From b145ed1ea4e614c5aaa2af273b00cd815d17e13f Mon Sep 17 00:00:00 2001 From: Sebastian Lorenz Date: Sun, 15 Nov 2026 11:17:55 +0000 Subject: [PATCH 01/88] Add effect/dataframe Typed columnar data frames and lazy queries on top of effect/arrow, with in-memory, streaming and SQL engines. Split out of #10641. --- .changeset/dataframe.md | 5 + .gitignore | 3 + packages/effect/benchmark/dataframe/h2o.py | 237 +++ packages/effect/benchmark/dataframe/h2o.ts | 175 ++ packages/effect/package.json | 4 + packages/effect/src/dataframe/Catalog.ts | 170 ++ packages/effect/src/dataframe/DataFrame.ts | 437 +++++ packages/effect/src/dataframe/Engine.ts | 96 ++ packages/effect/src/dataframe/Expr.ts | 1417 +++++++++++++++++ packages/effect/src/dataframe/Plan.ts | 1103 +++++++++++++ packages/effect/src/dataframe/Query.ts | 901 +++++++++++ packages/effect/src/dataframe/Sql.ts | 150 ++ packages/effect/src/dataframe/index.ts | 48 + .../src/dataframe/internal/aggregate.ts | 401 +++++ .../effect/src/dataframe/internal/decimal.ts | 194 +++ .../effect/src/dataframe/internal/dtype.ts | 187 +++ .../effect/src/dataframe/internal/expr.ts | 526 ++++++ .../effect/src/dataframe/internal/frame.ts | 757 +++++++++ .../effect/src/dataframe/internal/hash.ts | 444 ++++++ .../effect/src/dataframe/internal/kernels.ts | 1255 +++++++++++++++ .../effect/src/dataframe/internal/nested.ts | 85 + .../effect/src/dataframe/internal/partial.ts | 152 ++ .../effect/src/dataframe/internal/query.ts | 650 ++++++++ .../effect/src/dataframe/internal/schema.ts | 177 ++ .../effect/src/dataframe/internal/sort.ts | 263 +++ packages/effect/src/dataframe/internal/sql.ts | 930 +++++++++++ .../effect/src/dataframe/internal/stream.ts | 230 +++ .../effect/src/dataframe/internal/temporal.ts | 320 ++++ .../effect/src/dataframe/internal/vector.ts | 900 +++++++++++ .../effect/src/dataframe/internal/window.ts | 313 ++++ .../effect/test/dataframe/DataFrame.test.ts | 148 ++ packages/effect/test/dataframe/Oracle.test.ts | 29 + packages/effect/test/dataframe/Query.test.ts | 299 ++++ packages/effect/test/dataframe/Scale.test.ts | 232 +++ packages/effect/test/dataframe/Stream.test.ts | 180 +++ .../effect/test/dataframe/oracle/Compare.ts | 95 ++ packages/effect/test/dataframe/oracle/Gold.ts | 152 ++ packages/effect/test/dataframe/oracle/Plan.ts | 351 ++++ .../test/fixtures/dataframe/compile_sql.ts | 26 + .../test/fixtures/dataframe/generate_gold.py | 1059 ++++++++++++ .../fixtures/dataframe/gold/extended.json | 1 + .../test/fixtures/dataframe/gold/random.json | 1 + .../test/fixtures/dataframe/gold/spec.json | 1 + .../test/fixtures/dataframe/sql_oracle.py | 133 ++ packages/effect/typetest/DataFrame.tst.ts | 280 ++++ packages/sql/pglite/test/DataFrameSql.test.ts | 158 ++ 46 files changed, 15675 insertions(+) create mode 100644 .changeset/dataframe.md create mode 100644 packages/effect/benchmark/dataframe/h2o.py create mode 100644 packages/effect/benchmark/dataframe/h2o.ts create mode 100644 packages/effect/src/dataframe/Catalog.ts create mode 100644 packages/effect/src/dataframe/DataFrame.ts create mode 100644 packages/effect/src/dataframe/Engine.ts create mode 100644 packages/effect/src/dataframe/Expr.ts create mode 100644 packages/effect/src/dataframe/Plan.ts create mode 100644 packages/effect/src/dataframe/Query.ts create mode 100644 packages/effect/src/dataframe/Sql.ts create mode 100644 packages/effect/src/dataframe/index.ts create mode 100644 packages/effect/src/dataframe/internal/aggregate.ts create mode 100644 packages/effect/src/dataframe/internal/decimal.ts create mode 100644 packages/effect/src/dataframe/internal/dtype.ts create mode 100644 packages/effect/src/dataframe/internal/expr.ts create mode 100644 packages/effect/src/dataframe/internal/frame.ts create mode 100644 packages/effect/src/dataframe/internal/hash.ts create mode 100644 packages/effect/src/dataframe/internal/kernels.ts create mode 100644 packages/effect/src/dataframe/internal/nested.ts create mode 100644 packages/effect/src/dataframe/internal/partial.ts create mode 100644 packages/effect/src/dataframe/internal/query.ts create mode 100644 packages/effect/src/dataframe/internal/schema.ts create mode 100644 packages/effect/src/dataframe/internal/sort.ts create mode 100644 packages/effect/src/dataframe/internal/sql.ts create mode 100644 packages/effect/src/dataframe/internal/stream.ts create mode 100644 packages/effect/src/dataframe/internal/temporal.ts create mode 100644 packages/effect/src/dataframe/internal/vector.ts create mode 100644 packages/effect/src/dataframe/internal/window.ts create mode 100644 packages/effect/test/dataframe/DataFrame.test.ts create mode 100644 packages/effect/test/dataframe/Oracle.test.ts create mode 100644 packages/effect/test/dataframe/Query.test.ts create mode 100644 packages/effect/test/dataframe/Scale.test.ts create mode 100644 packages/effect/test/dataframe/Stream.test.ts create mode 100644 packages/effect/test/dataframe/oracle/Compare.ts create mode 100644 packages/effect/test/dataframe/oracle/Gold.ts create mode 100644 packages/effect/test/dataframe/oracle/Plan.ts create mode 100644 packages/effect/test/fixtures/dataframe/compile_sql.ts create mode 100644 packages/effect/test/fixtures/dataframe/generate_gold.py create mode 100644 packages/effect/test/fixtures/dataframe/gold/extended.json create mode 100644 packages/effect/test/fixtures/dataframe/gold/random.json create mode 100644 packages/effect/test/fixtures/dataframe/gold/spec.json create mode 100644 packages/effect/test/fixtures/dataframe/sql_oracle.py create mode 100644 packages/effect/typetest/DataFrame.tst.ts create mode 100644 packages/sql/pglite/test/DataFrameSql.test.ts diff --git a/.changeset/dataframe.md b/.changeset/dataframe.md new file mode 100644 index 00000000000..500c135633f --- /dev/null +++ b/.changeset/dataframe.md @@ -0,0 +1,5 @@ +--- +"effect": patch +--- + +Add `effect/dataframe`: typed, columnar data frames on top of `effect/arrow`. `DataFrame` is the immutable table value; `Query` describes filters, derived columns, sorts, groups, aggregates, joins, window functions (`Expr.over`, cumulative, shift, rank, rolling), pivots, unpivots and list explosion lazily, with column names and types checked at compile time, and runs as an `Effect` after an optimiser rewrites the plan (predicate, slice and projection pushdown, top-k). Columns can hold numbers, strings, booleans, dates, timestamps, durations, decimals, lists and structs. Queries are plain data (`Plan`, with Schema codecs to and from JSON), read named tables from a `Catalog` service (frames, effects or streams of batches, processed batch by batch), and run on an `Engine`: in memory, or on a database through `Sql.layer` and `effect/sql`. Results follow Polars semantics and are checked against a Polars-generated gold corpus. diff --git a/.gitignore b/.gitignore index 26bba5ae55d..64045b61fb4 100644 --- a/.gitignore +++ b/.gitignore @@ -39,3 +39,6 @@ scratchpad/**/* # Agent config opencode.json + +# Python bytecode (dev scripts) +__pycache__/ diff --git a/packages/effect/benchmark/dataframe/h2o.py b/packages/effect/benchmark/dataframe/h2o.py new file mode 100644 index 00000000000..7a697343ca0 --- /dev/null +++ b/packages/effect/benchmark/dataframe/h2o.py @@ -0,0 +1,237 @@ +""" +Performance baseline for effect/dataframe: H2O.ai db-benchmark style group-by +and join queries, run with Polars and pandas on the same data the TypeScript +benchmark reads (benchmark/dataframe/h2o.ts). + + python h2o.py generate --rows 1000000 [--data DIR] # writes Arrow IPC files + python h2o.py run [--data DIR] [--reps 3] # writes DIR/baseline-.json + +`run` times every query with Polars (all threads and one thread, each in its +own process) and pandas, and records a checksum of every result so the +TypeScript side can check it computed the same thing before comparing times. + +Group-by data follows db-benchmark's groupby-datagen (K = 100 groups for +id1/id2/id4/id5, N/K for id3/id6). Integer columns are Int32 on every side. +The join queries are db-benchmark-style (small/medium/big right tables) on +synthetic keys; see JOIN_QUERIES. +""" + +import argparse +import json +import os +import subprocess +import sys +import time +from pathlib import Path + +HERE = Path(__file__).parent +DEFAULT_DATA = HERE / ".data" + + +def generate(rows, data): + import numpy as np + import pyarrow as pa + import pyarrow.ipc as ipc + + rng = np.random.default_rng(20261114) + n = rows + k = 100 + data.mkdir(parents=True, exist_ok=True) + + def ids(fmt, count, size): + pool = np.array([fmt % (i + 1) for i in range(count)], dtype=object) + return pa.array(pool[rng.integers(0, count, size)], type=pa.string()) + + v3_pool = np.round(rng.uniform(0, 100, 100), 6) + groupby = pa.table({ + "id1": ids("id%03d", k, n), + "id2": ids("id%03d", k, n), + "id3": ids("id%010d", n // k, n), + "id4": pa.array(rng.integers(1, k + 1, n), type=pa.int32()), + "id5": pa.array(rng.integers(1, k + 1, n), type=pa.int32()), + "id6": pa.array(rng.integers(1, n // k + 1, n), type=pa.int32()), + "v1": pa.array(rng.integers(1, 6, n), type=pa.int32()), + "v2": pa.array(rng.integers(1, 16, n), type=pa.int32()), + "v3": pa.array(v3_pool[rng.integers(0, 100, n)], type=pa.float64()), + }) + write(groupby, data / f"groupby-{rows}.arrow") + + x_id2 = rng.integers(1, 10_001, n) + x = pa.table({ + "id1": pa.array(rng.integers(1, 11, n), type=pa.int32()), + "id2": pa.array(x_id2, type=pa.int32()), + "id3": pa.array(rng.integers(1, n + 1, n), type=pa.int32()), + "id5": pa.array(np.array([f"id{i}" for i in range(10_001)], dtype=object)[x_id2], type=pa.string()), + "v1": pa.array(np.round(rng.uniform(0, 100, n), 6), type=pa.float64()), + }) + small = pa.table({"id1": pa.array(np.arange(1, 11), type=pa.int32()), "v2": pa.array(rng.uniform(0, 100, 10))}) + medium_ids = np.arange(1, 10_001) + medium = pa.table({ + "id2": pa.array(medium_ids, type=pa.int32()), + "id5": pa.array([f"id{i}" for i in medium_ids], type=pa.string()), + "v2": pa.array(rng.uniform(0, 100, 10_000)), + }) + big = pa.table({"id3": pa.array(rng.permutation(n) + 1, type=pa.int32()), "v2": pa.array(rng.uniform(0, 100, n))}) + for name, t in [("x", x), ("small", small), ("medium", medium), ("big", big)]: + write(t, data / f"join-{name}-{rows}.arrow") + print(f"wrote {rows} rows to {data}") + + +def write(table, path): + import pyarrow.ipc as ipc + + with ipc.new_file(str(path), table.schema) as w: + w.write_table(table) + + +# --------------------------------------------------------------------------- +# Queries (Polars) +# --------------------------------------------------------------------------- + + +def polars_queries(pl): + c = pl.col + gb = lambda df, keys, *aggs: df.group_by(keys, maintain_order=True).agg(*aggs) + return { + "q1 sum v1 by id1": lambda t: gb(t["g"], ["id1"], c("v1").sum()), + "q2 sum v1 by id1:id2": lambda t: gb(t["g"], ["id1", "id2"], c("v1").sum()), + "q3 sum v1, mean v3 by id3": lambda t: gb(t["g"], ["id3"], c("v1").sum(), c("v3").mean()), + "q4 mean v1:v3 by id4": lambda t: gb(t["g"], ["id4"], c("v1").mean(), c("v2").mean(), c("v3").mean()), + "q5 sum v1:v3 by id6": lambda t: gb(t["g"], ["id6"], c("v1").sum(), c("v2").sum(), c("v3").sum()), + "q6 median, sd v3 by id4:id5": lambda t: gb(t["g"], ["id4", "id5"], c("v3").median().alias("median"), c("v3").std().alias("sd")), + "q7 max v1 - min v2 by id3": lambda t: gb(t["g"], ["id3"], (c("v1").max() - c("v2").min()).alias("range")), + "q10 sum v3, count by id1:id6": lambda t: gb(t["g"], ["id1", "id2", "id3", "id4", "id5", "id6"], c("v3").sum(), pl.len()), + "filter v3 > 50, sum v3": lambda t: t["g"].filter(c("v3") > 50).select(c("v3").sum()), + "sort by v3 desc, id6": lambda t: t["g"].sort(["v3", "id6"], descending=[True, False], maintain_order=True), + "top 10 by v3 desc, id6": lambda t: t["g"].lazy().sort(["v3", "id6"], descending=[True, False], maintain_order=True).head(10).collect(), + "j1 inner small on id1": lambda t: t["x"].join(t["small"], on="id1", how="inner", maintain_order="left_right"), + "j2 inner medium on id2": lambda t: t["x"].join(t["medium"].drop("id5"), on="id2", how="inner", maintain_order="left_right"), + "j3 left medium on id2": lambda t: t["x"].join(t["medium"].drop("id5"), on="id2", how="left", maintain_order="left_right"), + "j4 inner medium on id5 (string)": lambda t: t["x"].drop("id2").join(t["medium"].drop("id2"), on="id5", how="inner", maintain_order="left_right"), + "j5 inner big on id3": lambda t: t["x"].join(t["big"], on="id3", how="inner", maintain_order="left_right"), + } + + +def pandas_queries(pd): + gb = lambda df, keys, spec: df.groupby(keys, sort=False, observed=True).agg(**spec).reset_index() + return { + "q1 sum v1 by id1": lambda t: gb(t["g"], ["id1"], {"v1": ("v1", "sum")}), + "q2 sum v1 by id1:id2": lambda t: gb(t["g"], ["id1", "id2"], {"v1": ("v1", "sum")}), + "q3 sum v1, mean v3 by id3": lambda t: gb(t["g"], ["id3"], {"v1": ("v1", "sum"), "v3": ("v3", "mean")}), + "q4 mean v1:v3 by id4": lambda t: gb(t["g"], ["id4"], {"v1": ("v1", "mean"), "v2": ("v2", "mean"), "v3": ("v3", "mean")}), + "q5 sum v1:v3 by id6": lambda t: gb(t["g"], ["id6"], {"v1": ("v1", "sum"), "v2": ("v2", "sum"), "v3": ("v3", "sum")}), + "q6 median, sd v3 by id4:id5": lambda t: gb(t["g"], ["id4", "id5"], {"median": ("v3", "median"), "sd": ("v3", "std")}), + "q7 max v1 - min v2 by id3": lambda t: (lambda r: r.assign(range=r["v1"] - r["v2"])[["id3", "range"]])( + gb(t["g"], ["id3"], {"v1": ("v1", "max"), "v2": ("v2", "min")})), + "q10 sum v3, count by id1:id6": lambda t: gb(t["g"], ["id1", "id2", "id3", "id4", "id5", "id6"], {"v3": ("v3", "sum"), "len": ("v1", "size")}), + "filter v3 > 50, sum v3": lambda t: t["g"]["v3"][t["g"]["v3"] > 50].sum(), + "sort by v3 desc, id6": lambda t: t["g"].sort_values(["v3", "id6"], ascending=[False, True], kind="stable"), + "top 10 by v3 desc, id6": lambda t: t["g"].sort_values(["v3", "id6"], ascending=[False, True], kind="stable").head(10), + "j1 inner small on id1": lambda t: t["x"].merge(t["small"], on="id1", how="inner"), + "j2 inner medium on id2": lambda t: t["x"].merge(t["medium"].drop(columns="id5"), on="id2", how="inner"), + "j3 left medium on id2": lambda t: t["x"].merge(t["medium"].drop(columns="id5"), on="id2", how="left"), + "j4 inner medium on id5 (string)": lambda t: t["x"].drop(columns="id2").merge(t["medium"].drop(columns="id2"), on="id5", how="inner"), + "j5 inner big on id3": lambda t: t["x"].merge(t["big"], on="id3", how="inner"), + } + + +def checksum_polars(df): + import polars as pl + + if not isinstance(df, pl.DataFrame): + return {"rows": 1, "sum": float(df)} + total = 0.0 + for name, dtype in df.schema.items(): + if dtype.is_numeric(): + s = df.get_column(name).cast(pl.Float64).fill_nan(None).sum() + total += s or 0.0 + return {"rows": df.height, "sum": total} + + +def load_polars(pl, data, rows): + read = lambda name: pl.read_ipc(data / name) + return { + "g": read(f"groupby-{rows}.arrow"), + "x": read(f"join-x-{rows}.arrow"), + "small": read(f"join-small-{rows}.arrow"), + "medium": read(f"join-medium-{rows}.arrow"), + "big": read(f"join-big-{rows}.arrow"), + } + + +def time_best(f, reps): + best = float("inf") + result = None + for _ in range(reps): + start = time.perf_counter() + result = f() + best = min(best, time.perf_counter() - start) + return best * 1000, result + + +def run_polars(data, rows, reps): + import polars as pl + + tables = load_polars(pl, data, rows) + out = {"version": pl.__version__, "threads": pl.thread_pool_size(), "queries": {}} + for name, q in polars_queries(pl).items(): + ms, result = time_best(lambda: q(tables), reps) + out["queries"][name] = {"ms": ms, "checksum": checksum_polars(result)} + return out + + +def run_pandas(data, rows, reps): + import pandas as pd + import pyarrow.ipc as ipc + + read = lambda name: ipc.open_file(str(data / name)).read_all().to_pandas() + tables = { + "g": read(f"groupby-{rows}.arrow"), + "x": read(f"join-x-{rows}.arrow"), + "small": read(f"join-small-{rows}.arrow"), + "medium": read(f"join-medium-{rows}.arrow"), + "big": read(f"join-big-{rows}.arrow"), + } + out = {"version": pd.__version__, "queries": {}} + for name, q in pandas_queries(pd).items(): + ms, _ = time_best(lambda: q(tables), reps) + out["queries"][name] = {"ms": ms} + return out + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("command", choices=["generate", "run", "polars", "pandas"]) + parser.add_argument("--rows", type=int, default=1_000_000) + parser.add_argument("--data", type=Path, default=DEFAULT_DATA) + parser.add_argument("--reps", type=int, default=3) + args = parser.parse_args() + if args.command == "generate": + generate(args.rows, args.data) + elif args.command in ("polars", "pandas"): + result = run_polars(args.data, args.rows, args.reps) if args.command == "polars" else run_pandas(args.data, args.rows, args.reps) + print(json.dumps(result)) + else: + def child(command, env=None): + out = subprocess.run( + [sys.executable, __file__, command, "--rows", str(args.rows), "--data", str(args.data), "--reps", str(args.reps)], + check=True, capture_output=True, text=True, env={**os.environ, **(env or {})}, + ) + return json.loads(out.stdout) + + baseline = { + "rows": args.rows, + "machine": {"cpus": os.cpu_count(), "platform": sys.platform}, + "polars": child("polars"), + "polars1": child("polars", {"POLARS_MAX_THREADS": "1"}), + "pandas": child("pandas"), + } + path = args.data / f"baseline-{args.rows}.json" + path.write_text(json.dumps(baseline, indent=2) + "\n") + print(f"wrote {path}") + for name, q in baseline["polars"]["queries"].items(): + print(f"{name:36} polars {q['ms']:9.1f} ms polars(1t) {baseline['polars1']['queries'][name]['ms']:9.1f} ms pandas {baseline['pandas']['queries'][name]['ms']:9.1f} ms") + + +if __name__ == "__main__": + main() diff --git a/packages/effect/benchmark/dataframe/h2o.ts b/packages/effect/benchmark/dataframe/h2o.ts new file mode 100644 index 00000000000..5a400ce23eb --- /dev/null +++ b/packages/effect/benchmark/dataframe/h2o.ts @@ -0,0 +1,175 @@ +/** + * effect/dataframe performance against Polars and pandas: the performance + * half of the dataframe oracle. + * + * python benchmark/dataframe/h2o.py generate --rows 1000000 + * python benchmark/dataframe/h2o.py run --rows 1000000 + * node benchmark/dataframe/h2o.ts [--rows=1000000] [--reps=3] [--only=] [--out=] + * + * Every query reads the same Arrow IPC files as the Python baseline, and runs + * as a lazy query (so the time includes optimising the plan). Before + * its time counts, each result's checksum (row count and the sum of its + * numeric columns) has to match the Polars result, so a fast wrong answer + * can't win. Times are the best of `--reps` runs, in milliseconds. + `node --expose-gc h2o.ts --gc` + * collects garbage before every timed run, so earlier queries' garbage + * doesn't count against later ones. + */ +import { Effect } from "effect" +import { ArrowIpc } from "effect/arrow" +import { DataFrame, Expr, Query } from "effect/dataframe" +import * as Result from "effect/Result" +import * as Fs from "node:fs" +import * as Path from "node:path" + +const arg = (name: string) => process.argv.find((a) => a.startsWith(`--${name}=`))?.slice(name.length + 3) +const rows = Number(arg("rows") ?? 1_000_000) +const reps = Number(arg("reps") ?? 3) +const only = arg("only") +const out = arg("out") +const data = arg("data") ?? Path.join(import.meta.dirname, ".data") +/** With `node --expose-gc ... --gc`, collect garbage before every timed run. */ +const gc = process.argv.includes("--gc") ? (globalThis as { gc?: () => void }).gc : undefined + +const load = (name: string) => { + const start = performance.now() + const bytes = new Uint8Array(Fs.readFileSync(Path.join(data, `${name}-${rows}.arrow`))) + const table = Result.getOrThrow(ArrowIpc.decodeFile(bytes)) + const df = DataFrame.fromArrowTable(table) + return { df: df as DataFrame.DataFrame, ms: performance.now() - start } +} + +const loads = { + g: load("groupby"), + x: load("join-x"), + small: load("join-small"), + medium: load("join-medium"), + big: load("join-big") +} +const t = Object.fromEntries(Object.entries(loads).map(([k, v]) => [k, Query.from(v.df)])) as Record< + keyof typeof loads, + Query.Query +> + +/** Untyped frames: joins are built as plan data. */ +const join = (left: Query.Query, right: Query.Query, on: string, how: Query.JoinHow = "inner") => + Query.fromPlan({ _tag: "Join", left: left.plan, right: right.plan, on: [on], how }) + +const queries: Record Query.Query> = { + "q1 sum v1 by id1": () => t.g.pipe(Query.groupBy("id1"), Query.agg((c) => ({ v1: Expr.sum(c.v1) }))), + "q2 sum v1 by id1:id2": () => t.g.pipe(Query.groupBy("id1", "id2"), Query.agg((c) => ({ v1: Expr.sum(c.v1) }))), + "q3 sum v1, mean v3 by id3": () => + t.g.pipe(Query.groupBy("id3"), Query.agg((c) => ({ v1: Expr.sum(c.v1), v3: Expr.mean(c.v3) }))), + "q4 mean v1:v3 by id4": () => + t.g.pipe( + Query.groupBy("id4"), + Query.agg((c) => ({ v1: Expr.mean(c.v1), v2: Expr.mean(c.v2), v3: Expr.mean(c.v3) })) + ), + "q5 sum v1:v3 by id6": () => + t.g.pipe( + Query.groupBy("id6"), + Query.agg((c) => ({ v1: Expr.sum(c.v1), v2: Expr.sum(c.v2), v3: Expr.sum(c.v3) })) + ), + "q6 median, sd v3 by id4:id5": () => + t.g.pipe( + Query.groupBy("id4", "id5"), + Query.agg((c) => ({ median: Expr.median(c.v3), sd: Expr.std(c.v3) })) + ), + "q7 max v1 - min v2 by id3": () => + t.g.pipe( + Query.groupBy("id3"), + Query.agg((c) => ({ v1: Expr.max(c.v1), v2: Expr.min(c.v2) })), + Query.select((c) => ({ id3: c.id3, range: Expr.subtract(c.v1, c.v2) })) + ), + "q10 sum v3, count by id1:id6": () => + t.g.pipe( + Query.groupBy("id1", "id2", "id3", "id4", "id5", "id6"), + Query.agg((c) => ({ v3: Expr.sum(c.v3), len: Expr.count() })) + ), + "filter v3 > 50, sum v3": () => + t.g.pipe(Query.filter((c) => Expr.gt(c.v3, 50)), Query.aggregate((c) => ({ v3: Expr.sum(c.v3) }))), + "sort by v3 desc, id6": () => t.g.pipe(Query.sort((c) => [Expr.desc(c.v3), c.id6])), + "top 10 by v3 desc, id6": () => t.g.pipe(Query.sort((c) => [Expr.desc(c.v3), c.id6]), Query.head(10)), + "j1 inner small on id1": () => join(t.x, t.small, "id1"), + "j2 inner medium on id2": () => join(t.x, t.medium.pipe(Query.omit("id5")), "id2"), + "j3 left medium on id2": () => join(t.x, t.medium.pipe(Query.omit("id5")), "id2", "left"), + "j4 inner medium on id5 (string)": () => join(t.x.pipe(Query.omit("id2")), t.medium.pipe(Query.omit("id2")), "id5"), + "j5 inner big on id3": () => join(t.x, t.big, "id3") +} + +/** + * Row count and the sum of every numeric column. Reading the columns also + * forces the late-materialised columns of a result, so the timed work + * includes producing every value, as it does for Polars. + */ +const checksum = (df: DataFrame.DataFrame) => { + let sum = 0 + for (const field of df.schema.fields) { + const t = field.type._tag + if (t !== "Int" && t !== "Float") continue + for (const v of DataFrame.toArray(df, field.name)) { + if (typeof v === "number" && !Number.isNaN(v)) sum += v + else if (typeof v === "bigint") sum += Number(v) + } + } + return { rows: df.length, sum } +} + +/** Forces every column of a result without converting values to JS. */ +const force = (df: DataFrame.DataFrame) => DataFrame.toArrowTable(df) + +const baselinePath = Path.join(data, `baseline-${rows}.json`) +const baseline = Fs.existsSync(baselinePath) ? JSON.parse(Fs.readFileSync(baselinePath, "utf8")) : undefined +if (baseline === undefined) console.warn(`no baseline at ${baselinePath}: run h2o.py run --rows ${rows}`) + +const results: Record = {} +const fmt = (ms: number | undefined) => (ms === undefined ? "—" : ms.toFixed(1)).padStart(9) +const ratio = (a: number, b: number | undefined) => (b === undefined ? "" : `${(a / b).toFixed(2)}x`).padStart(7) + +console.log( + `effect/dataframe vs Polars ${baseline?.polars.version ?? "?"} (${ + baseline?.polars.threads ?? "?" + } threads) and pandas ${baseline?.pandas.version ?? "?"}, ${rows} rows, best of ${reps}` +) +console.log( + `load (IPC decode + DataFrame): ${Object.entries(loads).map(([k, v]) => `${k} ${v.ms.toFixed(1)} ms`).join(", ")}` +) +console.log( + `${"query".padEnd(34)} ${"effect".padStart(9)} ${"polars".padStart(9)} ${"pl 1t".padStart(9)} ${ + "pandas".padStart(9) + } ${"vs pl".padStart(7)} ${"vs pl1t".padStart(7)} ${"vs pd".padStart(7)} check` +) +for (const [name, query] of Object.entries(queries)) { + if (only !== undefined && !name.includes(only)) continue + let best = Infinity + let first = 0 + let result: DataFrame.DataFrame | undefined + for (let r = 0; r < reps; r++) { + result = undefined + gc?.() + const start = performance.now() + result = Effect.runSync(query()) + force(result) + const ms = performance.now() - start + if (r === 0) first = ms + best = Math.min(best, ms) + } + const sum = checksum(result!) + const expected = baseline?.polars.queries[name]?.checksum + const ok = expected === undefined + ? "?" + : expected.rows === sum.rows && Math.abs(expected.sum - sum.sum) <= 1e-6 * Math.max(1, Math.abs(expected.sum)) + ? "ok" + : `MISMATCH rows ${sum.rows}/${expected.rows} sum ${sum.sum}/${expected.sum}` + const pl = baseline?.polars.queries[name]?.ms + const pl1 = baseline?.polars1.queries[name]?.ms + const pd = baseline?.pandas.queries[name]?.ms + console.log( + `${name.padEnd(34)} ${fmt(best)} ${fmt(pl)} ${fmt(pl1)} ${fmt(pd)} ${ratio(best, pl)} ${ratio(best, pl1)} ${ + ratio(best, pd) + } ${ok}` + ) + results[name] = { ms: best, first, checksum: sum, ok, polars: pl, polars1: pl1, pandas: pd } +} + +if (out !== undefined) Fs.writeFileSync(out, JSON.stringify({ rows, reps, results }, null, 2) + "\n") diff --git a/packages/effect/package.json b/packages/effect/package.json index fbc71f36176..32e50df3031 100644 --- a/packages/effect/package.json +++ b/packages/effect/package.json @@ -35,6 +35,7 @@ "./testing": "./src/testing/index.ts", "./ai": "./src/ai/index.ts", "./arrow": "./src/arrow/index.ts", + "./dataframe": "./src/dataframe/index.ts", "./cli": "./src/cli/index.ts", "./cluster": "./src/cluster/index.ts", "./crypto": "./src/crypto/index.ts", @@ -60,6 +61,7 @@ "./*": "./src/*.ts", "./ai/internal/*": null, "./arrow/internal/*": null, + "./dataframe/internal/*": null, "./cli/internal/*": null, "./cluster/internal/*": null, "./crypto/internal/*": null, @@ -97,6 +99,7 @@ "./testing": "./dist/testing/index.js", "./ai": "./dist/ai/index.js", "./arrow": "./dist/arrow/index.js", + "./dataframe": "./dist/dataframe/index.js", "./cli": "./dist/cli/index.js", "./cluster": "./dist/cluster/index.js", "./crypto": "./dist/crypto/index.js", @@ -126,6 +129,7 @@ "./crypto/internal/*": null, "./ai/internal/*": null, "./arrow/internal/*": null, + "./dataframe/internal/*": null, "./eventlog/internal/*": null, "./http/internal/*": null, "./http/FindMyWay/internal/*": null, diff --git a/packages/effect/src/dataframe/Catalog.ts b/packages/effect/src/dataframe/Catalog.ts new file mode 100644 index 00000000000..c0a9ffbd2d0 --- /dev/null +++ b/packages/effect/src/dataframe/Catalog.ts @@ -0,0 +1,170 @@ +/** + * The tables a query can read by name. + * + * **Mental model** + * + * - `Query.table("trips", Trip)` (or a `Table` node in a plan) refers to a + * table by name. When the query runs, the {@link Catalog} in the context + * provides it. + * - A table is a frame, an effect producing one (read a file, call a + * service, fetch over Arrow Flight), or a stream of frames (batches). Each + * run of a query reads each table it names once. + * - A query over a streamed table runs batch by batch: row-wise operations + * per batch, grouping and aggregation folded into partial aggregates, so a + * stream larger than memory can be filtered and summarised. + * - The default catalog is empty; provide one with {@link layer} or + * `Effect.provideService(Catalog, Catalog.make(...))`. + * + * **Example** (Providing tables) + * + * ```ts import.meta.vitest + * import { Effect, Schema } from "effect" + * import { Catalog, DataFrame, Expr, Query } from "effect/dataframe" + * + * const Trip = Schema.Struct({ city: Schema.String, km: Schema.Number }) + * + * const longTrips = Query.table("trips", Trip).pipe( + * Query.filter((c) => Expr.gt(c.km, 10)), + * Query.pick("city") + * ) + * + * const trips = DataFrame.fromRows(Trip, [ + * { city: "Vienna", km: 4 }, + * { city: "Berlin", km: 31 } + * ]) + * + * const rows = longTrips.pipe( + * Effect.provide(Catalog.layer({ trips })), + * Effect.map(DataFrame.toRows), + * Effect.runSync + * ) + * rows // => [{ city: "Berlin" }] + * ``` + * + * @stability experimental + * @since 4.1.5 + */ +import * as Context from "../Context.ts" +import * as Effect from "../Effect.ts" +import * as Layer from "../Layer.ts" +import * as Stream from "../Stream.ts" +import { type DataFrame, DataFrameError, isDataFrame } from "./DataFrame.ts" +import * as F from "./internal/frame.ts" + +/** + * Provides tables by name to running queries. Defaults to an empty catalog. + * + * @stability experimental + * @category services + * @since 4.1.5 + */ +export class Catalog extends Context.Reference("effect/dataframe/Catalog", { + defaultValue: () => empty +}) {} + +/** + * Types of the {@link Catalog} service. + * + * @stability experimental + * @since 4.1.5 + */ +export declare namespace Catalog { + /** + * The shape of a catalog. + * + * @stability experimental + * @category models + * @since 4.1.5 + */ + export interface Service { + /** The table called `name`, as one frame; fails when there is none. */ + readonly table: (name: string) => Effect.Effect, DataFrameError> + /** The table called `name`, as batches. */ + readonly stream: (name: string) => Stream.Stream, DataFrameError> + /** Whether the table called `name` is streamed (queries over it run batch by batch). */ + readonly streaming: (name: string) => boolean + } +} + +/** + * A table: a frame, or an effect producing one. + * + * @stability experimental + * @category models + * @since 4.1.5 + */ +export type Source = + | DataFrame + | Effect.Effect, DataFrameError> + | Stream.Stream, DataFrameError> + +const unknownTable = (name: string) => + Effect.fail(new DataFrameError({ message: `unknown table ${JSON.stringify(name)}` })) + +/** + * A catalog without tables. + * + * @stability experimental + * @category constructors + * @since 4.1.5 + */ +export const empty: Catalog.Service = { + table: unknownTable, + stream: (name) => Stream.fromEffect(unknownTable(name)), + streaming: () => false +} + +const concatenated = (frames: ReadonlyArray>): Effect.Effect, DataFrameError> => { + if (frames.length === 1) return Effect.succeed(frames[0]) + try { + return Effect.succeed(F.concat(frames as unknown as ReadonlyArray) as unknown as DataFrame) + } catch (cause) { + return Effect.fail(new DataFrameError({ message: cause instanceof Error ? cause.message : String(cause), cause })) + } +} + +/** + * A catalog of the given tables. + * + * @stability experimental + * @category constructors + * @since 4.1.5 + */ +export const make = (tables: { readonly [name: string]: Source }): Catalog.Service => { + const entries = new Map(Object.entries(tables)) + return { + table: (name) => { + const source = entries.get(name) + if (source === undefined) return unknownTable(name) + if (isDataFrame(source)) return Effect.succeed(source) + if (Stream.isStream(source)) { + return Effect.flatMap( + Stream.runCollect(source as Stream.Stream, DataFrameError>), + (frames) => + frames.length === 0 + ? Effect.fail(new DataFrameError({ message: `table ${JSON.stringify(name)} has no batches` })) + : concatenated(frames) + ) + } + return source as Effect.Effect, DataFrameError> + }, + stream: (name) => { + const source = entries.get(name) + if (source === undefined) return Stream.fromEffect(unknownTable(name)) + if (isDataFrame(source)) return Stream.succeed(source) + if (Stream.isStream(source)) return source as Stream.Stream, DataFrameError> + return Stream.fromEffect(source as Effect.Effect, DataFrameError>) + }, + streaming: (name) => Stream.isStream(entries.get(name)) + } +} + +/** + * A layer providing a catalog of the given tables. + * + * @stability experimental + * @category layers + * @since 4.1.5 + */ +export const layer = (tables: { readonly [name: string]: Source }): Layer.Layer => + Layer.succeed(Catalog, make(tables)) diff --git a/packages/effect/src/dataframe/DataFrame.ts b/packages/effect/src/dataframe/DataFrame.ts new file mode 100644 index 00000000000..6d46b9628d2 --- /dev/null +++ b/packages/effect/src/dataframe/DataFrame.ts @@ -0,0 +1,437 @@ +/** + * Typed, columnar tables: the values queries read and produce. + * + * **Mental model** + * + * - A `DataFrame` is an immutable table whose rows have type `Row`, + * stored column by column in Arrow layouts (typed arrays, UTF-8 bytes). + * Frames convert to and from `effect/arrow` tables without copying where + * the layouts agree. + * - Frames are built from rows, columns or Arrow tables, inspected, compared + * and converted back to rows. Transforming them is the job of `Query`: + * `Query.from(frame)` starts a lazy query, and running it produces a new + * frame. + * - Missing values are `null`. A nullable column's row type includes `null`. + * + * **Example** (Building and reading a frame) + * + * ```ts import.meta.vitest + * import { Schema } from "effect" + * import { DataFrame } from "effect/dataframe" + * + * const Reading = Schema.Struct({ city: Schema.String, temp: Schema.NullOr(Schema.Number) }) + * const readings = DataFrame.fromRows(Reading, [ + * { city: "Vienna", temp: 21.5 }, + * { city: "Berlin", temp: null } + * ]) + * + * readings.length // => 2 + * DataFrame.columnNames(readings) // => ["city", "temp"] + * DataFrame.toArray(readings, "temp") // => [21.5, null] + * ``` + * + * **Gotchas** + * + * - Row objects ({@link toRows}, iteration) map values as `effect/arrow` + * does: 64-bit integers are `bigint`, timestamps `DateTime`. + * + * @stability experimental + * @since 4.1.5 + */ +import * as ArrowIpc from "../arrow/ArrowIpc.ts" +import * as ArrowSchema from "../arrow/ArrowSchema.ts" +import * as ArrowTable from "../arrow/ArrowTable.ts" +import * as Data from "../Data.ts" +import * as DateTime from "../DateTime.ts" +import * as Duration from "../Duration.ts" +import * as Effect from "../Effect.ts" +import type * as Equal from "../Equal.ts" +import { dual } from "../Function.ts" +import type { Inspectable } from "../Inspectable.ts" +import type { Pipeable } from "../Pipeable.ts" +import * as Result from "../Result.ts" +import * as Stream from "../Stream.ts" +import type * as Types from "../Types.ts" +import * as F from "./internal/frame.ts" +import * as Schema_ from "./internal/schema.ts" +import * as V from "./internal/vector.ts" + +// ----------------------------------------------------------------------------- +// Models +// ----------------------------------------------------------------------------- + +/** + * Identifies values of this module's main type. + * + * @stability experimental + * @category type IDs + * @since 4.1.5 + */ +export const TypeId: "~effect/dataframe/DataFrame" = F.TypeId + +/** + * A table whose rows have type `Row`. + * + * @stability experimental + * @category models + * @since 4.1.5 + */ +export interface DataFrame extends Pipeable, Inspectable, Equal.Equal, Iterable { + readonly [TypeId]: { + readonly _Row: Types.Covariant + } + /** The number of rows. */ + readonly length: number + /** The column names and their Arrow types. */ + readonly schema: ArrowSchema.Schema +} + +/** + * The row type of a frame. + * + * @stability experimental + * @category type-level + * @since 4.1.5 + */ +export type Row = F extends DataFrame ? R : never + +type Simplify = { readonly [K in keyof A]: A[K] } extends infer B ? B : never + +/** + * Raised for invalid input that the types can't rule out: a row that doesn't + * match its schema, an Arrow table whose columns don't match a schema, or a + * column type the DataFrame doesn't support. + * + * @stability experimental + * @category errors + * @since 4.1.5 + */ +export class DataFrameError extends Data.TaggedError("DataFrameError")<{ + readonly message: string + readonly cause?: unknown +}> {} + +const fromDefect = (cause: unknown): DataFrameError => + new DataFrameError({ message: cause instanceof Error ? cause.message : String(cause), cause }) + +const unsafe = (f: () => A): A => { + try { + return f() + } catch (cause) { + throw cause instanceof V.DataFrameDefect ? fromDefect(cause) : cause + } +} + +const frame = (self: DataFrame): F.Frame => self as unknown as F.Frame +const wrap = (f: F.Frame): DataFrame => f as unknown as DataFrame +/** + * Returns `true` for data frames. + * + * @stability experimental + * @category guards + * @since 4.1.5 + */ +export const isDataFrame = (u: unknown): u is DataFrame => F.isFrame(u) + +// ----------------------------------------------------------------------------- +// Constructors +// ----------------------------------------------------------------------------- + +/** + * Builds a frame from rows described by a struct schema. Column types come + * from the schema's type side: `String` is Utf8, `Number` is Float64, + * `Boolean` is Bool, `BigInt` is Int64, and `NullOr`/optional fields are + * nullable. `options.types` overrides the Arrow type of individual columns + * (for example `ArrowSchema.int32`). + * + * **Details** + * + * Throws a {@link DataFrameError} when a value doesn't fit its column. + * + * @stability experimental + * @category constructors + * @since 4.1.5 + */ +export const fromRows = ( + schema: S, + rows: Iterable, + options?: { readonly types?: { readonly [K in keyof S["Type"]]?: ArrowSchema.DataType } | undefined } +): DataFrame> => + unsafe(() => { + const arrow = Schema_.toArrowSchema(schema, options?.types as any) + const table = Result.getOrElse( + ArrowTable.fromRows(arrow, Array.isArray(rows) ? rows : Array.from(rows) as any), + (error) => { + throw fromDefect(error) + } + ) + return wrap(F.fromArrowTable(table)) + }) + +type ColumnInput = + | Float64Array + | Float32Array + | Int32Array + | Int16Array + | Int8Array + | Uint32Array + | Uint16Array + | Uint8Array + | BigInt64Array + | BigUint64Array + | ReadonlyArray + +type ColumnValue = C extends BigInt64Array | BigUint64Array ? bigint + : C extends ArrayBufferView ? number + : C extends ReadonlyArray ? A + : never + +const typedArrayTypes: ReadonlyArray = [ + [Float64Array, ArrowSchema.float64], + [Float32Array, ArrowSchema.float32], + [Int32Array, ArrowSchema.int32], + [Int16Array, ArrowSchema.int16], + [Int8Array, ArrowSchema.int8], + [Uint32Array, ArrowSchema.uint32], + [Uint16Array, ArrowSchema.uint16], + [Uint8Array, ArrowSchema.uint8], + [BigInt64Array, ArrowSchema.int64], + [BigUint64Array, ArrowSchema.uint64] +] + +const inferType = (values: ReadonlyArray): ArrowSchema.DataType => { + for (const v of values) { + switch (typeof v) { + case "number": + return ArrowSchema.float64 + case "string": + return ArrowSchema.utf8 + case "boolean": + return ArrowSchema.bool + case "bigint": + return ArrowSchema.int64 + case "object": + if (DateTime.isDateTime(v)) return ArrowSchema.timestamp("microsecond") + if (Duration.isDuration(v)) return ArrowSchema.duration("microsecond") + if (v !== null) return V.fail(`cannot infer a column type for ${String(v)}: use fromRows with a schema`) + } + } + return ArrowSchema.nullType +} + +/** + * Builds a frame from columns. Typed arrays keep their element type + * (`Int32Array` is Int32, `Float64Array` is Float64, …) and are used without + * copying where possible; arrays infer their type from their values + * (`number` is Float64, `string` Utf8, `boolean` Bool, `bigint` Int64, + * `DateTime` a microsecond timestamp, `Duration` a microsecond duration) and + * use `null` for missing values. + * + * **Details** + * + * Throws a {@link DataFrameError} when columns have different lengths or an + * array mixes value types. + * + * @stability experimental + * @category constructors + * @since 4.1.5 + */ +export const fromColumns = ( + columns: C +): DataFrame }>> => + unsafe(() => { + const names = Object.keys(columns) + const fields: Array = [] + const data = names.map((name) => { + const c = columns[name] + if (ArrayBuffer.isView(c)) { + const type = typedArrayTypes.find(([ctor]) => c instanceof ctor)![1] + fields.push(ArrowSchema.field(name, type, { nullable: false })) + const bytes = new Uint8Array(c.buffer, c.byteOffset, c.byteLength) + return { + type, + length: (c as Float64Array).length, + offset: 0, + nullCount: 0, + validity: undefined, + buffers: [bytes], + children: [], + dictionary: undefined + } + } + const type = inferType(c as ReadonlyArray) + fields.push(ArrowSchema.field(name, type)) + return undefined + }) + const lengths = new Set(names.map((n) => (columns[n] as { length: number }).length)) + if (lengths.size > 1) V.fail(`columns have different lengths: ${[...lengths].join(", ")}`) + const length = names.length === 0 ? 0 : (columns[names[0]] as { length: number }).length + const vectors = names.map((name, i) => { + const d = data[i] + if (d !== undefined) return V.fromArrowData(d) + const schema = ArrowSchema.make([fields[i]]) + const values = columns[name] as ReadonlyArray + const rows = values.map((v) => ({ [name]: v })) + const table = Result.getOrElse(ArrowTable.fromRows(schema, rows), (error) => { + throw fromDefect(error) + }) + return F.fromArrowTable(table).columns[0].vector + }) + return wrap(F.fromVectors(names, vectors, length)) + }) + +/** + * A frame over an Arrow table, with untyped rows. Use {@link validate} to + * check it against a schema and recover the row type. + * + * **Details** + * + * Throws a {@link DataFrameError} for column types the DataFrame doesn't + * support yet (decimals, nested types, intervals). + * + * @stability experimental + * @category constructors + * @since 4.1.5 + */ +export const fromArrowTable = (table: ArrowTable.Table): DataFrame<{ readonly [name: string]: unknown }> => + unsafe(() => wrap(F.fromArrowTable(table))) + +/** + * A frame over one Arrow record batch, with untyped rows. + * + * @stability experimental + * @category constructors + * @since 4.1.5 + */ +export const fromRecordBatch = (batch: ArrowTable.RecordBatch): DataFrame<{ readonly [name: string]: unknown }> => + fromArrowTable(ArrowTable.make(batch.schema, [batch])) + +/** + * Decodes Arrow IPC stream bytes (from a file, a socket or Arrow Flight) + * into a stream of frames, one per record batch, ready to be a streamed + * `Catalog` table. + * + * @stability experimental + * @category constructors + * @since 4.1.5 + */ +export const decodeArrowStream = ( + bytes: Stream.Stream +): Stream.Stream, DataFrameError | E, R> => + bytes.pipe( + Stream.pipeThroughChannel(ArrowIpc.decode()), + Stream.mapError((error) => + ArrowIpc.isArrowIpcError(error) ? new DataFrameError({ message: error.message, cause: error }) : error as E + ), + Stream.mapEffect((batch) => { + try { + return Effect.succeed(fromRecordBatch(batch)) + } catch (cause) { + return Effect.fail(cause instanceof DataFrameError ? cause : fromDefect(cause)) + } + }) + ) + +/** + * An empty frame with the columns of a struct schema. + * + * @stability experimental + * @category constructors + * @since 4.1.5 + */ +export const empty = ( + schema: S +): DataFrame> => fromRows(schema, []) + +/** + * Checks that a frame has the columns of a struct schema, with compatible + * types, and returns it typed (with only the schema's columns, in its + * order). Integer columns are accepted for `Number` fields. + * + * @stability experimental + * @category constructors + * @since 4.1.5 + */ +export const validate: { + ( + schema: S + ): (self: DataFrame) => Result.Result>, DataFrameError> + ( + self: DataFrame, + schema: S + ): Result.Result>, DataFrameError> +} = dual(2, (self: DataFrame, schema: { readonly ast: any }) => { + try { + const columns = Schema_.tableColumns(Schema_.toArrowSchema(schema, undefined)) + return Result.succeed(wrap(Schema_.conform(frame(self), columns))) + } catch (cause) { + return Result.fail(fromDefect(cause)) + } +}) + +// ----------------------------------------------------------------------------- +// Conversions +// ----------------------------------------------------------------------------- + +/** + * The rows as plain objects. + * + * @stability experimental + * @category conversions + * @since 4.1.5 + */ +export const toRows = (self: DataFrame): Array => F.toRows(frame(self)) + +/** + * The values of one column. + * + * @stability experimental + * @category conversions + * @since 4.1.5 + */ +export const toArray: { + (name: K): (self: DataFrame) => Array + (self: DataFrame, name: K): Array +} = dual(2, (self: DataFrame, name: string) => F.toArray(frame(self), name)) as any + +/** + * The frame as an Arrow table with one record batch. + * + * @stability experimental + * @category conversions + * @since 4.1.5 + */ +export const toArrowTable = (self: DataFrame): ArrowTable.Table => F.toArrowTable(frame(self)) + +/** + * The column names, in order. + * + * @stability experimental + * @category getters + * @since 4.1.5 + */ +export const columnNames = (self: DataFrame): Array => + frame(self).columns.map((c) => c.name) as Array + +/** + * A text table of the frame (its first and last rows when it is long). + * + * @stability experimental + * @category conversions + * @since 4.1.5 + */ +export const format = (self: DataFrame, options?: { readonly maxRows?: number | undefined }): string => + F.render(frame(self), options?.maxRows ?? 10) + +/** + * Whether two frames have the same columns (names and types) and values. + * `NaN` equals `NaN`. + * + * @stability experimental + * @category equivalence + * @since 4.1.5 + */ +export const equals: { + (that: DataFrame): (self: DataFrame) => boolean + (self: DataFrame, that: DataFrame): boolean +} = dual(2, (self: DataFrame, that: DataFrame) => F.equals(frame(self), frame(that))) diff --git a/packages/effect/src/dataframe/Engine.ts b/packages/effect/src/dataframe/Engine.ts new file mode 100644 index 00000000000..7d346b18c4a --- /dev/null +++ b/packages/effect/src/dataframe/Engine.ts @@ -0,0 +1,96 @@ +/** + * What runs queries: the in-memory engine by default, or another engine + * (for example `Sql.layer`, which runs plans on a database) provided as a + * layer. + * + * **Mental model** + * + * - A query is a `Plan.Node`. Running it (`yield* query`, `Query.stream`, + * `Query.explain`) hands the plan to the {@link Engine} in the context. + * - The {@link memory} engine runs plans in this process: it reads tables + * from the `Catalog`, rewrites the plan, and runs it. + * - Other engines run the same plans elsewhere. Their results are frames + * like any other. + * + * @stability experimental + * @since 4.1.5 + */ +import * as Context from "../Context.ts" +import * as Effect from "../Effect.ts" +import * as Layer from "../Layer.ts" +import * as Stream from "../Stream.ts" +import { Catalog } from "./Catalog.ts" +import type { DataFrame, DataFrameError } from "./DataFrame.ts" +import * as Q from "./internal/query.ts" +import * as Streaming from "./internal/stream.ts" +import type * as Plan from "./Plan.ts" + +/** + * Types of the {@link Engine} service. + * + * @stability experimental + * @since 4.1.5 + */ +export declare namespace Engine { + /** + * An engine. + * + * @stability experimental + * @category models + * @since 4.1.5 + */ + export interface Service { + /** A short name, for diagnostics. */ + readonly name: string + /** The plan's result as one frame. */ + readonly collect: (plan: Plan.Node) => Effect.Effect, DataFrameError> + /** The plan's result as batches. */ + readonly stream: (plan: Plan.Node) => Stream.Stream, DataFrameError> + /** What running the plan does, as text. */ + readonly explain: (plan: Plan.Node) => Effect.Effect + } +} + +const frame = (effect: Effect.Effect): Effect.Effect, DataFrameError> => + effect as any + +/** + * The in-memory engine: tables from the `Catalog`, the plan rewritten + * (predicate, slice and projection pushdown, top-k), streamed tables batch by + * batch. + * + * @stability experimental + * @category engines + * @since 4.1.5 + */ +export const memory: Engine.Service = { + name: "memory", + collect: (plan) => Catalog.use((catalog) => frame(Q.collect(plan, catalog))), + stream: (plan) => + Stream.unwrap( + Catalog.use((catalog) => + Effect.succeed(Streaming.stream(plan, catalog) as unknown as Stream.Stream, DataFrameError>) + ) + ), + explain: (plan) => Catalog.use((catalog) => Q.explain(plan, catalog)) +} + +/** + * Runs queries. Defaults to the {@link memory} engine. + * + * @stability experimental + * @category services + * @since 4.1.5 + */ +export class Engine extends Context.Reference("effect/dataframe/Engine", { + defaultValue: () => memory +}) {} + +/** + * A layer that runs queries with `engine`. + * + * @stability experimental + * @category layers + * @since 4.1.5 + */ +export const layer = (engine: Engine.Service): Layer.Layer => Layer.succeed(Engine, engine) diff --git a/packages/effect/src/dataframe/Expr.ts b/packages/effect/src/dataframe/Expr.ts new file mode 100644 index 00000000000..6db051ff747 --- /dev/null +++ b/packages/effect/src/dataframe/Expr.ts @@ -0,0 +1,1417 @@ +/** + * Column expressions: typed, inspectable descriptions of a computation over + * the columns of a query. + * + * **Mental model** + * + * - An `Expr` describes one value per row of type `A`. `A` includes + * `null` when the value can be missing. + * - Expressions are data: a typed wrapper around a `Plan.Expression` + * ({@link toPlan}, {@link fromPlan}). `Query` functions take a callback + * that receives one typed reference per column (`c.temp`, `c.city`) and + * returns expressions; the callback runs once, when the query is built. + * The resulting tree runs over whole columns at a time. + * - Every function is dual: `Expr.gt(c.temp, 20)` and + * `c.temp.pipe(Expr.gt(20))` build the same expression. Plain values in + * argument position are literals. + * - Missing values propagate: an operation on `null` gives `null`. + * `and`/`or` use three-valued logic (`false && null` is `false`). + * - An {@link Aggregate} reduces many rows to one (`sum`, `mean`, …). It is + * only accepted by `Query.agg` and `Query.aggregate`. + * + * **Details** + * + * The semantics follow Polars: + * + * - Float comparisons use a total order: `NaN` equals `NaN` and is greater + * than every other number. `-0` equals `0`. + * - Strings compare by Unicode code point (UTF-8 byte order). + * - Integer arithmetic wraps on overflow; `divide` is true division and + * returns a float; `floorDivide` and `mod` round towards negative infinity, + * and an integer division by zero gives `null`. + * - An integer literal takes the integer type of the column it meets + * (`c.int8Column + 1` stays Int8). + * + * **Example** (Building expressions) + * + * ```ts import.meta.vitest + * import { Effect, Schema } from "effect" + * import { DataFrame, Expr, Query } from "effect/dataframe" + * + * const Reading = Schema.Struct({ city: Schema.String, temp: Schema.Number, rain: Schema.NullOr(Schema.Number) }) + * const df = DataFrame.fromRows(Reading, [ + * { city: "Vienna", temp: 21.5, rain: 0 }, + * { city: "Berlin", temp: 26.5, rain: null } + * ]) + * + * const query = Query.from(df).pipe( + * Query.select((c) => ({ + * city: c.city, + * fahrenheit: c.temp.pipe(Expr.multiply(1.8), Expr.add(32)), + * wet: Expr.gt(c.rain, 0) + * })) + * ) + * + * const rows = DataFrame.toRows(Effect.runSync(query)) + * rows[0] // => { city: "Vienna", fahrenheit: 70.7, wet: false } + * rows[1] // => { city: "Berlin", fahrenheit: 79.7, wet: null } + * ``` + * + * @stability experimental + * @since 4.1.5 + */ +import type * as BigDecimal from "../BigDecimal.ts" +import type * as DateTime from "../DateTime.ts" +import type * as Duration from "../Duration.ts" +import { dual } from "../Function.ts" +import type { Pipeable } from "../Pipeable.ts" +import type * as Types from "../Types.ts" +import * as internal from "./internal/expr.ts" +import type * as Plan from "./Plan.ts" + +// ----------------------------------------------------------------------------- +// Models +// ----------------------------------------------------------------------------- + +/** + * Identifies values of this module's main type. + * + * @stability experimental + * @category type IDs + * @since 4.1.5 + */ +export const TypeId: "~effect/dataframe/Expr" = internal.ExprTypeId + +/** + * A per-row expression whose value has type `A`. + * + * @stability experimental + * @category models + * @since 4.1.5 + */ +export interface Expr extends Pipeable { + readonly [TypeId]: { + readonly _A: Types.Covariant + } +} + +/** + * Identifies aggregate expressions. + * + * @stability experimental + * @category type IDs + * @since 4.1.5 + */ +export const AggregateTypeId: "~effect/dataframe/Expr/Aggregate" = internal.AggregateTypeId + +/** + * An expression that reduces the rows of a group to one value of type `A`. + * + * @stability experimental + * @category models + * @since 4.1.5 + */ +export interface Aggregate extends Pipeable { + readonly [AggregateTypeId]: { + readonly _A: Types.Covariant + } +} + +/** + * A sort key: an expression and a direction. + * + * @stability experimental + * @category models + * @since 4.1.5 + */ +export interface SortOrder { + readonly expr: Expr + readonly descending: boolean +} + +/** + * An expression or a plain value (a literal). + * + * @stability experimental + * @category models + * @since 4.1.5 + */ +export type Input = Expr | A + +/** + * The value type of an expression or aggregate. + * + * @stability experimental + * @category type-level + * @since 4.1.5 + */ +export type Type = E extends Expr ? A : E extends Aggregate ? A : never + +/** + * `null` when `A` includes it. + * + * @stability experimental + * @category type-level + * @since 4.1.5 + */ +export type Nullish = null extends A ? null : never + +/** + * The widened primitive of `A`: what `A` can be compared with. + * + * @stability experimental + * @category type-level + * @since 4.1.5 + */ +export type Base = [NonNullable] extends [never] ? never + : NonNullable extends number ? number + : NonNullable extends bigint ? bigint + : NonNullable extends string ? string + : NonNullable extends boolean ? boolean + : NonNullable + +type Numeric = number | bigint | BigDecimal.BigDecimal +/** The numeric family of `A`: bigint, BigDecimal or number. */ +type NumericOf = NonNullable extends bigint ? bigint + : NonNullable extends BigDecimal.BigDecimal ? BigDecimal.BigDecimal + : number +type InputType = I extends Expr ? A : I + +/** + * Names accepted by {@link cast}, and the value type each produces. + * + * @stability experimental + * @category models + * @since 4.1.5 + */ +export interface CastTargets { + readonly Int8: number + readonly Int16: number + readonly Int32: number + readonly Int64: bigint + readonly UInt8: number + readonly UInt16: number + readonly UInt32: number + readonly UInt64: bigint + readonly Float32: number + readonly Float64: number + readonly Utf8: string + readonly Bool: boolean + readonly Date: DateTime.Utc + readonly "Timestamp[s]": DateTime.Utc + readonly "Timestamp[ms]": DateTime.Utc + readonly "Timestamp[us]": DateTime.Utc + readonly "Timestamp[ns]": DateTime.Utc + readonly "Duration[s]": Duration.Duration + readonly "Duration[ms]": Duration.Duration + readonly "Duration[us]": Duration.Duration + readonly "Duration[ns]": Duration.Duration +} + +/** + * Returns `true` for expressions. + * + * @stability experimental + * @category guards + * @since 4.1.5 + */ +export const isExpr = (u: unknown): u is Expr => internal.isExpr(u) && !internal.isAggregate(u) + +/** + * Returns `true` for aggregate expressions. + * + * @stability experimental + * @category guards + * @since 4.1.5 + */ +export const isAggregate = (u: unknown): u is Aggregate => internal.isAggregate(u) + +/** + * The expression as plan data. + * + * @stability experimental + * @category conversions + * @since 4.1.5 + */ +export const toPlan = (self: Expr | Aggregate): Plan.Expression => + (self as unknown as internal.Handle).node + +/** + * An expression from plan data, with an unknown value type. Aggregate calls + * become aggregates. + * + * @stability experimental + * @category conversions + * @since 4.1.5 + */ +export const fromPlan: { + (expression: Plan.AggregateCall): Aggregate + (expression: Plan.Expression): Expr +} = (expression: Plan.Expression): any => + expression._tag === "AggregateCall" ? internal.makeAggregate(expression) : internal.makeExpr(expression) + +// ----------------------------------------------------------------------------- +// Constructors +// ----------------------------------------------------------------------------- + +/** + * A constant. Integers between -2^31 and 2^31 - 1 adopt the integer type of + * the column they meet; other numbers are Float64; bigints are Int64. + * + * @stability experimental + * @category constructors + * @since 4.1.5 + */ +export const lit = < + const A extends number | bigint | string | boolean | DateTime.Utc | Duration.Duration | BigDecimal.BigDecimal | null +>(value: A): Expr | Nullish> => internal.makeExpr(internal.literalNode(value)) + +// ----------------------------------------------------------------------------- +// Arithmetic +// ----------------------------------------------------------------------------- + +type ArithResult = Expr | Nullish | Nullish> + +const binary = (fn: Plan.FunctionName) => + dual(2, (self: unknown, that: unknown) => internal.call(fn, [self, that])) as any + +/** + * `self + that`. Integers wrap on overflow. + * + * @stability experimental + * @category arithmetic + * @since 4.1.5 + */ +export const add: { + ( + that: Input + ): | null>(self: Expr) => ArithResult + ( + that: Input + ): ( + self: Expr + ) => Expr | Nullish | Nullish> + | null>(self: Expr, that: Input): ArithResult + ( + self: Expr, + that: Input + ): Expr | Nullish | Nullish> +} = binary("add") + +/** + * `self - that`. Integers wrap on overflow. + * + * @stability experimental + * @category arithmetic + * @since 4.1.5 + */ +export const subtract: { + ( + that: Input + ): | null>(self: Expr) => ArithResult + ( + that: Input + ): (self: Expr) => Expr | Nullish> + ( + that: Input + ): ( + self: Expr + ) => Expr | Nullish | Nullish> + | null>(self: Expr, that: Input): ArithResult + ( + self: Expr, + that: Input + ): Expr | Nullish> + ( + self: Expr, + that: Input + ): Expr | Nullish | Nullish> +} = binary("subtract") + +/** + * `self * that`. Integers wrap on overflow. + * + * @stability experimental + * @category arithmetic + * @since 4.1.5 + */ +export const multiply: { + ( + that: Input + ): | null>(self: Expr) => ArithResult + | null>(self: Expr, that: Input): ArithResult +} = binary("multiply") + +/** + * True division: always a float (`7 / 2` is `3.5`). Division by zero gives + * an infinity or NaN. + * + * @stability experimental + * @category arithmetic + * @since 4.1.5 + */ +export const divide: { + ( + that: Input + ): (self: Expr) => Expr | Nullish> + ( + self: Expr, + that: Input + ): Expr | Nullish> +} = binary("divide") + +/** + * Division rounded towards negative infinity. Integer division by zero + * gives `null`. + * + * @stability experimental + * @category arithmetic + * @since 4.1.5 + */ +export const floorDivide: { + ( + that: Input + ): | null>(self: Expr) => Expr | null> + | null>(self: Expr, that: Input): Expr | null> +} = binary("floorDivide") + +/** + * The remainder of {@link floorDivide}: it has the sign of the divisor + * (`-7 mod 3` is `2`). Integer division by zero gives `null`. + * + * @stability experimental + * @category arithmetic + * @since 4.1.5 + */ +export const mod: { + ( + that: Input + ): | null>(self: Expr) => Expr | null> + | null>(self: Expr, that: Input): Expr | null> +} = binary("mod") + +/** + * `-self`. + * + * @stability experimental + * @category arithmetic + * @since 4.1.5 + */ +export const negate = (self: Expr): Expr => internal.call("negate", [self]) + +/** + * The absolute value. + * + * @stability experimental + * @category arithmetic + * @since 4.1.5 + */ +export const abs = (self: Expr): Expr => internal.call("abs", [self]) + +/** + * The square root, as a float. + * + * @stability experimental + * @category arithmetic + * @since 4.1.5 + */ +export const sqrt = (self: Expr): Expr> => internal.call("sqrt", [self]) + +/** + * Rounds to `decimals` places (default 0), halves to even. + * + * @stability experimental + * @category arithmetic + * @since 4.1.5 + */ +export const round: { + (decimals?: number): (self: Expr) => Expr + (self: Expr, decimals?: number): Expr +} = dual( + (args) => isExpr(args[0]), + (self: Expr, decimals = 0) => internal.call("round", [self], { decimals }) +) as any + +// ----------------------------------------------------------------------------- +// Comparisons +// ----------------------------------------------------------------------------- + +type Comparison = { + (that: Input): | null>(self: Expr) => Expr | Nullish> + | null>(self: Expr, that: Input): Expr | Nullish> +} + +/** + * `self === that`. `null` when either side is `null`; `NaN` equals `NaN`. + * + * @stability experimental + * @category comparisons + * @since 4.1.5 + */ +export const eq: Comparison = binary("eq") + +/** + * `self !== that`. + * + * @stability experimental + * @category comparisons + * @since 4.1.5 + */ +export const neq: Comparison = binary("neq") + +/** + * `self < that`. + * + * @stability experimental + * @category comparisons + * @since 4.1.5 + */ +export const lt: Comparison = binary("lt") + +/** + * `self <= that`. + * + * @stability experimental + * @category comparisons + * @since 4.1.5 + */ +export const lte: Comparison = binary("lte") + +/** + * `self > that`. + * + * @stability experimental + * @category comparisons + * @since 4.1.5 + */ +export const gt: Comparison = binary("gt") + +/** + * `self >= that`. + * + * @stability experimental + * @category comparisons + * @since 4.1.5 + */ +export const gte: Comparison = binary("gte") + +/** + * `lower <= self && self <= upper`. + * + * @stability experimental + * @category comparisons + * @since 4.1.5 + */ +export const between: { + (lower: Input, upper: Input): | null>( + self: Expr + ) => Expr | Nullish | Nullish> + | null, C extends Base | null>( + self: Expr, + lower: Input, + upper: Input + ): Expr | Nullish | Nullish> +} = dual(3, (self: unknown, lower: unknown, upper: unknown) => internal.call("between", [self, lower, upper])) as any + +/** + * Whether the value is one of `values`. + * + * @stability experimental + * @category comparisons + * @since 4.1.5 + */ +export const isIn: { + (values: ReadonlyArray): | null>(self: Expr) => Expr> + (self: Expr, values: ReadonlyArray>): Expr> +} = dual(2, (self: unknown, values: ReadonlyArray) => internal.call("isIn", [self], { values })) as any + +// ----------------------------------------------------------------------------- +// Logic +// ----------------------------------------------------------------------------- + +/** + * Three-valued AND: `false` if either side is `false`, otherwise `null` if + * either side is `null`. + * + * @stability experimental + * @category logic + * @since 4.1.5 + */ +export const and: { + ( + that: Input + ): (self: Expr) => Expr | Nullish> + ( + self: Expr, + that: Input + ): Expr | Nullish> +} = binary("and") + +/** + * Three-valued OR: `true` if either side is `true`, otherwise `null` if + * either side is `null`. + * + * @stability experimental + * @category logic + * @since 4.1.5 + */ +export const or: { + ( + that: Input + ): (self: Expr) => Expr | Nullish> + ( + self: Expr, + that: Input + ): Expr | Nullish> +} = binary("or") + +/** + * Negation; `null` stays `null`. + * + * @stability experimental + * @category logic + * @since 4.1.5 + */ +export const not = (self: Expr): Expr => internal.call("not", [self]) + +// ----------------------------------------------------------------------------- +// Missing values +// ----------------------------------------------------------------------------- + +/** + * Whether the value is missing. + * + * @stability experimental + * @category missing values + * @since 4.1.5 + */ +export const isNull = (self: Expr): Expr => internal.call("isNull", [self]) + +/** + * Whether the value is present. + * + * @stability experimental + * @category missing values + * @since 4.1.5 + */ +export const isNotNull = (self: Expr): Expr => internal.call("isNotNull", [self]) + +/** + * Whether the value is `NaN` (`false` for integers; `null` stays `null`). + * + * @stability experimental + * @category missing values + * @since 4.1.5 + */ +export const isNaN = (self: Expr): Expr> => + internal.call("isNaN", [self]) + +/** + * Replaces missing values with `value`. + * + * @stability experimental + * @category missing values + * @since 4.1.5 + */ +export const fillNull: { + (value: Input): | null>(self: Expr) => Expr | Base | Nullish> + | null>(self: Expr, value: Input): Expr | Base | Nullish> +} = binary("fillNull") + +type Coalesced> = Expr> | AllNullable> +type AllNullable> = Es extends readonly [infer H, ...infer T] + ? null extends InputType ? AllNullable : never + : null + +/** + * The first argument that is not `null`, per row. + * + * @stability experimental + * @category missing values + * @since 4.1.5 + */ +export const coalesce = , ...Array>]>(...exprs: Es): Coalesced => + internal.call("coalesce", exprs) + +/** + * `then` where `condition` is `true`, `otherwise` where it is `false` or + * `null`. + * + * @stability experimental + * @category conditionals + * @since 4.1.5 + */ +export const when = | null>( + condition: Expr, + then: Input, + otherwise: Input +): Expr | Nullish | Base | Nullish> => internal.call("when", [condition, then, otherwise]) + +// ----------------------------------------------------------------------------- +// Casts +// ----------------------------------------------------------------------------- + +/** + * Converts to another type. Values that don't fit (out of range, unparsable + * strings, NaN to an integer) become `null`; floats truncate towards zero + * when converted to integers. + * + * @stability experimental + * @category conversions + * @since 4.1.5 + */ +export const cast: { + (to: To): (self: Expr) => Expr + (self: Expr, to: To): Expr +} = dual(2, (self: unknown, to: string) => internal.call("cast", [self], { to })) as any + +/** + * Converts to a decimal with `precision` digits, `scale` of them after the + * point. Values that don't fit become null; extra digits round half to even. + * + * @stability experimental + * @category casts + * @since 4.1.5 + */ +export const castDecimal: { + (precision: number, scale: number): (self: Expr) => Expr + (self: Expr, precision: number, scale: number): Expr +} = dual( + 3, + (self: unknown, precision: number, scale: number) => + internal.call("cast", [self], { to: "Decimal", precision, scale }) +) as any + +// ----------------------------------------------------------------------------- +// Strings +// ----------------------------------------------------------------------------- + +/** + * The number of Unicode code points. + * + * @stability experimental + * @category strings + * @since 4.1.5 + */ +export const length = (self: Expr): Expr> => + internal.call("length", [self]) + +/** + * Lower case. + * + * @stability experimental + * @category strings + * @since 4.1.5 + */ +export const toLowerCase = (self: Expr): Expr => internal.call("toLowerCase", [self]) + +/** + * Upper case. + * + * @stability experimental + * @category strings + * @since 4.1.5 + */ +export const toUpperCase = (self: Expr): Expr => internal.call("toUpperCase", [self]) + +const pattern = (fn: "includes" | "startsWith" | "endsWith") => + dual(2, (self: unknown, text: string) => internal.call(fn, [self], { pattern: text })) as any + +/** + * Whether the string contains `text` (literally). + * + * @stability experimental + * @category strings + * @since 4.1.5 + */ +export const includes: { + (text: string): (self: Expr) => Expr> + (self: Expr, text: string): Expr> +} = pattern("includes") + +/** + * Whether the string starts with `text`. + * + * @stability experimental + * @category strings + * @since 4.1.5 + */ +export const startsWith: { + (text: string): (self: Expr) => Expr> + (self: Expr, text: string): Expr> +} = pattern("startsWith") + +/** + * Whether the string ends with `text`. + * + * @stability experimental + * @category strings + * @since 4.1.5 + */ +export const endsWith: { + (text: string): (self: Expr) => Expr> + (self: Expr, text: string): Expr> +} = pattern("endsWith") + +/** + * Concatenates the arguments as strings; `null` if any argument is `null`. + * + * @stability experimental + * @category strings + * @since 4.1.5 + */ +export const concat = , ...Array>]>( + ...parts: Es +): Expr> }[number]> => internal.call("concat", parts) + +// ----------------------------------------------------------------------------- +// Sorting +// ----------------------------------------------------------------------------- + +/** + * Sorts by `self`, smallest first. + * + * @stability experimental + * @category sorting + * @since 4.1.5 + */ +export const asc = (self: Expr): SortOrder => ({ expr: self, descending: false }) + +/** + * Sorts by `self`, largest first. + * + * @stability experimental + * @category sorting + * @since 4.1.5 + */ +export const desc = (self: Expr): SortOrder => ({ expr: self, descending: true }) + +// ----------------------------------------------------------------------------- +// Aggregates +// ----------------------------------------------------------------------------- + +const agg = (fn: Plan.AggregateName, self?: unknown) => + internal.makeAggregate({ + _tag: "AggregateCall", + fn, + arg: self === undefined ? undefined : internal.toNode(self) + }) + +/** + * The sum of the values; 0 for a group without values. Sums of 8- and 16-bit + * integers are Int32. + * + * @stability experimental + * @category aggregates + * @since 4.1.5 + */ +export const sum = ( + self: Expr +): Aggregate> => agg("sum", self) + +/** + * The mean of the values; `null` for a group without values. + * + * @stability experimental + * @category aggregates + * @since 4.1.5 + */ +export const mean = (self: Expr): Aggregate => agg("mean", self) + +/** + * The median of the values (the mean of the middle two for an even count). + * + * @stability experimental + * @category aggregates + * @since 4.1.5 + */ +export const median = (self: Expr): Aggregate => agg("median", self) + +/** + * The sample standard deviation (`n - 1` denominator); `null` for fewer than + * two values. + * + * @stability experimental + * @category aggregates + * @since 4.1.5 + */ +export const std = (self: Expr): Aggregate => agg("std", self) + +/** + * The sample variance (`n - 1` denominator); `null` for fewer than two + * values. + * + * @stability experimental + * @category aggregates + * @since 4.1.5 + */ +export const variance = (self: Expr): Aggregate => agg("variance", self) + +/** + * The smallest value. Float `NaN`s are skipped unless every value is `NaN`. + * + * @stability experimental + * @category aggregates + * @since 4.1.5 + */ +export const min = (self: Expr): Aggregate | null> => agg("min", self) + +/** + * The largest value. Float `NaN`s are skipped unless every value is `NaN`. + * + * @stability experimental + * @category aggregates + * @since 4.1.5 + */ +export const max = (self: Expr): Aggregate | null> => agg("max", self) + +/** + * Without an argument: the number of rows. With one: the number of rows + * where the expression is not `null`. + * + * @stability experimental + * @category aggregates + * @since 4.1.5 + */ +export const count = (self?: Expr): Aggregate => agg("count", self) + +/** + * The number of distinct values, `null` included. + * + * @stability experimental + * @category aggregates + * @since 4.1.5 + */ +export const countDistinct = (self: Expr): Aggregate => agg("countDistinct", self) + +/** + * The value in the group's first row (`null` if the group is empty). + * + * @stability experimental + * @category aggregates + * @since 4.1.5 + */ +export const first = (self: Expr): Aggregate => agg("first", self) + +/** + * The value in the group's last row (`null` if the group is empty). + * + * @stability experimental + * @category aggregates + * @since 4.1.5 + */ +export const last = (self: Expr): Aggregate => agg("last", self) + +// ----------------------------------------------------------------------------- +// Windows +// ----------------------------------------------------------------------------- + +/** + * Evaluates `self` separately within each partition of rows with equal + * `partitionBy` values. An aggregate gives its partition's value on every + * row; a window function (`cumSum`, `shift`, `rank`, the rolling functions) + * runs over its partition's rows in row order. + * + * **Example** (Share of each city's total) + * + * ```ts import.meta.vitest + * import { Effect } from "effect" + * import { DataFrame, Expr, Query } from "effect/dataframe" + * + * const sales = DataFrame.fromColumns({ city: ["Vienna", "Berlin", "Vienna"], units: [2, 5, 6] }) + * + * const rows = Query.from(sales).pipe( + * Query.withColumns((c) => ({ + * share: Expr.divide(c.units, Expr.over(Expr.sum(c.units), [c.city])), + * running: Expr.over(Expr.cumSum(c.units), [c.city]) + * })), + * Effect.map(DataFrame.toRows), + * Effect.runSync + * ) + * rows[2] // => { city: "Vienna", units: 6, share: 0.75, running: 8 } + * ``` + * + * @stability experimental + * @category windows + * @since 4.1.5 + */ +export const over: { + (partitionBy: ReadonlyArray>): (self: Expr | Aggregate) => Expr + (self: Expr | Aggregate, partitionBy: ReadonlyArray>): Expr +} = dual( + 2, + (self: unknown, partitionBy: ReadonlyArray) => + internal.makeExpr({ + _tag: "Over", + expression: internal.toNode(self), + partitionBy: partitionBy.map(internal.toNode) + }) +) as any + +type CumSum = Expr | Nullish> + +/** + * The running sum: on each row, the sum of the values so far (in the + * partition, with {@link over}). Nulls stay null and are skipped; NaN + * propagates. + * + * @stability experimental + * @category windows + * @since 4.1.5 + */ +export const cumSum = (self: Expr): CumSum => internal.call("cumSum", [self]) + +/** + * The running minimum. Nulls stay null; NaN is skipped. + * + * @stability experimental + * @category windows + * @since 4.1.5 + */ +export const cumMin = (self: Expr): Expr => internal.call("cumMin", [self]) + +/** + * The running maximum. Nulls stay null; NaN is skipped. + * + * @stability experimental + * @category windows + * @since 4.1.5 + */ +export const cumMax = (self: Expr): Expr => internal.call("cumMax", [self]) + +/** + * The number of non-null values so far. + * + * @stability experimental + * @category windows + * @since 4.1.5 + */ +export const cumCount = (self: Expr): Expr => internal.call("cumCount", [self]) + +/** + * The value `n` rows earlier (1 by default); a negative `n` looks ahead. + * Null where there is no such row. + * + * @stability experimental + * @category windows + * @since 4.1.5 + */ +export const shift: { + (n?: number): (self: Expr) => Expr + (self: Expr, n?: number): Expr +} = dual( + (args) => isExpr(args[0]), + (self: unknown, n = 1) => internal.call("shift", [self], { n }) +) as any + +/** + * The difference to the value `n` rows earlier (1 by default). + * + * @stability experimental + * @category windows + * @since 4.1.5 + */ +export const diff: { + (n?: number): (self: Expr) => Expr + (self: Expr, n?: number): Expr +} = dual( + (args) => isExpr(args[0]), + (self: unknown, n = 1) => internal.call("diff", [self], { n }) +) as any + +/** + * The rank of each value, 1 for the smallest (largest with `descending`). + * Ties get the mean of their positions by default; `method` picks `min`, + * `max`, `dense` (consecutive) or `ordinal` (by row order). Nulls get no + * rank; NaN ranks above every number. + * + * @stability experimental + * @category windows + * @since 4.1.5 + */ +export const rank: { + (options?: { + readonly method?: Plan.RankMethod | undefined + readonly descending?: boolean | undefined + }): (self: Expr) => Expr> + (self: Expr, options?: { + readonly method?: Plan.RankMethod | undefined + readonly descending?: boolean | undefined + }): Expr> +} = dual( + (args) => isExpr(args[0]), + (self: unknown, options?: { readonly method?: Plan.RankMethod; readonly descending?: boolean }) => + internal.call("rank", [self], { + method: options?.method ?? "average", + descending: options?.descending ?? false + }) +) as any + +type Rolling = { + (window: number, options?: { readonly minPeriods?: number | undefined }): ( + self: Expr + ) => Expr | null> + ( + self: Expr, + window: number, + options?: { readonly minPeriods?: number | undefined } + ): Expr | null> +} + +const rolling = (fn: "rollingSum" | "rollingMean" | "rollingMin" | "rollingMax"): Rolling => + dual( + (args) => isExpr(args[0]), + (self: unknown, window: number, options?: { readonly minPeriods?: number | undefined }) => + internal.call(fn, [self], { window, minPeriods: options?.minPeriods ?? window }) + ) as any + +/** + * The sum of the last `window` values (the current one included). Null + * while fewer than `minPeriods` (default: `window`) of them are non-null. + * + * @stability experimental + * @category windows + * @since 4.1.5 + */ +export const rollingSum: Rolling = rolling("rollingSum") + +/** + * The mean of the last `window` values; see {@link rollingSum}. + * + * @stability experimental + * @category windows + * @since 4.1.5 + */ +export const rollingMean: { + (window: number, options?: { readonly minPeriods?: number | undefined }): ( + self: Expr + ) => Expr + ( + self: Expr, + window: number, + options?: { readonly minPeriods?: number | undefined } + ): Expr +} = rolling("rollingMean") as any + +/** + * The minimum of the last `window` values; see {@link rollingSum}. + * + * @stability experimental + * @category windows + * @since 4.1.5 + */ +export const rollingMin: Rolling = rolling("rollingMin") + +/** + * The maximum of the last `window` values; see {@link rollingSum}. + * + * @stability experimental + * @category windows + * @since 4.1.5 + */ +export const rollingMax: Rolling = rolling("rollingMax") + +// ----------------------------------------------------------------------------- +// Dates and times +// ----------------------------------------------------------------------------- + +type Field = (self: Expr) => Expr> +const datePart = (fn: Plan.FunctionName): Field => (self) => internal.call(fn, [self]) + +/** + * The calendar year of a date or timestamp (UTC). + * + * **Example** (Calendar fields) + * + * ```ts import.meta.vitest + * import { DateTime, Effect } from "effect" + * import { DataFrame, Expr, Query } from "effect/dataframe" + * + * const events = DataFrame.fromColumns({ at: [DateTime.makeUnsafe("2024-02-29T13:45:07Z")] }) + * + * const rows = Query.from(events).pipe( + * Query.select((c) => ({ year: Expr.year(c.at), weekday: Expr.weekday(c.at), hour: Expr.hour(c.at) })), + * Effect.map(DataFrame.toRows), + * Effect.runSync + * ) + * rows // => [{ year: 2024, weekday: 4, hour: 13 }] + * ``` + * + * @stability experimental + * @category dates and times + * @since 4.1.5 + */ +export const year: Field = datePart("year") + +/** + * The month (1–12) of a date or timestamp. + * + * @stability experimental + * @category dates and times + * @since 4.1.5 + */ +export const month: Field = datePart("month") + +/** + * The day of the month (1–31) of a date or timestamp. + * + * @stability experimental + * @category dates and times + * @since 4.1.5 + */ +export const day: Field = datePart("day") + +/** + * The hour (0–23) of a timestamp. + * + * @stability experimental + * @category dates and times + * @since 4.1.5 + */ +export const hour: Field = datePart("hour") + +/** + * The minute (0–59) of a timestamp. + * + * @stability experimental + * @category dates and times + * @since 4.1.5 + */ +export const minute: Field = datePart("minute") + +/** + * The second (0–59) of a timestamp. + * + * @stability experimental + * @category dates and times + * @since 4.1.5 + */ +export const second: Field = datePart("second") + +/** + * The millisecond within the second (0–999) of a timestamp. + * + * @stability experimental + * @category dates and times + * @since 4.1.5 + */ +export const millisecond: Field = datePart("millisecond") + +/** + * The ISO weekday of a date or timestamp: 1 for Monday to 7 for Sunday. + * + * @stability experimental + * @category dates and times + * @since 4.1.5 + */ +export const weekday: Field = datePart("weekday") + +/** + * The day of the year (1–366) of a date or timestamp. + * + * @stability experimental + * @category dates and times + * @since 4.1.5 + */ +export const ordinalDay: Field = datePart("ordinalDay") + +/** + * Rounds dates and timestamps down to a multiple of `every`: `"30s"`, + * `"15m"`, `"1h"`, `"1d"` (from the epoch), `"1w"` (weeks start on Monday), + * `"1mo"`, `"3mo"` or `"1y"` (calendar months and years). + * + * @stability experimental + * @category dates and times + * @since 4.1.5 + */ +export const truncate: { + (every: string): (self: Expr) => Expr + (self: Expr, every: string): Expr +} = dual(2, (self: unknown, every: string) => internal.call("truncate", [self], { every })) as any + +type Total = (self: Expr) => Expr> +const total = (fn: Plan.FunctionName): Total => (self) => internal.call(fn, [self]) + +/** + * A duration in whole days, rounded toward zero. + * + * @stability experimental + * @category dates and times + * @since 4.1.5 + */ +export const totalDays: Total = total("totalDays") + +/** + * A duration in whole hours, rounded toward zero. + * + * @stability experimental + * @category dates and times + * @since 4.1.5 + */ +export const totalHours: Total = total("totalHours") + +/** + * A duration in whole minutes, rounded toward zero. + * + * @stability experimental + * @category dates and times + * @since 4.1.5 + */ +export const totalMinutes: Total = total("totalMinutes") + +/** + * A duration in whole seconds, rounded toward zero. + * + * @stability experimental + * @category dates and times + * @since 4.1.5 + */ +export const totalSeconds: Total = total("totalSeconds") + +/** + * A duration in whole milliseconds, rounded toward zero. + * + * @stability experimental + * @category dates and times + * @since 4.1.5 + */ +export const totalMilliseconds: Total = total("totalMilliseconds") + +// ----------------------------------------------------------------------------- +// Lists and structs +// ----------------------------------------------------------------------------- + +type Element = NonNullable extends ReadonlyArray ? E : never + +/** + * The number of elements of a list; null for a null list. + * + * **Example** (Working with list columns) + * + * ```ts import.meta.vitest + * import { Effect, Schema } from "effect" + * import { DataFrame, Expr, Query } from "effect/dataframe" + * + * const Order = Schema.Struct({ id: Schema.String, items: Schema.Array(Schema.Number) }) + * const orders = DataFrame.fromRows(Order, [{ id: "a", items: [3, 5] }, { id: "b", items: [] }]) + * + * const rows = Query.from(orders).pipe( + * Query.select((c) => ({ id: c.id, n: Expr.listLength(c.items), total: Expr.listSum(c.items) })), + * Effect.map(DataFrame.toRows), + * Effect.runSync + * ) + * rows // => [{ id: "a", n: 2, total: 8 }, { id: "b", n: 0, total: 0 }] + * ``` + * + * @stability experimental + * @category lists and structs + * @since 4.1.5 + */ +export const listLength = | null>(self: Expr): Expr> => + internal.call("listLength", [self]) + +/** + * The element at `index` (negative counts from the end); null when the + * list is null or too short. + * + * @stability experimental + * @category lists and structs + * @since 4.1.5 + */ +export const listGet: { + (index: number): | null>(self: Expr) => Expr | null> + | null>(self: Expr, index: number): Expr | null> +} = dual(2, (self: unknown, index: number) => internal.call("listGet", [self], { index })) as any + +/** + * The sum of each list's elements: 0 for an empty list, null for a null + * list. + * + * @stability experimental + * @category lists and structs + * @since 4.1.5 + */ +export const listSum = | null>( + self: Expr +): Expr> | Nullish> => internal.call("listSum", [self]) + +/** + * The mean of each list's elements; null for an empty or null list. + * + * @stability experimental + * @category lists and structs + * @since 4.1.5 + */ +export const listMean = | null>( + self: Expr +): Expr => internal.call("listMean", [self]) + +/** + * The smallest element of each list; null for an empty or null list. + * + * @stability experimental + * @category lists and structs + * @since 4.1.5 + */ +export const listMin = | null>(self: Expr): Expr> | null> => + internal.call("listMin", [self]) + +/** + * The largest element of each list; null for an empty or null list. + * + * @stability experimental + * @category lists and structs + * @since 4.1.5 + */ +export const listMax = | null>(self: Expr): Expr> | null> => + internal.call("listMax", [self]) + +/** + * Whether each list holds `item`; null for a null list. + * + * @stability experimental + * @category lists and structs + * @since 4.1.5 + */ +export const listContains: { + (item: Input): | null>(self: Expr) => Expr> + | null>(self: Expr, item: Input>>): Expr> +} = dual(2, (self: unknown, item: unknown) => internal.call("listContains", [self, item])) as any + +/** + * A field of a struct; null where the struct is null. + * + * @stability experimental + * @category lists and structs + * @since 4.1.5 + */ +export const field: { + & string>( + name: K + ): (self: Expr) => Expr[K] | Nullish> + & string>( + self: Expr, + name: K + ): Expr[K] | Nullish> +} = dual(2, (self: unknown, name: string) => internal.call("field", [self], { name })) as any + +/** + * A struct of the given expressions; the record's keys name its fields. + * + * @stability experimental + * @category lists and structs + * @since 4.1.5 + */ +export const struct = }>( + fields: O +): Expr<{ readonly [K in keyof O]: Type }> => + internal.call("struct", Object.values(fields), { names: Object.keys(fields) }) + +/** + * Each group's values as a list, in row order. + * + * @stability experimental + * @category aggregates + * @since 4.1.5 + */ +export const list = (self: Expr): Aggregate> => agg("list", self) diff --git a/packages/effect/src/dataframe/Plan.ts b/packages/effect/src/dataframe/Plan.ts new file mode 100644 index 00000000000..09cbeeef283 --- /dev/null +++ b/packages/effect/src/dataframe/Plan.ts @@ -0,0 +1,1103 @@ +/** + * Queries as plain data: the nodes a `Query` is made of, and schemas + * that validate them and encode them to JSON. + * + * **Mental model** + * + * - A plan is a tree of {@link Node}s. Leaves are inputs ({@link Values} + * holds a frame, {@link Table} names a table the `Catalog` provides); + * every other node transforms the rows of its input(s). + * - Column values are computed by {@link Expression}s: column references, + * literals, function calls and aggregate calls. Function and aggregate + * names are the names of the `Expr` constructors (`add`, `gt`, `mean`, …). + * - Plans are what `Query` builds, what `Query.explain` prints and what + * `Query.fromPlan` runs. Building them from data (a request body, a stored + * report definition, a tool call) needs no TypeScript types: decode with + * {@link Node} or {@link NodeFromJson}, then run. + * + * **Example** (A plan from JSON) + * + * ```ts import.meta.vitest + * import { Effect, Schema } from "effect" + * import { Catalog, DataFrame, Plan, Query } from "effect/dataframe" + * + * const json = { + * _tag: "Filter", + * input: { _tag: "Table", name: "readings" }, + * predicate: { + * _tag: "Call", + * fn: "gt", + * args: [{ _tag: "Column", name: "temp" }, { _tag: "Literal", value: 20 }] + * } + * } + * + * const plan = Schema.decodeUnknownSync(Plan.NodeFromJson)(json) + * const readings = DataFrame.fromColumns({ temp: [18, 21.5, 24] }) + * + * const result = Query.fromPlan(plan).pipe( + * Effect.provide(Catalog.layer({ readings })), + * Effect.map(DataFrame.toRows), + * Effect.runSync + * ) + * result // => [{ temp: 21.5 }, { temp: 24 }] + * ``` + * + * **Details** + * + * In JSON, literal values are JSON values where that is unambiguous: strings, + * booleans, `null` and finite numbers. A `bigint` is written as + * `{ "bigint": "" }` and `NaN`, `Infinity`, `-Infinity` and `-0` as + * `{ "float": "NaN" }` and so on. A {@link Values} frame is written as a + * base64 Arrow IPC file. + * + * @stability experimental + * @since 4.1.5 + */ +import * as ArrowIpc from "../arrow/ArrowIpc.ts" +import * as Effect from "../Effect.ts" +import * as Result from "../Result.ts" +import * as Schema from "../Schema.ts" +import * as SchemaIssue from "../SchemaIssue.ts" +import * as SchemaTransformation from "../SchemaTransformation.ts" +import type { DataFrame } from "./DataFrame.ts" +import * as F from "./internal/frame.ts" + +// ----------------------------------------------------------------------------- +// Values +// ----------------------------------------------------------------------------- + +/** + * A literal value. + * + * @stability experimental + * @category models + * @since 4.1.5 + */ +export type Value = number | bigint | string | boolean | null + +const isValue = (u: unknown): u is Value => + u === null || typeof u === "number" || typeof u === "bigint" || typeof u === "string" || typeof u === "boolean" + +const FloatName = Schema.Literals(["NaN", "Infinity", "-Infinity", "-0"]) + +const ValueJson = Schema.Union([ + Schema.String, + Schema.Boolean, + Schema.Null, + Schema.Finite, + Schema.Struct({ bigint: Schema.String }), + Schema.Struct({ float: FloatName }) +]) + +const floatValues: Record = { + NaN: NaN, + Infinity: Infinity, + "-Infinity": -Infinity, + "-0": -0 +} + +const valueFromJson: SchemaTransformation.Transformation = SchemaTransformation + .transformEffect({ + decode: (json, options): Effect.Effect => { + if (json === null || typeof json !== "object") return Effect.succeed(json) + if ("float" in json) return Effect.succeed(floatValues[json.float]) + try { + return Effect.succeed(BigInt(json.bigint)) + } catch { + return Effect.fail(new SchemaIssue.InvalidValue({ expected: "an integer" }, json, options)) + } + }, + encode: (value) => { + if (typeof value === "bigint") return Effect.succeed({ bigint: value.toString() }) + if (typeof value === "number" && (!Number.isFinite(value) || Object.is(value, -0))) { + return Effect.succeed({ + float: Number.isNaN(value) ? "NaN" as const : value === Infinity ? "Infinity" as const : value === -Infinity + ? "-Infinity" as const + : "-0" as const + }) + } + return Effect.succeed(value as string | number | boolean | null) + } + }) + +/** + * Schema of {@link Value}. + * + * @stability experimental + * @category schemas + * @since 4.1.5 + */ +export const Value: Schema.declare = Schema.declare(isValue, { + expected: "number | bigint | string | boolean | null", + toCodecJson: () => Schema.link()(ValueJson, valueFromJson) +}) + +const isDataFrame = (u: unknown): u is DataFrame => F.isFrame(u) + +const frameFromBytes: SchemaTransformation.Transformation, Uint8Array> = SchemaTransformation + .transformEffect({ + decode: (bytes, options) => { + const table = ArrowIpc.decodeFile(bytes) + if (Result.isFailure(table)) { + return Effect.fail(new SchemaIssue.InvalidValue({ expected: "an Arrow IPC file" }, bytes, options)) + } + try { + return Effect.succeed(F.fromArrowTable(table.success) as unknown as DataFrame) + } catch (cause) { + return Effect.fail(new SchemaIssue.InvalidValue({ expected: `a supported table: ${String(cause)}` }, bytes)) + } + }, + encode: (frame, options) => { + const bytes = ArrowIpc.encodeFile(F.toArrowTable(frame as unknown as F.Frame)) + return Result.isSuccess(bytes) + ? Effect.succeed(bytes.success) + : Effect.fail(new SchemaIssue.InvalidValue({ expected: "a frame Arrow can encode" }, frame, options)) + } + }) + +/** + * Schema of a {@link DataFrame} inside a plan. Its JSON form is a base64 + * Arrow IPC file. + * + * @stability experimental + * @category schemas + * @since 4.1.5 + */ +export const Frame: Schema.declare> = Schema.declare(isDataFrame, { + expected: "DataFrame", + toCodecJson: () => Schema.link>()(Schema.Uint8ArrayFromBase64, frameFromBytes) +}) + +// ----------------------------------------------------------------------------- +// Expressions +// ----------------------------------------------------------------------------- + +/** + * Names of the functions a {@link Call} can apply; each is the `Expr` + * constructor of the same name. + * + * @stability experimental + * @category models + * @since 4.1.5 + */ +export type FunctionName = typeof FunctionName.Type + +/** + * Schema of {@link FunctionName}. + * + * @stability experimental + * @category schemas + * @since 4.1.5 + */ +export const FunctionName = Schema.Literals([ + "add", + "subtract", + "multiply", + "divide", + "floorDivide", + "mod", + "negate", + "abs", + "sqrt", + "round", + "eq", + "neq", + "lt", + "lte", + "gt", + "gte", + "between", + "isIn", + "and", + "or", + "not", + "isNull", + "isNotNull", + "isNaN", + "fillNull", + "coalesce", + "when", + "cast", + "length", + "toLowerCase", + "toUpperCase", + "includes", + "startsWith", + "endsWith", + "concat", + "cumSum", + "cumMin", + "cumMax", + "cumCount", + "shift", + "diff", + "rank", + "rollingSum", + "rollingMean", + "rollingMin", + "rollingMax", + "year", + "month", + "day", + "hour", + "minute", + "second", + "millisecond", + "weekday", + "ordinalDay", + "truncate", + "totalDays", + "totalHours", + "totalMinutes", + "totalSeconds", + "totalMilliseconds", + "listLength", + "listGet", + "listSum", + "listMean", + "listMin", + "listMax", + "listContains", + "field", + "struct" +]) + +/** + * Names of the aggregates an {@link AggregateCall} can compute; each is the + * `Expr` constructor of the same name. `count` without an argument counts + * rows. + * + * @stability experimental + * @category models + * @since 4.1.5 + */ +export type AggregateName = typeof AggregateName.Type + +/** + * Schema of {@link AggregateName}. + * + * @stability experimental + * @category schemas + * @since 4.1.5 + */ +export const AggregateName = Schema.Literals([ + "sum", + "mean", + "median", + "std", + "variance", + "min", + "max", + "count", + "countDistinct", + "first", + "last", + "list" +]) + +/** + * Names of the types `cast` converts to. + * + * @stability experimental + * @category models + * @since 4.1.5 + */ +export type CastTarget = typeof CastTarget.Type + +/** + * Schema of {@link CastTarget}. + * + * @stability experimental + * @category schemas + * @since 4.1.5 + */ +export const CastTarget = Schema.Literals([ + "Int8", + "Int16", + "Int32", + "Int64", + "UInt8", + "UInt16", + "UInt32", + "UInt64", + "Float32", + "Float64", + "Utf8", + "Bool", + "Date", + "Timestamp[s]", + "Timestamp[ms]", + "Timestamp[us]", + "Timestamp[ns]", + "Duration[s]", + "Duration[ms]", + "Duration[us]", + "Duration[ns]", + "Decimal" +]) + +/** + * The type of a {@link Literal} whose value is the integer storage of a + * richer type: days since the epoch for `Date`, units since the epoch for + * `Timestamp`, units for `Duration`, and the unscaled integer for `Decimal`. + * + * @stability experimental + * @category models + * @since 4.1.5 + */ +export type LiteralType = + | { readonly _tag: "Date" } + | { readonly _tag: "Timestamp"; readonly unit: TimeUnit; readonly timezone?: string | undefined } + | { readonly _tag: "Duration"; readonly unit: TimeUnit } + | { readonly _tag: "Decimal"; readonly precision: number; readonly scale: number } + +/** + * Units of timestamps and durations. + * + * @stability experimental + * @category models + * @since 4.1.5 + */ +export type TimeUnit = typeof TimeUnit.Type + +/** + * Schema of {@link TimeUnit}. + * + * @stability experimental + * @category schemas + * @since 4.1.5 + */ +export const TimeUnit = Schema.Literals(["second", "millisecond", "microsecond", "nanosecond"]) + +const LiteralTypeSchema: Schema.Codec = Schema.Union([ + Schema.Struct({ _tag: Schema.tag("Date") }), + Schema.Struct({ _tag: Schema.tag("Timestamp"), unit: TimeUnit, timezone: Schema.optional(Schema.String) }), + Schema.Struct({ _tag: Schema.tag("Duration"), unit: TimeUnit }), + Schema.Struct({ _tag: Schema.tag("Decimal"), precision: Schema.Int, scale: Schema.Int }) +]) as any + +/** + * A value computed per row (or per group, for aggregate calls). + * + * @stability experimental + * @category models + * @since 4.1.5 + */ +export type Expression = Column | Literal | Call | AggregateCall | Over + +/** + * The value of a column. + * + * @stability experimental + * @category models + * @since 4.1.5 + */ +export interface Column { + readonly _tag: "Column" + readonly name: string +} + +/** + * A constant. Integral numbers take the integer type of the column they meet, + * as Polars integer literals do. + * + * @stability experimental + * @category models + * @since 4.1.5 + */ +export interface Literal { + readonly _tag: "Literal" + readonly value: Value + readonly type?: LiteralType | undefined +} + +/** + * Options of the functions that take more than expressions: `round` + * (`decimals`), `cast` (`to`), `isIn` (`values`), `includes`, `startsWith` + * and `endsWith` (`pattern`), `shift` and `diff` (`n`), `rank` (`method`, + * `descending`), the rolling functions (`window`, `minPeriods`) and + * `truncate` (`every`: `"1h"`, `"1d"`, `"1w"`, `"1mo"`, `"1y"`, …), `listGet` + * (`index`), `field` (`name`), `struct` (`names`, one per argument) and + * `cast` to `Decimal` (`precision`, `scale`). + * + * @stability experimental + * @category models + * @since 4.1.5 + */ +export interface CallOptions { + readonly decimals?: number | undefined + readonly to?: CastTarget | undefined + readonly values?: ReadonlyArray | undefined + readonly pattern?: string | undefined + readonly n?: number | undefined + readonly method?: RankMethod | undefined + readonly descending?: boolean | undefined + readonly window?: number | undefined + readonly minPeriods?: number | undefined + readonly every?: string | undefined + readonly index?: number | undefined + readonly name?: string | undefined + readonly names?: ReadonlyArray | undefined + readonly precision?: number | undefined + readonly scale?: number | undefined +} + +/** + * How `rank` numbers ties: `average` (the mean of their positions, a float), + * `min`, `max`, `dense` (consecutive ranks) or `ordinal` (by row order). + * + * @stability experimental + * @category models + * @since 4.1.5 + */ +export type RankMethod = typeof RankMethod.Type + +/** + * Schema of {@link RankMethod}. + * + * @stability experimental + * @category schemas + * @since 4.1.5 + */ +export const RankMethod = Schema.Literals(["average", "min", "max", "dense", "ordinal"]) + +/** + * A function applied to argument expressions, row by row. + * + * @stability experimental + * @category models + * @since 4.1.5 + */ +export interface Call { + readonly _tag: "Call" + readonly fn: FunctionName + readonly args: ReadonlyArray + readonly options?: CallOptions | undefined +} + +/** + * An aggregate over the rows of a group. Only valid as an output of + * {@link GroupBy} or {@link Aggregate}. + * + * @stability experimental + * @category models + * @since 4.1.5 + */ +export interface AggregateCall { + readonly _tag: "AggregateCall" + readonly fn: AggregateName + readonly arg?: Expression | undefined +} + +/** + * An expression evaluated separately within each partition of rows with + * equal `partitionBy` values, its results in row order: aggregates give + * their partition's value on every row, window functions (`cumSum`, + * `shift`, `rank`, …) run over their partition's rows. + * + * @stability experimental + * @category models + * @since 4.1.5 + */ +export interface Over { + readonly _tag: "Over" + readonly expression: Expression + readonly partitionBy: ReadonlyArray +} + +const ExpressionSchema = Schema.suspend((): Schema.Codec => ExpressionUnion) + +const ColumnSchema = Schema.Struct({ _tag: Schema.tag("Column"), name: Schema.String }) +const LiteralSchema = Schema.Struct({ + _tag: Schema.tag("Literal"), + value: Value, + type: Schema.optional(LiteralTypeSchema) +}) +const PositiveInt = Schema.Int.check(Schema.isGreaterThanOrEqualTo(1)) +const CallOptionsSchema = Schema.Struct({ + decimals: Schema.optional(Schema.Int), + to: Schema.optional(CastTarget), + values: Schema.optional(Schema.Array(Value)), + pattern: Schema.optional(Schema.String), + n: Schema.optional(Schema.Int), + method: Schema.optional(RankMethod), + descending: Schema.optional(Schema.Boolean), + window: Schema.optional(PositiveInt), + minPeriods: Schema.optional(PositiveInt), + every: Schema.optional(Schema.String), + index: Schema.optional(Schema.Int), + name: Schema.optional(Schema.String), + names: Schema.optional(Schema.Array(Schema.String)), + precision: Schema.optional(PositiveInt), + scale: Schema.optional(Schema.Int.check(Schema.isGreaterThanOrEqualTo(0))) +}) +const CallSchema = Schema.Struct({ + _tag: Schema.tag("Call"), + fn: FunctionName, + args: Schema.Array(ExpressionSchema), + options: Schema.optional(CallOptionsSchema) +}) +const AggregateCallSchema = Schema.Struct({ + _tag: Schema.tag("AggregateCall"), + fn: AggregateName, + arg: Schema.optional(ExpressionSchema) +}) +const OverSchema = Schema.Struct({ + _tag: Schema.tag("Over"), + expression: ExpressionSchema, + partitionBy: Schema.Array(ExpressionSchema) +}) +const ExpressionUnion = Schema.Union([ColumnSchema, LiteralSchema, CallSchema, AggregateCallSchema, OverSchema]) + +/** + * Schema of {@link Expression}. + * + * @stability experimental + * @category schemas + * @since 4.1.5 + */ +export const Expression: Schema.Codec = ExpressionSchema + +/** + * An expression and the name of the column it produces. + * + * @stability experimental + * @category models + * @since 4.1.5 + */ +export interface Named { + readonly name: string + readonly expression: Expression +} + +const NamedSchema = Schema.Struct({ name: Schema.String, expression: ExpressionSchema }) + +/** + * A sort key. + * + * @stability experimental + * @category models + * @since 4.1.5 + */ +export interface SortKey { + readonly expression: Expression + readonly descending: boolean +} + +const SortKeySchema = Schema.Struct({ expression: ExpressionSchema, descending: Schema.Boolean }) + +// ----------------------------------------------------------------------------- +// Nodes +// ----------------------------------------------------------------------------- + +/** + * A query plan. + * + * @stability experimental + * @category models + * @since 4.1.5 + */ +export type Node = + | Values + | Table + | Filter + | Select + | WithColumns + | Sort + | Slice + | Unique + | DropNulls + | Pick + | Omit + | Rename + | GroupBy + | Aggregate + | Join + | Concat + | Pivot + | Unpivot + | Explode + | Unnest + +/** + * The rows of a frame. + * + * @stability experimental + * @category models + * @since 4.1.5 + */ +export interface Values { + readonly _tag: "Values" + readonly frame: DataFrame +} + +/** + * The type names a {@link Table} can declare for its columns. + * + * @stability experimental + * @category models + * @since 4.1.5 + */ +export type TypeName = typeof TypeName.Type + +/** + * Schema of {@link TypeName}. + * + * @stability experimental + * @category schemas + * @since 4.1.5 + */ +export const TypeName = Schema.Literals([ + "Int8", + "Int16", + "Int32", + "Int64", + "UInt8", + "UInt16", + "UInt32", + "UInt64", + "Float16", + "Float32", + "Float64", + "Utf8", + "Binary", + "Bool", + "Null", + "Date", + "Time", + "Timestamp", + "Duration", + "Decimal" +]) + +/** + * A column a {@link Table} is expected to have. + * + * @stability experimental + * @category models + * @since 4.1.5 + */ +export interface TableColumn { + readonly name: string + readonly type: TypeName + readonly nullable: boolean +} + +/** + * The rows of a table the `Catalog` provides by name. + * + * **Details** + * + * When `columns` is given, the table must have these columns, with types of + * the same kind (any integer width for an integer column, and integers below + * 64 bits for a float column), and no nulls in non-nullable ones; the input + * is narrowed to these columns, in this order. + * + * @stability experimental + * @category models + * @since 4.1.5 + */ +export interface Table { + readonly _tag: "Table" + readonly name: string + readonly columns?: ReadonlyArray | undefined +} + +/** + * The rows where `predicate` is `true`. + * + * @stability experimental + * @category models + * @since 4.1.5 + */ +export interface Filter { + readonly _tag: "Filter" + readonly input: Node + readonly predicate: Expression +} + +/** + * Only the given computed columns. + * + * @stability experimental + * @category models + * @since 4.1.5 + */ +export interface Select { + readonly _tag: "Select" + readonly input: Node + readonly columns: ReadonlyArray +} + +/** + * The input's columns plus the given computed columns (replacing columns of + * the same name in place). + * + * @stability experimental + * @category models + * @since 4.1.5 + */ +export interface WithColumns { + readonly _tag: "WithColumns" + readonly input: Node + readonly columns: ReadonlyArray +} + +/** + * The rows in key order, stable, nulls first unless `nullsLast`. + * + * @stability experimental + * @category models + * @since 4.1.5 + */ +export interface Sort { + readonly _tag: "Sort" + readonly input: Node + readonly keys: ReadonlyArray + readonly nullsLast: boolean +} + +/** + * `length` rows (all remaining when absent) from `offset`; a negative offset + * counts from the end. + * + * @stability experimental + * @category models + * @since 4.1.5 + */ +export interface Slice { + readonly _tag: "Slice" + readonly input: Node + readonly offset: number + readonly length?: number | undefined +} + +/** + * The first row of every distinct combination of `columns` (all when absent). + * + * @stability experimental + * @category models + * @since 4.1.5 + */ +export interface Unique { + readonly _tag: "Unique" + readonly input: Node + readonly columns?: ReadonlyArray | undefined +} + +/** + * The rows without nulls in `columns` (all when absent). + * + * @stability experimental + * @category models + * @since 4.1.5 + */ +export interface DropNulls { + readonly _tag: "DropNulls" + readonly input: Node + readonly columns?: ReadonlyArray | undefined +} + +/** + * Only the given columns, in the given order. + * + * @stability experimental + * @category models + * @since 4.1.5 + */ +export interface Pick { + readonly _tag: "Pick" + readonly input: Node + readonly columns: ReadonlyArray +} + +/** + * All columns but the given ones. + * + * @stability experimental + * @category models + * @since 4.1.5 + */ +export interface Omit { + readonly _tag: "Omit" + readonly input: Node + readonly columns: ReadonlyArray +} + +/** + * The input with columns renamed: `{ old: "new" }`. + * + * @stability experimental + * @category models + * @since 4.1.5 + */ +export interface Rename { + readonly _tag: "Rename" + readonly input: Node + readonly mapping: { readonly [name: string]: string } +} + +/** + * One row per distinct key, in order of first appearance: the keys, then the + * aggregates. + * + * @stability experimental + * @category models + * @since 4.1.5 + */ +export interface GroupBy { + readonly _tag: "GroupBy" + readonly input: Node + readonly keys: ReadonlyArray + readonly aggregates: ReadonlyArray +} + +/** + * One row of aggregates over all rows. + * + * @stability experimental + * @category models + * @since 4.1.5 + */ +export interface Aggregate { + readonly _tag: "Aggregate" + readonly input: Node + readonly aggregates: ReadonlyArray +} + +/** + * How a {@link Join} combines rows. + * + * **Details** + * + * - `inner`: rows whose key exists on both sides + * - `left`: every left row; right columns are `null` where nothing matched + * - `full`: every row of both sides; keys are taken from whichever side has them + * - `semi`: left rows that have a match (left columns only) + * - `anti`: left rows without a match (left columns only) + * + * Null keys never match. + * + * @stability experimental + * @category models + * @since 4.1.5 + */ +export type JoinHow = typeof JoinHow.Type + +/** + * Schema of {@link JoinHow}. + * + * @stability experimental + * @category schemas + * @since 4.1.5 + */ +export const JoinHow = Schema.Literals(["inner", "left", "full", "semi", "anti"]) + +/** + * Rows of `left` and `right` with equal `on` columns, in left row order. + * + * @stability experimental + * @category models + * @since 4.1.5 + */ +export interface Join { + readonly _tag: "Join" + readonly left: Node + readonly right: Node + readonly on: ReadonlyArray + readonly how: JoinHow +} + +/** + * The rows of every input, one after the other. + * + * @stability experimental + * @category models + * @since 4.1.5 + */ +export interface Concat { + readonly _tag: "Concat" + readonly inputs: ReadonlyArray +} + +/** + * Aggregates a {@link Pivot} can put in its cells: the {@link AggregateName}s + * and `len` (the number of rows). + * + * @stability experimental + * @category models + * @since 4.1.5 + */ +export type PivotAggregate = AggregateName | "len" + +/** + * One row per distinct `index`, one column per distinct value of `on`, and + * in each cell the aggregate of `values` over the rows with that index and + * `on` value. + * + * **Details** + * + * Columns are named by the `on` values as strings (`null` for a null), in + * order of first appearance; with `columns`, exactly those columns, in that + * order. A cell without rows holds the aggregate of no values: 0 for `sum`, + * `count` and `len`, otherwise null. + * + * @stability experimental + * @category models + * @since 4.1.5 + */ +export interface Pivot { + readonly _tag: "Pivot" + readonly input: Node + readonly on: string + readonly index: ReadonlyArray + readonly values: string + readonly aggregate: PivotAggregate + readonly columns?: ReadonlyArray | undefined +} + +/** + * The `on` columns stacked into one `value` column, one block of rows per + * column, with the column's name in `variable` and the `index` columns + * repeated. + * + * @stability experimental + * @category models + * @since 4.1.5 + */ +export interface Unpivot { + readonly _tag: "Unpivot" + readonly input: Node + readonly index: ReadonlyArray + readonly on: ReadonlyArray + readonly variable: string + readonly value: string +} + +/** + * One row per element of the list column `column`, the other columns + * repeated. A null list gives one row with a null element; an empty list + * gives no rows. + * + * @stability experimental + * @category models + * @since 4.1.5 + */ +export interface Explode { + readonly _tag: "Explode" + readonly input: Node + readonly column: string +} + +/** + * The fields of the struct column `column` as columns, in its place. + * + * @stability experimental + * @category models + * @since 4.1.5 + */ +export interface Unnest { + readonly _tag: "Unnest" + readonly input: Node + readonly column: string +} + +const NodeSchema = Schema.suspend((): Schema.Codec => NodeUnion) +const Columns = Schema.Array(Schema.String) +const NonNegativeInt = Schema.Int.check(Schema.isGreaterThanOrEqualTo(0)) + +const NodeUnion = Schema.Union([ + Schema.Struct({ _tag: Schema.tag("Values"), frame: Frame }), + Schema.Struct({ + _tag: Schema.tag("Table"), + name: Schema.String, + columns: Schema.optional( + Schema.Array(Schema.Struct({ name: Schema.String, type: TypeName, nullable: Schema.Boolean })) + ) + }), + Schema.Struct({ _tag: Schema.tag("Filter"), input: NodeSchema, predicate: ExpressionSchema }), + Schema.Struct({ _tag: Schema.tag("Select"), input: NodeSchema, columns: Schema.Array(NamedSchema) }), + Schema.Struct({ _tag: Schema.tag("WithColumns"), input: NodeSchema, columns: Schema.Array(NamedSchema) }), + Schema.Struct({ + _tag: Schema.tag("Sort"), + input: NodeSchema, + keys: Schema.Array(SortKeySchema), + nullsLast: Schema.Boolean + }), + Schema.Struct({ + _tag: Schema.tag("Slice"), + input: NodeSchema, + offset: Schema.Int, + length: Schema.optional(NonNegativeInt) + }), + Schema.Struct({ _tag: Schema.tag("Unique"), input: NodeSchema, columns: Schema.optional(Columns) }), + Schema.Struct({ _tag: Schema.tag("DropNulls"), input: NodeSchema, columns: Schema.optional(Columns) }), + Schema.Struct({ _tag: Schema.tag("Pick"), input: NodeSchema, columns: Columns }), + Schema.Struct({ _tag: Schema.tag("Omit"), input: NodeSchema, columns: Columns }), + Schema.Struct({ + _tag: Schema.tag("Rename"), + input: NodeSchema, + mapping: Schema.Record(Schema.String, Schema.String) + }), + Schema.Struct({ + _tag: Schema.tag("GroupBy"), + input: NodeSchema, + keys: Columns, + aggregates: Schema.Array(NamedSchema) + }), + Schema.Struct({ _tag: Schema.tag("Aggregate"), input: NodeSchema, aggregates: Schema.Array(NamedSchema) }), + Schema.Struct({ + _tag: Schema.tag("Join"), + left: NodeSchema, + right: NodeSchema, + on: Columns, + how: JoinHow + }), + Schema.Struct({ _tag: Schema.tag("Concat"), inputs: Schema.Array(NodeSchema) }), + Schema.Struct({ + _tag: Schema.tag("Pivot"), + input: NodeSchema, + on: Schema.String, + index: Columns, + values: Schema.String, + aggregate: Schema.Union([AggregateName, Schema.Literal("len")]), + columns: Schema.optional(Columns) + }), + Schema.Struct({ _tag: Schema.tag("Explode"), input: NodeSchema, column: Schema.String }), + Schema.Struct({ _tag: Schema.tag("Unnest"), input: NodeSchema, column: Schema.String }), + Schema.Struct({ + _tag: Schema.tag("Unpivot"), + input: NodeSchema, + index: Columns, + on: Columns, + variable: Schema.String, + value: Schema.String + }) +]) + +/** + * Schema of {@link Node}. + * + * @stability experimental + * @category schemas + * @since 4.1.5 + */ +export const Node: Schema.Codec = NodeSchema + +/** + * A {@link Node} from its JSON form. + * + * @stability experimental + * @category schemas + * @since 4.1.5 + */ +export const NodeFromJson: Schema.Codec = Schema.toCodecJson(Node) as any + +/** + * An {@link Expression} from its JSON form. + * + * @stability experimental + * @category schemas + * @since 4.1.5 + */ +export const ExpressionFromJson: Schema.Codec = Schema.toCodecJson(Expression) as any diff --git a/packages/effect/src/dataframe/Query.ts b/packages/effect/src/dataframe/Query.ts new file mode 100644 index 00000000000..b67867e9b15 --- /dev/null +++ b/packages/effect/src/dataframe/Query.ts @@ -0,0 +1,901 @@ +/** + * Lazy, typed queries over data frames: describe filters, derived columns, + * sorts, groups and joins, then run them as an `Effect`. + * + * **Mental model** + * + * - A `Query` describes how to compute a table whose rows have type + * `Row`. Building one does no work; every function returns a new query + * with one more operation, and the row type follows along. + * - A query is a {@link Plan.Node} underneath, plain data that can be + * inspected, stored, encoded to JSON and run again ({@link fromPlan}). + * - A query is an `Effect, DataFrameError>`: `yield*` it, + * or pass it to any `Effect` function. Running it reads the tables it + * names from the `Catalog`, rewrites the plan to do less work, and + * executes it. + * - Expressions come from {@link Expr}. Functions that take expressions take + * a callback that receives one typed reference per column; it runs once, + * when the query is built. + * + * **Example** (Summarising readings per city) + * + * ```ts import.meta.vitest + * import { Effect, Schema } from "effect" + * import { DataFrame, Expr, Query } from "effect/dataframe" + * + * const Reading = Schema.Struct({ city: Schema.String, temp: Schema.Number }) + * const readings = DataFrame.fromRows(Reading, [ + * { city: "Vienna", temp: 21.5 }, + * { city: "Berlin", temp: 26.5 }, + * { city: "Vienna", temp: 19 }, + * { city: "Zurich", temp: 15 } + * ]) + * + * const summary = Query.from(readings).pipe( + * Query.filter((c) => Expr.gt(c.temp, 16)), + * Query.groupBy("city"), + * Query.agg((c) => ({ readings: Expr.count(), mean: Expr.mean(c.temp) })), + * Query.sort((c) => [Expr.desc(c.mean)]) + * ) + * + * const rows = Effect.runSync(Effect.map(summary, DataFrame.toRows)) + * rows[0] // => { city: "Berlin", readings: 1, mean: 26.5 } + * rows[1] // => { city: "Vienna", readings: 2, mean: 20.25 } + * ``` + * + * **Details** + * + * Results match Polars for the same operations: groups and `unique` keep + * first-appearance order, sorting is stable with nulls first unless + * `nullsLast` is set, and joins keep the left row order (with right rows in + * their order within a key). See {@link Expr} for value semantics. + * + * Before running, the plan is rewritten: predicates move below the + * operations they commute with, filters merge, slices move below row-wise + * operations (and a slice of a sort keeps only the rows it needs), and + * columns and aggregates nothing reads are not computed. {@link explain} + * shows the plan that runs. + * + * **Gotchas** + * + * - The column references passed to callbacks are created on access, so + * spreading or enumerating them yields nothing. + * - Joins require the non-key columns of both sides to have different + * names; rename first. + * - A long query yields to other fibers between operations, so it can be + * interrupted or timed out, but not in the middle of one operation. + * + * @stability experimental + * @since 4.1.5 + */ +import type * as BigDecimal from "../BigDecimal.ts" +import * as Effect from "../Effect.ts" +import * as Effectable from "../Effectable.ts" +import { dual } from "../Function.ts" +import { NodeInspectSymbol } from "../Inspectable.ts" +import * as Stream from "../Stream.ts" +import type * as Types from "../Types.ts" +import type { DataFrame, DataFrameError } from "./DataFrame.ts" +import { Engine } from "./Engine.ts" +import type * as Expr from "./Expr.ts" +import * as E from "./internal/expr.ts" +import * as Q from "./internal/query.ts" +import * as Schema_ from "./internal/schema.ts" +import type * as Plan from "./Plan.ts" + +// ----------------------------------------------------------------------------- +// Models +// ----------------------------------------------------------------------------- + +/** + * Identifies queries. + * + * @stability experimental + * @category type IDs + * @since 4.1.5 + */ +export const TypeId = "~effect/dataframe/Query" + +/** + * A lazy computation of a table whose rows have type `Row`. + * + * @stability experimental + * @category models + * @since 4.1.5 + */ +export interface Query extends Effect.Effect, DataFrameError> { + readonly [TypeId]: { + readonly _Row: Types.Covariant + } + /** The query as data. */ + readonly plan: Plan.Node +} + +/** + * One typed column reference per column, as passed to expression callbacks. + * + * @stability experimental + * @category models + * @since 4.1.5 + */ +export type Columns = { readonly [K in keyof Row & string]: Expr.Expr } + +/** + * A query split into groups by key columns, waiting for {@link agg}. + * + * @stability experimental + * @category models + * @since 4.1.5 + */ +export interface GroupBy { + readonly _tag: "GroupBy" + readonly query: Query + readonly keys: ReadonlyArray +} + +/** + * The row type of a query. + * + * @stability experimental + * @category type-level + * @since 4.1.5 + */ +export type Row = Q extends Query ? R : never + +/** + * How {@link join} combines rows; see {@link Plan.JoinHow}. + * + * @stability experimental + * @category models + * @since 4.1.5 + */ +export type JoinHow = Plan.JoinHow + +/** + * The row type of a join. + * + * @stability experimental + * @category type-level + * @since 4.1.5 + */ +export type Joined = Simplify< + How extends "semi" | "anti" ? R + : How extends "inner" ? R & Omit + : How extends "left" ? R & Nullable> + : Pick & Nullable> & Nullable> +> + +type Simplify = { readonly [K in keyof A]: A[K] } extends infer B ? B : never +type Nullable = { readonly [K in keyof R]: R[K] | null } + +type JoinCheck = [Exclude] extends [never] ? unknown + : { readonly "~error": `rename the columns that exist on both sides: ${Exclude & string}` } + +interface QueryImpl extends Query {} + +const Proto = { + ...Effectable.Prototype({ + label: "Query", + evaluate(fiber) { + return fiber.getRef(Engine).collect(this.plan) as any + } + }), + [TypeId]: { _Row: (_: never) => _ }, + toJSON(this: QueryImpl) { + return { _id: "Query", plan: this.plan } + }, + toString(this: QueryImpl) { + return Q.show(this.plan) + }, + [NodeInspectSymbol](this: QueryImpl) { + return this.toJSON() + } +} + +const make = (plan: Plan.Node): Query => { + const self = Object.create(Proto) + self.plan = plan + return self +} + +/** + * Returns `true` for queries. + * + * @stability experimental + * @category guards + * @since 4.1.5 + */ +export const isQuery = (u: unknown): u is Query => typeof u === "object" && u !== null && TypeId in u + +const columns: any = new Proxy({}, { + get: (_, name) => typeof name === "string" ? E.makeExpr({ _tag: "Column", name }) : undefined +}) + +const named = (exprs: { readonly [name: string]: unknown }): Array => + Object.keys(exprs).map((name) => ({ name, expression: E.toNode(exprs[name]) })) + +const isGroupBy = (u: unknown): u is GroupBy => + typeof u === "object" && u !== null && (u as { _tag?: unknown })._tag === "GroupBy" && "query" in u + +// ----------------------------------------------------------------------------- +// Constructors +// ----------------------------------------------------------------------------- + +/** + * A query over the rows of a frame. + * + * @stability experimental + * @category constructors + * @since 4.1.5 + */ +export const from = (frame: DataFrame): Query => make({ _tag: "Values", frame: frame as DataFrame }) + +/** + * A query over the table called `name`, provided by the `Catalog` when + * the query runs. The table must have the columns of the struct schema (see + * `DataFrame.validate`); the query sees only those, in the schema's order. + * + * @stability experimental + * @category constructors + * @since 4.1.5 + */ +export const table = ( + name: string, + schema: S +): Query> => + make({ _tag: "Table", name, columns: Schema_.tableColumns(Schema_.toArrowSchema(schema, undefined)) }) + +/** + * A query from its plan, with untyped rows. Decode untrusted plans with + * {@link Plan.Node} or {@link Plan.NodeFromJson} first. + * + * @stability experimental + * @category constructors + * @since 4.1.5 + */ +export const fromPlan = (plan: Plan.Node): Query<{ readonly [name: string]: unknown }> => make(plan) + +/** + * The query's result as a stream of batches. + * + * **Details** + * + * Over streamed tables (a `Stream` in the `Catalog`), row-wise operations + * run on each batch as it arrives and a `head` stops reading once it has its + * rows; grouping and aggregation fold the batches into one result. Over + * frames, the result is one batch. + * + * **Example** (Filtering a stream of batches) + * + * ```ts import.meta.vitest + * import { Effect, Stream } from "effect" + * import { Catalog, DataFrame, Expr, Query } from "effect/dataframe" + * + * const batch = (from: number) => DataFrame.fromColumns({ n: Int32Array.from({ length: 3 }, (_, i) => from + i) }) + * const numbers = Stream.make(batch(0), batch(3), batch(6)) + * + * const sizes = Query.fromPlan({ _tag: "Table", name: "numbers" }).pipe( + * Query.filter((c) => Expr.gt(c.n as Expr.Expr, 2)), + * Query.stream, + * Stream.map((frame) => frame.length), + * Stream.runCollect, + * Effect.provide(Catalog.layer({ numbers })), + * Effect.runSync + * ) + * sizes // => [3, 3] + * ``` + * + * @stability experimental + * @category conversions + * @since 4.1.5 + */ +export const stream = (self: Query): Stream.Stream, DataFrameError> => + Stream.unwrap(Engine.use((engine) => Effect.succeed(engine.stream(self.plan) as any))) + +/** + * The plan that runs for a query, after tables are resolved and the plan is + * rewritten, as an indented tree with one operation per line. + * + * @stability experimental + * @category conversions + * @since 4.1.5 + */ +export const explain = (self: Query): Effect.Effect => + Engine.use((engine) => engine.explain(self.plan)) + +// ----------------------------------------------------------------------------- +// Rows +// ----------------------------------------------------------------------------- + +/** + * Keeps the rows where the predicate is `true` (`null` counts as `false`). + * + * @stability experimental + * @category filtering + * @since 4.1.5 + */ +export const filter: { + (predicate: (columns: Columns) => Expr.Expr): (self: Query) => Query + (self: Query, predicate: (columns: Columns) => Expr.Expr): Query +} = dual( + 2, + (self: Query, predicate: (columns: Columns) => Expr.Expr): Query => + make({ _tag: "Filter", input: self.plan, predicate: E.toNode(predicate(columns)) }) +) + +/** + * Sorts by the given keys, first key first; ties keep their order. Nulls + * come first unless `nullsLast` is set. + * + * @stability experimental + * @category sorting + * @since 4.1.5 + */ +export const sort: { + ( + keys: (columns: Columns) => ReadonlyArray | Expr.SortOrder>, + options?: { readonly nullsLast?: boolean | undefined } + ): (self: Query) => Query + ( + self: Query, + keys: (columns: Columns) => ReadonlyArray | Expr.SortOrder>, + options?: { readonly nullsLast?: boolean | undefined } + ): Query +} = dual((args) => isQuery(args[0]), ( + self: Query, + keys: (columns: Columns) => ReadonlyArray | Expr.SortOrder>, + options?: { readonly nullsLast?: boolean | undefined } +): Query => + make({ + _tag: "Sort", + input: self.plan, + keys: keys(columns).map((k) => + E.isExpr(k) + ? { expression: k.node, descending: false } + : { + expression: E.toNode((k as Expr.SortOrder).expr), + descending: (k as Expr.SortOrder).descending + } + ), + nullsLast: options?.nullsLast ?? false + })) + +/** + * `length` rows (all remaining when omitted) starting at `offset`; a negative + * offset counts from the end. + * + * @stability experimental + * @category slicing + * @since 4.1.5 + */ +export const slice: { + (offset: number, length?: number): (self: Query) => Query + (self: Query, offset: number, length?: number): Query +} = dual( + (args) => isQuery(args[0]), + (self: Query, offset: number, length?: number): Query => + make({ _tag: "Slice", input: self.plan, offset, length }) +) + +/** + * The first `n` rows. + * + * @stability experimental + * @category slicing + * @since 4.1.5 + */ +export const head: { + (n: number): (self: Query) => Query + (self: Query, n: number): Query +} = dual(2, (self: Query, n: number): Query => slice(self, 0, Math.max(0, n))) + +/** + * The last `n` rows. + * + * @stability experimental + * @category slicing + * @since 4.1.5 + */ +export const tail: { + (n: number): (self: Query) => Query + (self: Query, n: number): Query +} = dual(2, (self: Query, n: number): Query => n <= 0 ? slice(self, 0, 0) : slice(self, -n)) + +/** + * Keeps the first row of every distinct combination of the given columns + * (all columns by default). + * + * @stability experimental + * @category filtering + * @since 4.1.5 + */ +export const unique: { + (columns?: ReadonlyArray): (self: Query) => Query + (self: Query, columns?: ReadonlyArray): Query +} = dual( + (args) => isQuery(args[0]), + (self: Query, columns?: ReadonlyArray): Query => + make({ _tag: "Unique", input: self.plan, columns }) +) + +/** + * Removes the rows that are `null` in any of the given columns (all + * columns by default). + * + * @stability experimental + * @category filtering + * @since 4.1.5 + */ +export const dropNulls: { + ( + columns?: ReadonlyArray + ): (self: Query) => Query & { [P in K]: NonNullable }>> + ( + self: Query, + columns?: ReadonlyArray + ): Query & { [P in K]: NonNullable }>> +} = dual( + (args) => isQuery(args[0]), + (self: Query, columns?: ReadonlyArray) => make({ _tag: "DropNulls", input: self.plan, columns }) +) + +/** + * The rows of every query, one after the other. The queries must have the + * same columns. + * + * @stability experimental + * @category combining + * @since 4.1.5 + */ +export const concat = (queries: readonly [Query, ...Array>]): Query => + make({ _tag: "Concat", inputs: queries.map((q) => q.plan) }) + +/** + * The rows of `self`, then the rows of `that`. + * + * @stability experimental + * @category combining + * @since 4.1.5 + */ +export const append: { + (that: Query): (self: Query) => Query + (self: Query, that: Query): Query +} = dual(2, (self: Query, that: Query) => concat([self, that])) + +// ----------------------------------------------------------------------------- +// Columns +// ----------------------------------------------------------------------------- + +/** + * Keeps the given columns, in the given order. + * + * @stability experimental + * @category columns + * @since 4.1.5 + */ +export const pick: { + >( + ...names: K + ): (self: Query) => Query>> +} = (...names: ReadonlyArray) => (self: Query) => + make({ _tag: "Pick", input: self.plan, columns: names }) as any + +/** + * Removes the given columns. + * + * @stability experimental + * @category columns + * @since 4.1.5 + */ +export const omit: { + >( + ...names: K + ): (self: Query) => Query>> +} = (...names: ReadonlyArray) => (self: Query) => + make({ _tag: "Omit", input: self.plan, columns: names }) as any + +/** + * Renames columns: `{ old: "new" }`. + * + * @stability experimental + * @category columns + * @since 4.1.5 + */ +export const rename: { + ( + mapping: M + ): (self: Query) => Query< + Simplify & { [K in keyof M & keyof R as M[K] extends string ? M[K] : never]: R[K] }> + > + ( + self: Query, + mapping: M + ): Query & { [K in keyof M & keyof R as M[K] extends string ? M[K] : never]: R[K] }>> +} = dual( + 2, + (self: Query, mapping: { readonly [name: string]: string | undefined }) => { + const defined: { [name: string]: string } = {} + for (const [from, to] of Object.entries(mapping)) if (to !== undefined) defined[from] = to + return make({ _tag: "Rename", input: self.plan, mapping: defined }) + } +) + +/** + * Replaces the columns with the given expressions; the record's keys name + * the new columns. + * + * @stability experimental + * @category columns + * @since 4.1.5 + */ +export const select: { + }>( + f: (columns: Columns) => O + ): (self: Query) => Query }>> + }>( + self: Query, + f: (columns: Columns) => O + ): Query }>> +} = dual( + 2, + (self: Query, f: (columns: any) => { readonly [name: string]: unknown }) => + make({ _tag: "Select", input: self.plan, columns: named(f(columns)) }) +) + +/** + * Adds columns, or replaces columns of the same name in place. + * + * @stability experimental + * @category columns + * @since 4.1.5 + */ +export const withColumns: { + }>( + f: (columns: Columns) => O + ): (self: Query) => Query & { [K in keyof O]: Expr.Type }>> + }>( + self: Query, + f: (columns: Columns) => O + ): Query & { [K in keyof O]: Expr.Type }>> +} = dual( + 2, + (self: Query, f: (columns: any) => { readonly [name: string]: unknown }) => + make({ _tag: "WithColumns", input: self.plan, columns: named(f(columns)) }) +) + +// ----------------------------------------------------------------------------- +// Grouping +// ----------------------------------------------------------------------------- + +/** + * Groups rows by the given key columns; finish with {@link agg}. + * + * @stability experimental + * @category grouping + * @since 4.1.5 + */ +export const groupBy: { + >(...keys: K): (self: Query) => GroupBy +} = (...keys: ReadonlyArray) => (self: Query) => ({ _tag: "GroupBy", query: self, keys }) as any + +/** + * Computes aggregates per group: one row per distinct key, in order of first + * appearance, with the key columns first. + * + * @stability experimental + * @category grouping + * @since 4.1.5 + */ +export const agg: { + }>( + f: (columns: Columns) => O + ): (self: GroupBy) => Query & { [P in keyof O]: Expr.Type }>> + }>( + self: GroupBy, + f: (columns: Columns) => O + ): Query & { [P in keyof O]: Expr.Type }>> +} = dual( + (args) => isGroupBy(args[0]), + (self: GroupBy, f: (columns: any) => { readonly [name: string]: unknown }) => + make({ + _tag: "GroupBy", + input: self.query.plan, + keys: self.keys as ReadonlyArray, + aggregates: named(f(columns)) + }) +) + +/** + * Computes aggregates over all rows: a table of one row. + * + * @stability experimental + * @category grouping + * @since 4.1.5 + */ +export const aggregate: { + }>( + f: (columns: Columns) => O + ): (self: Query) => Query }>> + }>( + self: Query, + f: (columns: Columns) => O + ): Query }>> +} = dual( + 2, + (self: Query, f: (columns: any) => { readonly [name: string]: unknown }) => + make({ _tag: "Aggregate", input: self.plan, aggregates: named(f(columns)) }) +) + +// ----------------------------------------------------------------------------- +// Joins +// ----------------------------------------------------------------------------- + +/** + * Joins `that` onto `self` by equal key columns. Output rows keep the left + * row order; several right matches follow in their order. The non-key + * columns of the two sides must have different names. + * + * @stability experimental + * @category combining + * @since 4.1.5 + */ +export const join: { + ( + that: Query, + options: { readonly on: On | ReadonlyArray; readonly how?: How | undefined } & JoinCheck + ): (self: Query) => Query> + ( + self: Query, + that: Query, + options: { readonly on: On | ReadonlyArray; readonly how?: How | undefined } & JoinCheck + ): Query> +} = dual( + 3, + ( + self: Query, + that: Query, + options: { readonly on: string | ReadonlyArray; readonly how?: JoinHow | undefined } + ) => + make({ + _tag: "Join", + left: self.plan, + right: that.plan, + on: typeof options.on === "string" ? [options.on] : options.on, + how: options.how ?? "inner" + }) +) as any + +// ----------------------------------------------------------------------------- +// Reshaping +// ----------------------------------------------------------------------------- + +/** + * The value type of a pivot cell for an aggregate of values of type `A`. + * + * @stability experimental + * @category type-level + * @since 4.1.5 + */ +export type PivotCell = F extends "len" | "count" | "countDistinct" ? number + : F extends "sum" ? (NonNullable extends bigint ? bigint + : NonNullable extends BigDecimal.BigDecimal ? BigDecimal.BigDecimal + : number) + : F extends "mean" | "median" | "std" | "variance" ? number | null + : NonNullable | null + +interface PivotOptions { + /** The column whose distinct values become columns. */ + readonly on: On + /** The columns whose distinct combinations become rows. */ + readonly index: Index + /** The column aggregated into the cells. */ + readonly values: Values + /** The aggregate of each cell. */ + readonly aggregate: F + readonly "~row"?: R +} + +/** + * One row per distinct `index`, one column per distinct value of `on`, and + * in each cell the `aggregate` of `values` over the rows with that index and + * value (`Plan.Pivot` has the details). + * + * **Details** + * + * Without `columns`, the output columns depend on the data, so the rows are + * untyped. With `columns`, the output has exactly those value columns, in + * that order, and is typed. + * + * **Example** (Units per region and quarter) + * + * ```ts import.meta.vitest + * import { Effect } from "effect" + * import { DataFrame, Query } from "effect/dataframe" + * + * const sales = DataFrame.fromColumns({ + * region: ["north", "north", "south"], + * quarter: ["q1", "q2", "q1"], + * units: [10, 20, 5] + * }) + * + * const rows = Query.from(sales).pipe( + * Query.pivot({ on: "quarter", index: ["region"], values: "units", aggregate: "sum", columns: ["q1", "q2"] }), + * Effect.map(DataFrame.toRows), + * Effect.runSync + * ) + * rows // => [{ region: "north", q1: 10, q2: 20 }, { region: "south", q1: 5, q2: 0 }] + * ``` + * + * @stability experimental + * @category reshaping + * @since 4.1.5 + */ +export const pivot: { + < + R, + const On extends keyof R & string, + const Index extends ReadonlyArray, + const Values extends keyof R & string, + const F extends Plan.PivotAggregate, + const Columns extends ReadonlyArray + >( + options: PivotOptions & { readonly columns: Columns } + ): ( + self: Query + ) => Query & { readonly [C in Columns[number]]: PivotCell }>> + < + R, + const On extends keyof R & string, + const Index extends ReadonlyArray, + const Values extends keyof R & string, + const F extends Plan.PivotAggregate + >( + options: PivotOptions + ): (self: Query) => Query<{ readonly [name: string]: unknown }> + < + R, + const On extends keyof R & string, + const Index extends ReadonlyArray, + const Values extends keyof R & string, + const F extends Plan.PivotAggregate, + const Columns extends ReadonlyArray + >( + self: Query, + options: PivotOptions & { readonly columns: Columns } + ): Query & { readonly [C in Columns[number]]: PivotCell }>> + < + R, + const On extends keyof R & string, + const Index extends ReadonlyArray, + const Values extends keyof R & string, + const F extends Plan.PivotAggregate + >( + self: Query, + options: PivotOptions + ): Query<{ readonly [name: string]: unknown }> +} = dual(2, (self: Query, options: { + readonly on: string + readonly index: ReadonlyArray + readonly values: string + readonly aggregate: Plan.PivotAggregate + readonly columns?: ReadonlyArray +}) => + make({ + _tag: "Pivot", + input: self.plan, + on: options.on, + index: options.index, + values: options.values, + aggregate: options.aggregate, + columns: options.columns + })) as any + +/** + * Stacks the `on` columns into one `value` column (`"value"` by default), + * one block of rows per column, with the column's name in `variable` + * (`"variable"` by default) and the `index` columns repeated. The values + * take the common type of the stacked columns. + * + * @stability experimental + * @category reshaping + * @since 4.1.5 + */ +export const unpivot: { + < + R, + const On extends ReadonlyArray, + const Index extends ReadonlyArray = readonly [], + const Variable extends string = "variable", + const Value extends string = "value" + >(options: { + readonly on: On + readonly index?: Index | undefined + readonly variable?: Variable | undefined + readonly value?: Value | undefined + }): ( + self: Query + ) => Query< + Simplify< + & Pick + & { readonly [K in Variable]: On[number] } + & { readonly [K in Value]: R[On[number]] } + > + > + < + R, + const On extends ReadonlyArray, + const Index extends ReadonlyArray = readonly [], + const Variable extends string = "variable", + const Value extends string = "value" + >(self: Query, options: { + readonly on: On + readonly index?: Index | undefined + readonly variable?: Variable | undefined + readonly value?: Value | undefined + }): Query< + Simplify< + & Pick + & { readonly [K in Variable]: On[number] } + & { readonly [K in Value]: R[On[number]] } + > + > +} = dual(2, (self: Query, options: { + readonly on: ReadonlyArray + readonly index?: ReadonlyArray | undefined + readonly variable?: string | undefined + readonly value?: string | undefined +}) => + make({ + _tag: "Unpivot", + input: self.plan, + index: options.index ?? [], + on: options.on, + variable: options.variable ?? "variable", + value: options.value ?? "value" + })) as any + +/** + * One row per element of the list column `column`, the other columns + * repeated. A null list gives one row with a null element; an empty list + * gives no rows. + * + * @stability experimental + * @category reshaping + * @since 4.1.5 + */ +export const explode: { + >( + column: K + ): (self: Query) => Query | null : R[P] }>> + >( + self: Query, + column: K + ): Query | null : R[P] }>> +} = dual(2, (self: Query, column: string) => make({ _tag: "Explode", input: self.plan, column })) as any + +/** + * The fields of the struct column `column` as columns, in its place; they + * are null where the struct is null. + * + * @stability experimental + * @category reshaping + * @since 4.1.5 + */ +export const unnest: { + >( + column: K + ): (self: Query) => Query & Fields>> + >(self: Query, column: K): Query & Fields>> +} = dual(2, (self: Query, column: string) => make({ _tag: "Unnest", input: self.plan, column })) as any + +type ListKeys = { + [K in keyof R & string]: NonNullable extends ReadonlyArray ? K : never +}[keyof R & string] +type ElementOf = NonNullable extends ReadonlyArray ? E : never +type StructKeys = { + [K in keyof R & string]: NonNullable extends ReadonlyArray ? never + : NonNullable extends object ? K + : never +}[keyof R & string] +type Fields = null extends A ? { readonly [K in keyof NonNullable]: NonNullable[K] | null } + : { readonly [K in keyof NonNullable]: NonNullable[K] } diff --git a/packages/effect/src/dataframe/Sql.ts b/packages/effect/src/dataframe/Sql.ts new file mode 100644 index 00000000000..7d8a929f49a --- /dev/null +++ b/packages/effect/src/dataframe/Sql.ts @@ -0,0 +1,150 @@ +/** + * Running queries on a database: plans compiled to SQL and run through the + * `effect/sql` client in the context. + * + * **Mental model** + * + * - {@link compile} turns a plan into one SQL query whose rows come back in + * the plan's row order: stable sorts, groups in order of first appearance, + * joins in left row order, as the in-memory engine. + * - {@link layer} provides an `Engine` that compiles every query and runs it + * with the `SqlClient` in the context, so `yield* query` reads from the + * database. `Query.table` names become table names. + * - Two dialects: `postgres` (PostgreSQL, PGlite) and `duckdb`. + * + * **Example** (Compiling a query) + * + * ```ts import.meta.vitest + * import { Result, Schema } from "effect" + * import { Expr, Query, Sql } from "effect/dataframe" + * + * const Trip = Schema.Struct({ city: Schema.String, km: Schema.Number }) + * const query = Query.table("trips", Trip).pipe( + * Query.filter((c) => Expr.gt(c.km, 10)), + * Query.groupBy("city"), + * Query.agg((c) => ({ total: Expr.sum(c.km) })) + * ) + * + * const compiled = Result.getOrThrow(Sql.compile(query.plan, { dialect: "postgres" })) + * compiled.columns // => ["city", "total"] + * ``` + * + * **Details** + * + * Values follow the database's semantics where they differ from the + * in-memory engine's: integer overflow raises an error instead of wrapping, + * division by zero follows the database, string order follows its collation + * (binary in DuckDB), `round` rounds half away from zero in PostgreSQL, and + * `min`/`max` of floats treat NaN as the largest value. Pivots, unpivots, + * `explode` and `unnest` are not supported yet; tables need declared + * columns (`Query.table` with a schema). + * + * @stability experimental + * @since 4.1.5 + */ +import * as DateTime from "../DateTime.ts" +import * as Effect from "../Effect.ts" +import * as Layer from "../Layer.ts" +import * as Result from "../Result.ts" +import { SqlClient } from "../sql/SqlClient.ts" +import * as Stream from "../Stream.ts" +import { DataFrameError, fromColumns } from "./DataFrame.ts" +import type { DataFrame } from "./DataFrame.ts" +import { Engine } from "./Engine.ts" +import * as internal from "./internal/sql.ts" +import * as V from "./internal/vector.ts" +import type * as Plan from "./Plan.ts" + +/** + * The SQL dialects plans compile to. + * + * @stability experimental + * @category models + * @since 4.1.5 + */ +export type Dialect = "duckdb" | "postgres" + +/** + * A compiled plan: the query, and the names of its result columns in order. + * + * @stability experimental + * @category models + * @since 4.1.5 + */ +export interface Compiled { + readonly sql: string + readonly columns: ReadonlyArray +} + +const toError = (cause: unknown) => + new DataFrameError({ message: cause instanceof Error ? cause.message : String(cause), cause }) + +/** + * Compiles a plan to one SQL query (dialect `postgres` by default). Fails + * for operations the SQL engine doesn't support. + * + * @stability experimental + * @category compiling + * @since 4.1.5 + */ +export const compile = ( + plan: Plan.Node, + options?: { readonly dialect?: Dialect | undefined } +): Result.Result => { + try { + return Result.succeed(internal.compile(plan, options?.dialect ?? "postgres")) + } catch (cause) { + if (cause instanceof V.DataFrameDefect) return Result.fail(toError(cause)) + throw cause + } +} + +const frameOf = (columns: ReadonlyArray, rows: ReadonlyArray>): DataFrame => { + const data: Record> = {} + // Drivers return JS Dates for timestamps and dates. + const value = (v: unknown) => (v === undefined ? null : v instanceof Date ? DateTime.makeUnsafe(v) : v) + for (const name of columns) data[name] = rows.map((r) => value(r[name])) + return fromColumns(data as any) as DataFrame +} + +/** + * An engine that compiles queries to SQL and runs them with the + * `SqlClient` in the context. + * + * @stability experimental + * @category engines + * @since 4.1.5 + */ +export const engine = (options?: { readonly dialect?: Dialect | undefined }): Effect.Effect< + Engine.Service, + never, + SqlClient +> => + Effect.gen(function*() { + const sql = yield* SqlClient + const dialect = options?.dialect ?? "postgres" + const compiled = (plan: Plan.Node) => Effect.fromResult(compile(plan, { dialect })) + const collect = (plan: Plan.Node): Effect.Effect, DataFrameError> => + Effect.gen(function*() { + const query = yield* compiled(plan) + const rows = yield* sql.unsafe>(query.sql).pipe(Effect.mapError(toError)) + return yield* Effect.try({ try: () => frameOf(query.columns, rows), catch: toError }) + }) + return { + name: `sql (${dialect})`, + collect, + stream: (plan) => Stream.fromEffect(collect(plan)), + explain: (plan) => Effect.map(compiled(plan), (c) => c.sql) + } satisfies Engine.Service + }) + +/** + * A layer that runs queries on the database of the `SqlClient` in the + * context. + * + * @stability experimental + * @category layers + * @since 4.1.5 + */ +export const layer = (options?: { readonly dialect?: Dialect | undefined }): Layer.Layer => + Layer.effect(Engine, engine(options)) diff --git a/packages/effect/src/dataframe/index.ts b/packages/effect/src/dataframe/index.ts new file mode 100644 index 00000000000..2594495cb34 --- /dev/null +++ b/packages/effect/src/dataframe/index.ts @@ -0,0 +1,48 @@ +/** + * @stability experimental + * @since 4.1.5 + */ + +// @barrel: Auto-generated exports. Do not edit manually. + +/** + * @stability experimental + * @since 4.1.5 + */ +export * as Catalog from "./Catalog.ts" + +/** + * @stability experimental + * @since 4.1.5 + */ +export * as DataFrame from "./DataFrame.ts" + +/** + * @stability experimental + * @since 4.1.5 + */ +export * as Engine from "./Engine.ts" + +/** + * @stability experimental + * @since 4.1.5 + */ +export * as Expr from "./Expr.ts" + +/** + * @stability experimental + * @since 4.1.5 + */ +export * as Plan from "./Plan.ts" + +/** + * @stability experimental + * @since 4.1.5 + */ +export * as Query from "./Query.ts" + +/** + * @stability experimental + * @since 4.1.5 + */ +export * as Sql from "./Sql.ts" diff --git a/packages/effect/src/dataframe/internal/aggregate.ts b/packages/effect/src/dataframe/internal/aggregate.ts new file mode 100644 index 00000000000..b62677342dd --- /dev/null +++ b/packages/effect/src/dataframe/internal/aggregate.ts @@ -0,0 +1,401 @@ +/** + * Grouped aggregation kernels. Every kernel takes the input column (one value + * per row), the group id of every row and the group count, and returns one + * value per group. + * + * Null inputs are skipped. Over a group without values `sum` is 0, `count` + * is 0 and the other aggregates are null. `min`/`max` skip NaN unless every + * value is NaN; `sum` and `mean` propagate it. This matches Polars. + * + * @internal + */ +import * as S from "../../arrow/ArrowSchema.ts" +import * as Dec from "./decimal.ts" +import * as D from "./dtype.ts" +import * as Hash from "./hash.ts" +import { compareBytes } from "./kernels.ts" +import * as V from "./vector.ts" + +const I0 = BigInt(0) + +/** @internal */ +export type AggFn = + | "sum" + | "mean" + | "min" + | "max" + | "count" + | "len" + | "first" + | "last" + | "countDistinct" + | "std" + | "variance" + | "median" + | "list" + +/** @internal */ +export interface Grouping { + readonly ids: Int32Array + readonly count: number +} + +const validCounts = (v: V.Vector, g: Grouping): Float64Array => { + const out = new Float64Array(g.count) + const ids = g.ids + const n = ids.length + if (v.kind === "null") return out + const valid = v.valid + if (valid === undefined) { for (let i = 0; i < n; i++) out[ids[i]]++ } + else for (let i = 0; i < n; i++) out[ids[i]] += valid[i] + return out +} + +const invalidWhereZero = (counts: Float64Array): Uint8Array | undefined => { + let valid: Uint8Array | undefined + for (let g = 0; g < counts.length; g++) { + if (counts[g] === 0) { + if (valid === undefined) valid = new Uint8Array(counts.length).fill(1) + valid[g] = 0 + } + } + return valid +} + +/** @internal */ +export const aggregate = (fn: AggFn, v: V.Vector | undefined, g: Grouping): V.Vector => { + const ids = g.ids + const n = ids.length + const groups = g.count + if (fn === "len") { + const out = new Float64Array(groups) + for (let i = 0; i < n; i++) out[ids[i]]++ + return V.f64(S.uint32, out) + } + if (v === undefined) return V.fail(`${fn} needs an argument`) + switch (fn) { + case "list": { + // Each group's values, in row order. + const starts = new Int32Array(groups + 1) + for (let i = 0; i < n; i++) starts[ids[i] + 1]++ + for (let k = 0; k < groups; k++) starts[k + 1] += starts[k] + const at = starts.slice(0, groups) + const rows = new Int32Array(n) + for (let i = 0; i < n; i++) rows[at[ids[i]]++] = i + return V.list(S.list(S.field("item", v.type)), starts, V.take(v, rows)) + } + case "count": + return V.f64(S.uint32, validCounts(v, g)) + case "first": + case "last": { + const rows = new Int32Array(groups).fill(-1) + if (fn === "first") { for (let i = n - 1; i >= 0; i--) rows[ids[i]] = i } + else for (let i = 0; i < n; i++) rows[ids[i]] = i + return V.take(v, rows) + } + case "countDistinct": { + const values = Hash.ids(v) + const distinct = Hash.groups([V.i32(S.int32, ids), V.i32(S.int32, values.ids)], n) + const out = new Float64Array(groups) + for (let d = 0; d < distinct.count; d++) out[ids[distinct.firstRow[d]]]++ + return V.f64(S.uint32, out) + } + case "sum": + return sum(v, g) + case "mean": + return mean(v, g) + case "min": + case "max": + return minMax(fn, v, g) + case "std": + case "variance": + return variance(fn, v, g) + case "median": + return median(v, g) + } +} + +const sum = (v: V.Vector, g: Grouping): V.Vector => { + const ids = g.ids + const n = ids.length + const groups = g.count + const type = D.sumType(v.type) + switch (v.kind) { + case "null": + return V.i32(S.int32, new Int32Array(groups)) + case "f64": { + const x = v.values + const acc = new Float64Array(groups) + const valid = v.valid + if (valid === undefined) { for (let i = 0; i < n; i++) acc[ids[i]] += x[i] } + else for (let i = 0; i < n; i++) if (valid[i] === 1) acc[ids[i]] += x[i] + V.wrapInPlace(type, acc) + V.roundFloat32InPlace(type, acc) + return V.f64(type, acc) + } + case "i32": { + const x = v.values + // Accumulate in Float64 (exact up to 2^53), then wrap to Int32. + const acc = new Float64Array(groups) + const valid = v.valid + if (valid === undefined) { for (let i = 0; i < n; i++) acc[ids[i]] += x[i] } + else for (let i = 0; i < n; i++) if (valid[i] === 1) acc[ids[i]] += x[i] + const out = new Int32Array(groups) + for (let k = 0; k < groups; k++) out[k] = Number(BigInt.asIntN(32, BigInt(acc[k]))) + return V.i32(type, out) + } + case "bool": { + const x = v.values + const acc = new Float64Array(groups) + for (let i = 0; i < n; i++) if (V.isValid(v, i)) acc[ids[i]] += x[i] + return V.f64(S.uint32, acc) + } + case "i64": { + const x = v.values + const acc = new Array(groups).fill(I0) + const valid = v.valid + for (let i = 0; i < n; i++) if (valid === undefined || valid[i] === 1) acc[ids[i]] += x[i] + const out = new BigInt64Array(groups) + for (let k = 0; k < groups; k++) out[k] = BigInt.asIntN(64, acc[k]) + return V.i64(type, out) + } + default: + return V.fail(`sum is not defined for ${D.describe(v.type)}`) + } +} + +/** Every value as a float, with validity. */ +const asFloat = (v: V.Vector): { readonly x: Float64Array; readonly valid: Uint8Array | undefined } => { + switch (v.kind) { + case "f64": + return { x: v.values, valid: v.valid } + case "i32": + return { x: Float64Array.from(v.values), valid: v.valid } + case "i64": + if (v.type._tag === "Decimal") return { x: Dec.toFloat(v.values, v.type.scale), valid: v.valid } + return { + x: V.isUnsigned64(v.type) + ? Float64Array.from(new BigUint64Array(v.values.buffer, v.values.byteOffset, v.length), Number) + : Float64Array.from(v.values, Number), + valid: v.valid + } + case "bool": + return { x: Float64Array.from(v.values), valid: v.valid } + case "null": + return { x: new Float64Array(v.length), valid: new Uint8Array(v.length) } + default: + return V.fail(`expected a numeric column, got ${D.describe(v.type)}`) + } +} + +/** + * Per-group count and sum of the valid values, as floats: the parts of a + * mean that combine across row ranges. + * + * @internal + */ +export const sumCount = ( + v: V.Vector, + g: Grouping +): { readonly sum: Float64Array; readonly count: Float64Array } => { + const ids = g.ids + const n = ids.length + const { valid, x } = asFloat(v) + const sum = new Float64Array(g.count) + const count = new Float64Array(g.count) + if (valid === undefined) { + for (let i = 0; i < n; i++) { + const k = ids[i] + sum[k] += x[i] + count[k]++ + } + } else { + for (let i = 0; i < n; i++) { + if (valid[i] === 0) continue + const k = ids[i] + sum[k] += x[i] + count[k]++ + } + } + return { sum, count } +} + +/** The mean column from per-group sums and counts (consumes `sum`). @internal */ +export const meanFrom = (inputType: S.DataType, sum: Float64Array, count: Float64Array): V.Vector => { + for (let k = 0; k < sum.length; k++) sum[k] /= count[k] + const type = D.meanType(inputType) + V.roundFloat32InPlace(type, sum) + return V.f64(type, sum, invalidWhereZero(count)) +} + +const mean = (v: V.Vector, g: Grouping): V.Vector => { + const { count, sum } = sumCount(v, g) + return meanFrom(v.type, sum, count) +} + +/** + * Per-group count, mean and sum of squared deviations from the mean (M2): + * the parts of a variance that combine across row ranges. + * + * @internal + */ +export const moments = ( + v: V.Vector, + g: Grouping +): { readonly count: Float64Array; readonly mean: Float64Array; readonly m2: Float64Array } => { + const ids = g.ids + const n = ids.length + const { valid, x } = asFloat(v) + const groups = g.count + const { count, sum } = sumCount(v, g) + const m = sum + for (let k = 0; k < groups; k++) m[k] = sum[k] / count[k] + const m2 = new Float64Array(groups) + for (let i = 0; i < n; i++) { + if (valid !== undefined && valid[i] === 0) continue + const d = x[i] - m[ids[i]] + m2[ids[i]] += d * d + } + return { count, mean: m, m2 } +} + +/** The std or variance column from per-group counts and M2. @internal */ +export const varianceFrom = ( + fn: "std" | "variance", + inputType: S.DataType, + count: Float64Array, + m2: Float64Array +): V.Vector => { + const groups = count.length + const out = new Float64Array(groups) + let outValid: Uint8Array | undefined + for (let k = 0; k < groups; k++) { + if (count[k] < 2) { + if (outValid === undefined) outValid = new Uint8Array(groups).fill(1) + outValid[k] = 0 + continue + } + const vr = m2[k] / (count[k] - 1) + out[k] = fn === "std" ? Math.sqrt(vr) : vr + } + const type = D.meanType(inputType) + V.roundFloat32InPlace(type, out) + return V.f64(type, out, outValid) +} + +const variance = (fn: "std" | "variance", v: V.Vector, g: Grouping): V.Vector => { + const { count, m2 } = moments(v, g) + return varianceFrom(fn, v.type, count, m2) +} + +const minMax = (fn: "min" | "max", v: V.Vector, g: Grouping): V.Vector => { + if (v.kind === "list" || v.kind === "struct") return V.fail(`${fn} is not defined for ${v.type._tag} columns`) + const ids = g.ids + const n = ids.length + const groups = g.count + const isMin = fn === "min" + switch (v.kind) { + case "null": + return V.nulls(groups) + case "f64": { + const x = v.values + const valid = v.valid + const acc = new Float64Array(groups).fill(NaN) + const seen = new Uint8Array(groups) + for (let i = 0; i < n; i++) { + if (valid !== undefined && valid[i] === 0) continue + const k = ids[i] + const a = x[i] + seen[k] = 1 + if (a !== a) continue + const c = acc[k] + if (c !== c || (isMin ? a < c : a > c)) acc[k] = a + } + return V.f64(v.type, acc, V.compactValid(seen)) + } + case "i32": { + const x = v.values + const valid = v.valid + const acc = new Int32Array(groups) + const seen = new Uint8Array(groups) + for (let i = 0; i < n; i++) { + if (valid !== undefined && valid[i] === 0) continue + const k = ids[i] + const a = x[i] + if (seen[k] === 0) { + seen[k] = 1 + acc[k] = a + } else if (isMin ? a < acc[k] : a > acc[k]) acc[k] = a + } + return V.i32(v.type, acc, V.compactValid(seen)) + } + case "bool": { + const x = v.values + const acc = new Uint8Array(groups).fill(isMin ? 1 : 0) + const seen = new Uint8Array(groups) + for (let i = 0; i < n; i++) { + if (!V.isValid(v, i)) continue + const k = ids[i] + seen[k] = 1 + acc[k] = isMin ? acc[k] & x[i] : acc[k] | x[i] + } + return V.bool(acc, V.compactValid(seen)) + } + case "i64": + case "str": { + // Track the best row per group, then gather it. + const best = new Int32Array(groups).fill(-1) + const cmp = rowComparator(v) + for (let i = 0; i < n; i++) { + if (!V.isValid(v, i)) continue + const k = ids[i] + const b = best[k] + if (b < 0 || (isMin ? cmp(i, b) < 0 : cmp(i, b) > 0)) best[k] = i + } + return V.take(v, best) + } + } +} + +const rowComparator = (v: V.I64 | V.Str): (a: number, b: number) => number => { + if (v.kind === "str") { + const d = v.data + const o = v.offsets + return (a, b) => compareBytes(d, o[a], o[a + 1], d, o[b], o[b + 1]) + } + const x = v.values + if (V.isUnsigned64(v.type)) { + const u = new BigUint64Array(x.buffer, x.byteOffset, x.length) + return (a, b) => (u[a] < u[b] ? -1 : u[a] > u[b] ? 1 : 0) + } + return (a, b) => (x[a] < x[b] ? -1 : x[a] > x[b] ? 1 : 0) +} + +const median = (v: V.Vector, g: Grouping): V.Vector => { + const ids = g.ids + const n = ids.length + const groups = g.count + const { valid, x } = asFloat(v) + // Counting sort of the present values by group, then a numeric sort per group. + const start = new Int32Array(groups + 1) + for (let i = 0; i < n; i++) if (valid === undefined || valid[i] === 1) start[ids[i] + 1]++ + for (let k = 0; k < groups; k++) start[k + 1] += start[k] + const values = new Float64Array(start[groups]) + const fill = start.slice(0, groups) + for (let i = 0; i < n; i++) if (valid === undefined || valid[i] === 1) values[fill[ids[i]]++] = x[i] + const out = new Float64Array(groups) + const outValid = new Uint8Array(groups) + for (let k = 0; k < groups; k++) { + const s = start[k] + const len = start[k + 1] - s + if (len === 0) continue + const group = values.subarray(s, s + len) + group.sort() + const mid = len >> 1 + out[k] = len % 2 === 1 ? group[mid] : (group[mid - 1] + group[mid]) / 2 + outValid[k] = 1 + } + const type = D.meanType(v.type) + V.roundFloat32InPlace(type, out) + return V.f64(type, out, V.compactValid(outValid)) +} diff --git a/packages/effect/src/dataframe/internal/decimal.ts b/packages/effect/src/dataframe/internal/decimal.ts new file mode 100644 index 00000000000..dd944426afd --- /dev/null +++ b/packages/effect/src/dataframe/internal/decimal.ts @@ -0,0 +1,194 @@ +/** + * Decimals: unscaled 64-bit integers with a scale (`value / 10^scale`), so + * up to 18 significant digits. Arithmetic follows Polars' result types + * (precision 38, scales added for products, the larger scale for sums) and + * fails instead of overflowing 64 bits. Rescaling down rounds half to even. + * + * @internal + */ +import * as S from "../../arrow/ArrowSchema.ts" +import * as BigDecimal from "../../BigDecimal.ts" +import * as V from "./vector.ts" + +const powers: Array = [] +/** `10^k` as a bigint. @internal */ +export const pow10 = (k: number): bigint => (powers[k] ??= BigInt(10) ** BigInt(k)) + +const MAX = BigInt("9223372036854775807") +const MIN = -BigInt("9223372036854775808") + +/** @internal */ +export const isDecimal = (t: S.DataType): t is S.Decimal => t._tag === "Decimal" + +/** Precision of results, as in Polars. @internal */ +export const PRECISION = 38 + +/** @internal */ +export const decimal = (scale: number): S.Decimal => S.decimal(PRECISION, scale) + +/** The common type of a decimal and another numeric type. @internal */ +export const supertype = (a: S.DataType, b: S.DataType): S.DataType | undefined => { + if (isDecimal(a) && isDecimal(b)) { + return a.scale === b.scale && a.precision === b.precision + ? a + : decimal(Math.max(a.scale, b.scale)) + } + const [d, other] = isDecimal(a) ? [a, b] : [b as S.Decimal, a] + if (other._tag === "Int") return decimal(d.scale) + if (other._tag === "Float") return S.float64 + return undefined +} + +const checked = (x: bigint): bigint => (x > MAX || x < MIN ? V.fail("decimal value does not fit 18 digits") : x) + +/** Rounds `x / 10^k` half to even (`BigDecimal.round`). */ +const divRound = (x: bigint, k: number): bigint => + k === 0 ? x : BigDecimal.round(BigDecimal.make(x, k), { scale: 0, mode: "half-even" }).value + +/** `unscaled / 10^scale` as a decimal string, all `scale` digits shown. @internal */ +export const format = (unscaled: bigint, scale: number): string => { + const negative = unscaled < BigInt(0) + const digits = (negative ? -unscaled : unscaled).toString().padStart(scale + 1, "0") + const text = scale === 0 ? digits : `${digits.slice(0, digits.length - scale)}.${digits.slice(digits.length - scale)}` + return negative ? `-${text}` : text +} + +const DECIMAL_SYNTAX = /^\s*([+-]?)(\d*)(?:\.(\d*))?\s*$/ + +/** Parses a decimal string to unscaled units of `scale`; `undefined` when invalid. @internal */ +export const parse = (s: string, scale: number): bigint | undefined => { + const m = DECIMAL_SYNTAX.exec(s) + if (m === null || (m[2] === "" && (m[3] ?? "") === "")) return undefined + const fraction = m[3] ?? "" + const all = BigInt(`${m[2] || "0"}${fraction}`) + const x = fraction.length >= scale ? divRound(all, fraction.length - scale) : all * pow10(scale - fraction.length) + return m[1] === "-" ? -x : x +} + +const digitsOf = (x: bigint): number => (x < BigInt(0) ? -x : x).toString().length + +const fits = (x: bigint, t: S.Decimal): boolean => x <= MAX && x >= MIN && digitsOf(x) <= t.precision + +/** + * Casts to and from decimals; `undefined` when neither side is a decimal. + * Values that don't fit the target become null. + * + * @internal + */ +export const cast = (v: V.Vector, target: S.DataType): V.Vector | undefined => { + const from = v.type + if (!isDecimal(from) && !isDecimal(target)) return undefined + const n = v.length + const valid = new Uint8Array(n) + if (isDecimal(target)) { + const out = new BigInt64Array(n) + const put = (i: number, x: bigint | undefined) => { + if (x !== undefined && fits(x, target)) { + out[i] = x + valid[i] = 1 + } + } + for (let i = 0; i < n; i++) { + if (!V.isValid(v, i)) continue + switch (v.kind) { + case "i64": + put( + i, + isDecimal(from) + ? from.scale >= target.scale + ? divRound(v.values[i], from.scale - target.scale) + : v.values[i] * pow10(target.scale - from.scale) + : v.values[i] * pow10(target.scale) + ) + break + case "i32": + put(i, BigInt(v.values[i]) * pow10(target.scale)) + break + case "f64": { + const x = v.values[i] + if (Number.isFinite(x)) { + const scaled = x * 10 ** target.scale + const r = Math.round(scaled) + // round half to even + const even = Math.abs(scaled - Math.trunc(scaled)) === 0.5 && r % 2 !== 0 ? r - Math.sign(scaled) : r + put(i, BigInt(even)) + } + break + } + case "str": + put(i, parse(V.decodeString(v, i), target.scale)) + break + case "bool": + put(i, BigInt(v.values[i]) * pow10(target.scale)) + break + } + } + return V.i64(target, out, valid) + } + // from a decimal + if (v.kind !== "i64" || !isDecimal(from)) return V.fail("expected decimal storage") + const scale = pow10(from.scale) + switch (target._tag) { + case "Float": { + const out = new Float64Array(n) + for (let i = 0; i < n; i++) out[i] = Number(v.values[i]) / 10 ** from.scale + return V.f64(target, out, v.valid) + } + case "Int": { + if (target.bitWidth === 64) { + return V.i64(target, BigInt64Array.from(v.values, (x) => x / scale), v.valid) + } + const lo = target.signed ? -(2 ** (target.bitWidth - 1)) : 0 + const hi = target.signed ? 2 ** (target.bitWidth - 1) - 1 : 2 ** target.bitWidth - 1 + const out = new Float64Array(n) + for (let i = 0; i < n; i++) { + if (!V.isValid(v, i)) continue + const x = Number(v.values[i] / scale) + if (x >= lo && x <= hi) { + out[i] = x + valid[i] = 1 + } + } + return V.kindOf(target) === "f64" ? V.f64(target, out, valid) : V.i32(target, Int32Array.from(out), valid) + } + case "Utf8": + return V.stringsFrom( + S.utf8, + Array.from({ length: n }, (_, i) => (V.isValid(v, i) ? format(v.values[i], from.scale) : null)) + ) + case "Bool": + return V.bool(Uint8Array.from(v.values, (x) => (x === BigInt(0) ? 0 : 1)), v.valid) + default: + return V.fail(`cannot cast a decimal to ${target._tag}`) + } +} + +/** Decimal addition, subtraction and multiplication of unscaled values. @internal */ +export const arith = ( + op: "add" | "sub" | "mul", + a: BigInt64Array, + aScale: number, + b: BigInt64Array, + bScale: number, + scalar: boolean +): { readonly values: BigInt64Array; readonly type: S.Decimal } => { + const n = a.length + const out = new BigInt64Array(n) + if (op === "mul") { + for (let i = 0; i < n; i++) out[i] = checked(a[i] * (scalar ? b[0] : b[i])) + return { values: out, type: decimal(aScale + bScale) } + } + const scale = Math.max(aScale, bScale) + const fa = pow10(scale - aScale) + const fb = pow10(scale - bScale) + for (let i = 0; i < n; i++) { + const x = a[i] * fa + const y = (scalar ? b[0] : b[i]) * fb + out[i] = checked(op === "add" ? x + y : x - y) + } + return { values: out, type: decimal(scale) } +} + +/** Unscaled values as floats. @internal */ +export const toFloat = (values: BigInt64Array, scale: number): Float64Array => + Float64Array.from(values, (x) => Number(x) / 10 ** scale) diff --git a/packages/effect/src/dataframe/internal/dtype.ts b/packages/effect/src/dataframe/internal/dtype.ts new file mode 100644 index 00000000000..b423ebda5fd --- /dev/null +++ b/packages/effect/src/dataframe/internal/dtype.ts @@ -0,0 +1,187 @@ +/** + * Result types of expressions: numeric supertypes and literal adoption, + * following Polars where the TypeScript types allow it. + * + * @internal + */ +import * as S from "../../arrow/ArrowSchema.ts" +import * as Dec from "./decimal.ts" +import * as T from "./temporal.ts" + +/** + * A literal that has not committed to a width yet. Like Polars' dynamic + * literals, an integer literal adopts the integer type of the column it + * meets, and a float literal adopts Float32 next to a Float32 column. + * + * @internal + */ +export type Dynamic = "int" | "float" | undefined + +/** @internal */ +export const isNumeric = (t: S.DataType): t is S.Int | S.Float => t._tag === "Int" || t._tag === "Float" +/** @internal */ +export const isInteger = (t: S.DataType): t is S.Int => t._tag === "Int" +/** @internal */ +export const isFloat = (t: S.DataType): t is S.Float => t._tag === "Float" +/** @internal */ +export const isBinaryLike = (t: S.DataType): boolean => t._tag === "Utf8" || t._tag === "Binary" + +/** @internal */ +export const sameType = (a: S.DataType, b: S.DataType): boolean => { + if (a._tag !== b._tag) return false + switch (a._tag) { + case "Int": + return a.bitWidth === (b as S.Int).bitWidth && a.signed === (b as S.Int).signed + case "Float": + return a.bitWidth === (b as S.Float).bitWidth + case "Date": + return a.unit === (b as S.Date).unit + case "Time": + return a.unit === (b as S.Time).unit + case "Timestamp": + return a.unit === (b as S.Timestamp).unit && (a.timezone ?? null) === ((b as S.Timestamp).timezone ?? null) + case "Duration": + return a.unit === (b as S.Duration).unit + case "Decimal": + return a.precision === (b as S.Decimal).precision && a.scale === (b as S.Decimal).scale + case "List": + case "LargeList": + return sameType(a.child.type, (b as S.List).child.type) + case "Struct": { + const fields = (b as S.Struct).fields + return a.fields.length === fields.length && + a.fields.every((f, i) => f.name === fields[i].name && sameType(f.type, fields[i].type)) + } + default: + return true + } +} + +/** The default type of a literal value. @internal */ +export const literalType = (value: unknown): { readonly type: S.DataType; readonly dynamic: Dynamic } => { + switch (typeof value) { + case "number": + return Number.isInteger(value) && value >= -2147483648 && value <= 2147483647 && !Object.is(value, -0) + ? { type: S.int32, dynamic: "int" } + : { type: S.float64, dynamic: "float" } + case "bigint": + return { type: S.int64, dynamic: undefined } + case "string": + return { type: S.utf8, dynamic: undefined } + case "boolean": + return { type: S.bool, dynamic: undefined } + default: + return { type: S.nullType, dynamic: undefined } + } +} + +const intSupertype = (a: S.Int, b: S.Int): S.DataType => { + if (a.signed === b.signed) return a.bitWidth >= b.bitWidth ? a : b + const [s, u] = a.signed ? [a, b] : [b, a] + if (s.bitWidth > u.bitWidth) return s + // Polars widens to the next signed width; beyond 32 bits that is a + // bigint, which the number-typed API can't express, so use Float64. + return u.bitWidth === 8 ? S.int16 : u.bitWidth === 16 ? S.int32 : u.bitWidth === 32 ? S.float64 : S.int64 +} + +/** + * The common type of two operands. Returns `undefined` when they have none. + * + * @internal + */ +export const supertype = ( + a: S.DataType, + ad: Dynamic, + b: S.DataType, + bd: Dynamic +): { readonly type: S.DataType; readonly dynamic: Dynamic } | undefined => { + if (a._tag === "Null") return { type: b, dynamic: bd } + if (b._tag === "Null") return { type: a, dynamic: ad } + if (Dec.isDecimal(a) || Dec.isDecimal(b)) { + const t = Dec.supertype(a, b) + return t === undefined ? undefined : { type: t, dynamic: undefined } + } + if (isNumeric(a) && isNumeric(b)) { + if (ad !== undefined && bd !== undefined) { + return ad === "float" || bd === "float" + ? { type: S.float64, dynamic: "float" } + : { type: S.int32, dynamic: "int" } + } + if (ad !== undefined || bd !== undefined) { + const [lit, col] = ad !== undefined ? [ad, b] : [bd, a] + if (lit === "int") return { type: col, dynamic: undefined } + // float literal + return { type: col._tag === "Float" ? col : S.float64, dynamic: undefined } + } + if (a._tag === "Int" && b._tag === "Int") return { type: intSupertype(a, b), dynamic: undefined } + const f32 = (t: S.DataType) => t._tag === "Float" ? t.bitWidth === 32 : (t as S.Int).bitWidth <= 16 + return { type: f32(a) && f32(b) ? S.float32 : S.float64, dynamic: undefined } + } + if (T.isTemporal(a) || T.isTemporal(b)) { + if (sameType(a, b)) return { type: a, dynamic: undefined } + const t = T.supertype(a, b) + return t === undefined ? undefined : { type: t, dynamic: undefined } + } + if (sameType(a, b)) return { type: a, dynamic: undefined } + return undefined +} + +/** Type of a true division. @internal */ +export const divisionType = (t: S.DataType): S.DataType => + t._tag === "Float" && t.bitWidth === 32 ? S.float32 : S.float64 + +/** Type of `sum` over a column of type `t`. @internal */ +export const sumType = (t: S.DataType): S.DataType => { + if (t._tag === "Bool") return S.uint32 + if (t._tag === "Decimal") return Dec.decimal(t.scale) + if (t._tag === "Int" && t.bitWidth < 32) return S.int32 + return t +} + +/** Type of `mean`, `median`, `std` and `var` over a column of type `t`. @internal */ +export const meanType = (t: S.DataType): S.DataType => t._tag === "Float" && t.bitWidth === 32 ? S.float32 : S.float64 + +/** @internal */ +export const describe = (t: S.DataType): string => { + switch (t._tag) { + case "Int": + return `${t.signed ? "Int" : "UInt"}${t.bitWidth}` + case "Float": + return `Float${t.bitWidth}` + case "Timestamp": + return `Timestamp[${unitName[t.unit]}${t.timezone === undefined ? "" : `, ${t.timezone}`}]` + case "Duration": + return `Duration[${unitName[t.unit]}]` + case "Decimal": + return `Decimal[${t.precision}, ${t.scale}]` + default: + return t._tag + } +} + +const unitName: Record = { second: "s", millisecond: "ms", microsecond: "us", nanosecond: "ns" } + +/** Names accepted by `Expr.cast`. @internal */ +export const castTargets: Record = { + Int8: S.int8, + Int16: S.int16, + Int32: S.int32, + Int64: S.int64, + UInt8: S.uint8, + UInt16: S.uint16, + UInt32: S.uint32, + UInt64: S.uint64, + Float32: S.float32, + Float64: S.float64, + Utf8: S.utf8, + Bool: S.bool, + Date: S.date("day"), + "Timestamp[s]": S.timestamp("second"), + "Timestamp[ms]": S.timestamp("millisecond"), + "Timestamp[us]": S.timestamp("microsecond"), + "Timestamp[ns]": S.timestamp("nanosecond"), + "Duration[s]": S.duration("second"), + "Duration[ms]": S.duration("millisecond"), + "Duration[us]": S.duration("microsecond"), + "Duration[ns]": S.duration("nanosecond") +} diff --git a/packages/effect/src/dataframe/internal/expr.ts b/packages/effect/src/dataframe/internal/expr.ts new file mode 100644 index 00000000000..9cc548ecbe2 --- /dev/null +++ b/packages/effect/src/dataframe/internal/expr.ts @@ -0,0 +1,526 @@ +/** + * Expression trees and their evaluation over a frame. + * + * @internal + */ +import * as S from "../../arrow/ArrowSchema.ts" +import * as BigDecimal from "../../BigDecimal.ts" +import * as DateTime from "../../DateTime.ts" +import * as Duration from "../../Duration.ts" +import { pipeArguments } from "../../Pipeable.ts" +import type * as Plan from "../Plan.ts" +import * as Agg from "./aggregate.ts" +import * as D from "./dtype.ts" +import * as Hash from "./hash.ts" +import * as K from "./kernels.ts" +import * as Nested from "./nested.ts" +import * as T from "./temporal.ts" +import * as V from "./vector.ts" +import * as W from "./window.ts" + +/** @internal */ +export const ExprTypeId = "~effect/dataframe/Expr" +/** @internal */ +export const AggregateTypeId = "~effect/dataframe/Expr/Aggregate" + +/** @internal */ +export type Node = Plan.Expression + +/** @internal */ +export interface Handle { + readonly node: Node +} + +const ExprProto = { + [ExprTypeId]: { _A: (_: never) => _ }, + pipe() { + return pipeArguments(this, arguments) + }, + toJSON(this: Handle) { + return { _id: "Expr", node: this.node } + }, + toString(this: Handle) { + return show(this.node) + } +} + +const AggregateProto = { + ...ExprProto, + [AggregateTypeId]: { _A: (_: never) => _ }, + toJSON(this: Handle) { + return { _id: "Aggregate", node: this.node } + } +} + +/** @internal */ +export const makeExpr = (node: Node): E => { + const self = Object.create(ExprProto) + self.node = node + return self +} + +/** @internal */ +export const makeAggregate = (node: Node): E => { + const self = Object.create(AggregateProto) + self.node = node + return self +} + +/** @internal */ +export const isExpr = (u: unknown): u is Handle => typeof u === "object" && u !== null && ExprTypeId in u + +/** @internal */ +export const isAggregate = (u: unknown): u is Handle => typeof u === "object" && u !== null && AggregateTypeId in u + +/** @internal */ +export const toNode = (u: unknown): Node => isExpr(u) ? u.node : literalNode(u) + +/** + * A plan literal for a JS value: `DateTime`s become microsecond timestamps, + * `Duration`s microsecond durations and `BigDecimal`s decimals. + * + * @internal + */ +export const literalNode = (u: unknown): Plan.Literal => { + if (DateTime.isDateTime(u)) { + return { + _tag: "Literal", + value: BigInt(DateTime.toEpochMillis(u)) * BigInt(1000), + type: { _tag: "Timestamp", unit: "microsecond" } + } + } + if (Duration.isDuration(u)) { + const nanos = Duration.toNanos(u) + if (nanos._tag === "None") return V.fail("infinite durations cannot be literals") + const micros = nanos.value / BigInt(1000) + return { _tag: "Literal", value: micros, type: { _tag: "Duration", unit: "microsecond" } } + } + if (BigDecimal.isBigDecimal(u)) { + const digits = (u.value < BigInt(0) ? -u.value : u.value).toString().length + return { + _tag: "Literal", + value: u.value, + type: { _tag: "Decimal", precision: Math.max(digits, u.scale, 1), scale: u.scale } + } + } + return { _tag: "Literal", value: u as Plan.Value } +} + +/** @internal */ +export const call = ( + fn: Plan.FunctionName, + args: ReadonlyArray, + options?: { readonly [key: string]: unknown } +): E => makeExpr({ _tag: "Call", fn, args: args.map(toNode), options }) + +/** Readable form of an expression, used by `toString`. @internal */ +export const show = (node: Node): string => { + switch (node._tag) { + case "Column": + return `col(${JSON.stringify(node.name)})` + case "Literal": { + const value = typeof node.value === "bigint" ? `${node.value}n` : JSON.stringify(node.value) ?? "undefined" + return node.type === undefined ? value : `${node.type._tag}(${value})` + } + case "Call": + return `${node.fn}(${node.args.map(show).join(", ")}${ + node.options === undefined + ? "" + : `, ${JSON.stringify(node.options, (_, v) => typeof v === "bigint" ? `${v}n` : v)}` + })` + case "AggregateCall": + return `${node.fn}(${node.arg === undefined ? "" : show(node.arg)})` + case "Over": + return `over(${show(node.expression)}, [${node.partitionBy.map(show).join(", ")}])` + } +} + +// ----------------------------------------------------------------------------- +// Evaluation +// ----------------------------------------------------------------------------- + +/** What an expression is evaluated against. @internal */ +export interface Context { + readonly length: number + readonly column: (name: string) => V.Vector + /** The partitions of an enclosing `Over`: window functions run per partition. */ + readonly partition?: W.Partition | undefined +} + +const literalOfType = (value: unknown, type: Plan.LiteralType): K.Value => { + const big = typeof value === "bigint" ? value : BigInt(value as number) + switch (type._tag) { + case "Date": + return K.scalar(V.i32(S.date("day"), Int32Array.of(Number(big)))) + case "Timestamp": + return K.scalar(V.i64(S.timestamp(type.unit, type.timezone), BigInt64Array.of(big))) + case "Duration": + return K.scalar(V.i64(S.duration(type.unit), BigInt64Array.of(big))) + case "Decimal": + return K.scalar(V.i64(S.decimal(type.precision, type.scale), BigInt64Array.of(big))) + } +} + +const literal = (value: unknown): K.Value => { + const { dynamic, type } = D.literalType(value) + switch (typeof value) { + case "number": + return K.scalar( + type._tag === "Int" ? V.i32(type, Int32Array.of(value)) : V.f64(type, Float64Array.of(value)), + dynamic + ) + case "bigint": + return K.scalar(V.i64(type, BigInt64Array.of(BigInt.asIntN(64, value)))) + case "string": + return K.scalar(V.stringsFrom(S.utf8, [value])) + case "boolean": + return K.scalar(V.bool(Uint8Array.of(value ? 1 : 0))) + default: + return K.scalar(V.nulls(1)) + } +} + +/** @internal */ +export const evaluate = (node: Node, ctx: Context): K.Value => { + switch (node._tag) { + case "Column": + return K.column(ctx.column(node.name)) + case "Literal": + return node.type === undefined || node.value === null ? literal(node.value) : literalOfType(node.value, node.type) + case "AggregateCall": { + // Inside `over`: the partition's aggregate on each of its rows. + if (ctx.partition === undefined) return V.fail(`aggregate ${show(node)} can only be used in agg or over`) + if (node.arg === undefined && node.fn !== "count") return V.fail(`${node.fn} needs an argument`) + const arg = node.arg === undefined ? undefined : evaluateColumn(node.arg, { ...ctx, partition: undefined }) + const values = Agg.aggregate(node.arg === undefined ? "len" : node.fn, arg, ctx.partition) + return K.column(V.take(values, ctx.partition.ids)) + } + case "Call": + return evaluateCall(node, ctx) + case "Over": { + const keys = node.partitionBy.map((k) => evaluateColumn(k, { ...ctx, partition: undefined })) + const g = Hash.groups(keys, ctx.length) + return evaluate(node.expression, { ...ctx, partition: { ids: g.ids, count: g.count } }) + } + } +} + +const arity: { readonly [F in Plan.FunctionName]: readonly [number, number] } = { + add: [2, 2], + subtract: [2, 2], + multiply: [2, 2], + divide: [2, 2], + floorDivide: [2, 2], + mod: [2, 2], + negate: [1, 1], + abs: [1, 1], + sqrt: [1, 1], + round: [1, 1], + eq: [2, 2], + neq: [2, 2], + lt: [2, 2], + lte: [2, 2], + gt: [2, 2], + gte: [2, 2], + between: [3, 3], + isIn: [1, 1], + and: [2, 2], + or: [2, 2], + not: [1, 1], + isNull: [1, 1], + isNotNull: [1, 1], + isNaN: [1, 1], + fillNull: [2, 2], + coalesce: [1, Infinity], + when: [3, 3], + cast: [1, 1], + length: [1, 1], + toLowerCase: [1, 1], + toUpperCase: [1, 1], + includes: [1, 1], + startsWith: [1, 1], + endsWith: [1, 1], + concat: [1, Infinity], + cumSum: [1, 1], + cumMin: [1, 1], + cumMax: [1, 1], + cumCount: [1, 1], + shift: [1, 1], + diff: [1, 1], + rank: [1, 1], + rollingSum: [1, 1], + rollingMean: [1, 1], + rollingMin: [1, 1], + rollingMax: [1, 1], + year: [1, 1], + month: [1, 1], + day: [1, 1], + hour: [1, 1], + minute: [1, 1], + second: [1, 1], + millisecond: [1, 1], + weekday: [1, 1], + ordinalDay: [1, 1], + truncate: [1, 1], + totalDays: [1, 1], + totalHours: [1, 1], + totalMinutes: [1, 1], + totalSeconds: [1, 1], + totalMilliseconds: [1, 1], + listLength: [1, 1], + listGet: [1, 1], + listSum: [1, 1], + listMean: [1, 1], + listMin: [1, 1], + listMax: [1, 1], + listContains: [2, 2], + field: [1, 1], + struct: [1, Infinity] +} + +const windowFunctions: ReadonlySet = new Set([ + "cumSum", + "cumMin", + "cumMax", + "cumCount", + "shift", + "diff", + "rank", + "rollingSum", + "rollingMean", + "rollingMin", + "rollingMax" +]) + +/** + * Whether an expression's value on a row depends only on that row: no + * window functions, `over` or aggregates. Only such expressions commute + * with filters and slices. + * + * @internal + */ +export const isRowWise = (node: Node): boolean => { + switch (node._tag) { + case "Column": + case "Literal": + return true + case "Call": + return !windowFunctions.has(node.fn) && node.args.every(isRowWise) + case "AggregateCall": + case "Over": + return false + } +} + +const arithmetic = { + add: "add", + subtract: "sub", + multiply: "mul", + divide: "div", + floorDivide: "floorDiv", + mod: "mod" +} as const + +const evaluateCall = (node: Plan.Call, ctx: Context): K.Value => { + const n = ctx.length + const bounds = arity[node.fn] + if (bounds === undefined) return V.fail(`unknown function ${String(node.fn)}`) + if (node.args.length < bounds[0] || node.args.length > bounds[1]) { + return V.fail( + `${node.fn} takes ${ + bounds[0] === bounds[1] + ? bounds[0] + : bounds[1] === Infinity + ? `at least ${bounds[0]}` + : `${bounds[0]}–${bounds[1]}` + } argument(s), got ${node.args.length}` + ) + } + const arg = (i: number) => evaluate(node.args[i], ctx) + const options = node.options ?? {} + const option = (name: keyof Plan.CallOptions): A => { + const value = options[name] + return value === undefined ? V.fail(`${node.fn} needs the ${name} option`) : value as A + } + switch (node.fn) { + case "add": + case "subtract": + case "multiply": + case "divide": + case "floorDivide": + case "mod": + return K.arith(arithmetic[node.fn], arg(0), arg(1), n) + case "negate": + return K.unaryNumeric("neg", arg(0)) + case "abs": + case "sqrt": + return K.unaryNumeric(node.fn, arg(0)) + case "round": + return K.unaryNumeric("round", arg(0), options.decimals ?? 0) + case "eq": + case "neq": + case "lt": + case "lte": + case "gt": + case "gte": + return K.compare(node.fn, arg(0), arg(1), n) + case "and": + case "or": + return K.logical(node.fn, arg(0), arg(1), n) + case "not": + return K.not(arg(0)) + case "isNull": + return K.isNull(arg(0), false) + case "isNotNull": + return K.isNull(arg(0), true) + case "isNaN": + return K.isNaN_(arg(0)) + case "fillNull": + case "coalesce": + return K.coalesce(node.args.map((a) => evaluate(a, ctx)), n) + case "when": + return K.when(arg(0), arg(1), arg(2), n) + case "cast": { + const to = option("to") + if (to === "Decimal") { + return K.castValue(arg(0), S.decimal(option("precision"), option("scale"))) + } + const target = D.castTargets[to] + if (target === undefined) return V.fail(`unknown cast target ${to}`) + return K.castValue(arg(0), target) + } + case "isIn": + return K.isIn(arg(0), option>("values")) + case "between": { + const x = arg(0) + return K.logical("and", K.compare("gte", x, arg(1), n), K.compare("lte", x, arg(2), n), n) + } + case "length": + return K.strLength(arg(0)) + case "toLowerCase": + return K.strMap(arg(0), (s) => s.toLowerCase()) + case "toUpperCase": + return K.strMap(arg(0), (s) => s.toUpperCase()) + case "includes": + case "startsWith": + case "endsWith": + return K.strMatch(arg(0), node.fn, option("pattern")) + case "concat": + return K.strConcat(node.args.map((a) => evaluate(a, ctx)), n) + case "year": + case "month": + case "day": + case "hour": + case "minute": + case "second": + case "millisecond": + case "weekday": + case "ordinalDay": + return K.column(T.field(node.fn, K.materialize(arg(0), n))) + case "truncate": + return K.column(T.truncate(K.materialize(arg(0), n), option("every"))) + case "listLength": + return K.column(Nested.length(K.materialize(arg(0), n))) + case "listGet": + return K.column(Nested.get(K.materialize(arg(0), n), option("index"))) + case "listSum": + case "listMean": + case "listMin": + case "listMax": { + const fn = node.fn === "listSum" ? "sum" : node.fn === "listMean" ? "mean" : node.fn === "listMin" ? "min" : "max" + return K.column(Nested.aggregate(fn, K.materialize(arg(0), n))) + } + case "listContains": + return K.column(Nested.contains(K.materialize(arg(0), n), arg(1))) + case "field": + return K.column(Nested.field(K.materialize(arg(0), n), option("name"))) + case "struct": { + const names = option>("names") + if (names.length !== node.args.length) return V.fail("struct needs one name per argument") + return K.column(Nested.struct(names, node.args.map((a) => K.materialize(evaluate(a, ctx), n)), n)) + } + case "totalDays": + case "totalHours": + case "totalMinutes": + case "totalSeconds": + case "totalMilliseconds": + return K.column(T.total(node.fn, K.materialize(arg(0), n))) + case "cumSum": + case "cumMin": + case "cumMax": + case "cumCount": + case "shift": + case "diff": + case "rank": + case "rollingSum": + case "rollingMean": + case "rollingMin": + case "rollingMax": { + // The argument is evaluated over all rows, the window per partition. + const x = K.materialize(evaluate(node.args[0], { ...ctx, partition: undefined }), n) + const p = ctx.partition ?? W.whole(n) + switch (node.fn) { + case "cumSum": + return K.column(W.cumSum(x, p)) + case "cumMin": + case "cumMax": + return K.column(W.cumMinMax(node.fn, x, p)) + case "cumCount": + return K.column(W.cumCount(x, p)) + case "shift": + return K.column(W.shift(x, p, options.n ?? 1)) + case "diff": + return K.arith("sub", K.column(x), K.column(W.shift(x, p, options.n ?? 1)), n) + case "rank": + return K.column(W.rank(x, p, options.method ?? "average", options.descending ?? false)) + default: { + const window = option("window") + return K.column(W.rolling(node.fn, x, p, window, options.minPeriods ?? window)) + } + } + } + } +} + +/** Column names an expression reads. @internal */ +export const columns = (node: Node, out: Set = new Set()): Set => { + switch (node._tag) { + case "Column": + out.add(node.name) + break + case "Call": + for (const a of node.args) columns(a, out) + break + case "AggregateCall": + if (node.arg !== undefined) columns(node.arg, out) + break + case "Over": + columns(node.expression, out) + for (const k of node.partitionBy) columns(k, out) + break + } + return out +} + +/** Replaces column references. @internal */ +export const mapColumns = (node: Node, f: (name: string) => Node): Node => { + switch (node._tag) { + case "Column": + return f(node.name) + case "Literal": + return node + case "Call": + return { ...node, args: node.args.map((a) => mapColumns(a, f)) } + case "AggregateCall": + return node.arg === undefined ? node : { ...node, arg: mapColumns(node.arg, f) } + case "Over": + return { + ...node, + expression: mapColumns(node.expression, f), + partitionBy: node.partitionBy.map((k) => mapColumns(k, f)) + } + } +} + +/** Evaluates to a column of the context's length. @internal */ +export const evaluateColumn = (node: Node, ctx: Context): V.Vector => K.materialize(evaluate(node, ctx), ctx.length) diff --git a/packages/effect/src/dataframe/internal/frame.ts b/packages/effect/src/dataframe/internal/frame.ts new file mode 100644 index 00000000000..e98966250bf --- /dev/null +++ b/packages/effect/src/dataframe/internal/frame.ts @@ -0,0 +1,757 @@ +/** + * The DataFrame representation and its operations. + * + * A frame is a list of named columns of equal length. Each column is a + * physical vector plus an optional *selection*: the row indices of the + * vector that make up the column. `filter`, `sort`, `take` and joins only + * compose selections; the values are gathered the first time a column is + * read, and the gathered vector is cached on the column. Columns sharing a + * selection keep sharing it, so composing it happens once per operation. + * + * @internal + */ +import * as ArrowData from "../../arrow/ArrowData.ts" +import * as S from "../../arrow/ArrowSchema.ts" +import * as ArrowTable from "../../arrow/ArrowTable.ts" +import * as Equal from "../../Equal.ts" +import { format } from "../../Formatter.ts" +import * as Hash_ from "../../Hash.ts" +import { NodeInspectSymbol, toJson } from "../../Inspectable.ts" +import { pipeArguments } from "../../Pipeable.ts" +import * as Agg from "./aggregate.ts" +import * as D from "./dtype.ts" +import * as E from "./expr.ts" +import * as Hash from "./hash.ts" +import * as K from "./kernels.ts" +import * as Sort from "./sort.ts" +import * as V from "./vector.ts" + +/** @internal */ +export const TypeId = "~effect/dataframe/DataFrame" + +/** @internal */ +export class Column { + readonly name: string + readonly vector: V.Vector + readonly selection: Int32Array | undefined + /** Whether the column may hold nulls (the Arrow field's `nullable`). */ + readonly nullable: boolean + private gathered: V.Vector | undefined + constructor(name: string, vector: V.Vector, selection: Int32Array | undefined, nullable?: boolean) { + this.name = name + this.vector = vector + this.selection = selection + this.nullable = nullable ?? (vector.kind === "null" || vector.valid !== undefined) + } + get type(): S.DataType { + return this.vector.type + } + /** The column's values, gathered through its selection (cached). */ + values(): V.Vector { + if (this.selection === undefined) return this.vector + if (this.gathered === undefined) this.gathered = V.take(this.vector, this.selection) + return this.gathered + } + rename(name: string): Column { + const c = new Column(name, this.vector, this.selection, this.nullable) + c.gathered = this.gathered + return c + } +} + +/** @internal */ +export interface Frame { + readonly [TypeId]: unknown + readonly columns: ReadonlyArray + readonly length: number + readonly schema: S.Schema +} + +const Proto = { + [TypeId]: { _Row: (_: never) => _ }, + pipe() { + return pipeArguments(this, arguments) + }, + [Symbol.iterator](this: Frame) { + return rows(this)[Symbol.iterator]() + }, + toJSON(this: Frame) { + return { + _id: "DataFrame", + schema: this.columns.map((c) => ({ name: c.name, type: D.describe(c.type) })), + length: this.length, + rows: toJson(toRows(head(this, 10))) + } + }, + toString(this: Frame) { + return render(this) + }, + [NodeInspectSymbol](this: Frame) { + return render(this) + }, + [Equal.symbol](this: Frame, that: unknown): boolean { + return isFrame(that) && equals(this, that) + }, + [Hash_.symbol](this: Frame): number { + let h = Hash_.number(this.length) + for (const c of this.columns) h = Hash_.combine(h)(Hash_.string(c.name)) + return h + } +} + +/** @internal */ +export const isFrame = (u: unknown): u is Frame => typeof u === "object" && u !== null && TypeId in u + +/** @internal */ +export const make = (columns: ReadonlyArray, length: number): Frame => { + const self = Object.create(Proto) + self.columns = columns + self.length = length + let schema: S.Schema | undefined + Object.defineProperty(self, "schema", { + enumerable: true, + get: () => schema ??= S.make(columns.map((c) => S.field(c.name, c.type, { nullable: c.nullable }))) + }) + return self +} + +/** @internal */ +export const fromVectors = (names: ReadonlyArray, vectors: ReadonlyArray, length?: number): Frame => { + const n = length ?? vectors[0]?.length ?? 0 + return make(names.map((name, i) => new Column(name, vectors[i], undefined)), n) +} + +/** @internal */ +export const columnIndex = (self: Frame, name: string): number => { + const i = self.columns.findIndex((c) => c.name === name) + return i < 0 ? V.fail(`unknown column ${JSON.stringify(name)}`) : i +} + +/** @internal */ +export const column = (self: Frame, name: string): Column => self.columns[columnIndex(self, name)] + +/** @internal */ +export const context = (self: Frame): E.Context => ({ + length: self.length, + column: (name) => column(self, name).values() +}) + +// ----------------------------------------------------------------------------- +// Arrow interop and rows +// ----------------------------------------------------------------------------- + +/** @internal */ +export const fromArrowTable = (table: ArrowTable.Table): Frame => { + const length = table.batches.reduce((s, b) => s + b.length, 0) + const vectors = table.batches.length === 0 + ? table.schema.fields.map((f) => K.nullsOf(V.storageType(f.type), 0)) + : table.schema.fields.map((_, c) => V.concat(table.batches.map((b) => V.fromArrowData(b.columns[c])))) + return make( + table.schema.fields.map((f, c) => + new Column( + f.name, + vectors[c], + undefined, + f.nullable || vectors[c].kind === "null" || vectors[c].valid !== undefined + ) + ), + length + ) +} + +/** @internal */ +export const toArrowTable = (self: Frame): ArrowTable.Table => { + const schema = S.make(self.columns.map((c) => S.field(c.name, c.type, { nullable: c.nullable }))) + const data = self.columns.map((c) => V.toArrowData(c.values())) + return ArrowTable.make(schema, [ArrowTable.recordBatch(schema, self.length, data)]) +} + +/** @internal */ +export const rows = (self: Frame): Iterable => ArrowTable.rows(toArrowTable(self)) + +/** @internal */ +export const toRows = (self: Frame): Array => ArrowTable.toRows(toArrowTable(self)) + +/** @internal */ +export const toArray = (self: Frame, name: string): Array => { + const vector = column(self, name).values() + const field = S.field(name, vector.type) + const t = ArrowTable.make(S.make([field]), [ + ArrowTable.recordBatch(S.make([field]), vector.length, [V.toArrowData(vector)]) + ]) + return ArrowTable.toRows(t).map((r) => r[name]) +} + +void ArrowData + +// ----------------------------------------------------------------------------- +// Row selection +// ----------------------------------------------------------------------------- + +/** Re-selects every column through `indices` (positions in the current frame; -1 = null). @internal */ +export const take = (self: Frame, indices: Int32Array): Frame => { + let missing = false + for (let i = 0; i < indices.length; i++) { + if (indices[i] < 0) { + missing = true + break + } + } + const composed = new Map() + const columns = self.columns.map((c) => { + let sel = composed.get(c.selection) + if (sel === undefined) { + if (c.selection === undefined) sel = indices + else { + const base = c.selection + sel = new Int32Array(indices.length) + for (let i = 0; i < indices.length; i++) { + const j = indices[i] + sel[i] = j < 0 ? -1 : base[j] + } + } + composed.set(c.selection, sel) + } + return new Column(c.name, c.vector, sel, c.nullable || missing) + }) + return make(columns, indices.length) +} + +const maskIndices = (mask: V.Bool): Int32Array => { + const n = mask.length + const m = mask.values + const v = mask.valid + const out = new Int32Array(n) + let k = 0 + if (v === undefined) { + for (let i = 0; i < n; i++) { + out[k] = i + k += m[i] + } + } else { + for (let i = 0; i < n; i++) { + out[k] = i + k += m[i] & v[i] + } + } + return out.subarray(0, k) +} + +/** @internal */ +export const filter = (self: Frame, predicate: E.Node): Frame => { + const value = E.evaluate(predicate, context(self)) + const mask = K.materialize(value, self.length) + if (mask.kind === "null") return take(self, new Int32Array(0)) + if (mask.kind !== "bool") return V.fail(`filter predicate must be boolean, got ${D.describe(mask.type)}`) + const indices = maskIndices(mask) + return indices.length === self.length ? self : take(self, indices) +} + +/** @internal */ +export const slice = (self: Frame, start: number, end: number): Frame => { + const n = self.length + const s = Math.max(0, Math.min(n, start)) + const e = Math.max(s, Math.min(n, end)) + if (s === 0 && e === n) return self + return make( + self.columns.map((c) => + c.selection === undefined + ? new Column(c.name, V.slice(c.vector, s, e), undefined, c.nullable) + : new Column(c.name, c.vector, c.selection.subarray(s, e), c.nullable) + ), + e - s + ) +} + +/** @internal */ +export const head = (self: Frame, n: number): Frame => slice(self, 0, n) + +/** @internal */ +export const tail = (self: Frame, n: number): Frame => slice(self, Math.max(0, self.length - n), self.length) + +/** @internal */ +export const sort = ( + self: Frame, + keys: ReadonlyArray<{ readonly node: E.Node; readonly descending: boolean }>, + nullsLast: boolean, + limit?: number +): Frame => { + const ctx = context(self) + const sortKeys = keys.map((k) => ({ vector: E.evaluateColumn(k.node, ctx), descending: k.descending })) + return take( + self, + limit === undefined + ? Sort.permutation(sortKeys, nullsLast, self.length) + : Sort.topK(sortKeys, nullsLast, self.length, limit) + ) +} + +// ----------------------------------------------------------------------------- +// Projection +// ----------------------------------------------------------------------------- + +/** @internal */ +export const pick = (self: Frame, names: ReadonlyArray): Frame => + make(names.map((n) => column(self, n)), self.length) + +/** @internal */ +export const omit = (self: Frame, names: ReadonlyArray): Frame => { + const drop = new Set(names) + return make(self.columns.filter((c) => !drop.has(c.name)), self.length) +} + +/** @internal */ +export const rename = (self: Frame, mapping: { readonly [name: string]: string | undefined }): Frame => + make( + self.columns.map((c) => { + const to = mapping[c.name] + return to === undefined ? c : c.rename(to) + }), + self.length + ) + +const evaluateAll = (self: Frame, exprs: { readonly [name: string]: E.Node }): Array => { + const ctx = context(self) + return Object.keys(exprs).map((name) => new Column(name, E.evaluateColumn(exprs[name], ctx), undefined)) +} + +/** @internal */ +export const select = (self: Frame, exprs: { readonly [name: string]: E.Node }): Frame => { + const columns = evaluateAll(self, exprs) + // A frame of only scalars (e.g. `select(() => ({ one: Expr.lit(1) }))`) keeps the frame's length. + return make(columns, self.length) +} + +/** @internal */ +export const withColumns = (self: Frame, exprs: { readonly [name: string]: E.Node }): Frame => { + const added = evaluateAll(self, exprs) + const byName = new Map(added.map((c) => [c.name, c])) + const columns: Array = self.columns.map((c) => { + const replacement = byName.get(c.name) + if (replacement !== undefined) byName.delete(c.name) + return replacement ?? c + }) + for (const c of added) if (byName.has(c.name)) columns.push(c) + return make(columns, self.length) +} + +// ----------------------------------------------------------------------------- +// Aggregation +// ----------------------------------------------------------------------------- + +const aggregateColumns = ( + self: Frame, + aggs: { readonly [name: string]: E.Node }, + grouping: Agg.Grouping +): Array => { + const ctx = context(self) + return Object.keys(aggs).map((name) => { + const node = aggs[name] + if (node._tag !== "AggregateCall") return V.fail(`${name} is not an aggregate expression`) + if (node.arg === undefined && node.fn !== "count") return V.fail(`${node.fn} needs an argument`) + const input = node.arg === undefined ? undefined : E.evaluateColumn(node.arg, ctx) + return new Column(name, Agg.aggregate(node.arg === undefined ? "len" : node.fn, input, grouping), undefined) + }) +} + +/** @internal */ +export const groupBy = (self: Frame, keys: ReadonlyArray, aggs: { readonly [name: string]: E.Node }): Frame => { + const keyVectors = keys.map((k) => column(self, k).values()) + const groups = Hash.groups(keyVectors, self.length) + const keyColumns = keys.map((k, i) => + new Column(k, V.take(keyVectors[i], groups.firstRow), undefined, column(self, k).nullable) + ) + return make([...keyColumns, ...aggregateColumns(self, aggs, groups)], groups.count) +} + +/** @internal */ +export const aggregate = (self: Frame, aggs: { readonly [name: string]: E.Node }): Frame => + make(aggregateColumns(self, aggs, { ids: new Int32Array(self.length), count: 1 }), 1) + +// ----------------------------------------------------------------------------- +// Joins +// ----------------------------------------------------------------------------- + +/** @internal */ +export type JoinHow = "inner" | "left" | "full" | "semi" | "anti" + +const unifyKeys = (l: V.Vector, r: V.Vector): [V.Vector, V.Vector] => { + if (D.sameType(l.type, r.type)) return [l, r] + const st = D.supertype(l.type, undefined, r.type, undefined) + if (st === undefined) return V.fail(`cannot join ${D.describe(l.type)} with ${D.describe(r.type)}`) + return [K.cast(l, st.type), K.cast(r, st.type)] +} + +/** + * Key ids for both sides of a join; equal keys get equal ids and rows with a + * null key (or, on the left, a key the right side lacks) get -1. + */ +const joinIds = ( + pairs: ReadonlyArray<[V.Vector, V.Vector]>, + nl: number, + nr: number +): { leftIds: Int32Array; rightIds: Int32Array; count: number; nulls: undefined } => { + if (pairs.length === 1 && pairs[0][0].kind === "i32" && pairs[0][1].kind === "i32") { + // One Int32 key: index the right keys directly by value when their range is small. + const l = pairs[0][0] as V.I32 + const r = pairs[0][1] as V.I32 + let min = 2147483647 + let max = -2147483648 + for (let j = 0; j < nr; j++) { + if (r.valid !== undefined && r.valid[j] === 0) continue + const v = r.values[j] + if (v < min) min = v + if (v > max) max = v + } + if (max < min || max - min < Math.max(1 << 16, 2 * (nl + nr))) { + const range = max < min ? 0 : max - min + 1 + const table = new Int32Array(range).fill(-1) + const rightIds = new Int32Array(nr) + let count = 0 + for (let j = 0; j < nr; j++) { + if (r.valid !== undefined && r.valid[j] === 0) { + rightIds[j] = -1 + continue + } + const k = r.values[j] - min + let id = table[k] + if (id < 0) id = table[k] = count++ + rightIds[j] = id + } + const leftIds = new Int32Array(nl) + const lv = l.values + const la = l.valid + for (let i = 0; i < nl; i++) { + const k = lv[i] - min + leftIds[i] = (la !== undefined && la[i] === 0) || k < 0 || k >= range ? -1 : table[k] + } + return { leftIds, rightIds, count, nulls: undefined } + } + } + // General case: ids over both sides at once. + const joint = pairs.map(([l, r]) => V.concat([l, r])) + const g = Hash.groups(joint, nl + nr) + const nulls = Hash.anyNull(joint, nl + nr) + const leftIds = g.ids.slice(0, nl) + const rightIds = g.ids.slice(nl) + if (nulls !== undefined) { + for (let i = 0; i < nl; i++) if (nulls[i] === 1) leftIds[i] = -1 + for (let j = 0; j < nr; j++) if (nulls[nl + j] === 1) rightIds[j] = -1 + } + return { leftIds, rightIds, count: g.count, nulls: undefined } +} + +/** @internal */ +export const join = (self: Frame, that: Frame, on: ReadonlyArray, how: JoinHow): Frame => { + const nl = self.length + const nr = that.length + const pairs = on.map((k) => unifyKeys(column(self, k).values(), column(that, k).values())) + const { leftIds, nulls, rightIds, count } = joinIds(pairs, nl, nr) + // CSR buckets of right rows per id + const start = new Int32Array(count + 1) + for (let j = 0; j < nr; j++) if (rightIds[j] >= 0) start[rightIds[j] + 1]++ + for (let k = 0; k < count; k++) start[k + 1] += start[k] + const rows = new Int32Array(start[count]) + const fill = start.slice(0, count) + for (let j = 0; j < nr; j++) if (rightIds[j] >= 0) rows[fill[rightIds[j]]++] = j + const ids = leftIds + void nulls + + if (how === "semi" || how === "anti") { + const want = how === "semi" ? 1 : 0 + const keep = new Int32Array(nl) + let kept = 0 + for (let i = 0; i < nl; i++) { + const id = ids[i] + const has = id >= 0 && start[id + 1] > start[id] ? 1 : 0 + keep[kept] = i + kept += has === want ? 1 : 0 + } + return kept === nl ? self : take(self, keep.subarray(0, kept)) + } + + let total = 0 + const keepUnmatched = how === "inner" ? 0 : 1 + for (let i = 0; i < nl; i++) { + const id = ids[i] + const m = id < 0 ? 0 : start[id + 1] - start[id] + total += m === 0 ? keepUnmatched : m + } + let matchedRight: Uint8Array | undefined + if (how === "full") matchedRight = new Uint8Array(nr) + let li = new Int32Array(total) + let ri = new Int32Array(total) + let k = 0 + for (let i = 0; i < nl; i++) { + const id = ids[i] + const s = id < 0 ? 0 : start[id] + const e = id < 0 ? 0 : start[id + 1] + if (s === e) { + if (keepUnmatched === 1) { + li[k] = i + ri[k++] = -1 + } + continue + } + for (let t = s; t < e; t++) { + li[k] = i + ri[k++] = rows[t] + } + if (matchedRight !== undefined) { for (let t = s; t < e; t++) matchedRight[rows[t]] = 1 } + } + if (matchedRight !== undefined) { + let extra = 0 + for (let j = 0; j < nr; j++) extra += matchedRight[j] ^ 1 + if (extra > 0) { + const l2 = new Int32Array(total + extra) + const r2 = new Int32Array(total + extra) + l2.set(li) + r2.set(ri) + for (let j = 0; j < nr; j++) { + if (matchedRight[j] === 0) { + l2[k] = -1 + r2[k++] = j + } + } + li = l2 + ri = r2 + } + } + const left = take(self, li) + const keySet = new Set(on) + const right = take(make(that.columns.filter((c) => !keySet.has(c.name)), nr), ri) + let leftColumns: ReadonlyArray = left.columns + if (how === "full") { + // Coalesce the key columns: rows that only exist on the right take their keys from there. + const rightKeys = take(make(on.map((k) => column(that, k)), nr), ri) + leftColumns = left.columns.map((c) => { + if (!keySet.has(c.name)) return c + const [l, r] = unifyKeys(c.values(), column(rightKeys, c.name).values()) + return new Column(c.name, K.coalesce([K.column(l), K.column(r)], li.length).vector, undefined) + }) + } + return make([...leftColumns, ...right.columns], li.length) +} + +// ----------------------------------------------------------------------------- +// Other row operations +// ----------------------------------------------------------------------------- + +/** @internal */ +export const unique = (self: Frame, names: ReadonlyArray | undefined): Frame => { + const keys = (names ?? self.columns.map((c) => c.name)).map((n) => column(self, n).values()) + const g = Hash.groups(keys, self.length) + return g.count === self.length ? self : take(self, g.firstRow) +} + +/** @internal */ +export const dropNulls = (self: Frame, names: ReadonlyArray | undefined): Frame => { + const keys = (names ?? self.columns.map((c) => c.name)).map((n) => column(self, n).values()) + const nulls = Hash.anyNull(keys, self.length) + if (nulls === undefined) return self + const keep: Array = [] + for (let i = 0; i < self.length; i++) if (nulls[i] === 0) keep.push(i) + return take(self, Int32Array.from(keep)) +} + +/** @internal */ +export const concat = (frames: ReadonlyArray): Frame => { + const first = frames[0] + const columns = first.columns.map((c) => { + const parts = frames.map((f) => column(f, c.name).values()) + const type = parts.reduce((t, p) => { + const st = D.supertype(t, undefined, p.type, undefined) + return st === undefined ? V.fail(`cannot concatenate ${D.describe(t)} with ${D.describe(p.type)}`) : st.type + }, parts[0].type) + return new Column(c.name, V.concat(parts.map((p) => K.cast(p, type))), undefined) + }) + return make(columns, frames.reduce((s, f) => s + f.length, 0)) +} + +// ----------------------------------------------------------------------------- +// Equality and rendering +// ----------------------------------------------------------------------------- + +/** @internal */ +export const equals = (a: Frame, b: Frame): boolean => { + if (a === b) return true + if (a.length !== b.length || a.columns.length !== b.columns.length) return false + for (let c = 0; c < a.columns.length; c++) { + const x = a.columns[c] + const y = b.columns[c] + if (x.name !== y.name || !D.sameType(x.type, y.type)) return false + const vx = x.values() + const vy = y.values() + const eq = K.compare("eq", K.column(vx), K.column(vy), a.length).vector as V.Bool + for (let i = 0; i < a.length; i++) { + const px = V.isValid(vx, i) + const py = V.isValid(vy, i) + if (px !== py || (px && eq.values[i] === 0)) return false + } + } + return true +} + +const cell = (u: unknown): string => { + if (u === null || u === undefined) return "null" + if (typeof u === "string") return JSON.stringify(u) + if (typeof u === "bigint") return `${u}` + if (typeof u === "number") return Object.is(u, -0) ? "-0" : String(u) + return format(u) +} + +/** A text table of the first and last rows. @internal */ +export const render = (self: Frame, maxRows = 10): string => { + const n = self.length + const shown = n <= maxRows ? toRows(self) : [...toRows(head(self, maxRows / 2)), ...toRows(tail(self, maxRows / 2))] + const header = self.columns.map((c) => c.name) + const types = self.columns.map((c) => D.describe(c.type)) + const body = shown.map((r) => header.map((h) => cell(r[h]))) + if (n > maxRows) body.splice(maxRows / 2, 0, header.map(() => "…")) + const widths = header.map((h, i) => Math.max(h.length, types[i].length, ...body.map((r) => r[i].length))) + const line = (cells: ReadonlyArray) => `│ ${cells.map((c, i) => c.padEnd(widths[i])).join(" │ ")} │` + const rule = (l: string, m: string, r: string) => `${l}${widths.map((w) => "─".repeat(w + 2)).join(m)}${r}` + return [ + `DataFrame (${n} ${n === 1 ? "row" : "rows"} × ${header.length} ${header.length === 1 ? "column" : "columns"})`, + rule("┌", "┬", "┐"), + line(header), + line(types), + rule("├", "┼", "┤"), + ...body.map(line), + rule("└", "┴", "┘") + ].join("\n") +} + +// ----------------------------------------------------------------------------- +// Reshaping +// ----------------------------------------------------------------------------- + +/** @internal */ +export const pivot = ( + self: Frame, + options: { + readonly on: string + readonly index: ReadonlyArray + readonly values: string + readonly aggregate: Agg.AggFn + readonly columns?: ReadonlyArray | undefined + } +): Frame => { + const n = self.length + const indexVectors = options.index.map((k) => column(self, k).values()) + const rows = Hash.groups(indexVectors, n) + const on = column(self, options.on).values() + const cols = Hash.groups([on], n) + const cells = Hash.groups([V.i32(S.int32, rows.ids), V.i32(S.int32, cols.ids)], n) + const input = column(self, options.values).values() + const fn = options.aggregate + const values = Agg.aggregate(fn, fn === "len" ? undefined : input, { ids: cells.ids, count: cells.count }) + // The aggregate of no rows, for cells without rows, at index `cells.count`. + const empty = Agg.aggregate(fn, fn === "len" ? undefined : V.slice(input, 0, 0), { + ids: new Int32Array(0), + count: 1 + }) + const all = V.concat([values, K.cast(empty, values.type)]) + const cellOf = new Int32Array(rows.count * cols.count).fill(cells.count) + for (let c = 0; c < cells.count; c++) { + const r = cells.firstRow[c] + cellOf[rows.ids[r] * cols.count + cols.ids[r]] = c + } + const names = K.cast(V.take(on, cols.firstRow), S.utf8) + const nameOf = (j: number): string => V.isValid(names, j) ? V.decodeString(names as V.Str, j) : "null" + const byName = new Map() + for (let j = 0; j < cols.count; j++) if (!byName.has(nameOf(j))) byName.set(nameOf(j), j) + const outputs = options.columns ?? Array.from({ length: cols.count }, (_, j) => nameOf(j)) + const columns = outputs.map((name) => { + const j = byName.get(name) + const indices = new Int32Array(rows.count) + for (let r = 0; r < rows.count; r++) indices[r] = j === undefined ? cells.count : cellOf[r * cols.count + j] + return new Column(name, V.take(all, indices), undefined) + }) + const keys = options.index.map((k, i) => + new Column(k, V.take(indexVectors[i], rows.firstRow), undefined, column(self, k).nullable) + ) + return make([...keys, ...columns], rows.count) +} + +/** + * One row per list element, the other columns repeated. A null list gives one + * row with a null element; an empty list gives no rows (Polars). + * + * @internal + */ +export const explode = (self: Frame, name: string): Frame => { + const v = column(self, name).values() + if (v.kind !== "list") return V.fail(`explode is defined for list columns, not ${D.describe(v.type)}`) + const rows: Array = [] + const elements: Array = [] + for (let i = 0; i < v.length; i++) { + if (!V.isValid(v, i)) { + rows.push(i) + elements.push(-1) + continue + } + for (let e = v.offsets[i]; e < v.offsets[i + 1]; e++) { + rows.push(i) + elements.push(e) + } + } + const repeated = take(self, Int32Array.from(rows)) + const exploded = new Column(name, V.take(v.child, Int32Array.from(elements)), undefined) + return make(repeated.columns.map((c) => (c.name === name ? exploded : c)), rows.length) +} + +/** The fields of a struct column as columns, in its place. @internal */ +export const unnest = (self: Frame, name: string): Frame => { + const v = column(self, name).values() + if (v.kind !== "struct") return V.fail(`unnest is defined for struct columns, not ${D.describe(v.type)}`) + const fields = (v.type as S.Struct).fields + const existing = new Set(self.columns.map((c) => c.name).filter((n) => n !== name)) + const columns: Array = [] + for (const c of self.columns) { + if (c.name !== name) { + columns.push(c) + continue + } + fields.forEach((f, k) => { + if (existing.has(f.name)) V.fail(`unnest would duplicate the column ${JSON.stringify(f.name)}`) + const child = v.children[k] + const valid = v.valid === undefined + ? (child.kind === "null" ? undefined : child.valid) + : V.andValid(child.kind === "null" ? undefined : child.valid, v.valid, v.length) + columns.push(new Column(f.name, V.withValid(child, valid), undefined)) + }) + } + return make(columns, self.length) +} + +/** @internal */ +export const unpivot = ( + self: Frame, + options: { + readonly index: ReadonlyArray + readonly on: ReadonlyArray + readonly variable: string + readonly value: string + } +): Frame => { + const n = self.length + const k = options.on.length + const repeat = new Int32Array(n * k) + for (let b = 0; b < k; b++) for (let i = 0; i < n; i++) repeat[b * n + i] = i + const keys = options.index.map((name) => { + const c = column(self, name) + return new Column(name, V.take(c.values(), repeat), undefined, c.nullable) + }) + const parts = options.on.map((name) => column(self, name).values()) + const type = parts.reduce((t, p) => { + const st = D.supertype(t, undefined, p.type, undefined) + return st === undefined ? V.fail(`cannot unpivot ${D.describe(t)} with ${D.describe(p.type)}`) : st.type + }, parts.length === 0 ? S.nullType : parts[0].type) + const variable = V.stringsFrom(S.utf8, options.on.flatMap((name) => Array.from({ length: n }, () => name))) + const value = k === 0 ? V.nulls(0) : V.concat(parts.map((p) => K.cast(p, type))) + return make( + [...keys, new Column(options.variable, variable, undefined, false), new Column(options.value, value, undefined)], + n * k + ) +} diff --git a/packages/effect/src/dataframe/internal/hash.ts b/packages/effect/src/dataframe/internal/hash.ts new file mode 100644 index 00000000000..67c8e27ba4f --- /dev/null +++ b/packages/effect/src/dataframe/internal/hash.ts @@ -0,0 +1,444 @@ +/** + * Dense group ids: every distinct key gets an id in order of first + * appearance. Grouping, joins, `unique` and `countDistinct` all build on it. + * + * Keys hash with open addressing over typed arrays. Multi-column keys are + * grouped column by column and the per-column ids combined, so every column + * type needs only its own single-column table. + * + * Float keys follow Polars: `-0` and `0` are one key, and so are all NaNs. + * Null is a key of its own. + * + * @internal + */ +import * as V from "./vector.ts" + +/** @internal */ +export interface Groups { + /** Group id of every row. */ + readonly ids: Int32Array + /** Number of groups. */ + readonly count: number + /** The first row of every group, by group id. */ + readonly firstRow: Int32Array +} + +const mix = (h: number): number => { + h = Math.imul(h ^ (h >>> 16), 0x85ebca6b) + h = Math.imul(h ^ (h >>> 13), 0xc2b2ae35) + return (h ^ (h >>> 16)) >>> 0 +} + +/** Largest table the direct (array-indexed) strategy may allocate for `n` rows. */ +const directLimit = (n: number) => Math.max(1 << 16, n * 2) + +/** + * Ids for keys in `[0, range)`, by direct indexing: no hashing, one array + * read per row. + */ +const directIds = ( + x: Int32Array, + offset: number, + range: number, + valid: Uint8Array | undefined, + n: number +): Groups => { + const table = new Int32Array(range + 1).fill(-1) + const ids = new Int32Array(n) + let firstRow: Int32Array = new Int32Array(Math.min(range + 1, Math.max(64, n))) + let count = 0 + for (let i = 0; i < n; i++) { + const k = valid !== undefined && valid[i] === 0 ? range : x[i] - offset + let id = table[k] + if (id < 0) { + if (count === firstRow.length) firstRow = growInt(firstRow) + id = table[k] = count + firstRow[count++] = i + } + ids[i] = id + } + return { ids, count, firstRow: firstRow.subarray(0, count) } +} + +/** Ids for one Int32 column. @internal */ +const int32Ids = (x: Int32Array, valid: Uint8Array | undefined, n: number): Groups => { + let min = 2147483647 + let max = -2147483648 + if (valid === undefined) { + for (let i = 0; i < n; i++) { + const v = x[i] + if (v < min) min = v + if (v > max) max = v + } + } else { + for (let i = 0; i < n; i++) { + if (valid[i] === 0) continue + const v = x[i] + if (v < min) min = v + if (v > max) max = v + } + } + if (max >= min && max - min < directLimit(n)) return directIds(x, min, max - min + 1, valid, n) + return hashInt32Ids(x, valid, n) +} + +const hashInt32Ids = (x: Int32Array, valid: Uint8Array | undefined, n: number): Groups => { + const ids = new Int32Array(n) + let firstRow: Int32Array = new Int32Array(64) + let count = 0 + let nullId = -1 + let cap = 1024 + let mask = cap - 1 + let keys = new Int32Array(cap) + let slots = new Int32Array(cap).fill(-1) + for (let i = 0; i < n; i++) { + if (valid !== undefined && valid[i] === 0) { + if (nullId < 0) { + if (count === firstRow.length) firstRow = growInt(firstRow) + nullId = count + firstRow[count++] = i + } + ids[i] = nullId + continue + } + const k = x[i] + let s = mix(k) & mask + while (true) { + const id = slots[s] + if (id === -1) { + if (count === firstRow.length) firstRow = growInt(firstRow) + firstRow[count] = i + slots[s] = count + keys[s] = k + ids[i] = count++ + if (count * 2 > cap) { + const ok = keys + const os = slots + cap <<= 1 + mask = cap - 1 + keys = new Int32Array(cap) + slots = new Int32Array(cap).fill(-1) + for (let t = 0; t < os.length; t++) { + const id2 = os[t] + if (id2 === -1) continue + let u = mix(ok[t]) & mask + while (slots[u] !== -1) u = (u + 1) & mask + slots[u] = id2 + keys[u] = ok[t] + } + } + break + } + if (keys[s] === k) { + ids[i] = id + break + } + s = (s + 1) & mask + } + } + return { ids, count, firstRow: firstRow.subarray(0, count) } +} + +/** Ids for 64-bit keys given as (lo, hi) pairs. @internal */ +const pairIds = (pairs: Int32Array, valid: Uint8Array | undefined, n: number): Groups => { + const ids = new Int32Array(n) + let firstRow: Int32Array = new Int32Array(64) + let count = 0 + let nullId = -1 + let cap = 1024 + let mask = cap - 1 + let keys = new Int32Array(cap * 2) + let slots = new Int32Array(cap).fill(-1) + for (let i = 0; i < n; i++) { + if (valid !== undefined && valid[i] === 0) { + if (nullId < 0) { + if (count === firstRow.length) firstRow = growInt(firstRow) + nullId = count + firstRow[count++] = i + } + ids[i] = nullId + continue + } + const lo = pairs[2 * i] + const hi = pairs[2 * i + 1] + let s = mix(lo ^ Math.imul(hi, 0x9e3779b1)) & mask + while (true) { + const id = slots[s] + if (id === -1) { + if (count === firstRow.length) firstRow = growInt(firstRow) + firstRow[count] = i + slots[s] = count + keys[2 * s] = lo + keys[2 * s + 1] = hi + ids[i] = count++ + if (count * 2 > cap) { + const ok = keys + const os = slots + cap <<= 1 + mask = cap - 1 + keys = new Int32Array(cap * 2) + slots = new Int32Array(cap).fill(-1) + for (let t = 0; t < os.length; t++) { + const id2 = os[t] + if (id2 === -1) continue + const l = ok[2 * t] + const h = ok[2 * t + 1] + let u = mix(l ^ Math.imul(h, 0x9e3779b1)) & mask + while (slots[u] !== -1) u = (u + 1) & mask + slots[u] = id2 + keys[2 * u] = l + keys[2 * u + 1] = h + } + } + break + } + if (keys[2 * s] === lo && keys[2 * s + 1] === hi) { + ids[i] = id + break + } + s = (s + 1) & mask + } + } + return { ids, count, firstRow: firstRow.subarray(0, count) } +} + +const canonicalBits = (x: Float64Array, n: number): Int32Array => { + const out = new Float64Array(n) + for (let i = 0; i < n; i++) { + const v = x[i] + out[i] = v === 0 ? 0 : v !== v ? NaN : v + } + return new Int32Array(out.buffer) +} + +const stringIds = (v: V.Str, n: number): Groups => { + const ids = new Int32Array(n) + let firstRow: Int32Array = new Int32Array(64) + let count = 0 + let nullId = -1 + const valid = v.valid + const d = v.data + const off = v.offsets + let cap = 1024 + let mask = cap - 1 + let slots = new Int32Array(cap).fill(-1) + let hashes = new Int32Array(cap) + for (let i = 0; i < n; i++) { + if (valid !== undefined && valid[i] === 0) { + if (nullId < 0) { + if (count === firstRow.length) firstRow = growInt(firstRow) + nullId = count + firstRow[count++] = i + } + ids[i] = nullId + continue + } + const s0 = off[i] + const e0 = off[i + 1] + let h = 0x811c9dc5 | 0 + for (let k = s0; k < e0; k++) h = Math.imul(h ^ d[k], 0x01000193) + h = mix(h ^ (e0 - s0)) + let s = h & mask + while (true) { + const id = slots[s] + if (id === -1) { + if (count === firstRow.length) firstRow = growInt(firstRow) + firstRow[count] = i + slots[s] = count + hashes[s] = h + ids[i] = count++ + if (count * 2 > cap) { + const os = slots + const oh = hashes + cap <<= 1 + mask = cap - 1 + slots = new Int32Array(cap).fill(-1) + hashes = new Int32Array(cap) + for (let t = 0; t < os.length; t++) { + if (os[t] === -1) continue + let u = oh[t] & mask + while (slots[u] !== -1) u = (u + 1) & mask + slots[u] = os[t] + hashes[u] = oh[t] + } + } + break + } + if (hashes[s] === (h | 0)) { + const r = firstRow[id] + const s1 = off[r] + const e1 = off[r + 1] + if (e1 - s1 === e0 - s0) { + let same = true + for (let k = 0; k < e0 - s0; k++) { + if (d[s0 + k] !== d[s1 + k]) { + same = false + break + } + } + if (same) { + ids[i] = id + break + } + } + } + s = (s + 1) & mask + } + } + return { ids, count, firstRow: firstRow.subarray(0, count) } +} + +const smallIds = (values: Uint8Array, valid: Uint8Array | undefined, n: number): Groups => { + const ids = new Int32Array(n) + const map = new Int32Array(3).fill(-1) + const firstRow = new Int32Array(3) + let count = 0 + for (let i = 0; i < n; i++) { + const k = valid !== undefined && valid[i] === 0 ? 2 : values[i] + let id = map[k] + if (id < 0) { + id = map[k] = count + firstRow[count++] = i + } + ids[i] = id + } + return { ids, count, firstRow: firstRow.subarray(0, count) } +} + +const growInt = (a: Int32Array): Int32Array => { + const out = new Int32Array(a.length * 2) + out.set(a) + return out +} + +/** Dense ids for one column. @internal */ +export const ids = (v: V.Vector): Groups => { + const n = v.length + switch (v.kind) { + case "i32": + return int32Ids(v.values, v.valid, n) + case "f64": + return pairIds(canonicalBits(v.values, n), v.valid, n) + case "i64": + return pairIds(new Int32Array(v.values.buffer, v.values.byteOffset, n * 2), v.valid, n) + case "bool": + return smallIds(v.values, v.valid, n) + case "str": + return stringIds(v, n) + case "null": + return { ids: new Int32Array(n), count: n > 0 ? 1 : 0, firstRow: new Int32Array(n > 0 ? 1 : 0) } + case "list": + case "struct": + return V.fail(`cannot group, join or deduplicate by ${v.type._tag} columns yet`) + } +} + +/** Combines two id columns into ids of the pair. @internal */ +const combine = (a: Groups, b: Groups, n: number): Groups => { + if (a.count === 1) return b + if (b.count === 1) return a + const bc = b.count + const ai = a.ids + const bi = b.ids + if (a.count * bc <= directLimit(n)) { + const keys = new Int32Array(n) + for (let i = 0; i < n; i++) keys[i] = ai[i] * bc + bi[i] + return directIds(keys, 0, a.count * bc, undefined, n) + } + const pairs = new Int32Array(n * 2) + for (let i = 0; i < n; i++) { + pairs[2 * i] = ai[i] + pairs[2 * i + 1] = bi[i] + } + return pairIds(pairs, undefined, n) +} + +/** + * Ids of tuples of per-column ids, in one hashing pass: the tuple hashes as a + * whole and compares column by column against its group's first row. + */ +const tupleIds = (cols: ReadonlyArray, n: number): Groups => { + const ids = new Int32Array(n) + let firstRow: Int32Array = new Int32Array(64) + let count = 0 + let cap = 1024 + let mask = cap - 1 + let slots = new Int32Array(cap).fill(-1) + let hashes = new Int32Array(cap) + const k = cols.length + for (let i = 0; i < n; i++) { + let h = 0x2545f491 + for (let c = 0; c < k; c++) h = Math.imul(h ^ cols[c][i], 0x9e3779b1) ^ (h >>> 15) + h = mix(h) + let s = h & mask + while (true) { + const id = slots[s] + if (id === -1) { + if (count === firstRow.length) firstRow = growInt(firstRow) + firstRow[count] = i + slots[s] = count + hashes[s] = h + ids[i] = count++ + if (count * 2 > cap) { + const os = slots + const oh = hashes + cap <<= 1 + mask = cap - 1 + slots = new Int32Array(cap).fill(-1) + hashes = new Int32Array(cap) + for (let t = 0; t < os.length; t++) { + if (os[t] === -1) continue + let u = oh[t] & mask + while (slots[u] !== -1) u = (u + 1) & mask + slots[u] = os[t] + hashes[u] = oh[t] + } + } + break + } + if (hashes[s] === (h | 0)) { + const r = firstRow[id] + let same = true + for (let c = 0; c < k; c++) { + if (cols[c][i] !== cols[c][r]) { + same = false + break + } + } + if (same) { + ids[i] = id + break + } + } + s = (s + 1) & mask + } + } + return { ids, count, firstRow: firstRow.subarray(0, count) } +} + +/** Dense ids for the rows of several key columns together. @internal */ +export const groups = (keys: ReadonlyArray, n: number): Groups => { + if (keys.length === 0) { + return { ids: new Int32Array(n), count: n > 0 ? 1 : 0, firstRow: new Int32Array(n > 0 ? 1 : 0) } + } + const parts = keys.map(ids) + // Combine by direct indexing while the key space stays small, then hash + // the remaining columns as one tuple. + let acc = parts[0] + let k = 1 + while (k < parts.length && acc.count * parts[k].count <= directLimit(n)) acc = combine(acc, parts[k++], n) + if (k === parts.length) return acc + return tupleIds([acc.ids, ...parts.slice(k).map((p) => p.ids)], n) +} + +/** Rows where any key column is null. @internal */ +export const anyNull = (keys: ReadonlyArray, n: number): Uint8Array | undefined => { + let out: Uint8Array | undefined + for (const k of keys) { + if (k.kind === "null") return new Uint8Array(n).fill(1) + if (k.valid === undefined) continue + if (out === undefined) out = new Uint8Array(n) + for (let i = 0; i < n; i++) out[i] |= k.valid[i] ^ 1 + } + return out +} diff --git a/packages/effect/src/dataframe/internal/kernels.ts b/packages/effect/src/dataframe/internal/kernels.ts new file mode 100644 index 00000000000..10c6b70b11f --- /dev/null +++ b/packages/effect/src/dataframe/internal/kernels.ts @@ -0,0 +1,1255 @@ +/** + * Element-wise kernels. Each kernel has one loop per physical layout and + * operand shape (column/column, column/scalar), written out so that V8 sees + * monomorphic typed-array access in every loop. + * + * @internal + */ +import * as S from "../../arrow/ArrowSchema.ts" +import * as Dec from "./decimal.ts" +import * as D from "./dtype.ts" +import * as T from "./temporal.ts" +import * as V from "./vector.ts" + +const I0 = BigInt(0) +const I1 = BigInt(1) +const I2 = BigInt(2) +const I63 = BigInt(63) +const I64 = BigInt(64) + +/** + * An evaluated expression: a vector of the frame's length, or a scalar (a + * vector of length 1) that broadcasts. + * + * @internal + */ +export interface Value { + readonly vector: V.Vector + readonly scalar: boolean + readonly dynamic: D.Dynamic +} + +/** @internal */ +export const column = (vector: V.Vector): Value => ({ vector, scalar: false, dynamic: undefined }) + +/** @internal */ +export const scalar = (vector: V.Vector, dynamic: D.Dynamic = undefined): Value => ({ vector, scalar: true, dynamic }) + +/** Expands a scalar to `length` slots. @internal */ +export const materialize = (value: Value, length: number): V.Vector => + value.scalar ? V.broadcast(value.vector, length) : value.vector + +const scalarNull = (value: Value): boolean => value.scalar && !V.isValid(value.vector, 0) + +/** Validity of a binary result. @internal */ +const resultValid = (a: Value, b: Value, length: number): Uint8Array | undefined | "all-null" => { + if (scalarNull(a) || scalarNull(b)) return "all-null" + const av = a.scalar ? undefined : a.vector.valid + const bv = b.scalar ? undefined : b.vector.valid + return V.andValid(av, bv, length) +} + +// ----------------------------------------------------------------------------- +// Null vectors of any type +// ----------------------------------------------------------------------------- + +/** A vector of `length` nulls with the layout of `type`. @internal */ +export const nullsOf = (type: S.DataType, length: number): V.Vector => { + const valid = new Uint8Array(length) + switch (V.kindOf(type)) { + case "f64": + return V.f64(type, new Float64Array(length), valid) + case "i32": + return V.i32(type, new Int32Array(length), valid) + case "i64": + return V.i64(type, new BigInt64Array(length), valid) + case "bool": + return V.bool(new Uint8Array(length), valid) + case "str": + return V.str(type, new Int32Array(length + 1), new Uint8Array(0), valid) + case "list": + return V.list(type, new Int32Array(length + 1), nullsOf(V.elementType(type), 0), valid) + case "struct": { + const fields = (type as S.Struct).fields + return V.struct(type, fields.map((f) => nullsOf(f.type, length)), length, valid) + } + default: + return V.nulls(length) + } +} + +// ----------------------------------------------------------------------------- +// Casts +// ----------------------------------------------------------------------------- + +const intRange = (t: S.DataType): [number, number] => { + if (t._tag !== "Int") return [-Infinity, Infinity] + if (t.signed) { + const max = 2 ** (t.bitWidth - 1) + return [-max, max - 1] + } + return [0, 2 ** t.bitWidth - 1] +} + +const FLOAT_SYNTAX = /^[+-]?(?:\d+\.?\d*(?:[eE][+-]?\d+)?|\.\d+(?:[eE][+-]?\d+)?|inf|infinity|nan)$/i +const INT_SYNTAX = /^[+-]?\d+$/ + +const formatNumber = (x: number): string => String(x) + +/** + * Converts `v` to `target`. Values that don't fit become null (Polars' + * non-strict casts). + * + * @internal + */ +export const cast = (v: V.Vector, target: S.DataType): V.Vector => { + if (D.sameType(v.type, target)) return v + const n = v.length + if (v.kind === "null") return nullsOf(target, n) + const temporal = T.cast(v, target) + if (temporal !== undefined) return temporal + const decimal = Dec.cast(v, target) + if (decimal !== undefined) return decimal + const targetKind = V.kindOf(target) + if (targetKind === undefined) return V.fail(`cannot cast to ${target._tag}`) + let valid = v.valid === undefined ? undefined : v.valid.slice() + const invalidate = (i: number) => { + if (valid === undefined) valid = new Uint8Array(n).fill(1) + valid[i] = 0 + } + // Same physical layout and both temporal/integer: reinterpret. + if ( + targetKind === v.kind && (v.kind === "i32" || v.kind === "i64") && + (!D.isInteger(target) || !D.isInteger(v.type)) && target._tag !== "Float" + ) { + return V.withType(v, target) + } + if (targetKind === "str") { + if (target._tag === "Binary" && v.kind === "str") return V.withType(v, target) + const out: Array = new Array(n) + for (let i = 0; i < n; i++) { + if (!V.isValid(v, i)) out[i] = null + else { + switch (v.kind) { + case "f64": + out[i] = formatNumber(v.values[i]) + break + case "i32": + out[i] = String(v.values[i]) + break + case "i64": + out[i] = (V.isUnsigned64(v.type) ? BigInt.asUintN(64, v.values[i]) : v.values[i]).toString() + break + case "bool": + out[i] = v.values[i] === 1 ? "true" : "false" + break + case "str": + out[i] = V.decodeString(v, i) + break + } + } + } + return V.stringsFrom(target, out) + } + if (targetKind === "bool") { + const out = new Uint8Array(n) + switch (v.kind) { + case "f64": + case "i32": { + const a = v.values + for (let i = 0; i < n; i++) out[i] = a[i] !== 0 ? 1 : 0 + break + } + case "i64": { + const a = v.values + for (let i = 0; i < n; i++) out[i] = a[i] !== I0 ? 1 : 0 + break + } + case "str": + for (let i = 0; i < n; i++) { + if (!V.isValid(v, i)) continue + const s = V.decodeString(v, i) + if (s === "true") out[i] = 1 + else if (s !== "false") invalidate(i) + } + break + case "bool": + return v + } + return V.bool(out, V.compactValid(valid)) + } + // numeric targets: read every source slot as a number (or bigint for i64 targets) + const [lo, hi] = intRange(target) + const isIntTarget = D.isInteger(target) || (target._tag !== "Float" && targetKind !== "f64") + if (targetKind === "i64") { + const out = new BigInt64Array(n) + const unsigned = V.isUnsigned64(target) + for (let i = 0; i < n; i++) { + if (valid !== undefined && valid[i] === 0) continue + switch (v.kind) { + case "i64": { + const x = v.values[i] + if (unsigned && !V.isUnsigned64(v.type) && x < I0) invalidate(i) + else if (!unsigned && V.isUnsigned64(v.type) && x < I0) invalidate(i) + else out[i] = x + break + } + case "f64": + case "i32": { + const x = Math.trunc(v.values[i]) + if (!Number.isFinite(x) || x < (unsigned ? 0 : -(2 ** 63)) || x >= (unsigned ? 2 ** 64 : 2 ** 63)) { + invalidate(i) + } else out[i] = BigInt.asIntN(64, BigInt(x)) + break + } + case "bool": + out[i] = BigInt(v.values[i]) + break + case "str": { + const s = V.decodeString(v, i) + if (!INT_SYNTAX.test(s)) invalidate(i) + else { + const x = BigInt(s) + if (x < (unsigned ? I0 : -(I2 ** I63)) || x >= (unsigned ? I2 ** I64 : I2 ** I63)) invalidate(i) + else out[i] = BigInt.asIntN(64, x) + } + break + } + } + } + return V.i64(target, out, V.compactValid(valid)) + } + const out = targetKind === "i32" ? new Int32Array(n) : new Float64Array(n) + for (let i = 0; i < n; i++) { + if (valid !== undefined && valid[i] === 0) continue + let x: number + switch (v.kind) { + case "f64": + case "i32": + x = v.values[i] + break + case "i64": + x = Number(V.isUnsigned64(v.type) ? BigInt.asUintN(64, v.values[i]) : v.values[i]) + break + case "bool": + x = v.values[i] + break + case "str": { + const s = V.decodeString(v, i) + if (isIntTarget ? !INT_SYNTAX.test(s) : !FLOAT_SYNTAX.test(s)) { + invalidate(i) + continue + } + const lower = s.toLowerCase() + x = lower.endsWith("nan") ? NaN : lower.endsWith("inf") || lower.endsWith("infinity") + ? (s.startsWith("-") ? -Infinity : Infinity) + : Number(s) + break + } + case "list": + case "struct": + return V.fail(`cannot cast ${D.describe(v.type)} to ${D.describe(target)}`) + } + if (isIntTarget) { + const t = Math.trunc(x) + if (!Number.isFinite(t) || t < lo || t > hi) { + invalidate(i) + continue + } + out[i] = t + } else { + out[i] = x + } + } + if (target._tag === "Float" && target.bitWidth === 32) V.roundFloat32InPlace(target, out as Float64Array) + return targetKind === "i32" + ? V.i32(target, out as Int32Array, V.compactValid(valid)) + : V.f64(target, out as Float64Array, V.compactValid(valid)) +} + +/** Casts a value, keeping its shape. @internal */ +export const castValue = (value: Value, target: S.DataType): Value => + D.sameType(value.vector.type, target) + ? value + : { vector: cast(value.vector, target), scalar: value.scalar, dynamic: undefined } + +// ----------------------------------------------------------------------------- +// Arithmetic +// ----------------------------------------------------------------------------- + +/** @internal */ +export type ArithOp = "add" | "sub" | "mul" | "div" | "floorDiv" | "mod" + +const unify = (op: string, a: Value, b: Value) => { + const st = D.supertype(a.vector.type, a.dynamic, b.vector.type, b.dynamic) + if (st === undefined || !(D.isNumeric(st.type) || st.type._tag === "Null")) { + return V.fail(`${op} is not defined for ${D.describe(a.vector.type)} and ${D.describe(b.vector.type)}`) + } + return st +} + +/** Relabels a value's vector (same storage). */ +const relabel = (value: Value, type: S.DataType): Value => ({ ...value, vector: V.withType(value.vector, type) }) + +/** + * Arithmetic with dates, timestamps and durations, on their integers: + * differences of dates or timestamps are durations, durations shift + * timestamps, durations add up and scale by integers. Mixed units use the + * coarser one. + */ +const temporalArith = (op: ArithOp, a: Value, b: Value, length: number): Value => { + const at = a.vector.type + const bt = b.vector.type + const unsupported = () => V.fail(`${op} is not defined for ${D.describe(at)} and ${D.describe(bt)}`) + const asInt = (v: Value, type: S.DataType) => relabel(castValue(v, type), S.int64) + const point = (t: S.DataType) => t._tag === "Date" || t._tag === "Timestamp" + if (op === "sub" && point(at) && point(bt)) { + if (at._tag === "Date" && bt._tag === "Date") { + const ts = S.timestamp("microsecond") + return relabel(arith("sub", asInt(a, ts), asInt(b, ts), length), S.duration("microsecond")) + } + const st = T.supertype(at, bt) as S.Timestamp | undefined + if (st === undefined) return unsupported() + return relabel(arith("sub", asInt(a, st), asInt(b, st), length), S.duration(st.unit)) + } + if ((op === "add" || op === "sub") && at._tag === "Timestamp" && bt._tag === "Duration") { + const unit = T.coarser(at.unit, bt.unit) + const ts = S.timestamp(unit, at.timezone) + return relabel(arith(op, asInt(a, ts), asInt(b, S.duration(unit)), length), ts) + } + if (op === "add" && at._tag === "Duration" && bt._tag === "Timestamp") return temporalArith(op, b, a, length) + if ((op === "add" || op === "sub") && at._tag === "Duration" && bt._tag === "Duration") { + const d = S.duration(T.coarser(at.unit, bt.unit)) + return relabel(arith(op, asInt(a, d), asInt(b, d), length), d) + } + if (op === "mul" && at._tag === "Duration" && D.isInteger(bt)) { + return relabel(arith("mul", relabel(a, S.int64), castValue(b, S.int64), length), at) + } + if (op === "mul" && bt._tag === "Duration" && D.isInteger(at)) return temporalArith(op, b, a, length) + return unsupported() +} + +/** + * Arithmetic with decimals: exact for `add`, `sub` and `mul` (Polars result + * types); other operations and mixes with floats run on floats. + */ +const decimalArith = (op: ArithOp, a: Value, b: Value, length: number): Value => { + const st = D.supertype(a.vector.type, a.dynamic, b.vector.type, b.dynamic) + if (st === undefined) { + return V.fail(`${op} is not defined for ${D.describe(a.vector.type)} and ${D.describe(b.vector.type)}`) + } + if (!Dec.isDecimal(st.type) || (op !== "add" && op !== "sub" && op !== "mul")) { + return arith(op, castValue(a, S.float64), castValue(b, S.float64), length) + } + const asDecimal = (v: Value): Value => (Dec.isDecimal(v.vector.type) ? v : castValue(v, Dec.decimal(0))) + let left = asDecimal(a) + const right = asDecimal(b) + const n = left.scalar && right.scalar ? 1 : length + if (left.scalar && !right.scalar) left = column(materialize(left, n)) + const valid = resultValid(left, right, n) + const ls = (left.vector.type as S.Decimal).scale + const rs = (right.vector.type as S.Decimal).scale + const result = Dec.arith( + op, + (left.vector as V.I64).values, + ls, + (right.vector as V.I64).values, + rs, + right.scalar && !left.scalar + ) + if (valid === "all-null") return { vector: nullsOf(result.type, n), scalar: a.scalar && b.scalar, dynamic: undefined } + return { vector: V.i64(result.type, result.values, valid), scalar: a.scalar && b.scalar, dynamic: undefined } +} + +/** @internal */ +export const arith = (op: ArithOp, a: Value, b: Value, length: number): Value => { + if (T.isTemporal(a.vector.type) || T.isTemporal(b.vector.type)) return temporalArith(op, a, b, length) + if (Dec.isDecimal(a.vector.type) || Dec.isDecimal(b.vector.type)) return decimalArith(op, a, b, length) + let { dynamic, type } = unify(op, a, b) + if (op === "div") { + type = D.divisionType(type) + dynamic = dynamic === undefined ? undefined : "float" + } + if (type._tag === "Null") type = S.int32 + const n = a.scalar && b.scalar ? 1 : length + const valid = resultValid(a, b, n) + if (valid === "all-null") return { vector: nullsOf(type, n), scalar: a.scalar && b.scalar, dynamic } + // A scalar on the left: materialise it (rare) so kernels only see col op col / col op scalar. + let left = castValue(a, type) + const right = castValue(b, type) + if (left.scalar && !right.scalar) left = column(materialize(left, n)) + const lv = left.vector + const rv = right.vector + const rs = right.scalar + let out: V.Vector + let nullOut: Uint8Array | undefined + switch (lv.kind) { + case "f64": { + const x = lv.values + const o = new Float64Array(n) + const unsigned32 = type._tag === "Int" + if (rs) arithF64Scalar(op, x, (rv as V.F64).values[0], o, n, unsigned32) + else arithF64(op, x, (rv as V.F64).values, o, n, unsigned32) + V.roundFloat32InPlace(type, o) + out = V.f64(type, o, valid) + break + } + case "i32": { + const x = lv.values + const o = new Int32Array(n) + if (op === "floorDiv" || op === "mod") { + nullOut = new Uint8Array(n).fill(1) + intDivMod(op, x, (rv as V.I32).values, rs, o, n, nullOut) + } else if (rs) arithI32Scalar(op, x, (rv as V.I32).values[0], o, n) + else arithI32(op, x, (rv as V.I32).values, o, n) + V.wrapInPlace(type, o) + out = V.i32(type, o, valid) + break + } + case "i64": { + const x = lv.values + const y = (rv as V.I64).values + const o = new BigInt64Array(n) + const unsigned = V.isUnsigned64(type) + for (let i = 0; i < n; i++) { + let p = x[i] + let q = rs ? y[0] : y[i] + if (unsigned) { + p = BigInt.asUintN(64, p) + q = BigInt.asUintN(64, q) + } + let r: bigint + switch (op) { + case "add": + r = p + q + break + case "sub": + r = p - q + break + case "mul": + r = p * q + break + default: { + if (q === I0) { + if (nullOut === undefined) nullOut = new Uint8Array(n).fill(1) + nullOut[i] = 0 + r = I0 + break + } + let d = p / q + if ((p % q !== I0) && ((p < I0) !== (q < I0))) d -= I1 + r = op === "floorDiv" ? d : p - d * q + } + } + o[i] = BigInt.asIntN(64, r) + } + out = V.i64(type, o, valid) + break + } + default: + return V.fail(`${op} is not defined for ${D.describe(type)}`) + } + if (nullOut !== undefined) out = V.withValid(out, V.compactValid(V.andValid(out.valid, nullOut, n))) + return { vector: out, scalar: a.scalar && b.scalar, dynamic } +} + +/** Float modulo as Polars computes it: `a - b * floor(a / b)`. */ +const floatMod = (a: number, b: number): number => a - b * Math.floor(a / b) + +const floorMod = (a: number, b: number): number => { + const r = a % b + return r !== 0 && (r < 0) !== (b < 0) ? r + b : r +} + +const arithF64 = (op: ArithOp, x: Float64Array, y: Float64Array, o: Float64Array, n: number, u32: boolean) => { + switch (op) { + case "add": + for (let i = 0; i < n; i++) o[i] = x[i] + y[i] + break + case "sub": + for (let i = 0; i < n; i++) o[i] = x[i] - y[i] + break + case "mul": + if (u32) { for (let i = 0; i < n; i++) o[i] = Math.imul(x[i], y[i]) >>> 0 } + else for (let i = 0; i < n; i++) o[i] = x[i] * y[i] + break + case "div": + for (let i = 0; i < n; i++) o[i] = x[i] / y[i] + break + case "floorDiv": + if (u32) { for (let i = 0; i < n; i++) o[i] = Math.floor(x[i] / y[i]) } + else for (let i = 0; i < n; i++) o[i] = Math.floor(x[i] / y[i]) + break + case "mod": + if (u32) { for (let i = 0; i < n; i++) o[i] = floorMod(x[i], y[i]) } + else for (let i = 0; i < n; i++) o[i] = floatMod(x[i], y[i]) + break + } + if (u32) { for (let i = 0; i < n; i++) o[i] = o[i] >>> 0 } +} + +const arithF64Scalar = (op: ArithOp, x: Float64Array, y: number, o: Float64Array, n: number, u32: boolean) => { + switch (op) { + case "add": + for (let i = 0; i < n; i++) o[i] = x[i] + y + break + case "sub": + for (let i = 0; i < n; i++) o[i] = x[i] - y + break + case "mul": + if (u32) { for (let i = 0; i < n; i++) o[i] = Math.imul(x[i], y) >>> 0 } + else for (let i = 0; i < n; i++) o[i] = x[i] * y + break + case "div": + for (let i = 0; i < n; i++) o[i] = x[i] / y + break + case "floorDiv": + for (let i = 0; i < n; i++) o[i] = Math.floor(x[i] / y) + break + case "mod": + if (u32) { for (let i = 0; i < n; i++) o[i] = floorMod(x[i], y) } + else for (let i = 0; i < n; i++) o[i] = floatMod(x[i], y) + break + } + if (u32) { for (let i = 0; i < n; i++) o[i] = o[i] >>> 0 } +} + +const arithI32 = (op: ArithOp, x: Int32Array, y: Int32Array, o: Int32Array, n: number) => { + switch (op) { + case "add": + for (let i = 0; i < n; i++) o[i] = (x[i] + y[i]) | 0 + break + case "sub": + for (let i = 0; i < n; i++) o[i] = (x[i] - y[i]) | 0 + break + case "mul": + for (let i = 0; i < n; i++) o[i] = Math.imul(x[i], y[i]) + break + } +} + +const arithI32Scalar = (op: ArithOp, x: Int32Array, y: number, o: Int32Array, n: number) => { + switch (op) { + case "add": + for (let i = 0; i < n; i++) o[i] = (x[i] + y) | 0 + break + case "sub": + for (let i = 0; i < n; i++) o[i] = (x[i] - y) | 0 + break + case "mul": + for (let i = 0; i < n; i++) o[i] = Math.imul(x[i], y) + break + } +} + +const intDivMod = ( + op: "floorDiv" | "mod", + x: Int32Array, + y: Int32Array, + ys: boolean, + o: Int32Array, + n: number, + valid: Uint8Array +) => { + for (let i = 0; i < n; i++) { + const a = x[i] + const b = ys ? y[0] : y[i] + if (b === 0) { + valid[i] = 0 + continue + } + if (op === "floorDiv") o[i] = Math.floor(a / b) | 0 + else o[i] = floorMod(a, b) | 0 + } +} + +/** Unary numeric operations. @internal */ +export const unaryNumeric = (op: "neg" | "abs" | "sqrt" | "round", a: Value, decimals = 0): Value => { + const v = a.vector + const n = v.length + if (v.kind === "null") return a + if (op === "sqrt" && v.kind !== "f64") return unaryNumeric(op, castValue(a, S.float64)) + switch (v.kind) { + case "f64": { + const x = v.values + const o = new Float64Array(n) + if (op === "neg") { for (let i = 0; i < n; i++) o[i] = -x[i] } + else if (op === "abs") { for (let i = 0; i < n; i++) o[i] = Math.abs(x[i]) } + else if (op === "sqrt") { for (let i = 0; i < n; i++) o[i] = Math.sqrt(x[i]) } + else if (D.isFloat(v.type)) roundHalfEven(x, o, decimals) + else o.set(x) + if (v.type._tag === "Int") V.wrapInPlace(v.type, o) + V.roundFloat32InPlace(v.type, o) + return { vector: V.f64(v.type, o, v.valid), scalar: a.scalar, dynamic: a.dynamic } + } + case "i32": { + const x = v.values + const o = new Int32Array(n) + if (op === "neg") { for (let i = 0; i < n; i++) o[i] = -x[i] | 0 } + else if (op === "abs") { for (let i = 0; i < n; i++) o[i] = Math.abs(x[i]) | 0 } + else o.set(x) + V.wrapInPlace(v.type, o) + return { vector: V.i32(v.type, o, v.valid), scalar: a.scalar, dynamic: a.dynamic } + } + case "i64": { + const x = v.values + const o = new BigInt64Array(n) + if (op === "neg") { for (let i = 0; i < n; i++) o[i] = BigInt.asIntN(64, -x[i]) } + else if (op === "abs") { for (let i = 0; i < n; i++) o[i] = BigInt.asIntN(64, x[i] < I0 ? -x[i] : x[i]) } + else o.set(x) + return { vector: V.i64(v.type, o, v.valid), scalar: a.scalar, dynamic: a.dynamic } + } + default: + return V.fail(`${op} is not defined for ${D.describe(v.type)}`) + } +} + +const roundHalfEven = (x: Float64Array, o: Float64Array, decimals: number) => { + const scale = 10 ** decimals + for (let i = 0; i < x.length; i++) { + const v = x[i] + if (!Number.isFinite(v)) { + o[i] = v + continue + } + const y = decimals === 0 ? v : v * scale + let r = Math.round(y) + // Math.round rounds halves up; move exact halves to the even neighbour. + if (r - y === 0.5 && r % 2 !== 0) r -= 1 + if (r === 0 && (y < 0 || Object.is(y, -0))) r = -0 + o[i] = decimals === 0 ? r : r / scale + } +} + +// ----------------------------------------------------------------------------- +// Comparisons +// ----------------------------------------------------------------------------- + +/** @internal */ +export type CompareOp = "eq" | "neq" | "lt" | "lte" | "gt" | "gte" + +const mirror: Record = { eq: "eq", neq: "neq", lt: "gt", lte: "gte", gt: "lt", gte: "lte" } + +/** Lexicographic comparison of two byte ranges. @internal */ +export const compareBytes = ( + a: Uint8Array, + as: number, + ae: number, + b: Uint8Array, + bs: number, + be: number +): number => { + const la = ae - as + const lb = be - bs + const l = la < lb ? la : lb + for (let k = 0; k < l; k++) { + const d = a[as + k] - b[bs + k] + if (d !== 0) return d + } + return la - lb +} + +const fromOrdering = (op: CompareOp, c: number): number => { + switch (op) { + case "eq": + return c === 0 ? 1 : 0 + case "neq": + return c !== 0 ? 1 : 0 + case "lt": + return c < 0 ? 1 : 0 + case "lte": + return c <= 0 ? 1 : 0 + case "gt": + return c > 0 ? 1 : 0 + case "gte": + return c >= 0 ? 1 : 0 + } +} + +/** @internal */ +export const compare = (op: CompareOp, a: Value, b: Value, length: number): Value => { + if (a.scalar && !b.scalar) return compare(mirror[op], b, a, length) + const at = a.vector.type + const bt = b.vector.type + const n = a.scalar ? 1 : length + let left = a + let right = b + if (D.isNumeric(at) && D.isNumeric(bt) && !D.sameType(at, bt)) { + const st = D.supertype(at, a.dynamic, bt, b.dynamic)!.type + // Compare mixed widths as Float64 unless both are 64-bit integers. + const target = V.kindOf(st) === "i64" && V.kindOf(at) === "i64" && V.kindOf(bt) === "i64" ? st : S.float64 + left = castValue(a, V.kindOf(st) === "i32" ? st : target) + right = castValue(b, V.kindOf(st) === "i32" ? st : target) + } else if (at._tag === "Null" || bt._tag === "Null") { + return { vector: nullsOf(S.bool, n), scalar: a.scalar, dynamic: undefined } + } else if ((Dec.isDecimal(at) || Dec.isDecimal(bt)) && !D.sameType(at, bt)) { + const st = D.supertype(at, a.dynamic, bt, b.dynamic) + if (st === undefined) return V.fail(`cannot compare ${D.describe(at)} with ${D.describe(bt)}`) + left = castValue(a, st.type) + right = castValue(b, st.type) + } else if ((T.isTemporal(at) || T.isTemporal(bt)) && !D.sameType(at, bt)) { + const st = T.supertype(at, bt) + if (st === undefined) return V.fail(`cannot compare ${D.describe(at)} with ${D.describe(bt)}`) + left = castValue(a, st) + right = castValue(b, st) + } else if (!D.sameType(at, bt) && !(D.isBinaryLike(at) && D.isBinaryLike(bt))) { + return V.fail(`cannot compare ${D.describe(at)} with ${D.describe(bt)}`) + } + const valid = resultValid(left, right, n) + if (valid === "all-null") return { vector: nullsOf(S.bool, n), scalar: a.scalar, dynamic: undefined } + const o = new Uint8Array(n) + const lv = left.vector + const rv = right.vector + const rs = right.scalar + switch (lv.kind) { + case "f64": + if (rs) compareF64Scalar(op, lv.values, (rv as V.F64).values[0], o, n) + else compareF64(op, lv.values, (rv as V.F64).values, o, n) + break + case "i32": + if (rs) compareI32Scalar(op, lv.values, (rv as V.I32).values[0], o, n) + else compareI32(op, lv.values, (rv as V.I32).values, o, n) + break + case "bool": { + const x = lv.values + const y = (rv as V.Bool).values + for (let i = 0; i < n; i++) o[i] = fromOrdering(op, x[i] - y[rs ? 0 : i]) + break + } + case "i64": { + const unsigned = V.isUnsigned64(lv.type) + const x = new Int32Array(lv.values.buffer, lv.values.byteOffset, n * 2) + const yv = (rv as V.I64).values + const y = new Int32Array(yv.buffer, yv.byteOffset, yv.length * 2) + for (let i = 0; i < n; i++) { + const j = rs ? 0 : i + const xh = x[2 * i + 1] + const yh = y[2 * j + 1] + let c: number + if (xh !== yh) c = unsigned ? (xh >>> 0) - (yh >>> 0) : xh - yh + else c = (x[2 * i] >>> 0) - (y[2 * j] >>> 0) + o[i] = fromOrdering(op, c) + } + break + } + case "str": { + const r = rv as V.Str + const ao = lv.offsets + const ad = lv.data + const bo = r.offsets + const bd = r.data + if (rs && (op === "eq" || op === "neq")) { + const bs = bo[0] + const blen = bo[1] - bs + const yes = op === "eq" ? 1 : 0 + for (let i = 0; i < n; i++) { + const s = ao[i] + if (ao[i + 1] - s !== blen) { + o[i] = 1 - yes + continue + } + let same = 1 + for (let k = 0; k < blen; k++) { + if (ad[s + k] !== bd[bs + k]) { + same = 0 + break + } + } + o[i] = same === 1 ? yes : 1 - yes + } + } else { + for (let i = 0; i < n; i++) { + const j = rs ? 0 : i + o[i] = fromOrdering(op, compareBytes(ad, ao[i], ao[i + 1], bd, bo[j], bo[j + 1])) + } + } + break + } + default: + return V.fail(`cannot compare ${D.describe(at)}`) + } + return { vector: V.bool(o, valid), scalar: a.scalar, dynamic: undefined } +} + +// Float64 comparisons use a total order: NaN equals NaN and sorts above every +// other value, as in Polars. +const compareF64 = (op: CompareOp, x: Float64Array, y: Float64Array, o: Uint8Array, n: number) => { + switch (op) { + case "eq": + for (let i = 0; i < n; i++) { + const a = x[i] + const b = y[i] + o[i] = +(a === b || (a !== a && b !== b)) + } + break + case "neq": + for (let i = 0; i < n; i++) { + const a = x[i] + const b = y[i] + o[i] = +!(a === b || (a !== a && b !== b)) + } + break + case "lt": + for (let i = 0; i < n; i++) { + const a = x[i] + const b = y[i] + o[i] = +(a < b || (b !== b && a === a)) + } + break + case "gte": + for (let i = 0; i < n; i++) { + const a = x[i] + const b = y[i] + o[i] = +!(a < b || (b !== b && a === a)) + } + break + case "gt": + for (let i = 0; i < n; i++) { + const a = x[i] + const b = y[i] + o[i] = +(a > b || (a !== a && b === b)) + } + break + case "lte": + for (let i = 0; i < n; i++) { + const a = x[i] + const b = y[i] + o[i] = +!(a > b || (a !== a && b === b)) + } + break + } +} + +const compareF64Scalar = (op: CompareOp, x: Float64Array, b: number, o: Uint8Array, n: number) => { + if (b !== b) { + // Against NaN: only NaN is equal; everything else is smaller. + for (let i = 0; i < n; i++) { + const isNaN = x[i] !== x[i] + o[i] = op === "eq" || op === "gte" || op === "lte" + ? +(op === "lte" ? true : isNaN) + : op === "neq" || op === "lt" + ? +!isNaN + : 0 + } + return + } + switch (op) { + case "eq": + for (let i = 0; i < n; i++) o[i] = +(x[i] === b) + break + case "neq": + for (let i = 0; i < n; i++) o[i] = +(x[i] !== b) + break + case "lt": + for (let i = 0; i < n; i++) o[i] = +(x[i] < b) + break + case "lte": + for (let i = 0; i < n; i++) o[i] = +(x[i] <= b) + break + case "gt": + for (let i = 0; i < n; i++) { + const a = x[i] + o[i] = +(a > b || a !== a) + } + break + case "gte": + for (let i = 0; i < n; i++) { + const a = x[i] + o[i] = +(a >= b || a !== a) + } + break + } +} + +const compareI32 = (op: CompareOp, x: Int32Array, y: Int32Array, o: Uint8Array, n: number) => { + switch (op) { + case "eq": + for (let i = 0; i < n; i++) o[i] = +(x[i] === y[i]) + break + case "neq": + for (let i = 0; i < n; i++) o[i] = +(x[i] !== y[i]) + break + case "lt": + for (let i = 0; i < n; i++) o[i] = +(x[i] < y[i]) + break + case "lte": + for (let i = 0; i < n; i++) o[i] = +(x[i] <= y[i]) + break + case "gt": + for (let i = 0; i < n; i++) o[i] = +(x[i] > y[i]) + break + case "gte": + for (let i = 0; i < n; i++) o[i] = +(x[i] >= y[i]) + break + } +} + +const compareI32Scalar = (op: CompareOp, x: Int32Array, b: number, o: Uint8Array, n: number) => { + switch (op) { + case "eq": + for (let i = 0; i < n; i++) o[i] = +(x[i] === b) + break + case "neq": + for (let i = 0; i < n; i++) o[i] = +(x[i] !== b) + break + case "lt": + for (let i = 0; i < n; i++) o[i] = +(x[i] < b) + break + case "lte": + for (let i = 0; i < n; i++) o[i] = +(x[i] <= b) + break + case "gt": + for (let i = 0; i < n; i++) o[i] = +(x[i] > b) + break + case "gte": + for (let i = 0; i < n; i++) o[i] = +(x[i] >= b) + break + } +} + +// ----------------------------------------------------------------------------- +// Boolean logic (Kleene) +// ----------------------------------------------------------------------------- + +const asBool = (value: Value, length: number): V.Bool => { + const v = materialize(value, length) + if (v.kind === "bool") return v + if (v.kind === "null") return V.bool(new Uint8Array(length), new Uint8Array(length)) + return V.fail(`expected a boolean expression, got ${D.describe(v.type)}`) +} + +/** @internal */ +export const logical = (op: "and" | "or", a: Value, b: Value, length: number): Value => { + const n = a.scalar && b.scalar ? 1 : length + const x = asBool(a, n) + const y = asBool(b, n) + const o = new Uint8Array(n) + const xv = x.values + const yv = y.values + if (x.valid === undefined && y.valid === undefined) { + if (op === "and") { for (let i = 0; i < n; i++) o[i] = xv[i] & yv[i] } + else for (let i = 0; i < n; i++) o[i] = xv[i] | yv[i] + return { vector: V.bool(o), scalar: a.scalar && b.scalar, dynamic: undefined } + } + const valid = new Uint8Array(n) + const xa = x.valid + const ya = y.valid + for (let i = 0; i < n; i++) { + const p = xa === undefined ? 1 : xa[i] + const q = ya === undefined ? 1 : ya[i] + const pv = xv[i] & p + const qv = yv[i] & q + if (op === "and") { + // false wins over null + const definitelyFalse = (p === 1 && pv === 0) || (q === 1 && qv === 0) + o[i] = pv & qv + valid[i] = definitelyFalse || (p === 1 && q === 1) ? 1 : 0 + } else { + const definitelyTrue = pv === 1 || qv === 1 + o[i] = pv | qv + valid[i] = definitelyTrue || (p === 1 && q === 1) ? 1 : 0 + } + } + return { vector: V.bool(o, V.compactValid(valid)), scalar: a.scalar && b.scalar, dynamic: undefined } +} + +/** @internal */ +export const not = (a: Value): Value => { + const v = a.vector + if (v.kind === "null") return a + if (v.kind !== "bool") return V.fail(`not is not defined for ${D.describe(v.type)}`) + const o = new Uint8Array(v.length) + const x = v.values + for (let i = 0; i < o.length; i++) o[i] = x[i] ^ 1 + return { vector: V.bool(o, v.valid), scalar: a.scalar, dynamic: undefined } +} + +/** @internal */ +export const isNull = (a: Value, negate: boolean): Value => { + const v = a.vector + const o = new Uint8Array(v.length) + if (v.kind === "null") o.fill(negate ? 0 : 1) + else if (v.valid === undefined) o.fill(negate ? 1 : 0) + else if (negate) o.set(v.valid) + else for (let i = 0; i < o.length; i++) o[i] = v.valid[i] ^ 1 + return { vector: V.bool(o), scalar: a.scalar, dynamic: undefined } +} + +/** @internal */ +export const isNaN_ = (a: Value): Value => { + const v = a.vector + const o = new Uint8Array(v.length) + if (v.kind === "f64" && v.type._tag === "Float") { + const x = v.values + for (let i = 0; i < o.length; i++) o[i] = +(x[i] !== x[i]) + } else if (v.kind !== "f64" && v.kind !== "i32" && v.kind !== "i64" && v.kind !== "null") { + return V.fail(`isNaN is not defined for ${D.describe(v.type)}`) + } + return { + vector: V.bool(o, v.kind === "null" ? new Uint8Array(v.length) : v.valid), + scalar: a.scalar, + dynamic: undefined + } +} + +// ----------------------------------------------------------------------------- +// Conditionals +// ----------------------------------------------------------------------------- + +/** + * `cond ? then : otherwise` per slot; a null condition takes `otherwise`. + * + * @internal + */ +export const when = (cond: Value, then: Value, otherwise: Value, length: number): Value => { + const st = D.supertype(then.vector.type, then.dynamic, otherwise.vector.type, otherwise.dynamic) + if (st === undefined) { + return V.fail( + `when branches have incompatible types ${D.describe(then.vector.type)} and ${D.describe(otherwise.vector.type)}` + ) + } + const allScalar = cond.scalar && then.scalar && otherwise.scalar + const n = allScalar ? 1 : length + const c = asBool(cond, n) + const mask = new Int32Array(n) + const cv = c.values + const ca = c.valid + for (let i = 0; i < n; i++) mask[i] = (cv[i] & (ca === undefined ? 1 : ca[i])) === 1 ? 0 : 1 + const branches = [materialize(castValue(then, st.type), n), materialize(castValue(otherwise, st.type), n)] + return { vector: select(branches, mask), scalar: allScalar, dynamic: st.dynamic } +} + +/** Picks slot `i` from `branches[choice[i]]`; all branches share a type. @internal */ +export const select = (branches: ReadonlyArray, choice: Int32Array): V.Vector => { + const n = choice.length + const first = branches.find((b) => b.kind !== "null") ?? branches[0] + if (first.kind === "null") return V.nulls(n) + if (first.kind === "list" || first.kind === "struct") { + // Nested values: gather slot i from the chosen branch. + const all = V.concat(branches.map((b) => (b.kind === "null" ? nullsOf(first.type, n) : b))) + const indices = new Int32Array(n) + for (let i = 0; i < n; i++) indices[i] = choice[i] * n + i + return V.take(all, indices) + } + const valid = new Uint8Array(n) + for (let i = 0; i < n; i++) { + const b = branches[choice[i]] + valid[i] = V.isValid(b, i) ? 1 : 0 + } + switch (first.kind) { + case "f64": + case "i32": + case "bool": + case "i64": { + const out = first.kind === "f64" + ? new Float64Array(n) + : first.kind === "i32" + ? new Int32Array(n) + : first.kind === "bool" + ? new Uint8Array(n) + : new BigInt64Array(n) + for (let i = 0; i < n; i++) { + const b = branches[choice[i]] + if (b.kind !== "null") (out as any)[i] = (b as V.F64).values[i] + } + const vector = first.kind === "f64" + ? V.f64(first.type, out as Float64Array) + : first.kind === "i32" + ? V.i32(first.type, out as Int32Array) + : first.kind === "bool" + ? V.bool(out as Uint8Array) + : V.i64(first.type, out as BigInt64Array) + return V.withValid(vector, V.compactValid(valid)) + } + case "str": { + const indices = new Int32Array(n) + // Concatenate branches, then gather from the concatenation. + const strs = branches.map((b) => b.kind === "str" ? b : nullsOf(first.type, n)) + const all = V.concat(strs) + for (let i = 0; i < n; i++) indices[i] = choice[i] * n + i + return V.withValid(V.take(all, indices), V.compactValid(valid)) + } + } +} + +/** First non-null value per slot. @internal */ +export const coalesce = (values: ReadonlyArray, length: number): Value => { + let acc = values[0] + let type = acc.vector.type + let dynamic = acc.dynamic + for (const v of values.slice(1)) { + const st = D.supertype(type, dynamic, v.vector.type, v.dynamic) + if (st === undefined) return V.fail(`coalesce arguments have incompatible types`) + type = st.type + dynamic = st.dynamic + } + const allScalar = values.every((v) => v.scalar) + const n = allScalar ? 1 : length + const vectors = values.map((v) => materialize(castValue(v, type), n)) + const choice = new Int32Array(n) + for (let i = 0; i < n; i++) { + let k = 0 + while (k < vectors.length - 1 && !V.isValid(vectors[k], i)) k++ + choice[i] = k + } + acc = { vector: select(vectors, choice), scalar: allScalar, dynamic } + return acc +} + +// ----------------------------------------------------------------------------- +// Membership +// ----------------------------------------------------------------------------- + +/** @internal */ +export const isIn = (a: Value, candidates: ReadonlyArray): Value => { + const v = a.vector + const n = v.length + const o = new Uint8Array(n) + switch (v.kind) { + case "str": { + const encoded = candidates.filter((c): c is string => typeof c === "string").map(V.encodeString) + for (let i = 0; i < n; i++) { + const s = v.offsets[i] + const e = v.offsets[i + 1] + for (const c of encoded) { + if (compareBytes(v.data, s, e, c, 0, c.length) === 0) { + o[i] = 1 + break + } + } + } + break + } + case "f64": + case "i32": { + const set = new Set(candidates.filter((c) => typeof c === "number").map((c) => (c as number) === 0 ? 0 : c)) + const hasNaN = candidates.some((c) => typeof c === "number" && Number.isNaN(c)) + const x = v.values + for (let i = 0; i < n; i++) o[i] = +(set.has(x[i] === 0 ? 0 : x[i]) || (hasNaN && x[i] !== x[i])) + break + } + case "i64": { + const set = new Set( + candidates.filter((c) => typeof c === "bigint" || typeof c === "number").map((c) => BigInt(c as any)) + ) + for (let i = 0; i < n; i++) o[i] = +set.has(v.values[i]) + break + } + case "bool": { + const t = candidates.includes(true) + const f = candidates.includes(false) + for (let i = 0; i < n; i++) o[i] = +(v.values[i] === 1 ? t : f) + break + } + case "null": + return { vector: nullsOf(S.bool, n), scalar: a.scalar, dynamic: undefined } + } + return { vector: V.bool(o, v.valid), scalar: a.scalar, dynamic: undefined } +} + +// ----------------------------------------------------------------------------- +// Strings +// ----------------------------------------------------------------------------- + +const asStr = (value: Value, name: string): V.Str => { + const v = value.vector + if (v.kind === "str") return v + return V.fail(`${name} is not defined for ${D.describe(v.type)}`) +} + +/** Number of code points. @internal */ +export const strLength = (a: Value): Value => { + if (a.vector.kind === "null") { + return { vector: nullsOf(S.uint32, a.vector.length), scalar: a.scalar, dynamic: undefined } + } + const v = asStr(a, "length") + const n = v.length + const o = new Float64Array(n) + const d = v.data + const off = v.offsets + for (let i = 0; i < n; i++) { + let c = 0 + for (let k = off[i]; k < off[i + 1]; k++) c += (d[k] & 0xc0) !== 0x80 ? 1 : 0 + o[i] = c + } + return { vector: V.f64(S.uint32, o, v.valid), scalar: a.scalar, dynamic: undefined } +} + +/** @internal */ +export const strMap = (a: Value, f: (s: string) => string): Value => { + if (a.vector.kind === "null") return a + const v = asStr(a, "string function") + const out: Array = new Array(v.length) + for (let i = 0; i < v.length; i++) out[i] = V.isValid(v, i) ? f(V.decodeString(v, i)) : null + return { vector: V.stringsFrom(v.type, out), scalar: a.scalar, dynamic: undefined } +} + +/** @internal */ +export const strMatch = (a: Value, mode: "includes" | "startsWith" | "endsWith", pattern: string): Value => { + if (a.vector.kind === "null") { + return { vector: nullsOf(S.bool, a.vector.length), scalar: a.scalar, dynamic: undefined } + } + const v = asStr(a, mode) + const p = V.encodeString(pattern) + const m = p.length + const n = v.length + const o = new Uint8Array(n) + const d = v.data + const off = v.offsets + for (let i = 0; i < n; i++) { + const s = off[i] + const e = off[i + 1] + if (e - s < m) continue + if (mode === "startsWith" || mode === "endsWith") { + const at = mode === "startsWith" ? s : e - m + let ok = 1 + for (let k = 0; k < m; k++) { + if (d[at + k] !== p[k]) { + ok = 0 + break + } + } + o[i] = ok + } else { + outer: for (let at = s; at <= e - m; at++) { + for (let k = 0; k < m; k++) if (d[at + k] !== p[k]) continue outer + o[i] = 1 + break + } + } + } + return { vector: V.bool(o, v.valid), scalar: a.scalar, dynamic: undefined } +} + +/** Concatenation; a null argument makes the slot null. @internal */ +export const strConcat = (values: ReadonlyArray, length: number): Value => { + const allScalar = values.every((v) => v.scalar) + const n = allScalar ? 1 : length + const parts = values.map((v) => { + const vector = materialize(v, n) + return vector.kind === "str" ? vector : (cast(vector, S.utf8) as V.Str | V.Null) + }) + let valid: Uint8Array | undefined + for (const p of parts) { + if (p.kind === "null") valid = new Uint8Array(n) + else valid = V.andValid(valid, p.valid, n) + } + const offsets = new Int32Array(n + 1) + let total = 0 + for (let i = 0; i < n; i++) { + for (const p of parts) if (p.kind === "str") total += p.offsets[i + 1] - p.offsets[i] + offsets[i + 1] = total + } + const data = new Uint8Array(total) + for (let i = 0; i < n; i++) { + let at = offsets[i] + for (const p of parts) { + if (p.kind !== "str") continue + const s = p.offsets[i] + const e = p.offsets[i + 1] + data.set(p.data.subarray(s, e), at) + at += e - s + } + } + return { vector: V.str(S.utf8, offsets, data, valid), scalar: allScalar, dynamic: undefined } +} diff --git a/packages/effect/src/dataframe/internal/nested.ts b/packages/effect/src/dataframe/internal/nested.ts new file mode 100644 index 00000000000..498d4459cc9 --- /dev/null +++ b/packages/effect/src/dataframe/internal/nested.ts @@ -0,0 +1,85 @@ +/** + * Lists and structs: element access and aggregates of list columns, struct + * fields and construction (Polars semantics). + * + * @internal + */ +import * as S from "../../arrow/ArrowSchema.ts" +import * as Agg from "./aggregate.ts" +import * as D from "./dtype.ts" +import * as K from "./kernels.ts" +import * as V from "./vector.ts" + +const asList = (v: V.Vector, fn: string): V.List => + v.kind === "list" ? v : V.fail(`${fn} is defined for lists, not ${D.describe(v.type)}`) + +/** The row of every element of a list column. */ +const elementRows = (v: V.List): Int32Array => { + const ids = new Int32Array(v.child.length) + for (let i = 0; i < v.length; i++) for (let e = v.offsets[i]; e < v.offsets[i + 1]; e++) ids[e] = i + return ids +} + +/** Number of elements; null for a null list. @internal */ +export const length = (x: V.Vector): V.Vector => { + const v = asList(x, "listLength") + const out = new Float64Array(v.length) + for (let i = 0; i < v.length; i++) out[i] = v.offsets[i + 1] - v.offsets[i] + return V.f64(S.uint32, out, v.valid) +} + +/** The element at `index` (negative counts from the end); null when out of range. @internal */ +export const get = (x: V.Vector, index: number): V.Vector => { + const v = asList(x, "listGet") + const idx = new Int32Array(v.length) + for (let i = 0; i < v.length; i++) { + const start = v.offsets[i] + const end = v.offsets[i + 1] + const at = index < 0 ? end + index : start + index + idx[i] = V.isValid(v, i) && at >= start && at < end ? at : -1 + } + return V.take(v.child, idx) +} + +/** An aggregate of every list's elements; null for a null list. @internal */ +export const aggregate = (fn: "sum" | "mean" | "min" | "max", x: V.Vector): V.Vector => { + const v = asList(x, `list${fn[0].toUpperCase()}${fn.slice(1)}`) + const result = Agg.aggregate(fn, v.child, { ids: elementRows(v), count: v.length }) + return v.valid === undefined ? result : V.withValid(result, V.andValid(result.valid, v.valid, v.length)) +} + +/** Whether a list holds `item`; null for a null list. @internal */ +export const contains = (x: V.Vector, item: K.Value): V.Vector => { + const v = asList(x, "listContains") + const n = v.length + const items = V.take(K.materialize(item, n), elementRows(v)) + const eq = K.materialize(K.compare("eq", K.column(v.child), K.column(items), v.child.length), v.child.length) + const out = new Uint8Array(n) + if (eq.kind === "bool") { + for (let i = 0; i < n; i++) { + for (let e = v.offsets[i]; e < v.offsets[i + 1]; e++) { + if (V.isValid(eq, e) && eq.values[e] === 1) { + out[i] = 1 + break + } + } + } + } + return V.bool(out, v.valid) +} + +/** A struct field; null where the struct is null. @internal */ +export const field = (x: V.Vector, name: string): V.Vector => { + if (x.kind !== "struct") return V.fail(`field is defined for structs, not ${D.describe(x.type)}`) + const fields = (x.type as S.Struct).fields + const k = fields.findIndex((f) => f.name === name) + if (k < 0) return V.fail(`struct has no field ${JSON.stringify(name)}`) + const child = x.children[k] + return x.valid === undefined + ? child + : V.withValid(child, V.andValid(child.kind === "null" ? undefined : child.valid, x.valid, x.length)) +} + +/** A struct of the given columns. @internal */ +export const struct = (names: ReadonlyArray, values: ReadonlyArray, n: number): V.Vector => + V.struct(S.struct(values.map((v, k) => S.field(names[k], v.type))), values, n) diff --git a/packages/effect/src/dataframe/internal/partial.ts b/packages/effect/src/dataframe/internal/partial.ts new file mode 100644 index 00000000000..0fb27a1246a --- /dev/null +++ b/packages/effect/src/dataframe/internal/partial.ts @@ -0,0 +1,152 @@ +/** + * Partial aggregation: grouping a part of the rows (a row range, a stream + * batch) into partial aggregates that combine with other parts' into the + * aggregates of all rows. + * + * Sums, counts, minima, maxima, firsts and lasts combine by aggregating the + * partials again; means combine from sums and counts; variances from counts, + * means and M2 (Chan et al.). Medians and distinct counts don't combine. + * + * @internal + */ +import * as Agg from "./aggregate.ts" +import * as Hash from "./hash.ts" +import * as V from "./vector.ts" + +/** @internal */ +export type Part = + | { readonly _tag: "vector"; readonly vector: V.Vector } + | { readonly _tag: "mean"; readonly sum: Float64Array; readonly count: Float64Array } + | { + readonly _tag: "moments" + readonly count: Float64Array + readonly mean: Float64Array + readonly m2: Float64Array + } + | { readonly _tag: "none" } + +const reaggregated = new Set(["sum", "count", "len", "min", "max", "first", "last"]) + +/** Whether partial aggregates of `fn` combine. @internal */ +export const combinable = (fn: Agg.AggFn): boolean => + reaggregated.has(fn) || fn === "mean" || fn === "std" || fn === "variance" + +/** The partial aggregate of `fn` over `v` grouped by `g`. @internal */ +export const partial = (fn: Agg.AggFn, v: V.Vector | undefined, g: Agg.Grouping): Part => { + if (reaggregated.has(fn)) return { _tag: "vector", vector: Agg.aggregate(fn, v, g) } + if (v === undefined) return V.fail(`${fn} needs an argument`) + switch (fn) { + case "mean": { + const { count, sum } = Agg.sumCount(v, g) + return { _tag: "mean", sum, count } + } + case "std": + case "variance": + return { _tag: "moments", ...Agg.moments(v, g) } + default: + return { _tag: "none" } + } +} + +/** + * Combines the partials of several parts: `parts[c]` holds part `c`'s + * partials for its groups, `ids` maps the concatenated groups of all parts + * to `count` combined groups. + * + * @internal + */ +export const combine = (fn: Agg.AggFn, parts: ReadonlyArray, ids: Int32Array, count: number): Part => { + const grouping: Agg.Grouping = { ids, count } + if (reaggregated.has(fn)) { + const vectors = parts.map((p) => (p as Extract).vector) + const merge = fn === "count" || fn === "len" ? "sum" : fn + return { _tag: "vector", vector: Agg.aggregate(merge, V.concat(vectors), grouping) } + } + if (fn === "mean") { + const sum = new Float64Array(count) + const total = new Float64Array(count) + let at = 0 + for (const p of parts as ReadonlyArray>) { + for (let j = 0; j < p.sum.length; j++) { + sum[ids[at + j]] += p.sum[j] + total[ids[at + j]] += p.count[j] + } + at += p.sum.length + } + return { _tag: "mean", sum, count: total } + } + if (fn === "std" || fn === "variance") { + const n = new Float64Array(count) + const mean = new Float64Array(count) + const m2 = new Float64Array(count) + let at = 0 + for (const p of parts as ReadonlyArray>) { + for (let j = 0; j < p.count.length; j++) { + const nb = p.count[j] + if (nb === 0) continue + const g = ids[at + j] + const na = n[g] + const nab = na + nb + const delta = p.mean[j] - mean[g] + mean[g] += delta * nb / nab + m2[g] += p.m2[j] + delta * delta * na * nb / nab + n[g] = nab + } + at += p.count.length + } + return { _tag: "moments", count: n, mean, m2 } + } + return V.fail(`${fn} cannot be computed from partial aggregates`) +} + +/** The aggregate column of a combined partial. @internal */ +export const finish = (fn: Agg.AggFn, part: Part, input: V.Vector | undefined): V.Vector => { + switch (part._tag) { + case "vector": + return part.vector + case "mean": + return Agg.meanFrom(input!.type, part.sum, part.count) + case "moments": + return Agg.varianceFrom(fn as "std" | "variance", input!.type, part.count, part.m2) + case "none": + return V.fail(`${fn} cannot be computed from partial aggregates`) + } +} + +// ----------------------------------------------------------------------------- +// Grouped states +// ----------------------------------------------------------------------------- + +/** The groups of some rows: their keys (one row per group) and partial aggregates. @internal */ +export interface State { + readonly keys: ReadonlyArray + readonly parts: ReadonlyArray + readonly count: number +} + +/** Groups some rows. @internal */ +export const group = ( + keys: ReadonlyArray, + inputs: ReadonlyArray<{ readonly fn: Agg.AggFn; readonly vector: V.Vector | undefined }>, + n: number +): State => { + const g = Hash.groups(keys, n) + return { + keys: keys.map((k) => V.take(k, g.firstRow)), + parts: inputs.map(({ fn, vector }) => partial(fn, vector, g)), + count: g.count + } +} + +/** Combines states, keeping groups in order of first appearance. @internal */ +export const merge = (fns: ReadonlyArray, states: ReadonlyArray): State => { + if (states.length === 1) return states[0] + const total = states.reduce((s, x) => s + x.count, 0) + const keys = states[0].keys.map((_, k) => V.concat(states.map((s) => s.keys[k]))) + const g = Hash.groups(keys, total) + return { + keys: keys.map((k) => V.take(k, g.firstRow)), + parts: fns.map((fn, a) => combine(fn, states.map((s) => s.parts[a]), g.ids, g.count)), + count: g.count + } +} diff --git a/packages/effect/src/dataframe/internal/query.ts b/packages/effect/src/dataframe/internal/query.ts new file mode 100644 index 00000000000..30e3fdcabe6 --- /dev/null +++ b/packages/effect/src/dataframe/internal/query.ts @@ -0,0 +1,650 @@ +/** + * Running plans: resolving tables, rewriting the plan, and executing it. + * + * Execution is a post-order walk that calls the frame operations. Before it, + * `optimize` rewrites the plan so less work happens: + * + * - predicates move below the operations they commute with (sorts, + * projections, derived columns they don't read, group keys, either side of + * a join, every input of a concat), and filters that meet merge; + * - slices move below row-wise operations, and a slice of a sort becomes a + * top-k selection; + * - columns nothing reads are dropped at the inputs, and derived columns and + * aggregates nothing reads are not computed. + * + * @internal + */ +import * as Effect from "../../Effect.ts" +import type { Catalog } from "../Catalog.ts" +import { DataFrameError } from "../DataFrame.ts" +import type * as Plan from "../Plan.ts" +import * as E from "./expr.ts" +import * as F from "./frame.ts" +import { conform } from "./schema.ts" +import * as Streaming from "./stream.ts" +import * as V from "./vector.ts" + +/** A sort that only keeps its first `limit` rows. @internal */ +export interface TopK { + readonly _tag: "TopK" + readonly input: Node + readonly keys: ReadonlyArray + readonly nullsLast: boolean + readonly limit: number +} + +/** Plans after optimisation. @internal */ +export type Node = + | Exclude + | Rebuild> + | Omit & { readonly left: Node; readonly right: Node } + | Omit & { readonly inputs: ReadonlyArray } + | TopK + +type Rebuild = N extends { readonly input: Plan.Node } ? Omit & { readonly input: Node } : never + +/** @internal */ +export const toError = (cause: unknown): DataFrameError => + new DataFrameError({ message: cause instanceof Error ? cause.message : String(cause), cause }) + +/** Runs a kernel computation, turning its failures into `DataFrameError`s. @internal */ +export const attempt = (f: () => A): Effect.Effect => { + try { + return Effect.succeed(f()) + } catch (cause) { + if (cause instanceof V.DataFrameDefect) return Effect.fail(toError(cause)) + if (cause instanceof DataFrameError) return Effect.fail(cause) + throw cause + } +} + +// ----------------------------------------------------------------------------- +// Resolving tables +// ----------------------------------------------------------------------------- + +const values = (frame: F.Frame): Node => ({ _tag: "Values", frame: frame as any }) +const frameOf = (node: { readonly frame: unknown }): F.Frame => node.frame as F.Frame + +/** Replaces every `Table` with the catalog's frame. @internal */ +export const resolve = (plan: Plan.Node, catalog: Catalog.Service): Effect.Effect => { + const names = new Set() + const collect = (node: Plan.Node): void => { + switch (node._tag) { + case "Table": + names.add(node.name) + return + case "Values": + return + case "Join": + collect(node.left) + collect(node.right) + return + case "Concat": + for (const n of node.inputs) collect(n) + return + default: + collect(node.input) + } + } + collect(plan) + if (names.size === 0) return Effect.succeed(plan as Node) + return Effect.forEach([...names], (name) => + catalog.table(name).pipe( + Effect.map((df) => [name, df as unknown as F.Frame] as const) + )).pipe( + Effect.flatMap((entries) => { + const tables = new Map(entries) + const replace = (node: Plan.Node): Node => { + switch (node._tag) { + case "Table": { + const frame = tables.get(node.name)! + return values(node.columns === undefined ? frame : conform(frame, node.columns)) + } + case "Values": + return node + case "Join": + return { ...node, left: replace(node.left), right: replace(node.right) } + case "Concat": + return { ...node, inputs: node.inputs.map(replace) } + default: + return { ...node, input: replace(node.input) } as Node + } + } + return attempt(() => replace(plan)) + }) + ) +} + +// ----------------------------------------------------------------------------- +// Column names +// ----------------------------------------------------------------------------- + +const namesCache = new WeakMap | undefined }>() + +/** + * The output column names of a resolved plan, or `undefined` when they + * depend on the data (a pivot without declared columns). + * + * @internal + */ +export const names = (node: Node): ReadonlyArray | undefined => { + const cached = namesCache.get(node) + if (cached !== undefined) return cached.names + const result = computeNames(node) + namesCache.set(node, { names: result }) + return result +} + +const computeNames = (node: Node): ReadonlyArray | undefined => { + switch (node._tag) { + case "Values": + return frameOf(node).columns.map((c) => c.name) + case "Table": + return node.columns?.map((c) => c.name) + case "Filter": + case "Sort": + case "TopK": + case "Slice": + case "Unique": + case "DropNulls": + return names(node.input) + case "Select": + return node.columns.map((c) => c.name) + case "WithColumns": { + const input = names(node.input) + if (input === undefined) return undefined + const existing = new Set(input) + return [...input, ...node.columns.map((c) => c.name).filter((n) => !existing.has(n))] + } + case "Pick": + return node.columns + case "Omit": { + const drop = new Set(node.columns) + return names(node.input)?.filter((n) => !drop.has(n)) + } + case "Rename": + return names(node.input)?.map((n) => node.mapping[n] ?? n) + case "GroupBy": + return [...node.keys, ...node.aggregates.map((a) => a.name)] + case "Aggregate": + return node.aggregates.map((a) => a.name) + case "Join": { + const left = names(node.left) + if (node.how === "semi" || node.how === "anti") return left + const right = names(node.right) + if (left === undefined || right === undefined) return undefined + const on = new Set(node.on) + return [...left, ...right.filter((n) => !on.has(n))] + } + case "Concat": + return node.inputs.length === 0 ? [] : names(node.inputs[0]) + case "Pivot": + return node.columns === undefined ? undefined : [...node.index, ...node.columns] + case "Explode": + return names(node.input) + case "Unnest": + return undefined + case "Unpivot": + return [...node.index, node.variable, node.value] + } +} + +// ----------------------------------------------------------------------------- +// Optimisation +// ----------------------------------------------------------------------------- + +/** @internal */ +export const optimize = (plan: Node): Node => prune(pushSlices(pushFilters(plan)), undefined) + +const conjuncts = (e: Plan.Expression): Array => + e._tag === "Call" && e.fn === "and" && e.args.length === 2 + ? [...conjuncts(e.args[0]), ...conjuncts(e.args[1])] + : [e] + +const and = (es: ReadonlyArray): Plan.Expression => + es.reduce((a, b) => ({ _tag: "Call", fn: "and", args: [a, b] })) + +const subset = (a: Set, b: ReadonlySet | ReadonlyArray): boolean => { + const s = b instanceof Set ? b : new Set(b as ReadonlyArray) + for (const x of a) if (!s.has(x)) return false + return true +} + +const mapChildren = (node: Node, f: (child: Node) => Node): Node => { + switch (node._tag) { + case "Values": + case "Table": + return node + case "Join": + return { ...node, left: f(node.left), right: f(node.right) } + case "Concat": + return { ...node, inputs: node.inputs.map(f) } + default: + return { ...node, input: f(node.input) } as Node + } +} + +const pushFilters = (node: Node): Node => { + const optimized = mapChildren(node, pushFilters) + return optimized._tag === "Filter" ? push(optimized.input, conjuncts(optimized.predicate)) : optimized +} + +const filtered = (input: Node, predicates: ReadonlyArray): Node => + predicates.length === 0 ? input : { _tag: "Filter", input, predicate: and(predicates) } + +/** Places each predicate as deep as it can go below `node`. */ +const push = (node: Node, predicates: ReadonlyArray): Node => { + if (predicates.length === 0) return node + const split = (canPush: (refs: Set, p: Plan.Expression) => boolean) => { + const down: Array = [] + const here: Array = [] + for (const p of predicates) (canPush(E.columns(p), p) ? down : here).push(p) + return { down, here } + } + switch (node._tag) { + case "Filter": + return push(node.input, [...conjuncts(node.predicate), ...predicates]) + case "Sort": + case "DropNulls": + case "Pick": + case "Omit": + return { ...node, input: push(node.input, predicates) } + case "Rename": { + const inverse = new Map(Object.entries(node.mapping).map(([from, to]) => [to, from])) + const renamed = predicates.map((p) => + E.mapColumns(p, (name) => ({ _tag: "Column", name: inverse.get(name) ?? name })) + ) + return { ...node, input: push(node.input, renamed) } + } + case "WithColumns": { + if (!node.columns.every((c) => E.isRowWise(c.expression))) return filtered(node, predicates) + const added = new Set(node.columns.map((c) => c.name)) + const { down, here } = split((refs) => [...refs].every((r) => !added.has(r))) + return filtered({ ...node, input: push(node.input, down) }, here) + } + case "Select": { + if (!node.columns.every((c) => E.isRowWise(c.expression))) return filtered(node, predicates) + const byName = new Map(node.columns.map((c) => [c.name, c.expression])) + const { down, here } = split((refs) => [...refs].every((r) => byName.get(r)?._tag === "Column")) + const inlined = down.map((p) => E.mapColumns(p, (name) => byName.get(name)!)) + return filtered({ ...node, input: push(node.input, inlined) }, here) + } + case "Unique": { + const columns = node.columns + const { down, here } = split((refs) => columns === undefined || subset(refs, columns)) + return filtered({ ...node, input: push(node.input, down) }, here) + } + case "GroupBy": { + const { down, here } = split((refs) => subset(refs, node.keys)) + return filtered({ ...node, input: push(node.input, down) }, here) + } + case "Concat": + return { ...node, inputs: node.inputs.map((input) => push(input, predicates)) } + case "Join": { + const leftNames = names(node.left) + const rightNames = names(node.right) + if (node.how === "full" || leftNames === undefined || rightNames === undefined) return filtered(node, predicates) + const left = new Set(leftNames) + const right = new Set(rightNames) + const toLeft: Array = [] + const toRight: Array = [] + const here: Array = [] + for (const p of predicates) { + const refs = E.columns(p) + if (node.how === "inner" && subset(refs, node.on)) { + toLeft.push(p) + toRight.push(p) + } else if (subset(refs, left)) toLeft.push(p) + else if (node.how === "inner" && subset(refs, right)) toRight.push(p) + else here.push(p) + } + return filtered({ ...node, left: push(node.left, toLeft), right: push(node.right, toRight) }, here) + } + default: + return filtered(node, predicates) + } +} + +const isRowWise = (node: Node): boolean => + ((node._tag === "Select" || node._tag === "WithColumns") && node.columns.every((c) => E.isRowWise(c.expression))) || + node._tag === "Rename" || node._tag === "Pick" || node._tag === "Omit" + +const pushSlices = (node: Node): Node => { + const optimized = mapChildren(node, pushSlices) + return optimized._tag === "Slice" ? slice(optimized.input, optimized.offset, optimized.length) : optimized +} + +const slice = (input: Node, offset: number, length: number | undefined): Node => { + if (isRowWise(input)) { + return { ...input, input: slice((input as { readonly input: Node }).input, offset, length) } as Node + } + if (input._tag === "Slice" && input.offset >= 0 && offset >= 0) { + const inner = input.length === undefined ? Infinity : input.length + const remaining = Math.max(0, inner - offset) + const combined = Math.min(remaining, length === undefined ? Infinity : length) + return slice(input.input, input.offset + offset, combined === Infinity ? undefined : combined) + } + if (input._tag === "Sort" && offset >= 0 && length !== undefined) { + const topK: Node = { + _tag: "TopK", + input: input.input, + keys: input.keys, + nullsLast: input.nullsLast, + limit: offset + length + } + return offset === 0 ? topK : { _tag: "Slice", input: topK, offset, length } + } + return { _tag: "Slice", input, offset, length } +} + +type Required = ReadonlySet | undefined + +const union = (required: Required, ...more: ReadonlyArray>): Required => { + if (required === undefined) return undefined + const out = new Set(required) + for (const m of more) for (const x of m) out.add(x) + return out +} + +const exprColumns = (es: ReadonlyArray): Set => { + const out = new Set() + for (const e of es) E.columns(e, out) + return out +} + +/** Keeps at least one item, so a frame never loses its row count. */ +const keep = (all: ReadonlyArray, wanted: (a: A) => boolean): ReadonlyArray => { + const kept = all.filter(wanted) + return kept.length === 0 && all.length > 0 ? [all[0]] : kept +} + +/** Drops columns nothing reads. `required` undefined means all. */ +const prune = (node: Node, required: Required): Node => { + switch (node._tag) { + case "Values": { + if (required === undefined) return node + const all = names(node)! + const kept = keep(all, (n) => required.has(n)) + return kept.length === all.length ? node : { _tag: "Pick", input: node, columns: kept } + } + case "Table": + return node + case "Filter": + return { ...node, input: prune(node.input, union(required, E.columns(node.predicate))) } + case "Sort": + case "TopK": + return { ...node, input: prune(node.input, union(required, exprColumns(node.keys.map((k) => k.expression)))) } + case "Slice": + return { ...node, input: prune(node.input, required) } + case "Unique": + case "DropNulls": + return { + ...node, + input: prune(node.input, node.columns === undefined ? undefined : union(required, node.columns)) + } + case "Select": { + const columns = required === undefined ? node.columns : keep(node.columns, (c) => required.has(c.name)) + return { ...node, columns, input: prune(node.input, exprColumns(columns.map((c) => c.expression))) } + } + case "WithColumns": { + if (required === undefined) return { ...node, input: prune(node.input, undefined) } + const columns = node.columns.filter((c) => required.has(c.name)) + const produced = new Set(columns.map((c) => c.name)) + const below = new Set([...required].filter((n) => !produced.has(n))) + const input = prune(node.input, union(below, exprColumns(columns.map((c) => c.expression)))) + return columns.length === 0 ? input : { ...node, columns, input } + } + case "Pick": { + const columns = required === undefined ? node.columns : keep(node.columns, (c) => required.has(c)) + return { ...node, columns, input: prune(node.input, new Set(columns)) } + } + case "Omit": { + const all = names(node.input) + if (all === undefined) return { ...node, input: prune(node.input, undefined) } + const drop = new Set(node.columns) + const below = all.filter((n) => !drop.has(n) && (required === undefined || required.has(n))) + return { ...node, input: prune(node.input, new Set(below.length === 0 ? all.slice(0, 1) : below)) } + } + case "Rename": { + if (required === undefined) return { ...node, input: prune(node.input, undefined) } + const inverse = new Map(Object.entries(node.mapping).map(([from, to]) => [to, from])) + return { ...node, input: prune(node.input, new Set([...required].map((n) => inverse.get(n) ?? n))) } + } + case "GroupBy": { + const aggregates = required === undefined + ? node.aggregates + : node.aggregates.filter((a) => required.has(a.name)) + return { + ...node, + aggregates, + input: prune(node.input, new Set([...node.keys, ...exprColumns(aggregates.map((a) => a.expression))])) + } + } + case "Aggregate": { + const aggregates = required === undefined + ? node.aggregates + : keep(node.aggregates, (a) => required.has(a.name)) + return { ...node, aggregates, input: prune(node.input, exprColumns(aggregates.map((a) => a.expression))) } + } + case "Join": { + const side = (input: Node): Required => { + const all = names(input) + if (required === undefined || all === undefined) return undefined + const available = new Set(all) + return new Set([...node.on, ...[...required].filter((n) => available.has(n))]) + } + const right = node.how === "semi" || node.how === "anti" ? new Set(node.on) : side(node.right) + return { ...node, left: prune(node.left, side(node.left)), right: prune(node.right, right) } + } + case "Concat": + return { ...node, inputs: node.inputs.map((input) => prune(input, required)) } + case "Pivot": + return { ...node, input: prune(node.input, new Set([...node.index, node.on, node.values])) } + case "Explode": + return { ...node, input: prune(node.input, union(required, [node.column])) } + case "Unnest": + return { ...node, input: prune(node.input, undefined) } + case "Unpivot": + return { ...node, input: prune(node.input, new Set([...node.index, ...node.on])) } + } +} + +// ----------------------------------------------------------------------------- +// Explaining +// ----------------------------------------------------------------------------- + +const list = (items: ReadonlyArray) => `[${items.join(", ")}]` +const named = (items: ReadonlyArray) => list(items.map((c) => `${c.name} = ${E.show(c.expression)}`)) +const keys = (items: ReadonlyArray) => + list(items.map((k) => `${k.descending ? "desc" : "asc"}(${E.show(k.expression)})`)) + +const line = (node: Node): string => { + switch (node._tag) { + case "Values": { + const f = frameOf(node) + return `Values ${f.length} rows ${list(f.columns.map((c) => c.name))}` + } + case "Table": + return `Table ${node.name}${node.columns === undefined ? "" : ` ${list(node.columns.map((c) => c.name))}`}` + case "Filter": + return `Filter ${E.show(node.predicate)}` + case "Select": + return `Select ${named(node.columns)}` + case "WithColumns": + return `WithColumns ${named(node.columns)}` + case "Sort": + return `Sort ${keys(node.keys)}${node.nullsLast ? " nulls last" : ""}` + case "TopK": + return `TopK ${node.limit} ${keys(node.keys)}${node.nullsLast ? " nulls last" : ""}` + case "Slice": + return `Slice offset ${node.offset}${node.length === undefined ? "" : ` length ${node.length}`}` + case "Unique": + return `Unique${node.columns === undefined ? "" : ` ${list(node.columns)}`}` + case "DropNulls": + return `DropNulls${node.columns === undefined ? "" : ` ${list(node.columns)}`}` + case "Pick": + return `Pick ${list(node.columns)}` + case "Omit": + return `Omit ${list(node.columns)}` + case "Rename": + return `Rename ${list(Object.entries(node.mapping).map(([a, b]) => `${a} -> ${b}`))}` + case "GroupBy": + return `GroupBy ${list(node.keys)} ${named(node.aggregates)}` + case "Aggregate": + return `Aggregate ${named(node.aggregates)}` + case "Join": + return `Join ${node.how} on ${list(node.on)}` + case "Concat": + return `Concat` + case "Pivot": + return `Pivot on ${node.on} index ${list(node.index)} ${node.aggregate}(${node.values})${ + node.columns === undefined ? "" : ` columns ${list(node.columns)}` + }` + case "Unpivot": + return `Unpivot ${list(node.on)} index ${list(node.index)} as ${node.variable}, ${node.value}` + case "Explode": + return `Explode ${node.column}` + case "Unnest": + return `Unnest ${node.column}` + } +} + +/** @internal */ +export const children = (node: Node): ReadonlyArray => { + switch (node._tag) { + case "Values": + case "Table": + return [] + case "Join": + return [node.left, node.right] + case "Concat": + return node.inputs + default: + return [node.input] + } +} + +/** A plan as an indented tree, one operation per line. @internal */ +export const show = (node: Node | Plan.Node, indent = ""): string => + [indent + line(node as Node), ...children(node as Node).map((c) => show(c, indent + " "))].join("\n") + +// ----------------------------------------------------------------------------- +// Execution +// ----------------------------------------------------------------------------- + +const exprs = (items: ReadonlyArray): { [name: string]: Plan.Expression } => { + const out: { [name: string]: Plan.Expression } = {} + for (const c of items) out[c.name] = c.expression + return out +} + +const sortKeys = (items: ReadonlyArray) => + items.map((k) => ({ node: k.expression, descending: k.descending })) + +/** Runs one operation over its inputs. @internal */ +export const step = (node: Node, inputs: ReadonlyArray): F.Frame => { + const input = inputs[0] + switch (node._tag) { + case "Values": + return frameOf(node) + case "Table": + return V.fail(`table ${node.name} is not resolved`) + case "Filter": + return F.filter(input, node.predicate) + case "Select": + return F.select(input, exprs(node.columns)) + case "WithColumns": + return F.withColumns(input, exprs(node.columns)) + case "Sort": + return F.sort(input, sortKeys(node.keys), node.nullsLast) + case "TopK": + return F.sort(input, sortKeys(node.keys), node.nullsLast, node.limit) + case "Slice": { + const n = input.length + const start = node.offset < 0 ? Math.max(0, n + node.offset) : Math.min(node.offset, n) + return F.slice(input, start, node.length === undefined ? n : start + node.length) + } + case "Unique": + return F.unique(input, node.columns) + case "DropNulls": + return F.dropNulls(input, node.columns) + case "Pick": + return F.pick(input, node.columns) + case "Omit": + return F.omit(input, node.columns) + case "Rename": + return F.rename(input, node.mapping) + case "GroupBy": + return F.groupBy(input, node.keys, exprs(node.aggregates)) + case "Aggregate": + return F.aggregate(input, exprs(node.aggregates)) + case "Join": + return F.join(inputs[0], inputs[1], node.on, node.how) + case "Concat": + return inputs.length === 0 ? V.fail("concat needs at least one input") : F.concat(inputs) + case "Pivot": + return F.pivot(input, node) + case "Unpivot": + return F.unpivot(input, node) + case "Explode": + return F.explode(input, node.column) + case "Unnest": + return F.unnest(input, node.column) + } +} + +/** Yield to other fibers when an operation ran for longer than this (ms). */ +const SLICE = 10 + +/** Runs a resolved plan, yielding between long-running operations. @internal */ +export const execute = (plan: Node): Effect.Effect => { + let last = performance.now() + const run = (node: Node): Effect.Effect => + Effect.flatMap( + Effect.forEach(children(node), run), + (inputs) => { + const result = attempt(() => step(node, inputs)) + const now = performance.now() + if (now - last < SLICE) return result + last = now + return Effect.tap(result, () => Effect.yieldNow) + } + ) + return run(plan) +} + +/** The names of the tables a plan reads. @internal */ +export const tables = (plan: Plan.Node): Set => { + const out = new Set() + const walk = (node: Node): void => { + if (node._tag === "Table") out.add(node.name) + for (const c of children(node)) walk(c) + } + walk(plan as Node) + return out +} + +/** + * Resolves, optimises and runs a plan. Plans that read streamed tables run + * batch by batch (see `stream.ts`). + * + * @internal + */ +export const collect = (plan: Plan.Node, catalog: Catalog.Service): Effect.Effect => + [...tables(plan)].some(catalog.streaming) + ? Streaming.collect(plan, catalog).pipe( + Effect.withSpan("dataframe.Query", { + attributes: { "dataframe.operations": count(plan), "dataframe.streaming": true } + }) + ) + : resolve(plan, catalog).pipe( + Effect.flatMap((resolved) => attempt(() => optimize(resolved))), + Effect.flatMap(execute), + Effect.withSpan("dataframe.Query", { attributes: { "dataframe.operations": count(plan) } }) + ) + +/** The optimised plan, as text. @internal */ +export const explain = (plan: Plan.Node, catalog: Catalog.Service): Effect.Effect => + resolve(plan, catalog).pipe(Effect.flatMap((resolved) => attempt(() => show(optimize(resolved))))) + +const count = (node: Plan.Node): number => 1 + children(node as Node).reduce((n, c) => n + count(c as Plan.Node), 0) diff --git a/packages/effect/src/dataframe/internal/schema.ts b/packages/effect/src/dataframe/internal/schema.ts new file mode 100644 index 00000000000..6d7b7c55853 --- /dev/null +++ b/packages/effect/src/dataframe/internal/schema.ts @@ -0,0 +1,177 @@ +/** + * Arrow types for the fields of an Effect Schema struct. + * + * @internal + */ +import * as S from "../../arrow/ArrowSchema.ts" +import * as AST from "../../SchemaAST.ts" +import type * as Plan from "../Plan.ts" +import * as D from "./dtype.ts" +import * as F from "./frame.ts" +import * as V from "./vector.ts" + +interface Mapped { + readonly type: S.DataType + readonly nullable: boolean +} + +const literalType = (literal: unknown): S.DataType | undefined => { + switch (typeof literal) { + case "string": + return S.utf8 + case "number": + return S.float64 + case "boolean": + return S.bool + case "bigint": + return S.int64 + default: + return undefined + } +} + +const map = (ast: AST.AST, path: string): Mapped => { + switch (ast._tag) { + case "String": + case "TemplateLiteral": + return { type: S.utf8, nullable: false } + case "Number": + return { type: S.float64, nullable: false } + case "Boolean": + return { type: S.bool, nullable: false } + case "BigInt": + return { type: S.int64, nullable: false } + case "Null": + case "Undefined": + return { type: S.nullType, nullable: true } + case "Literal": { + const type = literalType(ast.literal) + return type === undefined ? unsupported(ast, path) : { type, nullable: false } + } + case "Enum": { + const kinds = new Set(ast.enums.map(([, v]) => typeof v)) + return kinds.size === 1 && kinds.has("string") + ? { type: S.utf8, nullable: false } + : kinds.size === 1 + ? { type: S.float64, nullable: false } + : unsupported(ast, path) + } + case "Union": { + let type: S.DataType | undefined + let nullable = false + for (const member of ast.types) { + const m = map(member, path) + nullable ||= m.nullable + if (m.type._tag === "Null") continue + if (type === undefined) type = m.type + else if (type._tag !== m.type._tag) return unsupported(ast, path) + } + return { type: type ?? S.nullType, nullable } + } + case "Suspend": + return map(ast.thunk(), path) + case "Arrays": { + const element = ast.rest[0] + if (ast.elements.length > 0 || element === undefined || ast.rest.length > 1) return unsupported(ast, path) + const m = map(element, `${path}[]`) + return { type: S.list(S.field("item", m.type, { nullable: m.nullable })), nullable: false } + } + case "Objects": { + if (ast.indexSignatures.length > 0) return unsupported(ast, path) + const fields = ast.propertySignatures.map((ps) => { + const name = String(ps.name) + const m = map(ps.type, `${path}.${name}`) + return S.field(name, m.type, { nullable: m.nullable || AST.isOptional(ps.type) }) + }) + return { type: S.struct(fields), nullable: false } + } + case "Declaration": { + const id = (ast.annotations as { readonly representation?: { readonly id?: unknown } } | undefined) + ?.representation?.id + if (id === "effect/schema/DateTimeUtc") return { type: S.timestamp("microsecond"), nullable: false } + if (id === "effect/schema/Duration") return { type: S.duration("microsecond"), nullable: false } + if (id === "effect/schema/BigDecimal") { + return V.fail( + `field ${ + JSON.stringify(path) + }: decimal columns need a precision and scale: pass options.types (for example ArrowSchema.decimal(18, 2))` + ) + } + return unsupported(ast, path) + } + default: + return unsupported(ast, path) + } +} + +const unsupported = (ast: AST.AST, path: string): never => + V.fail(`field ${JSON.stringify(path)}: schemas of kind ${ast._tag} have no DataFrame column type yet`) + +/** The Arrow schema of a struct schema's type side. @internal */ +export const toArrowSchema = ( + schema: { readonly ast: AST.AST }, + overrides: { readonly [name: string]: S.DataType | undefined } | undefined +): S.Schema => { + const ast = AST.toType(schema.ast) + if (ast._tag !== "Objects") return V.fail("DataFrame rows must be described by a struct schema") + return S.make(ast.propertySignatures.map((ps) => { + const name = String(ps.name) + const m = map(ps.type, name) + const type = overrides?.[name] ?? m.type + return S.field(name, type, { nullable: m.nullable || AST.isOptional(ps.type) }) + })) +} + +// ----------------------------------------------------------------------------- +// Types of table columns +// ----------------------------------------------------------------------------- + +/** The plan type name of an Arrow type. @internal */ +export const typeName = (t: S.DataType): Plan.TypeName => { + switch (t._tag) { + case "Int": + case "Float": + return D.describe(t) as Plan.TypeName + case "Utf8": + case "LargeUtf8": + case "Utf8View": + return "Utf8" + case "Binary": + case "LargeBinary": + case "BinaryView": + case "FixedSizeBinary": + return "Binary" + default: + return t._tag as Plan.TypeName + } +} + +const kind = (name: string): string => + name.startsWith("Int") || name.startsWith("UInt") ? "Int" : name.startsWith("Float") ? "Float" : name + +/** Columns of an Effect-Schema-derived Arrow schema, as table columns. @internal */ +export const tableColumns = (schema: S.Schema): Array => + schema.fields.map((f) => ({ name: f.name, type: typeName(f.type), nullable: f.nullable })) + +/** + * Checks that a frame has the expected columns and narrows it to them. + * + * @internal + */ +export const conform = (frame: F.Frame, columns: ReadonlyArray): F.Frame => { + for (const expected of columns) { + const c = frame.columns.find((c) => c.name === expected.name) + if (c === undefined) return V.fail(`missing column ${JSON.stringify(expected.name)}`) + const actual = typeName(c.type) + const compatible = kind(actual) === kind(expected.type) || + (kind(expected.type) === "Float" && c.type._tag === "Int" && c.type.bitWidth < 64) || + (expected.type === "Null" && expected.nullable) + if (!compatible) { + return V.fail(`column ${JSON.stringify(expected.name)} is ${actual}, expected ${expected.type}`) + } + if (!expected.nullable && V.nullCount(c.values()) > 0) { + return V.fail(`column ${JSON.stringify(expected.name)} has missing values`) + } + } + return F.pick(frame, columns.map((c) => c.name)) +} diff --git a/packages/effect/src/dataframe/internal/sort.ts b/packages/effect/src/dataframe/internal/sort.ts new file mode 100644 index 00000000000..db4877b7162 --- /dev/null +++ b/packages/effect/src/dataframe/internal/sort.ts @@ -0,0 +1,263 @@ +/** + * Stable multi-key sorting. + * + * Keys are sorted least significant first, each with a stable pass, so the + * result is ordered by the first key, then the second, and so on. Numeric + * keys turn into unsigned words whose order matches the key order (Float64 + * keys: NaN last, `-0` equal to `0`) and go through an LSD radix sort with + * 16-bit digits. String keys are ranked: their distinct values are sorted + * once by UTF-8 bytes and the rows radix-sorted by rank. + * + * @internal + */ +import * as Hash from "./hash.ts" +import { compareBytes } from "./kernels.ts" +import * as V from "./vector.ts" + +/** @internal */ +export interface SortKey { + readonly vector: V.Vector + readonly descending: boolean +} + +const SMALL = 512 + +/** The row order that sorts `keys`; nulls first unless `nullsLast`. @internal */ +export const permutation = (keys: ReadonlyArray, nullsLast: boolean, n: number): Int32Array => { + let perm: Int32Array = new Int32Array(n) + for (let i = 0; i < n; i++) perm[i] = i + if (n < 2) return perm + if (n <= SMALL) return smallSort(keys, nullsLast, perm) + for (let k = keys.length - 1; k >= 0; k--) perm = sortByKey(keys[k], nullsLast, perm) + return perm +} + +// ----------------------------------------------------------------------------- +// Keys as unsigned words +// ----------------------------------------------------------------------------- + +/** One or two 32-bit words per row (most significant first), in row order. @internal */ +export const words = (key: SortKey): { readonly hi: Uint32Array | undefined; readonly lo: Uint32Array } => { + const v = key.vector + const n = v.length + if (v.kind === "list" || v.kind === "struct") return V.fail(`cannot sort by ${v.type._tag} columns`) + const flip = key.descending ? 0xffffffff : 0 + switch (v.kind) { + case "i32": { + const lo = new Uint32Array(n) + const x = v.values + for (let i = 0; i < n; i++) lo[i] = ((x[i] ^ 0x80000000) ^ flip) >>> 0 + return { hi: undefined, lo } + } + case "bool": { + const lo = new Uint32Array(n) + const x = v.values + for (let i = 0; i < n; i++) lo[i] = (x[i] ^ flip) >>> 0 + return { hi: undefined, lo } + } + case "f64": { + const tmp = new Float64Array(n) + const x = v.values + for (let i = 0; i < n; i++) { + const a = x[i] + tmp[i] = a === 0 ? 0 : a !== a ? NaN : a + } + const bits = new Uint32Array(tmp.buffer) + const hi = new Uint32Array(n) + const lo = new Uint32Array(n) + for (let i = 0; i < n; i++) { + let l = bits[2 * i] + let h = bits[2 * i + 1] + if ((h & 0x80000000) !== 0) { + h = ~h + l = ~l + } else h ^= 0x80000000 + hi[i] = (h ^ flip) >>> 0 + lo[i] = (l ^ flip) >>> 0 + } + return { hi, lo } + } + case "i64": { + const bits = new Uint32Array(v.values.buffer, v.values.byteOffset, n * 2) + const signFlip = V.isUnsigned64(v.type) ? 0 : 0x80000000 + const hi = new Uint32Array(n) + const lo = new Uint32Array(n) + for (let i = 0; i < n; i++) { + hi[i] = ((bits[2 * i + 1] ^ signFlip) ^ flip) >>> 0 + lo[i] = (bits[2 * i] ^ flip) >>> 0 + } + return { hi, lo } + } + case "str": { + const g = Hash.ids(v) + const order = new Int32Array(g.count) + for (let i = 0; i < g.count; i++) order[i] = i + const off = v.offsets + const d = v.data + const sorted = Array.from(order).sort((a, b) => { + const ra = g.firstRow[a] + const rb = g.firstRow[b] + return compareBytes(d, off[ra], off[ra + 1], d, off[rb], off[rb + 1]) + }) + const rank = new Uint32Array(g.count) + for (let r = 0; r < sorted.length; r++) rank[sorted[r]] = r + const lo = new Uint32Array(n) + for (let i = 0; i < n; i++) lo[i] = (rank[g.ids[i]] ^ flip) >>> 0 + return { hi: undefined, lo } + } + case "null": + return { hi: undefined, lo: new Uint32Array(n) } + } +} + +// ----------------------------------------------------------------------------- +// Radix passes +// ----------------------------------------------------------------------------- + +const counts = new Uint32Array(65537) + +/** + * Stable LSD radix sort of `perm` by `keys` (in perm order), 16 bits per + * pass; skips passes whose digit is constant. + */ +const radix = (perm: Int32Array, hi: Uint32Array | undefined, lo: Uint32Array): Int32Array => { + const n = perm.length + let p: Int32Array = perm + let h: Uint32Array | undefined = hi + let l: Uint32Array = lo + let p2: Int32Array = new Int32Array(n) + let h2: Uint32Array | undefined = hi === undefined ? undefined : new Uint32Array(n) + let l2: Uint32Array = new Uint32Array(n) + const passes: Array<[boolean, number]> = [[false, 0], [false, 16]] + if (hi !== undefined) passes.push([true, 0], [true, 16]) + for (const [useHi, shift] of passes) { + const src = useHi ? h! : l + counts.fill(0) + for (let i = 0; i < n; i++) counts[((src[i] >>> shift) & 0xffff) + 1]++ + // constant digit: nothing to do + let constant = false + for (let b = 1; b <= 65536; b++) { + if (counts[b] === n) { + constant = true + break + } + if (counts[b] !== 0) break + } + if (constant) continue + for (let b = 1; b <= 65536; b++) counts[b] += counts[b - 1] + for (let i = 0; i < n; i++) { + const at = counts[(src[i] >>> shift) & 0xffff]++ + p2[at] = p[i] + l2[at] = l[i] + if (h !== undefined) h2![at] = h[i] + } + ;[p, p2] = [p2, p] + ;[l, l2] = [l2, l] + if (h !== undefined) [h, h2] = [h2, h] + } + return p +} + +const sortByKey = (key: SortKey, nullsLast: boolean, perm: Int32Array): Int32Array => { + const n = perm.length + const w = words(key) + // keys in current perm order + const lo = new Uint32Array(n) + const hi = w.hi === undefined ? undefined : new Uint32Array(n) + for (let i = 0; i < n; i++) { + const r = perm[i] + lo[i] = w.lo[r] + if (hi !== undefined) hi[i] = w.hi![r] + } + const sorted = radix(perm, hi, lo) + return partitionNulls(key.vector, sorted, nullsLast) +} + +/** Moves null keys to the front (or back), keeping order on both sides. @internal */ +export const partitionNulls = (v: V.Vector, perm: Int32Array, nullsLast: boolean): Int32Array => { + if (v.kind !== "null" && v.valid === undefined) return perm + const n = perm.length + const out = new Int32Array(n) + let nulls = 0 + for (let i = 0; i < n; i++) if (!V.isValid(v, perm[i])) nulls++ + let a = nullsLast ? 0 : nulls + let b = nullsLast ? n - nulls : 0 + for (let i = 0; i < n; i++) { + const r = perm[i] + if (V.isValid(v, r)) out[a++] = r + else out[b++] = r + } + return out +} + +// ----------------------------------------------------------------------------- +// Small inputs: one comparator sort +// ----------------------------------------------------------------------------- + +/** Compares two rows by the keys; ties go to the earlier row. */ +const comparator = (keys: ReadonlyArray, nullsLast: boolean): (a: number, b: number) => number => { + const prepared = keys.map((k) => ({ k, w: words(k) })) + return (a, b) => { + for (const { k, w } of prepared) { + const av = V.isValid(k.vector, a) + const bv = V.isValid(k.vector, b) + if (!av || !bv) { + if (av === bv) continue + return (av ? 1 : -1) * (nullsLast ? -1 : 1) + } + if (w.hi !== undefined && w.hi[a] !== w.hi[b]) return w.hi[a] < w.hi[b] ? -1 : 1 + if (w.lo[a] !== w.lo[b]) return w.lo[a] < w.lo[b] ? -1 : 1 + } + return a - b + } +} + +const smallSort = (keys: ReadonlyArray, nullsLast: boolean, perm: Int32Array): Int32Array => + Int32Array.from(Array.from(perm).sort(comparator(keys, nullsLast))) + +/** + * The first `k` rows of the order that sorts `keys`, without sorting the + * rest: one pass that keeps the best `k` rows in a heap. + * + * @internal + */ +export const topK = (keys: ReadonlyArray, nullsLast: boolean, n: number, k: number): Int32Array => { + if (k <= 0) return new Int32Array(0) + if (n <= SMALL || k * 8 > n) return permutation(keys, nullsLast, n).subarray(0, Math.min(k, n)) + const cmp = comparator(keys, nullsLast) + // A max-heap: the root is the worst of the best `k` rows so far. + const heap = new Int32Array(k) + const down = (size: number) => { + let i = 0 + while (true) { + const l = 2 * i + 1 + if (l >= size) return + const r = l + 1 + const c = r < size && cmp(heap[r], heap[l]) > 0 ? r : l + if (cmp(heap[c], heap[i]) <= 0) return + const t = heap[c] + heap[c] = heap[i] + heap[i] = t + i = c + } + } + for (let i = 0; i < k; i++) { + heap[i] = i + let j = i + while (j > 0) { + const p = (j - 1) >> 1 + if (cmp(heap[j], heap[p]) <= 0) break + const t = heap[p] + heap[p] = heap[j] + heap[j] = t + j = p + } + } + for (let i = k; i < n; i++) { + if (cmp(i, heap[0]) < 0) { + heap[0] = i + down(k) + } + } + return Int32Array.from(Array.from(heap).sort(cmp)) +} diff --git a/packages/effect/src/dataframe/internal/sql.ts b/packages/effect/src/dataframe/internal/sql.ts new file mode 100644 index 00000000000..cc5e51cf926 --- /dev/null +++ b/packages/effect/src/dataframe/internal/sql.ts @@ -0,0 +1,930 @@ +/** + * Compiling plans to SQL. + * + * Every relation the compiler builds carries a hidden `__row` column: a key + * that increases in row order. Operations keep it (filters, projections), + * redefine it (sorts, joins, concatenations: `ROW_NUMBER()` over the new + * order) or derive it (groups: the first row's `__row`), and the final + * query orders by it. This reproduces the row order of the in-memory + * engine: stable sorts, groups in order of first appearance, joins in left + * row order, `first`/`last` by row order, windows over rows in order. + * + * @internal + */ +import * as S from "../../arrow/ArrowSchema.ts" +import type * as Plan from "../Plan.ts" +import * as F from "./frame.ts" +import { civil } from "./temporal.ts" +import * as V from "./vector.ts" + +/** @internal */ +export type Dialect = "duckdb" | "postgres" + +/** What the compiler knows about a column's type (enough to pick SQL that keeps the engine's semantics). */ +type Kind = "int" | "float" | "string" | "bool" | "date" | "timestamp" | "duration" | "decimal" | undefined + +interface Relation { + /** A SELECT whose columns are `columns` plus `__row`. */ + readonly sql: string + readonly columns: ReadonlyArray + readonly kinds: ReadonlyArray +} + +const kindOfType = (t: S.DataType): Kind => { + switch (t._tag) { + case "Int": + return "int" + case "Float": + return "float" + case "Utf8": + case "LargeUtf8": + case "Utf8View": + return "string" + case "Bool": + return "bool" + case "Date": + return "date" + case "Timestamp": + return "timestamp" + case "Duration": + return "duration" + case "Decimal": + return "decimal" + default: + return undefined + } +} + +const kindOfName = (name: Plan.TypeName): Kind => + name.startsWith("Int") || name.startsWith("UInt") + ? "int" + : name.startsWith("Float") + ? "float" + : name === "Utf8" + ? "string" + : name === "Bool" + ? "bool" + : name === "Date" + ? "date" + : name === "Timestamp" + ? "timestamp" + : name === "Duration" + ? "duration" + : name === "Decimal" + ? "decimal" + : undefined + +const ROW = `"__row"` + +/** @internal */ +export const quote = (name: string): string => `"${name.replaceAll(`"`, `""`)}"` + +const list = (names: ReadonlyArray, prefix = "") => names.map((n) => `${prefix}${quote(n)}`).join(", ") + +const unsupported = (what: string, dialect: Dialect): never => + V.fail(`${what} is not supported by the SQL engine (${dialect})`) + +// ----------------------------------------------------------------------------- +// Types and literals +// ----------------------------------------------------------------------------- + +const sqlType = (t: S.DataType, dialect: Dialect): string => { + switch (t._tag) { + case "Int": + if (dialect === "duckdb") { + return `${t.signed ? "" : "U"}${ + ({ 8: "TINYINT", 16: "SMALLINT", 32: "INTEGER", 64: "BIGINT" } as const)[t.bitWidth] + }` + } + return t.bitWidth <= 16 && t.signed ? "SMALLINT" : t.bitWidth <= 16 || (t.bitWidth === 32 && t.signed) + ? "INTEGER" + : t.bitWidth === 32 + ? "BIGINT" + : t.signed + ? "BIGINT" + : "NUMERIC(20, 0)" + case "Float": + return t.bitWidth === 32 ? "REAL" : "DOUBLE PRECISION" + case "Utf8": + case "LargeUtf8": + case "Utf8View": + return dialect === "duckdb" ? "VARCHAR" : "TEXT" + case "Binary": + case "LargeBinary": + case "BinaryView": + return dialect === "duckdb" ? "BLOB" : "BYTEA" + case "Bool": + return "BOOLEAN" + case "Date": + return "DATE" + case "Timestamp": + return t.timezone === undefined ? "TIMESTAMP" : "TIMESTAMPTZ" + case "Duration": + return "INTERVAL" + case "Decimal": + return `DECIMAL(${t.precision}, ${t.scale})` + case "Null": + return "INTEGER" + default: + return unsupported(`columns of type ${t._tag}`, dialect) + } +} + +const castTypes: Record = { + Int8: S.int8, + Int16: S.int16, + Int32: S.int32, + Int64: S.int64, + UInt8: S.uint8, + UInt16: S.uint16, + UInt32: S.uint32, + UInt64: S.uint64, + Float32: S.float32, + Float64: S.float64, + Utf8: S.utf8, + Bool: S.bool, + Date: S.date("day"), + "Timestamp[s]": S.timestamp("second"), + "Timestamp[ms]": S.timestamp("millisecond"), + "Timestamp[us]": S.timestamp("microsecond"), + "Timestamp[ns]": S.timestamp("nanosecond"), + "Duration[s]": S.duration("second"), + "Duration[ms]": S.duration("millisecond"), + "Duration[us]": S.duration("microsecond"), + "Duration[ns]": S.duration("nanosecond") +} + +const string = (s: string): string => `'${s.replaceAll("'", "''")}'` + +const pad = (n: number, width = 2) => String(n).padStart(width, "0") + +const perSecond: Record = { + second: BigInt(1), + millisecond: BigInt(1000), + microsecond: BigInt(1_000_000), + nanosecond: BigInt(1_000_000_000) +} + +const floorDiv = (a: bigint, b: bigint) => (a < BigInt(0) ? -((-a + b - BigInt(1)) / b) : a / b) + +const timestampText = (value: bigint, unit: Plan.TimeUnit): string => { + const per = perSecond[unit] + const perDay = per * BigInt(86400) + const days = floorDiv(value, perDay) + const rem = value - days * perDay + const [y, m, d] = civil(Number(days)) + const seconds = Number(rem / per) + const micros = Number((rem % per) * BigInt(1_000_000) / per) + return `${pad(y, 4)}-${pad(m)}-${pad(d)} ${pad(Math.floor(seconds / 3600))}:${pad(Math.floor(seconds / 60) % 60)}:${ + pad(seconds % 60) + }.${pad(micros, 6)}` +} + +const literal = (value: unknown, type: Plan.LiteralType | undefined, dialect: Dialect): string => { + if (value === null || value === undefined) return "NULL" + if (type !== undefined) { + const big = BigInt(value as number | bigint) + switch (type._tag) { + case "Date": { + const [y, m, d] = civil(Number(big)) + return `DATE '${pad(y, 4)}-${pad(m)}-${pad(d)}'` + } + case "Timestamp": + return `TIMESTAMP '${timestampText(big, type.unit)}'` + case "Duration": { + const micros = big * BigInt(1_000_000) / perSecond[type.unit] + return dialect === "duckdb" + ? `TO_MICROSECONDS(CAST(${micros} AS BIGINT))` + : `(${micros} * INTERVAL '1 microsecond')` + } + case "Decimal": { + const negative = big < BigInt(0) + const digits = (negative ? -big : big).toString().padStart(type.scale + 1, "0") + const text = type.scale === 0 ? digits : `${digits.slice(0, -type.scale)}.${digits.slice(-type.scale)}` + return `CAST('${negative ? "-" : ""}${text}' AS DECIMAL(${type.precision}, ${type.scale}))` + } + } + } + switch (typeof value) { + case "number": + if (Number.isNaN(value)) return `CAST('NaN' AS DOUBLE PRECISION)` + if (!Number.isFinite(value)) return `CAST('${value > 0 ? "" : "-"}Infinity' AS DOUBLE PRECISION)` + if (Object.is(value, -0)) return `CAST('-0' AS DOUBLE PRECISION)` + return Number.isInteger(value) ? String(value) : `CAST(${value} AS DOUBLE PRECISION)` + case "bigint": + return `CAST(${value} AS BIGINT)` + case "boolean": + return value ? "TRUE" : "FALSE" + case "string": + return string(value) + default: + return unsupported(`literals like ${String(value)}`, dialect) + } +} + +// ----------------------------------------------------------------------------- +// Expressions +// ----------------------------------------------------------------------------- + +interface Scope { + readonly dialect: Dialect + /** The PARTITION BY of an enclosing `over`. */ + readonly partition: string | undefined + /** Kinds of the input's columns. */ + readonly kinds: ReadonlyMap +} + +const castKinds: Record = { + Utf8: "string", + Bool: "bool", + Date: "date", + Decimal: "decimal" +} + +/** The kind of an expression's result, when the compiler can tell. */ +const kindOf = (node: Plan.Expression, scope: Scope): Kind => { + switch (node._tag) { + case "Column": + return scope.kinds.get(node.name) + case "Literal": + if (node.type !== undefined) return node.type._tag.toLowerCase() as Kind + return typeof node.value === "number" + ? Number.isInteger(node.value) && !Object.is(node.value, -0) ? "int" : "float" + : typeof node.value === "bigint" + ? "int" + : typeof node.value === "string" + ? "string" + : typeof node.value === "boolean" + ? "bool" + : undefined + case "Over": + return kindOf(node.expression, scope) + case "AggregateCall": + return node.fn === "count" || node.fn === "countDistinct" + ? "int" + : node.fn === "mean" || node.fn === "median" || node.fn === "std" || node.fn === "variance" + ? "float" + : node.arg === undefined + ? undefined + : kindOf(node.arg, scope) + case "Call": { + const a = node.args.map((x) => kindOf(x, scope)) + switch (node.fn) { + case "add": + case "subtract": + case "multiply": + case "floorDivide": + case "mod": + if (a[0] === "timestamp" || a[1] === "timestamp") { + return node.fn === "subtract" && a[0] === a[1] + ? "duration" + : "timestamp" + } + if (a[0] === "date" && a[1] === "date") return "duration" + return a[0] === "int" && a[1] === "int" + ? "int" + : a[0] === "decimal" || a[1] === "decimal" + ? "decimal" + : a[0] && a[1] + ? "float" + : undefined + case "divide": + case "sqrt": + case "rollingMean": + return "float" + case "negate": + case "abs": + case "round": + case "cumSum": + case "cumMin": + case "cumMax": + case "shift": + case "diff": + case "rollingSum": + case "rollingMin": + case "rollingMax": + case "fillNull": + case "coalesce": + case "truncate": + return a[0] + case "when": + return a[1] ?? a[2] + case "cast": { + const to = node.options?.to ?? "" + return castKinds[to] ?? + (to.startsWith("Int") || to.startsWith("UInt") + ? "int" + : to.startsWith("Float") + ? "float" + : to.startsWith("Timestamp") + ? "timestamp" + : to.startsWith("Duration") + ? "duration" + : undefined) + } + case "length": + case "cumCount": + case "rank": + case "year": + case "month": + case "day": + case "hour": + case "minute": + case "second": + case "millisecond": + case "weekday": + case "ordinalDay": + case "totalDays": + case "totalHours": + case "totalMinutes": + case "totalSeconds": + case "totalMilliseconds": + case "listLength": + return "int" + case "toLowerCase": + case "toUpperCase": + case "concat": + return "string" + default: + return node.fn === "eq" || node.fn === "neq" || node.fn === "lt" || node.fn === "lte" || node.fn === "gt" || + node.fn === "gte" || node.fn === "and" || node.fn === "or" || node.fn === "not" || node.fn === "isNull" || + node.fn === "isNotNull" || node.fn === "isNaN" || node.fn === "isIn" || node.fn === "between" + ? "bool" + : undefined + } + } + } +} + +const isNaNSql = ( + x: string, + d: Dialect +) => (d === "duckdb" ? `ISNAN(${x})` : `(${x} = CAST('NaN' AS DOUBLE PRECISION))`) + +const window = (scope: Scope, frame?: string): string => + `OVER (${scope.partition === undefined ? "" : `PARTITION BY ${scope.partition} `}ORDER BY ${ROW}${ + frame === undefined ? "" : ` ${frame}` + })` + +const running = "ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW" + +const float = (x: string) => `CAST(${x} AS DOUBLE PRECISION)` + +const aggregate = (node: Plan.AggregateCall, scope: Scope, over: string | undefined): string => { + const d = scope.dialect + const x = node.arg === undefined ? "" : expression(node.arg, { ...scope, partition: undefined }) + const kind = node.arg === undefined ? undefined : kindOf(node.arg, scope) + const o = over === undefined ? "" : ` ${over}` + switch (node.fn) { + case "count": + return node.arg === undefined ? `COUNT(*)${o}` : `COUNT(${x})${o}` + case "sum": + if (kind === "duration" && d === "duckdb") { + return `TO_MICROSECONDS(CAST(COALESCE(SUM(EXTRACT(EPOCH FROM ${x}) * 1000000)${o}, 0) AS BIGINT))` + } + return `COALESCE(SUM(${x})${o}, 0)` + case "mean": + return `AVG(${x})${o}` + case "min": + case "max": { + const fn = node.fn.toUpperCase() + // Floats: NaN is skipped unless every value is NaN. + return kind === "float" + ? `COALESCE(${fn}(CASE WHEN ${isNaNSql(x, d)} THEN NULL ELSE ${x} END)${o}, ${fn}(${x})${o})` + : `${fn}(${x})${o}` + } + case "std": + return `STDDEV_SAMP(${x})${o}` + case "variance": + return `VAR_SAMP(${x})${o}` + case "median": + if (over !== undefined) return unsupported("median over partitions", d) + return d === "duckdb" ? `MEDIAN(${x})` : `PERCENTILE_CONT(0.5) WITHIN GROUP (ORDER BY ${x})` + case "countDistinct": + if (over !== undefined) return unsupported("countDistinct over partitions", d) + return `(COUNT(DISTINCT ${x}) + MAX(CASE WHEN ${x} IS NULL THEN 1 ELSE 0 END))` + case "first": + case "last": { + if (over !== undefined) { + return `${node.fn === "first" ? "FIRST_VALUE" : "LAST_VALUE"}(${x}) OVER (${ + scope.partition === undefined ? "" : `PARTITION BY ${scope.partition} ` + }ORDER BY ${ROW} ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING)` + } + return d === "duckdb" + ? `${node.fn.toUpperCase()}(${x} ORDER BY ${ROW})` + : `(ARRAY_AGG(${x} ORDER BY ${ROW} ${node.fn === "first" ? "ASC" : "DESC"}))[1]` + } + case "list": + if (over !== undefined) return unsupported("list over partitions", d) + return d === "duckdb" ? `LIST(${x} ORDER BY ${ROW})` : `ARRAY_AGG(${x} ORDER BY ${ROW})` + } +} + +const truncateUnits: Record = { "1h": "hour", "1d": "day", "1w": "week", "1mo": "month", "1y": "year" } +const totalSeconds: Record = { + totalDays: 86400, + totalHours: 3600, + totalMinutes: 60, + totalSeconds: 1, + totalMilliseconds: 0.001 +} + +const call = (node: Plan.Call, scope: Scope): string => { + const d = scope.dialect + const a = node.args.map((x) => expression(x, scope)) + const k = node.args.map((x) => kindOf(x, scope)) + const ints = k[0] === "int" && k[1] === "int" + const o = node.options ?? {} + // Floored modulo of integers, exact, null for a zero divisor. + const intMod = () => `((${a[0]} % NULLIF(${a[1]}, 0)) + ${a[1]}) % NULLIF(${a[1]}, 0)` + switch (node.fn) { + case "add": + return `(${a[0]} + ${a[1]})` + case "subtract": + if (k[0] === "date" && k[1] === "date") return `((${a[0]} - ${a[1]}) * INTERVAL '1 day')` + return `(${a[0]} - ${a[1]})` + case "multiply": + return `(${a[0]} * ${a[1]})` + case "divide": + return `(${float(a[0])} / ${a[1]})` + case "floorDivide": + return ints + ? `((${a[0]} - (${intMod()})) ${d === "duckdb" ? "//" : "/"} NULLIF(${a[1]}, 0))` + : `FLOOR(${float(a[0])} / ${a[1]})` + case "mod": + return ints ? `(${intMod()})` : `(${a[0]} - ${a[1]} * FLOOR(${float(a[0])} / ${a[1]}))` + case "negate": + return `(-(${a[0]}))` + case "abs": + return `ABS(${a[0]})` + case "sqrt": + return `SQRT(${a[0]})` + case "round": + return d === "duckdb" ? `ROUND_EVEN(${a[0]}, ${o.decimals ?? 0})` : `ROUND(${a[0]}, ${o.decimals ?? 0})` + case "eq": + return `(${a[0]} = ${a[1]})` + case "neq": + return `(${a[0]} <> ${a[1]})` + case "lt": + return `(${a[0]} < ${a[1]})` + case "lte": + return `(${a[0]} <= ${a[1]})` + case "gt": + return `(${a[0]} > ${a[1]})` + case "gte": + return `(${a[0]} >= ${a[1]})` + case "between": + return `(${a[0]} BETWEEN ${a[1]} AND ${a[2]})` + case "isIn": { + const values = (o.values ?? []).filter((v) => v !== null) + return values.length === 0 ? + `(${a[0]} IS NULL AND NULL)` : + `(${a[0]} IN (${values.map((v) => literal(v, undefined, d)).join(", ")}))` + } + case "and": + return `(${a[0]} AND ${a[1]})` + case "or": + return `(${a[0]} OR ${a[1]})` + case "not": + return `(NOT ${a[0]})` + case "isNull": + return `(${a[0]} IS NULL)` + case "isNotNull": + return `(${a[0]} IS NOT NULL)` + case "isNaN": + return d === "duckdb" ? `ISNAN(${a[0]})` : `(${a[0]} = CAST('NaN' AS DOUBLE PRECISION))` + case "fillNull": + case "coalesce": + return `COALESCE(${a.join(", ")})` + case "when": + return `(CASE WHEN ${a[0]} THEN ${a[1]} ELSE ${a[2]} END)` + case "cast": { + const to = o.to ?? "" + const target = to === "Decimal" + ? `DECIMAL(${o.precision}, ${o.scale})` + : sqlType(castTypes[to] ?? V.fail(`unknown cast target ${to}`), d) + const integer = to.startsWith("Int") || to.startsWith("UInt") + let x = a[0] + // Temporal values become their integers; decimals truncate (as the engine does). + if (integer && k[0] === "timestamp") { + x = d === "duckdb" + ? `EPOCH_US(${x})` + : `CAST(EXTRACT(EPOCH FROM ${x}) * 1000000 AS BIGINT)` + } else if (integer && k[0] === "date") x = `(${x} - DATE '1970-01-01')` + else if (integer && k[0] === "duration") x = `CAST(EXTRACT(EPOCH FROM ${x}) * 1000000 AS BIGINT)` + else if (integer && k[0] === "decimal") x = `TRUNC(${x})` + return d === "duckdb" ? `TRY_CAST(${x} AS ${target})` : `CAST(${x} AS ${target})` + } + case "length": + return `LENGTH(${a[0]})` + case "toLowerCase": + return `LOWER(${a[0]})` + case "toUpperCase": + return `UPPER(${a[0]})` + case "includes": + return `(STRPOS(${a[0]}, ${string(o.pattern ?? "")}) > 0)` + case "startsWith": + return `(LEFT(${a[0]}, ${[...(o.pattern ?? "")].length}) = ${string(o.pattern ?? "")})` + case "endsWith": + return `(RIGHT(${a[0]}, ${[...(o.pattern ?? "")].length}) = ${string(o.pattern ?? "")})` + case "concat": + return `(${a.join(" || ")})` + case "cumSum": + return `(CASE WHEN ${a[0]} IS NULL THEN NULL ELSE SUM(${a[0]}) ${window(scope, running)} END)` + case "cumMin": + case "cumMax": { + const fn = node.fn === "cumMin" ? "MIN" : "MAX" + // Floats: NaN is skipped (it stays only while no other value was seen). + const value = k[0] === "float" + ? `COALESCE(${fn}(CASE WHEN ${isNaNSql(a[0], d)} THEN NULL ELSE ${a[0]} END) ${window(scope, running)}, ${ + a[0] + })` + : `${fn}(${a[0]}) ${window(scope, running)}` + return `(CASE WHEN ${a[0]} IS NULL THEN NULL ELSE ${value} END)` + } + case "cumCount": + return `COUNT(${a[0]}) ${window(scope, running)}` + case "shift": { + const n = o.n ?? 1 + return n >= 0 ? `LAG(${a[0]}, ${n}) ${window(scope)}` : `LEAD(${a[0]}, ${-n}) ${window(scope)}` + } + case "diff": + return `(${a[0]} - LAG(${a[0]}, ${o.n ?? 1}) ${window(scope)})` + case "rank": { + const order = `${a[0]} ${o.descending ? "DESC" : "ASC"} NULLS LAST` + const part = scope.partition === undefined ? "" : `PARTITION BY ${scope.partition} ` + const ties = `COUNT(*) OVER (PARTITION BY ${scope.partition === undefined ? "" : `${scope.partition}, `}${a[0]})` + const ranked = (() => { + switch (o.method ?? "average") { + case "min": + return `RANK() OVER (${part}ORDER BY ${order})` + case "max": + return `(RANK() OVER (${part}ORDER BY ${order}) + ${ties} - 1)` + case "dense": + return `DENSE_RANK() OVER (${part}ORDER BY ${order})` + case "ordinal": + return `ROW_NUMBER() OVER (${part}ORDER BY ${order}, ${ROW})` + case "average": + return `(RANK() OVER (${part}ORDER BY ${order}) + (${ties} - 1) / 2.0)` + } + })() + return `(CASE WHEN ${a[0]} IS NULL THEN NULL ELSE ${ranked} END)` + } + case "rollingSum": + case "rollingMean": + case "rollingMin": + case "rollingMax": { + const frame = `ROWS BETWEEN ${(o.window ?? 1) - 1} PRECEDING AND CURRENT ROW` + const fn = { rollingSum: "SUM", rollingMean: "AVG", rollingMin: "MIN", rollingMax: "MAX" }[node.fn] + const enough = `COUNT(${a[0]}) ${window(scope, frame)} >= ${o.minPeriods ?? o.window ?? 1}` + // Floats: a NaN in the window makes the minimum or maximum NaN. + const value = k[0] === "float" && (fn === "MIN" || fn === "MAX") + ? `CASE WHEN MAX(CASE WHEN ${isNaNSql(a[0], d)} THEN 1 ELSE 0 END) ${ + window(scope, frame) + } = 1 THEN CAST('NaN' AS DOUBLE PRECISION) ELSE ${fn}(${a[0]}) ${window(scope, frame)} END` + : `${fn}(${a[0]}) ${window(scope, frame)}` + return `(CASE WHEN ${enough} THEN ${value} END)` + } + case "year": + case "month": + case "day": + case "hour": + case "minute": + return `CAST(EXTRACT(${node.fn.toUpperCase()} FROM ${a[0]}) AS INTEGER)` + case "second": + return `CAST(FLOOR(EXTRACT(SECOND FROM ${a[0]})) AS INTEGER)` + case "millisecond": + return `CAST(MOD(FLOOR(EXTRACT(MILLISECONDS FROM ${a[0]})), 1000) AS INTEGER)` + case "weekday": + return `CAST(EXTRACT(ISODOW FROM ${a[0]}) AS INTEGER)` + case "ordinalDay": + return `CAST(EXTRACT(DOY FROM ${a[0]}) AS INTEGER)` + case "truncate": { + const unit = truncateUnits[o.every ?? ""] ?? unsupported(`truncate to ${o.every}`, d) + return k[0] === "date" ? `CAST(DATE_TRUNC('${unit}', ${a[0]}) AS DATE)` : `DATE_TRUNC('${unit}', ${a[0]})` + } + case "totalDays": + case "totalHours": + case "totalMinutes": + case "totalSeconds": + case "totalMilliseconds": + return `CAST(TRUNC(EXTRACT(EPOCH FROM ${a[0]}) / ${totalSeconds[node.fn]}) AS BIGINT)` + case "listLength": + return d === "duckdb" + ? `CAST(LEN(${a[0]}) AS BIGINT)` + : `COALESCE(CARDINALITY(${a[0]}), CASE WHEN ${a[0]} IS NULL THEN NULL ELSE 0 END)` + case "listGet": { + const i = o.index ?? 0 + return d === "duckdb" + ? `${a[0]}[${i >= 0 ? i + 1 : i}]` + : `${a[0]}[${i >= 0 ? `${i + 1}` : `CARDINALITY(${a[0]}) + ${i + 1}`}]` + } + case "listSum": + case "listMean": + case "listMin": + case "listMax": { + if (d !== "duckdb") return unsupported(node.fn, d) + const fn = { listSum: "LIST_SUM", listMean: "LIST_AVG", listMin: "LIST_MIN", listMax: "LIST_MAX" }[node.fn] + return node.fn === "listSum" + ? `(CASE WHEN ${a[0]} IS NULL THEN NULL ELSE COALESCE(${fn}(${a[0]}), 0) END)` + : `${fn}(${a[0]})` + } + case "listContains": + return d === "duckdb" + ? `LIST_CONTAINS(${a[0]}, ${a[1]})` + : `(CASE WHEN ${a[0]} IS NULL THEN NULL ELSE ${a[1]} = ANY(${a[0]}) END)` + case "field": + return `(${a[0]}).${quote(o.name ?? "")}` + case "struct": { + if (d !== "duckdb") return unsupported("struct", d) + const names = o.names ?? [] + return `STRUCT_PACK(${a.map((x, i) => `${quote(names[i])} := ${x}`).join(", ")})` + } + } +} + +const expression = (node: Plan.Expression, scope: Scope): string => { + switch (node._tag) { + case "Column": + return quote(node.name) + case "Literal": + return literal(node.value, node.type, scope.dialect) + case "Call": + return call(node, scope) + case "AggregateCall": + return scope.partition === undefined + ? V.fail(`aggregate ${node.fn} can only be used in agg or over`) + : aggregate(node, scope, `OVER (PARTITION BY ${scope.partition})`) + case "Over": { + const partition = node.partitionBy.map((k) => expression(k, { ...scope, partition: undefined })).join(", ") + return expression(node.expression, { ...scope, partition: partition === "" ? undefined : partition }) + } + } +} + +// ----------------------------------------------------------------------------- +// Relations +// ----------------------------------------------------------------------------- + +const from = (r: Relation, alias = "t") => `(${r.sql}) AS ${alias}` + +const rowOrdered = ( + columns: ReadonlyArray, + kinds: ReadonlyArray, + input: Relation, + order: string +): Relation => ({ + sql: `SELECT ${list(columns)}${columns.length > 0 ? ", " : ""}ROW_NUMBER() OVER (ORDER BY ${order}) AS ${ROW} FROM ${ + from(input) + }`, + columns, + kinds +}) + +const values = (frame: F.Frame, dialect: Dialect): Relation => { + const columns = frame.columns.map((c) => c.name) + const kinds = frame.columns.map((c) => kindOfType(c.type)) + const types = frame.columns.map((c) => sqlType(c.type, dialect)) + const rows = F.toRows(frame) + const cell = (v: unknown, i: number): string => { + const t = frame.columns[i].type + if (v === null || v === undefined) return `CAST(NULL AS ${types[i]})` + if (t._tag === "Timestamp" || t._tag === "Date" || t._tag === "Duration" || t._tag === "Decimal") { + return `CAST(${string(String(v instanceof Object && "toJSON" in v ? (v as any).toJSON() : v))} AS ${types[i]})` + } + return `CAST(${literal(v, undefined, dialect)} AS ${types[i]})` + } + if (rows.length === 0) { + return { + sql: `SELECT ${columns.map((c, i) => `CAST(NULL AS ${types[i]}) AS ${quote(c)}`).join(", ")}${ + columns.length > 0 ? ", " : "" + }CAST(0 AS BIGINT) AS ${ROW} WHERE FALSE`, + columns, + kinds + } + } + const tuples = rows.map((row, r) => `(${[...columns.map((c, i) => cell(row[c], i)), String(r + 1)].join(", ")})`) + return { + sql: `SELECT * FROM (VALUES ${tuples.join(", ")}) AS v(${list([...columns, "__row"])})`, + columns, + kinds + } +} + +const named = (items: ReadonlyArray, scope: Scope) => + items.map((c) => `${expression(c.expression, scope)} AS ${quote(c.name)}`) + +const same = (r: Relation, sql: string): Relation => ({ sql, columns: r.columns, kinds: r.kinds }) + +const pick = (r: Relation, columns: ReadonlyArray): ReadonlyArray => + columns.map((c) => r.kinds[r.columns.indexOf(c)]) + +const scopeOf = (r: Relation, dialect: Dialect): Scope => ({ + dialect, + partition: undefined, + kinds: new Map(r.columns.map((c, i) => [c, r.kinds[i]])) +}) + +/** @internal */ +export const relation = (node: Plan.Node, dialect: Dialect): Relation => { + const input = () => relation((node as { readonly input: Plan.Node }).input, dialect) + switch (node._tag) { + case "Table": { + if (node.columns === undefined) { + return V.fail( + `the SQL engine needs the columns of table ${JSON.stringify(node.name)}: use Query.table with a schema` + ) + } + const columns = node.columns.map((c) => c.name) + return { + sql: `SELECT ${list(columns)}, ROW_NUMBER() OVER () AS ${ROW} FROM ${quote(node.name)}`, + columns, + kinds: node.columns.map((c) => kindOfName(c.type)) + } + } + case "Values": + return values(node.frame as unknown as F.Frame, dialect) + case "Filter": { + const r = input() + return same(r, `SELECT * FROM ${from(r)} WHERE ${expression(node.predicate, scopeOf(r, dialect))}`) + } + case "Select": { + const r = input() + const scope = scopeOf(r, dialect) + return { + sql: `SELECT ${[...named(node.columns, scope), ROW].join(", ")} FROM ${from(r)}`, + columns: node.columns.map((c) => c.name), + kinds: node.columns.map((c) => kindOf(c.expression, scope)) + } + } + case "WithColumns": { + const r = input() + const scope = scopeOf(r, dialect) + const added = new Map(node.columns.map((c) => [c.name, c.expression])) + const kept = r.columns.map((c) => { + const e = added.get(c) + return e === undefined ? quote(c) : `${expression(e, scope)} AS ${quote(c)}` + }) + const extra = node.columns.filter((c) => !r.columns.includes(c.name)) + return { + sql: `SELECT ${[...kept, ...named(extra, scope), ROW].join(", ")} FROM ${from(r)}`, + columns: [...r.columns, ...extra.map((c) => c.name)], + kinds: [ + ...r.columns.map((c, i) => { + const e = added.get(c) + return e === undefined ? r.kinds[i] : kindOf(e, scope) + }), + ...extra.map((c) => kindOf(c.expression, scope)) + ] + } + } + case "Pick": { + const r = input() + return { + sql: `SELECT ${list([...node.columns, "__row"])} FROM ${from(r)}`, + columns: node.columns, + kinds: pick(r, node.columns) + } + } + case "Omit": { + const r = input() + const columns = r.columns.filter((c) => !node.columns.includes(c)) + return { sql: `SELECT ${list([...columns, "__row"])} FROM ${from(r)}`, columns, kinds: pick(r, columns) } + } + case "Rename": { + const r = input() + const columns = r.columns.map((c) => node.mapping[c] ?? c) + return { + sql: `SELECT ${[...r.columns.map((c, i) => `${quote(c)} AS ${quote(columns[i])}`), ROW].join(", ")} FROM ${ + from(r) + }`, + columns, + kinds: r.kinds + } + } + case "Sort": { + const r = input() + const scope = scopeOf(r, dialect) + const keys = node.keys.map((k) => + `${expression(k.expression, scope)} ${k.descending ? "DESC" : "ASC"} NULLS ${node.nullsLast ? "LAST" : "FIRST"}` + ) + return rowOrdered(r.columns, r.kinds, r, [...keys, ROW].join(", ")) + } + case "Slice": { + const r = input() + if (node.offset >= 0) { + return same( + r, + `SELECT * FROM ${from(r)} ORDER BY ${ROW}${ + node.length === undefined ? "" : ` LIMIT ${node.length}` + } OFFSET ${node.offset}` + ) + } + const upper = node.length === undefined ? "" : ` AND "__i" <= "__n" + ${node.offset} + ${node.length}` + return same( + r, + `SELECT ${ + list([...r.columns, "__row"]) + } FROM (SELECT *, COUNT(*) OVER () AS "__n", ROW_NUMBER() OVER (ORDER BY ${ROW}) AS "__i" FROM ${ + from(r, "s") + }) AS t WHERE "__i" > "__n" + ${node.offset}${upper}` + ) + } + case "Unique": { + const r = input() + const keys = node.columns ?? r.columns + return same( + r, + `SELECT ${list([...r.columns, "__row"])} FROM (SELECT *, ROW_NUMBER() OVER (PARTITION BY ${ + list(keys) + } ORDER BY ${ROW}) AS "__k" FROM ${from(r, "s")}) AS t WHERE "__k" = 1` + ) + } + case "DropNulls": { + const r = input() + const keys = node.columns ?? r.columns + return same( + r, + `SELECT * FROM ${from(r)}${ + keys.length === 0 ? "" : ` WHERE ${keys.map((k) => `${quote(k)} IS NOT NULL`).join(" AND ")}` + }` + ) + } + case "GroupBy": + case "Aggregate": { + const r = input() + const scope = scopeOf(r, dialect) + const keys = node._tag === "GroupBy" ? node.keys : [] + const aggs = node.aggregates.map((a) => + a.expression._tag === "AggregateCall" + ? `${aggregate(a.expression, scope, undefined)} AS ${quote(a.name)}` + : V.fail(`${a.name} is not an aggregate expression`) + ) + const row = keys.length === 0 ? `1 AS ${ROW}` : `MIN(${ROW}) AS ${ROW}` + return { + sql: `SELECT ${[...keys.map(quote), ...aggs, row].join(", ")} FROM ${from(r)}${ + keys.length === 0 ? "" : ` GROUP BY ${list(keys)}` + }`, + columns: [...keys, ...node.aggregates.map((a) => a.name)], + kinds: [...pick(r, keys), ...node.aggregates.map((a) => kindOf(a.expression, scope))] + } + } + case "Join": { + const l = relation(node.left, dialect) + const rr = relation(node.right, dialect) + const on = node.on.map((k) => `l.${quote(k)} = r.${quote(k)}`).join(" AND ") + if (node.how === "semi" || node.how === "anti") { + return same( + l, + `SELECT * FROM (${l.sql}) AS l WHERE ${ + node.how === "anti" ? "NOT " : "" + }EXISTS (SELECT 1 FROM (${rr.sql}) AS r WHERE ${on})` + ) + } + const rightOnly = rr.columns.filter((c) => !node.on.includes(c)) + const clash = rightOnly.find((c) => l.columns.includes(c)) + if (clash !== undefined) return V.fail(`both sides of the join have a column ${JSON.stringify(clash)}`) + const left = l.columns.map((c) => + node.how === "full" && node.on.includes(c) + ? `COALESCE(l.${quote(c)}, r.${quote(c)}) AS ${quote(c)}` + : `l.${quote(c)}` + ) + const join = node.how === "inner" ? "INNER JOIN" : node.how === "left" ? "LEFT JOIN" : "FULL JOIN" + return { + sql: `SELECT ${ + [...left, ...rightOnly.map((c) => `r.${quote(c)}`)].join(", ") + }, ROW_NUMBER() OVER (ORDER BY l.${ROW}${ + node.how === "full" ? " NULLS LAST" : "" + }, r.${ROW}) AS ${ROW} FROM (${l.sql}) AS l ${join} (${rr.sql}) AS r ON ${on}`, + columns: [...l.columns, ...rightOnly], + kinds: [...l.kinds, ...pick(rr, rightOnly)] + } + } + case "Concat": { + const inputs = node.inputs.map((n) => relation(n, dialect)) + if (inputs.length === 0) return V.fail("concat needs at least one input") + const columns = inputs[0].columns + const union = inputs + .map((r, i) => `SELECT ${list([...columns, "__row"])}, ${i} AS "__source" FROM (${r.sql}) AS t${i}`) + .join(" UNION ALL ") + return rowOrdered(columns, inputs[0].kinds, { sql: union, columns, kinds: inputs[0].kinds }, `"__source", ${ROW}`) + } + case "Pivot": + case "Unpivot": + case "Explode": + case "Unnest": + return unsupported(node._tag, dialect) + } +} + +/** A plan as one SQL query, its rows in the plan's row order. @internal */ +export const compile = ( + plan: Plan.Node, + dialect: Dialect +): { readonly sql: string; readonly columns: ReadonlyArray } => { + const r = relation(plan, dialect) + return { + sql: `SELECT ${r.columns.length === 0 ? "1" : list(r.columns)} FROM (${r.sql}) AS t ORDER BY ${ROW}`, + columns: r.columns + } +} diff --git a/packages/effect/src/dataframe/internal/stream.ts b/packages/effect/src/dataframe/internal/stream.ts new file mode 100644 index 00000000000..62cb0467f5c --- /dev/null +++ b/packages/effect/src/dataframe/internal/stream.ts @@ -0,0 +1,230 @@ +/** + * Running plans batch by batch over streamed tables. + * + * Row-wise operations (filters, projections, derived columns without + * windows, renames, `dropNulls`, `explode`, `unnest`) run on every batch as + * it arrives, a slice from the start stops pulling batches once it has its + * rows, joins stream their left side against the collected right side, and + * concatenations stream their inputs one after the other. Grouping and whole- + * table aggregation fold every batch into partial aggregates, so memory grows + * with the number of groups, not rows (medians and distinct counts collect + * instead). Every other operation collects its inputs first. + * + * @internal + */ +import * as S from "../../arrow/ArrowSchema.ts" +import * as Effect from "../../Effect.ts" +import * as Stream from "../../Stream.ts" +import type { Catalog } from "../Catalog.ts" +import type { DataFrameError } from "../DataFrame.ts" +import type * as Plan from "../Plan.ts" +import type * as Agg from "./aggregate.ts" +import * as E from "./expr.ts" +import * as F from "./frame.ts" +import * as K from "./kernels.ts" +import * as Partial from "./partial.ts" +import * as Q from "./query.ts" +import { conform } from "./schema.ts" +import * as V from "./vector.ts" + +type Batches = Stream.Stream + +const typeOf: Record = { + Int8: S.int8, + Int16: S.int16, + Int32: S.int32, + Int64: S.int64, + UInt8: S.uint8, + UInt16: S.uint16, + UInt32: S.uint32, + UInt64: S.uint64, + Float32: S.float32, + Float64: S.float64, + Utf8: S.utf8, + Binary: S.binary, + Bool: S.bool, + Null: S.nullType, + Date: S.date("day"), + Timestamp: S.timestamp("microsecond"), + Duration: S.duration("microsecond") +} + +/** A table without rows: its declared columns, or none. */ +const emptyTable = (node: Extract): F.Frame => + F.make( + (node.columns ?? []).map((c) => + new F.Column(c.name, K.nullsOf(typeOf[c.type] ?? S.nullType, 0), undefined, c.nullable) + ), + 0 + ) + +/** The plan over no rows (every input empty), run in place. */ +const runEmpty = (node: Q.Node): Effect.Effect => { + const empty = (n: Q.Node): Q.Node => { + switch (n._tag) { + case "Table": + return { _tag: "Values", frame: emptyTable(n) as any } + case "Values": + return { _tag: "Values", frame: F.slice(n.frame as unknown as F.Frame, 0, 0) as any } + case "Join": + return { ...n, left: empty(n.left), right: empty(n.right) } + case "Concat": + return { ...n, inputs: n.inputs.map(empty) } + default: + return { ...n, input: empty((n as { readonly input: Q.Node }).input) } as Q.Node + } + } + return Q.execute(empty(node)) +} + +const collectFrames = (batches: Batches, node: Q.Node): Effect.Effect => + Effect.flatMap(Stream.runCollect(batches), (frames) => + frames.length === 0 + ? runEmpty(node) + : Q.attempt(() => frames.length === 1 ? frames[0] : F.concat(frames))) + +const isRowWise = (node: Q.Node): boolean => { + switch (node._tag) { + case "Filter": + case "Pick": + case "Omit": + case "Rename": + case "DropNulls": + case "Explode": + case "Unnest": + return true + case "Select": + case "WithColumns": + return node.columns.every((c) => E.isRowWise(c.expression)) + default: + return false + } +} + +/** Runs a node by collecting its inputs (an operation that needs all rows). */ +const collected = (node: Q.Node, catalog: Catalog.Service): Batches => + Stream.unwrap( + Effect.forEach(Q.children(node), (c) => collectFrames(batches(c, catalog), c)).pipe( + Effect.flatMap((inputs) => Q.attempt(() => Q.step(node, inputs))), + Effect.map(Stream.succeed) + ) + ) + +const aggregate = ( + node: Extract, + catalog: Catalog.Service +): Batches => { + const keys = node._tag === "GroupBy" ? node.keys : [] + const specs = node.aggregates.map(({ expression, name }) => { + if (expression._tag !== "AggregateCall") return V.fail(`${name} is not an aggregate expression`) + if (expression.arg === undefined && expression.fn !== "count") return V.fail(`${expression.fn} needs an argument`) + const fn: Agg.AggFn = expression.arg === undefined ? "len" : expression.fn + return { name, fn, arg: expression.arg } + }) + if (!specs.every((s) => Partial.combinable(s.fn))) return collected(node, catalog) + const fns = specs.map((s) => s.fn) + // The first batch's input vectors and key columns, for result types. + let template: { readonly inputs: ReadonlyArray; readonly frame: F.Frame } | undefined + return Stream.unwrap( + Stream.runFoldEffect( + batches(node.input, catalog), + (): Partial.State | undefined => undefined, + (state, batch) => + Q.attempt(() => { + const ctx = F.context(batch) + const inputs = specs.map((s) => (s.arg === undefined ? undefined : E.evaluateColumn(s.arg, ctx))) + template ??= { inputs, frame: batch } + const part = Partial.group( + keys.map((k) => F.column(batch, k).values()), + fns.map((fn, a) => ({ fn, vector: inputs[a] })), + batch.length + ) + return state === undefined ? part : Partial.merge(fns, [state, part]) + }) + ).pipe( + Effect.flatMap((state) => { + if (state === undefined || template === undefined) return runEmpty(node) + const t = template + return Q.attempt(() => { + const keyColumns = keys.map((k, i) => + new F.Column(k, state.keys[i], undefined, F.column(t.frame, k).nullable) + ) + const columns = specs.map((s, a) => + new F.Column(s.name, Partial.finish(s.fn, state.parts[a], t.inputs[a]), undefined) + ) + return F.make([...keyColumns, ...columns], node._tag === "Aggregate" ? 1 : state.count) + }) + }), + Effect.map(Stream.succeed) + ) + ) +} + +const slice = (input: Batches, offset: number, length: number | undefined): Batches => + Stream.suspend(() => { + let skip = offset + let left = length ?? Infinity + return input.pipe( + Stream.takeWhile(() => left > 0), + Stream.map((batch) => { + const start = Math.min(skip, batch.length) + skip -= start + const end = Math.min(batch.length, start + left) + left -= end - start + return F.slice(batch, start, end) + }), + Stream.filter((batch) => batch.length > 0) + ) + }) + +/** The batches of a node's result. @internal */ +export const batches = (node: Q.Node, catalog: Catalog.Service): Batches => { + if (isRowWise(node)) { + return Stream.mapEffect( + batches((node as { readonly input: Q.Node }).input, catalog), + (batch) => Q.attempt(() => Q.step(node, [batch])) + ).pipe(Stream.filter((batch) => batch.length > 0)) + } + switch (node._tag) { + case "Table": + return Stream.suspend(() => catalog.stream(node.name)).pipe( + Stream.mapEffect((frame) => { + const batch = frame as unknown as F.Frame + return Q.attempt(() => (node.columns === undefined ? batch : conform(batch, node.columns))) + }) + ) + case "Values": + return Stream.succeed(node.frame as unknown as F.Frame) + case "Slice": + return node.offset >= 0 ? slice(batches(node.input, catalog), node.offset, node.length) : collected(node, catalog) + case "Join": { + if (node.how === "full") return collected(node, catalog) + const right = collectFrames(batches(node.right, catalog), node.right) + return Stream.unwrap(Effect.map(right, (r) => + Stream.mapEffect( + batches(node.left, catalog), + (batch) => Q.attempt(() => F.join(batch, r, node.on, node.how)) + ))) + } + case "Concat": + return node.inputs.length === 0 + ? collected(node, catalog) + : node.inputs.map((input) => batches(input, catalog)).reduce((a, b) => Stream.concat(a, b)) + case "GroupBy": + case "Aggregate": + return aggregate(node, catalog) + default: + return collected(node, catalog) + } +} + +/** The batches of a plan's result, after optimisation. @internal */ +export const stream = (plan: Plan.Node, catalog: Catalog.Service): Batches => + Stream.unwrap(Effect.map(Q.attempt(() => Q.optimize(plan as Q.Node)), (optimized) => batches(optimized, catalog))) + +/** A plan's result as one frame, computed batch by batch. @internal */ +export const collect = (plan: Plan.Node, catalog: Catalog.Service): Effect.Effect => + Effect.flatMap( + Q.attempt(() => Q.optimize(plan as Q.Node)), + (optimized) => collectFrames(batches(optimized, catalog), optimized) + ) diff --git a/packages/effect/src/dataframe/internal/temporal.ts b/packages/effect/src/dataframe/internal/temporal.ts new file mode 100644 index 00000000000..d1957e77981 --- /dev/null +++ b/packages/effect/src/dataframe/internal/temporal.ts @@ -0,0 +1,320 @@ +/** + * Dates, timestamps and durations: unit conversions, calendar fields, + * truncation and casts, on their integer storage (days since the epoch for + * dates, units since the epoch for timestamps, units for durations). + * + * Calendar arithmetic uses Howard Hinnant's day-count algorithms, exact for + * negative days. Timestamps are read as UTC: their calendar fields are UTC + * fields (timestamps with another time zone are not supported yet). + * + * @internal + */ +import * as S from "../../arrow/ArrowSchema.ts" +import * as V from "./vector.ts" + +/** @internal */ +export const isTemporal = (t: S.DataType): boolean => + t._tag === "Date" || t._tag === "Timestamp" || t._tag === "Duration" || t._tag === "Time" + +const perSecond: Record = { + second: 1, + millisecond: 1_000, + microsecond: 1_000_000, + nanosecond: 1_000_000_000 +} +const fineness: Record = { second: 0, millisecond: 1, microsecond: 2, nanosecond: 3 } + +/** The coarser of two units (the common unit of mixed operands, as in Polars). @internal */ +export const coarser = (a: S.TimeUnit, b: S.TimeUnit): S.TimeUnit => (fineness[a] <= fineness[b] ? a : b) + +const zoned = (t: S.Timestamp): boolean => t.timezone !== undefined && t.timezone !== "UTC" && t.timezone !== "+00:00" + +/** Whether two timestamps' zones are compatible (no zone counts as UTC). */ +const sameZone = (a: S.Timestamp, b: S.Timestamp): boolean => (zoned(a) || zoned(b) ? a.timezone === b.timezone : true) + +/** The common type of two temporal types, if they have one. @internal */ +export const supertype = (a: S.DataType, b: S.DataType): S.DataType | undefined => { + if (a._tag === "Timestamp" && b._tag === "Timestamp") { + return sameZone(a, b) ? S.timestamp(coarser(a.unit, b.unit), a.timezone ?? b.timezone) : undefined + } + if (a._tag === "Timestamp" && b._tag === "Date") return a + if (a._tag === "Date" && b._tag === "Timestamp") return b + if (a._tag === "Date" && b._tag === "Date") return S.date("day") + if (a._tag === "Duration" && b._tag === "Duration") return S.duration(coarser(a.unit, b.unit)) + return undefined +} + +// ----------------------------------------------------------------------------- +// Integer helpers +// ----------------------------------------------------------------------------- + +const floorDiv = (a: bigint, b: bigint): bigint => { + const q = a / b + return (a % b !== BigInt(0)) && ((a < BigInt(0)) !== (b < BigInt(0))) ? q - BigInt(1) : q +} + +/** Rescales integer counts of `from` units to `to` units (flooring). */ +const rescale = (values: BigInt64Array, from: S.TimeUnit, to: S.TimeUnit): BigInt64Array => { + if (from === to) return values + const out = new BigInt64Array(values.length) + if (fineness[to] > fineness[from]) { + const f = BigInt(perSecond[to] / perSecond[from]) + for (let i = 0; i < values.length; i++) out[i] = BigInt.asIntN(64, values[i] * f) + } else { + const f = BigInt(perSecond[from] / perSecond[to]) + for (let i = 0; i < values.length; i++) out[i] = floorDiv(values[i], f) + } + return out +} + +const dayUnits = (unit: S.TimeUnit): bigint => BigInt(86_400 * perSecond[unit]) + +/** Days since the epoch of every row (dates and timestamps). */ +const days = (v: V.Vector): Float64Array => { + const out = new Float64Array(v.length) + if (v.kind === "i32") { + for (let i = 0; i < v.length; i++) out[i] = v.values[i] + } else if (v.kind === "i64") { + const per = v.type._tag === "Timestamp" ? dayUnits(v.type.unit) : BigInt(86_400_000) // Date64: milliseconds + for (let i = 0; i < v.length; i++) out[i] = Number(floorDiv(v.values[i], per)) + } + return out +} + +// ----------------------------------------------------------------------------- +// Civil dates +// ----------------------------------------------------------------------------- + +/** Year, month (1-12) and day (1-31) of a day count. @internal */ +export const civil = (z: number): readonly [number, number, number] => { + z += 719468 + const era = Math.floor(z / 146097) + const doe = z - era * 146097 + const yoe = Math.floor((doe - Math.floor(doe / 1460) + Math.floor(doe / 36524) - Math.floor(doe / 146096)) / 365) + const doy = doe - (365 * yoe + Math.floor(yoe / 4) - Math.floor(yoe / 100)) + const mp = Math.floor((5 * doy + 2) / 153) + const d = doy - Math.floor((153 * mp + 2) / 5) + 1 + const m = mp < 10 ? mp + 3 : mp - 9 + return [yoe + era * 400 + (m <= 2 ? 1 : 0), m, d] +} + +/** The day count of a date. @internal */ +export const daysFromCivil = (year: number, month: number, day: number): number => { + const y = month <= 2 ? year - 1 : year + const era = Math.floor(y / 400) + const yoe = y - era * 400 + const doy = Math.floor((153 * (month + (month > 2 ? -3 : 9)) + 2) / 5) + day - 1 + const doe = yoe * 365 + Math.floor(yoe / 4) - Math.floor(yoe / 100) + doy + return era * 146097 + doe - 719468 +} + +// ----------------------------------------------------------------------------- +// Fields +// ----------------------------------------------------------------------------- + +/** @internal */ +export type Field = + | "year" + | "month" + | "day" + | "hour" + | "minute" + | "second" + | "millisecond" + | "weekday" + | "ordinalDay" + +const fieldType: Record = { + year: S.int32, + month: S.int8, + day: S.int8, + hour: S.int8, + minute: S.int8, + second: S.int8, + millisecond: S.int32, + weekday: S.int8, + ordinalDay: S.int16 +} + +const checkTimestamp = (v: V.Vector, fn: string): void => { + if (v.type._tag !== "Date" && v.type._tag !== "Timestamp") { + V.fail(`${fn} is defined for dates and timestamps, not ${v.type._tag}`) + } + if (v.type._tag === "Timestamp" && zoned(v.type)) V.fail(`${fn} of timestamps with a time zone is not supported yet`) +} + +/** A calendar or clock field of every date or timestamp. @internal */ +export const field = (fn: Field, v: V.Vector): V.Vector => { + checkTimestamp(v, fn) + const n = v.length + const out = new Int32Array(n) + const d = days(v) + if (fn === "hour" || fn === "minute" || fn === "second" || fn === "millisecond") { + if (v.type._tag !== "Timestamp" || v.kind !== "i64") return V.i32(fieldType[fn], out, v.valid) + const unit = v.type.unit + const per = dayUnits(unit) + const second = BigInt(perSecond[unit]) + for (let i = 0; i < n; i++) { + const rem = v.values[i] - floorDiv(v.values[i], per) * per + const seconds = Number(rem / second) + out[i] = fn === "hour" + ? Math.floor(seconds / 3600) + : fn === "minute" + ? Math.floor(seconds / 60) % 60 + : fn === "second" + ? seconds % 60 + : Number((rem % second) * BigInt(1000) / second) + } + return V.i32(fieldType[fn], out, v.valid) + } + for (let i = 0; i < n; i++) { + const z = d[i] + switch (fn) { + case "weekday": + out[i] = (((z + 3) % 7) + 7) % 7 + 1 + break + case "ordinalDay": { + const [y] = civil(z) + out[i] = z - daysFromCivil(y, 1, 1) + 1 + break + } + default: { + const [y, m, day] = civil(z) + out[i] = fn === "year" ? y : fn === "month" ? m : day + } + } + } + return V.i32(fieldType[fn], out, v.valid) +} + +// ----------------------------------------------------------------------------- +// Truncation +// ----------------------------------------------------------------------------- + +const parseEvery = (every: string): { readonly n: number; readonly unit: string } => { + const match = /^(\d+)(ns|us|ms|s|m|h|d|w|mo|y)$/.exec(every) + if (match === null) return V.fail(`truncate expects an interval like "1h", "1d", "1w", "1mo" or "1y", got ${every}`) + const n = Number(match[1]) + if (n < 1) return V.fail(`truncate needs a positive interval, got ${every}`) + return { n, unit: match[2] } +} + +const fixed: Record = { s: 1, m: 60, h: 3600, d: 86_400, w: 7 * 86_400 } + +/** + * Every date or timestamp rounded down to a multiple of `every`: seconds, + * minutes, hours and days from the epoch, weeks from a Monday, months and + * years from the start of the calendar month or year. + * + * @internal + */ +export const truncate = (v: V.Vector, every: string): V.Vector => { + checkTimestamp(v, "truncate") + const { n: k, unit } = parseEvery(every) + const n = v.length + if (unit === "mo" || unit === "y") { + const months = unit === "y" ? 12 * k : k + const d = days(v) + const starts = new Float64Array(n) + for (let i = 0; i < n; i++) { + const [y, m] = civil(d[i]) + const index = Math.floor((y * 12 + (m - 1)) / months) * months + starts[i] = daysFromCivil(Math.floor(index / 12), (index % 12) + 1, 1) + } + return fromDays(v, starts) + } + if (v.kind === "i32") { + // dates: whole days + if (unit !== "d" && unit !== "w") return v + const size = k * (unit === "w" ? 7 : 1) + const offset = unit === "w" ? 3 : 0 // 1970-01-01 is a Thursday; weeks start on Monday + const out = new Int32Array(n) + for (let i = 0; i < n; i++) out[i] = Math.floor((v.values[i] + offset) / size) * size - offset + return V.i32(v.type, out, v.valid) + } + if (v.kind !== "i64" || v.type._tag !== "Timestamp") return V.fail(`truncate is not defined for ${v.type._tag}`) + const per = perSecond[v.type.unit] + const sub: Record = { ns: 1e-9, us: 1e-6, ms: 1e-3 } + const size = BigInt(Math.round((fixed[unit] ?? sub[unit]) * k * per)) + if (size <= BigInt(0)) return v + const offset = unit === "w" ? BigInt(3) * dayUnits(v.type.unit) : BigInt(0) + const out = new BigInt64Array(n) + for (let i = 0; i < n; i++) out[i] = floorDiv(v.values[i] + offset, size) * size - offset + return V.i64(v.type, out, v.valid) +} + +/** Day counts as the input's type (dates stay dates). */ +const fromDays = (v: V.Vector, d: Float64Array): V.Vector => { + if (v.kind === "i32") return V.i32(v.type, Int32Array.from(d), v.valid) + const per = v.type._tag === "Timestamp" ? dayUnits(v.type.unit) : BigInt(86_400_000) + return V.i64(v.type, BigInt64Array.from(d, (x) => BigInt(x) * per), v.valid) +} + +// ----------------------------------------------------------------------------- +// Durations +// ----------------------------------------------------------------------------- + +/** @internal */ +export type Total = "totalDays" | "totalHours" | "totalMinutes" | "totalSeconds" | "totalMilliseconds" + +const totalSeconds: Record = { + totalDays: 86_400, + totalHours: 3600, + totalMinutes: 60, + totalSeconds: 1, + totalMilliseconds: 0.001 +} + +/** A duration in whole units, rounded toward zero, as Int64. @internal */ +export const total = (fn: Total, v: V.Vector): V.Vector => { + if (v.type._tag !== "Duration" || v.kind !== "i64") { + return V.fail(`${fn} is defined for durations, not ${v.type._tag}`) + } + const per = BigInt(Math.round(totalSeconds[fn] * perSecond[v.type.unit])) + const out = new BigInt64Array(v.length) + for (let i = 0; i < v.length; i++) out[i] = v.values[i] / per + return V.i64(S.int64, out, v.valid) +} + +// ----------------------------------------------------------------------------- +// Casts +// ----------------------------------------------------------------------------- + +const asI64 = (v: V.Vector): BigInt64Array => + v.kind === "i64" ? v.values : v.kind === "i32" ? BigInt64Array.from(v.values, BigInt) : V.fail("expected integers") + +/** + * Casts between dates, timestamps, durations and their integers; `undefined` + * when neither side is temporal. + * + * @internal + */ +export const cast = (v: V.Vector, target: S.DataType): V.Vector | undefined => { + const from = v.type + if (!isTemporal(from) && !isTemporal(target)) return undefined + if (from._tag === "Timestamp" && target._tag === "Timestamp") { + return V.i64(target, rescale(asI64(v), from.unit, target.unit), v.valid) + } + if (from._tag === "Duration" && target._tag === "Duration") { + return V.i64(target, rescale(asI64(v), from.unit, target.unit), v.valid) + } + if (from._tag === "Timestamp" && target._tag === "Date") { + return target.unit === "day" + ? V.i32(target, Int32Array.from(days(v)), v.valid) + : V.i64(target, rescale(asI64(v), from.unit, "millisecond"), v.valid) + } + if (from._tag === "Date" && target._tag === "Timestamp") { + const per = from.unit === "day" ? dayUnits(target.unit) : BigInt(perSecond[target.unit] / 1000) + return V.i64(target, BigInt64Array.from(asI64(v), (x) => BigInt.asIntN(64, x * per)), v.valid) + } + // Integers and temporal values of the same width reinterpret each other. + if (target._tag === "Int" || isTemporal(target)) { + const kind = V.kindOf(target) + if (kind === "i64") return V.i64(target, asI64(v), v.valid) + if (kind === "i32" && v.kind === "i32") return V.i32(target, v.values, v.valid) + if (kind === "i32" && v.kind === "i64") { + return V.i32(target, Int32Array.from(v.values, (x) => Number(BigInt.asIntN(32, x))), v.valid) + } + } + return undefined +} diff --git a/packages/effect/src/dataframe/internal/vector.ts b/packages/effect/src/dataframe/internal/vector.ts new file mode 100644 index 00000000000..760446363e0 --- /dev/null +++ b/packages/effect/src/dataframe/internal/vector.ts @@ -0,0 +1,900 @@ +/** + * Physical columns used by the DataFrame kernels. + * + * Every Arrow type the DataFrame supports maps onto one of five physical + * layouts, so kernels only need a loop per layout: + * + * - `f64`: `Float64Array` (Float32, Float64 and UInt32) + * - `i32`: `Int32Array` (Int8, Int16, Int32, UInt8, UInt16, Date32, Time32) + * - `i64`: `BigInt64Array` (Int64, UInt64 (bit pattern), Date64, Timestamp, + * Time64, Duration) + * - `bool`: `Uint8Array` with one byte per value (0 or 1) + * - `str`: UTF-8 bytes plus `Int32Array` offsets (Utf8, LargeUtf8, Utf8View + * and Binary variants) + * - `null`: no buffers + * + * Validity is one byte per slot (1 = valid), `undefined` when no slot is + * null. Values behind null slots are unspecified but always readable. + * + * The logical Arrow type rides along so results keep their type (an Int8 sum + * stays Int8, a timestamp stays a timestamp). + * + * @internal + */ +import type * as ArrowData from "../../arrow/ArrowData.ts" +import * as S from "../../arrow/ArrowSchema.ts" +import { typed } from "../../arrow/internal/buffers.ts" + +/** @internal */ +export type Vector = F64 | I32 | I64 | Bool | Str | Null | List | Struct + +/** @internal */ +export interface F64 { + readonly kind: "f64" + readonly type: S.DataType + readonly length: number + readonly values: Float64Array + readonly valid: Uint8Array | undefined +} +/** @internal */ +export interface I32 { + readonly kind: "i32" + readonly type: S.DataType + readonly length: number + readonly values: Int32Array + readonly valid: Uint8Array | undefined +} +/** @internal */ +export interface I64 { + readonly kind: "i64" + readonly type: S.DataType + readonly length: number + readonly values: BigInt64Array + readonly valid: Uint8Array | undefined +} +/** @internal */ +export interface Bool { + readonly kind: "bool" + readonly type: S.DataType + readonly length: number + readonly values: Uint8Array + readonly valid: Uint8Array | undefined +} +/** @internal */ +export interface Str { + readonly kind: "str" + readonly type: S.DataType + readonly length: number + /** `offsets[i]..offsets[i + 1]` is slot `i`'s byte range in `data`. */ + readonly offsets: Int32Array + readonly data: Uint8Array + readonly valid: Uint8Array | undefined +} +/** @internal */ +export interface Null { + readonly kind: "null" + readonly type: S.DataType + readonly length: number + readonly valid: Uint8Array | undefined +} + +/** @internal */ +/** @internal */ +export interface List { + readonly kind: "list" + readonly type: S.DataType + readonly length: number + /** `offsets[i]..offsets[i + 1]` are slot `i`'s elements in `child`. */ + readonly offsets: Int32Array + readonly child: Vector + readonly valid: Uint8Array | undefined +} +/** @internal */ +export interface Struct { + readonly kind: "struct" + readonly type: S.DataType + readonly length: number + /** One vector per field, in the type's field order. */ + readonly children: ReadonlyArray + readonly valid: Uint8Array | undefined +} + +/** @internal */ +export class DataFrameDefect extends Error { + readonly _tag = "DataFrameDefect" +} + +/** @internal */ +export const fail = (message: string): never => { + throw new DataFrameDefect(message) +} + +// ----------------------------------------------------------------------------- +// Constructors +// ----------------------------------------------------------------------------- + +/** @internal */ +export const f64 = (type: S.DataType, values: Float64Array, valid?: Uint8Array): F64 => ({ + kind: "f64", + type, + length: values.length, + values, + valid +}) +/** @internal */ +export const i32 = (type: S.DataType, values: Int32Array, valid?: Uint8Array): I32 => ({ + kind: "i32", + type, + length: values.length, + values, + valid +}) +/** @internal */ +export const i64 = (type: S.DataType, values: BigInt64Array, valid?: Uint8Array): I64 => ({ + kind: "i64", + type, + length: values.length, + values, + valid +}) +/** @internal */ +export const bool = (values: Uint8Array, valid?: Uint8Array): Bool => ({ + kind: "bool", + type: S.bool, + length: values.length, + values, + valid +}) +/** @internal */ +export const str = (type: S.DataType, offsets: Int32Array, data: Uint8Array, valid?: Uint8Array): Str => ({ + kind: "str", + type, + length: offsets.length - 1, + offsets, + data, + valid +}) +/** @internal */ +export const list = (type: S.DataType, offsets: Int32Array, child: Vector, valid?: Uint8Array): List => ({ + kind: "list", + type, + length: offsets.length - 1, + offsets, + child, + valid +}) +/** @internal */ +export const struct = ( + type: S.DataType, + children: ReadonlyArray, + length: number, + valid?: Uint8Array +): Struct => ({ + kind: "struct", + type, + length, + children, + valid +}) +/** The element type of a list type. @internal */ +export const elementType = (type: S.DataType): S.DataType => + type._tag === "List" || type._tag === "LargeList" ? type.child.type : fail(`expected a list, got ${type._tag}`) +/** @internal */ +export const nulls = (length: number): Null => ({ kind: "null", type: S.nullType, length, valid: undefined }) + +/** @internal */ +export const withValid = (v: V, valid: Uint8Array | undefined): V => ({ ...v, valid }) + +/** @internal */ +export const withType = (v: V, type: S.DataType): V => ({ ...v, type }) + +/** Whether slot `i` holds a value. @internal */ +export const isValid = (v: Vector, i: number): boolean => + v.kind === "null" ? false : v.valid === undefined || v.valid[i] === 1 + +/** Number of null slots. @internal */ +export const nullCount = (v: Vector): number => { + if (v.kind === "null") return v.length + const valid = v.valid + if (valid === undefined) return 0 + let n = 0 + for (let i = 0; i < v.length; i++) n += valid[i] + return v.length - n +} + +/** Drops an all-ones validity mask. @internal */ +export const compactValid = (valid: Uint8Array | undefined): Uint8Array | undefined => { + if (valid === undefined) return undefined + for (let i = 0; i < valid.length; i++) if (valid[i] === 0) return valid + return undefined +} + +/** Validity of a slot-wise AND of two masks. @internal */ +export const andValid = ( + a: Uint8Array | undefined, + b: Uint8Array | undefined, + length: number +): Uint8Array | undefined => { + if (a === undefined) return b + if (b === undefined) return a + const out = new Uint8Array(length) + for (let i = 0; i < length; i++) out[i] = a[i] & b[i] + return out +} + +// ----------------------------------------------------------------------------- +// Physical classification of Arrow types +// ----------------------------------------------------------------------------- + +/** @internal */ +export type Kind = Vector["kind"] + +/** The physical layout of a supported logical type, or `undefined`. @internal */ +export const kindOf = (type: S.DataType): Kind | undefined => { + switch (type._tag) { + case "Null": + return "null" + case "Bool": + return "bool" + case "Int": + return type.bitWidth === 64 ? "i64" : type.bitWidth === 32 && !type.signed ? "f64" : "i32" + case "Float": + return type.bitWidth === 16 ? undefined : "f64" + case "Utf8": + case "LargeUtf8": + case "Utf8View": + case "Binary": + case "LargeBinary": + case "BinaryView": + return "str" + case "Date": + return type.unit === "day" ? "i32" : "i64" + case "Time": + return type.bitWidth === 32 ? "i32" : "i64" + case "Timestamp": + case "Duration": + return "i64" + case "Dictionary": + return kindOf(type.valueType) + case "Decimal": + return "i64" + case "List": + case "LargeList": + return kindOf(type.child.type) === undefined ? undefined : "list" + case "Struct": + return type.fields.every((f) => kindOf(f.type) !== undefined) ? "struct" : undefined + default: + return undefined + } +} + +/** The type a column of `type` has inside a DataFrame (string variants collapse). @internal */ +export const storageType = (type: S.DataType): S.DataType => { + switch (type._tag) { + case "LargeUtf8": + case "Utf8View": + return S.utf8 + case "LargeBinary": + case "BinaryView": + return S.binary + case "Dictionary": + return storageType(type.valueType) + case "List": + case "LargeList": + return S.list(S.field(type.child.name, storageType(type.child.type), { nullable: type.child.nullable })) + case "Struct": + return S.struct(type.fields.map((f) => S.field(f.name, storageType(f.type), { nullable: f.nullable }))) + default: + return type + } +} + +/** Wraps every value of an integer column to its logical type, in place. @internal */ +export const wrapInPlace = (type: S.DataType, values: Int32Array | Float64Array): void => { + if (type._tag !== "Int") return + const n = values.length + if (type.bitWidth === 8) { + if (type.signed) { for (let i = 0; i < n; i++) values[i] = (values[i] << 24) >> 24 } + else for (let i = 0; i < n; i++) values[i] = values[i] & 0xff + } else if (type.bitWidth === 16) { + if (type.signed) { for (let i = 0; i < n; i++) values[i] = (values[i] << 16) >> 16 } + else for (let i = 0; i < n; i++) values[i] = values[i] & 0xffff + } else if (type.bitWidth === 32 && !type.signed) { + for (let i = 0; i < n; i++) values[i] = values[i] >>> 0 + } else if (type.bitWidth === 32) { + for (let i = 0; i < n; i++) values[i] = values[i] | 0 + } +} + +/** Rounds every value of a Float32 column, in place. @internal */ +export const roundFloat32InPlace = (type: S.DataType, values: Float64Array): void => { + if (type._tag === "Float" && type.bitWidth === 32) { + for (let i = 0; i < values.length; i++) values[i] = Math.fround(values[i]) + } +} + +/** Wraps 64-bit values to an unsigned type's bit pattern view. @internal */ +export const isUnsigned64 = (type: S.DataType): boolean => type._tag === "Int" && type.bitWidth === 64 && !type.signed + +// ----------------------------------------------------------------------------- +// Arrow interop +// ----------------------------------------------------------------------------- + +const unpackBits = (bits: Uint8Array, offset: number, length: number): Uint8Array => { + const out = new Uint8Array(length) + if ((offset & 7) === 0) { + const base = offset >> 3 + const full = length >> 3 + for (let b = 0; b < full; b++) { + const byte = bits[base + b] + const o = b << 3 + out[o] = byte & 1 + out[o + 1] = (byte >> 1) & 1 + out[o + 2] = (byte >> 2) & 1 + out[o + 3] = (byte >> 3) & 1 + out[o + 4] = (byte >> 4) & 1 + out[o + 5] = (byte >> 5) & 1 + out[o + 6] = (byte >> 6) & 1 + out[o + 7] = (byte >> 7) & 1 + } + for (let i = full << 3; i < length; i++) out[i] = (bits[base + (i >> 3)] >> (i & 7)) & 1 + return out + } + for (let i = 0; i < length; i++) { + const j = offset + i + out[i] = (bits[j >> 3] >> (j & 7)) & 1 + } + return out +} + +/** Packs a byte-per-slot mask into an Arrow bitmap. @internal */ +export const packBits = (bytes: Uint8Array): Uint8Array => { + const n = bytes.length + const out = new Uint8Array((n + 7) >> 3) + const full = n >> 3 + for (let b = 0; b < full; b++) { + const o = b << 3 + out[b] = bytes[o] | (bytes[o + 1] << 1) | (bytes[o + 2] << 2) | (bytes[o + 3] << 3) | + (bytes[o + 4] << 4) | (bytes[o + 5] << 5) | (bytes[o + 6] << 6) | (bytes[o + 7] << 7) + } + for (let i = full << 3; i < n; i++) out[i >> 3] |= bytes[i] << (i & 7) + return out +} + +const encoder = new TextEncoder() +const decoder = new TextDecoder() + +/** Decodes slot `i` of a string vector. @internal */ +export const decodeString = (v: Str, i: number): string => { + const start = v.offsets[i] + const end = v.offsets[i + 1] + if (end === start) return "" + // Short ASCII strings are cheaper to build by hand than through TextDecoder. + if (end - start <= 16) { + let s = "" + const data = v.data + for (let k = start; k < end; k++) { + const c = data[k] + if (c >= 0x80) return decoder.decode(data.subarray(start, end)) + s += String.fromCharCode(c) + } + return s + } + return decoder.decode(v.data.subarray(start, end)) +} + +/** Encodes a string to UTF-8. @internal */ +export const encodeString = (s: string): Uint8Array => encoder.encode(s) + +/** Builds a string vector from JS values (`null` for missing slots). @internal */ +export const stringsFrom = (type: S.DataType, values: ReadonlyArray): Str => { + const n = values.length + const offsets = new Int32Array(n + 1) + let data: Uint8Array = new Uint8Array(Math.max(16, n * 8)) + let valid: Uint8Array | undefined + let at = 0 + for (let i = 0; i < n; i++) { + const v = values[i] + if (v === null || v === undefined) { + if (valid === undefined) valid = new Uint8Array(n).fill(1) + valid[i] = 0 + } else if (typeof v === "string") { + // At most 3 bytes per UTF-16 code unit. + const need = at + v.length * 3 + if (need > data.length) data = grow(data, need) + at += encoder.encodeInto(v, data.subarray(at)).written + } else { + const need = at + v.length + if (need > data.length) data = grow(data, need) + data.set(v, at) + at += v.length + } + offsets[i + 1] = at + } + return str(type, offsets, data.subarray(0, at), valid) +} + +const grow = (data: Uint8Array, need: number): Uint8Array => { + const out = new Uint8Array(Math.max(need, data.length * 2)) + out.set(data) + return out +} + +const viewOffsets = (data: ArrowData.ArrowData, large: boolean): { offsets: Int32Array; bytes: Uint8Array } => { + const n = data.length + if (!large) { + const all = typed(Int32Array, data.buffers[0]) + const first = all[data.offset] + const bytes = data.buffers[1] + if (first === 0) return { offsets: all.subarray(data.offset, data.offset + n + 1), bytes } + const offsets = new Int32Array(n + 1) + for (let i = 0; i <= n; i++) offsets[i] = all[data.offset + i] - first + return { offsets, bytes: bytes.subarray(first) } + } + const all = typed(BigInt64Array, data.buffers[0]) + const first = Number(all[data.offset]) + const last = Number(all[data.offset + n]) + if (last - first > 0x7fffffff) fail("string column larger than 2 GiB") + const offsets = new Int32Array(n + 1) + for (let i = 0; i <= n; i++) offsets[i] = Number(all[data.offset + i]) - first + return { offsets, bytes: data.buffers[1].subarray(first) } +} + +const fromViews = (type: S.DataType, data: ArrowData.ArrowData, valid: Uint8Array | undefined): Str => { + const n = data.length + const views = data.buffers[0] + const dv = new DataView(views.buffer, views.byteOffset, views.byteLength) + let total = 0 + for (let i = 0; i < n; i++) total += dv.getInt32((data.offset + i) * 16, true) + const out = new Uint8Array(total) + const offsets = new Int32Array(n + 1) + let at = 0 + for (let i = 0; i < n; i++) { + const base = (data.offset + i) * 16 + const len = dv.getInt32(base, true) + if (len <= 12) out.set(views.subarray(base + 4, base + 4 + len), at) + else { + const buffer = data.buffers[1 + dv.getInt32(base + 8, true)] + const off = dv.getInt32(base + 12, true) + out.set(buffer.subarray(off, off + len), at) + } + at += len + offsets[i + 1] = at + } + return str(type, offsets, out, valid) +} + +/** Converts array data to a vector. Copies only where the layouts differ. @internal */ +export const fromArrowData = (data: ArrowData.ArrowData): Vector => { + const type = data.type + if (type._tag === "Dictionary") { + const dictionary = data.dictionary === undefined + ? fail("dictionary-encoded column without a dictionary") + : fromArrowData(data.dictionary) + const indexData: ArrowData.ArrowData = { ...data, type: type.indexType, dictionary: undefined } + const indices = fromArrowData(indexData) + const idx = indices.kind === "i64" + ? Int32Array.from(indices.values, Number) + : indices.kind === "f64" + ? Int32Array.from(indices.values) + : (indices as I32).values + return take(dictionary, idx, indices.valid) + } + const n = data.length + const valid = data.validity === undefined || data.nullCount === 0 + ? undefined + : unpackBits(data.validity, data.offset, n) + const storage = storageType(type) + switch (type._tag) { + case "Null": + return nulls(n) + case "Bool": + return bool(unpackBits(data.buffers[0], data.offset, n), valid) + case "Int": { + const o = data.offset + const b = data.buffers[0] + switch (type.bitWidth) { + case 8: + return i32( + type, + type.signed + ? Int32Array.from(typed(Int8Array, b).subarray(o, o + n)) + : Int32Array.from(b.subarray(o, o + n)), + valid + ) + case 16: + return i32( + type, + Int32Array.from((type.signed ? typed(Int16Array, b) : typed(Uint16Array, b)).subarray(o, o + n)), + valid + ) + case 32: + return type.signed + ? i32(type, typed(Int32Array, b).subarray(o, o + n), valid) + : f64(type, Float64Array.from(typed(Uint32Array, b).subarray(o, o + n)), valid) + default: + return i64(type, typed(BigInt64Array, b).subarray(o, o + n), valid) + } + } + case "Float": + if (type.bitWidth === 64) { + return f64(type, typed(Float64Array, data.buffers[0]).subarray(data.offset, data.offset + n), valid) + } + if (type.bitWidth === 32) { + return f64( + type, + Float64Array.from(typed(Float32Array, data.buffers[0]).subarray(data.offset, data.offset + n)), + valid + ) + } + return fail("Float16 columns are not supported") + case "Date": + case "Time": + case "Timestamp": + case "Duration": { + const k = kindOf(type) + return k === "i32" + ? i32(type, typed(Int32Array, data.buffers[0]).subarray(data.offset, data.offset + n), valid) + : i64(type, typed(BigInt64Array, data.buffers[0]).subarray(data.offset, data.offset + n), valid) + } + case "Utf8": + case "Binary": + case "LargeUtf8": + case "LargeBinary": { + const { bytes, offsets } = viewOffsets(data, type._tag === "LargeUtf8" || type._tag === "LargeBinary") + return str(storage, offsets, bytes, valid) + } + case "Utf8View": + case "BinaryView": + return fromViews(storage, data, valid) + case "Decimal": { + const o = data.offset + const b = data.buffers[0] + if (type.bitWidth === 32) { + return i64(type, BigInt64Array.from(typed(Int32Array, b).subarray(o, o + n), BigInt), valid) + } + if (type.bitWidth === 64) return i64(type, typed(BigInt64Array, b).subarray(o, o + n), valid) + // 128 or 256 bits: keep the low word when the rest only extends its sign. + const words = type.bitWidth / 64 + const all = typed(BigInt64Array, b) + const out = new BigInt64Array(n) + for (let i = 0; i < n; i++) { + const base = (o + i) * words + const lo = all[base] + const sign = lo < BigInt(0) ? BigInt(-1) : BigInt(0) + for (let w = 1; w < words; w++) { + if (all[base + w] !== sign && (valid === undefined || valid[i] === 1)) { + return fail(`decimal values beyond 18 digits are not supported (column of ${type.bitWidth}-bit decimals)`) + } + } + out[i] = lo + } + return i64(type, out, valid) + } + case "List": + case "LargeList": { + const raw = type._tag === "LargeList" + ? Int32Array.from(typed(BigInt64Array, data.buffers[0]).subarray(data.offset, data.offset + n + 1), Number) + : typed(Int32Array, data.buffers[0]).subarray(data.offset, data.offset + n + 1) + const child = fromArrowData(data.children[0]) + const first = raw[0] + if (first === 0) return list(storage, raw, child, valid) + const offsets = Int32Array.from(raw, (o) => o - first) + return list(storage, offsets, slice(child, first, raw[n]), valid) + } + case "Struct": { + const children = data.children.map((c) => { + const v = fromArrowData(c) + return v.length === n && data.offset === 0 ? v : slice(v, data.offset, data.offset + n) + }) + return struct(storage, children, n, valid) + } + default: + return fail(`columns of type ${type._tag} are not supported`) + } +} + +/** Converts a vector to Arrow array data (offset 0). @internal */ +export const toArrowData = (v: Vector): ArrowData.ArrowData => { + const n = v.length + const nullCount_ = nullCount(v) + const validity = v.kind !== "null" && v.valid !== undefined && nullCount_ > 0 ? packBits(v.valid) : undefined + const base = { + type: v.type, + length: n, + offset: 0, + nullCount: nullCount_, + validity, + children: [], + dictionary: undefined + } + const bytes = (a: ArrayBufferView) => new Uint8Array(a.buffer, a.byteOffset, a.byteLength) + switch (v.kind) { + case "null": + return { ...base, validity: undefined, buffers: [] } + case "list": + return { ...base, buffers: [bytes(v.offsets)], children: [toArrowData(v.child)] } + case "struct": + return { ...base, buffers: [], children: v.children.map(toArrowData) } + case "bool": + return { ...base, buffers: [packBits(v.values)] } + case "str": + return { ...base, buffers: [bytes(v.offsets), v.data] } + case "i64": { + const t = v.type + if (t._tag === "Decimal" && t.bitWidth !== 64) { + if (t.bitWidth === 32) return { ...base, buffers: [bytes(Int32Array.from(v.values, Number))] } + const words = t.bitWidth / 64 + const out = new BigInt64Array(n * words) + for (let i = 0; i < n; i++) { + out[i * words] = v.values[i] + const sign = v.values[i] < BigInt(0) ? BigInt(-1) : BigInt(0) + for (let w = 1; w < words; w++) out[i * words + w] = sign + } + return { ...base, buffers: [bytes(out)] } + } + return { ...base, buffers: [bytes(v.values)] } + } + case "i32": { + const t = v.type + if (t._tag === "Int" && t.bitWidth === 8) { + return { ...base, buffers: [bytes(t.signed ? Int8Array.from(v.values) : Uint8Array.from(v.values))] } + } + if (t._tag === "Int" && t.bitWidth === 16) { + return { ...base, buffers: [bytes(t.signed ? Int16Array.from(v.values) : Uint16Array.from(v.values))] } + } + return { ...base, buffers: [bytes(v.values)] } + } + case "f64": { + const t = v.type + if (t._tag === "Int") return { ...base, buffers: [bytes(Uint32Array.from(v.values))] } + if (t._tag === "Float" && t.bitWidth === 32) return { ...base, buffers: [bytes(Float32Array.from(v.values))] } + return { ...base, buffers: [bytes(v.values)] } + } + } +} + +// ----------------------------------------------------------------------------- +// Gather, slice, concat +// ----------------------------------------------------------------------------- + +/** + * Gathers `indices` from `v`. A negative index produces a null slot. `valid` + * (optional) marks additional null output slots. + * + * @internal + */ +export const take = (v: Vector, indices: Int32Array, extraValid?: Uint8Array): Vector => { + const n = indices.length + let hasMissing = false + for (let i = 0; i < n; i++) { + if (indices[i] < 0) { + hasMissing = true + break + } + } + if (v.kind === "null") return nulls(n) + let valid: Uint8Array | undefined + const src = v.valid + if (src !== undefined || hasMissing || extraValid !== undefined) { + valid = new Uint8Array(n) + if (src === undefined && !hasMissing) valid.set(extraValid!) + else { + for (let i = 0; i < n; i++) { + const j = indices[i] + valid[i] = j < 0 ? 0 : src === undefined ? 1 : src[j] + } + if (extraValid !== undefined) { for (let i = 0; i < n; i++) valid[i] &= extraValid[i] } + } + } + switch (v.kind) { + case "list": { + const so = v.offsets + const offsets = new Int32Array(n + 1) + let total = 0 + for (let i = 0; i < n; i++) { + const j = indices[i] + if (j >= 0) total += so[j + 1] - so[j] + offsets[i + 1] = total + } + const elements = new Int32Array(total) + let k = 0 + for (let i = 0; i < n; i++) { + const j = indices[i] + if (j < 0) continue + for (let e = so[j]; e < so[j + 1]; e++) elements[k++] = e + } + return list(v.type, offsets, take(v.child, elements), valid) + } + case "struct": + return struct(v.type, v.children.map((c) => take(c, indices)), n, valid) + case "f64": { + const a = v.values + const o = new Float64Array(n) + if (hasMissing) { for (let i = 0; i < n; i++) o[i] = indices[i] < 0 ? 0 : a[indices[i]] } + else for (let i = 0; i < n; i++) o[i] = a[indices[i]] + return f64(v.type, o, valid) + } + case "i32": { + const a = v.values + const o = new Int32Array(n) + if (hasMissing) { for (let i = 0; i < n; i++) o[i] = indices[i] < 0 ? 0 : a[indices[i]] } + else for (let i = 0; i < n; i++) o[i] = a[indices[i]] + return i32(v.type, o, valid) + } + case "i64": { + // Gather 64-bit values as pairs of 32-bit halves: much faster than bigint reads. + const a = new Int32Array(v.values.buffer, v.values.byteOffset, v.values.length * 2) + const o = new Int32Array(n * 2) + for (let i = 0; i < n; i++) { + const j = indices[i] + if (j >= 0) { + o[2 * i] = a[2 * j] + o[2 * i + 1] = a[2 * j + 1] + } + } + return i64(v.type, new BigInt64Array(o.buffer), valid) + } + case "bool": { + const a = v.values + const o = new Uint8Array(n) + if (hasMissing) { for (let i = 0; i < n; i++) o[i] = indices[i] < 0 ? 0 : a[indices[i]] } + else for (let i = 0; i < n; i++) o[i] = a[indices[i]] + return bool(o, valid) + } + case "str": { + const offsets = new Int32Array(n + 1) + const so = v.offsets + let total = 0 + for (let i = 0; i < n; i++) { + const j = indices[i] + if (j >= 0) total += so[j + 1] - so[j] + offsets[i + 1] = total + } + const data = new Uint8Array(total) + const sd = v.data + let i = 0 + while (i < n) { + const j = indices[i] + if (j < 0) { + i++ + continue + } + // Copy runs of consecutive source rows in one go. + let e = i + 1 + while (e < n && indices[e] === indices[e - 1] + 1) e++ + const s = so[j] + const end = so[indices[e - 1] + 1] + const at = offsets[i] + const len = end - s + if (len < 16) { for (let k = 0; k < len; k++) data[at + k] = sd[s + k] } + else data.set(sd.subarray(s, end), at) + i = e + } + return str(v.type, offsets, data, valid) + } + } +} + +/** Slots `[start, end)`, sharing buffers. @internal */ +export const slice = (v: Vector, start: number, end: number): Vector => { + const valid = v.valid?.subarray(start, end) + switch (v.kind) { + case "list": { + const first = v.offsets[start] + const offsets = first === 0 + ? v.offsets.subarray(start, end + 1) + : Int32Array.from(v.offsets.subarray(start, end + 1), (o) => o - first) + return list(v.type, offsets, slice(v.child, first, v.offsets[end]), valid) + } + case "struct": + return struct(v.type, v.children.map((c) => slice(c, start, end)), end - start, valid) + case "null": + return nulls(end - start) + case "f64": + return f64(v.type, v.values.subarray(start, end), valid) + case "i32": + return i32(v.type, v.values.subarray(start, end), valid) + case "i64": + return i64(v.type, v.values.subarray(start, end), valid) + case "bool": + return bool(v.values.subarray(start, end), valid) + case "str": { + const first = v.offsets[start] + if (first === 0) return str(v.type, v.offsets.subarray(start, end + 1), v.data, valid) + const offsets = new Int32Array(end - start + 1) + for (let i = 0; i <= end - start; i++) offsets[i] = v.offsets[start + i] - first + return str(v.type, offsets, v.data.subarray(first, v.offsets[end]), valid) + } + } +} + +/** Concatenates vectors of the same kind and type. @internal */ +export const concat = (parts: ReadonlyArray): Vector => { + if (parts.length === 1) return parts[0] + const first = parts[0] + const n = parts.reduce((s, p) => s + p.length, 0) + let valid: Uint8Array | undefined + if (parts.some((p) => p.kind === "null" || p.valid !== undefined)) { + valid = new Uint8Array(n) + let at = 0 + for (const p of parts) { + if (p.kind !== "null") { + if (p.valid === undefined) valid.fill(1, at, at + p.length) + else valid.set(p.valid, at) + } + at += p.length + } + } + const kind = parts.find((p) => p.kind !== "null")?.kind ?? "null" + if (kind === "null") return nulls(n) + const type = parts.find((p) => p.kind !== "null")!.type + if (kind === "list") { + const offsets = new Int32Array(n + 1) + const children: Array = [] + let row = 0 + let at = 0 + for (const p of parts) { + if (p.kind === "list") { + for (let i = 1; i <= p.length; i++) offsets[row + i] = at + p.offsets[i] + at += p.offsets[p.length] + children.push(p.child) + } else for (let i = 1; i <= p.length; i++) offsets[row + i] = at + row += p.length + } + return list(type, offsets, concat(children.length === 0 ? [nulls(0)] : children), valid) + } + if (kind === "struct") { + const fields = (parts.find((p) => p.kind === "struct") as Struct).children.length + const children = Array.from( + { length: fields }, + (_, f) => concat(parts.map((p) => (p.kind === "struct" ? p.children[f] : nulls(p.length)))) + ) + return struct(type, children, n, valid) + } + const fill = (out: T): T => { + let at = 0 + for (const p of parts) { + if (p.kind !== "null") out.set((p as F64).values as any, at) + at += p.length + } + return out + } + switch (kind) { + case "f64": + return f64(type, fill(new Float64Array(n)), valid) + case "i32": + return i32(type, fill(new Int32Array(n)), valid) + case "i64": + return i64(type, fill(new BigInt64Array(n)), valid) + case "bool": + return bool(fill(new Uint8Array(n)), valid) + case "str": { + const total = parts.reduce((s, p) => s + (p.kind === "str" ? p.offsets[p.length] - p.offsets[0] : 0), 0) + const data = new Uint8Array(total) + const offsets = new Int32Array(n + 1) + let at = 0 + let row = 0 + for (const p of parts) { + if (p.kind === "str") { + const base = p.offsets[0] + data.set(p.data.subarray(base, p.offsets[p.length]), at) + for (let i = 1; i <= p.length; i++) offsets[row + i] = at + p.offsets[i] - base + at += p.offsets[p.length] - base + } else { + for (let i = 1; i <= p.length; i++) offsets[row + i] = at + } + row += p.length + } + return str(type, offsets, data, valid) + } + } + return first +} + +/** A vector of `length` copies of one value. @internal */ +export const broadcast = (v: Vector, length: number): Vector => { + if (v.length === length) return v + if (v.length !== 1) return fail(`cannot broadcast a column of length ${v.length} to ${length}`) + return take(v, new Int32Array(length)) +} diff --git a/packages/effect/src/dataframe/internal/window.ts b/packages/effect/src/dataframe/internal/window.ts new file mode 100644 index 00000000000..8e0851ddaf5 --- /dev/null +++ b/packages/effect/src/dataframe/internal/window.ts @@ -0,0 +1,313 @@ +/** + * Window functions: values computed from the other rows of a row's + * partition, in row order (Polars semantics). Without a partition, the whole + * column is one partition. + * + * @internal + */ +import * as S from "../../arrow/ArrowSchema.ts" +import type * as Plan from "../Plan.ts" +import * as D from "./dtype.ts" +import { compareBytes } from "./kernels.ts" +import * as Sort from "./sort.ts" +import * as V from "./vector.ts" + +/** Rows' partition ids (dense, in order of first appearance). @internal */ +export interface Partition { + readonly ids: Int32Array + readonly count: number +} + +/** One partition holding every row. @internal */ +export const whole = (n: number): Partition => ({ ids: new Int32Array(n), count: n > 0 ? 1 : 0 }) + +/** The rows of every partition, in row order: `rows[starts[g]..starts[g + 1])`. */ +const rowsByPartition = (p: Partition): { readonly starts: Int32Array; readonly rows: Int32Array } => { + const n = p.ids.length + const starts = new Int32Array(p.count + 1) + for (let i = 0; i < n; i++) starts[p.ids[i] + 1]++ + for (let g = 0; g < p.count; g++) starts[g + 1] += starts[g] + const at = starts.slice(0, p.count) + const rows = new Int32Array(n) + for (let i = 0; i < n; i++) rows[at[p.ids[i]]++] = i + return { starts, rows } +} + +const validOf = (v: V.Vector): Uint8Array | undefined => + v.kind === "null" ? new Uint8Array(v.length) : v.valid === undefined ? undefined : v.valid.slice() + +// ----------------------------------------------------------------------------- +// Cumulative +// ----------------------------------------------------------------------------- + +/** Running sum per partition; nulls stay null and are skipped. @internal */ +export const cumSum = (v: V.Vector, p: Partition): V.Vector => { + const n = v.length + const ids = p.ids + const type = D.sumType(v.type) + const valid = validOf(v) + switch (v.kind) { + case "null": + return V.nulls(n) + case "f64": { + const x = v.values + const acc = new Float64Array(p.count) + const out = new Float64Array(n) + for (let i = 0; i < n; i++) if (valid === undefined || valid[i] === 1) out[i] = acc[ids[i]] += x[i] + V.wrapInPlace(type, out) + V.roundFloat32InPlace(type, out) + return V.f64(type, out, valid) + } + case "i32": { + const x = v.values + const acc = new Int32Array(p.count) + const out = new Int32Array(n) + for (let i = 0; i < n; i++) { + if (valid === undefined || valid[i] === 1) out[i] = acc[ids[i]] = (acc[ids[i]] + x[i]) | 0 + } + return V.i32(type, out, valid) + } + case "bool": { + const x = v.values + const acc = new Float64Array(p.count) + const out = new Float64Array(n) + for (let i = 0; i < n; i++) if (valid === undefined || valid[i] === 1) out[i] = acc[ids[i]] += x[i] + return V.f64(S.uint32, out, valid) + } + case "i64": { + const x = v.values + const acc = new BigInt64Array(p.count) + const out = new BigInt64Array(n) + for (let i = 0; i < n; i++) { + if (valid === undefined || valid[i] === 1) out[i] = acc[ids[i]] = BigInt.asIntN(64, acc[ids[i]] + x[i]) + } + return V.i64(type, out, valid) + } + default: + return V.fail(`cumSum is not defined for ${D.describe(v.type)}`) + } +} + +/** Number of non-null values so far per partition (no nulls). @internal */ +export const cumCount = (v: V.Vector, p: Partition): V.Vector => { + const n = v.length + const acc = new Float64Array(p.count) + const out = new Float64Array(n) + for (let i = 0; i < n; i++) { + if (V.isValid(v, i)) acc[p.ids[i]]++ + out[i] = acc[p.ids[i]] + } + return V.f64(S.uint32, out) +} + +/** `compare(i, j) < 0` when row i sorts before row j (valid, non-NaN rows). */ +const rowOrder = (v: V.Vector): (i: number, j: number) => number => { + switch (v.kind) { + case "list": + case "struct": + return V.fail(`cannot order ${v.type._tag} values`) + case "f64": { + const x = v.values + return (i, j) => (x[i] < x[j] ? -1 : x[i] > x[j] ? 1 : 0) + } + case "i32": + case "bool": { + const x = v.values + return (i, j) => x[i] - x[j] + } + case "i64": { + const x = v.values + const unsigned = V.isUnsigned64(v.type) + return unsigned + ? ( + i, + j + ) => (BigInt.asUintN(64, x[i]) < BigInt.asUintN(64, x[j]) + ? -1 + : BigInt.asUintN(64, x[i]) > BigInt.asUintN(64, x[j]) + ? 1 + : 0) + : (i, j) => (x[i] < x[j] ? -1 : x[i] > x[j] ? 1 : 0) + } + case "str": { + const o = v.offsets + const d = v.data + return (i, j) => compareBytes(d, o[i], o[i + 1], d, o[j], o[j + 1]) + } + case "null": + return () => 0 + } +} + +const isNaNAt = (v: V.Vector, i: number): boolean => v.kind === "f64" && v.values[i] !== v.values[i] + +/** Running minimum or maximum per partition; NaNs are skipped, nulls stay null. @internal */ +export const cumMinMax = (fn: "cumMin" | "cumMax", v: V.Vector, p: Partition): V.Vector => { + const n = v.length + const order = rowOrder(v) + const sign = fn === "cumMin" ? -1 : 1 + const best = new Int32Array(p.count).fill(-1) + const rows = new Int32Array(n) + for (let i = 0; i < n; i++) { + if (!V.isValid(v, i)) { + rows[i] = -1 + continue + } + const g = p.ids[i] + if (!isNaNAt(v, i) && (best[g] < 0 || order(i, best[g]) * sign > 0)) best[g] = i + rows[i] = best[g] < 0 ? i : best[g] + } + return V.take(v, rows) +} + +// ----------------------------------------------------------------------------- +// Shifts +// ----------------------------------------------------------------------------- + +/** The value `n` rows earlier in the partition (later for negative `n`). @internal */ +export const shift = (v: V.Vector, p: Partition, n: number): V.Vector => { + const { rows, starts } = rowsByPartition(p) + const idx = new Int32Array(v.length).fill(-1) + for (let g = 0; g < p.count; g++) { + const s = starts[g] + const e = starts[g + 1] + for (let j = s; j < e; j++) { + const k = j - n + if (k >= s && k < e) idx[rows[j]] = rows[k] + } + } + return V.take(v, idx) +} + +// ----------------------------------------------------------------------------- +// Rank +// ----------------------------------------------------------------------------- + +/** Rank within the partition; nulls get no rank. @internal */ +export const rank = (v: V.Vector, p: Partition, method: Plan.RankMethod, descending: boolean): V.Vector => { + const n = v.length + const perm = Sort.permutation( + [{ vector: V.i32(S.int32, p.ids), descending: false }, { vector: v, descending }], + true, + n + ) + const words = Sort.words({ vector: v, descending }) + const same = (a: number, b: number) => + words.lo[a] === words.lo[b] && (words.hi === undefined || words.hi[a] === words.hi[b]) + const out = new Float64Array(n) + const valid = validOf(v) + let i = 0 + while (i < n) { + // one partition: perm[i..end) + const g = p.ids[perm[i]] + let end = i + while (end < n && p.ids[perm[end]] === g) end++ + let position = 0 + let dense = 0 + let s = i + while (s < end && V.isValid(v, perm[s])) { + let e = s + 1 + while (e < end && V.isValid(v, perm[e]) && same(perm[s], perm[e])) e++ + dense++ + const first = position + 1 + const last = position + (e - s) + for (let k = s; k < e; k++) { + const row = perm[k] + out[row] = method === "average" + ? (first + last) / 2 + : method === "min" + ? first + : method === "max" + ? last + : method === "dense" + ? dense + : position + (k - s) + 1 + } + position = last + s = e + } + i = end + } + return V.f64(method === "average" ? S.float64 : S.uint32, out, valid) +} + +// ----------------------------------------------------------------------------- +// Rolling +// ----------------------------------------------------------------------------- + +/** + * An aggregate over the last `window` rows of the partition (the current row + * included). Null when fewer than `minPeriods` of them are non-null; NaN + * propagates. + * + * @internal + */ +export const rolling = ( + fn: "rollingSum" | "rollingMean" | "rollingMin" | "rollingMax", + v: V.Vector, + p: Partition, + window: number, + minPeriods: number +): V.Vector => { + const n = v.length + const { rows, starts } = rowsByPartition(p) + const ok = new Uint8Array(n) + if (fn === "rollingMin" || fn === "rollingMax") { + const order = rowOrder(v) + const sign = fn === "rollingMin" ? -1 : 1 + const idx = new Int32Array(n).fill(-1) + for (let g = 0; g < p.count; g++) { + for (let j = starts[g]; j < starts[g + 1]; j++) { + let best = -1 + let count = 0 + for (let k = Math.max(starts[g], j - window + 1); k <= j; k++) { + const r = rows[k] + if (!V.isValid(v, r)) continue + count++ + if (best >= 0 && isNaNAt(v, best)) continue + if (isNaNAt(v, r) || best < 0 || order(r, best) * sign > 0) best = r + } + if (count >= minPeriods) idx[rows[j]] = best + } + } + return V.take(v, idx) + } + if (v.kind === "str" || v.kind === "null" || v.kind === "list" || v.kind === "struct") { + return V.fail(`${fn} is not defined for ${D.describe(v.type)}`) + } + const x = v.kind === "i64" + ? Float64Array.from(v.values, Number) + : v.kind === "f64" + ? v.values + : Float64Array.from(v.values as Int32Array | Uint8Array) + const out = new Float64Array(n) + for (let g = 0; g < p.count; g++) { + for (let j = starts[g]; j < starts[g + 1]; j++) { + let sum = 0 + let count = 0 + for (let k = Math.max(starts[g], j - window + 1); k <= j; k++) { + const r = rows[k] + if (!V.isValid(v, r)) continue + sum += x[r] + count++ + } + if (count >= minPeriods) { + ok[rows[j]] = 1 + out[rows[j]] = fn === "rollingMean" ? sum / count : sum + } + } + } + if (fn === "rollingMean") return V.f64(D.meanType(v.type), out, ok) + const type = D.sumType(v.type) + switch (v.kind) { + case "i32": + return V.i32(type, Int32Array.from(out, (a) => a | 0), ok) + case "i64": + return V.i64(type, BigInt64Array.from(out, (a) => BigInt.asIntN(64, BigInt(a))), ok) + case "bool": + return V.f64(S.uint32, out, ok) + default: + V.wrapInPlace(type, out) + return V.f64(type, out, ok) + } +} diff --git a/packages/effect/test/dataframe/DataFrame.test.ts b/packages/effect/test/dataframe/DataFrame.test.ts new file mode 100644 index 00000000000..dbec78b874c --- /dev/null +++ b/packages/effect/test/dataframe/DataFrame.test.ts @@ -0,0 +1,148 @@ +import { assert, describe, it } from "@effect/vitest" +import { Effect, Equal, Result, Schema } from "effect" +import { ArrowIpc, ArrowSchema } from "effect/arrow" +import { DataFrame, Expr, Query } from "effect/dataframe" + +const Reading = Schema.Struct({ + city: Schema.String, + temp: Schema.Number, + rain: Schema.NullOr(Schema.Number), + station: Schema.optional(Schema.String) +}) + +const readings = DataFrame.fromRows(Reading, [ + { city: "Vienna", temp: 21.5, rain: 0, station: "s1" }, + { city: "Berlin", temp: 26.5, rain: null }, + { city: "Vienna", temp: 19, rain: 2 } +]) + +const types = (df: DataFrame.DataFrame) => + df.schema.fields.map((f) => `${f.name}:${f.type._tag}:${f.nullable}`) + +describe("DataFrame", () => { + describe("constructors", () => { + it("fromRows maps schema fields to Arrow types", () => { + assert.deepStrictEqual(types(readings), [ + "city:Utf8:false", + "temp:Float:false", + "rain:Float:true", + "station:Utf8:true" + ]) + assert.deepStrictEqual(DataFrame.toArray(readings, "station") as Array, ["s1", null, null]) + }) + + it("fromRows takes per-column type overrides", () => { + const df = DataFrame.fromRows(Schema.Struct({ n: Schema.Number }), [{ n: 1 }, { n: 2 }], { + types: { n: ArrowSchema.int32 } + }) + assert.deepStrictEqual(df.schema.fields[0].type, ArrowSchema.int32) + }) + + it("fromRows rejects schemas without a column type", () => { + try { + DataFrame.fromRows(Schema.Struct({ xs: Schema.Record(Schema.String, Schema.Number) }), []) + assert.fail("expected an error") + } catch (e) { + assert.instanceOf(e, DataFrame.DataFrameError) + assert.include((e as DataFrame.DataFrameError).message, `"xs"`) + } + }) + + it("fromColumns keeps typed array element types", () => { + const df = DataFrame.fromColumns({ i: Int32Array.of(1, 2), b: BigInt64Array.of(1n, 2n), s: ["a", null] }) + assert.deepStrictEqual(types(df), ["i:Int:false", "b:Int:false", "s:Utf8:true"]) + assert.deepStrictEqual(DataFrame.toRows(df), [{ i: 1, b: 1n, s: "a" }, { i: 2, b: 2n, s: null }]) + }) + + it("fromColumns rejects columns of different lengths", () => { + assert.throws(() => DataFrame.fromColumns({ a: [1, 2], b: [1] }), DataFrame.DataFrameError) + }) + + it("empty has the schema's columns and no rows", () => { + const df = DataFrame.empty(Reading) + assert.strictEqual(df.length, 0) + assert.deepStrictEqual(DataFrame.columnNames(df), ["city", "temp", "rain", "station"]) + }) + }) + + describe("validate", () => { + const untyped = DataFrame.fromArrowTable(DataFrame.toArrowTable(readings)) + + it("returns the frame typed by the schema, in schema order", () => { + const result = DataFrame.validate(untyped, Schema.Struct({ temp: Schema.Number, city: Schema.String })) + assert.deepStrictEqual(DataFrame.columnNames(Result.getOrThrow(result)), ["temp", "city"]) + }) + + it("fails on missing columns, wrong types and unexpected nulls", () => { + const cases: ReadonlyArray, string]> = [ + [Schema.Struct({ country: Schema.String }), `missing column "country"`], + [Schema.Struct({ city: Schema.Number }), `column "city" is Utf8, expected Float64`], + [Schema.Struct({ rain: Schema.Number }), `column "rain" has missing values`] + ] + for (const [schema, message] of cases) { + const result = DataFrame.validate(untyped, schema) + assert.isTrue(Result.isFailure(result)) + assert.strictEqual(Result.isFailure(result) && result.failure.message, message) + } + }) + }) + + describe("Arrow interop", () => { + it("round-trips through the IPC stream format", () => { + const bytes = Result.getOrThrow(ArrowIpc.encodeStream(DataFrame.toArrowTable(readings))) + const back = DataFrame.fromArrowTable(Result.getOrThrow(ArrowIpc.decodeStream(bytes))) + assert.isTrue(Equal.equals(back, readings)) + }) + + it("exports late-selected columns", () => { + const df = Query.from(readings).pipe( + Query.filter((c) => Expr.isNotNull(c.rain)), + Query.sort((c) => [c.temp]), + Effect.runSync + ) + const table = DataFrame.toArrowTable(df) + assert.strictEqual(table.batches[0].length, 2) + assert.deepStrictEqual(DataFrame.toRows(DataFrame.fromArrowTable(table)), DataFrame.toRows(df)) + }) + }) + + describe("values", () => { + it("compare structurally, NaN included", () => { + const a = DataFrame.fromColumns({ x: [1, NaN, null] }) + const b = DataFrame.fromColumns({ x: [1, NaN, null] }) + const c = DataFrame.fromColumns({ x: [1, NaN, 2] }) + assert.isTrue(Equal.equals(a, b)) + assert.isFalse(Equal.equals(a, c)) + }) + + it("iterate as rows", () => { + assert.deepStrictEqual(Array.from(DataFrame.fromColumns({ city: DataFrame.toArray(readings, "city") })), [ + { city: "Vienna" }, + { city: "Berlin" }, + { city: "Vienna" } + ]) + }) + + it("format as a text table", () => { + assert.strictEqual( + DataFrame.format( + DataFrame.fromColumns({ + city: DataFrame.toArray(readings, "city"), + rain: DataFrame.toArray(readings, "rain") + }) + ), + [ + "DataFrame (3 rows × 2 columns)", + "┌──────────┬─────────┐", + "│ city │ rain │", + "│ Utf8 │ Float64 │", + "├──────────┼─────────┤", + `│ "Vienna" │ 0 │`, + `│ "Berlin" │ null │`, + `│ "Vienna" │ 2 │`, + "└──────────┴─────────┘" + ].join("\n") + ) + }) + }) +}) diff --git a/packages/effect/test/dataframe/Oracle.test.ts b/packages/effect/test/dataframe/Oracle.test.ts new file mode 100644 index 00000000000..4695edefdfb --- /dev/null +++ b/packages/effect/test/dataframe/Oracle.test.ts @@ -0,0 +1,29 @@ +/** + * Semantic oracle: every case of the gold corpus (hand-written specification + * cases plus seeded random pipelines) turned into an effect/dataframe plan, + * encoded to JSON and back, run as a query (so the optimiser rewrites it) + * and compared with the result Python Polars produced for the same plan. See + * test/fixtures/dataframe/generate_gold.py and test/dataframe/oracle. + */ +import { assert, describe, it } from "@effect/vitest" +import * as Compare from "./oracle/Compare.ts" +import * as Gold from "./oracle/Gold.ts" +import * as Plan from "./oracle/Plan.ts" + +const check = (c: Gold.Case, inline: boolean) => () => { + assert.strictEqual(Compare.diff(c.id, Plan.run(c, inline), c.expected), undefined) +} + +describe("DataFrame oracle (Polars gold)", () => { + // Specification cases carry their inputs inline (Values nodes, encoded as + // Arrow IPC in the JSON plan); random pipelines read them from a catalog. + describe("specification", () => { + for (const c of Gold.spec().cases) it(`${c.id}: ${c.description}`, check(c, true)) + }) + describe("random pipelines", () => { + for (const c of Gold.random().cases) it(c.id, check(c, false)) + }) + describe("extended (windows, reshaping, types)", () => { + for (const c of Gold.extended().cases) it(`${c.id}: ${c.description}`, check(c, false)) + }) +}) diff --git a/packages/effect/test/dataframe/Query.test.ts b/packages/effect/test/dataframe/Query.test.ts new file mode 100644 index 00000000000..32edbb57962 --- /dev/null +++ b/packages/effect/test/dataframe/Query.test.ts @@ -0,0 +1,299 @@ +import { assert, describe, it } from "@effect/vitest" +import { Effect, Exit, Schema } from "effect" +import { Catalog, DataFrame, Expr, Plan, Query } from "effect/dataframe" + +const Reading = Schema.Struct({ + city: Schema.String, + temp: Schema.Number, + rain: Schema.NullOr(Schema.Number) +}) + +const readings = DataFrame.fromRows(Reading, [ + { city: "Vienna", temp: 21.5, rain: 0 }, + { city: "Berlin", temp: 26.5, rain: null }, + { city: "Vienna", temp: 19, rain: 2 } +]) + +const City = Schema.Struct({ city: Schema.String, country: Schema.String }) +const cities = DataFrame.fromRows(City, [ + { city: "Vienna", country: "AT" }, + { city: "Berlin", country: "DE" } +]) + +const tables = Catalog.layer({ readings, cities }) + +const rows = (query: Query.Query) => Effect.map(query, DataFrame.toRows) + +describe("Query", () => { + describe("running", () => { + it.effect("is an Effect producing a frame", () => + Effect.gen(function*() { + const df = yield* Query.from(readings).pipe(Query.filter((c) => Expr.gt(c.temp, 20))) + assert.deepStrictEqual(DataFrame.toArray(df, "city"), ["Vienna", "Berlin"]) + })) + + it.effect("does no work until it runs", () => + Effect.gen(function*() { + const query = Query.table("missing", City).pipe(Query.filter((c) => Expr.eq(c.city, "Vienna"))) + const error = yield* Effect.flip(query) + assert.strictEqual(error.message, `unknown table "missing"`) + })) + + it.effect("reads each table once per run, from effects too", () => + Effect.gen(function*() { + let loads = 0 + const catalog = Catalog.make({ cities: Effect.sync(() => (loads++, cities)) }) + const query = Query.table("cities", City).pipe( + Query.join(Query.table("cities", City).pipe(Query.rename({ country: "other" })), { on: "city" }) + ) + const df = yield* Effect.provideService(query, Catalog.Catalog, catalog) + assert.strictEqual(df.length, 2) + assert.strictEqual(loads, 1) + yield* Effect.provideService(query, Catalog.Catalog, catalog) + assert.strictEqual(loads, 2) + })) + + it.effect("tables can be queries (views)", () => + Effect.gen(function*() { + const warm = Query.table("readings", Reading).pipe(Query.filter((c) => Expr.gt(c.temp, 20))) + const query = Query.table("warm", Reading).pipe(Query.pick("city")) + const result = yield* rows(query).pipe(Effect.provide(Catalog.layer({ readings, warm }))) + assert.deepStrictEqual(result, [{ city: "Vienna" }, { city: "Berlin" }]) + })) + + it.effect("checks tables against their declared columns", () => + Effect.gen(function*() { + const cases: ReadonlyArray, string]> = [ + [Schema.Struct({ country: Schema.String }), `missing column "country"`], + [Schema.Struct({ city: Schema.Number }), `column "city" is Utf8, expected Float64`], + [Schema.Struct({ rain: Schema.Number }), `column "rain" has missing values`] + ] + for (const [schema, message] of cases) { + const error = yield* Effect.flip(Query.table("readings", schema).pipe(Effect.provide(tables))) + assert.strictEqual(error.message, message) + } + })) + + it.effect("narrows tables to their declared columns", () => + Effect.gen(function*() { + const Temps = Schema.Struct({ temp: Schema.Number, city: Schema.String }) + const df = yield* Query.table("readings", Temps).pipe(Effect.provide(tables)) + assert.deepStrictEqual(DataFrame.columnNames(df), ["temp", "city"]) + })) + + it.effect("fails with DataFrameError on invalid expressions", () => + Effect.gen(function*() { + const query = Query.from(readings).pipe(Query.filter((c) => c.city as any)) + const exit = yield* Effect.exit(query) + assert.isTrue(Exit.isFailure(exit)) + const error = yield* Effect.flip(query) + assert.instanceOf(error, DataFrame.DataFrameError) + assert.strictEqual(error.message, "filter predicate must be boolean, got Utf8") + })) + }) + + describe("operations", () => { + it.effect("slice takes an offset and a length; negative offsets count from the end", () => + Effect.gen(function*() { + const n = Query.from(DataFrame.fromColumns({ n: Int32Array.of(0, 1, 2, 3, 4) })) + const values = (q: Query.Query<{ readonly n: number }>) => Effect.map(q, (df) => DataFrame.toArray(df, "n")) + assert.deepStrictEqual(yield* values(Query.slice(n, 1, 2)), [1, 2]) + assert.deepStrictEqual(yield* values(Query.slice(n, -2)), [3, 4]) + assert.deepStrictEqual(yield* values(Query.slice(n, -2, 1)), [3]) + assert.deepStrictEqual(yield* values(Query.head(n, 2)), [0, 1]) + assert.deepStrictEqual(yield* values(Query.tail(n, 2)), [3, 4]) + assert.deepStrictEqual(yield* values(Query.tail(n, 0)), []) + assert.deepStrictEqual(yield* values(Query.slice(n, 9)), []) + })) + + it.effect("append stacks rows; dropNulls removes incomplete rows", () => + Effect.gen(function*() { + const df = yield* Query.from(readings).pipe(Query.append(Query.from(readings)), Query.dropNulls(["rain"])) + assert.deepStrictEqual(DataFrame.toArray(df, "rain"), [0, 2, 0, 2]) + })) + + it.effect("joins on several keys", () => + Effect.gen(function*() { + const right = DataFrame.fromColumns({ city: ["Vienna", "Vienna"], temp: [19, 21.5], note: ["cool", "warm"] }) + const result = yield* rows( + Query.from(readings).pipe(Query.pick("city", "temp"), Query.join(Query.from(right), { on: ["city", "temp"] })) + ) + assert.deepStrictEqual(result, [ + { city: "Vienna", temp: 21.5, note: "warm" }, + { city: "Vienna", temp: 19, note: "cool" } + ]) + })) + + it.effect("cast reports values that don't fit as null", () => + Effect.gen(function*() { + const df = DataFrame.fromColumns({ x: [1.9, -1.9, 3e10, NaN] }) + const result = yield* rows( + Query.from(df).pipe(Query.select((c) => ({ i: Expr.cast(c.x, "Int32"), s: Expr.cast(c.x, "Utf8") }))) + ) + assert.deepStrictEqual(result, [ + { i: 1, s: "1.9" }, + { i: -1, s: "-1.9" }, + { i: null, s: "30000000000" }, + { i: null, s: "NaN" } + ]) + })) + }) + + describe("explain", () => { + it.effect("shows the rewritten plan", () => + Effect.gen(function*() { + const query = Query.table("readings", Reading).pipe( + Query.withColumns((c) => ({ fahrenheit: Expr.add(Expr.multiply(c.temp, 1.8), 32), loud: Expr.lit(1) })), + Query.join(Query.table("cities", City), { on: "city" }), + Query.filter((c) => Expr.and(Expr.eq(c.country, "AT"), Expr.gt(c.temp, 20))), + Query.sort((c) => [Expr.desc(c.fahrenheit)]), + Query.select((c) => ({ city: c.city, fahrenheit: c.fahrenheit })), + Query.head(1) + ) + const plan = yield* Query.explain(query).pipe(Effect.provide(tables)) + assert.strictEqual( + plan, + [ + `Select [city = col("city"), fahrenheit = col("fahrenheit")]`, + ` TopK 1 [desc(col("fahrenheit"))]`, + ` Join inner on [city]`, + ` WithColumns [fahrenheit = add(multiply(col("temp"), 1.8), 32)]`, + ` Filter gt(col("temp"), 20)`, + ` Pick [city, temp]`, + ` Values 3 rows [city, temp, rain]`, + ` Filter eq(col("country"), "AT")`, + ` Values 2 rows [city, country]` + ].join("\n") + ) + assert.deepStrictEqual(yield* rows(query).pipe(Effect.provide(tables)), [{ city: "Vienna", fahrenheit: 70.7 }]) + })) + + it.effect("pushes key predicates below groups and through renames", () => + Effect.gen(function*() { + const query = Query.from(readings).pipe( + Query.rename({ city: "town" }), + Query.groupBy("town"), + Query.agg((c) => ({ n: Expr.count(), mean: Expr.mean(c.temp) })), + Query.filter((c) => Expr.and(Expr.neq(c.town, "Berlin"), Expr.gt(c.n, 1))) + ) + assert.strictEqual( + yield* Query.explain(query), + [ + `Filter gt(col("n"), 1)`, + ` GroupBy [town] [n = count(), mean = mean(col("temp"))]`, + ` Rename [city -> town]`, + ` Filter neq(col("city"), "Berlin")`, + ` Pick [city, temp]`, + ` Values 3 rows [city, temp, rain]` + ].join("\n") + ) + assert.deepStrictEqual(yield* rows(query), [{ town: "Vienna", n: 2, mean: 20.25 }]) + })) + + it.effect("keeps predicates above left joins when they read the right side", () => + Effect.gen(function*() { + const query = Query.from(readings).pipe( + Query.join(Query.from(cities), { on: "city", how: "left" }), + Query.filter((c) => Expr.isNull(c.country)) + ) + assert.strictEqual( + yield* Query.explain(query), + [ + `Filter isNull(col("country"))`, + ` Join left on [city]`, + ` Values 3 rows [city, temp, rain]`, + ` Values 2 rows [city, country]` + ].join("\n") + ) + })) + }) + + describe("plans", () => { + it("are plain data", () => { + const query = Query.from(readings).pipe(Query.filter((c) => Expr.gt(c.temp, 20)), Query.pick("city")) + assert.deepStrictEqual(query.plan, { + _tag: "Pick", + input: { + _tag: "Filter", + input: { _tag: "Values", frame: readings }, + predicate: { + _tag: "Call", + fn: "gt", + args: [{ _tag: "Column", name: "temp" }, { _tag: "Literal", value: 20 }], + options: undefined + } + }, + columns: ["city"] + }) + }) + + it("encode to JSON without losing literal types", () => { + const values: ReadonlyArray = ["NaN", NaN, -0, Infinity, 2n ** 70n, "12", 12, true, null] + const expression: Plan.Expression = { + _tag: "Call", + fn: "isIn", + args: [{ _tag: "Column", name: "x" }], + options: { values } + } + const json = Schema.encodeSync(Plan.ExpressionFromJson)(expression) + assert.deepStrictEqual(json, { + _tag: "Call", + fn: "isIn", + args: [{ _tag: "Column", name: "x" }], + options: { + values: [ + "NaN", + { float: "NaN" }, + { float: "-0" }, + { float: "Infinity" }, + { + bigint: "1180591620717411303424" + }, + "12", + 12, + true, + null + ] + } + }) + const back = Schema.decodeUnknownSync(Plan.ExpressionFromJson)(JSON.parse(JSON.stringify(json))) + assert.deepStrictEqual(back, expression) + }) + + it.effect("round-trip through JSON with their inputs", () => + Effect.gen(function*() { + const query = Query.from(readings).pipe(Query.groupBy("city"), Query.agg((c) => ({ max: Expr.max(c.temp) }))) + const json = JSON.stringify(Schema.encodeSync(Plan.NodeFromJson)(query.plan)) + const decoded = Query.fromPlan(Schema.decodeUnknownSync(Plan.NodeFromJson)(JSON.parse(json))) + assert.deepStrictEqual(yield* rows(decoded), yield* rows(query)) + })) + + it("reject malformed JSON when decoding", () => { + const json = { _tag: "Filter", input: { _tag: "Table", name: "t" }, predicate: { _tag: "Call", fn: "explode" } } + assert.throws(() => Schema.decodeUnknownSync(Plan.NodeFromJson)(json)) + }) + + it.effect("fail at run time on the wrong number of arguments", () => + Effect.gen(function*() { + const plan: Plan.Node = { + _tag: "Filter", + input: { _tag: "Values", frame: readings }, + predicate: { _tag: "Call", fn: "gt", args: [{ _tag: "Column", name: "temp" }] } + } + const error = yield* Effect.flip(Query.fromPlan(plan)) + assert.strictEqual(error.message, "gt takes 2 argument(s), got 1") + })) + + it("expressions convert to and from plans", () => { + const e = Expr.add(Expr.lit(1n), 2n) + assert.deepStrictEqual(Expr.toPlan(e), { + _tag: "Call", + fn: "add", + args: [{ _tag: "Literal", value: 1n }, { _tag: "Literal", value: 2n }], + options: undefined + }) + assert.strictEqual(String(Expr.fromPlan(Expr.toPlan(e))), "add(1n, 2n)") + assert.isTrue(Expr.isAggregate(Expr.fromPlan({ _tag: "AggregateCall", fn: "count" }))) + }) + }) +}) diff --git a/packages/effect/test/dataframe/Scale.test.ts b/packages/effect/test/dataframe/Scale.test.ts new file mode 100644 index 00000000000..f8422a4f407 --- /dev/null +++ b/packages/effect/test/dataframe/Scale.test.ts @@ -0,0 +1,232 @@ +/** + * Differential tests on inputs large enough to reach the code paths the gold + * corpus (small tables) can't: radix sorting, hash table growth and joins + * with many keys. Each operation is compared with a plain row-by-row + * reference implementation of the same (Polars) semantics. + */ +import { assert, describe, it } from "@effect/vitest" +import { Effect } from "effect" +import { ArrowSchema, ArrowTable } from "effect/arrow" +import { DataFrame, Expr, Query } from "effect/dataframe" +import * as Result from "effect/Result" + +type Row = { + readonly f: number | null + readonly i: number | null + readonly b: bigint | null + readonly s: string | null + readonly p: boolean | null +} + +const schema = ArrowSchema.make([ + ArrowSchema.field("f", ArrowSchema.float64), + ArrowSchema.field("i", ArrowSchema.int32), + ArrowSchema.field("b", ArrowSchema.int64), + ArrowSchema.field("s", ArrowSchema.utf8), + ArrowSchema.field("p", ArrowSchema.bool) +]) + +const strings = ["a", "b", "ab", "B", "é", "😀", "￿", "", "zz", "a\u0000"] + +const generate = (seed: number, n: number, cardinality: number): Array => { + let state = seed + const next = () => { + state ^= state << 13 + state ^= state >>> 17 + state ^= state << 5 + return (state >>> 0) / 4294967296 + } + const pick = (values: ReadonlyArray) => values[Math.floor(next() * values.length)] + const maybe = (a: A): A | null => (next() < 0.1 ? null : a) + return Array.from({ length: n }, () => ({ + f: maybe(pick([NaN, -0, 0, Infinity, -Infinity, Math.floor(next() * cardinality) - cardinality / 2, 0.5, -2.5])), + i: maybe(Math.floor(next() * cardinality) - Math.floor(cardinality / 2)), + b: maybe(BigInt(Math.floor(next() * cardinality)) * 4294967311n - 2n ** 40n), + s: maybe(`${pick(strings)}${Math.floor(next() * cardinality)}`), + p: maybe(next() < 0.5) + })) +} + +const frameOf = (rows: ReadonlyArray): DataFrame.DataFrame => + DataFrame.fromArrowTable(Result.getOrThrow(ArrowTable.fromRows(schema, rows as any))) as any + +const rowsOf = (query: Query.Query): Array => DataFrame.toRows(Effect.runSync(query)) + +// ----------------------------------------------------------------------------- +// Reference semantics +// ----------------------------------------------------------------------------- + +const compareCodePoints = (a: string, b: string): number => { + const x = Array.from(a, (c) => c.codePointAt(0)!) + const y = Array.from(b, (c) => c.codePointAt(0)!) + for (let k = 0; k < Math.min(x.length, y.length); k++) if (x[k] !== y[k]) return x[k] - y[k] + return x.length - y.length +} + +const compareValues = (a: unknown, b: unknown): number => { + if (typeof a === "number" && typeof b === "number") { + if (Number.isNaN(a) || Number.isNaN(b)) return Number.isNaN(a) ? (Number.isNaN(b) ? 0 : 1) : -1 + return a < b ? -1 : a > b ? 1 : 0 + } + if (typeof a === "string") return compareCodePoints(a, b as string) + if (typeof a === "boolean") return Number(a) - Number(b) + return (a as bigint) < (b as bigint) ? -1 : (a as bigint) > (b as bigint) ? 1 : 0 +} + +const referenceSort = ( + rows: ReadonlyArray, + keys: ReadonlyArray<{ readonly name: keyof Row; readonly descending: boolean }>, + nullsLast: boolean +): Array => + [...rows].sort((r1, r2) => { + for (const { descending, name } of keys) { + const a = r1[name] + const b = r2[name] + if (a === null || b === null) { + if (a === b) continue + return (a === null ? -1 : 1) * (nullsLast ? -1 : 1) + } + const c = compareValues(a, b) * (descending ? -1 : 1) + if (c !== 0) return c + } + return 0 + }) + +const keyOf = (values: ReadonlyArray) => + JSON.stringify( + values, + (_, v) => + typeof v === "bigint" ? `${v}n` : typeof v === "number" && !Number.isFinite(v) ? String(v) : v === 0 ? 0 : v + ) + +const referenceGroups = (rows: ReadonlyArray, keys: ReadonlyArray) => { + const groups = new Map>() + for (const r of rows) { + const k = keyOf(keys.map((name) => r[name])) + const g = groups.get(k) + if (g === undefined) groups.set(k, [r]) + else g.push(r) + } + return [...groups.values()] +} + +// ----------------------------------------------------------------------------- +// Tests +// ----------------------------------------------------------------------------- + +const rows = generate(0x2545f491, 6000, 3000) +const df = Query.from(frameOf(rows)) + +describe("DataFrame at scale (differential)", () => { + for (const name of ["f", "i", "b", "s", "p"] as const) { + for (const descending of [false, true]) { + for (const nullsLast of [false, true]) { + it(`sort by ${name} ${descending ? "desc" : "asc"} nulls ${nullsLast ? "last" : "first"}`, () => { + const key = Expr.fromPlan({ _tag: "Column", name }) + const sorted = Query.sort(df, () => [descending ? Expr.desc(key) : Expr.asc(key)], { nullsLast }) + assert.deepStrictEqual(rowsOf(sorted), referenceSort(rows, [{ name, descending }], nullsLast)) + }) + } + } + } + + it("multi-key sort is stable", () => { + const keys = [{ name: "p", descending: true }, { name: "s", descending: false }, { + name: "f", + descending: true + }] as const + const sorted = Query.sort(df, (c) => [Expr.desc(c.p), Expr.asc(c.s), Expr.desc(c.f)]) + assert.deepStrictEqual(rowsOf(sorted), referenceSort(rows, keys, false)) + }) + + // A slice of a sort runs as a top-k selection. + for ( + const [name, descending, nullsLast, offset, length] of [ + ["f", true, false, 0, 10], + ["i", false, true, 0, 1], + ["s", false, false, 5, 20], + ["b", true, true, 0, 700], + ["p", false, false, 0, 50] + ] as const + ) { + it(`sort by ${name} then slice ${offset}+${length} keeps the sorted rows`, () => { + const top = df.pipe( + Query.sort((c) => { + const key: Expr.Expr = c[name] + return [descending ? Expr.desc(key) : Expr.asc(key), Expr.asc(c.i)] + }, { nullsLast }), + Query.slice(offset, length) + ) + const keys = [{ name, descending }, { name: "i", descending: false }] as const + assert.deepStrictEqual(rowsOf(top), referenceSort(rows, keys, nullsLast).slice(offset, offset + length)) + }) + } + + for (const keys of [["i"], ["s"], ["b"], ["f"], ["p", "i"], ["s", "b", "p"]] as const) { + it(`groupBy ${keys.join(", ")}`, () => { + const result = df.pipe( + Query.groupBy(...keys), + Query.agg((c) => ({ n: Expr.count(), ni: Expr.count(c.i), max: Expr.max(c.s), first: Expr.first(c.b) })) + ) + const expected = referenceGroups(rows, keys).map((g) => { + const out: Record = {} + for (const k of keys) out[k] = g[0][k] + const present = g.map((r) => r.s).filter((s): s is string => s !== null) + out.n = g.length + out.ni = g.filter((r) => r.i !== null).length + out.max = present.length === 0 ? null : present.reduce((a, b) => (compareCodePoints(a, b) >= 0 ? a : b)) + out.first = g[0].b + return out + }) + assert.deepStrictEqual(rowsOf(result), expected) + }) + } + + it("unique keeps the first row per key", () => { + const result = Query.unique(df, ["i", "p"]) + assert.deepStrictEqual(rowsOf(result), referenceGroups(rows, ["i", "p"]).map((g) => g[0])) + }) + + for (const how of ["inner", "left", "semi", "anti"] as const) { + it(`${how} join on a high-cardinality key`, () => { + const right = generate(0x1b873593, 4000, 3000).map((r) => ({ i: r.i, rs: r.s })) + const rightFrame = DataFrame.fromColumns({ + i: Int32Array.from(right, (r) => r.i ?? 0), + rs: right.map((r) => r.rs) + }) + const result = Query.join(df.pipe(Query.pick("i", "s")), Query.from(rightFrame), { on: "i", how }) + const byKey = new Map>() + for (const r of right) { + const k = r.i ?? 0 + byKey.set(k, [...(byKey.get(k) ?? []), r.rs]) + } + const expected = rows.flatMap((r) => { + const matches = r.i === null ? [] : byKey.get(r.i) ?? [] + switch (how) { + case "inner": + return matches.map((rs) => ({ i: r.i, s: r.s, rs })) + case "left": + return matches.length === 0 ? [{ i: r.i, s: r.s, rs: null }] : matches.map((rs) => ({ i: r.i, s: r.s, rs })) + case "semi": + return matches.length > 0 ? [{ i: r.i, s: r.s }] : [] + case "anti": + return matches.length === 0 ? [{ i: r.i, s: r.s }] : [] + } + }) + assert.deepStrictEqual(rowsOf(result), expected) + }) + } + + it("filters compose with later column reads", () => { + const result = df.pipe( + Query.filter((c) => Expr.isNotNull(c.i)), + Query.sort((c) => [c.i]), + Query.filter((c) => Expr.gt(c.i, 0)), + Query.head(100) + ) + const expected = referenceSort(rows.filter((r) => r.i !== null), [{ name: "i", descending: false }], false) + .filter((r) => (r.i as number) > 0) + .slice(0, 100) + assert.deepStrictEqual(rowsOf(result), expected) + }) +}) diff --git a/packages/effect/test/dataframe/Stream.test.ts b/packages/effect/test/dataframe/Stream.test.ts new file mode 100644 index 00000000000..58a1697156b --- /dev/null +++ b/packages/effect/test/dataframe/Stream.test.ts @@ -0,0 +1,180 @@ +/** + * Queries over streamed tables (a `Stream` of frames in the `Catalog`) run + * batch by batch and give the same result as over the whole table. + */ +import { assert, describe, it } from "@effect/vitest" +import { Effect, Schema, Stream } from "effect" +import { ArrowIpc } from "effect/arrow" +import { Catalog, DataFrame, Expr, Query } from "effect/dataframe" +import * as Result from "effect/Result" + +const Row = Schema.Struct({ + k: Schema.NullOr(Schema.String), + g: Schema.Number, + x: Schema.NullOr(Schema.Number) +}) +type Row = typeof Row.Type + +const generate = (seed: number, n: number): Array => { + let state = seed + const next = () => { + state ^= state << 13 + state ^= state >>> 17 + state ^= state << 5 + return (state >>> 0) / 4294967296 + } + return Array.from({ length: n }, () => ({ + k: next() < 0.1 ? null : `k${Math.floor(next() * 40)}`, + g: Math.floor(next() * 5), + x: next() < 0.1 ? null : Math.round(next() * 2000 - 1000) / 8 + })) +} + +const rows = generate(0x5eed, 2000) +const whole = DataFrame.fromRows(Row, rows) +const batchRows = 250 +const batches = Array.from( + { length: rows.length / batchRows }, + (_, b) => DataFrame.fromRows(Row, rows.slice(b * batchRows, (b + 1) * batchRows)) +) + +const Lookup = Schema.Struct({ k: Schema.String, label: Schema.String }) +const lookup = DataFrame.fromRows(Lookup, Array.from({ length: 30 }, (_, i) => ({ k: `k${i}`, label: `#${i}` }))) + +const table = Query.table("t", Row) + +/** Runs a query over the streamed table and over the whole table. */ +const both = Effect.fnUntraced(function*(query: Query.Query) { + let pulled = 0 + const streamed = Stream.fromIterable(batches).pipe(Stream.tap(() => Effect.sync(() => pulled++))) + const fromStream = DataFrame.toRows(yield* query.pipe(Effect.provide(Catalog.layer({ t: streamed, lookup })))) + const fromFrame = DataFrame.toRows(yield* query.pipe(Effect.provide(Catalog.layer({ t: whole, lookup })))) + assertClose(fromStream, fromFrame) + return { rows: fromStream, pulled } +}) + +const assertClose = (actual: ReadonlyArray, expected: ReadonlyArray) => { + assert.strictEqual(actual.length, expected.length) + const close = (a: unknown, b: unknown) => + typeof a === "number" && typeof b === "number" + ? (Number.isNaN(a) && Number.isNaN(b)) || Math.abs(a - b) <= 1e-9 * Math.max(1, Math.abs(a), Math.abs(b)) + : a === b + for (let r = 0; r < actual.length; r++) { + const keys = Object.keys(expected[r]) + if (!keys.every((k) => close(actual[r][k], expected[r][k]))) { + assert.deepStrictEqual(actual[r], expected[r], `row ${r}`) + } + } +} + +describe("Streaming", () => { + it.effect("row-wise operations run per batch", () => + Effect.gen(function*() { + const { pulled, rows } = yield* both(table.pipe( + Query.filter((c) => Expr.gt(c.x, 0)), + Query.withColumns((c) => ({ y: Expr.multiply(c.x, 2), label: Expr.concat(c.k, "!") })), + Query.pick("k", "y", "label") + )) + assert.isTrue(rows.length > 0) + assert.strictEqual(pulled, batches.length) + })) + + it.effect("a head stops reading the stream", () => + Effect.gen(function*() { + const { pulled, rows } = yield* both(table.pipe(Query.filter((c) => Expr.isNotNull(c.k)), Query.head(7))) + assert.strictEqual(rows.length, 7) + assert.isTrue(pulled <= 2, `read ${pulled} batches`) + })) + + it.effect("slices across batch boundaries", () => both(table.pipe(Query.slice(240, 30)))) + + it.effect("grouping folds batches into partial aggregates", () => + both(table.pipe( + Query.groupBy("k", "g"), + Query.agg((c) => ({ + n: Expr.count(), + nx: Expr.count(c.x), + sum: Expr.sum(c.x), + mean: Expr.mean(c.x), + std: Expr.std(c.x), + min: Expr.min(c.x), + max: Expr.max(c.k), + first: Expr.first(c.x), + last: Expr.last(c.x) + })) + ))) + + it.effect("aggregates over the whole stream", () => + both(table.pipe(Query.aggregate((c) => ({ n: Expr.count(), total: Expr.sum(c.x), variance: Expr.variance(c.x) }))))) + + it.effect("medians and distinct counts collect the stream", () => + both(table.pipe( + Query.groupBy("g"), + Query.agg((c) => ({ median: Expr.median(c.x), distinct: Expr.countDistinct(c.k) })) + ))) + + it.effect("joins stream their left side", () => + both(table.pipe(Query.join(Query.table("lookup", Lookup), { on: "k", how: "left" })))) + + it.effect("sorts collect their input", () => + both( + table.pipe(Query.filter((c) => Expr.isNotNull(c.x)), Query.sort((c) => [Expr.desc(c.x), c.k]), Query.head(20)) + )) + + it.effect("concatenates streams in order", () => + both(Query.concat([table.pipe(Query.head(3)), table.pipe(Query.tail(3))]))) + + it.effect("an empty stream gives an empty result with the declared columns", () => + Effect.gen(function*() { + const catalog = Catalog.layer({ t: Stream.empty }) + const counted = yield* table.pipe( + Query.aggregate((c) => ({ n: Expr.count(), total: Expr.sum(c.x) })), + Effect.provide(catalog) + ) + assert.deepStrictEqual(DataFrame.toRows(counted), [{ n: 0, total: 0 }]) + const grouped = yield* table.pipe( + Query.groupBy("g"), + Query.agg(() => ({ n: Expr.count() })), + Effect.provide(catalog) + ) + assert.deepStrictEqual(DataFrame.columnNames(grouped), ["g", "n"]) + assert.strictEqual(grouped.length, 0) + })) + + it.effect("Query.stream emits batches", () => + Effect.gen(function*() { + const sizes = yield* table.pipe( + Query.filter((c) => Expr.isNotNull(c.k)), + Query.stream, + Stream.map((frame) => frame.length), + Stream.runCollect, + Effect.provide(Catalog.layer({ t: Stream.fromIterable(batches) })) + ) + assert.strictEqual(sizes.length, batches.length) + const single = yield* Query.from(whole).pipe(Query.stream, Stream.runCollect) + assert.strictEqual(single.length, 1) + })) + + it.effect("decodes Arrow IPC stream bytes into batches", () => + Effect.gen(function*() { + const bytes = Result.getOrThrow(ArrowIpc.encodeStream(DataFrame.toArrowTable(whole))) + // Split the bytes at arbitrary points, as a socket would deliver them. + const chunks = [bytes.subarray(0, 100), bytes.subarray(100, 5000), bytes.subarray(5000)] + const frames = yield* DataFrame.decodeArrowStream(Stream.fromIterable(chunks)).pipe(Stream.runCollect) + const total = frames.reduce((s, f) => s + f.length, 0) + assert.strictEqual(total, whole.length) + const viaCatalog = yield* table.pipe( + Query.aggregate((c) => ({ n: Expr.count(c.x) })), + Effect.provide(Catalog.layer({ t: DataFrame.decodeArrowStream(Stream.fromIterable(chunks)) as any })) + ) + assert.deepStrictEqual(DataFrame.toRows(viaCatalog), [{ n: rows.filter((r) => r.x !== null).length }]) + })) + + it.effect("fails with DataFrameError for invalid batches", () => + Effect.gen(function*() { + const bad = Stream.make(DataFrame.fromColumns({ k: ["a"], g: ["not a number"], x: [1] })) + const error = yield* Effect.flip(table.pipe(Effect.provide(Catalog.layer({ t: bad })))) + assert.instanceOf(error, DataFrame.DataFrameError) + assert.strictEqual(error.message, `column "g" is Utf8, expected Float64`) + })) +}) diff --git a/packages/effect/test/dataframe/oracle/Compare.ts b/packages/effect/test/dataframe/oracle/Compare.ts new file mode 100644 index 00000000000..d236ab261c0 --- /dev/null +++ b/packages/effect/test/dataframe/oracle/Compare.ts @@ -0,0 +1,95 @@ +/** + * Compares a DataFrame with a gold (Polars) result: column names, column + * types and values, in row order. + * + * Floats compare with a relative tolerance of 1e-9 (summation order differs + * from Polars), NaN equals NaN, and -0 equals 0. + */ +import { BigDecimal, Effect } from "effect" +import { DataFrame, Expr, Query } from "effect/dataframe" +import * as Gold from "./Gold.ts" +import * as Plan from "./Plan.ts" + +/** + * Deliberate differences from Polars, by case id: the column types + * effect/dataframe produces instead. Values still have to match. + */ +export const deviations: { + readonly [id: string]: { readonly reason: string; readonly types: { readonly [column: string]: string } } +} = { + "group/small-ints": { + reason: "sums of 8/16-bit integers are Int32 (Polars: Int64, which would be a bigint in TypeScript)", + types: { i8: "i32", u8: "i32" } + } +} + +const close = (a: number, b: number): boolean => { + if (Number.isNaN(a) || Number.isNaN(b)) return Number.isNaN(a) && Number.isNaN(b) + if (a === b) return true + if (!Number.isFinite(a) || !Number.isFinite(b)) return false + return Math.abs(a - b) <= 1e-9 * Math.max(Math.abs(a), Math.abs(b)) + 1e-300 +} + +const same = (actual: unknown, expected: unknown): boolean => { + if (actual === null || expected === null) return actual === expected + if (BigDecimal.isBigDecimal(expected)) return BigDecimal.isBigDecimal(actual) && BigDecimal.equals(actual, expected) + if (Array.isArray(expected)) { + return Array.isArray(actual) && actual.length === expected.length && actual.every((a, i) => same(a, expected[i])) + } + if (typeof expected === "object" && typeof actual === "object") { + const keys = Object.keys(expected) + return keys.length === Object.keys(actual).length && + keys.every((k) => same((actual as any)[k] ?? null, (expected as any)[k])) + } + if (typeof actual === "number" && typeof expected === "number") return close(actual, expected) + if (typeof actual === "bigint" || typeof expected === "bigint") { + return typeof actual === typeof expected ? actual === expected : Number(actual) === Number(expected) + } + return actual === expected +} + +const show = (u: unknown): string => + BigDecimal.isBigDecimal(u) + ? `${BigDecimal.format(u)}d` + : typeof u === "bigint" + ? `${u}n` + : typeof u === "number" + ? (Object.is(u, -0) ? "-0" : String(u)) + : JSON.stringify(u) ?? String(u) + +/** `undefined` when equal, otherwise a description of the first difference. */ +export const diff = (id: string, actual: DataFrame.DataFrame, expected: Gold.Table): string | undefined => { + const names = actual.schema.fields.map((f) => f.name) + const expectedNames = expected.fields.map((f) => f.name) + if (JSON.stringify(names) !== JSON.stringify(expectedNames)) { + return `columns: got ${JSON.stringify(names)}, expected ${JSON.stringify(expectedNames)}` + } + const deviation = deviations[id] + for (const [i, f] of expected.fields.entries()) { + const got = Plan.typeName(actual.schema.fields[i].type) + const want = deviation?.types[f.name] ?? f.type + if (got !== want) return `column ${f.name}: type ${got}, expected ${want}` + } + const length = expected.fields.length === 0 ? 0 : expected.columns[expected.fields[0].name].length + if (actual.length !== length) return `length ${actual.length}, expected ${length}` + // Temporal columns compare through their integers (days or units since the epoch). + const rows = DataFrame.toRows(Effect.runSync( + Query.from(actual).pipe(Query.select((c: any) => { + const out: Record> = {} + for (const f of expected.fields) { + const t = Plan.temporal[f.type] + out[f.name] = t === undefined ? c[f.name] : Expr.cast(c[f.name], t.storage) + } + return out + })) + )) + for (const f of expected.fields) { + const column = expected.columns[f.name] + for (let i = 0; i < length; i++) { + const want = Gold.decodeValue(column[i], f.type) + const got = rows[i][f.name] + if (!same(got, want)) return `column ${f.name}, row ${i}: got ${show(got)}, expected ${show(want)}` + } + } + return undefined +} diff --git a/packages/effect/test/dataframe/oracle/Gold.ts b/packages/effect/test/dataframe/oracle/Gold.ts new file mode 100644 index 00000000000..e5fe3d576d5 --- /dev/null +++ b/packages/effect/test/dataframe/oracle/Gold.ts @@ -0,0 +1,152 @@ +/** + * The gold corpus: plans and Polars results from + * test/fixtures/dataframe/generate_gold.py. + */ +import * as BigDecimal from "effect/BigDecimal" +import * as Fs from "node:fs" + +export type TypeName = + | "bool" + | "i8" + | "i16" + | "i32" + | "i64" + | "u8" + | "u16" + | "u32" + | "u64" + | "f32" + | "f64" + | "utf8" + | "null" + | "date" + | "datetime[us]" + | "datetime[ms]" + | "duration[us]" + | "duration[ms]" + | `decimal[${string}]` + | `list<${string}>` + | `struct<${string}>` + +export interface Table { + readonly fields: ReadonlyArray<{ readonly name: string; readonly type: TypeName }> + readonly columns: { readonly [name: string]: ReadonlyArray } +} + +export type Expr = + | { readonly over: Expr; readonly partitionBy: ReadonlyArray } + | { readonly col: string } + | { readonly lit: unknown; readonly type?: TypeName } + | { readonly fn: string; readonly args: ReadonlyArray; readonly [option: string]: unknown } + | { readonly agg: string; readonly arg?: Expr } + +export type Op = + | { readonly op: "filter"; readonly predicate: Expr } + | { readonly op: "select" | "withColumns"; readonly exprs: { readonly [name: string]: Expr } } + | { readonly op: "pick" | "omit"; readonly columns: ReadonlyArray } + | { readonly op: "rename"; readonly mapping: { readonly [name: string]: string } } + | { + readonly op: "sort" + readonly keys: ReadonlyArray<{ readonly expr: Expr; readonly descending?: boolean }> + readonly nullsLast?: boolean + } + | { readonly op: "head" | "tail"; readonly n: number } + | { readonly op: "slice"; readonly offset: number; readonly length?: number } + | { readonly op: "groupBy"; readonly keys: ReadonlyArray; readonly aggs: { readonly [name: string]: Expr } } + | { readonly op: "agg"; readonly aggs: { readonly [name: string]: Expr } } + | { + readonly op: "join" + readonly right: string + readonly on: ReadonlyArray + readonly how: "inner" | "left" | "full" | "semi" | "anti" + } + | { readonly op: "unique" | "dropNulls"; readonly columns?: ReadonlyArray } + | { readonly op: "concat"; readonly with: string } + | { + readonly op: "pivot" + readonly on: string + readonly index: ReadonlyArray + readonly values: string + readonly aggregate: string + readonly columns?: ReadonlyArray + } + | { readonly op: "explode" | "unnest"; readonly column: string } + | { + readonly op: "unpivot" + readonly index: ReadonlyArray + readonly on: ReadonlyArray + readonly variable?: string + readonly value?: string + } + +export interface Case { + readonly id: string + readonly description: string + readonly inputs: { readonly [name: string]: Table } + readonly ops: ReadonlyArray + readonly expected: Table +} + +export interface Corpus { + readonly polars: string + readonly deviations: ReadonlyArray + readonly cases: ReadonlyArray +} + +const load = (name: string): Corpus => + JSON.parse(Fs.readFileSync(new URL(`../../fixtures/dataframe/gold/${name}.json`, import.meta.url), "utf8")) + +export const spec = (): Corpus => load("spec") +export const random = (): Corpus => load("random") +export const extended = (): Corpus => load("extended") + +/** Splits `a:x,b:list` on top-level commas. */ +export const splitTop = (s: string): Array => { + const parts: Array = [] + let depth = 0 + let current = "" + for (const ch of s) { + if (ch === "<") depth++ + if (ch === ">") depth-- + if (ch === "," && depth === 0) { + parts.push(current) + current = "" + } else current += ch + } + if (current !== "") parts.push(current) + return parts +} + +/** The fields of a `struct<...>` type name. */ +export const structFields = (type: string): Array => + splitTop(type.slice(7, -1)).map((f) => { + const at = f.indexOf(":") + return [f.slice(0, at), f.slice(at + 1) as TypeName] as const + }) + +/** Decodes a JSON-encoded gold value. */ +export const decodeValue = (value: unknown, type: TypeName): unknown => { + if (value === null) return null + if (type.startsWith("decimal[")) return BigDecimal.fromStringUnsafe(value as string) + if (type.startsWith("list<")) { + return (value as ReadonlyArray).map((v) => decodeValue(v, type.slice(5, -1) as TypeName)) + } + if (type.startsWith("struct<")) { + const out: Record = {} + for (const [name, t] of structFields(type)) out[name] = decodeValue((value as any)[name] ?? null, t) + return out + } + switch (type) { + case "f32": + case "f64": + if (typeof value === "string") { + return ({ NaN: NaN, Infinity: Infinity, "-Infinity": -Infinity, "-0": -0 } as Record)[value] + } + return value + case "i64": + case "u64": + return BigInt(value as string) + default: + return value + } +} diff --git a/packages/effect/test/dataframe/oracle/Plan.ts b/packages/effect/test/dataframe/oracle/Plan.ts new file mode 100644 index 00000000000..b45d0ba1a22 --- /dev/null +++ b/packages/effect/test/dataframe/oracle/Plan.ts @@ -0,0 +1,351 @@ +/** + * Gold plans as effect/dataframe plans, run through the public API. + */ +import { BigDecimal, Effect, Schema } from "effect" +import { ArrowSchema, ArrowTable } from "effect/arrow" +import { Catalog, DataFrame, Plan, Query } from "effect/dataframe" +import * as Result from "effect/Result" +import * as Gold from "./Gold.ts" + +const arrowTypes = { + bool: ArrowSchema.bool, + i8: ArrowSchema.int8, + i16: ArrowSchema.int16, + i32: ArrowSchema.int32, + i64: ArrowSchema.int64, + u8: ArrowSchema.uint8, + u16: ArrowSchema.uint16, + u32: ArrowSchema.uint32, + u64: ArrowSchema.uint64, + f32: ArrowSchema.float32, + f64: ArrowSchema.float64, + utf8: ArrowSchema.utf8, + null: ArrowSchema.nullType, + date: ArrowSchema.date("day"), + "datetime[us]": ArrowSchema.timestamp("microsecond"), + "datetime[ms]": ArrowSchema.timestamp("millisecond"), + "duration[us]": ArrowSchema.duration("microsecond"), + "duration[ms]": ArrowSchema.duration("millisecond") +} + +/** Temporal gold types: their integer storage type, and the literal type. */ +export const temporal: Partial< + Record +> = { + date: { storage: "Int32", literal: { _tag: "Date" } }, + "datetime[us]": { storage: "Int64", literal: { _tag: "Timestamp", unit: "microsecond" } }, + "datetime[ms]": { storage: "Int64", literal: { _tag: "Timestamp", unit: "millisecond" } }, + "duration[us]": { storage: "Int64", literal: { _tag: "Duration", unit: "microsecond" } }, + "duration[ms]": { storage: "Int64", literal: { _tag: "Duration", unit: "millisecond" } } +} + +/** Precision and scale of a `decimal[p,s]` name. */ +export const decimalOf = (name: string): readonly [number, number] => { + const [p, s] = name.slice(8, -1).split(",").map(Number) + return [p, s] +} + +export const arrowType = (name: Gold.TypeName): ArrowSchema.DataType => { + if (name.startsWith("decimal[")) return ArrowSchema.decimal(...decimalOf(name)) + if (name.startsWith("list<")) { + return ArrowSchema.list(ArrowSchema.field("item", arrowType(name.slice(5, -1) as Gold.TypeName))) + } + if (name.startsWith("struct<")) { + return ArrowSchema.struct(Gold.structFields(name).map(([n, t]) => ArrowSchema.field(n, arrowType(t)))) + } + return (arrowTypes as Record)[name] +} + +/** The gold type name of an Arrow type. */ +export const typeName = (type: ArrowSchema.DataType): string => { + if (type._tag === "Decimal") return `decimal[${type.precision},${type.scale}]` + if (type._tag === "List" || type._tag === "LargeList") return `list<${typeName(type.child.type)}>` + if (type._tag === "Struct") return `struct<${type.fields.map((f) => `${f.name}:${typeName(f.type)}`).join(",")}>` + for (const [name, t] of Object.entries(arrowTypes)) { + if (JSON.stringify(t) === JSON.stringify(type)) return name + } + return type._tag +} + +/** The table column type name of an Arrow type (nested types as Null: not checked). */ +const tableType = (t: ArrowSchema.DataType): Plan.TypeName => { + switch (t._tag) { + case "Int": + return `${t.signed ? "Int" : "UInt"}${t.bitWidth}` as Plan.TypeName + case "Float": + return `Float${t.bitWidth}` as Plan.TypeName + case "Utf8": + case "Bool": + case "Date": + case "Timestamp": + case "Duration": + case "Null": + case "Decimal": + return t._tag + default: + return "Null" + } +} + +/** A gold input table as a frame. */ +export const frameOf = (table: Gold.Table): DataFrame.DataFrame => { + const schema = ArrowSchema.make(table.fields.map((f) => ArrowSchema.field(f.name, arrowType(f.type)))) + const length = table.fields.length === 0 ? 0 : table.columns[table.fields[0].name].length + const rows = Array.from({ length }, (_, i) => { + const row: Record = {} + for (const f of table.fields) row[f.name] = Gold.decodeValue(table.columns[f.name][i], f.type) + return row + }) + return DataFrame.fromArrowTable(Result.getOrThrow(ArrowTable.fromRows(schema, rows))) +} + +const castNames: Partial> = { + i8: "Int8", + i16: "Int16", + i32: "Int32", + i64: "Int64", + u8: "UInt8", + u16: "UInt16", + u32: "UInt32", + u64: "UInt64", + f32: "Float32", + f64: "Float64", + utf8: "Utf8", + bool: "Bool", + date: "Date", + "datetime[us]": "Timestamp[us]", + "datetime[ms]": "Timestamp[ms]", + "duration[us]": "Duration[us]", + "duration[ms]": "Duration[ms]" +} + +const floatSpecials = new Set(["NaN", "Infinity", "-Infinity", "-0"]) + +const literal = (e: { readonly lit: unknown; readonly type?: Gold.TypeName }): Plan.Value => + (e.type !== undefined + ? Gold.decodeValue(e.lit, e.type) + : typeof e.lit === "string" && floatSpecials.has(e.lit) + ? Gold.decodeValue(e.lit, "f64") + : e.lit) as Plan.Value + +/** Gold function names that differ from the plan's. */ +const functionNames: Record = { + sub: "subtract", + mul: "multiply", + div: "divide", + floorDiv: "floorDivide", + neg: "negate", + strLength: "length", + strLower: "toLowerCase", + strUpper: "toUpperCase", + strContains: "includes", + strStartsWith: "startsWith", + strEndsWith: "endsWith", + strConcat: "concat" +} + +const options = (e: { readonly [option: string]: unknown }): Plan.CallOptions | undefined => { + switch (e.fn) { + case "round": + return { decimals: (e.decimals as number | undefined) ?? 0 } + case "cast": { + const to = e.to as string + if (to.startsWith("decimal[")) { + const [precision, scale] = decimalOf(to) + return { to: "Decimal", precision, scale } + } + return { to: castNames[to as Gold.TypeName] } + } + case "truncate": + return { every: e.every as string } + case "listGet": + return { index: e.index as number } + case "field": + return { name: e.field as string } + case "struct": + return { names: e.names as ReadonlyArray } + case "isIn": + return { + values: (e.values as ReadonlyArray).map((v) => + Gold.decodeValue(v, (e.type as Gold.TypeName) ?? "utf8") as Plan.Value + ) + } + case "strContains": + return { pattern: e.pattern as string } + case "strStartsWith": + return { pattern: e.prefix as string } + case "strEndsWith": + return { pattern: e.suffix as string } + case "shift": + case "diff": + return { n: (e.n as number | undefined) ?? 1 } + case "rank": + return { + method: (e.method as Plan.RankMethod | undefined) ?? "average", + descending: (e.descending as boolean | undefined) ?? false + } + case "rollingSum": + case "rollingMean": + case "rollingMin": + case "rollingMax": + return { window: e.window as number, minPeriods: (e.minPeriods as number | undefined) ?? (e.window as number) } + default: + return undefined + } +} + +const aggregateNames: Record = { len: "count", nUnique: "countDistinct", var: "variance" } + +/** A gold expression as plan data. */ +export const expression = (e: Gold.Expr): Plan.Expression => { + if ("over" in e) { + const o = e as { readonly over: Gold.Expr; readonly partitionBy: ReadonlyArray } + return { _tag: "Over", expression: expression(o.over), partitionBy: o.partitionBy.map(expression) } + } + if ("col" in e) return { _tag: "Column", name: e.col as string } + if ("lit" in e) { + const l = e as { readonly lit: unknown; readonly type?: Gold.TypeName } + const t = l.type === undefined ? undefined : temporal[l.type] + if (l.type !== undefined && l.type.startsWith("decimal[")) { + const [precision, scale] = decimalOf(l.type) + const value = BigDecimal.scale(BigDecimal.fromStringUnsafe(l.lit as string), scale).value + return { _tag: "Literal", value, type: { _tag: "Decimal", precision, scale } } + } + return t === undefined + ? { _tag: "Literal", value: literal(l) } + : { _tag: "Literal", value: l.lit as number, type: t.literal } + } + if ("agg" in e) { + const a = e as { readonly agg: string; readonly arg?: Gold.Expr } + return { + _tag: "AggregateCall", + fn: aggregateNames[a.agg] ?? a.agg as Plan.AggregateName, + arg: a.arg === undefined || a.agg === "len" ? undefined : expression(a.arg) + } + } + const call = e as { readonly fn: string; readonly args: ReadonlyArray; readonly [option: string]: unknown } + return { + _tag: "Call", + fn: functionNames[call.fn] ?? call.fn as Plan.FunctionName, + args: call.args.map(expression), + options: options(call) + } +} + +const named = (record: { readonly [name: string]: Gold.Expr }): Array => + Object.entries(record).map(([name, e]) => ({ name, expression: expression(e) })) + +/** + * A case's operations as one plan over its inputs. Inputs are `Table` nodes + * read from the catalog, or with `inline`, `Values` nodes holding the frames. + */ +export const plan = (gold: Gold.Case, inline: boolean, declare = false): Plan.Node => { + const columns = (name: string): Array => + gold.inputs[name].fields.map((f) => ({ name: f.name, type: tableType(arrowType(f.type)), nullable: true })) + const input = (name: string): Plan.Node => + inline + ? { _tag: "Values", frame: frameOf(gold.inputs[name]) } + : declare + ? { _tag: "Table", name, columns: columns(name) } + : { _tag: "Table", name } + let node = input("left") + for (const op of gold.ops) { + switch (op.op) { + case "filter": + node = { _tag: "Filter", input: node, predicate: expression(op.predicate) } + break + case "select": + node = { _tag: "Select", input: node, columns: named(op.exprs) } + break + case "withColumns": + node = { _tag: "WithColumns", input: node, columns: named(op.exprs) } + break + case "pick": + node = { _tag: "Pick", input: node, columns: op.columns } + break + case "omit": + node = { _tag: "Omit", input: node, columns: op.columns } + break + case "rename": + node = { _tag: "Rename", input: node, mapping: op.mapping } + break + case "sort": + node = { + _tag: "Sort", + input: node, + keys: op.keys.map((k) => ({ expression: expression(k.expr), descending: k.descending ?? false })), + nullsLast: op.nullsLast ?? false + } + break + case "head": + node = { _tag: "Slice", input: node, offset: 0, length: op.n } + break + case "tail": + node = op.n === 0 + ? { _tag: "Slice", input: node, offset: 0, length: 0 } + : { _tag: "Slice", input: node, offset: -op.n } + break + case "slice": + node = { _tag: "Slice", input: node, offset: op.offset, length: op.length } + break + case "groupBy": + node = { _tag: "GroupBy", input: node, keys: op.keys, aggregates: named(op.aggs) } + break + case "agg": + node = { _tag: "Aggregate", input: node, aggregates: named(op.aggs) } + break + case "join": + node = { _tag: "Join", left: node, right: input(op.right), on: op.on, how: op.how } + break + case "unique": + node = { _tag: "Unique", input: node, columns: op.columns } + break + case "dropNulls": + node = { _tag: "DropNulls", input: node, columns: op.columns } + break + case "concat": + node = { _tag: "Concat", inputs: [node, input(op.with)] } + break + case "pivot": + node = { + _tag: "Pivot", + input: node, + on: op.on, + index: op.index, + values: op.values, + aggregate: + (op.aggregate === "len" ? "len" : aggregateNames[op.aggregate] ?? op.aggregate) as Plan.PivotAggregate, + columns: op.columns + } + break + case "explode": + node = { _tag: "Explode", input: node, column: op.column } + break + case "unnest": + node = { _tag: "Unnest", input: node, column: op.column } + break + case "unpivot": + node = { + _tag: "Unpivot", + input: node, + index: op.index, + on: op.on, + variable: op.variable ?? "variable", + value: op.value ?? "value" + } + break + } + } + return node +} + +/** + * Runs a case: its plan goes through its JSON form (so the codecs are part + * of what is checked) and then runs as a query. + */ +export const run = (gold: Gold.Case, inline: boolean): DataFrame.DataFrame => { + const json = JSON.parse(JSON.stringify(Schema.encodeSync(Plan.NodeFromJson)(plan(gold, inline)))) + const decoded = Schema.decodeUnknownSync(Plan.NodeFromJson)(json) + const tables = Object.fromEntries(Object.entries(gold.inputs).map(([name, t]) => [name, frameOf(t)])) + return Query.fromPlan(decoded).pipe(Effect.provide(Catalog.layer(tables)), Effect.runSync) +} diff --git a/packages/effect/test/fixtures/dataframe/compile_sql.ts b/packages/effect/test/fixtures/dataframe/compile_sql.ts new file mode 100644 index 00000000000..1341f4313d4 --- /dev/null +++ b/packages/effect/test/fixtures/dataframe/compile_sql.ts @@ -0,0 +1,26 @@ +/** + * Compiles every gold plan to SQL for the DuckDB cross-check + * (sql_oracle.py): + * + * node test/fixtures/dataframe/compile_sql.ts [dialect] + * + * Writes .sql/sql-.json (git-ignored): per case, the SQL or why it + * can't compile. + */ +import { Result } from "effect" +import { Sql } from "effect/dataframe" +import * as Fs from "node:fs" +import * as Gold from "../../dataframe/oracle/Gold.ts" +import * as Plan from "../../dataframe/oracle/Plan.ts" + +const dialect = (process.argv[2] ?? "duckdb") as Sql.Dialect +const cases = [...Gold.spec().cases, ...Gold.random().cases, ...Gold.extended().cases] +const out = cases.map((c) => { + const compiled = Sql.compile(Plan.plan(c, false, true), { dialect }) + return Result.isSuccess(compiled) + ? { id: c.id, sql: compiled.success.sql, columns: compiled.success.columns } + : { id: c.id, error: compiled.failure.message } +}) +const file = new URL(`.sql/sql-${dialect}.json`, import.meta.url) +Fs.writeFileSync(file, JSON.stringify(out) + "\n") +console.log(`${out.filter((c) => "sql" in c).length}/${out.length} plans compiled to ${dialect}`) diff --git a/packages/effect/test/fixtures/dataframe/generate_gold.py b/packages/effect/test/fixtures/dataframe/generate_gold.py new file mode 100644 index 00000000000..b1007ad2c0f --- /dev/null +++ b/packages/effect/test/fixtures/dataframe/generate_gold.py @@ -0,0 +1,1059 @@ +""" +Gold corpus for the effect/dataframe oracle. + +Every case is a small input table (or two), a pipeline written in a neutral +JSON plan language, and the result Python Polars produces for that pipeline. +The TypeScript oracle (test/dataframe/oracle) replays each plan through the +public effect/dataframe API and compares the result with the Polars output. + + python generate_gold.py # writes gold/spec.json and gold/random.json + +Needs polars (tested with 2.1). Polars is the semantic reference: where +effect/dataframe deliberately differs, the case is left out here and the +difference is listed in DEVIATIONS below. +""" + +import json +import math +import random +import sys +from pathlib import Path + +import polars as pl + +HERE = Path(__file__).parent + +# Deliberate differences from Polars. Cases that would exercise them are not +# generated. +DEVIATIONS = [ + "JavaScript has no float literal distinct from an integer: an integral number literal behaves like a Polars integer literal (it takes the integer type of the column it meets), so cases never combine an integer column with an integral float literal.", + "Float to string casts use JavaScript number formatting (Polars prints 5.0 as '5.0', JavaScript as '5').", + "Joins never add '_right' suffixes: overlapping non-key columns are a type error.", + "Arithmetic between 64-bit integers (bigint) and other numeric types requires an explicit cast.", +] + +# --------------------------------------------------------------------------- +# Types and value encoding +# --------------------------------------------------------------------------- + +DTYPES = { + "bool": pl.Boolean, + "i8": pl.Int8, + "i16": pl.Int16, + "i32": pl.Int32, + "i64": pl.Int64, + "u8": pl.UInt8, + "u16": pl.UInt16, + "u32": pl.UInt32, + "u64": pl.UInt64, + "f32": pl.Float32, + "f64": pl.Float64, + "utf8": pl.String, + "null": pl.Null, + "date": pl.Date, + "datetime[us]": pl.Datetime("us"), + "datetime[ms]": pl.Datetime("ms"), + "duration[us]": pl.Duration("us"), + "duration[ms]": pl.Duration("ms"), +} + +# Temporal columns travel as their epoch integers (days, or units since the +# epoch): exact, and independent of how either side represents them. +TEMPORAL = {"date": pl.Int32, "datetime[us]": pl.Int64, "datetime[ms]": pl.Int64, "duration[us]": pl.Int64, "duration[ms]": pl.Int64} + + +def split_top(s): + """Splits on commas outside angle brackets.""" + parts, depth, cur = [], 0, "" + for ch in s: + if ch == "<": + depth += 1 + elif ch == ">": + depth -= 1 + if ch == "," and depth == 0: + parts.append(cur) + cur = "" + else: + cur += ch + if cur: + parts.append(cur) + return parts + + +def dtype_of(name): + if name.startswith("decimal["): + p, sc = name[8:-1].split(",") + return pl.Decimal(int(p), int(sc)) + if name.startswith("list<"): + return pl.List(dtype_of(name[5:-1])) + if name.startswith("struct<"): + return pl.Struct({f.split(":", 1)[0]: dtype_of(f.split(":", 1)[1]) for f in split_top(name[7:-1])}) + return DTYPES[name] + + +def type_name(dtype): + if isinstance(dtype, pl.Decimal): + return f"decimal[{dtype.precision},{dtype.scale}]" + if isinstance(dtype, pl.List): + return f"list<{type_name(dtype.inner)}>" + if isinstance(dtype, pl.Struct): + return "struct<" + ",".join(f"{f.name}:{type_name(f.dtype)}" for f in dtype.fields) + ">" + for name, d in DTYPES.items(): + if dtype == d: + return name + raise ValueError(f"unsupported dtype {dtype}") + + +def encode_value(value, t): + if value is None: + return None + if t.startswith("decimal["): + return str(value) + if t.startswith("list<"): + return [encode_value(v, t[5:-1]) for v in value] + if t.startswith("struct<"): + fields = [f.split(":", 1) for f in split_top(t[7:-1])] + return {n: encode_value(value.get(n), ft) for n, ft in fields} + if t in ("f32", "f64"): + if math.isnan(value): + return "NaN" + if math.isinf(value): + return "Infinity" if value > 0 else "-Infinity" + if value == 0 and math.copysign(1.0, value) < 0: + return "-0" + return value + if t in ("i64", "u64"): + return str(value) + return value + + +def decode_value(value, t): + if value is None: + return None + if t.startswith("decimal["): + from decimal import Decimal as PyDecimal + return PyDecimal(value) + if t.startswith("list<"): + return [decode_value(v, t[5:-1]) for v in value] + if t.startswith("struct<"): + fields = [f.split(":", 1) for f in split_top(t[7:-1])] + return {n: decode_value(value.get(n), ft) for n, ft in fields} + if t in ("f32", "f64"): + if isinstance(value, str): + return {"NaN": float("nan"), "Infinity": float("inf"), "-Infinity": float("-inf"), "-0": -0.0}[value] + return float(value) + if t in ("i64", "u64"): + return int(value) + return value + + +def table(fields, columns): + """fields: [(name, type)], columns: {name: [python values]}""" + return { + "fields": [{"name": n, "type": t} for n, t in fields], + "columns": {n: [encode_value(v, t) for v in columns[n]] for n, t in fields}, + } + + +def to_polars(t): + columns = [] + for f in t["fields"]: + values = [decode_value(v, f["type"]) for v in t["columns"][f["name"]]] + if f["type"] in TEMPORAL: + columns.append(pl.Series(f["name"], values, dtype=TEMPORAL[f["type"]]).cast(DTYPES[f["type"]])) + else: + columns.append(pl.Series(f["name"], values, dtype=dtype_of(f["type"]))) + return pl.DataFrame(columns) + + +def from_polars(df): + fields = [(name, type_name(dtype)) for name, dtype in df.schema.items()] + def values(name, t): + series = df.get_column(name) + return (series.cast(TEMPORAL[t]) if t in TEMPORAL else series).to_list() + return table(fields, {name: values(name, t) for name, t in fields}) + + +# --------------------------------------------------------------------------- +# Plan language -> Polars +# --------------------------------------------------------------------------- + + +def lit(value, t=None): + out = {"lit": encode_value(value, t or ("f64" if isinstance(value, float) else "")) if value is not None else None} + if t is not None: + out["type"] = t + return out + + +def col(name): + return {"col": name} + + +def fn(name, *args, **params): + return {"fn": name, "args": list(args), **params} + + +def agg(name, arg=None): + return {"agg": name} if arg is None else {"agg": name, "arg": arg} + + +def p_lit(e): + t = e.get("type") + v = e["lit"] + if t is not None and t.startswith("decimal["): + return pl.lit(decode_value(v, t), dtype=dtype_of(t)) + if t is not None and t in TEMPORAL: + return pl.lit(decode_value(v, t), dtype=TEMPORAL[t]).cast(DTYPES[t]) + if t is not None: + return pl.lit(decode_value(v, t), dtype=DTYPES[t]) + if isinstance(v, str) and v in ("NaN", "Infinity", "-Infinity", "-0"): + return pl.lit(decode_value(v, "f64")) + return pl.lit(v) + + +def p_expr(e): + if "over" in e: + return p_expr(e["over"]).over([p_expr(k) for k in e["partitionBy"]]) + if "col" in e: + return pl.col(e["col"]) + if "lit" in e: + return p_lit(e) + if "agg" in e: + return p_agg(e) + f = e["fn"] + a = [p_expr(x) for x in e["args"]] + binary = { + "add": lambda: a[0] + a[1], + "sub": lambda: a[0] - a[1], + "mul": lambda: a[0] * a[1], + "div": lambda: a[0] / a[1], + "floorDiv": lambda: a[0] // a[1], + "mod": lambda: a[0] % a[1], + "eq": lambda: a[0] == a[1], + "neq": lambda: a[0] != a[1], + "lt": lambda: a[0] < a[1], + "lte": lambda: a[0] <= a[1], + "gt": lambda: a[0] > a[1], + "gte": lambda: a[0] >= a[1], + "and": lambda: a[0] & a[1], + "or": lambda: a[0] | a[1], + } + if f in binary: + return binary[f]() + if f == "neg": + return -a[0] + if f == "abs": + return a[0].abs() + if f == "not": + return ~a[0] + if f == "isNull": + return a[0].is_null() + if f == "isNotNull": + return a[0].is_not_null() + if f == "isNaN": + return a[0].is_nan() + if f == "fillNull": + return a[0].fill_null(a[1]) + if f == "coalesce": + return pl.coalesce(a) + if f == "when": + return pl.when(a[0]).then(a[1]).otherwise(a[2]) + if f == "cast": + return a[0].cast(dtype_of(e["to"]), strict=False) + if f == "isIn": + return a[0].is_in([decode_value(v, e.get("type", "")) for v in e["values"]]) + if f == "between": + return a[0].is_between(a[1], a[2]) + if f == "round": + return a[0].round(e.get("decimals", 0)) + if f == "sqrt": + return a[0].sqrt() + if f == "strLength": + return a[0].str.len_chars() + if f == "strLower": + return a[0].str.to_lowercase() + if f == "strUpper": + return a[0].str.to_uppercase() + if f == "strContains": + return a[0].str.contains(e["pattern"], literal=True) + if f == "strStartsWith": + return a[0].str.starts_with(e["prefix"]) + if f == "strEndsWith": + return a[0].str.ends_with(e["suffix"]) + if f == "strConcat": + return pl.concat_str(a) + window = { + "cumSum": lambda: a[0].cum_sum(), + "cumMin": lambda: a[0].cum_min(), + "cumMax": lambda: a[0].cum_max(), + "cumCount": lambda: a[0].cum_count(), + "shift": lambda: a[0].shift(e.get("n", 1)), + "diff": lambda: a[0].diff(e.get("n", 1)), + "rank": lambda: a[0].rank(method=e.get("method", "average"), descending=e.get("descending", False)), + "rollingSum": lambda: a[0].rolling_sum(e["window"], min_samples=e.get("minPeriods")), + "rollingMean": lambda: a[0].rolling_mean(e["window"], min_samples=e.get("minPeriods")), + "rollingMin": lambda: a[0].rolling_min(e["window"], min_samples=e.get("minPeriods")), + "rollingMax": lambda: a[0].rolling_max(e["window"], min_samples=e.get("minPeriods")), + } + if f in window: + return window[f]() + temporal = { + "year": lambda: a[0].dt.year(), + "month": lambda: a[0].dt.month(), + "day": lambda: a[0].dt.day(), + "hour": lambda: a[0].dt.hour(), + "minute": lambda: a[0].dt.minute(), + "second": lambda: a[0].dt.second(), + "millisecond": lambda: a[0].dt.millisecond(), + "weekday": lambda: a[0].dt.weekday(), + "ordinalDay": lambda: a[0].dt.ordinal_day(), + "truncate": lambda: a[0].dt.truncate(e["every"]), + "totalDays": lambda: a[0].dt.total_days(), + "totalHours": lambda: a[0].dt.total_hours(), + "totalMinutes": lambda: a[0].dt.total_minutes(), + "totalSeconds": lambda: a[0].dt.total_seconds(), + "totalMilliseconds": lambda: a[0].dt.total_milliseconds(), + } + if f in temporal: + return temporal[f]() + nested = { + "listLength": lambda: a[0].list.len(), + "listGet": lambda: a[0].list.get(e["index"], null_on_oob=True), + "listSum": lambda: a[0].list.sum(), + "listMean": lambda: a[0].list.mean(), + "listMin": lambda: a[0].list.min(), + "listMax": lambda: a[0].list.max(), + "listContains": lambda: a[0].list.contains(a[1]), + "field": lambda: a[0].struct.field(e["field"]), + "struct": lambda: pl.struct([x.alias(n) for x, n in zip(a, e["names"])]), + } + if f in nested: + return nested[f]() + raise ValueError(f"unknown function {f}") + + +def p_agg(e, grouped=False): + name = e["agg"] + if name == "len": + return pl.len() + x = p_expr(e["arg"]) + if name == "list": + return x if grouped else x.implode() + return { + "sum": x.sum, + "mean": x.mean, + "min": x.min, + "max": x.max, + "count": x.count, + "first": x.first, + "last": x.last, + "nUnique": x.n_unique, + "std": x.std, + "var": x.var, + "median": x.median, + }[name]() + + +def run(inputs, ops): + frames = {name: to_polars(t) for name, t in inputs.items()} + df = frames["left"] + for op in ops: + kind = op["op"] + if kind == "filter": + df = df.filter(p_expr(op["predicate"])) + elif kind == "select": + df = df.select([p_expr(e).alias(n) for n, e in op["exprs"].items()]) + elif kind == "withColumns": + df = df.with_columns([p_expr(e).alias(n) for n, e in op["exprs"].items()]) + elif kind == "pick": + df = df.select(op["columns"]) + elif kind == "omit": + df = df.drop(op["columns"]) + elif kind == "rename": + df = df.rename(op["mapping"]) + elif kind == "sort": + df = df.sort( + [p_expr(k["expr"]) for k in op["keys"]], + descending=[k.get("descending", False) for k in op["keys"]], + nulls_last=op.get("nullsLast", False), + maintain_order=True, + ) + elif kind == "head": + df = df.head(op["n"]) + elif kind == "tail": + df = df.tail(op["n"]) + elif kind == "slice": + df = df.slice(op["offset"], op.get("length")) + elif kind == "groupBy": + df = df.group_by(op["keys"], maintain_order=True).agg( + [p_agg(a, grouped=True).alias(n) for n, a in op["aggs"].items()] + ) + elif kind == "agg": + df = df.select([p_agg(a).alias(n) for n, a in op["aggs"].items()]) + elif kind == "join": + right = frames[op["right"]] + how = op["how"] + kw = {} + if how in ("inner", "left", "full"): + kw["maintain_order"] = "left_right" + if how == "full": + kw["coalesce"] = True + df = df.join(right, on=op["on"], how=how, **kw) + elif kind == "unique": + df = df.unique(subset=op.get("columns"), keep="first", maintain_order=True) + elif kind == "concat": + df = pl.concat([df, frames[op["with"]]], how="vertical") + elif kind == "dropNulls": + df = df.drop_nulls(subset=op.get("columns")) + elif kind == "pivot": + df = df.pivot(on=op["on"], index=op["index"], values=op["values"], aggregate_function=op["aggregate"], sort_columns=False) + if "columns" in op: + # Declared columns that never occur are cells without rows: + # the aggregate of an empty set (0 for sums and counts). + empty = run({"left": inputs["left"]}, [o for o in ops[: ops.index(op)]]).clear().select( + p_agg(agg(op["aggregate"], col(op["values"])) if op["aggregate"] != "len" else agg("len")).alias("v") + ) + for c in op["columns"]: + if c not in df.columns: + df = df.with_columns(pl.lit(empty.item(0, 0), dtype=empty.schema["v"]).alias(c)) + df = df.select(op["index"] + op["columns"]) + elif kind == "explode": + df = df.explode(op["column"]) + elif kind == "unnest": + df = df.unnest(op["column"]) + elif kind == "unpivot": + df = df.unpivot(on=op["on"], index=op["index"], variable_name=op.get("variable", "variable"), value_name=op.get("value", "value")) + else: + raise ValueError(f"unknown op {kind}") + return df + + +def case(id, description, inputs, ops): + expected = run(inputs, ops) + return {"id": id, "description": description, "inputs": inputs, "ops": ops, "expected": from_polars(expected)} + + +# --------------------------------------------------------------------------- +# Hand-written specification cases +# --------------------------------------------------------------------------- + +NAN = float("nan") +INF = float("inf") + +readings = table( + [("station", "utf8"), ("city", "utf8"), ("temp", "f64"), ("rain", "f64"), ("count", "i32"), ("id", "i64")], + { + "station": ["s1", "s2", "s3", "s4", "s5", "s6", "s7"], + "city": ["Vienna", "Vienna", "Berlin", "Berlin", "Zürich", None, "Vienna"], + "temp": [21.5, 19.0, 24.0, 26.5, 17.0, NAN, None], + "rain": [0.0, None, 3.5, 1.0, 7.0, 2.0, 0.5], + "count": [3, 1, None, 7, 2, 5, 4], + "id": [10, 20, 30, 40, 9007199254740993, -1, 0], + }, +) +stations = table( + [("station", "utf8"), ("elevation", "f64"), ("active", "bool")], + {"station": ["s1", "s3", "s3", "s9", None], "elevation": [190.0, 34.0, 35.0, 500.0, 1.0], "active": [True, False, None, True, True]}, +) +mixed = table( + [("a", "f64"), ("b", "f64"), ("i", "i32"), ("j", "i32"), ("s", "utf8"), ("t", "utf8"), ("p", "bool"), ("q", "bool")], + { + "a": [1.0, -0.0, 0.0, NAN, INF, -INF, None, 2.5, -2.5, 1e300], + "b": [2.0, 0.0, -0.0, NAN, 1.0, None, 3.0, 0.0, 2.0, 1e300], + "i": [7, -7, 0, 2147483647, -2147483648, None, 3, -3, 100, 5], + "j": [2, 2, 3, 1, -1, 4, None, 2, -7, 0], + "s": ["b", "a", "", "é", "\U0001F600", "￿", None, "B", "ab", "a\u0000b"], + "t": ["b", "A", "x", "É", "😀", "z", "n", None, "AB", "a"], + "p": [True, True, True, False, False, False, None, None, None, True], + "q": [True, False, None, True, False, None, True, False, None, True], + }, +) +ints = table( + [("i8", "i8"), ("u8", "u8"), ("i16", "i16"), ("u16", "u16"), ("u32", "u32"), ("i64", "i64"), ("u64", "u64"), ("f32", "f32")], + { + "i8": [127, -128, 0, None, 5], + "u8": [255, 0, 1, None, 5], + "i16": [32767, -32768, 0, 12, None], + "u16": [65535, 0, 7, 12, None], + "u32": [4294967295, 0, 1, None, 3], + "i64": [9223372036854775807, -9223372036854775808, 0, None, 42], + "u64": [18446744073709551615, 0, 1, None, 42], + "f32": [1.5, -0.25, None, 3.0, 1e10], + }, +) +empty = table([("k", "utf8"), ("v", "f64")], {"k": [], "v": []}) +allnull = table([("k", "utf8"), ("v", "f64"), ("i", "i32")], {"k": ["x", "x", "y"], "v": [None, None, 1.0], "i": [None, None, None]}) + + +def spec_cases(): + R = {"left": readings} + RS = {"left": readings, "right": stations} + M = {"left": mixed} + out = [] + add = lambda *a: out.append(case(*a)) + + # filter + add("filter/gt", "numeric comparison with a literal", R, [{"op": "filter", "predicate": fn("gt", col("temp"), lit(20.0))}]) + add("filter/null-is-false", "rows where the predicate is null are dropped", R, [{"op": "filter", "predicate": fn("gt", col("rain"), lit(0.5))}]) + add("filter/nan-total-order", "NaN compares greater than every number and equal to itself", M, + [{"op": "filter", "predicate": fn("gte", col("a"), col("b"))}]) + add("filter/string-eq", "string equality", R, [{"op": "filter", "predicate": fn("eq", col("city"), lit("Vienna"))}]) + add("filter/string-order", "strings compare by code point (UTF-8 byte order)", M, + [{"op": "filter", "predicate": fn("lt", col("s"), col("t"))}]) + add("filter/kleene", "and/or use three-valued logic", M, + [{"op": "select", "exprs": {"and": fn("and", col("p"), col("q")), "or": fn("or", col("p"), col("q")), "not": fn("not", col("p"))}}]) + add("filter/is-null", "isNull and isNotNull", R, + [{"op": "filter", "predicate": fn("or", fn("isNull", col("city")), fn("isNull", col("temp")))}]) + add("filter/is-in", "membership in a literal list", R, + [{"op": "filter", "predicate": fn("isIn", col("city"), values=["Berlin", "Zürich"])}]) + add("filter/between", "inclusive range", R, + [{"op": "filter", "predicate": fn("between", col("temp"), lit(19.0), lit(24.0))}]) + add("filter/compound", "compound predicate", R, + [{"op": "filter", "predicate": fn("and", fn("gt", col("count"), lit(1)), fn("neq", col("city"), lit("Berlin")))}]) + add("filter/i64", "64-bit integer comparison beyond 2^53", R, + [{"op": "filter", "predicate": fn("gt", col("id"), lit(9007199254740992, "i64"))}]) + add("filter/empty-result", "a predicate that matches nothing", R, + [{"op": "filter", "predicate": fn("gt", col("temp"), lit(1000.0))}]) + add("filter/all-null-literal", "a null literal predicate drops everything", R, + [{"op": "filter", "predicate": lit(None, "bool")}]) + + # arithmetic + add("arith/float", "float arithmetic with nulls, NaN and infinities", M, + [{"op": "select", "exprs": {"add": fn("add", col("a"), col("b")), "sub": fn("sub", col("a"), col("b")), + "mul": fn("mul", col("a"), col("b")), "div": fn("div", col("a"), col("b"))}}]) + add("arith/int32-wrap", "Int32 arithmetic wraps on overflow", M, + [{"op": "select", "exprs": {"add": fn("add", col("i"), col("j")), "sub": fn("sub", col("i"), col("j")), + "mul": fn("mul", col("i"), col("j")), "neg": fn("neg", col("i"))}}]) + add("arith/int32-div", "integer true division gives Float64; division by zero gives infinity", M, + [{"op": "select", "exprs": {"div": fn("div", col("i"), col("j"))}}]) + add("arith/int32-floor", "floor division and floored modulo", M, + [{"op": "filter", "predicate": fn("neq", col("j"), lit(0))}, + {"op": "select", "exprs": {"floor": fn("floorDiv", col("i"), col("j")), "mod": fn("mod", col("i"), col("j"))}}]) + add("arith/float-floor", "floor division and floored modulo on floats", M, + [{"op": "select", "exprs": {"floor": fn("floorDiv", col("a"), col("b")), "mod": fn("mod", col("a"), col("b"))}}]) + add("arith/mixed", "Int32 with a float literal widens to Float64", M, + [{"op": "select", "exprs": {"x": fn("mul", col("i"), lit(1.5)), "y": fn("add", col("i"), col("a"))}}]) + add("arith/int-literal", "an integer literal adopts the integer column type", M, + [{"op": "select", "exprs": {"x": fn("add", col("i"), lit(1)), "y": fn("mul", col("j"), lit(3))}}]) + add("arith/abs-sqrt", "abs and sqrt", M, + [{"op": "select", "exprs": {"abs": fn("abs", col("a")), "iabs": fn("abs", col("j")), "sqrt": fn("sqrt", col("b"))}}]) + add("arith/i64", "Int64 arithmetic", R, + [{"op": "select", "exprs": {"x": fn("add", col("id"), lit(1, "i64")), "y": fn("mul", col("id"), lit(2, "i64")), + "z": fn("sub", col("id"), col("id"))}}]) + add("arith/round", "round half to even", {"left": table([("x", "f64")], {"x": [0.5, 1.5, 2.5, -0.5, -1.5, 1.25, 2.675, None, NAN]})}, + [{"op": "select", "exprs": {"r0": fn("round", col("x")), "r1": fn("round", col("x"), decimals=1)}}]) + add("arith/small-ints", "8/16-bit arithmetic wraps in the column type", {"left": ints}, + [{"op": "select", "exprs": {"i8": fn("add", col("i8"), lit(1)), "u8": fn("add", col("u8"), lit(1)), + "i16": fn("mul", col("i16"), lit(2)), "u16": fn("sub", col("u16"), lit(1)), + "u32": fn("add", col("u32"), lit(1)), "f32": fn("mul", col("f32"), lit(2.0))}}]) + + # null handling + add("null/fill", "fillNull with a literal", R, + [{"op": "select", "exprs": {"city": fn("fillNull", col("city"), lit("?")), "temp": fn("fillNull", col("temp"), lit(0.0))}}]) + add("null/coalesce", "first non-null", M, + [{"op": "select", "exprs": {"x": fn("coalesce", col("a"), col("b"), lit(-1.0))}}]) + add("null/when", "when/then/otherwise", R, + [{"op": "select", "exprs": {"label": fn("when", fn("gt", col("temp"), lit(20.0)), lit("warm"), lit("cold"))}}]) + add("null/drop", "drop rows with nulls", R, [{"op": "dropNulls"}]) + add("null/drop-subset", "drop rows with nulls in some columns", R, [{"op": "dropNulls", "columns": ["city"]}]) + add("null/is-nan", "isNaN is distinct from isNull", M, [{"op": "select", "exprs": {"nan": fn("isNaN", col("a"))}}]) + + # casts + add("cast/float-int", "float to Int32 truncates; out of range is null", M, + [{"op": "select", "exprs": {"i": fn("cast", col("a"), to="i32"), "f": fn("cast", col("i"), to="f64")}}]) + add("cast/int-string", "integers to strings", M, [{"op": "select", "exprs": {"s": fn("cast", col("i"), to="utf8")}}]) + add("cast/string-number", "strings to numbers; unparsable is null", + {"left": table([("s", "utf8")], {"s": ["1", "-2", "3.5", "x", "", None, " 4", "1e3"]})}, + [{"op": "select", "exprs": {"f": fn("cast", col("s"), to="f64"), "i": fn("cast", col("s"), to="i32")}}]) + add("cast/bool", "booleans to and from numbers", M, + [{"op": "select", "exprs": {"b": fn("cast", col("j"), to="bool"), "n": fn("cast", col("p"), to="i32")}}]) + add("cast/i64", "Int64 to and from other types", R, + [{"op": "select", "exprs": {"f": fn("cast", col("id"), to="f64"), "i": fn("cast", col("count"), to="i64"), + "s": fn("cast", col("id"), to="utf8")}}]) + + # strings + add("str/length", "length in characters", M, [{"op": "select", "exprs": {"n": fn("strLength", col("s"))}}]) + add("str/case", "upper and lower case", M, + [{"op": "select", "exprs": {"u": fn("strUpper", col("s")), "l": fn("strLower", col("t"))}}]) + add("str/match", "contains, startsWith, endsWith with literals", M, + [{"op": "select", "exprs": {"c": fn("strContains", col("s"), pattern="a"), "p": fn("strStartsWith", col("t"), prefix="A"), + "e": fn("strEndsWith", col("s"), suffix="b")}}]) + add("str/concat", "concatenation; null in any argument gives null", M, + [{"op": "select", "exprs": {"c": fn("strConcat", col("s"), lit("-"), col("t"))}}]) + + # projection + add("project/pick", "pick columns in the given order", R, [{"op": "pick", "columns": ["temp", "station"]}]) + add("project/omit", "omit columns", R, [{"op": "omit", "columns": ["rain", "id"]}]) + add("project/rename", "rename columns", R, [{"op": "rename", "mapping": {"temp": "celsius", "city": "town"}}]) + add("project/with-replace", "withColumns replaces an existing column in place", R, + [{"op": "withColumns", "exprs": {"temp": fn("mul", col("temp"), lit(2.0)), "f": fn("add", fn("mul", col("temp"), lit(1.8)), lit(32.0))}}]) + add("project/literal-column", "a literal broadcasts to every row", R, + [{"op": "select", "exprs": {"station": col("station"), "half": lit(0.5), "one": lit(1), "tag": lit("x")}}]) + + # sorting and slicing + add("sort/asc", "ascending, nulls first, NaN last among numbers", R, [{"op": "sort", "keys": [{"expr": col("temp")}]}]) + add("sort/desc-nulls-last", "descending with nulls last", R, + [{"op": "sort", "keys": [{"expr": col("temp"), "descending": True}], "nullsLast": True}]) + add("sort/multi", "multi-key sort is stable", R, + [{"op": "sort", "keys": [{"expr": col("city")}, {"expr": col("count"), "descending": True}]}]) + add("sort/strings", "string sort by code point", M, [{"op": "sort", "keys": [{"expr": col("s")}]}]) + add("sort/zeros", "-0 and 0 sort as equal (stable)", M, [{"op": "sort", "keys": [{"expr": col("a")}]}]) + add("sort/i64", "Int64 sort", R, [{"op": "sort", "keys": [{"expr": col("id")}]}]) + add("sort/expr", "sort by an expression", R, [{"op": "sort", "keys": [{"expr": fn("neg", col("count"))}]}]) + add("sort/bool", "boolean sort", M, [{"op": "sort", "keys": [{"expr": col("p")}, {"expr": col("q"), "descending": True}]}]) + add("slice/head", "first rows", R, [{"op": "head", "n": 3}]) + add("slice/tail", "last rows", R, [{"op": "tail", "n": 2}]) + add("slice/offset", "slice in the middle", R, [{"op": "slice", "offset": 2, "length": 3}]) + add("slice/after-filter", "slicing after a filter", R, + [{"op": "filter", "predicate": fn("isNotNull", col("city"))}, {"op": "slice", "offset": 1, "length": 2}]) + + # aggregation + add("agg/global", "aggregates over the whole frame", R, + [{"op": "agg", "aggs": {"n": agg("len"), "c": agg("count", col("temp")), "sum": agg("sum", col("rain")), + "mean": agg("mean", col("rain")), "min": agg("min", col("count")), "max": agg("max", col("count")), + "first": agg("first", col("city")), "last": agg("last", col("city"))}}]) + add("agg/nan", "sum includes NaN; min and max skip it", R, + [{"op": "agg", "aggs": {"sum": agg("sum", col("temp")), "min": agg("min", col("temp")), "max": agg("max", col("temp"))}}]) + add("agg/stats", "std, var and median", R, + [{"op": "agg", "aggs": {"std": agg("std", col("rain")), "var": agg("var", col("rain")), "median": agg("median", col("rain")), + "imed": agg("median", col("count"))}}]) + add("agg/strings", "min, max and nUnique on strings", R, + [{"op": "agg", "aggs": {"min": agg("min", col("city")), "max": agg("max", col("city")), "n": agg("nUnique", col("city"))}}]) + add("agg/empty", "aggregates over an empty frame", {"left": empty}, + [{"op": "agg", "aggs": {"n": agg("len"), "sum": agg("sum", col("v")), "mean": agg("mean", col("v")), "min": agg("min", col("v"))}}]) + add("group/basic", "group by a string key in first-appearance order", R, + [{"op": "groupBy", "keys": ["city"], "aggs": {"n": agg("len"), "temp": agg("mean", col("temp")), "rain": agg("sum", col("rain"))}}]) + add("group/null-key", "null is its own group", R, + [{"op": "groupBy", "keys": ["city"], "aggs": {"stations": agg("count", col("station"))}}]) + add("group/multi-key", "group by two keys", R, + [{"op": "withColumns", "exprs": {"wet": fn("gt", col("rain"), lit(1.0))}}, + {"op": "groupBy", "keys": ["city", "wet"], "aggs": {"n": agg("len"), "max": agg("max", col("temp"))}}]) + add("group/all-null", "aggregates over all-null groups", {"left": allnull}, + [{"op": "groupBy", "keys": ["k"], "aggs": {"sum": agg("sum", col("v")), "mean": agg("mean", col("v")), "count": agg("count", col("v")), + "n": agg("len"), "min": agg("min", col("v")), "std": agg("std", col("v")), + "nu": agg("nUnique", col("v")), "first": agg("first", col("v")), "isum": agg("sum", col("i")), + "imax": agg("max", col("i"))}}]) + add("group/float-key", "NaN, -0 and 0 as group keys", M, + [{"op": "groupBy", "keys": ["a"], "aggs": {"n": agg("len")}}]) + add("group/i64-key", "Int64 group keys and sums", R, + [{"op": "withColumns", "exprs": {"k": fn("mod", col("id"), lit(3, "i64"))}}, + {"op": "groupBy", "keys": ["k"], "aggs": {"n": agg("len"), "s": agg("sum", col("id")), "mx": agg("max", col("id"))}}]) + add("group/bool-key", "boolean group keys", M, [{"op": "groupBy", "keys": ["p"], "aggs": {"n": agg("len"), "s": agg("sum", col("i"))}}]) + add("group/int-sum", "Int32 sums stay Int32; means are Float64", M, + [{"op": "groupBy", "keys": ["p"], "aggs": {"s": agg("sum", col("j")), "m": agg("mean", col("j")), "mn": agg("min", col("j"))}}]) + add("group/after-filter", "group a filtered frame", R, + [{"op": "filter", "predicate": fn("gt", col("rain"), lit(0.0))}, + {"op": "groupBy", "keys": ["city"], "aggs": {"n": agg("len"), "last": agg("last", col("station"))}}]) + add("group/expression-agg", "aggregate of an expression", R, + [{"op": "groupBy", "keys": ["city"], "aggs": {"x": agg("sum", fn("mul", col("rain"), lit(2.0)))}}]) + add("group/empty", "grouping an empty frame", {"left": empty}, [{"op": "groupBy", "keys": ["k"], "aggs": {"n": agg("len")}}]) + add("group/small-ints", "sums of small integer types", {"left": ints}, + [{"op": "agg", "aggs": {"i8": agg("sum", col("i8")), "u8": agg("sum", col("u8")), "u32": agg("sum", col("u32")), + "i64": agg("sum", col("i64")), "u64": agg("max", col("u64")), "f32": agg("mean", col("f32"))}}]) + + # joins + for how in ("inner", "left", "full", "semi", "anti"): + add(f"join/{how}", f"{how} join with duplicate and null keys", RS, + [{"op": "pick", "columns": ["station", "temp"]}, {"op": "join", "right": "right", "on": ["station"], "how": how}]) + add("join/multi-key", "join on two keys", + {"left": table([("a", "i32"), ("b", "utf8"), ("x", "f64")], {"a": [1, 1, 2, 2, None], "b": ["x", "y", "x", None, "x"], "x": [1.0, 2.0, 3.0, 4.0, 5.0]}), + "right": table([("a", "i32"), ("b", "utf8"), ("y", "bool")], {"a": [1, 2, 2, 1], "b": ["y", "x", "x", "z"], "y": [True, False, True, None]})}, + [{"op": "join", "right": "right", "on": ["a", "b"], "how": "left"}]) + add("join/i64-key", "join on Int64 keys", + {"left": table([("k", "i64"), ("x", "f64")], {"k": [9007199254740993, 9007199254740992, 1], "x": [1.0, 2.0, 3.0]}), + "right": table([("k", "i64"), ("y", "utf8")], {"k": [9007199254740993, 1, 1], "y": ["a", "b", "c"]})}, + [{"op": "join", "right": "right", "on": ["k"], "how": "inner"}]) + add("join/f64-key", "join on Float64 keys: NaN matches NaN, -0 matches 0", + {"left": table([("k", "f64"), ("x", "i32")], {"k": [NAN, -0.0, 1.5, None], "x": [1, 2, 3, 4]}), + "right": table([("k", "f64"), ("y", "i32")], {"k": [0.0, NAN, None, 1.5], "y": [10, 20, 30, 40]})}, + [{"op": "join", "right": "right", "on": ["k"], "how": "inner"}]) + + # unique / concat + add("unique/all", "distinct rows, keeping the first", {"left": table([("a", "i32"), ("b", "utf8")], {"a": [1, 1, 2, None, None, 1], "b": ["x", "x", "y", None, None, "y"]})}, + [{"op": "unique"}]) + add("unique/subset", "distinct by a subset", R, [{"op": "unique", "columns": ["city"]}]) + add("concat/vertical", "stack frames with the same schema", {"left": readings, "right": readings}, + [{"op": "concat", "with": "right"}, {"op": "pick", "columns": ["station", "temp"]}]) + + # pipelines + add("pipeline/summary", "filter, derive, group, sort", R, + [{"op": "filter", "predicate": fn("gt", col("temp"), lit(18.0))}, + {"op": "withColumns", "exprs": {"tempF": fn("add", fn("mul", col("temp"), lit(1.8)), lit(32.0))}}, + {"op": "groupBy", "keys": ["city"], "aggs": {"stations": agg("len"), "avgF": agg("mean", col("tempF")), "maxRain": agg("max", col("rain"))}}, + {"op": "sort", "keys": [{"expr": col("avgF"), "descending": True}]}]) + add("pipeline/join-group", "join then aggregate the joined columns", RS, + [{"op": "join", "right": "right", "on": ["station"], "how": "left"}, + {"op": "groupBy", "keys": ["active"], "aggs": {"n": agg("len"), "elev": agg("mean", col("elevation"))}}, + {"op": "sort", "keys": [{"expr": col("active")}]}]) + return out + + +# --------------------------------------------------------------------------- +# Random differential cases +# --------------------------------------------------------------------------- + +NUMERIC = ("i32", "f64") +STRINGS = ["a", "b", "c", "aa", "ab", "B", "é", "😀", "", "zz"] + + +def random_value(rng, t, null_rate): + if rng.random() < null_rate: + return None + if t == "bool": + return rng.random() < 0.5 + if t == "i32": + return rng.choice([rng.randint(-5, 5), rng.randint(-1000, 1000), 2147483647, -2147483648]) if rng.random() < 0.1 else rng.randint(-5, 5) + if t == "i64": + return rng.choice([rng.randint(-5, 5), 2**53 + rng.randint(0, 3), -(2**62)]) + if t == "f64": + r = rng.random() + if r < 0.03: + return float("nan") + if r < 0.05: + return rng.choice([float("inf"), float("-inf"), -0.0]) + return rng.choice([float(rng.randint(-4, 4)), round(rng.uniform(-100, 100), 3)]) + if t == "utf8": + return rng.choice(STRINGS) + raise ValueError(t) + + +def random_table(rng, prefix, n_rows, types): + fields = [(f"{prefix}{i}_{t}", t) for i, t in enumerate(types)] + null_rate = rng.choice([0.0, 0.1, 0.3]) + return table(fields, {name: [random_value(rng, t, null_rate) for _ in range(n_rows)] for name, t in fields}) + + +def random_expr(rng, cols, want, depth=0): + """cols: {name: type}; want: 'num' | 'bool' | 'utf8' | 'i64'""" + by = lambda ts: [c for c, t in cols.items() if t in ts] + if want == "num": + choices = by(NUMERIC) + if depth >= 2 or rng.random() < 0.4: + if choices and rng.random() < 0.8: + return col(rng.choice(choices)) + return lit(rng.choice([2, -1, 0.5, 2.5])) + op = rng.choice(["add", "sub", "mul", "div", "neg", "abs", "fillNull", "when", "mod", "floorDiv"]) + if op in ("neg", "abs"): + return fn(op, random_expr(rng, cols, "num", depth + 1)) + if op == "fillNull": + return fn(op, random_expr(rng, cols, "num", depth + 1), lit(rng.choice([0, 1.5]))) + if op == "when": + return fn("when", random_expr(rng, cols, "bool", depth + 1), random_expr(rng, cols, "num", depth + 1), random_expr(rng, cols, "num", depth + 1)) + return fn(op, random_expr(rng, cols, "num", depth + 1), random_expr(rng, cols, "num", depth + 1)) + if want == "bool": + r = rng.random() + if r < 0.45 and by(NUMERIC): + return fn(rng.choice(["gt", "lt", "gte", "lte", "eq", "neq"]), random_expr(rng, cols, "num", depth + 1), random_expr(rng, cols, "num", depth + 1)) + if r < 0.6 and by(("utf8",)): + c = col(rng.choice(by(("utf8",)))) + return rng.choice([ + fn(rng.choice(["eq", "lt", "gte"]), c, lit(rng.choice(STRINGS))), + fn("isIn", c, values=rng.sample(STRINGS, 3)), + fn("strStartsWith", c, prefix=rng.choice(["a", "é", ""])), + ]) + if r < 0.7 and by(("i64",)): + return fn(rng.choice(["gt", "eq"]), col(rng.choice(by(("i64",)))), lit(rng.randint(-3, 3), "i64")) + if r < 0.8: + return fn(rng.choice(["isNull", "isNotNull"]), col(rng.choice(list(cols)))) + if r < 0.9 and by(("bool",)) and depth < 2: + return col(rng.choice(by(("bool",)))) + if depth < 2: + return fn(rng.choice(["and", "or"]), random_expr(rng, cols, "bool", depth + 1), random_expr(rng, cols, "bool", depth + 1)) + return fn("isNull", col(rng.choice(list(cols)))) + raise ValueError(want) + + +AGGS_NUM = ["sum", "mean", "min", "max", "count", "first", "last", "nUnique", "std", "median"] +AGGS_ANY = ["count", "first", "last", "nUnique", "min", "max"] + + +def random_aggs(rng, cols, exclude, tag): + aggs = {f"n{tag}": agg("len")} + for i in range(rng.randint(1, 3)): + candidates = [c for c in cols if c not in exclude] + if not candidates: + break + c = rng.choice(candidates) + t = cols[c] + name = rng.choice(AGGS_NUM if t in NUMERIC else AGGS_ANY) + if t == "bool" and name in ("min", "max"): + name = "count" + aggs[f"a{tag}_{i}"] = agg(name, col(c)) + return aggs + + +def schema_after(inputs, ops): + df = run(inputs, ops) + return {name: type_name(dtype) for name, dtype in df.schema.items()} + + +def random_case(rng, index): + try: + return random_case_unsafe(rng, index) + except Exception as error: # Polars rejected the plan: not a valid case + print(f"skipping random/{index:04d}: {type(error).__name__}: {str(error)[:120]}", file=sys.stderr) + return None + + +def random_case_unsafe(rng, index): + n_rows = rng.choice([0, 1, 5, 20, 60]) + types = [rng.choice(["i32", "f64", "utf8", "bool", "i64", "f64", "utf8"]) for _ in range(rng.randint(2, 5))] + left = random_table(rng, "l", n_rows, ["utf8", "i32"] + types) + inputs = {"left": left} + ops = [] + for _ in range(rng.randint(1, 4)): + cols = schema_after(inputs, ops) + kind = rng.choice(["filter", "withColumns", "sort", "groupBy", "join", "unique", "head", "filter", "withColumns"]) + if kind == "filter": + ops.append({"op": "filter", "predicate": random_expr(rng, cols, "bool")}) + elif kind == "withColumns": + ops.append({"op": "withColumns", "exprs": {f"x{len(ops)}": random_expr(rng, cols, "num") if rng.random() < 0.7 else random_expr(rng, cols, "bool")}}) + elif kind == "sort": + keys = rng.sample(list(cols), min(len(cols), rng.randint(1, 2))) + ops.append({"op": "sort", "keys": [{"expr": col(k), "descending": rng.random() < 0.5} for k in keys], "nullsLast": rng.random() < 0.5}) + elif kind == "groupBy": + keys = rng.sample([c for c, t in cols.items() if t != "f64"] or list(cols), 1) + ops.append({"op": "groupBy", "keys": keys, "aggs": random_aggs(rng, cols, keys, len(ops))}) + elif kind == "join" and "right" not in inputs: + key_candidates = [c for c, t in cols.items() if t in ("utf8", "i32", "i64")] + if not key_candidates: + continue + key = rng.choice(key_candidates) + right = random_table(rng, "r", rng.choice([0, 3, 10, 30]), [rng.choice(["f64", "utf8", "bool"])]) + # reuse the key column's values so that rows match + values = [v for v in inputs["left"]["columns"].get(key, [])] if key in inputs["left"]["columns"] else [] + pool = values or [encode_value(random_value(rng, cols[key], 0.2), cols[key]) for _ in range(5)] + right["fields"].insert(0, {"name": key, "type": cols[key]}) + n = len(next(iter(right["columns"].values()))) + right["columns"][key] = [rng.choice(pool) if pool else None for _ in range(n)] + inputs["right"] = right + ops.append({"op": "join", "right": "right", "on": [key], "how": rng.choice(["inner", "left", "full", "semi", "anti"])}) + elif kind == "unique": + ops.append({"op": "unique", "columns": rng.sample(list(cols), 1)} if rng.random() < 0.5 else {"op": "unique"}) + elif kind == "head": + ops.append({"op": "head", "n": rng.randint(0, 10)}) + return case(f"random/{index:04d}", "seeded random pipeline", inputs, ops) + + +# --------------------------------------------------------------------------- +# Extended cases: window functions, pivots, more types +# --------------------------------------------------------------------------- + +series = table( + [("g", "utf8"), ("t", "i32"), ("x", "f64"), ("i", "i32"), ("b", "bool")], + { + "g": ["a", "b", "a", "a", "b", None, "a", "b", "b", "a"], + "t": [1, 1, 2, 3, 2, 1, 4, 3, 4, 5], + "x": [1.5, 2.0, None, 4.0, NAN, 3.0, -1.0, 2.0, 0.5, 2.0], + "i": [3, 1, 4, 1, 5, 9, None, 6, 5, 3], + "b": [True, False, None, True, True, False, True, None, False, True], + }, +) + +from datetime import datetime, date, timedelta + +EPOCH = datetime(1970, 1, 1) + + +def _us(dt): + return int((dt - EPOCH).total_seconds() * 1_000_000) if dt is not None else None + + +def _days(d): + return (d - date(1970, 1, 1)).days if d is not None else None + + +times = table( + [("k", "utf8"), ("ts", "datetime[us]"), ("ms", "datetime[ms]"), ("d", "date"), ("dur", "duration[us]")], + { + "k": ["a", "b", "c", "d", "e", "f"], + "ts": [_us(datetime(2024, 2, 29, 13, 45, 7, 123456)), _us(datetime(2023, 12, 31, 23, 59, 59, 999000)), _us(datetime(2024, 3, 4, 0, 0, 0)), + None, _us(datetime(1969, 7, 20, 20, 17, 40)), _us(datetime(2024, 3, 10, 6, 30, 0, 500))], + "ms": [_us(datetime(2024, 2, 29, 13, 45, 7, 123000)) // 1000, _us(datetime(2024, 1, 1)) // 1000, None, + _us(datetime(2024, 3, 1)) // 1000, _us(datetime(1970, 1, 1)) // 1000, _us(datetime(2024, 3, 9)) // 1000], + "d": [_days(date(2024, 2, 29)), _days(date(2023, 12, 31)), _days(date(2024, 3, 4)), _days(date(2024, 2, 29)), None, _days(date(2024, 3, 10))], + "dur": [90 * 60 * 1_000_000, -1_500_000, None, 3 * 86_400_000_000 + 1, 0, 25 * 3_600_000_000], + }, +) +holidays = table( + [("d", "date"), ("name", "utf8")], + {"d": [_days(date(2024, 2, 29)), _days(date(2024, 3, 10)), _days(date(2025, 1, 1))], "name": ["leap", "sunday", "new year"]}, +) + +nested = table( + [("k", "utf8"), ("g", "i32"), ("xs", "list"), ("ws", "list"), ("s", "struct")], + { + "k": ["a", "b", "c", "d", "e"], + "g": [1, 2, 1, 2, 1], + "xs": [[1, 2, 3], [], None, [4, None, 2], [7]], + "ws": [["a", "b"], ["c"], [], None, ["z", "a"]], + "s": [{"a": 1, "b": "x"}, {"a": None, "b": "y"}, None, {"a": 4, "b": None}, {"a": 5, "b": "z"}], + }, +) +nested_right = table( + [("k", "utf8"), ("tags", "list")], + {"k": ["a", "d", "x"], "tags": [["t1"], ["t2", "t3"], []]}, +) + +money = table( + [("k", "utf8"), ("price", "decimal[10,2]"), ("fee", "decimal[6,3]"), ("qty", "decimal[4,0]"), ("x", "f64")], + { + "k": ["a", "b", "a", "c", "b", "a"], + "price": ["12.34", "0.10", "-5.00", None, "99999999.99", "10.01"], + "fee": ["0.125", "1.000", None, "2.500", "-0.001", "10.010"], + "qty": ["3", "10", "2", "1", None, "7"], + "x": [1.005, -2.5, 123.456, None, 1e6, 0.0], + }, +) + +sales = table( + [("region", "utf8"), ("quarter", "utf8"), ("units", "i32"), ("price", "f64")], + { + "region": ["north", "north", "south", "south", "north", "east", "south", "north"], + "quarter": ["q1", "q2", "q1", "q3", "q1", "q2", "q1", None], + "units": [10, 20, 5, None, 7, 3, 8, 1], + "price": [1.5, 2.0, 1.0, 3.0, None, 2.5, 1.0, 9.0], + }, +) + + +def extended_cases(): + S = {"left": series} + P = {"left": sales} + out = [] + add = lambda *a: out.append(case(*a)) + over = lambda e, *keys: {"over": e, "partitionBy": [col(k) for k in keys]} + + # window functions over the whole frame + add("window/cum", "cumulative sum, min, max and count; nulls stay null", S, + [{"op": "select", "exprs": {"sx": fn("cumSum", col("x")), "si": fn("cumSum", col("i")), "mn": fn("cumMin", col("i")), + "mx": fn("cumMax", col("x")), "n": fn("cumCount", col("x")), "sb": fn("cumSum", col("b"))}}]) + add("window/shift-diff", "shift (lag and lead) and diff", S, + [{"op": "select", "exprs": {"lag": fn("shift", col("i")), "lead": fn("shift", col("x"), n=-2), "d": fn("diff", col("i")), + "d2": fn("diff", col("x"), n=2), "ls": fn("shift", col("g"))}}]) + add("window/rank", "rank methods, nulls and NaN", S, + [{"op": "select", "exprs": {"avg": fn("rank", col("x")), "min": fn("rank", col("i"), method="min"), + "max": fn("rank", col("i"), method="max"), "dense": fn("rank", col("i"), method="dense", descending=True), + "ord": fn("rank", col("x"), method="ordinal"), "s": fn("rank", col("g"), method="dense")}}]) + add("window/rolling", "rolling sum, mean, min and max", S, + [{"op": "select", "exprs": {"s": fn("rollingSum", col("x"), window=3), "m": fn("rollingMean", col("i"), window=2), + "mn": fn("rollingMin", col("x"), window=3, minPeriods=1), "mx": fn("rollingMax", col("i"), window=4, minPeriods=2)}}]) + # window functions per partition + add("window/over-agg", "aggregates broadcast over partitions", S, + [{"op": "withColumns", "exprs": {"total": over(agg("sum", col("x")), "g"), "n": over(agg("len"), "g"), + "mean": over(agg("mean", col("i")), "g"), "first": over(agg("first", col("t")), "g", "b")}}]) + add("window/over-cum", "cumulative and shifted values per partition, in row order", S, + [{"op": "withColumns", "exprs": {"cs": over(fn("cumSum", col("i")), "g"), "prev": over(fn("shift", col("x")), "g"), + "d": over(fn("diff", col("t")), "g"), "r": over(fn("rank", col("x"), method="min"), "g"), + "roll": over(fn("rollingSum", col("i"), window=2, minPeriods=1), "g")}}]) + add("window/over-filter", "filters after windows see the windowed values", S, + [{"op": "withColumns", "exprs": {"cs": over(fn("cumSum", col("t")), "g")}}, + {"op": "filter", "predicate": fn("gt", col("t"), lit(2))}]) + add("window/filter-first", "windows after a filter only see the remaining rows", S, + [{"op": "filter", "predicate": fn("isNotNull", col("x"))}, + {"op": "withColumns", "exprs": {"cs": fn("cumSum", col("x")), "r": over(fn("rank", col("x")), "g")}}, + {"op": "head", "n": 4}]) + + # pivots + add("pivot/sum", "pivot with a sum per cell; missing cells are null", P, + [{"op": "pivot", "on": "quarter", "index": ["region"], "values": "units", "aggregate": "sum"}]) + add("pivot/first", "pivot with the first value per cell", P, + [{"op": "pivot", "on": "region", "index": ["quarter"], "values": "price", "aggregate": "first"}]) + add("pivot/len", "pivot counting rows per cell", P, + [{"op": "pivot", "on": "quarter", "index": ["region"], "values": "units", "aggregate": "len"}]) + add("pivot/columns", "pivot with declared output columns", P, + [{"op": "pivot", "on": "quarter", "index": ["region"], "values": "price", "aggregate": "max", "columns": ["q1", "q2", "q4"]}]) + add("pivot/numeric-on", "pivot on a numeric column names columns by value", S, + [{"op": "filter", "predicate": fn("isNotNull", col("g"))}, + {"op": "pivot", "on": "t", "index": ["g"], "values": "i", "aggregate": "min"}]) + add("unpivot/basic", "unpivot stacks columns, column by column", P, + [{"op": "unpivot", "index": ["region"], "on": ["units", "price"]}]) + add("unpivot/names", "unpivot with named variable and value columns", P, + [{"op": "pick", "columns": ["region", "quarter"]}, + {"op": "unpivot", "index": [], "on": ["region", "quarter"], "variable": "field", "value": "text"}]) + T = {"left": times, "right": holidays} + us = lambda dt: int((dt - EPOCH).total_seconds() * 1_000_000) + add("temporal/extract", "calendar fields of timestamps and dates", T, + [{"op": "select", "exprs": {"y": fn("year", col("ts")), "mo": fn("month", col("ts")), "d": fn("day", col("ts")), + "h": fn("hour", col("ts")), "mi": fn("minute", col("ts")), "s": fn("second", col("ts")), + "ms": fn("millisecond", col("ts")), "wd": fn("weekday", col("ts")), "od": fn("ordinalDay", col("ts")), + "dy": fn("year", col("d")), "dwd": fn("weekday", col("d")), "dod": fn("ordinalDay", col("d"))}}]) + add("temporal/arith", "timestamp differences are durations; durations shift timestamps", T, + [{"op": "select", "exprs": {"gap": fn("sub", col("ts"), col("ms")), "later": fn("add", col("ts"), col("dur")), + "earlier": fn("sub", col("ts"), col("dur")), "days": fn("sub", col("d"), lit(18000, "date")), + "twice": fn("add", col("dur"), col("dur"))}}]) + add("temporal/truncate", "truncate to a calendar unit", T, + [{"op": "select", "exprs": {"h": fn("truncate", col("ts"), every="1h"), "d": fn("truncate", col("ts"), every="1d"), + "w": fn("truncate", col("ts"), every="1w"), "mo": fn("truncate", col("ts"), every="1mo"), + "y": fn("truncate", col("ts"), every="1y"), "dmo": fn("truncate", col("d"), every="1mo")}}]) + add("temporal/duration", "durations in whole units", T, + [{"op": "select", "exprs": {"d": fn("totalDays", col("dur")), "h": fn("totalHours", col("dur")), "m": fn("totalMinutes", col("dur")), + "s": fn("totalSeconds", col("dur")), "ms": fn("totalMilliseconds", col("dur"))}}]) + add("temporal/cast", "casts between dates, timestamps and integers", T, + [{"op": "select", "exprs": {"date": fn("cast", col("ts"), to="date"), "ts": fn("cast", col("d"), to="datetime[us]"), + "us": fn("cast", col("ts"), to="i64"), "days": fn("cast", col("d"), to="i32"), + "ms": fn("cast", col("ts"), to="datetime[ms]")}}]) + add("temporal/compare", "comparisons with timestamp literals and between units", T, + [{"op": "filter", "predicate": fn("and", fn("gte", col("ts"), lit(us(datetime(2024, 3, 1)), "datetime[us]")), fn("lt", col("ms"), col("ts")))}]) + add("temporal/group-sort", "group by date, aggregate timestamps, sort", T, + [{"op": "groupBy", "keys": ["d"], "aggs": {"first": agg("min", col("ts")), "last": agg("max", col("ts")), "n": agg("len"), + "total": agg("sum", col("dur"))}}, + {"op": "sort", "keys": [{"expr": col("last"), "descending": True}]}]) + add("temporal/join", "join on a date", T, [{"op": "join", "right": "right", "on": ["d"], "how": "left"}]) + N = {"left": nested, "right": nested_right} + add("nested/list-ops", "list length, element access and aggregates", N, + [{"op": "select", "exprs": {"n": fn("listLength", col("xs")), "first": fn("listGet", col("xs"), index=0), + "last": fn("listGet", col("xs"), index=-1), "oob": fn("listGet", col("xs"), index=5), + "sum": fn("listSum", col("xs")), "mean": fn("listMean", col("xs")), "min": fn("listMin", col("xs")), + "max": fn("listMax", col("ws")), "has2": fn("listContains", col("xs"), lit(2)), + "hasA": fn("listContains", col("ws"), lit("a"))}}]) + add("nested/struct", "struct fields and struct construction", N, + [{"op": "select", "exprs": {"a": fn("field", col("s"), field="a"), "b": fn("field", col("s"), field="b"), + "pair": fn("struct", col("k"), fn("listLength", col("xs")), names=["key", "size"])}}]) + add("nested/explode", "explode lists into rows; a null list gives a null row, an empty list no rows", N, + [{"op": "pick", "columns": ["k", "xs"]}, {"op": "explode", "column": "xs"}]) + add("nested/unnest", "unnest a struct into columns", N, [{"op": "pick", "columns": ["k", "s"]}, {"op": "unnest", "column": "s"}]) + add("nested/agg-list", "collect values into lists per group", N, + [{"op": "groupBy", "keys": ["g"], "aggs": {"ks": agg("list", col("k")), "firsts": agg("first", col("xs")), + "n": agg("len")}}]) + add("nested/carry", "lists and structs survive filters, sorts, joins and slices", N, + [{"op": "filter", "predicate": fn("neq", col("k"), lit("c"))}, + {"op": "sort", "keys": [{"expr": col("g"), "descending": True}, {"expr": col("k")}]}, + {"op": "join", "right": "right", "on": ["k"], "how": "left"}, + {"op": "head", "n": 4}]) + D = {"left": money} + add("decimal/arith", "decimal addition, subtraction and multiplication", D, + [{"op": "select", "exprs": {"sum": fn("add", col("price"), col("fee")), "diff": fn("sub", col("price"), col("fee")), + "prod": fn("mul", col("price"), col("qty")), "plus": fn("add", col("price"), lit("0.5", "decimal[3,1]"))}}]) + add("decimal/compare", "decimal comparisons, also across scales", D, + [{"op": "filter", "predicate": fn("and", fn("gt", col("price"), lit("10.00", "decimal[10,2]")), fn("gte", col("price"), col("fee")))}]) + add("decimal/aggregate", "sums, extremes and means of decimals per group", D, + [{"op": "groupBy", "keys": ["k"], "aggs": {"total": agg("sum", col("price")), "min": agg("min", col("price")), + "max": agg("max", col("fee")), "mean": agg("mean", col("price")), "n": agg("count", col("price"))}}]) + add("decimal/cast", "casts between decimals, floats, integers and strings", D, + [{"op": "select", "exprs": {"f": fn("cast", col("price"), to="f64"), "i": fn("cast", col("price"), to="i32"), + "s": fn("cast", col("fee"), to="utf8"), "d": fn("cast", col("x"), to="decimal[10,2]"), + "r": fn("cast", col("price"), to="decimal[12,1]")}}]) + add("decimal/sort", "sort by a decimal", D, [{"op": "sort", "keys": [{"expr": col("price"), "descending": True}], "nullsLast": True}]) + add("decimal/round-ties", "narrowing the scale rounds exact ties", {"left": table([("t", "decimal[6,3]")], {"t": ["0.250", "0.350", "-0.250", "-0.350", "1.050", "2.500", "0.249"]})}, + [{"op": "select", "exprs": {"one": fn("cast", col("t"), to="decimal[6,1]"), "zero": fn("cast", col("t"), to="decimal[6,0]")}}]) + return out + + +def main(): + gold = HERE / "gold" + gold.mkdir(exist_ok=True) + header = {"generator": "generate_gold.py", "polars": pl.__version__, "deviations": DEVIATIONS} + spec = spec_cases() + (gold / "spec.json").write_text(json.dumps({**header, "cases": spec}, ensure_ascii=False, indent=None, separators=(",", ":")) + "\n") + rng = random.Random(20261114) + cases = [c for c in (random_case(rng, i) for i in range(400)) if c is not None] + (gold / "random.json").write_text(json.dumps({**header, "cases": cases}, ensure_ascii=False, indent=None, separators=(",", ":")) + "\n") + extended = extended_cases() + (gold / "extended.json").write_text(json.dumps({**header, "cases": extended}, ensure_ascii=False, indent=None, separators=(",", ":")) + "\n") + print(f"spec: {len(spec)} cases, random: {len(cases)} cases, extended: {len(extended)} cases") + + +if __name__ == "__main__": + main() diff --git a/packages/effect/test/fixtures/dataframe/gold/extended.json b/packages/effect/test/fixtures/dataframe/gold/extended.json new file mode 100644 index 00000000000..5528f1600e1 --- /dev/null +++ b/packages/effect/test/fixtures/dataframe/gold/extended.json @@ -0,0 +1 @@ +{"generator":"generate_gold.py","polars":"2.1.0","deviations":["JavaScript has no float literal distinct from an integer: an integral number literal behaves like a Polars integer literal (it takes the integer type of the column it meets), so cases never combine an integer column with an integral float literal.","Float to string casts use JavaScript number formatting (Polars prints 5.0 as '5.0', JavaScript as '5').","Joins never add '_right' suffixes: overlapping non-key columns are a type error.","Arithmetic between 64-bit integers (bigint) and other numeric types requires an explicit cast."],"cases":[{"id":"window/cum","description":"cumulative sum, min, max and count; nulls stay null","inputs":{"left":{"fields":[{"name":"g","type":"utf8"},{"name":"t","type":"i32"},{"name":"x","type":"f64"},{"name":"i","type":"i32"},{"name":"b","type":"bool"}],"columns":{"g":["a","b","a","a","b",null,"a","b","b","a"],"t":[1,1,2,3,2,1,4,3,4,5],"x":[1.5,2.0,null,4.0,"NaN",3.0,-1.0,2.0,0.5,2.0],"i":[3,1,4,1,5,9,null,6,5,3],"b":[true,false,null,true,true,false,true,null,false,true]}}},"ops":[{"op":"select","exprs":{"sx":{"fn":"cumSum","args":[{"col":"x"}]},"si":{"fn":"cumSum","args":[{"col":"i"}]},"mn":{"fn":"cumMin","args":[{"col":"i"}]},"mx":{"fn":"cumMax","args":[{"col":"x"}]},"n":{"fn":"cumCount","args":[{"col":"x"}]},"sb":{"fn":"cumSum","args":[{"col":"b"}]}}}],"expected":{"fields":[{"name":"sx","type":"f64"},{"name":"si","type":"i32"},{"name":"mn","type":"i32"},{"name":"mx","type":"f64"},{"name":"n","type":"u32"},{"name":"sb","type":"u32"}],"columns":{"sx":[1.5,3.5,null,7.5,"NaN","NaN","NaN","NaN","NaN","NaN"],"si":[3,4,8,9,14,23,null,29,34,37],"mn":[3,1,1,1,1,1,null,1,1,1],"mx":[1.5,2.0,null,4.0,4.0,4.0,4.0,4.0,4.0,4.0],"n":[1,2,2,3,4,5,6,7,8,9],"sb":[1,1,null,2,3,3,4,null,4,5]}}},{"id":"window/shift-diff","description":"shift (lag and lead) and diff","inputs":{"left":{"fields":[{"name":"g","type":"utf8"},{"name":"t","type":"i32"},{"name":"x","type":"f64"},{"name":"i","type":"i32"},{"name":"b","type":"bool"}],"columns":{"g":["a","b","a","a","b",null,"a","b","b","a"],"t":[1,1,2,3,2,1,4,3,4,5],"x":[1.5,2.0,null,4.0,"NaN",3.0,-1.0,2.0,0.5,2.0],"i":[3,1,4,1,5,9,null,6,5,3],"b":[true,false,null,true,true,false,true,null,false,true]}}},"ops":[{"op":"select","exprs":{"lag":{"fn":"shift","args":[{"col":"i"}]},"lead":{"fn":"shift","args":[{"col":"x"}],"n":-2},"d":{"fn":"diff","args":[{"col":"i"}]},"d2":{"fn":"diff","args":[{"col":"x"}],"n":2},"ls":{"fn":"shift","args":[{"col":"g"}]}}}],"expected":{"fields":[{"name":"lag","type":"i32"},{"name":"lead","type":"f64"},{"name":"d","type":"i32"},{"name":"d2","type":"f64"},{"name":"ls","type":"utf8"}],"columns":{"lag":[null,3,1,4,1,5,9,null,6,5],"lead":[null,4.0,"NaN",3.0,-1.0,2.0,0.5,2.0,null,null],"d":[null,-2,3,-3,4,4,null,null,-1,-2],"d2":[null,null,null,2.0,null,-1.0,"NaN",-1.0,1.5,0.0],"ls":[null,"a","b","a","a","b",null,"a","b","b"]}}},{"id":"window/rank","description":"rank methods, nulls and NaN","inputs":{"left":{"fields":[{"name":"g","type":"utf8"},{"name":"t","type":"i32"},{"name":"x","type":"f64"},{"name":"i","type":"i32"},{"name":"b","type":"bool"}],"columns":{"g":["a","b","a","a","b",null,"a","b","b","a"],"t":[1,1,2,3,2,1,4,3,4,5],"x":[1.5,2.0,null,4.0,"NaN",3.0,-1.0,2.0,0.5,2.0],"i":[3,1,4,1,5,9,null,6,5,3],"b":[true,false,null,true,true,false,true,null,false,true]}}},"ops":[{"op":"select","exprs":{"avg":{"fn":"rank","args":[{"col":"x"}]},"min":{"fn":"rank","args":[{"col":"i"}],"method":"min"},"max":{"fn":"rank","args":[{"col":"i"}],"method":"max"},"dense":{"fn":"rank","args":[{"col":"i"}],"method":"dense","descending":true},"ord":{"fn":"rank","args":[{"col":"x"}],"method":"ordinal"},"s":{"fn":"rank","args":[{"col":"g"}],"method":"dense"}}}],"expected":{"fields":[{"name":"avg","type":"f64"},{"name":"min","type":"u32"},{"name":"max","type":"u32"},{"name":"dense","type":"u32"},{"name":"ord","type":"u32"},{"name":"s","type":"u32"}],"columns":{"avg":[3.0,5.0,null,8.0,9.0,7.0,1.0,5.0,2.0,5.0],"min":[3,1,5,1,6,9,null,8,6,3],"max":[4,2,5,2,7,9,null,8,7,4],"dense":[5,6,4,6,3,1,null,2,3,5],"ord":[3,4,null,8,9,7,1,5,2,6],"s":[1,2,1,1,2,null,1,2,2,1]}}},{"id":"window/rolling","description":"rolling sum, mean, min and max","inputs":{"left":{"fields":[{"name":"g","type":"utf8"},{"name":"t","type":"i32"},{"name":"x","type":"f64"},{"name":"i","type":"i32"},{"name":"b","type":"bool"}],"columns":{"g":["a","b","a","a","b",null,"a","b","b","a"],"t":[1,1,2,3,2,1,4,3,4,5],"x":[1.5,2.0,null,4.0,"NaN",3.0,-1.0,2.0,0.5,2.0],"i":[3,1,4,1,5,9,null,6,5,3],"b":[true,false,null,true,true,false,true,null,false,true]}}},"ops":[{"op":"select","exprs":{"s":{"fn":"rollingSum","args":[{"col":"x"}],"window":3},"m":{"fn":"rollingMean","args":[{"col":"i"}],"window":2},"mn":{"fn":"rollingMin","args":[{"col":"x"}],"window":3,"minPeriods":1},"mx":{"fn":"rollingMax","args":[{"col":"i"}],"window":4,"minPeriods":2}}}],"expected":{"fields":[{"name":"s","type":"f64"},{"name":"m","type":"f64"},{"name":"mn","type":"f64"},{"name":"mx","type":"i32"}],"columns":{"s":[null,null,null,null,null,"NaN","NaN",4.0,1.5,4.5],"m":[null,2.0,2.5,2.5,3.0,7.0,null,null,5.5,4.0],"mn":[1.5,1.5,1.5,2.0,"NaN","NaN","NaN",-1.0,-1.0,0.5],"mx":[null,3,4,4,5,9,9,9,9,6]}}},{"id":"window/over-agg","description":"aggregates broadcast over partitions","inputs":{"left":{"fields":[{"name":"g","type":"utf8"},{"name":"t","type":"i32"},{"name":"x","type":"f64"},{"name":"i","type":"i32"},{"name":"b","type":"bool"}],"columns":{"g":["a","b","a","a","b",null,"a","b","b","a"],"t":[1,1,2,3,2,1,4,3,4,5],"x":[1.5,2.0,null,4.0,"NaN",3.0,-1.0,2.0,0.5,2.0],"i":[3,1,4,1,5,9,null,6,5,3],"b":[true,false,null,true,true,false,true,null,false,true]}}},"ops":[{"op":"withColumns","exprs":{"total":{"over":{"agg":"sum","arg":{"col":"x"}},"partitionBy":[{"col":"g"}]},"n":{"over":{"agg":"len"},"partitionBy":[{"col":"g"}]},"mean":{"over":{"agg":"mean","arg":{"col":"i"}},"partitionBy":[{"col":"g"}]},"first":{"over":{"agg":"first","arg":{"col":"t"}},"partitionBy":[{"col":"g"},{"col":"b"}]}}}],"expected":{"fields":[{"name":"g","type":"utf8"},{"name":"t","type":"i32"},{"name":"x","type":"f64"},{"name":"i","type":"i32"},{"name":"b","type":"bool"},{"name":"total","type":"f64"},{"name":"n","type":"u32"},{"name":"mean","type":"f64"},{"name":"first","type":"i32"}],"columns":{"g":["a","b","a","a","b",null,"a","b","b","a"],"t":[1,1,2,3,2,1,4,3,4,5],"x":[1.5,2.0,null,4.0,"NaN",3.0,-1.0,2.0,0.5,2.0],"i":[3,1,4,1,5,9,null,6,5,3],"b":[true,false,null,true,true,false,true,null,false,true],"total":[6.5,"NaN",6.5,6.5,"NaN",3.0,6.5,"NaN","NaN",6.5],"n":[5,4,5,5,4,1,5,4,4,5],"mean":[2.75,4.25,2.75,2.75,4.25,9.0,2.75,4.25,4.25,2.75],"first":[1,1,2,1,2,1,1,3,1,1]}}},{"id":"window/over-cum","description":"cumulative and shifted values per partition, in row order","inputs":{"left":{"fields":[{"name":"g","type":"utf8"},{"name":"t","type":"i32"},{"name":"x","type":"f64"},{"name":"i","type":"i32"},{"name":"b","type":"bool"}],"columns":{"g":["a","b","a","a","b",null,"a","b","b","a"],"t":[1,1,2,3,2,1,4,3,4,5],"x":[1.5,2.0,null,4.0,"NaN",3.0,-1.0,2.0,0.5,2.0],"i":[3,1,4,1,5,9,null,6,5,3],"b":[true,false,null,true,true,false,true,null,false,true]}}},"ops":[{"op":"withColumns","exprs":{"cs":{"over":{"fn":"cumSum","args":[{"col":"i"}]},"partitionBy":[{"col":"g"}]},"prev":{"over":{"fn":"shift","args":[{"col":"x"}]},"partitionBy":[{"col":"g"}]},"d":{"over":{"fn":"diff","args":[{"col":"t"}]},"partitionBy":[{"col":"g"}]},"r":{"over":{"fn":"rank","args":[{"col":"x"}],"method":"min"},"partitionBy":[{"col":"g"}]},"roll":{"over":{"fn":"rollingSum","args":[{"col":"i"}],"window":2,"minPeriods":1},"partitionBy":[{"col":"g"}]}}}],"expected":{"fields":[{"name":"g","type":"utf8"},{"name":"t","type":"i32"},{"name":"x","type":"f64"},{"name":"i","type":"i32"},{"name":"b","type":"bool"},{"name":"cs","type":"i32"},{"name":"prev","type":"f64"},{"name":"d","type":"i32"},{"name":"r","type":"u32"},{"name":"roll","type":"i32"}],"columns":{"g":["a","b","a","a","b",null,"a","b","b","a"],"t":[1,1,2,3,2,1,4,3,4,5],"x":[1.5,2.0,null,4.0,"NaN",3.0,-1.0,2.0,0.5,2.0],"i":[3,1,4,1,5,9,null,6,5,3],"b":[true,false,null,true,true,false,true,null,false,true],"cs":[3,1,7,8,6,9,null,12,17,11],"prev":[null,null,1.5,null,2.0,null,4.0,"NaN",2.0,-1.0],"d":[null,null,1,1,1,null,1,1,1,1],"r":[2,2,null,4,4,1,1,2,1,3],"roll":[3,1,7,5,6,9,1,11,11,3]}}},{"id":"window/over-filter","description":"filters after windows see the windowed values","inputs":{"left":{"fields":[{"name":"g","type":"utf8"},{"name":"t","type":"i32"},{"name":"x","type":"f64"},{"name":"i","type":"i32"},{"name":"b","type":"bool"}],"columns":{"g":["a","b","a","a","b",null,"a","b","b","a"],"t":[1,1,2,3,2,1,4,3,4,5],"x":[1.5,2.0,null,4.0,"NaN",3.0,-1.0,2.0,0.5,2.0],"i":[3,1,4,1,5,9,null,6,5,3],"b":[true,false,null,true,true,false,true,null,false,true]}}},"ops":[{"op":"withColumns","exprs":{"cs":{"over":{"fn":"cumSum","args":[{"col":"t"}]},"partitionBy":[{"col":"g"}]}}},{"op":"filter","predicate":{"fn":"gt","args":[{"col":"t"},{"lit":2}]}}],"expected":{"fields":[{"name":"g","type":"utf8"},{"name":"t","type":"i32"},{"name":"x","type":"f64"},{"name":"i","type":"i32"},{"name":"b","type":"bool"},{"name":"cs","type":"i32"}],"columns":{"g":["a","a","b","b","a"],"t":[3,4,3,4,5],"x":[4.0,-1.0,2.0,0.5,2.0],"i":[1,null,6,5,3],"b":[true,true,null,false,true],"cs":[6,10,6,10,15]}}},{"id":"window/filter-first","description":"windows after a filter only see the remaining rows","inputs":{"left":{"fields":[{"name":"g","type":"utf8"},{"name":"t","type":"i32"},{"name":"x","type":"f64"},{"name":"i","type":"i32"},{"name":"b","type":"bool"}],"columns":{"g":["a","b","a","a","b",null,"a","b","b","a"],"t":[1,1,2,3,2,1,4,3,4,5],"x":[1.5,2.0,null,4.0,"NaN",3.0,-1.0,2.0,0.5,2.0],"i":[3,1,4,1,5,9,null,6,5,3],"b":[true,false,null,true,true,false,true,null,false,true]}}},"ops":[{"op":"filter","predicate":{"fn":"isNotNull","args":[{"col":"x"}]}},{"op":"withColumns","exprs":{"cs":{"fn":"cumSum","args":[{"col":"x"}]},"r":{"over":{"fn":"rank","args":[{"col":"x"}]},"partitionBy":[{"col":"g"}]}}},{"op":"head","n":4}],"expected":{"fields":[{"name":"g","type":"utf8"},{"name":"t","type":"i32"},{"name":"x","type":"f64"},{"name":"i","type":"i32"},{"name":"b","type":"bool"},{"name":"cs","type":"f64"},{"name":"r","type":"f64"}],"columns":{"g":["a","b","a","b"],"t":[1,1,3,2],"x":[1.5,2.0,4.0,"NaN"],"i":[3,1,1,5],"b":[true,false,true,true],"cs":[1.5,3.5,7.5,"NaN"],"r":[2.0,2.5,4.0,4.0]}}},{"id":"pivot/sum","description":"pivot with a sum per cell; missing cells are null","inputs":{"left":{"fields":[{"name":"region","type":"utf8"},{"name":"quarter","type":"utf8"},{"name":"units","type":"i32"},{"name":"price","type":"f64"}],"columns":{"region":["north","north","south","south","north","east","south","north"],"quarter":["q1","q2","q1","q3","q1","q2","q1",null],"units":[10,20,5,null,7,3,8,1],"price":[1.5,2.0,1.0,3.0,null,2.5,1.0,9.0]}}},"ops":[{"op":"pivot","on":"quarter","index":["region"],"values":"units","aggregate":"sum"}],"expected":{"fields":[{"name":"region","type":"utf8"},{"name":"q1","type":"i32"},{"name":"q2","type":"i32"},{"name":"q3","type":"i32"},{"name":"null","type":"i32"}],"columns":{"region":["north","south","east"],"q1":[17,13,0],"q2":[20,0,3],"q3":[0,0,0],"null":[1,0,0]}}},{"id":"pivot/first","description":"pivot with the first value per cell","inputs":{"left":{"fields":[{"name":"region","type":"utf8"},{"name":"quarter","type":"utf8"},{"name":"units","type":"i32"},{"name":"price","type":"f64"}],"columns":{"region":["north","north","south","south","north","east","south","north"],"quarter":["q1","q2","q1","q3","q1","q2","q1",null],"units":[10,20,5,null,7,3,8,1],"price":[1.5,2.0,1.0,3.0,null,2.5,1.0,9.0]}}},"ops":[{"op":"pivot","on":"region","index":["quarter"],"values":"price","aggregate":"first"}],"expected":{"fields":[{"name":"quarter","type":"utf8"},{"name":"north","type":"f64"},{"name":"south","type":"f64"},{"name":"east","type":"f64"}],"columns":{"quarter":["q1","q2","q3",null],"north":[1.5,2.0,null,9.0],"south":[1.0,null,3.0,null],"east":[null,2.5,null,null]}}},{"id":"pivot/len","description":"pivot counting rows per cell","inputs":{"left":{"fields":[{"name":"region","type":"utf8"},{"name":"quarter","type":"utf8"},{"name":"units","type":"i32"},{"name":"price","type":"f64"}],"columns":{"region":["north","north","south","south","north","east","south","north"],"quarter":["q1","q2","q1","q3","q1","q2","q1",null],"units":[10,20,5,null,7,3,8,1],"price":[1.5,2.0,1.0,3.0,null,2.5,1.0,9.0]}}},"ops":[{"op":"pivot","on":"quarter","index":["region"],"values":"units","aggregate":"len"}],"expected":{"fields":[{"name":"region","type":"utf8"},{"name":"q1","type":"u32"},{"name":"q2","type":"u32"},{"name":"q3","type":"u32"},{"name":"null","type":"u32"}],"columns":{"region":["north","south","east"],"q1":[2,2,0],"q2":[1,0,1],"q3":[0,1,0],"null":[1,0,0]}}},{"id":"pivot/columns","description":"pivot with declared output columns","inputs":{"left":{"fields":[{"name":"region","type":"utf8"},{"name":"quarter","type":"utf8"},{"name":"units","type":"i32"},{"name":"price","type":"f64"}],"columns":{"region":["north","north","south","south","north","east","south","north"],"quarter":["q1","q2","q1","q3","q1","q2","q1",null],"units":[10,20,5,null,7,3,8,1],"price":[1.5,2.0,1.0,3.0,null,2.5,1.0,9.0]}}},"ops":[{"op":"pivot","on":"quarter","index":["region"],"values":"price","aggregate":"max","columns":["q1","q2","q4"]}],"expected":{"fields":[{"name":"region","type":"utf8"},{"name":"q1","type":"f64"},{"name":"q2","type":"f64"},{"name":"q4","type":"f64"}],"columns":{"region":["north","south","east"],"q1":[1.5,1.0,null],"q2":[2.0,null,2.5],"q4":[null,null,null]}}},{"id":"pivot/numeric-on","description":"pivot on a numeric column names columns by value","inputs":{"left":{"fields":[{"name":"g","type":"utf8"},{"name":"t","type":"i32"},{"name":"x","type":"f64"},{"name":"i","type":"i32"},{"name":"b","type":"bool"}],"columns":{"g":["a","b","a","a","b",null,"a","b","b","a"],"t":[1,1,2,3,2,1,4,3,4,5],"x":[1.5,2.0,null,4.0,"NaN",3.0,-1.0,2.0,0.5,2.0],"i":[3,1,4,1,5,9,null,6,5,3],"b":[true,false,null,true,true,false,true,null,false,true]}}},"ops":[{"op":"filter","predicate":{"fn":"isNotNull","args":[{"col":"g"}]}},{"op":"pivot","on":"t","index":["g"],"values":"i","aggregate":"min"}],"expected":{"fields":[{"name":"g","type":"utf8"},{"name":"1","type":"i32"},{"name":"2","type":"i32"},{"name":"3","type":"i32"},{"name":"4","type":"i32"},{"name":"5","type":"i32"}],"columns":{"g":["a","b"],"1":[3,1],"2":[4,5],"3":[1,6],"4":[null,5],"5":[3,null]}}},{"id":"unpivot/basic","description":"unpivot stacks columns, column by column","inputs":{"left":{"fields":[{"name":"region","type":"utf8"},{"name":"quarter","type":"utf8"},{"name":"units","type":"i32"},{"name":"price","type":"f64"}],"columns":{"region":["north","north","south","south","north","east","south","north"],"quarter":["q1","q2","q1","q3","q1","q2","q1",null],"units":[10,20,5,null,7,3,8,1],"price":[1.5,2.0,1.0,3.0,null,2.5,1.0,9.0]}}},"ops":[{"op":"unpivot","index":["region"],"on":["units","price"]}],"expected":{"fields":[{"name":"region","type":"utf8"},{"name":"variable","type":"utf8"},{"name":"value","type":"f64"}],"columns":{"region":["north","north","south","south","north","east","south","north","north","north","south","south","north","east","south","north"],"variable":["units","units","units","units","units","units","units","units","price","price","price","price","price","price","price","price"],"value":[10.0,20.0,5.0,null,7.0,3.0,8.0,1.0,1.5,2.0,1.0,3.0,null,2.5,1.0,9.0]}}},{"id":"unpivot/names","description":"unpivot with named variable and value columns","inputs":{"left":{"fields":[{"name":"region","type":"utf8"},{"name":"quarter","type":"utf8"},{"name":"units","type":"i32"},{"name":"price","type":"f64"}],"columns":{"region":["north","north","south","south","north","east","south","north"],"quarter":["q1","q2","q1","q3","q1","q2","q1",null],"units":[10,20,5,null,7,3,8,1],"price":[1.5,2.0,1.0,3.0,null,2.5,1.0,9.0]}}},"ops":[{"op":"pick","columns":["region","quarter"]},{"op":"unpivot","index":[],"on":["region","quarter"],"variable":"field","value":"text"}],"expected":{"fields":[{"name":"field","type":"utf8"},{"name":"text","type":"utf8"}],"columns":{"field":["region","region","region","region","region","region","region","region","quarter","quarter","quarter","quarter","quarter","quarter","quarter","quarter"],"text":["north","north","south","south","north","east","south","north","q1","q2","q1","q3","q1","q2","q1",null]}}},{"id":"temporal/extract","description":"calendar fields of timestamps and dates","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"ts","type":"datetime[us]"},{"name":"ms","type":"datetime[ms]"},{"name":"d","type":"date"},{"name":"dur","type":"duration[us]"}],"columns":{"k":["a","b","c","d","e","f"],"ts":[1709214307123456,1704067199999000,1709510400000000,null,-14182940000000,1710052200000500],"ms":[1709214307123,1704067200000,null,1709251200000,0,1709942400000],"d":[19782,19722,19786,19782,null,19792],"dur":[5400000000,-1500000,null,259200000001,0,90000000000]}},"right":{"fields":[{"name":"d","type":"date"},{"name":"name","type":"utf8"}],"columns":{"d":[19782,19792,20089],"name":["leap","sunday","new year"]}}},"ops":[{"op":"select","exprs":{"y":{"fn":"year","args":[{"col":"ts"}]},"mo":{"fn":"month","args":[{"col":"ts"}]},"d":{"fn":"day","args":[{"col":"ts"}]},"h":{"fn":"hour","args":[{"col":"ts"}]},"mi":{"fn":"minute","args":[{"col":"ts"}]},"s":{"fn":"second","args":[{"col":"ts"}]},"ms":{"fn":"millisecond","args":[{"col":"ts"}]},"wd":{"fn":"weekday","args":[{"col":"ts"}]},"od":{"fn":"ordinalDay","args":[{"col":"ts"}]},"dy":{"fn":"year","args":[{"col":"d"}]},"dwd":{"fn":"weekday","args":[{"col":"d"}]},"dod":{"fn":"ordinalDay","args":[{"col":"d"}]}}}],"expected":{"fields":[{"name":"y","type":"i32"},{"name":"mo","type":"i8"},{"name":"d","type":"i8"},{"name":"h","type":"i8"},{"name":"mi","type":"i8"},{"name":"s","type":"i8"},{"name":"ms","type":"i32"},{"name":"wd","type":"i8"},{"name":"od","type":"i16"},{"name":"dy","type":"i32"},{"name":"dwd","type":"i8"},{"name":"dod","type":"i16"}],"columns":{"y":[2024,2023,2024,null,1969,2024],"mo":[2,12,3,null,7,3],"d":[29,31,4,null,20,10],"h":[13,23,0,null,20,6],"mi":[45,59,0,null,17,30],"s":[7,59,0,null,40,0],"ms":[123,999,0,null,0,0],"wd":[4,7,1,null,7,7],"od":[60,365,64,null,201,70],"dy":[2024,2023,2024,2024,null,2024],"dwd":[4,7,1,4,null,7],"dod":[60,365,64,60,null,70]}}},{"id":"temporal/arith","description":"timestamp differences are durations; durations shift timestamps","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"ts","type":"datetime[us]"},{"name":"ms","type":"datetime[ms]"},{"name":"d","type":"date"},{"name":"dur","type":"duration[us]"}],"columns":{"k":["a","b","c","d","e","f"],"ts":[1709214307123456,1704067199999000,1709510400000000,null,-14182940000000,1710052200000500],"ms":[1709214307123,1704067200000,null,1709251200000,0,1709942400000],"d":[19782,19722,19786,19782,null,19792],"dur":[5400000000,-1500000,null,259200000001,0,90000000000]}},"right":{"fields":[{"name":"d","type":"date"},{"name":"name","type":"utf8"}],"columns":{"d":[19782,19792,20089],"name":["leap","sunday","new year"]}}},"ops":[{"op":"select","exprs":{"gap":{"fn":"sub","args":[{"col":"ts"},{"col":"ms"}]},"later":{"fn":"add","args":[{"col":"ts"},{"col":"dur"}]},"earlier":{"fn":"sub","args":[{"col":"ts"},{"col":"dur"}]},"days":{"fn":"sub","args":[{"col":"d"},{"lit":18000,"type":"date"}]},"twice":{"fn":"add","args":[{"col":"dur"},{"col":"dur"}]}}}],"expected":{"fields":[{"name":"gap","type":"duration[ms]"},{"name":"later","type":"datetime[us]"},{"name":"earlier","type":"datetime[us]"},{"name":"days","type":"duration[us]"},{"name":"twice","type":"duration[us]"}],"columns":{"gap":[0,-1,null,null,-14182940000,109800000],"later":[1709219707123456,1704067198499000,null,null,-14182940000000,1710142200000500],"earlier":[1709208907123456,1704067201499000,null,null,-14182940000000,1709962200000500],"days":[153964800000000,148780800000000,154310400000000,153964800000000,null,154828800000000],"twice":[10800000000,-3000000,null,518400000002,0,180000000000]}}},{"id":"temporal/truncate","description":"truncate to a calendar unit","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"ts","type":"datetime[us]"},{"name":"ms","type":"datetime[ms]"},{"name":"d","type":"date"},{"name":"dur","type":"duration[us]"}],"columns":{"k":["a","b","c","d","e","f"],"ts":[1709214307123456,1704067199999000,1709510400000000,null,-14182940000000,1710052200000500],"ms":[1709214307123,1704067200000,null,1709251200000,0,1709942400000],"d":[19782,19722,19786,19782,null,19792],"dur":[5400000000,-1500000,null,259200000001,0,90000000000]}},"right":{"fields":[{"name":"d","type":"date"},{"name":"name","type":"utf8"}],"columns":{"d":[19782,19792,20089],"name":["leap","sunday","new year"]}}},"ops":[{"op":"select","exprs":{"h":{"fn":"truncate","args":[{"col":"ts"}],"every":"1h"},"d":{"fn":"truncate","args":[{"col":"ts"}],"every":"1d"},"w":{"fn":"truncate","args":[{"col":"ts"}],"every":"1w"},"mo":{"fn":"truncate","args":[{"col":"ts"}],"every":"1mo"},"y":{"fn":"truncate","args":[{"col":"ts"}],"every":"1y"},"dmo":{"fn":"truncate","args":[{"col":"d"}],"every":"1mo"}}}],"expected":{"fields":[{"name":"h","type":"datetime[us]"},{"name":"d","type":"datetime[us]"},{"name":"w","type":"datetime[us]"},{"name":"mo","type":"datetime[us]"},{"name":"y","type":"datetime[us]"},{"name":"dmo","type":"date"}],"columns":{"h":[1709211600000000,1704063600000000,1709510400000000,null,-14184000000000,1710050400000000],"d":[1709164800000000,1703980800000000,1709510400000000,null,-14256000000000,1710028800000000],"w":[1708905600000000,1703462400000000,1709510400000000,null,-14774400000000,1709510400000000],"mo":[1706745600000000,1701388800000000,1709251200000000,null,-15897600000000,1709251200000000],"y":[1704067200000000,1672531200000000,1704067200000000,null,-31536000000000,1704067200000000],"dmo":[19754,19692,19783,19754,null,19783]}}},{"id":"temporal/duration","description":"durations in whole units","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"ts","type":"datetime[us]"},{"name":"ms","type":"datetime[ms]"},{"name":"d","type":"date"},{"name":"dur","type":"duration[us]"}],"columns":{"k":["a","b","c","d","e","f"],"ts":[1709214307123456,1704067199999000,1709510400000000,null,-14182940000000,1710052200000500],"ms":[1709214307123,1704067200000,null,1709251200000,0,1709942400000],"d":[19782,19722,19786,19782,null,19792],"dur":[5400000000,-1500000,null,259200000001,0,90000000000]}},"right":{"fields":[{"name":"d","type":"date"},{"name":"name","type":"utf8"}],"columns":{"d":[19782,19792,20089],"name":["leap","sunday","new year"]}}},"ops":[{"op":"select","exprs":{"d":{"fn":"totalDays","args":[{"col":"dur"}]},"h":{"fn":"totalHours","args":[{"col":"dur"}]},"m":{"fn":"totalMinutes","args":[{"col":"dur"}]},"s":{"fn":"totalSeconds","args":[{"col":"dur"}]},"ms":{"fn":"totalMilliseconds","args":[{"col":"dur"}]}}}],"expected":{"fields":[{"name":"d","type":"i64"},{"name":"h","type":"i64"},{"name":"m","type":"i64"},{"name":"s","type":"i64"},{"name":"ms","type":"i64"}],"columns":{"d":["0","0",null,"3","0","1"],"h":["1","0",null,"72","0","25"],"m":["90","0",null,"4320","0","1500"],"s":["5400","-1",null,"259200","0","90000"],"ms":["5400000","-1500",null,"259200000","0","90000000"]}}},{"id":"temporal/cast","description":"casts between dates, timestamps and integers","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"ts","type":"datetime[us]"},{"name":"ms","type":"datetime[ms]"},{"name":"d","type":"date"},{"name":"dur","type":"duration[us]"}],"columns":{"k":["a","b","c","d","e","f"],"ts":[1709214307123456,1704067199999000,1709510400000000,null,-14182940000000,1710052200000500],"ms":[1709214307123,1704067200000,null,1709251200000,0,1709942400000],"d":[19782,19722,19786,19782,null,19792],"dur":[5400000000,-1500000,null,259200000001,0,90000000000]}},"right":{"fields":[{"name":"d","type":"date"},{"name":"name","type":"utf8"}],"columns":{"d":[19782,19792,20089],"name":["leap","sunday","new year"]}}},"ops":[{"op":"select","exprs":{"date":{"fn":"cast","args":[{"col":"ts"}],"to":"date"},"ts":{"fn":"cast","args":[{"col":"d"}],"to":"datetime[us]"},"us":{"fn":"cast","args":[{"col":"ts"}],"to":"i64"},"days":{"fn":"cast","args":[{"col":"d"}],"to":"i32"},"ms":{"fn":"cast","args":[{"col":"ts"}],"to":"datetime[ms]"}}}],"expected":{"fields":[{"name":"date","type":"date"},{"name":"ts","type":"datetime[us]"},{"name":"us","type":"i64"},{"name":"days","type":"i32"},{"name":"ms","type":"datetime[ms]"}],"columns":{"date":[19782,19722,19786,null,-165,19792],"ts":[1709164800000000,1703980800000000,1709510400000000,1709164800000000,null,1710028800000000],"us":["1709214307123456","1704067199999000","1709510400000000",null,"-14182940000000","1710052200000500"],"days":[19782,19722,19786,19782,null,19792],"ms":[1709214307123,1704067199999,1709510400000,null,-14182940000,1710052200000]}}},{"id":"temporal/compare","description":"comparisons with timestamp literals and between units","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"ts","type":"datetime[us]"},{"name":"ms","type":"datetime[ms]"},{"name":"d","type":"date"},{"name":"dur","type":"duration[us]"}],"columns":{"k":["a","b","c","d","e","f"],"ts":[1709214307123456,1704067199999000,1709510400000000,null,-14182940000000,1710052200000500],"ms":[1709214307123,1704067200000,null,1709251200000,0,1709942400000],"d":[19782,19722,19786,19782,null,19792],"dur":[5400000000,-1500000,null,259200000001,0,90000000000]}},"right":{"fields":[{"name":"d","type":"date"},{"name":"name","type":"utf8"}],"columns":{"d":[19782,19792,20089],"name":["leap","sunday","new year"]}}},"ops":[{"op":"filter","predicate":{"fn":"and","args":[{"fn":"gte","args":[{"col":"ts"},{"lit":1709251200000000,"type":"datetime[us]"}]},{"fn":"lt","args":[{"col":"ms"},{"col":"ts"}]}]}}],"expected":{"fields":[{"name":"k","type":"utf8"},{"name":"ts","type":"datetime[us]"},{"name":"ms","type":"datetime[ms]"},{"name":"d","type":"date"},{"name":"dur","type":"duration[us]"}],"columns":{"k":["f"],"ts":[1710052200000500],"ms":[1709942400000],"d":[19792],"dur":[90000000000]}}},{"id":"temporal/group-sort","description":"group by date, aggregate timestamps, sort","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"ts","type":"datetime[us]"},{"name":"ms","type":"datetime[ms]"},{"name":"d","type":"date"},{"name":"dur","type":"duration[us]"}],"columns":{"k":["a","b","c","d","e","f"],"ts":[1709214307123456,1704067199999000,1709510400000000,null,-14182940000000,1710052200000500],"ms":[1709214307123,1704067200000,null,1709251200000,0,1709942400000],"d":[19782,19722,19786,19782,null,19792],"dur":[5400000000,-1500000,null,259200000001,0,90000000000]}},"right":{"fields":[{"name":"d","type":"date"},{"name":"name","type":"utf8"}],"columns":{"d":[19782,19792,20089],"name":["leap","sunday","new year"]}}},"ops":[{"op":"groupBy","keys":["d"],"aggs":{"first":{"agg":"min","arg":{"col":"ts"}},"last":{"agg":"max","arg":{"col":"ts"}},"n":{"agg":"len"},"total":{"agg":"sum","arg":{"col":"dur"}}}},{"op":"sort","keys":[{"expr":{"col":"last"},"descending":true}]}],"expected":{"fields":[{"name":"d","type":"date"},{"name":"first","type":"datetime[us]"},{"name":"last","type":"datetime[us]"},{"name":"n","type":"u32"},{"name":"total","type":"duration[us]"}],"columns":{"d":[19792,19786,19782,19722,null],"first":[1710052200000500,1709510400000000,1709214307123456,1704067199999000,-14182940000000],"last":[1710052200000500,1709510400000000,1709214307123456,1704067199999000,-14182940000000],"n":[1,1,2,1,1],"total":[90000000000,0,264600000001,-1500000,0]}}},{"id":"temporal/join","description":"join on a date","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"ts","type":"datetime[us]"},{"name":"ms","type":"datetime[ms]"},{"name":"d","type":"date"},{"name":"dur","type":"duration[us]"}],"columns":{"k":["a","b","c","d","e","f"],"ts":[1709214307123456,1704067199999000,1709510400000000,null,-14182940000000,1710052200000500],"ms":[1709214307123,1704067200000,null,1709251200000,0,1709942400000],"d":[19782,19722,19786,19782,null,19792],"dur":[5400000000,-1500000,null,259200000001,0,90000000000]}},"right":{"fields":[{"name":"d","type":"date"},{"name":"name","type":"utf8"}],"columns":{"d":[19782,19792,20089],"name":["leap","sunday","new year"]}}},"ops":[{"op":"join","right":"right","on":["d"],"how":"left"}],"expected":{"fields":[{"name":"k","type":"utf8"},{"name":"ts","type":"datetime[us]"},{"name":"ms","type":"datetime[ms]"},{"name":"d","type":"date"},{"name":"dur","type":"duration[us]"},{"name":"name","type":"utf8"}],"columns":{"k":["a","b","c","d","e","f"],"ts":[1709214307123456,1704067199999000,1709510400000000,null,-14182940000000,1710052200000500],"ms":[1709214307123,1704067200000,null,1709251200000,0,1709942400000],"d":[19782,19722,19786,19782,null,19792],"dur":[5400000000,-1500000,null,259200000001,0,90000000000],"name":["leap",null,null,"leap",null,"sunday"]}}},{"id":"nested/list-ops","description":"list length, element access and aggregates","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"g","type":"i32"},{"name":"xs","type":"list"},{"name":"ws","type":"list"},{"name":"s","type":"struct"}],"columns":{"k":["a","b","c","d","e"],"g":[1,2,1,2,1],"xs":[[1,2,3],[],null,[4,null,2],[7]],"ws":[["a","b"],["c"],[],null,["z","a"]],"s":[{"a":1,"b":"x"},{"a":null,"b":"y"},null,{"a":4,"b":null},{"a":5,"b":"z"}]}},"right":{"fields":[{"name":"k","type":"utf8"},{"name":"tags","type":"list"}],"columns":{"k":["a","d","x"],"tags":[["t1"],["t2","t3"],[]]}}},"ops":[{"op":"select","exprs":{"n":{"fn":"listLength","args":[{"col":"xs"}]},"first":{"fn":"listGet","args":[{"col":"xs"}],"index":0},"last":{"fn":"listGet","args":[{"col":"xs"}],"index":-1},"oob":{"fn":"listGet","args":[{"col":"xs"}],"index":5},"sum":{"fn":"listSum","args":[{"col":"xs"}]},"mean":{"fn":"listMean","args":[{"col":"xs"}]},"min":{"fn":"listMin","args":[{"col":"xs"}]},"max":{"fn":"listMax","args":[{"col":"ws"}]},"has2":{"fn":"listContains","args":[{"col":"xs"},{"lit":2}]},"hasA":{"fn":"listContains","args":[{"col":"ws"},{"lit":"a"}]}}}],"expected":{"fields":[{"name":"n","type":"u32"},{"name":"first","type":"i32"},{"name":"last","type":"i32"},{"name":"oob","type":"i32"},{"name":"sum","type":"i32"},{"name":"mean","type":"f64"},{"name":"min","type":"i32"},{"name":"max","type":"utf8"},{"name":"has2","type":"bool"},{"name":"hasA","type":"bool"}],"columns":{"n":[3,0,null,3,1],"first":[1,null,null,4,7],"last":[3,null,null,2,7],"oob":[null,null,null,null,null],"sum":[6,0,null,6,7],"mean":[2.0,null,null,3.0,7.0],"min":[1,null,null,2,7],"max":["b","c",null,null,"z"],"has2":[true,false,null,true,false],"hasA":[true,false,false,null,true]}}},{"id":"nested/struct","description":"struct fields and struct construction","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"g","type":"i32"},{"name":"xs","type":"list"},{"name":"ws","type":"list"},{"name":"s","type":"struct"}],"columns":{"k":["a","b","c","d","e"],"g":[1,2,1,2,1],"xs":[[1,2,3],[],null,[4,null,2],[7]],"ws":[["a","b"],["c"],[],null,["z","a"]],"s":[{"a":1,"b":"x"},{"a":null,"b":"y"},null,{"a":4,"b":null},{"a":5,"b":"z"}]}},"right":{"fields":[{"name":"k","type":"utf8"},{"name":"tags","type":"list"}],"columns":{"k":["a","d","x"],"tags":[["t1"],["t2","t3"],[]]}}},"ops":[{"op":"select","exprs":{"a":{"fn":"field","args":[{"col":"s"}],"field":"a"},"b":{"fn":"field","args":[{"col":"s"}],"field":"b"},"pair":{"fn":"struct","args":[{"col":"k"},{"fn":"listLength","args":[{"col":"xs"}]}],"names":["key","size"]}}}],"expected":{"fields":[{"name":"a","type":"i32"},{"name":"b","type":"utf8"},{"name":"pair","type":"struct"}],"columns":{"a":[1,null,null,4,5],"b":["x","y",null,null,"z"],"pair":[{"key":"a","size":3},{"key":"b","size":0},{"key":"c","size":null},{"key":"d","size":3},{"key":"e","size":1}]}}},{"id":"nested/explode","description":"explode lists into rows; a null list gives a null row, an empty list no rows","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"g","type":"i32"},{"name":"xs","type":"list"},{"name":"ws","type":"list"},{"name":"s","type":"struct"}],"columns":{"k":["a","b","c","d","e"],"g":[1,2,1,2,1],"xs":[[1,2,3],[],null,[4,null,2],[7]],"ws":[["a","b"],["c"],[],null,["z","a"]],"s":[{"a":1,"b":"x"},{"a":null,"b":"y"},null,{"a":4,"b":null},{"a":5,"b":"z"}]}},"right":{"fields":[{"name":"k","type":"utf8"},{"name":"tags","type":"list"}],"columns":{"k":["a","d","x"],"tags":[["t1"],["t2","t3"],[]]}}},"ops":[{"op":"pick","columns":["k","xs"]},{"op":"explode","column":"xs"}],"expected":{"fields":[{"name":"k","type":"utf8"},{"name":"xs","type":"i32"}],"columns":{"k":["a","a","a","c","d","d","d","e"],"xs":[1,2,3,null,4,null,2,7]}}},{"id":"nested/unnest","description":"unnest a struct into columns","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"g","type":"i32"},{"name":"xs","type":"list"},{"name":"ws","type":"list"},{"name":"s","type":"struct"}],"columns":{"k":["a","b","c","d","e"],"g":[1,2,1,2,1],"xs":[[1,2,3],[],null,[4,null,2],[7]],"ws":[["a","b"],["c"],[],null,["z","a"]],"s":[{"a":1,"b":"x"},{"a":null,"b":"y"},null,{"a":4,"b":null},{"a":5,"b":"z"}]}},"right":{"fields":[{"name":"k","type":"utf8"},{"name":"tags","type":"list"}],"columns":{"k":["a","d","x"],"tags":[["t1"],["t2","t3"],[]]}}},"ops":[{"op":"pick","columns":["k","s"]},{"op":"unnest","column":"s"}],"expected":{"fields":[{"name":"k","type":"utf8"},{"name":"a","type":"i32"},{"name":"b","type":"utf8"}],"columns":{"k":["a","b","c","d","e"],"a":[1,null,null,4,5],"b":["x","y",null,null,"z"]}}},{"id":"nested/agg-list","description":"collect values into lists per group","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"g","type":"i32"},{"name":"xs","type":"list"},{"name":"ws","type":"list"},{"name":"s","type":"struct"}],"columns":{"k":["a","b","c","d","e"],"g":[1,2,1,2,1],"xs":[[1,2,3],[],null,[4,null,2],[7]],"ws":[["a","b"],["c"],[],null,["z","a"]],"s":[{"a":1,"b":"x"},{"a":null,"b":"y"},null,{"a":4,"b":null},{"a":5,"b":"z"}]}},"right":{"fields":[{"name":"k","type":"utf8"},{"name":"tags","type":"list"}],"columns":{"k":["a","d","x"],"tags":[["t1"],["t2","t3"],[]]}}},"ops":[{"op":"groupBy","keys":["g"],"aggs":{"ks":{"agg":"list","arg":{"col":"k"}},"firsts":{"agg":"first","arg":{"col":"xs"}},"n":{"agg":"len"}}}],"expected":{"fields":[{"name":"g","type":"i32"},{"name":"ks","type":"list"},{"name":"firsts","type":"list"},{"name":"n","type":"u32"}],"columns":{"g":[1,2],"ks":[["a","c","e"],["b","d"]],"firsts":[[1,2,3],[]],"n":[3,2]}}},{"id":"nested/carry","description":"lists and structs survive filters, sorts, joins and slices","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"g","type":"i32"},{"name":"xs","type":"list"},{"name":"ws","type":"list"},{"name":"s","type":"struct"}],"columns":{"k":["a","b","c","d","e"],"g":[1,2,1,2,1],"xs":[[1,2,3],[],null,[4,null,2],[7]],"ws":[["a","b"],["c"],[],null,["z","a"]],"s":[{"a":1,"b":"x"},{"a":null,"b":"y"},null,{"a":4,"b":null},{"a":5,"b":"z"}]}},"right":{"fields":[{"name":"k","type":"utf8"},{"name":"tags","type":"list"}],"columns":{"k":["a","d","x"],"tags":[["t1"],["t2","t3"],[]]}}},"ops":[{"op":"filter","predicate":{"fn":"neq","args":[{"col":"k"},{"lit":"c"}]}},{"op":"sort","keys":[{"expr":{"col":"g"},"descending":true},{"expr":{"col":"k"}}]},{"op":"join","right":"right","on":["k"],"how":"left"},{"op":"head","n":4}],"expected":{"fields":[{"name":"k","type":"utf8"},{"name":"g","type":"i32"},{"name":"xs","type":"list"},{"name":"ws","type":"list"},{"name":"s","type":"struct"},{"name":"tags","type":"list"}],"columns":{"k":["b","d","a","e"],"g":[2,2,1,1],"xs":[[],[4,null,2],[1,2,3],[7]],"ws":[["c"],null,["a","b"],["z","a"]],"s":[{"a":null,"b":"y"},{"a":4,"b":null},{"a":1,"b":"x"},{"a":5,"b":"z"}],"tags":[null,["t2","t3"],["t1"],null]}}},{"id":"decimal/arith","description":"decimal addition, subtraction and multiplication","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"price","type":"decimal[10,2]"},{"name":"fee","type":"decimal[6,3]"},{"name":"qty","type":"decimal[4,0]"},{"name":"x","type":"f64"}],"columns":{"k":["a","b","a","c","b","a"],"price":["12.34","0.10","-5.00",null,"99999999.99","10.01"],"fee":["0.125","1.000",null,"2.500","-0.001","10.010"],"qty":["3","10","2","1",null,"7"],"x":[1.005,-2.5,123.456,null,1000000.0,0.0]}}},"ops":[{"op":"select","exprs":{"sum":{"fn":"add","args":[{"col":"price"},{"col":"fee"}]},"diff":{"fn":"sub","args":[{"col":"price"},{"col":"fee"}]},"prod":{"fn":"mul","args":[{"col":"price"},{"col":"qty"}]},"plus":{"fn":"add","args":[{"col":"price"},{"lit":"0.5","type":"decimal[3,1]"}]}}}],"expected":{"fields":[{"name":"sum","type":"decimal[38,3]"},{"name":"diff","type":"decimal[38,3]"},{"name":"prod","type":"decimal[38,2]"},{"name":"plus","type":"decimal[38,2]"}],"columns":{"sum":["12.465","1.100",null,null,"99999999.989","20.020"],"diff":["12.215","-0.900",null,null,"99999999.991","0.000"],"prod":["37.02","1.00","-10.00",null,null,"70.07"],"plus":["12.84","0.60","-4.50",null,"100000000.49","10.51"]}}},{"id":"decimal/compare","description":"decimal comparisons, also across scales","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"price","type":"decimal[10,2]"},{"name":"fee","type":"decimal[6,3]"},{"name":"qty","type":"decimal[4,0]"},{"name":"x","type":"f64"}],"columns":{"k":["a","b","a","c","b","a"],"price":["12.34","0.10","-5.00",null,"99999999.99","10.01"],"fee":["0.125","1.000",null,"2.500","-0.001","10.010"],"qty":["3","10","2","1",null,"7"],"x":[1.005,-2.5,123.456,null,1000000.0,0.0]}}},"ops":[{"op":"filter","predicate":{"fn":"and","args":[{"fn":"gt","args":[{"col":"price"},{"lit":"10.00","type":"decimal[10,2]"}]},{"fn":"gte","args":[{"col":"price"},{"col":"fee"}]}]}}],"expected":{"fields":[{"name":"k","type":"utf8"},{"name":"price","type":"decimal[10,2]"},{"name":"fee","type":"decimal[6,3]"},{"name":"qty","type":"decimal[4,0]"},{"name":"x","type":"f64"}],"columns":{"k":["a","b","a"],"price":["12.34","99999999.99","10.01"],"fee":["0.125","-0.001","10.010"],"qty":["3",null,"7"],"x":[1.005,1000000.0,0.0]}}},{"id":"decimal/aggregate","description":"sums, extremes and means of decimals per group","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"price","type":"decimal[10,2]"},{"name":"fee","type":"decimal[6,3]"},{"name":"qty","type":"decimal[4,0]"},{"name":"x","type":"f64"}],"columns":{"k":["a","b","a","c","b","a"],"price":["12.34","0.10","-5.00",null,"99999999.99","10.01"],"fee":["0.125","1.000",null,"2.500","-0.001","10.010"],"qty":["3","10","2","1",null,"7"],"x":[1.005,-2.5,123.456,null,1000000.0,0.0]}}},"ops":[{"op":"groupBy","keys":["k"],"aggs":{"total":{"agg":"sum","arg":{"col":"price"}},"min":{"agg":"min","arg":{"col":"price"}},"max":{"agg":"max","arg":{"col":"fee"}},"mean":{"agg":"mean","arg":{"col":"price"}},"n":{"agg":"count","arg":{"col":"price"}}}}],"expected":{"fields":[{"name":"k","type":"utf8"},{"name":"total","type":"decimal[38,2]"},{"name":"min","type":"decimal[10,2]"},{"name":"max","type":"decimal[6,3]"},{"name":"mean","type":"f64"},{"name":"n","type":"u32"}],"columns":{"k":["a","b","c"],"total":["17.35","100000000.09","0.00"],"min":["-5.00","0.10",null],"max":["10.010","1.000","2.500"],"mean":[5.783333333333334,50000000.044999994,null],"n":[3,2,0]}}},{"id":"decimal/cast","description":"casts between decimals, floats, integers and strings","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"price","type":"decimal[10,2]"},{"name":"fee","type":"decimal[6,3]"},{"name":"qty","type":"decimal[4,0]"},{"name":"x","type":"f64"}],"columns":{"k":["a","b","a","c","b","a"],"price":["12.34","0.10","-5.00",null,"99999999.99","10.01"],"fee":["0.125","1.000",null,"2.500","-0.001","10.010"],"qty":["3","10","2","1",null,"7"],"x":[1.005,-2.5,123.456,null,1000000.0,0.0]}}},"ops":[{"op":"select","exprs":{"f":{"fn":"cast","args":[{"col":"price"}],"to":"f64"},"i":{"fn":"cast","args":[{"col":"price"}],"to":"i32"},"s":{"fn":"cast","args":[{"col":"fee"}],"to":"utf8"},"d":{"fn":"cast","args":[{"col":"x"}],"to":"decimal[10,2]"},"r":{"fn":"cast","args":[{"col":"price"}],"to":"decimal[12,1]"}}}],"expected":{"fields":[{"name":"f","type":"f64"},{"name":"i","type":"i32"},{"name":"s","type":"utf8"},{"name":"d","type":"decimal[10,2]"},{"name":"r","type":"decimal[12,1]"}],"columns":{"f":[12.34,0.1,-5.0,null,99999999.99,10.01],"i":[12,0,-5,null,99999999,10],"s":["0.125","1.000",null,"2.500","-0.001","10.010"],"d":["1.00","-2.50","123.46",null,"1000000.00","0.00"],"r":["12.3","0.1","-5.0",null,"100000000.0","10.0"]}}},{"id":"decimal/sort","description":"sort by a decimal","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"price","type":"decimal[10,2]"},{"name":"fee","type":"decimal[6,3]"},{"name":"qty","type":"decimal[4,0]"},{"name":"x","type":"f64"}],"columns":{"k":["a","b","a","c","b","a"],"price":["12.34","0.10","-5.00",null,"99999999.99","10.01"],"fee":["0.125","1.000",null,"2.500","-0.001","10.010"],"qty":["3","10","2","1",null,"7"],"x":[1.005,-2.5,123.456,null,1000000.0,0.0]}}},"ops":[{"op":"sort","keys":[{"expr":{"col":"price"},"descending":true}],"nullsLast":true}],"expected":{"fields":[{"name":"k","type":"utf8"},{"name":"price","type":"decimal[10,2]"},{"name":"fee","type":"decimal[6,3]"},{"name":"qty","type":"decimal[4,0]"},{"name":"x","type":"f64"}],"columns":{"k":["b","a","a","b","a","c"],"price":["99999999.99","12.34","10.01","0.10","-5.00",null],"fee":["-0.001","0.125","10.010","1.000",null,"2.500"],"qty":[null,"3","7","10","2","1"],"x":[1000000.0,1.005,0.0,-2.5,123.456,null]}}},{"id":"decimal/round-ties","description":"narrowing the scale rounds exact ties","inputs":{"left":{"fields":[{"name":"t","type":"decimal[6,3]"}],"columns":{"t":["0.250","0.350","-0.250","-0.350","1.050","2.500","0.249"]}}},"ops":[{"op":"select","exprs":{"one":{"fn":"cast","args":[{"col":"t"}],"to":"decimal[6,1]"},"zero":{"fn":"cast","args":[{"col":"t"}],"to":"decimal[6,0]"}}}],"expected":{"fields":[{"name":"one","type":"decimal[6,1]"},{"name":"zero","type":"decimal[6,0]"}],"columns":{"one":["0.2","0.4","-0.2","-0.4","1.0","2.5","0.2"],"zero":["0","0","0","0","1","2","0"]}}}]} diff --git a/packages/effect/test/fixtures/dataframe/gold/random.json b/packages/effect/test/fixtures/dataframe/gold/random.json new file mode 100644 index 00000000000..e3aaaaed919 --- /dev/null +++ b/packages/effect/test/fixtures/dataframe/gold/random.json @@ -0,0 +1 @@ +{"generator":"generate_gold.py","polars":"2.1.0","deviations":["JavaScript has no float literal distinct from an integer: an integral number literal behaves like a Polars integer literal (it takes the integer type of the column it meets), so cases never combine an integer column with an integral float literal.","Float to string casts use JavaScript number formatting (Polars prints 5.0 as '5.0', JavaScript as '5').","Joins never add '_right' suffixes: overlapping non-key columns are a type error.","Arithmetic between 64-bit integers (bigint) and other numeric types requires an explicit cast."],"cases":[{"id":"random/0000","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i64","type":"i64"},{"name":"l3_i64","type":"i64"},{"name":"l4_bool","type":"bool"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_i64":[],"l3_i64":[],"l4_bool":[]}}},"ops":[{"op":"filter","predicate":{"fn":"lt","args":[{"fn":"mul","args":[{"col":"l1_i32"},{"lit":2.5}]},{"col":"l1_i32"}]}},{"op":"filter","predicate":{"fn":"eq","args":[{"fn":"abs","args":[{"lit":-1}]},{"col":"l1_i32"}]}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i64","type":"i64"},{"name":"l3_i64","type":"i64"},{"name":"l4_bool","type":"bool"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_i64":[],"l3_i64":[],"l4_bool":[]}}},{"id":"random/0001","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_f64","type":"f64"},{"name":"l4_bool","type":"bool"},{"name":"l5_f64","type":"f64"}],"columns":{"l0_utf8":["c"],"l1_i32":[-3],"l2_utf8":[""],"l3_f64":[63.689],"l4_bool":[true],"l5_f64":[-1.0]}}},"ops":[{"op":"filter","predicate":{"fn":"isIn","args":[{"col":"l2_utf8"}],"values":["c","B",""]}},{"op":"withColumns","exprs":{"x1":{"fn":"or","args":[{"fn":"or","args":[{"fn":"isIn","args":[{"col":"l0_utf8"}],"values":["😀","aa","zz"]},{"fn":"eq","args":[{"col":"l3_f64"},{"col":"l1_i32"}]}]},{"fn":"isNotNull","args":[{"col":"l2_utf8"}]}]}}},{"op":"groupBy","keys":["l4_bool"],"aggs":{"n2":{"agg":"len"},"a2_0":{"agg":"min","arg":{"col":"l3_f64"}},"a2_1":{"agg":"count","arg":{"col":"l5_f64"}},"a2_2":{"agg":"first","arg":{"col":"l5_f64"}}}},{"op":"unique","columns":["l4_bool"]}],"expected":{"fields":[{"name":"l4_bool","type":"bool"},{"name":"n2","type":"u32"},{"name":"a2_0","type":"f64"},{"name":"a2_1","type":"u32"},{"name":"a2_2","type":"f64"}],"columns":{"l4_bool":[true],"n2":[1],"a2_0":[63.689],"a2_1":[1],"a2_2":[-1.0]}}},{"id":"random/0002","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_f64","type":"f64"},{"name":"l4_f64","type":"f64"},{"name":"l5_bool","type":"bool"},{"name":"l6_bool","type":"bool"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_utf8":[],"l3_f64":[],"l4_f64":[],"l5_bool":[],"l6_bool":[]}}},"ops":[{"op":"groupBy","keys":["l5_bool"],"aggs":{"n0":{"agg":"len"},"a0_0":{"agg":"max","arg":{"col":"l4_f64"}}}}],"expected":{"fields":[{"name":"l5_bool","type":"bool"},{"name":"n0","type":"u32"},{"name":"a0_0","type":"f64"}],"columns":{"l5_bool":[],"n0":[],"a0_0":[]}}},{"id":"random/0003","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_utf8","type":"utf8"}],"columns":{"l0_utf8":["b"],"l1_i32":[-1],"l2_f64":[-4.0],"l3_utf8":["ab"],"l4_utf8":[null],"l5_utf8":["😀"]}},"right":{"fields":[{"name":"l4_utf8","type":"utf8"},{"name":"r0_bool","type":"bool"}],"columns":{"r0_bool":[null,null,null,true,false,false,true,true,null,null],"l4_utf8":[null,null,null,null,null,null,null,null,null,null]}}},"ops":[{"op":"join","right":"right","on":["l4_utf8"],"how":"semi"},{"op":"withColumns","exprs":{"x1":{"fn":"lt","args":[{"fn":"abs","args":[{"lit":0.5}]},{"fn":"neg","args":[{"col":"l2_f64"}]}]}}},{"op":"unique"}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_utf8","type":"utf8"},{"name":"x1","type":"bool"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_f64":[],"l3_utf8":[],"l4_utf8":[],"l5_utf8":[],"x1":[]}}},{"id":"random/0004","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_i64","type":"i64"},{"name":"l4_f64","type":"f64"},{"name":"l5_bool","type":"bool"},{"name":"l6_bool","type":"bool"}],"columns":{"l0_utf8":["aa","zz","😀","a","zz",null,"b","B","a","b","aa","b","ab","é",null,"b",null,"ab","b","é"],"l1_i32":[4,-3,-2,-2147483648,-4,-4,3,-4,null,-4,1,null,0,-2,4,5,0,0,5,4],"l2_bool":[true,false,false,true,true,false,true,false,null,true,false,false,false,true,true,true,false,true,false,true],"l3_i64":["4","5",null,"-4611686018427387904","9007199254740995",null,"-4611686018427387904","-4611686018427387904","-4611686018427387904","9007199254740993","9007199254740992","-5","3","9007199254740993","-1","9007199254740993","-2","-2","1","9007199254740995"],"l4_f64":[-89.526,0.0,-1.0,-0.645,2.0,1.0,-42.666,"NaN",-17.917,73.944,null,26.072,2.0,-15.736,null,"NaN",2.615,-3.0,3.0,0.0],"l5_bool":[false,true,false,false,false,true,true,true,true,true,true,true,false,false,false,true,true,false,false,null],"l6_bool":[true,false,false,false,false,false,true,false,true,true,true,true,false,true,false,true,false,false,true,true]}}},"ops":[{"op":"groupBy","keys":["l1_i32"],"aggs":{"n0":{"agg":"len"},"a0_0":{"agg":"first","arg":{"col":"l0_utf8"}},"a0_1":{"agg":"min","arg":{"col":"l3_i64"}},"a0_2":{"agg":"last","arg":{"col":"l0_utf8"}}}},{"op":"withColumns","exprs":{"x1":{"fn":"neg","args":[{"fn":"neg","args":[{"lit":-1}]}]}}},{"op":"head","n":9},{"op":"unique","columns":["a0_0"]}],"expected":{"fields":[{"name":"l1_i32","type":"i32"},{"name":"n0","type":"u32"},{"name":"a0_0","type":"utf8"},{"name":"a0_1","type":"i64"},{"name":"a0_2","type":"utf8"},{"name":"x1","type":"i32"}],"columns":{"l1_i32":[4,-3,-2,-2147483648,3,0],"n0":[3,1,2,1,1,3],"a0_0":["aa","zz","😀","a","b","ab"],"a0_1":["-1","5","9007199254740993","-4611686018427387904","-4611686018427387904","-2"],"a0_2":["é","zz","é","a","b","ab"],"x1":[-1,-1,-1,-1,-1,-1]}}},{"id":"random/0005","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_f64","type":"f64"},{"name":"l4_utf8","type":"utf8"}],"columns":{"l0_utf8":["b"],"l1_i32":[4],"l2_i32":[null],"l3_f64":[1.0],"l4_utf8":["😀"]}}},"ops":[{"op":"head","n":0},{"op":"head","n":2},{"op":"filter","predicate":{"fn":"neq","args":[{"fn":"floorDiv","args":[{"col":"l2_i32"},{"col":"l1_i32"}]},{"col":"l2_i32"}]}},{"op":"unique","columns":["l4_utf8"]}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_f64","type":"f64"},{"name":"l4_utf8","type":"utf8"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_i32":[],"l3_f64":[],"l4_utf8":[]}}},{"id":"random/0006","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_f64","type":"f64"},{"name":"l4_i32","type":"i32"}],"columns":{"l0_utf8":[null,null,"é",null,"😀","zz","B",null,null,"é","ab","c","B",null,"é","ab","b",null,null,"a"],"l1_i32":[-5,null,null,null,null,null,4,-3,-4,4,5,2,1,0,3,4,161,null,null,null],"l2_bool":[true,true,true,true,true,null,true,false,true,true,true,false,null,null,null,null,true,false,false,null],"l3_f64":[-4.0,2.0,null,3.0,null,22.128,-40.946,1.0,-3.0,-2.0,null,-74.816,-3.0,null,4.0,"NaN",2.0,-2.0,-12.995,69.001],"l4_i32":[null,null,1,4,3,null,null,null,2,1,3,3,1,null,null,-2,-1,null,340,3]}}},"ops":[{"op":"unique","columns":["l1_i32"]},{"op":"sort","keys":[{"expr":{"col":"l4_i32"},"descending":false},{"expr":{"col":"l1_i32"},"descending":true}],"nullsLast":false},{"op":"filter","predicate":{"fn":"lte","args":[{"col":"l4_i32"},{"col":"l1_i32"}]}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_f64","type":"f64"},{"name":"l4_i32","type":"i32"}],"columns":{"l0_utf8":["b","B","ab"],"l1_i32":[161,1,5],"l2_bool":[true,null,true],"l3_f64":[2.0,-3.0,null],"l4_i32":[-1,1,3]}}},{"id":"random/0007","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_i32","type":"i32"},{"name":"l4_f64","type":"f64"},{"name":"l5_utf8","type":"utf8"},{"name":"l6_i32","type":"i32"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_bool":[],"l3_i32":[],"l4_f64":[],"l5_utf8":[],"l6_i32":[]}}},"ops":[{"op":"sort","keys":[{"expr":{"col":"l6_i32"},"descending":true}],"nullsLast":false}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_i32","type":"i32"},{"name":"l4_f64","type":"f64"},{"name":"l5_utf8","type":"utf8"},{"name":"l6_i32","type":"i32"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_bool":[],"l3_i32":[],"l4_f64":[],"l5_utf8":[],"l6_i32":[]}}},{"id":"random/0008","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_i32","type":"i32"},{"name":"l4_bool","type":"bool"}],"columns":{"l0_utf8":["zz","b","B",null,null],"l1_i32":[-4,2,null,0,-1],"l2_utf8":["c","c",null,null,"é"],"l3_i32":[1,-1,1,-2147483648,5],"l4_bool":[null,false,false,false,true]}}},"ops":[{"op":"unique"},{"op":"withColumns","exprs":{"x1":{"lit":0.5}}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_i32","type":"i32"},{"name":"l4_bool","type":"bool"},{"name":"x1","type":"f64"}],"columns":{"l0_utf8":["zz","b","B",null,null],"l1_i32":[-4,2,null,0,-1],"l2_utf8":["c","c",null,null,"é"],"l3_i32":[1,-1,1,-2147483648,5],"l4_bool":[null,false,false,false,true],"x1":[0.5,0.5,0.5,0.5,0.5]}}},{"id":"random/0009","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_f64","type":"f64"},{"name":"l4_f64","type":"f64"}],"columns":{"l0_utf8":["","c","zz",null,"é"],"l1_i32":[-3,1,-5,-3,-2],"l2_utf8":["c","😀",null,"zz","a"],"l3_f64":[null,-44.767,3.0,-55.17,"Infinity"],"l4_f64":[-97.678,4.0,24.637,null,null]}},"right":{"fields":[{"name":"l2_utf8","type":"utf8"},{"name":"r0_utf8","type":"utf8"}],"columns":{"r0_utf8":["B","","zz","","c","b","aa","a","é","c","B","B","a","é","B","","ab","B","zz",null,"é","é","","aa","c","c","😀","B","😀","ab"],"l2_utf8":["a","zz","c",null,"😀","a","😀","zz","c","c",null,null,null,"zz","zz","a","a","zz","c","c","😀","😀","😀","a",null,"😀","a","c","😀",null]}}},"ops":[{"op":"join","right":"right","on":["l2_utf8"],"how":"left"},{"op":"groupBy","keys":["l2_utf8"],"aggs":{"n1":{"agg":"len"},"a1_0":{"agg":"count","arg":{"col":"l4_f64"}},"a1_1":{"agg":"median","arg":{"col":"l3_f64"}},"a1_2":{"agg":"last","arg":{"col":"l1_i32"}}}}],"expected":{"fields":[{"name":"l2_utf8","type":"utf8"},{"name":"n1","type":"u32"},{"name":"a1_0","type":"u32"},{"name":"a1_1","type":"f64"},{"name":"a1_2","type":"i32"}],"columns":{"l2_utf8":["c","😀",null,"zz","a"],"n1":[6,7,1,5,6],"a1_0":[6,7,1,0,0],"a1_1":[null,-44.767,3.0,-55.17,"Infinity"],"a1_2":[-3,1,-5,-3,-2]}}},{"id":"random/0010","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i64","type":"i64"},{"name":"l3_i64","type":"i64"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_i32","type":"i32"}],"columns":{"l0_utf8":["","","b","😀","c","zz","aa","b","ab","zz","b","B","B","","a","é","zz","é","a","é"],"l1_i32":[-488,null,null,-4,5,-2,null,null,null,1,-5,-4,1,-3,-2,-3,-2,-5,-4,5],"l2_i64":["9007199254740993","-5","-3","9007199254740993","9007199254740994","-4611686018427387904","-4611686018427387904","9007199254740993","-4611686018427387904",null,"-3","-1","1","0","0","-1","2","9007199254740994","5","-4611686018427387904"],"l3_i64":["9007199254740995","-4611686018427387904","0","9007199254740993","-4611686018427387904","-4611686018427387904","-1","-4611686018427387904","-4611686018427387904","9007199254740992","9007199254740995","-4611686018427387904","-4611686018427387904","9007199254740994","-4611686018427387904","5","3","-4611686018427387904","9007199254740993","9007199254740994"],"l4_utf8":["aa","zz","😀","é","","é","B","B","b","",null,"zz","ab","B","b","é","é",null,"B","aa"],"l5_i32":[null,null,-3,3,2,0,3,1,-3,0,null,4,-2,4,null,2,-1,0,3,5]}}},"ops":[{"op":"groupBy","keys":["l5_i32"],"aggs":{"n0":{"agg":"len"},"a0_0":{"agg":"mean","arg":{"col":"l1_i32"}},"a0_1":{"agg":"first","arg":{"col":"l3_i64"}},"a0_2":{"agg":"mean","arg":{"col":"l1_i32"}}}},{"op":"withColumns","exprs":{"x1":{"col":"a0_0"}}},{"op":"groupBy","keys":["l5_i32"],"aggs":{"n2":{"agg":"len"},"a2_0":{"agg":"min","arg":{"col":"n0"}},"a2_1":{"agg":"min","arg":{"col":"a0_2"}}}},{"op":"sort","keys":[{"expr":{"col":"a2_1"},"descending":false},{"expr":{"col":"l5_i32"},"descending":false}],"nullsLast":false}],"expected":{"fields":[{"name":"l5_i32","type":"i32"},{"name":"n2","type":"u32"},{"name":"a2_0","type":"u32"},{"name":"a2_1","type":"f64"}],"columns":{"l5_i32":[-3,1,null,3,4,-1,0,-2,2,5],"n2":[1,1,1,1,1,1,1,1,1,1],"a2_0":[2,1,4,3,2,1,3,1,2,1],"a2_1":[null,null,-165.0,-4.0,-3.5,-2.0,-2.0,1.0,1.0,5.0]}}},{"id":"random/0011","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_i64","type":"i64"}],"columns":{"l0_utf8":["c","a","","B","a"],"l1_i32":[2,2,2,-4,-4],"l2_utf8":["ab","B","b","","c"],"l3_i64":["-4","-4611686018427387904","2","-3","9007199254740994"]}}},"ops":[{"op":"withColumns","exprs":{"x0":{"col":"l1_i32"}}},{"op":"withColumns","exprs":{"x1":{"fn":"gte","args":[{"col":"x0"},{"lit":-1}]}}},{"op":"groupBy","keys":["x1"],"aggs":{"n2":{"agg":"len"},"a2_0":{"agg":"std","arg":{"col":"l1_i32"}},"a2_1":{"agg":"sum","arg":{"col":"l1_i32"}}}},{"op":"withColumns","exprs":{"x3":{"fn":"abs","args":[{"fn":"fillNull","args":[{"col":"a2_1"},{"lit":0}]}]}}}],"expected":{"fields":[{"name":"x1","type":"bool"},{"name":"n2","type":"u32"},{"name":"a2_0","type":"f64"},{"name":"a2_1","type":"i32"},{"name":"x3","type":"i32"}],"columns":{"x1":[true,false],"n2":[3,2],"a2_0":[0.0,0.0],"a2_1":[6,-8],"x3":[6,8]}}},{"id":"random/0012","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i64","type":"i64"},{"name":"l3_bool","type":"bool"},{"name":"l4_bool","type":"bool"},{"name":"l5_f64","type":"f64"},{"name":"l6_i64","type":"i64"}],"columns":{"l0_utf8":["aa","b","aa","ab","b"],"l1_i32":[5,-1,2,-1,null],"l2_i64":["-4611686018427387904","-4611686018427387904","4","-4611686018427387904","9007199254740995"],"l3_bool":[false,true,true,true,null],"l4_bool":[null,false,null,false,false],"l5_f64":[-62.07,0.0,-85.177,-32.354,53.167],"l6_i64":["-1","-4611686018427387904","-3","0","9007199254740995"]}}},"ops":[{"op":"head","n":7}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i64","type":"i64"},{"name":"l3_bool","type":"bool"},{"name":"l4_bool","type":"bool"},{"name":"l5_f64","type":"f64"},{"name":"l6_i64","type":"i64"}],"columns":{"l0_utf8":["aa","b","aa","ab","b"],"l1_i32":[5,-1,2,-1,null],"l2_i64":["-4611686018427387904","-4611686018427387904","4","-4611686018427387904","9007199254740995"],"l3_bool":[false,true,true,true,null],"l4_bool":[null,false,null,false,false],"l5_f64":[-62.07,0.0,-85.177,-32.354,53.167],"l6_i64":["-1","-4611686018427387904","-3","0","9007199254740995"]}}},{"id":"random/0013","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_bool","type":"bool"}],"columns":{"l0_utf8":["aa","c","aa","zz","aa"],"l1_i32":[-4,-3,4,0,5],"l2_utf8":["zz","é","é","c","a"],"l3_bool":[true,true,false,false,false]}}},"ops":[{"op":"withColumns","exprs":{"x0":{"col":"l1_i32"}}},{"op":"head","n":0}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_bool","type":"bool"},{"name":"x0","type":"i32"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_utf8":[],"l3_bool":[],"x0":[]}}},{"id":"random/0014","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_utf8","type":"utf8"}],"columns":{"l0_utf8":["a"],"l1_i32":[3],"l2_f64":[1.0],"l3_utf8":["B"],"l4_utf8":["c"]}}},"ops":[{"op":"sort","keys":[{"expr":{"col":"l2_f64"},"descending":true},{"expr":{"col":"l3_utf8"},"descending":false}],"nullsLast":true}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_utf8","type":"utf8"}],"columns":{"l0_utf8":["a"],"l1_i32":[3],"l2_f64":[1.0],"l3_utf8":["B"],"l4_utf8":["c"]}}},{"id":"random/0015","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_i64","type":"i64"},{"name":"l4_i32","type":"i32"}],"columns":{"l0_utf8":["ab","c","é","😀","aa","zz","aa","aa","","","B","a","zz","c","ab","😀","B","b","c","aa"],"l1_i32":[5,-2,3,-4,-2147483648,-4,-1,-3,4,1,4,2,4,-2,5,-2,-4,2,0,4],"l2_utf8":["B","zz","","zz","😀","","é","B","c","ab","😀","é","aa","","zz","b","B","é","é","a"],"l3_i64":["9007199254740995","-4611686018427387904","9007199254740995","9007199254740995","9007199254740993","-1","-4611686018427387904","-4","-4611686018427387904","9007199254740995","-4611686018427387904","9007199254740992","4","-4611686018427387904","9007199254740993","2","-4611686018427387904","-2","9007199254740994","9007199254740994"],"l4_i32":[-5,0,0,1,4,5,5,2,-5,-3,5,1,-2,2147483647,-3,3,-475,0,5,4]}},"right":{"fields":[{"name":"l1_i32","type":"i32"},{"name":"r0_bool","type":"bool"}],"columns":{"r0_bool":[],"l1_i32":[]}}},"ops":[{"op":"head","n":3},{"op":"join","right":"right","on":["l1_i32"],"how":"left"}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_i64","type":"i64"},{"name":"l4_i32","type":"i32"},{"name":"r0_bool","type":"bool"}],"columns":{"l0_utf8":["ab","c","é"],"l1_i32":[5,-2,3],"l2_utf8":["B","zz",""],"l3_i64":["9007199254740995","-4611686018427387904","9007199254740995"],"l4_i32":[-5,0,0],"r0_bool":[null,null,null]}}},{"id":"random/0016","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i64","type":"i64"},{"name":"l3_i64","type":"i64"},{"name":"l4_i32","type":"i32"},{"name":"l5_i32","type":"i32"},{"name":"l6_i32","type":"i32"}],"columns":{"l0_utf8":["","aa",null,"B","a"],"l1_i32":[-3,0,null,null,null],"l2_i64":["9007199254740992","4","9007199254740993","-4611686018427387904","9007199254740992"],"l3_i64":["9007199254740993","4",null,"9007199254740994",null],"l4_i32":[0,-4,-1,null,2],"l5_i32":[5,-2,-1,null,1],"l6_i32":[null,5,null,null,-1]}}},"ops":[{"op":"groupBy","keys":["l1_i32"],"aggs":{"n0":{"agg":"len"},"a0_0":{"agg":"std","arg":{"col":"l4_i32"}},"a0_1":{"agg":"count","arg":{"col":"l0_utf8"}}}},{"op":"head","n":3},{"op":"filter","predicate":{"fn":"and","args":[{"fn":"or","args":[{"fn":"isNull","args":[{"col":"l1_i32"}]},{"fn":"isNull","args":[{"col":"n0"}]}]},{"fn":"isNotNull","args":[{"col":"a0_1"}]}]}},{"op":"filter","predicate":{"fn":"isNotNull","args":[{"col":"l1_i32"}]}}],"expected":{"fields":[{"name":"l1_i32","type":"i32"},{"name":"n0","type":"u32"},{"name":"a0_0","type":"f64"},{"name":"a0_1","type":"u32"}],"columns":{"l1_i32":[],"n0":[],"a0_0":[],"a0_1":[]}}},{"id":"random/0017","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_bool","type":"bool"},{"name":"l4_bool","type":"bool"},{"name":"l5_utf8","type":"utf8"}],"columns":{"l0_utf8":["aa","zz","",null,"😀"],"l1_i32":[null,0,869,1,-4],"l2_f64":[4.0,46.085,-53.295,4.0,-43.418],"l3_bool":[true,false,true,true,true],"l4_bool":[false,false,false,true,false],"l5_utf8":["b",null,"😀","😀","zz"]}}},"ops":[{"op":"filter","predicate":{"fn":"eq","args":[{"fn":"sub","args":[{"col":"l1_i32"},{"lit":2}]},{"col":"l2_f64"}]}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_bool","type":"bool"},{"name":"l4_bool","type":"bool"},{"name":"l5_utf8","type":"utf8"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_f64":[],"l3_bool":[],"l4_bool":[],"l5_utf8":[]}}},{"id":"random/0018","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_utf8","type":"utf8"}],"columns":{"l0_utf8":["c","ab","B","c","zz",null,"aa","é","B","a","zz","b","ab","ab","","b","c","b","a","zz","ab","B","é","a","B","c","","",null,"😀","b","c","b","b","ab","zz","é","ab","ab","aa",null,"B","B","aa","é","é","","","😀","B","","","ab","B","é","","B","a","","b"],"l1_i32":[-1,2147483647,4,-1,-4,5,0,5,-5,-1,1,2,1,-3,-4,0,1,-4,4,1,null,5,-2,-2,5,2,1,2,null,-1,2,null,-3,-2,-1,3,null,2147483647,-2147483648,1,-5,3,4,-5,0,-5,4,-5,-5,3,-7,1,-2,0,-2,-2,-5,3,1,1],"l2_utf8":["é","ab","a","ab",null,"B","b","a","ab","b","b","é","😀","c",null,"b",null,null,"b","zz","","","é","😀","b",null,"😀","b","é","é","c","a","","a","c","b","é","","ab","",null,"",null,"b","zz","b","","zz","é",null,"é","B","é","b","b","zz","zz","B","zz",""],"l3_utf8":["😀","😀","ab","b","c",null,"c","a",null,"b",null,"B","😀","","😀","ab","é","c","b","😀","zz","","aa","a","b","B","","😀","b","é","ab","aa",null,"ab","é","a","é","é",null,null,"","😀","aa","a","aa",null,"é","zz",null,"a","aa","","😀",null,"é","c","c",null,"é","b"],"l4_utf8":["é","b","é","","a",null,"","😀","é","b","c","zz","😀","aa","😀","ab",null,"c","😀","é","c","b",null,"😀","b","c","é","","c","c","😀","aa","b","é","","aa","zz","é","zz","","aa","c","é","",null,null,null,"ab","😀","b","é",null,"é","a","a","","a","ab","","ab"]}}},"ops":[{"op":"filter","predicate":{"fn":"and","args":[{"fn":"neq","args":[{"lit":0.5},{"lit":2.5}]},{"fn":"isIn","args":[{"col":"l2_utf8"}],"values":["a","c","é"]}]}},{"op":"unique"},{"op":"head","n":0},{"op":"sort","keys":[{"expr":{"col":"l1_i32"},"descending":false},{"expr":{"col":"l2_utf8"},"descending":true}],"nullsLast":false}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_utf8","type":"utf8"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_utf8":[],"l3_utf8":[],"l4_utf8":[]}}},{"id":"random/0019","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_f64","type":"f64"},{"name":"l4_f64","type":"f64"},{"name":"l5_i32","type":"i32"},{"name":"l6_i64","type":"i64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_utf8":[],"l3_f64":[],"l4_f64":[],"l5_i32":[],"l6_i64":[]}}},"ops":[{"op":"head","n":6},{"op":"filter","predicate":{"fn":"and","args":[{"fn":"strStartsWith","args":[{"col":"l2_utf8"}],"prefix":""},{"fn":"lt","args":[{"col":"l3_f64"},{"lit":2.5}]}]}},{"op":"withColumns","exprs":{"x2":{"fn":"gt","args":[{"fn":"when","args":[{"fn":"strStartsWith","args":[{"col":"l0_utf8"}],"prefix":"é"},{"col":"l5_i32"},{"col":"l5_i32"}]},{"col":"l3_f64"}]}}},{"op":"withColumns","exprs":{"x3":{"fn":"strStartsWith","args":[{"col":"l0_utf8"}],"prefix":"a"}}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_f64","type":"f64"},{"name":"l4_f64","type":"f64"},{"name":"l5_i32","type":"i32"},{"name":"l6_i64","type":"i64"},{"name":"x2","type":"bool"},{"name":"x3","type":"bool"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_utf8":[],"l3_f64":[],"l4_f64":[],"l5_i32":[],"l6_i64":[],"x2":[],"x3":[]}}},{"id":"random/0020","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_f64","type":"f64"},{"name":"l4_i64","type":"i64"},{"name":"l5_f64","type":"f64"}],"columns":{"l0_utf8":["a","b","é","b","é","aa","b","","é","b","ab","","é","zz","a","aa","aa","c","","","😀","","B","😀","a","c","aa","a","b","aa","c","","é","c","B","","é","b","B","aa","B","c","é","b","b","é","é","","","aa","é","é","","😀","é","B","","","aa",""],"l1_i32":[2,4,0,2147483647,948,3,4,-2,-1,3,0,-2,3,-2,3,-1,2,-3,1,0,0,5,1,-2,-3,0,-4,-5,1,-1,5,-4,-3,1,5,2,-5,-4,-2,-1,0,4,5,0,4,5,-4,0,2,2147483647,0,-3,0,-1,-3,428,-3,-2,-5,2],"l2_utf8":["😀","B","zz","b","zz","b","😀","b","😀","B","zz","c","😀","","😀","é","b","é","","zz","ab","B","B","aa","b","b","","zz","","a","b","a","ab","é","","a","a","aa","","c","b","","😀","a","ab","ab","😀","c","a","zz","","aa","zz","B","","😀","é","😀","a","b"],"l3_f64":[3.0,1.0,-2.0,4.0,97.251,19.689,4.0,"NaN",-3.0,1.0,3.0,-73.663,0.0,52.468,-4.0,-14.157,89.235,"NaN",1.0,-3.0,55.601,-56.044,2.0,-86.905,-3.0,4.985,-3.0,1.0,0.0,75.342,3.0,-2.0,-4.0,-8.861,-2.0,"NaN",80.403,3.0,-2.0,93.017,-15.641,87.214,-1.0,52.961,4.0,-81.853,2.0,69.03,-2.0,1.0,-3.0,9.047,-90.203,-3.0,3.0,0.0,2.0,3.0,"-0",19.893],"l4_i64":["-4611686018427387904","-4611686018427387904","9007199254740994","9007199254740992","-5","-5","-4611686018427387904","9007199254740994","5","9007199254740993","-4611686018427387904","9007199254740995","9007199254740992","9007199254740995","0","4","-4611686018427387904","-4611686018427387904","5","9007199254740995","-3","-4611686018427387904","-4611686018427387904","9007199254740992","9007199254740995","2","-4611686018427387904","-4611686018427387904","9007199254740995","-4611686018427387904","-5","-4611686018427387904","9007199254740992","-4611686018427387904","-4611686018427387904","9007199254740995","-4611686018427387904","-4611686018427387904","-4611686018427387904","9007199254740993","-1","3","9007199254740995","1","9007199254740993","9007199254740995","-4611686018427387904","2","-4611686018427387904","2","-4","3","-5","-3","-4611686018427387904","2","9007199254740992","-4611686018427387904","-4","-3"],"l5_f64":[4.0,2.0,-17.865,58.447,3.0,-1.0,-12.94,"NaN",-78.234,-2.0,42.432,4.0,-3.0,90.222,65.581,-2.0,33.722,-3.0,-91.672,76.232,-2.0,-77.772,98.768,12.205,-82.769,-3.0,-63.61,-1.0,2.622,2.0,-14.858,34.225,"NaN",45.927,0.0,-3.0,-26.07,1.0,68.377,-9.065,-4.0,10.179,-98.477,-0.848,-61.907,1.0,71.479,-44.591,-37.657,"NaN",-45.997,-95.902,-4.0,-4.0,81.418,3.0,3.0,-8.697,0.0,-3.0]}}},"ops":[{"op":"filter","predicate":{"fn":"eq","args":[{"fn":"neg","args":[{"col":"l3_f64"}]},{"fn":"add","args":[{"lit":2},{"col":"l5_f64"}]}]}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_f64","type":"f64"},{"name":"l4_i64","type":"i64"},{"name":"l5_f64","type":"f64"}],"columns":{"l0_utf8":["","B"],"l1_i32":[-2,5],"l2_utf8":["b",""],"l3_f64":["NaN",-2.0],"l4_i64":["9007199254740994","-4611686018427387904"],"l5_f64":["NaN",0.0]}}},{"id":"random/0021","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_f64","type":"f64"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_utf8","type":"utf8"}],"columns":{"l0_utf8":["aa","é","zz","ab",null,"a","a","é","zz","é","a",null,null,"ab",null,"é","ab","","",null],"l1_i32":[-3,2,-2,-3,null,0,-1,-1,5,null,-2,-4,-2,-3,3,null,3,-2,-5,-2147483648],"l2_f64":[null,null,-56.018,null,-71.852,61.795,-4.0,null,1.0,1.0,-97.515,null,-2.0,3.0,25.17,null,null,null,0.0,3.0],"l3_f64":[null,-60.567,null,"-0",0.0,null,0.0,null,null,-0.895,3.0,88.531,null,null,51.913,-57.123,29.998,1.0,null,null],"l4_utf8":["a","aa","b","a",null,"a",null,"é","c","aa","B","a",null,"é","zz","😀","a","zz","é",null],"l5_utf8":["","a",null,null,"","aa",null,"zz","😀","b","b",null,"é","b","zz",null,null,"é","😀",null]}},"right":{"fields":[{"name":"l4_utf8","type":"utf8"},{"name":"r0_utf8","type":"utf8"}],"columns":{"r0_utf8":[null,"","zz","a",null,"c","b","c","c",null],"l4_utf8":["B",null,null,"a","b","é","é","😀",null,"b"]}}},"ops":[{"op":"withColumns","exprs":{"x0":{"fn":"fillNull","args":[{"col":"l1_i32"},{"lit":1.5}]}}},{"op":"sort","keys":[{"expr":{"col":"l4_utf8"},"descending":false},{"expr":{"col":"l3_f64"},"descending":false}],"nullsLast":true},{"op":"join","right":"right","on":["l4_utf8"],"how":"full"},{"op":"filter","predicate":{"fn":"and","args":[{"fn":"lt","args":[{"col":"l2_f64"},{"col":"l1_i32"}]},{"fn":"gte","args":[{"col":"l2_f64"},{"col":"l3_f64"}]}]}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_f64","type":"f64"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_utf8","type":"utf8"},{"name":"x0","type":"f64"},{"name":"r0_utf8","type":"utf8"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_f64":[],"l3_f64":[],"l4_utf8":[],"l5_utf8":[],"x0":[],"r0_utf8":[]}}},{"id":"random/0022","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_f64","type":"f64"},{"name":"l4_i32","type":"i32"}],"columns":{"l0_utf8":["aa","aa","a","ab","","ab","zz","aa","a","c","a",null,"😀","zz","ab","b","aa","B","é",null,"é","B","ab","zz","é","zz",null,null,"a","B","zz","a","B","","ab","zz","é","b","ab","a","a","a","é","ab","c","a","a",null,"","c","","aa","😀","a","aa","aa","😀","é",null,"aa"],"l1_i32":[-2,5,-2,5,4,4,1,-1,-1,4,-1,null,5,2,3,2,4,null,4,-1,null,4,1,2,4,4,-3,2,-2,-4,2,-2,4,-3,-5,1,-3,3,0,null,5,4,-4,2,5,-3,4,2,2,1,1,-2,0,0,1,0,-2,1,1,1],"l2_utf8":["ab","ab","B",null,"c",null,"zz","zz","aa","","b","😀",null,"zz","é","B","c","😀","c","zz","","B","zz","b","é","B","","c","c","é",null,"ab","😀","c","é","aa","a","b","B","B","a","aa","a","😀",null,"B","😀",null,"c","","","c","",null,"😀","B","zz","zz","ab","ab"],"l3_f64":[4.0,null,-3.0,-4.0,3.0,-2.0,-48.326,-75.891,-2.0,-2.0,-3.0,"NaN",-1.0,40.544,"NaN","-Infinity",77.27,-15.791,2.0,-3.0,1.0,3.0,-54.348,-36.805,-99.084,2.0,11.72,null,-24.091,-80.429,-3.0,2.0,null,-21.412,null,-2.0,-3.787,-6.581,-2.0,0.0,70.105,2.0,-49.222,62.704,2.076,-76.167,96.743,-1.0,-4.0,58.692,0.0,-4.0,-95.33,0.0,1.0,"NaN",-98.212,1.0,"Infinity",-26.467],"l4_i32":[4,-3,1,-4,-3,1,5,-3,5,2,4,0,4,3,0,-1,-3,-5,1,5,-5,0,5,-5,-4,2,5,-5,2147483647,-5,2,-5,-4,-3,2,3,-1,-2,2,-2,5,3,null,-5,null,-3,-2,0,3,-5,-5,-1,-5,5,null,1,2,5,5,2]}}},"ops":[{"op":"groupBy","keys":["l1_i32"],"aggs":{"n0":{"agg":"len"},"a0_0":{"agg":"max","arg":{"col":"l3_f64"}}}}],"expected":{"fields":[{"name":"l1_i32","type":"i32"},{"name":"n0","type":"u32"},{"name":"a0_0","type":"f64"}],"columns":{"l1_i32":[-2,5,4,1,-1,null,2,3,-3,-4,-5,0],"n0":[6,5,11,9,4,4,8,2,4,2,1,4],"a0_0":[4.0,70.105,96.743,"Infinity",-2.0,1.0,62.704,-6.581,11.72,-49.222,null,0.0]}}},{"id":"random/0023","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i64","type":"i64"},{"name":"l3_i32","type":"i32"}],"columns":{"l0_utf8":["c","","b","😀","zz","ab","a","a","c","b","a","b","zz","a","","é","b","ab","😀","😀","b","","c","B","a","é","😀","aa","","b","a","aa","zz","c","zz","B","ab","ab","B","a","B","a","a","aa","b","ab","é","ab","zz","c","b","aa","aa","c","B","c","é","aa","","c"],"l1_i32":[-2147483648,-5,-2,-4,-2,-4,-1,-4,0,0,0,-2,5,2,0,-4,5,4,-1,0,-2,4,0,5,1,2,1,4,2,-4,-2,-3,-3,3,4,1,3,-2,0,-1,1,-3,-4,-3,0,0,-4,4,-5,1,-4,-3,3,-2,2,-4,2,0,-3,2],"l2_i64":["-4","9007199254740992","-4611686018427387904","-1","9007199254740993","-4611686018427387904","-1","3","-1","-4611686018427387904","4","9007199254740992","-3","1","-4611686018427387904","5","-2","-2","-4611686018427387904","4","-2","-4611686018427387904","5","5","-4611686018427387904","-4611686018427387904","-1","3","9007199254740993","9007199254740992","-4611686018427387904","5","-5","-4611686018427387904","-4611686018427387904","5","9007199254740994","9007199254740992","-4611686018427387904","9007199254740995","9007199254740995","-1","-3","1","9007199254740992","-4","9007199254740992","9007199254740992","-5","-4611686018427387904","1","-4611686018427387904","-1","9007199254740993","-4611686018427387904","9007199254740994","9007199254740993","-4","9007199254740995","-2"],"l3_i32":[4,-2,4,-3,2147483647,-3,0,4,4,5,5,-3,-3,0,-1,0,5,-4,-1,4,-5,-3,1,-5,-4,2,-3,5,4,2147483647,1,0,-4,-1,3,-3,4,4,3,3,1,5,-3,-5,-3,-2147483648,1,5,1,0,2,-5,1,0,2,5,2,-3,3,5]}},"right":{"fields":[{"name":"l2_i64","type":"i64"},{"name":"r0_utf8","type":"utf8"}],"columns":{"r0_utf8":["😀","a","B"],"l2_i64":["-4","9007199254740992","9007199254740992"]}}},"ops":[{"op":"head","n":1},{"op":"withColumns","exprs":{"x1":{"fn":"mul","args":[{"fn":"mod","args":[{"col":"l1_i32"},{"col":"l1_i32"}]},{"fn":"sub","args":[{"lit":2},{"col":"l3_i32"}]}]}}},{"op":"filter","predicate":{"fn":"gt","args":[{"fn":"when","args":[{"fn":"neq","args":[{"col":"l3_i32"},{"col":"l1_i32"}]},{"col":"l3_i32"},{"col":"l3_i32"}]},{"fn":"div","args":[{"col":"x1"},{"col":"l1_i32"}]}]}},{"op":"join","right":"right","on":["l2_i64"],"how":"full"}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i64","type":"i64"},{"name":"l3_i32","type":"i32"},{"name":"x1","type":"i32"},{"name":"r0_utf8","type":"utf8"}],"columns":{"l0_utf8":["c",null,null],"l1_i32":[-2147483648,null,null],"l2_i64":["-4","9007199254740992","9007199254740992"],"l3_i32":[4,null,null],"x1":[0,null,null],"r0_utf8":["😀","a","B"]}}},{"id":"random/0024","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_f64","type":"f64"}],"columns":{"l0_utf8":[null],"l1_i32":[-5],"l2_f64":[53.062],"l3_f64":[-4.0]}}},"ops":[{"op":"filter","predicate":{"fn":"and","args":[{"fn":"strStartsWith","args":[{"col":"l0_utf8"}],"prefix":"a"},{"fn":"eq","args":[{"col":"l3_f64"},{"col":"l3_f64"}]}]}},{"op":"unique","columns":["l0_utf8"]},{"op":"filter","predicate":{"fn":"lte","args":[{"fn":"mod","args":[{"col":"l1_i32"},{"col":"l3_f64"}]},{"fn":"sub","args":[{"col":"l1_i32"},{"col":"l3_f64"}]}]}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_f64","type":"f64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_f64":[],"l3_f64":[]}}},{"id":"random/0025","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_f64","type":"f64"},{"name":"l5_i32","type":"i32"}],"columns":{"l0_utf8":[null,"zz","é","B","b","B","B",null,"aa","b","","b","zz",null,"😀",null,"",null,"a","zz","c","😀",null,null,null,null,"B","B","zz","c","😀",null,null,"","a","😀","ab","B","😀","ab",null,"a","ab","c","😀","zz","ab",null,"zz","c",null,null,null,null,"","a","",null,null,"zz"],"l1_i32":[null,-4,null,null,-4,1,null,null,null,null,-4,null,2147483647,1,2,3,3,null,null,-2,2,2147483647,1,-2,-1,2147483647,3,120,null,5,0,3,5,null,5,2,null,-3,null,null,-5,null,3,4,null,-4,3,0,-4,948,2,2,null,5,null,null,2,-5,null,-1],"l2_bool":[true,null,false,true,true,false,null,null,true,null,true,false,null,null,null,true,true,false,false,true,false,false,false,true,false,false,true,null,true,false,false,true,false,true,true,true,null,null,true,false,null,true,true,true,false,true,null,null,false,false,null,null,true,false,false,false,true,null,null,true],"l3_utf8":[null,"c","B",null,"é","é","c",null,null,null,"",null,"B","😀",null,"ab","","é","B","c","aa","B","b","zz","B","zz","😀","aa","",null,"😀",null,null,"😀","",null,null,"","aa","zz","é",null,"aa",null,null,null,"B",null,"😀","aa","😀","B","ab","","a","","a",null,"é",""],"l4_f64":[1.0,null,29.922,null,-4.0,38.289,null,-3.0,47.375,2.0,58.277,null,null,50.086,-53.776,-23.472,-91.921,null,null,57.871,null,null,-20.056,-90.534,-62.375,null,null,39.949,28.397,-74.628,15.051,null,-73.49,-38.607,null,-63.471,null,-1.0,3.0,13.435,null,78.442,1.0,35.901,null,66.921,null,-50.246,2.0,-2.0,2.0,99.447,null,null,null,null,null,45.355,2.0,4.0],"l5_i32":[3,0,null,2,-3,null,null,-4,-3,null,4,4,-5,-5,5,1,null,-4,1,-5,2,4,null,634,3,2,-1,-4,-1,null,-5,-3,null,5,null,-4,1,4,4,null,3,1,-2,2,null,null,3,null,-2,-3,null,-578,-2,3,-2,4,-2,4,null,0]}},"right":{"fields":[{"name":"l1_i32","type":"i32"},{"name":"r0_utf8","type":"utf8"}],"columns":{"r0_utf8":["zz","c","B"],"l1_i32":[null,null,5]}}},"ops":[{"op":"join","right":"right","on":["l1_i32"],"how":"semi"},{"op":"filter","predicate":{"fn":"or","args":[{"col":"l2_bool"},{"fn":"and","args":[{"fn":"eq","args":[{"col":"l4_f64"},{"col":"l4_f64"}]},{"fn":"lte","args":[{"lit":0.5},{"lit":2.5}]}]}]}},{"op":"withColumns","exprs":{"x2":{"col":"l5_i32"}}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_f64","type":"f64"},{"name":"l5_i32","type":"i32"},{"name":"x2","type":"i32"}],"columns":{"l0_utf8":["c",null,"a"],"l1_i32":[5,5,5],"l2_bool":[false,false,true],"l3_utf8":[null,null,""],"l4_f64":[-74.628,-73.49,null],"l5_i32":[null,null,null],"x2":[null,null,null]}}},{"id":"random/0026","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_i64","type":"i64"},{"name":"l4_bool","type":"bool"}],"columns":{"l0_utf8":["aa","😀","B","é","","b","B","B","c","😀","é","c","c","zz","aa","ab","a","zz","b",null],"l1_i32":[4,1,5,4,null,-1,null,3,0,null,4,0,-2,-2,-4,-3,-2147483648,-1,-4,4],"l2_utf8":[null,"a","c","b","","a","zz","ab","é","B","c","ab","😀","é","😀","😀",null,"zz","😀","😀"],"l3_i64":["2","-4","-4611686018427387904","-4611686018427387904","1","9007199254740993","-4611686018427387904","5","9007199254740992","-4611686018427387904","9007199254740993","9007199254740992","-4611686018427387904","4","9007199254740992","9007199254740993","9007199254740992",null,"9007199254740995","9007199254740993"],"l4_bool":[false,true,false,false,true,true,false,true,false,true,false,true,false,true,null,true,false,false,false,true]}}},"ops":[{"op":"sort","keys":[{"expr":{"col":"l0_utf8"},"descending":false}],"nullsLast":false},{"op":"withColumns","exprs":{"x1":{"fn":"mul","args":[{"fn":"mod","args":[{"col":"l1_i32"},{"col":"l1_i32"}]},{"col":"l1_i32"}]}}},{"op":"withColumns","exprs":{"x2":{"fn":"mul","args":[{"fn":"floorDiv","args":[{"col":"l1_i32"},{"col":"l1_i32"}]},{"col":"x1"}]}}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_i64","type":"i64"},{"name":"l4_bool","type":"bool"},{"name":"x1","type":"i32"},{"name":"x2","type":"i32"}],"columns":{"l0_utf8":[null,"","B","B","B","a","aa","aa","ab","b","b","c","c","c","zz","zz","é","é","😀","😀"],"l1_i32":[4,null,5,null,3,-2147483648,4,-4,-3,-1,-4,0,0,-2,-2,-1,4,4,1,null],"l2_utf8":["😀","","c","zz","ab",null,null,"😀","😀","a","😀","é","ab","😀","é","zz","b","c","a","B"],"l3_i64":["9007199254740993","1","-4611686018427387904","-4611686018427387904","5","9007199254740992","2","9007199254740992","9007199254740993","9007199254740993","9007199254740995","9007199254740992","9007199254740992","-4611686018427387904","4",null,"-4611686018427387904","9007199254740993","-4","-4611686018427387904"],"l4_bool":[true,true,false,false,true,false,false,null,true,true,false,false,true,false,true,false,false,false,true,true],"x1":[0,null,0,null,0,0,0,0,0,0,0,null,null,0,0,0,0,0,0,null],"x2":[0,null,0,null,0,0,0,0,0,0,0,null,null,0,0,0,0,0,0,null]}}},{"id":"random/0027","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_f64","type":"f64"},{"name":"l5_i32","type":"i32"},{"name":"l6_i32","type":"i32"}],"columns":{"l0_utf8":["😀"],"l1_i32":[4],"l2_i32":[-5],"l3_utf8":[""],"l4_f64":[3.0],"l5_i32":[-4],"l6_i32":[-3]}}},"ops":[{"op":"groupBy","keys":["l1_i32"],"aggs":{"n0":{"agg":"len"},"a0_0":{"agg":"count","arg":{"col":"l5_i32"}},"a0_1":{"agg":"last","arg":{"col":"l3_utf8"}},"a0_2":{"agg":"sum","arg":{"col":"l6_i32"}}}},{"op":"unique","columns":["a0_0"]}],"expected":{"fields":[{"name":"l1_i32","type":"i32"},{"name":"n0","type":"u32"},{"name":"a0_0","type":"u32"},{"name":"a0_1","type":"utf8"},{"name":"a0_2","type":"i32"}],"columns":{"l1_i32":[4],"n0":[1],"a0_0":[1],"a0_1":[""],"a0_2":[-3]}}},{"id":"random/0028","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_f64","type":"f64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_bool":[],"l3_f64":[]}}},"ops":[{"op":"filter","predicate":{"fn":"gt","args":[{"col":"l1_i32"},{"fn":"abs","args":[{"col":"l3_f64"}]}]}},{"op":"withColumns","exprs":{"x1":{"fn":"fillNull","args":[{"col":"l3_f64"},{"lit":0}]}}},{"op":"withColumns","exprs":{"x2":{"fn":"fillNull","args":[{"fn":"mod","args":[{"col":"l1_i32"},{"col":"x1"}]},{"lit":1.5}]}}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_f64","type":"f64"},{"name":"x1","type":"f64"},{"name":"x2","type":"f64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_bool":[],"l3_f64":[],"x1":[],"x2":[]}}},{"id":"random/0029","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_utf8","type":"utf8"}],"columns":{"l0_utf8":["b"],"l1_i32":[1],"l2_i32":[2147483647],"l3_utf8":["😀"],"l4_utf8":["zz"]}}},"ops":[{"op":"unique","columns":["l3_utf8"]},{"op":"withColumns","exprs":{"x1":{"fn":"mod","args":[{"fn":"when","args":[{"fn":"gt","args":[{"col":"l2_i32"},{"lit":-1}]},{"lit":2.5},{"col":"l2_i32"}]},{"fn":"div","args":[{"col":"l2_i32"},{"col":"l2_i32"}]}]}}},{"op":"withColumns","exprs":{"x2":{"fn":"or","args":[{"fn":"strStartsWith","args":[{"col":"l0_utf8"}],"prefix":""},{"fn":"isNotNull","args":[{"col":"l4_utf8"}]}]}}},{"op":"withColumns","exprs":{"x3":{"fn":"mul","args":[{"fn":"floorDiv","args":[{"col":"l1_i32"},{"col":"l2_i32"}]},{"fn":"abs","args":[{"col":"x1"}]}]}}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_utf8","type":"utf8"},{"name":"x1","type":"f64"},{"name":"x2","type":"bool"},{"name":"x3","type":"f64"}],"columns":{"l0_utf8":["b"],"l1_i32":[1],"l2_i32":[2147483647],"l3_utf8":["😀"],"l4_utf8":["zz"],"x1":[0.5],"x2":[true],"x3":[0.0]}}},{"id":"random/0030","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_f64","type":"f64"},{"name":"l4_i32","type":"i32"},{"name":"l5_utf8","type":"utf8"},{"name":"l6_f64","type":"f64"}],"columns":{"l0_utf8":[null,"zz","b","c",null,"😀","aa","a","ab","😀","a","😀",null,null,"B","é","😀","😀","aa",null],"l1_i32":[5,3,4,1,3,3,2,0,-4,-1,-2,3,-2,null,1,-1,-2,3,-2147483648,2],"l2_bool":[false,null,true,true,null,null,null,true,false,false,false,true,null,null,null,false,true,false,false,false],"l3_f64":[null,-32.047,null,4.0,null,-2.0,null,-1.0,1.0,12.887,null,-4.0,4.0,null,null,null,47.3,"NaN",null,1.0],"l4_i32":[1,null,null,2,-5,null,5,-2,4,-3,null,-2,4,null,-832,-3,-4,-1,null,null],"l5_utf8":["é","","aa",null,"","b","B","é","b","c","aa","ab",null,"b",null,null,"B","","B","zz"],"l6_f64":[null,null,20.937,74.376,-2.0,-2.0,1.0,1.0,4.0,null,3.0,-87.89,-4.0,-4.0,-2.0,2.0,18.81,-3.0,-57.539,-49.886]}}},"ops":[{"op":"head","n":10}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_f64","type":"f64"},{"name":"l4_i32","type":"i32"},{"name":"l5_utf8","type":"utf8"},{"name":"l6_f64","type":"f64"}],"columns":{"l0_utf8":[null,"zz","b","c",null,"😀","aa","a","ab","😀"],"l1_i32":[5,3,4,1,3,3,2,0,-4,-1],"l2_bool":[false,null,true,true,null,null,null,true,false,false],"l3_f64":[null,-32.047,null,4.0,null,-2.0,null,-1.0,1.0,12.887],"l4_i32":[1,null,null,2,-5,null,5,-2,4,-3],"l5_utf8":["é","","aa",null,"","b","B","é","b","c"],"l6_f64":[null,null,20.937,74.376,-2.0,-2.0,1.0,1.0,4.0,null]}}},{"id":"random/0031","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_f64","type":"f64"},{"name":"l5_f64","type":"f64"}],"columns":{"l0_utf8":["ab","a","B","","B","","😀","B","a","B","b","a",null,"ab","😀","a","a","😀","a","","B","c","ab","","a",null,"é",null,"B","zz","😀","😀","ab","aa","B","😀","é","b","","ab","é","b","aa","zz","a","b","c","ab","aa","b","zz","a","B","aa","zz","B","B","a","a","😀"],"l1_i32":[5,-3,null,0,-5,-4,4,-1,-2,null,3,1,0,-1,-2,null,null,-1,null,4,null,null,0,3,-2,-902,5,3,-2147483648,5,-2147483648,0,5,-2,2,0,null,-2147483648,null,4,-4,4,3,-2147483648,-5,null,-1,-3,1,-3,-4,0,2,1,3,0,-5,2,-4,-1],"l2_i32":[-1,1,1,-1,3,3,3,-2147483648,4,2,null,1,-5,-5,1,-4,-4,3,null,1,3,-2147483648,1,-5,-4,null,-2,2,-2,3,1,3,-1,-4,-2,-4,-5,5,4,-5,1,-4,-2,-5,null,5,0,4,5,0,2147483647,-5,-1,2,null,-4,1,null,-5,0],"l3_utf8":["a","zz","c","😀","b","",null,null,"😀","zz","B","aa","aa","c","😀","c",null,"B","é","c","😀","b","","😀","c","c","aa","é","ab","","a","aa","zz","B","😀","aa","😀","😀","zz","😀","é","aa","b","B","é","é","a","b","B","é","","b","","b","ab","ab","","","c","ab"],"l4_f64":[-28.119,0.0,77.697,"NaN",0.0,-50.427,-75.674,3.0,-57.073,null,4.0,34.717,82.971,-1.0,-2.0,-27.543,4.0,null,4.0,"NaN",0.213,-76.344,null,-3.0,-2.0,-83.76,4.0,-68.275,-27.346,-83.495,-3.0,0.0,-1.0,0.0,-3.0,8.143,-4.0,30.266,-4.0,7.966,4.0,61.037,-1.0,4.0,75.555,3.0,null,-70.192,-24.1,71.091,89.907,-11.614,64.357,4.0,0.0,-29.494,2.0,-2.0,null,1.0],"l5_f64":[66.043,1.0,-56.96,null,92.024,51.449,-22.086,-47.795,31.068,0.0,4.0,-2.0,2.0,-43.733,7.769,-89.954,-3.0,-25.39,4.0,-3.0,-29.676,-3.0,-29.299,0.0,-61.1,-92.605,"NaN",-83.779,-1.0,0.0,0.0,null,-37.859,null,null,-1.0,-3.0,59.246,null,-62.022,45.098,-67.778,21.841,-4.0,29.508,-43.894,-41.614,-1.0,-86.858,4.0,-11.462,null,-18.429,2.0,3.0,42.576,-98.8,-39.391,1.0,4.0]}}},"ops":[{"op":"sort","keys":[{"expr":{"col":"l0_utf8"},"descending":false}],"nullsLast":true},{"op":"filter","predicate":{"fn":"lt","args":[{"fn":"floorDiv","args":[{"col":"l5_f64"},{"lit":-1}]},{"fn":"floorDiv","args":[{"col":"l5_f64"},{"col":"l2_i32"}]}]}},{"op":"head","n":0}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_f64","type":"f64"},{"name":"l5_f64","type":"f64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_i32":[],"l3_utf8":[],"l4_f64":[],"l5_f64":[]}}},{"id":"random/0032","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_i64","type":"i64"},{"name":"l4_f64","type":"f64"},{"name":"l5_utf8","type":"utf8"}],"columns":{"l0_utf8":["😀","zz","b","","ab",null,"zz","","zz","zz","aa","zz","zz","B","a","é","zz","aa","ab","zz","","b","a","ab","ab","aa","B","","c",null,"b","","é","c","B","a",null,"😀","","é",null,"c","","a","b","a","b","B","ab","B","😀","aa","aa","zz",null,"b","aa","é","😀","ab"],"l1_i32":[null,-1,-2147483648,0,-3,-4,1,1,2,-2,2,-3,0,null,-3,1,1,-2147483648,-5,-5,5,2,null,-3,-1,null,4,3,0,-3,-810,-3,1,-2,3,-2147483648,5,2,2,0,4,5,-2,-2,null,2147483647,2,-4,-5,-3,3,null,5,null,null,-4,-5,1,null,-3],"l2_i32":[-1,2,4,0,-3,-3,1,-3,-2,5,1,-2,2,null,2147483647,4,-4,-4,null,2,-5,-2,5,-4,-4,-2,4,null,0,5,0,-2,2,4,null,-4,2,null,-3,-4,3,-4,null,null,4,2147483647,3,-3,-4,4,null,-4,-1,-4,1,0,-5,-2,null,2],"l3_i64":["9007199254740994","-4611686018427387904","-4","9007199254740993",null,"9007199254740994","9007199254740994","-4611686018427387904","-3","9007199254740993","9007199254740993","9007199254740995","-4611686018427387904","0","-4","-4611686018427387904","-4611686018427387904","-4611686018427387904","-4611686018427387904","-4611686018427387904","9007199254740992","9007199254740994","-1","-5","-2","9007199254740992","9007199254740993","5","9007199254740995",null,"-4611686018427387904","9007199254740992","2","9007199254740994","9007199254740993","0","9007199254740994","-4611686018427387904","9007199254740994","-4611686018427387904","-4611686018427387904","5","9007199254740994","-4611686018427387904","9007199254740993","9007199254740993","9007199254740993","-4611686018427387904","-5",null,"-1","4","9007199254740995","9007199254740993","2","5","9007199254740992","-4611686018427387904","-4","-4611686018427387904"],"l4_f64":[-4.0,-1.0,0.0,-70.308,77.387,-28.825,50.593,-4.0,-82.514,-32.141,null,-2.0,-3.0,4.0,-4.0,-1.0,null,53.602,65.505,-91.32,82.616,-5.335,-2.0,-18.204,1.0,-3.0,-32.172,-3.0,0.0,88.273,"-Infinity",4.0,12.631,1.0,null,-58.168,-50.84,-5.522,43.594,96.263,4.0,4.0,14.091,3.0,5.058,-3.0,-4.0,23.268,null,10.379,78.576,42.139,36.501,6.087,-34.883,-1.0,1.0,4.0,16.477,null],"l5_utf8":["a","","c","B","c","a","zz","a","é","a","aa","aa","zz","B","zz",null,"😀","","a","é","c",null,"aa","é","c","c","","c","zz","😀",null,"","b","😀","a","c","ab","B",null,"😀","b","B","ab","B","","","ab","b","é","a",null,"ab","zz","","😀","b","c","zz","c","B"]}},"right":{"fields":[{"name":"l1_i32","type":"i32"},{"name":"r0_bool","type":"bool"}],"columns":{"r0_bool":[true,null,true,true,null,true,true,null,false,false],"l1_i32":[-3,3,5,-5,null,1,1,1,1,-4]}}},"ops":[{"op":"unique"},{"op":"filter","predicate":{"fn":"or","args":[{"fn":"eq","args":[{"col":"l4_f64"},{"col":"l4_f64"}]},{"fn":"isIn","args":[{"col":"l0_utf8"}],"values":["c","","a"]}]}},{"op":"join","right":"right","on":["l1_i32"],"how":"left"}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_i64","type":"i64"},{"name":"l4_f64","type":"f64"},{"name":"l5_utf8","type":"utf8"},{"name":"r0_bool","type":"bool"}],"columns":{"l0_utf8":["😀","zz","b","","ab",null,"zz","zz","zz","zz","","","","","zz","zz","zz","zz","B","a","é","é","é","é","aa","ab","zz","","b","a","ab","ab","aa","B","","c",null,"b","","é","é","é","é","c","a",null,"😀","","é",null,"c","","a","b","a","b","B","B","😀","aa","aa","zz",null,"b","aa","é","é","é","é","😀"],"l1_i32":[null,-1,-2147483648,0,-3,-4,1,1,1,1,1,1,1,1,2,-2,-3,0,null,-3,1,1,1,1,-2147483648,-5,-5,5,2,null,-3,-1,null,4,3,0,-3,-810,-3,1,1,1,1,-2,-2147483648,5,2,2,0,4,5,-2,-2,null,2147483647,2,-4,-3,3,null,5,null,null,-4,-5,1,1,1,1,null],"l2_i32":[-1,2,4,0,-3,-3,1,1,1,1,-3,-3,-3,-3,-2,5,-2,2,null,2147483647,4,4,4,4,-4,null,2,-5,-2,5,-4,-4,-2,4,null,0,5,0,-2,2,2,2,2,4,-4,2,null,-3,-4,3,-4,null,null,4,2147483647,3,-3,4,null,-4,-1,-4,1,0,-5,-2,-2,-2,-2,null],"l3_i64":["9007199254740994","-4611686018427387904","-4","9007199254740993",null,"9007199254740994","9007199254740994","9007199254740994","9007199254740994","9007199254740994","-4611686018427387904","-4611686018427387904","-4611686018427387904","-4611686018427387904","-3","9007199254740993","9007199254740995","-4611686018427387904","0","-4","-4611686018427387904","-4611686018427387904","-4611686018427387904","-4611686018427387904","-4611686018427387904","-4611686018427387904","-4611686018427387904","9007199254740992","9007199254740994","-1","-5","-2","9007199254740992","9007199254740993","5","9007199254740995",null,"-4611686018427387904","9007199254740992","2","2","2","2","9007199254740994","0","9007199254740994","-4611686018427387904","9007199254740994","-4611686018427387904","-4611686018427387904","5","9007199254740994","-4611686018427387904","9007199254740993","9007199254740993","9007199254740993","-4611686018427387904",null,"-1","4","9007199254740995","9007199254740993","2","5","9007199254740992","-4611686018427387904","-4611686018427387904","-4611686018427387904","-4611686018427387904","-4"],"l4_f64":[-4.0,-1.0,0.0,-70.308,77.387,-28.825,50.593,50.593,50.593,50.593,-4.0,-4.0,-4.0,-4.0,-82.514,-32.141,-2.0,-3.0,4.0,-4.0,-1.0,-1.0,-1.0,-1.0,53.602,65.505,-91.32,82.616,-5.335,-2.0,-18.204,1.0,-3.0,-32.172,-3.0,0.0,88.273,"-Infinity",4.0,12.631,12.631,12.631,12.631,1.0,-58.168,-50.84,-5.522,43.594,96.263,4.0,4.0,14.091,3.0,5.058,-3.0,-4.0,23.268,10.379,78.576,42.139,36.501,6.087,-34.883,-1.0,1.0,4.0,4.0,4.0,4.0,16.477],"l5_utf8":["a","","c","B","c","a","zz","zz","zz","zz","a","a","a","a","é","a","aa","zz","B","zz",null,null,null,null,"","a","é","c",null,"aa","é","c","c","","c","zz","😀",null,"","b","b","b","b","😀","c","ab","B",null,"😀","b","B","ab","B","","","ab","b","a",null,"ab","zz","","😀","b","c","zz","zz","zz","zz","c"],"r0_bool":[null,null,null,null,true,false,true,true,null,false,true,true,null,false,null,null,true,null,null,true,true,true,null,false,null,true,true,true,null,null,true,null,null,null,null,null,true,null,true,true,true,null,false,null,null,true,null,null,null,null,true,null,null,null,null,null,false,true,null,null,true,null,null,false,true,true,true,null,false,null]}}},{"id":"random/0033","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_i32","type":"i32"},{"name":"l5_bool","type":"bool"},{"name":"l6_utf8","type":"utf8"}],"columns":{"l0_utf8":["a",null,null,"ab","ab"],"l1_i32":[null,-5,0,null,0],"l2_f64":[1.57,61.787,4.0,null,-55.982],"l3_utf8":["b","ab","😀","B","aa"],"l4_i32":[2,2,null,null,-2147483648],"l5_bool":[false,true,false,null,true],"l6_utf8":["é","B","b","aa","aa"]}}},"ops":[{"op":"unique","columns":["l0_utf8"]},{"op":"filter","predicate":{"fn":"lte","args":[{"fn":"mul","args":[{"lit":2},{"col":"l2_f64"}]},{"fn":"fillNull","args":[{"col":"l4_i32"},{"lit":0}]}]}},{"op":"withColumns","exprs":{"x2":{"fn":"add","args":[{"col":"l4_i32"},{"fn":"mul","args":[{"col":"l1_i32"},{"col":"l4_i32"}]}]}}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_i32","type":"i32"},{"name":"l5_bool","type":"bool"},{"name":"l6_utf8","type":"utf8"},{"name":"x2","type":"i32"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_f64":[],"l3_utf8":[],"l4_i32":[],"l5_bool":[],"l6_utf8":[],"x2":[]}}},{"id":"random/0034","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i64","type":"i64"},{"name":"l3_f64","type":"f64"},{"name":"l4_i32","type":"i32"},{"name":"l5_bool","type":"bool"}],"columns":{"l0_utf8":["😀","ab","c","a","a","B","😀","ab","c","B","","","é","B","aa","zz","","c","b","é"],"l1_i32":[2,-4,null,511,4,-2,5,-3,-1,-2,1,4,0,null,1,null,-4,1,-1,3],"l2_i64":["9007199254740992","-4611686018427387904","5","-4611686018427387904","-4611686018427387904","5",null,"-2","-1","4","-1",null,"9007199254740992","1","9007199254740995",null,"9007199254740993","-4611686018427387904","9007199254740994","-4611686018427387904"],"l3_f64":[3.0,-4.0,null,-21.919,-2.0,-59.552,4.0,null,3.0,null,1.573,3.0,2.0,-24.37,-1.0,1.0,-61.886,-71.333,"-0",null],"l4_i32":[4,-1,1,-4,1,0,5,null,2,2147483647,2,-2,2,-3,-2,-1,2,-2,-4,-2147483648],"l5_bool":[true,false,false,false,false,true,null,true,null,false,true,true,false,true,false,false,true,false,true,true]}}},"ops":[{"op":"withColumns","exprs":{"x0":{"col":"l3_f64"}}},{"op":"groupBy","keys":["l2_i64"],"aggs":{"n1":{"agg":"len"},"a1_0":{"agg":"max","arg":{"col":"l1_i32"}},"a1_1":{"agg":"min","arg":{"col":"l4_i32"}},"a1_2":{"agg":"last","arg":{"col":"l5_bool"}}}}],"expected":{"fields":[{"name":"l2_i64","type":"i64"},{"name":"n1","type":"u32"},{"name":"a1_0","type":"i32"},{"name":"a1_1","type":"i32"},{"name":"a1_2","type":"bool"}],"columns":{"l2_i64":["9007199254740992","-4611686018427387904","5",null,"-2","-1","4","1","9007199254740995","9007199254740993","9007199254740994"],"n1":[2,5,2,3,1,2,1,1,1,1,1],"a1_0":[2,511,-2,5,-3,1,-2,null,1,-4,-1],"a1_1":[2,-2147483648,0,-2,null,2,2147483647,-3,-2,2,-4],"a1_2":[false,true,true,false,true,true,false,true,false,true,true]}}},{"id":"random/0035","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_f64","type":"f64"},{"name":"l4_i64","type":"i64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_f64":[],"l3_f64":[],"l4_i64":[]}},"right":{"fields":[{"name":"l4_i64","type":"i64"},{"name":"r0_utf8","type":"utf8"}],"columns":{"r0_utf8":["a","aa","😀"],"l4_i64":[null,"-4","-2"]}}},"ops":[{"op":"sort","keys":[{"expr":{"col":"l0_utf8"},"descending":true},{"expr":{"col":"l4_i64"},"descending":true}],"nullsLast":true},{"op":"join","right":"right","on":["l4_i64"],"how":"anti"}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_f64","type":"f64"},{"name":"l4_i64","type":"i64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_f64":[],"l3_f64":[],"l4_i64":[]}}},{"id":"random/0036","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_bool","type":"bool"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_f64","type":"f64"},{"name":"l6_f64","type":"f64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_utf8":[],"l3_bool":[],"l4_utf8":[],"l5_f64":[],"l6_f64":[]}}},"ops":[{"op":"filter","predicate":{"fn":"isNotNull","args":[{"col":"l3_bool"}]}},{"op":"withColumns","exprs":{"x1":{"fn":"sub","args":[{"fn":"fillNull","args":[{"col":"l5_f64"},{"lit":1.5}]},{"col":"l5_f64"}]}}},{"op":"sort","keys":[{"expr":{"col":"l4_utf8"},"descending":true},{"expr":{"col":"l0_utf8"},"descending":true}],"nullsLast":true}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_bool","type":"bool"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_f64","type":"f64"},{"name":"l6_f64","type":"f64"},{"name":"x1","type":"f64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_utf8":[],"l3_bool":[],"l4_utf8":[],"l5_f64":[],"l6_f64":[],"x1":[]}}},{"id":"random/0037","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_i32","type":"i32"}],"columns":{"l0_utf8":["zz"],"l1_i32":[3],"l2_utf8":[""],"l3_utf8":[null],"l4_utf8":[""],"l5_i32":[-5]}}},"ops":[{"op":"groupBy","keys":["l4_utf8"],"aggs":{"n0":{"agg":"len"},"a0_0":{"agg":"sum","arg":{"col":"l5_i32"}},"a0_1":{"agg":"std","arg":{"col":"l5_i32"}}}},{"op":"groupBy","keys":["l4_utf8"],"aggs":{"n1":{"agg":"len"},"a1_0":{"agg":"count","arg":{"col":"a0_0"}}}}],"expected":{"fields":[{"name":"l4_utf8","type":"utf8"},{"name":"n1","type":"u32"},{"name":"a1_0","type":"u32"}],"columns":{"l4_utf8":[""],"n1":[1],"a1_0":[1]}}},{"id":"random/0038","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_utf8","type":"utf8"},{"name":"l6_bool","type":"bool"}],"columns":{"l0_utf8":["","aa","a","zz","c","B","é","aa","a","c","😀","aa","b","a","B","aa","a","aa","é","ab"],"l1_i32":[5,-5,4,2,-5,3,-1,3,4,-4,-5,5,4,3,2,4,3,-5,-1,-1],"l2_f64":[4.0,2.0,-57.377,-6.232,59.544,-76.314,14.56,1.0,13.915,-4.667,14.517,0.0,82.457,4.0,-44.608,-2.0,3.0,-34.946,29.93,-92.605],"l3_utf8":["ab","aa","é","aa","aa","b","😀","b","b","😀","c","B","ab","zz","ab","B","c","","😀","😀"],"l4_utf8":["","c","aa","aa","a","ab","b","zz","a","zz","c","c","","zz","a","c","B","ab","aa","B"],"l5_utf8":["c","😀","ab","b","B","c","c","a","aa","B","b","","zz","a","c","c","B","aa","a","ab"],"l6_bool":[true,false,true,true,true,false,true,false,false,true,true,false,true,false,false,false,false,true,true,true]}}},"ops":[{"op":"sort","keys":[{"expr":{"col":"l1_i32"},"descending":true}],"nullsLast":true},{"op":"withColumns","exprs":{"x1":{"fn":"neg","args":[{"col":"l1_i32"}]}}},{"op":"filter","predicate":{"fn":"or","args":[{"fn":"gte","args":[{"col":"l5_utf8"},{"lit":"😀"}]},{"fn":"gt","args":[{"col":"x1"},{"col":"l1_i32"}]}]}},{"op":"filter","predicate":{"fn":"lte","args":[{"fn":"neg","args":[{"lit":2.5}]},{"fn":"abs","args":[{"lit":2.5}]}]}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_utf8","type":"utf8"},{"name":"l6_bool","type":"bool"},{"name":"x1","type":"i32"}],"columns":{"l0_utf8":["é","é","ab","c","aa","c","😀","aa"],"l1_i32":[-1,-1,-1,-4,-5,-5,-5,-5],"l2_f64":[14.56,29.93,-92.605,-4.667,2.0,59.544,14.517,-34.946],"l3_utf8":["😀","😀","😀","😀","aa","aa","c",""],"l4_utf8":["b","aa","B","zz","c","a","c","ab"],"l5_utf8":["c","a","ab","B","😀","B","b","aa"],"l6_bool":[true,true,true,true,false,true,true,true],"x1":[1,1,1,4,5,5,5,5]}}},{"id":"random/0039","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_utf8","type":"utf8"}],"columns":{"l0_utf8":[null,"b",null,"","😀",null,"b",null,"aa","b",null,null,"",null,"a",null,"c","aa","é","a",null,"c",null,null,null,"é","😀","é","",null,null,null,"é","B",null,"B",null,"c",null,null,"b","aa","aa","B","B","zz","é",null,"é",null,"😀","B","",null,"B",null,"b",null,null,"b"],"l1_i32":[null,null,-5,null,null,1,null,null,-4,null,-1,-2,-2,0,-2,5,null,2147483647,null,null,-1,-5,-1,4,0,null,4,3,null,2,3,-4,null,null,null,-3,755,-1,-1,null,3,5,-1,5,-3,null,null,4,null,2,1,859,2147483647,null,4,-2,3,-5,null,0],"l2_f64":[null,null,4.0,-77.618,0.0,95.472,-88.487,-53.32,-95.121,-1.0,null,88.779,null,null,-17.286,82.866,98.264,null,null,-1.0,null,"-Infinity",-85.425,4.0,null,"NaN",-27.37,-3.0,"NaN",null,null,-4.0,-1.0,null,61.758,null,-82.626,-3.0,null,27.566,null,4.0,-4.0,2.0,null,null,-4.0,94.261,-4.0,52.594,91.992,null,64.752,null,20.549,81.434,null,"NaN",-18.987,null],"l3_utf8":["é","B","b","b","zz",null,null,"B",null,"a","é",null,"","zz","ab","😀","a","😀","c",null,"ab","a","aa",null,"😀","b","😀","B","aa","","",null,null,"c","","a","😀","a",null,"c","aa","😀","aa","c","aa","aa","B",null,"B","zz","c",null,"","😀","",null,null,"","aa","B"]}}},"ops":[{"op":"filter","predicate":{"fn":"eq","args":[{"col":"l0_utf8"},{"lit":"zz"}]}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_utf8","type":"utf8"}],"columns":{"l0_utf8":["zz"],"l1_i32":[null],"l2_f64":[null],"l3_utf8":["aa"]}}},{"id":"random/0040","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_i32","type":"i32"}],"columns":{"l0_utf8":["é"],"l1_i32":[-5],"l2_utf8":["zz"],"l3_utf8":["ab"],"l4_utf8":["B"],"l5_i32":[1]}}},"ops":[{"op":"groupBy","keys":["l5_i32"],"aggs":{"n0":{"agg":"len"},"a0_0":{"agg":"max","arg":{"col":"l3_utf8"}},"a0_1":{"agg":"min","arg":{"col":"l4_utf8"}}}},{"op":"withColumns","exprs":{"x1":{"fn":"isIn","args":[{"col":"a0_0"}],"values":["zz","","aa"]}}},{"op":"head","n":7}],"expected":{"fields":[{"name":"l5_i32","type":"i32"},{"name":"n0","type":"u32"},{"name":"a0_0","type":"utf8"},{"name":"a0_1","type":"utf8"},{"name":"x1","type":"bool"}],"columns":{"l5_i32":[1],"n0":[1],"a0_0":["ab"],"a0_1":["B"],"x1":[false]}}},{"id":"random/0041","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_utf8","type":"utf8"}],"columns":{"l0_utf8":[null,null,null,"a","aa","c",null,"","c","c","😀",null,"zz","","aa","😀","aa","😀","ab","b",null,"zz","c","aa","","","b","aa","😀","",null,null,"c",null,"😀","zz","😀","c",null,"ab",null,null,null,"B","c",null,"a","😀","c",null,"é","a","aa",null,"aa","é","c","B","ab",null],"l1_i32":[-5,null,2,1,-4,-2147483648,-4,null,-2,null,null,3,null,null,-1,5,-4,null,2,null,0,null,null,null,3,null,null,-2147483648,-3,-5,null,5,4,-2,null,-2,null,null,null,-2,null,null,null,null,null,368,null,4,-2,4,3,2,0,null,1,null,339,null,4,null],"l2_i32":[-3,-3,null,null,2,-1,5,null,null,0,null,null,null,-1,-3,2,null,null,3,-2,2,3,null,2147483647,4,-5,0,0,4,null,-1,32,-2,0,null,5,-5,-3,-1,null,-3,4,null,-2,null,null,null,-2,1,null,-2,-777,2,null,-5,2,1,-4,-5,-3],"l3_utf8":["c","","😀","ab","B",null,"😀","aa","a","aa","B","B",null,"😀",null,null,null,null,null,"c","zz",null,"aa",null,null,"",null,"aa","","é",null,null,null,"aa","c",null,null,"zz",null,"b","c",null,null,"😀","b","a",null,"b",null,null,"ab","b","é",null,null,"c","c",null,"a",null]}}},"ops":[{"op":"filter","predicate":{"fn":"neq","args":[{"fn":"mul","args":[{"col":"l1_i32"},{"col":"l1_i32"}]},{"fn":"mul","args":[{"lit":-1},{"col":"l1_i32"}]}]}},{"op":"groupBy","keys":["l1_i32"],"aggs":{"n1":{"agg":"len"},"a1_0":{"agg":"last","arg":{"col":"l2_i32"}}}},{"op":"sort","keys":[{"expr":{"col":"n1"},"descending":true}],"nullsLast":false}],"expected":{"fields":[{"name":"l1_i32","type":"i32"},{"name":"n1","type":"u32"},{"name":"a1_0","type":"i32"}],"columns":{"l1_i32":[-2,4,2,-4,3,-5,1,-2147483648,5,-3,368,339],"n1":[5,4,3,3,3,2,2,2,2,1,1,1],"a1_0":[1,-5,-777,null,-2,null,-5,0,32,4,null,1]}}},{"id":"random/0042","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_f64","type":"f64"},{"name":"l5_utf8","type":"utf8"},{"name":"l6_i64","type":"i64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_utf8":[],"l3_utf8":[],"l4_f64":[],"l5_utf8":[],"l6_i64":[]}}},"ops":[{"op":"withColumns","exprs":{"x0":{"col":"l4_f64"}}},{"op":"filter","predicate":{"fn":"neq","args":[{"col":"l4_f64"},{"lit":2.5}]}},{"op":"head","n":3}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_f64","type":"f64"},{"name":"l5_utf8","type":"utf8"},{"name":"l6_i64","type":"i64"},{"name":"x0","type":"f64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_utf8":[],"l3_utf8":[],"l4_f64":[],"l5_utf8":[],"l6_i64":[],"x0":[]}}},{"id":"random/0043","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_utf8","type":"utf8"}],"columns":{"l0_utf8":[""],"l1_i32":[null],"l2_utf8":["B"],"l3_utf8":["a"],"l4_utf8":["B"],"l5_utf8":[""]}}},"ops":[{"op":"unique"},{"op":"withColumns","exprs":{"x1":{"fn":"add","args":[{"fn":"sub","args":[{"col":"l1_i32"},{"col":"l1_i32"}]},{"fn":"div","args":[{"col":"l1_i32"},{"col":"l1_i32"}]}]}}},{"op":"sort","keys":[{"expr":{"col":"l4_utf8"},"descending":false},{"expr":{"col":"l5_utf8"},"descending":false}],"nullsLast":false}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_utf8","type":"utf8"},{"name":"x1","type":"f64"}],"columns":{"l0_utf8":[""],"l1_i32":[null],"l2_utf8":["B"],"l3_utf8":["a"],"l4_utf8":["B"],"l5_utf8":[""],"x1":[null]}}},{"id":"random/0044","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_i32","type":"i32"}],"columns":{"l0_utf8":["b","aa","b","ab","a","a","ab","ab","b","ab","b","b","é","b","a","zz","zz","😀","ab","é"],"l1_i32":[3,-5,-5,-4,5,4,4,2,5,-1,-3,4,5,0,1,2,2,5,-5,-2147483648],"l2_f64":[-51.166,-16.398,-4.0,54.999,"NaN",0.0,0.0,-3.0,-5.676,3.0,4.0,-3.0,50.057,-4.0,-1.0,3.0,-2.0,-1.0,-4.0,-2.0],"l3_i32":[0,1,0,-2,-1,-1,3,2,5,-2147483648,5,-5,-2147483648,-4,3,-1,-3,-2,3,2]}}},"ops":[{"op":"head","n":2},{"op":"filter","predicate":{"fn":"lt","args":[{"fn":"abs","args":[{"col":"l3_i32"}]},{"fn":"fillNull","args":[{"lit":2},{"lit":1.5}]}]}},{"op":"filter","predicate":{"fn":"lte","args":[{"fn":"sub","args":[{"col":"l3_i32"},{"col":"l3_i32"}]},{"fn":"fillNull","args":[{"col":"l2_f64"},{"lit":1.5}]}]}},{"op":"withColumns","exprs":{"x3":{"fn":"add","args":[{"fn":"fillNull","args":[{"col":"l3_i32"},{"lit":0}]},{"fn":"mod","args":[{"col":"l2_f64"},{"col":"l3_i32"}]}]}}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_i32","type":"i32"},{"name":"x3","type":"f64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_f64":[],"l3_i32":[],"x3":[]}}},{"id":"random/0045","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_bool","type":"bool"}],"columns":{"l0_utf8":["b"],"l1_i32":[-3],"l2_i32":[null],"l3_bool":[false]}}},"ops":[{"op":"head","n":10}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_bool","type":"bool"}],"columns":{"l0_utf8":["b"],"l1_i32":[-3],"l2_i32":[null],"l3_bool":[false]}}},{"id":"random/0046","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_utf8","type":"utf8"}],"columns":{"l0_utf8":["","","zz","zz","a","","c","😀","zz","aa","zz","a","b","😀","c","é","😀","","😀",""],"l1_i32":[-5,4,5,3,1,213,-1,-3,-4,0,-3,2,4,-1,-2,5,-5,-2,0,4],"l2_f64":[2.0,71.906,9.534,0.0,86.177,1.774,-1.0,51.274,64.903,-3.0,-2.0,64.553,-3.0,-4.0,99.817,-38.348,-1.844,-56.828,23.585,35.754],"l3_utf8":["B","a","😀","ab","é","zz","B","a","😀","ab","a","c","c","zz","é","é","é","ab","B","c"]}},"right":{"fields":[{"name":"l3_utf8","type":"utf8"},{"name":"r0_utf8","type":"utf8"}],"columns":{"r0_utf8":["😀","é","B","B","aa","é","zz","😀","ab","aa","a","c","zz","😀","zz","aa","zz","","aa","","B","😀","c","aa","zz","ab","b","b","zz","c"],"l3_utf8":["ab","ab","c","é","é","B","c","é","ab","é","😀","B","a","c","B","ab","é","ab","c","c","a","ab","é","😀","c","é","B","ab","a","é"]}}},"ops":[{"op":"groupBy","keys":["l3_utf8"],"aggs":{"n0":{"agg":"len"},"a0_0":{"agg":"count","arg":{"col":"l1_i32"}}}},{"op":"sort","keys":[{"expr":{"col":"l3_utf8"},"descending":false},{"expr":{"col":"n0"},"descending":false}],"nullsLast":true},{"op":"join","right":"right","on":["l3_utf8"],"how":"semi"},{"op":"unique"}],"expected":{"fields":[{"name":"l3_utf8","type":"utf8"},{"name":"n0","type":"u32"},{"name":"a0_0","type":"u32"}],"columns":{"l3_utf8":["B","a","ab","c","é","😀"],"n0":[3,3,3,3,4,2],"a0_0":[3,3,3,3,4,2]}}},{"id":"random/0047","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_f64","type":"f64"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_bool","type":"bool"},{"name":"l6_utf8","type":"utf8"}],"columns":{"l0_utf8":["a"],"l1_i32":[-2],"l2_f64":[-12.35],"l3_f64":[-58.837],"l4_utf8":["é"],"l5_bool":[false],"l6_utf8":[null]}}},"ops":[{"op":"groupBy","keys":["l6_utf8"],"aggs":{"n0":{"agg":"len"},"a0_0":{"agg":"nUnique","arg":{"col":"l3_f64"}},"a0_1":{"agg":"median","arg":{"col":"l1_i32"}},"a0_2":{"agg":"min","arg":{"col":"l1_i32"}}}},{"op":"head","n":3},{"op":"groupBy","keys":["a0_0"],"aggs":{"n2":{"agg":"len"},"a2_0":{"agg":"last","arg":{"col":"n0"}},"a2_1":{"agg":"count","arg":{"col":"a0_2"}}}}],"expected":{"fields":[{"name":"a0_0","type":"u32"},{"name":"n2","type":"u32"},{"name":"a2_0","type":"u32"},{"name":"a2_1","type":"u32"}],"columns":{"a0_0":[1],"n2":[1],"a2_0":[1],"a2_1":[1]}}},{"id":"random/0048","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_f64","type":"f64"},{"name":"l4_i64","type":"i64"},{"name":"l5_i32","type":"i32"},{"name":"l6_f64","type":"f64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_bool":[],"l3_f64":[],"l4_i64":[],"l5_i32":[],"l6_f64":[]}}},"ops":[{"op":"withColumns","exprs":{"x0":{"fn":"add","args":[{"lit":0.5},{"fn":"neg","args":[{"col":"l5_i32"}]}]}}},{"op":"filter","predicate":{"fn":"gte","args":[{"col":"l6_f64"},{"fn":"mul","args":[{"lit":2},{"col":"l1_i32"}]}]}},{"op":"withColumns","exprs":{"x2":{"lit":2}}},{"op":"filter","predicate":{"fn":"eq","args":[{"col":"l4_i64"},{"lit":"-1","type":"i64"}]}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_f64","type":"f64"},{"name":"l4_i64","type":"i64"},{"name":"l5_i32","type":"i32"},{"name":"l6_f64","type":"f64"},{"name":"x0","type":"f64"},{"name":"x2","type":"i32"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_bool":[],"l3_f64":[],"l4_i64":[],"l5_i32":[],"l6_f64":[],"x0":[],"x2":[]}}},{"id":"random/0049","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_f64","type":"f64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_i32":[],"l3_f64":[]}},"right":{"fields":[{"name":"l1_i32","type":"i32"},{"name":"r0_bool","type":"bool"}],"columns":{"r0_bool":[],"l1_i32":[]}}},"ops":[{"op":"groupBy","keys":["l1_i32"],"aggs":{"n0":{"agg":"len"},"a0_0":{"agg":"count","arg":{"col":"l3_f64"}},"a0_1":{"agg":"first","arg":{"col":"l3_f64"}},"a0_2":{"agg":"last","arg":{"col":"l3_f64"}}}},{"op":"join","right":"right","on":["l1_i32"],"how":"left"}],"expected":{"fields":[{"name":"l1_i32","type":"i32"},{"name":"n0","type":"u32"},{"name":"a0_0","type":"u32"},{"name":"a0_1","type":"f64"},{"name":"a0_2","type":"f64"},{"name":"r0_bool","type":"bool"}],"columns":{"l1_i32":[],"n0":[],"a0_0":[],"a0_1":[],"a0_2":[],"r0_bool":[]}}},{"id":"random/0050","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_f64","type":"f64"}],"columns":{"l0_utf8":["b","B","😀","ab","é"],"l1_i32":[-2,2,-2147483648,4,3],"l2_utf8":["","","aa","zz",""],"l3_utf8":["","zz","ab","","ab"],"l4_f64":[3.0,-17.346,-4.0,43.76,3.0]}}},"ops":[{"op":"filter","predicate":{"fn":"or","args":[{"fn":"lt","args":[{"col":"l4_f64"},{"col":"l1_i32"}]},{"fn":"isNull","args":[{"col":"l1_i32"}]}]}},{"op":"sort","keys":[{"expr":{"col":"l1_i32"},"descending":false}],"nullsLast":true}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_f64","type":"f64"}],"columns":{"l0_utf8":["B"],"l1_i32":[2],"l2_utf8":[""],"l3_utf8":["zz"],"l4_f64":[-17.346]}}},{"id":"random/0051","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_f64","type":"f64"}],"columns":{"l0_utf8":["😀"],"l1_i32":[-2],"l2_f64":[57.877],"l3_f64":[0.0]}}},"ops":[{"op":"withColumns","exprs":{"x0":{"fn":"and","args":[{"fn":"strStartsWith","args":[{"col":"l0_utf8"}],"prefix":"é"},{"fn":"gte","args":[{"col":"l3_f64"},{"col":"l1_i32"}]}]}}},{"op":"sort","keys":[{"expr":{"col":"l3_f64"},"descending":false},{"expr":{"col":"l2_f64"},"descending":false}],"nullsLast":true}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_f64","type":"f64"},{"name":"x0","type":"bool"}],"columns":{"l0_utf8":["😀"],"l1_i32":[-2],"l2_f64":[57.877],"l3_f64":[0.0],"x0":[false]}}},{"id":"random/0052","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_f64","type":"f64"},{"name":"l4_i32","type":"i32"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_utf8":[],"l3_f64":[],"l4_i32":[]}}},"ops":[{"op":"filter","predicate":{"fn":"neq","args":[{"fn":"neg","args":[{"col":"l1_i32"}]},{"col":"l3_f64"}]}},{"op":"groupBy","keys":["l4_i32"],"aggs":{"n1":{"agg":"len"},"a1_0":{"agg":"nUnique","arg":{"col":"l1_i32"}},"a1_1":{"agg":"nUnique","arg":{"col":"l3_f64"}}}}],"expected":{"fields":[{"name":"l4_i32","type":"i32"},{"name":"n1","type":"u32"},{"name":"a1_0","type":"u32"},{"name":"a1_1","type":"u32"}],"columns":{"l4_i32":[],"n1":[],"a1_0":[],"a1_1":[]}}},{"id":"random/0053","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i64","type":"i64"},{"name":"l3_utf8","type":"utf8"}],"columns":{"l0_utf8":["B"],"l1_i32":[2],"l2_i64":["9007199254740993"],"l3_utf8":["zz"]}}},"ops":[{"op":"sort","keys":[{"expr":{"col":"l3_utf8"},"descending":true}],"nullsLast":true}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i64","type":"i64"},{"name":"l3_utf8","type":"utf8"}],"columns":{"l0_utf8":["B"],"l1_i32":[2],"l2_i64":["9007199254740993"],"l3_utf8":["zz"]}}},{"id":"random/0054","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_f64","type":"f64"}],"columns":{"l0_utf8":["aa","a","b","zz","B","zz","a","b","","zz","é","a","c","B","aa",null,"","aa","zz","ab"],"l1_i32":[3,1,null,null,-5,null,3,-1,-1,-4,null,3,null,4,-3,null,-5,-4,5,0],"l2_f64":[4.45,42.701,null,null,4.0,-61.538,-72.423,null,null,32.325,null,null,-1.0,null,null,-4.052,-1.0,-1.0,-1.0,-4.0],"l3_f64":[-2.0,-3.0,0.0,0.0,-2.0,null,0.0,null,null,4.0,1.0,-2.0,-1.0,-3.0,84.114,null,79.785,null,null,1.0]}},"right":{"fields":[{"name":"l1_i32","type":"i32"},{"name":"r0_bool","type":"bool"}],"columns":{"r0_bool":[false,false,true,true,false,true,false,false,true,null],"l1_i32":[-4,1,-3,3,null,-1,null,-1,-1,null]}}},"ops":[{"op":"sort","keys":[{"expr":{"col":"l3_f64"},"descending":true},{"expr":{"col":"l2_f64"},"descending":false}],"nullsLast":false},{"op":"withColumns","exprs":{"x1":{"col":"l2_f64"}}},{"op":"join","right":"right","on":["l1_i32"],"how":"full"}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_f64","type":"f64"},{"name":"x1","type":"f64"},{"name":"r0_bool","type":"bool"}],"columns":{"l0_utf8":["b","b","b","","","","zz",null,"aa","zz","aa","","zz","é","ab","b","zz","a","c","a","B","aa","B","a",null,null,null],"l1_i32":[-1,-1,-1,-1,-1,-1,null,null,-4,5,-3,-5,-4,null,0,null,null,3,null,3,-5,3,4,1,null,null,null],"l2_f64":[null,null,null,null,null,null,-61.538,-4.052,-1.0,-1.0,null,-1.0,32.325,null,-4.0,null,null,-72.423,-1.0,null,4.0,4.45,null,42.701,null,null,null],"l3_f64":[null,null,null,null,null,null,null,null,null,null,84.114,79.785,4.0,1.0,1.0,0.0,0.0,0.0,-1.0,-2.0,-2.0,-2.0,-3.0,-3.0,null,null,null],"x1":[null,null,null,null,null,null,-61.538,-4.052,-1.0,-1.0,null,-1.0,32.325,null,-4.0,null,null,-72.423,-1.0,null,4.0,4.45,null,42.701,null,null,null],"r0_bool":[true,false,true,true,false,true,null,null,false,null,true,null,false,null,null,null,null,true,null,true,null,true,null,false,false,false,null]}}},{"id":"random/0055","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_utf8","type":"utf8"}],"columns":{"l0_utf8":["zz","aa","é",null,"aa"],"l1_i32":[2,5,-1,5,5],"l2_utf8":["B","B","B","é","a"],"l3_utf8":["zz","ab","b","aa","😀"],"l4_utf8":["ab","é","é","é","ab"]}}},"ops":[{"op":"groupBy","keys":["l0_utf8"],"aggs":{"n0":{"agg":"len"},"a0_0":{"agg":"min","arg":{"col":"l3_utf8"}}}},{"op":"sort","keys":[{"expr":{"col":"n0"},"descending":true},{"expr":{"col":"l0_utf8"},"descending":true}],"nullsLast":true},{"op":"groupBy","keys":["l0_utf8"],"aggs":{"n2":{"agg":"len"},"a2_0":{"agg":"count","arg":{"col":"n0"}},"a2_1":{"agg":"min","arg":{"col":"a0_0"}},"a2_2":{"agg":"first","arg":{"col":"a0_0"}}}},{"op":"filter","predicate":{"fn":"eq","args":[{"col":"a2_1"},{"lit":""}]}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"n2","type":"u32"},{"name":"a2_0","type":"u32"},{"name":"a2_1","type":"utf8"},{"name":"a2_2","type":"utf8"}],"columns":{"l0_utf8":[],"n2":[],"a2_0":[],"a2_1":[],"a2_2":[]}}},{"id":"random/0056","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i64","type":"i64"},{"name":"l3_i32","type":"i32"},{"name":"l4_f64","type":"f64"},{"name":"l5_bool","type":"bool"},{"name":"l6_utf8","type":"utf8"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_i64":[],"l3_i32":[],"l4_f64":[],"l5_bool":[],"l6_utf8":[]}},"right":{"fields":[{"name":"l6_utf8","type":"utf8"},{"name":"r0_utf8","type":"utf8"}],"columns":{"r0_utf8":["😀","c","c","B","😀","b","zz",null,"",null,"aa","😀","b","b",null,"a","","","é","",null,"é","a","B","😀","ab","😀",null,"é","é"],"l6_utf8":["😀","ab","a","😀","😀","c","ab","c","c","a","c","ab","a","ab","😀","c","a","","a","","ab","","😀","ab","a","","😀","c","a",""]}}},"ops":[{"op":"groupBy","keys":["l6_utf8"],"aggs":{"n0":{"agg":"len"},"a0_0":{"agg":"count","arg":{"col":"l0_utf8"}},"a0_1":{"agg":"last","arg":{"col":"l4_f64"}}}},{"op":"join","right":"right","on":["l6_utf8"],"how":"anti"},{"op":"head","n":1},{"op":"head","n":3}],"expected":{"fields":[{"name":"l6_utf8","type":"utf8"},{"name":"n0","type":"u32"},{"name":"a0_0","type":"u32"},{"name":"a0_1","type":"f64"}],"columns":{"l6_utf8":[],"n0":[],"a0_0":[],"a0_1":[]}}},{"id":"random/0057","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_bool","type":"bool"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_bool","type":"bool"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_utf8":[],"l3_bool":[],"l4_utf8":[],"l5_bool":[]}}},"ops":[{"op":"head","n":2},{"op":"head","n":8}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_bool","type":"bool"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_bool","type":"bool"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_utf8":[],"l3_bool":[],"l4_utf8":[],"l5_bool":[]}}},{"id":"random/0058","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_i64","type":"i64"},{"name":"l4_utf8","type":"utf8"}],"columns":{"l0_utf8":["zz"],"l1_i32":[-2],"l2_i32":[null],"l3_i64":["-2"],"l4_utf8":["aa"]}}},"ops":[{"op":"unique"},{"op":"head","n":0}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_i64","type":"i64"},{"name":"l4_utf8","type":"utf8"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_i32":[],"l3_i64":[],"l4_utf8":[]}}},{"id":"random/0059","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_utf8","type":"utf8"}],"columns":{"l0_utf8":["a","",null,null,"ab",null,"B","é","",null,null,null,null,"c","",null,"é","zz",null,null],"l1_i32":[5,-5,null,2,null,0,0,null,-2,-1,null,383,-4,-3,5,0,4,-3,null,-4],"l2_bool":[true,false,null,true,true,null,true,null,false,false,false,false,false,null,false,false,false,false,false,false],"l3_utf8":["","aa","ab","a","aa","","é","b","","a",null,"aa","b",null,null,null,"zz","aa","b",null]}}},"ops":[{"op":"unique","columns":["l0_utf8"]}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_utf8","type":"utf8"}],"columns":{"l0_utf8":["a","",null,"ab","B","é","c","zz"],"l1_i32":[5,-5,null,null,0,null,-3,-3],"l2_bool":[true,false,null,true,true,null,null,false],"l3_utf8":["","aa","ab","aa","é","b",null,"aa"]}}},{"id":"random/0060","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_f64","type":"f64"},{"name":"l4_f64","type":"f64"},{"name":"l5_i64","type":"i64"},{"name":"l6_i32","type":"i32"}],"columns":{"l0_utf8":["aa",null,"a",null,"ab",null,null,null,"é",null,"",null,null,null,null,"ab","😀","","😀",null],"l1_i32":[4,null,null,-3,null,null,4,null,5,-3,-4,-5,null,-1,3,0,-2,4,-3,3],"l2_utf8":["😀","",null,"aa",null,null,null,"a","",null,"a","ab",null,"é",null,"é","aa","b","😀","c"],"l3_f64":[null,null,-14.078,58.329,"NaN",null,null,null,1.0,2.0,0.0,0.0,-83.448,-76.53,null,-97.317,16.945,1.0,3.0,null],"l4_f64":[-45.717,null,1.0,"NaN",2.0,43.686,"NaN",0.0,2.0,null,null,89.606,null,2.0,-27.805,-3.845,72.75,7.614,84.651,-3.0],"l5_i64":[null,"9007199254740993","2","-5","-1",null,null,"-4611686018427387904","4","9007199254740992","9007199254740993",null,null,null,"9007199254740993",null,"-1",null,null,"9007199254740992"],"l6_i32":[null,null,-1,5,2,3,3,null,5,3,3,2,null,null,null,null,-4,-1,null,null]}}},"ops":[{"op":"unique"}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_f64","type":"f64"},{"name":"l4_f64","type":"f64"},{"name":"l5_i64","type":"i64"},{"name":"l6_i32","type":"i32"}],"columns":{"l0_utf8":["aa",null,"a",null,"ab",null,null,null,"é",null,"",null,null,null,null,"ab","😀","","😀",null],"l1_i32":[4,null,null,-3,null,null,4,null,5,-3,-4,-5,null,-1,3,0,-2,4,-3,3],"l2_utf8":["😀","",null,"aa",null,null,null,"a","",null,"a","ab",null,"é",null,"é","aa","b","😀","c"],"l3_f64":[null,null,-14.078,58.329,"NaN",null,null,null,1.0,2.0,0.0,0.0,-83.448,-76.53,null,-97.317,16.945,1.0,3.0,null],"l4_f64":[-45.717,null,1.0,"NaN",2.0,43.686,"NaN",0.0,2.0,null,null,89.606,null,2.0,-27.805,-3.845,72.75,7.614,84.651,-3.0],"l5_i64":[null,"9007199254740993","2","-5","-1",null,null,"-4611686018427387904","4","9007199254740992","9007199254740993",null,null,null,"9007199254740993",null,"-1",null,null,"9007199254740992"],"l6_i32":[null,null,-1,5,2,3,3,null,5,3,3,2,null,null,null,null,-4,-1,null,null]}}},{"id":"random/0061","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_f64","type":"f64"},{"name":"l4_f64","type":"f64"},{"name":"l5_i64","type":"i64"}],"columns":{"l0_utf8":["b",null,null,"ab",""],"l1_i32":[-5,-1,null,4,-4],"l2_utf8":["a",null,null,"ab","B"],"l3_f64":[-22.867,0.0,-4.0,null,3.0],"l4_f64":[-1.0,-3.0,2.0,3.0,33.788],"l5_i64":[null,null,"-4611686018427387904","-4611686018427387904","-4611686018427387904"]}},"right":{"fields":[{"name":"l5_i64","type":"i64"},{"name":"r0_utf8","type":"utf8"}],"columns":{"r0_utf8":[],"l5_i64":[]}}},"ops":[{"op":"unique"},{"op":"filter","predicate":{"fn":"or","args":[{"fn":"lt","args":[{"col":"l3_f64"},{"col":"l1_i32"}]},{"fn":"or","args":[{"fn":"isNull","args":[{"col":"l0_utf8"}]},{"fn":"gt","args":[{"col":"l5_i64"},{"lit":"0","type":"i64"}]}]}]}},{"op":"join","right":"right","on":["l5_i64"],"how":"inner"},{"op":"groupBy","keys":["l0_utf8"],"aggs":{"n3":{"agg":"len"},"a3_0":{"agg":"max","arg":{"col":"l5_i64"}},"a3_1":{"agg":"min","arg":{"col":"l2_utf8"}},"a3_2":{"agg":"nUnique","arg":{"col":"l2_utf8"}}}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"n3","type":"u32"},{"name":"a3_0","type":"i64"},{"name":"a3_1","type":"utf8"},{"name":"a3_2","type":"u32"}],"columns":{"l0_utf8":[],"n3":[],"a3_0":[],"a3_1":[],"a3_2":[]}}},{"id":"random/0062","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_bool","type":"bool"}],"columns":{"l0_utf8":["😀",null,"é","B","😀"],"l1_i32":[2,-3,-4,4,null],"l2_bool":[true,true,true,true,false],"l3_bool":[true,null,false,null,true]}}},"ops":[{"op":"head","n":1},{"op":"filter","predicate":{"fn":"eq","args":[{"fn":"fillNull","args":[{"col":"l1_i32"},{"lit":1.5}]},{"fn":"sub","args":[{"col":"l1_i32"},{"col":"l1_i32"}]}]}},{"op":"withColumns","exprs":{"x2":{"col":"l1_i32"}}},{"op":"filter","predicate":{"fn":"isNull","args":[{"col":"l0_utf8"}]}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_bool","type":"bool"},{"name":"x2","type":"i32"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_bool":[],"l3_bool":[],"x2":[]}}},{"id":"random/0063","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_f64","type":"f64"},{"name":"l4_bool","type":"bool"}],"columns":{"l0_utf8":["c"],"l1_i32":[-2],"l2_utf8":["zz"],"l3_f64":["NaN"],"l4_bool":[true]}}},"ops":[{"op":"sort","keys":[{"expr":{"col":"l2_utf8"},"descending":true},{"expr":{"col":"l4_bool"},"descending":true}],"nullsLast":false},{"op":"filter","predicate":{"fn":"neq","args":[{"fn":"mod","args":[{"col":"l3_f64"},{"col":"l1_i32"}]},{"fn":"mul","args":[{"lit":2.5},{"col":"l3_f64"}]}]}},{"op":"sort","keys":[{"expr":{"col":"l2_utf8"},"descending":false},{"expr":{"col":"l1_i32"},"descending":true}],"nullsLast":true},{"op":"withColumns","exprs":{"x3":{"col":"l1_i32"}}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_f64","type":"f64"},{"name":"l4_bool","type":"bool"},{"name":"x3","type":"i32"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_utf8":[],"l3_f64":[],"l4_bool":[],"x3":[]}}},{"id":"random/0064","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_i64","type":"i64"},{"name":"l5_bool","type":"bool"}],"columns":{"l0_utf8":["B"],"l1_i32":[null],"l2_i32":[null],"l3_utf8":["zz"],"l4_i64":["9007199254740992"],"l5_bool":[true]}}},"ops":[{"op":"filter","predicate":{"fn":"gte","args":[{"col":"l1_i32"},{"fn":"div","args":[{"lit":2},{"col":"l1_i32"}]}]}},{"op":"filter","predicate":{"fn":"eq","args":[{"fn":"mod","args":[{"col":"l1_i32"},{"col":"l2_i32"}]},{"lit":2.5}]}},{"op":"head","n":9},{"op":"filter","predicate":{"fn":"gt","args":[{"col":"l2_i32"},{"col":"l1_i32"}]}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_i64","type":"i64"},{"name":"l5_bool","type":"bool"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_i32":[],"l3_utf8":[],"l4_i64":[],"l5_bool":[]}}},{"id":"random/0065","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_i32","type":"i32"},{"name":"l4_f64","type":"f64"},{"name":"l5_bool","type":"bool"},{"name":"l6_utf8","type":"utf8"}],"columns":{"l0_utf8":["c","","B","ab","aa"],"l1_i32":[3,3,-4,2,778],"l2_utf8":["","b",null,"é",""],"l3_i32":[null,-3,3,-2147483648,1],"l4_f64":[1.0,-2.0,"NaN",54.391,null],"l5_bool":[true,false,false,true,true],"l6_utf8":["é","B","c",null,"zz"]}}},"ops":[{"op":"withColumns","exprs":{"x0":{"fn":"mod","args":[{"fn":"sub","args":[{"col":"l4_f64"},{"lit":-1}]},{"fn":"mod","args":[{"col":"l3_i32"},{"col":"l3_i32"}]}]}}},{"op":"filter","predicate":{"fn":"isNull","args":[{"col":"l2_utf8"}]}},{"op":"unique","columns":["l4_f64"]}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_i32","type":"i32"},{"name":"l4_f64","type":"f64"},{"name":"l5_bool","type":"bool"},{"name":"l6_utf8","type":"utf8"},{"name":"x0","type":"f64"}],"columns":{"l0_utf8":["B"],"l1_i32":[-4],"l2_utf8":[null],"l3_i32":[3],"l4_f64":["NaN"],"l5_bool":[false],"l6_utf8":["c"],"x0":["NaN"]}}},{"id":"random/0066","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_i32","type":"i32"},{"name":"l4_f64","type":"f64"}],"columns":{"l0_utf8":["😀"],"l1_i32":[null],"l2_utf8":[null],"l3_i32":[1],"l4_f64":[null]}}},"ops":[{"op":"head","n":7}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_i32","type":"i32"},{"name":"l4_f64","type":"f64"}],"columns":{"l0_utf8":["😀"],"l1_i32":[null],"l2_utf8":[null],"l3_i32":[1],"l4_f64":[null]}}},{"id":"random/0067","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_i64","type":"i64"},{"name":"l4_f64","type":"f64"}],"columns":{"l0_utf8":["😀","","zz","a","c","c","aa","zz","b","B","zz","zz","c","a","","😀","zz","ab","aa","b","c","a","😀",null,null,"",null,null,"é","ab","zz","zz","b","","aa","","zz","é","","ab","a",null,"zz","😀","c","b","c","b","aa","a","ab","b","ab",null,"","ab","a","c","",""],"l1_i32":[null,0,-3,-1,1,1,-4,5,2,null,5,null,-5,null,-91,0,-3,0,-3,0,null,-4,null,-5,5,-5,-5,5,2,0,-3,2147483647,-1,null,null,null,-2,-1,-2,0,-4,-2,-2,null,2,2,5,1,5,0,-2,5,-5,-3,3,5,5,-444,4,5],"l2_bool":[true,false,false,false,true,true,false,false,false,true,false,true,false,true,false,false,true,true,true,false,true,null,true,true,false,true,true,true,true,false,true,null,false,false,false,true,true,false,false,true,true,true,false,true,false,false,false,true,true,false,true,true,false,null,false,false,null,false,null,true],"l3_i64":[null,"-2","9007199254740995","-1","-4611686018427387904","9007199254740992","9007199254740994","-3","-2","9007199254740993","-4611686018427387904","-4611686018427387904",null,"9007199254740994","9007199254740994","4",null,null,"-2","9007199254740995","1","-4611686018427387904","3","-4611686018427387904","-3",null,"-4611686018427387904","9007199254740994","4","-2","-4611686018427387904","-4611686018427387904","-4611686018427387904",null,"9007199254740992",null,"-4611686018427387904","9007199254740992","4","9007199254740995","5","1","-4611686018427387904","9007199254740994","9007199254740994",null,"-4611686018427387904","-4611686018427387904","-5","9007199254740995","4","1","-4611686018427387904","4","9007199254740992",null,"9007199254740993","9007199254740994","-4611686018427387904","9007199254740992"],"l4_f64":[-1.0,-3.0,-12.791,16.858,2.0,2.0,-77.552,-18.725,-4.0,null,3.0,-94.869,null,-1.0,96.328,-3.0,-3.0,null,2.0,-47.486,2.0,null,-2.0,-24.75,-84.96,-2.0,7.667,81.083,4.117,3.0,0.0,2.0,82.878,0.0,-2.151,2.0,30.616,0.484,-4.0,1.0,-1.0,2.0,3.0,53.676,-38.084,2.0,4.0,-55.776,-56.741,25.898,-95.433,-4.0,-77.119,-4.0,2.0,-91.497,76.809,-70.094,null,-62.333]}},"right":{"fields":[{"name":"l3_i64","type":"i64"},{"name":"r0_utf8","type":"utf8"}],"columns":{"r0_utf8":["zz","","b","a","zz","ab","c","b","B","ab"],"l3_i64":["-4611686018427387904","9007199254740992","1","1","-2",null,"9007199254740993","-4611686018427387904","3","9007199254740994"]}}},"ops":[{"op":"join","right":"right","on":["l3_i64"],"how":"anti"},{"op":"unique","columns":["l1_i32"]}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_i64","type":"i64"},{"name":"l4_f64","type":"f64"}],"columns":{"l0_utf8":["😀","zz","a","zz","c","😀","é","","a"],"l1_i32":[null,-3,-1,5,-5,0,2,-2,-4],"l2_bool":[true,false,false,false,false,false,true,false,true],"l3_i64":[null,"9007199254740995","-1","-3",null,"4","4","4","5"],"l4_f64":[-1.0,-12.791,16.858,-18.725,null,-3.0,4.117,-4.0,-1.0]}}},{"id":"random/0068","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_i64","type":"i64"},{"name":"l5_f64","type":"f64"}],"columns":{"l0_utf8":["B","c","aa","😀","zz"],"l1_i32":[-4,3,1,-3,1],"l2_f64":[73.87,null,84.426,-1.0,null],"l3_utf8":[null,"zz",null,"a",null],"l4_i64":[null,"-5","-4611686018427387904","9007199254740992","9007199254740995"],"l5_f64":[-35.732,-1.0,null,6.722,4.0]}},"right":{"fields":[{"name":"l4_i64","type":"i64"},{"name":"r0_bool","type":"bool"}],"columns":{"r0_bool":[true,false,false,null,false,true,false,true,false,false],"l4_i64":["-5","9007199254740995","-5",null,"9007199254740992","9007199254740995","9007199254740992","-5",null,"-4611686018427387904"]}}},"ops":[{"op":"join","right":"right","on":["l4_i64"],"how":"left"}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_i64","type":"i64"},{"name":"l5_f64","type":"f64"},{"name":"r0_bool","type":"bool"}],"columns":{"l0_utf8":["B","c","c","c","aa","😀","😀","zz","zz"],"l1_i32":[-4,3,3,3,1,-3,-3,1,1],"l2_f64":[73.87,null,null,null,84.426,-1.0,-1.0,null,null],"l3_utf8":[null,"zz","zz","zz",null,"a","a",null,null],"l4_i64":[null,"-5","-5","-5","-4611686018427387904","9007199254740992","9007199254740992","9007199254740995","9007199254740995"],"l5_f64":[-35.732,-1.0,-1.0,-1.0,null,6.722,6.722,4.0,4.0],"r0_bool":[null,true,false,true,false,false,false,false,true]}}},{"id":"random/0069","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_f64","type":"f64"},{"name":"l4_f64","type":"f64"}],"columns":{"l0_utf8":["ab","😀","a","ab","é","zz","😀","a","zz","😀","B","B",null,"B","é","é","B","aa",null,"ab","😀","a","B","b","b","😀","B","",null,"a","B",null,"é","b",null,"ab","b","zz","b","b","b","é","b","ab","é","B","a",null,"ab","é","B","B","zz","aa","B",null,"",null,"B",null],"l1_i32":[-2147483648,2,null,-4,null,-3,4,4,-3,2147483647,-3,null,-3,1,1,null,-1,1,-1,-2147483648,-3,1,null,-2,2147483647,-3,4,-2147483648,3,-2,-4,-3,null,-3,4,-2,5,0,4,null,-2147483648,null,5,-1,1,4,-4,-1,-2,-3,-2,0,-1,5,-2,4,-4,null,null,-2],"l2_f64":[2.0,-88.772,-42.94,-22.512,null,-2.0,-41.04,null,-88.668,3.0,null,null,4.594,-4.0,32.416,1.0,71.105,-75.714,"Infinity",null,-47.726,3.0,-16.846,71.361,52.15,null,null,94.593,-25.42,-2.0,4.0,null,"NaN",3.0,null,-92.705,-96.427,3.0,-4.0,-15.385,2.0,-16.761,null,-62.691,-37.721,2.0,-94.352,80.945,-90.385,29.012,-21.229,74.907,1.0,"NaN",88.608,-42.186,-43.28,null,-27.2,0.0],"l3_f64":[-4.0,-4.0,72.528,70.408,60.168,0.0,-93.32,-1.0,-91.246,3.0,-2.0,3.0,2.0,-98.946,4.0,0.0,-4.0,null,2.0,-3.0,-4.0,-3.0,75.506,-73.662,-3.0,4.0,-2.0,4.0,1.0,null,96.558,-4.0,null,null,"Infinity",3.0,-82.404,-1.0,-24.563,78.754,"NaN",-4.0,-4.0,-3.0,90.136,-4.0,-3.0,4.0,null,null,-24.638,-1.0,4.0,-4.0,-28.057,43.994,-3.0,4.0,-95.748,null],"l4_f64":[-1.0,"NaN",2.0,"NaN",0.0,3.0,null,55.683,32.731,0.0,-75.018,-3.0,-2.0,null,"NaN",1.0,-16.965,-2.0,-1.0,null,-3.0,82.058,4.0,0.0,"NaN",-1.0,81.169,-3.0,4.0,23.218,-8.889,null,null,-80.473,-66.253,78.524,1.0,-65.198,90.632,2.0,-51.322,-17.538,-3.0,1.0,3.0,-4.0,-3.0,3.0,"-0",-45.295,3.0,5.028,47.162,null,null,-3.0,-35.381,42.283,89.886,2.0]}}},"ops":[{"op":"filter","predicate":{"fn":"lte","args":[{"fn":"sub","args":[{"col":"l3_f64"},{"col":"l3_f64"}]},{"fn":"when","args":[{"fn":"lte","args":[{"col":"l3_f64"},{"col":"l4_f64"}]},{"col":"l2_f64"},{"col":"l3_f64"}]}]}},{"op":"sort","keys":[{"expr":{"col":"l1_i32"},"descending":false},{"expr":{"col":"l0_utf8"},"descending":true}],"nullsLast":true},{"op":"filter","predicate":{"fn":"or","args":[{"fn":"gt","args":[{"lit":2.5},{"col":"l3_f64"}]},{"fn":"eq","args":[{"lit":0.5},{"col":"l2_f64"}]}]}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_f64","type":"f64"},{"name":"l4_f64","type":"f64"}],"columns":{"l0_utf8":["ab",null,"b",null,"B","a","B","b","é"],"l1_i32":[-2147483648,-3,-2,-1,0,1,4,2147483647,null],"l2_f64":[2.0,4.594,71.361,"Infinity",74.907,3.0,2.0,52.15,1.0],"l3_f64":[-4.0,2.0,-73.662,2.0,-1.0,-3.0,-4.0,-3.0,0.0],"l4_f64":[-1.0,-2.0,0.0,-1.0,5.028,82.058,-4.0,"NaN",1.0]}}},{"id":"random/0070","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_i32","type":"i32"}],"columns":{"l0_utf8":["é","é","ab","B","ab","ab","é","a","aa","ab","a","zz","c","","zz","😀","a","ab","c","😀"],"l1_i32":[2,-1,3,-5,0,-5,-1,-1,-4,-45,3,2,-1,-3,-3,3,-3,-1,-2,2],"l2_utf8":["é","B","B","c","a","a","é","b","","","zz","😀","b","c","","a","B","c","é",""],"l3_i32":[3,-1,4,346,-3,-3,-5,-3,0,-2,-4,-4,0,3,-1,2,-2,-5,-2,-2]}}},"ops":[{"op":"unique","columns":["l0_utf8"]},{"op":"sort","keys":[{"expr":{"col":"l1_i32"},"descending":false},{"expr":{"col":"l3_i32"},"descending":false}],"nullsLast":false},{"op":"withColumns","exprs":{"x2":{"fn":"neg","args":[{"fn":"mod","args":[{"col":"l3_i32"},{"col":"l1_i32"}]}]}}},{"op":"unique","columns":["l1_i32"]}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_i32","type":"i32"},{"name":"x2","type":"i32"}],"columns":{"l0_utf8":["B","aa","","a","zz","😀"],"l1_i32":[-5,-4,-3,-1,2,3],"l2_utf8":["c","","c","b","😀","a"],"l3_i32":[346,0,3,-3,-4,2],"x2":[4,0,0,0,0,-2]}}},{"id":"random/0071","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_bool","type":"bool"},{"name":"l4_i64","type":"i64"},{"name":"l5_f64","type":"f64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_utf8":[],"l3_bool":[],"l4_i64":[],"l5_f64":[]}}},"ops":[{"op":"sort","keys":[{"expr":{"col":"l2_utf8"},"descending":true}],"nullsLast":false}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_bool","type":"bool"},{"name":"l4_i64","type":"i64"},{"name":"l5_f64","type":"f64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_utf8":[],"l3_bool":[],"l4_i64":[],"l5_f64":[]}}},{"id":"random/0072","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_f64","type":"f64"},{"name":"l5_i64","type":"i64"},{"name":"l6_f64","type":"f64"}],"columns":{"l0_utf8":["a","B","aa","c","B","é","zz","aa","B","😀","b","é","c","😀","a","aa","😀","zz","b","ab","c","b","zz","zz","b","zz","zz","a","a","B","é","é","b","é","é","zz","b","a","b","c","zz","B","zz","é","😀","B","ab","ab","b","aa","a","ab","a","a","","é","zz","zz","zz","aa"],"l1_i32":[1,0,-5,-4,5,0,1,-1,-5,0,-4,-3,1,1,5,-2,0,-1,-3,-5,-3,-1,-1,2,-2,-5,3,-5,4,-4,0,-4,-3,-2,-4,2,-2,1,-4,-1,2,-2,-2147483648,2,4,0,-4,-657,5,-3,-3,5,-5,-1,3,5,0,5,2,4],"l2_f64":[3.0,2.0,0.0,1.0,62.583,4.0,28.034,1.0,-1.0,63.127,4.0,"NaN",2.419,4.0,86.754,-3.0,84.651,-37.701,15.35,4.0,"NaN",77.174,-2.0,3.0,1.0,-80.937,-20.164,-2.0,3.0,7.038,-72.793,-2.0,"Infinity",-79.999,1.0,-3.0,2.0,-21.207,-90.84,"NaN",-3.0,-37.282,1.0,-1.0,1.0,55.814,-1.0,2.0,4.0,-1.0,-79.272,-99.055,2.0,3.0,93.0,3.0,63.422,3.0,-14.207,-3.0],"l3_utf8":["","zz","ab","😀","b","zz","aa","ab","aa","b","😀","B","é","","zz","c","zz","ab","c","😀","c","zz","ab","a","","é","B","ab","😀","ab","aa","😀","😀","zz","aa","a","zz","a","b","ab","","a","B","zz","😀","c","a","a","b","b","a","ab","zz","B","a","","B","c","é","é"],"l4_f64":[-77.483,-3.0,0.0,"Infinity",-1.0,57.049,-3.0,2.0,-1.0,75.3,34.184,4.0,0.0,25.856,-2.0,-2.0,39.886,12.863,-95.829,54.204,59.898,-3.0,4.0,1.0,"NaN",0.0,-42.762,3.0,"NaN",74.394,-55.311,-4.0,-18.89,60.932,-1.0,"-0",1.0,35.967,-92.403,75.5,0.0,1.0,51.843,1.0,14.051,-3.0,-3.0,-82.294,39.996,26.341,72.514,97.067,1.0,-69.065,-4.0,73.583,-4.0,34.37,53.009,3.0],"l5_i64":["9007199254740992","-2","9007199254740993","-4611686018427387904","9007199254740995","-4611686018427387904","9007199254740995","-4611686018427387904","9007199254740994","-5","-5","3","-4611686018427387904","1","-4611686018427387904","3","9007199254740992","9007199254740994","-4611686018427387904","-4611686018427387904","-4611686018427387904","-2","-5","9007199254740994","-4","-3","1","9007199254740994","9007199254740995","9007199254740993","-4611686018427387904","9007199254740994","9007199254740994","-4611686018427387904","-4611686018427387904","9007199254740993","-4611686018427387904","-4611686018427387904","9007199254740993","9007199254740992","-3","1","9007199254740992","-4611686018427387904","4","4","-4611686018427387904","9007199254740994","9007199254740993","9007199254740993","9007199254740992","9007199254740993","9007199254740995","-4611686018427387904","9007199254740992","9007199254740995","9007199254740995","-4611686018427387904","9007199254740995","-4611686018427387904"],"l6_f64":[-77.955,4.0,-2.0,-3.0,"Infinity",-14.667,-1.0,0.0,-5.382,-68.384,4.0,86.842,-3.0,0.0,-1.0,-75.106,2.0,-1.0,1.0,7.215,1.0,-80.72,12.674,-1.0,-1.0,0.0,3.0,-20.917,0.0,-2.0,2.0,97.303,-3.0,21.228,3.0,-4.0,30.092,25.766,4.0,2.0,-2.0,-1.0,2.0,"-Infinity",-3.0,-2.0,-4.0,18.634,64.738,-4.0,0.0,-2.0,-8.785,-1.112,-55.423,-2.0,-2.0,-36.497,53.025,-39.096]}}},"ops":[{"op":"filter","predicate":{"fn":"eq","args":[{"fn":"div","args":[{"col":"l1_i32"},{"col":"l1_i32"}]},{"col":"l1_i32"}]}},{"op":"unique"},{"op":"withColumns","exprs":{"x2":{"fn":"mod","args":[{"fn":"mul","args":[{"col":"l6_f64"},{"col":"l6_f64"}]},{"col":"l2_f64"}]}}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_f64","type":"f64"},{"name":"l5_i64","type":"i64"},{"name":"l6_f64","type":"f64"},{"name":"x2","type":"f64"}],"columns":{"l0_utf8":["a","zz","c","😀","a"],"l1_i32":[1,1,1,1,1],"l2_f64":[3.0,28.034,2.419,4.0,-21.207],"l3_utf8":["","aa","é","","a"],"l4_f64":[-77.483,-3.0,0.0,25.856,35.967],"l5_i64":["9007199254740992","9007199254740995","-4611686018427387904","1","-4611686018427387904"],"l6_f64":[-77.955,-1.0,-3.0,0.0,25.766],"x2":[1.9820249999993393,1.0,1.7430000000000003,0.0,-14.737244000000146]}}},{"id":"random/0073","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i64","type":"i64"},{"name":"l3_utf8","type":"utf8"}],"columns":{"l0_utf8":["b","b","😀","","a"],"l1_i32":[-811,0,-2,3,5],"l2_i64":["-2","-4611686018427387904","-2","-4611686018427387904","2"],"l3_utf8":["aa","aa","b","c","😀"]}}},"ops":[{"op":"head","n":2},{"op":"unique"},{"op":"withColumns","exprs":{"x2":{"fn":"when","args":[{"fn":"lte","args":[{"col":"l1_i32"},{"col":"l1_i32"}]},{"fn":"fillNull","args":[{"col":"l1_i32"},{"lit":1.5}]},{"lit":2}]}}},{"op":"unique"}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i64","type":"i64"},{"name":"l3_utf8","type":"utf8"},{"name":"x2","type":"f64"}],"columns":{"l0_utf8":["b","b"],"l1_i32":[-811,0],"l2_i64":["-2","-4611686018427387904"],"l3_utf8":["aa","aa"],"x2":[-811.0,0.0]}}},{"id":"random/0074","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_i64","type":"i64"}],"columns":{"l0_utf8":["é"],"l1_i32":[-5],"l2_i32":[-3],"l3_i64":["9007199254740994"]}}},"ops":[{"op":"head","n":2},{"op":"groupBy","keys":["l1_i32"],"aggs":{"n1":{"agg":"len"},"a1_0":{"agg":"max","arg":{"col":"l0_utf8"}}}}],"expected":{"fields":[{"name":"l1_i32","type":"i32"},{"name":"n1","type":"u32"},{"name":"a1_0","type":"utf8"}],"columns":{"l1_i32":[-5],"n1":[1],"a1_0":["é"]}}},{"id":"random/0075","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_i64","type":"i64"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_f64","type":"f64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_bool":[],"l3_i64":[],"l4_utf8":[],"l5_f64":[]}},"right":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"r0_utf8","type":"utf8"}],"columns":{"r0_utf8":["aa","é","ab"],"l0_utf8":["a","😀","b"]}}},"ops":[{"op":"join","right":"right","on":["l0_utf8"],"how":"anti"},{"op":"withColumns","exprs":{"x1":{"fn":"gt","args":[{"col":"l3_i64"},{"lit":"-2","type":"i64"}]}}},{"op":"sort","keys":[{"expr":{"col":"l1_i32"},"descending":false},{"expr":{"col":"l2_bool"},"descending":false}],"nullsLast":true}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_i64","type":"i64"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_f64","type":"f64"},{"name":"x1","type":"bool"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_bool":[],"l3_i64":[],"l4_utf8":[],"l5_f64":[],"x1":[]}}},{"id":"random/0076","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_i32","type":"i32"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_f64","type":"f64"},{"name":"l6_f64","type":"f64"}],"columns":{"l0_utf8":["a","B","c","a",""],"l1_i32":[-1,3,-2,1,3],"l2_utf8":["","B","a","é","😀"],"l3_i32":[2,4,4,2,0],"l4_utf8":["aa","ab","","aa","c"],"l5_f64":[59.826,-1.0,-53.594,60.915,-53.714],"l6_f64":[2.0,12.124,3.0,41.271,"NaN"]}},"right":{"fields":[{"name":"l3_i32","type":"i32"},{"name":"r0_bool","type":"bool"}],"columns":{"r0_bool":[],"l3_i32":[]}}},"ops":[{"op":"join","right":"right","on":["l3_i32"],"how":"semi"},{"op":"head","n":2}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_i32","type":"i32"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_f64","type":"f64"},{"name":"l6_f64","type":"f64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_utf8":[],"l3_i32":[],"l4_utf8":[],"l5_f64":[],"l6_f64":[]}}},{"id":"random/0077","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_f64","type":"f64"},{"name":"l4_i32","type":"i32"},{"name":"l5_bool","type":"bool"}],"columns":{"l0_utf8":["é","b","","😀","c","aa","é","","aa",null,"c","","","B","é",null,"a","","é","B"],"l1_i32":[5,-5,1,-4,null,1,null,0,null,-3,null,0,null,2147483647,1,-2,-5,-1,5,-3],"l2_utf8":[null,null,"B",null,"B","zz","aa","","c","é","","zz","zz",null,"c",null,"c","zz","😀","aa"],"l3_f64":[null,null,4.0,-63.695,0.0,9.115,2.0,null,-4.728,-59.593,-1.0,"NaN",30.796,1.0,null,-76.129,3.0,30.357,-56.62,-93.974],"l4_i32":[3,-3,2,-2,3,null,null,-2,-5,null,null,-3,5,2,null,-4,2147483647,null,-1,null],"l5_bool":[null,true,false,false,false,null,null,false,null,true,null,false,false,null,null,null,null,null,true,true]}},"right":{"fields":[{"name":"a0_0","type":"utf8"},{"name":"r0_f64","type":"f64"}],"columns":{"r0_f64":[],"a0_0":[]}}},"ops":[{"op":"groupBy","keys":["l1_i32"],"aggs":{"n0":{"agg":"len"},"a0_0":{"agg":"first","arg":{"col":"l2_utf8"}}}},{"op":"filter","predicate":{"fn":"gt","args":[{"col":"l1_i32"},{"col":"l1_i32"}]}},{"op":"join","right":"right","on":["a0_0"],"how":"semi"}],"expected":{"fields":[{"name":"l1_i32","type":"i32"},{"name":"n0","type":"u32"},{"name":"a0_0","type":"utf8"}],"columns":{"l1_i32":[],"n0":[],"a0_0":[]}}},{"id":"random/0078","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_f64","type":"f64"},{"name":"l4_utf8","type":"utf8"}],"columns":{"l0_utf8":[""],"l1_i32":[-1],"l2_f64":[null],"l3_f64":[-1.0],"l4_utf8":["b"]}}},"ops":[{"op":"withColumns","exprs":{"x0":{"fn":"gt","args":[{"fn":"mod","args":[{"lit":2.5},{"col":"l3_f64"}]},{"col":"l1_i32"}]}}},{"op":"head","n":10},{"op":"groupBy","keys":["l4_utf8"],"aggs":{"n2":{"agg":"len"},"a2_0":{"agg":"count","arg":{"col":"x0"}},"a2_1":{"agg":"first","arg":{"col":"l0_utf8"}}}},{"op":"filter","predicate":{"fn":"strStartsWith","args":[{"col":"a2_1"}],"prefix":"a"}}],"expected":{"fields":[{"name":"l4_utf8","type":"utf8"},{"name":"n2","type":"u32"},{"name":"a2_0","type":"u32"},{"name":"a2_1","type":"utf8"}],"columns":{"l4_utf8":[],"n2":[],"a2_0":[],"a2_1":[]}}},{"id":"random/0079","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_f64","type":"f64"},{"name":"l4_i64","type":"i64"},{"name":"l5_bool","type":"bool"},{"name":"l6_f64","type":"f64"}],"columns":{"l0_utf8":[null],"l1_i32":[null],"l2_i32":[4],"l3_f64":[null],"l4_i64":["9007199254740995"],"l5_bool":[true],"l6_f64":[21.361]}}},"ops":[{"op":"groupBy","keys":["l5_bool"],"aggs":{"n0":{"agg":"len"},"a0_0":{"agg":"last","arg":{"col":"l4_i64"}}}},{"op":"groupBy","keys":["l5_bool"],"aggs":{"n1":{"agg":"len"},"a1_0":{"agg":"nUnique","arg":{"col":"a0_0"}}}},{"op":"withColumns","exprs":{"x2":{"fn":"mul","args":[{"fn":"sub","args":[{"lit":0.5},{"lit":2.5}]},{"fn":"neg","args":[{"lit":2}]}]}}}],"expected":{"fields":[{"name":"l5_bool","type":"bool"},{"name":"n1","type":"u32"},{"name":"a1_0","type":"u32"},{"name":"x2","type":"f64"}],"columns":{"l5_bool":[true],"n1":[1],"a1_0":[1],"x2":[4.0]}}},{"id":"random/0080","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_utf8","type":"utf8"}],"columns":{"l0_utf8":["zz",null,"","😀",""],"l1_i32":[-4,-5,4,null,4],"l2_i32":[null,3,null,-2,0],"l3_utf8":["é","B","aa","a",null]}}},"ops":[{"op":"withColumns","exprs":{"x0":{"fn":"mul","args":[{"fn":"mod","args":[{"lit":0.5},{"col":"l1_i32"}]},{"fn":"fillNull","args":[{"col":"l2_i32"},{"lit":0}]}]}}},{"op":"filter","predicate":{"fn":"isIn","args":[{"col":"l3_utf8"}],"values":["é","ab","B"]}},{"op":"withColumns","exprs":{"x2":{"lit":2.5}}},{"op":"filter","predicate":{"fn":"isNotNull","args":[{"col":"x2"}]}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_utf8","type":"utf8"},{"name":"x0","type":"f64"},{"name":"x2","type":"f64"}],"columns":{"l0_utf8":["zz",null],"l1_i32":[-4,-5],"l2_i32":[null,3],"l3_utf8":["é","B"],"x0":["-0",-13.5],"x2":[2.5,2.5]}}},{"id":"random/0081","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_i32","type":"i32"}],"columns":{"l0_utf8":["😀",null,"c",null,"ab","ab","B",null,null,"c","ab","zz","aa",null,"aa","c","😀","ab",null,""],"l1_i32":[5,-3,-1,0,null,2,null,-4,null,-5,null,-3,2,3,5,-3,null,-5,4,-3],"l2_f64":[-4.802,75.504,-21.884,null,null,2.0,-21.413,-80.905,97.777,null,-1.0,55.903,1.0,-51.178,null,-85.919,-2.0,1.0,39.072,6.785],"l3_utf8":["b","zz",null,"ab","😀","a","aa","😀","zz","","é","a","b","b",null,"é","aa","","a",""],"l4_i32":[null,null,-5,-5,0,null,-5,-1,1,3,null,4,-3,null,-3,4,-2,5,0,null]}}},"ops":[{"op":"groupBy","keys":["l4_i32"],"aggs":{"n0":{"agg":"len"},"a0_0":{"agg":"median","arg":{"col":"l1_i32"}},"a0_1":{"agg":"last","arg":{"col":"l0_utf8"}},"a0_2":{"agg":"count","arg":{"col":"l1_i32"}}}}],"expected":{"fields":[{"name":"l4_i32","type":"i32"},{"name":"n0","type":"u32"},{"name":"a0_0","type":"f64"},{"name":"a0_1","type":"utf8"},{"name":"a0_2","type":"u32"}],"columns":{"l4_i32":[null,-5,0,-1,1,3,4,-3,-2,5],"n0":[6,3,2,1,1,1,2,2,1,1],"a0_0":[2.0,-0.5,4.0,-4.0,null,-5.0,-3.0,3.5,null,-5.0],"a0_1":["","B",null,null,null,"c","c","aa","😀","ab"],"a0_2":[5,2,1,1,0,1,2,2,0,1]}}},{"id":"random/0082","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_f64","type":"f64"},{"name":"l4_f64","type":"f64"}],"columns":{"l0_utf8":["b","é","b","a","a","ab","zz","","ab","B","zz","","ab","a","zz","b","B","é","zz","aa","b","c","a","ab","😀","é","zz","😀","a","ab","","é","aa","b","ab","c","ab","zz","zz","c","a","aa","aa","b","c","c","a","aa","a","c","é","é","é","é","c","😀","B","c","b","B"],"l1_i32":[2147483647,-2,0,5,0,4,-339,1,-2147483648,-2,1,-4,1,2,4,-5,0,-2,0,4,-147,0,-1,3,-3,-4,3,0,5,-5,0,-2147483648,4,-2,5,5,3,3,-4,-2,-5,4,-2,2,-4,-2147483648,-2,-2,-5,-2147483648,-1,5,2,3,-5,1,0,1,5,0],"l2_i32":[-4,-2,-870,0,2,0,-4,2,3,0,-3,-2,-1,4,-3,4,-507,-3,3,1,0,598,-2,2,-350,-1,-2,2,-3,2,5,-4,5,-5,-5,0,0,-3,-3,-5,5,-5,2,2,1,4,-2,0,-1,1,-4,5,3,4,5,3,3,-3,2,-2147483648],"l3_f64":[-3.0,33.838,-2.0,-91.595,-92.14,1.0,-3.0,-4.0,1.0,-1.0,8.86,57.037,"NaN",63.483,52.37,-1.0,4.0,-4.0,-7.472,-28.24,-18.534,-1.0,-64.607,2.0,-3.0,-42.092,25.723,69.112,0.0,2.0,4.0,2.182,4.0,"NaN",-2.0,-3.0,3.0,-2.0,-10.813,2.0,3.0,2.0,-91.656,"NaN",2.0,0.0,-3.0,1.0,1.0,-3.0,70.652,-3.0,51.01,-4.0,-3.0,-96.0,-81.575,2.0,-3.0,-2.0],"l4_f64":[1.0,99.592,-4.0,-72.414,-2.0,"NaN",-74.537,1.0,-3.0,-39.215,-7.284,2.0,-50.389,16.421,-68.618,-9.229,88.097,-18.361,97.282,-47.624,-99.102,-66.447,3.0,0.0,-24.103,2.0,4.0,4.0,78.008,1.0,45.528,-2.0,66.377,14.043,2.0,-83.24,-68.059,65.037,2.0,-2.0,"-0",-2.0,-4.0,2.0,2.0,3.0,-3.0,-12.036,-0.095,-79.071,13.979,-5.327,-43.259,-3.0,-50.46,-62.016,-27.288,"-Infinity",-1.013,0.0]}}},"ops":[{"op":"withColumns","exprs":{"x0":{"fn":"when","args":[{"fn":"isNotNull","args":[{"col":"l2_i32"}]},{"fn":"mod","args":[{"col":"l1_i32"},{"col":"l1_i32"}]},{"col":"l3_f64"}]}}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_f64","type":"f64"},{"name":"l4_f64","type":"f64"},{"name":"x0","type":"f64"}],"columns":{"l0_utf8":["b","é","b","a","a","ab","zz","","ab","B","zz","","ab","a","zz","b","B","é","zz","aa","b","c","a","ab","😀","é","zz","😀","a","ab","","é","aa","b","ab","c","ab","zz","zz","c","a","aa","aa","b","c","c","a","aa","a","c","é","é","é","é","c","😀","B","c","b","B"],"l1_i32":[2147483647,-2,0,5,0,4,-339,1,-2147483648,-2,1,-4,1,2,4,-5,0,-2,0,4,-147,0,-1,3,-3,-4,3,0,5,-5,0,-2147483648,4,-2,5,5,3,3,-4,-2,-5,4,-2,2,-4,-2147483648,-2,-2,-5,-2147483648,-1,5,2,3,-5,1,0,1,5,0],"l2_i32":[-4,-2,-870,0,2,0,-4,2,3,0,-3,-2,-1,4,-3,4,-507,-3,3,1,0,598,-2,2,-350,-1,-2,2,-3,2,5,-4,5,-5,-5,0,0,-3,-3,-5,5,-5,2,2,1,4,-2,0,-1,1,-4,5,3,4,5,3,3,-3,2,-2147483648],"l3_f64":[-3.0,33.838,-2.0,-91.595,-92.14,1.0,-3.0,-4.0,1.0,-1.0,8.86,57.037,"NaN",63.483,52.37,-1.0,4.0,-4.0,-7.472,-28.24,-18.534,-1.0,-64.607,2.0,-3.0,-42.092,25.723,69.112,0.0,2.0,4.0,2.182,4.0,"NaN",-2.0,-3.0,3.0,-2.0,-10.813,2.0,3.0,2.0,-91.656,"NaN",2.0,0.0,-3.0,1.0,1.0,-3.0,70.652,-3.0,51.01,-4.0,-3.0,-96.0,-81.575,2.0,-3.0,-2.0],"l4_f64":[1.0,99.592,-4.0,-72.414,-2.0,"NaN",-74.537,1.0,-3.0,-39.215,-7.284,2.0,-50.389,16.421,-68.618,-9.229,88.097,-18.361,97.282,-47.624,-99.102,-66.447,3.0,0.0,-24.103,2.0,4.0,4.0,78.008,1.0,45.528,-2.0,66.377,14.043,2.0,-83.24,-68.059,65.037,2.0,-2.0,"-0",-2.0,-4.0,2.0,2.0,3.0,-3.0,-12.036,-0.095,-79.071,13.979,-5.327,-43.259,-3.0,-50.46,-62.016,-27.288,"-Infinity",-1.013,0.0],"x0":[0.0,0.0,null,0.0,null,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,null,0.0,null,0.0,0.0,null,0.0,0.0,0.0,0.0,0.0,null,0.0,0.0,null,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,null,0.0,0.0,null]}}},{"id":"random/0083","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_bool","type":"bool"},{"name":"l5_f64","type":"f64"},{"name":"l6_utf8","type":"utf8"}],"columns":{"l0_utf8":["B"],"l1_i32":[-4],"l2_utf8":["b"],"l3_utf8":["b"],"l4_bool":[false],"l5_f64":[-4.0],"l6_utf8":["a"]}}},"ops":[{"op":"withColumns","exprs":{"x0":{"fn":"mul","args":[{"col":"l1_i32"},{"col":"l1_i32"}]}}},{"op":"withColumns","exprs":{"x1":{"fn":"isNull","args":[{"col":"l2_utf8"}]}}},{"op":"unique"},{"op":"head","n":3}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_bool","type":"bool"},{"name":"l5_f64","type":"f64"},{"name":"l6_utf8","type":"utf8"},{"name":"x0","type":"i32"},{"name":"x1","type":"bool"}],"columns":{"l0_utf8":["B"],"l1_i32":[-4],"l2_utf8":["b"],"l3_utf8":["b"],"l4_bool":[false],"l5_f64":[-4.0],"l6_utf8":["a"],"x0":[16],"x1":[false]}}},{"id":"random/0084","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_i32","type":"i32"},{"name":"l4_f64","type":"f64"},{"name":"l5_f64","type":"f64"},{"name":"l6_i64","type":"i64"}],"columns":{"l0_utf8":["c","é","B","","a"],"l1_i32":[null,0,null,1,null],"l2_f64":["NaN",2.318,null,1.0,null],"l3_i32":[null,-5,null,1,null],"l4_f64":[3.0,-3.0,"NaN","NaN",null],"l5_f64":[-36.112,93.106,-3.698,-1.0,-54.703],"l6_i64":[null,"9007199254740993",null,"9007199254740992","9007199254740992"]}},"right":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"r0_f64","type":"f64"}],"columns":{"r0_f64":[-1.0,"NaN",4.0,-4.0,3.0,-1.886,3.0,33.672,59.986,4.785,87.187,-97.497,-3.0,-3.0,-45.445,22.282,67.684,"NaN",-1.0,-2.0,21.07,-4.0,99.142,73.946,1.0,-21.858,-2.0,6.228,2.0,-60.12],"l0_utf8":["B","a","é","B","a","a","c","é","B","é","c","","a","B","é","B","a","","","a","","","c","","c","B","","a","a","é"]}}},"ops":[{"op":"filter","predicate":{"fn":"eq","args":[{"col":"l6_i64"},{"lit":"2","type":"i64"}]}},{"op":"join","right":"right","on":["l0_utf8"],"how":"anti"},{"op":"sort","keys":[{"expr":{"col":"l5_f64"},"descending":true},{"expr":{"col":"l0_utf8"},"descending":true}],"nullsLast":false}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_i32","type":"i32"},{"name":"l4_f64","type":"f64"},{"name":"l5_f64","type":"f64"},{"name":"l6_i64","type":"i64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_f64":[],"l3_i32":[],"l4_f64":[],"l5_f64":[],"l6_i64":[]}}},{"id":"random/0085","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_i64","type":"i64"},{"name":"l4_f64","type":"f64"},{"name":"l5_i64","type":"i64"},{"name":"l6_bool","type":"bool"}],"columns":{"l0_utf8":["a"],"l1_i32":[1],"l2_i32":[-5],"l3_i64":["-4611686018427387904"],"l4_f64":[-4.0],"l5_i64":["-4611686018427387904"],"l6_bool":[true]}}},"ops":[{"op":"sort","keys":[{"expr":{"col":"l0_utf8"},"descending":true}],"nullsLast":true}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_i64","type":"i64"},{"name":"l4_f64","type":"f64"},{"name":"l5_i64","type":"i64"},{"name":"l6_bool","type":"bool"}],"columns":{"l0_utf8":["a"],"l1_i32":[1],"l2_i32":[-5],"l3_i64":["-4611686018427387904"],"l4_f64":[-4.0],"l5_i64":["-4611686018427387904"],"l6_bool":[true]}}},{"id":"random/0086","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_bool","type":"bool"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_f64":[],"l3_utf8":[],"l4_bool":[]}},"right":{"fields":[{"name":"l3_utf8","type":"utf8"},{"name":"r0_utf8","type":"utf8"}],"columns":{"r0_utf8":["c","B",""],"l3_utf8":[null,null,null]}}},"ops":[{"op":"filter","predicate":{"fn":"strStartsWith","args":[{"col":"l3_utf8"}],"prefix":""}},{"op":"filter","predicate":{"col":"l4_bool"}},{"op":"join","right":"right","on":["l3_utf8"],"how":"anti"},{"op":"filter","predicate":{"fn":"gte","args":[{"fn":"when","args":[{"fn":"isNull","args":[{"col":"l3_utf8"}]},{"col":"l1_i32"},{"col":"l1_i32"}]},{"fn":"div","args":[{"col":"l2_f64"},{"col":"l1_i32"}]}]}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_bool","type":"bool"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_f64":[],"l3_utf8":[],"l4_bool":[]}}},{"id":"random/0087","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_f64","type":"f64"},{"name":"l5_utf8","type":"utf8"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_utf8":[],"l3_utf8":[],"l4_f64":[],"l5_utf8":[]}}},"ops":[{"op":"withColumns","exprs":{"x0":{"fn":"fillNull","args":[{"fn":"when","args":[{"fn":"isNull","args":[{"col":"l5_utf8"}]},{"col":"l4_f64"},{"col":"l4_f64"}]},{"lit":0}]}}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_f64","type":"f64"},{"name":"l5_utf8","type":"utf8"},{"name":"x0","type":"f64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_utf8":[],"l3_utf8":[],"l4_f64":[],"l5_utf8":[],"x0":[]}}},{"id":"random/0088","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_f64","type":"f64"}],"columns":{"l0_utf8":["é","B","","zz","😀"],"l1_i32":[5,-1,-4,-4,-943],"l2_utf8":["a","aa","c","aa","a"],"l3_f64":[-3.0,4.0,-28.158,"-Infinity",2.0]}}},"ops":[{"op":"withColumns","exprs":{"x0":{"fn":"mod","args":[{"col":"l3_f64"},{"col":"l3_f64"}]}}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_f64","type":"f64"},{"name":"x0","type":"f64"}],"columns":{"l0_utf8":["é","B","","zz","😀"],"l1_i32":[5,-1,-4,-4,-943],"l2_utf8":["a","aa","c","aa","a"],"l3_f64":[-3.0,4.0,-28.158,"-Infinity",2.0],"x0":[0.0,0.0,0.0,"NaN",0.0]}}},{"id":"random/0089","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_f64","type":"f64"}],"columns":{"l0_utf8":["aa"],"l1_i32":[0],"l2_i32":[1],"l3_utf8":["aa"],"l4_f64":[-2.0]}}},"ops":[{"op":"head","n":5},{"op":"filter","predicate":{"fn":"lte","args":[{"fn":"floorDiv","args":[{"col":"l2_i32"},{"col":"l2_i32"}]},{"fn":"sub","args":[{"col":"l1_i32"},{"col":"l1_i32"}]}]}},{"op":"filter","predicate":{"fn":"isNotNull","args":[{"col":"l2_i32"}]}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_f64","type":"f64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_i32":[],"l3_utf8":[],"l4_f64":[]}}},{"id":"random/0090","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_utf8","type":"utf8"}],"columns":{"l0_utf8":["B","","","ab","B","ab","c","b","a","b","😀",null,"zz","ab","😀","b","😀","b","é","a","😀",null,"c","","aa","zz","zz","ab","b","a","zz","😀","é","aa","é","😀","aa",null,"b","zz","zz",null,"a","b","a","","","b","",null,"é","zz","c","aa","b","","b","😀","","B"],"l1_i32":[-1,0,3,-3,-1,2,2,1,-3,-4,null,-4,-5,4,2147483647,3,3,-2147483648,-2,-5,2147483647,5,5,-3,-3,-4,null,0,-4,0,-3,5,-2,-4,1,2,4,0,-1,-2147483648,-3,-2,4,null,-5,-3,-5,-2,-1,-2,-2,3,0,-2,-1,3,null,1,3,2],"l2_bool":[false,false,false,null,false,true,true,true,null,false,false,false,null,false,true,true,null,false,true,false,true,true,true,true,true,null,false,false,true,true,false,false,false,null,false,false,false,false,true,false,false,false,false,true,true,false,false,false,true,true,false,null,true,false,false,false,true,true,null,false],"l3_utf8":["B",null,"","c","b","😀","","c","c","aa","aa","aa","a","b","c",null,"😀","ab","b","b","😀","zz","é","","c","b","zz","ab","a","B","é","B","é","b","a",null,"zz","B",null,"zz","😀","zz","aa","zz","c","zz","c","zz","é","c","zz","ab","a","b","zz",null,"c","ab","c","😀"]}},"right":{"fields":[{"name":"l3_utf8","type":"utf8"},{"name":"r0_f64","type":"f64"}],"columns":{"r0_f64":[-50.26,-12.185,52.145],"l3_utf8":["c","ab","c"]}}},"ops":[{"op":"filter","predicate":{"fn":"gte","args":[{"fn":"abs","args":[{"col":"l1_i32"}]},{"fn":"floorDiv","args":[{"col":"l1_i32"},{"col":"l1_i32"}]}]}},{"op":"withColumns","exprs":{"x1":{"fn":"isNull","args":[{"col":"l1_i32"}]}}},{"op":"join","right":"right","on":["l3_utf8"],"how":"left"}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_utf8","type":"utf8"},{"name":"x1","type":"bool"},{"name":"r0_f64","type":"f64"}],"columns":{"l0_utf8":["B","","ab","ab","B","ab","c","b","b","a","a","b",null,"zz","ab","😀","😀","b","😀","é","a","😀",null,"c","","aa","aa","zz","b","zz","😀","é","aa","é","😀","aa","b","zz",null,"a","a","a","","","","b","",null,null,"é","zz","aa","b","","😀","","","B"],"l1_i32":[-1,3,-3,-3,-1,2,2,1,1,-3,-3,-4,-4,-5,4,2147483647,2147483647,3,3,-2,-5,2147483647,5,5,-3,-3,-3,-4,-4,-3,5,-2,-4,1,2,4,-1,-3,-2,4,-5,-5,-3,-5,-5,-2,-1,-2,-2,-2,3,-2,-1,3,1,3,3,2],"l2_bool":[false,false,null,null,false,true,true,true,true,null,null,false,false,null,false,true,true,true,null,true,false,true,true,true,true,true,true,null,true,false,false,false,null,false,false,false,true,false,false,false,true,true,false,false,false,false,true,true,true,false,null,false,false,false,true,null,null,false],"l3_utf8":["B","","c","c","b","😀","","c","c","c","c","aa","aa","a","b","c","c",null,"😀","b","b","😀","zz","é","","c","c","b","a","é","B","é","b","a",null,"zz",null,"😀","zz","aa","c","c","zz","c","c","zz","é","c","c","zz","ab","b","zz",null,"ab","c","c","😀"],"x1":[false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false],"r0_f64":[null,null,-50.26,52.145,null,null,null,-50.26,52.145,-50.26,52.145,null,null,null,null,-50.26,52.145,null,null,null,null,null,null,null,null,-50.26,52.145,null,null,null,null,null,null,null,null,null,null,null,null,null,-50.26,52.145,null,-50.26,52.145,null,null,-50.26,52.145,null,-12.185,null,null,null,-12.185,-50.26,52.145,null]}}},{"id":"random/0091","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_bool","type":"bool"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_f64","type":"f64"},{"name":"l6_f64","type":"f64"}],"columns":{"l0_utf8":[null],"l1_i32":[-2147483648],"l2_utf8":["c"],"l3_bool":[true],"l4_utf8":["é"],"l5_f64":[38.674],"l6_f64":[-4.481]}}},"ops":[{"op":"unique"},{"op":"groupBy","keys":["l4_utf8"],"aggs":{"n1":{"agg":"len"},"a1_0":{"agg":"count","arg":{"col":"l0_utf8"}},"a1_1":{"agg":"max","arg":{"col":"l5_f64"}},"a1_2":{"agg":"mean","arg":{"col":"l1_i32"}}}},{"op":"head","n":2}],"expected":{"fields":[{"name":"l4_utf8","type":"utf8"},{"name":"n1","type":"u32"},{"name":"a1_0","type":"u32"},{"name":"a1_1","type":"f64"},{"name":"a1_2","type":"f64"}],"columns":{"l4_utf8":["é"],"n1":[1],"a1_0":[0],"a1_1":[38.674],"a1_2":[-2147483648.0]}}},{"id":"random/0092","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_i64","type":"i64"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_i32","type":"i32"},{"name":"l6_i64","type":"i64"}],"columns":{"l0_utf8":[null,"aa",null,null,null,"😀","a","a","b","aa","b",null,null,"B","😀","é","B","",null,"ab"],"l1_i32":[-2,-5,null,null,null,-3,null,1,null,null,2,-3,-1,-4,null,5,-3,null,-4,-1],"l2_i32":[2,0,null,-4,-4,5,-5,0,null,4,2,-5,-5,1,3,null,2,-5,0,null],"l3_i64":["9007199254740994","9007199254740993","9007199254740993","-4611686018427387904",null,"5","-4611686018427387904","0",null,null,null,"9007199254740994","-4611686018427387904","9007199254740995","9007199254740995","9007199254740992",null,"1","9007199254740995","9007199254740992"],"l4_utf8":[null,"a",null,"","c","",null,"é","😀","zz",null,null,"","aa",null,"B","a","é","B","aa"],"l5_i32":[-2,null,null,null,null,-319,null,1,null,1,1,null,-5,null,2,null,null,-1,3,-5],"l6_i64":["9007199254740992","1","9007199254740992","-4611686018427387904","-3","2","3","-4611686018427387904","9007199254740995","-4611686018427387904","-3","9007199254740993",null,null,"9007199254740995","9007199254740992","9007199254740992","-4611686018427387904","4","-4611686018427387904"]}}},"ops":[{"op":"head","n":1}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_i64","type":"i64"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_i32","type":"i32"},{"name":"l6_i64","type":"i64"}],"columns":{"l0_utf8":[null],"l1_i32":[-2],"l2_i32":[2],"l3_i64":["9007199254740994"],"l4_utf8":[null],"l5_i32":[-2],"l6_i64":["9007199254740992"]}}},{"id":"random/0093","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_bool","type":"bool"},{"name":"l4_f64","type":"f64"}],"columns":{"l0_utf8":["😀"],"l1_i32":[-2],"l2_bool":[true],"l3_bool":[false],"l4_f64":["NaN"]}}},"ops":[{"op":"sort","keys":[{"expr":{"col":"l1_i32"},"descending":false},{"expr":{"col":"l3_bool"},"descending":false}],"nullsLast":false},{"op":"unique","columns":["l1_i32"]},{"op":"head","n":2}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_bool","type":"bool"},{"name":"l4_f64","type":"f64"}],"columns":{"l0_utf8":["😀"],"l1_i32":[-2],"l2_bool":[true],"l3_bool":[false],"l4_f64":["NaN"]}}},{"id":"random/0094","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i64","type":"i64"},{"name":"l3_f64","type":"f64"},{"name":"l4_bool","type":"bool"},{"name":"l5_i32","type":"i32"}],"columns":{"l0_utf8":["😀","","😀","aa","b","","b","é","😀","ab","","c","c","ab","b",null,"c",null,"😀","aa"],"l1_i32":[0,2,5,null,4,null,-4,0,5,-2,null,1,2,-5,5,1,4,5,null,5],"l2_i64":["2","0","-4611686018427387904","9007199254740992","9007199254740994","9007199254740992","9007199254740995","0","9007199254740993","-4611686018427387904","9007199254740993","-4611686018427387904","5","-4611686018427387904","9007199254740995","9007199254740994","2","3","9007199254740995",null],"l3_f64":[83.107,-2.0,4.0,-1.0,2.0,null,34.601,null,26.729,4.0,-10.973,-3.0,-16.656,-4.0,null,3.0,77.078,-2.0,-12.471,"NaN"],"l4_bool":[true,true,false,false,false,true,false,true,false,false,false,false,true,false,true,false,false,false,true,false],"l5_i32":[-4,-5,1,null,2,-2,null,-1,-5,-2,-2,2,-4,-2,-3,2,-4,-2147483648,2147483647,2]}}},"ops":[{"op":"withColumns","exprs":{"x0":{"col":"l3_f64"}}},{"op":"head","n":0},{"op":"head","n":2}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i64","type":"i64"},{"name":"l3_f64","type":"f64"},{"name":"l4_bool","type":"bool"},{"name":"l5_i32","type":"i32"},{"name":"x0","type":"f64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_i64":[],"l3_f64":[],"l4_bool":[],"l5_i32":[],"x0":[]}}},{"id":"random/0095","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_i64","type":"i64"},{"name":"l4_i32","type":"i32"},{"name":"l5_i32","type":"i32"}],"columns":{"l0_utf8":["aa","c","b","é",null],"l1_i32":[0,5,5,5,3],"l2_f64":[-97.596,3.519,null,null,4.0],"l3_i64":["-4611686018427387904","9007199254740995","9007199254740994","9007199254740992","-3"],"l4_i32":[1,null,-4,3,4],"l5_i32":[2147483647,5,5,null,null]}}},"ops":[{"op":"withColumns","exprs":{"x0":{"col":"l4_i32"}}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_i64","type":"i64"},{"name":"l4_i32","type":"i32"},{"name":"l5_i32","type":"i32"},{"name":"x0","type":"i32"}],"columns":{"l0_utf8":["aa","c","b","é",null],"l1_i32":[0,5,5,5,3],"l2_f64":[-97.596,3.519,null,null,4.0],"l3_i64":["-4611686018427387904","9007199254740995","9007199254740994","9007199254740992","-3"],"l4_i32":[1,null,-4,3,4],"l5_i32":[2147483647,5,5,null,null],"x0":[1,null,-4,3,4]}}},{"id":"random/0096","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_i32","type":"i32"},{"name":"l6_i64","type":"i64"}],"columns":{"l0_utf8":["","aa","😀","B","B"],"l1_i32":[5,5,5,-1,5],"l2_f64":[28.568,93.458,96.591,22.159,3.0],"l3_utf8":["b","B","😀","😀","B"],"l4_utf8":["a","","b","a","a"],"l5_i32":[-4,-4,-5,-5,-1],"l6_i64":["-5","4","9007199254740993","-4611686018427387904","-4611686018427387904"]}}},"ops":[{"op":"head","n":7},{"op":"groupBy","keys":["l0_utf8"],"aggs":{"n1":{"agg":"len"},"a1_0":{"agg":"last","arg":{"col":"l1_i32"}},"a1_1":{"agg":"nUnique","arg":{"col":"l5_i32"}}}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"n1","type":"u32"},{"name":"a1_0","type":"i32"},{"name":"a1_1","type":"u32"}],"columns":{"l0_utf8":["","aa","😀","B"],"n1":[1,1,1,2],"a1_0":[5,5,5,5],"a1_1":[1,1,1,2]}}},{"id":"random/0097","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_f64","type":"f64"}],"columns":{"l0_utf8":["b",null,"é","a","B"],"l1_i32":[-5,null,3,null,-3],"l2_utf8":["😀","c","ab","😀","ab"],"l3_f64":[null,null,-48.169,-66.332,87.716]}},"right":{"fields":[{"name":"l2_utf8","type":"utf8"},{"name":"r0_f64","type":"f64"}],"columns":{"r0_f64":[null,-15.99,0.0,37.436,null,0.0,-4.0,-24.324,-2.0,-1.0],"l2_utf8":["c","ab","ab","😀","😀","😀","😀","😀","c","ab"]}}},"ops":[{"op":"join","right":"right","on":["l2_utf8"],"how":"full"},{"op":"sort","keys":[{"expr":{"col":"l1_i32"},"descending":false},{"expr":{"col":"l2_utf8"},"descending":true}],"nullsLast":false},{"op":"unique","columns":["l0_utf8"]}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_f64","type":"f64"},{"name":"r0_f64","type":"f64"}],"columns":{"l0_utf8":["a",null,"b","B","é"],"l1_i32":[null,null,-5,-3,3],"l2_utf8":["😀","c","😀","ab","ab"],"l3_f64":[-66.332,null,null,87.716,-48.169],"r0_f64":[37.436,null,37.436,-15.99,-15.99]}}},{"id":"random/0098","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_f64","type":"f64"},{"name":"l4_f64","type":"f64"}],"columns":{"l0_utf8":["ab","B","a","ab","😀"],"l1_i32":[null,null,4,null,-3],"l2_f64":[99.465,null,3.413,4.0,0.0],"l3_f64":[null,null,-4.0,4.0,null],"l4_f64":[-3.0,-4.0,-63.113,-4.0,-2.0]}}},"ops":[{"op":"filter","predicate":{"fn":"strStartsWith","args":[{"col":"l0_utf8"}],"prefix":"a"}},{"op":"filter","predicate":{"fn":"isNull","args":[{"col":"l3_f64"}]}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_f64","type":"f64"},{"name":"l4_f64","type":"f64"}],"columns":{"l0_utf8":["ab"],"l1_i32":[null],"l2_f64":[99.465],"l3_f64":[null],"l4_f64":[-3.0]}}},{"id":"random/0099","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_f64","type":"f64"}],"columns":{"l0_utf8":[null,null,null,null,"ab","zz","😀",null,null,null,"",null,"b","zz","ab","😀",null,"😀",null,"aa"],"l1_i32":[-4,null,null,null,4,null,null,null,3,-5,-3,-4,2,5,0,null,2,null,null,5],"l2_f64":[4.0,null,14.586,3.0,-1.0,null,3.0,-3.0,null,4.0,0.0,75.037,null,null,3.0,null,1.0,68.395,-81.302,-32.419],"l3_f64":[3.0,69.85,null,-3.0,3.0,null,null,43.911,0.0,80.338,1.0,-28.491,null,null,-14.784,null,-6.82,-2.0,3.0,null]}},"right":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"r0_f64","type":"f64"}],"columns":{"r0_f64":[],"l0_utf8":[]}}},"ops":[{"op":"unique","columns":["l0_utf8"]},{"op":"groupBy","keys":["l0_utf8"],"aggs":{"n1":{"agg":"len"},"a1_0":{"agg":"mean","arg":{"col":"l1_i32"}},"a1_1":{"agg":"max","arg":{"col":"l2_f64"}}}},{"op":"join","right":"right","on":["l0_utf8"],"how":"left"}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"n1","type":"u32"},{"name":"a1_0","type":"f64"},{"name":"a1_1","type":"f64"},{"name":"r0_f64","type":"f64"}],"columns":{"l0_utf8":[null,"ab","zz","😀","","b","aa"],"n1":[1,1,1,1,1,1,1],"a1_0":[-4.0,4.0,null,null,-3.0,2.0,5.0],"a1_1":[4.0,-1.0,null,3.0,0.0,null,-32.419],"r0_f64":[null,null,null,null,null,null,null]}}},{"id":"random/0100","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i64","type":"i64"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_utf8","type":"utf8"}],"columns":{"l0_utf8":["B","é","c",null,null],"l1_i32":[-3,5,4,-3,3],"l2_i64":["-4611686018427387904","-4611686018427387904","9007199254740992",null,"-2"],"l3_utf8":["","é","é","aa",null],"l4_utf8":["zz","😀","😀","B","b"]}}},"ops":[{"op":"sort","keys":[{"expr":{"col":"l2_i64"},"descending":true}],"nullsLast":true},{"op":"head","n":2},{"op":"withColumns","exprs":{"x2":{"fn":"isIn","args":[{"col":"l3_utf8"}],"values":["","b","ab"]}}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i64","type":"i64"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_utf8","type":"utf8"},{"name":"x2","type":"bool"}],"columns":{"l0_utf8":["c",null],"l1_i32":[4,3],"l2_i64":["9007199254740992","-2"],"l3_utf8":["é",null],"l4_utf8":["😀","b"],"x2":[false,null]}}},{"id":"random/0101","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_i32","type":"i32"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_bool","type":"bool"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_f64":[],"l3_i32":[],"l4_utf8":[],"l5_bool":[]}}},"ops":[{"op":"unique","columns":["l4_utf8"]},{"op":"filter","predicate":{"col":"l5_bool"}},{"op":"filter","predicate":{"fn":"gte","args":[{"fn":"neg","args":[{"col":"l1_i32"}]},{"fn":"floorDiv","args":[{"col":"l1_i32"},{"col":"l1_i32"}]}]}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_i32","type":"i32"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_bool","type":"bool"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_f64":[],"l3_i32":[],"l4_utf8":[],"l5_bool":[]}}},{"id":"random/0102","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_i32","type":"i32"},{"name":"l4_bool","type":"bool"}],"columns":{"l0_utf8":[""],"l1_i32":[3],"l2_bool":[true],"l3_i32":[-3],"l4_bool":[false]}},"right":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"r0_bool","type":"bool"}],"columns":{"r0_bool":[null,false,true,true,null,null,false,null,false,false],"l0_utf8":["","","","","","","","","",""]}}},"ops":[{"op":"withColumns","exprs":{"x0":{"fn":"neg","args":[{"fn":"mod","args":[{"lit":2.5},{"lit":2.5}]}]}}},{"op":"filter","predicate":{"fn":"isNull","args":[{"col":"l3_i32"}]}},{"op":"join","right":"right","on":["l0_utf8"],"how":"anti"},{"op":"unique","columns":["l1_i32"]}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_i32","type":"i32"},{"name":"l4_bool","type":"bool"},{"name":"x0","type":"f64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_bool":[],"l3_i32":[],"l4_bool":[],"x0":[]}}},{"id":"random/0103","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_f64","type":"f64"},{"name":"l4_i64","type":"i64"}],"columns":{"l0_utf8":["😀",null,"b","é",null],"l1_i32":[3,0,1,-3,0],"l2_f64":[-76.38,-2.0,-54.446,95.132,-4.0],"l3_f64":[-26.664,2.0,3.0,"NaN",-38.242],"l4_i64":["-4611686018427387904",null,"-4611686018427387904","-4611686018427387904","-4611686018427387904"]}}},"ops":[{"op":"sort","keys":[{"expr":{"col":"l1_i32"},"descending":false},{"expr":{"col":"l2_f64"},"descending":true}],"nullsLast":false},{"op":"withColumns","exprs":{"x1":{"fn":"sub","args":[{"fn":"div","args":[{"col":"l1_i32"},{"col":"l1_i32"}]},{"col":"l2_f64"}]}}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_f64","type":"f64"},{"name":"l4_i64","type":"i64"},{"name":"x1","type":"f64"}],"columns":{"l0_utf8":["é",null,null,"b","😀"],"l1_i32":[-3,0,0,1,3],"l2_f64":[95.132,-2.0,-4.0,-54.446,-76.38],"l3_f64":["NaN",2.0,-38.242,3.0,-26.664],"l4_i64":["-4611686018427387904",null,"-4611686018427387904","-4611686018427387904","-4611686018427387904"],"x1":[-94.132,"NaN","NaN",55.446,77.38]}}},{"id":"random/0104","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_f64","type":"f64"},{"name":"l4_f64","type":"f64"}],"columns":{"l0_utf8":["zz","é","ab","B","😀","","b","😀","B","b","é","","a","ab","b","B","é","B","","c"],"l1_i32":[-2,1,-3,0,-1,3,-3,-5,0,-5,1,5,1,1,1,-2,-1,-2,-2,-3],"l2_f64":[-20.036,4.0,"NaN",-1.0,0.0,-4.0,-3.0,4.0,-76.874,-4.0,0.0,-2.0,2.0,81.913,2.0,-4.0,-47.67,1.0,3.0,-26.66],"l3_f64":[93.831,-97.018,0.0,98.231,-43.895,17.645,-3.0,3.0,-53.12,2.0,2.0,"NaN",4.0,81.133,0.0,10.613,1.0,32.628,68.959,"-Infinity"],"l4_f64":[4.0,80.392,-4.0,-37.92,"-0",-3.0,2.0,-4.0,94.57,0.0,2.0,93.977,67.582,2.0,-2.0,-2.0,-1.0,2.0,87.844,1.0]}},"right":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"r0_utf8","type":"utf8"}],"columns":{"r0_utf8":["b","é","ab","ab","zz","zz","a","zz","b","zz"],"l0_utf8":["zz","B","B","c","zz","B","é","é","c","😀"]}}},"ops":[{"op":"withColumns","exprs":{"x0":{"fn":"mod","args":[{"fn":"abs","args":[{"col":"l2_f64"}]},{"fn":"when","args":[{"fn":"eq","args":[{"col":"l1_i32"},{"col":"l3_f64"}]},{"col":"l4_f64"},{"col":"l3_f64"}]}]}}},{"op":"join","right":"right","on":["l0_utf8"],"how":"left"}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_f64","type":"f64"},{"name":"l4_f64","type":"f64"},{"name":"x0","type":"f64"},{"name":"r0_utf8","type":"utf8"}],"columns":{"l0_utf8":["zz","zz","é","é","ab","B","B","B","😀","","b","😀","B","B","B","b","é","é","","a","ab","b","B","B","B","é","é","B","B","B","","c","c"],"l1_i32":[-2,-2,1,1,-3,0,0,0,-1,3,-3,-5,0,0,0,-5,1,1,5,1,1,1,-2,-2,-2,-1,-1,-2,-2,-2,-2,-3,-3],"l2_f64":[-20.036,-20.036,4.0,4.0,"NaN",-1.0,-1.0,-1.0,0.0,-4.0,-3.0,4.0,-76.874,-76.874,-76.874,-4.0,0.0,0.0,-2.0,2.0,81.913,2.0,-4.0,-4.0,-4.0,-47.67,-47.67,1.0,1.0,1.0,3.0,-26.66,-26.66],"l3_f64":[93.831,93.831,-97.018,-97.018,0.0,98.231,98.231,98.231,-43.895,17.645,-3.0,3.0,-53.12,-53.12,-53.12,2.0,2.0,2.0,"NaN",4.0,81.133,0.0,10.613,10.613,10.613,1.0,1.0,32.628,32.628,32.628,68.959,"-Infinity","-Infinity"],"l4_f64":[4.0,4.0,80.392,80.392,-4.0,-37.92,-37.92,-37.92,"-0",-3.0,2.0,-4.0,94.57,94.57,94.57,0.0,2.0,2.0,93.977,67.582,2.0,-2.0,-2.0,-2.0,-2.0,-1.0,-1.0,2.0,2.0,2.0,87.844,1.0,1.0],"x0":[20.036,20.036,-93.018,-93.018,"NaN",1.0,1.0,1.0,0.0,4.0,1.0,1.0,-29.366,-29.366,-29.366,0.0,0.0,0.0,"NaN",2.0,0.7800000000000011,"NaN",4.0,4.0,4.0,0.6700000000000017,0.6700000000000017,1.0,1.0,1.0,3.0,"NaN","NaN"],"r0_utf8":["b","zz","a","zz",null,"é","ab","zz","zz",null,null,"zz","é","ab","zz",null,"a","zz",null,null,null,null,"é","ab","zz","a","zz","é","ab","zz",null,"ab","b"]}}},{"id":"random/0105","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_i64","type":"i64"}],"columns":{"l0_utf8":[null],"l1_i32":[-4],"l2_utf8":["a"],"l3_i64":["-4611686018427387904"]}}},"ops":[{"op":"head","n":1},{"op":"unique","columns":["l0_utf8"]},{"op":"sort","keys":[{"expr":{"col":"l2_utf8"},"descending":false},{"expr":{"col":"l1_i32"},"descending":true}],"nullsLast":false},{"op":"head","n":2}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_i64","type":"i64"}],"columns":{"l0_utf8":[null],"l1_i32":[-4],"l2_utf8":["a"],"l3_i64":["-4611686018427387904"]}}},{"id":"random/0106","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_f64","type":"f64"}],"columns":{"l0_utf8":["ab","","B","é","ab"],"l1_i32":[5,-2147483648,-1,3,-1],"l2_f64":[-49.089,-58.946,-91.914,86.019,-53.887],"l3_f64":[2.0,3.0,0.0,3.0,-37.86]}}},"ops":[{"op":"groupBy","keys":["l1_i32"],"aggs":{"n0":{"agg":"len"},"a0_0":{"agg":"min","arg":{"col":"l0_utf8"}}}},{"op":"filter","predicate":{"fn":"eq","args":[{"fn":"mod","args":[{"col":"l1_i32"},{"col":"l1_i32"}]},{"fn":"mul","args":[{"col":"l1_i32"},{"lit":2}]}]}},{"op":"sort","keys":[{"expr":{"col":"a0_0"},"descending":true}],"nullsLast":false},{"op":"unique","columns":["l1_i32"]}],"expected":{"fields":[{"name":"l1_i32","type":"i32"},{"name":"n0","type":"u32"},{"name":"a0_0","type":"utf8"}],"columns":{"l1_i32":[-2147483648],"n0":[1],"a0_0":[""]}}},{"id":"random/0107","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_i64","type":"i64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_utf8":[],"l3_i64":[]}}},"ops":[{"op":"unique"}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_i64","type":"i64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_utf8":[],"l3_i64":[]}}},{"id":"random/0108","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_i32","type":"i32"},{"name":"l4_i32","type":"i32"}],"columns":{"l0_utf8":["a","a","","aa",null],"l1_i32":[597,null,-4,null,null],"l2_utf8":["b",null,null,null,"B"],"l3_i32":[5,null,5,-1,-306],"l4_i32":[null,-1,null,null,-3]}},"right":{"fields":[{"name":"l4_i32","type":"i32"},{"name":"r0_f64","type":"f64"}],"columns":{"r0_f64":["-0",44.618,41.662,null,null,-21.543,-2.0,-1.0,-2.0,-3.0,null,-50.709,3.0,12.306,0.0,null,null,-2.0,-3.0,3.0,null,"NaN",-4.0,3.0,42.326,null,0.0,null,null,-63.517],"l4_i32":[null,null,-1,-1,-1,null,null,null,null,-3,-3,null,-3,-1,-1,null,null,-3,-1,null,-3,-3,null,null,null,null,-1,null,null,null]}}},"ops":[{"op":"filter","predicate":{"fn":"isIn","args":[{"col":"l2_utf8"}],"values":["","é","😀"]}},{"op":"unique","columns":["l3_i32"]},{"op":"join","right":"right","on":["l4_i32"],"how":"semi"}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_i32","type":"i32"},{"name":"l4_i32","type":"i32"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_utf8":[],"l3_i32":[],"l4_i32":[]}}},{"id":"random/0109","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_i64","type":"i64"}],"columns":{"l0_utf8":["zz","zz","c","é","a","zz","a","B","B","😀","B","aa","b","","ab","B","ab","c","c","😀"],"l1_i32":[0,-4,0,-1,5,1,1,-5,5,0,0,-712,-824,5,-3,-4,-4,-2,-4,-2],"l2_utf8":["zz","zz","zz","","c","aa","😀","","zz","a","","é","😀","😀","aa","B","zz","aa","aa","a"],"l3_i64":["-4611686018427387904","4","9007199254740993","2","9007199254740993","-2","9007199254740995","-4611686018427387904","4","-4","9007199254740992","-4611686018427387904","2","-4611686018427387904","-4611686018427387904","-4611686018427387904","9007199254740994","9007199254740993","9007199254740992","9007199254740994"]}}},"ops":[{"op":"filter","predicate":{"fn":"eq","args":[{"col":"l3_i64"},{"lit":"3","type":"i64"}]}},{"op":"withColumns","exprs":{"x1":{"fn":"and","args":[{"fn":"lte","args":[{"col":"l1_i32"},{"col":"l1_i32"}]},{"fn":"neq","args":[{"col":"l1_i32"},{"col":"l1_i32"}]}]}}},{"op":"withColumns","exprs":{"x2":{"col":"l1_i32"}}},{"op":"withColumns","exprs":{"x3":{"fn":"mod","args":[{"fn":"when","args":[{"fn":"gte","args":[{"lit":2.5},{"col":"l1_i32"}]},{"col":"x2"},{"col":"x2"}]},{"col":"l1_i32"}]}}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_i64","type":"i64"},{"name":"x1","type":"bool"},{"name":"x2","type":"i32"},{"name":"x3","type":"i32"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_utf8":[],"l3_i64":[],"x1":[],"x2":[],"x3":[]}}},{"id":"random/0110","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_i64","type":"i64"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_bool","type":"bool"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_i32":[],"l3_i64":[],"l4_utf8":[],"l5_bool":[]}}},"ops":[{"op":"head","n":9},{"op":"filter","predicate":{"fn":"eq","args":[{"fn":"fillNull","args":[{"lit":0.5},{"lit":0}]},{"fn":"sub","args":[{"col":"l1_i32"},{"col":"l2_i32"}]}]}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_i64","type":"i64"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_bool","type":"bool"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_i32":[],"l3_i64":[],"l4_utf8":[],"l5_bool":[]}}},{"id":"random/0111","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_f64","type":"f64"},{"name":"l4_bool","type":"bool"},{"name":"l5_i64","type":"i64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_utf8":[],"l3_f64":[],"l4_bool":[],"l5_i64":[]}}},"ops":[{"op":"filter","predicate":{"fn":"lt","args":[{"fn":"sub","args":[{"lit":0.5},{"col":"l3_f64"}]},{"col":"l3_f64"}]}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_f64","type":"f64"},{"name":"l4_bool","type":"bool"},{"name":"l5_i64","type":"i64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_utf8":[],"l3_f64":[],"l4_bool":[],"l5_i64":[]}}},{"id":"random/0112","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_utf8","type":"utf8"}],"columns":{"l0_utf8":["é","aa","aa","😀","B"],"l1_i32":[-5,1,-2,3,3],"l2_f64":[97.428,0.0,-26.499,90.545,0.0],"l3_utf8":["","zz","","😀","é"]}}},"ops":[{"op":"withColumns","exprs":{"x0":{"fn":"eq","args":[{"col":"l3_utf8"},{"lit":"a"}]}}},{"op":"sort","keys":[{"expr":{"col":"l0_utf8"},"descending":false},{"expr":{"col":"l1_i32"},"descending":true}],"nullsLast":false},{"op":"withColumns","exprs":{"x2":{"col":"l2_f64"}}},{"op":"withColumns","exprs":{"x3":{"fn":"gte","args":[{"col":"l1_i32"},{"fn":"floorDiv","args":[{"col":"l1_i32"},{"col":"l2_f64"}]}]}}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_utf8","type":"utf8"},{"name":"x0","type":"bool"},{"name":"x2","type":"f64"},{"name":"x3","type":"bool"}],"columns":{"l0_utf8":["B","aa","aa","é","😀"],"l1_i32":[3,1,-2,-5,3],"l2_f64":[0.0,0.0,-26.499,97.428,90.545],"l3_utf8":["é","zz","","","😀"],"x0":[false,false,false,false,false],"x2":[0.0,0.0,-26.499,97.428,90.545],"x3":[false,false,false,false,true]}}},{"id":"random/0113","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_f64","type":"f64"},{"name":"l4_i32","type":"i32"},{"name":"l5_utf8","type":"utf8"},{"name":"l6_i64","type":"i64"}],"columns":{"l0_utf8":["😀","aa","aa","B","ab"],"l1_i32":[-3,4,2,-438,3],"l2_utf8":["zz","","a","zz","ab"],"l3_f64":[2.0,91.628,2.0,3.0,2.0],"l4_i32":[-5,2,-5,-4,-2],"l5_utf8":["c","é","c","B","b"],"l6_i64":["9007199254740995","-2","-5","-3","-4611686018427387904"]}}},"ops":[{"op":"groupBy","keys":["l6_i64"],"aggs":{"n0":{"agg":"len"},"a0_0":{"agg":"min","arg":{"col":"l4_i32"}},"a0_1":{"agg":"max","arg":{"col":"l0_utf8"}},"a0_2":{"agg":"min","arg":{"col":"l5_utf8"}}}},{"op":"head","n":9},{"op":"withColumns","exprs":{"x2":{"fn":"abs","args":[{"lit":2.5}]}}}],"expected":{"fields":[{"name":"l6_i64","type":"i64"},{"name":"n0","type":"u32"},{"name":"a0_0","type":"i32"},{"name":"a0_1","type":"utf8"},{"name":"a0_2","type":"utf8"},{"name":"x2","type":"f64"}],"columns":{"l6_i64":["9007199254740995","-2","-5","-3","-4611686018427387904"],"n0":[1,1,1,1,1],"a0_0":[-5,2,-5,-4,-2],"a0_1":["😀","aa","aa","B","ab"],"a0_2":["c","é","c","B","b"],"x2":[2.5,2.5,2.5,2.5,2.5]}}},{"id":"random/0114","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_f64","type":"f64"},{"name":"l4_i64","type":"i64"},{"name":"l5_f64","type":"f64"},{"name":"l6_i32","type":"i32"}],"columns":{"l0_utf8":["aa","😀","b","","","B","é","b","😀","zz","c","aa","é","c","😀","B","c","b","B","c","c","B","B","aa","ab","ab","aa","c","zz","ab","b","a","aa","ab","B","B","b","c","a","aa","zz","zz","c","zz","é","aa","zz","","😀","ab","c","","😀","é","","😀","😀","é","aa","zz"],"l1_i32":[-3,2147483647,-3,4,5,-2147483648,-157,-4,0,-4,3,-1,5,3,5,3,-2,3,2,-3,-2,-1,-4,3,-4,-3,-5,0,1,-4,3,-3,-1,-4,-4,-4,-3,4,5,3,4,3,-4,5,4,-1,4,2,5,2,-2147483648,5,5,4,3,2,4,-1,-3,2],"l2_i32":[2,-1,-4,5,-5,0,-2,-5,4,-1,-5,0,5,5,-1,0,-1,681,-3,2,350,3,1,2,2,-5,5,4,-2,1,3,5,-2147483648,2,2,2,1,0,4,0,-2,5,-4,-4,3,-3,3,-1,-4,-4,0,4,-5,-2147483648,3,0,-2,1,-1,-4],"l3_f64":[26.801,"NaN",-4.0,-1.0,0.0,1.0,2.0,4.0,1.0,3.0,2.0,4.0,-4.0,4.0,1.0,-1.0,3.0,-22.3,2.331,-5.4,-14.111,-94.213,-2.0,-27.078,"Infinity",46.674,60.941,23.922,-38.117,1.0,99.344,-41.243,19.693,4.0,2.0,-74.51,-65.303,3.0,-2.0,1.0,"NaN",-4.0,3.0,-2.0,4.0,3.0,3.0,-98.575,0.0,-58.964,-4.0,1.0,0.0,73.924,-1.0,2.0,3.0,-90.645,3.0,3.0],"l4_i64":["-4611686018427387904","-4","9007199254740993","9007199254740995","9007199254740995","9007199254740992","9007199254740994","-4611686018427387904","-4611686018427387904","9007199254740995","-4611686018427387904","-4611686018427387904","9007199254740992","9007199254740994","3","2","-4611686018427387904","-1","-4611686018427387904","9007199254740993","-4611686018427387904","0","-4611686018427387904","-4611686018427387904","-4611686018427387904","-4611686018427387904","-1","5","9007199254740992","-4611686018427387904","3","9007199254740992","9007199254740992","-4611686018427387904","-3","9007199254740993","9007199254740993","-3","-4611686018427387904","-1","3","9007199254740995","-4","-4611686018427387904","-4","-3","9007199254740993","9007199254740993","9007199254740993","9007199254740995","9007199254740995","2","-4611686018427387904","5","0","-4611686018427387904","-4","-4611686018427387904","0","9007199254740993"],"l5_f64":[-44.856,35.521,-31.52,-70.008,-1.0,0.0,-38.421,0.0,2.0,-4.0,"-Infinity",66.639,-78.513,12.231,-46.561,-61.45,-4.0,-41.91,-2.0,-2.0,89.202,-1.0,-91.788,-2.0,2.0,"NaN",-2.0,9.489,90.872,-76.153,-16.603,-4.0,-61.137,4.0,1.0,-47.606,3.0,-36.824,-45.113,30.015,-40.606,-77.345,46.223,2.0,90.838,-99.508,-6.783,-17.157,14.248,-3.0,88.217,69.461,-43.519,51.257,-2.0,-22.795,64.935,72.697,3.05,-3.0],"l6_i32":[1,-1,1,-1,1,1,3,2147483647,-1,-2,4,-3,4,4,2,2,-3,2,0,2,0,5,-5,3,2,-3,3,3,-4,2147483647,1,0,0,4,0,-3,928,-4,-4,-1,-3,-5,-3,-3,5,1,-3,4,4,-5,5,0,-5,5,3,-4,2,5,5,-3]}}},"ops":[{"op":"sort","keys":[{"expr":{"col":"l5_f64"},"descending":false}],"nullsLast":true}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_f64","type":"f64"},{"name":"l4_i64","type":"i64"},{"name":"l5_f64","type":"f64"},{"name":"l6_i32","type":"i32"}],"columns":{"l0_utf8":["c","aa","B","é","zz","ab","","B","aa","B","😀","a","aa","😀","b","zz","é","c","b","😀","","b","zz","zz","c","a","ab","zz","B","c","aa","aa","","","B","B","b","B","😀","ab","zz","b","aa","ab","c","c","😀","aa","😀","c","é","😀","aa","","é","c","c","é","zz","ab"],"l1_i32":[3,-1,-4,5,3,-4,4,3,-1,-4,5,5,-3,5,3,4,-157,4,-3,2,2,3,4,-4,-2,-3,2,2,2,-3,3,-5,3,5,-1,-2147483648,-4,-4,0,-4,5,-3,-3,-4,0,3,5,3,2147483647,-4,4,4,-1,5,-1,-2147483648,-2,4,1,-3],"l2_i32":[-5,-3,1,5,5,1,5,0,-2147483648,2,-1,4,2,-5,681,-2,-2,0,-4,0,-1,3,3,-1,-1,5,-4,-4,-3,2,2,5,3,-5,3,0,-5,2,4,2,-4,1,-1,2,4,5,-4,0,-1,-4,-2147483648,-2,0,4,1,0,350,3,-2,-5],"l3_f64":[2.0,3.0,-2.0,-4.0,-4.0,1.0,-1.0,-1.0,19.693,-74.51,1.0,-2.0,26.801,0.0,-22.3,"NaN",2.0,3.0,-4.0,2.0,-98.575,99.344,3.0,3.0,3.0,-41.243,-58.964,3.0,2.331,-5.4,-27.078,60.941,-1.0,0.0,-94.213,1.0,4.0,2.0,1.0,"Infinity",-2.0,-65.303,3.0,4.0,23.922,4.0,0.0,1.0,"NaN",3.0,73.924,3.0,4.0,1.0,-90.645,-4.0,-14.111,4.0,-38.117,46.674],"l4_i64":["-4611686018427387904","-3","-4611686018427387904","9007199254740992","9007199254740995","-4611686018427387904","9007199254740995","2","9007199254740992","9007199254740993","3","-4611686018427387904","-4611686018427387904","-4611686018427387904","-1","3","9007199254740994","-3","9007199254740993","-4611686018427387904","9007199254740993","3","9007199254740993","9007199254740995","-4611686018427387904","9007199254740992","9007199254740995","9007199254740993","-4611686018427387904","9007199254740993","-4611686018427387904","-1","0","9007199254740995","0","9007199254740992","-4611686018427387904","-3","-4611686018427387904","-4611686018427387904","-4611686018427387904","9007199254740993","0","-4611686018427387904","5","9007199254740994","9007199254740993","-1","-4","-4","5","-4","-4611686018427387904","2","-4611686018427387904","9007199254740995","-4611686018427387904","-4","9007199254740992","-4611686018427387904"],"l5_f64":["-Infinity",-99.508,-91.788,-78.513,-77.345,-76.153,-70.008,-61.45,-61.137,-47.606,-46.561,-45.113,-44.856,-43.519,-41.91,-40.606,-38.421,-36.824,-31.52,-22.795,-17.157,-16.603,-6.783,-4.0,-4.0,-4.0,-3.0,-3.0,-2.0,-2.0,-2.0,-2.0,-2.0,-1.0,-1.0,0.0,0.0,1.0,2.0,2.0,2.0,3.0,3.05,4.0,9.489,12.231,14.248,30.015,35.521,46.223,51.257,64.935,66.639,69.461,72.697,88.217,89.202,90.838,90.872,"NaN"],"l6_i32":[4,1,-5,4,-5,2147483647,-1,2,0,-3,2,-4,1,-5,2,-3,3,-4,1,-4,4,1,-3,-2,-3,0,-5,-3,0,2,3,3,3,1,5,1,2147483647,0,-1,2,-3,928,5,4,3,4,4,-1,-1,-3,5,2,-3,0,5,5,0,5,-4,-3]}}},{"id":"random/0115","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_utf8","type":"utf8"}],"columns":{"l0_utf8":["😀","é","ab",null,"b","a","b","B","B","aa","zz","b","c","é","zz","","é","B","ab","c","","b","zz",null,"","😀","é","ab",null,"zz","é","a","c","c","zz","aa","😀","B","é","c","b","😀","zz","aa","b","😀","b","B","c",null,"😀","ab","c","zz","b","ab",null,"zz","c","é"],"l1_i32":[5,3,1,-2147483648,-1,-3,-2,-4,0,4,1,5,4,4,-914,-2,-3,0,-5,-2,1,-700,2147483647,-5,-5,3,-1,1,4,4,-5,4,5,null,1,5,-3,4,null,0,-1,1,0,1,2,0,-2,null,3,1,-1,-2,2,null,-1,4,-5,3,1,5],"l2_i32":[-2,-5,4,-4,-2,-4,-2,618,-5,-318,-2,-1,null,1,-4,null,-4,-306,-5,0,-2,4,2,-4,3,0,-4,-3,-2147483648,-5,2147483647,3,-4,4,null,-2,4,1,2,-4,2,2,5,0,4,-5,-2,1,0,0,null,2147483647,1,2,-4,-2,-2,-3,-3,5],"l3_utf8":["aa",null,"😀","B","c","é","é","aa","aa","😀","B","ab","aa","é","é","aa",null,"aa","zz","😀","ab","é","zz","a","😀","a","a","",null,"aa","a","B","zz","c","zz","é","ab","c","a","ab","zz","zz","","a","ab","😀","zz","é",null,"aa",null,"😀","😀","c","b","","c",null,null,""]}}},"ops":[{"op":"unique","columns":["l1_i32"]},{"op":"filter","predicate":{"fn":"lte","args":[{"fn":"sub","args":[{"col":"l2_i32"},{"col":"l1_i32"}]},{"lit":2.5}]}},{"op":"withColumns","exprs":{"x2":{"col":"l1_i32"}}},{"op":"sort","keys":[{"expr":{"col":"l0_utf8"},"descending":false}],"nullsLast":true}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_utf8","type":"utf8"},{"name":"x2","type":"i32"}],"columns":{"l0_utf8":["B","a","aa","ab","b","b","b","zz","é","😀"],"l1_i32":[0,-3,4,-5,-1,-2,2,2147483647,3,5],"l2_i32":[-5,-4,-318,-5,-2,-2,4,2,-5,-2],"l3_utf8":["aa","é","😀","zz","c","é","ab","zz",null,"aa"],"x2":[0,-3,4,-5,-1,-2,2,2147483647,3,5]}}},{"id":"random/0116","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_i64","type":"i64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_f64":[],"l3_utf8":[],"l4_i64":[]}}},"ops":[{"op":"head","n":2},{"op":"filter","predicate":{"fn":"gte","args":[{"fn":"sub","args":[{"col":"l1_i32"},{"col":"l1_i32"}]},{"fn":"add","args":[{"col":"l2_f64"},{"col":"l1_i32"}]}]}},{"op":"filter","predicate":{"fn":"neq","args":[{"fn":"sub","args":[{"col":"l2_f64"},{"col":"l2_f64"}]},{"fn":"sub","args":[{"col":"l2_f64"},{"col":"l2_f64"}]}]}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_i64","type":"i64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_f64":[],"l3_utf8":[],"l4_i64":[]}}},{"id":"random/0117","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_f64","type":"f64"}],"columns":{"l0_utf8":["aa","b","","😀","B","aa","aa","zz","a","aa","a","😀","😀","zz","a","b","b","c","c","b"],"l1_i32":[-4,5,1,4,-3,3,4,5,1,2,1,3,2,-3,2,0,1,-2,2,-4],"l2_utf8":["zz","zz","c","é","é","B","é","zz","aa","😀","ab","b","é","a","zz","😀","c","ab","😀","c"],"l3_utf8":["😀","a","é","é","a","zz","zz","c","c","","😀","a","B","b","aa","a","","aa","c","aa"],"l4_f64":[4.0,2.0,"NaN",79.849,1.0,-4.0,56.742,29.192,-23.865,2.0,"-0",-68.343,1.0,-54.662,-1.0,53.017,7.166,67.556,4.0,-83.429]}}},"ops":[{"op":"filter","predicate":{"fn":"isNotNull","args":[{"col":"l0_utf8"}]}},{"op":"unique","columns":["l1_i32"]}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_f64","type":"f64"}],"columns":{"l0_utf8":["aa","b","","😀","B","aa","aa","b","c"],"l1_i32":[-4,5,1,4,-3,3,2,0,-2],"l2_utf8":["zz","zz","c","é","é","B","😀","😀","ab"],"l3_utf8":["😀","a","é","é","a","zz","","a","aa"],"l4_f64":[4.0,2.0,"NaN",79.849,1.0,-4.0,2.0,53.017,67.556]}}},{"id":"random/0118","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_i64","type":"i64"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_bool","type":"bool"},{"name":"l6_bool","type":"bool"}],"columns":{"l0_utf8":["😀","a","B","b","😀","zz","B","aa","b","zz","c","","zz","aa","é","zz","c","c","a","","a","b","a",null,"zz","c","",null,"b","ab","a","c","c","a","B",null,"b","é","😀","","b","é",null,"é",null,"",null,"a",null,null,"é","B","a","c","aa","aa","zz",null,"","é"],"l1_i32":[-1,-599,-4,-1,3,3,-5,3,4,2,-3,-4,null,4,-2,3,0,-4,-5,1,-932,1,5,2,-5,-2147483648,null,3,5,0,-1,3,0,3,-2,4,-2,-4,5,1,-2,0,5,-1,5,975,0,4,null,-4,3,-2,5,0,null,-1,-1,-3,0,4],"l2_bool":[null,false,false,false,true,true,true,true,false,true,true,true,true,true,false,false,false,false,false,null,true,null,null,false,true,false,false,true,true,null,true,true,false,false,false,false,false,false,false,false,true,true,false,true,false,true,null,true,true,null,false,null,false,true,true,true,true,null,false,false],"l3_i64":["-3","9007199254740995","9007199254740992","-2","3","-3","3",null,"9007199254740995","9007199254740993","-5","9007199254740995","-4611686018427387904",null,"5","-4611686018427387904",null,null,"9007199254740994","9007199254740995","-1","9007199254740992","9007199254740995",null,"1","-4611686018427387904","-4611686018427387904","-4611686018427387904","-4611686018427387904","-4611686018427387904","9007199254740994","-4611686018427387904","-3","3","9007199254740992","9007199254740993","-4611686018427387904","9007199254740995","9007199254740992","-4611686018427387904","9007199254740993","9007199254740995","9007199254740994","-4","-4611686018427387904","-4611686018427387904","-4","5","-4611686018427387904","-4611686018427387904","9007199254740995","3","-3","-2","0","-1",null,"4","9007199254740993","-4"],"l4_utf8":["zz","","zz","B","zz","ab","c","é","é","aa","zz","b","B","zz","b","zz","zz","zz","😀","zz","c",null,"aa","a","b",null,"ab","b",null,"b","😀","é","c","a","ab","b","ab","ab","c","ab","a",null,"B",null,"é",null,"","zz","B","B",null,"zz","B","a","B","é","b","a","","b"],"l5_bool":[null,null,true,null,false,true,true,true,false,false,false,false,false,null,false,true,true,true,false,true,true,false,true,true,false,true,true,null,false,true,true,null,false,true,true,true,false,true,true,true,true,false,true,true,false,null,true,true,false,false,false,false,false,true,true,true,true,null,false,false],"l6_bool":[false,true,false,true,true,true,true,false,true,true,true,true,true,true,null,false,null,true,true,false,false,true,false,false,true,false,false,true,false,true,false,true,true,false,true,false,true,null,true,true,false,false,true,false,true,true,true,true,true,true,false,false,false,true,false,null,true,false,true,false]}}},"ops":[{"op":"withColumns","exprs":{"x0":{"col":"l1_i32"}}},{"op":"head","n":2},{"op":"head","n":8},{"op":"filter","predicate":{"fn":"eq","args":[{"fn":"mul","args":[{"lit":2},{"col":"x0"}]},{"fn":"sub","args":[{"col":"l1_i32"},{"col":"l1_i32"}]}]}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_i64","type":"i64"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_bool","type":"bool"},{"name":"l6_bool","type":"bool"},{"name":"x0","type":"i32"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_bool":[],"l3_i64":[],"l4_utf8":[],"l5_bool":[],"l6_bool":[],"x0":[]}}},{"id":"random/0119","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_i64","type":"i64"},{"name":"l4_utf8","type":"utf8"}],"columns":{"l0_utf8":["😀","","zz","aa","aa","aa","ab","b","ab","zz","","a","B","ab","aa","😀","zz","b","ab","ab","","c","aa","B","b","zz","a","B","aa","b","B","","B","a","😀","","zz","é","B","😀","b","b","aa","zz","zz","B","zz","aa","c","a","ab","","c","b","b","B","😀","","c","zz"],"l1_i32":[3,2,1,-2,-1,1,5,2,-431,2147483647,3,-4,3,-2147483648,5,-5,-1,1,-3,4,0,5,-3,-5,5,3,0,3,-2,5,-2,2,-2147483648,4,3,-5,4,3,2,-5,-1,-2,-3,-4,-1,2,-2147483648,-3,5,3,-2,0,-5,1,-5,5,1,0,0,2147483647],"l2_f64":[64.964,-74.044,-4.0,-4.0,-55.263,-41.263,-3.0,3.0,-25.216,66.879,88.456,-4.0,-1.0,-16.014,-4.0,-38.809,-22.499,-6.775,4.0,67.084,4.0,26.549,-1.0,51.604,-2.0,2.0,2.0,-1.0,12.152,-29.916,4.0,3.0,0.0,-4.606,-70.227,-1.0,0.0,-68.512,0.0,86.079,75.548,-55.789,-36.068,-59.076,-29.762,4.0,1.0,-14.626,-87.119,-3.0,-2.0,-1.0,-3.0,19.792,82.692,0.0,-58.674,50.462,3.0,-73.246],"l3_i64":["9007199254740993","9007199254740993","9007199254740993","-4611686018427387904","-1","9007199254740995","-4611686018427387904","-4611686018427387904","-4611686018427387904","5","-4611686018427387904","9007199254740992","9007199254740994","9007199254740992","9007199254740993","-4611686018427387904","9007199254740992","-4611686018427387904","-4611686018427387904","-4611686018427387904","9007199254740994","-5","-4611686018427387904","-1","9007199254740993","9007199254740994","9007199254740992","9007199254740995","9007199254740995","-4611686018427387904","-4611686018427387904","3","-3","9007199254740992","-2","-4","-1","4","-3","-3","3","-4611686018427387904","2","-4611686018427387904","2","9007199254740995","-4611686018427387904","-4611686018427387904","-1","-4611686018427387904","9007199254740993","5","-4611686018427387904","-4611686018427387904","-4611686018427387904","3","-4611686018427387904","-4611686018427387904","9007199254740995","9007199254740994"],"l4_utf8":["b","a","ab","é","b","b","ab","zz","😀","é","","é","","B","ab","a","é","c","é","B","aa","","c","zz","","","a","a","","B","B","😀","c","aa","b","aa","é","b","B","B","","zz","é","😀","é","zz","B","é","b","aa","b","zz","B","B","","B","B","b","B","aa"]}},"right":{"fields":[{"name":"x0","type":"i32"},{"name":"r0_f64","type":"f64"}],"columns":{"r0_f64":[-1.0,-4.0,0.0],"x0":[-275,1,1]}}},"ops":[{"op":"withColumns","exprs":{"x0":{"fn":"sub","args":[{"col":"l1_i32"},{"fn":"sub","args":[{"col":"l1_i32"},{"lit":2}]}]}}},{"op":"filter","predicate":{"fn":"lt","args":[{"col":"x0"},{"fn":"floorDiv","args":[{"lit":0.5},{"lit":2}]}]}},{"op":"join","right":"right","on":["x0"],"how":"semi"}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_i64","type":"i64"},{"name":"l4_utf8","type":"utf8"},{"name":"x0","type":"i32"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_f64":[],"l3_i64":[],"l4_utf8":[],"x0":[]}}},{"id":"random/0120","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i64","type":"i64"},{"name":"l3_f64","type":"f64"}],"columns":{"l0_utf8":["c"],"l1_i32":[-2],"l2_i64":["-4611686018427387904"],"l3_f64":[-2.0]}}},"ops":[{"op":"filter","predicate":{"fn":"strStartsWith","args":[{"col":"l0_utf8"}],"prefix":"a"}},{"op":"filter","predicate":{"fn":"lte","args":[{"lit":2},{"fn":"mul","args":[{"col":"l3_f64"},{"col":"l3_f64"}]}]}},{"op":"sort","keys":[{"expr":{"col":"l1_i32"},"descending":true},{"expr":{"col":"l0_utf8"},"descending":true}],"nullsLast":true}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i64","type":"i64"},{"name":"l3_f64","type":"f64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_i64":[],"l3_f64":[]}}},{"id":"random/0121","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_f64","type":"f64"},{"name":"l5_i32","type":"i32"}],"columns":{"l0_utf8":["aa","","","b","aa"],"l1_i32":[3,2,0,2147483647,2],"l2_utf8":["","é","b","b","aa"],"l3_utf8":["a","b","","c","b"],"l4_f64":[11.189,-32.572,-97.26,-4.0,-16.338],"l5_i32":[-4,-1,-5,-5,-2]}}},"ops":[{"op":"filter","predicate":{"fn":"and","args":[{"fn":"isIn","args":[{"col":"l3_utf8"}],"values":["b","a","zz"]},{"fn":"gte","args":[{"col":"l0_utf8"},{"lit":"zz"}]}]}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_f64","type":"f64"},{"name":"l5_i32","type":"i32"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_utf8":[],"l3_utf8":[],"l4_f64":[],"l5_i32":[]}}},{"id":"random/0122","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_i32","type":"i32"},{"name":"l4_i64","type":"i64"},{"name":"l5_f64","type":"f64"}],"columns":{"l0_utf8":["zz","c","zz","é","ab"],"l1_i32":[4,-1,3,-3,5],"l2_utf8":["ab","a","zz","","b"],"l3_i32":[-2,-5,0,-4,2],"l4_i64":["-4611686018427387904","1","3","9007199254740995","-5"],"l5_f64":[34.832,1.0,-2.0,0.0,-67.208]}}},"ops":[{"op":"head","n":1},{"op":"withColumns","exprs":{"x1":{"fn":"mod","args":[{"fn":"add","args":[{"col":"l3_i32"},{"col":"l1_i32"}]},{"col":"l5_f64"}]}}},{"op":"withColumns","exprs":{"x2":{"fn":"neg","args":[{"fn":"when","args":[{"fn":"lte","args":[{"col":"l3_i32"},{"col":"x1"}]},{"col":"l3_i32"},{"col":"x1"}]}]}}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_i32","type":"i32"},{"name":"l4_i64","type":"i64"},{"name":"l5_f64","type":"f64"},{"name":"x1","type":"f64"},{"name":"x2","type":"f64"}],"columns":{"l0_utf8":["zz"],"l1_i32":[4],"l2_utf8":["ab"],"l3_i32":[-2],"l4_i64":["-4611686018427387904"],"l5_f64":[34.832],"x1":[2.0],"x2":[2.0]}}},{"id":"random/0123","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_i32","type":"i32"},{"name":"l4_f64","type":"f64"}],"columns":{"l0_utf8":["aa","😀","","B","😀","","B","c","c","b","aa","b","😀","B","c","B","😀","ab","c","B","b","c","😀","ab","😀","ab","é","aa","zz","","😀","zz","zz","c","a","é","B","B","B","b","b","b","zz","B","","ab","B","😀","é","b","a","😀","zz","😀","ab","c","😀","é","aa","é"],"l1_i32":[5,0,0,-1,1,-2147483648,-1,2,2147483647,-2,1,0,-5,-3,-1,-3,4,-5,5,-3,-341,0,-1,-4,-2,-5,1,-2,4,-4,5,-1,2,1,-4,-1,-4,3,4,-4,0,-4,-2,5,-2,-5,3,3,1,3,4,2,4,-5,-1,0,-3,0,2,4],"l2_utf8":["é","B","ab","ab","é","","zz","a","ab","a","B","c","a","","aa","zz","","é","😀","é","😀","c","a","ab","B","😀","","zz","ab","ab","b","a","c","aa","a","a","c","é","😀","c","aa","aa","c","😀","aa","aa","b","b","a","B","B","B","a","aa","😀","aa","b","","aa","B"],"l3_i32":[-1,-2,-5,-1,-4,-1,5,-5,5,2147483647,1,2,2147483647,-1,1,0,-1,759,4,-2147483648,3,-2147483648,-2,4,-2,-5,4,-2,2,5,5,1,-2,-3,-4,0,-2,4,5,-3,-3,-5,0,1,-3,615,1,-1,-4,1,4,0,-2,-4,2147483647,1,0,5,2147483647,-4],"l4_f64":[-91.18,2.0,18.149,-23.311,-26.807,2.0,"-0",1.0,-45.434,-4.0,55.245,-1.0,-80.654,-48.111,-58.468,4.0,10.762,1.0,86.399,0.0,4.0,53.839,-1.0,-2.0,-2.0,0.0,-83.807,79.569,1.0,"NaN",-3.0,2.0,2.0,8.026,4.0,71.591,41.785,-68.405,45.166,-2.0,3.0,-2.0,-3.0,"NaN",4.0,67.932,-48.252,0.0,0.0,0.0,-37.406,0.0,95.931,15.502,0.0,9.56,4.0,-2.0,-49.612,-1.0]}},"right":{"fields":[{"name":"l1_i32","type":"i32"},{"name":"r0_bool","type":"bool"}],"columns":{"r0_bool":[],"l1_i32":[]}}},"ops":[{"op":"join","right":"right","on":["l1_i32"],"how":"full"},{"op":"groupBy","keys":["l1_i32"],"aggs":{"n1":{"agg":"len"},"a1_0":{"agg":"max","arg":{"col":"l2_utf8"}},"a1_1":{"agg":"first","arg":{"col":"l3_i32"}}}},{"op":"filter","predicate":{"fn":"lte","args":[{"col":"a1_1"},{"fn":"mod","args":[{"col":"a1_1"},{"col":"a1_1"}]}]}}],"expected":{"fields":[{"name":"l1_i32","type":"i32"},{"name":"n1","type":"u32"},{"name":"a1_0","type":"utf8"},{"name":"a1_1","type":"i32"}],"columns":{"l1_i32":[5,0,-1,1,-2147483648,2,-3,4],"n1":[4,7,7,5,1,4,4,6],"a1_0":["😀","c","😀","é","","c","é","😀"],"a1_1":[-1,-2,-1,-4,-1,-5,-1,-1]}}},{"id":"random/0124","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_i64","type":"i64"},{"name":"l4_i32","type":"i32"},{"name":"l5_i64","type":"i64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_i32":[],"l3_i64":[],"l4_i32":[],"l5_i64":[]}},"right":{"fields":[{"name":"l3_i64","type":"i64"},{"name":"r0_bool","type":"bool"}],"columns":{"r0_bool":[null,true,true],"l3_i64":["9007199254740992","-4611686018427387904","-4611686018427387904"]}}},"ops":[{"op":"groupBy","keys":["l3_i64"],"aggs":{"n0":{"agg":"len"},"a0_0":{"agg":"max","arg":{"col":"l5_i64"}}}},{"op":"withColumns","exprs":{"x1":{"fn":"mul","args":[{"lit":-1},{"fn":"div","args":[{"lit":-1},{"lit":2.5}]}]}}},{"op":"join","right":"right","on":["l3_i64"],"how":"semi"}],"expected":{"fields":[{"name":"l3_i64","type":"i64"},{"name":"n0","type":"u32"},{"name":"a0_0","type":"i64"},{"name":"x1","type":"f64"}],"columns":{"l3_i64":[],"n0":[],"a0_0":[],"x1":[]}}},{"id":"random/0125","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_utf8","type":"utf8"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_utf8":[],"l3_utf8":[]}}},"ops":[{"op":"sort","keys":[{"expr":{"col":"l3_utf8"},"descending":false}],"nullsLast":false},{"op":"withColumns","exprs":{"x1":{"col":"l1_i32"}}},{"op":"withColumns","exprs":{"x2":{"fn":"add","args":[{"col":"l1_i32"},{"col":"l1_i32"}]}}},{"op":"withColumns","exprs":{"x3":{"fn":"add","args":[{"fn":"mul","args":[{"col":"l1_i32"},{"lit":2}]},{"col":"x2"}]}}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_utf8","type":"utf8"},{"name":"x1","type":"i32"},{"name":"x2","type":"i32"},{"name":"x3","type":"i32"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_utf8":[],"l3_utf8":[],"x1":[],"x2":[],"x3":[]}}},{"id":"random/0126","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_i64","type":"i64"},{"name":"l5_bool","type":"bool"}],"columns":{"l0_utf8":[null,null,"aa",null,"é","",null,null,"c",null,null,"","ab",null,null,"",null,"","b","zz",null,"zz",null,"😀","a","b","é",null,"",null,"c","aa","ab","😀","aa",null,null,"é","😀","a",null,null,null,"B","ab",null,null,"b","zz","B","B",null,"zz","aa","aa","",null,"B","B",null],"l1_i32":[-2,null,1,0,null,1,5,0,null,-1,-1,-1,0,null,0,1,548,2,2,-4,null,null,5,null,4,1,0,null,-1,-5,1,2147483647,-2147483648,null,null,null,null,1,3,null,2,-4,0,null,-3,null,null,-3,4,null,0,4,-3,null,null,1,-5,4,null,0],"l2_utf8":[null,"a","",null,"","é","zz","ab","a","B","B","","ab","😀","😀","c","b",null,"ab","b","B","zz","zz",null,"é",null,"aa","aa","😀",null,null,"ab",null,"😀","b",null,"zz","c",null,"","a","B","c",null,null,"",null,"","b",null,"B",null,null,"😀",null,"","é","😀","ab","a"],"l3_utf8":[null,null,"aa",null,"ab","aa","c","b",null,"a","b","B","B","aa","😀","aa",null,"c",null,"c","é","é","a",null,"zz","B","B","zz",null,"B","b","c","a",null,"c","c","c","",null,null,"ab","é","",null,"B",null,null,"","b","b","aa",null,"é","zz",null,"aa","zz",null,"aa",null],"l4_i64":["9007199254740993","2","-5","2","9007199254740994","9007199254740993",null,"-3",null,"9007199254740995","1",null,"-1",null,"-4611686018427387904",null,"0","9007199254740995",null,"-4611686018427387904","-4611686018427387904","9007199254740994",null,"9007199254740992","3","-4611686018427387904","-1",null,"5",null,"9007199254740995","9007199254740994",null,null,"-4611686018427387904",null,null,"3","-4611686018427387904","-4611686018427387904","-4611686018427387904","1","9007199254740995","-4611686018427387904",null,"9007199254740994","-4611686018427387904","-5",null,null,"9007199254740995","9007199254740992","-4611686018427387904",null,"-4611686018427387904",null,"9007199254740992","-2","-2","-4611686018427387904"],"l5_bool":[false,true,null,null,null,true,false,false,false,true,false,false,false,false,true,false,null,true,null,true,false,false,null,false,false,true,null,true,null,null,null,true,false,null,true,false,false,null,true,true,null,true,true,true,true,true,true,false,null,null,true,null,null,null,null,false,true,null,false,true]}},"right":{"fields":[{"name":"l1_i32","type":"i32"},{"name":"r0_utf8","type":"utf8"}],"columns":{"r0_utf8":[null,"ab","zz"],"l1_i32":[2,0,0]}}},"ops":[{"op":"withColumns","exprs":{"x0":{"col":"l1_i32"}}},{"op":"join","right":"right","on":["l1_i32"],"how":"inner"},{"op":"groupBy","keys":["l5_bool"],"aggs":{"n2":{"agg":"len"},"a2_0":{"agg":"max","arg":{"col":"l4_i64"}},"a2_1":{"agg":"std","arg":{"col":"l1_i32"}}}}],"expected":{"fields":[{"name":"l5_bool","type":"bool"},{"name":"n2","type":"u32"},{"name":"a2_0","type":"i64"},{"name":"a2_1","type":"f64"}],"columns":{"l5_bool":[null,false,true],"n2":[6,4,9],"a2_0":["2","-1","9007199254740995"],"a2_1":[1.0327955589886446,0.0,0.6666666666666667]}}},{"id":"random/0127","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_bool","type":"bool"},{"name":"l4_i32","type":"i32"},{"name":"l5_i64","type":"i64"},{"name":"l6_bool","type":"bool"}],"columns":{"l0_utf8":["a","a",null,"😀",null,null,"a","c","c","é","","B",null,"aa","c",null,"aa",null,null,"aa","aa","ab","a",null,"c","aa","","😀",null,"c",null,null,null,"ab","","","ab","zz",null,"ab","ab","a",null,"é","ab","zz","ab",null,"","a","c","zz",null,"","",null,null,null,"","aa"],"l1_i32":[null,-3,1,1,-4,null,null,null,0,null,1,5,-1,-4,null,-2,0,null,-3,-2,2,1,-3,null,-4,3,null,-5,-3,-2147483648,0,-2,-3,null,5,-2,null,-5,0,-3,-2,null,2147483647,-3,null,null,null,-2,null,0,-2,null,1,-2,null,1,3,1,null,null],"l2_utf8":[null,null,null,null,"aa","a",null,"c",null,"ab","B","b","b","é","B",null,"ab","aa","é","é",null,null,"aa","b","😀","b","",null,"B","","B",null,"aa",null,"😀",null,null,"aa",null,"😀","a","a","zz","a",null,"B","c",null,"😀",null,null,"b",null,"zz","é","c",null,null,"😀","é"],"l3_bool":[null,null,true,false,true,false,false,null,true,false,true,true,null,true,true,null,null,true,null,false,false,true,true,true,true,null,false,false,false,false,true,null,null,false,false,false,null,true,true,true,false,null,false,null,false,false,false,false,null,false,true,true,false,false,true,null,null,true,null,null],"l4_i32":[4,-5,3,3,null,null,2,null,-3,null,-1,-3,3,-2,null,-2,-3,3,-2,-5,-4,-3,0,null,-5,0,5,-2,-4,-3,-1,1,null,-2,4,-1,0,3,3,null,0,2147483647,-4,-5,-4,null,-2,null,0,2,1,null,4,3,null,-1,-5,0,null,-2],"l5_i64":[null,null,"9007199254740993",null,null,"-4611686018427387904",null,null,"5","9007199254740995",null,"-2","9007199254740992",null,"-3","-1","-4611686018427387904","1",null,"9007199254740994","9007199254740993",null,"-4611686018427387904",null,"9007199254740995","9007199254740993","9007199254740993","-3","9007199254740993","-5",null,"0",null,"9007199254740993",null,null,"-2",null,"0","-2","-4611686018427387904","-4611686018427387904","9007199254740995",null,"5","9007199254740994",null,"9007199254740993","9007199254740993","5",null,"-3","-4611686018427387904","-4611686018427387904","-4611686018427387904","-4","-3","9007199254740993","-4611686018427387904","3"],"l6_bool":[true,true,null,false,true,null,true,true,null,null,true,true,false,true,null,null,true,true,false,null,null,false,false,true,null,null,false,true,null,null,false,null,null,false,null,true,null,null,false,null,null,true,false,false,true,null,false,false,false,null,false,false,false,true,false,true,true,false,true,false]}},"right":{"fields":[{"name":"l1_i32","type":"i32"},{"name":"r0_f64","type":"f64"}],"columns":{"r0_f64":[3.0,4.0,-43.28,-72.491,2.0,29.017,4.0,null,-57.398,-3.0],"l1_i32":[-1,5,null,null,-2,-3,1,-3,-3,5]}}},"ops":[{"op":"join","right":"right","on":["l1_i32"],"how":"left"},{"op":"filter","predicate":{"fn":"eq","args":[{"fn":"div","args":[{"col":"r0_f64"},{"col":"r0_f64"}]},{"col":"l1_i32"}]}},{"op":"withColumns","exprs":{"x2":{"fn":"lt","args":[{"fn":"floorDiv","args":[{"col":"l4_i32"},{"lit":-1}]},{"fn":"div","args":[{"lit":-1},{"col":"r0_f64"}]}]}}},{"op":"withColumns","exprs":{"x3":{"lit":2}}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_bool","type":"bool"},{"name":"l4_i32","type":"i32"},{"name":"l5_i64","type":"i64"},{"name":"l6_bool","type":"bool"},{"name":"r0_f64","type":"f64"},{"name":"x2","type":"bool"},{"name":"x3","type":"i32"}],"columns":{"l0_utf8":[null,"😀","","ab",null,null,null],"l1_i32":[1,1,1,1,1,1,1],"l2_utf8":[null,null,"B",null,null,"c",null],"l3_bool":[true,false,true,true,false,null,true],"l4_i32":[3,3,-1,-3,4,-1,0],"l5_i64":["9007199254740993",null,null,null,"-4611686018427387904","-4","9007199254740993"],"l6_bool":[null,false,true,false,false,true,false],"r0_f64":[4.0,4.0,4.0,4.0,4.0,4.0,4.0],"x2":[true,true,false,false,true,false,false],"x3":[2,2,2,2,2,2,2]}}},{"id":"random/0128","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_utf8","type":"utf8"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_utf8":[],"l3_utf8":[]}}},"ops":[{"op":"sort","keys":[{"expr":{"col":"l3_utf8"},"descending":false},{"expr":{"col":"l2_utf8"},"descending":true}],"nullsLast":false},{"op":"unique","columns":["l2_utf8"]},{"op":"head","n":7},{"op":"sort","keys":[{"expr":{"col":"l1_i32"},"descending":false},{"expr":{"col":"l2_utf8"},"descending":false}],"nullsLast":true}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_utf8","type":"utf8"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_utf8":[],"l3_utf8":[]}}},{"id":"random/0129","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_f64","type":"f64"},{"name":"l5_f64","type":"f64"},{"name":"l6_utf8","type":"utf8"}],"columns":{"l0_utf8":["","a","B","b","ab"],"l1_i32":[null,2147483647,-2,1,-3],"l2_f64":["-0",73.898,4.0,-26.96,-19.2],"l3_utf8":["ab","ab","a","c","ab"],"l4_f64":[-91.115,3.0,-47.084,-77.853,4.0],"l5_f64":[2.0,-4.0,95.601,-2.0,74.686],"l6_utf8":["b",null,"😀","a","ab"]}}},"ops":[{"op":"filter","predicate":{"fn":"isNotNull","args":[{"col":"l4_f64"}]}},{"op":"withColumns","exprs":{"x1":{"fn":"isNotNull","args":[{"col":"l5_f64"}]}}},{"op":"withColumns","exprs":{"x2":{"fn":"floorDiv","args":[{"fn":"div","args":[{"col":"l4_f64"},{"col":"l2_f64"}]},{"fn":"mul","args":[{"col":"l1_i32"},{"col":"l5_f64"}]}]}}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_f64","type":"f64"},{"name":"l5_f64","type":"f64"},{"name":"l6_utf8","type":"utf8"},{"name":"x1","type":"bool"},{"name":"x2","type":"f64"}],"columns":{"l0_utf8":["","a","B","b","ab"],"l1_i32":[null,2147483647,-2,1,-3],"l2_f64":["-0",73.898,4.0,-26.96,-19.2],"l3_utf8":["ab","ab","a","c","ab"],"l4_f64":[-91.115,3.0,-47.084,-77.853,4.0],"l5_f64":[2.0,-4.0,95.601,-2.0,74.686],"l6_utf8":["b",null,"😀","a","ab"],"x1":[true,true,true,true,true],"x2":[null,-1.0,0.0,-2.0,0.0]}}},{"id":"random/0130","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_i64","type":"i64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_f64":[],"l3_utf8":[],"l4_utf8":[],"l5_i64":[]}}},"ops":[{"op":"withColumns","exprs":{"x0":{"fn":"floorDiv","args":[{"fn":"neg","args":[{"col":"l2_f64"}]},{"fn":"when","args":[{"fn":"lte","args":[{"col":"l1_i32"},{"lit":0.5}]},{"col":"l1_i32"},{"col":"l2_f64"}]}]}}},{"op":"unique","columns":["l4_utf8"]}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_i64","type":"i64"},{"name":"x0","type":"f64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_f64":[],"l3_utf8":[],"l4_utf8":[],"l5_i64":[],"x0":[]}}},{"id":"random/0131","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_f64","type":"f64"},{"name":"l4_f64","type":"f64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_f64":[],"l3_f64":[],"l4_f64":[]}}},"ops":[{"op":"unique"},{"op":"head","n":5}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_f64","type":"f64"},{"name":"l4_f64","type":"f64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_f64":[],"l3_f64":[],"l4_f64":[]}}},{"id":"random/0132","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_i64","type":"i64"},{"name":"l4_i32","type":"i32"}],"columns":{"l0_utf8":["b","","","a","b"],"l1_i32":[null,4,0,-4,-1],"l2_i32":[-3,4,-4,1,5],"l3_i64":["-4611686018427387904","9007199254740993","-4","9007199254740993","9007199254740994"],"l4_i32":[2147483647,1,-1,1,null]}}},"ops":[{"op":"filter","predicate":{"fn":"lt","args":[{"lit":0.5},{"fn":"abs","args":[{"col":"l4_i32"}]}]}},{"op":"withColumns","exprs":{"x1":{"fn":"isIn","args":[{"col":"l0_utf8"}],"values":["B","a","é"]}}},{"op":"filter","predicate":{"fn":"eq","args":[{"lit":-1},{"col":"l2_i32"}]}},{"op":"head","n":7}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_i64","type":"i64"},{"name":"l4_i32","type":"i32"},{"name":"x1","type":"bool"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_i32":[],"l3_i64":[],"l4_i32":[],"x1":[]}}},{"id":"random/0133","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_i32","type":"i32"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_utf8":[],"l3_i32":[]}}},"ops":[{"op":"sort","keys":[{"expr":{"col":"l2_utf8"},"descending":true},{"expr":{"col":"l0_utf8"},"descending":true}],"nullsLast":true},{"op":"withColumns","exprs":{"x1":{"col":"l1_i32"}}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_i32","type":"i32"},{"name":"x1","type":"i32"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_utf8":[],"l3_i32":[],"x1":[]}}},{"id":"random/0134","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_bool","type":"bool"}],"columns":{"l0_utf8":["c","c","é","a","zz","c","","😀","é","a","zz","B","c","b","B","a","B","zz","c","B"],"l1_i32":[3,-557,3,0,-2,1,-1,3,0,-3,-5,1,-4,-3,3,-1,4,2147483647,-2,4],"l2_utf8":["a","zz","b","aa","c","c","ab","ab","ab","b","b","zz","zz","a","b","B","","é","zz","a"],"l3_bool":[true,false,false,false,true,true,false,false,false,false,true,false,true,false,true,false,false,false,false,false]}}},"ops":[{"op":"filter","predicate":{"fn":"neq","args":[{"fn":"abs","args":[{"col":"l1_i32"}]},{"col":"l1_i32"}]}},{"op":"filter","predicate":{"fn":"isNull","args":[{"col":"l1_i32"}]}},{"op":"withColumns","exprs":{"x2":{"fn":"and","args":[{"fn":"and","args":[{"fn":"gt","args":[{"col":"l1_i32"},{"col":"l1_i32"}]},{"fn":"isIn","args":[{"col":"l2_utf8"}],"values":["ab","c","😀"]}]},{"fn":"or","args":[{"fn":"isNotNull","args":[{"col":"l3_bool"}]},{"fn":"eq","args":[{"col":"l1_i32"},{"lit":-1}]}]}]}}},{"op":"filter","predicate":{"fn":"lt","args":[{"lit":0.5},{"col":"l1_i32"}]}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_bool","type":"bool"},{"name":"x2","type":"bool"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_utf8":[],"l3_bool":[],"x2":[]}}},{"id":"random/0135","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_i64","type":"i64"}],"columns":{"l0_utf8":["a","a","😀","ab","aa","a","B",null,null,"ab","ab",null,"a","","zz","b","B","é","B","B","b","é","B","ab","b","😀","ab","aa","",null,"a","😀","ab","b",null,"😀","B","é","c","b","a","a","zz","😀","zz","é","é",null,"ab","a",null,"aa",null,"b","zz","zz","b","ab","a","😀"],"l1_i32":[-1,3,3,0,null,null,-3,1,1,-4,119,1,4,-4,2,-4,2,null,2,2,2,3,null,3,-4,null,-1,5,-4,2147483647,0,-1,-3,4,4,-3,-1,1,-5,2,-1,1,3,955,4,-1,-1,-1,5,-4,null,5,-3,null,1,2147483647,-2,5,-4,5],"l2_i32":[-2,-2,0,2,-4,-5,3,2,-4,null,3,5,-4,2,4,1,-3,3,789,-5,3,2147483647,4,-811,null,-5,5,1,-3,-1,4,-5,null,-2,-4,1,1,5,null,-1,-2,-3,5,1,1,0,1,3,3,4,1,5,2,-137,-608,1,-2,4,null,1],"l3_i64":["-4611686018427387904","2","-4611686018427387904","9007199254740992","-4611686018427387904","0","5","-4611686018427387904","5","-4611686018427387904","9007199254740995","-4611686018427387904","9007199254740992","-4611686018427387904","9007199254740993",null,"-4611686018427387904","-4611686018427387904","-4611686018427387904","-4611686018427387904","9007199254740995",null,"-4611686018427387904",null,"-2","-4611686018427387904","9007199254740995","9007199254740992","5","9007199254740995","-4611686018427387904","-4611686018427387904","-4611686018427387904","9007199254740992","9007199254740995","9007199254740992","0","-4611686018427387904","-4611686018427387904","-2","-4611686018427387904","9007199254740994","5","9007199254740993","-4","-4611686018427387904","-4611686018427387904","-4611686018427387904","-1",null,"2","-4611686018427387904","9007199254740993","-5","9007199254740994","-4611686018427387904","9007199254740992","4","-5","-4611686018427387904"]}}},"ops":[{"op":"withColumns","exprs":{"x0":{"fn":"fillNull","args":[{"fn":"sub","args":[{"col":"l2_i32"},{"col":"l1_i32"}]},{"lit":0}]}}},{"op":"filter","predicate":{"fn":"neq","args":[{"fn":"div","args":[{"col":"l2_i32"},{"col":"l2_i32"}]},{"fn":"mul","args":[{"col":"l1_i32"},{"col":"x0"}]}]}},{"op":"sort","keys":[{"expr":{"col":"l0_utf8"},"descending":false}],"nullsLast":false}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_i64","type":"i64"},{"name":"x0","type":"i32"}],"columns":{"l0_utf8":[null,null,null,null,null,null,"","","B","B","B","B","B","a","a","a","a","a","aa","aa","ab","ab","ab","ab","ab","ab","b","b","b","b","b","zz","zz","zz","zz","zz","é","é","é","é","😀","😀","😀","😀","😀"],"l1_i32":[1,1,2147483647,4,-1,-3,-4,-4,-3,2,2,2,-1,3,4,0,1,-4,5,5,0,119,3,-1,5,5,-4,2,4,2,-2,2,3,4,1,2147483647,3,1,-1,-1,3,-1,-3,955,5],"l2_i32":[-4,5,-1,-4,3,2,2,-3,3,-3,789,-5,1,-2,-4,4,-3,4,1,5,2,3,-811,5,3,4,1,3,-2,-1,-2,4,5,1,-608,1,2147483647,5,0,1,0,-5,1,1,1],"l3_i64":["5","-4611686018427387904","9007199254740995","9007199254740995","-4611686018427387904","9007199254740993","-4611686018427387904","5","5","-4611686018427387904","-4611686018427387904","-4611686018427387904","0","2","9007199254740992","-4611686018427387904","9007199254740994",null,"9007199254740992","-4611686018427387904","9007199254740992","9007199254740995",null,"9007199254740995","-1","4",null,"9007199254740995","9007199254740992","-2","9007199254740992","9007199254740993","5","-4","9007199254740994","-4611686018427387904",null,"-4611686018427387904","-4611686018427387904","-4611686018427387904","-4611686018427387904","-4611686018427387904","9007199254740992","9007199254740993","-4611686018427387904"],"x0":[-5,4,-2147483648,-8,4,5,6,1,6,-5,787,-7,2,-5,-8,4,-4,8,-4,0,2,-116,-814,6,-2,-1,5,1,-6,-3,0,2,2,-3,-609,-2147483646,2147483644,4,1,2,-3,-4,4,-954,-4]}}},{"id":"random/0136","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_i32","type":"i32"},{"name":"l6_i64","type":"i64"}],"columns":{"l0_utf8":["a","😀","","ab","a"],"l1_i32":[-5,1,4,4,-1],"l2_i32":[-2,2,-1,3,2],"l3_utf8":["B","c","B","c","é"],"l4_utf8":["B","a","ab","b","aa"],"l5_i32":[1,3,0,-3,-5],"l6_i64":["-4611686018427387904","3","4","-4","9007199254740994"]}}},"ops":[{"op":"filter","predicate":{"fn":"and","args":[{"fn":"gt","args":[{"lit":0.5},{"lit":2.5}]},{"fn":"gt","args":[{"col":"l2_i32"},{"col":"l5_i32"}]}]}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_i32","type":"i32"},{"name":"l6_i64","type":"i64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_i32":[],"l3_utf8":[],"l4_utf8":[],"l5_i32":[],"l6_i64":[]}}},{"id":"random/0137","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_i32","type":"i32"},{"name":"l4_i32","type":"i32"}],"columns":{"l0_utf8":["é",null,"aa","a",null],"l1_i32":[4,1,-1,1,-1],"l2_f64":[-53.175,3.0,-2.0,3.0,null],"l3_i32":[-5,3,-3,4,-5],"l4_i32":[1,1,-3,null,-1]}}},"ops":[{"op":"head","n":3},{"op":"filter","predicate":{"fn":"eq","args":[{"fn":"add","args":[{"col":"l1_i32"},{"col":"l2_f64"}]},{"fn":"mod","args":[{"lit":0.5},{"col":"l1_i32"}]}]}},{"op":"withColumns","exprs":{"x2":{"fn":"abs","args":[{"col":"l4_i32"}]}}},{"op":"filter","predicate":{"fn":"eq","args":[{"fn":"when","args":[{"fn":"isNotNull","args":[{"col":"l0_utf8"}]},{"lit":2.5},{"col":"l3_i32"}]},{"col":"l4_i32"}]}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_i32","type":"i32"},{"name":"l4_i32","type":"i32"},{"name":"x2","type":"i32"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_f64":[],"l3_i32":[],"l4_i32":[],"x2":[]}}},{"id":"random/0138","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_utf8","type":"utf8"}],"columns":{"l0_utf8":["b","é","a","","😀","a","","zz","é","aa","b","ab","zz","aa","aa","aa","zz","zz","B","b"],"l1_i32":[5,1,2,-2,-3,-4,3,5,-4,-2,-2147483648,0,-1,0,5,-4,2147483647,0,4,0],"l2_utf8":["c","","c","a","aa","B","😀","B","😀","aa","é","zz","😀","a","é","ab","aa","😀","B",""],"l3_utf8":["ab","c","c","c","b","é","ab","","","B","ab","ab","B","a","ab","zz","a","😀","a","ab"],"l4_utf8":["a","aa","zz","ab","c","B","","aa","b","zz","c","b","","b","c","b","zz","","zz","b"]}}},"ops":[{"op":"withColumns","exprs":{"x0":{"fn":"isIn","args":[{"col":"l4_utf8"}],"values":["ab","B","😀"]}}},{"op":"groupBy","keys":["l2_utf8"],"aggs":{"n1":{"agg":"len"},"a1_0":{"agg":"max","arg":{"col":"l0_utf8"}}}},{"op":"withColumns","exprs":{"x2":{"fn":"abs","args":[{"lit":-1}]}}},{"op":"sort","keys":[{"expr":{"col":"n1"},"descending":false}],"nullsLast":true}],"expected":{"fields":[{"name":"l2_utf8","type":"utf8"},{"name":"n1","type":"u32"},{"name":"a1_0","type":"utf8"},{"name":"x2","type":"i32"}],"columns":{"l2_utf8":["zz","ab","c","","a","é","aa","B","😀"],"n1":[1,1,2,2,2,2,3,3,4],"a1_0":["ab","aa","b","é","aa","b","😀","zz","é"],"x2":[1,1,1,1,1,1,1,1,1]}}},{"id":"random/0139","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_i32","type":"i32"},{"name":"l4_bool","type":"bool"},{"name":"l5_bool","type":"bool"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_f64":[],"l3_i32":[],"l4_bool":[],"l5_bool":[]}}},"ops":[{"op":"groupBy","keys":["l4_bool"],"aggs":{"n0":{"agg":"len"},"a0_0":{"agg":"max","arg":{"col":"l2_f64"}}}}],"expected":{"fields":[{"name":"l4_bool","type":"bool"},{"name":"n0","type":"u32"},{"name":"a0_0","type":"f64"}],"columns":{"l4_bool":[],"n0":[],"a0_0":[]}}},{"id":"random/0140","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i64","type":"i64"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_f64","type":"f64"}],"columns":{"l0_utf8":["😀"],"l1_i32":[4],"l2_i64":["9007199254740993"],"l3_utf8":["b"],"l4_f64":[-42.28]}}},"ops":[{"op":"sort","keys":[{"expr":{"col":"l3_utf8"},"descending":false},{"expr":{"col":"l4_f64"},"descending":false}],"nullsLast":false},{"op":"unique"}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i64","type":"i64"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_f64","type":"f64"}],"columns":{"l0_utf8":["😀"],"l1_i32":[4],"l2_i64":["9007199254740993"],"l3_utf8":["b"],"l4_f64":[-42.28]}}},{"id":"random/0141","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_bool","type":"bool"},{"name":"l4_utf8","type":"utf8"}],"columns":{"l0_utf8":[null,null,null,"é","c","zz","aa","a","aa","aa","é","B","B","c","b","😀","😀","","","aa",null,"é","😀","","é","B","aa","ab",null,"c","aa",null,"B",null,"é",null,"","a","zz",null,"zz",null,null,null,"B","é","zz",null,"ab","ab",null,null,"zz","B","c","c",null,null,null,"😀"],"l1_i32":[-1,5,4,null,4,3,0,1,null,-3,0,null,-4,5,0,null,5,4,992,null,-2,1,-1,2,-2,-4,2,124,null,-5,null,-3,-3,null,-5,-4,null,-3,4,-2,-1,4,null,2,null,null,0,5,0,null,-2147483648,null,-1,2,-1,null,3,-5,-5,1],"l2_f64":[null,2.0,95.44,80.659,55.69,-3.0,null,null,-3.767,-4.0,-3.0,-4.0,-1.0,3.0,null,-4.0,null,4.0,1.0,null,-59.737,-46.968,-50.905,null,-81.427,null,null,-2.0,3.0,-14.915,null,-82.116,0.593,97.652,51.958,1.0,null,67.806,-9.207,3.0,-81.453,"NaN",null,null,null,0.0,3.0,null,-4.0,23.348,null,null,null,-53.527,null,0.0,-94.58,2.0,-83.113,-55.022],"l3_bool":[true,true,null,true,true,null,true,true,false,false,false,false,false,true,false,true,true,null,null,null,true,false,null,false,true,false,true,false,true,false,true,false,true,false,null,true,null,true,true,false,false,false,true,true,null,false,null,false,null,false,true,true,null,false,null,null,false,true,false,null],"l4_utf8":[null,"b","","😀",null,"😀",null,"",null,"aa","B","a",null,"a",null,"😀","zz","zz","aa","😀",null,"","c","aa","B","a","a",null,"ab","",null,null,"ab","aa","b",null,"ab",null,"a","a","c","ab","aa",null,"ab","c",null,"a",null,"B",null,"b","b",null,"aa","b","é","zz",null,null]}}},"ops":[{"op":"withColumns","exprs":{"x0":{"col":"l1_i32"}}},{"op":"filter","predicate":{"fn":"neq","args":[{"fn":"add","args":[{"col":"l1_i32"},{"col":"l1_i32"}]},{"fn":"floorDiv","args":[{"col":"l2_f64"},{"col":"x0"}]}]}},{"op":"filter","predicate":{"fn":"strStartsWith","args":[{"col":"l0_utf8"}],"prefix":"é"}},{"op":"sort","keys":[{"expr":{"col":"l4_utf8"},"descending":false},{"expr":{"col":"l0_utf8"},"descending":false}],"nullsLast":true}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_bool","type":"bool"},{"name":"l4_utf8","type":"utf8"},{"name":"x0","type":"i32"}],"columns":{"l0_utf8":["é","é","é","é"],"l1_i32":[1,0,-2,-5],"l2_f64":[-46.968,-3.0,-81.427,51.958],"l3_bool":[false,false,true,null],"l4_utf8":["","B","B","b"],"x0":[1,0,-2,-5]}}},{"id":"random/0142","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_bool","type":"bool"}],"columns":{"l0_utf8":["aa",null,"aa","B",null,"ab","B",null,"B","a","é","B",null,"a","B","a","é","zz","aa","zz"],"l1_i32":[2,5,1,3,-2,null,0,-1,2,5,2,0,2,4,-77,-855,1,-238,-1,2147483647],"l2_utf8":["c","b","B","zz","😀","zz","é",null,"zz","b","aa","","","aa","ab","c","ab","😀","😀","zz"],"l3_utf8":["B","B","aa","c",null,"zz","ab","😀","c","é","aa","","a",null,"aa","B","é","ab",null,"a"],"l4_bool":[true,true,false,true,false,true,true,null,false,true,false,false,true,true,false,false,true,false,null,true]}}},"ops":[{"op":"filter","predicate":{"fn":"and","args":[{"fn":"isNull","args":[{"col":"l4_bool"}]},{"fn":"gt","args":[{"lit":-1},{"col":"l1_i32"}]}]}},{"op":"unique"}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_bool","type":"bool"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_utf8":[],"l3_utf8":[],"l4_bool":[]}}},{"id":"random/0143","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i64","type":"i64"},{"name":"l3_i32","type":"i32"}],"columns":{"l0_utf8":["😀","aa",null,null,null,"B",null,"a","","b","B","b","",null,"b","😀","é",null,"aa","c",null,"ab","a","aa","c",null,"aa","aa","",null,null,"B","c","b","b","ab","","zz","😀",null,"B",null,null,null,"c",null,"aa",null,"c",null,"ab","c",null,"ab","zz","é","","","ab",""],"l1_i32":[-4,null,-4,null,3,5,3,-3,-1,null,4,null,-1,5,5,1,2,-5,null,1,1,-4,2,2,0,4,-2,null,4,null,-2,3,4,5,-1,null,null,5,-4,1,-2,-1,2147483647,-3,3,2147483647,3,2,null,null,null,-1,-4,null,3,1,null,-4,3,1],"l2_i64":["9007199254740995","-4611686018427387904","-4611686018427387904","-4611686018427387904",null,null,"9007199254740992",null,null,null,"9007199254740993","-4611686018427387904","-5",null,"4","-4611686018427387904","-4611686018427387904","-4611686018427387904",null,"-4611686018427387904","-4","-4611686018427387904",null,"5","-4611686018427387904","-4611686018427387904","-4611686018427387904",null,"-4611686018427387904","-4611686018427387904","-4611686018427387904",null,"9007199254740992","-4611686018427387904","-5","-3",null,null,"-4611686018427387904","-4611686018427387904","-4611686018427387904","9007199254740993","-4611686018427387904","2",null,"-4611686018427387904",null,"9007199254740994","-4","-4611686018427387904",null,"5","9007199254740992",null,"-4611686018427387904","0","-4611686018427387904","9007199254740992","9007199254740994","9007199254740994"],"l3_i32":[-2,-5,-3,-2,2,4,3,-5,null,0,-5,null,4,3,-5,null,-1,0,-4,1,null,-3,1,null,0,-3,-2,null,-2147483648,-4,4,1,null,-3,null,-1,-2,4,2,null,-2,1,1,null,0,null,-5,null,3,null,2,null,-2,-2,-1,5,1,0,null,1]}},"right":{"fields":[{"name":"l3_i32","type":"i32"},{"name":"r0_f64","type":"f64"}],"columns":{"r0_f64":[1.0,1.0,-2.0,-1.0,-2.0,10.676,96.286,45.979,-40.883,-1.0,1.0,-16.329,-17.196,0.0,-4.0,-4.0,"NaN",3.0,"-0",-3.0,-99.901,13.946,0.0,34.747,-4.0,-77.636,3.0,93.732,3.0,14.443],"l3_i32":[null,-1,-5,null,null,-2,-2,-3,3,1,-5,2,-5,4,-2,-5,1,-4,4,-2,-2,-2,0,4,-2,0,-3,1,-3,null]}}},"ops":[{"op":"filter","predicate":{"fn":"gt","args":[{"col":"l1_i32"},{"col":"l1_i32"}]}},{"op":"withColumns","exprs":{"x1":{"fn":"fillNull","args":[{"lit":2.5},{"lit":1.5}]}}},{"op":"join","right":"right","on":["l3_i32"],"how":"inner"}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i64","type":"i64"},{"name":"l3_i32","type":"i32"},{"name":"x1","type":"f64"},{"name":"r0_f64","type":"f64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_i64":[],"l3_i32":[],"x1":[],"r0_f64":[]}}},{"id":"random/0144","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_f64","type":"f64"}],"columns":{"l0_utf8":["zz","b","","","aa"],"l1_i32":[1,4,2,null,2],"l2_f64":[null,95.323,97.124,2.0,0.338],"l3_f64":["-0",4.0,-77.463,-1.0,-2.0]}}},"ops":[{"op":"sort","keys":[{"expr":{"col":"l3_f64"},"descending":false},{"expr":{"col":"l0_utf8"},"descending":true}],"nullsLast":true},{"op":"filter","predicate":{"fn":"lt","args":[{"col":"l1_i32"},{"col":"l1_i32"}]}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_f64","type":"f64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_f64":[],"l3_f64":[]}}},{"id":"random/0145","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_i64","type":"i64"},{"name":"l5_i32","type":"i32"},{"name":"l6_utf8","type":"utf8"}],"columns":{"l0_utf8":["zz"],"l1_i32":[-3],"l2_utf8":["B"],"l3_utf8":["B"],"l4_i64":["-4611686018427387904"],"l5_i32":[0],"l6_utf8":["zz"]}},"right":{"fields":[{"name":"l6_utf8","type":"utf8"},{"name":"r0_utf8","type":"utf8"}],"columns":{"r0_utf8":[null,"😀","a","a","ab","b","ab","😀","","😀","😀","é","b","","ab","aa","c",null,"é","c",null,"é","aa","b","aa","é","","B","aa","😀"],"l6_utf8":["zz","zz","zz","zz","zz","zz","zz","zz","zz","zz","zz","zz","zz","zz","zz","zz","zz","zz","zz","zz","zz","zz","zz","zz","zz","zz","zz","zz","zz","zz"]}}},"ops":[{"op":"unique"},{"op":"join","right":"right","on":["l6_utf8"],"how":"anti"},{"op":"groupBy","keys":["l0_utf8"],"aggs":{"n2":{"agg":"len"},"a2_0":{"agg":"first","arg":{"col":"l5_i32"}},"a2_1":{"agg":"count","arg":{"col":"l1_i32"}},"a2_2":{"agg":"count","arg":{"col":"l2_utf8"}}}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"n2","type":"u32"},{"name":"a2_0","type":"i32"},{"name":"a2_1","type":"u32"},{"name":"a2_2","type":"u32"}],"columns":{"l0_utf8":[],"n2":[],"a2_0":[],"a2_1":[],"a2_2":[]}}},{"id":"random/0146","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_bool","type":"bool"},{"name":"l4_utf8","type":"utf8"}],"columns":{"l0_utf8":["aa"],"l1_i32":[0],"l2_f64":[-68.128],"l3_bool":[false],"l4_utf8":["b"]}}},"ops":[{"op":"head","n":1}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_bool","type":"bool"},{"name":"l4_utf8","type":"utf8"}],"columns":{"l0_utf8":["aa"],"l1_i32":[0],"l2_f64":[-68.128],"l3_bool":[false],"l4_utf8":["b"]}}},{"id":"random/0147","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_bool","type":"bool"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_bool","type":"bool"}],"columns":{"l0_utf8":["aa"],"l1_i32":[-5],"l2_bool":[true],"l3_bool":[true],"l4_utf8":[null],"l5_bool":[null]}}},"ops":[{"op":"withColumns","exprs":{"x0":{"col":"l1_i32"}}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_bool","type":"bool"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_bool","type":"bool"},{"name":"x0","type":"i32"}],"columns":{"l0_utf8":["aa"],"l1_i32":[-5],"l2_bool":[true],"l3_bool":[true],"l4_utf8":[null],"l5_bool":[null],"x0":[-5]}}},{"id":"random/0148","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_f64","type":"f64"},{"name":"l5_f64","type":"f64"},{"name":"l6_utf8","type":"utf8"}],"columns":{"l0_utf8":["ab","c",null,null,null,"c","B","aa","ab","b",null,null,"B","ab",null,null,"zz",null,"b","B"],"l1_i32":[-4,-2,1,null,-4,2,null,-5,2,null,null,null,4,2,-1,-4,-1,-2147483648,null,null],"l2_f64":[86.534,-53.607,-66.98,15.709,0.0,90.824,-3.0,null,-14.19,-80.437,null,null,3.778,null,-47.743,-62.376,-1.0,null,2.0,3.0],"l3_utf8":["B",null,null,null,"c","aa","",null,"","B","zz","",null,null,null,"B",null,"a",null,""],"l4_f64":[null,-2.0,-1.0,"NaN",-13.437,-97.496,57.442,null,36.855,69.561,3.0,null,-4.0,3.0,-4.0,-15.522,5.934,null,91.203,2.0],"l5_f64":[null,97.873,0.0,null,4.0,1.0,-1.0,-82.395,55.378,1.467,-46.733,-93.682,null,71.854,0.0,-1.0,-2.0,null,null,1.0],"l6_utf8":["c","","é","b","😀","B",null,"",null,null,"ab",null,"a",null,"😀","😀","","B","😀",null]}}},"ops":[{"op":"withColumns","exprs":{"x0":{"col":"l1_i32"}}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_f64","type":"f64"},{"name":"l5_f64","type":"f64"},{"name":"l6_utf8","type":"utf8"},{"name":"x0","type":"i32"}],"columns":{"l0_utf8":["ab","c",null,null,null,"c","B","aa","ab","b",null,null,"B","ab",null,null,"zz",null,"b","B"],"l1_i32":[-4,-2,1,null,-4,2,null,-5,2,null,null,null,4,2,-1,-4,-1,-2147483648,null,null],"l2_f64":[86.534,-53.607,-66.98,15.709,0.0,90.824,-3.0,null,-14.19,-80.437,null,null,3.778,null,-47.743,-62.376,-1.0,null,2.0,3.0],"l3_utf8":["B",null,null,null,"c","aa","",null,"","B","zz","",null,null,null,"B",null,"a",null,""],"l4_f64":[null,-2.0,-1.0,"NaN",-13.437,-97.496,57.442,null,36.855,69.561,3.0,null,-4.0,3.0,-4.0,-15.522,5.934,null,91.203,2.0],"l5_f64":[null,97.873,0.0,null,4.0,1.0,-1.0,-82.395,55.378,1.467,-46.733,-93.682,null,71.854,0.0,-1.0,-2.0,null,null,1.0],"l6_utf8":["c","","é","b","😀","B",null,"",null,null,"ab",null,"a",null,"😀","😀","","B","😀",null],"x0":[-4,-2,1,null,-4,2,null,-5,2,null,null,null,4,2,-1,-4,-1,-2147483648,null,null]}}},{"id":"random/0149","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_i64","type":"i64"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_bool","type":"bool"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_bool":[],"l3_i64":[],"l4_utf8":[],"l5_bool":[]}}},"ops":[{"op":"withColumns","exprs":{"x0":{"fn":"gte","args":[{"fn":"neg","args":[{"lit":2}]},{"fn":"add","args":[{"col":"l1_i32"},{"lit":2.5}]}]}}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_i64","type":"i64"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_bool","type":"bool"},{"name":"x0","type":"bool"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_bool":[],"l3_i64":[],"l4_utf8":[],"l5_bool":[],"x0":[]}}},{"id":"random/0150","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_i32","type":"i32"},{"name":"l4_f64","type":"f64"},{"name":"l5_f64","type":"f64"}],"columns":{"l0_utf8":["ab","😀",null,"B","é"],"l1_i32":[-4,3,2,1,3],"l2_f64":[-4.0,4.0,null,29.092,2.0],"l3_i32":[1,null,null,-2,null],"l4_f64":[null,48.813,89.755,-8.91,null],"l5_f64":[87.465,null,-4.0,null,null]}},"right":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"r0_f64","type":"f64"}],"columns":{"r0_f64":[-85.126,60.31,-5.522,-4.0,75.31,4.0,0.0,null,-1.0,null],"l0_utf8":[null,"B","B","B","ab","B","B","😀","😀","é"]}}},"ops":[{"op":"sort","keys":[{"expr":{"col":"l1_i32"},"descending":false}],"nullsLast":false},{"op":"unique","columns":["l2_f64"]},{"op":"filter","predicate":{"fn":"and","args":[{"fn":"isNotNull","args":[{"col":"l2_f64"}]},{"fn":"neq","args":[{"col":"l4_f64"},{"col":"l3_i32"}]}]}},{"op":"join","right":"right","on":["l0_utf8"],"how":"inner"}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_i32","type":"i32"},{"name":"l4_f64","type":"f64"},{"name":"l5_f64","type":"f64"},{"name":"r0_f64","type":"f64"}],"columns":{"l0_utf8":["B","B","B","B","B"],"l1_i32":[1,1,1,1,1],"l2_f64":[29.092,29.092,29.092,29.092,29.092],"l3_i32":[-2,-2,-2,-2,-2],"l4_f64":[-8.91,-8.91,-8.91,-8.91,-8.91],"l5_f64":[null,null,null,null,null],"r0_f64":[60.31,-5.522,-4.0,4.0,0.0]}}},{"id":"random/0151","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_i32","type":"i32"},{"name":"l4_f64","type":"f64"},{"name":"l5_i64","type":"i64"}],"columns":{"l0_utf8":["b","😀","😀","","ab",null,"","a","c",null,"😀","é","c","ab","😀","","a","é","B",null],"l1_i32":[1,1,-1,-3,1,-1,1,-1,-1,0,null,null,-5,1,5,null,-5,1,2147483647,null],"l2_utf8":["a",null,"B","ab","c","","😀","c","zz","B","😀","aa","zz","zz","zz","c",null,"😀","b",null],"l3_i32":[-4,0,-4,-3,-1,-4,0,4,null,-932,1,3,-89,null,-3,1,-2,-4,0,3],"l4_f64":[-2.0,-96.715,64.633,-2.0,34.835,-2.126,null,"NaN",-28.808,4.0,null,3.0,78.393,15.949,-2.0,-78.44,-4.0,2.0,72.641,-4.0],"l5_i64":["-4611686018427387904","9007199254740995","-4611686018427387904",null,"-4611686018427387904","1",null,"-4611686018427387904","4","9007199254740995","9007199254740994","9007199254740993",null,"9007199254740994","-4611686018427387904","-4611686018427387904","9007199254740993","-4611686018427387904","-2",null]}}},"ops":[{"op":"unique","columns":["l3_i32"]},{"op":"sort","keys":[{"expr":{"col":"l4_f64"},"descending":false},{"expr":{"col":"l1_i32"},"descending":false}],"nullsLast":false},{"op":"filter","predicate":{"fn":"gt","args":[{"col":"l5_i64"},{"lit":"0","type":"i64"}]}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_i32","type":"i32"},{"name":"l4_f64","type":"f64"},{"name":"l5_i64","type":"i64"}],"columns":{"l0_utf8":["😀","😀","c","a","é",null],"l1_i32":[null,1,-1,-5,null,0],"l2_utf8":["😀",null,"zz",null,"aa","B"],"l3_i32":[1,0,null,-2,3,-932],"l4_f64":[null,-96.715,-28.808,-4.0,3.0,4.0],"l5_i64":["9007199254740994","9007199254740995","4","9007199254740993","9007199254740993","9007199254740995"]}}},{"id":"random/0152","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_bool","type":"bool"},{"name":"l4_i32","type":"i32"},{"name":"l5_bool","type":"bool"}],"columns":{"l0_utf8":["","","B","a","ab","zz","a","é","b","c","c","","","B","aa","c","é","😀","😀","😀","zz","B","","a","c","B","aa","B","zz","B","é","é","zz","ab","aa","é","aa","zz","c","aa","a","B","a","c","zz","B","ab","zz","b","c","ab","zz","B","c","c","aa","","é","ab","zz"],"l1_i32":[0,-2147483648,2,-4,3,0,5,-3,1,-5,1,-5,-2,-3,2,0,-2,0,-4,-5,131,-4,-3,-2,-2,-2147483648,5,-2,5,3,-3,1,0,2,-2,-651,0,-1,0,2,-2,-3,-3,-2,4,0,-1,-1,0,3,2,5,-272,-4,1,-5,-5,2,1,-3],"l2_f64":[-3.0,-3.0,-1.0,"-Infinity",4.0,-1.0,0.0,2.0,84.359,98.241,"NaN",2.0,-4.0,3.0,-65.684,64.024,-59.022,-66.915,-18.931,-2.0,-3.0,76.356,-25.823,1.0,-4.0,4.0,-4.0,0.0,-65.241,2.0,2.0,-3.0,3.872,2.0,64.983,2.0,2.0,2.0,3.0,4.0,59.986,-44.445,"-Infinity",2.0,15.858,-3.0,-13.691,-61.578,-2.0,-1.0,-58.97,4.0,-92.025,4.0,3.0,3.0,3.0,-4.0,2.0,"NaN"],"l3_bool":[false,false,true,true,false,false,true,true,true,true,true,false,false,false,true,false,true,false,false,false,false,true,false,false,true,false,true,false,true,true,false,true,true,true,false,true,false,false,false,false,false,false,true,false,true,false,false,true,false,true,true,true,true,false,true,true,false,false,false,false],"l4_i32":[2,-1,-3,-5,5,-3,-1,-1,-2,5,-4,0,-2,-2,-4,1,0,-4,-4,-2,5,5,5,-4,2,-3,4,-4,4,-4,0,-4,2,2,4,-1,-2147483648,-2,-1,0,-3,3,-5,-5,-3,2,3,-4,5,0,5,-5,4,-2147483648,2147483647,-4,5,1,-5,5],"l5_bool":[true,true,true,false,false,true,false,true,false,false,true,true,false,false,true,true,false,true,true,false,false,false,false,true,false,true,false,false,false,false,true,false,true,false,false,true,false,false,false,true,false,false,false,false,false,false,false,false,true,false,false,true,true,false,false,true,true,true,false,false]}}},"ops":[{"op":"sort","keys":[{"expr":{"col":"l1_i32"},"descending":false}],"nullsLast":false},{"op":"filter","predicate":{"fn":"or","args":[{"fn":"gt","args":[{"col":"l2_f64"},{"col":"l2_f64"}]},{"fn":"isIn","args":[{"col":"l0_utf8"}],"values":["b","B","ab"]}]}},{"op":"groupBy","keys":["l5_bool"],"aggs":{"n2":{"agg":"len"},"a2_0":{"agg":"first","arg":{"col":"l1_i32"}},"a2_1":{"agg":"count","arg":{"col":"l3_bool"}},"a2_2":{"agg":"count","arg":{"col":"l4_i32"}}}}],"expected":{"fields":[{"name":"l5_bool","type":"bool"},{"name":"n2","type":"u32"},{"name":"a2_0","type":"i32"},{"name":"a2_1","type":"u32"},{"name":"a2_2","type":"u32"}],"columns":{"l5_bool":[true,false],"n2":[4,12],"a2_0":[-2147483648,-4],"a2_1":[4,12],"a2_2":[4,12]}}},{"id":"random/0153","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_i32","type":"i32"},{"name":"l5_i32","type":"i32"}],"columns":{"l0_utf8":["ab","é","b","aa",null,"a","😀","é","B",null,null,null,null,"é","aa","zz","B",null,null,"",null,null,"é","","B","é","zz",null,null,"é","😀","ab","c","zz",null,"c","ab","b",null,"aa",null,"é","😀",null,null,null,null,null,"😀","a","😀","aa","a",null,"é",null,"zz",null,"b",""],"l1_i32":[-4,1,4,-2,0,-2147483648,-1,-2,-4,-3,3,0,4,3,-2,3,5,-5,-2,5,-5,4,-4,5,null,1,3,-4,-4,null,-3,430,1,null,-2,3,5,null,5,3,null,4,null,null,-4,null,null,null,null,null,-4,-3,null,0,null,1,-1,null,-3,-1],"l2_i32":[2,5,5,null,2,null,-4,464,1,3,-4,-62,-4,4,-2,null,1,null,null,null,-5,0,-3,null,4,1,null,null,null,1,2,null,-3,null,0,null,null,5,-1,null,null,-2,null,3,-2,-4,5,null,null,-5,-5,null,3,null,2,null,null,null,null,-2147483648],"l3_utf8":[null,"B","ab",null,"ab","é",null,"ab","aa",null,"B","",null,null,"ab","aa","é","😀","a","é","😀",null,"b","a","😀",null,null,null,null,null,"😀","😀",null,"ab",null,"😀",null,null,"B","ab",null,"b",null,null,"c",null,"B","😀","é","aa","😀",null,"ab","aa","B","aa","😀",null,"","aa"],"l4_i32":[4,3,-4,-672,null,5,null,2147483647,null,-5,4,0,null,null,null,-1,4,null,2147483647,0,null,null,null,4,0,-2,null,null,3,0,2147483647,5,null,2,2,-1,null,-1,-5,2,1,null,5,-3,-2147483648,-4,1,-4,3,null,null,3,-3,4,null,null,null,null,-4,null],"l5_i32":[-4,null,-4,-5,-3,-4,2,2147483647,null,null,0,4,null,null,null,2,null,null,-5,-2,-2,null,null,null,null,-3,180,-4,null,-1,null,0,null,0,4,-2,0,-2,null,-2147483648,-2147483648,null,-2,4,4,-1,-1,1,null,null,2,-5,3,0,489,null,3,4,-5,-1]}}},"ops":[{"op":"filter","predicate":{"fn":"and","args":[{"fn":"or","args":[{"fn":"lte","args":[{"col":"l5_i32"},{"lit":0.5}]},{"fn":"lt","args":[{"lit":0.5},{"col":"l5_i32"}]}]},{"fn":"isNull","args":[{"col":"l5_i32"}]}]}},{"op":"filter","predicate":{"fn":"gte","args":[{"col":"l2_i32"},{"col":"l2_i32"}]}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_i32","type":"i32"},{"name":"l5_i32","type":"i32"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_i32":[],"l3_utf8":[],"l4_i32":[],"l5_i32":[]}}},{"id":"random/0154","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_bool","type":"bool"},{"name":"l4_utf8","type":"utf8"}],"columns":{"l0_utf8":["ab","ab","aa","aa","c","😀","","aa","c","zz","ab","zz","é","zz","😀","","😀","a","b","a"],"l1_i32":[-3,5,5,-4,-3,0,-2,3,-4,-3,-3,2,0,1,-2,4,5,-5,-3,-3],"l2_f64":[-78.172,2.0,-31.358,"-Infinity",54.072,-95.226,-4.0,54.577,-43.75,4.0,-1.0,-4.0,-47.045,2.0,86.183,98.903,-3.0,-86.161,0.0,-3.0],"l3_bool":[false,true,false,true,false,false,false,true,true,true,false,false,false,true,true,true,true,false,true,false],"l4_utf8":["ab","aa","ab","aa","zz","é","aa","","c","é","ab","c","a","é","b","aa","ab","aa","zz","b"]}},"right":{"fields":[{"name":"l4_utf8","type":"utf8"},{"name":"r0_utf8","type":"utf8"}],"columns":{"r0_utf8":[],"l4_utf8":[]}}},"ops":[{"op":"sort","keys":[{"expr":{"col":"l3_bool"},"descending":false},{"expr":{"col":"l1_i32"},"descending":true}],"nullsLast":false},{"op":"join","right":"right","on":["l4_utf8"],"how":"anti"},{"op":"filter","predicate":{"fn":"neq","args":[{"fn":"mul","args":[{"col":"l1_i32"},{"col":"l2_f64"}]},{"fn":"when","args":[{"fn":"isNull","args":[{"col":"l3_bool"}]},{"col":"l1_i32"},{"lit":0.5}]}]}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_bool","type":"bool"},{"name":"l4_utf8","type":"utf8"}],"columns":{"l0_utf8":["aa","zz","😀","é","","ab","c","ab","a","a","ab","😀","","aa","zz","😀","zz","b","aa","c"],"l1_i32":[5,2,0,0,-2,-3,-3,-3,-3,-5,5,5,4,3,1,-2,-3,-3,-4,-4],"l2_f64":[-31.358,-4.0,-95.226,-47.045,-4.0,-78.172,54.072,-1.0,-3.0,-86.161,2.0,-3.0,98.903,54.577,2.0,86.183,4.0,0.0,"-Infinity",-43.75],"l3_bool":[false,false,false,false,false,false,false,false,false,false,true,true,true,true,true,true,true,true,true,true],"l4_utf8":["ab","c","é","a","aa","ab","zz","ab","b","aa","aa","ab","aa","","é","b","é","zz","aa","c"]}}},{"id":"random/0155","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i64","type":"i64"},{"name":"l3_f64","type":"f64"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_utf8","type":"utf8"},{"name":"l6_i64","type":"i64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_i64":[],"l3_f64":[],"l4_utf8":[],"l5_utf8":[],"l6_i64":[]}},"right":{"fields":[{"name":"l6_i64","type":"i64"},{"name":"r0_utf8","type":"utf8"}],"columns":{"r0_utf8":["a","zz","c","zz",null,null,null,null,null,null],"l6_i64":["9007199254740995",null,"9007199254740993","-4611686018427387904",null,"-4611686018427387904","9007199254740995","9007199254740995","9007199254740995","-4611686018427387904"]}}},"ops":[{"op":"join","right":"right","on":["l6_i64"],"how":"semi"},{"op":"filter","predicate":{"fn":"eq","args":[{"fn":"fillNull","args":[{"col":"l1_i32"},{"lit":1.5}]},{"lit":2.5}]}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i64","type":"i64"},{"name":"l3_f64","type":"f64"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_utf8","type":"utf8"},{"name":"l6_i64","type":"i64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_i64":[],"l3_f64":[],"l4_utf8":[],"l5_utf8":[],"l6_i64":[]}}},{"id":"random/0156","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_f64","type":"f64"},{"name":"l4_i32","type":"i32"},{"name":"l5_utf8","type":"utf8"}],"columns":{"l0_utf8":["","b","ab","a","b"],"l1_i32":[3,1,-4,-2,0],"l2_f64":[2.0,-3.0,50.839,-1.0,-2.0],"l3_f64":[3.0,-8.132,2.0,"NaN",-43.937],"l4_i32":[-2,-5,2,2147483647,-2],"l5_utf8":["a","b","B","b",""]}},"right":{"fields":[{"name":"l1_i32","type":"i32"},{"name":"r0_utf8","type":"utf8"}],"columns":{"r0_utf8":[],"l1_i32":[]}}},"ops":[{"op":"join","right":"right","on":["l1_i32"],"how":"left"},{"op":"unique","columns":["r0_utf8"]},{"op":"filter","predicate":{"fn":"isNull","args":[{"col":"l0_utf8"}]}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_f64","type":"f64"},{"name":"l4_i32","type":"i32"},{"name":"l5_utf8","type":"utf8"},{"name":"r0_utf8","type":"utf8"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_f64":[],"l3_f64":[],"l4_i32":[],"l5_utf8":[],"r0_utf8":[]}}},{"id":"random/0157","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_f64","type":"f64"},{"name":"l4_f64","type":"f64"},{"name":"l5_utf8","type":"utf8"},{"name":"l6_i64","type":"i64"}],"columns":{"l0_utf8":["c",null,"B",null,null,null,null,null,"aa","😀","zz","aa",null,"aa","","ab",null,"b","b",null],"l1_i32":[-3,null,1,null,-4,5,2147483647,null,1,null,2,null,null,null,212,-4,3,0,-1,null],"l2_i32":[null,null,5,2,null,3,2,1,-2,1,3,2,-5,null,-4,null,2,0,-2147483648,null],"l3_f64":[88.229,null,3.0,null,null,null,43.13,-48.14,-1.0,-36.69,-41.651,54.971,3.0,-26.911,4.0,-4.0,"-Infinity",0.0,60.721,-27.958],"l4_f64":[1.0,-4.653,-3.0,2.0,80.555,null,null,"NaN",null,-1.0,"NaN",2.0,3.0,null,-50.776,null,-74.384,-4.0,58.135,"NaN"],"l5_utf8":[null,null,"é","a","aa",null,"ab",null,"",null,"c","😀",null,"😀","b","B","😀",null,"zz","é"],"l6_i64":[null,"-5",null,"9007199254740995",null,"9007199254740994","9007199254740992","2","-4611686018427387904","9007199254740995","-4611686018427387904","9007199254740993",null,"-4611686018427387904","9007199254740992","-1","-5","-4611686018427387904","-4611686018427387904","-4611686018427387904"]}}},"ops":[{"op":"sort","keys":[{"expr":{"col":"l1_i32"},"descending":true},{"expr":{"col":"l0_utf8"},"descending":true}],"nullsLast":true},{"op":"withColumns","exprs":{"x1":{"fn":"gte","args":[{"fn":"abs","args":[{"col":"l2_i32"}]},{"col":"l2_i32"}]}}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_f64","type":"f64"},{"name":"l4_f64","type":"f64"},{"name":"l5_utf8","type":"utf8"},{"name":"l6_i64","type":"i64"},{"name":"x1","type":"bool"}],"columns":{"l0_utf8":[null,"",null,null,"zz","aa","B","b","b","c","ab",null,"😀","aa","aa",null,null,null,null,null],"l1_i32":[2147483647,212,5,3,2,1,1,0,-1,-3,-4,-4,null,null,null,null,null,null,null,null],"l2_i32":[2,-4,3,2,3,-2,5,0,-2147483648,null,null,null,1,2,null,null,2,1,-5,null],"l3_f64":[43.13,4.0,null,"-Infinity",-41.651,-1.0,3.0,0.0,60.721,88.229,-4.0,null,-36.69,54.971,-26.911,null,null,-48.14,3.0,-27.958],"l4_f64":[null,-50.776,null,-74.384,"NaN",null,-3.0,-4.0,58.135,1.0,null,80.555,-1.0,2.0,null,-4.653,2.0,"NaN",3.0,"NaN"],"l5_utf8":["ab","b",null,"😀","c","","é",null,"zz",null,"B","aa",null,"😀","😀",null,"a",null,null,"é"],"l6_i64":["9007199254740992","9007199254740992","9007199254740994","-5","-4611686018427387904","-4611686018427387904",null,"-4611686018427387904","-4611686018427387904",null,"-1",null,"9007199254740995","9007199254740993","-4611686018427387904","-5","9007199254740995","2",null,"-4611686018427387904"],"x1":[true,true,true,true,true,true,true,true,true,null,null,null,true,true,null,null,true,true,true,null]}}},{"id":"random/0158","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_i64","type":"i64"},{"name":"l4_utf8","type":"utf8"}],"columns":{"l0_utf8":["a","","aa","","zz","aa","B","a","b","ab","","é","c","a","","a","zz","ab","aa","aa","a","é","b","ab","c","c","aa","😀","b","","c","zz","😀","😀","a","b","zz","","é","😀","😀","aa","zz","b","c","zz","😀","aa","é","c","zz","zz","aa","c","","zz","é","a","B","B"],"l1_i32":[-5,2,-2,-5,0,5,-1,4,4,3,-3,3,3,-2,-1,2,-4,1,-3,2147483647,1,-5,-1,2,2,2,-3,-1,-4,-79,-1,3,4,5,-4,0,5,5,-5,-4,-2,4,-2,1,-3,1,-5,0,3,-3,4,-2147483648,-5,1,0,-4,201,-2,5,-2],"l2_i32":[3,2,-4,2,5,1,2,4,-5,-1,3,-4,1,-4,1,-5,-1,0,1,2,-43,1,4,3,-1,3,-4,-3,2147483647,-2147483648,-1,0,-3,1,-2,-4,2,5,191,-5,4,-2,4,-3,-2,-4,3,4,-4,-1,-2,2,3,0,1,5,-5,-5,-3,2147483647],"l3_i64":["-4611686018427387904","2","-4611686018427387904","-2","-3","-2","1","9007199254740992","-1","9007199254740995","-5","-4611686018427387904","9007199254740995","-4611686018427387904","-4611686018427387904","9007199254740992","9007199254740993","9007199254740993","-4611686018427387904","-3","9007199254740993","-4611686018427387904","-4611686018427387904","0","1","9007199254740992","9007199254740994","-4611686018427387904","-4611686018427387904","9007199254740994","-4611686018427387904","-4611686018427387904","2","9007199254740992","9007199254740992","2","9007199254740995","-4611686018427387904","2","-4611686018427387904","9007199254740992","9007199254740995","-2","-4611686018427387904","9007199254740992","-5","9007199254740993","-4611686018427387904","9007199254740993","-4611686018427387904","-2","-4611686018427387904","2","9007199254740992","2","-4611686018427387904","4","-4","3","-4611686018427387904"],"l4_utf8":["zz","b","é","c","c","aa","c","B","aa","a","c","a","b","😀","a","é","c","a","a","é","b","b","aa","B","","aa","a","é","b","aa","😀","","zz","ab","","😀","B","c","B","B","","","","ab","😀","aa","","","aa","aa","b","c","c","aa","B","b","aa","b","é",""]}}},"ops":[{"op":"head","n":10},{"op":"unique"},{"op":"groupBy","keys":["l1_i32"],"aggs":{"n2":{"agg":"len"},"a2_0":{"agg":"max","arg":{"col":"l2_i32"}},"a2_1":{"agg":"nUnique","arg":{"col":"l3_i64"}},"a2_2":{"agg":"sum","arg":{"col":"l2_i32"}}}},{"op":"filter","predicate":{"fn":"or","args":[{"fn":"and","args":[{"fn":"lt","args":[{"lit":-1},{"col":"a2_2"}]},{"fn":"gt","args":[{"col":"a2_2"},{"col":"a2_2"}]}]},{"fn":"neq","args":[{"lit":2},{"col":"a2_0"}]}]}}],"expected":{"fields":[{"name":"l1_i32","type":"i32"},{"name":"n2","type":"u32"},{"name":"a2_0","type":"i32"},{"name":"a2_1","type":"u32"},{"name":"a2_2","type":"i32"}],"columns":{"l1_i32":[-5,-2,0,5,4,3],"n2":[2,1,1,1,2,1],"a2_0":[3,-4,5,1,4,-1],"a2_1":[2,1,1,1,2,1],"a2_2":[5,-4,5,1,-1,-1]}}},{"id":"random/0159","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_f64","type":"f64"},{"name":"l4_i32","type":"i32"},{"name":"l5_utf8","type":"utf8"},{"name":"l6_i64","type":"i64"}],"columns":{"l0_utf8":["zz"],"l1_i32":[2147483647],"l2_f64":[0.0],"l3_f64":[97.147],"l4_i32":[null],"l5_utf8":[null],"l6_i64":["-4611686018427387904"]}},"right":{"fields":[{"name":"l1_i32","type":"i32"},{"name":"r0_f64","type":"f64"}],"columns":{"r0_f64":[],"l1_i32":[]}}},"ops":[{"op":"withColumns","exprs":{"x0":{"fn":"mul","args":[{"fn":"fillNull","args":[{"col":"l3_f64"},{"lit":1.5}]},{"fn":"abs","args":[{"col":"l4_i32"}]}]}}},{"op":"filter","predicate":{"fn":"gte","args":[{"fn":"abs","args":[{"col":"l3_f64"}]},{"col":"l3_f64"}]}},{"op":"join","right":"right","on":["l1_i32"],"how":"inner"},{"op":"sort","keys":[{"expr":{"col":"l4_i32"},"descending":false},{"expr":{"col":"l3_f64"},"descending":false}],"nullsLast":true}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_f64","type":"f64"},{"name":"l4_i32","type":"i32"},{"name":"l5_utf8","type":"utf8"},{"name":"l6_i64","type":"i64"},{"name":"x0","type":"f64"},{"name":"r0_f64","type":"f64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_f64":[],"l3_f64":[],"l4_i32":[],"l5_utf8":[],"l6_i64":[],"x0":[],"r0_f64":[]}}},{"id":"random/0160","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_bool","type":"bool"},{"name":"l4_i64","type":"i64"},{"name":"l5_i32","type":"i32"},{"name":"l6_i64","type":"i64"}],"columns":{"l0_utf8":["B"],"l1_i32":[-1],"l2_bool":[null],"l3_bool":[null],"l4_i64":["5"],"l5_i32":[4],"l6_i64":["4"]}}},"ops":[{"op":"withColumns","exprs":{"x0":{"col":"l1_i32"}}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_bool","type":"bool"},{"name":"l4_i64","type":"i64"},{"name":"l5_i32","type":"i32"},{"name":"l6_i64","type":"i64"},{"name":"x0","type":"i32"}],"columns":{"l0_utf8":["B"],"l1_i32":[-1],"l2_bool":[null],"l3_bool":[null],"l4_i64":["5"],"l5_i32":[4],"l6_i64":["4"],"x0":[-1]}}},{"id":"random/0161","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_f64","type":"f64"},{"name":"l4_utf8","type":"utf8"}],"columns":{"l0_utf8":["é","ab",null,"c",null],"l1_i32":[5,null,5,5,null],"l2_f64":[null,1.0,54.199,2.0,3.206],"l3_f64":[-37.154,0.0,null,-52.505,1.0],"l4_utf8":["","zz",null,"é","c"]}}},"ops":[{"op":"head","n":10},{"op":"groupBy","keys":["l4_utf8"],"aggs":{"n1":{"agg":"len"},"a1_0":{"agg":"max","arg":{"col":"l3_f64"}}}},{"op":"sort","keys":[{"expr":{"col":"n1"},"descending":false},{"expr":{"col":"a1_0"},"descending":false}],"nullsLast":false}],"expected":{"fields":[{"name":"l4_utf8","type":"utf8"},{"name":"n1","type":"u32"},{"name":"a1_0","type":"f64"}],"columns":{"l4_utf8":[null,"é","","zz","c"],"n1":[1,1,1,1,1],"a1_0":[null,-52.505,-37.154,0.0,1.0]}}},{"id":"random/0162","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_f64","type":"f64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_f64":[],"l3_f64":[]}},"right":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"r0_utf8","type":"utf8"}],"columns":{"r0_utf8":["ab","c","zz","é","😀","é","a","b","b","","a","ab","a",null,"c","aa","aa","aa","é","zz","😀","a","ab","é","😀","B","é","ab","zz","aa"],"l0_utf8":["","ab",null,"a","😀","a","a","😀","","a",null,null,"ab","a","😀",null,"😀","😀",null,"a","","a","",null,"a",null,"ab","a","",null]}}},"ops":[{"op":"join","right":"right","on":["l0_utf8"],"how":"left"},{"op":"sort","keys":[{"expr":{"col":"l2_f64"},"descending":true},{"expr":{"col":"l1_i32"},"descending":true}],"nullsLast":true},{"op":"filter","predicate":{"fn":"gte","args":[{"fn":"floorDiv","args":[{"col":"l1_i32"},{"col":"l1_i32"}]},{"col":"l2_f64"}]}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_f64","type":"f64"},{"name":"r0_utf8","type":"utf8"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_f64":[],"l3_f64":[],"r0_utf8":[]}}},{"id":"random/0163","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_i32","type":"i32"},{"name":"l4_i64","type":"i64"},{"name":"l5_bool","type":"bool"}],"columns":{"l0_utf8":["zz","a","b","é","c"],"l1_i32":[null,null,2,-5,null],"l2_utf8":["ab",null,"b",null,"aa"],"l3_i32":[-4,-1,-1,null,5],"l4_i64":[null,null,null,null,null],"l5_bool":[null,false,true,null,false]}}},"ops":[{"op":"filter","predicate":{"fn":"or","args":[{"fn":"eq","args":[{"col":"l3_i32"},{"col":"l1_i32"}]},{"fn":"eq","args":[{"col":"l1_i32"},{"col":"l1_i32"}]}]}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_i32","type":"i32"},{"name":"l4_i64","type":"i64"},{"name":"l5_bool","type":"bool"}],"columns":{"l0_utf8":["b","é"],"l1_i32":[2,-5],"l2_utf8":["b",null],"l3_i32":[-1,null],"l4_i64":[null,null],"l5_bool":[true,null]}}},{"id":"random/0164","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_f64","type":"f64"}],"columns":{"l0_utf8":[null,"c",null,"c","😀","B",null,"aa","b","a",null,"zz","ab","é",null,"","","😀","",""],"l1_i32":[2,3,1,null,-2,null,null,3,-4,2,-3,-2,null,0,null,null,null,-1,null,-5],"l2_f64":[-25.4,null,59.931,-95.535,null,0.0,-28.035,null,null,null,null,75.582,null,null,89.818,-46.36,null,-4.0,50.941,21.846],"l3_utf8":["B","a",null,"B","B",null,null,null,"","é","😀","é","b",null,null,"ab","c","a","é","b"],"l4_f64":[83.783,-1.0,96.957,4.0,1.0,null,-1.0,-47.318,null,null,-52.309,1.0,0.0,-3.0,4.0,70.654,null,-4.0,-1.0,"NaN"]}}},"ops":[{"op":"sort","keys":[{"expr":{"col":"l1_i32"},"descending":false}],"nullsLast":true}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_f64","type":"f64"}],"columns":{"l0_utf8":["","b",null,"😀","zz","😀","é",null,null,"a","c","aa","c","B",null,"ab",null,"","",""],"l1_i32":[-5,-4,-3,-2,-2,-1,0,1,2,2,3,3,null,null,null,null,null,null,null,null],"l2_f64":[21.846,null,null,null,75.582,-4.0,null,59.931,-25.4,null,null,null,-95.535,0.0,-28.035,null,89.818,-46.36,null,50.941],"l3_utf8":["b","","😀","B","é","a",null,null,"B","é","a",null,"B",null,null,"b",null,"ab","c","é"],"l4_f64":["NaN",null,-52.309,1.0,1.0,-4.0,-3.0,96.957,83.783,null,-1.0,-47.318,4.0,null,-1.0,0.0,4.0,70.654,null,-1.0]}}},{"id":"random/0165","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_i64","type":"i64"}],"columns":{"l0_utf8":["c"],"l1_i32":[0],"l2_utf8":["aa"],"l3_i64":["-4611686018427387904"]}}},"ops":[{"op":"groupBy","keys":["l2_utf8"],"aggs":{"n0":{"agg":"len"},"a0_0":{"agg":"count","arg":{"col":"l3_i64"}},"a0_1":{"agg":"nUnique","arg":{"col":"l0_utf8"}},"a0_2":{"agg":"count","arg":{"col":"l0_utf8"}}}},{"op":"filter","predicate":{"fn":"strStartsWith","args":[{"col":"l2_utf8"}],"prefix":"a"}}],"expected":{"fields":[{"name":"l2_utf8","type":"utf8"},{"name":"n0","type":"u32"},{"name":"a0_0","type":"u32"},{"name":"a0_1","type":"u32"},{"name":"a0_2","type":"u32"}],"columns":{"l2_utf8":["aa"],"n0":[1],"a0_0":[1],"a0_1":[1],"a0_2":[1]}}},{"id":"random/0166","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_i32","type":"i32"},{"name":"l4_i64","type":"i64"},{"name":"l5_bool","type":"bool"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_utf8":[],"l3_i32":[],"l4_i64":[],"l5_bool":[]}}},"ops":[{"op":"head","n":1},{"op":"groupBy","keys":["l5_bool"],"aggs":{"n1":{"agg":"len"},"a1_0":{"agg":"last","arg":{"col":"l0_utf8"}}}},{"op":"withColumns","exprs":{"x2":{"fn":"add","args":[{"lit":2.5},{"lit":2}]}}}],"expected":{"fields":[{"name":"l5_bool","type":"bool"},{"name":"n1","type":"u32"},{"name":"a1_0","type":"utf8"},{"name":"x2","type":"f64"}],"columns":{"l5_bool":[],"n1":[],"a1_0":[],"x2":[]}}},{"id":"random/0167","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_utf8","type":"utf8"}],"columns":{"l0_utf8":["b","ab","😀",null,"","","","😀","c","","a","b",null,"ab","c","","zz","","zz",null],"l1_i32":[-1,-3,0,0,5,-5,0,-2147483648,-3,-5,683,1,5,null,2,-2,-5,-5,3,2147483647],"l2_f64":[76.958,-0.784,3.0,null,90.124,4.0,-4.0,"NaN",-86.322,-90.133,-2.0,null,"NaN",89.251,52.872,-3.0,-1.0,0.0,-2.0,null],"l3_utf8":["b","","😀","😀","😀","😀","é","c","aa","ab","b","b","aa","zz","é",null,"zz","b","😀","zz"]}}},"ops":[{"op":"sort","keys":[{"expr":{"col":"l2_f64"},"descending":true}],"nullsLast":false},{"op":"groupBy","keys":["l1_i32"],"aggs":{"n1":{"agg":"len"},"a1_0":{"agg":"min","arg":{"col":"l0_utf8"}}}}],"expected":{"fields":[{"name":"l1_i32","type":"i32"},{"name":"n1","type":"u32"},{"name":"a1_0","type":"utf8"}],"columns":{"l1_i32":[0,1,2147483647,-2147483648,5,null,-1,2,-5,-3,683,3,-2],"n1":[3,1,1,1,2,1,1,1,4,2,1,1,1],"a1_0":["","b",null,"😀","","ab","b","c","","ab","a","zz",""]}}},{"id":"random/0168","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_i64","type":"i64"}],"columns":{"l0_utf8":["a"],"l1_i32":[1],"l2_i32":[4],"l3_i64":["-2"]}}},"ops":[{"op":"filter","predicate":{"fn":"eq","args":[{"col":"l3_i64"},{"lit":"-1","type":"i64"}]}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_i64","type":"i64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_i32":[],"l3_i64":[]}}},{"id":"random/0169","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_i64","type":"i64"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_i64","type":"i64"}],"columns":{"l0_utf8":["B"],"l1_i32":[-1],"l2_f64":[32.563],"l3_i64":["9007199254740992"],"l4_utf8":["é"],"l5_i64":["-4611686018427387904"]}},"right":{"fields":[{"name":"l4_utf8","type":"utf8"},{"name":"r0_utf8","type":"utf8"}],"columns":{"r0_utf8":["aa","c","ab"],"l4_utf8":["é","é","é"]}}},"ops":[{"op":"filter","predicate":{"fn":"lte","args":[{"fn":"mul","args":[{"col":"l2_f64"},{"col":"l2_f64"}]},{"col":"l2_f64"}]}},{"op":"join","right":"right","on":["l4_utf8"],"how":"full"}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_i64","type":"i64"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_i64","type":"i64"},{"name":"r0_utf8","type":"utf8"}],"columns":{"l0_utf8":[null,null,null],"l1_i32":[null,null,null],"l2_f64":[null,null,null],"l3_i64":[null,null,null],"l4_utf8":["é","é","é"],"l5_i64":[null,null,null],"r0_utf8":["aa","c","ab"]}}},{"id":"random/0170","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_f64","type":"f64"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_utf8","type":"utf8"}],"columns":{"l0_utf8":["c","","c","ab","b","c","ab","B","😀","","zz",null,"é","b","ab","ab","b","ab",null,null],"l1_i32":[0,3,null,-2,-850,null,null,null,2147483647,null,1,null,511,null,null,-1,-4,-1,null,2],"l2_i32":[-2,3,null,0,-2,-3,-4,3,-3,1,2,4,-783,null,null,null,null,5,-3,0],"l3_f64":[-49.32,"NaN",null,null,71.79,-4.0,null,-2.0,1.0,-3.794,37.924,-4.0,4.0,4.0,-2.0,97.559,-47.037,49.061,-28.537,-28.574],"l4_utf8":[null,"c","ab","ab","a",null,"ab","B",null,"a","B","😀",null,null,"b","aa",null,"ab","é","zz"],"l5_utf8":["","ab","b",null,"","ab","B",null,null,"B",null,"b","ab","ab","aa","","zz","B","é",""]}}},"ops":[{"op":"groupBy","keys":["l4_utf8"],"aggs":{"n0":{"agg":"len"},"a0_0":{"agg":"min","arg":{"col":"l2_i32"}},"a0_1":{"agg":"max","arg":{"col":"l0_utf8"}}}},{"op":"withColumns","exprs":{"x1":{"fn":"when","args":[{"fn":"or","args":[{"fn":"lte","args":[{"col":"a0_0"},{"col":"a0_0"}]},{"fn":"isNull","args":[{"col":"l4_utf8"}]}]},{"col":"a0_0"},{"fn":"fillNull","args":[{"col":"a0_0"},{"lit":1.5}]}]}}},{"op":"filter","predicate":{"fn":"gt","args":[{"col":"a0_0"},{"fn":"sub","args":[{"col":"a0_0"},{"col":"a0_0"}]}]}},{"op":"sort","keys":[{"expr":{"col":"n0"},"descending":false}],"nullsLast":false}],"expected":{"fields":[{"name":"l4_utf8","type":"utf8"},{"name":"n0","type":"u32"},{"name":"a0_0","type":"i32"},{"name":"a0_1","type":"utf8"},{"name":"x1","type":"f64"}],"columns":{"l4_utf8":["c","😀","B"],"n0":[1,1,2],"a0_0":[3,4,2],"a0_1":["",null,"zz"],"x1":[3.0,4.0,2.0]}}},{"id":"random/0171","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_utf8","type":"utf8"}],"columns":{"l0_utf8":["b","aa","b","b","b"],"l1_i32":[-1,-3,2,1,-3],"l2_utf8":[null,"😀","ab","b","aa"],"l3_utf8":["aa","a","B","zz","zz"]}},"right":{"fields":[{"name":"l1_i32","type":"i32"},{"name":"r0_bool","type":"bool"}],"columns":{"r0_bool":[false,false,false],"l1_i32":[1,-1,-3]}}},"ops":[{"op":"withColumns","exprs":{"x0":{"fn":"gt","args":[{"col":"l1_i32"},{"col":"l1_i32"}]}}},{"op":"join","right":"right","on":["l1_i32"],"how":"left"},{"op":"sort","keys":[{"expr":{"col":"l2_utf8"},"descending":false}],"nullsLast":false},{"op":"sort","keys":[{"expr":{"col":"l1_i32"},"descending":true},{"expr":{"col":"l2_utf8"},"descending":false}],"nullsLast":false}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_utf8","type":"utf8"},{"name":"x0","type":"bool"},{"name":"r0_bool","type":"bool"}],"columns":{"l0_utf8":["b","b","b","b","aa"],"l1_i32":[2,1,-1,-3,-3],"l2_utf8":["ab","b",null,"aa","😀"],"l3_utf8":["B","zz","aa","zz","a"],"x0":[false,false,false,false,false],"r0_bool":[null,false,false,false,false]}}},{"id":"random/0172","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_i32","type":"i32"},{"name":"l5_utf8","type":"utf8"},{"name":"l6_utf8","type":"utf8"}],"columns":{"l0_utf8":["zz","ab","é","c","c"],"l1_i32":[-2147483648,-4,2,-5,1],"l2_f64":[2.0,-42.236,"NaN",4.0,20.388],"l3_utf8":["B","a","é","","zz"],"l4_i32":[-5,-2147483648,1,-1,3],"l5_utf8":["zz","","ab","aa","aa"],"l6_utf8":["","ab","c","","😀"]}}},"ops":[{"op":"sort","keys":[{"expr":{"col":"l6_utf8"},"descending":true},{"expr":{"col":"l0_utf8"},"descending":true}],"nullsLast":true},{"op":"groupBy","keys":["l0_utf8"],"aggs":{"n1":{"agg":"len"},"a1_0":{"agg":"first","arg":{"col":"l6_utf8"}},"a1_1":{"agg":"min","arg":{"col":"l2_f64"}},"a1_2":{"agg":"last","arg":{"col":"l1_i32"}}}},{"op":"sort","keys":[{"expr":{"col":"a1_2"},"descending":false},{"expr":{"col":"a1_1"},"descending":false}],"nullsLast":false},{"op":"withColumns","exprs":{"x3":{"fn":"lte","args":[{"col":"a1_2"},{"fn":"div","args":[{"lit":2},{"col":"a1_1"}]}]}}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"n1","type":"u32"},{"name":"a1_0","type":"utf8"},{"name":"a1_1","type":"f64"},{"name":"a1_2","type":"i32"},{"name":"x3","type":"bool"}],"columns":{"l0_utf8":["zz","c","ab","é"],"n1":[1,2,1,1],"a1_0":["","😀","ab","c"],"a1_1":[2.0,4.0,-42.236,"NaN"],"a1_2":[-2147483648,-5,-4,2],"x3":[true,true,true,true]}}},{"id":"random/0173","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_utf8","type":"utf8"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_f64":[],"l3_utf8":[],"l4_utf8":[]}}},"ops":[{"op":"withColumns","exprs":{"x0":{"lit":0.5}}},{"op":"unique","columns":["x0"]}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_utf8","type":"utf8"},{"name":"x0","type":"f64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_f64":[],"l3_utf8":[],"l4_utf8":[],"x0":[]}}},{"id":"random/0174","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_i32","type":"i32"}],"columns":{"l0_utf8":["a","a","B","a","é","c","b","ab","c","é","😀","B","aa",null,"","b","aa","😀","a","b"],"l1_i32":[3,1,-5,-3,-4,3,0,-1,-4,-3,-4,2,-2147483648,-974,-4,-5,2147483647,2,5,null],"l2_f64":[-1.0,null,0.593,null,18.255,70.064,4.0,42.864,-2.0,-72.168,"NaN",-4.0,39.957,-90.685,-32.366,4.0,-3.0,2.0,"-0",-22.617],"l3_i32":[4,-5,-4,-5,3,-1,2,-5,-4,-947,-5,-4,2,0,-4,-1,null,null,5,3]}}},"ops":[{"op":"filter","predicate":{"fn":"gt","args":[{"col":"l3_i32"},{"fn":"floorDiv","args":[{"col":"l2_f64"},{"col":"l3_i32"}]}]}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_i32","type":"i32"}],"columns":{"l0_utf8":["a","c","ab",null,"b","a","b"],"l1_i32":[3,3,-1,-974,-5,5,null],"l2_f64":[-1.0,70.064,42.864,-90.685,4.0,"-0",-22.617],"l3_i32":[4,-1,-5,0,-1,5,3]}}},{"id":"random/0175","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_bool","type":"bool"},{"name":"l4_f64","type":"f64"}],"columns":{"l0_utf8":[null,null,"zz",null,null,"ab","😀","ab","a","aa","zz",null,null,"a","ab","ab","","😀","a","ab"],"l1_i32":[0,null,null,-3,null,-4,-1,1,null,null,-3,-5,-1,3,3,-464,-3,3,-4,null],"l2_utf8":[null,"B","B","b","b","aa","a",null,"a","c","😀","B","zz","ab","b","a","c","c","B","é"],"l3_bool":[true,true,null,true,true,false,true,true,true,null,null,false,null,true,null,true,false,true,false,false],"l4_f64":[null,-4.0,null,null,-0.679,-4.0,null,35.308,4.0,"NaN",87.138,35.986,-15.632,-48.286,-44.037,null,-75.71,1.0,4.0,null]}}},"ops":[{"op":"withColumns","exprs":{"x0":{"fn":"when","args":[{"fn":"neq","args":[{"col":"l4_f64"},{"col":"l1_i32"}]},{"col":"l1_i32"},{"col":"l1_i32"}]}}},{"op":"withColumns","exprs":{"x1":{"fn":"isNull","args":[{"col":"l3_bool"}]}}},{"op":"filter","predicate":{"fn":"strStartsWith","args":[{"col":"l0_utf8"}],"prefix":"a"}},{"op":"filter","predicate":{"fn":"lt","args":[{"fn":"sub","args":[{"lit":0.5},{"col":"l1_i32"}]},{"fn":"mul","args":[{"col":"l4_f64"},{"col":"l4_f64"}]}]}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_bool","type":"bool"},{"name":"l4_f64","type":"f64"},{"name":"x0","type":"i32"},{"name":"x1","type":"bool"}],"columns":{"l0_utf8":["ab","ab","a","ab","a"],"l1_i32":[-4,1,3,3,-4],"l2_utf8":["aa",null,"ab","b","B"],"l3_bool":[false,true,true,null,false],"l4_f64":[-4.0,35.308,-48.286,-44.037,4.0],"x0":[-4,1,3,3,-4],"x1":[false,false,false,true,false]}}},{"id":"random/0176","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_bool","type":"bool"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_f64","type":"f64"},{"name":"l6_i64","type":"i64"}],"columns":{"l0_utf8":["",null,"zz",null,"","B",null,null,"😀",null,null,null,"😀",null,"a","ab",null,"ab","ab","ab","","c","","😀","aa",null,null,"😀",null,null,"a",null,"B","a",null,null,"ab","zz","B","ab","B","😀","😀",null,"a","zz","aa","aa","b",null,"zz",null,null,null,"😀",null,null,"😀",null,"B"],"l1_i32":[-3,null,-5,null,null,null,5,null,null,null,-3,2,null,4,null,3,-5,4,4,-5,null,null,null,null,2147483647,-2147483648,null,-2,null,null,-3,-5,0,-3,4,-3,5,null,-3,0,1,1,null,-5,-2,null,-1,-5,-5,-3,0,null,4,3,0,1,-5,-5,4,-2],"l2_f64":[41.511,-3.0,"NaN",null,null,74.465,null,-16.684,4.0,24.067,null,-4.0,50.247,2.0,26.984,49.046,-3.0,30.488,-36.623,-2.0,null,-43.541,-4.0,null,-11.235,null,null,4.0,-87.602,null,null,4.0,-77.958,3.0,null,null,-2.0,3.0,-61.422,27.453,3.0,1.0,null,3.0,2.0,null,1.0,-4.0,-3.0,null,null,41.178,null,-4.0,null,4.0,36.737,62.647,-98.38,-4.0],"l3_bool":[null,null,true,true,null,true,null,true,false,true,true,true,true,null,false,true,false,true,null,false,false,false,true,null,false,null,null,null,null,false,true,true,null,true,true,true,true,false,false,false,null,true,null,true,null,null,true,null,true,false,null,false,false,false,false,true,true,null,false,null],"l4_utf8":["zz","aa","","a",null,"b","a","😀","a","zz","aa",null,"B","zz","aa",null,null,null,"zz","é",null,"c",null,null,"b","B","c","a","b",null,"ab","a","a","ab","","😀","aa","c","c","😀",null,"zz","aa",null,"a","ab","😀",null,null,"😀","b",null,"ab","c","zz","é","B","B","a","B"],"l5_f64":[-1.0,-1.0,null,null,4.0,null,"-Infinity",45.817,null,-2.0,-1.0,3.0,-2.977,3.0,-72.494,null,-1.0,null,-35.721,-1.0,"-Infinity",-3.0,-94.137,0.0,null,-1.0,null,null,-84.754,-1.0,"NaN",71.179,null,3.0,4.193,78.118,-19.278,null,58.217,-45.336,58.631,2.0,4.0,-67.984,null,null,null,1.0,null,-4.0,3.0,-1.0,null,null,null,null,3.0,-15.707,-4.0,90.511],"l6_i64":["-4611686018427387904",null,"-2","-4611686018427387904","-4611686018427387904","9007199254740994","-2",null,null,"-4611686018427387904",null,"-2","-4611686018427387904","-4611686018427387904","9007199254740995",null,null,"-3","-4611686018427387904","4","-5","-4611686018427387904","9007199254740992","9007199254740993","9007199254740995",null,"9007199254740994","9007199254740995",null,null,"-4611686018427387904","9007199254740993","9007199254740993","-4611686018427387904","9007199254740994","9007199254740995","9007199254740993","9007199254740992","-2","0","2","-4611686018427387904","9007199254740994","4","-5","4","-4611686018427387904","5",null,"-4611686018427387904","9007199254740992","-4611686018427387904","9007199254740994","-4611686018427387904","-4611686018427387904","-4611686018427387904","9007199254740994",null,"1","-4611686018427387904"]}}},"ops":[{"op":"filter","predicate":{"col":"l3_bool"}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_bool","type":"bool"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_f64","type":"f64"},{"name":"l6_i64","type":"i64"}],"columns":{"l0_utf8":["zz",null,"B",null,null,null,null,"😀","ab","ab","","a",null,"a",null,null,"ab","😀",null,"aa","b",null,null],"l1_i32":[-5,null,null,null,null,-3,2,null,3,4,null,-3,-5,-3,4,-3,5,1,-5,-1,-5,1,-5],"l2_f64":["NaN",null,74.465,-16.684,24.067,null,-4.0,50.247,49.046,30.488,-4.0,null,4.0,3.0,null,null,-2.0,1.0,3.0,1.0,-3.0,4.0,36.737],"l3_bool":[true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true],"l4_utf8":["","a","b","😀","zz","aa",null,"B",null,null,null,"ab","a","ab","","😀","aa","zz",null,"😀",null,"é","B"],"l5_f64":[null,null,null,45.817,-2.0,-1.0,3.0,-2.977,null,null,-94.137,"NaN",71.179,3.0,4.193,78.118,-19.278,2.0,-67.984,null,null,null,3.0],"l6_i64":["-2","-4611686018427387904","9007199254740994",null,"-4611686018427387904",null,"-2","-4611686018427387904",null,"-3","9007199254740992","-4611686018427387904","9007199254740993","-4611686018427387904","9007199254740994","9007199254740995","9007199254740993","-4611686018427387904","4","-4611686018427387904",null,"-4611686018427387904","9007199254740994"]}}},{"id":"random/0177","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_i64","type":"i64"}],"columns":{"l0_utf8":["é","b",null,"ab",null,"ab",null,"aa","a","c",null,"a","B","aa","a",null,"aa","c",null,"b"],"l1_i32":[-3,-2147483648,-1,-4,null,0,2,-1,5,3,2147483647,1,3,1,-1,-2,-2,0,-2,-5],"l2_f64":[0.0,"NaN",96.439,2.0,null,null,63.372,31.718,-53.943,31.912,2.0,"NaN","-Infinity",-97.364,2.0,null,1.0,0.0,-33.8,0.0],"l3_utf8":["c","c",null,"B","zz",null,"a","é","b","ab","c","ab","aa","é","","a","c","aa","ab",null],"l4_utf8":["",null,null,null,"c",null,"c","aa","b","😀",null,"aa",null,"c","c","aa",null,"😀","B","aa"],"l5_i64":[null,"9007199254740993","-4611686018427387904","9007199254740992",null,"4","9007199254740995","9007199254740994","-4611686018427387904",null,"2","-4611686018427387904",null,null,null,"-4611686018427387904",null,"9007199254740992","9007199254740995",null]}}},"ops":[{"op":"filter","predicate":{"fn":"eq","args":[{"col":"l5_i64"},{"lit":"-3","type":"i64"}]}},{"op":"unique","columns":["l2_f64"]},{"op":"filter","predicate":{"fn":"gte","args":[{"fn":"div","args":[{"lit":-1},{"lit":2}]},{"fn":"abs","args":[{"lit":0.5}]}]}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_i64","type":"i64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_f64":[],"l3_utf8":[],"l4_utf8":[],"l5_i64":[]}}},{"id":"random/0178","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_i64","type":"i64"}],"columns":{"l0_utf8":["b","c",null,"aa","é","b",null,"ab","zz","😀","😀",null,null,"aa","a",null,"aa","aa",null,"ab","b","b","b","a",null,"ab","","ab","ab","aa","é","","","B","zz","😀","ab","a",null,null,"a",null,"ab","😀",null,"aa",null,null,"","",null,null,"zz",null,"","😀","é","b","B","zz"],"l1_i32":[0,-2,null,null,null,5,null,null,3,-2,5,-5,4,null,null,5,null,-3,null,null,2,null,null,-4,null,3,2,2147483647,null,0,null,5,5,-2,null,2,null,null,-5,null,1,-2147483648,null,null,null,3,null,-1,3,null,-5,null,4,1,1,null,-4,5,-3,4],"l2_i32":[-237,-3,318,null,null,4,null,1,5,null,null,0,-3,5,4,2,-3,4,null,1,null,-4,null,3,3,3,3,5,null,4,null,null,-5,-2,null,-4,5,-3,1,-5,-1,null,0,-4,-1,2,5,-1,null,-2,-3,null,null,4,0,null,null,null,-1,0],"l3_i64":["-3","9007199254740992",null,null,"9007199254740995","-4611686018427387904","9007199254740995","2","9007199254740994","1","-4611686018427387904","-4611686018427387904","-4","-4611686018427387904",null,"-4611686018427387904","3",null,"3","4","-5",null,"-4","-2","-4611686018427387904","-4","3",null,null,"-4611686018427387904",null,"-4611686018427387904","5",null,null,null,"5",null,"-3","2","9007199254740994",null,"5","-3","-4611686018427387904","9007199254740994","-4611686018427387904","9007199254740993","9007199254740995","-4611686018427387904",null,"-4611686018427387904","9007199254740992",null,null,null,"9007199254740994","-1","-4611686018427387904",null]}}},"ops":[{"op":"groupBy","keys":["l3_i64"],"aggs":{"n0":{"agg":"len"},"a0_0":{"agg":"last","arg":{"col":"l0_utf8"}}}},{"op":"head","n":9}],"expected":{"fields":[{"name":"l3_i64","type":"i64"},{"name":"n0","type":"u32"},{"name":"a0_0","type":"utf8"}],"columns":{"l3_i64":["-3","9007199254740992",null,"9007199254740995","-4611686018427387904","2","9007199254740994","1","-4"],"n0":[3,2,18,3,13,2,4,1,3],"a0_0":["😀","zz","zz","","B",null,"é","😀","ab"]}}},{"id":"random/0179","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_f64","type":"f64"},{"name":"l5_bool","type":"bool"},{"name":"l6_f64","type":"f64"}],"columns":{"l0_utf8":["","b","aa",null,"B","ab","aa","ab","😀",null,null,"é","B","B","c","","😀","B","zz","😀","zz",null,"",null,"B","é","😀","zz",null,"",null,"ab",null,"c","ab","😀","c","c","😀","b","c",null,"","a","B",null,"B",null,"c","","é","😀","B","é","é","B","😀",null,"c","😀"],"l1_i32":[-5,0,-4,-1,null,2,null,0,null,0,-5,-5,null,-4,null,-4,-2147483648,-4,-4,-3,0,-5,null,null,null,1,2,-1,null,0,null,null,-2147483648,null,null,2147483647,-2,-2,-3,null,null,0,null,-1,-4,3,null,5,null,-5,4,null,1,5,null,-1,-217,null,null,5],"l2_utf8":[null,"B","zz","","aa",null,"😀","é","c",null,"B",null,null,null,"ab","a","a","a","ab","","","c",null,"😀","a",null,null,"aa",null,"ab","é","é",null,null,"a","","😀","zz",null,"c","",null,null,"B","c",null,"zz","c",null,"é","zz",null,"é",null,null,"c","😀","a",null,null],"l3_utf8":[null,"c","zz",null,"zz","aa","ab","zz","aa","😀","b","c","c",null,null,"",null,null,null,"zz","zz",null,"c",null,"é","c","a","aa","😀","","é","é","a",null,null,"B","b","ab","c","zz",null,null,"c","aa","a",null,"B","aa","é","ab","zz","","😀","",null,"é","a","zz","b",null],"l4_f64":[-3.0,null,null,2.0,null,-3.0,"NaN",11.817,null,-63.499,null,null,4.0,null,-4.0,76.822,3.0,-71.436,null,55.604,null,-1.0,"NaN",-5.044,null,4.0,50.228,null,null,-5.686,1.0,4.0,-8.518,"Infinity",93.661,-66.411,-1.0,null,57.63,null,-5.153,31.942,38.776,null,0.0,-3.0,-34.805,0.0,69.303,-91.914,-18.616,-15.251,null,-29.761,1.0,3.0,-4.0,-82.802,4.0,-1.0],"l5_bool":[true,true,true,false,false,true,null,false,false,true,null,true,true,true,null,null,true,null,true,true,null,false,true,true,false,true,null,null,false,true,false,true,false,null,false,null,true,false,null,true,true,null,false,true,null,null,false,true,true,null,true,true,true,true,true,true,true,false,false,false],"l6_f64":[null,8.691,null,null,3.0,24.87,-3.0,21.096,null,null,3.0,null,null,2.0,null,null,1.0,null,4.0,-9.151,21.632,null,-1.0,-77.064,94.815,null,-54.955,3.0,29.347,1.0,null,93.951,-1.0,1.0,94.065,4.0,27.558,null,null,-15.281,1.0,3.0,-16.829,-11.592,null,-17.289,null,-1.0,2.0,-4.0,null,-1.0,0.0,null,-74.999,-3.0,null,null,-1.0,4.0]}}},"ops":[{"op":"filter","predicate":{"fn":"lt","args":[{"fn":"when","args":[{"fn":"strStartsWith","args":[{"col":"l0_utf8"}],"prefix":"a"},{"col":"l1_i32"},{"col":"l6_f64"}]},{"fn":"floorDiv","args":[{"col":"l1_i32"},{"col":"l4_f64"}]}]}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_f64","type":"f64"},{"name":"l5_bool","type":"bool"},{"name":"l6_f64","type":"f64"}],"columns":{"l0_utf8":["😀","😀",null,null,null,"","B"],"l1_i32":[-3,2,-2147483648,3,5,-5,-1],"l2_utf8":["",null,null,null,"c","é","c"],"l3_utf8":["zz","a","a",null,"aa","ab","é"],"l4_f64":[55.604,50.228,-8.518,-3.0,0.0,-91.914,3.0],"l5_bool":[true,null,false,null,true,null,true],"l6_f64":[-9.151,-54.955,-1.0,-17.289,-1.0,-4.0,-3.0]}}},{"id":"random/0180","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_i64","type":"i64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_i32":[],"l3_i64":[]}}},"ops":[{"op":"head","n":4},{"op":"unique","columns":["l2_i32"]},{"op":"filter","predicate":{"fn":"and","args":[{"fn":"neq","args":[{"col":"l2_i32"},{"col":"l1_i32"}]},{"fn":"lte","args":[{"col":"l2_i32"},{"col":"l1_i32"}]}]}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_i64","type":"i64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_i32":[],"l3_i64":[]}}},{"id":"random/0181","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_i64","type":"i64"}],"columns":{"l0_utf8":[null],"l1_i32":[4],"l2_f64":[-2.0],"l3_i64":["-3"]}},"right":{"fields":[{"name":"l3_i64","type":"i64"},{"name":"r0_bool","type":"bool"}],"columns":{"r0_bool":[false,false,true,true,false,false,true,true,false,false,false,true,true,false,null,true,true,false,true,true,true,false,false,true,true,true,true,true,true,true],"l3_i64":["-3","-3","-3","-3","-3","-3","-3","-3","-3","-3","-3","-3","-3","-3","-3","-3","-3","-3","-3","-3","-3","-3","-3","-3","-3","-3","-3","-3","-3","-3"]}}},"ops":[{"op":"join","right":"right","on":["l3_i64"],"how":"full"},{"op":"sort","keys":[{"expr":{"col":"l1_i32"},"descending":false},{"expr":{"col":"l2_f64"},"descending":true}],"nullsLast":true},{"op":"unique","columns":["l2_f64"]},{"op":"head","n":4}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_i64","type":"i64"},{"name":"r0_bool","type":"bool"}],"columns":{"l0_utf8":[null],"l1_i32":[4],"l2_f64":[-2.0],"l3_i64":["-3"],"r0_bool":[false]}}},{"id":"random/0182","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_i32","type":"i32"},{"name":"l5_utf8","type":"utf8"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_utf8":[],"l3_utf8":[],"l4_i32":[],"l5_utf8":[]}}},"ops":[{"op":"unique"},{"op":"unique","columns":["l1_i32"]},{"op":"filter","predicate":{"fn":"isNull","args":[{"col":"l2_utf8"}]}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_i32","type":"i32"},{"name":"l5_utf8","type":"utf8"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_utf8":[],"l3_utf8":[],"l4_i32":[],"l5_utf8":[]}}},{"id":"random/0183","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_bool","type":"bool"},{"name":"l4_bool","type":"bool"},{"name":"l5_utf8","type":"utf8"},{"name":"l6_f64","type":"f64"}],"columns":{"l0_utf8":["ab","zz","ab",null,"aa","aa","zz",null,"😀",null,"","😀","aa","😀","zz","b","é","é","zz","😀"],"l1_i32":[-3,0,4,-2,1,-4,0,5,4,1,2,-5,-3,-3,-3,5,-1,null,null,4],"l2_i32":[4,3,1,-1,4,-2,-497,-3,-74,null,-5,-5,-4,-3,1,-2,2147483647,null,1,-2],"l3_bool":[true,false,false,false,false,null,false,true,false,true,false,true,false,false,false,false,false,true,null,true],"l4_bool":[true,false,true,true,true,true,true,false,false,true,false,null,true,false,false,false,true,false,true,false],"l5_utf8":["","aa","aa","B","ab","é",null,"zz","b","é","","zz","b",null,"😀","ab","a","","b","c"],"l6_f64":[-4.0,2.0,2.109,3.0,-1.0,96.202,1.0,null,-1.0,74.777,1.0,71.745,-4.0,"NaN",-50.369,16.428,-3.0,null,-2.0,-56.69]}},"right":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"r0_f64","type":"f64"}],"columns":{"r0_f64":[],"l0_utf8":[]}}},"ops":[{"op":"unique","columns":["l6_f64"]},{"op":"join","right":"right","on":["l0_utf8"],"how":"left"},{"op":"head","n":9}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_bool","type":"bool"},{"name":"l4_bool","type":"bool"},{"name":"l5_utf8","type":"utf8"},{"name":"l6_f64","type":"f64"},{"name":"r0_f64","type":"f64"}],"columns":{"l0_utf8":["ab","zz","ab",null,"aa","aa","zz",null,null],"l1_i32":[-3,0,4,-2,1,-4,0,5,1],"l2_i32":[4,3,1,-1,4,-2,-497,-3,null],"l3_bool":[true,false,false,false,false,null,false,true,true],"l4_bool":[true,false,true,true,true,true,true,false,true],"l5_utf8":["","aa","aa","B","ab","é",null,"zz","é"],"l6_f64":[-4.0,2.0,2.109,3.0,-1.0,96.202,1.0,null,74.777],"r0_f64":[null,null,null,null,null,null,null,null,null]}}},{"id":"random/0184","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_f64","type":"f64"},{"name":"l4_i32","type":"i32"},{"name":"l5_i64","type":"i64"}],"columns":{"l0_utf8":["a",null,"é",null,"😀",null,null,"é","a","é",null,"ab","B","B","é","c","é","😀","a","B","b",null,"c","B","ab",null,"aa","b","c","é","ab","ab","zz","😀","😀","","zz","a","","aa","B","😀","😀","B","aa","","b","😀","","zz","b","aa","B",null,"b","","aa","b","é",""],"l1_i32":[1,0,2,3,null,3,5,-4,-5,-3,-5,-3,0,-1,-1,-2147483648,-1,1,4,5,-2147483648,-1,0,-2147483648,-2,2,0,-3,0,2,0,1,-3,-1,2147483647,-4,3,null,-2,4,2,-4,-5,-1,-2,-1,-1,5,-3,-2,-2147483648,-1,-1,-2147483648,-3,4,-1,-3,-1,-5],"l2_f64":[4.0,-3.0,-65.19,-92.118,-15.573,-64.016,null,0.0,0.0,"NaN",4.0,87.605,60.195,"NaN",-3.0,80.614,null,6.365,4.0,"Infinity",-4.0,-32.301,20.736,-3.0,-0.294,null,2.0,0.0,-45.75,2.0,"NaN",-92.801,null,"NaN",-18.803,-9.658,69.782,56.547,-3.0,86.524,"NaN",1.0,-47.966,2.0,1.0,-1.0,94.149,-2.0,1.0,-62.636,3.0,-83.386,3.0,-64.836,-84.221,"NaN",-4.0,0.0,58.95,-84.119],"l3_f64":[3.0,59.876,1.0,21.773,-4.0,3.0,-23.168,-24.849,79.674,-15.735,-8.364,63.536,-77.007,-6.574,1.0,6.346,"Infinity",44.342,-49.961,3.0,-3.0,-65.053,-65.201,83.539,-4.0,-3.0,-21.815,-4.0,-2.0,-17.633,98.676,-23.7,null,31.561,-3.0,0.0,29.272,3.0,-2.0,"NaN",-3.0,null,null,1.0,42.086,1.0,-13.623,-35.905,"NaN",-65.353,-87.631,null,-49.222,-2.0,-6.288,3.0,-1.0,-87.624,-4.0,null],"l4_i32":[-1,0,5,4,1,-5,-2,-1,-1,3,-2,null,2147483647,69,1,0,-3,5,4,1,-2,-436,-3,-5,5,-4,-4,null,null,-1,872,4,-3,0,0,2,5,null,-1,-1,-2147483648,2,2,-3,-2,1,-1,3,null,-5,-4,-3,-1,5,-2,2147483647,-3,1,null,1],"l5_i64":["9007199254740993","-4611686018427387904","-4611686018427387904","0","9007199254740992","0",null,"9007199254740992","-3","9007199254740992","9007199254740992","4","-4611686018427387904","2","9007199254740992","-3","-4611686018427387904","4","-4","9007199254740992","-4611686018427387904","-4611686018427387904","-4611686018427387904","-4611686018427387904","4",null,"0","-4611686018427387904","-4611686018427387904","9007199254740994","0","9007199254740994","5","5",null,"0",null,"-4611686018427387904",null,null,"-4611686018427387904","-4611686018427387904","5","9007199254740992","9007199254740992","-4611686018427387904","9007199254740995","5","-3","-4611686018427387904","9007199254740993","3","-4611686018427387904","9007199254740995",null,"0",null,"-4611686018427387904","-4611686018427387904","-4611686018427387904"]}},"right":{"fields":[{"name":"l5_i64","type":"i64"},{"name":"r0_bool","type":"bool"}],"columns":{"r0_bool":[],"l5_i64":[]}}},"ops":[{"op":"join","right":"right","on":["l5_i64"],"how":"full"},{"op":"filter","predicate":{"fn":"isIn","args":[{"col":"l0_utf8"}],"values":["b","ab","a"]}},{"op":"withColumns","exprs":{"x2":{"fn":"eq","args":[{"fn":"mod","args":[{"col":"l2_f64"},{"lit":2}]},{"fn":"mod","args":[{"col":"l4_i32"},{"lit":2.5}]}]}}},{"op":"head","n":9}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_f64","type":"f64"},{"name":"l4_i32","type":"i32"},{"name":"l5_i64","type":"i64"},{"name":"r0_bool","type":"bool"},{"name":"x2","type":"bool"}],"columns":{"l0_utf8":["a","a","ab","a","b","ab","b","ab","ab"],"l1_i32":[1,-5,-3,4,-2147483648,-2,-3,0,1],"l2_f64":[4.0,0.0,87.605,4.0,-4.0,-0.294,0.0,"NaN",-92.801],"l3_f64":[3.0,79.674,63.536,-49.961,-3.0,-4.0,-4.0,98.676,-23.7],"l4_i32":[-1,-1,null,4,-2,5,null,872,4],"l5_i64":["9007199254740993","-3","4","-4","-4611686018427387904","4","-4611686018427387904","0","9007199254740994"],"r0_bool":[null,null,null,null,null,null,null,null,null],"x2":[false,false,null,false,false,false,null,false,false]}}},{"id":"random/0185","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_f64","type":"f64"},{"name":"l4_bool","type":"bool"},{"name":"l5_f64","type":"f64"},{"name":"l6_utf8","type":"utf8"}],"columns":{"l0_utf8":["B"],"l1_i32":[1],"l2_utf8":["ab"],"l3_f64":[3.0],"l4_bool":[false],"l5_f64":[15.563],"l6_utf8":["c"]}},"right":{"fields":[{"name":"l2_utf8","type":"utf8"},{"name":"r0_utf8","type":"utf8"}],"columns":{"r0_utf8":["zz","é","a","a",null,null,"c","é","",null],"l2_utf8":["ab","ab","ab","ab","ab","ab","ab","ab","ab","ab"]}}},"ops":[{"op":"join","right":"right","on":["l2_utf8"],"how":"inner"},{"op":"withColumns","exprs":{"x1":{"fn":"abs","args":[{"fn":"sub","args":[{"col":"l3_f64"},{"col":"l1_i32"}]}]}}},{"op":"filter","predicate":{"fn":"eq","args":[{"col":"l6_utf8"},{"lit":"aa"}]}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_f64","type":"f64"},{"name":"l4_bool","type":"bool"},{"name":"l5_f64","type":"f64"},{"name":"l6_utf8","type":"utf8"},{"name":"r0_utf8","type":"utf8"},{"name":"x1","type":"f64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_utf8":[],"l3_f64":[],"l4_bool":[],"l5_f64":[],"l6_utf8":[],"r0_utf8":[],"x1":[]}}},{"id":"random/0186","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_f64","type":"f64"}],"columns":{"l0_utf8":[null,null,null,"B",null,null,null,"ab","B",null,"zz","zz",null,"B",null,"a","😀","ab","😀","ab","é","aa","aa",null,null,null,"é","B","B","B",null,null,"c","é","zz","B","aa","a",null,"ab","c",null,"aa","ab",null,null,"zz",null,"zz","c",null,"zz",null,null,"B","",null,null,null,"ab"],"l1_i32":[5,-2,-4,null,-1,1,5,3,-5,4,2,1,-1,0,-1,-3,null,null,1,-1,2147483647,null,2,null,5,3,null,1,5,null,-1,2,2,0,0,0,-2,4,null,0,3,0,3,0,-4,-2,1,5,null,85,-4,0,null,0,-1,-3,null,3,5,2],"l2_utf8":[null,"a","é",null,"","b","😀","a","a","b",null,"c","b","b","a","zz",null,null,"aa","😀","é","é","c","c","B","aa","aa",null,"ab",null,"","b",null,"B","aa","B","c","B",null,null,"a",null,"é","c","c","😀",null,null,"B","aa",null,"😀","a",null,null,null,null,"zz","é","zz"],"l3_f64":[null,-80.618,46.704,-4.0,4.0,47.307,null,"-0",-96.618,null,null,-3.254,62.393,null,4.0,null,-4.0,91.465,null,-3.0,-87.054,null,-3.0,null,14.261,-1.0,53.966,3.0,null,null,-4.0,-3.0,null,null,-48.155,-20.805,null,null,1.056,null,95.428,14.488,null,null,null,-4.0,92.499,2.0,-1.621,null,4.0,null,-16.197,null,40.93,-82.865,77.192,4.0,-95.184,-1.0]}}},"ops":[{"op":"withColumns","exprs":{"x0":{"fn":"gt","args":[{"fn":"floorDiv","args":[{"col":"l1_i32"},{"lit":2}]},{"col":"l3_f64"}]}}},{"op":"filter","predicate":{"fn":"eq","args":[{"fn":"div","args":[{"col":"l3_f64"},{"col":"l1_i32"}]},{"fn":"div","args":[{"lit":0.5},{"lit":2.5}]}]}},{"op":"filter","predicate":{"fn":"gte","args":[{"fn":"div","args":[{"col":"l3_f64"},{"col":"l1_i32"}]},{"col":"l1_i32"}]}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_f64","type":"f64"},{"name":"x0","type":"bool"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_utf8":[],"l3_f64":[],"x0":[]}}},{"id":"random/0187","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_utf8","type":"utf8"}],"columns":{"l0_utf8":["B","😀","B","","B","😀",null,"é",null,"","aa","é",null,"zz","ab",null,"c","ab","B","é"],"l1_i32":[0,4,null,0,5,-2,-5,-4,5,0,-2,1,2,null,-3,-4,-3,3,-5,1],"l2_f64":[2.0,-26.144,-61.449,2.0,1.0,-3.0,-80.039,-92.818,-54.194,-1.0,4.0,-1.0,3.0,2.0,2.0,38.867,-3.0,null,4.0,-59.41],"l3_utf8":["c","😀","aa",null,"B","aa","","","zz",null,"zz","ab","","zz","","aa","ab","b","a","B"],"l4_utf8":["ab","zz","b","b","a","c","aa","a","😀",null,"ab","zz",null,null,"b","a","c","B","B","aa"]}}},"ops":[{"op":"filter","predicate":{"fn":"lt","args":[{"fn":"fillNull","args":[{"col":"l2_f64"},{"lit":1.5}]},{"col":"l2_f64"}]}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_utf8","type":"utf8"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_f64":[],"l3_utf8":[],"l4_utf8":[]}}},{"id":"random/0188","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_bool","type":"bool"},{"name":"l4_i32","type":"i32"}],"columns":{"l0_utf8":["c"],"l1_i32":[null],"l2_f64":["NaN"],"l3_bool":[null],"l4_i32":[-5]}},"right":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"r0_utf8","type":"utf8"}],"columns":{"r0_utf8":["aa","ab","é"],"l0_utf8":["c","c","c"]}}},"ops":[{"op":"join","right":"right","on":["l0_utf8"],"how":"inner"},{"op":"groupBy","keys":["l0_utf8"],"aggs":{"n1":{"agg":"len"},"a1_0":{"agg":"mean","arg":{"col":"l4_i32"}}}},{"op":"filter","predicate":{"fn":"lt","args":[{"fn":"fillNull","args":[{"col":"a1_0"},{"lit":0}]},{"fn":"mod","args":[{"col":"a1_0"},{"col":"a1_0"}]}]}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"n1","type":"u32"},{"name":"a1_0","type":"f64"}],"columns":{"l0_utf8":["c"],"n1":[3],"a1_0":[-5.0]}}},{"id":"random/0189","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_bool","type":"bool"},{"name":"l4_f64","type":"f64"},{"name":"l5_utf8","type":"utf8"},{"name":"l6_f64","type":"f64"}],"columns":{"l0_utf8":["😀"],"l1_i32":[2],"l2_i32":[-2],"l3_bool":[false],"l4_f64":["-0"],"l5_utf8":["é"],"l6_f64":["NaN"]}}},"ops":[{"op":"head","n":9},{"op":"filter","predicate":{"fn":"and","args":[{"col":"l3_bool"},{"fn":"gt","args":[{"col":"l4_f64"},{"col":"l4_f64"}]}]}},{"op":"sort","keys":[{"expr":{"col":"l0_utf8"},"descending":true},{"expr":{"col":"l6_f64"},"descending":true}],"nullsLast":true},{"op":"withColumns","exprs":{"x3":{"fn":"sub","args":[{"fn":"fillNull","args":[{"col":"l2_i32"},{"lit":1.5}]},{"fn":"neg","args":[{"col":"l1_i32"}]}]}}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_bool","type":"bool"},{"name":"l4_f64","type":"f64"},{"name":"l5_utf8","type":"utf8"},{"name":"l6_f64","type":"f64"},{"name":"x3","type":"f64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_i32":[],"l3_bool":[],"l4_f64":[],"l5_utf8":[],"l6_f64":[],"x3":[]}}},{"id":"random/0190","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_f64","type":"f64"}],"columns":{"l0_utf8":["ab"],"l1_i32":[2],"l2_utf8":["zz"],"l3_f64":[0.0]}}},"ops":[{"op":"unique"}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_f64","type":"f64"}],"columns":{"l0_utf8":["ab"],"l1_i32":[2],"l2_utf8":["zz"],"l3_f64":[0.0]}}},{"id":"random/0191","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_i64","type":"i64"},{"name":"l4_bool","type":"bool"},{"name":"l5_f64","type":"f64"},{"name":"l6_f64","type":"f64"}],"columns":{"l0_utf8":[null,null,"é",null,"b"],"l1_i32":[null,-2,-1,-3,564],"l2_f64":[null,4.0,94.073,null,51.67],"l3_i64":["-4611686018427387904","-4611686018427387904","-3","-4611686018427387904","-4"],"l4_bool":[false,null,false,false,true],"l5_f64":[3.0,31.502,4.0,3.0,null],"l6_f64":[null,null,3.0,1.0,-1.0]}}},"ops":[{"op":"sort","keys":[{"expr":{"col":"l6_f64"},"descending":true}],"nullsLast":false}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_i64","type":"i64"},{"name":"l4_bool","type":"bool"},{"name":"l5_f64","type":"f64"},{"name":"l6_f64","type":"f64"}],"columns":{"l0_utf8":[null,null,"é",null,"b"],"l1_i32":[null,-2,-1,-3,564],"l2_f64":[null,4.0,94.073,null,51.67],"l3_i64":["-4611686018427387904","-4611686018427387904","-3","-4611686018427387904","-4"],"l4_bool":[false,null,false,false,true],"l5_f64":[3.0,31.502,4.0,3.0,null],"l6_f64":[null,null,3.0,1.0,-1.0]}}},{"id":"random/0192","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_f64","type":"f64"},{"name":"l4_bool","type":"bool"},{"name":"l5_bool","type":"bool"},{"name":"l6_i32","type":"i32"}],"columns":{"l0_utf8":["😀","a","ab","aa","😀","","","aa","B","zz","😀","é","ab","é","","a","😀","aa","B","a"],"l1_i32":[5,-4,-4,-2,0,-5,5,2147483647,2,-3,-2,3,3,-2,-4,-3,2,-804,-3,-3],"l2_utf8":["ab","c","b","B","zz","a","B","b","zz","B","😀","B","aa","a","B","é","c","","zz","ab"],"l3_f64":[2.0,"NaN",2.0,4.0,-79.802,-42.574,-16.129,-2.0,2.0,20.741,-1.771,-1.0,-65.11,1.0,-1.0,-22.167,79.233,-2.0,4.0,76.8],"l4_bool":[false,false,true,true,false,true,false,false,true,false,true,false,false,true,false,false,true,false,false,false],"l5_bool":[true,false,true,false,true,false,true,false,false,false,false,false,false,false,true,true,false,false,false,true],"l6_i32":[0,0,5,0,2,1,-1,3,3,1,0,-3,-5,-4,1,-2147483648,-1,0,3,-2147483648]}}},"ops":[{"op":"sort","keys":[{"expr":{"col":"l4_bool"},"descending":true}],"nullsLast":false},{"op":"sort","keys":[{"expr":{"col":"l5_bool"},"descending":false}],"nullsLast":true}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_f64","type":"f64"},{"name":"l4_bool","type":"bool"},{"name":"l5_bool","type":"bool"},{"name":"l6_i32","type":"i32"}],"columns":{"l0_utf8":["aa","","B","😀","é","😀","a","aa","zz","é","ab","aa","B","ab","😀","😀","","","a","a"],"l1_i32":[-2,-5,2,-2,-2,2,-4,2147483647,-3,3,3,-804,-3,-4,5,0,5,-4,-3,-3],"l2_utf8":["B","a","zz","😀","a","c","c","b","B","B","aa","","zz","b","ab","zz","B","B","é","ab"],"l3_f64":[4.0,-42.574,2.0,-1.771,1.0,79.233,"NaN",-2.0,20.741,-1.0,-65.11,-2.0,4.0,2.0,2.0,-79.802,-16.129,-1.0,-22.167,76.8],"l4_bool":[true,true,true,true,true,true,false,false,false,false,false,false,false,true,false,false,false,false,false,false],"l5_bool":[false,false,false,false,false,false,false,false,false,false,false,false,false,true,true,true,true,true,true,true],"l6_i32":[0,1,3,0,-4,-1,0,3,1,-3,-5,0,3,5,0,2,-1,1,-2147483648,-2147483648]}}},{"id":"random/0193","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_i64","type":"i64"},{"name":"l4_bool","type":"bool"}],"columns":{"l0_utf8":["😀","😀","a","c","c"],"l1_i32":[-4,-3,-4,-2,2147483647],"l2_bool":[true,false,false,false,false],"l3_i64":["0","9007199254740995","-4611686018427387904",null,"9007199254740995"],"l4_bool":[false,true,false,false,false]}}},"ops":[{"op":"unique","columns":["l3_i64"]}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_i64","type":"i64"},{"name":"l4_bool","type":"bool"}],"columns":{"l0_utf8":["😀","😀","a","c"],"l1_i32":[-4,-3,-4,-2],"l2_bool":[true,false,false,false],"l3_i64":["0","9007199254740995","-4611686018427387904",null],"l4_bool":[false,true,false,false]}}},{"id":"random/0194","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i64","type":"i64"},{"name":"l3_f64","type":"f64"},{"name":"l4_f64","type":"f64"},{"name":"l5_f64","type":"f64"},{"name":"l6_f64","type":"f64"}],"columns":{"l0_utf8":["é"],"l1_i32":[1],"l2_i64":["-4611686018427387904"],"l3_f64":[-4.0],"l4_f64":[-67.348],"l5_f64":[3.0],"l6_f64":[2.0]}},"right":{"fields":[{"name":"l1_i32","type":"i32"},{"name":"r0_bool","type":"bool"}],"columns":{"r0_bool":[],"l1_i32":[]}}},"ops":[{"op":"withColumns","exprs":{"x0":{"fn":"fillNull","args":[{"fn":"neg","args":[{"col":"l5_f64"}]},{"lit":0}]}}},{"op":"join","right":"right","on":["l1_i32"],"how":"left"}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i64","type":"i64"},{"name":"l3_f64","type":"f64"},{"name":"l4_f64","type":"f64"},{"name":"l5_f64","type":"f64"},{"name":"l6_f64","type":"f64"},{"name":"x0","type":"f64"},{"name":"r0_bool","type":"bool"}],"columns":{"l0_utf8":["é"],"l1_i32":[1],"l2_i64":["-4611686018427387904"],"l3_f64":[-4.0],"l4_f64":[-67.348],"l5_f64":[3.0],"l6_f64":[2.0],"x0":[-3.0],"r0_bool":[null]}}},{"id":"random/0195","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i64","type":"i64"},{"name":"l3_f64","type":"f64"},{"name":"l4_utf8","type":"utf8"}],"columns":{"l0_utf8":["é",null,"aa","c",null,null,"ab","aa",null,null,"","c","aa","c","",null,null,"b",null,"B","😀","é","b",null,"é","zz","😀","ab","zz","zz","","a","c","","c","B","ab","b",null,"c","a","😀",null,"😀","zz","zz",null,null,"aa",null,"ab","é",null,null,"b","",null,"","b","😀"],"l1_i32":[1,-4,null,null,-2,null,420,4,-1,null,0,null,-3,5,-3,-740,null,-4,-4,2,3,null,0,-2,-2,null,-2147483648,null,-1,null,null,-5,1,-3,2147483647,3,-1,-3,0,null,3,1,-1,null,1,-1,-2,-5,-5,-2,5,-1,-2,4,null,null,4,-3,-2147483648,-2],"l2_i64":["-1",null,"-4611686018427387904","-4","-4611686018427387904",null,"-4611686018427387904",null,null,null,"-3","9007199254740994",null,"-4","9007199254740992","9007199254740993","9007199254740994","-4611686018427387904","-4611686018427387904","-2","9007199254740992","9007199254740993",null,"-4611686018427387904","1","9007199254740995",null,null,"9007199254740995",null,"-4611686018427387904","9007199254740995","-4611686018427387904","-1",null,null,null,"9007199254740992","9007199254740993","4",null,"9007199254740994","9007199254740994","9007199254740995","5","-4611686018427387904",null,null,"9007199254740994","1","-4611686018427387904",null,null,"9007199254740994","9007199254740992","9007199254740995","-4611686018427387904","-4611686018427387904","-4611686018427387904",null],"l3_f64":[null,1.0,-1.0,2.0,23.609,-14.493,4.0,"-0","NaN",-77.55,4.0,-1.0,0.684,null,null,null,null,11.974,4.0,null,null,1.0,86.202,-32.586,null,null,0.0,"-0",null,null,null,3.0,0.0,null,null,null,-3.0,2.0,-43.209,null,null,null,null,-3.0,null,-18.291,"NaN",-54.214,70.865,-1.0,null,null,4.0,2.0,4.0,-2.0,1.0,-27.053,2.0,null],"l4_utf8":["b","aa","B","a","b",null,null,null,"b","B",null,"a",null,null,"aa","zz",null,"é","aa","a","😀",null,"aa","aa",null,null,"😀","a","ab","c","é","😀","c",null,null,"aa","a",null,null,null,null,"😀","aa","b","aa","aa","zz",null,"B","","b","B","é","ab","ab",null,null,"é",null,null]}}},"ops":[{"op":"filter","predicate":{"fn":"and","args":[{"fn":"lte","args":[{"lit":0.5},{"col":"l1_i32"}]},{"fn":"isIn","args":[{"col":"l0_utf8"}],"values":["a","aa","c"]}]}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i64","type":"i64"},{"name":"l3_f64","type":"f64"},{"name":"l4_utf8","type":"utf8"}],"columns":{"l0_utf8":["aa","c","c","c","a"],"l1_i32":[4,5,1,2147483647,3],"l2_i64":[null,"-4","-4611686018427387904",null,null],"l3_f64":["-0",null,0.0,null,null],"l4_utf8":[null,null,"c",null,null]}}},{"id":"random/0196","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_f64","type":"f64"},{"name":"l5_f64","type":"f64"},{"name":"l6_f64","type":"f64"}],"columns":{"l0_utf8":["a","é","😀","é","aa"],"l1_i32":[null,null,-1,-3,2],"l2_bool":[true,false,false,false,true],"l3_utf8":["é","b","ab","b","ab"],"l4_f64":[null,null,null,null,0.0],"l5_f64":[53.478,null,-25.778,1.0,65.7],"l6_f64":[2.0,1.0,-43.169,"-0","-0"]}}},"ops":[{"op":"sort","keys":[{"expr":{"col":"l6_f64"},"descending":true},{"expr":{"col":"l2_bool"},"descending":true}],"nullsLast":false},{"op":"withColumns","exprs":{"x1":{"lit":-1}}},{"op":"unique","columns":["l6_f64"]},{"op":"head","n":7}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_f64","type":"f64"},{"name":"l5_f64","type":"f64"},{"name":"l6_f64","type":"f64"},{"name":"x1","type":"i32"}],"columns":{"l0_utf8":["a","é","aa","😀"],"l1_i32":[null,null,2,-1],"l2_bool":[true,false,true,false],"l3_utf8":["é","b","ab","ab"],"l4_f64":[null,null,0.0,null],"l5_f64":[53.478,null,65.7,-25.778],"l6_f64":[2.0,1.0,"-0",-43.169],"x1":[-1,-1,-1,-1]}}},{"id":"random/0197","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_i32","type":"i32"}],"columns":{"l0_utf8":[null,"aa","ab","b","","","😀","c","aa","aa","ab","a","","zz","B","é","é","a","a","😀","","😀","é","é","ab","é","","ab","é","zz","zz","aa","","",null,"b","a","c","aa","ab","é","b","ab","B","",null,"c","b","aa","zz","","c","c","zz","ab","aa","c","zz","a","zz"],"l1_i32":[2,-2,5,null,-2147483648,2,null,-5,null,3,-1,5,3,4,-4,-4,-5,2,-4,2147483647,1,1,-3,4,null,4,null,-5,0,-5,-1,-4,2147483647,-2,null,2,5,2,-671,null,2,null,-2,1,4,-2147483648,0,4,-1,5,3,-2,-5,1,1,4,0,-2147483648,-2,-4],"l2_f64":[1.0,2.0,-85.911,1.0,47.608,21.428,null,null,-27.594,40.902,1.0,-95.911,-95.194,88.774,-4.0,32.199,1.0,-30.514,-15.871,10.805,42.693,null,43.274,4.0,-96.566,94.018,3.0,4.0,-3.0,-4.0,"NaN",-2.0,-61.991,4.0,2.0,-3.0,71.736,-1.0,null,2.0,-1.0,-27.609,3.0,-1.0,-53.676,-91.928,2.0,38.243,2.0,4.0,"-Infinity",82.87,null,0.0,-87.721,96.855,93.878,-3.0,29.723,null],"l3_utf8":["a","b","b","c","b","c","é","😀","c","B","B","",null,"B",null,"aa","B","ab",null,"aa","b","ab","","aa",null,"B","c","a","b","a","zz","B","zz","😀","😀","aa","😀","aa","a","é","B","a","zz","é","a","zz","b","","ab",null,"aa","B","B","c","","c","é","","😀","c"],"l4_i32":[2147483647,4,-3,2147483647,5,5,-1,1,-3,3,null,-2,3,-1,5,-2,-1,-1,null,-1,3,0,3,-4,-4,-2,2,-2,-3,-1,-2,-8,null,280,-5,1,0,-4,0,1,-1,null,2147483647,4,5,-5,5,2147483647,2,5,3,4,-5,-5,-3,3,-2,5,-2,5]}}},"ops":[{"op":"sort","keys":[{"expr":{"col":"l4_i32"},"descending":false},{"expr":{"col":"l2_f64"},"descending":true}],"nullsLast":true},{"op":"sort","keys":[{"expr":{"col":"l0_utf8"},"descending":true},{"expr":{"col":"l2_f64"},"descending":false}],"nullsLast":false},{"op":"filter","predicate":{"fn":"and","args":[{"fn":"isNotNull","args":[{"col":"l0_utf8"}]},{"fn":"eq","args":[{"col":"l1_i32"},{"col":"l1_i32"}]}]}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_i32","type":"i32"}],"columns":{"l0_utf8":["😀","😀","é","é","é","é","é","é","é","zz","zz","zz","zz","zz","zz","zz","c","c","c","c","c","c","b","b","ab","ab","ab","ab","ab","aa","aa","aa","aa","aa","aa","a","a","a","a","a","B","B","","","","","","","",""],"l1_i32":[1,2147483647,0,2,-5,4,-4,-3,4,-4,-5,-2147483648,1,5,4,-1,-5,-5,2,0,-2,0,2,4,1,5,-1,-2,-5,-671,-4,-1,-2,3,4,5,2,-4,-2,5,-4,1,3,3,2147483647,4,-2,2,1,-2147483648],"l2_f64":[null,10.805,-3.0,-1.0,1.0,4.0,32.199,43.274,94.018,null,-4.0,-3.0,0.0,4.0,88.774,"NaN",null,null,-1.0,2.0,82.87,93.878,-3.0,38.243,-87.721,-85.911,1.0,3.0,4.0,null,-2.0,2.0,2.0,40.902,96.855,-95.911,-30.514,-15.871,29.723,71.736,-4.0,-1.0,"-Infinity",-95.194,-61.991,-53.676,4.0,21.428,42.693,47.608],"l3_utf8":["ab","aa","b","B","B","aa","aa","","B","c","a","","c",null,"B","zz","B","😀","aa","b","B","é","aa","","","b","B","zz","a","a","B","ab","b","B","c","","ab",null,"😀","😀",null,"é","aa",null,"zz","a","😀","c","b","b"],"l4_i32":[0,-1,-3,-1,-1,-4,-2,3,-2,5,-1,5,-5,5,-1,-2,-5,1,-4,5,4,-2,1,2147483647,-3,-3,null,2147483647,-2,0,-8,2,4,3,3,-2,-1,null,-2,0,5,4,3,3,null,5,280,5,3,5]}}},{"id":"random/0198","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_utf8","type":"utf8"},{"name":"l6_utf8","type":"utf8"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_bool":[],"l3_utf8":[],"l4_utf8":[],"l5_utf8":[],"l6_utf8":[]}}},"ops":[{"op":"filter","predicate":{"col":"l2_bool"}},{"op":"groupBy","keys":["l5_utf8"],"aggs":{"n1":{"agg":"len"},"a1_0":{"agg":"std","arg":{"col":"l1_i32"}},"a1_1":{"agg":"first","arg":{"col":"l4_utf8"}}}},{"op":"unique"},{"op":"withColumns","exprs":{"x3":{"fn":"mul","args":[{"fn":"add","args":[{"col":"a1_0"},{"col":"a1_0"}]},{"fn":"sub","args":[{"col":"a1_0"},{"lit":2}]}]}}}],"expected":{"fields":[{"name":"l5_utf8","type":"utf8"},{"name":"n1","type":"u32"},{"name":"a1_0","type":"f64"},{"name":"a1_1","type":"utf8"},{"name":"x3","type":"f64"}],"columns":{"l5_utf8":[],"n1":[],"a1_0":[],"a1_1":[],"x3":[]}}},{"id":"random/0199","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_utf8","type":"utf8"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_i32":[],"l3_utf8":[]}},"right":{"fields":[{"name":"l2_i32","type":"i32"},{"name":"r0_utf8","type":"utf8"}],"columns":{"r0_utf8":["é","😀","c","B","b",null,"b","a","aa","😀","zz","ab","a",null,"B","é","é","ab","","é","a",null,"a","b","b","é","é","b","","a"],"l2_i32":[-4,1,null,null,null,-4,-4,-4,null,4,-4,-4,-4,null,null,-4,null,null,null,4,4,1,-4,4,-4,null,-4,-3,1,null]}}},"ops":[{"op":"head","n":10},{"op":"join","right":"right","on":["l2_i32"],"how":"left"},{"op":"filter","predicate":{"fn":"isNull","args":[{"col":"l2_i32"}]}},{"op":"filter","predicate":{"fn":"eq","args":[{"fn":"mul","args":[{"col":"l2_i32"},{"col":"l2_i32"}]},{"fn":"neg","args":[{"col":"l2_i32"}]}]}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_utf8","type":"utf8"},{"name":"r0_utf8","type":"utf8"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_i32":[],"l3_utf8":[],"r0_utf8":[]}}},{"id":"random/0200","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_bool","type":"bool"},{"name":"l4_bool","type":"bool"},{"name":"l5_utf8","type":"utf8"},{"name":"l6_bool","type":"bool"}],"columns":{"l0_utf8":["😀"],"l1_i32":[1],"l2_f64":[-2.0],"l3_bool":[true],"l4_bool":[false],"l5_utf8":["a"],"l6_bool":[null]}}},"ops":[{"op":"unique","columns":["l0_utf8"]},{"op":"withColumns","exprs":{"x1":{"lit":2}}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_bool","type":"bool"},{"name":"l4_bool","type":"bool"},{"name":"l5_utf8","type":"utf8"},{"name":"l6_bool","type":"bool"},{"name":"x1","type":"i32"}],"columns":{"l0_utf8":["😀"],"l1_i32":[1],"l2_f64":[-2.0],"l3_bool":[true],"l4_bool":[false],"l5_utf8":["a"],"l6_bool":[null],"x1":[2]}}},{"id":"random/0201","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i64","type":"i64"},{"name":"l3_f64","type":"f64"},{"name":"l4_f64","type":"f64"}],"columns":{"l0_utf8":["é","zz","é","b","aa",null,"ab","😀","c",null,"😀","ab","a","a",null,"b",null,"aa","a","😀",null,"zz",null,"😀","","😀","ab","B","é","a","ab","b","aa",null,"é",null,"",null,"ab","😀",null,null,"zz",null,"ab",null,"B","B","","c",null,null,null,null,null,"ab",null,"","ab",""],"l1_i32":[null,-5,null,-5,null,-1,2,null,null,null,null,null,null,-5,-1,-4,-4,3,-2,-2147483648,-3,null,null,0,null,1,null,null,null,1,-5,0,-4,-3,-4,null,0,null,0,null,1,null,-3,-1,3,-2,null,-3,-1,null,0,623,5,2147483647,null,0,null,2147483647,-5,-1],"l2_i64":["-4611686018427387904","-4611686018427387904","-1","-4611686018427387904","-4611686018427387904","9007199254740995","9007199254740993",null,"9007199254740995","-4611686018427387904","4","9007199254740995","-4611686018427387904","9007199254740995",null,"9007199254740995","9007199254740995",null,null,"9007199254740992","-4611686018427387904","9007199254740994","-4611686018427387904","9007199254740992","-4611686018427387904","-4611686018427387904",null,"-5","-4611686018427387904",null,"9007199254740992",null,null,"-4611686018427387904",null,"-2","-4611686018427387904","0","-4","9007199254740992",null,"9007199254740994","-4611686018427387904",null,null,"9007199254740994","-2","-4611686018427387904","-5","1",null,"-4611686018427387904",null,"1","9007199254740994",null,"-2",null,null,"-4611686018427387904"],"l3_f64":[97.832,-33.702,61.607,-4.0,-0.679,-90.259,-3.0,47.6,53.769,null,4.0,26.312,3.0,4.0,24.95,0.0,null,null,-2.0,-2.0,null,null,-27.435,null,3.0,null,null,-5.829,null,-89.209,null,"Infinity",null,1.0,null,-84.841,51.677,-2.0,null,-48.557,-79.214,3.0,93.112,null,-64.79,null,"NaN",null,null,-42.789,93.731,33.596,null,-3.0,2.0,-3.0,-52.018,4.0,33.687,null],"l4_f64":[-43.244,-1.0,null,4.0,null,null,-3.376,74.328,0.0,null,null,-33.249,2.0,null,null,null,null,"NaN",4.0,53.329,80.045,-4.0,4.0,92.42,-4.0,24.234,14.611,-3.0,-3.0,null,-4.0,null,7.815,33.421,null,0.0,null,-16.38,-3.0,62.388,-1.0,0.0,null,null,65.676,null,null,-2.0,25.51,null,-52.689,43.536,null,69.697,null,null,86.29,-23.279,null,1.0]}}},"ops":[{"op":"filter","predicate":{"fn":"lt","args":[{"fn":"floorDiv","args":[{"col":"l3_f64"},{"col":"l3_f64"}]},{"fn":"abs","args":[{"col":"l1_i32"}]}]}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i64","type":"i64"},{"name":"l3_f64","type":"f64"},{"name":"l4_f64","type":"f64"}],"columns":{"l0_utf8":["zz","b","ab","a","a",null,"zz","ab",null,null,"","ab"],"l1_i32":[-5,-5,2,-5,-2,-3,-3,3,623,2147483647,2147483647,-5],"l2_i64":["-4611686018427387904","-4611686018427387904","9007199254740993","9007199254740995",null,"-4611686018427387904","-4611686018427387904",null,"-4611686018427387904","1",null,null],"l3_f64":[-33.702,-4.0,-3.0,4.0,-2.0,1.0,93.112,-64.79,33.596,-3.0,4.0,33.687],"l4_f64":[-1.0,4.0,-3.376,null,4.0,33.421,null,65.676,43.536,69.697,-23.279,null]}}},{"id":"random/0202","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_i64","type":"i64"},{"name":"l4_f64","type":"f64"},{"name":"l5_i64","type":"i64"},{"name":"l6_utf8","type":"utf8"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_bool":[],"l3_i64":[],"l4_f64":[],"l5_i64":[],"l6_utf8":[]}}},"ops":[{"op":"groupBy","keys":["l2_bool"],"aggs":{"n0":{"agg":"len"},"a0_0":{"agg":"mean","arg":{"col":"l1_i32"}}}}],"expected":{"fields":[{"name":"l2_bool","type":"bool"},{"name":"n0","type":"u32"},{"name":"a0_0","type":"f64"}],"columns":{"l2_bool":[],"n0":[],"a0_0":[]}}},{"id":"random/0203","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_i32","type":"i32"},{"name":"l4_i64","type":"i64"},{"name":"l5_i64","type":"i64"},{"name":"l6_bool","type":"bool"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_f64":[],"l3_i32":[],"l4_i64":[],"l5_i64":[],"l6_bool":[]}},"right":{"fields":[{"name":"l4_i64","type":"i64"},{"name":"r0_utf8","type":"utf8"}],"columns":{"r0_utf8":["é","B","ab","ab","a","aa","zz","é","a","a"],"l4_i64":["4","9007199254740995","9007199254740995","9007199254740995","9007199254740995","9007199254740995","9007199254740992","9007199254740995","1","1"]}}},"ops":[{"op":"filter","predicate":{"fn":"lt","args":[{"fn":"div","args":[{"col":"l3_i32"},{"lit":0.5}]},{"lit":2.5}]}},{"op":"unique"},{"op":"join","right":"right","on":["l4_i64"],"how":"semi"}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_i32","type":"i32"},{"name":"l4_i64","type":"i64"},{"name":"l5_i64","type":"i64"},{"name":"l6_bool","type":"bool"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_f64":[],"l3_i32":[],"l4_i64":[],"l5_i64":[],"l6_bool":[]}}},{"id":"random/0204","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_bool","type":"bool"},{"name":"l5_utf8","type":"utf8"}],"columns":{"l0_utf8":["aa","c","b",null,"c","😀","é",null,null,"b",null,null,"😀","aa",null,"B","zz",null,"aa",null,"B","zz","😀","aa","é","b","","","😀","ab",null,"b","b","ab",null,"é",null,"😀",null,null,"c","zz",null,"b",null,null,null,"😀","é","aa","😀","b","","ab","a","",null,"😀",null,"c"],"l1_i32":[null,-3,null,-4,1,-5,1,-5,2,4,null,5,-3,4,-3,-4,0,null,4,-5,-3,1,4,-2,0,-5,4,3,5,-4,null,null,1,0,-4,null,1,0,-1,-5,-4,null,-5,-4,-2,2,5,-3,-1,1,null,2147483647,null,null,4,null,-3,3,null,null],"l2_i32":[5,-5,-4,3,null,-3,1,0,0,4,null,-4,3,-3,-2,4,4,459,null,4,null,4,5,-5,-4,-2,2,-710,null,null,5,null,-2,2,1,2,null,null,2,-2147483648,-4,2,-4,2,-2,5,2,3,5,2147483647,-4,0,null,1,4,-4,0,-1,3,-2],"l3_utf8":[null,"é","😀","b",null,"",null,"c","é","",null,"aa",null,"b","B","b","é","B","","ab",null,"","B","","a","B","aa","ab","😀","c","b",null,"","","é","","ab",null,"😀","aa",null,"B","zz","c","ab","c","","aa",null,"",null,null,null,"😀","aa","b","ab",null,"b",null],"l4_bool":[null,true,null,false,null,true,null,true,null,false,null,true,true,false,true,true,null,true,false,true,false,true,false,true,false,false,true,true,null,null,false,false,true,true,null,true,null,false,null,null,true,null,true,false,null,false,false,null,true,false,true,null,true,false,true,null,true,true,true,true],"l5_utf8":["a","ab","c",null,"b",null,"","zz","é","","é","ab","",null,null,"é","a","","😀","aa","B","b","","","B","😀",null,"ab","b","ab","b",null,"aa","","aa",null,"é","😀","c","B",null,"B",null,"😀",null,"ab","b","😀",null,null,"a","ab",null,null,"c","b",null,"a","B","b"]}}},"ops":[{"op":"head","n":0}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_bool","type":"bool"},{"name":"l5_utf8","type":"utf8"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_i32":[],"l3_utf8":[],"l4_bool":[],"l5_utf8":[]}}},{"id":"random/0205","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i64","type":"i64"},{"name":"l3_f64","type":"f64"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_i64","type":"i64"},{"name":"l6_utf8","type":"utf8"}],"columns":{"l0_utf8":["c","ab","c","zz","zz"],"l1_i32":[0,1,-4,0,-2147483648],"l2_i64":["9007199254740992","-4611686018427387904","-1",null,"-4611686018427387904"],"l3_f64":[0.558,-43.336,-57.192,null,-3.0],"l4_utf8":["a","B","ab","é","c"],"l5_i64":["0","9007199254740994","2","-1","-4611686018427387904"],"l6_utf8":["zz","a","zz",null,null]}}},"ops":[{"op":"withColumns","exprs":{"x0":{"fn":"lte","args":[{"fn":"div","args":[{"col":"l1_i32"},{"col":"l3_f64"}]},{"fn":"mod","args":[{"col":"l1_i32"},{"col":"l3_f64"}]}]}}},{"op":"unique"}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i64","type":"i64"},{"name":"l3_f64","type":"f64"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_i64","type":"i64"},{"name":"l6_utf8","type":"utf8"},{"name":"x0","type":"bool"}],"columns":{"l0_utf8":["c","ab","c","zz","zz"],"l1_i32":[0,1,-4,0,-2147483648],"l2_i64":["9007199254740992","-4611686018427387904","-1",null,"-4611686018427387904"],"l3_f64":[0.558,-43.336,-57.192,null,-3.0],"l4_utf8":["a","B","ab","é","c"],"l5_i64":["0","9007199254740994","2","-1","-4611686018427387904"],"l6_utf8":["zz","a","zz",null,null],"x0":[true,false,false,null,false]}}},{"id":"random/0206","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_f64","type":"f64"},{"name":"l4_bool","type":"bool"},{"name":"l5_f64","type":"f64"},{"name":"l6_bool","type":"bool"}],"columns":{"l0_utf8":["zz","b","a","zz","zz","B","","aa","a","B","😀","a","zz","B","B","","ab","b","ab","","é","a","c","ab","b","ab","a","a","ab","é","c","aa","😀","é","c","","aa","ab","é","aa","c","B","","zz","aa","c","a","a","é","c","é","aa","B","ab","","b","a","é","ab","zz"],"l1_i32":[-3,-1,-1,1,-1,2,-5,25,3,4,-5,0,3,0,1,-2147483648,-4,-1,1,5,4,3,-1,0,-5,-5,1,1,-5,-5,5,-1,-5,-3,-1,4,2,-2,3,4,12,5,3,2,-2,4,0,3,2,2,-4,2,-5,-4,1,-2,0,1,5,-3],"l2_f64":[1.0,38.593,-63.191,-99.704,0.0,-78.5,3.0,-2.0,-2.0,-4.0,-96.736,58.778,37.631,1.0,-93.254,1.0,-0.868,-0.826,0.0,1.0,-3.0,21.399,"NaN",2.0,-1.0,48.38,-70.187,0.0,85.86,"-Infinity",67.757,-7.196,13.937,"NaN",50.461,73.241,-81.186,-3.0,-4.0,-1.867,-67.255,71.539,-79.551,45.863,15.527,0.0,78.287,"-Infinity","NaN",-2.0,-16.858,-23.292,11.734,-21.906,2.0,4.0,-36.747,4.0,-1.0,0.0],"l3_f64":[68.252,1.0,-4.0,-3.0,39.731,-8.76,17.226,-2.0,-82.681,-1.0,5.043,-6.784,68.285,49.183,-85.1,-39.214,2.0,37.264,-37.637,0.0,0.0,52.363,-18.341,-1.0,23.267,-47.354,1.0,2.0,-2.0,-4.0,-4.0,-3.0,24.086,-62.928,1.0,-60.783,89.114,-69.633,-1.0,-2.0,96.015,-65.122,-4.784,-4.0,97.493,3.0,-4.0,"NaN",74.193,"-Infinity",0.0,59.531,-3.0,0.513,"NaN",71.304,"NaN",-2.0,1.0,-1.0],"l4_bool":[false,false,true,false,true,false,false,false,true,true,true,true,false,true,false,false,true,false,true,false,true,false,false,true,false,false,false,false,false,false,true,false,true,true,true,false,true,true,true,false,false,false,false,false,false,false,true,true,false,true,false,false,true,false,true,false,false,true,true,true],"l5_f64":["Infinity",-2.0,-43.573,4.0,70.412,-2.0,1.0,2.0,-9.775,-81.927,-2.0,2.0,-4.0,3.0,-79.73,-42.157,4.0,"NaN",-30.62,0.0,-90.737,54.892,-76.408,17.278,-40.126,-4.0,60.679,1.0,-2.0,-1.0,60.671,-1.0,15.392,-9.168,87.451,-21.446,2.0,-1.0,4.0,-75.187,54.033,-37.172,-99.921,-20.228,1.0,-98.529,3.0,2.0,1.0,-4.0,-3.0,-75.127,-30.512,43.807,20.774,0.0,9.318,11.938,-67.214,-56.725],"l6_bool":[false,true,false,false,true,true,false,false,false,false,false,false,true,false,true,false,false,true,true,false,true,false,true,true,false,true,true,false,false,false,false,true,false,false,true,false,true,false,true,false,false,false,true,false,false,true,true,false,true,true,true,false,true,true,true,true,true,true,false,true]}}},"ops":[{"op":"head","n":7},{"op":"head","n":7}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_f64","type":"f64"},{"name":"l4_bool","type":"bool"},{"name":"l5_f64","type":"f64"},{"name":"l6_bool","type":"bool"}],"columns":{"l0_utf8":["zz","b","a","zz","zz","B",""],"l1_i32":[-3,-1,-1,1,-1,2,-5],"l2_f64":[1.0,38.593,-63.191,-99.704,0.0,-78.5,3.0],"l3_f64":[68.252,1.0,-4.0,-3.0,39.731,-8.76,17.226],"l4_bool":[false,false,true,false,true,false,false],"l5_f64":["Infinity",-2.0,-43.573,4.0,70.412,-2.0,1.0],"l6_bool":[false,true,false,false,true,true,false]}}},{"id":"random/0207","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_i64","type":"i64"},{"name":"l4_i64","type":"i64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_f64":[],"l3_i64":[],"l4_i64":[]}}},"ops":[{"op":"sort","keys":[{"expr":{"col":"l0_utf8"},"descending":true}],"nullsLast":true},{"op":"sort","keys":[{"expr":{"col":"l0_utf8"},"descending":false}],"nullsLast":false},{"op":"filter","predicate":{"fn":"isNull","args":[{"col":"l1_i32"}]}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_i64","type":"i64"},{"name":"l4_i64","type":"i64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_f64":[],"l3_i64":[],"l4_i64":[]}}},{"id":"random/0208","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_i32","type":"i32"},{"name":"l4_i64","type":"i64"},{"name":"l5_bool","type":"bool"}],"columns":{"l0_utf8":["c"],"l1_i32":[-1],"l2_i32":[4],"l3_i32":[3],"l4_i64":["9007199254740995"],"l5_bool":[true]}},"right":{"fields":[{"name":"l4_i64","type":"i64"},{"name":"r0_utf8","type":"utf8"}],"columns":{"r0_utf8":["zz","","B","ab","aa","ab","ab","ab","a","c"],"l4_i64":["9007199254740995","9007199254740995","9007199254740995","9007199254740995","9007199254740995","9007199254740995","9007199254740995","9007199254740995","9007199254740995","9007199254740995"]}}},"ops":[{"op":"join","right":"right","on":["l4_i64"],"how":"anti"},{"op":"groupBy","keys":["l3_i32"],"aggs":{"n1":{"agg":"len"},"a1_0":{"agg":"count","arg":{"col":"l4_i64"}},"a1_1":{"agg":"min","arg":{"col":"l2_i32"}}}},{"op":"sort","keys":[{"expr":{"col":"n1"},"descending":true}],"nullsLast":true}],"expected":{"fields":[{"name":"l3_i32","type":"i32"},{"name":"n1","type":"u32"},{"name":"a1_0","type":"u32"},{"name":"a1_1","type":"i32"}],"columns":{"l3_i32":[],"n1":[],"a1_0":[],"a1_1":[]}}},{"id":"random/0209","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i64","type":"i64"},{"name":"l3_f64","type":"f64"},{"name":"l4_i64","type":"i64"},{"name":"l5_i32","type":"i32"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_i64":[],"l3_f64":[],"l4_i64":[],"l5_i32":[]}}},"ops":[{"op":"filter","predicate":{"fn":"and","args":[{"fn":"and","args":[{"fn":"lt","args":[{"col":"l3_f64"},{"col":"l1_i32"}]},{"fn":"eq","args":[{"col":"l1_i32"},{"col":"l1_i32"}]}]},{"fn":"lte","args":[{"col":"l5_i32"},{"col":"l5_i32"}]}]}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i64","type":"i64"},{"name":"l3_f64","type":"f64"},{"name":"l4_i64","type":"i64"},{"name":"l5_i32","type":"i32"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_i64":[],"l3_f64":[],"l4_i64":[],"l5_i32":[]}}},{"id":"random/0210","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_i64","type":"i64"}],"columns":{"l0_utf8":["","ab",null,"b",null,"","c","c","ab","zz",null,"c","é","😀","😀","ab","","a","zz","c"],"l1_i32":[3,2,-1,-4,1,-94,null,-5,3,-2147483648,-4,4,5,-4,-5,0,3,2,0,3],"l2_i32":[5,0,3,-3,3,-5,2,5,4,1,2,4,4,-3,5,4,-5,null,-4,-5],"l3_i64":["9007199254740994","9007199254740993","4","-3","-4611686018427387904","3","4","-4611686018427387904","2","-4611686018427387904","-2","3","1","-4611686018427387904",null,null,"9007199254740994","-4611686018427387904","9007199254740995","9007199254740994"]}}},"ops":[{"op":"unique"},{"op":"filter","predicate":{"fn":"lt","args":[{"col":"l0_utf8"},{"lit":"b"}]}},{"op":"groupBy","keys":["l3_i64"],"aggs":{"n2":{"agg":"len"},"a2_0":{"agg":"nUnique","arg":{"col":"l1_i32"}}}}],"expected":{"fields":[{"name":"l3_i64","type":"i64"},{"name":"n2","type":"u32"},{"name":"a2_0","type":"u32"}],"columns":{"l3_i64":["9007199254740994","9007199254740993","3","2",null,"-4611686018427387904"],"n2":[2,1,1,1,1,1],"a2_0":[1,1,1,1,1,1]}}},{"id":"random/0211","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_i64","type":"i64"}],"columns":{"l0_utf8":["B","é","a","a","aa"],"l1_i32":[2,3,-4,-4,null],"l2_f64":[-0.071,4.836,-73.244,10.929,-16.182],"l3_i64":["9007199254740995","0","-4611686018427387904","9007199254740992","-4611686018427387904"]}}},"ops":[{"op":"head","n":5},{"op":"filter","predicate":{"fn":"and","args":[{"fn":"isIn","args":[{"col":"l0_utf8"}],"values":["B","c","zz"]},{"fn":"isNotNull","args":[{"col":"l1_i32"}]}]}},{"op":"groupBy","keys":["l3_i64"],"aggs":{"n2":{"agg":"len"},"a2_0":{"agg":"std","arg":{"col":"l2_f64"}},"a2_1":{"agg":"mean","arg":{"col":"l1_i32"}}}}],"expected":{"fields":[{"name":"l3_i64","type":"i64"},{"name":"n2","type":"u32"},{"name":"a2_0","type":"f64"},{"name":"a2_1","type":"f64"}],"columns":{"l3_i64":["9007199254740995"],"n2":[1],"a2_0":[null],"a2_1":[2.0]}}},{"id":"random/0212","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_i64","type":"i64"},{"name":"l4_i32","type":"i32"},{"name":"l5_i64","type":"i64"},{"name":"l6_f64","type":"f64"}],"columns":{"l0_utf8":["😀"],"l1_i32":[-4],"l2_utf8":["😀"],"l3_i64":["5"],"l4_i32":[-5],"l5_i64":["3"],"l6_f64":[-4.0]}},"right":{"fields":[{"name":"l1_i32","type":"i32"},{"name":"r0_utf8","type":"utf8"}],"columns":{"r0_utf8":["a",null,"zz"],"l1_i32":[-4,-4,-4]}}},"ops":[{"op":"join","right":"right","on":["l1_i32"],"how":"full"},{"op":"sort","keys":[{"expr":{"col":"l0_utf8"},"descending":true},{"expr":{"col":"l4_i32"},"descending":false}],"nullsLast":false}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_i64","type":"i64"},{"name":"l4_i32","type":"i32"},{"name":"l5_i64","type":"i64"},{"name":"l6_f64","type":"f64"},{"name":"r0_utf8","type":"utf8"}],"columns":{"l0_utf8":["😀","😀","😀"],"l1_i32":[-4,-4,-4],"l2_utf8":["😀","😀","😀"],"l3_i64":["5","5","5"],"l4_i32":[-5,-5,-5],"l5_i64":["3","3","3"],"l6_f64":[-4.0,-4.0,-4.0],"r0_utf8":["a",null,"zz"]}}},{"id":"random/0213","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_f64","type":"f64"},{"name":"l4_f64","type":"f64"},{"name":"l5_f64","type":"f64"},{"name":"l6_f64","type":"f64"}],"columns":{"l0_utf8":["😀",null,"a",null,null,"","a","c",null,"a","é","c",null,null,null,"b",null,"c","aa","😀",null,"é","",null,null,null,"c","aa","b","é",null,"é","é",null,null,"a",null,"c","😀",null,"aa","ab",null,"b",null,"ab","é","c","aa","c","b",null,null,null,null,"","B",null,"b",null],"l1_i32":[4,-2147483648,null,2,-1,308,null,-4,1,-4,5,null,null,-4,5,null,4,2,-5,5,1,-2,-3,3,-1,2,-1,-2,-5,0,null,0,null,null,5,null,5,4,1,-4,null,-3,5,-3,-2,2,2,null,0,-2,null,3,null,null,null,2,null,5,-2,1],"l2_utf8":["aa","c","a","b","ab","ab",null,"é","😀","😀","😀","a","b","b","b","",null,"a","aa","","aa","ab",null,"aa","zz","a",null,"é","é",null,"😀",null,"B","aa","é","c",null,null,"a","b","a","ab",null,"b",null,"a","ab","c",null,null,null,"B","ab",null,"a","zz","é","",null,null],"l3_f64":[null,null,25.895,null,2.839,4.0,-4.0,3.0,-88.774,-21.16,-91.207,null,76.192,4.0,null,null,3.0,null,null,-80.575,null,-1.0,76.616,null,1.0,2.0,null,3.0,null,null,null,null,-4.0,null,86.765,null,-3.0,null,-1.0,-3.0,null,-3.0,null,4.0,-46.78,74.941,-32.155,4.0,4.0,-2.732,-95.332,"NaN",-56.878,null,null,-3.0,-2.0,1.0,-4.32,67.783],"l4_f64":[-1.0,4.0,3.0,null,null,63.374,1.0,50.043,-22.708,-1.0,-3.0,33.92,-1.0,-14.174,null,-4.0,28.34,null,-26.677,53.186,-1.0,null,null,4.0,-57.442,null,1.0,-82.252,null,0.0,-1.0,null,82.802,99.932,null,null,-2.0,27.107,-58.632,2.0,null,0.0,-2.0,null,88.035,null,"NaN",null,null,null,null,3.0,-67.932,-96.907,null,-84.683,-33.927,null,-67.959,-75.226],"l5_f64":[null,1.0,-4.0,null,-2.0,16.909,-3.0,null,-18.741,null,-3.0,88.097,-1.0,-4.0,88.849,76.421,-3.042,31.853,1.0,18.612,-39.35,null,null,0.0,-3.0,-10.269,77.401,-3.0,null,null,null,-27.671,null,-91.003,3.0,4.0,65.696,-2.0,12.171,5.978,null,4.0,-1.0,-47.148,-5.348,null,-58.924,-60.944,35.267,1.0,null,1.0,-95.781,19.696,null,-3.0,34.837,32.993,null,null],"l6_f64":[-44.109,-4.0,97.54,null,null,null,null,null,61.356,-86.173,-3.976,-61.305,54.262,1.0,null,46.635,-33.824,null,null,null,null,null,2.0,92.121,80.197,-3.0,-23.282,null,62.116,77.404,-70.053,-4.0,-2.0,8.181,null,-89.572,4.0,null,1.0,-36.077,-79.747,39.497,-4.0,1.0,null,3.0,"NaN",-4.0,null,98.458,-11.873,2.0,55.533,null,0.0,null,null,1.0,3.0,-14.624]}},"right":{"fields":[{"name":"l2_utf8","type":"utf8"},{"name":"r0_f64","type":"f64"}],"columns":{"r0_f64":[58.001,-2.0,29.667,4.0,"-0",88.62,-72.689,-2.0,45.252,-40.089],"l2_utf8":["ab","a","aa","😀","ab","B","aa",null,"😀",null]}}},"ops":[{"op":"unique"},{"op":"join","right":"right","on":["l2_utf8"],"how":"semi"}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_f64","type":"f64"},{"name":"l4_f64","type":"f64"},{"name":"l5_f64","type":"f64"},{"name":"l6_f64","type":"f64"}],"columns":{"l0_utf8":["😀","a",null,"",null,"a","é","c","c","aa",null,"é",null,null,null,"é",null,"😀","aa","ab","ab","é",null,null,null],"l1_i32":[4,null,-1,308,1,-4,5,null,2,-5,1,-2,3,2,null,null,null,1,null,-3,2,2,3,null,null],"l2_utf8":["aa","a","ab","ab","😀","😀","😀","a","a","aa","aa","ab","aa","a","😀","B","aa","a","a","ab","a","ab","B","ab","a"],"l3_f64":[null,25.895,2.839,4.0,-88.774,-21.16,-91.207,null,null,null,null,-1.0,null,2.0,null,-4.0,null,-1.0,null,-3.0,74.941,-32.155,"NaN",-56.878,null],"l4_f64":[-1.0,3.0,null,63.374,-22.708,-1.0,-3.0,33.92,null,-26.677,-1.0,null,4.0,null,-1.0,82.802,99.932,-58.632,null,0.0,null,"NaN",3.0,-67.932,null],"l5_f64":[null,-4.0,-2.0,16.909,-18.741,null,-3.0,88.097,31.853,1.0,-39.35,null,0.0,-10.269,null,null,-91.003,12.171,null,4.0,null,-58.924,1.0,-95.781,null],"l6_f64":[-44.109,97.54,null,null,61.356,-86.173,-3.976,-61.305,null,null,null,null,92.121,-3.0,-70.053,-2.0,8.181,1.0,-79.747,39.497,3.0,"NaN",2.0,55.533,0.0]}}},{"id":"random/0214","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_bool","type":"bool"},{"name":"l4_i64","type":"i64"}],"columns":{"l0_utf8":[null,"aa","ab","é",""],"l1_i32":[0,3,-2147483648,-4,2],"l2_i32":[-4,-2,1,-4,-5],"l3_bool":[true,false,true,true,false],"l4_i64":["-4","9007199254740992","-4611686018427387904","9007199254740995","-5"]}}},"ops":[{"op":"head","n":10}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_bool","type":"bool"},{"name":"l4_i64","type":"i64"}],"columns":{"l0_utf8":[null,"aa","ab","é",""],"l1_i32":[0,3,-2147483648,-4,2],"l2_i32":[-4,-2,1,-4,-5],"l3_bool":[true,false,true,true,false],"l4_i64":["-4","9007199254740992","-4611686018427387904","9007199254740995","-5"]}}},{"id":"random/0215","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_f64","type":"f64"},{"name":"l4_f64","type":"f64"},{"name":"l5_i64","type":"i64"}],"columns":{"l0_utf8":["aa","é","",null,null,"zz","c",null,"ab",null,"😀","b",null,"aa",null,"😀","B","b",null,"😀","c",null,null,"a","aa","é","B","é","b","zz","b","ab","a","B",null,null,"",null,"zz","b","B","a",null,null,null,null,null,"b",null,"c",null,"ab",null,"é","b","",null,"",null,null],"l1_i32":[null,4,null,-3,-1,null,2147483647,3,-4,-5,-4,153,-4,3,0,0,null,-2,null,null,295,2147483647,-2,3,null,null,3,5,null,null,-2,2,-2,null,null,5,4,1,5,-2147483648,-4,3,null,-2147483648,4,4,5,0,0,null,1,-3,null,null,null,5,null,null,1,4],"l2_utf8":["ab","B",null,null,null,"aa","ab","b","ab",null,"aa","aa","b",null,null,"a","aa","ab",null,null,"é","aa","zz","é","c","c",null,"","aa",null,"ab",null,"aa","😀",null,null,"aa","a","b","b","aa","aa","B",null,"","B",null,null,"aa",null,"😀","😀","zz",null,null,"b","","B","zz",null],"l3_f64":[null,52.748,99.642,null,34.731,null,null,66.186,null,93.923,null,null,60.79,-2.0,4.0,null,"-Infinity",1.0,null,96.593,null,3.0,-98.716,73.762,-98.233,-1.0,null,2.0,null,4.0,null,null,42.111,2.505,null,-80.865,2.0,null,null,-4.0,null,-3.0,null,1.0,64.877,null,-83.345,-4.0,-5.338,-46.042,-2.0,-2.0,37.442,-70.122,-52.481,null,72.292,53.522,"NaN",null],"l4_f64":[3.062,56.649,29.412,-4.0,-1.0,null,-4.0,3.0,-92.394,null,-3.0,-33.367,2.0,-1.0,86.688,null,53.964,null,-2.0,-47.448,66.908,72.515,4.0,-32.809,null,-5.598,null,-4.0,1.0,2.0,-53.313,0.0,2.0,null,24.006,null,-2.0,1.0,2.0,null,"-Infinity",-2.0,-75.967,null,-2.0,null,14.605,-3.0,-2.0,null,null,98.536,-26.445,59.173,null,-16.507,null,"-Infinity",57.006,2.0],"l5_i64":[null,"-4611686018427387904",null,"-5","9007199254740995","9007199254740993","9007199254740995",null,"4","9007199254740992","-5",null,null,"-4611686018427387904","9007199254740992","-4611686018427387904","9007199254740994","-4","9007199254740995",null,"-4611686018427387904",null,null,"-4611686018427387904","-4611686018427387904","5","9007199254740993","-4611686018427387904","2","-5",null,null,"-4611686018427387904","-4611686018427387904","1","-4611686018427387904","-3","0","-4611686018427387904","4","9007199254740992","9007199254740992","9007199254740995","-4611686018427387904","9007199254740993","-5","9007199254740993",null,"-4611686018427387904","-5",null,"-4611686018427387904","9007199254740993","3","1","-5","9007199254740995","-5","-4611686018427387904",null]}}},"ops":[{"op":"unique","columns":["l1_i32"]},{"op":"filter","predicate":{"fn":"eq","args":[{"fn":"add","args":[{"col":"l3_f64"},{"col":"l3_f64"}]},{"fn":"add","args":[{"col":"l1_i32"},{"col":"l3_f64"}]}]}},{"op":"withColumns","exprs":{"x2":{"col":"l3_f64"}}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_f64","type":"f64"},{"name":"l4_f64","type":"f64"},{"name":"l5_i64","type":"i64"},{"name":"x2","type":"f64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_utf8":[],"l3_f64":[],"l4_f64":[],"l5_i64":[],"x2":[]}}},{"id":"random/0216","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_f64","type":"f64"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_f64","type":"f64"},{"name":"l6_i32","type":"i32"}],"columns":{"l0_utf8":["😀",null,"a","b","a","zz","a","aa","ab","c","ab",null,null,"b","B","aa",null,"zz","c","B"],"l1_i32":[-1,4,1,-2,null,-4,-2,0,1,0,5,2147483647,2,-5,-3,1,0,2,-425,null],"l2_utf8":["😀","aa","😀","B","zz","a","aa","a","ab","B","ab","zz","😀",null,null,"é","b","b","b",null],"l3_f64":[3.0,1.0,-7.796,2.0,-2.0,4.0,-3.0,48.888,3.0,3.0,null,-53.183,-2.0,56.105,-42.137,-4.0,-72.442,-1.0,null,null],"l4_utf8":["é","a","aa","é","c","é",null,"aa",null,"","😀","ab","","aa","ab","a","zz","B","ab",""],"l5_f64":[18.131,0.0,-2.0,2.0,-21.778,53.097,-3.0,-52.456,3.0,1.0,-2.0,-4.0,-1.0,43.727,3.977,null,91.858,-29.44,56.761,-31.553],"l6_i32":[-3,5,null,-2147483648,-5,4,4,-2147483648,0,-5,-3,0,3,2,1,-5,-4,0,-1,null]}},"right":{"fields":[{"name":"l2_utf8","type":"utf8"},{"name":"r0_bool","type":"bool"}],"columns":{"r0_bool":[null,false,null],"l2_utf8":["a","b","😀"]}}},"ops":[{"op":"join","right":"right","on":["l2_utf8"],"how":"left"},{"op":"withColumns","exprs":{"x1":{"col":"l3_f64"}}},{"op":"head","n":2}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_f64","type":"f64"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_f64","type":"f64"},{"name":"l6_i32","type":"i32"},{"name":"r0_bool","type":"bool"},{"name":"x1","type":"f64"}],"columns":{"l0_utf8":["😀",null],"l1_i32":[-1,4],"l2_utf8":["😀","aa"],"l3_f64":[3.0,1.0],"l4_utf8":["é","a"],"l5_f64":[18.131,0.0],"l6_i32":[-3,5],"r0_bool":[null,null],"x1":[3.0,1.0]}}},{"id":"random/0217","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i64","type":"i64"},{"name":"l3_i32","type":"i32"},{"name":"l4_f64","type":"f64"},{"name":"l5_i64","type":"i64"},{"name":"l6_utf8","type":"utf8"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_i64":[],"l3_i32":[],"l4_f64":[],"l5_i64":[],"l6_utf8":[]}}},"ops":[{"op":"head","n":7},{"op":"filter","predicate":{"fn":"isNull","args":[{"col":"l0_utf8"}]}},{"op":"sort","keys":[{"expr":{"col":"l5_i64"},"descending":true},{"expr":{"col":"l6_utf8"},"descending":false}],"nullsLast":false}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i64","type":"i64"},{"name":"l3_i32","type":"i32"},{"name":"l4_f64","type":"f64"},{"name":"l5_i64","type":"i64"},{"name":"l6_utf8","type":"utf8"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_i64":[],"l3_i32":[],"l4_f64":[],"l5_i64":[],"l6_utf8":[]}}},{"id":"random/0218","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_bool","type":"bool"},{"name":"l4_i32","type":"i32"},{"name":"l5_i32","type":"i32"}],"columns":{"l0_utf8":["zz","é","b","é","","c","😀","é","aa","ab","é","zz","zz","😀","a","a","a","","aa","B","zz","b","B","b","é","ab","b","B","","B","B","zz","zz","aa","é","c","😀","zz","B","é","b","ab","zz","zz","c","b","b","B","😀","zz","B","a","a","ab","a","é","😀","😀","","é"],"l1_i32":[-3,-3,5,-1,-2,2,3,-4,-4,3,-4,2,-2147483648,0,-4,-2,3,-2147483648,-4,-2,-2,3,2,-1,1,-4,-5,5,4,3,-1,4,-3,5,4,-2,4,-1,-2,3,2,0,0,-5,5,0,5,4,-4,1,-3,2,-4,-5,-5,-4,0,2,1,-3],"l2_i32":[2,4,1,-5,5,4,-3,0,-1,-2,1,-1,-1,4,-4,-2,2147483647,1,545,-2,-3,-666,2147483647,3,-3,-1,4,-5,-5,0,1,2,3,2,3,710,3,0,4,-1,-3,4,-4,-5,4,-3,5,2147483647,2,2147483647,3,2,2,4,-4,0,-1,2,2,-3],"l3_bool":[true,true,false,true,true,true,true,false,false,true,false,true,false,true,false,false,false,true,true,true,false,true,true,false,false,true,true,true,true,true,false,true,false,false,false,false,false,false,true,true,true,false,true,true,true,true,false,true,true,true,true,false,false,false,false,false,false,true,false,false],"l4_i32":[1,-1,2,4,0,-3,3,1,-3,-4,1,-2147483648,-3,-2,1,4,-5,312,-4,0,0,-5,3,2,0,3,1,5,2,4,0,5,-208,0,-5,-3,-3,0,3,1,3,0,-3,4,4,3,-3,2,-5,-1,-1,0,-4,0,-5,-5,-1,1,0,3],"l5_i32":[4,-5,-2,1,2,-4,-4,-5,-614,-5,0,2,3,5,3,-4,5,2,-1,379,5,4,-4,4,-1,5,1,-4,-3,-3,2,4,4,5,-4,-3,4,5,-4,0,-401,2,-3,3,-3,-5,5,-4,2,-3,4,3,2,-757,2,-886,-2147483648,-3,0,0]}},"right":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"r0_f64","type":"f64"}],"columns":{"r0_f64":[0.0,-1.0,3.0,4.0,3.0,"NaN",1.0,1.0,-91.739,-1.0],"l0_utf8":["c","ab","c","","","zz","b","é","zz","B"]}}},"ops":[{"op":"head","n":2},{"op":"join","right":"right","on":["l0_utf8"],"how":"full"}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_bool","type":"bool"},{"name":"l4_i32","type":"i32"},{"name":"l5_i32","type":"i32"},{"name":"r0_f64","type":"f64"}],"columns":{"l0_utf8":["zz","zz","é","c","ab","c","","","b","B"],"l1_i32":[-3,-3,-3,null,null,null,null,null,null,null],"l2_i32":[2,2,4,null,null,null,null,null,null,null],"l3_bool":[true,true,true,null,null,null,null,null,null,null],"l4_i32":[1,1,-1,null,null,null,null,null,null,null],"l5_i32":[4,4,-5,null,null,null,null,null,null,null],"r0_f64":["NaN",-91.739,1.0,0.0,-1.0,3.0,4.0,3.0,1.0,-1.0]}}},{"id":"random/0219","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_i32","type":"i32"}],"columns":{"l0_utf8":["ab",null,null,"😀","aa"],"l1_i32":[4,1,null,null,null],"l2_utf8":["é","c","b","ab",null],"l3_i32":[-2,4,-3,-5,-4]}}},"ops":[{"op":"groupBy","keys":["l2_utf8"],"aggs":{"n0":{"agg":"len"},"a0_0":{"agg":"mean","arg":{"col":"l1_i32"}}}}],"expected":{"fields":[{"name":"l2_utf8","type":"utf8"},{"name":"n0","type":"u32"},{"name":"a0_0","type":"f64"}],"columns":{"l2_utf8":["é","c","b","ab",null],"n0":[1,1,1,1,1],"a0_0":[4.0,1.0,null,null,null]}}},{"id":"random/0220","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_utf8","type":"utf8"}],"columns":{"l0_utf8":["B"],"l1_i32":[3],"l2_f64":[null],"l3_utf8":[null],"l4_utf8":["ab"]}},"right":{"fields":[{"name":"l4_utf8","type":"utf8"},{"name":"r0_f64","type":"f64"}],"columns":{"r0_f64":[],"l4_utf8":[]}}},"ops":[{"op":"join","right":"right","on":["l4_utf8"],"how":"left"},{"op":"filter","predicate":{"fn":"isNull","args":[{"col":"l0_utf8"}]}},{"op":"unique","columns":["l1_i32"]}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_utf8","type":"utf8"},{"name":"r0_f64","type":"f64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_f64":[],"l3_utf8":[],"l4_utf8":[],"r0_f64":[]}}},{"id":"random/0221","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_f64","type":"f64"},{"name":"l4_i32","type":"i32"}],"columns":{"l0_utf8":["😀","zz","c",null,"B","c","😀","c","ab","c",null,"ab","a","ab","b","aa","😀","é","😀","",null,null,"B","B","B","😀","","zz","B","😀",null,null,"ab","é","B","zz",null,"zz","zz","😀","ab","a","😀","😀","é","😀","","c","é","","b","é","zz","é","é","aa",null,"a","c","ab"],"l1_i32":[-2,-5,-1,1,1,1,-1,-3,4,-4,-4,3,-2,-2,4,1,1,-5,-4,-3,-1,3,null,-4,-1,-4,-4,-5,5,null,-1,-3,1,null,2147483647,null,-3,2,0,4,1,3,-5,-3,-3,0,null,2,null,2,4,-5,3,3,-3,2,-1,2,-2,-3],"l2_f64":[0.0,42.491,-1.0,-4.0,-26.008,3.0,0.0,42.481,78.047,"NaN",-4.0,3.0,12.125,-1.0,null,70.992,null,null,-91.794,-4.0,94.123,37.878,"NaN",null,null,-1.0,4.0,-1.315,null,-1.0,-63.981,35.741,-1.0,45.796,-3.0,-3.0,null,4.0,72.764,2.0,-3.0,null,-18.37,3.0,-16.12,"NaN",0.0,4.0,3.0,null,-11.094,54.595,3.0,-88.373,78.396,47.438,null,-47.668,51.065,-14.731],"l3_f64":[94.582,3.0,-3.0,-59.457,1.0,4.0,"NaN","-Infinity",-4.0,-11.029,68.956,-46.159,-42.456,-56.346,-4.0,null,0.0,null,"NaN",62.046,null,-35.436,-4.0,-3.0,-65.786,3.0,null,53.279,4.0,null,-34.303,88.949,1.0,-1.0,-66.53,-92.453,-81.095,-9.721,-27.32,0.0,-1.0,-98.068,17.653,-4.0,-4.104,81.062,-2.0,58.832,-2.0,null,null,4.0,-95.306,-59.747,"Infinity",-92.423,0.0,75.806,"-0",34.707],"l4_i32":[0,-3,-5,-4,1,-1,3,4,null,2,0,-5,-2,1,3,-2,-3,-1,5,2,4,0,2,2,535,4,4,0,0,0,-2,-1,-2,-3,-4,null,5,-4,-2,4,0,null,null,2,2147483647,4,2,2147483647,-4,4,4,-1,null,0,-4,-5,-5,4,0,null]}}},"ops":[{"op":"withColumns","exprs":{"x0":{"fn":"lte","args":[{"fn":"add","args":[{"col":"l2_f64"},{"col":"l4_i32"}]},{"fn":"abs","args":[{"col":"l1_i32"}]}]}}},{"op":"head","n":2}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_f64","type":"f64"},{"name":"l4_i32","type":"i32"},{"name":"x0","type":"bool"}],"columns":{"l0_utf8":["😀","zz"],"l1_i32":[-2,-5],"l2_f64":[0.0,42.491],"l3_f64":[94.582,3.0],"l4_i32":[0,-3],"x0":[true,false]}}},{"id":"random/0222","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i64","type":"i64"},{"name":"l3_i32","type":"i32"},{"name":"l4_i64","type":"i64"},{"name":"l5_i32","type":"i32"},{"name":"l6_f64","type":"f64"}],"columns":{"l0_utf8":["é","😀","zz","c","😀","c","a","B","c","zz","c","b","","b","aa","","zz","😀","a","😀"],"l1_i32":[-3,-3,-2,0,-3,-3,-3,-5,3,0,3,-5,-2,2147483647,891,-3,-5,4,-3,-717],"l2_i64":["-4611686018427387904","9007199254740995","-4611686018427387904","9007199254740993","-4611686018427387904","-4611686018427387904","3","1","2","-5","2","-4611686018427387904","-3","0","3","9007199254740995","-3","-2","9007199254740993","9007199254740995"],"l3_i32":[2,5,5,0,4,-1,2,-4,3,-1,1,3,-4,-3,-3,5,-5,-1,2,-2],"l4_i64":["-4611686018427387904","-4","9007199254740993","-4611686018427387904","-4611686018427387904","-4611686018427387904","5","9007199254740992","-3","0","-1","9007199254740994","4","9007199254740995","-4611686018427387904","-4611686018427387904","9007199254740994","-4611686018427387904","9007199254740994","-4"],"l5_i32":[4,4,2,0,3,-5,0,0,-3,3,-2,-653,-1,1,5,1,-5,5,-4,1],"l6_f64":[4.0,2.0,4.0,42.667,0.0,-1.0,51.507,2.0,-87.014,-19.101,-11.413,-19.665,-28.32,4.0,"-0",-1.0,-21.511,-1.0,-2.0,0.0]}}},"ops":[{"op":"withColumns","exprs":{"x0":{"fn":"neq","args":[{"col":"l5_i32"},{"fn":"when","args":[{"fn":"isNull","args":[{"col":"l6_f64"}]},{"col":"l5_i32"},{"col":"l5_i32"}]}]}}},{"op":"withColumns","exprs":{"x1":{"fn":"abs","args":[{"col":"l5_i32"}]}}},{"op":"groupBy","keys":["l5_i32"],"aggs":{"n2":{"agg":"len"},"a2_0":{"agg":"count","arg":{"col":"l4_i64"}},"a2_1":{"agg":"max","arg":{"col":"l2_i64"}},"a2_2":{"agg":"count","arg":{"col":"x0"}}}}],"expected":{"fields":[{"name":"l5_i32","type":"i32"},{"name":"n2","type":"u32"},{"name":"a2_0","type":"u32"},{"name":"a2_1","type":"i64"},{"name":"a2_2","type":"u32"}],"columns":{"l5_i32":[4,2,0,3,-5,-3,-2,-653,-1,1,5,-4],"n2":[2,1,3,2,2,1,1,1,1,3,2,1],"a2_0":[2,1,3,2,2,1,1,1,1,3,2,1],"a2_1":["9007199254740995","-4611686018427387904","9007199254740993","-5","-3","2","2","-4611686018427387904","-3","9007199254740995","3","9007199254740993"],"a2_2":[2,1,3,2,2,1,1,1,1,3,2,1]}}},{"id":"random/0223","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_bool","type":"bool"},{"name":"l4_f64","type":"f64"}],"columns":{"l0_utf8":["c"],"l1_i32":[2147483647],"l2_f64":[93.957],"l3_bool":[true],"l4_f64":[-44.174]}}},"ops":[{"op":"withColumns","exprs":{"x0":{"fn":"isNull","args":[{"col":"l2_f64"}]}}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_bool","type":"bool"},{"name":"l4_f64","type":"f64"},{"name":"x0","type":"bool"}],"columns":{"l0_utf8":["c"],"l1_i32":[2147483647],"l2_f64":[93.957],"l3_bool":[true],"l4_f64":[-44.174],"x0":[false]}}},{"id":"random/0224","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_f64","type":"f64"},{"name":"l4_i32","type":"i32"}],"columns":{"l0_utf8":["c",null,"B",null,"😀","😀","ab","B",null,"B",null,"c",null,"b","b",null,"a",null,null,"zz","B",null,"B","😀","é","😀","b","B","😀","😀","😀","aa","c","a",null,"zz","a",null,"a","é","😀","zz","aa","aa","b",null,"ab","b","c","","","é","a",null,"aa",null,"é",null,null,"ab"],"l1_i32":[3,-5,-4,null,3,4,-1,4,4,null,-2,null,null,4,-2,null,4,null,-4,5,5,2,-5,5,-3,-5,0,null,-4,null,4,5,null,5,null,null,1,0,null,null,null,2,-4,-3,-4,31,-5,-3,1000,3,null,null,-4,null,-3,0,null,-1,-5,3],"l2_f64":[null,2.0,1.0,3.0,-29.533,null,68.366,-12.378,-4.0,null,null,-1.0,3.0,4.0,null,-2.0,1.0,98.549,null,26.348,17.38,null,45.508,-11.136,null,null,2.0,-4.0,2.0,0.0,null,-68.745,4.0,22.039,-50.479,null,-20.232,43.639,null,38.77,-2.0,-4.0,4.0,2.0,-2.0,-52.457,-50.314,37.991,-1.0,null,-58.155,99.711,null,null,4.0,3.0,63.397,1.0,-1.0,-2.0],"l3_f64":[null,-2.0,null,2.0,13.707,-1.0,null,-1.0,48.235,24.045,null,0.0,2.0,-46.87,2.0,null,3.0,-43.354,-2.0,null,65.82,-29.526,4.0,38.266,4.0,-27.323,null,1.0,null,null,38.197,"-Infinity",-58.434,71.398,33.567,null,24.188,-1.0,0.0,null,-4.0,null,4.0,1.0,63.88,-2.0,-20.828,null,-4.478,null,null,null,null,-4.0,-72.593,null,null,56.85,-2.0,-1.0],"l4_i32":[3,-5,-2,4,null,0,-3,-2,null,-3,null,0,null,0,-2147483648,-5,0,-4,null,5,2,0,1,-2147483648,5,-5,4,-1,null,2,0,null,2,5,-3,-3,-5,1,null,5,-3,null,3,4,1,4,null,null,-5,-2,null,-1,null,4,-4,2,null,-1,-110,null]}}},"ops":[{"op":"groupBy","keys":["l4_i32"],"aggs":{"n0":{"agg":"len"},"a0_0":{"agg":"min","arg":{"col":"l0_utf8"}}}}],"expected":{"fields":[{"name":"l4_i32","type":"i32"},{"name":"n0","type":"u32"},{"name":"a0_0","type":"utf8"}],"columns":{"l4_i32":[3,-5,-2,4,null,0,-3,-2147483648,-4,5,2,1,-1,-110],"n0":[2,5,3,5,15,6,5,2,2,4,4,3,3,1],"a0_0":["aa","a","","aa","","a","B","b","aa","a","B","B","B",null]}}},{"id":"random/0225","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_i64","type":"i64"}],"columns":{"l0_utf8":["aa","ab","B","b","é"],"l1_i32":[5,273,5,-5,-2147483648],"l2_bool":[true,true,true,false,true],"l3_i64":["2","-4611686018427387904","9007199254740995","-2","9007199254740994"]}},"right":{"fields":[{"name":"l1_i32","type":"i32"},{"name":"r0_utf8","type":"utf8"}],"columns":{"r0_utf8":["b","B","b","B","aa","B","a","ab","aa","a"],"l1_i32":[-2147483648,5,-5,273,5,-2147483648,5,-5,-5,273]}}},"ops":[{"op":"join","right":"right","on":["l1_i32"],"how":"left"},{"op":"unique","columns":["l2_bool"]},{"op":"withColumns","exprs":{"x2":{"fn":"add","args":[{"col":"l1_i32"},{"col":"l1_i32"}]}}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_i64","type":"i64"},{"name":"r0_utf8","type":"utf8"},{"name":"x2","type":"i32"}],"columns":{"l0_utf8":["aa","b"],"l1_i32":[5,-5],"l2_bool":[true,false],"l3_i64":["2","-2"],"r0_utf8":["B","b"],"x2":[10,-10]}}},{"id":"random/0226","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_bool","type":"bool"}],"columns":{"l0_utf8":["a",null,"B","zz","aa"],"l1_i32":[-2,0,0,null,-5],"l2_bool":[false,null,true,true,true],"l3_utf8":["c","c","","b",null],"l4_utf8":["ab","é","c","c",null],"l5_bool":[true,false,null,false,true]}},"right":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"r0_bool","type":"bool"}],"columns":{"r0_bool":[null,true,true,true,true,false,true,true,null,false,false,false,true,true,true,false,null,null,true,false,false,false,true,false,true,true,false,false,true,true],"l0_utf8":["aa","a","a",null,"aa","B","aa","B","B","zz","zz","B","B",null,"zz","zz","zz","aa","zz",null,"a","B",null,"B","zz","a","zz","aa","aa","zz"]}}},"ops":[{"op":"unique","columns":["l2_bool"]},{"op":"head","n":4},{"op":"join","right":"right","on":["l0_utf8"],"how":"anti"}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_bool","type":"bool"}],"columns":{"l0_utf8":[null],"l1_i32":[0],"l2_bool":[null],"l3_utf8":["c"],"l4_utf8":["é"],"l5_bool":[false]}}},{"id":"random/0227","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_i32","type":"i32"}],"columns":{"l0_utf8":["😀"],"l1_i32":[0],"l2_bool":[true],"l3_i32":[-3]}}},"ops":[{"op":"unique","columns":["l0_utf8"]}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_i32","type":"i32"}],"columns":{"l0_utf8":["😀"],"l1_i32":[0],"l2_bool":[true],"l3_i32":[-3]}}},{"id":"random/0228","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_f64","type":"f64"},{"name":"l4_bool","type":"bool"},{"name":"l5_i32","type":"i32"},{"name":"l6_utf8","type":"utf8"}],"columns":{"l0_utf8":["aa",null,null,"aa",null],"l1_i32":[null,3,1,-1,-2],"l2_utf8":["ab",null,null,"😀","B"],"l3_f64":[null,null,null,null,94.131],"l4_bool":[true,true,null,null,false],"l5_i32":[1,null,3,-4,-1],"l6_utf8":["ab","é",null,"B","zz"]}}},"ops":[{"op":"head","n":1},{"op":"filter","predicate":{"fn":"neq","args":[{"fn":"floorDiv","args":[{"col":"l1_i32"},{"col":"l5_i32"}]},{"fn":"neg","args":[{"lit":2.5}]}]}},{"op":"withColumns","exprs":{"x2":{"fn":"gte","args":[{"col":"l6_utf8"},{"lit":"c"}]}}},{"op":"unique","columns":["x2"]}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_f64","type":"f64"},{"name":"l4_bool","type":"bool"},{"name":"l5_i32","type":"i32"},{"name":"l6_utf8","type":"utf8"},{"name":"x2","type":"bool"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_utf8":[],"l3_f64":[],"l4_bool":[],"l5_i32":[],"l6_utf8":[],"x2":[]}}},{"id":"random/0229","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_f64","type":"f64"}],"columns":{"l0_utf8":[null,"aa","aa","","a",null,"zz","aa",null,"B","B",null,"a",null,"b","zz",null,"",null,"","a","é",null,"aa",null,"a","😀",null,null,"","aa",null,null,"b","zz","ab","a",null,"ab","c","é",null,null,null,"é",null,"B",null,"é","b","c",null,"b","ab","ab","zz","ab","B","é",null],"l1_i32":[-1,null,-341,-2,-1,null,-2,5,2147483647,-2,-5,null,-3,3,null,-2,637,2147483647,null,null,5,958,3,null,-5,3,null,-4,3,-1,1,4,null,5,null,0,-5,null,-3,4,3,null,null,3,1,-2,5,null,null,null,0,1,1,2,null,2,2,null,0,0],"l2_bool":[false,null,true,false,true,true,null,false,true,null,null,null,false,null,null,true,false,null,null,null,true,false,null,false,true,null,true,false,false,false,false,true,true,true,null,true,true,true,true,false,null,null,false,true,null,true,null,true,true,null,false,null,true,false,true,true,null,false,false,false],"l3_f64":[-3.0,null,-4.0,3.0,-98.122,null,null,-16.845,16.978,null,null,-76.432,null,null,null,"NaN",null,-29.52,null,null,3.0,2.0,47.342,null,-45.524,null,"NaN",-1.0,null,1.0,null,null,2.0,null,-3.0,2.0,50.551,-4.0,56.546,-2.0,89.544,1.0,null,-65.012,null,null,2.0,-2.0,-1.0,1.0,-1.0,-4.0,2.0,33.84,3.0,"-0",null,-4.0,null,-4.0]}},"right":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"r0_f64","type":"f64"}],"columns":{"r0_f64":[],"l0_utf8":[]}}},"ops":[{"op":"join","right":"right","on":["l0_utf8"],"how":"semi"},{"op":"filter","predicate":{"fn":"neq","args":[{"col":"l1_i32"},{"fn":"mod","args":[{"col":"l3_f64"},{"col":"l1_i32"}]}]}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_f64","type":"f64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_bool":[],"l3_f64":[]}}},{"id":"random/0230","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i64","type":"i64"},{"name":"l3_bool","type":"bool"}],"columns":{"l0_utf8":["a","","b","a","ab","aa","B","b","b","😀","c","b","aa","c","B","a","b","B","é","B"],"l1_i32":[5,-2147483648,-4,3,2147483647,2,-2147483648,3,5,3,0,2147483647,5,4,-2,4,3,-2,3,-5],"l2_i64":["-4","-1","9007199254740993","9007199254740992","3","9007199254740995","9007199254740993","9007199254740992","0","-1","5","9007199254740993","9007199254740992","9007199254740993","9007199254740993","9007199254740994","9007199254740994","9007199254740995","-4611686018427387904","-5"],"l3_bool":[true,true,false,false,false,false,true,false,true,false,true,false,false,false,true,false,false,false,false,true]}}},"ops":[{"op":"head","n":6},{"op":"head","n":7},{"op":"filter","predicate":{"fn":"and","args":[{"fn":"gte","args":[{"col":"l0_utf8"},{"lit":"é"}]},{"fn":"neq","args":[{"col":"l1_i32"},{"col":"l1_i32"}]}]}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i64","type":"i64"},{"name":"l3_bool","type":"bool"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_i64":[],"l3_bool":[]}}},{"id":"random/0231","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_f64","type":"f64"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_f64","type":"f64"},{"name":"l6_f64","type":"f64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_f64":[],"l3_f64":[],"l4_utf8":[],"l5_f64":[],"l6_f64":[]}}},"ops":[{"op":"head","n":9}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_f64","type":"f64"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_f64","type":"f64"},{"name":"l6_f64","type":"f64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_f64":[],"l3_f64":[],"l4_utf8":[],"l5_f64":[],"l6_f64":[]}}},{"id":"random/0232","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_i64","type":"i64"},{"name":"l4_i32","type":"i32"}],"columns":{"l0_utf8":["aa","😀","a","aa","😀"],"l1_i32":[5,1,2,-3,-2],"l2_f64":[4.0,1.0,-98.713,-8.581,-3.0],"l3_i64":["-4611686018427387904","-3","-5","9007199254740994","-4611686018427387904"],"l4_i32":[3,3,-2,-1,3]}}},"ops":[{"op":"withColumns","exprs":{"x0":{"col":"l4_i32"}}},{"op":"withColumns","exprs":{"x1":{"col":"l2_f64"}}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_i64","type":"i64"},{"name":"l4_i32","type":"i32"},{"name":"x0","type":"i32"},{"name":"x1","type":"f64"}],"columns":{"l0_utf8":["aa","😀","a","aa","😀"],"l1_i32":[5,1,2,-3,-2],"l2_f64":[4.0,1.0,-98.713,-8.581,-3.0],"l3_i64":["-4611686018427387904","-3","-5","9007199254740994","-4611686018427387904"],"l4_i32":[3,3,-2,-1,3],"x0":[3,3,-2,-1,3],"x1":[4.0,1.0,-98.713,-8.581,-3.0]}}},{"id":"random/0233","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_i32","type":"i32"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_utf8","type":"utf8"},{"name":"l6_utf8","type":"utf8"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_utf8":[],"l3_i32":[],"l4_utf8":[],"l5_utf8":[],"l6_utf8":[]}},"right":{"fields":[{"name":"l6_utf8","type":"utf8"},{"name":"r0_bool","type":"bool"}],"columns":{"r0_bool":[false,true,null],"l6_utf8":["😀","ab",""]}}},"ops":[{"op":"sort","keys":[{"expr":{"col":"l4_utf8"},"descending":true}],"nullsLast":true},{"op":"join","right":"right","on":["l6_utf8"],"how":"inner"}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_i32","type":"i32"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_utf8","type":"utf8"},{"name":"l6_utf8","type":"utf8"},{"name":"r0_bool","type":"bool"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_utf8":[],"l3_i32":[],"l4_utf8":[],"l5_utf8":[],"l6_utf8":[],"r0_bool":[]}}},{"id":"random/0234","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_bool","type":"bool"},{"name":"l4_f64","type":"f64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_utf8":[],"l3_bool":[],"l4_f64":[]}}},"ops":[{"op":"head","n":2},{"op":"groupBy","keys":["l0_utf8"],"aggs":{"n1":{"agg":"len"},"a1_0":{"agg":"median","arg":{"col":"l4_f64"}}}},{"op":"withColumns","exprs":{"x2":{"fn":"and","args":[{"fn":"gt","args":[{"col":"a1_0"},{"lit":0.5}]},{"fn":"isNotNull","args":[{"col":"n1"}]}]}}},{"op":"filter","predicate":{"fn":"eq","args":[{"col":"a1_0"},{"col":"a1_0"}]}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"n1","type":"u32"},{"name":"a1_0","type":"f64"},{"name":"x2","type":"bool"}],"columns":{"l0_utf8":[],"n1":[],"a1_0":[],"x2":[]}}},{"id":"random/0235","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_f64","type":"f64"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_utf8","type":"utf8"},{"name":"l6_utf8","type":"utf8"}],"columns":{"l0_utf8":["","a","zz","b","zz","aa","B","ab","😀","b","b","c","","c",null,"é","B","c","B","","","😀","ab","é","c","B","é","B","c","","c","a","😀","c","b",null,null,"a","zz","B","é","a","c","ab","","aa","é","c","ab","zz","é","a","😀","","zz","zz","ab",null,"😀",""],"l1_i32":[0,-4,5,3,null,3,2,0,3,2,-3,-1,2,2,-5,4,2,-3,-3,5,-2,2,-3,3,-4,1,-1,2147483647,-3,5,2,2,-2147483648,5,-5,4,-5,2,null,3,5,-2,null,null,0,1,-2,5,-3,0,1,1,-4,-1,5,5,3,2,-3,-2],"l2_f64":[null,-4.0,-54.756,2.0,1.0,0.729,4.0,null,-33.134,-80.384,35.102,0.0,3.0,2.0,2.0,4.0,99.354,-13.696,-18.416,-2.0,-3.0,-8.874,14.178,null,76.35,"NaN",97.722,-2.28,"-0",-4.0,-3.0,85.173,8.127,4.0,86.32,-47.955,-4.0,-4.0,-3.0,null,-80.717,-0.989,null,4.0,1.0,null,16.717,-3.0,-21.783,-3.0,-20.893,88.62,42.84,null,3.0,2.0,-1.0,-4.0,0.0,43.989],"l3_f64":[-80.292,-1.0,-45.918,-10.035,"-Infinity",0.0,61.056,-37.111,null,-4.0,28.574,86.734,-32.981,-94.458,-3.0,0.0,-79.813,"NaN",88.501,-1.0,-59.624,0.0,-2.0,3.0,93.276,-51.682,23.688,null,1.0,null,-58.625,-1.0,-15.889,-1.0,-84.714,41.402,4.0,85.1,-34.633,75.944,4.0,47.116,"-0",0.0,19.453,2.0,-1.0,94.906,-62.922,80.246,-3.0,-4.0,46.232,-2.0,null,73.862,-49.725,-1.0,-50.358,-92.881],"l4_utf8":["é","a","ab","😀",null,"b","ab","",null,"😀","ab","😀","a","c","c","B","","😀","B","B","ab","c","ab",null,"zz","zz","é","b","a","B","","zz","B","b","c","","B","zz","b","zz",null,"","zz","é",null,"B","b","c","c","zz","zz","b","c","ab","aa","b","aa","😀","😀","ab"],"l5_utf8":["aa","a","","😀","c","aa","","ab","zz","","aa","B","B",null,"","c","zz","aa","c","B","B",null,"aa","a","B","B","c",null,"a","zz","","ab","ab","b",null,"c","zz","c","é","b","b","a","b","B","ab","é","a","aa","b","B","é",null,"a","aa","B","b","a","","😀","zz"],"l6_utf8":["","B","😀","c","","c","aa",null,"zz","c",null,"😀","ab","😀","😀",null,"😀",null,"é","aa",null,"zz","aa","b","","c","zz","b","😀","é","ab","😀","zz","😀","","aa","é","a","aa","é","b","aa",null,"b",null,"é","c","c","c",null,"c","B","c",null,"a","😀","aa",null,"é","c"]}}},"ops":[{"op":"withColumns","exprs":{"x0":{"col":"l3_f64"}}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_f64","type":"f64"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_utf8","type":"utf8"},{"name":"l6_utf8","type":"utf8"},{"name":"x0","type":"f64"}],"columns":{"l0_utf8":["","a","zz","b","zz","aa","B","ab","😀","b","b","c","","c",null,"é","B","c","B","","","😀","ab","é","c","B","é","B","c","","c","a","😀","c","b",null,null,"a","zz","B","é","a","c","ab","","aa","é","c","ab","zz","é","a","😀","","zz","zz","ab",null,"😀",""],"l1_i32":[0,-4,5,3,null,3,2,0,3,2,-3,-1,2,2,-5,4,2,-3,-3,5,-2,2,-3,3,-4,1,-1,2147483647,-3,5,2,2,-2147483648,5,-5,4,-5,2,null,3,5,-2,null,null,0,1,-2,5,-3,0,1,1,-4,-1,5,5,3,2,-3,-2],"l2_f64":[null,-4.0,-54.756,2.0,1.0,0.729,4.0,null,-33.134,-80.384,35.102,0.0,3.0,2.0,2.0,4.0,99.354,-13.696,-18.416,-2.0,-3.0,-8.874,14.178,null,76.35,"NaN",97.722,-2.28,"-0",-4.0,-3.0,85.173,8.127,4.0,86.32,-47.955,-4.0,-4.0,-3.0,null,-80.717,-0.989,null,4.0,1.0,null,16.717,-3.0,-21.783,-3.0,-20.893,88.62,42.84,null,3.0,2.0,-1.0,-4.0,0.0,43.989],"l3_f64":[-80.292,-1.0,-45.918,-10.035,"-Infinity",0.0,61.056,-37.111,null,-4.0,28.574,86.734,-32.981,-94.458,-3.0,0.0,-79.813,"NaN",88.501,-1.0,-59.624,0.0,-2.0,3.0,93.276,-51.682,23.688,null,1.0,null,-58.625,-1.0,-15.889,-1.0,-84.714,41.402,4.0,85.1,-34.633,75.944,4.0,47.116,"-0",0.0,19.453,2.0,-1.0,94.906,-62.922,80.246,-3.0,-4.0,46.232,-2.0,null,73.862,-49.725,-1.0,-50.358,-92.881],"l4_utf8":["é","a","ab","😀",null,"b","ab","",null,"😀","ab","😀","a","c","c","B","","😀","B","B","ab","c","ab",null,"zz","zz","é","b","a","B","","zz","B","b","c","","B","zz","b","zz",null,"","zz","é",null,"B","b","c","c","zz","zz","b","c","ab","aa","b","aa","😀","😀","ab"],"l5_utf8":["aa","a","","😀","c","aa","","ab","zz","","aa","B","B",null,"","c","zz","aa","c","B","B",null,"aa","a","B","B","c",null,"a","zz","","ab","ab","b",null,"c","zz","c","é","b","b","a","b","B","ab","é","a","aa","b","B","é",null,"a","aa","B","b","a","","😀","zz"],"l6_utf8":["","B","😀","c","","c","aa",null,"zz","c",null,"😀","ab","😀","😀",null,"😀",null,"é","aa",null,"zz","aa","b","","c","zz","b","😀","é","ab","😀","zz","😀","","aa","é","a","aa","é","b","aa",null,"b",null,"é","c","c","c",null,"c","B","c",null,"a","😀","aa",null,"é","c"],"x0":[-80.292,-1.0,-45.918,-10.035,"-Infinity",0.0,61.056,-37.111,null,-4.0,28.574,86.734,-32.981,-94.458,-3.0,0.0,-79.813,"NaN",88.501,-1.0,-59.624,0.0,-2.0,3.0,93.276,-51.682,23.688,null,1.0,null,-58.625,-1.0,-15.889,-1.0,-84.714,41.402,4.0,85.1,-34.633,75.944,4.0,47.116,"-0",0.0,19.453,2.0,-1.0,94.906,-62.922,80.246,-3.0,-4.0,46.232,-2.0,null,73.862,-49.725,-1.0,-50.358,-92.881]}}},{"id":"random/0236","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_bool","type":"bool"},{"name":"l4_f64","type":"f64"}],"columns":{"l0_utf8":["ab","zz","😀","ab","é","a","B","b","B","aa","b","ab","é","a","b","zz","zz","é","a","c","aa","a","aa","aa","","zz","B","😀","b","zz","aa","zz","zz","a","ab","zz","😀","B","aa","zz","","ab","c","","😀","aa","c","B","a","zz","zz","aa","ab","B","é","aa","a","aa","","aa"],"l1_i32":[0,-2,-1,4,3,2,-3,4,3,0,5,-3,-3,1,5,-5,4,4,2,1,-5,2147483647,-5,3,-1,1,3,3,5,2147483647,3,4,0,-3,-3,3,-2,0,-5,-4,-1,4,-4,3,1,-1,-3,4,0,0,4,-5,3,-4,1,2,-2,-2,-1,2],"l2_f64":[0.0,"Infinity",-3.0,91.966,"NaN",-1.0,-48.666,0.0,-4.0,-78.985,55.042,-63.404,-2.0,-57.506,-2.0,3.0,"NaN",4.0,4.0,-99.896,-2.0,4.0,"-0",-2.0,-48.387,4.0,-1.0,2.414,-82.947,15.064,-2.0,-4.0,-8.441,-1.0,-1.0,-7.999,-4.0,2.0,"-Infinity",2.0,-3.0,"-Infinity",0.0,4.0,1.0,"NaN",-65.336,-3.0,26.026,94.73,-89.729,-71.779,11.368,0.0,0.0,-90.587,-82.643,"NaN",0.0,54.153],"l3_bool":[false,false,true,false,false,false,true,true,true,false,false,false,false,true,true,false,false,true,false,false,false,true,true,true,true,true,false,true,false,true,false,false,true,false,false,true,true,true,true,false,true,false,true,true,false,true,true,false,true,true,false,false,true,true,true,true,true,false,true,true],"l4_f64":[3.0,-4.0,-48.295,-1.0,43.491,-15.89,43.34,-4.0,35.856,0.05,-99.277,-86.003,21.311,-3.0,-3.0,2.0,-71.88,0.0,-7.59,99.821,-1.0,"Infinity",24.225,4.0,-2.0,-4.0,-2.0,4.0,1.0,-96.062,4.0,-2.0,-3.0,1.0,-3.0,65.661,-3.0,63.601,4.0,0.0,0.0,57.254,72.563,-1.0,-4.0,-1.0,2.0,4.0,-2.0,-2.0,-1.0,-1.0,67.378,-39.861,-98.145,0.0,-74.499,"NaN",-69.03,0.0]}}},"ops":[{"op":"groupBy","keys":["l3_bool"],"aggs":{"n0":{"agg":"len"},"a0_0":{"agg":"first","arg":{"col":"l0_utf8"}},"a0_1":{"agg":"min","arg":{"col":"l0_utf8"}}}},{"op":"filter","predicate":{"fn":"strStartsWith","args":[{"col":"a0_1"}],"prefix":"a"}},{"op":"unique","columns":["n0"]},{"op":"filter","predicate":{"fn":"isNotNull","args":[{"col":"n0"}]}}],"expected":{"fields":[{"name":"l3_bool","type":"bool"},{"name":"n0","type":"u32"},{"name":"a0_0","type":"utf8"},{"name":"a0_1","type":"utf8"}],"columns":{"l3_bool":[],"n0":[],"a0_0":[],"a0_1":[]}}},{"id":"random/0237","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_i64","type":"i64"},{"name":"l4_utf8","type":"utf8"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_f64":[],"l3_i64":[],"l4_utf8":[]}}},"ops":[{"op":"withColumns","exprs":{"x0":{"lit":-1}}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_i64","type":"i64"},{"name":"l4_utf8","type":"utf8"},{"name":"x0","type":"i32"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_f64":[],"l3_i64":[],"l4_utf8":[],"x0":[]}}},{"id":"random/0238","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_i64","type":"i64"},{"name":"l5_utf8","type":"utf8"},{"name":"l6_bool","type":"bool"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_i32":[],"l3_utf8":[],"l4_i64":[],"l5_utf8":[],"l6_bool":[]}}},"ops":[{"op":"unique"}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_i64","type":"i64"},{"name":"l5_utf8","type":"utf8"},{"name":"l6_bool","type":"bool"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_i32":[],"l3_utf8":[],"l4_i64":[],"l5_utf8":[],"l6_bool":[]}}},{"id":"random/0239","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_f64","type":"f64"},{"name":"l4_f64","type":"f64"}],"columns":{"l0_utf8":["😀","c","b",null,"zz",null,null,null,null,"😀",null,null,"zz",null,"zz",null,null,"","","b","aa","😀",null,null,"a","é","c","aa","B","aa","ab","aa","é","a","ab",null,null,"é","é",null,"zz","aa","é","","a","a","😀","a","c","a","c","aa","ab",null,"","é","😀","aa","ab",""],"l1_i32":[1,-2,1,-2,null,3,null,-4,null,3,-3,4,4,2,null,-5,null,5,798,2,null,5,0,null,2147483647,-2,1,5,3,0,1,null,null,2,null,0,null,-1,null,0,null,-1,null,4,5,5,-4,3,-5,null,-4,-4,2,5,1,-1,4,null,3,null],"l2_utf8":["a","é","b",null,null,"é","é","é","é","b","B","aa","aa","B","b","a","",null,"","B","B","B",null,"","B",null,null,null,null,"b","a","","b","a","é","B",null,"B","a",null,"B",null,"ab",null,"ab","c","c","a","é","a",null,"😀",null,"😀",null,"B","a",null,"zz","c"],"l3_f64":[-99.232,-9.422,34.061,-69.56,null,null,50.713,0.0,-3.0,-4.0,2.0,-39.637,4.0,81.495,2.0,-2.0,0.0,-35.471,-4.0,2.0,28.241,-1.0,17.905,null,"-Infinity",null,47.089,31.218,null,"NaN",90.508,null,67.411,-1.0,null,null,null,-97.001,2.0,null,48.292,98.7,null,null,0.0,9.362,-2.0,-40.237,-1.0,"-Infinity","NaN",-83.774,-51.063,0.0,-96.234,-51.284,null,1.0,-6.491,-3.0],"l4_f64":[9.056,-3.0,null,-1.0,20.059,null,55.155,-1.0,-2.0,-37.396,1.604,-1.0,null,-35.521,27.489,null,4.0,null,3.0,-1.0,null,null,0.0,82.397,null,-3.0,60.581,23.866,null,-84.213,null,null,null,20.805,null,1.0,null,null,-2.0,null,3.0,null,-1.0,69.185,-3.0,null,-73.862,35.531,-3.376,-36.58,28.205,null,null,48.231,0.0,-4.0,"NaN",null,-76.614,null]}}},"ops":[{"op":"filter","predicate":{"fn":"isNull","args":[{"col":"l0_utf8"}]}},{"op":"head","n":1},{"op":"sort","keys":[{"expr":{"col":"l1_i32"},"descending":false}],"nullsLast":false},{"op":"unique"}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_f64","type":"f64"},{"name":"l4_f64","type":"f64"}],"columns":{"l0_utf8":[null],"l1_i32":[-2],"l2_utf8":[null],"l3_f64":[-69.56],"l4_f64":[-1.0]}}},{"id":"random/0240","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_f64","type":"f64"},{"name":"l4_i64","type":"i64"},{"name":"l5_f64","type":"f64"},{"name":"l6_i32","type":"i32"}],"columns":{"l0_utf8":["c","ab","a","c","aa","b","B","é","é","a","ab","b","ab","a","b","c","😀","aa","😀","é"],"l1_i32":[-1,0,-4,4,-4,-2147483648,4,-1,-5,1,5,1,4,-3,-1,929,-5,-5,2,3],"l2_i32":[0,-1,4,-4,-5,870,5,2,5,-2,-2,4,1,4,5,4,-4,-1,4,-2],"l3_f64":[4.0,-40.572,"NaN",-1.869,-3.0,17.415,"NaN",-4.0,7.265,4.0,-3.0,4.0,4.0,2.0,-1.0,-60.249,-1.0,0.0,-1.0,1.0],"l4_i64":["5","5","9007199254740993","9007199254740993","9007199254740994","4","-5","-4611686018427387904","5","9007199254740992","-4611686018427387904","3","1","-4611686018427387904","9007199254740995","-4611686018427387904","-4611686018427387904","0","-4611686018427387904","4"],"l5_f64":[25.856,"NaN",4.0,4.0,82.473,-52.108,15.155,2.0,0.0,-26.534,4.0,51.677,2.0,4.0,1.0,2.81,1.0,-49.718,2.0,0.0],"l6_i32":[-5,-3,-4,0,0,5,2147483647,-502,-1,-2,5,-3,4,-5,-1,4,-2147483648,0,-2,-1]}}},"ops":[{"op":"withColumns","exprs":{"x0":{"fn":"gte","args":[{"col":"l0_utf8"},{"lit":"aa"}]}}},{"op":"head","n":8},{"op":"filter","predicate":{"fn":"eq","args":[{"col":"l4_i64"},{"lit":"0","type":"i64"}]}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_f64","type":"f64"},{"name":"l4_i64","type":"i64"},{"name":"l5_f64","type":"f64"},{"name":"l6_i32","type":"i32"},{"name":"x0","type":"bool"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_i32":[],"l3_f64":[],"l4_i64":[],"l5_f64":[],"l6_i32":[],"x0":[]}}},{"id":"random/0241","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_i64","type":"i64"}],"columns":{"l0_utf8":["é","c","zz","b","ab","b",null,"B","aa","b","c","ab","😀","é","a","zz",null,"é","c","ab","😀","😀","ab","😀","","c","ab","B","aa","","é","c","ab","ab","😀","aa","😀","a","zz","😀","aa","aa","","b","ab","ab","","c",null,null,"aa","B","zz","c","B","","aa","B","ab","b"],"l1_i32":[3,661,1,-3,5,-4,2147483647,-1,3,null,4,2,-3,-2,2,-3,3,-1,2147483647,-2,-3,2147483647,-5,null,-2,1,0,0,4,-5,null,5,1,-4,3,-2147483648,-4,0,0,5,-2,-1,-5,-2,1,-4,1,-331,null,-2,3,-2,2,-1,-3,0,-4,0,4,4],"l2_bool":[false,true,false,false,true,true,false,false,false,false,true,true,false,false,false,true,null,false,true,false,true,true,null,true,false,false,false,true,false,true,true,null,true,false,false,false,false,true,true,true,false,null,false,true,true,true,true,false,true,false,true,false,true,null,false,false,true,false,true,false],"l3_i64":["-4611686018427387904","-2","5","9007199254740992","-4611686018427387904","-4611686018427387904","-3","9007199254740992","-4611686018427387904","-4611686018427387904","9007199254740994","-4611686018427387904","9007199254740994","9007199254740994","9007199254740995","-5","9007199254740995","-4611686018427387904","9007199254740995","9007199254740993","0","-1",null,"-4611686018427387904","-4611686018427387904","9007199254740993","-1","-4611686018427387904","-4611686018427387904",null,"0",null,"9007199254740995","0","-4611686018427387904","-4611686018427387904","-4611686018427387904","9007199254740995",null,null,"9007199254740993","2","5","9007199254740995","-4611686018427387904","-4611686018427387904",null,"-1","9007199254740994","3",null,null,"2","-4611686018427387904","-4611686018427387904","-4611686018427387904","0","-4611686018427387904","2","9007199254740995"]}}},"ops":[{"op":"unique"},{"op":"filter","predicate":{"fn":"isNull","args":[{"col":"l0_utf8"}]}},{"op":"withColumns","exprs":{"x2":{"fn":"neq","args":[{"col":"l1_i32"},{"fn":"abs","args":[{"col":"l1_i32"}]}]}}},{"op":"groupBy","keys":["x2"],"aggs":{"n3":{"agg":"len"},"a3_0":{"agg":"nUnique","arg":{"col":"l0_utf8"}},"a3_1":{"agg":"max","arg":{"col":"l3_i64"}}}}],"expected":{"fields":[{"name":"x2","type":"bool"},{"name":"n3","type":"u32"},{"name":"a3_0","type":"u32"},{"name":"a3_1","type":"i64"}],"columns":{"x2":[false,null,true],"n3":[2,1,1],"a3_0":[1,1,1],"a3_1":["9007199254740995","9007199254740994","3"]}}},{"id":"random/0242","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_bool","type":"bool"},{"name":"l5_bool","type":"bool"},{"name":"l6_utf8","type":"utf8"}],"columns":{"l0_utf8":["😀","ab","c","zz","😀"],"l1_i32":[0,-1,4,5,5],"l2_bool":[true,false,false,false,false],"l3_utf8":["","B","😀","é","é"],"l4_bool":[false,false,true,false,false],"l5_bool":[false,false,false,true,false],"l6_utf8":["zz","B","😀","ab","zz"]}}},"ops":[{"op":"unique"},{"op":"groupBy","keys":["l4_bool"],"aggs":{"n1":{"agg":"len"},"a1_0":{"agg":"last","arg":{"col":"l6_utf8"}},"a1_1":{"agg":"first","arg":{"col":"l6_utf8"}},"a1_2":{"agg":"count","arg":{"col":"l2_bool"}}}},{"op":"unique","columns":["l4_bool"]},{"op":"sort","keys":[{"expr":{"col":"l4_bool"},"descending":true}],"nullsLast":true}],"expected":{"fields":[{"name":"l4_bool","type":"bool"},{"name":"n1","type":"u32"},{"name":"a1_0","type":"utf8"},{"name":"a1_1","type":"utf8"},{"name":"a1_2","type":"u32"}],"columns":{"l4_bool":[true,false],"n1":[1,4],"a1_0":["😀","zz"],"a1_1":["😀","zz"],"a1_2":[1,4]}}},{"id":"random/0243","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_bool","type":"bool"}],"columns":{"l0_utf8":["","",null,"b","😀"],"l1_i32":[5,4,null,-4,null],"l2_bool":[false,true,true,true,false],"l3_bool":[false,false,true,true,true]}},"right":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"r0_utf8","type":"utf8"}],"columns":{"r0_utf8":["","B","B","","a","b","ab","",null,null,null,"B","aa","aa","",null,"a",null,"B","a","😀",null,"a",null,"B","B","ab","é",null,null],"l0_utf8":["b","b",null,"😀","","","😀","😀","","",null,"b","","","b","","b","","😀","","😀","b","😀","","😀","","","b","",null]}}},"ops":[{"op":"join","right":"right","on":["l0_utf8"],"how":"left"},{"op":"withColumns","exprs":{"x1":{"fn":"neq","args":[{"fn":"sub","args":[{"lit":2.5},{"lit":2.5}]},{"fn":"abs","args":[{"col":"l1_i32"}]}]}}},{"op":"filter","predicate":{"col":"l3_bool"}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_bool","type":"bool"},{"name":"r0_utf8","type":"utf8"},{"name":"x1","type":"bool"}],"columns":{"l0_utf8":[null,"b","b","b","b","b","b","b","😀","😀","😀","😀","😀","😀","😀"],"l1_i32":[null,-4,-4,-4,-4,-4,-4,-4,null,null,null,null,null,null,null],"l2_bool":[true,true,true,true,true,true,true,true,false,false,false,false,false,false,false],"l3_bool":[true,true,true,true,true,true,true,true,true,true,true,true,true,true,true],"r0_utf8":[null,"","B","B","","a",null,"é","","ab","","B","😀","a","B"],"x1":[null,true,true,true,true,true,true,true,null,null,null,null,null,null,null]}}},{"id":"random/0244","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_f64","type":"f64"},{"name":"l4_bool","type":"bool"},{"name":"l5_utf8","type":"utf8"}],"columns":{"l0_utf8":["c","ab",null,"c","é","","","a",null,null,"B","","","aa","é",null,null,null,null,"b","zz",null,null,null,"b","aa","","aa","😀","aa",null,"😀","","é",null,"😀","😀","b","a","ab",null,"😀","b",null,"zz","aa","é",null,"aa",null,"aa",null,"zz","b",null,"a","",null,"zz","é"],"l1_i32":[null,1,null,1,5,5,-2147483648,2147483647,null,3,null,-4,null,-5,1,-4,null,5,3,-4,null,null,0,null,1,3,-2147483648,-3,2,null,null,-5,null,-3,5,null,-1,null,null,null,3,-3,4,2,3,-5,null,-1,5,5,null,-5,-4,2,-4,null,null,463,3,-4],"l2_utf8":[null,"😀",null,"","","aa","aa","","é","é",null,"ab","b","ab","b","",null,"ab","😀","ab","",null,null,null,null,null,"aa",null,"aa","B",null,null,null,"aa","ab","aa","ab","é","é",null,"c",null,"B","é","b","aa",null,"c",null,"c",null,"c",null,"",null,"B","a","a","zz","zz"],"l3_f64":[4.0,null,3.0,4.0,2.0,4.0,39.481,null,null,2.0,3.0,-3.0,-12.291,-84.05,null,null,null,-58.788,-2.0,-15.772,-0.204,null,null,3.0,72.981,1.0,4.0,4.0,1.0,1.0,null,-1.0,"NaN",-3.0,null,75.078,0.0,null,2.0,45.946,3.0,4.0,68.869,-12.299,-3.0,-2.0,11.648,null,-61.455,null,0.0,76.771,-2.0,1.0,null,-93.902,4.0,null,99.297,"Infinity"],"l4_bool":[true,false,false,false,false,false,false,null,null,null,false,null,true,null,true,false,false,true,false,null,null,true,true,true,true,null,true,true,true,true,false,true,null,null,null,true,false,null,true,true,true,false,false,true,true,null,true,true,false,true,true,true,false,true,null,false,true,false,true,null],"l5_utf8":["ab","B","ab","B","c",null,"😀","","","","aa","é","a",null,null,"ab",null,"a","","B","b","b","B","b",null,"a","zz","a",null,"zz","ab","é","zz","B",null,"ab","😀",null,"a","é",null,null,"zz",null,"aa","a","B","",null,"a","","a","zz",null,null,"aa","zz","zz",null,"zz"]}}},"ops":[{"op":"filter","predicate":{"fn":"strStartsWith","args":[{"col":"l0_utf8"}],"prefix":""}},{"op":"withColumns","exprs":{"x1":{"fn":"div","args":[{"col":"l1_i32"},{"fn":"mul","args":[{"col":"l3_f64"},{"col":"l3_f64"}]}]}}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_f64","type":"f64"},{"name":"l4_bool","type":"bool"},{"name":"l5_utf8","type":"utf8"},{"name":"x1","type":"f64"}],"columns":{"l0_utf8":["c","ab","c","é","","","a","B","","","aa","é","b","zz","b","aa","","aa","😀","aa","😀","","é","😀","😀","b","a","ab","😀","b","zz","aa","é","aa","aa","zz","b","a","","zz","é"],"l1_i32":[null,1,1,5,5,-2147483648,2147483647,null,-4,null,-5,1,-4,null,1,3,-2147483648,-3,2,null,-5,null,-3,null,-1,null,null,null,-3,4,3,-5,null,5,null,-4,2,null,null,3,-4],"l2_utf8":[null,"😀","","","aa","aa","",null,"ab","b","ab","b","ab","",null,null,"aa",null,"aa","B",null,null,"aa","aa","ab","é","é",null,null,"B","b","aa",null,null,null,null,"","B","a","zz","zz"],"l3_f64":[4.0,null,4.0,2.0,4.0,39.481,null,3.0,-3.0,-12.291,-84.05,null,-15.772,-0.204,72.981,1.0,4.0,4.0,1.0,1.0,-1.0,"NaN",-3.0,75.078,0.0,null,2.0,45.946,4.0,68.869,-3.0,-2.0,11.648,-61.455,0.0,-2.0,1.0,-93.902,4.0,99.297,"Infinity"],"l4_bool":[true,false,false,false,false,false,null,false,null,true,null,true,null,null,true,null,true,true,true,true,true,null,null,true,false,null,true,true,false,false,true,null,true,false,true,false,true,false,true,true,null],"l5_utf8":["ab","B","B","c",null,"😀","","aa","é","a",null,null,"B","b",null,"a","zz","a",null,"zz","é","zz","B","ab","😀",null,"a","é",null,"zz","aa","a","B",null,"","zz",null,"aa","zz",null,"zz"],"x1":[null,null,0.0625,1.25,0.3125,-1377696.5699105745,null,null,-0.4444444444444444,null,-0.0007077739412498085,null,-0.016080015184679938,null,0.00018775018788579049,3.0,-134217728.0,-0.1875,2.0,null,-5.0,null,-0.3333333333333333,null,"-Infinity",null,null,null,-0.1875,0.0008433589097855181,0.3333333333333333,-1.25,null,0.0013239011466579232,null,-1.0,2.0,null,null,0.0003042628994195124,"-0"]}}},{"id":"random/0245","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_i32","type":"i32"},{"name":"l4_bool","type":"bool"},{"name":"l5_f64","type":"f64"},{"name":"l6_f64","type":"f64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_utf8":[],"l3_i32":[],"l4_bool":[],"l5_f64":[],"l6_f64":[]}}},"ops":[{"op":"groupBy","keys":["l3_i32"],"aggs":{"n0":{"agg":"len"},"a0_0":{"agg":"min","arg":{"col":"l0_utf8"}},"a0_1":{"agg":"min","arg":{"col":"l5_f64"}}}},{"op":"head","n":7}],"expected":{"fields":[{"name":"l3_i32","type":"i32"},{"name":"n0","type":"u32"},{"name":"a0_0","type":"utf8"},{"name":"a0_1","type":"f64"}],"columns":{"l3_i32":[],"n0":[],"a0_0":[],"a0_1":[]}}},{"id":"random/0246","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_f64","type":"f64"},{"name":"l6_i64","type":"i64"}],"columns":{"l0_utf8":["B","b","😀","B","zz"],"l1_i32":[null,-2147483648,-2,-2,1],"l2_utf8":["B","c","a","b","😀"],"l3_utf8":[null,"aa","c","zz","B"],"l4_utf8":["ab","b","😀","😀","c"],"l5_f64":[null,0.0,2.0,-45.254,18.6],"l6_i64":["-4611686018427387904",null,null,"2","9007199254740995"]}},"right":{"fields":[{"name":"l1_i32","type":"i32"},{"name":"r0_utf8","type":"utf8"}],"columns":{"r0_utf8":[],"l1_i32":[]}}},"ops":[{"op":"join","right":"right","on":["l1_i32"],"how":"anti"}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_f64","type":"f64"},{"name":"l6_i64","type":"i64"}],"columns":{"l0_utf8":["B","b","😀","B","zz"],"l1_i32":[null,-2147483648,-2,-2,1],"l2_utf8":["B","c","a","b","😀"],"l3_utf8":[null,"aa","c","zz","B"],"l4_utf8":["ab","b","😀","😀","c"],"l5_f64":[null,0.0,2.0,-45.254,18.6],"l6_i64":["-4611686018427387904",null,null,"2","9007199254740995"]}}},{"id":"random/0247","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_i32","type":"i32"},{"name":"l4_f64","type":"f64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_bool":[],"l3_i32":[],"l4_f64":[]}}},"ops":[{"op":"withColumns","exprs":{"x0":{"fn":"gte","args":[{"fn":"neg","args":[{"col":"l1_i32"}]},{"fn":"abs","args":[{"col":"l1_i32"}]}]}}},{"op":"filter","predicate":{"fn":"eq","args":[{"col":"l0_utf8"},{"lit":"aa"}]}},{"op":"head","n":0},{"op":"sort","keys":[{"expr":{"col":"l2_bool"},"descending":true}],"nullsLast":false}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_i32","type":"i32"},{"name":"l4_f64","type":"f64"},{"name":"x0","type":"bool"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_bool":[],"l3_i32":[],"l4_f64":[],"x0":[]}}},{"id":"random/0248","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_i64","type":"i64"}],"columns":{"l0_utf8":["aa","b","zz","B","a",null,"a","B","a","",null,"😀","ab","B","ab","a","é","ab",null,"ab","zz","a","aa",null,"é","b",null,"é",null,"c","aa","c","B","é","zz","c","ab",null,"ab","",null,"a","aa","zz",null,"ab","zz","B","zz","aa","ab","","a","ab","aa","zz","a","b","B","zz"],"l1_i32":[3,5,-2147483648,3,2,0,5,3,5,4,0,1,2,3,-3,5,1,-1,-3,-2,0,-1,-4,-2,-4,-2,5,null,0,4,3,-3,-4,4,null,-3,4,4,-3,-2,-4,-2147483648,5,-3,1,-1,-4,-5,1,0,4,1,4,4,0,1,3,0,-281,-4],"l2_f64":[-3.0,15.128,null,64.526,62.551,89.957,-77.7,48.174,-1.0,null,-19.471,43.356,-32.53,null,75.568,-2.0,34.511,80.155,-95.603,2.0,-4.0,4.0,2.0,"-0",-1.0,4.0,-89.485,-2.0,4.0,0.0,-75.144,1.595,null,90.841,20.587,-86.741,4.0,81.398,2.0,-2.0,-42.336,-31.887,null,-23.467,-2.0,"Infinity",-64.381,2.0,-31.847,4.0,-1.655,-3.0,null,1.0,null,-4.0,-3.0,0.0,5.333,73.821],"l3_i64":[null,"-4611686018427387904","-4611686018427387904",null,"-4611686018427387904","9007199254740993","9007199254740993","-1","9007199254740995","9007199254740994","9007199254740993","9007199254740994","-3","9007199254740993","5","-1","2","3","-3","-4611686018427387904","-4611686018427387904","9007199254740993","1","-4611686018427387904","9007199254740995","0","5","-4","-1",null,"9007199254740992","-5","9007199254740992","3","-2","9007199254740995","9007199254740994","9007199254740995","-4611686018427387904","-4611686018427387904","-5","2","9007199254740993","-4611686018427387904","9007199254740993","9007199254740992","4","9007199254740992","-1","9007199254740995","9007199254740994","2",null,"9007199254740994","9007199254740995","9007199254740995","3","-4","-4611686018427387904",null]}}},"ops":[{"op":"groupBy","keys":["l0_utf8"],"aggs":{"n0":{"agg":"len"},"a0_0":{"agg":"nUnique","arg":{"col":"l3_i64"}}}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"n0","type":"u32"},{"name":"a0_0","type":"u32"}],"columns":{"l0_utf8":["aa","b","zz","B","a",null,"","😀","ab","é","c"],"n0":[6,3,8,6,8,9,3,1,9,4,3],"a0_0":[5,3,6,5,7,7,3,1,6,4,3]}}},{"id":"random/0249","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_f64","type":"f64"},{"name":"l4_bool","type":"bool"},{"name":"l5_f64","type":"f64"},{"name":"l6_f64","type":"f64"}],"columns":{"l0_utf8":["a","aa","c","aa","","c","b","zz","a","a","é","c","ab","c","zz","B","zz","zz","B","zz"],"l1_i32":[-4,5,5,2,-2,5,4,-2,-1,244,4,1,4,1,2147483647,-4,-5,3,0,3],"l2_utf8":["b","B","aa","ab","b","zz","","b","c","😀","aa","B","a","😀","😀","😀","c","aa","zz","ab"],"l3_f64":[57.945,82.487,44.9,8.595,-1.0,47.409,2.0,-1.0,9.477,0.0,-3.0,3.0,4.0,96.591,-2.0,"NaN",-1.0,20.051,31.566,1.0],"l4_bool":[true,false,false,true,true,true,false,false,true,true,false,false,true,false,true,false,false,false,false,true],"l5_f64":[1.0,78.699,-64.51,-2.0,3.0,33.522,97.317,2.0,-99.269,82.531,2.0,-1.0,-66.606,2.0,-4.0,-4.0,"Infinity",-4.0,-4.0,-85.09],"l6_f64":[-4.0,93.604,-4.0,7.561,-54.185,3.0,1.0,2.0,-1.0,0.0,-61.743,-2.0,99.685,-4.334,-10.862,40.795,1.0,-75.639,-1.0,-2.0]}}},"ops":[{"op":"head","n":3}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_f64","type":"f64"},{"name":"l4_bool","type":"bool"},{"name":"l5_f64","type":"f64"},{"name":"l6_f64","type":"f64"}],"columns":{"l0_utf8":["a","aa","c"],"l1_i32":[-4,5,5],"l2_utf8":["b","B","aa"],"l3_f64":[57.945,82.487,44.9],"l4_bool":[true,false,false],"l5_f64":[1.0,78.699,-64.51],"l6_f64":[-4.0,93.604,-4.0]}}},{"id":"random/0250","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_f64","type":"f64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_i32":[],"l3_f64":[]}}},"ops":[{"op":"filter","predicate":{"fn":"isNull","args":[{"col":"l0_utf8"}]}},{"op":"sort","keys":[{"expr":{"col":"l2_i32"},"descending":true}],"nullsLast":true},{"op":"withColumns","exprs":{"x2":{"fn":"fillNull","args":[{"fn":"fillNull","args":[{"col":"l2_i32"},{"lit":1.5}]},{"lit":0}]}}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_f64","type":"f64"},{"name":"x2","type":"f64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_i32":[],"l3_f64":[],"x2":[]}}},{"id":"random/0251","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i64","type":"i64"},{"name":"l3_f64","type":"f64"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_i32","type":"i32"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_i64":[],"l3_f64":[],"l4_utf8":[],"l5_i32":[]}}},"ops":[{"op":"unique","columns":["l2_i64"]},{"op":"groupBy","keys":["l2_i64"],"aggs":{"n1":{"agg":"len"},"a1_0":{"agg":"min","arg":{"col":"l5_i32"}}}}],"expected":{"fields":[{"name":"l2_i64","type":"i64"},{"name":"n1","type":"u32"},{"name":"a1_0","type":"i32"}],"columns":{"l2_i64":[],"n1":[],"a1_0":[]}}},{"id":"random/0252","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_i64","type":"i64"},{"name":"l4_f64","type":"f64"},{"name":"l5_bool","type":"bool"}],"columns":{"l0_utf8":["ab"],"l1_i32":[-2],"l2_f64":[98.978],"l3_i64":["-4611686018427387904"],"l4_f64":[-18.877],"l5_bool":[true]}}},"ops":[{"op":"withColumns","exprs":{"x0":{"fn":"sub","args":[{"fn":"neg","args":[{"col":"l4_f64"}]},{"fn":"abs","args":[{"lit":-1}]}]}}},{"op":"withColumns","exprs":{"x1":{"fn":"gt","args":[{"fn":"add","args":[{"col":"l2_f64"},{"col":"l4_f64"}]},{"col":"l4_f64"}]}}},{"op":"withColumns","exprs":{"x2":{"fn":"gte","args":[{"fn":"sub","args":[{"col":"l4_f64"},{"col":"x0"}]},{"col":"x0"}]}}},{"op":"groupBy","keys":["l0_utf8"],"aggs":{"n3":{"agg":"len"},"a3_0":{"agg":"first","arg":{"col":"x2"}},"a3_1":{"agg":"first","arg":{"col":"l2_f64"}}}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"n3","type":"u32"},{"name":"a3_0","type":"bool"},{"name":"a3_1","type":"f64"}],"columns":{"l0_utf8":["ab"],"n3":[1],"a3_0":[false],"a3_1":[98.978]}}},{"id":"random/0253","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_utf8","type":"utf8"}],"columns":{"l0_utf8":["c"],"l1_i32":[4],"l2_utf8":["b"],"l3_utf8":["a"]}}},"ops":[{"op":"groupBy","keys":["l0_utf8"],"aggs":{"n0":{"agg":"len"},"a0_0":{"agg":"count","arg":{"col":"l3_utf8"}},"a0_1":{"agg":"max","arg":{"col":"l2_utf8"}},"a0_2":{"agg":"min","arg":{"col":"l3_utf8"}}}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"n0","type":"u32"},{"name":"a0_0","type":"u32"},{"name":"a0_1","type":"utf8"},{"name":"a0_2","type":"utf8"}],"columns":{"l0_utf8":["c"],"n0":[1],"a0_0":[1],"a0_1":["b"],"a0_2":["a"]}}},{"id":"random/0254","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_utf8","type":"utf8"}],"columns":{"l0_utf8":["😀","c","ab","a","ab"],"l1_i32":[-3,-2,0,2,-4],"l2_f64":[-75.476,2.0,2.0,-83.161,1.0],"l3_utf8":["a","aa","B","aa","😀"]}}},"ops":[{"op":"head","n":0},{"op":"sort","keys":[{"expr":{"col":"l2_f64"},"descending":false},{"expr":{"col":"l1_i32"},"descending":true}],"nullsLast":true},{"op":"head","n":4},{"op":"head","n":3}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_utf8","type":"utf8"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_f64":[],"l3_utf8":[]}}},{"id":"random/0255","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_f64","type":"f64"},{"name":"l4_i32","type":"i32"},{"name":"l5_utf8","type":"utf8"},{"name":"l6_bool","type":"bool"}],"columns":{"l0_utf8":["😀","é",null,"aa","B","ab","😀","a","B",null,"é","c","c","","é","",null,"c","😀","zz"],"l1_i32":[2,null,-1,-1,5,-4,null,-1,-2,-1,-4,null,null,null,-1,null,null,4,1,0],"l2_bool":[null,false,true,true,false,null,false,false,true,false,true,null,null,false,true,false,true,null,true,true],"l3_f64":[-69.949,-1.0,null,-13.598,83.66,65.863,-3.0,null,-2.0,"NaN",null,null,null,63.318,-79.414,null,4.0,3.0,60.217,-4.0],"l4_i32":[5,-3,-4,-5,4,-2,5,-2,null,4,-3,2147483647,-2,null,2147483647,null,-3,4,-2,-2],"l5_utf8":["😀","ab","a","😀","ab","aa","b",null,null,null,null,"é","","zz","aa","😀","zz","","😀",null],"l6_bool":[null,true,false,true,true,false,true,false,false,false,true,true,false,null,true,null,false,null,false,false]}}},"ops":[{"op":"sort","keys":[{"expr":{"col":"l2_bool"},"descending":true},{"expr":{"col":"l5_utf8"},"descending":true}],"nullsLast":false},{"op":"groupBy","keys":["l0_utf8"],"aggs":{"n1":{"agg":"len"},"a1_0":{"agg":"nUnique","arg":{"col":"l1_i32"}}}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"n1","type":"u32"},{"name":"a1_0","type":"u32"}],"columns":{"l0_utf8":["😀","c","ab","B","é","zz","aa",null,"a",""],"n1":[3,3,1,2,3,1,1,3,1,2],"a1_0":[3,2,1,2,3,1,1,2,1,1]}}},{"id":"random/0256","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_i64","type":"i64"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_utf8","type":"utf8"}],"columns":{"l0_utf8":["ab"],"l1_i32":[-1],"l2_utf8":[null],"l3_i64":["3"],"l4_utf8":["aa"],"l5_utf8":["aa"]}},"right":{"fields":[{"name":"l4_utf8","type":"utf8"},{"name":"r0_bool","type":"bool"}],"columns":{"r0_bool":[],"l4_utf8":[]}}},"ops":[{"op":"join","right":"right","on":["l4_utf8"],"how":"inner"}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_i64","type":"i64"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_utf8","type":"utf8"},{"name":"r0_bool","type":"bool"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_utf8":[],"l3_i64":[],"l4_utf8":[],"l5_utf8":[],"r0_bool":[]}}},{"id":"random/0257","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_f64","type":"f64"},{"name":"l4_i64","type":"i64"},{"name":"l5_f64","type":"f64"},{"name":"l6_f64","type":"f64"}],"columns":{"l0_utf8":["B",null,"B","ab","zz","ab","é","😀",null,"B","zz","é",null,"B","aa","aa","aa","ab","c","a"],"l1_i32":[5,-1,5,-2147483648,-4,-2,-1,5,0,-1,-2,4,4,-2147483648,-1,1,1,3,3,5],"l2_i32":[2,4,3,3,-4,5,null,-2,1,0,null,-4,2,null,-3,0,5,3,0,-795],"l3_f64":[-83.741,"NaN",0.0,-2.0,0.0,0.0,-5.782,1.0,61.608,-1.0,"Infinity",-1.0,-32.064,-4.0,1.0,0.0,-82.663,-84.411,-2.0,31.84],"l4_i64":["-3","9007199254740993","-4611686018427387904","-4",null,null,null,"-4611686018427387904","-4611686018427387904","9007199254740994",null,"-4611686018427387904","-4611686018427387904","-4611686018427387904","9007199254740994","9007199254740995","3",null,"9007199254740993","0"],"l5_f64":[38.438,3.0,-39.995,-71.935,-1.0,-72.475,null,-83.312,"NaN",-15.703,52.806,8.177,3.0,-69.348,null,null,-80.234,-2.0,62.93,null],"l6_f64":["NaN",71.247,6.244,49.364,-4.0,-43.941,null,-4.909,30.102,4.0,null,-3.0,-2.0,51.341,-2.0,-47.749,3.0,-99.466,14.958,66.319]}}},"ops":[{"op":"sort","keys":[{"expr":{"col":"l4_i64"},"descending":false},{"expr":{"col":"l0_utf8"},"descending":false}],"nullsLast":false},{"op":"unique"},{"op":"withColumns","exprs":{"x2":{"lit":-1}}},{"op":"filter","predicate":{"fn":"neq","args":[{"col":"l5_f64"},{"col":"l3_f64"}]}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_f64","type":"f64"},{"name":"l4_i64","type":"i64"},{"name":"l5_f64","type":"f64"},{"name":"l6_f64","type":"f64"},{"name":"x2","type":"i32"}],"columns":{"l0_utf8":["ab","ab","zz","zz",null,null,"B","B","é","😀","ab","B","aa",null,"c","B"],"l1_i32":[-2,3,-4,-2,0,4,5,-2147483648,4,5,-2147483648,5,1,-1,3,-1],"l2_i32":[5,3,-4,null,1,2,3,null,-4,-2,3,2,5,4,0,0],"l3_f64":[0.0,-84.411,0.0,"Infinity",61.608,-32.064,0.0,-4.0,-1.0,1.0,-2.0,-83.741,-82.663,"NaN",-2.0,-1.0],"l4_i64":[null,null,null,null,"-4611686018427387904","-4611686018427387904","-4611686018427387904","-4611686018427387904","-4611686018427387904","-4611686018427387904","-4","-3","3","9007199254740993","9007199254740993","9007199254740994"],"l5_f64":[-72.475,-2.0,-1.0,52.806,"NaN",3.0,-39.995,-69.348,8.177,-83.312,-71.935,38.438,-80.234,3.0,62.93,-15.703],"l6_f64":[-43.941,-99.466,-4.0,null,30.102,-2.0,6.244,51.341,-3.0,-4.909,49.364,"NaN",3.0,71.247,14.958,4.0],"x2":[-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1]}}},{"id":"random/0258","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_utf8","type":"utf8"}],"columns":{"l0_utf8":[null],"l1_i32":[3],"l2_f64":["NaN"],"l3_utf8":["B"]}}},"ops":[{"op":"withColumns","exprs":{"x0":{"col":"l2_f64"}}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_utf8","type":"utf8"},{"name":"x0","type":"f64"}],"columns":{"l0_utf8":[null],"l1_i32":[3],"l2_f64":["NaN"],"l3_utf8":["B"],"x0":["NaN"]}}},{"id":"random/0259","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_i64","type":"i64"}],"columns":{"l0_utf8":["a","c","b","ab","a"],"l1_i32":[-5,3,3,4,2],"l2_utf8":["c","aa","b","aa",""],"l3_i64":["-5","-1","-2","-4611686018427387904","1"]}}},"ops":[{"op":"head","n":10},{"op":"groupBy","keys":["l2_utf8"],"aggs":{"n1":{"agg":"len"},"a1_0":{"agg":"min","arg":{"col":"l0_utf8"}},"a1_1":{"agg":"max","arg":{"col":"l1_i32"}},"a1_2":{"agg":"median","arg":{"col":"l1_i32"}}}},{"op":"sort","keys":[{"expr":{"col":"l2_utf8"},"descending":false}],"nullsLast":true},{"op":"withColumns","exprs":{"x3":{"fn":"neg","args":[{"fn":"sub","args":[{"col":"a1_1"},{"col":"a1_1"}]}]}}}],"expected":{"fields":[{"name":"l2_utf8","type":"utf8"},{"name":"n1","type":"u32"},{"name":"a1_0","type":"utf8"},{"name":"a1_1","type":"i32"},{"name":"a1_2","type":"f64"},{"name":"x3","type":"i32"}],"columns":{"l2_utf8":["","aa","b","c"],"n1":[1,2,1,1],"a1_0":["a","ab","b","a"],"a1_1":[2,4,3,-5],"a1_2":[2.0,3.5,3.0,-5.0],"x3":[0,0,0,0]}}},{"id":"random/0260","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i64","type":"i64"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_utf8","type":"utf8"}],"columns":{"l0_utf8":["c"],"l1_i32":[-417],"l2_i64":["9007199254740995"],"l3_utf8":["B"],"l4_utf8":["B"],"l5_utf8":["aa"]}},"right":{"fields":[{"name":"l4_utf8","type":"utf8"},{"name":"r0_f64","type":"f64"}],"columns":{"r0_f64":[84.015,-30.607,1.0],"l4_utf8":["B","B","B"]}}},"ops":[{"op":"join","right":"right","on":["l4_utf8"],"how":"semi"},{"op":"head","n":5},{"op":"groupBy","keys":["l4_utf8"],"aggs":{"n2":{"agg":"len"},"a2_0":{"agg":"count","arg":{"col":"l1_i32"}}}},{"op":"groupBy","keys":["n2"],"aggs":{"n3":{"agg":"len"},"a3_0":{"agg":"min","arg":{"col":"l4_utf8"}},"a3_1":{"agg":"last","arg":{"col":"a2_0"}}}}],"expected":{"fields":[{"name":"n2","type":"u32"},{"name":"n3","type":"u32"},{"name":"a3_0","type":"utf8"},{"name":"a3_1","type":"u32"}],"columns":{"n2":[1],"n3":[1],"a3_0":["B"],"a3_1":[1]}}},{"id":"random/0261","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_bool","type":"bool"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_i64","type":"i64"}],"columns":{"l0_utf8":["ab","a","b",null,"b","😀","zz","zz","é","aa","ab","b","ab","a","aa","b","é",null,"é","😀"],"l1_i32":[-3,-4,-1,4,387,-4,-2,-2,4,2,1,-5,-4,5,-4,2,4,-2,1,224],"l2_f64":[67.664,45.897,0.0,null,-1.0,3.0,-1.0,-2.0,2.0,-2.0,58.538,3.0,-81.989,50.055,null,-13.973,44.091,"NaN",2.0,-47.32],"l3_bool":[false,false,true,false,true,true,false,true,false,false,null,false,true,true,false,true,null,true,null,true],"l4_utf8":["B","","c","😀","é","aa",null,"é",null,"é","zz","b","ab","b","zz",null,null,"","b","a"],"l5_i64":["9007199254740992","2",null,"-4611686018427387904","1","-4611686018427387904",null,"9007199254740992","-1","9007199254740992","-2","-2","9007199254740993","-4611686018427387904","9007199254740992","-4611686018427387904","9007199254740995","-4611686018427387904","2",null]}}},"ops":[{"op":"head","n":0},{"op":"sort","keys":[{"expr":{"col":"l1_i32"},"descending":false},{"expr":{"col":"l5_i64"},"descending":false}],"nullsLast":false}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_bool","type":"bool"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_i64","type":"i64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_f64":[],"l3_bool":[],"l4_utf8":[],"l5_i64":[]}}},{"id":"random/0262","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_f64","type":"f64"}],"columns":{"l0_utf8":[null,"c","b","ab",""],"l1_i32":[-5,261,-4,3,2],"l2_utf8":["zz","é","B","é",""],"l3_f64":[-99.638,-73.602,"NaN",-1.0,-1.0]}},"right":{"fields":[{"name":"l2_utf8","type":"utf8"},{"name":"r0_bool","type":"bool"}],"columns":{"r0_bool":[],"l2_utf8":[]}}},"ops":[{"op":"sort","keys":[{"expr":{"col":"l0_utf8"},"descending":false}],"nullsLast":true},{"op":"join","right":"right","on":["l2_utf8"],"how":"full"},{"op":"unique","columns":["l1_i32"]},{"op":"filter","predicate":{"fn":"or","args":[{"fn":"lte","args":[{"col":"l3_f64"},{"col":"l1_i32"}]},{"fn":"isNotNull","args":[{"col":"l0_utf8"}]}]}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_f64","type":"f64"},{"name":"r0_bool","type":"bool"}],"columns":{"l0_utf8":["","ab","b","c",null],"l1_i32":[2,3,-4,261,-5],"l2_utf8":["","é","B","é","zz"],"l3_f64":[-1.0,-1.0,"NaN",-73.602,-99.638],"r0_bool":[null,null,null,null,null]}}},{"id":"random/0263","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_f64","type":"f64"},{"name":"l4_f64","type":"f64"},{"name":"l5_utf8","type":"utf8"}],"columns":{"l0_utf8":["😀","é","aa","","B"],"l1_i32":[0,2,1,-5,-3],"l2_bool":[true,true,null,false,null],"l3_f64":[-72.163,-0.197,-1.0,31.045,3.0],"l4_f64":[28.357,71.659,"Infinity",-22.611,33.707],"l5_utf8":["",null,"B","é","c"]}}},"ops":[{"op":"groupBy","keys":["l0_utf8"],"aggs":{"n0":{"agg":"len"},"a0_0":{"agg":"median","arg":{"col":"l4_f64"}},"a0_1":{"agg":"min","arg":{"col":"l5_utf8"}},"a0_2":{"agg":"min","arg":{"col":"l5_utf8"}}}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"n0","type":"u32"},{"name":"a0_0","type":"f64"},{"name":"a0_1","type":"utf8"},{"name":"a0_2","type":"utf8"}],"columns":{"l0_utf8":["😀","é","aa","","B"],"n0":[1,1,1,1,1],"a0_0":[28.357,71.659,"Infinity",-22.611,33.707],"a0_1":["",null,"B","é","c"],"a0_2":["",null,"B","é","c"]}}},{"id":"random/0264","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_utf8","type":"utf8"}],"columns":{"l0_utf8":[null,null,"ab","B","zz"],"l1_i32":[-879,4,null,-4,-2],"l2_utf8":[null,null,"aa","ab","c"],"l3_utf8":[null,"c","B","a","B"],"l4_utf8":[null,null,null,"zz","aa"]}}},"ops":[{"op":"unique"},{"op":"filter","predicate":{"fn":"lt","args":[{"col":"l2_utf8"},{"lit":"ab"}]}},{"op":"filter","predicate":{"fn":"isNotNull","args":[{"col":"l0_utf8"}]}},{"op":"unique","columns":["l0_utf8"]}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_utf8","type":"utf8"}],"columns":{"l0_utf8":["ab"],"l1_i32":[null],"l2_utf8":["aa"],"l3_utf8":["B"],"l4_utf8":[null]}}},{"id":"random/0265","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_i64","type":"i64"},{"name":"l5_bool","type":"bool"},{"name":"l6_bool","type":"bool"}],"columns":{"l0_utf8":["","é","a","B","😀"],"l1_i32":[-2147483648,4,-3,-2,1],"l2_f64":[-94.735,-4.0,-93.718,-5.125,-6.849],"l3_utf8":["aa","😀","aa","aa","zz"],"l4_i64":["-4611686018427387904","9007199254740993","9007199254740993","4","9007199254740992"],"l5_bool":[true,false,false,false,true],"l6_bool":[false,false,true,true,false]}}},"ops":[{"op":"filter","predicate":{"fn":"and","args":[{"fn":"eq","args":[{"lit":2},{"col":"l1_i32"}]},{"fn":"isNull","args":[{"col":"l1_i32"}]}]}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_i64","type":"i64"},{"name":"l5_bool","type":"bool"},{"name":"l6_bool","type":"bool"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_f64":[],"l3_utf8":[],"l4_i64":[],"l5_bool":[],"l6_bool":[]}}},{"id":"random/0266","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_i64","type":"i64"},{"name":"l4_i64","type":"i64"},{"name":"l5_utf8","type":"utf8"},{"name":"l6_i64","type":"i64"}],"columns":{"l0_utf8":["c",null,null,null,"b"],"l1_i32":[null,1,-5,-2147483648,-2],"l2_bool":[true,false,true,true,false],"l3_i64":["-4611686018427387904",null,"3","-4611686018427387904","-4611686018427387904"],"l4_i64":["-1","-4611686018427387904","9007199254740995","2","-4"],"l5_utf8":["😀","ab","","é","zz"],"l6_i64":["-1",null,"-4611686018427387904",null,"-4611686018427387904"]}}},"ops":[{"op":"filter","predicate":{"fn":"lte","args":[{"fn":"div","args":[{"col":"l1_i32"},{"col":"l1_i32"}]},{"fn":"sub","args":[{"col":"l1_i32"},{"col":"l1_i32"}]}]}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_i64","type":"i64"},{"name":"l4_i64","type":"i64"},{"name":"l5_utf8","type":"utf8"},{"name":"l6_i64","type":"i64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_bool":[],"l3_i64":[],"l4_i64":[],"l5_utf8":[],"l6_i64":[]}}},{"id":"random/0267","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i64","type":"i64"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_i64","type":"i64"},{"name":"l6_utf8","type":"utf8"}],"columns":{"l0_utf8":["a","a","é","ab","B","c","","zz","c","a","b","b","","b","","a","B",null,"","c",null,"zz","aa","","","b","B","b",null,"b","a","aa","zz","😀","","ab","é","zz","zz","ab","zz","c","B","a","zz","aa","","b","b","ab",null,"b","","aa","é","😀","zz","a","","B"],"l1_i32":[1,0,-2147483648,1,3,-4,2,-5,-5,4,5,-98,-3,0,-3,3,4,5,-2147483648,4,3,-1,-1,2,-1,-4,-4,-5,0,null,5,2,0,2,4,-2,null,2,4,5,5,-2,-3,5,2,0,-4,-3,-5,5,1,-2,2,-5,5,5,2,-4,-1,2147483647],"l2_i64":["3","-4611686018427387904","-3","-4611686018427387904","-4611686018427387904","9007199254740993","2","9007199254740994","9007199254740994","-4611686018427387904","9007199254740993","-4611686018427387904","9007199254740994","9007199254740993","9007199254740993","9007199254740992","-5",null,"-2","-2","9007199254740993","-4611686018427387904","9007199254740993","-4","-4611686018427387904","-4611686018427387904",null,"-4611686018427387904","-4611686018427387904","9007199254740992","9007199254740995","9007199254740993","-4611686018427387904","-4611686018427387904","-4611686018427387904","-5","-4611686018427387904","9007199254740993","9007199254740995","-4611686018427387904","9007199254740994","4",null,"9007199254740995","9007199254740995","-3","9007199254740993","9007199254740994","9007199254740992","-4611686018427387904","9007199254740993",null,"9007199254740992","9007199254740993","-1","-4","-1","-4611686018427387904","9007199254740995","-4611686018427387904"],"l3_utf8":["aa","😀","c","😀",null,"b","aa","😀","b","c","","😀","zz","a","B","b","B","é","b","ab","ab","b","ab","B","aa","B","B",null,"",null,null,"b","😀","ab","zz","b","B","","ab",null,"ab","a",null,"B","b","zz","é","a","b",null,"😀","😀","c","a","aa","B","zz","","B",null],"l4_utf8":["ab",null,"b","","zz","a","B","😀",null,"","aa","",null,"zz",null,"a",null,"zz","B",null,null,"ab","zz","ab","😀","😀","b","ab","ab","c","c",null,"a","B",null,"B","ab","zz","é","B","a","c","zz",null,"b","c",null,"é","B","ab","aa","zz","aa","é","aa","ab","a","B","c","a"],"l5_i64":["3","2","4","-5","-4611686018427387904","-5","9007199254740993","-2","-1","-4611686018427387904","9007199254740995","-4611686018427387904","0","9007199254740994","-4611686018427387904","3","9007199254740993","1",null,"-4611686018427387904","-4611686018427387904","9007199254740995","9007199254740995","-4611686018427387904","-4611686018427387904",null,"-4611686018427387904","-4611686018427387904","-5","-4611686018427387904","-4611686018427387904","-3","-1","9007199254740993","-4611686018427387904","-5","-2","-4611686018427387904","-2","4","-4611686018427387904","9007199254740995","-4611686018427387904","9007199254740995","9007199254740994","9007199254740995","9007199254740993","-5","-1","-4611686018427387904","-2","9007199254740993","4",null,"9007199254740992","9007199254740993","1","-4611686018427387904","-2","-4611686018427387904"],"l6_utf8":["😀",null,"zz","é","é","zz","c","aa","c","zz",null,"c",null,"c",null,"","aa","ab","b","","😀","zz","😀",null,"a","zz",null,"é","ab",null,"c","zz","é","","B","c","c","","ab","",null,"b","aa","b","é","b","aa","","","ab","aa","é","ab",null,"ab","zz","😀","aa","é","B"]}}},"ops":[{"op":"groupBy","keys":["l0_utf8"],"aggs":{"n0":{"agg":"len"},"a0_0":{"agg":"nUnique","arg":{"col":"l2_i64"}}}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"n0","type":"u32"},{"name":"a0_0","type":"u32"}],"columns":{"l0_utf8":["a","é","ab","B","c","","zz","b",null,"aa","😀"],"n0":[7,3,4,5,4,10,8,9,4,4,2],"a0_0":[4,3,2,3,4,8,5,5,3,2,2]}}},{"id":"random/0268","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i64","type":"i64"},{"name":"l3_i32","type":"i32"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_i64":[],"l3_i32":[]}}},"ops":[{"op":"withColumns","exprs":{"x0":{"col":"l3_i32"}}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i64","type":"i64"},{"name":"l3_i32","type":"i32"},{"name":"x0","type":"i32"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_i64":[],"l3_i32":[],"x0":[]}}},{"id":"random/0269","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_i64","type":"i64"},{"name":"l4_bool","type":"bool"}],"columns":{"l0_utf8":[null,"ab",null,"","ab"],"l1_i32":[-5,null,4,2,2],"l2_utf8":["😀","😀",null,null,"B"],"l3_i64":[null,"-4611686018427387904","9007199254740995","2","9007199254740993"],"l4_bool":[true,null,false,false,false]}}},"ops":[{"op":"head","n":1},{"op":"filter","predicate":{"fn":"lte","args":[{"lit":-1},{"col":"l1_i32"}]}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_i64","type":"i64"},{"name":"l4_bool","type":"bool"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_utf8":[],"l3_i64":[],"l4_bool":[]}}},{"id":"random/0270","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_bool","type":"bool"},{"name":"l4_f64","type":"f64"},{"name":"l5_utf8","type":"utf8"},{"name":"l6_f64","type":"f64"}],"columns":{"l0_utf8":[null,"😀","é","zz","é"],"l1_i32":[null,-3,3,3,2],"l2_utf8":[null,"ab","ab","é","😀"],"l3_bool":[false,false,true,false,false],"l4_f64":[null,-62.83,"NaN",0.0,null],"l5_utf8":["aa","é",null,"c",""],"l6_f64":["NaN",-4.0,-30.748,-4.0,2.0]}}},"ops":[{"op":"filter","predicate":{"fn":"strStartsWith","args":[{"col":"l0_utf8"}],"prefix":""}},{"op":"withColumns","exprs":{"x1":{"lit":2}}},{"op":"withColumns","exprs":{"x2":{"col":"l1_i32"}}},{"op":"groupBy","keys":["l2_utf8"],"aggs":{"n3":{"agg":"len"},"a3_0":{"agg":"count","arg":{"col":"l5_utf8"}}}}],"expected":{"fields":[{"name":"l2_utf8","type":"utf8"},{"name":"n3","type":"u32"},{"name":"a3_0","type":"u32"}],"columns":{"l2_utf8":["ab","é","😀"],"n3":[2,1,1],"a3_0":[1,1,1]}}},{"id":"random/0271","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_bool","type":"bool"},{"name":"l4_i64","type":"i64"}],"columns":{"l0_utf8":["aa",null,"😀","é","ab","aa","ab","ab",null,"a",null,"a","b","B","","ab",null,"a",null,"zz"],"l1_i32":[null,-1,null,2,null,2,-5,null,null,4,-1,null,null,-1,-5,-4,-3,null,-5,0],"l2_f64":[null,88.906,0.0,1.131,29.86,24.466,null,-74.316,2.0,-61.404,null,-50.65,null,-33.186,-99.577,0.0,-2.0,null,-97.617,-50.398],"l3_bool":[true,false,true,true,true,false,true,null,false,false,false,true,false,false,false,null,true,true,null,true],"l4_i64":[null,null,null,"9007199254740994","9007199254740993",null,"1","-4611686018427387904","0",null,"9007199254740994","9007199254740992","-4611686018427387904","9007199254740994","9007199254740995","9007199254740994","-4611686018427387904","-4611686018427387904",null,"9007199254740992"]}}},"ops":[{"op":"sort","keys":[{"expr":{"col":"l3_bool"},"descending":false},{"expr":{"col":"l2_f64"},"descending":false}],"nullsLast":false}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_bool","type":"bool"},{"name":"l4_i64","type":"i64"}],"columns":{"l0_utf8":[null,"ab","ab",null,"b","","a","B",null,"aa",null,"aa","ab","a","a","zz",null,"😀","é","ab"],"l1_i32":[-5,null,-4,-1,null,-5,4,-1,null,2,-1,null,-5,null,null,0,-3,null,2,null],"l2_f64":[-97.617,-74.316,0.0,null,null,-99.577,-61.404,-33.186,2.0,24.466,88.906,null,null,null,-50.65,-50.398,-2.0,0.0,1.131,29.86],"l3_bool":[null,null,null,false,false,false,false,false,false,false,false,true,true,true,true,true,true,true,true,true],"l4_i64":[null,"-4611686018427387904","9007199254740994","9007199254740994","-4611686018427387904","9007199254740995",null,"9007199254740994","0",null,null,null,"1","-4611686018427387904","9007199254740992","9007199254740992","-4611686018427387904",null,"9007199254740994","9007199254740993"]}}},{"id":"random/0272","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_utf8","type":"utf8"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_i32":[],"l3_utf8":[]}}},"ops":[{"op":"filter","predicate":{"fn":"and","args":[{"fn":"lte","args":[{"col":"l1_i32"},{"col":"l1_i32"}]},{"fn":"gte","args":[{"col":"l1_i32"},{"col":"l1_i32"}]}]}},{"op":"filter","predicate":{"fn":"neq","args":[{"fn":"mod","args":[{"col":"l2_i32"},{"col":"l1_i32"}]},{"fn":"mul","args":[{"col":"l1_i32"},{"lit":0.5}]}]}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_utf8","type":"utf8"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_i32":[],"l3_utf8":[]}}},{"id":"random/0273","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_i64","type":"i64"},{"name":"l4_f64","type":"f64"},{"name":"l5_i32","type":"i32"}],"columns":{"l0_utf8":["c","","zz","B",null],"l1_i32":[-2,2,null,5,4],"l2_utf8":["ab","ab","b","c","😀"],"l3_i64":["-4611686018427387904","-4","-4611686018427387904","-4611686018427387904","9007199254740995"],"l4_f64":[26.061,1.0,80.89,80.11,-93.81],"l5_i32":[-1,null,null,0,null]}},"right":{"fields":[{"name":"l1_i32","type":"i32"},{"name":"r0_utf8","type":"utf8"}],"columns":{"r0_utf8":[],"l1_i32":[]}}},"ops":[{"op":"join","right":"right","on":["l1_i32"],"how":"full"},{"op":"head","n":10}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_i64","type":"i64"},{"name":"l4_f64","type":"f64"},{"name":"l5_i32","type":"i32"},{"name":"r0_utf8","type":"utf8"}],"columns":{"l0_utf8":["c","","zz","B",null],"l1_i32":[-2,2,null,5,4],"l2_utf8":["ab","ab","b","c","😀"],"l3_i64":["-4611686018427387904","-4","-4611686018427387904","-4611686018427387904","9007199254740995"],"l4_f64":[26.061,1.0,80.89,80.11,-93.81],"l5_i32":[-1,null,null,0,null],"r0_utf8":[null,null,null,null,null]}}},{"id":"random/0274","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_bool","type":"bool"}],"columns":{"l0_utf8":["c"],"l1_i32":[-3],"l2_f64":[null],"l3_bool":[null]}}},"ops":[{"op":"groupBy","keys":["l1_i32"],"aggs":{"n0":{"agg":"len"},"a0_0":{"agg":"last","arg":{"col":"l2_f64"}}}}],"expected":{"fields":[{"name":"l1_i32","type":"i32"},{"name":"n0","type":"u32"},{"name":"a0_0","type":"f64"}],"columns":{"l1_i32":[-3],"n0":[1],"a0_0":[null]}}},{"id":"random/0275","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i64","type":"i64"},{"name":"l3_i64","type":"i64"},{"name":"l4_i32","type":"i32"},{"name":"l5_i32","type":"i32"},{"name":"l6_f64","type":"f64"}],"columns":{"l0_utf8":["","ab","b","a","b"],"l1_i32":[1,0,-5,-4,-5],"l2_i64":["9007199254740995","-4611686018427387904","-1","-4611686018427387904","9007199254740992"],"l3_i64":["9007199254740993","-4611686018427387904","9007199254740994","-2","9007199254740992"],"l4_i32":[3,-2147483648,2,3,-3],"l5_i32":[5,-5,-2147483648,1,5],"l6_f64":[3.0,-3.0,-77.309,-64.987,-2.0]}},"right":{"fields":[{"name":"l4_i32","type":"i32"},{"name":"r0_f64","type":"f64"}],"columns":{"r0_f64":[-18.789,-68.118,4.0],"l4_i32":[3,-2147483648,2]}}},"ops":[{"op":"join","right":"right","on":["l4_i32"],"how":"left"}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i64","type":"i64"},{"name":"l3_i64","type":"i64"},{"name":"l4_i32","type":"i32"},{"name":"l5_i32","type":"i32"},{"name":"l6_f64","type":"f64"},{"name":"r0_f64","type":"f64"}],"columns":{"l0_utf8":["","ab","b","a","b"],"l1_i32":[1,0,-5,-4,-5],"l2_i64":["9007199254740995","-4611686018427387904","-1","-4611686018427387904","9007199254740992"],"l3_i64":["9007199254740993","-4611686018427387904","9007199254740994","-2","9007199254740992"],"l4_i32":[3,-2147483648,2,3,-3],"l5_i32":[5,-5,-2147483648,1,5],"l6_f64":[3.0,-3.0,-77.309,-64.987,-2.0],"r0_f64":[-18.789,-68.118,4.0,-18.789,null]}}},{"id":"random/0276","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i64","type":"i64"},{"name":"l3_i32","type":"i32"},{"name":"l4_f64","type":"f64"},{"name":"l5_i32","type":"i32"},{"name":"l6_i32","type":"i32"}],"columns":{"l0_utf8":["😀","c","aa","😀","","c","ab","ab","","c","","aa","aa","zz","zz","B","ab","aa","B","aa","zz","zz","zz","a","aa","ab","ab","c","ab","zz","b","aa","zz","zz","ab","B","zz","b","a","","B","a","b","b","😀","b","","é","😀","aa","c","b","é","c","b","aa","ab","é","c","é"],"l1_i32":[3,-4,0,-2,0,423,-1,-3,4,4,3,5,-5,3,4,-5,-2147483648,-2,5,-2,2,-5,5,-2147483648,-4,-2,4,4,0,0,3,-2147483648,1,5,-1,-3,2,-4,1,-3,-5,4,-1,-2,3,1,4,0,-4,3,4,-2,0,-2,5,473,0,611,-4,5],"l2_i64":["9007199254740995","-4611686018427387904","9007199254740992","-4611686018427387904","-4611686018427387904","9007199254740993","9007199254740992","9007199254740992","9007199254740995","4","-4611686018427387904","-4611686018427387904","-4611686018427387904","-4611686018427387904","-4611686018427387904","9007199254740994","-4611686018427387904","-4611686018427387904","-2","9007199254740992","3","9007199254740992","-1","0","0","-4611686018427387904","-5","0","9007199254740995","-4611686018427387904","9007199254740994","-4611686018427387904","9007199254740995","-4611686018427387904","-4611686018427387904","-4611686018427387904","-3","5","9007199254740992","3","-4611686018427387904","9007199254740995","-4611686018427387904","5","9007199254740992","5","-1","9007199254740994","-4611686018427387904","-4611686018427387904","-5","-4611686018427387904","9007199254740992","3","9007199254740994","9007199254740994","-4611686018427387904","-4611686018427387904","9007199254740992","9007199254740992"],"l3_i32":[5,-4,0,-2,-3,0,-5,-5,5,-2147483648,-3,2,-5,-1,-3,1,-5,-3,-2147483648,-4,3,2,-4,-2,4,4,2,3,-5,-4,1,-1,-1,-2,-4,2,-1,5,-4,-3,2147483647,-1,0,-3,3,2,2,2,-2,3,-4,2,5,-3,2,-5,2147483647,3,-3,-3],"l4_f64":[1.0,-58.91,2.0,0.0,-3.0,2.0,1.0,45.507,-45.335,0.0,4.0,2.0,"NaN",-1.0,46.833,-70.909,75.758,-2.0,4.0,1.0,69.296,-15.606,42.874,-1.0,-6.354,50.366,-8.082,9.719,-10.494,-54.182,48.055,4.0,"Infinity",59.099,-2.0,2.0,-2.0,30.238,74.281,34.887,-51.918,79.905,0.0,26.671,-48.914,84.759,0.0,-3.0,2.0,4.086,-24.467,-48.825,80.735,-3.0,-28.107,14.719,71.93,91.071,2.0,2.0],"l5_i32":[-3,0,-3,4,-3,3,-1,-1,3,-3,4,5,1,2,-1,-5,4,2,5,-1,1,-1,913,4,2,-1,-4,-5,3,-781,-3,0,4,-2,-4,-3,5,-3,-5,-1,-2,4,4,4,-338,-3,-1,2,3,1,1,1,-4,2147483647,5,3,1,3,2,3],"l6_i32":[2,-3,1,-2147483648,2147483647,4,1,2,0,-1,3,-2,-609,-2,-1,3,1,0,4,-2,-3,3,2147483647,-5,5,-5,5,-2,-2,5,1,-1,2147483647,-5,-2,5,3,3,-4,5,1,1,-1,4,2,2,-790,3,0,1,-1,-1,-3,1,2,2,4,-4,5,1]}}},"ops":[{"op":"filter","predicate":{"fn":"gte","args":[{"fn":"when","args":[{"fn":"lt","args":[{"col":"l5_i32"},{"lit":-1}]},{"col":"l6_i32"},{"col":"l6_i32"}]},{"col":"l6_i32"}]}},{"op":"head","n":3}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i64","type":"i64"},{"name":"l3_i32","type":"i32"},{"name":"l4_f64","type":"f64"},{"name":"l5_i32","type":"i32"},{"name":"l6_i32","type":"i32"}],"columns":{"l0_utf8":["😀","c","aa"],"l1_i32":[3,-4,0],"l2_i64":["9007199254740995","-4611686018427387904","9007199254740992"],"l3_i32":[5,-4,0],"l4_f64":[1.0,-58.91,2.0],"l5_i32":[-3,0,-3],"l6_i32":[2,-3,1]}}},{"id":"random/0277","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i64","type":"i64"},{"name":"l3_bool","type":"bool"}],"columns":{"l0_utf8":["","aa","aa","aa","a","c","ab","c","aa","zz","zz","aa","ab","c","","aa","a","c","é","zz"],"l1_i32":[0,-2,2,5,-5,5,1,0,2,1,-5,5,2,4,3,0,-4,2,-4,1],"l2_i64":["-3","9007199254740994","9007199254740995","-4611686018427387904","2","9007199254740993","-4611686018427387904","9007199254740992","2","-4611686018427387904","-4611686018427387904","-4611686018427387904","3","-2","-4611686018427387904","3","-4","9007199254740995","9007199254740992","-4611686018427387904"],"l3_bool":[true,true,true,true,true,true,false,false,false,false,false,true,false,false,true,false,false,true,true,true]}}},"ops":[{"op":"head","n":1},{"op":"filter","predicate":{"fn":"isNotNull","args":[{"col":"l0_utf8"}]}},{"op":"withColumns","exprs":{"x2":{"fn":"add","args":[{"col":"l1_i32"},{"fn":"abs","args":[{"lit":2.5}]}]}}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i64","type":"i64"},{"name":"l3_bool","type":"bool"},{"name":"x2","type":"f64"}],"columns":{"l0_utf8":[""],"l1_i32":[0],"l2_i64":["-3"],"l3_bool":[true],"x2":[2.5]}}},{"id":"random/0278","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_i32","type":"i32"},{"name":"l4_i64","type":"i64"},{"name":"l5_utf8","type":"utf8"},{"name":"l6_f64","type":"f64"}],"columns":{"l0_utf8":["zz",null,null,"B","b",null,"ab","a","c","a","ab","zz","a","😀",null,"ab",null,"é","b",null,"B","zz","","","😀","zz","😀",null,null,"c","a","aa","😀","B","a","😀","zz","","ab","","aa","😀","B","😀","é","😀","a","c","ab","c",null,"é","c","ab","b","é","b","zz",null,"😀"],"l1_i32":[3,0,-1,-3,5,-5,0,-5,null,3,3,null,4,0,3,-4,-2,4,0,-4,0,-5,1,1,5,5,2147483647,3,-4,1,3,2,-5,3,-5,-2,0,-3,3,4,-4,0,-1,-5,2,-2,-5,-2147483648,1,null,3,-3,-5,1,-3,5,0,-2,1,-4],"l2_utf8":["","b","aa","","😀","😀",null,"é","c","ab","","😀","zz","😀","aa","é","zz","B","","","",null,"B","ab","","a","é","B","c","","zz","a","a","ab","é","","B","aa","ab","é",null,"é","é","😀","","ab","",null,"b","","aa","ab","é",null,"B",null,"B","","😀","c"],"l3_i32":[3,2,-3,-2147483648,3,-1,2,3,-3,-3,-1,5,null,3,-3,-2,-4,0,1,2,2,1,null,-1,5,0,3,null,-5,-4,-1,2,-5,-3,-2,0,117,-4,-5,2,-2,-36,3,1,0,-5,4,4,-3,1,5,2,718,-4,-1,-5,5,0,-4,-3],"l4_i64":["-5","9007199254740994","-4611686018427387904",null,"-4611686018427387904","-5","9007199254740993","-4611686018427387904",null,"9007199254740993","9007199254740993","9007199254740994","-4611686018427387904",null,"-4611686018427387904","-4611686018427387904","-5","0","9007199254740994","-4611686018427387904",null,"-4611686018427387904","9007199254740993","-4611686018427387904","9007199254740992","-4611686018427387904","9007199254740993","9007199254740992","9007199254740995","-2","9007199254740994","-4611686018427387904","9007199254740994",null,"-4611686018427387904","9007199254740995","-4611686018427387904","9007199254740992","5","-4","-1","9007199254740994","9007199254740993","9007199254740994","-2","-4611686018427387904","5","4","9007199254740992","9007199254740995","-3","9007199254740994","3","-4611686018427387904","0","2","-4611686018427387904","-4611686018427387904","-3","9007199254740992"],"l5_utf8":[null,null,"é","zz","ab","😀","é","B","zz","zz",null,"ab","ab","","c","aa","😀","a","a",null,"b","c","c","b","aa",null,"aa","B",null,"a","B","é","B","é","B","a","😀","c","😀","","zz","ab","B","aa","a","c","aa",null,"","c","😀","b","",null,"zz","ab","é","ab","c",null],"l6_f64":[2.0,3.0,-1.0,0.0,39.381,1.0,-3.0,null,4.0,-4.0,2.0,-3.0,3.0,20.604,84.733,-99.626,-4.0,"NaN",24.842,74.001,-53.649,3.0,4.0,-11.963,61.192,null,"NaN",-3.0,4.0,-30.612,3.0,null,76.677,null,-2.0,3.0,-53.767,-3.0,92.667,80.557,null,-2.0,-3.0,-1.0,-4.0,5.069,-75.01,-3.0,"Infinity",3.0,-4.0,-25.584,0.0,3.0,-2.0,-2.0,-4.0,null,27.947,1.0]}}},"ops":[{"op":"filter","predicate":{"fn":"isNull","args":[{"col":"l1_i32"}]}},{"op":"filter","predicate":{"fn":"or","args":[{"fn":"gt","args":[{"lit":2.5},{"col":"l6_f64"}]},{"fn":"neq","args":[{"lit":-1},{"col":"l3_i32"}]}]}},{"op":"head","n":3}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_i32","type":"i32"},{"name":"l4_i64","type":"i64"},{"name":"l5_utf8","type":"utf8"},{"name":"l6_f64","type":"f64"}],"columns":{"l0_utf8":["c","zz","c"],"l1_i32":[null,null,null],"l2_utf8":["c","😀",""],"l3_i32":[-3,5,1],"l4_i64":[null,"9007199254740994","9007199254740995"],"l5_utf8":["zz","ab","c"],"l6_f64":[4.0,-3.0,3.0]}}},{"id":"random/0279","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_i32","type":"i32"},{"name":"l4_f64","type":"f64"},{"name":"l5_f64","type":"f64"},{"name":"l6_i32","type":"i32"}],"columns":{"l0_utf8":["","B","😀","zz","😀"],"l1_i32":[4,-5,-4,1,1],"l2_utf8":["B","c","é","","B"],"l3_i32":[-1,5,1,-1,3],"l4_f64":[-3.0,-37.236,"NaN",-3.0,3.0],"l5_f64":[-2.0,-1.0,2.0,3.0,50.502],"l6_i32":[1,-3,3,-4,5]}},"right":{"fields":[{"name":"l6_i32","type":"i32"},{"name":"r0_utf8","type":"utf8"}],"columns":{"r0_utf8":["zz","😀","","a","zz","B",null,"😀","é","😀"],"l6_i32":[-4,-3,3,3,3,-4,-3,1,5,1]}}},"ops":[{"op":"join","right":"right","on":["l6_i32"],"how":"inner"},{"op":"filter","predicate":{"fn":"isIn","args":[{"col":"l0_utf8"}],"values":["b","é","ab"]}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_i32","type":"i32"},{"name":"l4_f64","type":"f64"},{"name":"l5_f64","type":"f64"},{"name":"l6_i32","type":"i32"},{"name":"r0_utf8","type":"utf8"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_utf8":[],"l3_i32":[],"l4_f64":[],"l5_f64":[],"l6_i32":[],"r0_utf8":[]}}},{"id":"random/0280","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i64","type":"i64"},{"name":"l3_f64","type":"f64"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_utf8","type":"utf8"},{"name":"l6_i32","type":"i32"}],"columns":{"l0_utf8":["c","ab","😀",null,null,"a",null,null,"a","é","B","",null,"B",null,null,"ab","ab","aa","c",null,null,"b","ab","","aa",null,null,"😀","B","B",null,null,null,"",null,null,"b",null,"ab","a","ab",null,"b","b","b","😀","😀","B",null,"zz","😀",null,"a","aa","","B","b","",null],"l1_i32":[3,null,null,2,4,0,null,-2,4,null,4,-5,0,0,null,2,5,null,3,-1,3,-3,-3,-3,2,0,1,-4,3,2,-4,-4,3,1,null,4,1,-2147483648,1,2,1,null,4,-5,-1,4,2,3,null,5,null,-4,-1,-2,5,2147483647,5,-3,-3,0],"l2_i64":[null,"-5","0",null,"-1","-4611686018427387904",null,null,"9007199254740993","-4611686018427387904",null,null,null,null,"4","-4611686018427387904","-4611686018427387904","-4611686018427387904","2","-4","-4611686018427387904","-4611686018427387904","9007199254740993",null,null,"5","4",null,"-4611686018427387904","0","4","9007199254740994","9007199254740995","-4611686018427387904",null,null,"9007199254740994","-4611686018427387904","-4611686018427387904","-4611686018427387904","9007199254740994",null,"9007199254740994",null,null,"-4611686018427387904","-4611686018427387904","-4611686018427387904","3",null,"-4611686018427387904","9007199254740993",null,null,"9007199254740992","4",null,"-4","5","9007199254740992"],"l3_f64":[2.0,-2.0,0.0,null,42.04,null,-58.976,80.696,null,-44.734,0.0,null,0.0,-2.0,-36.532,null,44.922,null,0.0,1.0,1.0,-2.0,-4.0,null,14.109,-4.0,-86.251,null,-1.0,15.799,-56.317,-4.0,-2.0,null,null,-1.0,null,null,-3.0,-50.054,0.0,2.0,null,null,3.0,"Infinity",-86.636,"NaN",null,-96.171,null,null,-1.0,null,null,1.0,null,null,-31.124,15.469],"l4_utf8":["B","B",null,null,"é","c",null,"é","b","b","c","a","zz","B",null,"é","c",null,"é","B",null,"c",null,null,"zz","é","zz","c","c","","zz","é","zz","a","b",null,"é","zz","😀",null,"😀","c","a",null,"ab",null,"c","ab","zz","é",null,"ab","é","B","","😀","é",null,null,null],"l5_utf8":[null,"ab","é",null,null,"aa","b",null,"zz",null,"é","b",null,"c",null,"a","😀","😀",null,"😀","aa","zz",null,"c","aa","a","B",null,"ab",null,null,"",null,"zz",null,"a","é","a",null,null,"B","",null,null,"😀",null,null,"zz",null,"é","a","b",null,"😀","b","😀","😀","zz","é","c"],"l6_i32":[-4,-5,null,1,null,null,null,5,null,-5,-2147483648,-4,2,-4,3,-2,null,-2,null,-4,null,5,0,null,2,-2147483648,2,-4,null,1,null,5,null,-1,-1,null,-3,null,-2,null,null,2,-4,null,0,-5,-3,2,-4,null,2147483647,null,1,3,-1,null,null,-2,3,null]}},"right":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"r0_utf8","type":"utf8"}],"columns":{"r0_utf8":["ab","B","B","😀","aa","B","zz","aa","ab","a","b","B","😀","c","b","c","ab","ab","é","zz","ab","a","","é","ab","aa","","","aa",""],"l0_utf8":["c",null,"b","",null,"b","b",null,null,null,"😀","b","aa","B",null,null,null,null,"ab","a",null,null,"b",null,null,"","B","é",null,"😀"]}}},"ops":[{"op":"join","right":"right","on":["l0_utf8"],"how":"inner"},{"op":"head","n":5}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i64","type":"i64"},{"name":"l3_f64","type":"f64"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_utf8","type":"utf8"},{"name":"l6_i32","type":"i32"},{"name":"r0_utf8","type":"utf8"}],"columns":{"l0_utf8":["c","ab","😀","😀","a"],"l1_i32":[3,null,null,null,0],"l2_i64":[null,"-5","0","0","-4611686018427387904"],"l3_f64":[2.0,-2.0,0.0,0.0,null],"l4_utf8":["B","B",null,null,"c"],"l5_utf8":[null,"ab","é","é","aa"],"l6_i32":[-4,-5,null,null,null],"r0_utf8":["ab","é","b","","zz"]}}},{"id":"random/0281","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_i64","type":"i64"},{"name":"l4_bool","type":"bool"}],"columns":{"l0_utf8":["é","aa","c","😀","aa","ab","zz","ab","😀","zz","zz","B","zz","aa","😀","b","a","B","aa","😀"],"l1_i32":[4,5,-3,-3,2,-2,-2,3,-3,1,1,5,767,2,4,-4,2,-2,-3,1],"l2_f64":[95.949,-96.362,1.0,70.335,-4.0,-39.164,-35.858,-71.696,"-0",-4.778,3.0,29.286,-31.932,-18.787,-28.814,3.0,-4.0,-20.156,53.813,77.436],"l3_i64":["-1","9007199254740995","-5","9007199254740994","-4611686018427387904","4","-4611686018427387904","-1","2","-4","9007199254740992","9007199254740992","9007199254740994","-4611686018427387904","9007199254740992","-4611686018427387904","-3","-4611686018427387904","-4611686018427387904","9007199254740995"],"l4_bool":[false,true,true,true,false,true,true,false,false,false,true,false,false,true,false,true,false,false,false,false]}}},"ops":[{"op":"filter","predicate":{"fn":"isNull","args":[{"col":"l0_utf8"}]}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_i64","type":"i64"},{"name":"l4_bool","type":"bool"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_f64":[],"l3_i64":[],"l4_bool":[]}}},{"id":"random/0282","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_bool","type":"bool"},{"name":"l4_f64","type":"f64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_f64":[],"l3_bool":[],"l4_f64":[]}}},"ops":[{"op":"sort","keys":[{"expr":{"col":"l3_bool"},"descending":false}],"nullsLast":true},{"op":"sort","keys":[{"expr":{"col":"l0_utf8"},"descending":false},{"expr":{"col":"l2_f64"},"descending":false}],"nullsLast":false},{"op":"groupBy","keys":["l0_utf8"],"aggs":{"n2":{"agg":"len"},"a2_0":{"agg":"min","arg":{"col":"l2_f64"}}}},{"op":"filter","predicate":{"fn":"or","args":[{"fn":"isNull","args":[{"col":"a2_0"}]},{"fn":"and","args":[{"fn":"isNull","args":[{"col":"l0_utf8"}]},{"fn":"isNotNull","args":[{"col":"n2"}]}]}]}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"n2","type":"u32"},{"name":"a2_0","type":"f64"}],"columns":{"l0_utf8":[],"n2":[],"a2_0":[]}}},{"id":"random/0283","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_i32","type":"i32"},{"name":"l4_i32","type":"i32"},{"name":"l5_f64","type":"f64"},{"name":"l6_utf8","type":"utf8"}],"columns":{"l0_utf8":["😀","aa","aa","aa","😀","😀","c","zz","c","B","","b","zz","b","é","b","","é","zz","😀"],"l1_i32":[3,0,4,-5,-3,2,5,3,-5,4,-2147483648,-1,1,-1,5,2,-3,-3,-3,5],"l2_f64":[-37.142,3.0,64.921,6.902,87.139,95.625,4.0,4.0,-2.0,75.522,1.0,4.0,3.0,2.0,0.0,1.0,97.376,31.017,1.0,-71.814],"l3_i32":[-4,5,4,-2147483648,3,-3,-2147483648,-1,-5,2147483647,-1,-3,1,0,5,-4,0,5,2,-2147483648],"l4_i32":[-5,5,-2,3,1,-4,-3,5,-2147483648,-5,1,-3,2,-3,5,-3,-5,355,5,-5],"l5_f64":["NaN",23.923,0.0,-85.042,-87.429,0.0,-12.488,-4.0,-57.977,4.0,59.073,4.0,-2.0,47.171,0.0,3.0,-31.97,-3.0,16.292,34.942],"l6_utf8":["a","B","aa","😀","c","a","aa","B","zz","","c","c","a","c","😀","b","b","c","b","😀"]}},"right":{"fields":[{"name":"l6_utf8","type":"utf8"},{"name":"r0_bool","type":"bool"}],"columns":{"r0_bool":[false,true,false,false,true,false,null,true,null,false,null,null,false,true,true,true,false,false,false,true,true,false,null,null,false,true,null,false,null,false],"l6_utf8":["a","a","aa","a","b","c","b","aa","aa","B","c","😀","b","a","c","a","b","zz","a","c","b","c","b","c","😀","b","b","😀","B","zz"]}}},"ops":[{"op":"unique"},{"op":"join","right":"right","on":["l6_utf8"],"how":"inner"},{"op":"withColumns","exprs":{"x2":{"lit":-1}}},{"op":"withColumns","exprs":{"x3":{"fn":"mod","args":[{"col":"l2_f64"},{"fn":"neg","args":[{"col":"l5_f64"}]}]}}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_i32","type":"i32"},{"name":"l4_i32","type":"i32"},{"name":"l5_f64","type":"f64"},{"name":"l6_utf8","type":"utf8"},{"name":"r0_bool","type":"bool"},{"name":"x2","type":"i32"},{"name":"x3","type":"f64"}],"columns":{"l0_utf8":["😀","😀","😀","😀","😀","😀","aa","aa","aa","aa","aa","aa","aa","aa","😀","😀","😀","😀","😀","😀","😀","😀","😀","😀","😀","😀","c","c","c","zz","zz","c","c","","","","","","","b","b","b","b","b","b","zz","zz","zz","zz","zz","zz","b","b","b","b","b","b","é","é","é","b","b","b","b","b","b","b","b","","","","","","","","","é","é","é","é","é","é","zz","zz","zz","zz","zz","zz","zz","zz","😀","😀","😀"],"l1_i32":[3,3,3,3,3,3,0,0,4,4,4,-5,-5,-5,-3,-3,-3,-3,-3,-3,2,2,2,2,2,2,5,5,5,3,3,-5,-5,-2147483648,-2147483648,-2147483648,-2147483648,-2147483648,-2147483648,-1,-1,-1,-1,-1,-1,1,1,1,1,1,1,-1,-1,-1,-1,-1,-1,5,5,5,2,2,2,2,2,2,2,2,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,5,5,5],"l2_f64":[-37.142,-37.142,-37.142,-37.142,-37.142,-37.142,3.0,3.0,64.921,64.921,64.921,6.902,6.902,6.902,87.139,87.139,87.139,87.139,87.139,87.139,95.625,95.625,95.625,95.625,95.625,95.625,4.0,4.0,4.0,4.0,4.0,-2.0,-2.0,1.0,1.0,1.0,1.0,1.0,1.0,4.0,4.0,4.0,4.0,4.0,4.0,3.0,3.0,3.0,3.0,3.0,3.0,2.0,2.0,2.0,2.0,2.0,2.0,0.0,0.0,0.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,97.376,97.376,97.376,97.376,97.376,97.376,97.376,97.376,31.017,31.017,31.017,31.017,31.017,31.017,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,-71.814,-71.814,-71.814],"l3_i32":[-4,-4,-4,-4,-4,-4,5,5,4,4,4,-2147483648,-2147483648,-2147483648,3,3,3,3,3,3,-3,-3,-3,-3,-3,-3,-2147483648,-2147483648,-2147483648,-1,-1,-5,-5,-1,-1,-1,-1,-1,-1,-3,-3,-3,-3,-3,-3,1,1,1,1,1,1,0,0,0,0,0,0,5,5,5,-4,-4,-4,-4,-4,-4,-4,-4,0,0,0,0,0,0,0,0,5,5,5,5,5,5,2,2,2,2,2,2,2,2,-2147483648,-2147483648,-2147483648],"l4_i32":[-5,-5,-5,-5,-5,-5,5,5,-2,-2,-2,3,3,3,1,1,1,1,1,1,-4,-4,-4,-4,-4,-4,-3,-3,-3,5,5,-2147483648,-2147483648,1,1,1,1,1,1,-3,-3,-3,-3,-3,-3,2,2,2,2,2,2,-3,-3,-3,-3,-3,-3,5,5,5,-3,-3,-3,-3,-3,-3,-3,-3,-5,-5,-5,-5,-5,-5,-5,-5,355,355,355,355,355,355,5,5,5,5,5,5,5,5,-5,-5,-5],"l5_f64":["NaN","NaN","NaN","NaN","NaN","NaN",23.923,23.923,0.0,0.0,0.0,-85.042,-85.042,-85.042,-87.429,-87.429,-87.429,-87.429,-87.429,-87.429,0.0,0.0,0.0,0.0,0.0,0.0,-12.488,-12.488,-12.488,-4.0,-4.0,-57.977,-57.977,59.073,59.073,59.073,59.073,59.073,59.073,4.0,4.0,4.0,4.0,4.0,4.0,-2.0,-2.0,-2.0,-2.0,-2.0,-2.0,47.171,47.171,47.171,47.171,47.171,47.171,0.0,0.0,0.0,3.0,3.0,3.0,3.0,3.0,3.0,3.0,3.0,-31.97,-31.97,-31.97,-31.97,-31.97,-31.97,-31.97,-31.97,-3.0,-3.0,-3.0,-3.0,-3.0,-3.0,16.292,16.292,16.292,16.292,16.292,16.292,16.292,16.292,34.942,34.942,34.942],"l6_utf8":["a","a","a","a","a","a","B","B","aa","aa","aa","😀","😀","😀","c","c","c","c","c","c","a","a","a","a","a","a","aa","aa","aa","B","B","zz","zz","c","c","c","c","c","c","c","c","c","c","c","c","a","a","a","a","a","a","c","c","c","c","c","c","😀","😀","😀","b","b","b","b","b","b","b","b","b","b","b","b","b","b","b","b","c","c","c","c","c","c","b","b","b","b","b","b","b","b","😀","😀","😀"],"r0_bool":[false,true,false,true,true,false,false,null,false,true,null,null,false,false,false,null,true,true,false,null,false,true,false,true,true,false,false,true,null,false,null,false,false,false,null,true,true,false,null,false,null,true,true,false,null,false,true,false,true,true,false,false,null,true,true,false,null,null,false,false,true,null,false,false,true,null,true,null,true,null,false,false,true,null,true,null,false,null,true,true,false,null,true,null,false,false,true,null,true,null,null,false,false],"x2":[-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1],"x3":["NaN","NaN","NaN","NaN","NaN","NaN",-20.923,-20.923,"NaN","NaN","NaN",6.902,6.902,6.902,87.139,87.139,87.139,87.139,87.139,87.139,"NaN","NaN","NaN","NaN","NaN","NaN",4.0,4.0,4.0,0.0,0.0,55.977,55.977,-58.073,-58.073,-58.073,-58.073,-58.073,-58.073,0.0,0.0,0.0,0.0,0.0,0.0,1.0,1.0,1.0,1.0,1.0,1.0,-45.171,-45.171,-45.171,-45.171,-45.171,-45.171,"NaN","NaN","NaN",-2.0,-2.0,-2.0,-2.0,-2.0,-2.0,-2.0,-2.0,1.4660000000000082,1.4660000000000082,1.4660000000000082,1.4660000000000082,1.4660000000000082,1.4660000000000082,1.4660000000000082,1.4660000000000082,1.0169999999999995,1.0169999999999995,1.0169999999999995,1.0169999999999995,1.0169999999999995,1.0169999999999995,-15.292000000000002,-15.292000000000002,-15.292000000000002,-15.292000000000002,-15.292000000000002,-15.292000000000002,-15.292000000000002,-15.292000000000002,-1.9299999999999926,-1.9299999999999926,-1.9299999999999926]}}},{"id":"random/0284","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_i64","type":"i64"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_i32","type":"i32"},{"name":"l6_bool","type":"bool"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_i32":[],"l3_i64":[],"l4_utf8":[],"l5_i32":[],"l6_bool":[]}},"right":{"fields":[{"name":"l1_i32","type":"i32"},{"name":"r0_utf8","type":"utf8"}],"columns":{"r0_utf8":["é","😀","zz","é","aa","b","ab","B","aa","é"],"l1_i32":[null,-3,null,null,-3,-3,null,null,null,null]}}},"ops":[{"op":"unique","columns":["l5_i32"]},{"op":"join","right":"right","on":["l1_i32"],"how":"semi"},{"op":"sort","keys":[{"expr":{"col":"l4_utf8"},"descending":false},{"expr":{"col":"l0_utf8"},"descending":true}],"nullsLast":false}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_i64","type":"i64"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_i32","type":"i32"},{"name":"l6_bool","type":"bool"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_i32":[],"l3_i64":[],"l4_utf8":[],"l5_i32":[],"l6_bool":[]}}},{"id":"random/0285","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i64","type":"i64"},{"name":"l3_bool","type":"bool"},{"name":"l4_i32","type":"i32"}],"columns":{"l0_utf8":["a","b","","zz","B"],"l1_i32":[3,5,1,-4,-2],"l2_i64":["9007199254740994","9007199254740994","-4611686018427387904","0","-4611686018427387904"],"l3_bool":[true,true,false,true,true],"l4_i32":[0,-3,3,-2147483648,-3]}},"right":{"fields":[{"name":"a0_0","type":"i64"},{"name":"r0_f64","type":"f64"}],"columns":{"r0_f64":[1.0,-11.481,-3.0,-82.76,null,66.535,-1.0,null,4.0,null],"a0_0":["-1","9007199254740992","-5","9007199254740992","9007199254740992","-1","-1","5","9007199254740995","9007199254740995"]}}},"ops":[{"op":"groupBy","keys":["l1_i32"],"aggs":{"n0":{"agg":"len"},"a0_0":{"agg":"last","arg":{"col":"l2_i64"}}}},{"op":"groupBy","keys":["a0_0"],"aggs":{"n1":{"agg":"len"},"a1_0":{"agg":"std","arg":{"col":"l1_i32"}},"a1_1":{"agg":"last","arg":{"col":"n0"}}}},{"op":"join","right":"right","on":["a0_0"],"how":"left"},{"op":"filter","predicate":{"fn":"gte","args":[{"lit":-1},{"col":"a1_0"}]}}],"expected":{"fields":[{"name":"a0_0","type":"i64"},{"name":"n1","type":"u32"},{"name":"a1_0","type":"f64"},{"name":"a1_1","type":"u32"},{"name":"r0_f64","type":"f64"}],"columns":{"a0_0":[],"n1":[],"a1_0":[],"a1_1":[],"r0_f64":[]}}},{"id":"random/0286","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_f64","type":"f64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_f64":[],"l3_f64":[]}}},"ops":[{"op":"withColumns","exprs":{"x0":{"fn":"abs","args":[{"col":"l3_f64"}]}}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_f64","type":"f64"},{"name":"x0","type":"f64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_f64":[],"l3_f64":[],"x0":[]}}},{"id":"random/0287","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_bool","type":"bool"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_i32","type":"i32"}],"columns":{"l0_utf8":["é"],"l1_i32":[null],"l2_bool":[false],"l3_bool":[true],"l4_utf8":["😀"],"l5_i32":[null]}}},"ops":[{"op":"head","n":7},{"op":"head","n":4}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_bool","type":"bool"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_i32","type":"i32"}],"columns":{"l0_utf8":["é"],"l1_i32":[null],"l2_bool":[false],"l3_bool":[true],"l4_utf8":["😀"],"l5_i32":[null]}}},{"id":"random/0288","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_i32","type":"i32"},{"name":"l4_f64","type":"f64"}],"columns":{"l0_utf8":[null],"l1_i32":[null],"l2_utf8":[""],"l3_i32":[-3],"l4_f64":[null]}}},"ops":[{"op":"unique"},{"op":"sort","keys":[{"expr":{"col":"l0_utf8"},"descending":true},{"expr":{"col":"l2_utf8"},"descending":true}],"nullsLast":false}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_i32","type":"i32"},{"name":"l4_f64","type":"f64"}],"columns":{"l0_utf8":[null],"l1_i32":[null],"l2_utf8":[""],"l3_i32":[-3],"l4_f64":[null]}}},{"id":"random/0289","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_f64","type":"f64"}],"columns":{"l0_utf8":["😀",null,"a","B","b","B","aa","ab","B","ab","c","é","B",null,"","aa","a","aa",null,"","B","aa","a","c","é","é","B","zz","aa","a","c","😀","ab","é","ab","zz","a","ab",null,"é","aa","aa",null,"ab","c","a","B","B","aa","😀","a","aa","zz","","😀","","B","b","B",""],"l1_i32":[-1,-4,null,-3,1,1,null,-3,4,0,-2147483648,null,null,-5,3,null,-2,-5,2,2,0,-2147483648,4,5,3,4,0,4,4,1,5,2147483647,1,null,-4,-4,null,null,-4,3,4,2,-5,0,5,null,null,1,null,-5,3,-2,-3,-2,3,-1,-2,-2,-5,4],"l2_f64":[78.614,2.0,81.278,54.942,4.0,1.0,-43.712,-3.0,63.337,59.685,-1.0,4.0,-4.0,0.0,-84.52,-21.837,-62.322,48.897,1.0,-4.0,45.238,-4.0,74.091,-2.0,-51.584,"NaN",-57.912,-48.624,null,null,null,null,-68.349,-10.5,-1.0,0.0,1.0,31.772,null,-89.082,56.136,1.0,-4.0,0.469,38.959,-2.0,27.899,-1.0,-2.0,44.575,null,-4.0,"NaN",32.785,-70.022,1.0,3.0,null,14.619,14.95],"l3_f64":[-42.527,83.106,-4.0,null,null,-19.767,0.0,3.0,-89.853,2.0,58.895,-59.428,1.0,-99.055,null,"NaN",9.787,-4.0,-2.0,55.341,-24.85,33.511,-1.0,null,"NaN",-83.142,-4.0,-66.574,-2.0,null,74.024,-2.0,1.0,4.0,45.524,16.591,4.0,null,86.067,85.196,null,-98.946,71.407,-3.597,98.871,-68.416,2.0,-37.482,-66.238,24.771,91.751,4.0,94.149,"NaN",null,80.776,-3.0,null,-1.0,1.0]}},"right":{"fields":[{"name":"l1_i32","type":"i32"},{"name":"r0_utf8","type":"utf8"}],"columns":{"r0_utf8":["aa","","é","B","aa","a","c","ab","é","zz","","😀","b","","😀","zz","ab","é","a","B","b","a","a","aa","zz","b","aa","a","b","aa"],"l1_i32":[-2,4,2,null,null,5,null,2,4,1,-5,null,5,null,null,2,0,-2,-2,3,-3,-5,null,-5,1,1,2,3,4,1]}}},"ops":[{"op":"join","right":"right","on":["l1_i32"],"how":"semi"}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_f64","type":"f64"}],"columns":{"l0_utf8":["B","b","B","ab","B","ab",null,"","a","aa",null,"","B","a","c","é","é","B","zz","aa","a","c","ab","é","aa","aa",null,"ab","c","B","😀","a","aa","zz","","😀","B","b","B",""],"l1_i32":[-3,1,1,-3,4,0,-5,3,-2,-5,2,2,0,4,5,3,4,0,4,4,1,5,1,3,4,2,-5,0,5,1,-5,3,-2,-3,-2,3,-2,-2,-5,4],"l2_f64":[54.942,4.0,1.0,-3.0,63.337,59.685,0.0,-84.52,-62.322,48.897,1.0,-4.0,45.238,74.091,-2.0,-51.584,"NaN",-57.912,-48.624,null,null,null,-68.349,-89.082,56.136,1.0,-4.0,0.469,38.959,-1.0,44.575,null,-4.0,"NaN",32.785,-70.022,3.0,null,14.619,14.95],"l3_f64":[null,null,-19.767,3.0,-89.853,2.0,-99.055,null,9.787,-4.0,-2.0,55.341,-24.85,-1.0,null,"NaN",-83.142,-4.0,-66.574,-2.0,null,74.024,1.0,85.196,null,-98.946,71.407,-3.597,98.871,-37.482,24.771,91.751,4.0,94.149,"NaN",null,-3.0,null,-1.0,1.0]}}},{"id":"random/0290","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_i64","type":"i64"},{"name":"l4_bool","type":"bool"},{"name":"l5_utf8","type":"utf8"},{"name":"l6_i32","type":"i32"}],"columns":{"l0_utf8":["zz",null,"b","aa","a","","😀","aa",null,null,"é","a","zz","é",null,"B","zz",null,"B",null,"c","😀","zz","zz",null,"ab",null,"c","a","zz","zz","B",null,null,null,"😀",null,null,"B","b",null,"b","a","a","a",null,"a","ab","c",null,"c","zz","","B","b","aa",null,null,"a","zz"],"l1_i32":[-3,-3,2,-1,2,null,null,5,null,4,null,-2147483648,-4,-2,2,3,3,4,-2,-2147483648,-5,-2147483648,-4,-1,-5,-1,null,null,1,5,4,-3,0,0,-3,-4,null,2,null,5,0,null,4,1,2,-2,-2,0,null,-4,-1,-4,3,-1,4,1,null,-1,-1,-5],"l2_i32":[null,-4,-5,2,-4,null,0,null,4,0,3,5,null,null,1,-3,2,null,-1,2147483647,3,3,-2,null,-3,-1,null,0,null,0,-1,4,0,-2,-3,null,-2,1,null,-4,-5,-2,null,-4,5,3,3,2,1,0,0,null,null,4,1,5,0,null,null,4],"l3_i64":["-4",null,"9007199254740992","-4611686018427387904","5",null,"5",null,"-4611686018427387904","-4611686018427387904","-4","9007199254740995","0","1","9007199254740995",null,null,"9007199254740993","9007199254740995","-4","9007199254740992",null,"-4611686018427387904",null,"5",null,"-4611686018427387904","9007199254740992","2","0","9007199254740992","-4",null,"9007199254740992",null,null,"-4611686018427387904","-2","-4611686018427387904",null,"-5","-4611686018427387904",null,null,null,"3","0","9007199254740994","9007199254740993","9007199254740994","9007199254740995","-4611686018427387904","9007199254740994","9007199254740993",null,"9007199254740994","9007199254740994","-4611686018427387904","-4611686018427387904",null],"l4_bool":[null,null,true,null,null,null,false,null,false,false,null,false,true,false,false,null,false,true,false,false,false,null,false,false,true,false,true,null,null,null,true,false,false,false,true,false,null,null,null,true,null,true,null,false,true,true,null,false,true,false,null,true,false,null,false,true,false,null,null,true],"l5_utf8":["a","b","",null,"😀",null,"é",null,"c","B","é","😀","ab","zz","B","é",null,"","a","aa",null,null,null,"aa",null,"","😀",null,"zz","B","aa","😀",null,"é","c",null,"a","é","é","aa","é","aa","😀","c","zz","aa","ab","c",null,null,null,"c","😀","B","😀","b","a","é","a",null],"l6_i32":[null,0,-3,null,null,null,-5,null,null,2,5,-2,2,1,762,null,null,2147483647,2,null,-1,null,-5,-2147483648,null,-2,null,0,null,-5,1,-1,3,525,null,0,-5,-3,null,null,null,-5,-1,3,2,2,5,-3,5,null,-5,null,-1,null,-5,-1,-4,0,null,null]}}},"ops":[{"op":"withColumns","exprs":{"x0":{"col":"l1_i32"}}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_i64","type":"i64"},{"name":"l4_bool","type":"bool"},{"name":"l5_utf8","type":"utf8"},{"name":"l6_i32","type":"i32"},{"name":"x0","type":"i32"}],"columns":{"l0_utf8":["zz",null,"b","aa","a","","😀","aa",null,null,"é","a","zz","é",null,"B","zz",null,"B",null,"c","😀","zz","zz",null,"ab",null,"c","a","zz","zz","B",null,null,null,"😀",null,null,"B","b",null,"b","a","a","a",null,"a","ab","c",null,"c","zz","","B","b","aa",null,null,"a","zz"],"l1_i32":[-3,-3,2,-1,2,null,null,5,null,4,null,-2147483648,-4,-2,2,3,3,4,-2,-2147483648,-5,-2147483648,-4,-1,-5,-1,null,null,1,5,4,-3,0,0,-3,-4,null,2,null,5,0,null,4,1,2,-2,-2,0,null,-4,-1,-4,3,-1,4,1,null,-1,-1,-5],"l2_i32":[null,-4,-5,2,-4,null,0,null,4,0,3,5,null,null,1,-3,2,null,-1,2147483647,3,3,-2,null,-3,-1,null,0,null,0,-1,4,0,-2,-3,null,-2,1,null,-4,-5,-2,null,-4,5,3,3,2,1,0,0,null,null,4,1,5,0,null,null,4],"l3_i64":["-4",null,"9007199254740992","-4611686018427387904","5",null,"5",null,"-4611686018427387904","-4611686018427387904","-4","9007199254740995","0","1","9007199254740995",null,null,"9007199254740993","9007199254740995","-4","9007199254740992",null,"-4611686018427387904",null,"5",null,"-4611686018427387904","9007199254740992","2","0","9007199254740992","-4",null,"9007199254740992",null,null,"-4611686018427387904","-2","-4611686018427387904",null,"-5","-4611686018427387904",null,null,null,"3","0","9007199254740994","9007199254740993","9007199254740994","9007199254740995","-4611686018427387904","9007199254740994","9007199254740993",null,"9007199254740994","9007199254740994","-4611686018427387904","-4611686018427387904",null],"l4_bool":[null,null,true,null,null,null,false,null,false,false,null,false,true,false,false,null,false,true,false,false,false,null,false,false,true,false,true,null,null,null,true,false,false,false,true,false,null,null,null,true,null,true,null,false,true,true,null,false,true,false,null,true,false,null,false,true,false,null,null,true],"l5_utf8":["a","b","",null,"😀",null,"é",null,"c","B","é","😀","ab","zz","B","é",null,"","a","aa",null,null,null,"aa",null,"","😀",null,"zz","B","aa","😀",null,"é","c",null,"a","é","é","aa","é","aa","😀","c","zz","aa","ab","c",null,null,null,"c","😀","B","😀","b","a","é","a",null],"l6_i32":[null,0,-3,null,null,null,-5,null,null,2,5,-2,2,1,762,null,null,2147483647,2,null,-1,null,-5,-2147483648,null,-2,null,0,null,-5,1,-1,3,525,null,0,-5,-3,null,null,null,-5,-1,3,2,2,5,-3,5,null,-5,null,-1,null,-5,-1,-4,0,null,null],"x0":[-3,-3,2,-1,2,null,null,5,null,4,null,-2147483648,-4,-2,2,3,3,4,-2,-2147483648,-5,-2147483648,-4,-1,-5,-1,null,null,1,5,4,-3,0,0,-3,-4,null,2,null,5,0,null,4,1,2,-2,-2,0,null,-4,-1,-4,3,-1,4,1,null,-1,-1,-5]}}},{"id":"random/0291","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_i32","type":"i32"},{"name":"l4_i32","type":"i32"}],"columns":{"l0_utf8":["","zz","b","c","c","","😀","aa","zz","a","a","zz","ab","é","zz","é","b","B","c","b"],"l1_i32":[5,-5,2,-5,5,-4,-2,2,2147483647,-4,-4,2147483647,-2147483648,3,5,-2,-1,-1,0,4],"l2_f64":[0.0,2.0,-18.013,23.503,4.0,78.396,"NaN","Infinity",60.004,97.443,-2.0,-1.0,-1.0,-3.0,-4.0,2.0,0.0,24.082,-89.978,-2.0],"l3_i32":[-3,0,-3,2147483647,0,1,2,2,3,-2,-1,-2,1,2147483647,-2,2,0,-1,5,-1],"l4_i32":[2,1,4,3,1,0,1,-3,0,0,338,-4,2,5,-5,-2,-3,3,1,2147483647]}}},"ops":[{"op":"groupBy","keys":["l0_utf8"],"aggs":{"n0":{"agg":"len"},"a0_0":{"agg":"max","arg":{"col":"l3_i32"}},"a0_1":{"agg":"count","arg":{"col":"l2_f64"}},"a0_2":{"agg":"sum","arg":{"col":"l1_i32"}}}},{"op":"filter","predicate":{"fn":"eq","args":[{"fn":"abs","args":[{"col":"a0_2"}]},{"fn":"abs","args":[{"col":"a0_0"}]}]}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"n0","type":"u32"},{"name":"a0_0","type":"i32"},{"name":"a0_1","type":"u32"},{"name":"a0_2","type":"i32"}],"columns":{"l0_utf8":["","😀","aa","B"],"n0":[2,1,1,1],"a0_0":[1,2,2,-1],"a0_1":[2,1,1,1],"a0_2":[1,-2,2,-1]}}},{"id":"random/0292","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_f64","type":"f64"},{"name":"l4_i64","type":"i64"},{"name":"l5_utf8","type":"utf8"},{"name":"l6_f64","type":"f64"}],"columns":{"l0_utf8":["b"],"l1_i32":[null],"l2_f64":["Infinity"],"l3_f64":[-4.0],"l4_i64":["-1"],"l5_utf8":["é"],"l6_f64":[82.584]}}},"ops":[{"op":"filter","predicate":{"fn":"lt","args":[{"col":"l3_f64"},{"fn":"mul","args":[{"lit":2},{"col":"l1_i32"}]}]}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_f64","type":"f64"},{"name":"l4_i64","type":"i64"},{"name":"l5_utf8","type":"utf8"},{"name":"l6_f64","type":"f64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_f64":[],"l3_f64":[],"l4_i64":[],"l5_utf8":[],"l6_f64":[]}}},{"id":"random/0293","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_f64","type":"f64"},{"name":"l4_f64","type":"f64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_i32":[],"l3_f64":[],"l4_f64":[]}},"right":{"fields":[{"name":"l2_i32","type":"i32"},{"name":"r0_f64","type":"f64"}],"columns":{"r0_f64":[0.0,null,-21.881,-3.0,90.634,2.0,null,70.23,"NaN",null],"l2_i32":[2,1,null,1,1,2,null,-4,null,1]}}},"ops":[{"op":"join","right":"right","on":["l2_i32"],"how":"anti"},{"op":"head","n":4},{"op":"withColumns","exprs":{"x2":{"fn":"mod","args":[{"col":"l2_i32"},{"lit":2}]}}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_f64","type":"f64"},{"name":"l4_f64","type":"f64"},{"name":"x2","type":"i32"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_i32":[],"l3_f64":[],"l4_f64":[],"x2":[]}}},{"id":"random/0294","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i64","type":"i64"},{"name":"l3_i32","type":"i32"},{"name":"l4_f64","type":"f64"},{"name":"l5_i32","type":"i32"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_i64":[],"l3_i32":[],"l4_f64":[],"l5_i32":[]}}},"ops":[{"op":"withColumns","exprs":{"x0":{"fn":"div","args":[{"fn":"when","args":[{"fn":"gte","args":[{"col":"l4_f64"},{"col":"l5_i32"}]},{"lit":2},{"lit":2.5}]},{"fn":"when","args":[{"fn":"lte","args":[{"col":"l5_i32"},{"col":"l1_i32"}]},{"col":"l3_i32"},{"col":"l1_i32"}]}]}}},{"op":"unique"}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i64","type":"i64"},{"name":"l3_i32","type":"i32"},{"name":"l4_f64","type":"f64"},{"name":"l5_i32","type":"i32"},{"name":"x0","type":"f64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_i64":[],"l3_i32":[],"l4_f64":[],"l5_i32":[],"x0":[]}}},{"id":"random/0295","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_f64","type":"f64"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_f64","type":"f64"},{"name":"l6_utf8","type":"utf8"}],"columns":{"l0_utf8":["é",null,"a","é","","zz","é",null,"zz","a",null,"c","B",null,"","é",null,"B","B","b"],"l1_i32":[-5,-5,5,-5,null,3,3,null,-2147483648,null,-3,-2,-4,4,null,3,null,3,2,-5],"l2_utf8":["a","😀","😀","c","😀",null,null,"a",null,"ab","",null,"","aa","B","a","ab","aa","b",null],"l3_f64":[-3.0,-26.751,null,-1.0,-17.015,79.401,null,"-Infinity",null,null,4.0,null,0.0,-48.615,3.0,0.0,0.0,1.864,-74.026,null],"l4_utf8":["B",null,null,"b","😀",null,"é",null,"aa","😀",null,null,"c",null,"ab","aa","zz",null,"é","a"],"l5_f64":["-Infinity",-44.075,-66.765,60.654,4.0,null,4.0,-66.148,3.0,-71.796,3.0,4.0,-2.0,50.446,-1.303,3.0,3.0,1.0,22.653,74.43],"l6_utf8":[null,"aa",null,"a","zz",null,"B","b","aa",null,"zz","aa","ab","","aa","b","😀","ab","é",null]}},"right":{"fields":[{"name":"l4_utf8","type":"utf8"},{"name":"r0_bool","type":"bool"}],"columns":{"r0_bool":[true,true,true],"l4_utf8":["aa","zz","b"]}}},"ops":[{"op":"head","n":6},{"op":"join","right":"right","on":["l4_utf8"],"how":"anti"},{"op":"groupBy","keys":["l2_utf8"],"aggs":{"n2":{"agg":"len"},"a2_0":{"agg":"count","arg":{"col":"l5_f64"}},"a2_1":{"agg":"std","arg":{"col":"l5_f64"}},"a2_2":{"agg":"nUnique","arg":{"col":"l6_utf8"}}}},{"op":"sort","keys":[{"expr":{"col":"a2_0"},"descending":false}],"nullsLast":true}],"expected":{"fields":[{"name":"l2_utf8","type":"utf8"},{"name":"n2","type":"u32"},{"name":"a2_0","type":"u32"},{"name":"a2_1","type":"f64"},{"name":"a2_2","type":"u32"}],"columns":{"l2_utf8":[null,"a","😀"],"n2":[1,1,3],"a2_0":[0,1,3],"a2_1":[null,null,36.13338011220834],"a2_2":[1,1,3]}}},{"id":"random/0296","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_utf8","type":"utf8"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_utf8":[],"l3_utf8":[]}}},"ops":[{"op":"filter","predicate":{"fn":"gt","args":[{"fn":"mod","args":[{"col":"l1_i32"},{"col":"l1_i32"}]},{"col":"l1_i32"}]}},{"op":"unique","columns":["l2_utf8"]},{"op":"sort","keys":[{"expr":{"col":"l0_utf8"},"descending":false},{"expr":{"col":"l3_utf8"},"descending":false}],"nullsLast":true}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_utf8","type":"utf8"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_utf8":[],"l3_utf8":[]}}},{"id":"random/0297","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_f64","type":"f64"},{"name":"l4_i32","type":"i32"},{"name":"l5_utf8","type":"utf8"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_f64":[],"l3_f64":[],"l4_i32":[],"l5_utf8":[]}}},"ops":[{"op":"sort","keys":[{"expr":{"col":"l5_utf8"},"descending":false},{"expr":{"col":"l4_i32"},"descending":true}],"nullsLast":true}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_f64","type":"f64"},{"name":"l4_i32","type":"i32"},{"name":"l5_utf8","type":"utf8"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_f64":[],"l3_f64":[],"l4_i32":[],"l5_utf8":[]}}},{"id":"random/0298","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_f64","type":"f64"},{"name":"l4_i32","type":"i32"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_i32":[],"l3_f64":[],"l4_i32":[]}},"right":{"fields":[{"name":"l1_i32","type":"i32"},{"name":"r0_utf8","type":"utf8"}],"columns":{"r0_utf8":["b",null,null,"B",null,null,null,"a",null,"a",null,null,"B",null,null,"aa","😀",null,"","c","b","b","a",null,null,null,"c",null,null,"b"],"l1_i32":[4,-1,-2,-1,4,null,-2,-1,-1,-2,null,null,-1,null,-2,-1,4,-2,4,-1,4,-2,4,-1,-1,-1,-2,-1,-2,-1]}}},"ops":[{"op":"join","right":"right","on":["l1_i32"],"how":"left"}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_f64","type":"f64"},{"name":"l4_i32","type":"i32"},{"name":"r0_utf8","type":"utf8"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_i32":[],"l3_f64":[],"l4_i32":[],"r0_utf8":[]}}},{"id":"random/0299","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_i32","type":"i32"},{"name":"l4_f64","type":"f64"}],"columns":{"l0_utf8":["😀","é","a","zz","",null,"","ab","b","b","b","","é","zz","aa","ab","a","ab","a","b",null,"é","","😀","B","B","😀","é",null,"a","a","ab","é",null,null,"é","","","aa","b","é","a","é","a","é","zz","ab","aa","b","é","B","a","aa","é","c","b","zz","c",null,"b"],"l1_i32":[4,-1,-3,-3,1,5,-5,4,1,3,-5,1,-4,-1,-5,-1,-3,-2,3,-624,3,5,5,0,-1,5,-4,4,5,-4,1,2,4,-1,-5,4,2,3,null,2147483647,null,0,-1,3,5,-2,2,4,-2,null,0,5,5,4,-4,5,-3,-4,1,3],"l2_f64":[3.0,-89.222,-3.0,1.0,-13.516,4.0,-91.489,null,"-0",-1.0,3.0,-92.838,null,68.566,1.0,94.688,-47.088,1.0,34.821,-2.0,-0.594,3.926,-53.736,58.087,null,88.539,77.812,3.0,75.16,88.596,null,-2.0,-1.0,3.0,0.699,-69.835,-2.0,-2.0,-2.0,1.0,11.063,79.633,-2.0,-74.547,1.0,96.52,45.294,53.723,2.0,-4.0,-2.0,5.525,-96.454,3.0,null,4.0,2.0,4.78,"Infinity",4.0],"l3_i32":[-4,2,-2,-5,-3,1,-4,null,-4,-2147483648,2147483647,-1,-2,4,-4,2,3,-5,0,3,null,3,-1,4,-4,-5,1,5,1,1,-5,null,-1,-2147483648,-1,475,3,4,0,-4,-4,-2147483648,2,4,4,0,1,-2,5,0,4,4,-3,4,null,-1,-4,-4,-5,3],"l4_f64":[-81.288,1.0,-4.0,-40.736,-27.874,null,"-Infinity",73.25,-3.0,-87.428,1.0,-91.868,1.0,"NaN",3.0,1.0,-82.594,null,-1.0,81.486,-6.757,null,1.0,4.0,1.0,34.593,0.0,-25.71,4.0,null,-1.0,"NaN",4.0,-2.0,44.859,32.274,-2.0,null,6.368,-1.0,null,-4.0,-40.202,"NaN",null,null,0.0,-9.888,50.082,-83.599,2.0,null,"NaN",-3.0,83.74,-60.385,-2.0,-77.321,-65.482,-83.571]}}},"ops":[{"op":"withColumns","exprs":{"x0":{"fn":"add","args":[{"lit":2.5},{"fn":"sub","args":[{"col":"l4_f64"},{"lit":2.5}]}]}}},{"op":"groupBy","keys":["l0_utf8"],"aggs":{"n1":{"agg":"len"},"a1_0":{"agg":"median","arg":{"col":"l4_f64"}},"a1_1":{"agg":"nUnique","arg":{"col":"x0"}}}},{"op":"unique","columns":["l0_utf8"]}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"n1","type":"u32"},{"name":"a1_0","type":"f64"},{"name":"a1_1","type":"u32"}],"columns":{"l0_utf8":["😀","é","a","zz","",null,"ab","b","aa","B","c"],"n1":[3,11,8,4,6,6,5,8,4,3,2],"a1_0":[0.0,-1.0,-2.5,-2.0,-27.874,-2.0,37.125,-2.0,4.684,2.0,3.2094999999999914],"a1_1":[3,8,5,4,6,6,5,8,4,3,2]}}},{"id":"random/0300","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_f64","type":"f64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_utf8":[],"l3_f64":[]}}},"ops":[{"op":"sort","keys":[{"expr":{"col":"l3_f64"},"descending":false},{"expr":{"col":"l0_utf8"},"descending":true}],"nullsLast":false},{"op":"filter","predicate":{"fn":"eq","args":[{"col":"l2_utf8"},{"lit":"a"}]}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_f64","type":"f64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_utf8":[],"l3_f64":[]}}},{"id":"random/0301","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_f64","type":"f64"}],"columns":{"l0_utf8":["a"],"l1_i32":[-736],"l2_f64":[-2.0],"l3_f64":[9.993]}}},"ops":[{"op":"filter","predicate":{"fn":"gt","args":[{"col":"l2_f64"},{"lit":0.5}]}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_f64","type":"f64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_f64":[],"l3_f64":[]}}},{"id":"random/0302","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_f64","type":"f64"},{"name":"l4_i64","type":"i64"},{"name":"l5_utf8","type":"utf8"}],"columns":{"l0_utf8":["zz","B","zz","B","","","é","aa","","B","aa","é",null,"zz","aa",null,"a",null,"zz","ab","zz","zz","😀","😀","a","c",null,"zz","😀","B","a","a","c",null,null,"é","é","aa","B","😀","zz","b","zz","zz","zz","aa","ab","😀","ab","aa","aa","b","a","a","b","ab","😀","","ab","b"],"l1_i32":[null,5,-3,-5,null,2,-5,-2,3,3,-3,-2147483648,-3,-4,-1,-2147483648,-2147483648,3,1,-4,-4,2,-2,-3,2,-2147483648,4,3,-3,-5,4,null,3,-2,5,-2,null,5,0,-5,null,-3,4,-3,-1,4,-4,-3,3,0,4,4,-327,-1,-3,-5,4,0,-3,2],"l2_f64":[24.279,2.0,33.381,-1.0,71.719,1.0,-3.0,null,1.0,64.25,1.0,"NaN",-3.0,null,3.0,-45.003,36.915,-42.858,"NaN",4.0,3.0,16.196,53.754,50.722,-99.994,67.264,3.0,4.0,-3.0,94.468,3.0,-63.206,-4.0,3.0,-37.287,1.0,4.0,0.0,-3.0,null,4.0,71.534,-1.0,-87.435,1.0,4.0,3.0,-19.91,null,"-0",6.79,94.772,3.0,-3.0,3.0,"NaN",null,4.0,-2.0,-75.0],"l3_f64":[97.925,-1.0,2.0,32.875,4.0,4.0,-1.0,1.0,2.0,"-Infinity",4.773,71.286,-1.0,-81.403,-96.953,50.214,82.771,"NaN",-2.0,12.154,"NaN",-81.006,2.0,-37.136,-3.0,2.0,88.384,-19.163,-35.765,-30.188,1.0,null,4.0,4.0,null,3.0,-1.0,-93.227,-33.405,null,-28.687,-43.035,2.0,77.431,null,73.607,-4.0,null,0.0,74.969,"NaN",-1.0,"NaN",null,-43.08,-2.0,36.227,1.312,3.0,-48.59],"l4_i64":["9007199254740992","9007199254740994","9007199254740995","-4","-4611686018427387904","4","-4611686018427387904","-4611686018427387904","1","1","9007199254740992","-4611686018427387904",null,"-4611686018427387904","-4611686018427387904","5","9007199254740992","5","9007199254740993","-1","9007199254740995","4","2","0","2","5","9007199254740992","9007199254740993","0","3","-2","-5","-3","-4611686018427387904","-2","2","-1","9007199254740994","9007199254740992","9007199254740994","9007199254740995","-4611686018427387904","1","9007199254740993","-1","9007199254740995","9007199254740993","-5",null,"1","4","-4611686018427387904","1","-4611686018427387904","-4611686018427387904","3","9007199254740992","-4611686018427387904","-4611686018427387904","5"],"l5_utf8":["b","😀","b","b","b",null,"é","b","é","b","c",null,"aa","","","😀","aa","ab","a","c","aa","c","😀","zz","","ab",null,"😀","a",null,"a","b","zz","B","b","a","aa","","aa","b","😀","zz","ab","ab","aa","a","c","B","b","é","zz","b","😀","a","a","zz","a",null,"é",""]}},"right":{"fields":[{"name":"l4_i64","type":"i64"},{"name":"r0_utf8","type":"utf8"}],"columns":{"r0_utf8":[],"l4_i64":[]}}},"ops":[{"op":"head","n":10},{"op":"join","right":"right","on":["l4_i64"],"how":"inner"}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_f64","type":"f64"},{"name":"l4_i64","type":"i64"},{"name":"l5_utf8","type":"utf8"},{"name":"r0_utf8","type":"utf8"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_f64":[],"l3_f64":[],"l4_i64":[],"l5_utf8":[],"r0_utf8":[]}}},{"id":"random/0303","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_i64","type":"i64"}],"columns":{"l0_utf8":["B",null,"c",null,null,"aa",null,"B","aa","c",null,null,null,"ab","zz","aa",null,"c","zz","B"],"l1_i32":[-1,2,1,5,null,2147483647,-2,0,null,-5,5,-5,-3,1,null,-4,-4,0,-1,-4],"l2_f64":[null,3.0,80.379,-1.0,4.0,null,"-0",null,8.109,92.95,90.369,null,0.0,-84.745,null,72.89,"NaN",null,null,-26.295],"l3_i64":["-4","-4611686018427387904",null,"0","2",null,"-4611686018427387904","-4611686018427387904","9007199254740993","-4611686018427387904","-4611686018427387904",null,"9007199254740994","-2",null,null,null,"-4611686018427387904",null,null]}}},"ops":[{"op":"sort","keys":[{"expr":{"col":"l1_i32"},"descending":false}],"nullsLast":true},{"op":"groupBy","keys":["l3_i64"],"aggs":{"n1":{"agg":"len"},"a1_0":{"agg":"mean","arg":{"col":"l2_f64"}}}},{"op":"withColumns","exprs":{"x2":{"fn":"when","args":[{"fn":"neq","args":[{"col":"a1_0"},{"col":"a1_0"}]},{"col":"a1_0"},{"fn":"div","args":[{"col":"a1_0"},{"lit":-1}]}]}}}],"expected":{"fields":[{"name":"l3_i64","type":"i64"},{"name":"n1","type":"u32"},{"name":"a1_0","type":"f64"},{"name":"x2","type":"f64"}],"columns":{"l3_i64":["-4611686018427387904",null,"9007199254740994","-4","-2","0","2","9007199254740993"],"n1":[6,8,1,1,1,1,1,1],"a1_0":[46.579750000000004,"NaN",0.0,null,-84.745,-1.0,4.0,8.109],"x2":[-46.579750000000004,"NaN","-0",null,84.745,1.0,-4.0,-8.109]}}},{"id":"random/0304","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_i32","type":"i32"},{"name":"l5_i64","type":"i64"},{"name":"l6_f64","type":"f64"}],"columns":{"l0_utf8":["c",null,"😀","",""],"l1_i32":[null,-1,5,-3,-2],"l2_f64":[null,1.0,77.277,null,7.574],"l3_utf8":[null,null,null,"","B"],"l4_i32":[null,5,2,-2,-5],"l5_i64":["9007199254740994",null,"0","-4",null],"l6_f64":[2.0,-66.123,0.0,null,99.73]}}},"ops":[{"op":"unique","columns":["l4_i32"]},{"op":"filter","predicate":{"fn":"neq","args":[{"fn":"neg","args":[{"lit":2}]},{"lit":2.5}]}},{"op":"head","n":2},{"op":"head","n":8}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_i32","type":"i32"},{"name":"l5_i64","type":"i64"},{"name":"l6_f64","type":"f64"}],"columns":{"l0_utf8":["c",null],"l1_i32":[null,-1],"l2_f64":[null,1.0],"l3_utf8":[null,null],"l4_i32":[null,5],"l5_i64":["9007199254740994",null],"l6_f64":[2.0,-66.123]}}},{"id":"random/0305","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_i32","type":"i32"},{"name":"l5_i64","type":"i64"},{"name":"l6_f64","type":"f64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_utf8":[],"l3_utf8":[],"l4_i32":[],"l5_i64":[],"l6_f64":[]}}},"ops":[{"op":"filter","predicate":{"fn":"isNotNull","args":[{"col":"l6_f64"}]}},{"op":"sort","keys":[{"expr":{"col":"l0_utf8"},"descending":false}],"nullsLast":false}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_i32","type":"i32"},{"name":"l5_i64","type":"i64"},{"name":"l6_f64","type":"f64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_utf8":[],"l3_utf8":[],"l4_i32":[],"l5_i64":[],"l6_f64":[]}}},{"id":"random/0306","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_f64","type":"f64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_f64":[],"l3_utf8":[],"l4_f64":[]}}},"ops":[{"op":"filter","predicate":{"fn":"and","args":[{"fn":"isNull","args":[{"col":"l0_utf8"}]},{"fn":"eq","args":[{"col":"l2_f64"},{"col":"l2_f64"}]}]}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_f64","type":"f64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_f64":[],"l3_utf8":[],"l4_f64":[]}}},{"id":"random/0307","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_i64","type":"i64"},{"name":"l4_bool","type":"bool"},{"name":"l5_utf8","type":"utf8"}],"columns":{"l0_utf8":["aa","aa","c","ab","","","B","","a","a","","B","c","","é","zz","c","c","B","zz"],"l1_i32":[-2,-1,-186,3,-2,2,0,-1,-1,-3,4,0,5,-4,1,-2,-5,-5,4,4],"l2_utf8":["ab","c","ab","ab","ab","","zz","B","c","","b","zz","aa","b","B","zz","😀","zz","zz","zz"],"l3_i64":["-3","-4611686018427387904","9007199254740994","9007199254740994","-4611686018427387904","-3","4","0","9007199254740992","1","-4611686018427387904","-4611686018427387904","9007199254740993","0","3","9007199254740995","9007199254740993","-4611686018427387904","9007199254740995","9007199254740995"],"l4_bool":[true,false,true,false,true,true,false,false,false,false,true,false,true,true,false,false,false,false,false,true],"l5_utf8":["é","c","c","aa","aa","c","é","é","é","c","aa","ab","B","c","","B","😀","a","zz","a"]}}},"ops":[{"op":"groupBy","keys":["l5_utf8"],"aggs":{"n0":{"agg":"len"},"a0_0":{"agg":"first","arg":{"col":"l4_bool"}},"a0_1":{"agg":"last","arg":{"col":"l2_utf8"}}}},{"op":"withColumns","exprs":{"x1":{"fn":"floorDiv","args":[{"fn":"neg","args":[{"lit":-1}]},{"lit":-1}]}}},{"op":"withColumns","exprs":{"x2":{"col":"x1"}}},{"op":"head","n":9}],"expected":{"fields":[{"name":"l5_utf8","type":"utf8"},{"name":"n0","type":"u32"},{"name":"a0_0","type":"bool"},{"name":"a0_1","type":"utf8"},{"name":"x1","type":"i32"},{"name":"x2","type":"i32"}],"columns":{"l5_utf8":["é","c","aa","ab","B","","😀","a","zz"],"n0":[4,5,3,1,2,1,1,2,1],"a0_0":[true,false,false,false,true,false,false,false,false],"a0_1":["c","b","b","zz","zz","B","😀","zz","zz"],"x1":[-1,-1,-1,-1,-1,-1,-1,-1,-1],"x2":[-1,-1,-1,-1,-1,-1,-1,-1,-1]}}},{"id":"random/0308","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i64","type":"i64"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_utf8","type":"utf8"}],"columns":{"l0_utf8":["ab",null,"😀","aa","zz","aa","😀","ab","ab","a","",null,"😀","B","é","b","é","","ab","b","a","zz",null,"zz","B","c","b","",null,"ab","B","aa","zz","aa","aa","c","c","ab",null,null,null,"é","b","aa","a",null,null,null,"B","","B","zz","aa",null,"zz",null,"é","zz",null,"B"],"l1_i32":[3,null,null,null,null,4,null,null,null,1,-5,4,2,null,-5,3,null,-5,1,null,5,2,0,2147483647,2,null,1,5,0,5,3,4,-2,-3,-5,0,-1,null,2,5,1,null,1,-1,3,3,0,4,4,1,null,4,-2147483648,-4,-2,5,null,null,-4,2],"l2_i64":["1","-4611686018427387904",null,"9007199254740992","-4611686018427387904","2","9007199254740994","9007199254740993",null,null,"-4611686018427387904","-4611686018427387904","9007199254740992","-4611686018427387904",null,"-4611686018427387904","9007199254740992",null,"-1",null,"-4611686018427387904",null,"9007199254740992",null,"1","-3","-4611686018427387904","5","9007199254740994","9007199254740993","9007199254740993","-4611686018427387904","-5",null,null,"-4611686018427387904","9007199254740992","-4611686018427387904","5",null,"9007199254740995","0","-4611686018427387904","-4611686018427387904","-4611686018427387904","9007199254740993",null,"9007199254740994","9007199254740993","9007199254740993","9007199254740992","3",null,"4",null,"-4","-4611686018427387904","-5","-2","9007199254740992"],"l3_utf8":["a","aa","","zz","ab","B","B","","é","aa","zz","aa","zz","😀","B","aa",null,null,"zz","c","ab","é","b","B","é","B",null,"","",null,"é","b","ab","😀","c","aa","ab","a","zz","zz","b",null,"😀","","a","aa","b","é","ab","","b","zz",null,"a",null,null,null,"zz",null,"B"],"l4_utf8":["a","B",null,"zz","B",null,"zz","é","b","zz","😀","aa","aa","zz","zz","é","ab","é",null,"b","",null,null,"é",null,"b","a",null,"zz",null,null,"ab",null,null,"B","aa","b","😀",null,"zz","B","c","c",null,"b",null,"é","b","c",null,"",null,null,null,"é","b","","ab","é",null],"l5_utf8":["é","B",null,null,null,"aa",null,null,null,"ab",null,"b","c","é","😀","c",null,"B","ab",null,"",null,null,null,"a",null,"","",null,"B","zz","","a","😀","ab",null,null,null,"B","B","😀","aa","c","a","zz",null,"b",null,null,"c",null,"a",null,"B","b","a",null,"ab","ab",""]}}},"ops":[{"op":"filter","predicate":{"fn":"neq","args":[{"col":"l1_i32"},{"lit":-1}]}},{"op":"withColumns","exprs":{"x1":{"col":"l1_i32"}}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i64","type":"i64"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_utf8","type":"utf8"},{"name":"x1","type":"i32"}],"columns":{"l0_utf8":["ab","aa","a","",null,"😀","é","b","","ab","a","zz",null,"zz","B","b","",null,"ab","B","aa","zz","aa","aa","c",null,null,null,"b","a",null,null,null,"B","","zz","aa",null,"zz",null,null,"B"],"l1_i32":[3,4,1,-5,4,2,-5,3,-5,1,5,2,0,2147483647,2,1,5,0,5,3,4,-2,-3,-5,0,2,5,1,1,3,3,0,4,4,1,4,-2147483648,-4,-2,5,-4,2],"l2_i64":["1","2",null,"-4611686018427387904","-4611686018427387904","9007199254740992",null,"-4611686018427387904",null,"-1","-4611686018427387904",null,"9007199254740992",null,"1","-4611686018427387904","5","9007199254740994","9007199254740993","9007199254740993","-4611686018427387904","-5",null,null,"-4611686018427387904","5",null,"9007199254740995","-4611686018427387904","-4611686018427387904","9007199254740993",null,"9007199254740994","9007199254740993","9007199254740993","3",null,"4",null,"-4","-2","9007199254740992"],"l3_utf8":["a","B","aa","zz","aa","zz","B","aa",null,"zz","ab","é","b","B","é",null,"","",null,"é","b","ab","😀","c","aa","zz","zz","b","😀","a","aa","b","é","ab","","zz",null,"a",null,null,null,"B"],"l4_utf8":["a",null,"zz","😀","aa","aa","zz","é","é",null,"",null,null,"é",null,"a",null,"zz",null,null,"ab",null,null,"B","aa",null,"zz","B","c","b",null,"é","b","c",null,null,null,null,"é","b","é",null],"l5_utf8":["é","aa","ab",null,"b","c","😀","c","B","ab","",null,null,null,"a","","",null,"B","zz","","a","😀","ab",null,"B","B","😀","c","zz",null,"b",null,null,"c","a",null,"B","b","a","ab",""],"x1":[3,4,1,-5,4,2,-5,3,-5,1,5,2,0,2147483647,2,1,5,0,5,3,4,-2,-3,-5,0,2,5,1,1,3,3,0,4,4,1,4,-2147483648,-4,-2,5,-4,2]}}},{"id":"random/0309","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_i32","type":"i32"},{"name":"l5_f64","type":"f64"},{"name":"l6_f64","type":"f64"}],"columns":{"l0_utf8":[null,"a","B","a","zz","","","b","é","b","😀","😀",null,"é","B",null,"","😀",null,"B"],"l1_i32":[3,-1,0,2147483647,1,4,null,0,null,-4,-3,0,-2147483648,null,null,-3,3,-4,4,-3],"l2_i32":[null,null,-136,-2,0,4,-1,2,0,null,-1,null,2,0,0,4,4,-2,5,-3],"l3_utf8":[null,"é","zz","B","b","ab","zz",null,"","aa","ab","😀",null,"",null,null,"😀","","ab",""],"l4_i32":[-4,4,null,2147483647,null,2,3,-1,-3,1,null,null,-5,null,null,-4,null,5,-3,null],"l5_f64":[1.0,-4.0,2.0,null,null,-7.276,-32.276,0.0,0.0,3.0,-2.0,-90.833,-1.0,null,null,-1.0,2.0,2.0,2.0,null],"l6_f64":[null,62.979,null,null,null,"-0","Infinity",null,43.278,null,3.0,40.34,45.49,1.0,-2.0,-1.0,null,null,-3.0,0.0]}}},"ops":[{"op":"filter","predicate":{"fn":"strStartsWith","args":[{"col":"l0_utf8"}],"prefix":"a"}},{"op":"withColumns","exprs":{"x1":{"fn":"fillNull","args":[{"lit":2},{"lit":1.5}]}}},{"op":"withColumns","exprs":{"x2":{"fn":"add","args":[{"col":"l4_i32"},{"fn":"floorDiv","args":[{"col":"l5_f64"},{"col":"l2_i32"}]}]}}},{"op":"withColumns","exprs":{"x3":{"fn":"fillNull","args":[{"fn":"abs","args":[{"col":"l2_i32"}]},{"lit":0}]}}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_i32","type":"i32"},{"name":"l5_f64","type":"f64"},{"name":"l6_f64","type":"f64"},{"name":"x1","type":"f64"},{"name":"x2","type":"f64"},{"name":"x3","type":"i32"}],"columns":{"l0_utf8":["a","a"],"l1_i32":[-1,2147483647],"l2_i32":[null,-2],"l3_utf8":["é","B"],"l4_i32":[4,2147483647],"l5_f64":[-4.0,null],"l6_f64":[62.979,null],"x1":[2.0,2.0],"x2":[null,null],"x3":[0,2]}}},{"id":"random/0310","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_bool","type":"bool"}],"columns":{"l0_utf8":["ab","zz",null,"B","a","ab","B","B",null,null,null,"","",null,null,"😀","aa","",null,null,"c","b","😀","é","aa","zz","ab","b",null,"é",null,"é","ab","é","ab",null,"aa","","",null,"",null,null,"a",null,null,"aa","b","",null,"c","","c",null,"","c","😀","c","b","zz"],"l1_i32":[null,1,-5,-4,null,-2,null,1,2147483647,null,2,null,-3,-4,3,5,null,5,5,null,-4,1,-3,3,-4,0,null,null,0,-4,-2147483648,-2,-5,-3,2,-3,2,null,4,645,null,-1,5,null,null,null,-4,null,4,2,null,null,null,null,0,null,2,-2,-1,-2],"l2_f64":[-2.0,null,98.071,89.214,4.0,-50.322,95.241,null,-4.0,null,null,null,null,null,null,-2.0,null,29.67,70.479,-49.82,-14.163,1.0,null,null,null,null,-2.0,-4.0,4.0,-1.0,"NaN",null,33.093,2.0,2.0,-52.046,-4.0,"NaN",-61.733,null,-40.678,null,-17.927,0.0,3.0,2.0,99.723,72.53,3.0,1.0,-65.399,95.754,null,null,62.832,-75.369,44.201,null,null,99.544],"l3_bool":[null,null,null,true,null,true,false,false,false,true,true,false,null,null,true,false,false,true,true,false,false,true,true,false,null,null,false,false,false,true,null,false,null,true,null,false,null,true,false,false,false,true,false,null,true,true,true,null,null,true,null,null,false,false,false,null,null,null,false,false]}}},"ops":[{"op":"withColumns","exprs":{"x0":{"col":"l1_i32"}}},{"op":"groupBy","keys":["l3_bool"],"aggs":{"n1":{"agg":"len"},"a1_0":{"agg":"count","arg":{"col":"l1_i32"}},"a1_1":{"agg":"mean","arg":{"col":"x0"}}}},{"op":"filter","predicate":{"fn":"isNotNull","args":[{"col":"a1_0"}]}}],"expected":{"fields":[{"name":"l3_bool","type":"bool"},{"name":"n1","type":"u32"},{"name":"a1_0","type":"u32"},{"name":"a1_1","type":"f64"}],"columns":{"l3_bool":[null,true,false],"n1":[20,17,23],"a1_0":[13,13,14],"a1_1":[-165191050.76923078,-0.23076923076923078,153391735.57142857]}}},{"id":"random/0311","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_bool","type":"bool"},{"name":"l4_f64","type":"f64"}],"columns":{"l0_utf8":["c"],"l1_i32":[-5],"l2_f64":[68.974],"l3_bool":[true],"l4_f64":[2.0]}}},"ops":[{"op":"head","n":5},{"op":"sort","keys":[{"expr":{"col":"l2_f64"},"descending":false}],"nullsLast":false},{"op":"filter","predicate":{"fn":"isIn","args":[{"col":"l0_utf8"}],"values":["B","b","é"]}},{"op":"sort","keys":[{"expr":{"col":"l3_bool"},"descending":true}],"nullsLast":false}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_bool","type":"bool"},{"name":"l4_f64","type":"f64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_f64":[],"l3_bool":[],"l4_f64":[]}}},{"id":"random/0312","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_f64","type":"f64"},{"name":"l4_bool","type":"bool"}],"columns":{"l0_utf8":["zz","é","B","zz","😀","é","😀","ab",null,null,"a","","B",null,null,null,null,"😀",null,null],"l1_i32":[0,0,-4,5,5,null,1,-4,-2,-1,4,-2,-4,-3,-5,3,null,null,-2,4],"l2_f64":[null,null,0.0,1.0,-55.526,-63.151,40.087,-4.0,-44.212,85.092,67.789,3.0,-4.0,-84.354,null,49.33,-20.72,19.609,-48.678,null],"l3_f64":[-1.0,3.0,null,null,null,null,null,52.088,-4.0,-3.0,91.669,-67.019,-4.0,-4.0,null,-4.0,null,-48.751,0.0,-77.087],"l4_bool":[false,true,true,null,false,true,true,false,null,false,true,true,false,true,false,true,false,true,false,true]}}},"ops":[{"op":"unique"},{"op":"filter","predicate":{"fn":"or","args":[{"fn":"isIn","args":[{"col":"l0_utf8"}],"values":["B","ab","c"]},{"col":"l4_bool"}]}},{"op":"withColumns","exprs":{"x2":{"lit":0.5}}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_f64","type":"f64"},{"name":"l4_bool","type":"bool"},{"name":"x2","type":"f64"}],"columns":{"l0_utf8":["é","B","é","😀","ab","a","","B",null,null,"😀",null],"l1_i32":[0,-4,null,1,-4,4,-2,-4,-3,3,null,4],"l2_f64":[null,0.0,-63.151,40.087,-4.0,67.789,3.0,-4.0,-84.354,49.33,19.609,null],"l3_f64":[3.0,null,null,null,52.088,91.669,-67.019,-4.0,-4.0,-4.0,-48.751,-77.087],"l4_bool":[true,true,true,true,false,true,true,false,true,true,true,true],"x2":[0.5,0.5,0.5,0.5,0.5,0.5,0.5,0.5,0.5,0.5,0.5,0.5]}}},{"id":"random/0313","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_f64","type":"f64"},{"name":"l4_f64","type":"f64"},{"name":"l5_f64","type":"f64"},{"name":"l6_utf8","type":"utf8"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_i32":[],"l3_f64":[],"l4_f64":[],"l5_f64":[],"l6_utf8":[]}}},"ops":[{"op":"sort","keys":[{"expr":{"col":"l3_f64"},"descending":false},{"expr":{"col":"l0_utf8"},"descending":true}],"nullsLast":false}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_f64","type":"f64"},{"name":"l4_f64","type":"f64"},{"name":"l5_f64","type":"f64"},{"name":"l6_utf8","type":"utf8"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_i32":[],"l3_f64":[],"l4_f64":[],"l5_f64":[],"l6_utf8":[]}}},{"id":"random/0314","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i64","type":"i64"},{"name":"l3_i32","type":"i32"},{"name":"l4_f64","type":"f64"},{"name":"l5_i32","type":"i32"},{"name":"l6_f64","type":"f64"}],"columns":{"l0_utf8":["😀","é","B","B","b","aa","c","B","a","","B","a","zz","zz","c","ab","b","","é","aa"],"l1_i32":[4,-2,5,-5,-1,-1,0,4,-5,-4,5,2,-5,-2,-1,-1,4,640,4,2],"l2_i64":["9007199254740993","-5","3","9007199254740992","4","9007199254740995","1","-4611686018427387904","1","9007199254740993","1","-5","9007199254740994","-4611686018427387904","-4611686018427387904","9007199254740994","-4","9007199254740995","-4611686018427387904","9007199254740993"],"l3_i32":[-5,-3,5,3,4,0,-4,-5,1,-4,0,-4,-2147483648,-4,5,-4,-2,-4,3,-5],"l4_f64":[85.662,2.0,36.4,3.0,-1.0,4.0,-29.808,-50.279,-17.544,"-0",0.0,-3.0,"NaN",4.0,36.122,4.0,89.992,-63.832,-86.765,10.751],"l5_i32":[-2,5,5,-3,5,-5,5,2,-4,1,2,3,-4,3,0,2147483647,0,-4,1,1],"l6_f64":[1.0,-44.722,1.0,3.0,8.396,-92.965,4.0,4.0,76.008,0.0,89.842,-99.56,10.14,-1.0,1.0,-4.0,-43.084,4.0,82.845,4.0]}},"right":{"fields":[{"name":"l2_i64","type":"i64"},{"name":"r0_f64","type":"f64"}],"columns":{"r0_f64":[-30.34,1.0,null],"l2_i64":["-4611686018427387904","-4611686018427387904","9007199254740992"]}}},"ops":[{"op":"groupBy","keys":["l2_i64"],"aggs":{"n0":{"agg":"len"},"a0_0":{"agg":"mean","arg":{"col":"l4_f64"}}}},{"op":"filter","predicate":{"fn":"eq","args":[{"fn":"div","args":[{"col":"a0_0"},{"lit":2.5}]},{"fn":"neg","args":[{"col":"a0_0"}]}]}},{"op":"join","right":"right","on":["l2_i64"],"how":"full"}],"expected":{"fields":[{"name":"l2_i64","type":"i64"},{"name":"n0","type":"u32"},{"name":"a0_0","type":"f64"},{"name":"r0_f64","type":"f64"}],"columns":{"l2_i64":["9007199254740994","-4611686018427387904","-4611686018427387904","9007199254740992"],"n0":[2,null,null,null],"a0_0":["NaN",null,null,null],"r0_f64":[null,-30.34,1.0,null]}}},{"id":"random/0315","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_bool","type":"bool"}],"columns":{"l0_utf8":["aa","zz","","zz","","ab","B",null,"zz","ab","ab","",null,"aa","B","😀","","zz","é","😀"],"l1_i32":[1,-2147483648,0,null,2,-3,0,null,-4,2,null,-2147483648,-5,1,0,null,2,1,null,0],"l2_utf8":["c","b","ab","ab","😀","ab","b","😀","","é","B","ab","","b","B","a","é","a","b","é"],"l3_bool":[true,false,false,true,true,false,false,true,false,false,true,false,true,null,null,false,null,true,true,false]}}},"ops":[{"op":"head","n":4}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_bool","type":"bool"}],"columns":{"l0_utf8":["aa","zz","","zz"],"l1_i32":[1,-2147483648,0,null],"l2_utf8":["c","b","ab","ab"],"l3_bool":[true,false,false,true]}}},{"id":"random/0316","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_bool","type":"bool"}],"columns":{"l0_utf8":["","aa","","B","ab"],"l1_i32":[-3,3,4,1,-1],"l2_f64":[-4.0,"NaN",-41.699,-4.0,45.194],"l3_bool":[false,true,true,null,false]}}},"ops":[{"op":"groupBy","keys":["l0_utf8"],"aggs":{"n0":{"agg":"len"},"a0_0":{"agg":"std","arg":{"col":"l2_f64"}},"a0_1":{"agg":"max","arg":{"col":"l1_i32"}},"a0_2":{"agg":"nUnique","arg":{"col":"l2_f64"}}}},{"op":"withColumns","exprs":{"x1":{"fn":"neg","args":[{"fn":"div","args":[{"col":"a0_0"},{"col":"a0_0"}]}]}}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"n0","type":"u32"},{"name":"a0_0","type":"f64"},{"name":"a0_1","type":"i32"},{"name":"a0_2","type":"u32"},{"name":"x1","type":"f64"}],"columns":{"l0_utf8":["","aa","B","ab"],"n0":[2,1,1,1],"a0_0":[26.657218543951654,null,null,null],"a0_1":[4,3,1,-1],"a0_2":[2,1,1,1],"x1":[-1.0,null,null,null]}}},{"id":"random/0317","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_i32","type":"i32"}],"columns":{"l0_utf8":["a","😀","ab","é","c","zz","aa","aa","ab","b","B","c","","aa","😀","c","b","b","b","aa","B","b","B","ab","","é","ab","a","","aa","B","a","zz","zz","zz","c","😀","😀","é","zz","zz","😀","B","a","aa","é","zz","b","zz","aa","😀","😀","😀","b","zz","é","😀","c","c","😀"],"l1_i32":[5,0,5,4,-4,-4,-2,0,2,4,0,3,-3,-2147483648,-2,-4,1,1,-4,0,0,5,-1,5,-5,-4,-2,1,1,0,-1,-4,2,-1,-1,4,-4,2147483647,-5,5,-5,4,-2147483648,-2,-3,0,-5,5,5,2,-3,5,-2,1,-4,-1,2,5,-2,1],"l2_i32":[5,5,3,5,-1,4,3,2,-5,0,5,-3,4,3,5,-4,5,-4,-2,0,1,3,0,-3,1,4,-1,1,-5,2147483647,0,-2,5,3,-4,-3,2,4,-1,0,-4,-2147483648,1,0,-5,1,1,0,-3,3,-5,1,0,-4,0,-3,-1,-5,2147483647,0],"l3_i32":[2,3,3,2,2,2,0,0,-5,-4,-4,1,0,-1,5,-3,-3,1,-4,-5,0,0,5,-2,1,-2,4,0,4,2,-108,-1,-2,-4,1,-4,-3,5,-2,-5,2,-2147483648,2147483647,-1,1,5,-4,-1,4,-5,2,3,-3,-4,4,-4,-1,1,-1,4]}}},"ops":[{"op":"withColumns","exprs":{"x0":{"col":"l3_i32"}}},{"op":"withColumns","exprs":{"x1":{"fn":"fillNull","args":[{"fn":"sub","args":[{"col":"x0"},{"col":"l2_i32"}]},{"lit":1.5}]}}},{"op":"head","n":5}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_i32","type":"i32"},{"name":"x0","type":"i32"},{"name":"x1","type":"f64"}],"columns":{"l0_utf8":["a","😀","ab","é","c"],"l1_i32":[5,0,5,4,-4],"l2_i32":[5,5,3,5,-1],"l3_i32":[2,3,3,2,2],"x0":[2,3,3,2,2],"x1":[-3.0,-2.0,0.0,-3.0,3.0]}}},{"id":"random/0318","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_bool","type":"bool"},{"name":"l4_i64","type":"i64"},{"name":"l5_bool","type":"bool"},{"name":"l6_utf8","type":"utf8"}],"columns":{"l0_utf8":["aa"],"l1_i32":[2],"l2_utf8":["aa"],"l3_bool":[true],"l4_i64":["4"],"l5_bool":[false],"l6_utf8":["B"]}}},"ops":[{"op":"withColumns","exprs":{"x0":{"lit":2.5}}},{"op":"groupBy","keys":["l6_utf8"],"aggs":{"n1":{"agg":"len"},"a1_0":{"agg":"first","arg":{"col":"l1_i32"}},"a1_1":{"agg":"sum","arg":{"col":"x0"}},"a1_2":{"agg":"count","arg":{"col":"l5_bool"}}}},{"op":"unique"},{"op":"filter","predicate":{"fn":"eq","args":[{"fn":"mul","args":[{"col":"a1_0"},{"col":"a1_1"}]},{"fn":"fillNull","args":[{"lit":2.5},{"lit":1.5}]}]}}],"expected":{"fields":[{"name":"l6_utf8","type":"utf8"},{"name":"n1","type":"u32"},{"name":"a1_0","type":"i32"},{"name":"a1_1","type":"f64"},{"name":"a1_2","type":"u32"}],"columns":{"l6_utf8":[],"n1":[],"a1_0":[],"a1_1":[],"a1_2":[]}}},{"id":"random/0319","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_i32","type":"i32"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_f64":[],"l3_i32":[]}}},"ops":[{"op":"unique","columns":["l0_utf8"]},{"op":"withColumns","exprs":{"x1":{"col":"l1_i32"}}},{"op":"unique"},{"op":"withColumns","exprs":{"x3":{"fn":"isNotNull","args":[{"col":"l1_i32"}]}}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_i32","type":"i32"},{"name":"x1","type":"i32"},{"name":"x3","type":"bool"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_f64":[],"l3_i32":[],"x1":[],"x3":[]}}},{"id":"random/0320","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_i32","type":"i32"},{"name":"l4_f64","type":"f64"},{"name":"l5_bool","type":"bool"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_f64":[],"l3_i32":[],"l4_f64":[],"l5_bool":[]}}},"ops":[{"op":"head","n":5},{"op":"unique"}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_i32","type":"i32"},{"name":"l4_f64","type":"f64"},{"name":"l5_bool","type":"bool"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_f64":[],"l3_i32":[],"l4_f64":[],"l5_bool":[]}}},{"id":"random/0321","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i64","type":"i64"},{"name":"l3_f64","type":"f64"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_utf8","type":"utf8"}],"columns":{"l0_utf8":["ab","b",null,"aa","b","zz","ab","😀",null,"😀","é","B","b","aa","ab",null,"zz","c","b","c"],"l1_i32":[4,5,-2,null,2147483647,4,1,-4,null,-5,4,-3,null,3,null,4,null,-4,-4,-4],"l2_i64":["9007199254740995","-4","9007199254740994","-4611686018427387904","-4611686018427387904","-4611686018427387904","-4611686018427387904","-4611686018427387904","-4611686018427387904","2","-4611686018427387904","4","-4611686018427387904","-4611686018427387904","-4611686018427387904","-4611686018427387904","9007199254740992","9007199254740994","-1","-4611686018427387904"],"l3_f64":[-94.266,4.0,74.442,0.0,null,98.62,49.703,-91.639,7.616,-98.368,-17.88,1.962,52.256,-35.736,0.0,null,7.986,-2.0,null,"-0"],"l4_utf8":[null,null,"b","é","😀","c","aa","b","aa","B","c","B","😀","a","a","aa",null,"ab","é","a"],"l5_utf8":["a","B","c","zz","a","zz","c","zz","",null,"zz","zz","aa","c","ab","b","B","c","zz","B"]}}},"ops":[{"op":"sort","keys":[{"expr":{"col":"l5_utf8"},"descending":true}],"nullsLast":false}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i64","type":"i64"},{"name":"l3_f64","type":"f64"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_utf8","type":"utf8"}],"columns":{"l0_utf8":["😀","aa","zz","😀","é","B","b",null,"ab","aa","c",null,"ab","b","ab","b","b","zz","c",null],"l1_i32":[-5,null,4,-4,4,-3,-4,-2,1,3,-4,4,null,null,4,2147483647,5,null,-4,null],"l2_i64":["2","-4611686018427387904","-4611686018427387904","-4611686018427387904","-4611686018427387904","4","-1","9007199254740994","-4611686018427387904","-4611686018427387904","9007199254740994","-4611686018427387904","-4611686018427387904","-4611686018427387904","9007199254740995","-4611686018427387904","-4","9007199254740992","-4611686018427387904","-4611686018427387904"],"l3_f64":[-98.368,0.0,98.62,-91.639,-17.88,1.962,null,74.442,49.703,-35.736,-2.0,null,0.0,52.256,-94.266,null,4.0,7.986,"-0",7.616],"l4_utf8":["B","é","c","b","c","B","é","b","aa","a","ab","aa","a","😀",null,"😀",null,null,"a","aa"],"l5_utf8":[null,"zz","zz","zz","zz","zz","zz","c","c","c","c","b","ab","aa","a","a","B","B","B",""]}}},{"id":"random/0322","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_f64","type":"f64"}],"columns":{"l0_utf8":["B","B","",null,"😀",null,null,"b","c","c","aa","","aa","c","ab","aa",null,"😀","","","","a","ab","b","zz","é","é","😀","b","",null,"","c","aa","aa","a","B","c","😀","a","zz","b","B","B","zz","B","aa","ab","a","é","","é","b","aa","c","ab","b",null,"","a"],"l1_i32":[3,-2147483648,0,2147483647,4,null,4,5,null,2,-4,1,-3,-4,-4,-3,-3,-2,-3,-4,-5,null,-4,0,-2,3,48,null,-1,5,4,4,-4,-5,-1,-4,-2,null,-2,3,null,1,-3,null,1,-2147483648,null,5,-1,-5,1,3,-3,-5,-2,1,-4,3,-1,0],"l2_f64":[-4.0,-4.0,-3.0,-2.0,44.075,-17.403,0.0,-53.745,1.0,-42.752,89.707,-4.0,null,86.835,null,-1.0,-98.328,64.515,-3.0,59.602,-2.0,59.059,1.0,-3.0,-30.275,-74.895,-1.0,33.457,0.0,"NaN",-26.772,19.748,87.905,null,-97.472,null,null,null,-59.401,26.346,-82.706,-1.0,-34.487,-74.857,-36.659,-79.387,2.0,0.331,-12.907,-3.0,-61.585,"NaN",36.07,3.0,null,null,2.0,1.0,-4.0,-80.884],"l3_utf8":["",null,"aa","b","zz","é",null,"ab","a","aa","c","aa","","é","b","c","","aa",null,"c","aa","","B","c","ab","c","c","","c","b",null,"é","B","zz",null,"é","ab","ab","B","zz","aa","B","ab","c","b","a","b","","aa","a","aa",null,null,"a","😀","zz",null,"zz","b","é"],"l4_f64":[-56.966,0.0,-3.0,-10.992,-63.911,-0.351,null,4.0,2.0,1.0,-47.133,1.0,-5.18,-3.0,null,0.0,3.0,-96.854,0.0,-2.0,-74.906,30.148,-52.973,0.0,null,0.0,28.388,-2.0,"NaN",25.487,-4.0,null,3.234,-57.805,1.0,3.0,null,-8.562,-97.114,1.0,2.0,3.0,4.0,3.0,0.0,-4.0,3.0,6.789,null,2.0,2.0,22.428,-3.0,-2.0,2.0,4.0,-3.0,98.651,null,-3.0]}}},"ops":[{"op":"sort","keys":[{"expr":{"col":"l0_utf8"},"descending":true}],"nullsLast":false},{"op":"filter","predicate":{"fn":"gte","args":[{"col":"l4_f64"},{"fn":"neg","args":[{"col":"l1_i32"}]}]}},{"op":"sort","keys":[{"expr":{"col":"l4_f64"},"descending":true}],"nullsLast":false}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_f64","type":"f64"}],"columns":{"l0_utf8":["b",null,"é","","é","ab","b","ab","B",null,"b","c","","c","aa","a","","é","zz","b","B",null,"B",null],"l1_i32":[-1,3,48,5,3,5,5,1,-3,-3,1,-2,1,2,-1,3,1,3,1,0,-2147483648,4,-2147483648,2147483647],"l2_f64":[0.0,1.0,-1.0,"NaN","NaN",0.331,-53.745,null,-34.487,-98.328,-1.0,null,-61.585,-42.752,-97.472,26.346,-4.0,-74.895,-36.659,-3.0,-4.0,-26.772,-79.387,-2.0],"l3_utf8":["c","zz","c","b",null,"","ab","zz","ab","","B","😀","aa","aa",null,"zz","aa","c","b","c",null,null,"a","b"],"l4_f64":["NaN",98.651,28.388,25.487,22.428,6.789,4.0,4.0,4.0,3.0,3.0,2.0,2.0,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,-4.0,-4.0,-10.992]}}},{"id":"random/0323","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_f64","type":"f64"},{"name":"l4_f64","type":"f64"},{"name":"l5_utf8","type":"utf8"},{"name":"l6_f64","type":"f64"}],"columns":{"l0_utf8":["","a","B","c","c","c","a","😀","c","é","aa","ab","b","zz","B","😀","aa","ab","😀","a","aa","b","aa","B","zz","é","B","ab","B","ab","c","c","a","a","ab","b","B","aa","","😀","b","é","ab","c","zz","zz","😀","zz","aa","ab","b","zz","c","ab","ab","","😀","a","ab","zz"],"l1_i32":[4,-5,-1,5,4,3,2,-1,3,-4,0,3,4,1,-4,-2147483648,1,2,-1,-4,2,-3,-1,0,-3,4,2,-3,-2,4,0,-4,-4,0,4,-2,5,0,-5,4,-3,4,-5,-2147483648,-3,-4,5,3,4,-4,2,1,3,5,3,-2,2,-3,1,2147483647],"l2_f64":[-3.0,-2.0,-95.644,3.0,3.0,-31.119,54.996,65.428,22.954,-86.944,1.0,-1.0,"NaN",3.0,"NaN",1.0,"Infinity",76.302,2.0,-24.355,21.887,-95.501,-11.173,11.639,"-Infinity",-1.0,-83.71,4.0,63.768,-72.85,-19.932,3.0,"NaN",-3.0,-3.0,-3.0,"NaN",33.018,2.0,3.0,-2.0,-1.0,0.0,-2.0,65.578,-2.0,0.0,80.569,3.0,0.0,-2.0,1.0,-4.0,-2.693,-2.0,-3.0,-44.09,-26.687,36.001,2.0],"l3_f64":[3.0,-3.0,-14.914,13.753,-1.0,56.707,2.263,3.0,-67.117,-52.789,37.412,-2.0,0.0,-66.29,-1.0,1.0,-95.195,4.0,2.0,56.817,-1.0,-1.0,2.095,-1.0,16.128,0.0,92.065,1.0,3.0,-4.0,-36.127,-1.0,-4.0,-83.266,38.497,19.83,-4.0,3.0,1.0,-4.0,-64.866,-1.0,-56.858,-99.89,3.0,90.15,-4.0,-60.825,-3.0,2.0,-4.0,"NaN",-14.154,-82.85,21.796,-4.0,65.401,1.0,-4.0,"NaN"],"l4_f64":[-40.006,-1.0,80.817,-77.932,-37.408,4.0,64.523,10.347,2.0,82.35,58.229,86.016,-89.32,2.0,"-0",-1.0,3.0,-86.911,-1.0,24.049,51.948,-1.0,-1.0,4.0,41.107,-69.321,59.812,2.0,-1.0,48.809,71.321,-4.0,4.0,6.531,-35.026,-1.0,4.0,3.0,65.371,30.792,0.0,-65.286,30.487,1.0,0.0,-4.0,4.0,-91.068,"NaN",-4.0,4.0,1.116,2.0,58.102,2.0,3.0,14.635,-52.814,-73.368,-49.882],"l5_utf8":["a","é","ab","😀","","B","b","B","","b","é","b","é","é","c","aa","😀","aa","é","zz","😀","é","","😀","aa","B","zz","","b","a","é","a","","😀","","ab","é","b","","a","zz","a","zz","B","b","😀","😀","ab","zz","c","B","","","","😀","c","","","é","c"],"l6_f64":[-15.244,-1.0,50.592,0.0,-1.223,-3.0,-1.0,52.706,-41.383,4.0,-4.0,2.0,62.195,54.008,95.143,89.529,-50.65,-2.0,8.663,1.0,-4.0,"NaN",-76.464,-1.0,-24.512,-1.0,-71.786,-3.0,-43.843,-10.063,-4.0,3.0,-3.0,-2.0,94.356,-3.0,-1.0,-4.0,-40.981,73.162,"Infinity",-4.0,47.47,0.0,-57.74,3.0,4.0,1.828,4.0,-3.0,-1.0,-4.0,-3.0,3.0,-3.0,-96.122,49.208,29.534,2.0,-1.0]}},"right":{"fields":[{"name":"l5_utf8","type":"utf8"},{"name":"r0_f64","type":"f64"}],"columns":{"r0_f64":[58.317,"Infinity",2.0,null,1.832,null,1.0,null,54.71,4.0,46.141,-2.0,1.0,null,null,4.0,1.0,null,-15.16,"NaN",null,73.585,null,-1.0,-1.0,2.0,80.628,null,-56.186,-36.764],"l5_utf8":["😀","😀","","😀","é","b","😀","","é","é","aa","B","😀","😀","zz","b","zz","😀","","","b","B","😀","B","😀","","😀","zz","c","b"]}}},"ops":[{"op":"sort","keys":[{"expr":{"col":"l3_f64"},"descending":false},{"expr":{"col":"l6_f64"},"descending":true}],"nullsLast":false},{"op":"filter","predicate":{"fn":"eq","args":[{"fn":"mod","args":[{"col":"l6_f64"},{"col":"l6_f64"}]},{"fn":"abs","args":[{"col":"l3_f64"}]}]}},{"op":"join","right":"right","on":["l5_utf8"],"how":"semi"}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_f64","type":"f64"},{"name":"l4_f64","type":"f64"},{"name":"l5_utf8","type":"utf8"},{"name":"l6_f64","type":"f64"}],"columns":{"l0_utf8":["b","é"],"l1_i32":[4,4],"l2_f64":["NaN",-1.0],"l3_f64":[0.0,0.0],"l4_f64":[-89.32,-69.321],"l5_utf8":["é","B"],"l6_f64":[62.195,-1.0]}}},{"id":"random/0324","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_i32","type":"i32"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_bool","type":"bool"}],"columns":{"l0_utf8":["B","c","B","aa","c","c","aa","é","aa","a","é","😀","c","a","","😀","zz","a","😀","😀","c","😀","","aa","B","a","😀","ab","😀","😀","c","a","c","😀","","zz","aa","zz","a","B","é","😀","B","aa","B","zz","B","","aa","aa","aa","b","zz","b","ab","B","B","a","B","zz"],"l1_i32":[3,-3,-5,2,3,-1,-3,1,0,-2147483648,-2,-3,-2,3,-5,0,-2,-5,4,-5,4,-4,-1,3,-4,-5,-5,3,-4,-1,3,-2,3,3,-2,-1,2,4,5,-2147483648,0,-4,2,-1,4,0,-5,4,2147483647,4,-3,-5,0,3,-2147483648,2147483647,3,-1,0,-5],"l2_utf8":["b","é","aa","😀","aa","a","😀","ab","aa","😀","aa","é","B","B","é","c","zz","b","B","aa","ab","😀","B","b","","b","b","a","b","aa","ab","b","a","a","😀","😀","B","c","é","c","c","é","é","c","a","c","aa","a","😀","","é","","a","b","😀","é","ab","c","😀","😀"],"l3_i32":[5,-4,2,2,4,4,-5,0,-2,5,0,-5,0,5,-1,1,1,2,3,2,0,-939,-3,3,-3,-2,4,-2,-5,2,-4,0,-2147483648,1,4,-3,4,4,-1,1,2147483647,2147483647,2,-5,5,-786,5,-4,-1,2147483647,-5,-2,0,-2147483648,3,-1,5,-4,-4,4],"l4_utf8":["😀","zz","aa","é","a","b","c","B","zz","aa","c","é","ab","B","B","b","","zz","","c","a","","aa","a","aa","","b","a","B","B","ab","zz","b","ab","zz","zz","b","","😀","zz","a","c","B","B","b","é","😀","","b","é","b","b","b","é","","😀","zz","ab","c","aa"],"l5_bool":[false,true,true,true,false,true,true,false,false,true,true,false,false,true,true,false,true,false,false,true,true,true,false,true,true,false,false,false,true,false,false,true,false,true,true,false,false,false,true,true,true,false,true,false,false,true,true,true,false,true,false,true,true,false,true,true,true,false,true,true]}},"right":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"r0_utf8","type":"utf8"}],"columns":{"r0_utf8":[],"l0_utf8":[]}}},"ops":[{"op":"unique"},{"op":"join","right":"right","on":["l0_utf8"],"how":"inner"},{"op":"withColumns","exprs":{"x2":{"fn":"when","args":[{"fn":"neq","args":[{"col":"l3_i32"},{"col":"l3_i32"}]},{"col":"l3_i32"},{"fn":"abs","args":[{"lit":2.5}]}]}}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_i32","type":"i32"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_bool","type":"bool"},{"name":"r0_utf8","type":"utf8"},{"name":"x2","type":"f64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_utf8":[],"l3_i32":[],"l4_utf8":[],"l5_bool":[],"r0_utf8":[],"x2":[]}}},{"id":"random/0325","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_i32","type":"i32"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_f64","type":"f64"}],"columns":{"l0_utf8":["B","c","é","ab","zz","ab","B","b","zz",null,"b","","c","zz","b","ab","ab","c","a","c","b","B","c","é","😀","c","zz",null,"ab","","zz","zz","B","b","😀","","","c","aa","ab","aa","ab",null,"B",null,"zz","ab","ab",null,"a","zz","aa","",null,"ab","c",null,"a","ab","aa"],"l1_i32":[5,-1,-1,-1,4,5,1,2,4,-4,-98,2147483647,2147483647,-5,-2,-4,-1,5,0,1,1,null,2147483647,-3,1,-2147483648,-5,-1,-4,1,-4,0,5,null,2,4,5,2,0,-5,-1,5,3,-5,-4,-3,4,-3,3,0,-2147483648,2,-2147483648,2,4,2,792,-2,-2,1],"l2_utf8":["ab","aa","","aa","aa","a",null,"zz","😀",null,"é","😀","b",null,"c","ab","ab","B",null,"😀","aa","ab","B","a","","ab","a","c","c","é","b","é","","","B","b","b","é","b","c","zz",null,"zz","c","ab","é","zz","aa","ab","","é","B","ab","B","B","B","aa","b","ab","b"],"l3_i32":[2,4,-5,0,-3,1,5,1,-2,4,2,4,0,-4,-4,1,null,1,-5,0,-1,-2147483648,null,829,-1,0,1,-4,3,-2,-5,-5,null,-1,3,2,1,0,3,1,-5,3,-1,-3,2147483647,-1,4,-1,-5,null,-3,-5,1,-2,-4,1,2,-4,-5,-1],"l4_utf8":["ab","aa","😀","é","zz","ab","a","B","é","😀","b","B","c","b","zz","c","b","B","😀",null,null,"é","zz","a","","","c","é","b","b","😀","😀","zz","B","zz","c","b","","aa","","B","ab","","aa","aa","😀","","😀","b","😀",null,"zz",null,"c","c","c","😀","😀","aa","😀"],"l5_f64":[-1.0,-4.0,-0.785,86.736,-51.792,1.0,22.674,-1.0,3.0,"NaN",null,-40.495,-3.0,-1.0,-5.507,0.0,null,1.0,"Infinity",4.0,1.0,1.029,77.125,80.272,2.0,null,-97.61,null,0.162,2.0,-54.182,-4.0,-4.0,30.133,-25.351,0.0,-95.774,-21.763,50.805,-20.86,24.348,3.0,-35.245,-82.184,null,"NaN",-86.909,60.393,-2.0,-2.0,22.212,4.0,2.0,1.0,81.519,0.0,2.0,"NaN",3.0,2.0]}},"right":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"r0_bool","type":"bool"}],"columns":{"r0_bool":[],"l0_utf8":[]}}},"ops":[{"op":"withColumns","exprs":{"x0":{"fn":"when","args":[{"fn":"eq","args":[{"lit":2},{"col":"l3_i32"}]},{"fn":"mod","args":[{"col":"l1_i32"},{"col":"l3_i32"}]},{"lit":0.5}]}}},{"op":"join","right":"right","on":["l0_utf8"],"how":"full"},{"op":"unique"}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_i32","type":"i32"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_f64","type":"f64"},{"name":"x0","type":"f64"},{"name":"r0_bool","type":"bool"}],"columns":{"l0_utf8":["B","c","é","ab","zz","ab","B","b","zz",null,"b","","c","zz","b","ab","ab","c","a","c","b","B","c","é","😀","c","zz",null,"ab","","zz","zz","B","b","😀","","","c","aa","ab","aa","ab",null,"B",null,"zz","ab","ab",null,"a","zz","aa","",null,"ab","c",null,"a","ab","aa"],"l1_i32":[5,-1,-1,-1,4,5,1,2,4,-4,-98,2147483647,2147483647,-5,-2,-4,-1,5,0,1,1,null,2147483647,-3,1,-2147483648,-5,-1,-4,1,-4,0,5,null,2,4,5,2,0,-5,-1,5,3,-5,-4,-3,4,-3,3,0,-2147483648,2,-2147483648,2,4,2,792,-2,-2,1],"l2_utf8":["ab","aa","","aa","aa","a",null,"zz","😀",null,"é","😀","b",null,"c","ab","ab","B",null,"😀","aa","ab","B","a","","ab","a","c","c","é","b","é","","","B","b","b","é","b","c","zz",null,"zz","c","ab","é","zz","aa","ab","","é","B","ab","B","B","B","aa","b","ab","b"],"l3_i32":[2,4,-5,0,-3,1,5,1,-2,4,2,4,0,-4,-4,1,null,1,-5,0,-1,-2147483648,null,829,-1,0,1,-4,3,-2,-5,-5,null,-1,3,2,1,0,3,1,-5,3,-1,-3,2147483647,-1,4,-1,-5,null,-3,-5,1,-2,-4,1,2,-4,-5,-1],"l4_utf8":["ab","aa","😀","é","zz","ab","a","B","é","😀","b","B","c","b","zz","c","b","B","😀",null,null,"é","zz","a","","","c","é","b","b","😀","😀","zz","B","zz","c","b","","aa","","B","ab","","aa","aa","😀","","😀","b","😀",null,"zz",null,"c","c","c","😀","😀","aa","😀"],"l5_f64":[-1.0,-4.0,-0.785,86.736,-51.792,1.0,22.674,-1.0,3.0,"NaN",null,-40.495,-3.0,-1.0,-5.507,0.0,null,1.0,"Infinity",4.0,1.0,1.029,77.125,80.272,2.0,null,-97.61,null,0.162,2.0,-54.182,-4.0,-4.0,30.133,-25.351,0.0,-95.774,-21.763,50.805,-20.86,24.348,3.0,-35.245,-82.184,null,"NaN",-86.909,60.393,-2.0,-2.0,22.212,4.0,2.0,1.0,81.519,0.0,2.0,"NaN",3.0,2.0],"x0":[1.0,0.5,0.5,0.5,0.5,0.5,0.5,0.5,0.5,0.5,0.0,0.5,0.5,0.5,0.5,0.5,0.5,0.5,0.5,0.5,0.5,0.5,0.5,0.5,0.5,0.5,0.5,0.5,0.5,0.5,0.5,0.5,0.5,0.5,0.5,0.0,0.5,0.5,0.5,0.5,0.5,0.5,0.5,0.5,0.5,0.5,0.5,0.5,0.5,0.5,0.5,0.5,0.5,0.5,0.5,0.5,0.0,0.5,0.5,0.5],"r0_bool":[null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null]}}},{"id":"random/0326","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_i32","type":"i32"}],"columns":{"l0_utf8":["é","😀","é","aa","aa","B","😀","a","ab","aa","b","aa","aa","a","aa","b","é","😀","c","😀"],"l1_i32":[4,4,-4,-4,-5,-2,5,-4,-5,1,1,4,-5,-1,3,5,-4,0,-2,2],"l2_f64":[-2.0,3.0,93.997,-3.0,-3.0,-1.0,2.0,-2.0,-1.0,85.929,-51.236,2.731,0.0,88.98,-12.564,74.445,-3.0,0.0,-2.0,3.0],"l3_i32":[-2,-1,3,5,-1,4,0,-3,-2,1,4,0,-5,-5,-906,2147483647,-1,4,1,5]}},"right":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"r0_f64","type":"f64"}],"columns":{"r0_f64":[null,21.515,null,25.888,-51.528,null,5.198,-90.889,-4.0,"NaN",null,-1.0,-33.017,2.0,0.0,1.0,-10.947,-69.676,-3.0,-56.61,0.0,14.478,3.0,-48.5,-2.0,null,-3.0,null,-4.0,-82.267],"l0_utf8":["é","é","B","é","aa","B","c","é","aa","aa","aa","😀","a","é","a","😀","é","é","😀","a","b","a","aa","é","aa","é","aa","aa","b","aa"]}}},"ops":[{"op":"join","right":"right","on":["l0_utf8"],"how":"inner"},{"op":"sort","keys":[{"expr":{"col":"l3_i32"},"descending":false}],"nullsLast":false},{"op":"filter","predicate":{"fn":"isNotNull","args":[{"col":"l1_i32"}]}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_i32","type":"i32"},{"name":"r0_f64","type":"f64"}],"columns":{"l0_utf8":["aa","aa","aa","aa","aa","aa","aa","aa","aa","aa","aa","aa","aa","aa","aa","aa","aa","aa","a","a","a","a","a","a","a","a","é","é","é","é","é","é","é","é","é","😀","😀","😀","aa","aa","aa","aa","aa","aa","aa","aa","aa","é","é","é","é","é","é","é","é","é","😀","😀","😀","aa","aa","aa","aa","aa","aa","aa","aa","aa","aa","aa","aa","aa","aa","aa","aa","aa","aa","c","é","é","é","é","é","é","é","é","é","B","B","b","b","😀","😀","😀","aa","aa","aa","aa","aa","aa","aa","aa","aa","😀","😀","😀","b","b"],"l1_i32":[3,3,3,3,3,3,3,3,3,-5,-5,-5,-5,-5,-5,-5,-5,-5,-1,-1,-1,-1,-4,-4,-4,-4,4,4,4,4,4,4,4,4,4,4,4,4,-5,-5,-5,-5,-5,-5,-5,-5,-5,-4,-4,-4,-4,-4,-4,-4,-4,-4,5,5,5,4,4,4,4,4,4,4,4,4,1,1,1,1,1,1,1,1,1,-2,-4,-4,-4,-4,-4,-4,-4,-4,-4,-2,-2,1,1,0,0,0,-4,-4,-4,-4,-4,-4,-4,-4,-4,2,2,2,5,5],"l2_f64":[-12.564,-12.564,-12.564,-12.564,-12.564,-12.564,-12.564,-12.564,-12.564,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,88.98,88.98,88.98,88.98,-2.0,-2.0,-2.0,-2.0,-2.0,-2.0,-2.0,-2.0,-2.0,-2.0,-2.0,-2.0,-2.0,3.0,3.0,3.0,-3.0,-3.0,-3.0,-3.0,-3.0,-3.0,-3.0,-3.0,-3.0,-3.0,-3.0,-3.0,-3.0,-3.0,-3.0,-3.0,-3.0,-3.0,2.0,2.0,2.0,2.731,2.731,2.731,2.731,2.731,2.731,2.731,2.731,2.731,85.929,85.929,85.929,85.929,85.929,85.929,85.929,85.929,85.929,-2.0,93.997,93.997,93.997,93.997,93.997,93.997,93.997,93.997,93.997,-1.0,-1.0,-51.236,-51.236,0.0,0.0,0.0,-3.0,-3.0,-3.0,-3.0,-3.0,-3.0,-3.0,-3.0,-3.0,3.0,3.0,3.0,74.445,74.445],"l3_i32":[-906,-906,-906,-906,-906,-906,-906,-906,-906,-5,-5,-5,-5,-5,-5,-5,-5,-5,-5,-5,-5,-5,-3,-3,-3,-3,-2,-2,-2,-2,-2,-2,-2,-2,-2,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,0,0,0,0,0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1,1,1,3,3,3,3,3,3,3,3,3,4,4,4,4,4,4,4,5,5,5,5,5,5,5,5,5,5,5,5,2147483647,2147483647],"r0_f64":[-51.528,-4.0,"NaN",null,3.0,-2.0,-3.0,null,-82.267,-51.528,-4.0,"NaN",null,3.0,-2.0,-3.0,null,-82.267,-33.017,0.0,-56.61,14.478,-33.017,0.0,-56.61,14.478,null,21.515,25.888,-90.889,2.0,-10.947,-69.676,-48.5,null,-1.0,1.0,-3.0,-51.528,-4.0,"NaN",null,3.0,-2.0,-3.0,null,-82.267,null,21.515,25.888,-90.889,2.0,-10.947,-69.676,-48.5,null,-1.0,1.0,-3.0,-51.528,-4.0,"NaN",null,3.0,-2.0,-3.0,null,-82.267,-51.528,-4.0,"NaN",null,3.0,-2.0,-3.0,null,-82.267,5.198,null,21.515,25.888,-90.889,2.0,-10.947,-69.676,-48.5,null,null,null,0.0,-4.0,-1.0,1.0,-3.0,-51.528,-4.0,"NaN",null,3.0,-2.0,-3.0,null,-82.267,-1.0,1.0,-3.0,0.0,-4.0]}}},{"id":"random/0327","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i64","type":"i64"},{"name":"l3_f64","type":"f64"},{"name":"l4_f64","type":"f64"},{"name":"l5_utf8","type":"utf8"},{"name":"l6_i64","type":"i64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_i64":[],"l3_f64":[],"l4_f64":[],"l5_utf8":[],"l6_i64":[]}},"right":{"fields":[{"name":"l5_utf8","type":"utf8"},{"name":"r0_f64","type":"f64"}],"columns":{"r0_f64":[-48.354,null,-4.0,-15.727,null,30.4,null,null,2.0,null,null,1.0,null,-3.0,44.911,"NaN",null,3.0,2.0,null,4.0,47.398,58.981,null,-1.0,-1.0,-2.0,null,null,null],"l5_utf8":["ab","a","a","😀","a","zz",null,"ab",null,"zz","zz","ab","a","a",null,"ab","ab","😀","zz","zz",null,null,"zz","ab","😀",null,"😀","ab","ab",null]}}},"ops":[{"op":"unique","columns":["l3_f64"]},{"op":"head","n":5},{"op":"filter","predicate":{"fn":"isNull","args":[{"col":"l6_i64"}]}},{"op":"join","right":"right","on":["l5_utf8"],"how":"semi"}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i64","type":"i64"},{"name":"l3_f64","type":"f64"},{"name":"l4_f64","type":"f64"},{"name":"l5_utf8","type":"utf8"},{"name":"l6_i64","type":"i64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_i64":[],"l3_f64":[],"l4_f64":[],"l5_utf8":[],"l6_i64":[]}}},{"id":"random/0328","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_utf8","type":"utf8"},{"name":"l6_utf8","type":"utf8"}],"columns":{"l0_utf8":[null],"l1_i32":[-5],"l2_utf8":["ab"],"l3_utf8":["zz"],"l4_utf8":[""],"l5_utf8":["é"],"l6_utf8":[""]}}},"ops":[{"op":"withColumns","exprs":{"x0":{"fn":"sub","args":[{"fn":"neg","args":[{"col":"l1_i32"}]},{"col":"l1_i32"}]}}},{"op":"withColumns","exprs":{"x1":{"col":"x0"}}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_utf8","type":"utf8"},{"name":"l6_utf8","type":"utf8"},{"name":"x0","type":"i32"},{"name":"x1","type":"i32"}],"columns":{"l0_utf8":[null],"l1_i32":[-5],"l2_utf8":["ab"],"l3_utf8":["zz"],"l4_utf8":[""],"l5_utf8":["é"],"l6_utf8":[""],"x0":[10],"x1":[10]}}},{"id":"random/0329","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_f64","type":"f64"},{"name":"l4_bool","type":"bool"},{"name":"l5_bool","type":"bool"},{"name":"l6_i32","type":"i32"}],"columns":{"l0_utf8":["é","c","aa","c","B"],"l1_i32":[3,3,-4,null,-5],"l2_f64":[-13.792,null,null,"NaN",null],"l3_f64":[null,-1.0,-3.0,null,1.0],"l4_bool":[false,true,null,false,false],"l5_bool":[true,false,null,true,true],"l6_i32":[2,3,1,2,-1]}}},"ops":[{"op":"filter","predicate":{"fn":"isNull","args":[{"col":"l4_bool"}]}},{"op":"sort","keys":[{"expr":{"col":"l1_i32"},"descending":true},{"expr":{"col":"l6_i32"},"descending":true}],"nullsLast":true},{"op":"withColumns","exprs":{"x2":{"fn":"fillNull","args":[{"fn":"when","args":[{"fn":"isNotNull","args":[{"col":"l0_utf8"}]},{"col":"l6_i32"},{"col":"l6_i32"}]},{"lit":0}]}}},{"op":"sort","keys":[{"expr":{"col":"x2"},"descending":false},{"expr":{"col":"l3_f64"},"descending":true}],"nullsLast":true}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_f64","type":"f64"},{"name":"l4_bool","type":"bool"},{"name":"l5_bool","type":"bool"},{"name":"l6_i32","type":"i32"},{"name":"x2","type":"i32"}],"columns":{"l0_utf8":["aa"],"l1_i32":[-4],"l2_f64":[null],"l3_f64":[-3.0],"l4_bool":[null],"l5_bool":[null],"l6_i32":[1],"x2":[1]}}},{"id":"random/0330","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i64","type":"i64"},{"name":"l3_f64","type":"f64"},{"name":"l4_f64","type":"f64"}],"columns":{"l0_utf8":["c","aa",null,"b","a"],"l1_i32":[5,-4,5,5,null],"l2_i64":["-4611686018427387904","9007199254740995","1",null,"9007199254740995"],"l3_f64":[3.0,-63.171,-29.968,null,-51.323],"l4_f64":[7.766,null,1.0,null,null]}},"right":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"r0_utf8","type":"utf8"}],"columns":{"r0_utf8":["ab","ab",null,"zz","😀","a","c","c","😀","B"],"l0_utf8":["a",null,"b","b","a","aa","aa","a","b","c"]}}},"ops":[{"op":"join","right":"right","on":["l0_utf8"],"how":"left"}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i64","type":"i64"},{"name":"l3_f64","type":"f64"},{"name":"l4_f64","type":"f64"},{"name":"r0_utf8","type":"utf8"}],"columns":{"l0_utf8":["c","aa","aa",null,"b","b","b","a","a","a"],"l1_i32":[5,-4,-4,5,5,5,5,null,null,null],"l2_i64":["-4611686018427387904","9007199254740995","9007199254740995","1",null,null,null,"9007199254740995","9007199254740995","9007199254740995"],"l3_f64":[3.0,-63.171,-63.171,-29.968,null,null,null,-51.323,-51.323,-51.323],"l4_f64":[7.766,null,null,1.0,null,null,null,null,null,null],"r0_utf8":["B","a","c",null,null,"zz","😀","ab","😀","c"]}}},{"id":"random/0331","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_f64","type":"f64"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_f64","type":"f64"}],"columns":{"l0_utf8":["a","aa","zz","é","aa"],"l1_i32":[-2,-1,2,-1,-5],"l2_f64":[4.0,-90.04,0.0,3.0,-63.794],"l3_f64":[49.181,93.965,3.0,-2.0,8.462],"l4_utf8":["aa","a","😀","😀","B"],"l5_f64":[70.517,26.583,-11.654,34.858,2.0]}}},"ops":[{"op":"filter","predicate":{"fn":"gte","args":[{"col":"l4_utf8"},{"lit":""}]}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_f64","type":"f64"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_f64","type":"f64"}],"columns":{"l0_utf8":["a","aa","zz","é","aa"],"l1_i32":[-2,-1,2,-1,-5],"l2_f64":[4.0,-90.04,0.0,3.0,-63.794],"l3_f64":[49.181,93.965,3.0,-2.0,8.462],"l4_utf8":["aa","a","😀","😀","B"],"l5_f64":[70.517,26.583,-11.654,34.858,2.0]}}},{"id":"random/0332","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_bool","type":"bool"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_bool","type":"bool"},{"name":"l6_f64","type":"f64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_f64":[],"l3_bool":[],"l4_utf8":[],"l5_bool":[],"l6_f64":[]}},"right":{"fields":[{"name":"l1_i32","type":"i32"},{"name":"r0_f64","type":"f64"}],"columns":{"r0_f64":[-3.0,-1.0,24.233,null,null,30.994,-11.362,13.536,1.0,3.0,-44.785,63.041,-2.0,1.0,2.0,-2.0,42.463,null,2.0,null,0.0,-32.384,0.0,null,null,-1.0,-62.665,-40.386,"Infinity",-3.0],"l1_i32":[-5,null,137,-5,-5,-5,-2147483648,null,137,null,137,-2147483648,137,null,-2147483648,-5,null,-5,null,137,-5,-2147483648,137,null,null,-2147483648,137,-5,null,137]}}},"ops":[{"op":"join","right":"right","on":["l1_i32"],"how":"anti"},{"op":"unique","columns":["l3_bool"]}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_bool","type":"bool"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_bool","type":"bool"},{"name":"l6_f64","type":"f64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_f64":[],"l3_bool":[],"l4_utf8":[],"l5_bool":[],"l6_f64":[]}}},{"id":"random/0333","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_bool","type":"bool"},{"name":"l5_utf8","type":"utf8"}],"columns":{"l0_utf8":["","B","b","","aa","ab","a","zz","é","é","ab","ab","c","😀","b","😀","c","é","","aa"],"l1_i32":[-2,3,3,-5,1,0,5,0,4,-5,-1,1,2,1,3,-5,4,-3,-4,4],"l2_i32":[-2,1,1,1,-137,1,-4,-5,5,-5,1,-1,-4,-2,-3,0,4,-1,-1,5],"l3_utf8":["c","zz","a","aa","é","😀","a","a","😀","B","zz","","zz","😀","aa","","b","ab","aa",""],"l4_bool":[true,false,false,false,false,false,true,true,false,true,true,false,true,true,false,false,false,true,false,true],"l5_utf8":["zz","ab","","c","é","ab","B","zz","c","aa","","","","a","","zz","aa","b","😀","a"]}}},"ops":[{"op":"head","n":9},{"op":"head","n":2}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_bool","type":"bool"},{"name":"l5_utf8","type":"utf8"}],"columns":{"l0_utf8":["","B"],"l1_i32":[-2,3],"l2_i32":[-2,1],"l3_utf8":["c","zz"],"l4_bool":[true,false],"l5_utf8":["zz","ab"]}}},{"id":"random/0334","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_i32","type":"i32"}],"columns":{"l0_utf8":[""],"l1_i32":[-1],"l2_i32":[-2],"l3_i32":[1]}}},"ops":[{"op":"head","n":5}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_i32","type":"i32"}],"columns":{"l0_utf8":[""],"l1_i32":[-1],"l2_i32":[-2],"l3_i32":[1]}}},{"id":"random/0335","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_f64","type":"f64"},{"name":"l5_f64","type":"f64"}],"columns":{"l0_utf8":["aa","c","a","","b","","😀","","B","c","é","b","ab","é","a","ab","","","b","aa"],"l1_i32":[1,5,-2,-3,-5,-2,2,1,1,2,1,-3,2147483647,1,-4,4,3,-3,2,-4],"l2_utf8":["B","","a","😀","B","😀","😀","a","a","a","c","b","b","c","é","B","ab","😀","aa",""],"l3_utf8":["a","b","b","","B","","😀","é","ab","aa","aa","ab","😀","c","ab","é","ab","zz","zz","zz"],"l4_f64":[-1.0,-4.0,-4.0,4.0,68.766,3.0,-2.0,-79.718,2.0,-2.0,3.0,0.0,57.309,4.0,-2.0,2.0,4.0,80.324,-57.7,1.0],"l5_f64":[3.0,-83.617,-4.0,-44.595,92.618,-35.172,-29.783,0.0,4.0,2.0,1.0,53.314,-4.0,57.942,1.0,68.779,27.002,3.0,-4.0,-20.644]}}},"ops":[{"op":"sort","keys":[{"expr":{"col":"l5_f64"},"descending":false},{"expr":{"col":"l0_utf8"},"descending":true}],"nullsLast":false},{"op":"filter","predicate":{"fn":"isNull","args":[{"col":"l2_utf8"}]}},{"op":"sort","keys":[{"expr":{"col":"l3_utf8"},"descending":false}],"nullsLast":false}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_f64","type":"f64"},{"name":"l5_f64","type":"f64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_utf8":[],"l3_utf8":[],"l4_f64":[],"l5_f64":[]}}},{"id":"random/0336","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_utf8","type":"utf8"}],"columns":{"l0_utf8":["😀","é","ab","é",null],"l1_i32":[0,null,4,null,-5],"l2_utf8":[null,null,"zz","a","😀"],"l3_utf8":["b","aa","é",null,null]}}},"ops":[{"op":"filter","predicate":{"fn":"isNotNull","args":[{"col":"l3_utf8"}]}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_utf8","type":"utf8"}],"columns":{"l0_utf8":["😀","é","ab"],"l1_i32":[0,null,4],"l2_utf8":[null,null,"zz"],"l3_utf8":["b","aa","é"]}}},{"id":"random/0337","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_f64","type":"f64"},{"name":"l4_i32","type":"i32"},{"name":"l5_i64","type":"i64"},{"name":"l6_bool","type":"bool"}],"columns":{"l0_utf8":["é"],"l1_i32":[2],"l2_utf8":["c"],"l3_f64":[-72.194],"l4_i32":[-2],"l5_i64":["-4611686018427387904"],"l6_bool":[true]}},"right":{"fields":[{"name":"l5_i64","type":"i64"},{"name":"r0_utf8","type":"utf8"}],"columns":{"r0_utf8":["é","é","ab"],"l5_i64":["-4611686018427387904","-4611686018427387904","-4611686018427387904"]}}},"ops":[{"op":"filter","predicate":{"fn":"isNull","args":[{"col":"l1_i32"}]}},{"op":"join","right":"right","on":["l5_i64"],"how":"left"},{"op":"withColumns","exprs":{"x2":{"fn":"strStartsWith","args":[{"col":"r0_utf8"}],"prefix":""}}},{"op":"groupBy","keys":["r0_utf8"],"aggs":{"n3":{"agg":"len"},"a3_0":{"agg":"nUnique","arg":{"col":"l4_i32"}},"a3_1":{"agg":"min","arg":{"col":"l2_utf8"}},"a3_2":{"agg":"max","arg":{"col":"l2_utf8"}}}}],"expected":{"fields":[{"name":"r0_utf8","type":"utf8"},{"name":"n3","type":"u32"},{"name":"a3_0","type":"u32"},{"name":"a3_1","type":"utf8"},{"name":"a3_2","type":"utf8"}],"columns":{"r0_utf8":[],"n3":[],"a3_0":[],"a3_1":[],"a3_2":[]}}},{"id":"random/0338","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_bool","type":"bool"},{"name":"l4_f64","type":"f64"},{"name":"l5_utf8","type":"utf8"}],"columns":{"l0_utf8":["é","B","zz","😀","ab","","zz","","é","zz","zz","c","a","a","zz","zz","zz","a","zz","c","","","zz","B","B","","a","c","ab","zz","é","","B","😀","a","é","","B","c","é","😀","B","B","ab","c","é","","😀","aa","a","","aa","B","ab","c","a","","B","a","c"],"l1_i32":[-3,0,3,4,2147483647,0,3,-4,0,4,4,-2,2147483647,3,-4,2,-4,-4,4,-3,2,-1,1,0,4,-3,1,5,1,-4,-2,5,-1,-5,0,2,-2,0,1,2,4,1,-1,-1,-1,-4,-4,-186,0,1,0,-5,-2,1,457,-2,-4,0,4,0],"l2_bool":[true,true,true,true,true,false,false,false,false,true,false,true,true,true,true,true,true,true,true,true,false,false,false,false,false,false,true,true,true,false,false,true,false,false,false,true,true,false,false,false,true,false,true,true,true,true,false,true,false,true,false,true,true,true,false,false,true,false,false,false],"l3_bool":[true,false,false,true,false,true,false,true,true,true,true,true,true,true,true,false,false,true,false,false,true,false,true,false,true,true,true,true,false,true,false,false,true,false,true,false,true,false,false,false,true,true,false,true,true,false,true,false,true,true,false,true,true,false,true,true,true,false,true,true],"l4_f64":[-3.0,3.0,-2.0,1.0,74.616,-27.784,-39.18,0.0,3.0,"Infinity","-0",-3.0,-70.052,3.0,4.0,0.0,-0.823,-28.264,-2.0,"NaN",4.0,-3.0,61.994,2.0,4.0,-2.0,-4.0,-82.517,89.721,3.0,0.0,-72.568,3.0,"NaN",2.0,"-Infinity",4.0,1.0,-4.0,56.028,-2.0,0.0,-2.0,99.684,-54.377,-2.0,-2.0,3.0,-81.435,-63.381,-60.605,-92.898,3.0,-76.533,3.0,3.0,-1.0,-4.0,-1.0,14.964],"l5_utf8":["","","B","B","a","","c","😀","é","c","c","c","B","c","é","é","c","B","B","","b","aa","aa","😀","aa","B","B","é","","b","é","ab","é","😀","😀","aa","zz","","zz","aa","zz","c","😀","a","aa","","zz","😀","a","B","é","ab","B","zz","a","","","b","B","zz"]}},"right":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"r0_bool","type":"bool"}],"columns":{"r0_bool":[true,null,true,true,true,null,true,false,false,false,true,true,false,true,true,true,false,true,true,null,false,true,true,null,true,true,true,true,null,true],"l0_utf8":["B","zz","zz","zz","zz","zz","a","","B","c","c","c","zz","😀","","é","😀","a","","","a","","","é","","zz","","B","B",""]}}},"ops":[{"op":"join","right":"right","on":["l0_utf8"],"how":"inner"}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_bool","type":"bool"},{"name":"l4_f64","type":"f64"},{"name":"l5_utf8","type":"utf8"},{"name":"r0_bool","type":"bool"}],"columns":{"l0_utf8":["é","é","B","B","B","B","zz","zz","zz","zz","zz","zz","zz","😀","😀","","","","","","","","","","zz","zz","zz","zz","zz","zz","zz","","","","","","","","","","é","é","zz","zz","zz","zz","zz","zz","zz","zz","zz","zz","zz","zz","zz","zz","c","c","c","a","a","a","a","a","a","zz","zz","zz","zz","zz","zz","zz","zz","zz","zz","zz","zz","zz","zz","zz","zz","zz","zz","zz","zz","zz","a","a","a","zz","zz","zz","zz","zz","zz","zz","c","c","c","","","","","","","","","","","","","","","","","","","zz","zz","zz","zz","zz","zz","zz","B","B","B","B","B","B","B","B","","","","","","","","","","a","a","a","c","c","c","zz","zz","zz","zz","zz","zz","zz","é","é","","","","","","","","","","B","B","B","B","😀","😀","a","a","a","é","é","","","","","","","","","","B","B","B","B","c","c","c","é","é","😀","😀","B","B","B","B","B","B","B","B","c","c","c","é","é","","","","","","","","","","😀","😀","a","a","a","","","","","","","","","","B","B","B","B","c","c","c","a","a","a","","","","","","","","","","B","B","B","B","a","a","a","c","c","c"],"l1_i32":[-3,-3,0,0,0,0,3,3,3,3,3,3,3,4,4,0,0,0,0,0,0,0,0,0,3,3,3,3,3,3,3,-4,-4,-4,-4,-4,-4,-4,-4,-4,0,0,4,4,4,4,4,4,4,4,4,4,4,4,4,4,-2,-2,-2,2147483647,2147483647,2147483647,3,3,3,-4,-4,-4,-4,-4,-4,-4,2,2,2,2,2,2,2,-4,-4,-4,-4,-4,-4,-4,-4,-4,-4,4,4,4,4,4,4,4,-3,-3,-3,2,2,2,2,2,2,2,2,2,-1,-1,-1,-1,-1,-1,-1,-1,-1,1,1,1,1,1,1,1,0,0,0,0,4,4,4,4,-3,-3,-3,-3,-3,-3,-3,-3,-3,1,1,1,5,5,5,-4,-4,-4,-4,-4,-4,-4,-2,-2,5,5,5,5,5,5,5,5,5,-1,-1,-1,-1,-5,-5,0,0,0,2,2,-2,-2,-2,-2,-2,-2,-2,-2,-2,0,0,0,0,1,1,1,2,2,4,4,1,1,1,1,-1,-1,-1,-1,-1,-1,-1,-4,-4,-4,-4,-4,-4,-4,-4,-4,-4,-4,-186,-186,1,1,1,0,0,0,0,0,0,0,0,0,-2,-2,-2,-2,457,457,457,-2,-2,-2,-4,-4,-4,-4,-4,-4,-4,-4,-4,0,0,0,0,4,4,4,0,0,0],"l2_bool":[true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,true,true,true,true,true,true,true,false,false,false,false,false,false,false,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,true,true,true,true,true,true,false,false,false,false,false,false,false,false,false,true,true,true,true,true,true,true,true,true,false,false,false,false,false,false,false,false,false,true,true,true,true,true,true,true,true,true,true,true,false,false,false,false,false,false,false,false,false,true,true,false,false,false,false,true,true,true,true,true,true,true,true,true,false,false,false,false,false,false,false,false,false,true,true,true,true,true,false,false,false,false,false,false,false,false,false,true,true,true,true,false,false,false,false,false,false,true,true,true,true,true,true,true,true,true,false,false,false,false,false,false,false,false,false,false],"l3_bool":[true,true,false,false,false,false,false,false,false,false,false,false,false,true,true,true,true,true,true,true,true,true,true,true,false,false,false,false,false,false,false,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,false,false,false,false,false,false,false,false,false,false,false,false,false,false,true,true,true,false,false,false,false,false,false,false,false,false,false,true,true,true,true,true,true,true,true,true,false,false,false,false,false,false,false,false,false,true,true,true,true,true,true,true,false,false,false,false,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,false,false,false,false,false,false,false,false,false,false,false,true,true,true,true,false,false,true,true,true,false,false,true,true,true,true,true,true,true,true,true,false,false,false,false,false,false,false,false,false,true,true,true,true,true,true,false,false,false,false,true,true,true,false,false,true,true,true,true,true,true,true,true,true,false,false,true,true,true,false,false,false,false,false,false,false,false,false,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,false,false,false,false,true,true,true,true,true,true],"l4_f64":[-3.0,-3.0,3.0,3.0,3.0,3.0,-2.0,-2.0,-2.0,-2.0,-2.0,-2.0,-2.0,1.0,1.0,-27.784,-27.784,-27.784,-27.784,-27.784,-27.784,-27.784,-27.784,-27.784,-39.18,-39.18,-39.18,-39.18,-39.18,-39.18,-39.18,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,3.0,3.0,"Infinity","Infinity","Infinity","Infinity","Infinity","Infinity","Infinity","-0","-0","-0","-0","-0","-0","-0",-3.0,-3.0,-3.0,-70.052,-70.052,-70.052,3.0,3.0,3.0,4.0,4.0,4.0,4.0,4.0,4.0,4.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.823,-0.823,-0.823,-0.823,-0.823,-0.823,-0.823,-28.264,-28.264,-28.264,-2.0,-2.0,-2.0,-2.0,-2.0,-2.0,-2.0,"NaN","NaN","NaN",4.0,4.0,4.0,4.0,4.0,4.0,4.0,4.0,4.0,-3.0,-3.0,-3.0,-3.0,-3.0,-3.0,-3.0,-3.0,-3.0,61.994,61.994,61.994,61.994,61.994,61.994,61.994,2.0,2.0,2.0,2.0,4.0,4.0,4.0,4.0,-2.0,-2.0,-2.0,-2.0,-2.0,-2.0,-2.0,-2.0,-2.0,-4.0,-4.0,-4.0,-82.517,-82.517,-82.517,3.0,3.0,3.0,3.0,3.0,3.0,3.0,0.0,0.0,-72.568,-72.568,-72.568,-72.568,-72.568,-72.568,-72.568,-72.568,-72.568,3.0,3.0,3.0,3.0,"NaN","NaN",2.0,2.0,2.0,"-Infinity","-Infinity",4.0,4.0,4.0,4.0,4.0,4.0,4.0,4.0,4.0,1.0,1.0,1.0,1.0,-4.0,-4.0,-4.0,56.028,56.028,-2.0,-2.0,0.0,0.0,0.0,0.0,-2.0,-2.0,-2.0,-2.0,-54.377,-54.377,-54.377,-2.0,-2.0,-2.0,-2.0,-2.0,-2.0,-2.0,-2.0,-2.0,-2.0,-2.0,3.0,3.0,-63.381,-63.381,-63.381,-60.605,-60.605,-60.605,-60.605,-60.605,-60.605,-60.605,-60.605,-60.605,3.0,3.0,3.0,3.0,3.0,3.0,3.0,3.0,3.0,3.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-4.0,-4.0,-4.0,-4.0,-1.0,-1.0,-1.0,14.964,14.964,14.964],"l5_utf8":["","","","","","","B","B","B","B","B","B","B","B","B","","","","","","","","","","c","c","c","c","c","c","c","😀","😀","😀","😀","😀","😀","😀","😀","😀","é","é","c","c","c","c","c","c","c","c","c","c","c","c","c","c","c","c","c","B","B","B","c","c","c","é","é","é","é","é","é","é","é","é","é","é","é","é","é","c","c","c","c","c","c","c","B","B","B","B","B","B","B","B","B","B","","","","b","b","b","b","b","b","b","b","b","aa","aa","aa","aa","aa","aa","aa","aa","aa","aa","aa","aa","aa","aa","aa","aa","😀","😀","😀","😀","aa","aa","aa","aa","B","B","B","B","B","B","B","B","B","B","B","B","é","é","é","b","b","b","b","b","b","b","é","é","ab","ab","ab","ab","ab","ab","ab","ab","ab","é","é","é","é","😀","😀","😀","😀","😀","aa","aa","zz","zz","zz","zz","zz","zz","zz","zz","zz","","","","","zz","zz","zz","aa","aa","zz","zz","c","c","c","c","😀","😀","😀","😀","aa","aa","aa","","","zz","zz","zz","zz","zz","zz","zz","zz","zz","😀","😀","B","B","B","é","é","é","é","é","é","é","é","é","B","B","B","B","a","a","a","","","","","","","","","","","","","b","b","b","b","B","B","B","zz","zz","zz"],"r0_bool":[true,null,true,false,true,null,null,true,true,true,null,false,true,true,false,false,true,true,null,true,true,true,true,true,null,true,true,true,null,false,true,false,true,true,null,true,true,true,true,true,true,null,null,true,true,true,null,false,true,null,true,true,true,null,false,true,false,true,true,true,true,false,true,true,false,null,true,true,true,null,false,true,null,true,true,true,null,false,true,null,true,true,true,null,false,true,true,true,false,null,true,true,true,null,false,true,false,true,true,false,true,true,null,true,true,true,true,true,false,true,true,null,true,true,true,true,true,null,true,true,true,null,false,true,true,false,true,null,true,false,true,null,false,true,true,null,true,true,true,true,true,true,true,false,false,true,true,null,true,true,true,null,false,true,true,null,false,true,true,null,true,true,true,true,true,true,false,true,null,true,false,true,true,false,true,null,false,true,true,null,true,true,true,true,true,true,false,true,null,false,true,true,true,null,true,false,true,false,true,null,true,false,true,null,false,true,true,true,null,false,true,true,null,true,true,true,true,true,true,false,true,true,false,false,true,true,null,true,true,true,true,true,true,false,true,null,false,true,true,true,true,false,false,true,true,null,true,true,true,true,true,true,false,true,null,true,true,false,false,true,true]}}},{"id":"random/0339","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_i64","type":"i64"},{"name":"l4_i32","type":"i32"}],"columns":{"l0_utf8":[null,null,"a",null,"ab","","aa",null,null,"ab","B",null,null,"ab","é","é","zz",null,"c",null],"l1_i32":[2,null,5,1,null,2147483647,-4,-3,-5,2,null,null,null,-3,2,1,2,null,null,null],"l2_i32":[null,null,0,-5,3,3,0,1,-4,-5,null,0,-4,null,null,null,null,null,-4,3],"l3_i64":["9007199254740993","-4611686018427387904","-1",null,"4","-4611686018427387904","9007199254740992","9007199254740994","-1",null,null,"-4611686018427387904",null,"-4611686018427387904","3","-4611686018427387904","1",null,"-5","-4611686018427387904"],"l4_i32":[0,null,-1,null,-1,-1,-3,-1,null,1,null,null,3,-1,-2147483648,0,null,2,-2147483648,0]}}},"ops":[{"op":"unique"},{"op":"filter","predicate":{"fn":"neq","args":[{"fn":"sub","args":[{"col":"l2_i32"},{"col":"l2_i32"}]},{"col":"l2_i32"}]}},{"op":"unique","columns":["l4_i32"]},{"op":"groupBy","keys":["l2_i32"],"aggs":{"n3":{"agg":"len"},"a3_0":{"agg":"last","arg":{"col":"l0_utf8"}},"a3_1":{"agg":"mean","arg":{"col":"l4_i32"}},"a3_2":{"agg":"first","arg":{"col":"l3_i64"}}}}],"expected":{"fields":[{"name":"l2_i32","type":"i32"},{"name":"n3","type":"u32"},{"name":"a3_0","type":"utf8"},{"name":"a3_1","type":"f64"},{"name":"a3_2","type":"i64"}],"columns":{"l2_i32":[-5,3,-4],"n3":[2,2,2],"a3_0":["ab",null,"c"],"a3_1":[1.0,-0.5,-1073741822.5],"a3_2":[null,"4",null]}}},{"id":"random/0340","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_utf8","type":"utf8"}],"columns":{"l0_utf8":["😀","😀","B","b","b","aa",null,"b",null,null,"","é",null,"",null,"😀","","","B","","aa","a","b",null,"B","zz",null,"a","a",null,"c",null,"",null,"b","😀","","😀",null,"B","aa",null,"aa","b","😀","é","aa","a",null,"a","aa","😀","😀",null,"ab","a","c",null,"aa","aa"],"l1_i32":[null,null,null,null,-5,4,-2,-2,null,2,-1,4,5,5,5,-2,-4,551,-4,-1,4,2147483647,null,null,4,-3,null,null,4,1,null,null,4,null,null,5,0,5,3,null,5,-1,-5,37,null,2147483647,-1,null,2,0,2,-4,null,-2147483648,2,2,-4,0,-2,-3],"l2_bool":[null,null,true,true,null,false,null,true,null,true,false,false,null,false,false,true,false,false,false,false,false,false,true,false,null,false,false,false,true,null,null,null,false,false,false,false,false,true,true,null,true,null,false,true,false,true,null,null,true,true,false,null,true,null,true,true,true,true,false,false],"l3_utf8":["zz",null,"b",null,"c","B","a",null,"","aa",null,"😀","a",null,null,null,null,"b","aa","","é","","b","é","ab","c",null,"b","é","ab","é","b","B","zz","B",null,"ab","B",null,"","zz","c",null,"c","c","é","",null,"é",null,"ab","B","",null,null,null,"","b","ab","c"],"l4_utf8":[null,"a",null,"zz","aa",null,null,"😀",null,"ab","B","aa","B","ab",null,"a",null,null,"B","c","😀",null,"",null,null,"😀",null,"B","é","","zz","😀",null,"a",null,"a","b","",null,null,"","a","é",null,"","ab",null,"zz",null,"","c","é",null,null,"","c","aa",null,"b","😀"],"l5_utf8":["é",null,null,"ab","😀","B",null,null,"","a",null,null,"😀","é","aa",null,null,"","","zz","ab",null,"a",null,"c",null,null,null,null,null,"aa",null,null,"a",null,"b","a","aa","a","😀","ab","aa",null,null,"c",null,null,"b","B",null,null,"c","zz",null,null,"",null,null,null,null]}}},"ops":[{"op":"withColumns","exprs":{"x0":{"col":"l1_i32"}}},{"op":"filter","predicate":{"fn":"isNotNull","args":[{"col":"l3_utf8"}]}},{"op":"head","n":2}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_utf8","type":"utf8"},{"name":"x0","type":"i32"}],"columns":{"l0_utf8":["😀","B"],"l1_i32":[null,null],"l2_bool":[null,true],"l3_utf8":["zz","b"],"l4_utf8":[null,null],"l5_utf8":["é",null],"x0":[null,null]}}},{"id":"random/0341","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_i32","type":"i32"},{"name":"l4_utf8","type":"utf8"}],"columns":{"l0_utf8":["aa","","c","a","é","","B","aa","aa","b","zz","a","b","a","aa","ab","c","b","a","c","aa","b","","a","c","😀","b","b","ab","B","a","zz","aa","aa","","B","é","","ab","aa","😀","B","b","","é","ab","😀","","b","zz","B","b","é","é","b","aa","a","aa","aa","a"],"l1_i32":[-2,0,-1,2147483647,-2,-5,2,-2147483648,2,-983,5,4,0,-1,1,3,2,2,-2,-3,1,3,-1,-1,2147483647,-2,5,-1,4,1,-5,1,-2147483648,-2,2,2,3,-3,-5,3,-3,1,-3,-3,0,5,5,5,2,5,0,2,1,1,4,-1,-2147483648,0,-2,-1],"l2_bool":[true,true,false,true,false,true,false,false,true,true,true,true,false,false,false,false,false,false,true,true,true,true,true,false,false,false,false,false,true,true,true,false,false,false,false,true,true,false,false,true,true,true,true,false,false,true,false,true,false,true,true,false,false,true,true,true,true,false,false,false],"l3_i32":[-4,0,-1,3,-2,-1,0,3,0,2147483647,4,-4,-4,4,0,5,3,-3,0,-3,-4,-4,3,-5,5,1,-3,4,-5,3,0,4,-1,4,0,4,5,-3,-2147483648,1,4,1,-4,1,3,-2,-4,-2147483648,-2,-3,-2,-3,-2,5,-4,2,2147483647,1,1,-5],"l4_utf8":["c","a","ab","a","ab","😀","😀","","B","😀","é","B","c","é","a","c","ab","b","ab","B","😀","B","","😀","B","é","😀","","b","b","😀","aa","c","c","ab","","zz","a","a","b","ab","B","zz","a","B","zz","zz","aa","c","ab","😀","a","😀","é","aa","é","a","a","","c"]}},"right":{"fields":[{"name":"l1_i32","type":"i32"},{"name":"r0_utf8","type":"utf8"}],"columns":{"r0_utf8":["é","zz","c","😀","é","","zz","c","zz","ab"],"l1_i32":[-3,5,4,-1,-2,5,-2,2,2,0]}}},"ops":[{"op":"withColumns","exprs":{"x0":{"col":"l2_bool"}}},{"op":"sort","keys":[{"expr":{"col":"l4_utf8"},"descending":true},{"expr":{"col":"l2_bool"},"descending":false}],"nullsLast":false},{"op":"unique"},{"op":"join","right":"right","on":["l1_i32"],"how":"semi"}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_i32","type":"i32"},{"name":"l4_utf8","type":"utf8"},{"name":"x0","type":"bool"}],"columns":{"l0_utf8":["B","a","b","B","a","😀","zz","aa","😀","b","ab","b","aa","b","a","aa","b","ab","c","é","c","","a","😀","zz","","b","","","b","aa","","é","aa","a","c","b","aa","","B"],"l1_i32":[2,-1,5,0,-1,-2,5,-1,5,-3,5,0,-2,2,-1,-2,2,4,-1,-2,2,2,-2,-3,5,5,4,-3,-3,2,0,0,0,2,4,-3,-1,-2,-1,2],"l2_bool":[false,false,false,true,false,false,true,true,false,true,true,false,false,false,false,true,false,true,false,false,false,false,true,true,true,true,true,false,false,false,false,true,false,true,true,true,false,false,true,true],"l3_i32":[0,-5,-3,-2,4,1,4,2,-4,-4,-2,-4,4,-2,-5,-4,-3,-5,-1,-2,3,0,0,4,-3,-2147483648,-4,-3,1,-3,1,0,3,0,-4,-3,4,1,3,4],"l4_utf8":["😀","😀","😀","😀","é","é","é","é","zz","zz","zz","c","c","c","c","c","b","b","ab","ab","ab","ab","ab","ab","ab","aa","aa","a","a","a","a","a","B","B","B","B","","","",""],"x0":[false,false,false,true,false,false,true,true,false,true,true,false,false,false,false,true,false,true,false,false,false,false,true,true,true,true,true,false,false,false,false,true,false,true,true,true,false,false,true,true]}}},{"id":"random/0342","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_f64","type":"f64"}],"columns":{"l0_utf8":["c","é","é","ab","zz","aa","😀","é","B","B","b","😀","ab","a","c","😀","B","a","c","","","aa","b","b","B","zz","😀","a","c","é","é","zz","é",null,"b","ab","","B","b","é","b","b","zz","B","","a","😀","😀","ab","b","B","ab","😀","c","B",null,"c","b",null,null],"l1_i32":[0,-2,null,3,3,-2,4,null,4,-4,-4,1,-952,5,4,-2,0,-2,0,-4,5,null,null,-5,1,1,-1,3,1,-1,-1,4,-1,null,4,-2,0,-1,5,-5,2,4,-2,-4,5,4,-2,-2147483648,4,null,1,-1,2,-4,3,-4,4,5,-2,-5],"l2_f64":[-1.0,-88.042,null,-4.0,1.0,-1.0,99.455,-98.725,-2.0,-3.0,-4.0,-1.0,-95.145,1.0,-59.943,54.562,82.518,3.0,3.0,null,-1.0,46.246,-20.842,-30.231,-48.379,-1.0,22.54,38.5,94.032,null,-8.288,-39.017,77.237,-1.0,-92.143,null,3.0,null,4.0,22.119,-2.0,0.0,null,-3.0,-1.0,-3.0,1.0,-88.093,38.305,null,-53.892,4.0,0.0,3.0,0.0,null,3.0,44.688,-2.148,-42.835],"l3_f64":[null,2.0,-4.0,72.066,4.0,-3.0,-20.263,-69.479,2.0,-77.91,-57.52,-1.0,97.641,7.176,3.0,-57.867,-2.0,-4.0,81.071,-1.0,-4.0,-40.62,2.0,-3.0,37.187,-62.227,1.0,-20.351,-4.0,-3.0,-1.0,1.0,3.0,-4.0,-3.0,-71.383,-35.722,null,-4.0,null,3.0,1.0,-35.699,79.75,-32.474,1.0,-99.216,4.389,null,null,-53.364,3.0,0.0,"-0","NaN",-82.899,2.0,null,null,61.323]}}},"ops":[{"op":"filter","predicate":{"fn":"lt","args":[{"col":"l3_f64"},{"fn":"fillNull","args":[{"col":"l1_i32"},{"lit":1.5}]}]}},{"op":"filter","predicate":{"fn":"and","args":[{"fn":"lte","args":[{"col":"l1_i32"},{"col":"l2_f64"}]},{"fn":"and","args":[{"fn":"isNull","args":[{"col":"l2_f64"}]},{"fn":"gte","args":[{"lit":0.5},{"col":"l1_i32"}]}]}]}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_f64","type":"f64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_f64":[],"l3_f64":[]}}},{"id":"random/0343","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_utf8","type":"utf8"}],"columns":{"l0_utf8":["c","zz","c","é","c","ab","B","b","ab","é","",null,"😀","ab","aa","B","b","","😀","B","","b","","b","zz","B","c","a","c","😀",null,"ab","c","ab","é","zz","",null,null,"a","ab","a","ab","aa","😀","c",null,"a","aa","a","","c","a","a","B","a","",null,"","zz"],"l1_i32":[4,-2,-5,-4,1,-1,2,-2,-2,-2,-2147483648,3,-1,-3,2147483647,-4,-2147483648,-2,null,5,-1,0,null,5,1,3,-3,-2,2,null,-2,0,5,4,0,-2,-5,-5,-5,2,2,-1,-3,-4,-4,-2147483648,1,-2,-2,-1,3,3,-4,-5,-3,3,1,2147483647,0,2],"l2_f64":[1.0,-3.0,null,85.842,14.504,-86.062,0.0,1.0,-2.0,-2.0,0.0,-34.704,null,-3.0,-38.271,-2.0,-26.281,84.39,-20.42,61.652,5.481,-24.706,38.776,"-0",2.0,58.354,1.0,null,3.0,64.007,"NaN",null,2.0,null,1.0,4.0,-1.0,-73.277,-65.266,54.824,-68.6,4.0,-4.0,-1.0,83.67,89.417,45.086,76.31,9.927,null,2.0,-68.415,57.815,-4.0,-1.0,-1.0,7.538,4.928,"NaN","-Infinity"],"l3_utf8":["c","😀",null,"zz","c","é","b",null,null,"","a","a","zz","a","zz","c","c","a","é","ab","zz","b","B","","c","c","😀",null,"B","a","b","a","a",null,null,"c","zz","b","zz","b","","ab","ab","","c","b","😀",null,"aa","c","zz",null,"😀","😀","ab","","ab","aa","ab",""]}},"right":{"fields":[{"name":"l1_i32","type":"i32"},{"name":"r0_f64","type":"f64"}],"columns":{"r0_f64":[],"l1_i32":[]}}},"ops":[{"op":"head","n":0},{"op":"join","right":"right","on":["l1_i32"],"how":"semi"}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_utf8","type":"utf8"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_f64":[],"l3_utf8":[]}}},{"id":"random/0344","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_i32","type":"i32"},{"name":"l4_bool","type":"bool"},{"name":"l5_bool","type":"bool"},{"name":"l6_i64","type":"i64"}],"columns":{"l0_utf8":["c"],"l1_i32":[-4],"l2_f64":[-66.266],"l3_i32":[2],"l4_bool":[true],"l5_bool":[false],"l6_i64":["-4611686018427387904"]}},"right":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"r0_bool","type":"bool"}],"columns":{"r0_bool":[true,true,false,false,false,true,false,true,true,true,false,true,false,false,false,true,false,true,true,true,false,true,true,true,false,true,true,false,false,false],"l0_utf8":["c","c","c","c","c","c","c","c","c","c","c","c","c","c","c","c","c","c","c","c","c","c","c","c","c","c","c","c","c","c"]}}},"ops":[{"op":"join","right":"right","on":["l0_utf8"],"how":"left"},{"op":"filter","predicate":{"fn":"gt","args":[{"col":"l3_i32"},{"fn":"fillNull","args":[{"lit":0.5},{"lit":1.5}]}]}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_i32","type":"i32"},{"name":"l4_bool","type":"bool"},{"name":"l5_bool","type":"bool"},{"name":"l6_i64","type":"i64"},{"name":"r0_bool","type":"bool"}],"columns":{"l0_utf8":["c","c","c","c","c","c","c","c","c","c","c","c","c","c","c","c","c","c","c","c","c","c","c","c","c","c","c","c","c","c"],"l1_i32":[-4,-4,-4,-4,-4,-4,-4,-4,-4,-4,-4,-4,-4,-4,-4,-4,-4,-4,-4,-4,-4,-4,-4,-4,-4,-4,-4,-4,-4,-4],"l2_f64":[-66.266,-66.266,-66.266,-66.266,-66.266,-66.266,-66.266,-66.266,-66.266,-66.266,-66.266,-66.266,-66.266,-66.266,-66.266,-66.266,-66.266,-66.266,-66.266,-66.266,-66.266,-66.266,-66.266,-66.266,-66.266,-66.266,-66.266,-66.266,-66.266,-66.266],"l3_i32":[2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2],"l4_bool":[true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true],"l5_bool":[false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false],"l6_i64":["-4611686018427387904","-4611686018427387904","-4611686018427387904","-4611686018427387904","-4611686018427387904","-4611686018427387904","-4611686018427387904","-4611686018427387904","-4611686018427387904","-4611686018427387904","-4611686018427387904","-4611686018427387904","-4611686018427387904","-4611686018427387904","-4611686018427387904","-4611686018427387904","-4611686018427387904","-4611686018427387904","-4611686018427387904","-4611686018427387904","-4611686018427387904","-4611686018427387904","-4611686018427387904","-4611686018427387904","-4611686018427387904","-4611686018427387904","-4611686018427387904","-4611686018427387904","-4611686018427387904","-4611686018427387904"],"r0_bool":[true,true,false,false,false,true,false,true,true,true,false,true,false,false,false,true,false,true,true,true,false,true,true,true,false,true,true,false,false,false]}}},{"id":"random/0345","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_f64","type":"f64"}],"columns":{"l0_utf8":["zz","é","é","B","aa","","","zz","ab","B","c","","aa","zz","aa","ab","aa","ab","ab","a","c","b","aa","","ab","😀","😀","B","aa","aa","","é","","ab","B","a","B","zz","ab","zz","ab","b","😀","aa","c","B","ab","B","","aa","a","é","b","aa","ab","zz","zz","é","aa","é"],"l1_i32":[-324,-3,4,-3,-4,5,2,-5,-4,3,-3,0,-4,2,3,5,-4,0,3,0,-2,-5,4,-1,5,2147483647,-2,-551,2,2,-1,-1,3,-2,-5,1,2,5,1,4,4,5,2,-4,0,-3,4,-3,-3,-3,4,3,2,2,-2147483648,4,1,5,2,-4],"l2_bool":[true,false,true,false,false,true,true,true,false,false,true,true,false,true,true,true,false,false,false,false,false,true,true,false,false,true,true,true,true,true,false,true,true,true,false,false,true,true,false,false,false,true,true,false,true,true,false,true,true,true,false,true,false,true,true,false,true,true,false,false],"l3_utf8":["B","ab","zz","ab","a","zz","é","zz","zz","","ab","é","B","c","zz","ab","c","c","","","","ab","aa","zz","zz","é","a","B","zz","ab","c","é","a","zz","a","","c","aa","é","zz","😀","b","😀","ab","c","b","c","ab","ab","","B","B","b","😀","😀","b","aa","😀","é","é"],"l4_utf8":["","aa","zz","😀","c","😀","b","aa","zz","zz","B","","a","","zz","","c","😀","ab","é","","a","zz","😀","B","é","aa","","c","c","é","c","zz","é","zz","zz","B","😀","","b","😀","aa","aa","c","c","aa","😀","zz","é","c","","a","b","c","","ab","é","😀","B","c"],"l5_f64":[82.058,-2.0,-1.0,-46.863,-78.829,-55.809,95.667,-21.715,-24.689,-1.0,-1.0,1.0,50.176,-18.988,2.0,"NaN",0.0,28.655,-2.0,0.0,94.161,1.0,-14.313,-70.898,97.98,0.0,-82.732,1.0,75.229,-2.0,-1.0,3.0,70.41,-2.0,-2.0,1.0,37.354,1.0,-10.941,-4.0,-84.086,-4.0,-93.37,-88.595,86.611,3.0,61.699,4.0,1.0,48.982,65.049,2.0,-97.157,20.197,3.0,2.0,0.0,-3.0,-3.0,42.472]}}},"ops":[{"op":"unique"},{"op":"withColumns","exprs":{"x1":{"col":"l1_i32"}}},{"op":"withColumns","exprs":{"x2":{"lit":0.5}}},{"op":"unique","columns":["x1"]}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_f64","type":"f64"},{"name":"x1","type":"i32"},{"name":"x2","type":"f64"}],"columns":{"l0_utf8":["zz","é","é","aa","","","zz","B","","c","","😀","B","a","ab"],"l1_i32":[-324,-3,4,-4,5,2,-5,3,0,-2,-1,2147483647,-551,1,-2147483648],"l2_bool":[true,false,true,false,true,true,true,false,true,false,false,true,true,false,true],"l3_utf8":["B","ab","zz","a","zz","é","zz","","é","","zz","é","B","","😀"],"l4_utf8":["","aa","zz","c","😀","b","aa","zz","","","😀","é","","zz",""],"l5_f64":[82.058,-2.0,-1.0,-78.829,-55.809,95.667,-21.715,-1.0,1.0,94.161,-70.898,0.0,1.0,1.0,3.0],"x1":[-324,-3,4,-4,5,2,-5,3,0,-2,-1,2147483647,-551,1,-2147483648],"x2":[0.5,0.5,0.5,0.5,0.5,0.5,0.5,0.5,0.5,0.5,0.5,0.5,0.5,0.5,0.5]}}},{"id":"random/0346","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_bool","type":"bool"},{"name":"l4_f64","type":"f64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_bool":[],"l3_bool":[],"l4_f64":[]}}},"ops":[{"op":"filter","predicate":{"fn":"isIn","args":[{"col":"l0_utf8"}],"values":["ab","zz","😀"]}},{"op":"withColumns","exprs":{"x1":{"fn":"mod","args":[{"fn":"mul","args":[{"col":"l4_f64"},{"col":"l4_f64"}]},{"fn":"neg","args":[{"col":"l1_i32"}]}]}}},{"op":"filter","predicate":{"fn":"lte","args":[{"fn":"div","args":[{"col":"l4_f64"},{"col":"l1_i32"}]},{"fn":"abs","args":[{"lit":2.5}]}]}},{"op":"filter","predicate":{"col":"l3_bool"}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_bool","type":"bool"},{"name":"l4_f64","type":"f64"},{"name":"x1","type":"f64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_bool":[],"l3_bool":[],"l4_f64":[],"x1":[]}}},{"id":"random/0347","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i64","type":"i64"},{"name":"l3_i64","type":"i64"}],"columns":{"l0_utf8":["c","é","😀","aa",null,"b","aa","B","c","ab","é",null,"a","zz","a","é","😀","ab","c","","c","B","c","aa","c","ab","aa","b","B","é","b","ab","b","B","é","😀","","a","a","B","B","aa","c","b","zz","b",null,"aa","é","B","ab","zz","","ab","😀","zz","😀","a","b","B"],"l1_i32":[-5,4,null,5,-2,0,4,-2,-4,1,-2,0,-5,-194,-2147483648,2,-1,4,1,0,1,-3,0,-3,1,3,null,null,4,-3,2,2,-5,-5,-5,-5,5,3,-1,1,-4,1,-3,3,4,-1,5,null,5,0,-1,-2,0,2147483647,4,-1,-2,0,0,0],"l2_i64":["-4611686018427387904","9007199254740995","-4611686018427387904","9007199254740993","3",null,null,"-4611686018427387904","-4611686018427387904","-2","9007199254740994","-3","-4611686018427387904","5",null,"-4611686018427387904","-4","2","-4611686018427387904","-4","9007199254740994","-4","-2","-4611686018427387904","5","-4611686018427387904","5","9007199254740995","-4611686018427387904","9007199254740992","-4611686018427387904",null,null,"9007199254740995","9007199254740994","-5","-4611686018427387904","-2","-4611686018427387904","-4611686018427387904","-3","4","9007199254740992","5","9007199254740992","9007199254740992","9007199254740992","9007199254740993","9007199254740994","0","-4611686018427387904","-4611686018427387904","9007199254740995",null,"9007199254740992","-4611686018427387904","-4611686018427387904","-4611686018427387904","9007199254740995","-4611686018427387904"],"l3_i64":["-4","-4611686018427387904","9007199254740994","-4611686018427387904","0","2","9007199254740994","-4611686018427387904","-4611686018427387904","-4","-4611686018427387904",null,"-5",null,"9007199254740993","9007199254740992","4","0","-4","0","4",null,"9007199254740995","9007199254740993","-5","-4611686018427387904","-4611686018427387904","-4611686018427387904","-2","9007199254740993","9007199254740993","-4611686018427387904","9007199254740994","9007199254740995","9007199254740995","-2","5","9007199254740993","-1","-4611686018427387904","9007199254740993","-2","-4611686018427387904","-4611686018427387904","0",null,"-2","3","9007199254740994","-4611686018427387904","-4611686018427387904","-1","-4611686018427387904","-5","9007199254740993","-4611686018427387904","3","-4611686018427387904","-4611686018427387904","0"]}}},"ops":[{"op":"withColumns","exprs":{"x0":{"col":"l1_i32"}}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i64","type":"i64"},{"name":"l3_i64","type":"i64"},{"name":"x0","type":"i32"}],"columns":{"l0_utf8":["c","é","😀","aa",null,"b","aa","B","c","ab","é",null,"a","zz","a","é","😀","ab","c","","c","B","c","aa","c","ab","aa","b","B","é","b","ab","b","B","é","😀","","a","a","B","B","aa","c","b","zz","b",null,"aa","é","B","ab","zz","","ab","😀","zz","😀","a","b","B"],"l1_i32":[-5,4,null,5,-2,0,4,-2,-4,1,-2,0,-5,-194,-2147483648,2,-1,4,1,0,1,-3,0,-3,1,3,null,null,4,-3,2,2,-5,-5,-5,-5,5,3,-1,1,-4,1,-3,3,4,-1,5,null,5,0,-1,-2,0,2147483647,4,-1,-2,0,0,0],"l2_i64":["-4611686018427387904","9007199254740995","-4611686018427387904","9007199254740993","3",null,null,"-4611686018427387904","-4611686018427387904","-2","9007199254740994","-3","-4611686018427387904","5",null,"-4611686018427387904","-4","2","-4611686018427387904","-4","9007199254740994","-4","-2","-4611686018427387904","5","-4611686018427387904","5","9007199254740995","-4611686018427387904","9007199254740992","-4611686018427387904",null,null,"9007199254740995","9007199254740994","-5","-4611686018427387904","-2","-4611686018427387904","-4611686018427387904","-3","4","9007199254740992","5","9007199254740992","9007199254740992","9007199254740992","9007199254740993","9007199254740994","0","-4611686018427387904","-4611686018427387904","9007199254740995",null,"9007199254740992","-4611686018427387904","-4611686018427387904","-4611686018427387904","9007199254740995","-4611686018427387904"],"l3_i64":["-4","-4611686018427387904","9007199254740994","-4611686018427387904","0","2","9007199254740994","-4611686018427387904","-4611686018427387904","-4","-4611686018427387904",null,"-5",null,"9007199254740993","9007199254740992","4","0","-4","0","4",null,"9007199254740995","9007199254740993","-5","-4611686018427387904","-4611686018427387904","-4611686018427387904","-2","9007199254740993","9007199254740993","-4611686018427387904","9007199254740994","9007199254740995","9007199254740995","-2","5","9007199254740993","-1","-4611686018427387904","9007199254740993","-2","-4611686018427387904","-4611686018427387904","0",null,"-2","3","9007199254740994","-4611686018427387904","-4611686018427387904","-1","-4611686018427387904","-5","9007199254740993","-4611686018427387904","3","-4611686018427387904","-4611686018427387904","0"],"x0":[-5,4,null,5,-2,0,4,-2,-4,1,-2,0,-5,-194,-2147483648,2,-1,4,1,0,1,-3,0,-3,1,3,null,null,4,-3,2,2,-5,-5,-5,-5,5,3,-1,1,-4,1,-3,3,4,-1,5,null,5,0,-1,-2,0,2147483647,4,-1,-2,0,0,0]}}},{"id":"random/0348","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_utf8","type":"utf8"}],"columns":{"l0_utf8":["a","b","😀","","é",null,null,"zz","ab","aa",null,"zz","a","a","ab","c","é","","zz","aa"],"l1_i32":[-2,-4,0,1,2,-1,0,-5,5,0,null,3,null,-4,2,-3,-2,2147483647,5,0],"l2_i32":[2,0,3,-3,3,2,-1,4,2,null,-4,-2,0,-3,null,0,2,0,0,null],"l3_utf8":["B","b","B",null,"B","a","B",null,"ab","aa","é","ab",null,"ab","aa","a","B","ab",null,"😀"]}}},"ops":[{"op":"filter","predicate":{"fn":"lt","args":[{"fn":"mod","args":[{"col":"l2_i32"},{"col":"l2_i32"}]},{"fn":"mul","args":[{"col":"l1_i32"},{"col":"l2_i32"}]}]}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_utf8","type":"utf8"}],"columns":{"l0_utf8":["é","ab","a"],"l1_i32":[2,5,-4],"l2_i32":[3,2,-3],"l3_utf8":["B","ab","ab"]}}},{"id":"random/0349","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_f64","type":"f64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_f64":[],"l3_f64":[]}}},"ops":[{"op":"unique"},{"op":"withColumns","exprs":{"x1":{"fn":"mod","args":[{"fn":"mod","args":[{"col":"l1_i32"},{"col":"l2_f64"}]},{"col":"l3_f64"}]}}},{"op":"withColumns","exprs":{"x2":{"fn":"when","args":[{"fn":"isIn","args":[{"col":"l0_utf8"}],"values":["😀","ab","b"]},{"col":"l1_i32"},{"fn":"mod","args":[{"lit":2.5},{"col":"l2_f64"}]}]}}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_f64","type":"f64"},{"name":"x1","type":"f64"},{"name":"x2","type":"f64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_f64":[],"l3_f64":[],"x1":[],"x2":[]}}},{"id":"random/0350","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i64","type":"i64"},{"name":"l3_i32","type":"i32"}],"columns":{"l0_utf8":["zz","a","ab",null,null,"zz",null,"😀","ab",null,null,null,null,"ab","B","a","","é","b","ab","b","c","c",null,"a","😀",null,null,"c",null,"b","a","aa","aa",null,null,null,"é","ab","😀","","c","ab",null,"ab","a","aa",null,"é","é",null,"c","",null,"ab","ab",null,"a","b","B"],"l1_i32":[null,null,null,null,1,null,1,-4,-4,-5,null,-5,5,null,5,3,1,null,-1,2147483647,5,null,null,2147483647,null,null,null,0,null,3,-5,2147483647,1,-4,null,null,0,4,4,5,2147483647,null,null,-1,5,null,-5,5,null,null,null,2,-5,-5,-5,-4,null,-1,-5,-3],"l2_i64":["-4611686018427387904","-3","-5",null,"9007199254740993","3","-4611686018427387904","-4611686018427387904","-4611686018427387904",null,null,"-4611686018427387904","9007199254740993","-1","-2","-4611686018427387904",null,null,"9007199254740993","-4",null,"-4611686018427387904",null,null,"0","9007199254740993","9007199254740992",null,"9007199254740994","9007199254740994",null,"9007199254740993","9007199254740993","-4611686018427387904",null,"-1","-1","3",null,"-4611686018427387904","4","9007199254740992","9007199254740995",null,"-4611686018427387904",null,null,"-4611686018427387904",null,"-4611686018427387904","9007199254740993","-4611686018427387904","-4611686018427387904","-4611686018427387904","9007199254740992",null,"-4611686018427387904","-1","-4611686018427387904","2"],"l3_i32":[0,4,null,-4,4,-2,2,null,-1,-3,3,null,0,-2147483648,null,1,4,-3,5,null,-4,null,3,null,0,5,-2,0,null,null,2,5,null,-5,null,null,0,null,-2,3,5,null,4,-5,null,-5,1,null,null,-4,null,-4,-2,null,-3,1,1,3,1,null]}}},"ops":[{"op":"groupBy","keys":["l1_i32"],"aggs":{"n0":{"agg":"len"},"a0_0":{"agg":"last","arg":{"col":"l0_utf8"}},"a0_1":{"agg":"first","arg":{"col":"l3_i32"}}}}],"expected":{"fields":[{"name":"l1_i32","type":"i32"},{"name":"n0","type":"u32"},{"name":"a0_0","type":"utf8"},{"name":"a0_1","type":"i32"}],"columns":{"l1_i32":[null,1,-4,-5,5,3,-1,2147483647,0,4,2,-3],"n0":[23,4,4,8,6,2,3,4,2,2,1,1],"a0_0":[null,"aa","ab","b",null,null,"a","",null,"ab","c","B"],"a0_1":[0,4,null,-3,0,1,5,null,0,null,-4,null]}}},{"id":"random/0351","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_i32","type":"i32"},{"name":"l5_utf8","type":"utf8"}],"columns":{"l0_utf8":["b","aa","aa","","é"],"l1_i32":[3,2,5,2,1],"l2_i32":[-3,-2,3,-2,2147483647],"l3_utf8":["a","aa","B","é","é"],"l4_i32":[-4,2,3,-4,3],"l5_utf8":["zz","b","aa","a","é"]}},"right":{"fields":[{"name":"l5_utf8","type":"utf8"},{"name":"r0_bool","type":"bool"}],"columns":{"r0_bool":[true,false,null,null,false,false,false,false,false,true],"l5_utf8":["a","zz","aa","zz","é","aa","zz","b","b","é"]}}},"ops":[{"op":"unique","columns":["l2_i32"]},{"op":"join","right":"right","on":["l5_utf8"],"how":"left"}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_i32","type":"i32"},{"name":"l5_utf8","type":"utf8"},{"name":"r0_bool","type":"bool"}],"columns":{"l0_utf8":["b","b","b","aa","aa","aa","aa","é","é"],"l1_i32":[3,3,3,2,2,5,5,1,1],"l2_i32":[-3,-3,-3,-2,-2,3,3,2147483647,2147483647],"l3_utf8":["a","a","a","aa","aa","B","B","é","é"],"l4_i32":[-4,-4,-4,2,2,3,3,3,3],"l5_utf8":["zz","zz","zz","b","b","aa","aa","é","é"],"r0_bool":[false,null,false,false,false,null,false,false,true]}}},{"id":"random/0352","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_i64","type":"i64"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_utf8","type":"utf8"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_f64":[],"l3_i64":[],"l4_utf8":[],"l5_utf8":[]}},"right":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"r0_f64","type":"f64"}],"columns":{"r0_f64":[-2.0,72.578,-1.0,2.0,-71.148,59.274,-34.088,null,null,6.753,-1.0,null,4.0,65.255,"-Infinity",4.0,0.0,86.774,-3.0,78.656,null,2.0,4.0,-1.0,3.0,-24.017,19.725,15.561,-93.072,1.0],"l0_utf8":["zz",null,null,null,"c","zz","c","zz",null,null,null,"zz","c",null,"c",null,null,null,"zz",null,null,null,null,"c","c",null,null,"zz",null,null]}}},"ops":[{"op":"join","right":"right","on":["l0_utf8"],"how":"semi"},{"op":"withColumns","exprs":{"x1":{"fn":"gt","args":[{"fn":"abs","args":[{"lit":0.5}]},{"fn":"fillNull","args":[{"col":"l2_f64"},{"lit":0}]}]}}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_i64","type":"i64"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_utf8","type":"utf8"},{"name":"x1","type":"bool"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_f64":[],"l3_i64":[],"l4_utf8":[],"l5_utf8":[],"x1":[]}}},{"id":"random/0353","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_i64","type":"i64"},{"name":"l4_bool","type":"bool"}],"columns":{"l0_utf8":["","ab","B",null,"b",null,"a",null,"","a",null,null,"😀","b","a","a",null,null,"b","zz"],"l1_i32":[null,null,2,-1,5,null,null,-1,null,3,-2,4,-3,null,null,3,null,null,0,null],"l2_f64":[2.0,4.0,-1.0,null,-2.0,null,null,-76.337,2.0,null,1.0,-4.0,-1.0,-3.0,null,-4.915,null,null,null,null],"l3_i64":["-4611686018427387904","9007199254740993",null,"-2","9007199254740995",null,"1","-4611686018427387904","-4611686018427387904","9007199254740994","9007199254740992","-5",null,null,null,null,"9007199254740994","9007199254740994","-4611686018427387904","9007199254740993"],"l4_bool":[false,true,null,false,null,null,false,true,null,false,null,null,true,false,null,null,null,false,null,null]}}},"ops":[{"op":"filter","predicate":{"fn":"or","args":[{"fn":"lt","args":[{"lit":2.5},{"lit":2.5}]},{"fn":"lte","args":[{"col":"l1_i32"},{"col":"l2_f64"}]}]}},{"op":"head","n":8},{"op":"head","n":2},{"op":"withColumns","exprs":{"x3":{"fn":"mul","args":[{"col":"l1_i32"},{"fn":"div","args":[{"col":"l2_f64"},{"col":"l1_i32"}]}]}}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_i64","type":"i64"},{"name":"l4_bool","type":"bool"},{"name":"x3","type":"f64"}],"columns":{"l0_utf8":[null,"😀"],"l1_i32":[-2,-3],"l2_f64":[1.0,-1.0],"l3_i64":["9007199254740992",null],"l4_bool":[null,true],"x3":[1.0,-1.0]}}},{"id":"random/0354","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_f64","type":"f64"}],"columns":{"l0_utf8":["c","B","aa","zz","c","B","zz","b","","c","aa","aa","zz","b","ab","aa","zz","c","b","a"],"l1_i32":[-1,5,-177,-2147483648,-1,-2,4,-4,2,0,-2,4,5,0,-5,-3,3,2147483647,2,-2],"l2_utf8":["a","ab","😀","B","b","a","b","","a","","aa","😀","c","","c","B","é","b","","ab"],"l3_f64":[1.0,-83.144,4.0,-2.0,-2.0,-37.133,3.0,3.0,1.0,-4.0,2.0,3.243,-1.0,"Infinity",85.625,-9.517,53.079,1.0,88.34,3.0]}}},"ops":[{"op":"unique","columns":["l0_utf8"]}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_f64","type":"f64"}],"columns":{"l0_utf8":["c","B","aa","zz","b","","ab","a"],"l1_i32":[-1,5,-177,-2147483648,-4,2,-5,-2],"l2_utf8":["a","ab","😀","B","","a","c","ab"],"l3_f64":[1.0,-83.144,4.0,-2.0,3.0,1.0,85.625,3.0]}}},{"id":"random/0355","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_f64","type":"f64"},{"name":"l4_utf8","type":"utf8"}],"columns":{"l0_utf8":["é","aa","ab","",null,"é","ab","b","😀","b","B",null,"é","😀","zz",null,"B","aa","zz","aa","zz","😀",null,null,"zz","ab","aa","a","aa","a",null,"B","aa",null,null,null,"ab","😀","ab","b",null,"b",null,null,"c","a","c","a","c","c","b",null,null,"b","",null,"a","zz","zz",null],"l1_i32":[-4,-5,5,5,null,-1,-1,-3,0,-1,-3,-5,null,-2,null,-3,-3,null,-2147483648,2,3,null,-4,-1,null,1,2,null,2,2,5,5,0,2,-33,-3,null,-1,3,0,null,-3,-2,-1,2,-1,4,-4,null,-1,3,5,null,null,-1,-3,null,2,-4,2],"l2_i32":[2147483647,5,373,null,-3,2,null,1,230,-5,-2,null,-1,-4,-2,-1,null,null,-4,1,-1,null,2,1,-1,584,-1,null,1,null,0,0,-2,2,null,1,-386,1,4,null,null,null,2,-5,0,null,null,null,null,null,5,null,0,2,1,3,-1,null,3,-1],"l3_f64":[null,-3.0,-1.0,-1.0,70.582,-74.329,34.995,null,null,null,4.0,null,-80.667,"-Infinity",null,null,85.949,16.927,-15.204,null,-27.389,null,2.0,3.0,64.335,null,-2.0,null,45.655,-3.0,52.655,null,2.0,80.234,null,null,4.0,null,-3.0,null,1.0,null,null,81.853,3.0,-55.605,-1.0,2.0,null,80.967,15.024,-1.0,null,-4.0,2.0,null,-50.76,-1.0,null,-2.0],"l4_utf8":[null,"zz","c","B","c","b","é","c","b","B",null,"zz",null,"é",null,"B","é","é","é","a","c","b","a","ab",null,"aa","c","a",null,"B","c","aa","aa","é","","é",null,null,"ab","a",null,"b","B","ab","a",null,"aa",null,null,"",null,"😀","B","B",null,"zz","é","é",null,"é"]}}},"ops":[{"op":"unique","columns":["l1_i32"]}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_f64","type":"f64"},{"name":"l4_utf8","type":"utf8"}],"columns":{"l0_utf8":["é","aa","ab",null,"é","b","😀","😀","zz","aa","zz","ab",null,"c"],"l1_i32":[-4,-5,5,null,-1,-3,0,-2,-2147483648,2,3,1,-33,4],"l2_i32":[2147483647,5,373,-3,2,1,230,-4,-4,1,-1,584,null,null],"l3_f64":[null,-3.0,-1.0,70.582,-74.329,null,null,"-Infinity",-15.204,null,-27.389,null,null,-1.0],"l4_utf8":[null,"zz","c","c","b","c","b","é","é","a","c","aa","","aa"]}}},{"id":"random/0356","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_f64","type":"f64"},{"name":"l4_f64","type":"f64"},{"name":"l5_utf8","type":"utf8"},{"name":"l6_bool","type":"bool"}],"columns":{"l0_utf8":["a","ab","b","B","a","B","😀","ab","é","zz","a","c","ab","😀","ab","aa","B","😀","c","B","😀","c","B","ab","é","B","B","","B","é","c","é","c","c","B","é","c","é","","B","b","é","😀","c","c","é","b","zz","ab","B","é","c","B","","","ab","a","é","","zz"],"l1_i32":[2,-719,-1,-2,4,5,-3,-1,-3,2,-1,2147483647,3,0,-1,-3,3,-4,3,-5,-2,-5,-1,4,-4,-3,0,0,2,-1,5,-3,0,5,-2,-5,-2,-3,3,-3,4,1,0,-1,-4,2147483647,-4,-3,2,1,3,4,4,-2,4,-2,-1,-2,2147483647,-5],"l2_bool":[true,true,false,true,true,true,false,true,false,false,false,true,true,false,true,false,true,false,true,true,false,true,false,true,true,false,true,true,false,true,true,true,true,true,true,true,true,true,true,true,false,true,false,true,true,true,false,false,true,false,false,false,true,false,true,true,true,false,false,true],"l3_f64":[2.0,87.653,3.0,-4.0,76.157,-1.0,57.499,3.0,-8.773,1.0,70.916,2.0,73.256,-11.823,3.0,-4.0,4.0,4.0,-6.069,-1.0,49.332,4.0,94.435,56.141,-45.918,-1.0,2.0,46.248,4.0,2.0,-4.0,1.0,0.0,"NaN",4.0,-30.443,-3.0,-89.29,-3.0,-10.199,-93.638,-25.26,"NaN",47.655,3.0,-3.0,-36.12,-2.0,-4.0,-2.0,4.0,-51.796,1.0,3.0,-3.0,"Infinity",-13.928,4.0,-32.928,24.555],"l4_f64":[22.359,-3.0,-11.005,-81.129,18.19,-10.841,88.416,-89.606,"Infinity",6.099,1.0,-2.0,-44.209,"NaN","Infinity",-1.0,76.081,1.0,1.0,68.449,-1.0,2.0,4.0,69.834,0.0,73.307,-3.0,38.218,"-Infinity",68.903,-4.0,-2.0,2.0,-4.0,69.513,-4.0,"NaN",1.0,-71.237,-27.491,61.794,-1.0,3.0,-90.193,36.29,-24.872,1.0,-2.0,-43.485,-3.0,82.224,25.985,4.0,-7.983,2.0,2.0,"Infinity",-1.0,-57.222,"-Infinity"],"l5_utf8":["zz","é","é","aa","é","","","c","zz","ab","zz","a","","aa","c","zz","a","a","é","a","B","b","","ab","c","😀","é","😀","a","c","c","b","c","B","zz","😀","B","é","b","é","B","😀","😀","ab","B","aa","ab","a","B","B","b","c","é","zz","a","B","aa","😀","b","zz"],"l6_bool":[true,true,true,true,false,false,true,true,false,true,true,false,true,true,false,true,true,false,true,false,true,true,false,true,false,true,true,false,false,false,true,false,true,true,true,true,true,true,true,true,false,false,false,false,true,false,false,true,false,true,true,true,false,true,false,true,true,true,true,false]}}},"ops":[{"op":"filter","predicate":{"fn":"gte","args":[{"col":"l3_f64"},{"col":"l4_f64"}]}},{"op":"withColumns","exprs":{"x1":{"fn":"gte","args":[{"col":"l1_i32"},{"col":"l4_f64"}]}}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_f64","type":"f64"},{"name":"l4_f64","type":"f64"},{"name":"l5_utf8","type":"utf8"},{"name":"l6_bool","type":"bool"},{"name":"x1","type":"bool"}],"columns":{"l0_utf8":["ab","b","B","a","B","ab","a","c","ab","😀","😀","c","B","B","","B","c","é","c","","B","😀","c","é","zz","ab","B","","ab","é","","zz"],"l1_i32":[-719,-1,-2,4,5,-1,-1,2147483647,3,-4,-2,-5,-1,0,0,2,5,-3,5,3,-3,0,-1,2147483647,-3,2,1,-2,-2,-2,2147483647,-5],"l2_bool":[true,false,true,true,true,true,false,true,true,false,false,true,false,true,true,false,true,true,true,true,true,false,true,true,false,true,false,false,true,false,false,true],"l3_f64":[87.653,3.0,-4.0,76.157,-1.0,3.0,70.916,2.0,73.256,4.0,49.332,4.0,94.435,2.0,46.248,4.0,-4.0,1.0,"NaN",-3.0,-10.199,"NaN",47.655,-3.0,-2.0,-4.0,-2.0,3.0,"Infinity",4.0,-32.928,24.555],"l4_f64":[-3.0,-11.005,-81.129,18.19,-10.841,-89.606,1.0,-2.0,-44.209,1.0,-1.0,2.0,4.0,-3.0,38.218,"-Infinity",-4.0,-2.0,-4.0,-71.237,-27.491,3.0,-90.193,-24.872,-2.0,-43.485,-3.0,-7.983,2.0,-1.0,-57.222,"-Infinity"],"l5_utf8":["é","é","aa","é","","c","zz","a","","a","B","b","","é","😀","a","c","b","B","b","é","😀","ab","aa","a","B","B","zz","B","😀","b","zz"],"l6_bool":[true,true,true,false,false,true,true,false,true,false,true,true,false,true,false,false,true,false,true,true,true,false,false,false,true,false,true,true,true,true,true,false],"x1":[false,true,true,false,true,true,false,true,true,false,false,false,false,true,false,true,true,false,true,true,true,false,true,true,false,true,true,true,false,false,true,true]}}},{"id":"random/0357","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_f64","type":"f64"}],"columns":{"l0_utf8":["a"],"l1_i32":[-2],"l2_f64":[-63.258],"l3_f64":[-3.0]}}},"ops":[{"op":"head","n":1},{"op":"filter","predicate":{"fn":"neq","args":[{"col":"l1_i32"},{"col":"l1_i32"}]}},{"op":"sort","keys":[{"expr":{"col":"l1_i32"},"descending":true}],"nullsLast":false},{"op":"head","n":4}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_f64","type":"f64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_f64":[],"l3_f64":[]}}},{"id":"random/0358","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_f64","type":"f64"},{"name":"l4_f64","type":"f64"},{"name":"l5_i32","type":"i32"}],"columns":{"l0_utf8":["aa"],"l1_i32":[2],"l2_f64":[82.429],"l3_f64":[9.052],"l4_f64":[-1.0],"l5_i32":[null]}},"right":{"fields":[{"name":"l1_i32","type":"i32"},{"name":"r0_bool","type":"bool"}],"columns":{"r0_bool":[false,true,true],"l1_i32":[2,2,2]}}},"ops":[{"op":"filter","predicate":{"fn":"or","args":[{"fn":"gte","args":[{"col":"l3_f64"},{"col":"l2_f64"}]},{"fn":"or","args":[{"fn":"isNotNull","args":[{"col":"l0_utf8"}]},{"fn":"isNull","args":[{"col":"l1_i32"}]}]}]}},{"op":"withColumns","exprs":{"x1":{"fn":"abs","args":[{"fn":"fillNull","args":[{"lit":-1},{"lit":1.5}]}]}}},{"op":"unique","columns":["l3_f64"]},{"op":"join","right":"right","on":["l1_i32"],"how":"full"}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_f64","type":"f64"},{"name":"l4_f64","type":"f64"},{"name":"l5_i32","type":"i32"},{"name":"x1","type":"f64"},{"name":"r0_bool","type":"bool"}],"columns":{"l0_utf8":["aa","aa","aa"],"l1_i32":[2,2,2],"l2_f64":[82.429,82.429,82.429],"l3_f64":[9.052,9.052,9.052],"l4_f64":[-1.0,-1.0,-1.0],"l5_i32":[null,null,null],"x1":[1.0,1.0,1.0],"r0_bool":[false,true,true]}}},{"id":"random/0359","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i64","type":"i64"},{"name":"l3_bool","type":"bool"},{"name":"l4_f64","type":"f64"},{"name":"l5_i64","type":"i64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_i64":[],"l3_bool":[],"l4_f64":[],"l5_i64":[]}}},"ops":[{"op":"unique","columns":["l5_i64"]},{"op":"withColumns","exprs":{"x1":{"col":"l1_i32"}}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i64","type":"i64"},{"name":"l3_bool","type":"bool"},{"name":"l4_f64","type":"f64"},{"name":"l5_i64","type":"i64"},{"name":"x1","type":"i32"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_i64":[],"l3_bool":[],"l4_f64":[],"l5_i64":[],"x1":[]}}},{"id":"random/0360","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_bool","type":"bool"},{"name":"l5_f64","type":"f64"}],"columns":{"l0_utf8":["c","zz","","zz","a","b","a","aa",null,"zz",null,"B","é","😀","b","a","a","😀","zz","zz","B","B","é","é","b","zz","a","aa","😀","é","b","B","c","a","","c","😀","c","😀","a","😀",null,"","b",null,"b",null,null,"b","a",null,"a",null,"",null,"aa","b","B","",null],"l1_i32":[-2,-3,-821,0,0,4,-1,3,2,-4,-1,-197,null,5,-4,-2,null,1,-2147483648,-3,null,-1,0,null,0,2147483647,-2147483648,-3,-3,-2,null,2147483647,1,3,-3,5,2,-5,-2,-5,5,-5,null,-4,-1,0,4,5,-3,1,-5,0,4,4,null,3,-1,1,-2,null],"l2_f64":[null,0.0,null,3.0,0.0,-3.0,3.0,5.91,4.0,2.379,3.0,-4.0,-3.0,"NaN",3.0,-26.447,-3.0,3.0,null,-86.247,0.0,78.903,-3.0,23.67,-55.282,-61.11,4.0,null,-78.798,1.0,-4.0,0.0,82.054,"-Infinity",-85.757,19.351,-29.656,-59.259,-2.0,46.709,-1.0,3.0,-2.0,-49.252,-97.942,null,0.0,-2.0,83.354,1.0,-45.569,-58.389,-85.061,-85.979,-24.486,0.0,-7.18,-2.0,1.0,null],"l3_utf8":["😀","😀","😀","aa","B","","é","b","zz","aa","aa","c","a","zz","zz","c",null,"",null,"é","","zz","a",null,null,"zz","é","b","b","ab","ab","aa","é","zz",null,"B","b","zz","aa","ab","ab","b","é","😀","é","B",null,"zz","c","é","a","b","😀","😀","😀",null,"c","B","b","B"],"l4_bool":[true,false,false,false,false,true,true,null,true,false,true,true,false,false,true,false,true,false,null,false,false,true,false,false,false,true,true,true,true,true,false,true,true,true,false,false,true,false,true,true,true,null,false,false,false,false,true,true,true,null,null,true,true,null,true,true,false,null,false,true],"l5_f64":[null,-77.123,-86.068,3.0,4.0,-44.472,3.0,3.0,2.0,2.0,-3.0,-4.0,-67.576,-82.718,1.0,3.0,null,-3.0,4.0,0.0,null,88.863,-2.0,-40.152,-4.0,-18.522,0.0,1.0,53.61,"NaN",-1.0,2.0,-14.238,53.69,3.0,4.0,76.145,null,-3.0,55.596,-4.0,37.33,0.0,null,46.626,3.0,-73.747,-4.407,-16.886,-1.0,-0.8,"-0",-30.764,25.544,0.0,-17.526,null,null,-76.044,null]}}},"ops":[{"op":"unique","columns":["l3_utf8"]},{"op":"head","n":3},{"op":"groupBy","keys":["l4_bool"],"aggs":{"n2":{"agg":"len"},"a2_0":{"agg":"count","arg":{"col":"l3_utf8"}},"a2_1":{"agg":"std","arg":{"col":"l5_f64"}}}}],"expected":{"fields":[{"name":"l4_bool","type":"bool"},{"name":"n2","type":"u32"},{"name":"a2_0","type":"u32"},{"name":"a2_1","type":"f64"}],"columns":{"l4_bool":[true,false],"n2":[1,2],"a2_0":[1,2],"a2_1":[null,0.7071067811865476]}}},{"id":"random/0361","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i64","type":"i64"},{"name":"l3_bool","type":"bool"},{"name":"l4_f64","type":"f64"}],"columns":{"l0_utf8":["a"],"l1_i32":[-2],"l2_i64":["9007199254740992"],"l3_bool":[true],"l4_f64":[-1.0]}}},"ops":[{"op":"withColumns","exprs":{"x0":{"fn":"mul","args":[{"col":"l4_f64"},{"lit":-1}]}}},{"op":"filter","predicate":{"fn":"lte","args":[{"col":"x0"},{"fn":"mul","args":[{"lit":2},{"col":"x0"}]}]}},{"op":"groupBy","keys":["l0_utf8"],"aggs":{"n2":{"agg":"len"},"a2_0":{"agg":"count","arg":{"col":"l2_i64"}}}},{"op":"unique"}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"n2","type":"u32"},{"name":"a2_0","type":"u32"}],"columns":{"l0_utf8":["a"],"n2":[1],"a2_0":[1]}}},{"id":"random/0362","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_f64","type":"f64"}],"columns":{"l0_utf8":["ab",null,"",null,"😀",null,"ab",null,"a",null,null,"zz",null,null,"é",null,"ab","b","😀",""],"l1_i32":[3,5,null,0,-5,1,0,null,-3,-1,null,-2,null,-3,-3,-5,-2,1,-1,null],"l2_i32":[null,null,4,-3,-5,null,null,3,null,null,-3,null,-5,null,-1,2,null,-3,-1,0],"l3_f64":[null,null,-1.182,-3.0,null,-21.432,1.0,-70.023,-1.0,-41.194,62.733,null,-82.339,null,74.982,null,null,70.931,"NaN",null]}},"right":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"r0_bool","type":"bool"}],"columns":{"r0_bool":[true,false,true,true,true,false,true,true,false,true,false,true,true,true,false,true,true,true,false,false,true,false,false,false,true,false,true,true,false,true],"l0_utf8":["😀","é","zz","é","é","é",null,"😀",null,"a","ab",null,null,"a",null,"b","ab",null,null,"b","b",null,"😀",null,null,null,"a","a","a",null]}}},"ops":[{"op":"unique"},{"op":"join","right":"right","on":["l0_utf8"],"how":"inner"}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_f64","type":"f64"},{"name":"r0_bool","type":"bool"}],"columns":{"l0_utf8":["ab","ab","😀","😀","😀","ab","ab","a","a","a","a","a","zz","é","é","é","é","ab","ab","b","b","b","😀","😀","😀"],"l1_i32":[3,3,-5,-5,-5,0,0,-3,-3,-3,-3,-3,-2,-3,-3,-3,-3,-2,-2,1,1,1,-1,-1,-1],"l2_i32":[null,null,-5,-5,-5,null,null,null,null,null,null,null,null,-1,-1,-1,-1,null,null,-3,-3,-3,-1,-1,-1],"l3_f64":[null,null,null,null,null,1.0,1.0,-1.0,-1.0,-1.0,-1.0,-1.0,null,74.982,74.982,74.982,74.982,null,null,70.931,70.931,70.931,"NaN","NaN","NaN"],"r0_bool":[false,true,true,true,false,false,true,true,true,true,true,false,true,false,true,true,false,false,true,true,false,true,true,true,false]}}},{"id":"random/0363","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_bool","type":"bool"},{"name":"l4_f64","type":"f64"},{"name":"l5_bool","type":"bool"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_utf8":[],"l3_bool":[],"l4_f64":[],"l5_bool":[]}},"right":{"fields":[{"name":"l2_utf8","type":"utf8"},{"name":"r0_f64","type":"f64"}],"columns":{"r0_f64":[-2.0,2.0,3.0,81.286,-3.0,null,-3.0,-51.322,-3.105,-64.29,4.0,1.0,49.196,78.498,-4.0,null,null,75.322,null,11.381,-2.0,null,-1.0,null,-2.0,-4.0,44.238,-3.0,-2.0,78.262],"l2_utf8":["a","B","a",null,null,null,"B",null,"a","a","a",null,"a","a","a","a","B","B","B","B","B","a","B","B","B",null,"B",null,"a",null]}}},"ops":[{"op":"join","right":"right","on":["l2_utf8"],"how":"anti"},{"op":"unique"}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_bool","type":"bool"},{"name":"l4_f64","type":"f64"},{"name":"l5_bool","type":"bool"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_utf8":[],"l3_bool":[],"l4_f64":[],"l5_bool":[]}}},{"id":"random/0364","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i64","type":"i64"},{"name":"l3_f64","type":"f64"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_f64","type":"f64"},{"name":"l6_utf8","type":"utf8"}],"columns":{"l0_utf8":["😀",null,"b","a","😀",null,null,null,"c",null,"é",null,"zz","ab","B","",null,"ab",null,null],"l1_i32":[null,-4,null,4,5,-3,-3,-3,null,-5,null,4,5,null,-1,-4,null,-1,2147483647,null],"l2_i64":["3",null,"9007199254740995","-4611686018427387904","9007199254740994",null,"-4611686018427387904","5","-4611686018427387904","9007199254740995",null,null,"-4611686018427387904",null,null,"9007199254740994","-4","-1",null,"9007199254740994"],"l3_f64":[-50.694,-23.692,null,-3.0,null,84.871,3.0,null,4.0,-3.0,null,1.0,-3.0,null,-1.0,57.757,null,49.863,4.0,null],"l4_utf8":[null,null,"ab","😀","c","c","b","aa",null,"B",null,"é","a","ab","ab","c","ab","b",null,null],"l5_f64":[73.907,0.0,1.0,null,null,-48.306,null,null,27.27,null,-1.0,null,null,null,1.0,-30.214,null,-48.025,4.0,-3.0],"l6_utf8":["é","aa","ab","é","b","ab","😀","b","é","c",null,"ab","zz","B","😀","é",null,"B","ab","zz"]}},"right":{"fields":[{"name":"l6_utf8","type":"utf8"},{"name":"r0_f64","type":"f64"}],"columns":{"r0_f64":[-22.732,null,91.352,3.0,-2.0,null,null,null,-32.074,-22.024,2.0,null,2.0,3.0,4.0,-70.781,29.995,"NaN",null,4.0,null,-79.081,null,null,null,-2.0,"NaN",null,-1.0,97.862],"l6_utf8":["ab","é","c","zz","ab","é","b","é","ab","é","c","é","é","B",null,"é","b","B","é","ab","B","😀","😀","é","ab","é","ab","😀",null,"é"]}}},"ops":[{"op":"groupBy","keys":["l6_utf8"],"aggs":{"n0":{"agg":"len"},"a0_0":{"agg":"count","arg":{"col":"l2_i64"}}}},{"op":"join","right":"right","on":["l6_utf8"],"how":"anti"}],"expected":{"fields":[{"name":"l6_utf8","type":"utf8"},{"name":"n0","type":"u32"},{"name":"a0_0","type":"u32"}],"columns":{"l6_utf8":["aa",null],"n0":[1,2],"a0_0":[0,1]}}},{"id":"random/0365","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_bool","type":"bool"},{"name":"l4_f64","type":"f64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_f64":[],"l3_bool":[],"l4_f64":[]}}},"ops":[{"op":"withColumns","exprs":{"x0":{"fn":"isIn","args":[{"col":"l0_utf8"}],"values":["c","a","B"]}}},{"op":"unique"},{"op":"withColumns","exprs":{"x2":{"fn":"and","args":[{"fn":"eq","args":[{"col":"l0_utf8"},{"lit":"c"}]},{"fn":"lt","args":[{"lit":2},{"col":"l4_f64"}]}]}}},{"op":"withColumns","exprs":{"x3":{"col":"l4_f64"}}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_bool","type":"bool"},{"name":"l4_f64","type":"f64"},{"name":"x0","type":"bool"},{"name":"x2","type":"bool"},{"name":"x3","type":"f64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_f64":[],"l3_bool":[],"l4_f64":[],"x0":[],"x2":[],"x3":[]}}},{"id":"random/0366","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_f64","type":"f64"},{"name":"l5_i64","type":"i64"},{"name":"l6_utf8","type":"utf8"}],"columns":{"l0_utf8":[null,"","a",null,"c",null,"é","B",null,"ab","é","zz",null,null,null,null,null,"b","","",null,"b","c","","ab","é","B","ab","B","","a","😀","😀","aa","😀","😀","b","","😀","a","ab",null,"","c","😀","é","zz","aa","😀","zz","","ab","é","ab","aa","zz",null,null,null,"b"],"l1_i32":[-5,5,2,0,1,null,3,3,null,-2147483648,1,-3,5,null,-5,5,null,-3,5,null,5,null,-4,-1,-1,-5,-254,2147483647,null,5,3,1,4,0,2,null,2,null,null,3,1,null,null,-3,null,null,0,-1,null,2,null,5,0,-1,-5,0,0,-5,1,1],"l2_bool":[null,null,null,null,null,true,false,null,true,false,null,false,true,null,true,true,true,false,true,false,false,null,false,null,true,true,true,true,false,null,true,null,false,null,null,true,true,false,null,false,false,true,null,true,true,true,false,null,false,null,false,false,false,true,true,null,null,null,true,true],"l3_utf8":[null,"aa","ab","aa","ab","a","aa","ab",null,null,"aa","B",null,"aa",null,"b",null,null,null,"ab","","b","é",null,"zz","zz","b","a","aa","ab","ab",null,null,"a","😀","😀","b","ab","😀","zz","b","b",null,"a","ab","😀",null,"é","😀","aa","😀","😀",null,"😀",null,"ab","é","","ab",null],"l4_f64":["NaN",null,null,"NaN",87.24,"NaN",1.0,2.0,null,2.0,null,61.212,36.952,-2.0,0.0,"-0",null,57.019,null,null,-4.0,2.0,69.638,-4.0,null,-14.064,78.188,null,-3.0,null,4.0,null,55.408,0.0,"NaN",null,null,-1.0,-70.486,62.488,null,-9.905,41.272,-4.0,1.0,"NaN",null,0.0,-2.0,null,4.0,63.367,-15.665,-22.358,null,-4.0,-2.0,-39.799,-58.209,-15.887],"l5_i64":["-4","9007199254740992","-4611686018427387904",null,"9007199254740993","-3",null,"9007199254740995","-4611686018427387904","-4611686018427387904","9007199254740995","-4611686018427387904",null,null,"-4611686018427387904","0","0","9007199254740994","-5","-5",null,"9007199254740995","9007199254740993","0","-3","-4611686018427387904","9007199254740994","0",null,"-4611686018427387904","9007199254740994","0",null,"3","9007199254740993","9007199254740992","-1",null,null,"9007199254740992",null,"9007199254740994",null,"9007199254740992","-4611686018427387904",null,null,"3","3","3","-4611686018427387904",null,null,"-4611686018427387904",null,"-2","9007199254740994",null,"-2",null],"l6_utf8":["c","é",null,"",null,"aa","ab",null,null,"é","ab","c","é","B","aa","c","aa","zz","c","aa",null,"c","aa","",null,"aa",null,null,null,"a","é","a",null,"","aa","","😀",null,"c","é","a","ab",null,"B","","c","aa",null,null,null,"é",null,null,null,null,null,"c","b",null,""]}}},"ops":[{"op":"withColumns","exprs":{"x0":{"fn":"mul","args":[{"lit":0.5},{"fn":"abs","args":[{"col":"l4_f64"}]}]}}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_f64","type":"f64"},{"name":"l5_i64","type":"i64"},{"name":"l6_utf8","type":"utf8"},{"name":"x0","type":"f64"}],"columns":{"l0_utf8":[null,"","a",null,"c",null,"é","B",null,"ab","é","zz",null,null,null,null,null,"b","","",null,"b","c","","ab","é","B","ab","B","","a","😀","😀","aa","😀","😀","b","","😀","a","ab",null,"","c","😀","é","zz","aa","😀","zz","","ab","é","ab","aa","zz",null,null,null,"b"],"l1_i32":[-5,5,2,0,1,null,3,3,null,-2147483648,1,-3,5,null,-5,5,null,-3,5,null,5,null,-4,-1,-1,-5,-254,2147483647,null,5,3,1,4,0,2,null,2,null,null,3,1,null,null,-3,null,null,0,-1,null,2,null,5,0,-1,-5,0,0,-5,1,1],"l2_bool":[null,null,null,null,null,true,false,null,true,false,null,false,true,null,true,true,true,false,true,false,false,null,false,null,true,true,true,true,false,null,true,null,false,null,null,true,true,false,null,false,false,true,null,true,true,true,false,null,false,null,false,false,false,true,true,null,null,null,true,true],"l3_utf8":[null,"aa","ab","aa","ab","a","aa","ab",null,null,"aa","B",null,"aa",null,"b",null,null,null,"ab","","b","é",null,"zz","zz","b","a","aa","ab","ab",null,null,"a","😀","😀","b","ab","😀","zz","b","b",null,"a","ab","😀",null,"é","😀","aa","😀","😀",null,"😀",null,"ab","é","","ab",null],"l4_f64":["NaN",null,null,"NaN",87.24,"NaN",1.0,2.0,null,2.0,null,61.212,36.952,-2.0,0.0,"-0",null,57.019,null,null,-4.0,2.0,69.638,-4.0,null,-14.064,78.188,null,-3.0,null,4.0,null,55.408,0.0,"NaN",null,null,-1.0,-70.486,62.488,null,-9.905,41.272,-4.0,1.0,"NaN",null,0.0,-2.0,null,4.0,63.367,-15.665,-22.358,null,-4.0,-2.0,-39.799,-58.209,-15.887],"l5_i64":["-4","9007199254740992","-4611686018427387904",null,"9007199254740993","-3",null,"9007199254740995","-4611686018427387904","-4611686018427387904","9007199254740995","-4611686018427387904",null,null,"-4611686018427387904","0","0","9007199254740994","-5","-5",null,"9007199254740995","9007199254740993","0","-3","-4611686018427387904","9007199254740994","0",null,"-4611686018427387904","9007199254740994","0",null,"3","9007199254740993","9007199254740992","-1",null,null,"9007199254740992",null,"9007199254740994",null,"9007199254740992","-4611686018427387904",null,null,"3","3","3","-4611686018427387904",null,null,"-4611686018427387904",null,"-2","9007199254740994",null,"-2",null],"l6_utf8":["c","é",null,"",null,"aa","ab",null,null,"é","ab","c","é","B","aa","c","aa","zz","c","aa",null,"c","aa","",null,"aa",null,null,null,"a","é","a",null,"","aa","","😀",null,"c","é","a","ab",null,"B","","c","aa",null,null,null,"é",null,null,null,null,null,"c","b",null,""],"x0":["NaN",null,null,"NaN",43.62,"NaN",0.5,1.0,null,1.0,null,30.606,18.476,1.0,0.0,0.0,null,28.5095,null,null,2.0,1.0,34.819,2.0,null,7.032,39.094,null,1.5,null,2.0,null,27.704,0.0,"NaN",null,null,0.5,35.243,31.244,null,4.9525,20.636,2.0,0.5,"NaN",null,0.0,1.0,null,2.0,31.6835,7.8325,11.179,null,2.0,1.0,19.8995,29.1045,7.9435]}}},{"id":"random/0367","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_i64","type":"i64"},{"name":"l4_f64","type":"f64"},{"name":"l5_i64","type":"i64"},{"name":"l6_f64","type":"f64"}],"columns":{"l0_utf8":[null,"B","b",null,"zz","aa","b","b",null,null,null,"b",null,"é","ab","B","aa","b","","zz"],"l1_i32":[null,null,null,null,4,-1,3,-2,2,-983,null,2,2,3,0,2,2,-5,null,2],"l2_utf8":[null,null,"c","aa","c","b",null,"😀",null,null,null,"😀","zz","é","B","é","B","a","a","a"],"l3_i64":["-5","9007199254740994","9007199254740993","9007199254740994","5","3",null,"-4611686018427387904",null,"9007199254740992",null,"-1","-4611686018427387904",null,"4",null,null,"-4611686018427387904","-4611686018427387904",null],"l4_f64":[null,-2.0,58.184,null,null,90.503,-2.0,null,null,71.104,3.0,-1.0,-1.0,null,null,1.634,2.0,1.0,null,-33.622],"l5_i64":["-1",null,null,"-4611686018427387904","9007199254740993",null,"4","-4611686018427387904","-4611686018427387904","9007199254740995",null,null,"-4611686018427387904","-4611686018427387904","9007199254740995","9007199254740995","9007199254740994","-4",null,"-4"],"l6_f64":[-17.201,3.0,-21.711,-65.025,null,0.0,17.356,null,-2.0,1.0,1.0,2.0,null,null,27.118,4.0,4.0,"-0",51.47,-92.759]}}},"ops":[{"op":"unique","columns":["l0_utf8"]},{"op":"withColumns","exprs":{"x1":{"fn":"mod","args":[{"fn":"floorDiv","args":[{"col":"l4_f64"},{"col":"l1_i32"}]},{"col":"l1_i32"}]}}},{"op":"sort","keys":[{"expr":{"col":"l2_utf8"},"descending":true}],"nullsLast":true},{"op":"filter","predicate":{"fn":"lte","args":[{"col":"x1"},{"col":"l1_i32"}]}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_i64","type":"i64"},{"name":"l4_f64","type":"f64"},{"name":"l5_i64","type":"i64"},{"name":"l6_f64","type":"f64"},{"name":"x1","type":"f64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_utf8":[],"l3_i64":[],"l4_f64":[],"l5_i64":[],"l6_f64":[],"x1":[]}}},{"id":"random/0368","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_f64","type":"f64"},{"name":"l4_f64","type":"f64"},{"name":"l5_utf8","type":"utf8"}],"columns":{"l0_utf8":["zz","😀","a","","😀"],"l1_i32":[null,2,2147483647,2147483647,-4],"l2_i32":[2,4,-2,-1,-1],"l3_f64":[21.339,-34.706,14.51,-48.685,3.0],"l4_f64":[3.0,1.0,-31.516,null,null],"l5_utf8":[null,"é",null,"a","ab"]}},"right":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"r0_utf8","type":"utf8"}],"columns":{"r0_utf8":["😀","😀",null,"b","zz","B","a","b","😀","é","b","aa","a","ab","é","c","B","é","ab",null,"a",null,"é","😀","a","é","a","ab",null,null],"l0_utf8":["a","😀","😀","a","😀","a","😀","","a","a","","zz","😀","a","zz","zz","😀","😀","😀","😀","😀","😀","a","😀","zz","😀","a","zz","zz","zz"]}}},"ops":[{"op":"join","right":"right","on":["l0_utf8"],"how":"inner"},{"op":"sort","keys":[{"expr":{"col":"r0_utf8"},"descending":true},{"expr":{"col":"l1_i32"},"descending":false}],"nullsLast":false},{"op":"sort","keys":[{"expr":{"col":"l1_i32"},"descending":false},{"expr":{"col":"l5_utf8"},"descending":true}],"nullsLast":true}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_f64","type":"f64"},{"name":"l4_f64","type":"f64"},{"name":"l5_utf8","type":"utf8"},{"name":"r0_utf8","type":"utf8"}],"columns":{"l0_utf8":["😀","😀","😀","😀","😀","😀","😀","😀","😀","😀","😀","😀","😀","😀","😀","😀","😀","😀","😀","😀","😀","😀","😀","😀","😀","😀","","","a","a","a","a","a","a","a","a","zz","zz","zz","zz","zz","zz","zz"],"l1_i32":[-4,-4,-4,-4,-4,-4,-4,-4,-4,-4,-4,-4,-4,2,2,2,2,2,2,2,2,2,2,2,2,2,2147483647,2147483647,2147483647,2147483647,2147483647,2147483647,2147483647,2147483647,2147483647,2147483647,null,null,null,null,null,null,null],"l2_i32":[-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,4,4,4,4,4,4,4,4,4,4,4,4,4,-1,-1,-2,-2,-2,-2,-2,-2,-2,-2,2,2,2,2,2,2,2],"l3_f64":[3.0,3.0,3.0,3.0,3.0,3.0,3.0,3.0,3.0,3.0,3.0,3.0,3.0,-34.706,-34.706,-34.706,-34.706,-34.706,-34.706,-34.706,-34.706,-34.706,-34.706,-34.706,-34.706,-34.706,-48.685,-48.685,14.51,14.51,14.51,14.51,14.51,14.51,14.51,14.51,21.339,21.339,21.339,21.339,21.339,21.339,21.339],"l4_f64":[null,null,null,null,null,null,null,null,null,null,null,null,null,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,null,null,-31.516,-31.516,-31.516,-31.516,-31.516,-31.516,-31.516,-31.516,3.0,3.0,3.0,3.0,3.0,3.0,3.0],"l5_utf8":["ab","ab","ab","ab","ab","ab","ab","ab","ab","ab","ab","ab","ab","é","é","é","é","é","é","é","é","é","é","é","é","é","a","a",null,null,null,null,null,null,null,null,null,null,null,null,null,null,null],"r0_utf8":[null,null,null,"😀","😀","é","é","zz","ab","a","a","a","B",null,null,null,"😀","😀","é","é","zz","ab","a","a","a","B","b","b","😀","😀","é","é","b","ab","a","B",null,null,"é","c","ab","aa","a"]}}},{"id":"random/0369","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_i32","type":"i32"}],"columns":{"l0_utf8":["c","aa","é","b","c"],"l1_i32":[-3,-2,-4,3,5],"l2_f64":[80.017,-1.0,-26.669,0.0,70.019],"l3_utf8":["a","😀","a","aa","c"],"l4_utf8":["ab","a","B","c","c"],"l5_i32":[-3,4,-3,0,4]}}},"ops":[{"op":"unique","columns":["l1_i32"]}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_i32","type":"i32"}],"columns":{"l0_utf8":["c","aa","é","b","c"],"l1_i32":[-3,-2,-4,3,5],"l2_f64":[80.017,-1.0,-26.669,0.0,70.019],"l3_utf8":["a","😀","a","aa","c"],"l4_utf8":["ab","a","B","c","c"],"l5_i32":[-3,4,-3,0,4]}}},{"id":"random/0370","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_f64","type":"f64"},{"name":"l4_f64","type":"f64"}],"columns":{"l0_utf8":["😀","zz","b","é","aa","","a","c","zz","aa","é","ab","😀","😀","c","b","é","B","zz","B"],"l1_i32":[-3,-2,-2,-1,-3,5,5,158,3,2,5,-3,-3,-1,4,5,-3,2,2,-4],"l2_utf8":["c","zz","aa","zz","b","b","B","b","","c","aa","é","zz","zz","B","zz","b","😀","zz","a"],"l3_f64":[62.662,-32.088,2.0,-3.0,4.0,"-0",4.0,86.394,-3.0,22.222,-34.013,63.166,-68.276,0.0,-3.0,3.0,-1.0,0.0,-91.737,-69.461],"l4_f64":[0.0,-50.463,-1.0,-57.016,3.43,31.591,0.0,22.693,39.996,-29.512,4.0,6.292,4.0,15.69,-3.0,43.847,28.804,94.01,-51.104,16.502]}}},"ops":[{"op":"groupBy","keys":["l2_utf8"],"aggs":{"n0":{"agg":"len"},"a0_0":{"agg":"nUnique","arg":{"col":"l1_i32"}},"a0_1":{"agg":"first","arg":{"col":"l0_utf8"}},"a0_2":{"agg":"nUnique","arg":{"col":"l0_utf8"}}}}],"expected":{"fields":[{"name":"l2_utf8","type":"utf8"},{"name":"n0","type":"u32"},{"name":"a0_0","type":"u32"},{"name":"a0_1","type":"utf8"},{"name":"a0_2","type":"u32"}],"columns":{"l2_utf8":["c","zz","aa","b","B","","é","😀","a"],"n0":[2,6,2,4,2,1,1,1,1],"a0_0":[2,5,2,3,2,1,1,1,1],"a0_1":["😀","zz","b","aa","a","zz","ab","B","B"],"a0_2":[2,4,2,4,2,1,1,1,1]}}},{"id":"random/0371","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_i32","type":"i32"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_f64":[],"l3_i32":[]}}},"ops":[{"op":"withColumns","exprs":{"x0":{"fn":"add","args":[{"col":"l1_i32"},{"fn":"div","args":[{"col":"l2_f64"},{"col":"l1_i32"}]}]}}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_i32","type":"i32"},{"name":"x0","type":"f64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_f64":[],"l3_i32":[],"x0":[]}}},{"id":"random/0372","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_utf8","type":"utf8"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_f64":[],"l3_utf8":[]}}},"ops":[{"op":"unique","columns":["l3_utf8"]},{"op":"withColumns","exprs":{"x1":{"fn":"gte","args":[{"fn":"abs","args":[{"col":"l2_f64"}]},{"col":"l2_f64"}]}}},{"op":"unique"},{"op":"withColumns","exprs":{"x3":{"col":"l1_i32"}}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_utf8","type":"utf8"},{"name":"x1","type":"bool"},{"name":"x3","type":"i32"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_f64":[],"l3_utf8":[],"x1":[],"x3":[]}}},{"id":"random/0373","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_bool","type":"bool"},{"name":"l4_i64","type":"i64"},{"name":"l5_i32","type":"i32"}],"columns":{"l0_utf8":["","a","","ab",""],"l1_i32":[-3,5,-2,-2147483648,5],"l2_f64":[-3.224,97.391,81.367,69.525,-17.591],"l3_bool":[true,true,false,false,false],"l4_i64":["-3","-4611686018427387904","-3","-4611686018427387904","-5"],"l5_i32":[3,1,2,-3,-5]}}},"ops":[{"op":"unique"},{"op":"groupBy","keys":["l0_utf8"],"aggs":{"n1":{"agg":"len"},"a1_0":{"agg":"nUnique","arg":{"col":"l5_i32"}},"a1_1":{"agg":"max","arg":{"col":"l4_i64"}}}},{"op":"withColumns","exprs":{"x2":{"fn":"neg","args":[{"lit":0.5}]}}},{"op":"unique"}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"n1","type":"u32"},{"name":"a1_0","type":"u32"},{"name":"a1_1","type":"i64"},{"name":"x2","type":"f64"}],"columns":{"l0_utf8":["","a","ab"],"n1":[3,1,1],"a1_0":[3,1,1],"a1_1":["-3","-4611686018427387904","-4611686018427387904"],"x2":[-0.5,-0.5,-0.5]}}},{"id":"random/0374","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_bool","type":"bool"}],"columns":{"l0_utf8":["","b","c","é","c","b",null,"b","zz","c","é","b","","😀","😀","aa","é",null,"ab","aa",null,"c",null,"c","","zz","😀","","aa","zz","a","a","aa","B",null,"a","zz","B","B","😀","","😀","b","b","ab","é","é","","B","","a","a","zz","aa","B",null,"ab","","ab",null],"l1_i32":[0,3,-2,5,-1,5,null,2147483647,5,-4,3,1,0,0,null,4,-5,-4,2,4,-4,1,5,-5,2147483647,-4,null,4,3,-1,2,3,0,4,-3,null,3,null,5,-4,5,-3,-4,null,-1,-4,580,3,-5,-5,-5,0,3,-2,-124,5,-3,2147483647,2,-4],"l2_bool":[false,true,false,null,false,false,true,false,false,true,false,false,true,true,false,true,null,false,true,false,true,false,false,true,true,true,false,true,true,true,null,true,true,true,false,null,true,false,false,true,true,false,true,false,false,false,false,false,true,false,false,true,true,true,true,false,false,true,false,false],"l3_bool":[false,true,false,null,true,true,false,false,true,true,false,false,true,null,true,true,false,true,false,false,true,false,false,true,true,null,false,null,false,true,null,null,false,false,false,true,false,true,false,false,true,null,false,false,false,true,true,true,false,false,false,true,false,true,true,true,false,false,false,false]}}},"ops":[{"op":"head","n":8},{"op":"filter","predicate":{"fn":"gte","args":[{"col":"l1_i32"},{"lit":0.5}]}},{"op":"unique","columns":["l3_bool"]}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_bool","type":"bool"}],"columns":{"l0_utf8":["b","é","b"],"l1_i32":[3,5,2147483647],"l2_bool":[true,null,false],"l3_bool":[true,null,false]}}},{"id":"random/0375","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_f64","type":"f64"}],"columns":{"l0_utf8":["ab","c","😀",null,null,null,"b","ab",null,"aa","😀",null,null,null,null,"zz","aa",null,"c","é","B","aa","é",null,"B","é","B","😀","aa","a","ab","zz","b",null,"ab","😀",null,"b",null,null,"aa","aa",null,"ab",null,"aa","B","b","aa","b",null,null,null,"B",null,"😀","zz","c",null,"c"],"l1_i32":[0,4,-2,-5,null,4,509,-2,-5,-1,null,4,3,null,4,null,2,3,3,-4,-3,-1,-4,-5,null,4,-4,null,-3,1,-5,-5,null,null,3,null,1,null,4,1,-4,1,null,5,-3,5,-2,null,0,null,-3,null,0,-3,5,4,-5,-2,-5,null],"l2_utf8":["zz","a","zz","c",null,"ab","b",null,"b",null,"😀",null,"aa","a","","b",null,"zz",null,"é","😀","","zz","B",null,"B","ab","zz","😀","c","aa",null,"😀","c",null,"😀",null,"😀",null,"aa","é","é","aa","b","a","","b","c","aa",null,null,null,"aa",null,"a","a","😀","B","zz","ab"],"l3_f64":[-82.624,null,69.766,1.0,null,"Infinity",21.82,null,-23.211,43.465,null,"Infinity",-35.363,-4.0,3.0,4.0,null,-3.0,null,null,4.0,1.0,1.0,-19.733,-96.705,null,21.653,null,4.0,null,0.0,null,null,null,-0.128,-24.846,70.008,null,97.433,-4.0,-2.0,null,91.059,-59.189,null,4.0,0.0,-79.568,null,31.318,-76.594,null,2.0,null,14.886,33.306,-78.883,null,null,87.406]}},"right":{"fields":[{"name":"l2_utf8","type":"utf8"},{"name":"r0_f64","type":"f64"}],"columns":{"r0_f64":[null,-3.0,90.588,2.0,-2.0,"Infinity",0.0,90.013,-70.274,4.0,null,-1.0,-1.0,4.0,-2.0,-86.404,null,-25.606,null,null,-4.0,null,-2.0,0.0,null,80.253,-47.288,-3.0,null,3.0],"l2_utf8":["zz","aa","",null,"zz","B","ab","😀",null,"zz",null,null,"B",null,"ab",null,"B","c",null,"aa",null,"ab",null,"c","aa","aa","é","c","B",null]}}},"ops":[{"op":"join","right":"right","on":["l2_utf8"],"how":"left"},{"op":"unique"}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_f64","type":"f64"},{"name":"r0_f64","type":"f64"}],"columns":{"l0_utf8":["ab","ab","ab","c","😀","😀","😀",null,null,null,null,null,null,null,"b","ab",null,"aa","😀",null,null,null,null,null,null,"zz","aa",null,null,null,"c","é","B","aa","é","é","é",null,null,null,"B","é","é","é","B","B","B","😀","😀","😀","aa","a","a","a","ab","ab","ab","zz","b",null,null,null,"ab","😀",null,null,null,null,null,"aa","aa",null,null,null,"ab",null,"aa","B","b","b","b","aa","aa","aa","b",null,null,null,null,"B",null,"😀","zz","c","c","c",null,null,null,"c","c","c"],"l1_i32":[0,0,0,4,-2,-2,-2,-5,-5,-5,null,4,4,4,509,-2,-5,-1,null,4,3,3,3,null,4,null,2,3,3,3,3,-4,-3,-1,-4,-4,-4,-5,-5,-5,null,4,4,4,-4,-4,-4,null,null,null,-3,1,1,1,-5,-5,-5,-5,null,null,null,null,3,null,1,4,1,1,1,-4,1,null,null,null,5,-3,5,-2,null,null,null,0,0,0,null,-3,0,0,0,-3,5,4,-5,-2,-2,-2,-5,-5,-5,null,null,null],"l2_utf8":["zz","zz","zz","a","zz","zz","zz","c","c","c",null,"ab","ab","ab","b",null,"b",null,"😀",null,"aa","aa","aa","a","","b",null,"zz","zz","zz",null,"é","😀","","zz","zz","zz","B","B","B",null,"B","B","B","ab","ab","ab","zz","zz","zz","😀","c","c","c","aa","aa","aa",null,"😀","c","c","c",null,"😀",null,null,"aa","aa","aa","é","é","aa","aa","aa","b","a","","b","c","c","c","aa","aa","aa",null,null,"aa","aa","aa",null,"a","a","😀","B","B","B","zz","zz","zz","ab","ab","ab"],"l3_f64":[-82.624,-82.624,-82.624,null,69.766,69.766,69.766,1.0,1.0,1.0,null,"Infinity","Infinity","Infinity",21.82,null,-23.211,43.465,null,"Infinity",-35.363,-35.363,-35.363,-4.0,3.0,4.0,null,-3.0,-3.0,-3.0,null,null,4.0,1.0,1.0,1.0,1.0,-19.733,-19.733,-19.733,-96.705,null,null,null,21.653,21.653,21.653,null,null,null,4.0,null,null,null,0.0,0.0,0.0,null,null,null,null,null,-0.128,-24.846,70.008,97.433,-4.0,-4.0,-4.0,-2.0,null,91.059,91.059,91.059,-59.189,null,4.0,0.0,-79.568,-79.568,-79.568,null,null,null,31.318,-76.594,2.0,2.0,2.0,null,14.886,33.306,-78.883,null,null,null,null,null,null,87.406,87.406,87.406],"r0_f64":[null,-2.0,4.0,null,null,-2.0,4.0,-25.606,0.0,-3.0,null,0.0,-2.0,null,null,null,null,null,90.013,null,-3.0,null,80.253,null,90.588,null,null,null,-2.0,4.0,null,-47.288,90.013,90.588,null,-2.0,4.0,"Infinity",-1.0,null,null,"Infinity",-1.0,null,0.0,-2.0,null,null,-2.0,4.0,90.013,-25.606,0.0,-3.0,-3.0,null,80.253,null,90.013,-25.606,0.0,-3.0,null,90.013,null,null,-3.0,null,80.253,-47.288,-47.288,-3.0,null,80.253,null,null,90.588,null,-25.606,0.0,-3.0,-3.0,null,80.253,null,null,-3.0,null,80.253,null,null,null,90.013,"Infinity",-1.0,null,null,-2.0,4.0,0.0,-2.0,null]}}},{"id":"random/0376","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_i64","type":"i64"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_i32","type":"i32"}],"columns":{"l0_utf8":["B","c",null,"a","😀","B",null,null,"","c","é","B","b","aa","a","é","😀","😀","aa",null,"é","zz",null,"aa","","c","é","b","a","zz","a","b","B","a","c","","b",null,"a","b","a","","c","","B","c","é","","a","c","aa","aa","aa","B","é","aa","ab","😀","c","a"],"l1_i32":[-5,-3,null,2,3,-3,-1,-1,1,163,-3,4,-5,-2147483648,-3,0,null,4,5,-2,1,1,-5,-5,-1,1,4,-1,null,5,-3,5,2,null,-5,-3,1,3,-2147483648,4,-3,-1,5,-2,null,-3,135,-2,-5,137,null,null,4,-3,2,-2,-5,-3,4,-2],"l2_f64":[null,39.892,97.164,-1.0,-70.666,-4.0,4.0,"Infinity",-2.0,77.843,-16.664,29.839,"NaN",85.912,-1.0,-44.827,3.0,null,-30.099,-4.0,81.847,-62.871,4.0,"-Infinity",-38.092,-83.794,-4.0,2.0,-1.0,1.0,-4.0,"-0",3.0,"NaN",null,2.253,43.466,-98.923,28.898,4.0,null,37.342,-2.0,-76.652,"-0",47.819,3.0,91.661,-3.0,93.174,1.0,-13.203,0.0,17.27,10.722,1.0,null,2.0,67.061,4.0],"l3_i64":["0","2","9007199254740993","9007199254740992","-4611686018427387904","1","2","-2","-4611686018427387904","-2","-4611686018427387904","9007199254740995","-2","-4611686018427387904","9007199254740995","-4611686018427387904",null,"9007199254740993","9007199254740993","-4611686018427387904","-5","-4611686018427387904","-4611686018427387904","9007199254740995","9007199254740995","4",null,"-4","-2","9007199254740995","9007199254740995","5","-4611686018427387904","-4611686018427387904",null,"5","-4611686018427387904",null,"0","-4611686018427387904","-2","9007199254740992",null,"-4611686018427387904","-4611686018427387904","9007199254740994","9007199254740995","-4611686018427387904","-2","9007199254740994",null,"-4611686018427387904","-2",null,"-4611686018427387904","-4611686018427387904","-1",null,"3","9007199254740993"],"l4_utf8":["zz","é","c","😀",null,"aa",null,"zz",null,null,"c","😀","zz","b","c","zz","a","c","😀","B","é","c","","B","","ab","b","a","zz","😀","B","a","c","c","ab","c","😀","é","","a","b","aa",null,"b","c","a","zz",null,"aa","aa","aa","c","😀","c","é","c","B","aa",null,"c"],"l5_i32":[null,3,3,4,1,1,0,1,0,0,769,-1,3,2,5,-5,-2,4,3,4,-3,5,2,-5,null,-4,4,1,-2,-2,4,-3,5,4,2,-5,5,1,3,5,3,-4,-3,-1,null,5,-3,2147483647,1,-3,3,null,2147483647,-5,2,-1,0,3,3,1]}}},"ops":[{"op":"head","n":9},{"op":"withColumns","exprs":{"x1":{"fn":"lt","args":[{"col":"l5_i32"},{"col":"l2_f64"}]}}},{"op":"groupBy","keys":["l0_utf8"],"aggs":{"n2":{"agg":"len"},"a2_0":{"agg":"max","arg":{"col":"l5_i32"}},"a2_1":{"agg":"median","arg":{"col":"l2_f64"}},"a2_2":{"agg":"last","arg":{"col":"l5_i32"}}}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"n2","type":"u32"},{"name":"a2_0","type":"i32"},{"name":"a2_1","type":"f64"},{"name":"a2_2","type":"i32"}],"columns":{"l0_utf8":["B","c",null,"a","😀",""],"n2":[2,1,3,1,1,1],"a2_0":[1,3,3,4,1,0],"a2_1":[-4.0,39.892,97.164,-1.0,-70.666,-2.0],"a2_2":[1,3,1,4,1,0]}}},{"id":"random/0377","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_bool","type":"bool"},{"name":"l4_f64","type":"f64"}],"columns":{"l0_utf8":["aa","😀","é","aa","c"],"l1_i32":[-4,-2,0,5,-2],"l2_utf8":["","B","é","😀","aa"],"l3_bool":[false,true,false,true,true],"l4_f64":[74.859,6.729,3.0,-2.0,2.0]}},"right":{"fields":[{"name":"l2_utf8","type":"utf8"},{"name":"r0_bool","type":"bool"}],"columns":{"r0_bool":[false,true,true],"l2_utf8":["B","","é"]}}},"ops":[{"op":"sort","keys":[{"expr":{"col":"l2_utf8"},"descending":false},{"expr":{"col":"l1_i32"},"descending":false}],"nullsLast":false},{"op":"join","right":"right","on":["l2_utf8"],"how":"anti"},{"op":"withColumns","exprs":{"x2":{"fn":"mul","args":[{"fn":"sub","args":[{"lit":2},{"col":"l1_i32"}]},{"col":"l1_i32"}]}}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_bool","type":"bool"},{"name":"l4_f64","type":"f64"},{"name":"x2","type":"i32"}],"columns":{"l0_utf8":["c","aa"],"l1_i32":[-2,5],"l2_utf8":["aa","😀"],"l3_bool":[true,true],"l4_f64":[2.0,-2.0],"x2":[-8,-15]}}},{"id":"random/0378","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i64","type":"i64"},{"name":"l3_bool","type":"bool"}],"columns":{"l0_utf8":["😀","B","","a","zz"],"l1_i32":[-2,-1,3,5,-5],"l2_i64":["9007199254740995","-1","-4611686018427387904","-4611686018427387904","-4611686018427387904"],"l3_bool":[true,false,false,true,false]}}},"ops":[{"op":"withColumns","exprs":{"x0":{"fn":"mod","args":[{"fn":"neg","args":[{"col":"l1_i32"}]},{"lit":2}]}}},{"op":"filter","predicate":{"fn":"gt","args":[{"fn":"mul","args":[{"col":"l1_i32"},{"lit":2}]},{"fn":"floorDiv","args":[{"col":"x0"},{"lit":0.5}]}]}},{"op":"withColumns","exprs":{"x2":{"fn":"mul","args":[{"col":"l1_i32"},{"fn":"fillNull","args":[{"col":"x0"},{"lit":0}]}]}}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i64","type":"i64"},{"name":"l3_bool","type":"bool"},{"name":"x0","type":"i32"},{"name":"x2","type":"i32"}],"columns":{"l0_utf8":["","a"],"l1_i32":[3,5],"l2_i64":["-4611686018427387904","-4611686018427387904"],"l3_bool":[false,true],"x0":[1,1],"x2":[3,5]}}},{"id":"random/0379","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i64","type":"i64"},{"name":"l3_f64","type":"f64"},{"name":"l4_utf8","type":"utf8"}],"columns":{"l0_utf8":["aa","zz","aa",null,"b","b","b",null,"a","zz","a","é","ab",null,"zz","é","B","zz","aa","b","é","aa","c","","B","B","😀","zz","a","b","zz","aa","é","b","ab","","a","😀","","zz","😀","zz","ab","a","","zz","😀","B","ab","","zz","é","😀","a","😀","ab",null,"b","zz","zz"],"l1_i32":[-3,-4,-5,2,-4,-1,5,-2,-4,1,1,178,-5,0,5,-3,null,0,4,-1,0,5,-5,2147483647,null,2147483647,-5,null,1,-5,4,4,5,-1,-1,-2147483648,-1,2,5,null,2,-4,5,5,-4,-4,1,3,-2,1,-1,834,-3,4,-3,4,null,3,null,null],"l2_i64":["9007199254740992",null,null,"2","3","-1",null,"-4611686018427387904",null,"-4611686018427387904","4","-3","-4611686018427387904","4","-4611686018427387904","9007199254740994",null,"0",null,"-4611686018427387904",null,null,"-4611686018427387904","-3",null,"-4611686018427387904",null,"-4611686018427387904","3","-4611686018427387904","4","9007199254740993","9007199254740995","3",null,"9007199254740992","1","5","-4611686018427387904","0","9007199254740993","-5","9007199254740995","-4611686018427387904","-4611686018427387904","9007199254740994","9007199254740992","-3","9007199254740993","9007199254740992","-4611686018427387904","-5","-2","-4611686018427387904","9007199254740994","9007199254740994","-4611686018427387904","5","9007199254740994","9007199254740992"],"l3_f64":[-15.151,3.0,-4.0,-3.0,"NaN",65.997,-4.0,38.324,0.0,-86.048,36.861,null,null,47.053,-4.0,-69.156,null,-1.0,-2.0,-48.991,0.0,96.449,-27.568,20.163,-4.0,34.694,-2.0,-1.0,0.0,-96.488,2.0,1.0,-86.385,"NaN",-97.619,-68.609,82.633,4.0,null,-2.0,41.63,1.0,-21.976,0.0,-3.0,38.903,"-0",-1.0,-1.0,44.678,1.0,-43.061,58.309,-4.0,-99.753,null,-3.0,-4.0,-47.155,"NaN"],"l4_utf8":["aa","é","ab",null,"zz","b","zz","","😀",null,"a",null,"B","a","","é","","aa","😀","aa","😀","b","B","zz","é","ab","a","ab","","b","c","ab","","B","b","ab","b","ab","aa","B","😀","b","","aa","zz","ab","😀","é","é","😀","","a","B","😀",null,"c","B","B","c","ab"]}}},"ops":[{"op":"filter","predicate":{"fn":"isNull","args":[{"col":"l4_utf8"}]}},{"op":"filter","predicate":{"fn":"gte","args":[{"fn":"sub","args":[{"col":"l3_f64"},{"col":"l3_f64"}]},{"col":"l3_f64"}]}},{"op":"unique"},{"op":"groupBy","keys":["l2_i64"],"aggs":{"n3":{"agg":"len"},"a3_0":{"agg":"min","arg":{"col":"l0_utf8"}},"a3_1":{"agg":"max","arg":{"col":"l1_i32"}},"a3_2":{"agg":"max","arg":{"col":"l1_i32"}}}}],"expected":{"fields":[{"name":"l2_i64","type":"i64"},{"name":"n3","type":"u32"},{"name":"a3_0","type":"utf8"},{"name":"a3_1","type":"i32"},{"name":"a3_2","type":"i32"}],"columns":{"l2_i64":["2","-4611686018427387904","9007199254740994"],"n3":[1,1,1],"a3_0":[null,"zz","😀"],"a3_1":[2,1,-3],"a3_2":[2,1,-3]}}},{"id":"random/0380","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_f64","type":"f64"},{"name":"l4_f64","type":"f64"}],"columns":{"l0_utf8":["😀"],"l1_i32":[-5],"l2_bool":[true],"l3_f64":[-3.0],"l4_f64":[83.042]}}},"ops":[{"op":"head","n":2},{"op":"filter","predicate":{"fn":"isNotNull","args":[{"col":"l4_f64"}]}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_f64","type":"f64"},{"name":"l4_f64","type":"f64"}],"columns":{"l0_utf8":["😀"],"l1_i32":[-5],"l2_bool":[true],"l3_f64":[-3.0],"l4_f64":[83.042]}}},{"id":"random/0381","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_f64","type":"f64"}],"columns":{"l0_utf8":["a","😀","é","","ab"],"l1_i32":[1,5,3,1,2147483647],"l2_bool":[true,false,true,true,false],"l3_f64":[-2.0,-53.744,-86.46,-62.362,"-0"]}}},"ops":[{"op":"sort","keys":[{"expr":{"col":"l2_bool"},"descending":false},{"expr":{"col":"l3_f64"},"descending":true}],"nullsLast":true},{"op":"head","n":5},{"op":"withColumns","exprs":{"x2":{"fn":"lt","args":[{"fn":"when","args":[{"fn":"lt","args":[{"col":"l1_i32"},{"col":"l1_i32"}]},{"lit":0.5},{"col":"l1_i32"}]},{"fn":"floorDiv","args":[{"col":"l1_i32"},{"col":"l3_f64"}]}]}}},{"op":"sort","keys":[{"expr":{"col":"l0_utf8"},"descending":true}],"nullsLast":false}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_f64","type":"f64"},{"name":"x2","type":"bool"}],"columns":{"l0_utf8":["😀","é","ab","a",""],"l1_i32":[5,3,2147483647,1,1],"l2_bool":[false,true,false,true,true],"l3_f64":[-53.744,-86.46,"-0",-2.0,-62.362],"x2":[false,false,false,false,false]}}},{"id":"random/0382","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_i64","type":"i64"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_bool","type":"bool"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_f64":[],"l3_i64":[],"l4_utf8":[],"l5_bool":[]}}},"ops":[{"op":"sort","keys":[{"expr":{"col":"l4_utf8"},"descending":true},{"expr":{"col":"l0_utf8"},"descending":false}],"nullsLast":false},{"op":"withColumns","exprs":{"x1":{"col":"l5_bool"}}},{"op":"unique","columns":["l5_bool"]},{"op":"sort","keys":[{"expr":{"col":"l1_i32"},"descending":false},{"expr":{"col":"l0_utf8"},"descending":true}],"nullsLast":false}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_i64","type":"i64"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_bool","type":"bool"},{"name":"x1","type":"bool"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_f64":[],"l3_i64":[],"l4_utf8":[],"l5_bool":[],"x1":[]}}},{"id":"random/0383","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_i64","type":"i64"},{"name":"l4_i64","type":"i64"}],"columns":{"l0_utf8":["zz","😀","😀","c","a"],"l1_i32":[4,-5,-1,-4,3],"l2_f64":[-43.781,3.0,-20.504,-3.0,98.645],"l3_i64":["9007199254740992","9007199254740995","9007199254740995","-4","-4611686018427387904"],"l4_i64":["0","-4611686018427387904","-4611686018427387904","-3","-4611686018427387904"]}},"right":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"r0_utf8","type":"utf8"}],"columns":{"r0_utf8":["c","a","B","",null,"b","","c",null,"B","ab","B","c","B","","c","😀","zz","😀","b","B","zz","B","b","b","c","😀","a","b","😀"],"l0_utf8":["😀","zz","😀","😀","zz","zz","c","c","c","😀","zz","zz","c","😀","😀","😀","c","a","😀","😀","a","😀","c","c","a","c","c","a","zz","c"]}}},"ops":[{"op":"join","right":"right","on":["l0_utf8"],"how":"anti"},{"op":"head","n":2},{"op":"sort","keys":[{"expr":{"col":"l3_i64"},"descending":false}],"nullsLast":true}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_i64","type":"i64"},{"name":"l4_i64","type":"i64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_f64":[],"l3_i64":[],"l4_i64":[]}}},{"id":"random/0384","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_bool","type":"bool"},{"name":"l4_bool","type":"bool"},{"name":"l5_bool","type":"bool"},{"name":"l6_f64","type":"f64"}],"columns":{"l0_utf8":["aa"],"l1_i32":[-5],"l2_utf8":["B"],"l3_bool":[false],"l4_bool":[true],"l5_bool":[false],"l6_f64":[-9.728]}}},"ops":[{"op":"groupBy","keys":["l2_utf8"],"aggs":{"n0":{"agg":"len"},"a0_0":{"agg":"nUnique","arg":{"col":"l3_bool"}},"a0_1":{"agg":"nUnique","arg":{"col":"l6_f64"}}}},{"op":"unique"}],"expected":{"fields":[{"name":"l2_utf8","type":"utf8"},{"name":"n0","type":"u32"},{"name":"a0_0","type":"u32"},{"name":"a0_1","type":"u32"}],"columns":{"l2_utf8":["B"],"n0":[1],"a0_0":[1],"a0_1":[1]}}},{"id":"random/0385","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_f64","type":"f64"},{"name":"l4_bool","type":"bool"},{"name":"l5_i64","type":"i64"}],"columns":{"l0_utf8":["c","a","c","a","","c","zz","","ab","b","b","zz","zz","c","zz","aa","😀","ab","aa","zz"],"l1_i32":[-3,1,-4,5,4,1,-2,-3,-1,-4,3,1,1,-2,5,-4,3,-4,5,3],"l2_bool":[true,true,false,true,true,true,false,true,false,false,false,false,true,true,true,true,true,true,false,false],"l3_f64":[98.008,-56.997,-67.646,44.162,8.09,4.0,2.0,"NaN",1.0,-44.407,-4.0,"NaN",86.318,-78.083,3.0,-43.878,0.0,1.0,-2.0,1.0],"l4_bool":[false,false,true,true,false,true,true,true,true,false,false,true,true,false,true,true,false,false,false,false],"l5_i64":["-4","9007199254740994","9007199254740994","3","4","5","-4611686018427387904","4","9007199254740994","9007199254740995","2","-4611686018427387904","9007199254740995","9007199254740995","-4611686018427387904","9007199254740994","-4611686018427387904","-3","2","5"]}}},"ops":[{"op":"withColumns","exprs":{"x0":{"fn":"lte","args":[{"fn":"add","args":[{"col":"l3_f64"},{"col":"l3_f64"}]},{"col":"l3_f64"}]}}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_f64","type":"f64"},{"name":"l4_bool","type":"bool"},{"name":"l5_i64","type":"i64"},{"name":"x0","type":"bool"}],"columns":{"l0_utf8":["c","a","c","a","","c","zz","","ab","b","b","zz","zz","c","zz","aa","😀","ab","aa","zz"],"l1_i32":[-3,1,-4,5,4,1,-2,-3,-1,-4,3,1,1,-2,5,-4,3,-4,5,3],"l2_bool":[true,true,false,true,true,true,false,true,false,false,false,false,true,true,true,true,true,true,false,false],"l3_f64":[98.008,-56.997,-67.646,44.162,8.09,4.0,2.0,"NaN",1.0,-44.407,-4.0,"NaN",86.318,-78.083,3.0,-43.878,0.0,1.0,-2.0,1.0],"l4_bool":[false,false,true,true,false,true,true,true,true,false,false,true,true,false,true,true,false,false,false,false],"l5_i64":["-4","9007199254740994","9007199254740994","3","4","5","-4611686018427387904","4","9007199254740994","9007199254740995","2","-4611686018427387904","9007199254740995","9007199254740995","-4611686018427387904","9007199254740994","-4611686018427387904","-3","2","5"],"x0":[false,true,true,false,false,false,false,true,false,true,true,true,false,true,false,true,true,false,true,false]}}},{"id":"random/0386","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_bool","type":"bool"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_bool","type":"bool"},{"name":"l6_utf8","type":"utf8"}],"columns":{"l0_utf8":["é"],"l1_i32":[-5],"l2_bool":[false],"l3_bool":[false],"l4_utf8":["ab"],"l5_bool":[false],"l6_utf8":[""]}},"right":{"fields":[{"name":"l4_utf8","type":"utf8"},{"name":"r0_utf8","type":"utf8"}],"columns":{"r0_utf8":["ab","aa","","B","zz","b","zz",null,"c","zz","B","é","b","zz","c","","B","","ab","ab","é","zz","aa","b","é","b","ab","","😀","a"],"l4_utf8":["ab","ab","ab","ab","ab","ab","ab","ab","ab","ab","ab","ab","ab","ab","ab","ab","ab","ab","ab","ab","ab","ab","ab","ab","ab","ab","ab","ab","ab","ab"]}}},"ops":[{"op":"join","right":"right","on":["l4_utf8"],"how":"inner"},{"op":"unique"},{"op":"unique","columns":["l5_bool"]}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_bool","type":"bool"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_bool","type":"bool"},{"name":"l6_utf8","type":"utf8"},{"name":"r0_utf8","type":"utf8"}],"columns":{"l0_utf8":["é"],"l1_i32":[-5],"l2_bool":[false],"l3_bool":[false],"l4_utf8":["ab"],"l5_bool":[false],"l6_utf8":[""],"r0_utf8":["ab"]}}},{"id":"random/0387","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_i64","type":"i64"},{"name":"l6_f64","type":"f64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_f64":[],"l3_utf8":[],"l4_utf8":[],"l5_i64":[],"l6_f64":[]}}},"ops":[{"op":"sort","keys":[{"expr":{"col":"l1_i32"},"descending":false},{"expr":{"col":"l6_f64"},"descending":true}],"nullsLast":false},{"op":"head","n":5},{"op":"filter","predicate":{"fn":"gt","args":[{"col":"l2_f64"},{"fn":"add","args":[{"col":"l2_f64"},{"col":"l2_f64"}]}]}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_i64","type":"i64"},{"name":"l6_f64","type":"f64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_f64":[],"l3_utf8":[],"l4_utf8":[],"l5_i64":[],"l6_f64":[]}}},{"id":"random/0388","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_f64","type":"f64"},{"name":"l4_i32","type":"i32"}],"columns":{"l0_utf8":["b"],"l1_i32":[-1],"l2_bool":[null],"l3_f64":[-3.0],"l4_i32":[4]}}},"ops":[{"op":"sort","keys":[{"expr":{"col":"l0_utf8"},"descending":false}],"nullsLast":false},{"op":"unique","columns":["l3_f64"]},{"op":"sort","keys":[{"expr":{"col":"l3_f64"},"descending":true},{"expr":{"col":"l1_i32"},"descending":true}],"nullsLast":false},{"op":"withColumns","exprs":{"x3":{"fn":"neg","args":[{"fn":"floorDiv","args":[{"col":"l4_i32"},{"col":"l4_i32"}]}]}}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_f64","type":"f64"},{"name":"l4_i32","type":"i32"},{"name":"x3","type":"i32"}],"columns":{"l0_utf8":["b"],"l1_i32":[-1],"l2_bool":[null],"l3_f64":[-3.0],"l4_i32":[4],"x3":[-1]}}},{"id":"random/0389","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i64","type":"i64"},{"name":"l3_f64","type":"f64"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_utf8","type":"utf8"}],"columns":{"l0_utf8":[null,"zz","😀","é","c"],"l1_i32":[4,5,null,null,2],"l2_i64":["3","9007199254740992",null,"-4611686018427387904",null],"l3_f64":[1.0,null,-3.0,-79.931,-56.128],"l4_utf8":["","😀",null,null,"zz"],"l5_utf8":["aa","c","B",null,"a"]}},"right":{"fields":[{"name":"l2_i64","type":"i64"},{"name":"r0_f64","type":"f64"}],"columns":{"r0_f64":[],"l2_i64":[]}}},"ops":[{"op":"join","right":"right","on":["l2_i64"],"how":"left"},{"op":"unique"},{"op":"sort","keys":[{"expr":{"col":"r0_f64"},"descending":true}],"nullsLast":false}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i64","type":"i64"},{"name":"l3_f64","type":"f64"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_utf8","type":"utf8"},{"name":"r0_f64","type":"f64"}],"columns":{"l0_utf8":[null,"zz","😀","é","c"],"l1_i32":[4,5,null,null,2],"l2_i64":["3","9007199254740992",null,"-4611686018427387904",null],"l3_f64":[1.0,null,-3.0,-79.931,-56.128],"l4_utf8":["","😀",null,null,"zz"],"l5_utf8":["aa","c","B",null,"a"],"r0_f64":[null,null,null,null,null]}}},{"id":"random/0390","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_i64","type":"i64"},{"name":"l4_f64","type":"f64"},{"name":"l5_utf8","type":"utf8"},{"name":"l6_utf8","type":"utf8"}],"columns":{"l0_utf8":["B","c","é","ab","",null,"","aa","b","c","a",null,"zz","ab","😀","B","",null,"","😀","é","😀","zz","","😀","a","ab","","ab","B","B",null,"aa","b","B","é",null,"B",null,"é","ab","é","b",null,"😀","B","","😀","aa",null,"B",null,"b","",null,"😀","😀","😀",null,"B"],"l1_i32":[-4,-4,2,2,-4,null,-2,-2147483648,-4,1,-2,2,4,null,null,0,4,null,5,0,null,4,null,0,null,2,5,1,-2,-3,4,2,null,4,-1,null,null,null,1,null,5,2,-5,null,2,null,5,-2,2,1,null,null,-3,2,-2,-4,-2,-3,1,3],"l2_bool":[false,null,false,true,null,true,null,null,true,false,true,null,null,null,false,false,true,false,true,true,null,false,false,true,null,false,false,true,null,true,false,false,true,null,null,true,null,false,true,null,true,null,false,true,true,true,true,false,true,true,false,null,false,false,false,false,false,null,true,false],"l3_i64":["9007199254740994","-4611686018427387904","0","-3","-2",null,"-4611686018427387904","2","-3","9007199254740995","9007199254740992",null,"0",null,"-4611686018427387904","0","1",null,"-4611686018427387904",null,"9007199254740994","1","-4611686018427387904","-4611686018427387904","2","3",null,null,"9007199254740992","-4611686018427387904","-4611686018427387904","-4611686018427387904","9007199254740994","-3","3","9007199254740993",null,"9007199254740993","-5","-4611686018427387904",null,"-4611686018427387904","-5",null,null,null,"-5","3","-4611686018427387904",null,"-4611686018427387904","-4611686018427387904","-2","-4611686018427387904","-4611686018427387904","5","9007199254740992","2",null,"-4611686018427387904"],"l4_f64":[3.0,1.0,-3.0,-2.358,"NaN",45.769,-53.974,23.623,null,2.0,2.0,null,0.0,null,4.0,null,null,-2.0,-96.639,-2.0,-3.0,1.0,-1.0,-4.0,-3.0,-56.952,null,null,10.31,-3.0,94.615,-35.494,-26.213,-2.0,-57.762,null,null,33.267,null,3.0,-4.0,89.094,null,-1.0,-3.0,-63.585,3.0,1.0,23.643,-76.82,1.0,"Infinity",-1.0,null,null,-46.717,-2.0,65.951,null,null],"l5_utf8":["",null,"b","a","😀",null,"😀","ab","zz",null,null,"aa","c","aa",null,"a","ab","zz","😀",null,"b",null,"ab",null,"a",null,"a","aa",null,"c","b",null,null,"zz","ab",null,"b",null,"zz","",null,"é","zz","a",null,"zz","","zz","B","B",null,"zz","zz","ab","B","b","ab",null,"aa","ab"],"l6_utf8":["B","c","c","é",null,null,"é",null,"aa",null,"é",null,null,"c",null,"zz","b","b",null,null,"a","zz","é","B","","😀","é","B","aa","aa","B","c","aa",null,null,"aa","","b","zz","ab","é",null,"B",null,"aa","b","B","zz",null,"é",null,null,"zz",null,null,null,null,"","zz","é"]}}},"ops":[{"op":"head","n":9}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_i64","type":"i64"},{"name":"l4_f64","type":"f64"},{"name":"l5_utf8","type":"utf8"},{"name":"l6_utf8","type":"utf8"}],"columns":{"l0_utf8":["B","c","é","ab","",null,"","aa","b"],"l1_i32":[-4,-4,2,2,-4,null,-2,-2147483648,-4],"l2_bool":[false,null,false,true,null,true,null,null,true],"l3_i64":["9007199254740994","-4611686018427387904","0","-3","-2",null,"-4611686018427387904","2","-3"],"l4_f64":[3.0,1.0,-3.0,-2.358,"NaN",45.769,-53.974,23.623,null],"l5_utf8":["",null,"b","a","😀",null,"😀","ab","zz"],"l6_utf8":["B","c","c","é",null,null,"é",null,"aa"]}}},{"id":"random/0391","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_bool","type":"bool"},{"name":"l5_utf8","type":"utf8"},{"name":"l6_f64","type":"f64"}],"columns":{"l0_utf8":["a","zz","ab","é","😀",null,"zz",null,"b","é","c","zz","",null,"a","b","ab","é","zz","b"],"l1_i32":[-4,-5,-2147483648,4,3,-3,4,-2,-2,-3,132,1,1,0,5,3,0,5,-3,1],"l2_utf8":["",null,"b","B","zz","é","zz","b",null,"","a","a","aa","ab","B","a","","é","B","é"],"l3_utf8":["😀","zz","ab","😀","b","b","b","ab","ab","ab","é","a","B","aa","a","ab","c","é","a",null],"l4_bool":[false,true,true,true,true,true,null,false,false,false,true,true,true,null,false,true,true,false,true,true],"l5_utf8":["aa","a","😀","aa","a",null,"ab","zz","é","c","zz","ab","b","é","c","zz","B","é","😀","c"],"l6_f64":[-2.0,0.0,-1.0,-4.0,null,-2.0,-33.402,-67.905,"NaN",null,0.0,80.515,1.0,-88.39,"Infinity",-3.0,0.0,4.0,-49.545,3.0]}}},"ops":[{"op":"unique"},{"op":"unique","columns":["l1_i32"]}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_bool","type":"bool"},{"name":"l5_utf8","type":"utf8"},{"name":"l6_f64","type":"f64"}],"columns":{"l0_utf8":["a","zz","ab","é","😀",null,null,"c","zz",null,"a"],"l1_i32":[-4,-5,-2147483648,4,3,-3,-2,132,1,0,5],"l2_utf8":["",null,"b","B","zz","é","b","a","a","ab","B"],"l3_utf8":["😀","zz","ab","😀","b","b","ab","é","a","aa","a"],"l4_bool":[false,true,true,true,true,true,false,true,true,null,false],"l5_utf8":["aa","a","😀","aa","a",null,"zz","zz","ab","é","c"],"l6_f64":[-2.0,0.0,-1.0,-4.0,null,-2.0,-67.905,0.0,80.515,-88.39,"Infinity"]}}},{"id":"random/0392","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i64","type":"i64"},{"name":"l3_i64","type":"i64"}],"columns":{"l0_utf8":[""],"l1_i32":[2],"l2_i64":["9007199254740995"],"l3_i64":["2"]}}},"ops":[{"op":"withColumns","exprs":{"x0":{"fn":"isNotNull","args":[{"col":"l2_i64"}]}}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i64","type":"i64"},{"name":"l3_i64","type":"i64"},{"name":"x0","type":"bool"}],"columns":{"l0_utf8":[""],"l1_i32":[2],"l2_i64":["9007199254740995"],"l3_i64":["2"],"x0":[true]}}},{"id":"random/0393","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_i32","type":"i32"},{"name":"l4_i32","type":"i32"}],"columns":{"l0_utf8":["B","B",null,"ab","","a","","","B","c","c","é","B","aa",null,null,null,"😀","aa","a"],"l1_i32":[-3,-3,4,null,-4,null,-527,null,1,-4,5,3,null,3,1,-4,null,null,-3,5],"l2_i32":[-1,0,-3,null,null,null,-4,null,null,0,-2,2,-4,null,5,-5,4,0,null,2147483647],"l3_i32":[5,-2,null,-3,null,-539,-3,null,2,-2147483648,-5,null,null,4,null,1,2,-2,2,-1],"l4_i32":[3,3,-2,null,2,-5,-1,1,null,null,2,-337,3,1,4,null,0,-3,null,-1]}}},"ops":[{"op":"withColumns","exprs":{"x0":{"fn":"mul","args":[{"col":"l4_i32"},{"col":"l3_i32"}]}}},{"op":"head","n":7}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_i32","type":"i32"},{"name":"l4_i32","type":"i32"},{"name":"x0","type":"i32"}],"columns":{"l0_utf8":["B","B",null,"ab","","a",""],"l1_i32":[-3,-3,4,null,-4,null,-527],"l2_i32":[-1,0,-3,null,null,null,-4],"l3_i32":[5,-2,null,-3,null,-539,-3],"l4_i32":[3,3,-2,null,2,-5,-1],"x0":[15,-6,null,null,null,2695,3]}}},{"id":"random/0394","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_bool","type":"bool"}],"columns":{"l0_utf8":["a","","c","b",""],"l1_i32":[1,-976,-4,-1,3],"l2_utf8":["aa",null,"😀","zz","c"],"l3_utf8":["b","zz","zz","B","a"],"l4_bool":[null,null,true,true,true]}},"right":{"fields":[{"name":"a0_1","type":"utf8"},{"name":"r0_utf8","type":"utf8"}],"columns":{"r0_utf8":[null,"b","😀"],"a0_1":["😀","ab","ab"]}}},"ops":[{"op":"groupBy","keys":["l3_utf8"],"aggs":{"n0":{"agg":"len"},"a0_0":{"agg":"nUnique","arg":{"col":"l0_utf8"}},"a0_1":{"agg":"min","arg":{"col":"l0_utf8"}},"a0_2":{"agg":"first","arg":{"col":"l4_bool"}}}},{"op":"join","right":"right","on":["a0_1"],"how":"left"}],"expected":{"fields":[{"name":"l3_utf8","type":"utf8"},{"name":"n0","type":"u32"},{"name":"a0_0","type":"u32"},{"name":"a0_1","type":"utf8"},{"name":"a0_2","type":"bool"},{"name":"r0_utf8","type":"utf8"}],"columns":{"l3_utf8":["b","zz","B","a"],"n0":[1,2,1,1],"a0_0":[1,2,1,1],"a0_1":["a","","b",""],"a0_2":[null,null,true,true],"r0_utf8":[null,null,null,null]}}},{"id":"random/0395","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_f64","type":"f64"},{"name":"l4_i64","type":"i64"},{"name":"l5_i32","type":"i32"}],"columns":{"l0_utf8":["","😀","zz","a","B"],"l1_i32":[2,-2147483648,2147483647,1,-4],"l2_f64":[-2.0,-82.606,-0.035,-99.118,0.555],"l3_f64":[2.584,-2.0,91.867,69.91,27.93],"l4_i64":["9007199254740994","3","-4","9007199254740992","-3"],"l5_i32":[-5,-5,-3,1,-3]}}},"ops":[{"op":"unique","columns":["l2_f64"]},{"op":"filter","predicate":{"fn":"lte","args":[{"fn":"sub","args":[{"col":"l5_i32"},{"col":"l3_f64"}]},{"fn":"mod","args":[{"col":"l3_f64"},{"col":"l3_f64"}]}]}},{"op":"sort","keys":[{"expr":{"col":"l0_utf8"},"descending":true}],"nullsLast":false},{"op":"filter","predicate":{"fn":"gte","args":[{"fn":"when","args":[{"fn":"eq","args":[{"col":"l4_i64"},{"lit":"1","type":"i64"}]},{"col":"l5_i32"},{"col":"l2_f64"}]},{"fn":"mod","args":[{"lit":0.5},{"lit":0.5}]}]}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_f64","type":"f64"},{"name":"l4_i64","type":"i64"},{"name":"l5_i32","type":"i32"}],"columns":{"l0_utf8":["B"],"l1_i32":[-4],"l2_f64":[0.555],"l3_f64":[27.93],"l4_i64":["-3"],"l5_i32":[-3]}}},{"id":"random/0396","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i64","type":"i64"},{"name":"l3_utf8","type":"utf8"}],"columns":{"l0_utf8":["","a","c","ab","😀"],"l1_i32":[3,4,-3,-1,5],"l2_i64":["-4611686018427387904","-4611686018427387904","9007199254740992","-4611686018427387904","9007199254740992"],"l3_utf8":["aa","zz","c","","ab"]}},"right":{"fields":[{"name":"l2_i64","type":"i64"},{"name":"r0_f64","type":"f64"}],"columns":{"r0_f64":[-2.0,99.674,null,4.0,23.982,"-Infinity",76.323,1.0,-1.0,-95.789],"l2_i64":["-4611686018427387904","9007199254740992","-4611686018427387904","-4611686018427387904","-4611686018427387904","-4611686018427387904","-4611686018427387904","9007199254740992","-4611686018427387904","9007199254740992"]}}},"ops":[{"op":"unique"},{"op":"unique","columns":["l0_utf8"]},{"op":"withColumns","exprs":{"x2":{"fn":"isIn","args":[{"col":"l0_utf8"}],"values":["😀","ab",""]}}},{"op":"join","right":"right","on":["l2_i64"],"how":"inner"}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i64","type":"i64"},{"name":"l3_utf8","type":"utf8"},{"name":"x2","type":"bool"},{"name":"r0_f64","type":"f64"}],"columns":{"l0_utf8":["","","","","","","","a","a","a","a","a","a","a","c","c","c","ab","ab","ab","ab","ab","ab","ab","😀","😀","😀"],"l1_i32":[3,3,3,3,3,3,3,4,4,4,4,4,4,4,-3,-3,-3,-1,-1,-1,-1,-1,-1,-1,5,5,5],"l2_i64":["-4611686018427387904","-4611686018427387904","-4611686018427387904","-4611686018427387904","-4611686018427387904","-4611686018427387904","-4611686018427387904","-4611686018427387904","-4611686018427387904","-4611686018427387904","-4611686018427387904","-4611686018427387904","-4611686018427387904","-4611686018427387904","9007199254740992","9007199254740992","9007199254740992","-4611686018427387904","-4611686018427387904","-4611686018427387904","-4611686018427387904","-4611686018427387904","-4611686018427387904","-4611686018427387904","9007199254740992","9007199254740992","9007199254740992"],"l3_utf8":["aa","aa","aa","aa","aa","aa","aa","zz","zz","zz","zz","zz","zz","zz","c","c","c","","","","","","","","ab","ab","ab"],"x2":[true,true,true,true,true,true,true,false,false,false,false,false,false,false,false,false,false,true,true,true,true,true,true,true,true,true,true],"r0_f64":[-2.0,null,4.0,23.982,"-Infinity",76.323,-1.0,-2.0,null,4.0,23.982,"-Infinity",76.323,-1.0,99.674,1.0,-95.789,-2.0,null,4.0,23.982,"-Infinity",76.323,-1.0,99.674,1.0,-95.789]}}},{"id":"random/0397","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i64","type":"i64"},{"name":"l3_f64","type":"f64"}],"columns":{"l0_utf8":["b"],"l1_i32":[2147483647],"l2_i64":["9007199254740992"],"l3_f64":[-2.0]}}},"ops":[{"op":"sort","keys":[{"expr":{"col":"l0_utf8"},"descending":false},{"expr":{"col":"l2_i64"},"descending":false}],"nullsLast":true},{"op":"sort","keys":[{"expr":{"col":"l3_f64"},"descending":false},{"expr":{"col":"l1_i32"},"descending":true}],"nullsLast":false},{"op":"head","n":7}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i64","type":"i64"},{"name":"l3_f64","type":"f64"}],"columns":{"l0_utf8":["b"],"l1_i32":[2147483647],"l2_i64":["9007199254740992"],"l3_f64":[-2.0]}}},{"id":"random/0398","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_i64","type":"i64"},{"name":"l4_utf8","type":"utf8"}],"columns":{"l0_utf8":["é","ab","zz","ab","é","ab","é","a","é","ab","c","é","c","aa","zz","B","a","b","c","é","","aa",null,"c","aa","zz","B","c","é","é","c",null,"",null,"","😀","B","a","b",null,"c","ab","","zz","","ab","a","é","aa",null,"zz","zz","B","aa","😀","b","c","a","c","B"],"l1_i32":[-4,-4,null,5,2147483647,null,-1,0,-3,-4,-4,-1,0,-4,-3,3,null,-1,-3,2,1,2,2,1,null,793,-3,1,-1,0,5,-1,-1,0,-4,4,5,4,2,1,-1,-3,2,-2,0,-2,-4,null,5,null,-4,-3,-2,-2,-2,5,5,null,null,null],"l2_i32":[3,-5,2,-5,1,2,-2,-4,-3,-5,4,3,4,-2147483648,3,2,4,5,1,1,1,null,2147483647,null,4,5,3,4,-3,1,3,-2,2,null,5,5,4,2147483647,0,0,null,-2,-4,0,null,3,-2147483648,4,2,-2,-5,0,null,4,1,-1,5,-1,5,-1],"l3_i64":["4","2","-4611686018427387904","-4611686018427387904","9007199254740994","9007199254740992","4","2","-4611686018427387904","1","-4611686018427387904","-4611686018427387904","9007199254740995","9007199254740995","9007199254740994","9007199254740992","-4611686018427387904","-1","2","-4611686018427387904","-4611686018427387904","-4611686018427387904","3","-4611686018427387904","-4611686018427387904","9007199254740993","-3","-2","9007199254740995","-4611686018427387904","2","-4611686018427387904","0","9007199254740995","3","-4611686018427387904","-4611686018427387904","0","-4611686018427387904","-4","-4611686018427387904","5","9007199254740993","9007199254740994","2","5","-4611686018427387904","0","-4611686018427387904","-2","-4611686018427387904",null,"-5","9007199254740993","0","-4611686018427387904",null,"-4611686018427387904","-4","-1"],"l4_utf8":["a","é","a","",null,"ab","a","zz","","b","😀",null,"😀","B","B","ab","😀","B","ab","B","a",null,"b","B","a","b","b","b","b","b","a","zz","é","é","ab","b","","é","😀","b","aa","b","é","zz","é","B","ab","b","c","c","","b","c","ab","c","ab","😀","","","aa"]}}},"ops":[{"op":"filter","predicate":{"fn":"lt","args":[{"fn":"mod","args":[{"col":"l2_i32"},{"col":"l2_i32"}]},{"fn":"div","args":[{"lit":-1},{"col":"l1_i32"}]}]}},{"op":"head","n":5}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_i64","type":"i64"},{"name":"l4_utf8","type":"utf8"}],"columns":{"l0_utf8":["é","ab","é","é","ab"],"l1_i32":[-4,-4,-1,-3,-4],"l2_i32":[3,-5,-2,-3,-5],"l3_i64":["4","2","4","-4611686018427387904","1"],"l4_utf8":["a","é","a","","b"]}}},{"id":"random/0399","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_utf8","type":"utf8"}],"columns":{"l0_utf8":["a",null,"ab",null,null,"a",null,"b",null,"😀","zz","aa",null,"zz","ab","c",null,"aa",null,null],"l1_i32":[null,null,null,-3,null,4,3,null,null,-4,null,null,null,null,null,1,null,4,-3,null],"l2_f64":[48.242,null,null,-39.956,4.0,73.933,null,82.848,-4.0,null,-30.608,-4.0,-3.0,null,null,null,null,1.0,-33.925,null],"l3_utf8":["ab",null,"","a","aa",null,null,"é",null,"😀","ab","zz","","ab",null,"zz",null,"é","😀","B"]}}},"ops":[{"op":"head","n":6},{"op":"filter","predicate":{"fn":"eq","args":[{"col":"l2_f64"},{"fn":"sub","args":[{"col":"l1_i32"},{"col":"l2_f64"}]}]}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_utf8","type":"utf8"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_f64":[],"l3_utf8":[]}}}]} diff --git a/packages/effect/test/fixtures/dataframe/gold/spec.json b/packages/effect/test/fixtures/dataframe/gold/spec.json new file mode 100644 index 00000000000..d7e735dc6de --- /dev/null +++ b/packages/effect/test/fixtures/dataframe/gold/spec.json @@ -0,0 +1 @@ +{"generator":"generate_gold.py","polars":"2.1.0","deviations":["JavaScript has no float literal distinct from an integer: an integral number literal behaves like a Polars integer literal (it takes the integer type of the column it meets), so cases never combine an integer column with an integral float literal.","Float to string casts use JavaScript number formatting (Polars prints 5.0 as '5.0', JavaScript as '5').","Joins never add '_right' suffixes: overlapping non-key columns are a type error.","Arithmetic between 64-bit integers (bigint) and other numeric types requires an explicit cast."],"cases":[{"id":"filter/gt","description":"numeric comparison with a literal","inputs":{"left":{"fields":[{"name":"station","type":"utf8"},{"name":"city","type":"utf8"},{"name":"temp","type":"f64"},{"name":"rain","type":"f64"},{"name":"count","type":"i32"},{"name":"id","type":"i64"}],"columns":{"station":["s1","s2","s3","s4","s5","s6","s7"],"city":["Vienna","Vienna","Berlin","Berlin","Zürich",null,"Vienna"],"temp":[21.5,19.0,24.0,26.5,17.0,"NaN",null],"rain":[0.0,null,3.5,1.0,7.0,2.0,0.5],"count":[3,1,null,7,2,5,4],"id":["10","20","30","40","9007199254740993","-1","0"]}}},"ops":[{"op":"filter","predicate":{"fn":"gt","args":[{"col":"temp"},{"lit":20.0}]}}],"expected":{"fields":[{"name":"station","type":"utf8"},{"name":"city","type":"utf8"},{"name":"temp","type":"f64"},{"name":"rain","type":"f64"},{"name":"count","type":"i32"},{"name":"id","type":"i64"}],"columns":{"station":["s1","s3","s4","s6"],"city":["Vienna","Berlin","Berlin",null],"temp":[21.5,24.0,26.5,"NaN"],"rain":[0.0,3.5,1.0,2.0],"count":[3,null,7,5],"id":["10","30","40","-1"]}}},{"id":"filter/null-is-false","description":"rows where the predicate is null are dropped","inputs":{"left":{"fields":[{"name":"station","type":"utf8"},{"name":"city","type":"utf8"},{"name":"temp","type":"f64"},{"name":"rain","type":"f64"},{"name":"count","type":"i32"},{"name":"id","type":"i64"}],"columns":{"station":["s1","s2","s3","s4","s5","s6","s7"],"city":["Vienna","Vienna","Berlin","Berlin","Zürich",null,"Vienna"],"temp":[21.5,19.0,24.0,26.5,17.0,"NaN",null],"rain":[0.0,null,3.5,1.0,7.0,2.0,0.5],"count":[3,1,null,7,2,5,4],"id":["10","20","30","40","9007199254740993","-1","0"]}}},"ops":[{"op":"filter","predicate":{"fn":"gt","args":[{"col":"rain"},{"lit":0.5}]}}],"expected":{"fields":[{"name":"station","type":"utf8"},{"name":"city","type":"utf8"},{"name":"temp","type":"f64"},{"name":"rain","type":"f64"},{"name":"count","type":"i32"},{"name":"id","type":"i64"}],"columns":{"station":["s3","s4","s5","s6"],"city":["Berlin","Berlin","Zürich",null],"temp":[24.0,26.5,17.0,"NaN"],"rain":[3.5,1.0,7.0,2.0],"count":[null,7,2,5],"id":["30","40","9007199254740993","-1"]}}},{"id":"filter/nan-total-order","description":"NaN compares greater than every number and equal to itself","inputs":{"left":{"fields":[{"name":"a","type":"f64"},{"name":"b","type":"f64"},{"name":"i","type":"i32"},{"name":"j","type":"i32"},{"name":"s","type":"utf8"},{"name":"t","type":"utf8"},{"name":"p","type":"bool"},{"name":"q","type":"bool"}],"columns":{"a":[1.0,"-0",0.0,"NaN","Infinity","-Infinity",null,2.5,-2.5,1e+300],"b":[2.0,0.0,"-0","NaN",1.0,null,3.0,0.0,2.0,1e+300],"i":[7,-7,0,2147483647,-2147483648,null,3,-3,100,5],"j":[2,2,3,1,-1,4,null,2,-7,0],"s":["b","a","","é","😀","￿",null,"B","ab","a\u0000b"],"t":["b","A","x","É","😀","z","n",null,"AB","a"],"p":[true,true,true,false,false,false,null,null,null,true],"q":[true,false,null,true,false,null,true,false,null,true]}}},"ops":[{"op":"filter","predicate":{"fn":"gte","args":[{"col":"a"},{"col":"b"}]}}],"expected":{"fields":[{"name":"a","type":"f64"},{"name":"b","type":"f64"},{"name":"i","type":"i32"},{"name":"j","type":"i32"},{"name":"s","type":"utf8"},{"name":"t","type":"utf8"},{"name":"p","type":"bool"},{"name":"q","type":"bool"}],"columns":{"a":["-0",0.0,"NaN","Infinity",2.5,1e+300],"b":[0.0,"-0","NaN",1.0,0.0,1e+300],"i":[-7,0,2147483647,-2147483648,-3,5],"j":[2,3,1,-1,2,0],"s":["a","","é","😀","B","a\u0000b"],"t":["A","x","É","😀",null,"a"],"p":[true,true,false,false,null,true],"q":[false,null,true,false,false,true]}}},{"id":"filter/string-eq","description":"string equality","inputs":{"left":{"fields":[{"name":"station","type":"utf8"},{"name":"city","type":"utf8"},{"name":"temp","type":"f64"},{"name":"rain","type":"f64"},{"name":"count","type":"i32"},{"name":"id","type":"i64"}],"columns":{"station":["s1","s2","s3","s4","s5","s6","s7"],"city":["Vienna","Vienna","Berlin","Berlin","Zürich",null,"Vienna"],"temp":[21.5,19.0,24.0,26.5,17.0,"NaN",null],"rain":[0.0,null,3.5,1.0,7.0,2.0,0.5],"count":[3,1,null,7,2,5,4],"id":["10","20","30","40","9007199254740993","-1","0"]}}},"ops":[{"op":"filter","predicate":{"fn":"eq","args":[{"col":"city"},{"lit":"Vienna"}]}}],"expected":{"fields":[{"name":"station","type":"utf8"},{"name":"city","type":"utf8"},{"name":"temp","type":"f64"},{"name":"rain","type":"f64"},{"name":"count","type":"i32"},{"name":"id","type":"i64"}],"columns":{"station":["s1","s2","s7"],"city":["Vienna","Vienna","Vienna"],"temp":[21.5,19.0,null],"rain":[0.0,null,0.5],"count":[3,1,4],"id":["10","20","0"]}}},{"id":"filter/string-order","description":"strings compare by code point (UTF-8 byte order)","inputs":{"left":{"fields":[{"name":"a","type":"f64"},{"name":"b","type":"f64"},{"name":"i","type":"i32"},{"name":"j","type":"i32"},{"name":"s","type":"utf8"},{"name":"t","type":"utf8"},{"name":"p","type":"bool"},{"name":"q","type":"bool"}],"columns":{"a":[1.0,"-0",0.0,"NaN","Infinity","-Infinity",null,2.5,-2.5,1e+300],"b":[2.0,0.0,"-0","NaN",1.0,null,3.0,0.0,2.0,1e+300],"i":[7,-7,0,2147483647,-2147483648,null,3,-3,100,5],"j":[2,2,3,1,-1,4,null,2,-7,0],"s":["b","a","","é","😀","￿",null,"B","ab","a\u0000b"],"t":["b","A","x","É","😀","z","n",null,"AB","a"],"p":[true,true,true,false,false,false,null,null,null,true],"q":[true,false,null,true,false,null,true,false,null,true]}}},"ops":[{"op":"filter","predicate":{"fn":"lt","args":[{"col":"s"},{"col":"t"}]}}],"expected":{"fields":[{"name":"a","type":"f64"},{"name":"b","type":"f64"},{"name":"i","type":"i32"},{"name":"j","type":"i32"},{"name":"s","type":"utf8"},{"name":"t","type":"utf8"},{"name":"p","type":"bool"},{"name":"q","type":"bool"}],"columns":{"a":[0.0],"b":["-0"],"i":[0],"j":[3],"s":[""],"t":["x"],"p":[true],"q":[null]}}},{"id":"filter/kleene","description":"and/or use three-valued logic","inputs":{"left":{"fields":[{"name":"a","type":"f64"},{"name":"b","type":"f64"},{"name":"i","type":"i32"},{"name":"j","type":"i32"},{"name":"s","type":"utf8"},{"name":"t","type":"utf8"},{"name":"p","type":"bool"},{"name":"q","type":"bool"}],"columns":{"a":[1.0,"-0",0.0,"NaN","Infinity","-Infinity",null,2.5,-2.5,1e+300],"b":[2.0,0.0,"-0","NaN",1.0,null,3.0,0.0,2.0,1e+300],"i":[7,-7,0,2147483647,-2147483648,null,3,-3,100,5],"j":[2,2,3,1,-1,4,null,2,-7,0],"s":["b","a","","é","😀","￿",null,"B","ab","a\u0000b"],"t":["b","A","x","É","😀","z","n",null,"AB","a"],"p":[true,true,true,false,false,false,null,null,null,true],"q":[true,false,null,true,false,null,true,false,null,true]}}},"ops":[{"op":"select","exprs":{"and":{"fn":"and","args":[{"col":"p"},{"col":"q"}]},"or":{"fn":"or","args":[{"col":"p"},{"col":"q"}]},"not":{"fn":"not","args":[{"col":"p"}]}}}],"expected":{"fields":[{"name":"and","type":"bool"},{"name":"or","type":"bool"},{"name":"not","type":"bool"}],"columns":{"and":[true,false,null,false,false,false,null,false,null,true],"or":[true,true,true,true,false,null,true,null,null,true],"not":[false,false,false,true,true,true,null,null,null,false]}}},{"id":"filter/is-null","description":"isNull and isNotNull","inputs":{"left":{"fields":[{"name":"station","type":"utf8"},{"name":"city","type":"utf8"},{"name":"temp","type":"f64"},{"name":"rain","type":"f64"},{"name":"count","type":"i32"},{"name":"id","type":"i64"}],"columns":{"station":["s1","s2","s3","s4","s5","s6","s7"],"city":["Vienna","Vienna","Berlin","Berlin","Zürich",null,"Vienna"],"temp":[21.5,19.0,24.0,26.5,17.0,"NaN",null],"rain":[0.0,null,3.5,1.0,7.0,2.0,0.5],"count":[3,1,null,7,2,5,4],"id":["10","20","30","40","9007199254740993","-1","0"]}}},"ops":[{"op":"filter","predicate":{"fn":"or","args":[{"fn":"isNull","args":[{"col":"city"}]},{"fn":"isNull","args":[{"col":"temp"}]}]}}],"expected":{"fields":[{"name":"station","type":"utf8"},{"name":"city","type":"utf8"},{"name":"temp","type":"f64"},{"name":"rain","type":"f64"},{"name":"count","type":"i32"},{"name":"id","type":"i64"}],"columns":{"station":["s6","s7"],"city":[null,"Vienna"],"temp":["NaN",null],"rain":[2.0,0.5],"count":[5,4],"id":["-1","0"]}}},{"id":"filter/is-in","description":"membership in a literal list","inputs":{"left":{"fields":[{"name":"station","type":"utf8"},{"name":"city","type":"utf8"},{"name":"temp","type":"f64"},{"name":"rain","type":"f64"},{"name":"count","type":"i32"},{"name":"id","type":"i64"}],"columns":{"station":["s1","s2","s3","s4","s5","s6","s7"],"city":["Vienna","Vienna","Berlin","Berlin","Zürich",null,"Vienna"],"temp":[21.5,19.0,24.0,26.5,17.0,"NaN",null],"rain":[0.0,null,3.5,1.0,7.0,2.0,0.5],"count":[3,1,null,7,2,5,4],"id":["10","20","30","40","9007199254740993","-1","0"]}}},"ops":[{"op":"filter","predicate":{"fn":"isIn","args":[{"col":"city"}],"values":["Berlin","Zürich"]}}],"expected":{"fields":[{"name":"station","type":"utf8"},{"name":"city","type":"utf8"},{"name":"temp","type":"f64"},{"name":"rain","type":"f64"},{"name":"count","type":"i32"},{"name":"id","type":"i64"}],"columns":{"station":["s3","s4","s5"],"city":["Berlin","Berlin","Zürich"],"temp":[24.0,26.5,17.0],"rain":[3.5,1.0,7.0],"count":[null,7,2],"id":["30","40","9007199254740993"]}}},{"id":"filter/between","description":"inclusive range","inputs":{"left":{"fields":[{"name":"station","type":"utf8"},{"name":"city","type":"utf8"},{"name":"temp","type":"f64"},{"name":"rain","type":"f64"},{"name":"count","type":"i32"},{"name":"id","type":"i64"}],"columns":{"station":["s1","s2","s3","s4","s5","s6","s7"],"city":["Vienna","Vienna","Berlin","Berlin","Zürich",null,"Vienna"],"temp":[21.5,19.0,24.0,26.5,17.0,"NaN",null],"rain":[0.0,null,3.5,1.0,7.0,2.0,0.5],"count":[3,1,null,7,2,5,4],"id":["10","20","30","40","9007199254740993","-1","0"]}}},"ops":[{"op":"filter","predicate":{"fn":"between","args":[{"col":"temp"},{"lit":19.0},{"lit":24.0}]}}],"expected":{"fields":[{"name":"station","type":"utf8"},{"name":"city","type":"utf8"},{"name":"temp","type":"f64"},{"name":"rain","type":"f64"},{"name":"count","type":"i32"},{"name":"id","type":"i64"}],"columns":{"station":["s1","s2","s3"],"city":["Vienna","Vienna","Berlin"],"temp":[21.5,19.0,24.0],"rain":[0.0,null,3.5],"count":[3,1,null],"id":["10","20","30"]}}},{"id":"filter/compound","description":"compound predicate","inputs":{"left":{"fields":[{"name":"station","type":"utf8"},{"name":"city","type":"utf8"},{"name":"temp","type":"f64"},{"name":"rain","type":"f64"},{"name":"count","type":"i32"},{"name":"id","type":"i64"}],"columns":{"station":["s1","s2","s3","s4","s5","s6","s7"],"city":["Vienna","Vienna","Berlin","Berlin","Zürich",null,"Vienna"],"temp":[21.5,19.0,24.0,26.5,17.0,"NaN",null],"rain":[0.0,null,3.5,1.0,7.0,2.0,0.5],"count":[3,1,null,7,2,5,4],"id":["10","20","30","40","9007199254740993","-1","0"]}}},"ops":[{"op":"filter","predicate":{"fn":"and","args":[{"fn":"gt","args":[{"col":"count"},{"lit":1}]},{"fn":"neq","args":[{"col":"city"},{"lit":"Berlin"}]}]}}],"expected":{"fields":[{"name":"station","type":"utf8"},{"name":"city","type":"utf8"},{"name":"temp","type":"f64"},{"name":"rain","type":"f64"},{"name":"count","type":"i32"},{"name":"id","type":"i64"}],"columns":{"station":["s1","s5","s7"],"city":["Vienna","Zürich","Vienna"],"temp":[21.5,17.0,null],"rain":[0.0,7.0,0.5],"count":[3,2,4],"id":["10","9007199254740993","0"]}}},{"id":"filter/i64","description":"64-bit integer comparison beyond 2^53","inputs":{"left":{"fields":[{"name":"station","type":"utf8"},{"name":"city","type":"utf8"},{"name":"temp","type":"f64"},{"name":"rain","type":"f64"},{"name":"count","type":"i32"},{"name":"id","type":"i64"}],"columns":{"station":["s1","s2","s3","s4","s5","s6","s7"],"city":["Vienna","Vienna","Berlin","Berlin","Zürich",null,"Vienna"],"temp":[21.5,19.0,24.0,26.5,17.0,"NaN",null],"rain":[0.0,null,3.5,1.0,7.0,2.0,0.5],"count":[3,1,null,7,2,5,4],"id":["10","20","30","40","9007199254740993","-1","0"]}}},"ops":[{"op":"filter","predicate":{"fn":"gt","args":[{"col":"id"},{"lit":"9007199254740992","type":"i64"}]}}],"expected":{"fields":[{"name":"station","type":"utf8"},{"name":"city","type":"utf8"},{"name":"temp","type":"f64"},{"name":"rain","type":"f64"},{"name":"count","type":"i32"},{"name":"id","type":"i64"}],"columns":{"station":["s5"],"city":["Zürich"],"temp":[17.0],"rain":[7.0],"count":[2],"id":["9007199254740993"]}}},{"id":"filter/empty-result","description":"a predicate that matches nothing","inputs":{"left":{"fields":[{"name":"station","type":"utf8"},{"name":"city","type":"utf8"},{"name":"temp","type":"f64"},{"name":"rain","type":"f64"},{"name":"count","type":"i32"},{"name":"id","type":"i64"}],"columns":{"station":["s1","s2","s3","s4","s5","s6","s7"],"city":["Vienna","Vienna","Berlin","Berlin","Zürich",null,"Vienna"],"temp":[21.5,19.0,24.0,26.5,17.0,"NaN",null],"rain":[0.0,null,3.5,1.0,7.0,2.0,0.5],"count":[3,1,null,7,2,5,4],"id":["10","20","30","40","9007199254740993","-1","0"]}}},"ops":[{"op":"filter","predicate":{"fn":"gt","args":[{"col":"temp"},{"lit":1000.0}]}}],"expected":{"fields":[{"name":"station","type":"utf8"},{"name":"city","type":"utf8"},{"name":"temp","type":"f64"},{"name":"rain","type":"f64"},{"name":"count","type":"i32"},{"name":"id","type":"i64"}],"columns":{"station":["s6"],"city":[null],"temp":["NaN"],"rain":[2.0],"count":[5],"id":["-1"]}}},{"id":"filter/all-null-literal","description":"a null literal predicate drops everything","inputs":{"left":{"fields":[{"name":"station","type":"utf8"},{"name":"city","type":"utf8"},{"name":"temp","type":"f64"},{"name":"rain","type":"f64"},{"name":"count","type":"i32"},{"name":"id","type":"i64"}],"columns":{"station":["s1","s2","s3","s4","s5","s6","s7"],"city":["Vienna","Vienna","Berlin","Berlin","Zürich",null,"Vienna"],"temp":[21.5,19.0,24.0,26.5,17.0,"NaN",null],"rain":[0.0,null,3.5,1.0,7.0,2.0,0.5],"count":[3,1,null,7,2,5,4],"id":["10","20","30","40","9007199254740993","-1","0"]}}},"ops":[{"op":"filter","predicate":{"lit":null,"type":"bool"}}],"expected":{"fields":[{"name":"station","type":"utf8"},{"name":"city","type":"utf8"},{"name":"temp","type":"f64"},{"name":"rain","type":"f64"},{"name":"count","type":"i32"},{"name":"id","type":"i64"}],"columns":{"station":[],"city":[],"temp":[],"rain":[],"count":[],"id":[]}}},{"id":"arith/float","description":"float arithmetic with nulls, NaN and infinities","inputs":{"left":{"fields":[{"name":"a","type":"f64"},{"name":"b","type":"f64"},{"name":"i","type":"i32"},{"name":"j","type":"i32"},{"name":"s","type":"utf8"},{"name":"t","type":"utf8"},{"name":"p","type":"bool"},{"name":"q","type":"bool"}],"columns":{"a":[1.0,"-0",0.0,"NaN","Infinity","-Infinity",null,2.5,-2.5,1e+300],"b":[2.0,0.0,"-0","NaN",1.0,null,3.0,0.0,2.0,1e+300],"i":[7,-7,0,2147483647,-2147483648,null,3,-3,100,5],"j":[2,2,3,1,-1,4,null,2,-7,0],"s":["b","a","","é","😀","￿",null,"B","ab","a\u0000b"],"t":["b","A","x","É","😀","z","n",null,"AB","a"],"p":[true,true,true,false,false,false,null,null,null,true],"q":[true,false,null,true,false,null,true,false,null,true]}}},"ops":[{"op":"select","exprs":{"add":{"fn":"add","args":[{"col":"a"},{"col":"b"}]},"sub":{"fn":"sub","args":[{"col":"a"},{"col":"b"}]},"mul":{"fn":"mul","args":[{"col":"a"},{"col":"b"}]},"div":{"fn":"div","args":[{"col":"a"},{"col":"b"}]}}}],"expected":{"fields":[{"name":"add","type":"f64"},{"name":"sub","type":"f64"},{"name":"mul","type":"f64"},{"name":"div","type":"f64"}],"columns":{"add":[3.0,0.0,0.0,"NaN","Infinity",null,null,2.5,-0.5,2e+300],"sub":[-1.0,"-0",0.0,"NaN","Infinity",null,null,2.5,-4.5,0.0],"mul":[2.0,"-0","-0","NaN","Infinity",null,null,0.0,-5.0,"Infinity"],"div":[0.5,"NaN","NaN","NaN","Infinity",null,null,"Infinity",-1.25,1.0]}}},{"id":"arith/int32-wrap","description":"Int32 arithmetic wraps on overflow","inputs":{"left":{"fields":[{"name":"a","type":"f64"},{"name":"b","type":"f64"},{"name":"i","type":"i32"},{"name":"j","type":"i32"},{"name":"s","type":"utf8"},{"name":"t","type":"utf8"},{"name":"p","type":"bool"},{"name":"q","type":"bool"}],"columns":{"a":[1.0,"-0",0.0,"NaN","Infinity","-Infinity",null,2.5,-2.5,1e+300],"b":[2.0,0.0,"-0","NaN",1.0,null,3.0,0.0,2.0,1e+300],"i":[7,-7,0,2147483647,-2147483648,null,3,-3,100,5],"j":[2,2,3,1,-1,4,null,2,-7,0],"s":["b","a","","é","😀","￿",null,"B","ab","a\u0000b"],"t":["b","A","x","É","😀","z","n",null,"AB","a"],"p":[true,true,true,false,false,false,null,null,null,true],"q":[true,false,null,true,false,null,true,false,null,true]}}},"ops":[{"op":"select","exprs":{"add":{"fn":"add","args":[{"col":"i"},{"col":"j"}]},"sub":{"fn":"sub","args":[{"col":"i"},{"col":"j"}]},"mul":{"fn":"mul","args":[{"col":"i"},{"col":"j"}]},"neg":{"fn":"neg","args":[{"col":"i"}]}}}],"expected":{"fields":[{"name":"add","type":"i32"},{"name":"sub","type":"i32"},{"name":"mul","type":"i32"},{"name":"neg","type":"i32"}],"columns":{"add":[9,-5,3,-2147483648,2147483647,null,null,-1,93,5],"sub":[5,-9,-3,2147483646,-2147483647,null,null,-5,107,5],"mul":[14,-14,0,2147483647,-2147483648,null,null,-6,-700,0],"neg":[-7,7,0,-2147483647,-2147483648,null,-3,3,-100,-5]}}},{"id":"arith/int32-div","description":"integer true division gives Float64; division by zero gives infinity","inputs":{"left":{"fields":[{"name":"a","type":"f64"},{"name":"b","type":"f64"},{"name":"i","type":"i32"},{"name":"j","type":"i32"},{"name":"s","type":"utf8"},{"name":"t","type":"utf8"},{"name":"p","type":"bool"},{"name":"q","type":"bool"}],"columns":{"a":[1.0,"-0",0.0,"NaN","Infinity","-Infinity",null,2.5,-2.5,1e+300],"b":[2.0,0.0,"-0","NaN",1.0,null,3.0,0.0,2.0,1e+300],"i":[7,-7,0,2147483647,-2147483648,null,3,-3,100,5],"j":[2,2,3,1,-1,4,null,2,-7,0],"s":["b","a","","é","😀","￿",null,"B","ab","a\u0000b"],"t":["b","A","x","É","😀","z","n",null,"AB","a"],"p":[true,true,true,false,false,false,null,null,null,true],"q":[true,false,null,true,false,null,true,false,null,true]}}},"ops":[{"op":"select","exprs":{"div":{"fn":"div","args":[{"col":"i"},{"col":"j"}]}}}],"expected":{"fields":[{"name":"div","type":"f64"}],"columns":{"div":[3.5,-3.5,0.0,2147483647.0,2147483648.0,null,null,-1.5,-14.285714285714286,"Infinity"]}}},{"id":"arith/int32-floor","description":"floor division and floored modulo","inputs":{"left":{"fields":[{"name":"a","type":"f64"},{"name":"b","type":"f64"},{"name":"i","type":"i32"},{"name":"j","type":"i32"},{"name":"s","type":"utf8"},{"name":"t","type":"utf8"},{"name":"p","type":"bool"},{"name":"q","type":"bool"}],"columns":{"a":[1.0,"-0",0.0,"NaN","Infinity","-Infinity",null,2.5,-2.5,1e+300],"b":[2.0,0.0,"-0","NaN",1.0,null,3.0,0.0,2.0,1e+300],"i":[7,-7,0,2147483647,-2147483648,null,3,-3,100,5],"j":[2,2,3,1,-1,4,null,2,-7,0],"s":["b","a","","é","😀","￿",null,"B","ab","a\u0000b"],"t":["b","A","x","É","😀","z","n",null,"AB","a"],"p":[true,true,true,false,false,false,null,null,null,true],"q":[true,false,null,true,false,null,true,false,null,true]}}},"ops":[{"op":"filter","predicate":{"fn":"neq","args":[{"col":"j"},{"lit":0}]}},{"op":"select","exprs":{"floor":{"fn":"floorDiv","args":[{"col":"i"},{"col":"j"}]},"mod":{"fn":"mod","args":[{"col":"i"},{"col":"j"}]}}}],"expected":{"fields":[{"name":"floor","type":"i32"},{"name":"mod","type":"i32"}],"columns":{"floor":[3,-4,0,2147483647,-2147483648,null,-2,-15],"mod":[1,1,0,0,0,null,1,-5]}}},{"id":"arith/float-floor","description":"floor division and floored modulo on floats","inputs":{"left":{"fields":[{"name":"a","type":"f64"},{"name":"b","type":"f64"},{"name":"i","type":"i32"},{"name":"j","type":"i32"},{"name":"s","type":"utf8"},{"name":"t","type":"utf8"},{"name":"p","type":"bool"},{"name":"q","type":"bool"}],"columns":{"a":[1.0,"-0",0.0,"NaN","Infinity","-Infinity",null,2.5,-2.5,1e+300],"b":[2.0,0.0,"-0","NaN",1.0,null,3.0,0.0,2.0,1e+300],"i":[7,-7,0,2147483647,-2147483648,null,3,-3,100,5],"j":[2,2,3,1,-1,4,null,2,-7,0],"s":["b","a","","é","😀","￿",null,"B","ab","a\u0000b"],"t":["b","A","x","É","😀","z","n",null,"AB","a"],"p":[true,true,true,false,false,false,null,null,null,true],"q":[true,false,null,true,false,null,true,false,null,true]}}},"ops":[{"op":"select","exprs":{"floor":{"fn":"floorDiv","args":[{"col":"a"},{"col":"b"}]},"mod":{"fn":"mod","args":[{"col":"a"},{"col":"b"}]}}}],"expected":{"fields":[{"name":"floor","type":"f64"},{"name":"mod","type":"f64"}],"columns":{"floor":[0.0,"NaN","NaN","NaN","Infinity",null,null,"Infinity",-2.0,1.0],"mod":[1.0,"NaN","NaN","NaN","NaN",null,null,"NaN",1.5,0.0]}}},{"id":"arith/mixed","description":"Int32 with a float literal widens to Float64","inputs":{"left":{"fields":[{"name":"a","type":"f64"},{"name":"b","type":"f64"},{"name":"i","type":"i32"},{"name":"j","type":"i32"},{"name":"s","type":"utf8"},{"name":"t","type":"utf8"},{"name":"p","type":"bool"},{"name":"q","type":"bool"}],"columns":{"a":[1.0,"-0",0.0,"NaN","Infinity","-Infinity",null,2.5,-2.5,1e+300],"b":[2.0,0.0,"-0","NaN",1.0,null,3.0,0.0,2.0,1e+300],"i":[7,-7,0,2147483647,-2147483648,null,3,-3,100,5],"j":[2,2,3,1,-1,4,null,2,-7,0],"s":["b","a","","é","😀","￿",null,"B","ab","a\u0000b"],"t":["b","A","x","É","😀","z","n",null,"AB","a"],"p":[true,true,true,false,false,false,null,null,null,true],"q":[true,false,null,true,false,null,true,false,null,true]}}},"ops":[{"op":"select","exprs":{"x":{"fn":"mul","args":[{"col":"i"},{"lit":1.5}]},"y":{"fn":"add","args":[{"col":"i"},{"col":"a"}]}}}],"expected":{"fields":[{"name":"x","type":"f64"},{"name":"y","type":"f64"}],"columns":{"x":[10.5,-10.5,0.0,3221225470.5,-3221225472.0,null,4.5,-4.5,150.0,7.5],"y":[8.0,-7.0,0.0,"NaN","Infinity",null,null,-0.5,97.5,1e+300]}}},{"id":"arith/int-literal","description":"an integer literal adopts the integer column type","inputs":{"left":{"fields":[{"name":"a","type":"f64"},{"name":"b","type":"f64"},{"name":"i","type":"i32"},{"name":"j","type":"i32"},{"name":"s","type":"utf8"},{"name":"t","type":"utf8"},{"name":"p","type":"bool"},{"name":"q","type":"bool"}],"columns":{"a":[1.0,"-0",0.0,"NaN","Infinity","-Infinity",null,2.5,-2.5,1e+300],"b":[2.0,0.0,"-0","NaN",1.0,null,3.0,0.0,2.0,1e+300],"i":[7,-7,0,2147483647,-2147483648,null,3,-3,100,5],"j":[2,2,3,1,-1,4,null,2,-7,0],"s":["b","a","","é","😀","￿",null,"B","ab","a\u0000b"],"t":["b","A","x","É","😀","z","n",null,"AB","a"],"p":[true,true,true,false,false,false,null,null,null,true],"q":[true,false,null,true,false,null,true,false,null,true]}}},"ops":[{"op":"select","exprs":{"x":{"fn":"add","args":[{"col":"i"},{"lit":1}]},"y":{"fn":"mul","args":[{"col":"j"},{"lit":3}]}}}],"expected":{"fields":[{"name":"x","type":"i32"},{"name":"y","type":"i32"}],"columns":{"x":[8,-6,1,-2147483648,-2147483647,null,4,-2,101,6],"y":[6,6,9,3,-3,12,null,6,-21,0]}}},{"id":"arith/abs-sqrt","description":"abs and sqrt","inputs":{"left":{"fields":[{"name":"a","type":"f64"},{"name":"b","type":"f64"},{"name":"i","type":"i32"},{"name":"j","type":"i32"},{"name":"s","type":"utf8"},{"name":"t","type":"utf8"},{"name":"p","type":"bool"},{"name":"q","type":"bool"}],"columns":{"a":[1.0,"-0",0.0,"NaN","Infinity","-Infinity",null,2.5,-2.5,1e+300],"b":[2.0,0.0,"-0","NaN",1.0,null,3.0,0.0,2.0,1e+300],"i":[7,-7,0,2147483647,-2147483648,null,3,-3,100,5],"j":[2,2,3,1,-1,4,null,2,-7,0],"s":["b","a","","é","😀","￿",null,"B","ab","a\u0000b"],"t":["b","A","x","É","😀","z","n",null,"AB","a"],"p":[true,true,true,false,false,false,null,null,null,true],"q":[true,false,null,true,false,null,true,false,null,true]}}},"ops":[{"op":"select","exprs":{"abs":{"fn":"abs","args":[{"col":"a"}]},"iabs":{"fn":"abs","args":[{"col":"j"}]},"sqrt":{"fn":"sqrt","args":[{"col":"b"}]}}}],"expected":{"fields":[{"name":"abs","type":"f64"},{"name":"iabs","type":"i32"},{"name":"sqrt","type":"f64"}],"columns":{"abs":[1.0,0.0,0.0,"NaN","Infinity","Infinity",null,2.5,2.5,1e+300],"iabs":[2,2,3,1,1,4,null,2,7,0],"sqrt":[1.4142135623730951,0.0,"-0","NaN",1.0,null,1.7320508075688772,0.0,1.4142135623730951,1e+150]}}},{"id":"arith/i64","description":"Int64 arithmetic","inputs":{"left":{"fields":[{"name":"station","type":"utf8"},{"name":"city","type":"utf8"},{"name":"temp","type":"f64"},{"name":"rain","type":"f64"},{"name":"count","type":"i32"},{"name":"id","type":"i64"}],"columns":{"station":["s1","s2","s3","s4","s5","s6","s7"],"city":["Vienna","Vienna","Berlin","Berlin","Zürich",null,"Vienna"],"temp":[21.5,19.0,24.0,26.5,17.0,"NaN",null],"rain":[0.0,null,3.5,1.0,7.0,2.0,0.5],"count":[3,1,null,7,2,5,4],"id":["10","20","30","40","9007199254740993","-1","0"]}}},"ops":[{"op":"select","exprs":{"x":{"fn":"add","args":[{"col":"id"},{"lit":"1","type":"i64"}]},"y":{"fn":"mul","args":[{"col":"id"},{"lit":"2","type":"i64"}]},"z":{"fn":"sub","args":[{"col":"id"},{"col":"id"}]}}}],"expected":{"fields":[{"name":"x","type":"i64"},{"name":"y","type":"i64"},{"name":"z","type":"i64"}],"columns":{"x":["11","21","31","41","9007199254740994","0","1"],"y":["20","40","60","80","18014398509481986","-2","0"],"z":["0","0","0","0","0","0","0"]}}},{"id":"arith/round","description":"round half to even","inputs":{"left":{"fields":[{"name":"x","type":"f64"}],"columns":{"x":[0.5,1.5,2.5,-0.5,-1.5,1.25,2.675,null,"NaN"]}}},"ops":[{"op":"select","exprs":{"r0":{"fn":"round","args":[{"col":"x"}]},"r1":{"fn":"round","args":[{"col":"x"}],"decimals":1}}}],"expected":{"fields":[{"name":"r0","type":"f64"},{"name":"r1","type":"f64"}],"columns":{"r0":[0.0,2.0,2.0,"-0",-2.0,1.0,3.0,null,"NaN"],"r1":[0.5,1.5,2.5,-0.5,-1.5,1.2,2.7,null,"NaN"]}}},{"id":"arith/small-ints","description":"8/16-bit arithmetic wraps in the column type","inputs":{"left":{"fields":[{"name":"i8","type":"i8"},{"name":"u8","type":"u8"},{"name":"i16","type":"i16"},{"name":"u16","type":"u16"},{"name":"u32","type":"u32"},{"name":"i64","type":"i64"},{"name":"u64","type":"u64"},{"name":"f32","type":"f32"}],"columns":{"i8":[127,-128,0,null,5],"u8":[255,0,1,null,5],"i16":[32767,-32768,0,12,null],"u16":[65535,0,7,12,null],"u32":[4294967295,0,1,null,3],"i64":["9223372036854775807","-9223372036854775808","0",null,"42"],"u64":["18446744073709551615","0","1",null,"42"],"f32":[1.5,-0.25,null,3.0,10000000000.0]}}},"ops":[{"op":"select","exprs":{"i8":{"fn":"add","args":[{"col":"i8"},{"lit":1}]},"u8":{"fn":"add","args":[{"col":"u8"},{"lit":1}]},"i16":{"fn":"mul","args":[{"col":"i16"},{"lit":2}]},"u16":{"fn":"sub","args":[{"col":"u16"},{"lit":1}]},"u32":{"fn":"add","args":[{"col":"u32"},{"lit":1}]},"f32":{"fn":"mul","args":[{"col":"f32"},{"lit":2.0}]}}}],"expected":{"fields":[{"name":"i8","type":"i8"},{"name":"u8","type":"u8"},{"name":"i16","type":"i16"},{"name":"u16","type":"u16"},{"name":"u32","type":"u32"},{"name":"f32","type":"f32"}],"columns":{"i8":[-128,-127,1,null,6],"u8":[0,1,2,null,6],"i16":[-2,0,0,24,null],"u16":[65534,65535,6,11,null],"u32":[0,1,2,null,4],"f32":[3.0,-0.5,null,6.0,20000000000.0]}}},{"id":"null/fill","description":"fillNull with a literal","inputs":{"left":{"fields":[{"name":"station","type":"utf8"},{"name":"city","type":"utf8"},{"name":"temp","type":"f64"},{"name":"rain","type":"f64"},{"name":"count","type":"i32"},{"name":"id","type":"i64"}],"columns":{"station":["s1","s2","s3","s4","s5","s6","s7"],"city":["Vienna","Vienna","Berlin","Berlin","Zürich",null,"Vienna"],"temp":[21.5,19.0,24.0,26.5,17.0,"NaN",null],"rain":[0.0,null,3.5,1.0,7.0,2.0,0.5],"count":[3,1,null,7,2,5,4],"id":["10","20","30","40","9007199254740993","-1","0"]}}},"ops":[{"op":"select","exprs":{"city":{"fn":"fillNull","args":[{"col":"city"},{"lit":"?"}]},"temp":{"fn":"fillNull","args":[{"col":"temp"},{"lit":0.0}]}}}],"expected":{"fields":[{"name":"city","type":"utf8"},{"name":"temp","type":"f64"}],"columns":{"city":["Vienna","Vienna","Berlin","Berlin","Zürich","?","Vienna"],"temp":[21.5,19.0,24.0,26.5,17.0,"NaN",0.0]}}},{"id":"null/coalesce","description":"first non-null","inputs":{"left":{"fields":[{"name":"a","type":"f64"},{"name":"b","type":"f64"},{"name":"i","type":"i32"},{"name":"j","type":"i32"},{"name":"s","type":"utf8"},{"name":"t","type":"utf8"},{"name":"p","type":"bool"},{"name":"q","type":"bool"}],"columns":{"a":[1.0,"-0",0.0,"NaN","Infinity","-Infinity",null,2.5,-2.5,1e+300],"b":[2.0,0.0,"-0","NaN",1.0,null,3.0,0.0,2.0,1e+300],"i":[7,-7,0,2147483647,-2147483648,null,3,-3,100,5],"j":[2,2,3,1,-1,4,null,2,-7,0],"s":["b","a","","é","😀","￿",null,"B","ab","a\u0000b"],"t":["b","A","x","É","😀","z","n",null,"AB","a"],"p":[true,true,true,false,false,false,null,null,null,true],"q":[true,false,null,true,false,null,true,false,null,true]}}},"ops":[{"op":"select","exprs":{"x":{"fn":"coalesce","args":[{"col":"a"},{"col":"b"},{"lit":-1.0}]}}}],"expected":{"fields":[{"name":"x","type":"f64"}],"columns":{"x":[1.0,"-0",0.0,"NaN","Infinity","-Infinity",3.0,2.5,-2.5,1e+300]}}},{"id":"null/when","description":"when/then/otherwise","inputs":{"left":{"fields":[{"name":"station","type":"utf8"},{"name":"city","type":"utf8"},{"name":"temp","type":"f64"},{"name":"rain","type":"f64"},{"name":"count","type":"i32"},{"name":"id","type":"i64"}],"columns":{"station":["s1","s2","s3","s4","s5","s6","s7"],"city":["Vienna","Vienna","Berlin","Berlin","Zürich",null,"Vienna"],"temp":[21.5,19.0,24.0,26.5,17.0,"NaN",null],"rain":[0.0,null,3.5,1.0,7.0,2.0,0.5],"count":[3,1,null,7,2,5,4],"id":["10","20","30","40","9007199254740993","-1","0"]}}},"ops":[{"op":"select","exprs":{"label":{"fn":"when","args":[{"fn":"gt","args":[{"col":"temp"},{"lit":20.0}]},{"lit":"warm"},{"lit":"cold"}]}}}],"expected":{"fields":[{"name":"label","type":"utf8"}],"columns":{"label":["warm","cold","warm","warm","cold","warm","cold"]}}},{"id":"null/drop","description":"drop rows with nulls","inputs":{"left":{"fields":[{"name":"station","type":"utf8"},{"name":"city","type":"utf8"},{"name":"temp","type":"f64"},{"name":"rain","type":"f64"},{"name":"count","type":"i32"},{"name":"id","type":"i64"}],"columns":{"station":["s1","s2","s3","s4","s5","s6","s7"],"city":["Vienna","Vienna","Berlin","Berlin","Zürich",null,"Vienna"],"temp":[21.5,19.0,24.0,26.5,17.0,"NaN",null],"rain":[0.0,null,3.5,1.0,7.0,2.0,0.5],"count":[3,1,null,7,2,5,4],"id":["10","20","30","40","9007199254740993","-1","0"]}}},"ops":[{"op":"dropNulls"}],"expected":{"fields":[{"name":"station","type":"utf8"},{"name":"city","type":"utf8"},{"name":"temp","type":"f64"},{"name":"rain","type":"f64"},{"name":"count","type":"i32"},{"name":"id","type":"i64"}],"columns":{"station":["s1","s4","s5"],"city":["Vienna","Berlin","Zürich"],"temp":[21.5,26.5,17.0],"rain":[0.0,1.0,7.0],"count":[3,7,2],"id":["10","40","9007199254740993"]}}},{"id":"null/drop-subset","description":"drop rows with nulls in some columns","inputs":{"left":{"fields":[{"name":"station","type":"utf8"},{"name":"city","type":"utf8"},{"name":"temp","type":"f64"},{"name":"rain","type":"f64"},{"name":"count","type":"i32"},{"name":"id","type":"i64"}],"columns":{"station":["s1","s2","s3","s4","s5","s6","s7"],"city":["Vienna","Vienna","Berlin","Berlin","Zürich",null,"Vienna"],"temp":[21.5,19.0,24.0,26.5,17.0,"NaN",null],"rain":[0.0,null,3.5,1.0,7.0,2.0,0.5],"count":[3,1,null,7,2,5,4],"id":["10","20","30","40","9007199254740993","-1","0"]}}},"ops":[{"op":"dropNulls","columns":["city"]}],"expected":{"fields":[{"name":"station","type":"utf8"},{"name":"city","type":"utf8"},{"name":"temp","type":"f64"},{"name":"rain","type":"f64"},{"name":"count","type":"i32"},{"name":"id","type":"i64"}],"columns":{"station":["s1","s2","s3","s4","s5","s7"],"city":["Vienna","Vienna","Berlin","Berlin","Zürich","Vienna"],"temp":[21.5,19.0,24.0,26.5,17.0,null],"rain":[0.0,null,3.5,1.0,7.0,0.5],"count":[3,1,null,7,2,4],"id":["10","20","30","40","9007199254740993","0"]}}},{"id":"null/is-nan","description":"isNaN is distinct from isNull","inputs":{"left":{"fields":[{"name":"a","type":"f64"},{"name":"b","type":"f64"},{"name":"i","type":"i32"},{"name":"j","type":"i32"},{"name":"s","type":"utf8"},{"name":"t","type":"utf8"},{"name":"p","type":"bool"},{"name":"q","type":"bool"}],"columns":{"a":[1.0,"-0",0.0,"NaN","Infinity","-Infinity",null,2.5,-2.5,1e+300],"b":[2.0,0.0,"-0","NaN",1.0,null,3.0,0.0,2.0,1e+300],"i":[7,-7,0,2147483647,-2147483648,null,3,-3,100,5],"j":[2,2,3,1,-1,4,null,2,-7,0],"s":["b","a","","é","😀","￿",null,"B","ab","a\u0000b"],"t":["b","A","x","É","😀","z","n",null,"AB","a"],"p":[true,true,true,false,false,false,null,null,null,true],"q":[true,false,null,true,false,null,true,false,null,true]}}},"ops":[{"op":"select","exprs":{"nan":{"fn":"isNaN","args":[{"col":"a"}]}}}],"expected":{"fields":[{"name":"nan","type":"bool"}],"columns":{"nan":[false,false,false,true,false,false,null,false,false,false]}}},{"id":"cast/float-int","description":"float to Int32 truncates; out of range is null","inputs":{"left":{"fields":[{"name":"a","type":"f64"},{"name":"b","type":"f64"},{"name":"i","type":"i32"},{"name":"j","type":"i32"},{"name":"s","type":"utf8"},{"name":"t","type":"utf8"},{"name":"p","type":"bool"},{"name":"q","type":"bool"}],"columns":{"a":[1.0,"-0",0.0,"NaN","Infinity","-Infinity",null,2.5,-2.5,1e+300],"b":[2.0,0.0,"-0","NaN",1.0,null,3.0,0.0,2.0,1e+300],"i":[7,-7,0,2147483647,-2147483648,null,3,-3,100,5],"j":[2,2,3,1,-1,4,null,2,-7,0],"s":["b","a","","é","😀","￿",null,"B","ab","a\u0000b"],"t":["b","A","x","É","😀","z","n",null,"AB","a"],"p":[true,true,true,false,false,false,null,null,null,true],"q":[true,false,null,true,false,null,true,false,null,true]}}},"ops":[{"op":"select","exprs":{"i":{"fn":"cast","args":[{"col":"a"}],"to":"i32"},"f":{"fn":"cast","args":[{"col":"i"}],"to":"f64"}}}],"expected":{"fields":[{"name":"i","type":"i32"},{"name":"f","type":"f64"}],"columns":{"i":[1,0,0,null,null,null,null,2,-2,null],"f":[7.0,-7.0,0.0,2147483647.0,-2147483648.0,null,3.0,-3.0,100.0,5.0]}}},{"id":"cast/int-string","description":"integers to strings","inputs":{"left":{"fields":[{"name":"a","type":"f64"},{"name":"b","type":"f64"},{"name":"i","type":"i32"},{"name":"j","type":"i32"},{"name":"s","type":"utf8"},{"name":"t","type":"utf8"},{"name":"p","type":"bool"},{"name":"q","type":"bool"}],"columns":{"a":[1.0,"-0",0.0,"NaN","Infinity","-Infinity",null,2.5,-2.5,1e+300],"b":[2.0,0.0,"-0","NaN",1.0,null,3.0,0.0,2.0,1e+300],"i":[7,-7,0,2147483647,-2147483648,null,3,-3,100,5],"j":[2,2,3,1,-1,4,null,2,-7,0],"s":["b","a","","é","😀","￿",null,"B","ab","a\u0000b"],"t":["b","A","x","É","😀","z","n",null,"AB","a"],"p":[true,true,true,false,false,false,null,null,null,true],"q":[true,false,null,true,false,null,true,false,null,true]}}},"ops":[{"op":"select","exprs":{"s":{"fn":"cast","args":[{"col":"i"}],"to":"utf8"}}}],"expected":{"fields":[{"name":"s","type":"utf8"}],"columns":{"s":["7","-7","0","2147483647","-2147483648",null,"3","-3","100","5"]}}},{"id":"cast/string-number","description":"strings to numbers; unparsable is null","inputs":{"left":{"fields":[{"name":"s","type":"utf8"}],"columns":{"s":["1","-2","3.5","x","",null," 4","1e3"]}}},"ops":[{"op":"select","exprs":{"f":{"fn":"cast","args":[{"col":"s"}],"to":"f64"},"i":{"fn":"cast","args":[{"col":"s"}],"to":"i32"}}}],"expected":{"fields":[{"name":"f","type":"f64"},{"name":"i","type":"i32"}],"columns":{"f":[1.0,-2.0,3.5,null,null,null,null,1000.0],"i":[1,-2,null,null,null,null,null,null]}}},{"id":"cast/bool","description":"booleans to and from numbers","inputs":{"left":{"fields":[{"name":"a","type":"f64"},{"name":"b","type":"f64"},{"name":"i","type":"i32"},{"name":"j","type":"i32"},{"name":"s","type":"utf8"},{"name":"t","type":"utf8"},{"name":"p","type":"bool"},{"name":"q","type":"bool"}],"columns":{"a":[1.0,"-0",0.0,"NaN","Infinity","-Infinity",null,2.5,-2.5,1e+300],"b":[2.0,0.0,"-0","NaN",1.0,null,3.0,0.0,2.0,1e+300],"i":[7,-7,0,2147483647,-2147483648,null,3,-3,100,5],"j":[2,2,3,1,-1,4,null,2,-7,0],"s":["b","a","","é","😀","￿",null,"B","ab","a\u0000b"],"t":["b","A","x","É","😀","z","n",null,"AB","a"],"p":[true,true,true,false,false,false,null,null,null,true],"q":[true,false,null,true,false,null,true,false,null,true]}}},"ops":[{"op":"select","exprs":{"b":{"fn":"cast","args":[{"col":"j"}],"to":"bool"},"n":{"fn":"cast","args":[{"col":"p"}],"to":"i32"}}}],"expected":{"fields":[{"name":"b","type":"bool"},{"name":"n","type":"i32"}],"columns":{"b":[true,true,true,true,true,true,null,true,true,false],"n":[1,1,1,0,0,0,null,null,null,1]}}},{"id":"cast/i64","description":"Int64 to and from other types","inputs":{"left":{"fields":[{"name":"station","type":"utf8"},{"name":"city","type":"utf8"},{"name":"temp","type":"f64"},{"name":"rain","type":"f64"},{"name":"count","type":"i32"},{"name":"id","type":"i64"}],"columns":{"station":["s1","s2","s3","s4","s5","s6","s7"],"city":["Vienna","Vienna","Berlin","Berlin","Zürich",null,"Vienna"],"temp":[21.5,19.0,24.0,26.5,17.0,"NaN",null],"rain":[0.0,null,3.5,1.0,7.0,2.0,0.5],"count":[3,1,null,7,2,5,4],"id":["10","20","30","40","9007199254740993","-1","0"]}}},"ops":[{"op":"select","exprs":{"f":{"fn":"cast","args":[{"col":"id"}],"to":"f64"},"i":{"fn":"cast","args":[{"col":"count"}],"to":"i64"},"s":{"fn":"cast","args":[{"col":"id"}],"to":"utf8"}}}],"expected":{"fields":[{"name":"f","type":"f64"},{"name":"i","type":"i64"},{"name":"s","type":"utf8"}],"columns":{"f":[10.0,20.0,30.0,40.0,9007199254740992.0,-1.0,0.0],"i":["3","1",null,"7","2","5","4"],"s":["10","20","30","40","9007199254740993","-1","0"]}}},{"id":"str/length","description":"length in characters","inputs":{"left":{"fields":[{"name":"a","type":"f64"},{"name":"b","type":"f64"},{"name":"i","type":"i32"},{"name":"j","type":"i32"},{"name":"s","type":"utf8"},{"name":"t","type":"utf8"},{"name":"p","type":"bool"},{"name":"q","type":"bool"}],"columns":{"a":[1.0,"-0",0.0,"NaN","Infinity","-Infinity",null,2.5,-2.5,1e+300],"b":[2.0,0.0,"-0","NaN",1.0,null,3.0,0.0,2.0,1e+300],"i":[7,-7,0,2147483647,-2147483648,null,3,-3,100,5],"j":[2,2,3,1,-1,4,null,2,-7,0],"s":["b","a","","é","😀","￿",null,"B","ab","a\u0000b"],"t":["b","A","x","É","😀","z","n",null,"AB","a"],"p":[true,true,true,false,false,false,null,null,null,true],"q":[true,false,null,true,false,null,true,false,null,true]}}},"ops":[{"op":"select","exprs":{"n":{"fn":"strLength","args":[{"col":"s"}]}}}],"expected":{"fields":[{"name":"n","type":"u32"}],"columns":{"n":[1,1,0,1,1,1,null,1,2,3]}}},{"id":"str/case","description":"upper and lower case","inputs":{"left":{"fields":[{"name":"a","type":"f64"},{"name":"b","type":"f64"},{"name":"i","type":"i32"},{"name":"j","type":"i32"},{"name":"s","type":"utf8"},{"name":"t","type":"utf8"},{"name":"p","type":"bool"},{"name":"q","type":"bool"}],"columns":{"a":[1.0,"-0",0.0,"NaN","Infinity","-Infinity",null,2.5,-2.5,1e+300],"b":[2.0,0.0,"-0","NaN",1.0,null,3.0,0.0,2.0,1e+300],"i":[7,-7,0,2147483647,-2147483648,null,3,-3,100,5],"j":[2,2,3,1,-1,4,null,2,-7,0],"s":["b","a","","é","😀","￿",null,"B","ab","a\u0000b"],"t":["b","A","x","É","😀","z","n",null,"AB","a"],"p":[true,true,true,false,false,false,null,null,null,true],"q":[true,false,null,true,false,null,true,false,null,true]}}},"ops":[{"op":"select","exprs":{"u":{"fn":"strUpper","args":[{"col":"s"}]},"l":{"fn":"strLower","args":[{"col":"t"}]}}}],"expected":{"fields":[{"name":"u","type":"utf8"},{"name":"l","type":"utf8"}],"columns":{"u":["B","A","","É","😀","￿",null,"B","AB","A\u0000B"],"l":["b","a","x","é","😀","z","n",null,"ab","a"]}}},{"id":"str/match","description":"contains, startsWith, endsWith with literals","inputs":{"left":{"fields":[{"name":"a","type":"f64"},{"name":"b","type":"f64"},{"name":"i","type":"i32"},{"name":"j","type":"i32"},{"name":"s","type":"utf8"},{"name":"t","type":"utf8"},{"name":"p","type":"bool"},{"name":"q","type":"bool"}],"columns":{"a":[1.0,"-0",0.0,"NaN","Infinity","-Infinity",null,2.5,-2.5,1e+300],"b":[2.0,0.0,"-0","NaN",1.0,null,3.0,0.0,2.0,1e+300],"i":[7,-7,0,2147483647,-2147483648,null,3,-3,100,5],"j":[2,2,3,1,-1,4,null,2,-7,0],"s":["b","a","","é","😀","￿",null,"B","ab","a\u0000b"],"t":["b","A","x","É","😀","z","n",null,"AB","a"],"p":[true,true,true,false,false,false,null,null,null,true],"q":[true,false,null,true,false,null,true,false,null,true]}}},"ops":[{"op":"select","exprs":{"c":{"fn":"strContains","args":[{"col":"s"}],"pattern":"a"},"p":{"fn":"strStartsWith","args":[{"col":"t"}],"prefix":"A"},"e":{"fn":"strEndsWith","args":[{"col":"s"}],"suffix":"b"}}}],"expected":{"fields":[{"name":"c","type":"bool"},{"name":"p","type":"bool"},{"name":"e","type":"bool"}],"columns":{"c":[false,true,false,false,false,false,null,false,true,true],"p":[false,true,false,false,false,false,false,null,true,false],"e":[true,false,false,false,false,false,null,false,true,true]}}},{"id":"str/concat","description":"concatenation; null in any argument gives null","inputs":{"left":{"fields":[{"name":"a","type":"f64"},{"name":"b","type":"f64"},{"name":"i","type":"i32"},{"name":"j","type":"i32"},{"name":"s","type":"utf8"},{"name":"t","type":"utf8"},{"name":"p","type":"bool"},{"name":"q","type":"bool"}],"columns":{"a":[1.0,"-0",0.0,"NaN","Infinity","-Infinity",null,2.5,-2.5,1e+300],"b":[2.0,0.0,"-0","NaN",1.0,null,3.0,0.0,2.0,1e+300],"i":[7,-7,0,2147483647,-2147483648,null,3,-3,100,5],"j":[2,2,3,1,-1,4,null,2,-7,0],"s":["b","a","","é","😀","￿",null,"B","ab","a\u0000b"],"t":["b","A","x","É","😀","z","n",null,"AB","a"],"p":[true,true,true,false,false,false,null,null,null,true],"q":[true,false,null,true,false,null,true,false,null,true]}}},"ops":[{"op":"select","exprs":{"c":{"fn":"strConcat","args":[{"col":"s"},{"lit":"-"},{"col":"t"}]}}}],"expected":{"fields":[{"name":"c","type":"utf8"}],"columns":{"c":["b-b","a-A","-x","é-É","😀-😀","￿-z",null,null,"ab-AB","a\u0000b-a"]}}},{"id":"project/pick","description":"pick columns in the given order","inputs":{"left":{"fields":[{"name":"station","type":"utf8"},{"name":"city","type":"utf8"},{"name":"temp","type":"f64"},{"name":"rain","type":"f64"},{"name":"count","type":"i32"},{"name":"id","type":"i64"}],"columns":{"station":["s1","s2","s3","s4","s5","s6","s7"],"city":["Vienna","Vienna","Berlin","Berlin","Zürich",null,"Vienna"],"temp":[21.5,19.0,24.0,26.5,17.0,"NaN",null],"rain":[0.0,null,3.5,1.0,7.0,2.0,0.5],"count":[3,1,null,7,2,5,4],"id":["10","20","30","40","9007199254740993","-1","0"]}}},"ops":[{"op":"pick","columns":["temp","station"]}],"expected":{"fields":[{"name":"temp","type":"f64"},{"name":"station","type":"utf8"}],"columns":{"temp":[21.5,19.0,24.0,26.5,17.0,"NaN",null],"station":["s1","s2","s3","s4","s5","s6","s7"]}}},{"id":"project/omit","description":"omit columns","inputs":{"left":{"fields":[{"name":"station","type":"utf8"},{"name":"city","type":"utf8"},{"name":"temp","type":"f64"},{"name":"rain","type":"f64"},{"name":"count","type":"i32"},{"name":"id","type":"i64"}],"columns":{"station":["s1","s2","s3","s4","s5","s6","s7"],"city":["Vienna","Vienna","Berlin","Berlin","Zürich",null,"Vienna"],"temp":[21.5,19.0,24.0,26.5,17.0,"NaN",null],"rain":[0.0,null,3.5,1.0,7.0,2.0,0.5],"count":[3,1,null,7,2,5,4],"id":["10","20","30","40","9007199254740993","-1","0"]}}},"ops":[{"op":"omit","columns":["rain","id"]}],"expected":{"fields":[{"name":"station","type":"utf8"},{"name":"city","type":"utf8"},{"name":"temp","type":"f64"},{"name":"count","type":"i32"}],"columns":{"station":["s1","s2","s3","s4","s5","s6","s7"],"city":["Vienna","Vienna","Berlin","Berlin","Zürich",null,"Vienna"],"temp":[21.5,19.0,24.0,26.5,17.0,"NaN",null],"count":[3,1,null,7,2,5,4]}}},{"id":"project/rename","description":"rename columns","inputs":{"left":{"fields":[{"name":"station","type":"utf8"},{"name":"city","type":"utf8"},{"name":"temp","type":"f64"},{"name":"rain","type":"f64"},{"name":"count","type":"i32"},{"name":"id","type":"i64"}],"columns":{"station":["s1","s2","s3","s4","s5","s6","s7"],"city":["Vienna","Vienna","Berlin","Berlin","Zürich",null,"Vienna"],"temp":[21.5,19.0,24.0,26.5,17.0,"NaN",null],"rain":[0.0,null,3.5,1.0,7.0,2.0,0.5],"count":[3,1,null,7,2,5,4],"id":["10","20","30","40","9007199254740993","-1","0"]}}},"ops":[{"op":"rename","mapping":{"temp":"celsius","city":"town"}}],"expected":{"fields":[{"name":"station","type":"utf8"},{"name":"town","type":"utf8"},{"name":"celsius","type":"f64"},{"name":"rain","type":"f64"},{"name":"count","type":"i32"},{"name":"id","type":"i64"}],"columns":{"station":["s1","s2","s3","s4","s5","s6","s7"],"town":["Vienna","Vienna","Berlin","Berlin","Zürich",null,"Vienna"],"celsius":[21.5,19.0,24.0,26.5,17.0,"NaN",null],"rain":[0.0,null,3.5,1.0,7.0,2.0,0.5],"count":[3,1,null,7,2,5,4],"id":["10","20","30","40","9007199254740993","-1","0"]}}},{"id":"project/with-replace","description":"withColumns replaces an existing column in place","inputs":{"left":{"fields":[{"name":"station","type":"utf8"},{"name":"city","type":"utf8"},{"name":"temp","type":"f64"},{"name":"rain","type":"f64"},{"name":"count","type":"i32"},{"name":"id","type":"i64"}],"columns":{"station":["s1","s2","s3","s4","s5","s6","s7"],"city":["Vienna","Vienna","Berlin","Berlin","Zürich",null,"Vienna"],"temp":[21.5,19.0,24.0,26.5,17.0,"NaN",null],"rain":[0.0,null,3.5,1.0,7.0,2.0,0.5],"count":[3,1,null,7,2,5,4],"id":["10","20","30","40","9007199254740993","-1","0"]}}},"ops":[{"op":"withColumns","exprs":{"temp":{"fn":"mul","args":[{"col":"temp"},{"lit":2.0}]},"f":{"fn":"add","args":[{"fn":"mul","args":[{"col":"temp"},{"lit":1.8}]},{"lit":32.0}]}}}],"expected":{"fields":[{"name":"station","type":"utf8"},{"name":"city","type":"utf8"},{"name":"temp","type":"f64"},{"name":"rain","type":"f64"},{"name":"count","type":"i32"},{"name":"id","type":"i64"},{"name":"f","type":"f64"}],"columns":{"station":["s1","s2","s3","s4","s5","s6","s7"],"city":["Vienna","Vienna","Berlin","Berlin","Zürich",null,"Vienna"],"temp":[43.0,38.0,48.0,53.0,34.0,"NaN",null],"rain":[0.0,null,3.5,1.0,7.0,2.0,0.5],"count":[3,1,null,7,2,5,4],"id":["10","20","30","40","9007199254740993","-1","0"],"f":[70.7,66.2,75.2,79.7,62.6,"NaN",null]}}},{"id":"project/literal-column","description":"a literal broadcasts to every row","inputs":{"left":{"fields":[{"name":"station","type":"utf8"},{"name":"city","type":"utf8"},{"name":"temp","type":"f64"},{"name":"rain","type":"f64"},{"name":"count","type":"i32"},{"name":"id","type":"i64"}],"columns":{"station":["s1","s2","s3","s4","s5","s6","s7"],"city":["Vienna","Vienna","Berlin","Berlin","Zürich",null,"Vienna"],"temp":[21.5,19.0,24.0,26.5,17.0,"NaN",null],"rain":[0.0,null,3.5,1.0,7.0,2.0,0.5],"count":[3,1,null,7,2,5,4],"id":["10","20","30","40","9007199254740993","-1","0"]}}},"ops":[{"op":"select","exprs":{"station":{"col":"station"},"half":{"lit":0.5},"one":{"lit":1},"tag":{"lit":"x"}}}],"expected":{"fields":[{"name":"station","type":"utf8"},{"name":"half","type":"f64"},{"name":"one","type":"i32"},{"name":"tag","type":"utf8"}],"columns":{"station":["s1","s2","s3","s4","s5","s6","s7"],"half":[0.5,0.5,0.5,0.5,0.5,0.5,0.5],"one":[1,1,1,1,1,1,1],"tag":["x","x","x","x","x","x","x"]}}},{"id":"sort/asc","description":"ascending, nulls first, NaN last among numbers","inputs":{"left":{"fields":[{"name":"station","type":"utf8"},{"name":"city","type":"utf8"},{"name":"temp","type":"f64"},{"name":"rain","type":"f64"},{"name":"count","type":"i32"},{"name":"id","type":"i64"}],"columns":{"station":["s1","s2","s3","s4","s5","s6","s7"],"city":["Vienna","Vienna","Berlin","Berlin","Zürich",null,"Vienna"],"temp":[21.5,19.0,24.0,26.5,17.0,"NaN",null],"rain":[0.0,null,3.5,1.0,7.0,2.0,0.5],"count":[3,1,null,7,2,5,4],"id":["10","20","30","40","9007199254740993","-1","0"]}}},"ops":[{"op":"sort","keys":[{"expr":{"col":"temp"}}]}],"expected":{"fields":[{"name":"station","type":"utf8"},{"name":"city","type":"utf8"},{"name":"temp","type":"f64"},{"name":"rain","type":"f64"},{"name":"count","type":"i32"},{"name":"id","type":"i64"}],"columns":{"station":["s7","s5","s2","s1","s3","s4","s6"],"city":["Vienna","Zürich","Vienna","Vienna","Berlin","Berlin",null],"temp":[null,17.0,19.0,21.5,24.0,26.5,"NaN"],"rain":[0.5,7.0,null,0.0,3.5,1.0,2.0],"count":[4,2,1,3,null,7,5],"id":["0","9007199254740993","20","10","30","40","-1"]}}},{"id":"sort/desc-nulls-last","description":"descending with nulls last","inputs":{"left":{"fields":[{"name":"station","type":"utf8"},{"name":"city","type":"utf8"},{"name":"temp","type":"f64"},{"name":"rain","type":"f64"},{"name":"count","type":"i32"},{"name":"id","type":"i64"}],"columns":{"station":["s1","s2","s3","s4","s5","s6","s7"],"city":["Vienna","Vienna","Berlin","Berlin","Zürich",null,"Vienna"],"temp":[21.5,19.0,24.0,26.5,17.0,"NaN",null],"rain":[0.0,null,3.5,1.0,7.0,2.0,0.5],"count":[3,1,null,7,2,5,4],"id":["10","20","30","40","9007199254740993","-1","0"]}}},"ops":[{"op":"sort","keys":[{"expr":{"col":"temp"},"descending":true}],"nullsLast":true}],"expected":{"fields":[{"name":"station","type":"utf8"},{"name":"city","type":"utf8"},{"name":"temp","type":"f64"},{"name":"rain","type":"f64"},{"name":"count","type":"i32"},{"name":"id","type":"i64"}],"columns":{"station":["s6","s4","s3","s1","s2","s5","s7"],"city":[null,"Berlin","Berlin","Vienna","Vienna","Zürich","Vienna"],"temp":["NaN",26.5,24.0,21.5,19.0,17.0,null],"rain":[2.0,1.0,3.5,0.0,null,7.0,0.5],"count":[5,7,null,3,1,2,4],"id":["-1","40","30","10","20","9007199254740993","0"]}}},{"id":"sort/multi","description":"multi-key sort is stable","inputs":{"left":{"fields":[{"name":"station","type":"utf8"},{"name":"city","type":"utf8"},{"name":"temp","type":"f64"},{"name":"rain","type":"f64"},{"name":"count","type":"i32"},{"name":"id","type":"i64"}],"columns":{"station":["s1","s2","s3","s4","s5","s6","s7"],"city":["Vienna","Vienna","Berlin","Berlin","Zürich",null,"Vienna"],"temp":[21.5,19.0,24.0,26.5,17.0,"NaN",null],"rain":[0.0,null,3.5,1.0,7.0,2.0,0.5],"count":[3,1,null,7,2,5,4],"id":["10","20","30","40","9007199254740993","-1","0"]}}},"ops":[{"op":"sort","keys":[{"expr":{"col":"city"}},{"expr":{"col":"count"},"descending":true}]}],"expected":{"fields":[{"name":"station","type":"utf8"},{"name":"city","type":"utf8"},{"name":"temp","type":"f64"},{"name":"rain","type":"f64"},{"name":"count","type":"i32"},{"name":"id","type":"i64"}],"columns":{"station":["s6","s3","s4","s7","s1","s2","s5"],"city":[null,"Berlin","Berlin","Vienna","Vienna","Vienna","Zürich"],"temp":["NaN",24.0,26.5,null,21.5,19.0,17.0],"rain":[2.0,3.5,1.0,0.5,0.0,null,7.0],"count":[5,null,7,4,3,1,2],"id":["-1","30","40","0","10","20","9007199254740993"]}}},{"id":"sort/strings","description":"string sort by code point","inputs":{"left":{"fields":[{"name":"a","type":"f64"},{"name":"b","type":"f64"},{"name":"i","type":"i32"},{"name":"j","type":"i32"},{"name":"s","type":"utf8"},{"name":"t","type":"utf8"},{"name":"p","type":"bool"},{"name":"q","type":"bool"}],"columns":{"a":[1.0,"-0",0.0,"NaN","Infinity","-Infinity",null,2.5,-2.5,1e+300],"b":[2.0,0.0,"-0","NaN",1.0,null,3.0,0.0,2.0,1e+300],"i":[7,-7,0,2147483647,-2147483648,null,3,-3,100,5],"j":[2,2,3,1,-1,4,null,2,-7,0],"s":["b","a","","é","😀","￿",null,"B","ab","a\u0000b"],"t":["b","A","x","É","😀","z","n",null,"AB","a"],"p":[true,true,true,false,false,false,null,null,null,true],"q":[true,false,null,true,false,null,true,false,null,true]}}},"ops":[{"op":"sort","keys":[{"expr":{"col":"s"}}]}],"expected":{"fields":[{"name":"a","type":"f64"},{"name":"b","type":"f64"},{"name":"i","type":"i32"},{"name":"j","type":"i32"},{"name":"s","type":"utf8"},{"name":"t","type":"utf8"},{"name":"p","type":"bool"},{"name":"q","type":"bool"}],"columns":{"a":[null,0.0,2.5,"-0",1e+300,-2.5,1.0,"NaN","-Infinity","Infinity"],"b":[3.0,"-0",0.0,0.0,1e+300,2.0,2.0,"NaN",null,1.0],"i":[3,0,-3,-7,5,100,7,2147483647,null,-2147483648],"j":[null,3,2,2,0,-7,2,1,4,-1],"s":[null,"","B","a","a\u0000b","ab","b","é","￿","😀"],"t":["n","x",null,"A","a","AB","b","É","z","😀"],"p":[null,true,null,true,true,null,true,false,false,false],"q":[true,null,false,false,true,null,true,true,null,false]}}},{"id":"sort/zeros","description":"-0 and 0 sort as equal (stable)","inputs":{"left":{"fields":[{"name":"a","type":"f64"},{"name":"b","type":"f64"},{"name":"i","type":"i32"},{"name":"j","type":"i32"},{"name":"s","type":"utf8"},{"name":"t","type":"utf8"},{"name":"p","type":"bool"},{"name":"q","type":"bool"}],"columns":{"a":[1.0,"-0",0.0,"NaN","Infinity","-Infinity",null,2.5,-2.5,1e+300],"b":[2.0,0.0,"-0","NaN",1.0,null,3.0,0.0,2.0,1e+300],"i":[7,-7,0,2147483647,-2147483648,null,3,-3,100,5],"j":[2,2,3,1,-1,4,null,2,-7,0],"s":["b","a","","é","😀","￿",null,"B","ab","a\u0000b"],"t":["b","A","x","É","😀","z","n",null,"AB","a"],"p":[true,true,true,false,false,false,null,null,null,true],"q":[true,false,null,true,false,null,true,false,null,true]}}},"ops":[{"op":"sort","keys":[{"expr":{"col":"a"}}]}],"expected":{"fields":[{"name":"a","type":"f64"},{"name":"b","type":"f64"},{"name":"i","type":"i32"},{"name":"j","type":"i32"},{"name":"s","type":"utf8"},{"name":"t","type":"utf8"},{"name":"p","type":"bool"},{"name":"q","type":"bool"}],"columns":{"a":[null,"-Infinity",-2.5,"-0",0.0,1.0,2.5,1e+300,"Infinity","NaN"],"b":[3.0,null,2.0,0.0,"-0",2.0,0.0,1e+300,1.0,"NaN"],"i":[3,null,100,-7,0,7,-3,5,-2147483648,2147483647],"j":[null,4,-7,2,3,2,2,0,-1,1],"s":[null,"￿","ab","a","","b","B","a\u0000b","😀","é"],"t":["n","z","AB","A","x","b",null,"a","😀","É"],"p":[null,false,null,true,true,true,null,true,false,false],"q":[true,null,null,false,null,true,false,true,false,true]}}},{"id":"sort/i64","description":"Int64 sort","inputs":{"left":{"fields":[{"name":"station","type":"utf8"},{"name":"city","type":"utf8"},{"name":"temp","type":"f64"},{"name":"rain","type":"f64"},{"name":"count","type":"i32"},{"name":"id","type":"i64"}],"columns":{"station":["s1","s2","s3","s4","s5","s6","s7"],"city":["Vienna","Vienna","Berlin","Berlin","Zürich",null,"Vienna"],"temp":[21.5,19.0,24.0,26.5,17.0,"NaN",null],"rain":[0.0,null,3.5,1.0,7.0,2.0,0.5],"count":[3,1,null,7,2,5,4],"id":["10","20","30","40","9007199254740993","-1","0"]}}},"ops":[{"op":"sort","keys":[{"expr":{"col":"id"}}]}],"expected":{"fields":[{"name":"station","type":"utf8"},{"name":"city","type":"utf8"},{"name":"temp","type":"f64"},{"name":"rain","type":"f64"},{"name":"count","type":"i32"},{"name":"id","type":"i64"}],"columns":{"station":["s6","s7","s1","s2","s3","s4","s5"],"city":[null,"Vienna","Vienna","Vienna","Berlin","Berlin","Zürich"],"temp":["NaN",null,21.5,19.0,24.0,26.5,17.0],"rain":[2.0,0.5,0.0,null,3.5,1.0,7.0],"count":[5,4,3,1,null,7,2],"id":["-1","0","10","20","30","40","9007199254740993"]}}},{"id":"sort/expr","description":"sort by an expression","inputs":{"left":{"fields":[{"name":"station","type":"utf8"},{"name":"city","type":"utf8"},{"name":"temp","type":"f64"},{"name":"rain","type":"f64"},{"name":"count","type":"i32"},{"name":"id","type":"i64"}],"columns":{"station":["s1","s2","s3","s4","s5","s6","s7"],"city":["Vienna","Vienna","Berlin","Berlin","Zürich",null,"Vienna"],"temp":[21.5,19.0,24.0,26.5,17.0,"NaN",null],"rain":[0.0,null,3.5,1.0,7.0,2.0,0.5],"count":[3,1,null,7,2,5,4],"id":["10","20","30","40","9007199254740993","-1","0"]}}},"ops":[{"op":"sort","keys":[{"expr":{"fn":"neg","args":[{"col":"count"}]}}]}],"expected":{"fields":[{"name":"station","type":"utf8"},{"name":"city","type":"utf8"},{"name":"temp","type":"f64"},{"name":"rain","type":"f64"},{"name":"count","type":"i32"},{"name":"id","type":"i64"}],"columns":{"station":["s3","s4","s6","s7","s1","s5","s2"],"city":["Berlin","Berlin",null,"Vienna","Vienna","Zürich","Vienna"],"temp":[24.0,26.5,"NaN",null,21.5,17.0,19.0],"rain":[3.5,1.0,2.0,0.5,0.0,7.0,null],"count":[null,7,5,4,3,2,1],"id":["30","40","-1","0","10","9007199254740993","20"]}}},{"id":"sort/bool","description":"boolean sort","inputs":{"left":{"fields":[{"name":"a","type":"f64"},{"name":"b","type":"f64"},{"name":"i","type":"i32"},{"name":"j","type":"i32"},{"name":"s","type":"utf8"},{"name":"t","type":"utf8"},{"name":"p","type":"bool"},{"name":"q","type":"bool"}],"columns":{"a":[1.0,"-0",0.0,"NaN","Infinity","-Infinity",null,2.5,-2.5,1e+300],"b":[2.0,0.0,"-0","NaN",1.0,null,3.0,0.0,2.0,1e+300],"i":[7,-7,0,2147483647,-2147483648,null,3,-3,100,5],"j":[2,2,3,1,-1,4,null,2,-7,0],"s":["b","a","","é","😀","￿",null,"B","ab","a\u0000b"],"t":["b","A","x","É","😀","z","n",null,"AB","a"],"p":[true,true,true,false,false,false,null,null,null,true],"q":[true,false,null,true,false,null,true,false,null,true]}}},"ops":[{"op":"sort","keys":[{"expr":{"col":"p"}},{"expr":{"col":"q"},"descending":true}]}],"expected":{"fields":[{"name":"a","type":"f64"},{"name":"b","type":"f64"},{"name":"i","type":"i32"},{"name":"j","type":"i32"},{"name":"s","type":"utf8"},{"name":"t","type":"utf8"},{"name":"p","type":"bool"},{"name":"q","type":"bool"}],"columns":{"a":[-2.5,null,2.5,"-Infinity","NaN","Infinity",0.0,1.0,1e+300,"-0"],"b":[2.0,3.0,0.0,null,"NaN",1.0,"-0",2.0,1e+300,0.0],"i":[100,3,-3,null,2147483647,-2147483648,0,7,5,-7],"j":[-7,null,2,4,1,-1,3,2,0,2],"s":["ab",null,"B","￿","é","😀","","b","a\u0000b","a"],"t":["AB","n",null,"z","É","😀","x","b","a","A"],"p":[null,null,null,false,false,false,true,true,true,true],"q":[null,true,false,null,true,false,null,true,true,false]}}},{"id":"slice/head","description":"first rows","inputs":{"left":{"fields":[{"name":"station","type":"utf8"},{"name":"city","type":"utf8"},{"name":"temp","type":"f64"},{"name":"rain","type":"f64"},{"name":"count","type":"i32"},{"name":"id","type":"i64"}],"columns":{"station":["s1","s2","s3","s4","s5","s6","s7"],"city":["Vienna","Vienna","Berlin","Berlin","Zürich",null,"Vienna"],"temp":[21.5,19.0,24.0,26.5,17.0,"NaN",null],"rain":[0.0,null,3.5,1.0,7.0,2.0,0.5],"count":[3,1,null,7,2,5,4],"id":["10","20","30","40","9007199254740993","-1","0"]}}},"ops":[{"op":"head","n":3}],"expected":{"fields":[{"name":"station","type":"utf8"},{"name":"city","type":"utf8"},{"name":"temp","type":"f64"},{"name":"rain","type":"f64"},{"name":"count","type":"i32"},{"name":"id","type":"i64"}],"columns":{"station":["s1","s2","s3"],"city":["Vienna","Vienna","Berlin"],"temp":[21.5,19.0,24.0],"rain":[0.0,null,3.5],"count":[3,1,null],"id":["10","20","30"]}}},{"id":"slice/tail","description":"last rows","inputs":{"left":{"fields":[{"name":"station","type":"utf8"},{"name":"city","type":"utf8"},{"name":"temp","type":"f64"},{"name":"rain","type":"f64"},{"name":"count","type":"i32"},{"name":"id","type":"i64"}],"columns":{"station":["s1","s2","s3","s4","s5","s6","s7"],"city":["Vienna","Vienna","Berlin","Berlin","Zürich",null,"Vienna"],"temp":[21.5,19.0,24.0,26.5,17.0,"NaN",null],"rain":[0.0,null,3.5,1.0,7.0,2.0,0.5],"count":[3,1,null,7,2,5,4],"id":["10","20","30","40","9007199254740993","-1","0"]}}},"ops":[{"op":"tail","n":2}],"expected":{"fields":[{"name":"station","type":"utf8"},{"name":"city","type":"utf8"},{"name":"temp","type":"f64"},{"name":"rain","type":"f64"},{"name":"count","type":"i32"},{"name":"id","type":"i64"}],"columns":{"station":["s6","s7"],"city":[null,"Vienna"],"temp":["NaN",null],"rain":[2.0,0.5],"count":[5,4],"id":["-1","0"]}}},{"id":"slice/offset","description":"slice in the middle","inputs":{"left":{"fields":[{"name":"station","type":"utf8"},{"name":"city","type":"utf8"},{"name":"temp","type":"f64"},{"name":"rain","type":"f64"},{"name":"count","type":"i32"},{"name":"id","type":"i64"}],"columns":{"station":["s1","s2","s3","s4","s5","s6","s7"],"city":["Vienna","Vienna","Berlin","Berlin","Zürich",null,"Vienna"],"temp":[21.5,19.0,24.0,26.5,17.0,"NaN",null],"rain":[0.0,null,3.5,1.0,7.0,2.0,0.5],"count":[3,1,null,7,2,5,4],"id":["10","20","30","40","9007199254740993","-1","0"]}}},"ops":[{"op":"slice","offset":2,"length":3}],"expected":{"fields":[{"name":"station","type":"utf8"},{"name":"city","type":"utf8"},{"name":"temp","type":"f64"},{"name":"rain","type":"f64"},{"name":"count","type":"i32"},{"name":"id","type":"i64"}],"columns":{"station":["s3","s4","s5"],"city":["Berlin","Berlin","Zürich"],"temp":[24.0,26.5,17.0],"rain":[3.5,1.0,7.0],"count":[null,7,2],"id":["30","40","9007199254740993"]}}},{"id":"slice/after-filter","description":"slicing after a filter","inputs":{"left":{"fields":[{"name":"station","type":"utf8"},{"name":"city","type":"utf8"},{"name":"temp","type":"f64"},{"name":"rain","type":"f64"},{"name":"count","type":"i32"},{"name":"id","type":"i64"}],"columns":{"station":["s1","s2","s3","s4","s5","s6","s7"],"city":["Vienna","Vienna","Berlin","Berlin","Zürich",null,"Vienna"],"temp":[21.5,19.0,24.0,26.5,17.0,"NaN",null],"rain":[0.0,null,3.5,1.0,7.0,2.0,0.5],"count":[3,1,null,7,2,5,4],"id":["10","20","30","40","9007199254740993","-1","0"]}}},"ops":[{"op":"filter","predicate":{"fn":"isNotNull","args":[{"col":"city"}]}},{"op":"slice","offset":1,"length":2}],"expected":{"fields":[{"name":"station","type":"utf8"},{"name":"city","type":"utf8"},{"name":"temp","type":"f64"},{"name":"rain","type":"f64"},{"name":"count","type":"i32"},{"name":"id","type":"i64"}],"columns":{"station":["s2","s3"],"city":["Vienna","Berlin"],"temp":[19.0,24.0],"rain":[null,3.5],"count":[1,null],"id":["20","30"]}}},{"id":"agg/global","description":"aggregates over the whole frame","inputs":{"left":{"fields":[{"name":"station","type":"utf8"},{"name":"city","type":"utf8"},{"name":"temp","type":"f64"},{"name":"rain","type":"f64"},{"name":"count","type":"i32"},{"name":"id","type":"i64"}],"columns":{"station":["s1","s2","s3","s4","s5","s6","s7"],"city":["Vienna","Vienna","Berlin","Berlin","Zürich",null,"Vienna"],"temp":[21.5,19.0,24.0,26.5,17.0,"NaN",null],"rain":[0.0,null,3.5,1.0,7.0,2.0,0.5],"count":[3,1,null,7,2,5,4],"id":["10","20","30","40","9007199254740993","-1","0"]}}},"ops":[{"op":"agg","aggs":{"n":{"agg":"len"},"c":{"agg":"count","arg":{"col":"temp"}},"sum":{"agg":"sum","arg":{"col":"rain"}},"mean":{"agg":"mean","arg":{"col":"rain"}},"min":{"agg":"min","arg":{"col":"count"}},"max":{"agg":"max","arg":{"col":"count"}},"first":{"agg":"first","arg":{"col":"city"}},"last":{"agg":"last","arg":{"col":"city"}}}}],"expected":{"fields":[{"name":"n","type":"u32"},{"name":"c","type":"u32"},{"name":"sum","type":"f64"},{"name":"mean","type":"f64"},{"name":"min","type":"i32"},{"name":"max","type":"i32"},{"name":"first","type":"utf8"},{"name":"last","type":"utf8"}],"columns":{"n":[7],"c":[6],"sum":[14.0],"mean":[2.3333333333333335],"min":[1],"max":[7],"first":["Vienna"],"last":["Vienna"]}}},{"id":"agg/nan","description":"sum includes NaN; min and max skip it","inputs":{"left":{"fields":[{"name":"station","type":"utf8"},{"name":"city","type":"utf8"},{"name":"temp","type":"f64"},{"name":"rain","type":"f64"},{"name":"count","type":"i32"},{"name":"id","type":"i64"}],"columns":{"station":["s1","s2","s3","s4","s5","s6","s7"],"city":["Vienna","Vienna","Berlin","Berlin","Zürich",null,"Vienna"],"temp":[21.5,19.0,24.0,26.5,17.0,"NaN",null],"rain":[0.0,null,3.5,1.0,7.0,2.0,0.5],"count":[3,1,null,7,2,5,4],"id":["10","20","30","40","9007199254740993","-1","0"]}}},"ops":[{"op":"agg","aggs":{"sum":{"agg":"sum","arg":{"col":"temp"}},"min":{"agg":"min","arg":{"col":"temp"}},"max":{"agg":"max","arg":{"col":"temp"}}}}],"expected":{"fields":[{"name":"sum","type":"f64"},{"name":"min","type":"f64"},{"name":"max","type":"f64"}],"columns":{"sum":["NaN"],"min":[17.0],"max":[26.5]}}},{"id":"agg/stats","description":"std, var and median","inputs":{"left":{"fields":[{"name":"station","type":"utf8"},{"name":"city","type":"utf8"},{"name":"temp","type":"f64"},{"name":"rain","type":"f64"},{"name":"count","type":"i32"},{"name":"id","type":"i64"}],"columns":{"station":["s1","s2","s3","s4","s5","s6","s7"],"city":["Vienna","Vienna","Berlin","Berlin","Zürich",null,"Vienna"],"temp":[21.5,19.0,24.0,26.5,17.0,"NaN",null],"rain":[0.0,null,3.5,1.0,7.0,2.0,0.5],"count":[3,1,null,7,2,5,4],"id":["10","20","30","40","9007199254740993","-1","0"]}}},"ops":[{"op":"agg","aggs":{"std":{"agg":"std","arg":{"col":"rain"}},"var":{"agg":"var","arg":{"col":"rain"}},"median":{"agg":"median","arg":{"col":"rain"}},"imed":{"agg":"median","arg":{"col":"count"}}}}],"expected":{"fields":[{"name":"std","type":"f64"},{"name":"var","type":"f64"},{"name":"median","type":"f64"},{"name":"imed","type":"f64"}],"columns":{"std":[2.6012817353502227],"var":[6.766666666666666],"median":[1.5],"imed":[3.5]}}},{"id":"agg/strings","description":"min, max and nUnique on strings","inputs":{"left":{"fields":[{"name":"station","type":"utf8"},{"name":"city","type":"utf8"},{"name":"temp","type":"f64"},{"name":"rain","type":"f64"},{"name":"count","type":"i32"},{"name":"id","type":"i64"}],"columns":{"station":["s1","s2","s3","s4","s5","s6","s7"],"city":["Vienna","Vienna","Berlin","Berlin","Zürich",null,"Vienna"],"temp":[21.5,19.0,24.0,26.5,17.0,"NaN",null],"rain":[0.0,null,3.5,1.0,7.0,2.0,0.5],"count":[3,1,null,7,2,5,4],"id":["10","20","30","40","9007199254740993","-1","0"]}}},"ops":[{"op":"agg","aggs":{"min":{"agg":"min","arg":{"col":"city"}},"max":{"agg":"max","arg":{"col":"city"}},"n":{"agg":"nUnique","arg":{"col":"city"}}}}],"expected":{"fields":[{"name":"min","type":"utf8"},{"name":"max","type":"utf8"},{"name":"n","type":"u32"}],"columns":{"min":["Berlin"],"max":["Zürich"],"n":[4]}}},{"id":"agg/empty","description":"aggregates over an empty frame","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"v","type":"f64"}],"columns":{"k":[],"v":[]}}},"ops":[{"op":"agg","aggs":{"n":{"agg":"len"},"sum":{"agg":"sum","arg":{"col":"v"}},"mean":{"agg":"mean","arg":{"col":"v"}},"min":{"agg":"min","arg":{"col":"v"}}}}],"expected":{"fields":[{"name":"n","type":"u32"},{"name":"sum","type":"f64"},{"name":"mean","type":"f64"},{"name":"min","type":"f64"}],"columns":{"n":[0],"sum":[0.0],"mean":[null],"min":[null]}}},{"id":"group/basic","description":"group by a string key in first-appearance order","inputs":{"left":{"fields":[{"name":"station","type":"utf8"},{"name":"city","type":"utf8"},{"name":"temp","type":"f64"},{"name":"rain","type":"f64"},{"name":"count","type":"i32"},{"name":"id","type":"i64"}],"columns":{"station":["s1","s2","s3","s4","s5","s6","s7"],"city":["Vienna","Vienna","Berlin","Berlin","Zürich",null,"Vienna"],"temp":[21.5,19.0,24.0,26.5,17.0,"NaN",null],"rain":[0.0,null,3.5,1.0,7.0,2.0,0.5],"count":[3,1,null,7,2,5,4],"id":["10","20","30","40","9007199254740993","-1","0"]}}},"ops":[{"op":"groupBy","keys":["city"],"aggs":{"n":{"agg":"len"},"temp":{"agg":"mean","arg":{"col":"temp"}},"rain":{"agg":"sum","arg":{"col":"rain"}}}}],"expected":{"fields":[{"name":"city","type":"utf8"},{"name":"n","type":"u32"},{"name":"temp","type":"f64"},{"name":"rain","type":"f64"}],"columns":{"city":["Vienna","Berlin","Zürich",null],"n":[3,2,1,1],"temp":[20.25,25.25,17.0,"NaN"],"rain":[0.5,4.5,7.0,2.0]}}},{"id":"group/null-key","description":"null is its own group","inputs":{"left":{"fields":[{"name":"station","type":"utf8"},{"name":"city","type":"utf8"},{"name":"temp","type":"f64"},{"name":"rain","type":"f64"},{"name":"count","type":"i32"},{"name":"id","type":"i64"}],"columns":{"station":["s1","s2","s3","s4","s5","s6","s7"],"city":["Vienna","Vienna","Berlin","Berlin","Zürich",null,"Vienna"],"temp":[21.5,19.0,24.0,26.5,17.0,"NaN",null],"rain":[0.0,null,3.5,1.0,7.0,2.0,0.5],"count":[3,1,null,7,2,5,4],"id":["10","20","30","40","9007199254740993","-1","0"]}}},"ops":[{"op":"groupBy","keys":["city"],"aggs":{"stations":{"agg":"count","arg":{"col":"station"}}}}],"expected":{"fields":[{"name":"city","type":"utf8"},{"name":"stations","type":"u32"}],"columns":{"city":["Vienna","Berlin","Zürich",null],"stations":[3,2,1,1]}}},{"id":"group/multi-key","description":"group by two keys","inputs":{"left":{"fields":[{"name":"station","type":"utf8"},{"name":"city","type":"utf8"},{"name":"temp","type":"f64"},{"name":"rain","type":"f64"},{"name":"count","type":"i32"},{"name":"id","type":"i64"}],"columns":{"station":["s1","s2","s3","s4","s5","s6","s7"],"city":["Vienna","Vienna","Berlin","Berlin","Zürich",null,"Vienna"],"temp":[21.5,19.0,24.0,26.5,17.0,"NaN",null],"rain":[0.0,null,3.5,1.0,7.0,2.0,0.5],"count":[3,1,null,7,2,5,4],"id":["10","20","30","40","9007199254740993","-1","0"]}}},"ops":[{"op":"withColumns","exprs":{"wet":{"fn":"gt","args":[{"col":"rain"},{"lit":1.0}]}}},{"op":"groupBy","keys":["city","wet"],"aggs":{"n":{"agg":"len"},"max":{"agg":"max","arg":{"col":"temp"}}}}],"expected":{"fields":[{"name":"city","type":"utf8"},{"name":"wet","type":"bool"},{"name":"n","type":"u32"},{"name":"max","type":"f64"}],"columns":{"city":["Vienna","Vienna","Berlin","Berlin","Zürich",null],"wet":[false,null,true,false,true,true],"n":[2,1,1,1,1,1],"max":[21.5,19.0,24.0,26.5,17.0,"NaN"]}}},{"id":"group/all-null","description":"aggregates over all-null groups","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"v","type":"f64"},{"name":"i","type":"i32"}],"columns":{"k":["x","x","y"],"v":[null,null,1.0],"i":[null,null,null]}}},"ops":[{"op":"groupBy","keys":["k"],"aggs":{"sum":{"agg":"sum","arg":{"col":"v"}},"mean":{"agg":"mean","arg":{"col":"v"}},"count":{"agg":"count","arg":{"col":"v"}},"n":{"agg":"len"},"min":{"agg":"min","arg":{"col":"v"}},"std":{"agg":"std","arg":{"col":"v"}},"nu":{"agg":"nUnique","arg":{"col":"v"}},"first":{"agg":"first","arg":{"col":"v"}},"isum":{"agg":"sum","arg":{"col":"i"}},"imax":{"agg":"max","arg":{"col":"i"}}}}],"expected":{"fields":[{"name":"k","type":"utf8"},{"name":"sum","type":"f64"},{"name":"mean","type":"f64"},{"name":"count","type":"u32"},{"name":"n","type":"u32"},{"name":"min","type":"f64"},{"name":"std","type":"f64"},{"name":"nu","type":"u32"},{"name":"first","type":"f64"},{"name":"isum","type":"i32"},{"name":"imax","type":"i32"}],"columns":{"k":["x","y"],"sum":[0.0,1.0],"mean":[null,1.0],"count":[0,1],"n":[2,1],"min":[null,1.0],"std":[null,null],"nu":[1,1],"first":[null,1.0],"isum":[0,0],"imax":[null,null]}}},{"id":"group/float-key","description":"NaN, -0 and 0 as group keys","inputs":{"left":{"fields":[{"name":"a","type":"f64"},{"name":"b","type":"f64"},{"name":"i","type":"i32"},{"name":"j","type":"i32"},{"name":"s","type":"utf8"},{"name":"t","type":"utf8"},{"name":"p","type":"bool"},{"name":"q","type":"bool"}],"columns":{"a":[1.0,"-0",0.0,"NaN","Infinity","-Infinity",null,2.5,-2.5,1e+300],"b":[2.0,0.0,"-0","NaN",1.0,null,3.0,0.0,2.0,1e+300],"i":[7,-7,0,2147483647,-2147483648,null,3,-3,100,5],"j":[2,2,3,1,-1,4,null,2,-7,0],"s":["b","a","","é","😀","￿",null,"B","ab","a\u0000b"],"t":["b","A","x","É","😀","z","n",null,"AB","a"],"p":[true,true,true,false,false,false,null,null,null,true],"q":[true,false,null,true,false,null,true,false,null,true]}}},"ops":[{"op":"groupBy","keys":["a"],"aggs":{"n":{"agg":"len"}}}],"expected":{"fields":[{"name":"a","type":"f64"},{"name":"n","type":"u32"}],"columns":{"a":[1.0,"-0","NaN","Infinity","-Infinity",null,2.5,-2.5,1e+300],"n":[1,2,1,1,1,1,1,1,1]}}},{"id":"group/i64-key","description":"Int64 group keys and sums","inputs":{"left":{"fields":[{"name":"station","type":"utf8"},{"name":"city","type":"utf8"},{"name":"temp","type":"f64"},{"name":"rain","type":"f64"},{"name":"count","type":"i32"},{"name":"id","type":"i64"}],"columns":{"station":["s1","s2","s3","s4","s5","s6","s7"],"city":["Vienna","Vienna","Berlin","Berlin","Zürich",null,"Vienna"],"temp":[21.5,19.0,24.0,26.5,17.0,"NaN",null],"rain":[0.0,null,3.5,1.0,7.0,2.0,0.5],"count":[3,1,null,7,2,5,4],"id":["10","20","30","40","9007199254740993","-1","0"]}}},"ops":[{"op":"withColumns","exprs":{"k":{"fn":"mod","args":[{"col":"id"},{"lit":"3","type":"i64"}]}}},{"op":"groupBy","keys":["k"],"aggs":{"n":{"agg":"len"},"s":{"agg":"sum","arg":{"col":"id"}},"mx":{"agg":"max","arg":{"col":"id"}}}}],"expected":{"fields":[{"name":"k","type":"i64"},{"name":"n","type":"u32"},{"name":"s","type":"i64"},{"name":"mx","type":"i64"}],"columns":{"k":["1","2","0"],"n":[2,2,3],"s":["50","19","9007199254741023"],"mx":["40","20","9007199254740993"]}}},{"id":"group/bool-key","description":"boolean group keys","inputs":{"left":{"fields":[{"name":"a","type":"f64"},{"name":"b","type":"f64"},{"name":"i","type":"i32"},{"name":"j","type":"i32"},{"name":"s","type":"utf8"},{"name":"t","type":"utf8"},{"name":"p","type":"bool"},{"name":"q","type":"bool"}],"columns":{"a":[1.0,"-0",0.0,"NaN","Infinity","-Infinity",null,2.5,-2.5,1e+300],"b":[2.0,0.0,"-0","NaN",1.0,null,3.0,0.0,2.0,1e+300],"i":[7,-7,0,2147483647,-2147483648,null,3,-3,100,5],"j":[2,2,3,1,-1,4,null,2,-7,0],"s":["b","a","","é","😀","￿",null,"B","ab","a\u0000b"],"t":["b","A","x","É","😀","z","n",null,"AB","a"],"p":[true,true,true,false,false,false,null,null,null,true],"q":[true,false,null,true,false,null,true,false,null,true]}}},"ops":[{"op":"groupBy","keys":["p"],"aggs":{"n":{"agg":"len"},"s":{"agg":"sum","arg":{"col":"i"}}}}],"expected":{"fields":[{"name":"p","type":"bool"},{"name":"n","type":"u32"},{"name":"s","type":"i32"}],"columns":{"p":[true,false,null],"n":[4,3,3],"s":[5,-1,100]}}},{"id":"group/int-sum","description":"Int32 sums stay Int32; means are Float64","inputs":{"left":{"fields":[{"name":"a","type":"f64"},{"name":"b","type":"f64"},{"name":"i","type":"i32"},{"name":"j","type":"i32"},{"name":"s","type":"utf8"},{"name":"t","type":"utf8"},{"name":"p","type":"bool"},{"name":"q","type":"bool"}],"columns":{"a":[1.0,"-0",0.0,"NaN","Infinity","-Infinity",null,2.5,-2.5,1e+300],"b":[2.0,0.0,"-0","NaN",1.0,null,3.0,0.0,2.0,1e+300],"i":[7,-7,0,2147483647,-2147483648,null,3,-3,100,5],"j":[2,2,3,1,-1,4,null,2,-7,0],"s":["b","a","","é","😀","￿",null,"B","ab","a\u0000b"],"t":["b","A","x","É","😀","z","n",null,"AB","a"],"p":[true,true,true,false,false,false,null,null,null,true],"q":[true,false,null,true,false,null,true,false,null,true]}}},"ops":[{"op":"groupBy","keys":["p"],"aggs":{"s":{"agg":"sum","arg":{"col":"j"}},"m":{"agg":"mean","arg":{"col":"j"}},"mn":{"agg":"min","arg":{"col":"j"}}}}],"expected":{"fields":[{"name":"p","type":"bool"},{"name":"s","type":"i32"},{"name":"m","type":"f64"},{"name":"mn","type":"i32"}],"columns":{"p":[true,false,null],"s":[7,4,-5],"m":[1.75,1.3333333333333333,-2.5],"mn":[0,-1,-7]}}},{"id":"group/after-filter","description":"group a filtered frame","inputs":{"left":{"fields":[{"name":"station","type":"utf8"},{"name":"city","type":"utf8"},{"name":"temp","type":"f64"},{"name":"rain","type":"f64"},{"name":"count","type":"i32"},{"name":"id","type":"i64"}],"columns":{"station":["s1","s2","s3","s4","s5","s6","s7"],"city":["Vienna","Vienna","Berlin","Berlin","Zürich",null,"Vienna"],"temp":[21.5,19.0,24.0,26.5,17.0,"NaN",null],"rain":[0.0,null,3.5,1.0,7.0,2.0,0.5],"count":[3,1,null,7,2,5,4],"id":["10","20","30","40","9007199254740993","-1","0"]}}},"ops":[{"op":"filter","predicate":{"fn":"gt","args":[{"col":"rain"},{"lit":0.0}]}},{"op":"groupBy","keys":["city"],"aggs":{"n":{"agg":"len"},"last":{"agg":"last","arg":{"col":"station"}}}}],"expected":{"fields":[{"name":"city","type":"utf8"},{"name":"n","type":"u32"},{"name":"last","type":"utf8"}],"columns":{"city":["Berlin","Zürich",null,"Vienna"],"n":[2,1,1,1],"last":["s4","s5","s6","s7"]}}},{"id":"group/expression-agg","description":"aggregate of an expression","inputs":{"left":{"fields":[{"name":"station","type":"utf8"},{"name":"city","type":"utf8"},{"name":"temp","type":"f64"},{"name":"rain","type":"f64"},{"name":"count","type":"i32"},{"name":"id","type":"i64"}],"columns":{"station":["s1","s2","s3","s4","s5","s6","s7"],"city":["Vienna","Vienna","Berlin","Berlin","Zürich",null,"Vienna"],"temp":[21.5,19.0,24.0,26.5,17.0,"NaN",null],"rain":[0.0,null,3.5,1.0,7.0,2.0,0.5],"count":[3,1,null,7,2,5,4],"id":["10","20","30","40","9007199254740993","-1","0"]}}},"ops":[{"op":"groupBy","keys":["city"],"aggs":{"x":{"agg":"sum","arg":{"fn":"mul","args":[{"col":"rain"},{"lit":2.0}]}}}}],"expected":{"fields":[{"name":"city","type":"utf8"},{"name":"x","type":"f64"}],"columns":{"city":["Vienna","Berlin","Zürich",null],"x":[1.0,9.0,14.0,4.0]}}},{"id":"group/empty","description":"grouping an empty frame","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"v","type":"f64"}],"columns":{"k":[],"v":[]}}},"ops":[{"op":"groupBy","keys":["k"],"aggs":{"n":{"agg":"len"}}}],"expected":{"fields":[{"name":"k","type":"utf8"},{"name":"n","type":"u32"}],"columns":{"k":[],"n":[]}}},{"id":"group/small-ints","description":"sums of small integer types","inputs":{"left":{"fields":[{"name":"i8","type":"i8"},{"name":"u8","type":"u8"},{"name":"i16","type":"i16"},{"name":"u16","type":"u16"},{"name":"u32","type":"u32"},{"name":"i64","type":"i64"},{"name":"u64","type":"u64"},{"name":"f32","type":"f32"}],"columns":{"i8":[127,-128,0,null,5],"u8":[255,0,1,null,5],"i16":[32767,-32768,0,12,null],"u16":[65535,0,7,12,null],"u32":[4294967295,0,1,null,3],"i64":["9223372036854775807","-9223372036854775808","0",null,"42"],"u64":["18446744073709551615","0","1",null,"42"],"f32":[1.5,-0.25,null,3.0,10000000000.0]}}},"ops":[{"op":"agg","aggs":{"i8":{"agg":"sum","arg":{"col":"i8"}},"u8":{"agg":"sum","arg":{"col":"u8"}},"u32":{"agg":"sum","arg":{"col":"u32"}},"i64":{"agg":"sum","arg":{"col":"i64"}},"u64":{"agg":"max","arg":{"col":"u64"}},"f32":{"agg":"mean","arg":{"col":"f32"}}}}],"expected":{"fields":[{"name":"i8","type":"i64"},{"name":"u8","type":"i64"},{"name":"u32","type":"u32"},{"name":"i64","type":"i64"},{"name":"u64","type":"u64"},{"name":"f32","type":"f32"}],"columns":{"i8":["4"],"u8":["261"],"u32":[3],"i64":["41"],"u64":["18446744073709551615"],"f32":[2500000000.0]}}},{"id":"join/inner","description":"inner join with duplicate and null keys","inputs":{"left":{"fields":[{"name":"station","type":"utf8"},{"name":"city","type":"utf8"},{"name":"temp","type":"f64"},{"name":"rain","type":"f64"},{"name":"count","type":"i32"},{"name":"id","type":"i64"}],"columns":{"station":["s1","s2","s3","s4","s5","s6","s7"],"city":["Vienna","Vienna","Berlin","Berlin","Zürich",null,"Vienna"],"temp":[21.5,19.0,24.0,26.5,17.0,"NaN",null],"rain":[0.0,null,3.5,1.0,7.0,2.0,0.5],"count":[3,1,null,7,2,5,4],"id":["10","20","30","40","9007199254740993","-1","0"]}},"right":{"fields":[{"name":"station","type":"utf8"},{"name":"elevation","type":"f64"},{"name":"active","type":"bool"}],"columns":{"station":["s1","s3","s3","s9",null],"elevation":[190.0,34.0,35.0,500.0,1.0],"active":[true,false,null,true,true]}}},"ops":[{"op":"pick","columns":["station","temp"]},{"op":"join","right":"right","on":["station"],"how":"inner"}],"expected":{"fields":[{"name":"station","type":"utf8"},{"name":"temp","type":"f64"},{"name":"elevation","type":"f64"},{"name":"active","type":"bool"}],"columns":{"station":["s1","s3","s3"],"temp":[21.5,24.0,24.0],"elevation":[190.0,34.0,35.0],"active":[true,false,null]}}},{"id":"join/left","description":"left join with duplicate and null keys","inputs":{"left":{"fields":[{"name":"station","type":"utf8"},{"name":"city","type":"utf8"},{"name":"temp","type":"f64"},{"name":"rain","type":"f64"},{"name":"count","type":"i32"},{"name":"id","type":"i64"}],"columns":{"station":["s1","s2","s3","s4","s5","s6","s7"],"city":["Vienna","Vienna","Berlin","Berlin","Zürich",null,"Vienna"],"temp":[21.5,19.0,24.0,26.5,17.0,"NaN",null],"rain":[0.0,null,3.5,1.0,7.0,2.0,0.5],"count":[3,1,null,7,2,5,4],"id":["10","20","30","40","9007199254740993","-1","0"]}},"right":{"fields":[{"name":"station","type":"utf8"},{"name":"elevation","type":"f64"},{"name":"active","type":"bool"}],"columns":{"station":["s1","s3","s3","s9",null],"elevation":[190.0,34.0,35.0,500.0,1.0],"active":[true,false,null,true,true]}}},"ops":[{"op":"pick","columns":["station","temp"]},{"op":"join","right":"right","on":["station"],"how":"left"}],"expected":{"fields":[{"name":"station","type":"utf8"},{"name":"temp","type":"f64"},{"name":"elevation","type":"f64"},{"name":"active","type":"bool"}],"columns":{"station":["s1","s2","s3","s3","s4","s5","s6","s7"],"temp":[21.5,19.0,24.0,24.0,26.5,17.0,"NaN",null],"elevation":[190.0,null,34.0,35.0,null,null,null,null],"active":[true,null,false,null,null,null,null,null]}}},{"id":"join/full","description":"full join with duplicate and null keys","inputs":{"left":{"fields":[{"name":"station","type":"utf8"},{"name":"city","type":"utf8"},{"name":"temp","type":"f64"},{"name":"rain","type":"f64"},{"name":"count","type":"i32"},{"name":"id","type":"i64"}],"columns":{"station":["s1","s2","s3","s4","s5","s6","s7"],"city":["Vienna","Vienna","Berlin","Berlin","Zürich",null,"Vienna"],"temp":[21.5,19.0,24.0,26.5,17.0,"NaN",null],"rain":[0.0,null,3.5,1.0,7.0,2.0,0.5],"count":[3,1,null,7,2,5,4],"id":["10","20","30","40","9007199254740993","-1","0"]}},"right":{"fields":[{"name":"station","type":"utf8"},{"name":"elevation","type":"f64"},{"name":"active","type":"bool"}],"columns":{"station":["s1","s3","s3","s9",null],"elevation":[190.0,34.0,35.0,500.0,1.0],"active":[true,false,null,true,true]}}},"ops":[{"op":"pick","columns":["station","temp"]},{"op":"join","right":"right","on":["station"],"how":"full"}],"expected":{"fields":[{"name":"station","type":"utf8"},{"name":"temp","type":"f64"},{"name":"elevation","type":"f64"},{"name":"active","type":"bool"}],"columns":{"station":["s1","s2","s3","s3","s4","s5","s6","s7","s9",null],"temp":[21.5,19.0,24.0,24.0,26.5,17.0,"NaN",null,null,null],"elevation":[190.0,null,34.0,35.0,null,null,null,null,500.0,1.0],"active":[true,null,false,null,null,null,null,null,true,true]}}},{"id":"join/semi","description":"semi join with duplicate and null keys","inputs":{"left":{"fields":[{"name":"station","type":"utf8"},{"name":"city","type":"utf8"},{"name":"temp","type":"f64"},{"name":"rain","type":"f64"},{"name":"count","type":"i32"},{"name":"id","type":"i64"}],"columns":{"station":["s1","s2","s3","s4","s5","s6","s7"],"city":["Vienna","Vienna","Berlin","Berlin","Zürich",null,"Vienna"],"temp":[21.5,19.0,24.0,26.5,17.0,"NaN",null],"rain":[0.0,null,3.5,1.0,7.0,2.0,0.5],"count":[3,1,null,7,2,5,4],"id":["10","20","30","40","9007199254740993","-1","0"]}},"right":{"fields":[{"name":"station","type":"utf8"},{"name":"elevation","type":"f64"},{"name":"active","type":"bool"}],"columns":{"station":["s1","s3","s3","s9",null],"elevation":[190.0,34.0,35.0,500.0,1.0],"active":[true,false,null,true,true]}}},"ops":[{"op":"pick","columns":["station","temp"]},{"op":"join","right":"right","on":["station"],"how":"semi"}],"expected":{"fields":[{"name":"station","type":"utf8"},{"name":"temp","type":"f64"}],"columns":{"station":["s1","s3"],"temp":[21.5,24.0]}}},{"id":"join/anti","description":"anti join with duplicate and null keys","inputs":{"left":{"fields":[{"name":"station","type":"utf8"},{"name":"city","type":"utf8"},{"name":"temp","type":"f64"},{"name":"rain","type":"f64"},{"name":"count","type":"i32"},{"name":"id","type":"i64"}],"columns":{"station":["s1","s2","s3","s4","s5","s6","s7"],"city":["Vienna","Vienna","Berlin","Berlin","Zürich",null,"Vienna"],"temp":[21.5,19.0,24.0,26.5,17.0,"NaN",null],"rain":[0.0,null,3.5,1.0,7.0,2.0,0.5],"count":[3,1,null,7,2,5,4],"id":["10","20","30","40","9007199254740993","-1","0"]}},"right":{"fields":[{"name":"station","type":"utf8"},{"name":"elevation","type":"f64"},{"name":"active","type":"bool"}],"columns":{"station":["s1","s3","s3","s9",null],"elevation":[190.0,34.0,35.0,500.0,1.0],"active":[true,false,null,true,true]}}},"ops":[{"op":"pick","columns":["station","temp"]},{"op":"join","right":"right","on":["station"],"how":"anti"}],"expected":{"fields":[{"name":"station","type":"utf8"},{"name":"temp","type":"f64"}],"columns":{"station":["s2","s4","s5","s6","s7"],"temp":[19.0,26.5,17.0,"NaN",null]}}},{"id":"join/multi-key","description":"join on two keys","inputs":{"left":{"fields":[{"name":"a","type":"i32"},{"name":"b","type":"utf8"},{"name":"x","type":"f64"}],"columns":{"a":[1,1,2,2,null],"b":["x","y","x",null,"x"],"x":[1.0,2.0,3.0,4.0,5.0]}},"right":{"fields":[{"name":"a","type":"i32"},{"name":"b","type":"utf8"},{"name":"y","type":"bool"}],"columns":{"a":[1,2,2,1],"b":["y","x","x","z"],"y":[true,false,true,null]}}},"ops":[{"op":"join","right":"right","on":["a","b"],"how":"left"}],"expected":{"fields":[{"name":"a","type":"i32"},{"name":"b","type":"utf8"},{"name":"x","type":"f64"},{"name":"y","type":"bool"}],"columns":{"a":[1,1,2,2,2,null],"b":["x","y","x","x",null,"x"],"x":[1.0,2.0,3.0,3.0,4.0,5.0],"y":[null,true,false,true,null,null]}}},{"id":"join/i64-key","description":"join on Int64 keys","inputs":{"left":{"fields":[{"name":"k","type":"i64"},{"name":"x","type":"f64"}],"columns":{"k":["9007199254740993","9007199254740992","1"],"x":[1.0,2.0,3.0]}},"right":{"fields":[{"name":"k","type":"i64"},{"name":"y","type":"utf8"}],"columns":{"k":["9007199254740993","1","1"],"y":["a","b","c"]}}},"ops":[{"op":"join","right":"right","on":["k"],"how":"inner"}],"expected":{"fields":[{"name":"k","type":"i64"},{"name":"x","type":"f64"},{"name":"y","type":"utf8"}],"columns":{"k":["9007199254740993","1","1"],"x":[1.0,3.0,3.0],"y":["a","b","c"]}}},{"id":"join/f64-key","description":"join on Float64 keys: NaN matches NaN, -0 matches 0","inputs":{"left":{"fields":[{"name":"k","type":"f64"},{"name":"x","type":"i32"}],"columns":{"k":["NaN","-0",1.5,null],"x":[1,2,3,4]}},"right":{"fields":[{"name":"k","type":"f64"},{"name":"y","type":"i32"}],"columns":{"k":[0.0,"NaN",null,1.5],"y":[10,20,30,40]}}},"ops":[{"op":"join","right":"right","on":["k"],"how":"inner"}],"expected":{"fields":[{"name":"k","type":"f64"},{"name":"x","type":"i32"},{"name":"y","type":"i32"}],"columns":{"k":["NaN","-0",1.5],"x":[1,2,3],"y":[20,10,40]}}},{"id":"unique/all","description":"distinct rows, keeping the first","inputs":{"left":{"fields":[{"name":"a","type":"i32"},{"name":"b","type":"utf8"}],"columns":{"a":[1,1,2,null,null,1],"b":["x","x","y",null,null,"y"]}}},"ops":[{"op":"unique"}],"expected":{"fields":[{"name":"a","type":"i32"},{"name":"b","type":"utf8"}],"columns":{"a":[1,2,null,1],"b":["x","y",null,"y"]}}},{"id":"unique/subset","description":"distinct by a subset","inputs":{"left":{"fields":[{"name":"station","type":"utf8"},{"name":"city","type":"utf8"},{"name":"temp","type":"f64"},{"name":"rain","type":"f64"},{"name":"count","type":"i32"},{"name":"id","type":"i64"}],"columns":{"station":["s1","s2","s3","s4","s5","s6","s7"],"city":["Vienna","Vienna","Berlin","Berlin","Zürich",null,"Vienna"],"temp":[21.5,19.0,24.0,26.5,17.0,"NaN",null],"rain":[0.0,null,3.5,1.0,7.0,2.0,0.5],"count":[3,1,null,7,2,5,4],"id":["10","20","30","40","9007199254740993","-1","0"]}}},"ops":[{"op":"unique","columns":["city"]}],"expected":{"fields":[{"name":"station","type":"utf8"},{"name":"city","type":"utf8"},{"name":"temp","type":"f64"},{"name":"rain","type":"f64"},{"name":"count","type":"i32"},{"name":"id","type":"i64"}],"columns":{"station":["s1","s3","s5","s6"],"city":["Vienna","Berlin","Zürich",null],"temp":[21.5,24.0,17.0,"NaN"],"rain":[0.0,3.5,7.0,2.0],"count":[3,null,2,5],"id":["10","30","9007199254740993","-1"]}}},{"id":"concat/vertical","description":"stack frames with the same schema","inputs":{"left":{"fields":[{"name":"station","type":"utf8"},{"name":"city","type":"utf8"},{"name":"temp","type":"f64"},{"name":"rain","type":"f64"},{"name":"count","type":"i32"},{"name":"id","type":"i64"}],"columns":{"station":["s1","s2","s3","s4","s5","s6","s7"],"city":["Vienna","Vienna","Berlin","Berlin","Zürich",null,"Vienna"],"temp":[21.5,19.0,24.0,26.5,17.0,"NaN",null],"rain":[0.0,null,3.5,1.0,7.0,2.0,0.5],"count":[3,1,null,7,2,5,4],"id":["10","20","30","40","9007199254740993","-1","0"]}},"right":{"fields":[{"name":"station","type":"utf8"},{"name":"city","type":"utf8"},{"name":"temp","type":"f64"},{"name":"rain","type":"f64"},{"name":"count","type":"i32"},{"name":"id","type":"i64"}],"columns":{"station":["s1","s2","s3","s4","s5","s6","s7"],"city":["Vienna","Vienna","Berlin","Berlin","Zürich",null,"Vienna"],"temp":[21.5,19.0,24.0,26.5,17.0,"NaN",null],"rain":[0.0,null,3.5,1.0,7.0,2.0,0.5],"count":[3,1,null,7,2,5,4],"id":["10","20","30","40","9007199254740993","-1","0"]}}},"ops":[{"op":"concat","with":"right"},{"op":"pick","columns":["station","temp"]}],"expected":{"fields":[{"name":"station","type":"utf8"},{"name":"temp","type":"f64"}],"columns":{"station":["s1","s2","s3","s4","s5","s6","s7","s1","s2","s3","s4","s5","s6","s7"],"temp":[21.5,19.0,24.0,26.5,17.0,"NaN",null,21.5,19.0,24.0,26.5,17.0,"NaN",null]}}},{"id":"pipeline/summary","description":"filter, derive, group, sort","inputs":{"left":{"fields":[{"name":"station","type":"utf8"},{"name":"city","type":"utf8"},{"name":"temp","type":"f64"},{"name":"rain","type":"f64"},{"name":"count","type":"i32"},{"name":"id","type":"i64"}],"columns":{"station":["s1","s2","s3","s4","s5","s6","s7"],"city":["Vienna","Vienna","Berlin","Berlin","Zürich",null,"Vienna"],"temp":[21.5,19.0,24.0,26.5,17.0,"NaN",null],"rain":[0.0,null,3.5,1.0,7.0,2.0,0.5],"count":[3,1,null,7,2,5,4],"id":["10","20","30","40","9007199254740993","-1","0"]}}},"ops":[{"op":"filter","predicate":{"fn":"gt","args":[{"col":"temp"},{"lit":18.0}]}},{"op":"withColumns","exprs":{"tempF":{"fn":"add","args":[{"fn":"mul","args":[{"col":"temp"},{"lit":1.8}]},{"lit":32.0}]}}},{"op":"groupBy","keys":["city"],"aggs":{"stations":{"agg":"len"},"avgF":{"agg":"mean","arg":{"col":"tempF"}},"maxRain":{"agg":"max","arg":{"col":"rain"}}}},{"op":"sort","keys":[{"expr":{"col":"avgF"},"descending":true}]}],"expected":{"fields":[{"name":"city","type":"utf8"},{"name":"stations","type":"u32"},{"name":"avgF","type":"f64"},{"name":"maxRain","type":"f64"}],"columns":{"city":[null,"Berlin","Vienna"],"stations":[1,2,2],"avgF":["NaN",77.45,68.45],"maxRain":[2.0,3.5,0.0]}}},{"id":"pipeline/join-group","description":"join then aggregate the joined columns","inputs":{"left":{"fields":[{"name":"station","type":"utf8"},{"name":"city","type":"utf8"},{"name":"temp","type":"f64"},{"name":"rain","type":"f64"},{"name":"count","type":"i32"},{"name":"id","type":"i64"}],"columns":{"station":["s1","s2","s3","s4","s5","s6","s7"],"city":["Vienna","Vienna","Berlin","Berlin","Zürich",null,"Vienna"],"temp":[21.5,19.0,24.0,26.5,17.0,"NaN",null],"rain":[0.0,null,3.5,1.0,7.0,2.0,0.5],"count":[3,1,null,7,2,5,4],"id":["10","20","30","40","9007199254740993","-1","0"]}},"right":{"fields":[{"name":"station","type":"utf8"},{"name":"elevation","type":"f64"},{"name":"active","type":"bool"}],"columns":{"station":["s1","s3","s3","s9",null],"elevation":[190.0,34.0,35.0,500.0,1.0],"active":[true,false,null,true,true]}}},"ops":[{"op":"join","right":"right","on":["station"],"how":"left"},{"op":"groupBy","keys":["active"],"aggs":{"n":{"agg":"len"},"elev":{"agg":"mean","arg":{"col":"elevation"}}}},{"op":"sort","keys":[{"expr":{"col":"active"}}]}],"expected":{"fields":[{"name":"active","type":"bool"},{"name":"n","type":"u32"},{"name":"elev","type":"f64"}],"columns":{"active":[null,false,true],"n":[6,1,1],"elev":[35.0,34.0,190.0]}}}]} diff --git a/packages/effect/test/fixtures/dataframe/sql_oracle.py b/packages/effect/test/fixtures/dataframe/sql_oracle.py new file mode 100644 index 00000000000..e98544408c0 --- /dev/null +++ b/packages/effect/test/fixtures/dataframe/sql_oracle.py @@ -0,0 +1,133 @@ +""" +Cross-checks the SQL engine: runs every gold plan, compiled to DuckDB SQL by +compile_sql.ts, in DuckDB over the gold inputs, and compares the rows with +the Polars result. + + node test/fixtures/dataframe/compile_sql.ts duckdb + python test/fixtures/dataframe/sql_oracle.py + +Types are not compared (SQL engines widen integers and counts); values are, +in row order, floats with a relative tolerance. Prints a summary and writes +.sql/sql-duckdb-report.json (git-ignored) with every difference. +""" + +import json +import math +import sys +from decimal import Decimal +from pathlib import Path + +import duckdb + +sys.path.insert(0, str(Path(__file__).parent)) +import generate_gold as g # noqa: E402 + +HERE = Path(__file__).parent + + +def load(name): + return json.loads((HERE / "gold" / f"{name}.json").read_text())["cases"] + + +def same(a, b): + if a is None or b is None: + return a is None and b is None + if isinstance(b, list): + return isinstance(a, list) and len(a) == len(b) and all(same(x, y) for x, y in zip(a, b)) + if isinstance(b, dict): + return isinstance(a, dict) and all(same(a.get(k), v) for k, v in b.items()) + if isinstance(b, bool) or isinstance(a, bool): + return bool(a) == bool(b) and type(a) is type(b) + if isinstance(b, (int, float, Decimal)) and isinstance(a, (int, float, Decimal)): + a, b = float(a), float(b) + if math.isnan(a) or math.isnan(b): + return math.isnan(a) and math.isnan(b) + if a == b: + return True + return math.isfinite(a) and math.isfinite(b) and abs(a - b) <= 1e-9 * max(abs(a), abs(b)) + return a == b + + +def expected_rows(table): + names = [f["name"] for f in table["fields"]] + types = {f["name"]: f["type"] for f in table["fields"]} + n = len(table["columns"][names[0]]) if names else 0 + rows = [] + for i in range(n): + row = {} + for name in names: + v = table["columns"][name][i] + t = types[name] + row[name] = g.decode_value(v, t) if t not in g.TEMPORAL else v + rows.append(row) + return names, types, rows + + +def temporal_int(v, t): + """Gold temporal values are epoch integers; DuckDB returns Python objects.""" + import datetime as dt + if v is None: + return None + if t == "date": + return (v - dt.date(1970, 1, 1)).days + if t.startswith("datetime"): + us = int((v - dt.datetime(1970, 1, 1)) / dt.timedelta(microseconds=1)) + return us if t.endswith("[us]") else us // 1000 + if t.startswith("duration"): + us = int(v / dt.timedelta(microseconds=1)) + return us if t.endswith("[us]") else us // 1000 + return v + + +def run_case(case, compiled): + con = duckdb.connect() + con.execute("SET threads = 1") + con.execute("SET preserve_insertion_order = true") + for name, table in case["inputs"].items(): + df = g.to_polars(table) + con.register(f"_{name}", df) + con.execute(f'CREATE TABLE "{name}" AS SELECT * FROM "_{name}"') + result = con.execute(compiled["sql"]) + columns = [d[0] for d in result.description] + got = [dict(zip(columns, row)) for row in result.fetchall()] + names, types, want = expected_rows(case["expected"]) + if columns != names: + return f"columns {columns}, expected {names}" + if len(got) != len(want): + return f"{len(got)} rows, expected {len(want)}" + for i, (a, b) in enumerate(zip(got, want)): + for name in names: + x = a[name] + if types[name] in g.TEMPORAL: + x = temporal_int(x, types[name]) + if not same(x, b[name]): + return f"row {i} column {name}: got {x!r}, expected {b[name]!r}" + return None + + +def main(): + cases = {c["id"]: c for name in ("spec", "random", "extended") for c in load(name)} + compiled = json.loads((HERE / ".sql" / "sql-duckdb.json").read_text()) + report = {"pass": [], "fail": {}, "unsupported": {}, "error": {}} + for c in compiled: + if "error" in c: + report["unsupported"][c["id"]] = c["error"] + continue + try: + diff = run_case(cases[c["id"]], c) + except Exception as e: # DuckDB rejected the SQL or raised at run time + report["error"][c["id"]] = f"{type(e).__name__}: {str(e)[:300]}" + continue + if diff is None: + report["pass"].append(c["id"]) + else: + report["fail"][c["id"]] = diff + (HERE / ".sql" / "sql-duckdb-report.json").write_text(json.dumps(report, indent=1) + "\n") + print( + f"duckdb {duckdb.__version__}: {len(report['pass'])} pass, {len(report['fail'])} differ, " + f"{len(report['error'])} errors, {len(report['unsupported'])} unsupported" + ) + + +if __name__ == "__main__": + main() diff --git a/packages/effect/typetest/DataFrame.tst.ts b/packages/effect/typetest/DataFrame.tst.ts new file mode 100644 index 00000000000..1a6c3435abb --- /dev/null +++ b/packages/effect/typetest/DataFrame.tst.ts @@ -0,0 +1,280 @@ +import type { BigDecimal, Stream } from "effect" +import { DateTime, Duration, Effect, Schema } from "effect" +import { DataFrame, Expr, Query } from "effect/dataframe" +import { describe, expect, it } from "tstyche" + +const Reading = Schema.Struct({ + station: Schema.String, + city: Schema.String, + temp: Schema.Number, + rain: Schema.NullOr(Schema.Number), + id: Schema.BigInt +}) +declare const readings: Query.Query + +const Station = Schema.Struct({ station: Schema.String, elevation: Schema.Number }) +declare const stations: Query.Query + +describe("DataFrame", () => { + it("fromRows takes the row type from the schema", () => { + expect(DataFrame.fromRows(Station, [])).type.toBe< + DataFrame.DataFrame<{ readonly station: string; readonly elevation: number }> + >() + }) + + it("fromColumns infers column types", () => { + expect(DataFrame.fromColumns({ a: new Float64Array(1), b: ["x", null], c: new BigInt64Array(1) })).type.toBe< + DataFrame.DataFrame<{ readonly a: number; readonly b: string | null; readonly c: bigint }> + >() + }) +}) + +describe("Query", () => { + it("from and table take the row type from the frame or schema", () => { + expect(Query.from(DataFrame.fromRows(Station, []))).type.toBe< + Query.Query<{ readonly station: string; readonly elevation: number }> + >() + expect(Query.table("stations", Station)).type.toBe< + Query.Query<{ readonly station: string; readonly elevation: number }> + >() + }) + + it("is an Effect producing a frame of its rows", () => { + expect(Effect.runSync(stations)).type.toBe>() + expect(Effect.gen(function*() { + return yield* stations + })).type.toBe, DataFrame.DataFrameError>>() + }) + + it("filter keeps the row type", () => { + expect(readings.pipe(Query.filter((c) => Expr.gt(c.temp, 20)))).type.toBe() + }) + + it("select and withColumns compute the row type, nulls included", () => { + expect( + readings.pipe( + Query.select((c) => ({ + city: c.city, + wet: Expr.gt(c.rain, 1), + f: c.temp.pipe(Expr.multiply(1.8), Expr.add(32)) + })) + ) + ).type.toBe>() + expect(readings.pipe(Query.withColumns((c) => ({ temp: Expr.cast(c.temp, "Int32") })), Query.pick("temp"))) + .type.toBe>() + }) + + it("pick, omit and rename", () => { + expect(readings.pipe(Query.pick("city", "temp"))).type.toBe< + Query.Query<{ readonly city: string; readonly temp: number }> + >() + expect(readings.pipe(Query.omit("station", "rain", "id"))).type.toBe< + Query.Query<{ readonly city: string; readonly temp: number }> + >() + expect(readings.pipe(Query.pick("temp", "city"), Query.rename({ temp: "celsius" }))).type.toBe< + Query.Query<{ readonly city: string; readonly celsius: number }> + >() + }) + + it("groupBy and agg", () => { + expect( + readings.pipe( + Query.groupBy("city"), + Query.agg((c) => ({ + n: Expr.count(), + mean: Expr.mean(c.temp), + total: Expr.sum(c.id), + last: Expr.last(c.station) + })) + ) + ).type.toBe< + Query.Query< + { + readonly city: string + readonly n: number + readonly mean: number | null + readonly total: bigint + readonly last: string | null + } + > + >() + }) + + it("joins add the other side's columns; left joins make them nullable", () => { + expect(readings.pipe(Query.pick("station", "temp"), Query.join(stations, { on: "station" }))).type.toBe< + Query.Query<{ readonly station: string; readonly temp: number; readonly elevation: number }> + >() + expect(readings.pipe(Query.pick("station", "temp"), Query.join(stations, { on: "station", how: "left" }))) + .type.toBe< + Query.Query<{ readonly station: string; readonly temp: number; readonly elevation: number | null }> + >() + expect(readings.pipe(Query.pick("station", "temp"), Query.join(stations, { on: "station", how: "anti" }))) + .type.toBe>() + }) + + it("rejects mistakes", () => { + // no column "tmp" + // @ts-expect-error: Property 'tmp' does not exist + readings.pipe(Query.filter((c) => Expr.gt(c.tmp, 18))) + // a string compared with a number + // @ts-expect-error: Argument of type '18' is not assignable + readings.pipe(Query.filter((c) => Expr.gt(c.city, 18))) + // arithmetic on strings + // @ts-expect-error: Argument of type 'Expr' is not assignable + readings.pipe(Query.select((c) => ({ x: Expr.add(c.city, 1) }))) + // bigint and number don't mix without a cast + // @ts-expect-error: Argument of type 'Expr' is not assignable + readings.pipe(Query.select((c) => ({ x: Expr.add(c.id, c.temp) }))) + // agg needs aggregates + // @ts-expect-error: is missing in type 'Expr' but required in type 'Aggregate' + readings.pipe(Query.groupBy("city"), Query.agg((c) => ({ t: c.temp }))) + // a predicate can't be an aggregate + // @ts-expect-error: is missing in type 'Aggregate' + readings.pipe(Query.filter((c) => Expr.mean(c.temp))) + // unknown group key + // @ts-expect-error: Argument of type '"country"' is not assignable + readings.pipe(Query.groupBy("country")) + // "temp" exists on both sides and is not the key + // @ts-expect-error: rename the columns that exist on both sides: temp + readings.pipe(Query.join(readings.pipe(Query.pick("station", "temp")), { on: "station" })) + }) +}) + +declare const n: Expr.Expr +declare const nn: Expr.Expr +declare const s: Expr.Expr + +describe("Expr", () => { + it("propagates nullability", () => { + expect(Expr.add(n, 1)).type.toBe>() + expect(Expr.add(n, nn)).type.toBe>() + expect(Expr.gt(nn, 1)).type.toBe>() + expect(Expr.fillNull(nn, 0)).type.toBe>() + expect(Expr.coalesce(nn, s.pipe(Expr.length), 0)).type.toBe>() + expect(Expr.coalesce(nn, nn)).type.toBe>() + expect(Expr.when(Expr.gt(n, 0), "pos", "neg")).type.toBe>() + expect(Expr.concat(s, "-")).type.toBe>() + }) + + it("data-last forms", () => { + expect(n.pipe(Expr.gt(1))).type.toBe>() + expect(s.pipe(Expr.startsWith("a"))).type.toBe>() + expect(n.pipe(Expr.round(2))).type.toBe>() + }) +}) + +const Event = Schema.Struct({ + id: Schema.String, + at: Schema.DateTimeUtc, + took: Schema.Duration, + tags: Schema.Array(Schema.String), + place: Schema.NullOr(Schema.Struct({ city: Schema.String, zip: Schema.Number })), + amount: Schema.Number +}) +declare const events: Query.Query +declare const price: Expr.Expr + +describe("Windows and reshaping", () => { + it("over keeps the value type; window functions type their results", () => { + expect(events.pipe(Query.select((c) => ({ + total: Expr.over(Expr.sum(c.amount), [c.id]), + running: Expr.cumSum(c.amount), + previous: Expr.shift(c.amount), + rank: Expr.rank(c.amount, { method: "dense" }), + avg3: Expr.rollingMean(c.amount, 3) + })))).type.toBe< + Query.Query<{ + readonly total: number + readonly running: number + readonly previous: number | null + readonly rank: number + readonly avg3: number | null + }> + >() + }) + + it("pivot with declared columns is typed; without them rows are untyped", () => { + expect(events.pipe(Query.pivot({ on: "id", index: [], values: "amount", aggregate: "sum", columns: ["a", "b"] }))) + .type.toBe>() + expect(events.pipe(Query.pivot({ on: "id", index: [], values: "amount", aggregate: "max" }))) + .type.toBe>() + }) + + it("unpivot names the variable and value columns", () => { + expect(readings.pipe(Query.unpivot({ index: ["city"], on: ["temp", "rain"], variable: "measure" }))).type.toBe< + Query.Query<{ readonly city: string; readonly measure: "temp" | "rain"; readonly value: number | null }> + >() + }) + + it("explode and unnest", () => { + expect(events.pipe(Query.pick("id", "tags"), Query.explode("tags"))).type.toBe< + Query.Query<{ readonly id: string; readonly tags: string | null }> + >() + expect(events.pipe(Query.pick("id", "place"), Query.unnest("place"))).type.toBe< + Query.Query<{ readonly id: string; readonly city: string | null; readonly zip: number | null }> + >() + // @ts-expect-error: Argument of type '"amount"' is not assignable + events.pipe(Query.explode("amount")) + }) +}) + +describe("Types", () => { + it("dates, times and durations", () => { + expect(events.pipe(Query.select((c) => ({ + year: Expr.year(c.at), + day: Expr.truncate(c.at, "1d"), + later: Expr.add(c.at, c.took), + gap: Expr.subtract(c.at, c.at), + hours: Expr.totalHours(c.took), + recent: Expr.gt(c.at, DateTime.makeUnsafe(0)), + long: Expr.gt(c.took, Duration.seconds(5)) + })))).type.toBe< + Query.Query<{ + readonly year: number + readonly day: DateTime.Utc + readonly later: DateTime.Utc + readonly gap: Duration.Duration + readonly hours: bigint + readonly recent: boolean + readonly long: boolean + }> + >() + // @ts-expect-error: is not assignable to parameter of type 'Expr' + Expr.year(Expr.lit(1)) + }) + + it("lists and structs", () => { + expect(events.pipe(Query.select((c) => ({ + n: Expr.listLength(c.tags), + first: Expr.listGet(c.tags, 0), + hasA: Expr.listContains(c.tags, "a"), + city: Expr.field(c.place, "city"), + pair: Expr.struct({ id: c.id, n: Expr.listLength(c.tags) }) + })))).type.toBe< + Query.Query<{ + readonly n: number + readonly first: string | null + readonly hasA: boolean + readonly city: string | null + readonly pair: { readonly id: string; readonly n: number } + }> + >() + expect(events.pipe(Query.groupBy("id"), Query.agg((c) => ({ amounts: Expr.list(c.amount) })))).type.toBe< + Query.Query<{ readonly id: string; readonly amounts: ReadonlyArray }> + >() + }) + + it("decimals", () => { + expect(Expr.add(price, price)).type.toBe>() + expect(Expr.sum(price)).type.toBe>() + expect(Expr.castDecimal(Expr.lit(1.5), 10, 2)).type.toBe>() + }) +}) + +describe("Streaming", () => { + it("Query.stream emits frames of the row type", () => { + expect(Query.stream(stations)).type.toBe< + Stream.Stream, DataFrame.DataFrameError> + >() + }) +}) diff --git a/packages/sql/pglite/test/DataFrameSql.test.ts b/packages/sql/pglite/test/DataFrameSql.test.ts new file mode 100644 index 00000000000..f0730515ee1 --- /dev/null +++ b/packages/sql/pglite/test/DataFrameSql.test.ts @@ -0,0 +1,158 @@ +/** + * effect/dataframe queries run on PostgreSQL (PGlite) through `Sql.layer` + * give the rows the in-memory engine gives for the same data. + */ +import { PgliteClient } from "@effect/sql-pglite" +import { assert, describe, layer } from "@effect/vitest" +import { DateTime, Effect, Layer, Schema } from "effect" +import { Catalog, DataFrame, Expr, Query, Sql } from "effect/dataframe" +import { SqlClient } from "effect/sql/SqlClient" + +const Sale = Schema.Struct({ + id: Schema.Number, + region: Schema.NullOr(Schema.String), + units: Schema.NullOr(Schema.Number), + price: Schema.Number, + at: Schema.DateTimeUtc +}) +const Region = Schema.Struct({ region: Schema.String, manager: Schema.String }) + +const sales = Array.from({ length: 40 }, (_, i) => ({ + id: i, + region: i % 7 === 0 ? null : ["north", "south", "east", "west"][i % 4], + units: i % 5 === 0 ? null : (i * 37) % 23, + price: Math.round(((i * 53) % 97) * 1.25 * 100) / 100, + at: DateTime.makeUnsafe(Date.UTC(2024, i % 12, 1 + (i % 27), i % 24)) +})) +const regions = [ + { region: "north", manager: "Ada" }, + { region: "south", manager: "Grace" }, + { region: "east", manager: "Edsger" } +] + +const Setup = Layer.effectDiscard( + Effect.gen(function*() { + const sql = yield* SqlClient + yield* sql`DROP TABLE IF EXISTS sales` + yield* sql`DROP TABLE IF EXISTS regions` + yield* sql`CREATE TABLE sales (id DOUBLE PRECISION, region TEXT, units DOUBLE PRECISION, price DOUBLE PRECISION, at TIMESTAMP)` + yield* sql`CREATE TABLE regions (region TEXT, manager TEXT)` + for (const s of sales) { + yield* sql`INSERT INTO sales VALUES (${s.id}, ${s.region}, ${s.units}, ${s.price}, ${ + DateTime.formatIso(s.at).replace("T", " ").replace("Z", "") + })` + } + for (const r of regions) yield* sql`INSERT INTO regions VALUES (${r.region}, ${r.manager})` + }) +) + +const memory = Catalog.layer({ + sales: DataFrame.fromRows(Sale, sales), + regions: DataFrame.fromRows(Region, regions) +}) + +const table = Query.table("sales", Sale) + +/** Values as plain JS for comparison: numbers from numeric strings, dates as ISO strings. */ +const plain = (u: unknown): unknown => + DateTime.isDateTime(u) + ? DateTime.formatIso(u) + : u instanceof Date + ? u.toISOString() + : typeof u === "bigint" + ? Number(u) + : typeof u === "string" && /^-?\d+(\.\d+)?$/.test(u) + ? Number(u) + : u + +const close = (a: unknown, b: unknown) => + typeof a === "number" && typeof b === "number" ? Math.abs(a - b) <= 1e-9 * Math.max(1, Math.abs(a)) : a === b + +describe("effect/dataframe on PostgreSQL (PGlite)", { concurrent: false }, () => { + layer(Layer.provideMerge(Setup, PgliteClient.layer()), { timeout: "60 seconds" })((it) => { + const both = Effect.fnUntraced(function*(query: Query.Query) { + const expected = DataFrame.toRows(yield* query.pipe(Effect.provide(memory))) as Array> + const actual = DataFrame.toRows(yield* query.pipe(Effect.provide(Sql.layer({ dialect: "postgres" })))) as Array< + Record + > + assert.strictEqual(actual.length, expected.length) + for (let r = 0; r < expected.length; r++) { + for (const k of Object.keys(expected[r])) { + const a = plain(actual[r][k]) + const b = plain(expected[r][k]) + if (!close(a, b)) assert.deepStrictEqual({ row: r, column: k, value: a }, { row: r, column: k, value: b }) + } + } + }) + + it.effect("filters, derives and sorts", () => + both(table.pipe( + Query.filter((c) => Expr.and(Expr.isNotNull(c.units), Expr.gt(c.price, 20))), + Query.withColumns((c) => ({ revenue: Expr.multiply(c.units, c.price), tag: Expr.concat(c.region, "/", "x") })), + Query.sort((c) => [Expr.desc(c.revenue), c.id]), + Query.head(10) + ))) + + it.effect("groups in order of first appearance", () => + both(table.pipe( + Query.groupBy("region"), + Query.agg((c) => ({ + n: Expr.count(), + units: Expr.sum(c.units), + mean: Expr.mean(c.price), + min: Expr.min(c.price), + max: Expr.max(c.price), + first: Expr.first(c.id), + last: Expr.last(c.id), + median: Expr.median(c.price), + sd: Expr.std(c.price) + })) + ))) + + it.effect("joins keep the left row order", () => + both(table.pipe( + Query.pick("id", "region"), + Query.join(Query.table("regions", Region), { on: "region", how: "left" }), + Query.filter((c) => Expr.lt(c.id, 15)) + ))) + + it.effect("semi and anti joins, unique and slices", () => + Effect.gen(function*() { + yield* both(table.pipe(Query.join(Query.table("regions", Region), { on: "region", how: "anti" }))) + yield* both(table.pipe(Query.unique(["region"]), Query.pick("id", "region"))) + yield* both(table.pipe(Query.tail(5))) + yield* both(table.pipe(Query.slice(7, 4))) + })) + + it.effect("window functions over partitions", () => + both(table.pipe( + Query.select((c) => ({ + id: c.id, + region: c.region, + running: Expr.over(Expr.cumSum(c.units), [c.region]), + previous: Expr.over(Expr.shift(c.price), [c.region]), + rank: Expr.over(Expr.rank(c.price, { method: "min" }), [c.region]), + share: Expr.divide(c.price, Expr.over(Expr.sum(c.price), [c.region])) + })) + ))) + + it.effect("dates and times", () => + both(table.pipe( + Query.select((c) => ({ + id: c.id, + year: Expr.year(c.at), + month: Expr.month(c.at), + weekday: Expr.weekday(c.at), + hour: Expr.hour(c.at) + })) + ))) + + it.effect("explain shows the SQL", () => + Effect.gen(function*() { + const text = yield* Query.explain(table.pipe(Query.head(1))).pipe( + Effect.provide(Sql.layer({ dialect: "postgres" })) + ) + assert.include(text, `FROM "sales"`) + })) + }) +}) From 19b4a9c689bda8ffbb5280cb461ff3ac35a858fd Mon Sep 17 00:00:00 2001 From: Effect Bot Date: Sun, 15 Nov 2026 11:24:07 +0000 Subject: [PATCH 02/88] Focus effect/dataframe tests Run the Polars gold corpus with inputs inline, as catalog frames and as streams of small batches, which replaces the per-operator streaming tests. Fold the large-input and streaming checks into Query.test.ts, drop the 400-case random corpus and the DuckDB cross-check scripts, and exclude the generated gold fixtures from formatting. --- .gitignore | 3 - dprint.json | 1 + .../effect/test/dataframe/DataFrame.test.ts | 17 +- packages/effect/test/dataframe/Oracle.test.ts | 29 -- packages/effect/test/dataframe/Query.test.ts | 248 +++++++++++++++++- packages/effect/test/dataframe/Scale.test.ts | 232 ---------------- packages/effect/test/dataframe/Stream.test.ts | 180 ------------- packages/effect/test/dataframe/oracle/Gold.ts | 1 - packages/effect/test/dataframe/oracle/Plan.ts | 28 +- .../test/fixtures/dataframe/compile_sql.ts | 26 -- .../test/fixtures/dataframe/generate_gold.py | 155 +---------- .../test/fixtures/dataframe/gold/random.json | 1 - .../test/fixtures/dataframe/sql_oracle.py | 133 ---------- 13 files changed, 280 insertions(+), 774 deletions(-) delete mode 100644 packages/effect/test/dataframe/Oracle.test.ts delete mode 100644 packages/effect/test/dataframe/Scale.test.ts delete mode 100644 packages/effect/test/dataframe/Stream.test.ts delete mode 100644 packages/effect/test/fixtures/dataframe/compile_sql.ts delete mode 100644 packages/effect/test/fixtures/dataframe/gold/random.json delete mode 100644 packages/effect/test/fixtures/dataframe/sql_oracle.py diff --git a/.gitignore b/.gitignore index 64045b61fb4..26bba5ae55d 100644 --- a/.gitignore +++ b/.gitignore @@ -39,6 +39,3 @@ scratchpad/**/* # Agent config opencode.json - -# Python bytecode (dev scripts) -__pycache__/ diff --git a/dprint.json b/dprint.json index 0622d2e2642..41ad1584bb4 100644 --- a/dprint.json +++ b/dprint.json @@ -21,6 +21,7 @@ "**/*.tsbuildinfo", "**/CHANGELOG.md", "packages/effect/src/StandardSchema.ts", + "packages/effect/test/fixtures/dataframe/gold/*.json", "packages/tools/graphql-generator/test/generated/**/*.graphql.ts", "!scratchpad/**/*", "**/.*", diff --git a/packages/effect/test/dataframe/DataFrame.test.ts b/packages/effect/test/dataframe/DataFrame.test.ts index dbec78b874c..b17ce543086 100644 --- a/packages/effect/test/dataframe/DataFrame.test.ts +++ b/packages/effect/test/dataframe/DataFrame.test.ts @@ -1,6 +1,6 @@ import { assert, describe, it } from "@effect/vitest" -import { Effect, Equal, Result, Schema } from "effect" -import { ArrowIpc, ArrowSchema } from "effect/arrow" +import { Effect, Equal, Result, Schema, Stream } from "effect" +import { ArrowIpc, ArrowSchema, ArrowTable } from "effect/arrow" import { DataFrame, Expr, Query } from "effect/dataframe" const Reading = Schema.Struct({ @@ -94,6 +94,19 @@ describe("DataFrame", () => { assert.isTrue(Equal.equals(back, readings)) }) + it.effect("decodes IPC stream bytes into a frame per batch", () => + Effect.gen(function*() { + const table = DataFrame.toArrowTable(readings) + const bytes = Result.getOrThrow( + ArrowIpc.encodeStream(ArrowTable.make(table.schema, [table.batches[0], table.batches[0]])) + ) + // Split the bytes at arbitrary points, as a socket would deliver them. + const chunks = [bytes.subarray(0, 100), bytes.subarray(100, 300), bytes.subarray(300)] + const frames = yield* DataFrame.decodeArrowStream(Stream.fromIterable(chunks)).pipe(Stream.runCollect) + assert.strictEqual(frames.length, 2) + for (const frame of frames) assert.isTrue(Equal.equals(frame, readings)) + })) + it("exports late-selected columns", () => { const df = Query.from(readings).pipe( Query.filter((c) => Expr.isNotNull(c.rain)), diff --git a/packages/effect/test/dataframe/Oracle.test.ts b/packages/effect/test/dataframe/Oracle.test.ts deleted file mode 100644 index 4695edefdfb..00000000000 --- a/packages/effect/test/dataframe/Oracle.test.ts +++ /dev/null @@ -1,29 +0,0 @@ -/** - * Semantic oracle: every case of the gold corpus (hand-written specification - * cases plus seeded random pipelines) turned into an effect/dataframe plan, - * encoded to JSON and back, run as a query (so the optimiser rewrites it) - * and compared with the result Python Polars produced for the same plan. See - * test/fixtures/dataframe/generate_gold.py and test/dataframe/oracle. - */ -import { assert, describe, it } from "@effect/vitest" -import * as Compare from "./oracle/Compare.ts" -import * as Gold from "./oracle/Gold.ts" -import * as Plan from "./oracle/Plan.ts" - -const check = (c: Gold.Case, inline: boolean) => () => { - assert.strictEqual(Compare.diff(c.id, Plan.run(c, inline), c.expected), undefined) -} - -describe("DataFrame oracle (Polars gold)", () => { - // Specification cases carry their inputs inline (Values nodes, encoded as - // Arrow IPC in the JSON plan); random pipelines read them from a catalog. - describe("specification", () => { - for (const c of Gold.spec().cases) it(`${c.id}: ${c.description}`, check(c, true)) - }) - describe("random pipelines", () => { - for (const c of Gold.random().cases) it(c.id, check(c, false)) - }) - describe("extended (windows, reshaping, types)", () => { - for (const c of Gold.extended().cases) it(`${c.id}: ${c.description}`, check(c, false)) - }) -}) diff --git a/packages/effect/test/dataframe/Query.test.ts b/packages/effect/test/dataframe/Query.test.ts index 32edbb57962..01f4836ff1f 100644 --- a/packages/effect/test/dataframe/Query.test.ts +++ b/packages/effect/test/dataframe/Query.test.ts @@ -1,6 +1,10 @@ import { assert, describe, it } from "@effect/vitest" -import { Effect, Exit, Schema } from "effect" +import { Effect, Exit, Result, Schema, Stream } from "effect" +import { ArrowSchema, ArrowTable } from "effect/arrow" import { Catalog, DataFrame, Expr, Plan, Query } from "effect/dataframe" +import * as Compare from "./oracle/Compare.ts" +import * as Gold from "./oracle/Gold.ts" +import * as Oracle from "./oracle/Plan.ts" const Reading = Schema.Struct({ city: Schema.String, @@ -63,15 +67,11 @@ describe("Query", () => { it.effect("checks tables against their declared columns", () => Effect.gen(function*() { - const cases: ReadonlyArray, string]> = [ - [Schema.Struct({ country: Schema.String }), `missing column "country"`], - [Schema.Struct({ city: Schema.Number }), `column "city" is Utf8, expected Float64`], - [Schema.Struct({ rain: Schema.Number }), `column "rain" has missing values`] - ] - for (const [schema, message] of cases) { - const error = yield* Effect.flip(Query.table("readings", schema).pipe(Effect.provide(tables))) - assert.strictEqual(error.message, message) - } + const error = yield* Effect.flip(Query.table("readings", City).pipe(Effect.provide(tables))) + assert.strictEqual(error.message, `missing column "country"`) + const bad = Stream.make(DataFrame.fromColumns({ city: [1] })) + const streamed = yield* Effect.flip(Query.table("t", City).pipe(Effect.provide(Catalog.layer({ t: bad })))) + assert.strictEqual(streamed.message, `column "city" is Float64, expected Utf8`) })) it.effect("narrows tables to their declared columns", () => @@ -296,4 +296,232 @@ describe("Query", () => { assert.isTrue(Expr.isAggregate(Expr.fromPlan({ _tag: "AggregateCall", fn: "count" }))) }) }) + + // Every case of the Polars gold corpus (test/fixtures/dataframe/gold), with + // its inputs inline, as catalog frames and as streams of small batches. + describe("Polars oracle", () => { + const cases = [...Gold.spec().cases, ...Gold.extended().cases] + for (const source of ["values", "frames", "batches"] as const) { + describe(source, () => { + for (const c of cases) { + it(`${c.id}: ${c.description}`, () => { + assert.strictEqual(Compare.diff(c.id, Oracle.run(c, source), c.expected), undefined) + }) + } + }) + } + }) + + describe("streamed tables", () => { + const Row = Schema.Struct({ x: Schema.Number }) + const batches = Array.from({ length: 8 }, (_, b) => DataFrame.fromRows(Row, [{ x: 2 * b }, { x: 2 * b + 1 }])) + + it.effect("a head stops reading the stream", () => + Effect.gen(function*() { + let pulled = 0 + const source = Stream.fromIterable(batches).pipe(Stream.tap(() => Effect.sync(() => pulled++))) + const df = yield* Query.table("t", Row).pipe( + Query.filter((c) => Expr.gt(c.x, 0)), + Query.head(2), + Effect.provide(Catalog.layer({ t: source })) + ) + assert.deepStrictEqual(DataFrame.toArray(df, "x"), [1, 2]) + assert.isBelow(pulled, batches.length) + })) + + it.effect("Query.stream emits a frame per non-empty batch", () => + Effect.gen(function*() { + const sizes = yield* Query.table("t", Row).pipe( + Query.filter((c) => Expr.lt(c.x, 5)), + Query.stream, + Stream.map((frame) => frame.length), + Stream.runCollect, + Effect.provide(Catalog.layer({ t: Stream.fromIterable(batches) })) + ) + assert.deepStrictEqual(sizes, [2, 2, 1]) + })) + }) + + // Inputs large enough to reach radix sorting, hash table growth and joins + // with many keys, checked against row-by-row reference implementations. + describe("large inputs", () => { + type Row = { + readonly f: number | null + readonly i: number | null + readonly b: bigint | null + readonly s: string | null + readonly p: boolean | null + } + type Key = { readonly name: keyof Row; readonly descending: boolean } + + const schema = ArrowSchema.make([ + ArrowSchema.field("f", ArrowSchema.float64), + ArrowSchema.field("i", ArrowSchema.int32), + ArrowSchema.field("b", ArrowSchema.int64), + ArrowSchema.field("s", ArrowSchema.utf8), + ArrowSchema.field("p", ArrowSchema.bool) + ]) + const strings = ["a", "b", "ab", "B", "é", "😀", "\uffff", "", "zz", "a\u0000"] + + const generate = (seed: number, n: number, cardinality: number): Array => { + let state = seed + const next = () => { + state ^= state << 13 + state ^= state >>> 17 + state ^= state << 5 + return (state >>> 0) / 4294967296 + } + const pick = (values: ReadonlyArray) => values[Math.floor(next() * values.length)] + const maybe = (a: A): A | null => (next() < 0.1 ? null : a) + return Array.from({ length: n }, () => ({ + f: maybe( + pick([NaN, -0, 0, Infinity, -Infinity, Math.floor(next() * cardinality) - cardinality / 2, 0.5, -2.5]) + ), + i: maybe(Math.floor(next() * cardinality) - Math.floor(cardinality / 2)), + b: maybe(BigInt(Math.floor(next() * cardinality)) * 4294967311n - 2n ** 40n), + s: maybe(`${pick(strings)}${Math.floor(next() * cardinality)}`), + p: maybe(next() < 0.5) + })) + } + + const compareCodePoints = (a: string, b: string): number => { + const x = Array.from(a, (c) => c.codePointAt(0)!) + const y = Array.from(b, (c) => c.codePointAt(0)!) + for (let k = 0; k < Math.min(x.length, y.length); k++) if (x[k] !== y[k]) return x[k] - y[k] + return x.length - y.length + } + + const compareValues = (a: unknown, b: unknown): number => { + if (typeof a === "number" && typeof b === "number") { + if (Number.isNaN(a) || Number.isNaN(b)) return Number.isNaN(a) ? (Number.isNaN(b) ? 0 : 1) : -1 + return a < b ? -1 : a > b ? 1 : 0 + } + if (typeof a === "string") return compareCodePoints(a, b as string) + if (typeof a === "boolean") return Number(a) - Number(b) + return (a as bigint) < (b as bigint) ? -1 : (a as bigint) > (b as bigint) ? 1 : 0 + } + + const referenceSort = (rows: ReadonlyArray, keys: ReadonlyArray, nullsLast: boolean): Array => + [...rows].sort((r1, r2) => { + for (const { descending, name } of keys) { + const a = r1[name] + const b = r2[name] + if (a === null || b === null) { + if (a === b) continue + return (a === null ? -1 : 1) * (nullsLast ? -1 : 1) + } + const c = compareValues(a, b) * (descending ? -1 : 1) + if (c !== 0) return c + } + return 0 + }) + + const referenceGroups = (rows: ReadonlyArray, keys: ReadonlyArray) => { + const keyOf = (values: ReadonlyArray) => + JSON.stringify( + values, + (_, v) => + typeof v === "bigint" ? `${v}n` : typeof v === "number" && !Number.isFinite(v) ? String(v) : v === 0 ? 0 : v + ) + const groups = new Map>() + for (const r of rows) { + const k = keyOf(keys.map((name) => r[name])) + const g = groups.get(k) + if (g === undefined) groups.set(k, [r]) + else g.push(r) + } + return [...groups.values()] + } + + const rows = generate(0x2545f491, 6000, 3000) + const df = Query.from( + DataFrame.fromArrowTable(Result.getOrThrow(ArrowTable.fromRows(schema, rows as any))) as DataFrame.DataFrame + ) + const rowsOf = (query: Query.Query): Array => DataFrame.toRows(Effect.runSync(query)) + const sortKey = (c: { readonly [K in keyof Row]: Expr.Expr }, key: Key) => + key.descending ? Expr.desc(c[key.name] as Expr.Expr) : Expr.asc(c[key.name] as Expr.Expr) + + it("sorts by every column type, in both directions, nulls first or last", () => { + for (const name of ["f", "i", "b", "s", "p"] as const) { + for (const descending of [false, true]) { + for (const nullsLast of [false, true]) { + const sorted = Query.sort(df, (c) => [sortKey(c, { name, descending })], { nullsLast }) + assert.deepStrictEqual(rowsOf(sorted), referenceSort(rows, [{ name, descending }], nullsLast)) + } + } + } + }) + + it("sorts stably by several keys, also when sliced (top-k)", () => { + const keys: ReadonlyArray = [ + { name: "p", descending: true }, + { name: "s", descending: false }, + { name: "f", descending: true } + ] + const sorted = Query.sort(df, (c) => keys.map((key) => sortKey(c, key))) + assert.deepStrictEqual(rowsOf(sorted), referenceSort(rows, keys, false)) + for ( + const [name, descending, nullsLast, offset, length] of [ + ["f", true, false, 0, 10], + ["i", false, true, 0, 1], + ["s", false, false, 5, 20], + ["b", true, true, 0, 700] + ] as const + ) { + const top = df.pipe( + Query.sort((c) => [sortKey(c, { name, descending }), Expr.asc(c.i)], { nullsLast }), + Query.slice(offset, length) + ) + const expected = referenceSort(rows, [{ name, descending }, { name: "i", descending: false }], nullsLast) + assert.deepStrictEqual(rowsOf(top), expected.slice(offset, offset + length)) + } + }) + + it("groups by keys of every type", () => { + for (const keys of [["f"], ["b"], ["p", "i"], ["s", "b", "p"]] as const) { + const result = df.pipe( + Query.groupBy(...keys), + Query.agg((c) => ({ n: Expr.count(), ni: Expr.count(c.i), max: Expr.max(c.s), first: Expr.first(c.b) })) + ) + const expected = referenceGroups(rows, keys).map((g) => { + const out: Record = {} + for (const k of keys) out[k] = g[0][k] + const present = g.map((r) => r.s).filter((s): s is string => s !== null) + out.n = g.length + out.ni = g.filter((r) => r.i !== null).length + out.max = present.length === 0 ? null : present.reduce((a, b) => (compareCodePoints(a, b) >= 0 ? a : b)) + out.first = g[0].b + return out + }) + assert.deepStrictEqual(rowsOf(result), expected) + } + assert.deepStrictEqual(rowsOf(Query.unique(df, ["i", "p"])), referenceGroups(rows, ["i", "p"]).map((g) => g[0])) + }) + + it("joins on a high-cardinality key", () => { + const right = generate(0x1b873593, 4000, 3000).map((r) => ({ i: r.i ?? 0, rs: r.s })) + const rightFrame = DataFrame.fromColumns({ i: Int32Array.from(right, (r) => r.i), rs: right.map((r) => r.rs) }) + const byKey = new Map>() + for (const r of right) byKey.set(r.i, [...(byKey.get(r.i) ?? []), r.rs]) + for (const how of ["inner", "left", "semi", "anti"] as const) { + const result = Query.join(df.pipe(Query.pick("i", "s")), Query.from(rightFrame), { on: "i", how }) + const expected = rows.flatMap((r) => { + const matches = r.i === null ? [] : byKey.get(r.i) ?? [] + switch (how) { + case "inner": + return matches.map((rs) => ({ i: r.i, s: r.s, rs })) + case "left": + return matches.length === 0 + ? [{ i: r.i, s: r.s, rs: null }] + : matches.map((rs) => ({ i: r.i, s: r.s, rs })) + case "semi": + return matches.length > 0 ? [{ i: r.i, s: r.s }] : [] + case "anti": + return matches.length === 0 ? [{ i: r.i, s: r.s }] : [] + } + }) + assert.deepStrictEqual(rowsOf(result), expected) + } + }) + }) }) diff --git a/packages/effect/test/dataframe/Scale.test.ts b/packages/effect/test/dataframe/Scale.test.ts deleted file mode 100644 index f8422a4f407..00000000000 --- a/packages/effect/test/dataframe/Scale.test.ts +++ /dev/null @@ -1,232 +0,0 @@ -/** - * Differential tests on inputs large enough to reach the code paths the gold - * corpus (small tables) can't: radix sorting, hash table growth and joins - * with many keys. Each operation is compared with a plain row-by-row - * reference implementation of the same (Polars) semantics. - */ -import { assert, describe, it } from "@effect/vitest" -import { Effect } from "effect" -import { ArrowSchema, ArrowTable } from "effect/arrow" -import { DataFrame, Expr, Query } from "effect/dataframe" -import * as Result from "effect/Result" - -type Row = { - readonly f: number | null - readonly i: number | null - readonly b: bigint | null - readonly s: string | null - readonly p: boolean | null -} - -const schema = ArrowSchema.make([ - ArrowSchema.field("f", ArrowSchema.float64), - ArrowSchema.field("i", ArrowSchema.int32), - ArrowSchema.field("b", ArrowSchema.int64), - ArrowSchema.field("s", ArrowSchema.utf8), - ArrowSchema.field("p", ArrowSchema.bool) -]) - -const strings = ["a", "b", "ab", "B", "é", "😀", "￿", "", "zz", "a\u0000"] - -const generate = (seed: number, n: number, cardinality: number): Array => { - let state = seed - const next = () => { - state ^= state << 13 - state ^= state >>> 17 - state ^= state << 5 - return (state >>> 0) / 4294967296 - } - const pick = (values: ReadonlyArray) => values[Math.floor(next() * values.length)] - const maybe = (a: A): A | null => (next() < 0.1 ? null : a) - return Array.from({ length: n }, () => ({ - f: maybe(pick([NaN, -0, 0, Infinity, -Infinity, Math.floor(next() * cardinality) - cardinality / 2, 0.5, -2.5])), - i: maybe(Math.floor(next() * cardinality) - Math.floor(cardinality / 2)), - b: maybe(BigInt(Math.floor(next() * cardinality)) * 4294967311n - 2n ** 40n), - s: maybe(`${pick(strings)}${Math.floor(next() * cardinality)}`), - p: maybe(next() < 0.5) - })) -} - -const frameOf = (rows: ReadonlyArray): DataFrame.DataFrame => - DataFrame.fromArrowTable(Result.getOrThrow(ArrowTable.fromRows(schema, rows as any))) as any - -const rowsOf = (query: Query.Query): Array => DataFrame.toRows(Effect.runSync(query)) - -// ----------------------------------------------------------------------------- -// Reference semantics -// ----------------------------------------------------------------------------- - -const compareCodePoints = (a: string, b: string): number => { - const x = Array.from(a, (c) => c.codePointAt(0)!) - const y = Array.from(b, (c) => c.codePointAt(0)!) - for (let k = 0; k < Math.min(x.length, y.length); k++) if (x[k] !== y[k]) return x[k] - y[k] - return x.length - y.length -} - -const compareValues = (a: unknown, b: unknown): number => { - if (typeof a === "number" && typeof b === "number") { - if (Number.isNaN(a) || Number.isNaN(b)) return Number.isNaN(a) ? (Number.isNaN(b) ? 0 : 1) : -1 - return a < b ? -1 : a > b ? 1 : 0 - } - if (typeof a === "string") return compareCodePoints(a, b as string) - if (typeof a === "boolean") return Number(a) - Number(b) - return (a as bigint) < (b as bigint) ? -1 : (a as bigint) > (b as bigint) ? 1 : 0 -} - -const referenceSort = ( - rows: ReadonlyArray, - keys: ReadonlyArray<{ readonly name: keyof Row; readonly descending: boolean }>, - nullsLast: boolean -): Array => - [...rows].sort((r1, r2) => { - for (const { descending, name } of keys) { - const a = r1[name] - const b = r2[name] - if (a === null || b === null) { - if (a === b) continue - return (a === null ? -1 : 1) * (nullsLast ? -1 : 1) - } - const c = compareValues(a, b) * (descending ? -1 : 1) - if (c !== 0) return c - } - return 0 - }) - -const keyOf = (values: ReadonlyArray) => - JSON.stringify( - values, - (_, v) => - typeof v === "bigint" ? `${v}n` : typeof v === "number" && !Number.isFinite(v) ? String(v) : v === 0 ? 0 : v - ) - -const referenceGroups = (rows: ReadonlyArray, keys: ReadonlyArray) => { - const groups = new Map>() - for (const r of rows) { - const k = keyOf(keys.map((name) => r[name])) - const g = groups.get(k) - if (g === undefined) groups.set(k, [r]) - else g.push(r) - } - return [...groups.values()] -} - -// ----------------------------------------------------------------------------- -// Tests -// ----------------------------------------------------------------------------- - -const rows = generate(0x2545f491, 6000, 3000) -const df = Query.from(frameOf(rows)) - -describe("DataFrame at scale (differential)", () => { - for (const name of ["f", "i", "b", "s", "p"] as const) { - for (const descending of [false, true]) { - for (const nullsLast of [false, true]) { - it(`sort by ${name} ${descending ? "desc" : "asc"} nulls ${nullsLast ? "last" : "first"}`, () => { - const key = Expr.fromPlan({ _tag: "Column", name }) - const sorted = Query.sort(df, () => [descending ? Expr.desc(key) : Expr.asc(key)], { nullsLast }) - assert.deepStrictEqual(rowsOf(sorted), referenceSort(rows, [{ name, descending }], nullsLast)) - }) - } - } - } - - it("multi-key sort is stable", () => { - const keys = [{ name: "p", descending: true }, { name: "s", descending: false }, { - name: "f", - descending: true - }] as const - const sorted = Query.sort(df, (c) => [Expr.desc(c.p), Expr.asc(c.s), Expr.desc(c.f)]) - assert.deepStrictEqual(rowsOf(sorted), referenceSort(rows, keys, false)) - }) - - // A slice of a sort runs as a top-k selection. - for ( - const [name, descending, nullsLast, offset, length] of [ - ["f", true, false, 0, 10], - ["i", false, true, 0, 1], - ["s", false, false, 5, 20], - ["b", true, true, 0, 700], - ["p", false, false, 0, 50] - ] as const - ) { - it(`sort by ${name} then slice ${offset}+${length} keeps the sorted rows`, () => { - const top = df.pipe( - Query.sort((c) => { - const key: Expr.Expr = c[name] - return [descending ? Expr.desc(key) : Expr.asc(key), Expr.asc(c.i)] - }, { nullsLast }), - Query.slice(offset, length) - ) - const keys = [{ name, descending }, { name: "i", descending: false }] as const - assert.deepStrictEqual(rowsOf(top), referenceSort(rows, keys, nullsLast).slice(offset, offset + length)) - }) - } - - for (const keys of [["i"], ["s"], ["b"], ["f"], ["p", "i"], ["s", "b", "p"]] as const) { - it(`groupBy ${keys.join(", ")}`, () => { - const result = df.pipe( - Query.groupBy(...keys), - Query.agg((c) => ({ n: Expr.count(), ni: Expr.count(c.i), max: Expr.max(c.s), first: Expr.first(c.b) })) - ) - const expected = referenceGroups(rows, keys).map((g) => { - const out: Record = {} - for (const k of keys) out[k] = g[0][k] - const present = g.map((r) => r.s).filter((s): s is string => s !== null) - out.n = g.length - out.ni = g.filter((r) => r.i !== null).length - out.max = present.length === 0 ? null : present.reduce((a, b) => (compareCodePoints(a, b) >= 0 ? a : b)) - out.first = g[0].b - return out - }) - assert.deepStrictEqual(rowsOf(result), expected) - }) - } - - it("unique keeps the first row per key", () => { - const result = Query.unique(df, ["i", "p"]) - assert.deepStrictEqual(rowsOf(result), referenceGroups(rows, ["i", "p"]).map((g) => g[0])) - }) - - for (const how of ["inner", "left", "semi", "anti"] as const) { - it(`${how} join on a high-cardinality key`, () => { - const right = generate(0x1b873593, 4000, 3000).map((r) => ({ i: r.i, rs: r.s })) - const rightFrame = DataFrame.fromColumns({ - i: Int32Array.from(right, (r) => r.i ?? 0), - rs: right.map((r) => r.rs) - }) - const result = Query.join(df.pipe(Query.pick("i", "s")), Query.from(rightFrame), { on: "i", how }) - const byKey = new Map>() - for (const r of right) { - const k = r.i ?? 0 - byKey.set(k, [...(byKey.get(k) ?? []), r.rs]) - } - const expected = rows.flatMap((r) => { - const matches = r.i === null ? [] : byKey.get(r.i) ?? [] - switch (how) { - case "inner": - return matches.map((rs) => ({ i: r.i, s: r.s, rs })) - case "left": - return matches.length === 0 ? [{ i: r.i, s: r.s, rs: null }] : matches.map((rs) => ({ i: r.i, s: r.s, rs })) - case "semi": - return matches.length > 0 ? [{ i: r.i, s: r.s }] : [] - case "anti": - return matches.length === 0 ? [{ i: r.i, s: r.s }] : [] - } - }) - assert.deepStrictEqual(rowsOf(result), expected) - }) - } - - it("filters compose with later column reads", () => { - const result = df.pipe( - Query.filter((c) => Expr.isNotNull(c.i)), - Query.sort((c) => [c.i]), - Query.filter((c) => Expr.gt(c.i, 0)), - Query.head(100) - ) - const expected = referenceSort(rows.filter((r) => r.i !== null), [{ name: "i", descending: false }], false) - .filter((r) => (r.i as number) > 0) - .slice(0, 100) - assert.deepStrictEqual(rowsOf(result), expected) - }) -}) diff --git a/packages/effect/test/dataframe/Stream.test.ts b/packages/effect/test/dataframe/Stream.test.ts deleted file mode 100644 index 58a1697156b..00000000000 --- a/packages/effect/test/dataframe/Stream.test.ts +++ /dev/null @@ -1,180 +0,0 @@ -/** - * Queries over streamed tables (a `Stream` of frames in the `Catalog`) run - * batch by batch and give the same result as over the whole table. - */ -import { assert, describe, it } from "@effect/vitest" -import { Effect, Schema, Stream } from "effect" -import { ArrowIpc } from "effect/arrow" -import { Catalog, DataFrame, Expr, Query } from "effect/dataframe" -import * as Result from "effect/Result" - -const Row = Schema.Struct({ - k: Schema.NullOr(Schema.String), - g: Schema.Number, - x: Schema.NullOr(Schema.Number) -}) -type Row = typeof Row.Type - -const generate = (seed: number, n: number): Array => { - let state = seed - const next = () => { - state ^= state << 13 - state ^= state >>> 17 - state ^= state << 5 - return (state >>> 0) / 4294967296 - } - return Array.from({ length: n }, () => ({ - k: next() < 0.1 ? null : `k${Math.floor(next() * 40)}`, - g: Math.floor(next() * 5), - x: next() < 0.1 ? null : Math.round(next() * 2000 - 1000) / 8 - })) -} - -const rows = generate(0x5eed, 2000) -const whole = DataFrame.fromRows(Row, rows) -const batchRows = 250 -const batches = Array.from( - { length: rows.length / batchRows }, - (_, b) => DataFrame.fromRows(Row, rows.slice(b * batchRows, (b + 1) * batchRows)) -) - -const Lookup = Schema.Struct({ k: Schema.String, label: Schema.String }) -const lookup = DataFrame.fromRows(Lookup, Array.from({ length: 30 }, (_, i) => ({ k: `k${i}`, label: `#${i}` }))) - -const table = Query.table("t", Row) - -/** Runs a query over the streamed table and over the whole table. */ -const both = Effect.fnUntraced(function*(query: Query.Query) { - let pulled = 0 - const streamed = Stream.fromIterable(batches).pipe(Stream.tap(() => Effect.sync(() => pulled++))) - const fromStream = DataFrame.toRows(yield* query.pipe(Effect.provide(Catalog.layer({ t: streamed, lookup })))) - const fromFrame = DataFrame.toRows(yield* query.pipe(Effect.provide(Catalog.layer({ t: whole, lookup })))) - assertClose(fromStream, fromFrame) - return { rows: fromStream, pulled } -}) - -const assertClose = (actual: ReadonlyArray, expected: ReadonlyArray) => { - assert.strictEqual(actual.length, expected.length) - const close = (a: unknown, b: unknown) => - typeof a === "number" && typeof b === "number" - ? (Number.isNaN(a) && Number.isNaN(b)) || Math.abs(a - b) <= 1e-9 * Math.max(1, Math.abs(a), Math.abs(b)) - : a === b - for (let r = 0; r < actual.length; r++) { - const keys = Object.keys(expected[r]) - if (!keys.every((k) => close(actual[r][k], expected[r][k]))) { - assert.deepStrictEqual(actual[r], expected[r], `row ${r}`) - } - } -} - -describe("Streaming", () => { - it.effect("row-wise operations run per batch", () => - Effect.gen(function*() { - const { pulled, rows } = yield* both(table.pipe( - Query.filter((c) => Expr.gt(c.x, 0)), - Query.withColumns((c) => ({ y: Expr.multiply(c.x, 2), label: Expr.concat(c.k, "!") })), - Query.pick("k", "y", "label") - )) - assert.isTrue(rows.length > 0) - assert.strictEqual(pulled, batches.length) - })) - - it.effect("a head stops reading the stream", () => - Effect.gen(function*() { - const { pulled, rows } = yield* both(table.pipe(Query.filter((c) => Expr.isNotNull(c.k)), Query.head(7))) - assert.strictEqual(rows.length, 7) - assert.isTrue(pulled <= 2, `read ${pulled} batches`) - })) - - it.effect("slices across batch boundaries", () => both(table.pipe(Query.slice(240, 30)))) - - it.effect("grouping folds batches into partial aggregates", () => - both(table.pipe( - Query.groupBy("k", "g"), - Query.agg((c) => ({ - n: Expr.count(), - nx: Expr.count(c.x), - sum: Expr.sum(c.x), - mean: Expr.mean(c.x), - std: Expr.std(c.x), - min: Expr.min(c.x), - max: Expr.max(c.k), - first: Expr.first(c.x), - last: Expr.last(c.x) - })) - ))) - - it.effect("aggregates over the whole stream", () => - both(table.pipe(Query.aggregate((c) => ({ n: Expr.count(), total: Expr.sum(c.x), variance: Expr.variance(c.x) }))))) - - it.effect("medians and distinct counts collect the stream", () => - both(table.pipe( - Query.groupBy("g"), - Query.agg((c) => ({ median: Expr.median(c.x), distinct: Expr.countDistinct(c.k) })) - ))) - - it.effect("joins stream their left side", () => - both(table.pipe(Query.join(Query.table("lookup", Lookup), { on: "k", how: "left" })))) - - it.effect("sorts collect their input", () => - both( - table.pipe(Query.filter((c) => Expr.isNotNull(c.x)), Query.sort((c) => [Expr.desc(c.x), c.k]), Query.head(20)) - )) - - it.effect("concatenates streams in order", () => - both(Query.concat([table.pipe(Query.head(3)), table.pipe(Query.tail(3))]))) - - it.effect("an empty stream gives an empty result with the declared columns", () => - Effect.gen(function*() { - const catalog = Catalog.layer({ t: Stream.empty }) - const counted = yield* table.pipe( - Query.aggregate((c) => ({ n: Expr.count(), total: Expr.sum(c.x) })), - Effect.provide(catalog) - ) - assert.deepStrictEqual(DataFrame.toRows(counted), [{ n: 0, total: 0 }]) - const grouped = yield* table.pipe( - Query.groupBy("g"), - Query.agg(() => ({ n: Expr.count() })), - Effect.provide(catalog) - ) - assert.deepStrictEqual(DataFrame.columnNames(grouped), ["g", "n"]) - assert.strictEqual(grouped.length, 0) - })) - - it.effect("Query.stream emits batches", () => - Effect.gen(function*() { - const sizes = yield* table.pipe( - Query.filter((c) => Expr.isNotNull(c.k)), - Query.stream, - Stream.map((frame) => frame.length), - Stream.runCollect, - Effect.provide(Catalog.layer({ t: Stream.fromIterable(batches) })) - ) - assert.strictEqual(sizes.length, batches.length) - const single = yield* Query.from(whole).pipe(Query.stream, Stream.runCollect) - assert.strictEqual(single.length, 1) - })) - - it.effect("decodes Arrow IPC stream bytes into batches", () => - Effect.gen(function*() { - const bytes = Result.getOrThrow(ArrowIpc.encodeStream(DataFrame.toArrowTable(whole))) - // Split the bytes at arbitrary points, as a socket would deliver them. - const chunks = [bytes.subarray(0, 100), bytes.subarray(100, 5000), bytes.subarray(5000)] - const frames = yield* DataFrame.decodeArrowStream(Stream.fromIterable(chunks)).pipe(Stream.runCollect) - const total = frames.reduce((s, f) => s + f.length, 0) - assert.strictEqual(total, whole.length) - const viaCatalog = yield* table.pipe( - Query.aggregate((c) => ({ n: Expr.count(c.x) })), - Effect.provide(Catalog.layer({ t: DataFrame.decodeArrowStream(Stream.fromIterable(chunks)) as any })) - ) - assert.deepStrictEqual(DataFrame.toRows(viaCatalog), [{ n: rows.filter((r) => r.x !== null).length }]) - })) - - it.effect("fails with DataFrameError for invalid batches", () => - Effect.gen(function*() { - const bad = Stream.make(DataFrame.fromColumns({ k: ["a"], g: ["not a number"], x: [1] })) - const error = yield* Effect.flip(table.pipe(Effect.provide(Catalog.layer({ t: bad })))) - assert.instanceOf(error, DataFrame.DataFrameError) - assert.strictEqual(error.message, `column "g" is Utf8, expected Float64`) - })) -}) diff --git a/packages/effect/test/dataframe/oracle/Gold.ts b/packages/effect/test/dataframe/oracle/Gold.ts index e5fe3d576d5..86945da2300 100644 --- a/packages/effect/test/dataframe/oracle/Gold.ts +++ b/packages/effect/test/dataframe/oracle/Gold.ts @@ -97,7 +97,6 @@ const load = (name: string): Corpus => JSON.parse(Fs.readFileSync(new URL(`../../fixtures/dataframe/gold/${name}.json`, import.meta.url), "utf8")) export const spec = (): Corpus => load("spec") -export const random = (): Corpus => load("random") export const extended = (): Corpus => load("extended") /** Splits `a:x,b:list` on top-level commas. */ diff --git a/packages/effect/test/dataframe/oracle/Plan.ts b/packages/effect/test/dataframe/oracle/Plan.ts index b45d0ba1a22..51f57b975b7 100644 --- a/packages/effect/test/dataframe/oracle/Plan.ts +++ b/packages/effect/test/dataframe/oracle/Plan.ts @@ -1,7 +1,7 @@ /** * Gold plans as effect/dataframe plans, run through the public API. */ -import { BigDecimal, Effect, Schema } from "effect" +import { BigDecimal, Effect, Schema, Stream } from "effect" import { ArrowSchema, ArrowTable } from "effect/arrow" import { Catalog, DataFrame, Plan, Query } from "effect/dataframe" import * as Result from "effect/Result" @@ -339,13 +339,33 @@ export const plan = (gold: Gold.Case, inline: boolean, declare = false): Plan.No return node } +/** + * How a case's inputs reach the query: inline `Values` nodes, frames in the + * catalog, or streams of two-row batches in the catalog. + */ +export type Source = "values" | "frames" | "batches" + +const batchesOf = (table: Gold.Table): Stream.Stream> => { + const length = table.fields.length === 0 ? 0 : table.columns[table.fields[0].name].length + return Stream.fromIterable(Array.from({ length: Math.ceil(length / 2) }, (_, b) => + frameOf({ + fields: table.fields, + columns: Object.fromEntries( + Object.entries(table.columns).map(([name, values]) => [name, values.slice(b * 2, b * 2 + 2)]) + ) + }))) +} + /** * Runs a case: its plan goes through its JSON form (so the codecs are part * of what is checked) and then runs as a query. */ -export const run = (gold: Gold.Case, inline: boolean): DataFrame.DataFrame => { - const json = JSON.parse(JSON.stringify(Schema.encodeSync(Plan.NodeFromJson)(plan(gold, inline)))) +export const run = (gold: Gold.Case, source: Source): DataFrame.DataFrame => { + const node = plan(gold, source === "values", source === "batches") + const json = JSON.parse(JSON.stringify(Schema.encodeSync(Plan.NodeFromJson)(node))) const decoded = Schema.decodeUnknownSync(Plan.NodeFromJson)(json) - const tables = Object.fromEntries(Object.entries(gold.inputs).map(([name, t]) => [name, frameOf(t)])) + const tables = Object.fromEntries( + Object.entries(gold.inputs).map(([name, t]) => [name, source === "batches" ? batchesOf(t) : frameOf(t)]) + ) return Query.fromPlan(decoded).pipe(Effect.provide(Catalog.layer(tables)), Effect.runSync) } diff --git a/packages/effect/test/fixtures/dataframe/compile_sql.ts b/packages/effect/test/fixtures/dataframe/compile_sql.ts deleted file mode 100644 index 1341f4313d4..00000000000 --- a/packages/effect/test/fixtures/dataframe/compile_sql.ts +++ /dev/null @@ -1,26 +0,0 @@ -/** - * Compiles every gold plan to SQL for the DuckDB cross-check - * (sql_oracle.py): - * - * node test/fixtures/dataframe/compile_sql.ts [dialect] - * - * Writes .sql/sql-.json (git-ignored): per case, the SQL or why it - * can't compile. - */ -import { Result } from "effect" -import { Sql } from "effect/dataframe" -import * as Fs from "node:fs" -import * as Gold from "../../dataframe/oracle/Gold.ts" -import * as Plan from "../../dataframe/oracle/Plan.ts" - -const dialect = (process.argv[2] ?? "duckdb") as Sql.Dialect -const cases = [...Gold.spec().cases, ...Gold.random().cases, ...Gold.extended().cases] -const out = cases.map((c) => { - const compiled = Sql.compile(Plan.plan(c, false, true), { dialect }) - return Result.isSuccess(compiled) - ? { id: c.id, sql: compiled.success.sql, columns: compiled.success.columns } - : { id: c.id, error: compiled.failure.message } -}) -const file = new URL(`.sql/sql-${dialect}.json`, import.meta.url) -Fs.writeFileSync(file, JSON.stringify(out) + "\n") -console.log(`${out.filter((c) => "sql" in c).length}/${out.length} plans compiled to ${dialect}`) diff --git a/packages/effect/test/fixtures/dataframe/generate_gold.py b/packages/effect/test/fixtures/dataframe/generate_gold.py index b1007ad2c0f..0eeac75fb8c 100644 --- a/packages/effect/test/fixtures/dataframe/generate_gold.py +++ b/packages/effect/test/fixtures/dataframe/generate_gold.py @@ -6,7 +6,7 @@ The TypeScript oracle (test/dataframe/oracle) replays each plan through the public effect/dataframe API and compares the result with the Polars output. - python generate_gold.py # writes gold/spec.json and gold/random.json + uv run --with polars==2.1.0 python generate_gold.py # writes gold/spec.json and gold/extended.json Needs polars (tested with 2.1). Polars is the semantic reference: where effect/dataframe deliberately differs, the case is left out here and the @@ -15,7 +15,6 @@ import json import math -import random import sys from pathlib import Path @@ -689,153 +688,6 @@ def spec_cases(): return out -# --------------------------------------------------------------------------- -# Random differential cases -# --------------------------------------------------------------------------- - -NUMERIC = ("i32", "f64") -STRINGS = ["a", "b", "c", "aa", "ab", "B", "é", "😀", "", "zz"] - - -def random_value(rng, t, null_rate): - if rng.random() < null_rate: - return None - if t == "bool": - return rng.random() < 0.5 - if t == "i32": - return rng.choice([rng.randint(-5, 5), rng.randint(-1000, 1000), 2147483647, -2147483648]) if rng.random() < 0.1 else rng.randint(-5, 5) - if t == "i64": - return rng.choice([rng.randint(-5, 5), 2**53 + rng.randint(0, 3), -(2**62)]) - if t == "f64": - r = rng.random() - if r < 0.03: - return float("nan") - if r < 0.05: - return rng.choice([float("inf"), float("-inf"), -0.0]) - return rng.choice([float(rng.randint(-4, 4)), round(rng.uniform(-100, 100), 3)]) - if t == "utf8": - return rng.choice(STRINGS) - raise ValueError(t) - - -def random_table(rng, prefix, n_rows, types): - fields = [(f"{prefix}{i}_{t}", t) for i, t in enumerate(types)] - null_rate = rng.choice([0.0, 0.1, 0.3]) - return table(fields, {name: [random_value(rng, t, null_rate) for _ in range(n_rows)] for name, t in fields}) - - -def random_expr(rng, cols, want, depth=0): - """cols: {name: type}; want: 'num' | 'bool' | 'utf8' | 'i64'""" - by = lambda ts: [c for c, t in cols.items() if t in ts] - if want == "num": - choices = by(NUMERIC) - if depth >= 2 or rng.random() < 0.4: - if choices and rng.random() < 0.8: - return col(rng.choice(choices)) - return lit(rng.choice([2, -1, 0.5, 2.5])) - op = rng.choice(["add", "sub", "mul", "div", "neg", "abs", "fillNull", "when", "mod", "floorDiv"]) - if op in ("neg", "abs"): - return fn(op, random_expr(rng, cols, "num", depth + 1)) - if op == "fillNull": - return fn(op, random_expr(rng, cols, "num", depth + 1), lit(rng.choice([0, 1.5]))) - if op == "when": - return fn("when", random_expr(rng, cols, "bool", depth + 1), random_expr(rng, cols, "num", depth + 1), random_expr(rng, cols, "num", depth + 1)) - return fn(op, random_expr(rng, cols, "num", depth + 1), random_expr(rng, cols, "num", depth + 1)) - if want == "bool": - r = rng.random() - if r < 0.45 and by(NUMERIC): - return fn(rng.choice(["gt", "lt", "gte", "lte", "eq", "neq"]), random_expr(rng, cols, "num", depth + 1), random_expr(rng, cols, "num", depth + 1)) - if r < 0.6 and by(("utf8",)): - c = col(rng.choice(by(("utf8",)))) - return rng.choice([ - fn(rng.choice(["eq", "lt", "gte"]), c, lit(rng.choice(STRINGS))), - fn("isIn", c, values=rng.sample(STRINGS, 3)), - fn("strStartsWith", c, prefix=rng.choice(["a", "é", ""])), - ]) - if r < 0.7 and by(("i64",)): - return fn(rng.choice(["gt", "eq"]), col(rng.choice(by(("i64",)))), lit(rng.randint(-3, 3), "i64")) - if r < 0.8: - return fn(rng.choice(["isNull", "isNotNull"]), col(rng.choice(list(cols)))) - if r < 0.9 and by(("bool",)) and depth < 2: - return col(rng.choice(by(("bool",)))) - if depth < 2: - return fn(rng.choice(["and", "or"]), random_expr(rng, cols, "bool", depth + 1), random_expr(rng, cols, "bool", depth + 1)) - return fn("isNull", col(rng.choice(list(cols)))) - raise ValueError(want) - - -AGGS_NUM = ["sum", "mean", "min", "max", "count", "first", "last", "nUnique", "std", "median"] -AGGS_ANY = ["count", "first", "last", "nUnique", "min", "max"] - - -def random_aggs(rng, cols, exclude, tag): - aggs = {f"n{tag}": agg("len")} - for i in range(rng.randint(1, 3)): - candidates = [c for c in cols if c not in exclude] - if not candidates: - break - c = rng.choice(candidates) - t = cols[c] - name = rng.choice(AGGS_NUM if t in NUMERIC else AGGS_ANY) - if t == "bool" and name in ("min", "max"): - name = "count" - aggs[f"a{tag}_{i}"] = agg(name, col(c)) - return aggs - - -def schema_after(inputs, ops): - df = run(inputs, ops) - return {name: type_name(dtype) for name, dtype in df.schema.items()} - - -def random_case(rng, index): - try: - return random_case_unsafe(rng, index) - except Exception as error: # Polars rejected the plan: not a valid case - print(f"skipping random/{index:04d}: {type(error).__name__}: {str(error)[:120]}", file=sys.stderr) - return None - - -def random_case_unsafe(rng, index): - n_rows = rng.choice([0, 1, 5, 20, 60]) - types = [rng.choice(["i32", "f64", "utf8", "bool", "i64", "f64", "utf8"]) for _ in range(rng.randint(2, 5))] - left = random_table(rng, "l", n_rows, ["utf8", "i32"] + types) - inputs = {"left": left} - ops = [] - for _ in range(rng.randint(1, 4)): - cols = schema_after(inputs, ops) - kind = rng.choice(["filter", "withColumns", "sort", "groupBy", "join", "unique", "head", "filter", "withColumns"]) - if kind == "filter": - ops.append({"op": "filter", "predicate": random_expr(rng, cols, "bool")}) - elif kind == "withColumns": - ops.append({"op": "withColumns", "exprs": {f"x{len(ops)}": random_expr(rng, cols, "num") if rng.random() < 0.7 else random_expr(rng, cols, "bool")}}) - elif kind == "sort": - keys = rng.sample(list(cols), min(len(cols), rng.randint(1, 2))) - ops.append({"op": "sort", "keys": [{"expr": col(k), "descending": rng.random() < 0.5} for k in keys], "nullsLast": rng.random() < 0.5}) - elif kind == "groupBy": - keys = rng.sample([c for c, t in cols.items() if t != "f64"] or list(cols), 1) - ops.append({"op": "groupBy", "keys": keys, "aggs": random_aggs(rng, cols, keys, len(ops))}) - elif kind == "join" and "right" not in inputs: - key_candidates = [c for c, t in cols.items() if t in ("utf8", "i32", "i64")] - if not key_candidates: - continue - key = rng.choice(key_candidates) - right = random_table(rng, "r", rng.choice([0, 3, 10, 30]), [rng.choice(["f64", "utf8", "bool"])]) - # reuse the key column's values so that rows match - values = [v for v in inputs["left"]["columns"].get(key, [])] if key in inputs["left"]["columns"] else [] - pool = values or [encode_value(random_value(rng, cols[key], 0.2), cols[key]) for _ in range(5)] - right["fields"].insert(0, {"name": key, "type": cols[key]}) - n = len(next(iter(right["columns"].values()))) - right["columns"][key] = [rng.choice(pool) if pool else None for _ in range(n)] - inputs["right"] = right - ops.append({"op": "join", "right": "right", "on": [key], "how": rng.choice(["inner", "left", "full", "semi", "anti"])}) - elif kind == "unique": - ops.append({"op": "unique", "columns": rng.sample(list(cols), 1)} if rng.random() < 0.5 else {"op": "unique"}) - elif kind == "head": - ops.append({"op": "head", "n": rng.randint(0, 10)}) - return case(f"random/{index:04d}", "seeded random pipeline", inputs, ops) - - # --------------------------------------------------------------------------- # Extended cases: window functions, pivots, more types # --------------------------------------------------------------------------- @@ -1047,12 +899,9 @@ def main(): header = {"generator": "generate_gold.py", "polars": pl.__version__, "deviations": DEVIATIONS} spec = spec_cases() (gold / "spec.json").write_text(json.dumps({**header, "cases": spec}, ensure_ascii=False, indent=None, separators=(",", ":")) + "\n") - rng = random.Random(20261114) - cases = [c for c in (random_case(rng, i) for i in range(400)) if c is not None] - (gold / "random.json").write_text(json.dumps({**header, "cases": cases}, ensure_ascii=False, indent=None, separators=(",", ":")) + "\n") extended = extended_cases() (gold / "extended.json").write_text(json.dumps({**header, "cases": extended}, ensure_ascii=False, indent=None, separators=(",", ":")) + "\n") - print(f"spec: {len(spec)} cases, random: {len(cases)} cases, extended: {len(extended)} cases") + print(f"spec: {len(spec)} cases, extended: {len(extended)} cases") if __name__ == "__main__": diff --git a/packages/effect/test/fixtures/dataframe/gold/random.json b/packages/effect/test/fixtures/dataframe/gold/random.json deleted file mode 100644 index e3aaaaed919..00000000000 --- a/packages/effect/test/fixtures/dataframe/gold/random.json +++ /dev/null @@ -1 +0,0 @@ -{"generator":"generate_gold.py","polars":"2.1.0","deviations":["JavaScript has no float literal distinct from an integer: an integral number literal behaves like a Polars integer literal (it takes the integer type of the column it meets), so cases never combine an integer column with an integral float literal.","Float to string casts use JavaScript number formatting (Polars prints 5.0 as '5.0', JavaScript as '5').","Joins never add '_right' suffixes: overlapping non-key columns are a type error.","Arithmetic between 64-bit integers (bigint) and other numeric types requires an explicit cast."],"cases":[{"id":"random/0000","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i64","type":"i64"},{"name":"l3_i64","type":"i64"},{"name":"l4_bool","type":"bool"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_i64":[],"l3_i64":[],"l4_bool":[]}}},"ops":[{"op":"filter","predicate":{"fn":"lt","args":[{"fn":"mul","args":[{"col":"l1_i32"},{"lit":2.5}]},{"col":"l1_i32"}]}},{"op":"filter","predicate":{"fn":"eq","args":[{"fn":"abs","args":[{"lit":-1}]},{"col":"l1_i32"}]}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i64","type":"i64"},{"name":"l3_i64","type":"i64"},{"name":"l4_bool","type":"bool"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_i64":[],"l3_i64":[],"l4_bool":[]}}},{"id":"random/0001","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_f64","type":"f64"},{"name":"l4_bool","type":"bool"},{"name":"l5_f64","type":"f64"}],"columns":{"l0_utf8":["c"],"l1_i32":[-3],"l2_utf8":[""],"l3_f64":[63.689],"l4_bool":[true],"l5_f64":[-1.0]}}},"ops":[{"op":"filter","predicate":{"fn":"isIn","args":[{"col":"l2_utf8"}],"values":["c","B",""]}},{"op":"withColumns","exprs":{"x1":{"fn":"or","args":[{"fn":"or","args":[{"fn":"isIn","args":[{"col":"l0_utf8"}],"values":["😀","aa","zz"]},{"fn":"eq","args":[{"col":"l3_f64"},{"col":"l1_i32"}]}]},{"fn":"isNotNull","args":[{"col":"l2_utf8"}]}]}}},{"op":"groupBy","keys":["l4_bool"],"aggs":{"n2":{"agg":"len"},"a2_0":{"agg":"min","arg":{"col":"l3_f64"}},"a2_1":{"agg":"count","arg":{"col":"l5_f64"}},"a2_2":{"agg":"first","arg":{"col":"l5_f64"}}}},{"op":"unique","columns":["l4_bool"]}],"expected":{"fields":[{"name":"l4_bool","type":"bool"},{"name":"n2","type":"u32"},{"name":"a2_0","type":"f64"},{"name":"a2_1","type":"u32"},{"name":"a2_2","type":"f64"}],"columns":{"l4_bool":[true],"n2":[1],"a2_0":[63.689],"a2_1":[1],"a2_2":[-1.0]}}},{"id":"random/0002","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_f64","type":"f64"},{"name":"l4_f64","type":"f64"},{"name":"l5_bool","type":"bool"},{"name":"l6_bool","type":"bool"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_utf8":[],"l3_f64":[],"l4_f64":[],"l5_bool":[],"l6_bool":[]}}},"ops":[{"op":"groupBy","keys":["l5_bool"],"aggs":{"n0":{"agg":"len"},"a0_0":{"agg":"max","arg":{"col":"l4_f64"}}}}],"expected":{"fields":[{"name":"l5_bool","type":"bool"},{"name":"n0","type":"u32"},{"name":"a0_0","type":"f64"}],"columns":{"l5_bool":[],"n0":[],"a0_0":[]}}},{"id":"random/0003","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_utf8","type":"utf8"}],"columns":{"l0_utf8":["b"],"l1_i32":[-1],"l2_f64":[-4.0],"l3_utf8":["ab"],"l4_utf8":[null],"l5_utf8":["😀"]}},"right":{"fields":[{"name":"l4_utf8","type":"utf8"},{"name":"r0_bool","type":"bool"}],"columns":{"r0_bool":[null,null,null,true,false,false,true,true,null,null],"l4_utf8":[null,null,null,null,null,null,null,null,null,null]}}},"ops":[{"op":"join","right":"right","on":["l4_utf8"],"how":"semi"},{"op":"withColumns","exprs":{"x1":{"fn":"lt","args":[{"fn":"abs","args":[{"lit":0.5}]},{"fn":"neg","args":[{"col":"l2_f64"}]}]}}},{"op":"unique"}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_utf8","type":"utf8"},{"name":"x1","type":"bool"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_f64":[],"l3_utf8":[],"l4_utf8":[],"l5_utf8":[],"x1":[]}}},{"id":"random/0004","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_i64","type":"i64"},{"name":"l4_f64","type":"f64"},{"name":"l5_bool","type":"bool"},{"name":"l6_bool","type":"bool"}],"columns":{"l0_utf8":["aa","zz","😀","a","zz",null,"b","B","a","b","aa","b","ab","é",null,"b",null,"ab","b","é"],"l1_i32":[4,-3,-2,-2147483648,-4,-4,3,-4,null,-4,1,null,0,-2,4,5,0,0,5,4],"l2_bool":[true,false,false,true,true,false,true,false,null,true,false,false,false,true,true,true,false,true,false,true],"l3_i64":["4","5",null,"-4611686018427387904","9007199254740995",null,"-4611686018427387904","-4611686018427387904","-4611686018427387904","9007199254740993","9007199254740992","-5","3","9007199254740993","-1","9007199254740993","-2","-2","1","9007199254740995"],"l4_f64":[-89.526,0.0,-1.0,-0.645,2.0,1.0,-42.666,"NaN",-17.917,73.944,null,26.072,2.0,-15.736,null,"NaN",2.615,-3.0,3.0,0.0],"l5_bool":[false,true,false,false,false,true,true,true,true,true,true,true,false,false,false,true,true,false,false,null],"l6_bool":[true,false,false,false,false,false,true,false,true,true,true,true,false,true,false,true,false,false,true,true]}}},"ops":[{"op":"groupBy","keys":["l1_i32"],"aggs":{"n0":{"agg":"len"},"a0_0":{"agg":"first","arg":{"col":"l0_utf8"}},"a0_1":{"agg":"min","arg":{"col":"l3_i64"}},"a0_2":{"agg":"last","arg":{"col":"l0_utf8"}}}},{"op":"withColumns","exprs":{"x1":{"fn":"neg","args":[{"fn":"neg","args":[{"lit":-1}]}]}}},{"op":"head","n":9},{"op":"unique","columns":["a0_0"]}],"expected":{"fields":[{"name":"l1_i32","type":"i32"},{"name":"n0","type":"u32"},{"name":"a0_0","type":"utf8"},{"name":"a0_1","type":"i64"},{"name":"a0_2","type":"utf8"},{"name":"x1","type":"i32"}],"columns":{"l1_i32":[4,-3,-2,-2147483648,3,0],"n0":[3,1,2,1,1,3],"a0_0":["aa","zz","😀","a","b","ab"],"a0_1":["-1","5","9007199254740993","-4611686018427387904","-4611686018427387904","-2"],"a0_2":["é","zz","é","a","b","ab"],"x1":[-1,-1,-1,-1,-1,-1]}}},{"id":"random/0005","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_f64","type":"f64"},{"name":"l4_utf8","type":"utf8"}],"columns":{"l0_utf8":["b"],"l1_i32":[4],"l2_i32":[null],"l3_f64":[1.0],"l4_utf8":["😀"]}}},"ops":[{"op":"head","n":0},{"op":"head","n":2},{"op":"filter","predicate":{"fn":"neq","args":[{"fn":"floorDiv","args":[{"col":"l2_i32"},{"col":"l1_i32"}]},{"col":"l2_i32"}]}},{"op":"unique","columns":["l4_utf8"]}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_f64","type":"f64"},{"name":"l4_utf8","type":"utf8"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_i32":[],"l3_f64":[],"l4_utf8":[]}}},{"id":"random/0006","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_f64","type":"f64"},{"name":"l4_i32","type":"i32"}],"columns":{"l0_utf8":[null,null,"é",null,"😀","zz","B",null,null,"é","ab","c","B",null,"é","ab","b",null,null,"a"],"l1_i32":[-5,null,null,null,null,null,4,-3,-4,4,5,2,1,0,3,4,161,null,null,null],"l2_bool":[true,true,true,true,true,null,true,false,true,true,true,false,null,null,null,null,true,false,false,null],"l3_f64":[-4.0,2.0,null,3.0,null,22.128,-40.946,1.0,-3.0,-2.0,null,-74.816,-3.0,null,4.0,"NaN",2.0,-2.0,-12.995,69.001],"l4_i32":[null,null,1,4,3,null,null,null,2,1,3,3,1,null,null,-2,-1,null,340,3]}}},"ops":[{"op":"unique","columns":["l1_i32"]},{"op":"sort","keys":[{"expr":{"col":"l4_i32"},"descending":false},{"expr":{"col":"l1_i32"},"descending":true}],"nullsLast":false},{"op":"filter","predicate":{"fn":"lte","args":[{"col":"l4_i32"},{"col":"l1_i32"}]}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_f64","type":"f64"},{"name":"l4_i32","type":"i32"}],"columns":{"l0_utf8":["b","B","ab"],"l1_i32":[161,1,5],"l2_bool":[true,null,true],"l3_f64":[2.0,-3.0,null],"l4_i32":[-1,1,3]}}},{"id":"random/0007","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_i32","type":"i32"},{"name":"l4_f64","type":"f64"},{"name":"l5_utf8","type":"utf8"},{"name":"l6_i32","type":"i32"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_bool":[],"l3_i32":[],"l4_f64":[],"l5_utf8":[],"l6_i32":[]}}},"ops":[{"op":"sort","keys":[{"expr":{"col":"l6_i32"},"descending":true}],"nullsLast":false}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_i32","type":"i32"},{"name":"l4_f64","type":"f64"},{"name":"l5_utf8","type":"utf8"},{"name":"l6_i32","type":"i32"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_bool":[],"l3_i32":[],"l4_f64":[],"l5_utf8":[],"l6_i32":[]}}},{"id":"random/0008","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_i32","type":"i32"},{"name":"l4_bool","type":"bool"}],"columns":{"l0_utf8":["zz","b","B",null,null],"l1_i32":[-4,2,null,0,-1],"l2_utf8":["c","c",null,null,"é"],"l3_i32":[1,-1,1,-2147483648,5],"l4_bool":[null,false,false,false,true]}}},"ops":[{"op":"unique"},{"op":"withColumns","exprs":{"x1":{"lit":0.5}}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_i32","type":"i32"},{"name":"l4_bool","type":"bool"},{"name":"x1","type":"f64"}],"columns":{"l0_utf8":["zz","b","B",null,null],"l1_i32":[-4,2,null,0,-1],"l2_utf8":["c","c",null,null,"é"],"l3_i32":[1,-1,1,-2147483648,5],"l4_bool":[null,false,false,false,true],"x1":[0.5,0.5,0.5,0.5,0.5]}}},{"id":"random/0009","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_f64","type":"f64"},{"name":"l4_f64","type":"f64"}],"columns":{"l0_utf8":["","c","zz",null,"é"],"l1_i32":[-3,1,-5,-3,-2],"l2_utf8":["c","😀",null,"zz","a"],"l3_f64":[null,-44.767,3.0,-55.17,"Infinity"],"l4_f64":[-97.678,4.0,24.637,null,null]}},"right":{"fields":[{"name":"l2_utf8","type":"utf8"},{"name":"r0_utf8","type":"utf8"}],"columns":{"r0_utf8":["B","","zz","","c","b","aa","a","é","c","B","B","a","é","B","","ab","B","zz",null,"é","é","","aa","c","c","😀","B","😀","ab"],"l2_utf8":["a","zz","c",null,"😀","a","😀","zz","c","c",null,null,null,"zz","zz","a","a","zz","c","c","😀","😀","😀","a",null,"😀","a","c","😀",null]}}},"ops":[{"op":"join","right":"right","on":["l2_utf8"],"how":"left"},{"op":"groupBy","keys":["l2_utf8"],"aggs":{"n1":{"agg":"len"},"a1_0":{"agg":"count","arg":{"col":"l4_f64"}},"a1_1":{"agg":"median","arg":{"col":"l3_f64"}},"a1_2":{"agg":"last","arg":{"col":"l1_i32"}}}}],"expected":{"fields":[{"name":"l2_utf8","type":"utf8"},{"name":"n1","type":"u32"},{"name":"a1_0","type":"u32"},{"name":"a1_1","type":"f64"},{"name":"a1_2","type":"i32"}],"columns":{"l2_utf8":["c","😀",null,"zz","a"],"n1":[6,7,1,5,6],"a1_0":[6,7,1,0,0],"a1_1":[null,-44.767,3.0,-55.17,"Infinity"],"a1_2":[-3,1,-5,-3,-2]}}},{"id":"random/0010","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i64","type":"i64"},{"name":"l3_i64","type":"i64"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_i32","type":"i32"}],"columns":{"l0_utf8":["","","b","😀","c","zz","aa","b","ab","zz","b","B","B","","a","é","zz","é","a","é"],"l1_i32":[-488,null,null,-4,5,-2,null,null,null,1,-5,-4,1,-3,-2,-3,-2,-5,-4,5],"l2_i64":["9007199254740993","-5","-3","9007199254740993","9007199254740994","-4611686018427387904","-4611686018427387904","9007199254740993","-4611686018427387904",null,"-3","-1","1","0","0","-1","2","9007199254740994","5","-4611686018427387904"],"l3_i64":["9007199254740995","-4611686018427387904","0","9007199254740993","-4611686018427387904","-4611686018427387904","-1","-4611686018427387904","-4611686018427387904","9007199254740992","9007199254740995","-4611686018427387904","-4611686018427387904","9007199254740994","-4611686018427387904","5","3","-4611686018427387904","9007199254740993","9007199254740994"],"l4_utf8":["aa","zz","😀","é","","é","B","B","b","",null,"zz","ab","B","b","é","é",null,"B","aa"],"l5_i32":[null,null,-3,3,2,0,3,1,-3,0,null,4,-2,4,null,2,-1,0,3,5]}}},"ops":[{"op":"groupBy","keys":["l5_i32"],"aggs":{"n0":{"agg":"len"},"a0_0":{"agg":"mean","arg":{"col":"l1_i32"}},"a0_1":{"agg":"first","arg":{"col":"l3_i64"}},"a0_2":{"agg":"mean","arg":{"col":"l1_i32"}}}},{"op":"withColumns","exprs":{"x1":{"col":"a0_0"}}},{"op":"groupBy","keys":["l5_i32"],"aggs":{"n2":{"agg":"len"},"a2_0":{"agg":"min","arg":{"col":"n0"}},"a2_1":{"agg":"min","arg":{"col":"a0_2"}}}},{"op":"sort","keys":[{"expr":{"col":"a2_1"},"descending":false},{"expr":{"col":"l5_i32"},"descending":false}],"nullsLast":false}],"expected":{"fields":[{"name":"l5_i32","type":"i32"},{"name":"n2","type":"u32"},{"name":"a2_0","type":"u32"},{"name":"a2_1","type":"f64"}],"columns":{"l5_i32":[-3,1,null,3,4,-1,0,-2,2,5],"n2":[1,1,1,1,1,1,1,1,1,1],"a2_0":[2,1,4,3,2,1,3,1,2,1],"a2_1":[null,null,-165.0,-4.0,-3.5,-2.0,-2.0,1.0,1.0,5.0]}}},{"id":"random/0011","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_i64","type":"i64"}],"columns":{"l0_utf8":["c","a","","B","a"],"l1_i32":[2,2,2,-4,-4],"l2_utf8":["ab","B","b","","c"],"l3_i64":["-4","-4611686018427387904","2","-3","9007199254740994"]}}},"ops":[{"op":"withColumns","exprs":{"x0":{"col":"l1_i32"}}},{"op":"withColumns","exprs":{"x1":{"fn":"gte","args":[{"col":"x0"},{"lit":-1}]}}},{"op":"groupBy","keys":["x1"],"aggs":{"n2":{"agg":"len"},"a2_0":{"agg":"std","arg":{"col":"l1_i32"}},"a2_1":{"agg":"sum","arg":{"col":"l1_i32"}}}},{"op":"withColumns","exprs":{"x3":{"fn":"abs","args":[{"fn":"fillNull","args":[{"col":"a2_1"},{"lit":0}]}]}}}],"expected":{"fields":[{"name":"x1","type":"bool"},{"name":"n2","type":"u32"},{"name":"a2_0","type":"f64"},{"name":"a2_1","type":"i32"},{"name":"x3","type":"i32"}],"columns":{"x1":[true,false],"n2":[3,2],"a2_0":[0.0,0.0],"a2_1":[6,-8],"x3":[6,8]}}},{"id":"random/0012","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i64","type":"i64"},{"name":"l3_bool","type":"bool"},{"name":"l4_bool","type":"bool"},{"name":"l5_f64","type":"f64"},{"name":"l6_i64","type":"i64"}],"columns":{"l0_utf8":["aa","b","aa","ab","b"],"l1_i32":[5,-1,2,-1,null],"l2_i64":["-4611686018427387904","-4611686018427387904","4","-4611686018427387904","9007199254740995"],"l3_bool":[false,true,true,true,null],"l4_bool":[null,false,null,false,false],"l5_f64":[-62.07,0.0,-85.177,-32.354,53.167],"l6_i64":["-1","-4611686018427387904","-3","0","9007199254740995"]}}},"ops":[{"op":"head","n":7}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i64","type":"i64"},{"name":"l3_bool","type":"bool"},{"name":"l4_bool","type":"bool"},{"name":"l5_f64","type":"f64"},{"name":"l6_i64","type":"i64"}],"columns":{"l0_utf8":["aa","b","aa","ab","b"],"l1_i32":[5,-1,2,-1,null],"l2_i64":["-4611686018427387904","-4611686018427387904","4","-4611686018427387904","9007199254740995"],"l3_bool":[false,true,true,true,null],"l4_bool":[null,false,null,false,false],"l5_f64":[-62.07,0.0,-85.177,-32.354,53.167],"l6_i64":["-1","-4611686018427387904","-3","0","9007199254740995"]}}},{"id":"random/0013","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_bool","type":"bool"}],"columns":{"l0_utf8":["aa","c","aa","zz","aa"],"l1_i32":[-4,-3,4,0,5],"l2_utf8":["zz","é","é","c","a"],"l3_bool":[true,true,false,false,false]}}},"ops":[{"op":"withColumns","exprs":{"x0":{"col":"l1_i32"}}},{"op":"head","n":0}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_bool","type":"bool"},{"name":"x0","type":"i32"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_utf8":[],"l3_bool":[],"x0":[]}}},{"id":"random/0014","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_utf8","type":"utf8"}],"columns":{"l0_utf8":["a"],"l1_i32":[3],"l2_f64":[1.0],"l3_utf8":["B"],"l4_utf8":["c"]}}},"ops":[{"op":"sort","keys":[{"expr":{"col":"l2_f64"},"descending":true},{"expr":{"col":"l3_utf8"},"descending":false}],"nullsLast":true}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_utf8","type":"utf8"}],"columns":{"l0_utf8":["a"],"l1_i32":[3],"l2_f64":[1.0],"l3_utf8":["B"],"l4_utf8":["c"]}}},{"id":"random/0015","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_i64","type":"i64"},{"name":"l4_i32","type":"i32"}],"columns":{"l0_utf8":["ab","c","é","😀","aa","zz","aa","aa","","","B","a","zz","c","ab","😀","B","b","c","aa"],"l1_i32":[5,-2,3,-4,-2147483648,-4,-1,-3,4,1,4,2,4,-2,5,-2,-4,2,0,4],"l2_utf8":["B","zz","","zz","😀","","é","B","c","ab","😀","é","aa","","zz","b","B","é","é","a"],"l3_i64":["9007199254740995","-4611686018427387904","9007199254740995","9007199254740995","9007199254740993","-1","-4611686018427387904","-4","-4611686018427387904","9007199254740995","-4611686018427387904","9007199254740992","4","-4611686018427387904","9007199254740993","2","-4611686018427387904","-2","9007199254740994","9007199254740994"],"l4_i32":[-5,0,0,1,4,5,5,2,-5,-3,5,1,-2,2147483647,-3,3,-475,0,5,4]}},"right":{"fields":[{"name":"l1_i32","type":"i32"},{"name":"r0_bool","type":"bool"}],"columns":{"r0_bool":[],"l1_i32":[]}}},"ops":[{"op":"head","n":3},{"op":"join","right":"right","on":["l1_i32"],"how":"left"}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_i64","type":"i64"},{"name":"l4_i32","type":"i32"},{"name":"r0_bool","type":"bool"}],"columns":{"l0_utf8":["ab","c","é"],"l1_i32":[5,-2,3],"l2_utf8":["B","zz",""],"l3_i64":["9007199254740995","-4611686018427387904","9007199254740995"],"l4_i32":[-5,0,0],"r0_bool":[null,null,null]}}},{"id":"random/0016","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i64","type":"i64"},{"name":"l3_i64","type":"i64"},{"name":"l4_i32","type":"i32"},{"name":"l5_i32","type":"i32"},{"name":"l6_i32","type":"i32"}],"columns":{"l0_utf8":["","aa",null,"B","a"],"l1_i32":[-3,0,null,null,null],"l2_i64":["9007199254740992","4","9007199254740993","-4611686018427387904","9007199254740992"],"l3_i64":["9007199254740993","4",null,"9007199254740994",null],"l4_i32":[0,-4,-1,null,2],"l5_i32":[5,-2,-1,null,1],"l6_i32":[null,5,null,null,-1]}}},"ops":[{"op":"groupBy","keys":["l1_i32"],"aggs":{"n0":{"agg":"len"},"a0_0":{"agg":"std","arg":{"col":"l4_i32"}},"a0_1":{"agg":"count","arg":{"col":"l0_utf8"}}}},{"op":"head","n":3},{"op":"filter","predicate":{"fn":"and","args":[{"fn":"or","args":[{"fn":"isNull","args":[{"col":"l1_i32"}]},{"fn":"isNull","args":[{"col":"n0"}]}]},{"fn":"isNotNull","args":[{"col":"a0_1"}]}]}},{"op":"filter","predicate":{"fn":"isNotNull","args":[{"col":"l1_i32"}]}}],"expected":{"fields":[{"name":"l1_i32","type":"i32"},{"name":"n0","type":"u32"},{"name":"a0_0","type":"f64"},{"name":"a0_1","type":"u32"}],"columns":{"l1_i32":[],"n0":[],"a0_0":[],"a0_1":[]}}},{"id":"random/0017","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_bool","type":"bool"},{"name":"l4_bool","type":"bool"},{"name":"l5_utf8","type":"utf8"}],"columns":{"l0_utf8":["aa","zz","",null,"😀"],"l1_i32":[null,0,869,1,-4],"l2_f64":[4.0,46.085,-53.295,4.0,-43.418],"l3_bool":[true,false,true,true,true],"l4_bool":[false,false,false,true,false],"l5_utf8":["b",null,"😀","😀","zz"]}}},"ops":[{"op":"filter","predicate":{"fn":"eq","args":[{"fn":"sub","args":[{"col":"l1_i32"},{"lit":2}]},{"col":"l2_f64"}]}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_bool","type":"bool"},{"name":"l4_bool","type":"bool"},{"name":"l5_utf8","type":"utf8"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_f64":[],"l3_bool":[],"l4_bool":[],"l5_utf8":[]}}},{"id":"random/0018","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_utf8","type":"utf8"}],"columns":{"l0_utf8":["c","ab","B","c","zz",null,"aa","é","B","a","zz","b","ab","ab","","b","c","b","a","zz","ab","B","é","a","B","c","","",null,"😀","b","c","b","b","ab","zz","é","ab","ab","aa",null,"B","B","aa","é","é","","","😀","B","","","ab","B","é","","B","a","","b"],"l1_i32":[-1,2147483647,4,-1,-4,5,0,5,-5,-1,1,2,1,-3,-4,0,1,-4,4,1,null,5,-2,-2,5,2,1,2,null,-1,2,null,-3,-2,-1,3,null,2147483647,-2147483648,1,-5,3,4,-5,0,-5,4,-5,-5,3,-7,1,-2,0,-2,-2,-5,3,1,1],"l2_utf8":["é","ab","a","ab",null,"B","b","a","ab","b","b","é","😀","c",null,"b",null,null,"b","zz","","","é","😀","b",null,"😀","b","é","é","c","a","","a","c","b","é","","ab","",null,"",null,"b","zz","b","","zz","é",null,"é","B","é","b","b","zz","zz","B","zz",""],"l3_utf8":["😀","😀","ab","b","c",null,"c","a",null,"b",null,"B","😀","","😀","ab","é","c","b","😀","zz","","aa","a","b","B","","😀","b","é","ab","aa",null,"ab","é","a","é","é",null,null,"","😀","aa","a","aa",null,"é","zz",null,"a","aa","","😀",null,"é","c","c",null,"é","b"],"l4_utf8":["é","b","é","","a",null,"","😀","é","b","c","zz","😀","aa","😀","ab",null,"c","😀","é","c","b",null,"😀","b","c","é","","c","c","😀","aa","b","é","","aa","zz","é","zz","","aa","c","é","",null,null,null,"ab","😀","b","é",null,"é","a","a","","a","ab","","ab"]}}},"ops":[{"op":"filter","predicate":{"fn":"and","args":[{"fn":"neq","args":[{"lit":0.5},{"lit":2.5}]},{"fn":"isIn","args":[{"col":"l2_utf8"}],"values":["a","c","é"]}]}},{"op":"unique"},{"op":"head","n":0},{"op":"sort","keys":[{"expr":{"col":"l1_i32"},"descending":false},{"expr":{"col":"l2_utf8"},"descending":true}],"nullsLast":false}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_utf8","type":"utf8"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_utf8":[],"l3_utf8":[],"l4_utf8":[]}}},{"id":"random/0019","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_f64","type":"f64"},{"name":"l4_f64","type":"f64"},{"name":"l5_i32","type":"i32"},{"name":"l6_i64","type":"i64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_utf8":[],"l3_f64":[],"l4_f64":[],"l5_i32":[],"l6_i64":[]}}},"ops":[{"op":"head","n":6},{"op":"filter","predicate":{"fn":"and","args":[{"fn":"strStartsWith","args":[{"col":"l2_utf8"}],"prefix":""},{"fn":"lt","args":[{"col":"l3_f64"},{"lit":2.5}]}]}},{"op":"withColumns","exprs":{"x2":{"fn":"gt","args":[{"fn":"when","args":[{"fn":"strStartsWith","args":[{"col":"l0_utf8"}],"prefix":"é"},{"col":"l5_i32"},{"col":"l5_i32"}]},{"col":"l3_f64"}]}}},{"op":"withColumns","exprs":{"x3":{"fn":"strStartsWith","args":[{"col":"l0_utf8"}],"prefix":"a"}}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_f64","type":"f64"},{"name":"l4_f64","type":"f64"},{"name":"l5_i32","type":"i32"},{"name":"l6_i64","type":"i64"},{"name":"x2","type":"bool"},{"name":"x3","type":"bool"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_utf8":[],"l3_f64":[],"l4_f64":[],"l5_i32":[],"l6_i64":[],"x2":[],"x3":[]}}},{"id":"random/0020","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_f64","type":"f64"},{"name":"l4_i64","type":"i64"},{"name":"l5_f64","type":"f64"}],"columns":{"l0_utf8":["a","b","é","b","é","aa","b","","é","b","ab","","é","zz","a","aa","aa","c","","","😀","","B","😀","a","c","aa","a","b","aa","c","","é","c","B","","é","b","B","aa","B","c","é","b","b","é","é","","","aa","é","é","","😀","é","B","","","aa",""],"l1_i32":[2,4,0,2147483647,948,3,4,-2,-1,3,0,-2,3,-2,3,-1,2,-3,1,0,0,5,1,-2,-3,0,-4,-5,1,-1,5,-4,-3,1,5,2,-5,-4,-2,-1,0,4,5,0,4,5,-4,0,2,2147483647,0,-3,0,-1,-3,428,-3,-2,-5,2],"l2_utf8":["😀","B","zz","b","zz","b","😀","b","😀","B","zz","c","😀","","😀","é","b","é","","zz","ab","B","B","aa","b","b","","zz","","a","b","a","ab","é","","a","a","aa","","c","b","","😀","a","ab","ab","😀","c","a","zz","","aa","zz","B","","😀","é","😀","a","b"],"l3_f64":[3.0,1.0,-2.0,4.0,97.251,19.689,4.0,"NaN",-3.0,1.0,3.0,-73.663,0.0,52.468,-4.0,-14.157,89.235,"NaN",1.0,-3.0,55.601,-56.044,2.0,-86.905,-3.0,4.985,-3.0,1.0,0.0,75.342,3.0,-2.0,-4.0,-8.861,-2.0,"NaN",80.403,3.0,-2.0,93.017,-15.641,87.214,-1.0,52.961,4.0,-81.853,2.0,69.03,-2.0,1.0,-3.0,9.047,-90.203,-3.0,3.0,0.0,2.0,3.0,"-0",19.893],"l4_i64":["-4611686018427387904","-4611686018427387904","9007199254740994","9007199254740992","-5","-5","-4611686018427387904","9007199254740994","5","9007199254740993","-4611686018427387904","9007199254740995","9007199254740992","9007199254740995","0","4","-4611686018427387904","-4611686018427387904","5","9007199254740995","-3","-4611686018427387904","-4611686018427387904","9007199254740992","9007199254740995","2","-4611686018427387904","-4611686018427387904","9007199254740995","-4611686018427387904","-5","-4611686018427387904","9007199254740992","-4611686018427387904","-4611686018427387904","9007199254740995","-4611686018427387904","-4611686018427387904","-4611686018427387904","9007199254740993","-1","3","9007199254740995","1","9007199254740993","9007199254740995","-4611686018427387904","2","-4611686018427387904","2","-4","3","-5","-3","-4611686018427387904","2","9007199254740992","-4611686018427387904","-4","-3"],"l5_f64":[4.0,2.0,-17.865,58.447,3.0,-1.0,-12.94,"NaN",-78.234,-2.0,42.432,4.0,-3.0,90.222,65.581,-2.0,33.722,-3.0,-91.672,76.232,-2.0,-77.772,98.768,12.205,-82.769,-3.0,-63.61,-1.0,2.622,2.0,-14.858,34.225,"NaN",45.927,0.0,-3.0,-26.07,1.0,68.377,-9.065,-4.0,10.179,-98.477,-0.848,-61.907,1.0,71.479,-44.591,-37.657,"NaN",-45.997,-95.902,-4.0,-4.0,81.418,3.0,3.0,-8.697,0.0,-3.0]}}},"ops":[{"op":"filter","predicate":{"fn":"eq","args":[{"fn":"neg","args":[{"col":"l3_f64"}]},{"fn":"add","args":[{"lit":2},{"col":"l5_f64"}]}]}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_f64","type":"f64"},{"name":"l4_i64","type":"i64"},{"name":"l5_f64","type":"f64"}],"columns":{"l0_utf8":["","B"],"l1_i32":[-2,5],"l2_utf8":["b",""],"l3_f64":["NaN",-2.0],"l4_i64":["9007199254740994","-4611686018427387904"],"l5_f64":["NaN",0.0]}}},{"id":"random/0021","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_f64","type":"f64"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_utf8","type":"utf8"}],"columns":{"l0_utf8":["aa","é","zz","ab",null,"a","a","é","zz","é","a",null,null,"ab",null,"é","ab","","",null],"l1_i32":[-3,2,-2,-3,null,0,-1,-1,5,null,-2,-4,-2,-3,3,null,3,-2,-5,-2147483648],"l2_f64":[null,null,-56.018,null,-71.852,61.795,-4.0,null,1.0,1.0,-97.515,null,-2.0,3.0,25.17,null,null,null,0.0,3.0],"l3_f64":[null,-60.567,null,"-0",0.0,null,0.0,null,null,-0.895,3.0,88.531,null,null,51.913,-57.123,29.998,1.0,null,null],"l4_utf8":["a","aa","b","a",null,"a",null,"é","c","aa","B","a",null,"é","zz","😀","a","zz","é",null],"l5_utf8":["","a",null,null,"","aa",null,"zz","😀","b","b",null,"é","b","zz",null,null,"é","😀",null]}},"right":{"fields":[{"name":"l4_utf8","type":"utf8"},{"name":"r0_utf8","type":"utf8"}],"columns":{"r0_utf8":[null,"","zz","a",null,"c","b","c","c",null],"l4_utf8":["B",null,null,"a","b","é","é","😀",null,"b"]}}},"ops":[{"op":"withColumns","exprs":{"x0":{"fn":"fillNull","args":[{"col":"l1_i32"},{"lit":1.5}]}}},{"op":"sort","keys":[{"expr":{"col":"l4_utf8"},"descending":false},{"expr":{"col":"l3_f64"},"descending":false}],"nullsLast":true},{"op":"join","right":"right","on":["l4_utf8"],"how":"full"},{"op":"filter","predicate":{"fn":"and","args":[{"fn":"lt","args":[{"col":"l2_f64"},{"col":"l1_i32"}]},{"fn":"gte","args":[{"col":"l2_f64"},{"col":"l3_f64"}]}]}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_f64","type":"f64"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_utf8","type":"utf8"},{"name":"x0","type":"f64"},{"name":"r0_utf8","type":"utf8"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_f64":[],"l3_f64":[],"l4_utf8":[],"l5_utf8":[],"x0":[],"r0_utf8":[]}}},{"id":"random/0022","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_f64","type":"f64"},{"name":"l4_i32","type":"i32"}],"columns":{"l0_utf8":["aa","aa","a","ab","","ab","zz","aa","a","c","a",null,"😀","zz","ab","b","aa","B","é",null,"é","B","ab","zz","é","zz",null,null,"a","B","zz","a","B","","ab","zz","é","b","ab","a","a","a","é","ab","c","a","a",null,"","c","","aa","😀","a","aa","aa","😀","é",null,"aa"],"l1_i32":[-2,5,-2,5,4,4,1,-1,-1,4,-1,null,5,2,3,2,4,null,4,-1,null,4,1,2,4,4,-3,2,-2,-4,2,-2,4,-3,-5,1,-3,3,0,null,5,4,-4,2,5,-3,4,2,2,1,1,-2,0,0,1,0,-2,1,1,1],"l2_utf8":["ab","ab","B",null,"c",null,"zz","zz","aa","","b","😀",null,"zz","é","B","c","😀","c","zz","","B","zz","b","é","B","","c","c","é",null,"ab","😀","c","é","aa","a","b","B","B","a","aa","a","😀",null,"B","😀",null,"c","","","c","",null,"😀","B","zz","zz","ab","ab"],"l3_f64":[4.0,null,-3.0,-4.0,3.0,-2.0,-48.326,-75.891,-2.0,-2.0,-3.0,"NaN",-1.0,40.544,"NaN","-Infinity",77.27,-15.791,2.0,-3.0,1.0,3.0,-54.348,-36.805,-99.084,2.0,11.72,null,-24.091,-80.429,-3.0,2.0,null,-21.412,null,-2.0,-3.787,-6.581,-2.0,0.0,70.105,2.0,-49.222,62.704,2.076,-76.167,96.743,-1.0,-4.0,58.692,0.0,-4.0,-95.33,0.0,1.0,"NaN",-98.212,1.0,"Infinity",-26.467],"l4_i32":[4,-3,1,-4,-3,1,5,-3,5,2,4,0,4,3,0,-1,-3,-5,1,5,-5,0,5,-5,-4,2,5,-5,2147483647,-5,2,-5,-4,-3,2,3,-1,-2,2,-2,5,3,null,-5,null,-3,-2,0,3,-5,-5,-1,-5,5,null,1,2,5,5,2]}}},"ops":[{"op":"groupBy","keys":["l1_i32"],"aggs":{"n0":{"agg":"len"},"a0_0":{"agg":"max","arg":{"col":"l3_f64"}}}}],"expected":{"fields":[{"name":"l1_i32","type":"i32"},{"name":"n0","type":"u32"},{"name":"a0_0","type":"f64"}],"columns":{"l1_i32":[-2,5,4,1,-1,null,2,3,-3,-4,-5,0],"n0":[6,5,11,9,4,4,8,2,4,2,1,4],"a0_0":[4.0,70.105,96.743,"Infinity",-2.0,1.0,62.704,-6.581,11.72,-49.222,null,0.0]}}},{"id":"random/0023","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i64","type":"i64"},{"name":"l3_i32","type":"i32"}],"columns":{"l0_utf8":["c","","b","😀","zz","ab","a","a","c","b","a","b","zz","a","","é","b","ab","😀","😀","b","","c","B","a","é","😀","aa","","b","a","aa","zz","c","zz","B","ab","ab","B","a","B","a","a","aa","b","ab","é","ab","zz","c","b","aa","aa","c","B","c","é","aa","","c"],"l1_i32":[-2147483648,-5,-2,-4,-2,-4,-1,-4,0,0,0,-2,5,2,0,-4,5,4,-1,0,-2,4,0,5,1,2,1,4,2,-4,-2,-3,-3,3,4,1,3,-2,0,-1,1,-3,-4,-3,0,0,-4,4,-5,1,-4,-3,3,-2,2,-4,2,0,-3,2],"l2_i64":["-4","9007199254740992","-4611686018427387904","-1","9007199254740993","-4611686018427387904","-1","3","-1","-4611686018427387904","4","9007199254740992","-3","1","-4611686018427387904","5","-2","-2","-4611686018427387904","4","-2","-4611686018427387904","5","5","-4611686018427387904","-4611686018427387904","-1","3","9007199254740993","9007199254740992","-4611686018427387904","5","-5","-4611686018427387904","-4611686018427387904","5","9007199254740994","9007199254740992","-4611686018427387904","9007199254740995","9007199254740995","-1","-3","1","9007199254740992","-4","9007199254740992","9007199254740992","-5","-4611686018427387904","1","-4611686018427387904","-1","9007199254740993","-4611686018427387904","9007199254740994","9007199254740993","-4","9007199254740995","-2"],"l3_i32":[4,-2,4,-3,2147483647,-3,0,4,4,5,5,-3,-3,0,-1,0,5,-4,-1,4,-5,-3,1,-5,-4,2,-3,5,4,2147483647,1,0,-4,-1,3,-3,4,4,3,3,1,5,-3,-5,-3,-2147483648,1,5,1,0,2,-5,1,0,2,5,2,-3,3,5]}},"right":{"fields":[{"name":"l2_i64","type":"i64"},{"name":"r0_utf8","type":"utf8"}],"columns":{"r0_utf8":["😀","a","B"],"l2_i64":["-4","9007199254740992","9007199254740992"]}}},"ops":[{"op":"head","n":1},{"op":"withColumns","exprs":{"x1":{"fn":"mul","args":[{"fn":"mod","args":[{"col":"l1_i32"},{"col":"l1_i32"}]},{"fn":"sub","args":[{"lit":2},{"col":"l3_i32"}]}]}}},{"op":"filter","predicate":{"fn":"gt","args":[{"fn":"when","args":[{"fn":"neq","args":[{"col":"l3_i32"},{"col":"l1_i32"}]},{"col":"l3_i32"},{"col":"l3_i32"}]},{"fn":"div","args":[{"col":"x1"},{"col":"l1_i32"}]}]}},{"op":"join","right":"right","on":["l2_i64"],"how":"full"}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i64","type":"i64"},{"name":"l3_i32","type":"i32"},{"name":"x1","type":"i32"},{"name":"r0_utf8","type":"utf8"}],"columns":{"l0_utf8":["c",null,null],"l1_i32":[-2147483648,null,null],"l2_i64":["-4","9007199254740992","9007199254740992"],"l3_i32":[4,null,null],"x1":[0,null,null],"r0_utf8":["😀","a","B"]}}},{"id":"random/0024","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_f64","type":"f64"}],"columns":{"l0_utf8":[null],"l1_i32":[-5],"l2_f64":[53.062],"l3_f64":[-4.0]}}},"ops":[{"op":"filter","predicate":{"fn":"and","args":[{"fn":"strStartsWith","args":[{"col":"l0_utf8"}],"prefix":"a"},{"fn":"eq","args":[{"col":"l3_f64"},{"col":"l3_f64"}]}]}},{"op":"unique","columns":["l0_utf8"]},{"op":"filter","predicate":{"fn":"lte","args":[{"fn":"mod","args":[{"col":"l1_i32"},{"col":"l3_f64"}]},{"fn":"sub","args":[{"col":"l1_i32"},{"col":"l3_f64"}]}]}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_f64","type":"f64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_f64":[],"l3_f64":[]}}},{"id":"random/0025","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_f64","type":"f64"},{"name":"l5_i32","type":"i32"}],"columns":{"l0_utf8":[null,"zz","é","B","b","B","B",null,"aa","b","","b","zz",null,"😀",null,"",null,"a","zz","c","😀",null,null,null,null,"B","B","zz","c","😀",null,null,"","a","😀","ab","B","😀","ab",null,"a","ab","c","😀","zz","ab",null,"zz","c",null,null,null,null,"","a","",null,null,"zz"],"l1_i32":[null,-4,null,null,-4,1,null,null,null,null,-4,null,2147483647,1,2,3,3,null,null,-2,2,2147483647,1,-2,-1,2147483647,3,120,null,5,0,3,5,null,5,2,null,-3,null,null,-5,null,3,4,null,-4,3,0,-4,948,2,2,null,5,null,null,2,-5,null,-1],"l2_bool":[true,null,false,true,true,false,null,null,true,null,true,false,null,null,null,true,true,false,false,true,false,false,false,true,false,false,true,null,true,false,false,true,false,true,true,true,null,null,true,false,null,true,true,true,false,true,null,null,false,false,null,null,true,false,false,false,true,null,null,true],"l3_utf8":[null,"c","B",null,"é","é","c",null,null,null,"",null,"B","😀",null,"ab","","é","B","c","aa","B","b","zz","B","zz","😀","aa","",null,"😀",null,null,"😀","",null,null,"","aa","zz","é",null,"aa",null,null,null,"B",null,"😀","aa","😀","B","ab","","a","","a",null,"é",""],"l4_f64":[1.0,null,29.922,null,-4.0,38.289,null,-3.0,47.375,2.0,58.277,null,null,50.086,-53.776,-23.472,-91.921,null,null,57.871,null,null,-20.056,-90.534,-62.375,null,null,39.949,28.397,-74.628,15.051,null,-73.49,-38.607,null,-63.471,null,-1.0,3.0,13.435,null,78.442,1.0,35.901,null,66.921,null,-50.246,2.0,-2.0,2.0,99.447,null,null,null,null,null,45.355,2.0,4.0],"l5_i32":[3,0,null,2,-3,null,null,-4,-3,null,4,4,-5,-5,5,1,null,-4,1,-5,2,4,null,634,3,2,-1,-4,-1,null,-5,-3,null,5,null,-4,1,4,4,null,3,1,-2,2,null,null,3,null,-2,-3,null,-578,-2,3,-2,4,-2,4,null,0]}},"right":{"fields":[{"name":"l1_i32","type":"i32"},{"name":"r0_utf8","type":"utf8"}],"columns":{"r0_utf8":["zz","c","B"],"l1_i32":[null,null,5]}}},"ops":[{"op":"join","right":"right","on":["l1_i32"],"how":"semi"},{"op":"filter","predicate":{"fn":"or","args":[{"col":"l2_bool"},{"fn":"and","args":[{"fn":"eq","args":[{"col":"l4_f64"},{"col":"l4_f64"}]},{"fn":"lte","args":[{"lit":0.5},{"lit":2.5}]}]}]}},{"op":"withColumns","exprs":{"x2":{"col":"l5_i32"}}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_f64","type":"f64"},{"name":"l5_i32","type":"i32"},{"name":"x2","type":"i32"}],"columns":{"l0_utf8":["c",null,"a"],"l1_i32":[5,5,5],"l2_bool":[false,false,true],"l3_utf8":[null,null,""],"l4_f64":[-74.628,-73.49,null],"l5_i32":[null,null,null],"x2":[null,null,null]}}},{"id":"random/0026","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_i64","type":"i64"},{"name":"l4_bool","type":"bool"}],"columns":{"l0_utf8":["aa","😀","B","é","","b","B","B","c","😀","é","c","c","zz","aa","ab","a","zz","b",null],"l1_i32":[4,1,5,4,null,-1,null,3,0,null,4,0,-2,-2,-4,-3,-2147483648,-1,-4,4],"l2_utf8":[null,"a","c","b","","a","zz","ab","é","B","c","ab","😀","é","😀","😀",null,"zz","😀","😀"],"l3_i64":["2","-4","-4611686018427387904","-4611686018427387904","1","9007199254740993","-4611686018427387904","5","9007199254740992","-4611686018427387904","9007199254740993","9007199254740992","-4611686018427387904","4","9007199254740992","9007199254740993","9007199254740992",null,"9007199254740995","9007199254740993"],"l4_bool":[false,true,false,false,true,true,false,true,false,true,false,true,false,true,null,true,false,false,false,true]}}},"ops":[{"op":"sort","keys":[{"expr":{"col":"l0_utf8"},"descending":false}],"nullsLast":false},{"op":"withColumns","exprs":{"x1":{"fn":"mul","args":[{"fn":"mod","args":[{"col":"l1_i32"},{"col":"l1_i32"}]},{"col":"l1_i32"}]}}},{"op":"withColumns","exprs":{"x2":{"fn":"mul","args":[{"fn":"floorDiv","args":[{"col":"l1_i32"},{"col":"l1_i32"}]},{"col":"x1"}]}}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_i64","type":"i64"},{"name":"l4_bool","type":"bool"},{"name":"x1","type":"i32"},{"name":"x2","type":"i32"}],"columns":{"l0_utf8":[null,"","B","B","B","a","aa","aa","ab","b","b","c","c","c","zz","zz","é","é","😀","😀"],"l1_i32":[4,null,5,null,3,-2147483648,4,-4,-3,-1,-4,0,0,-2,-2,-1,4,4,1,null],"l2_utf8":["😀","","c","zz","ab",null,null,"😀","😀","a","😀","é","ab","😀","é","zz","b","c","a","B"],"l3_i64":["9007199254740993","1","-4611686018427387904","-4611686018427387904","5","9007199254740992","2","9007199254740992","9007199254740993","9007199254740993","9007199254740995","9007199254740992","9007199254740992","-4611686018427387904","4",null,"-4611686018427387904","9007199254740993","-4","-4611686018427387904"],"l4_bool":[true,true,false,false,true,false,false,null,true,true,false,false,true,false,true,false,false,false,true,true],"x1":[0,null,0,null,0,0,0,0,0,0,0,null,null,0,0,0,0,0,0,null],"x2":[0,null,0,null,0,0,0,0,0,0,0,null,null,0,0,0,0,0,0,null]}}},{"id":"random/0027","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_f64","type":"f64"},{"name":"l5_i32","type":"i32"},{"name":"l6_i32","type":"i32"}],"columns":{"l0_utf8":["😀"],"l1_i32":[4],"l2_i32":[-5],"l3_utf8":[""],"l4_f64":[3.0],"l5_i32":[-4],"l6_i32":[-3]}}},"ops":[{"op":"groupBy","keys":["l1_i32"],"aggs":{"n0":{"agg":"len"},"a0_0":{"agg":"count","arg":{"col":"l5_i32"}},"a0_1":{"agg":"last","arg":{"col":"l3_utf8"}},"a0_2":{"agg":"sum","arg":{"col":"l6_i32"}}}},{"op":"unique","columns":["a0_0"]}],"expected":{"fields":[{"name":"l1_i32","type":"i32"},{"name":"n0","type":"u32"},{"name":"a0_0","type":"u32"},{"name":"a0_1","type":"utf8"},{"name":"a0_2","type":"i32"}],"columns":{"l1_i32":[4],"n0":[1],"a0_0":[1],"a0_1":[""],"a0_2":[-3]}}},{"id":"random/0028","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_f64","type":"f64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_bool":[],"l3_f64":[]}}},"ops":[{"op":"filter","predicate":{"fn":"gt","args":[{"col":"l1_i32"},{"fn":"abs","args":[{"col":"l3_f64"}]}]}},{"op":"withColumns","exprs":{"x1":{"fn":"fillNull","args":[{"col":"l3_f64"},{"lit":0}]}}},{"op":"withColumns","exprs":{"x2":{"fn":"fillNull","args":[{"fn":"mod","args":[{"col":"l1_i32"},{"col":"x1"}]},{"lit":1.5}]}}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_f64","type":"f64"},{"name":"x1","type":"f64"},{"name":"x2","type":"f64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_bool":[],"l3_f64":[],"x1":[],"x2":[]}}},{"id":"random/0029","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_utf8","type":"utf8"}],"columns":{"l0_utf8":["b"],"l1_i32":[1],"l2_i32":[2147483647],"l3_utf8":["😀"],"l4_utf8":["zz"]}}},"ops":[{"op":"unique","columns":["l3_utf8"]},{"op":"withColumns","exprs":{"x1":{"fn":"mod","args":[{"fn":"when","args":[{"fn":"gt","args":[{"col":"l2_i32"},{"lit":-1}]},{"lit":2.5},{"col":"l2_i32"}]},{"fn":"div","args":[{"col":"l2_i32"},{"col":"l2_i32"}]}]}}},{"op":"withColumns","exprs":{"x2":{"fn":"or","args":[{"fn":"strStartsWith","args":[{"col":"l0_utf8"}],"prefix":""},{"fn":"isNotNull","args":[{"col":"l4_utf8"}]}]}}},{"op":"withColumns","exprs":{"x3":{"fn":"mul","args":[{"fn":"floorDiv","args":[{"col":"l1_i32"},{"col":"l2_i32"}]},{"fn":"abs","args":[{"col":"x1"}]}]}}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_utf8","type":"utf8"},{"name":"x1","type":"f64"},{"name":"x2","type":"bool"},{"name":"x3","type":"f64"}],"columns":{"l0_utf8":["b"],"l1_i32":[1],"l2_i32":[2147483647],"l3_utf8":["😀"],"l4_utf8":["zz"],"x1":[0.5],"x2":[true],"x3":[0.0]}}},{"id":"random/0030","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_f64","type":"f64"},{"name":"l4_i32","type":"i32"},{"name":"l5_utf8","type":"utf8"},{"name":"l6_f64","type":"f64"}],"columns":{"l0_utf8":[null,"zz","b","c",null,"😀","aa","a","ab","😀","a","😀",null,null,"B","é","😀","😀","aa",null],"l1_i32":[5,3,4,1,3,3,2,0,-4,-1,-2,3,-2,null,1,-1,-2,3,-2147483648,2],"l2_bool":[false,null,true,true,null,null,null,true,false,false,false,true,null,null,null,false,true,false,false,false],"l3_f64":[null,-32.047,null,4.0,null,-2.0,null,-1.0,1.0,12.887,null,-4.0,4.0,null,null,null,47.3,"NaN",null,1.0],"l4_i32":[1,null,null,2,-5,null,5,-2,4,-3,null,-2,4,null,-832,-3,-4,-1,null,null],"l5_utf8":["é","","aa",null,"","b","B","é","b","c","aa","ab",null,"b",null,null,"B","","B","zz"],"l6_f64":[null,null,20.937,74.376,-2.0,-2.0,1.0,1.0,4.0,null,3.0,-87.89,-4.0,-4.0,-2.0,2.0,18.81,-3.0,-57.539,-49.886]}}},"ops":[{"op":"head","n":10}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_f64","type":"f64"},{"name":"l4_i32","type":"i32"},{"name":"l5_utf8","type":"utf8"},{"name":"l6_f64","type":"f64"}],"columns":{"l0_utf8":[null,"zz","b","c",null,"😀","aa","a","ab","😀"],"l1_i32":[5,3,4,1,3,3,2,0,-4,-1],"l2_bool":[false,null,true,true,null,null,null,true,false,false],"l3_f64":[null,-32.047,null,4.0,null,-2.0,null,-1.0,1.0,12.887],"l4_i32":[1,null,null,2,-5,null,5,-2,4,-3],"l5_utf8":["é","","aa",null,"","b","B","é","b","c"],"l6_f64":[null,null,20.937,74.376,-2.0,-2.0,1.0,1.0,4.0,null]}}},{"id":"random/0031","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_f64","type":"f64"},{"name":"l5_f64","type":"f64"}],"columns":{"l0_utf8":["ab","a","B","","B","","😀","B","a","B","b","a",null,"ab","😀","a","a","😀","a","","B","c","ab","","a",null,"é",null,"B","zz","😀","😀","ab","aa","B","😀","é","b","","ab","é","b","aa","zz","a","b","c","ab","aa","b","zz","a","B","aa","zz","B","B","a","a","😀"],"l1_i32":[5,-3,null,0,-5,-4,4,-1,-2,null,3,1,0,-1,-2,null,null,-1,null,4,null,null,0,3,-2,-902,5,3,-2147483648,5,-2147483648,0,5,-2,2,0,null,-2147483648,null,4,-4,4,3,-2147483648,-5,null,-1,-3,1,-3,-4,0,2,1,3,0,-5,2,-4,-1],"l2_i32":[-1,1,1,-1,3,3,3,-2147483648,4,2,null,1,-5,-5,1,-4,-4,3,null,1,3,-2147483648,1,-5,-4,null,-2,2,-2,3,1,3,-1,-4,-2,-4,-5,5,4,-5,1,-4,-2,-5,null,5,0,4,5,0,2147483647,-5,-1,2,null,-4,1,null,-5,0],"l3_utf8":["a","zz","c","😀","b","",null,null,"😀","zz","B","aa","aa","c","😀","c",null,"B","é","c","😀","b","","😀","c","c","aa","é","ab","","a","aa","zz","B","😀","aa","😀","😀","zz","😀","é","aa","b","B","é","é","a","b","B","é","","b","","b","ab","ab","","","c","ab"],"l4_f64":[-28.119,0.0,77.697,"NaN",0.0,-50.427,-75.674,3.0,-57.073,null,4.0,34.717,82.971,-1.0,-2.0,-27.543,4.0,null,4.0,"NaN",0.213,-76.344,null,-3.0,-2.0,-83.76,4.0,-68.275,-27.346,-83.495,-3.0,0.0,-1.0,0.0,-3.0,8.143,-4.0,30.266,-4.0,7.966,4.0,61.037,-1.0,4.0,75.555,3.0,null,-70.192,-24.1,71.091,89.907,-11.614,64.357,4.0,0.0,-29.494,2.0,-2.0,null,1.0],"l5_f64":[66.043,1.0,-56.96,null,92.024,51.449,-22.086,-47.795,31.068,0.0,4.0,-2.0,2.0,-43.733,7.769,-89.954,-3.0,-25.39,4.0,-3.0,-29.676,-3.0,-29.299,0.0,-61.1,-92.605,"NaN",-83.779,-1.0,0.0,0.0,null,-37.859,null,null,-1.0,-3.0,59.246,null,-62.022,45.098,-67.778,21.841,-4.0,29.508,-43.894,-41.614,-1.0,-86.858,4.0,-11.462,null,-18.429,2.0,3.0,42.576,-98.8,-39.391,1.0,4.0]}}},"ops":[{"op":"sort","keys":[{"expr":{"col":"l0_utf8"},"descending":false}],"nullsLast":true},{"op":"filter","predicate":{"fn":"lt","args":[{"fn":"floorDiv","args":[{"col":"l5_f64"},{"lit":-1}]},{"fn":"floorDiv","args":[{"col":"l5_f64"},{"col":"l2_i32"}]}]}},{"op":"head","n":0}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_f64","type":"f64"},{"name":"l5_f64","type":"f64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_i32":[],"l3_utf8":[],"l4_f64":[],"l5_f64":[]}}},{"id":"random/0032","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_i64","type":"i64"},{"name":"l4_f64","type":"f64"},{"name":"l5_utf8","type":"utf8"}],"columns":{"l0_utf8":["😀","zz","b","","ab",null,"zz","","zz","zz","aa","zz","zz","B","a","é","zz","aa","ab","zz","","b","a","ab","ab","aa","B","","c",null,"b","","é","c","B","a",null,"😀","","é",null,"c","","a","b","a","b","B","ab","B","😀","aa","aa","zz",null,"b","aa","é","😀","ab"],"l1_i32":[null,-1,-2147483648,0,-3,-4,1,1,2,-2,2,-3,0,null,-3,1,1,-2147483648,-5,-5,5,2,null,-3,-1,null,4,3,0,-3,-810,-3,1,-2,3,-2147483648,5,2,2,0,4,5,-2,-2,null,2147483647,2,-4,-5,-3,3,null,5,null,null,-4,-5,1,null,-3],"l2_i32":[-1,2,4,0,-3,-3,1,-3,-2,5,1,-2,2,null,2147483647,4,-4,-4,null,2,-5,-2,5,-4,-4,-2,4,null,0,5,0,-2,2,4,null,-4,2,null,-3,-4,3,-4,null,null,4,2147483647,3,-3,-4,4,null,-4,-1,-4,1,0,-5,-2,null,2],"l3_i64":["9007199254740994","-4611686018427387904","-4","9007199254740993",null,"9007199254740994","9007199254740994","-4611686018427387904","-3","9007199254740993","9007199254740993","9007199254740995","-4611686018427387904","0","-4","-4611686018427387904","-4611686018427387904","-4611686018427387904","-4611686018427387904","-4611686018427387904","9007199254740992","9007199254740994","-1","-5","-2","9007199254740992","9007199254740993","5","9007199254740995",null,"-4611686018427387904","9007199254740992","2","9007199254740994","9007199254740993","0","9007199254740994","-4611686018427387904","9007199254740994","-4611686018427387904","-4611686018427387904","5","9007199254740994","-4611686018427387904","9007199254740993","9007199254740993","9007199254740993","-4611686018427387904","-5",null,"-1","4","9007199254740995","9007199254740993","2","5","9007199254740992","-4611686018427387904","-4","-4611686018427387904"],"l4_f64":[-4.0,-1.0,0.0,-70.308,77.387,-28.825,50.593,-4.0,-82.514,-32.141,null,-2.0,-3.0,4.0,-4.0,-1.0,null,53.602,65.505,-91.32,82.616,-5.335,-2.0,-18.204,1.0,-3.0,-32.172,-3.0,0.0,88.273,"-Infinity",4.0,12.631,1.0,null,-58.168,-50.84,-5.522,43.594,96.263,4.0,4.0,14.091,3.0,5.058,-3.0,-4.0,23.268,null,10.379,78.576,42.139,36.501,6.087,-34.883,-1.0,1.0,4.0,16.477,null],"l5_utf8":["a","","c","B","c","a","zz","a","é","a","aa","aa","zz","B","zz",null,"😀","","a","é","c",null,"aa","é","c","c","","c","zz","😀",null,"","b","😀","a","c","ab","B",null,"😀","b","B","ab","B","","","ab","b","é","a",null,"ab","zz","","😀","b","c","zz","c","B"]}},"right":{"fields":[{"name":"l1_i32","type":"i32"},{"name":"r0_bool","type":"bool"}],"columns":{"r0_bool":[true,null,true,true,null,true,true,null,false,false],"l1_i32":[-3,3,5,-5,null,1,1,1,1,-4]}}},"ops":[{"op":"unique"},{"op":"filter","predicate":{"fn":"or","args":[{"fn":"eq","args":[{"col":"l4_f64"},{"col":"l4_f64"}]},{"fn":"isIn","args":[{"col":"l0_utf8"}],"values":["c","","a"]}]}},{"op":"join","right":"right","on":["l1_i32"],"how":"left"}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_i64","type":"i64"},{"name":"l4_f64","type":"f64"},{"name":"l5_utf8","type":"utf8"},{"name":"r0_bool","type":"bool"}],"columns":{"l0_utf8":["😀","zz","b","","ab",null,"zz","zz","zz","zz","","","","","zz","zz","zz","zz","B","a","é","é","é","é","aa","ab","zz","","b","a","ab","ab","aa","B","","c",null,"b","","é","é","é","é","c","a",null,"😀","","é",null,"c","","a","b","a","b","B","B","😀","aa","aa","zz",null,"b","aa","é","é","é","é","😀"],"l1_i32":[null,-1,-2147483648,0,-3,-4,1,1,1,1,1,1,1,1,2,-2,-3,0,null,-3,1,1,1,1,-2147483648,-5,-5,5,2,null,-3,-1,null,4,3,0,-3,-810,-3,1,1,1,1,-2,-2147483648,5,2,2,0,4,5,-2,-2,null,2147483647,2,-4,-3,3,null,5,null,null,-4,-5,1,1,1,1,null],"l2_i32":[-1,2,4,0,-3,-3,1,1,1,1,-3,-3,-3,-3,-2,5,-2,2,null,2147483647,4,4,4,4,-4,null,2,-5,-2,5,-4,-4,-2,4,null,0,5,0,-2,2,2,2,2,4,-4,2,null,-3,-4,3,-4,null,null,4,2147483647,3,-3,4,null,-4,-1,-4,1,0,-5,-2,-2,-2,-2,null],"l3_i64":["9007199254740994","-4611686018427387904","-4","9007199254740993",null,"9007199254740994","9007199254740994","9007199254740994","9007199254740994","9007199254740994","-4611686018427387904","-4611686018427387904","-4611686018427387904","-4611686018427387904","-3","9007199254740993","9007199254740995","-4611686018427387904","0","-4","-4611686018427387904","-4611686018427387904","-4611686018427387904","-4611686018427387904","-4611686018427387904","-4611686018427387904","-4611686018427387904","9007199254740992","9007199254740994","-1","-5","-2","9007199254740992","9007199254740993","5","9007199254740995",null,"-4611686018427387904","9007199254740992","2","2","2","2","9007199254740994","0","9007199254740994","-4611686018427387904","9007199254740994","-4611686018427387904","-4611686018427387904","5","9007199254740994","-4611686018427387904","9007199254740993","9007199254740993","9007199254740993","-4611686018427387904",null,"-1","4","9007199254740995","9007199254740993","2","5","9007199254740992","-4611686018427387904","-4611686018427387904","-4611686018427387904","-4611686018427387904","-4"],"l4_f64":[-4.0,-1.0,0.0,-70.308,77.387,-28.825,50.593,50.593,50.593,50.593,-4.0,-4.0,-4.0,-4.0,-82.514,-32.141,-2.0,-3.0,4.0,-4.0,-1.0,-1.0,-1.0,-1.0,53.602,65.505,-91.32,82.616,-5.335,-2.0,-18.204,1.0,-3.0,-32.172,-3.0,0.0,88.273,"-Infinity",4.0,12.631,12.631,12.631,12.631,1.0,-58.168,-50.84,-5.522,43.594,96.263,4.0,4.0,14.091,3.0,5.058,-3.0,-4.0,23.268,10.379,78.576,42.139,36.501,6.087,-34.883,-1.0,1.0,4.0,4.0,4.0,4.0,16.477],"l5_utf8":["a","","c","B","c","a","zz","zz","zz","zz","a","a","a","a","é","a","aa","zz","B","zz",null,null,null,null,"","a","é","c",null,"aa","é","c","c","","c","zz","😀",null,"","b","b","b","b","😀","c","ab","B",null,"😀","b","B","ab","B","","","ab","b","a",null,"ab","zz","","😀","b","c","zz","zz","zz","zz","c"],"r0_bool":[null,null,null,null,true,false,true,true,null,false,true,true,null,false,null,null,true,null,null,true,true,true,null,false,null,true,true,true,null,null,true,null,null,null,null,null,true,null,true,true,true,null,false,null,null,true,null,null,null,null,true,null,null,null,null,null,false,true,null,null,true,null,null,false,true,true,true,null,false,null]}}},{"id":"random/0033","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_i32","type":"i32"},{"name":"l5_bool","type":"bool"},{"name":"l6_utf8","type":"utf8"}],"columns":{"l0_utf8":["a",null,null,"ab","ab"],"l1_i32":[null,-5,0,null,0],"l2_f64":[1.57,61.787,4.0,null,-55.982],"l3_utf8":["b","ab","😀","B","aa"],"l4_i32":[2,2,null,null,-2147483648],"l5_bool":[false,true,false,null,true],"l6_utf8":["é","B","b","aa","aa"]}}},"ops":[{"op":"unique","columns":["l0_utf8"]},{"op":"filter","predicate":{"fn":"lte","args":[{"fn":"mul","args":[{"lit":2},{"col":"l2_f64"}]},{"fn":"fillNull","args":[{"col":"l4_i32"},{"lit":0}]}]}},{"op":"withColumns","exprs":{"x2":{"fn":"add","args":[{"col":"l4_i32"},{"fn":"mul","args":[{"col":"l1_i32"},{"col":"l4_i32"}]}]}}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_i32","type":"i32"},{"name":"l5_bool","type":"bool"},{"name":"l6_utf8","type":"utf8"},{"name":"x2","type":"i32"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_f64":[],"l3_utf8":[],"l4_i32":[],"l5_bool":[],"l6_utf8":[],"x2":[]}}},{"id":"random/0034","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i64","type":"i64"},{"name":"l3_f64","type":"f64"},{"name":"l4_i32","type":"i32"},{"name":"l5_bool","type":"bool"}],"columns":{"l0_utf8":["😀","ab","c","a","a","B","😀","ab","c","B","","","é","B","aa","zz","","c","b","é"],"l1_i32":[2,-4,null,511,4,-2,5,-3,-1,-2,1,4,0,null,1,null,-4,1,-1,3],"l2_i64":["9007199254740992","-4611686018427387904","5","-4611686018427387904","-4611686018427387904","5",null,"-2","-1","4","-1",null,"9007199254740992","1","9007199254740995",null,"9007199254740993","-4611686018427387904","9007199254740994","-4611686018427387904"],"l3_f64":[3.0,-4.0,null,-21.919,-2.0,-59.552,4.0,null,3.0,null,1.573,3.0,2.0,-24.37,-1.0,1.0,-61.886,-71.333,"-0",null],"l4_i32":[4,-1,1,-4,1,0,5,null,2,2147483647,2,-2,2,-3,-2,-1,2,-2,-4,-2147483648],"l5_bool":[true,false,false,false,false,true,null,true,null,false,true,true,false,true,false,false,true,false,true,true]}}},"ops":[{"op":"withColumns","exprs":{"x0":{"col":"l3_f64"}}},{"op":"groupBy","keys":["l2_i64"],"aggs":{"n1":{"agg":"len"},"a1_0":{"agg":"max","arg":{"col":"l1_i32"}},"a1_1":{"agg":"min","arg":{"col":"l4_i32"}},"a1_2":{"agg":"last","arg":{"col":"l5_bool"}}}}],"expected":{"fields":[{"name":"l2_i64","type":"i64"},{"name":"n1","type":"u32"},{"name":"a1_0","type":"i32"},{"name":"a1_1","type":"i32"},{"name":"a1_2","type":"bool"}],"columns":{"l2_i64":["9007199254740992","-4611686018427387904","5",null,"-2","-1","4","1","9007199254740995","9007199254740993","9007199254740994"],"n1":[2,5,2,3,1,2,1,1,1,1,1],"a1_0":[2,511,-2,5,-3,1,-2,null,1,-4,-1],"a1_1":[2,-2147483648,0,-2,null,2,2147483647,-3,-2,2,-4],"a1_2":[false,true,true,false,true,true,false,true,false,true,true]}}},{"id":"random/0035","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_f64","type":"f64"},{"name":"l4_i64","type":"i64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_f64":[],"l3_f64":[],"l4_i64":[]}},"right":{"fields":[{"name":"l4_i64","type":"i64"},{"name":"r0_utf8","type":"utf8"}],"columns":{"r0_utf8":["a","aa","😀"],"l4_i64":[null,"-4","-2"]}}},"ops":[{"op":"sort","keys":[{"expr":{"col":"l0_utf8"},"descending":true},{"expr":{"col":"l4_i64"},"descending":true}],"nullsLast":true},{"op":"join","right":"right","on":["l4_i64"],"how":"anti"}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_f64","type":"f64"},{"name":"l4_i64","type":"i64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_f64":[],"l3_f64":[],"l4_i64":[]}}},{"id":"random/0036","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_bool","type":"bool"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_f64","type":"f64"},{"name":"l6_f64","type":"f64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_utf8":[],"l3_bool":[],"l4_utf8":[],"l5_f64":[],"l6_f64":[]}}},"ops":[{"op":"filter","predicate":{"fn":"isNotNull","args":[{"col":"l3_bool"}]}},{"op":"withColumns","exprs":{"x1":{"fn":"sub","args":[{"fn":"fillNull","args":[{"col":"l5_f64"},{"lit":1.5}]},{"col":"l5_f64"}]}}},{"op":"sort","keys":[{"expr":{"col":"l4_utf8"},"descending":true},{"expr":{"col":"l0_utf8"},"descending":true}],"nullsLast":true}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_bool","type":"bool"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_f64","type":"f64"},{"name":"l6_f64","type":"f64"},{"name":"x1","type":"f64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_utf8":[],"l3_bool":[],"l4_utf8":[],"l5_f64":[],"l6_f64":[],"x1":[]}}},{"id":"random/0037","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_i32","type":"i32"}],"columns":{"l0_utf8":["zz"],"l1_i32":[3],"l2_utf8":[""],"l3_utf8":[null],"l4_utf8":[""],"l5_i32":[-5]}}},"ops":[{"op":"groupBy","keys":["l4_utf8"],"aggs":{"n0":{"agg":"len"},"a0_0":{"agg":"sum","arg":{"col":"l5_i32"}},"a0_1":{"agg":"std","arg":{"col":"l5_i32"}}}},{"op":"groupBy","keys":["l4_utf8"],"aggs":{"n1":{"agg":"len"},"a1_0":{"agg":"count","arg":{"col":"a0_0"}}}}],"expected":{"fields":[{"name":"l4_utf8","type":"utf8"},{"name":"n1","type":"u32"},{"name":"a1_0","type":"u32"}],"columns":{"l4_utf8":[""],"n1":[1],"a1_0":[1]}}},{"id":"random/0038","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_utf8","type":"utf8"},{"name":"l6_bool","type":"bool"}],"columns":{"l0_utf8":["","aa","a","zz","c","B","é","aa","a","c","😀","aa","b","a","B","aa","a","aa","é","ab"],"l1_i32":[5,-5,4,2,-5,3,-1,3,4,-4,-5,5,4,3,2,4,3,-5,-1,-1],"l2_f64":[4.0,2.0,-57.377,-6.232,59.544,-76.314,14.56,1.0,13.915,-4.667,14.517,0.0,82.457,4.0,-44.608,-2.0,3.0,-34.946,29.93,-92.605],"l3_utf8":["ab","aa","é","aa","aa","b","😀","b","b","😀","c","B","ab","zz","ab","B","c","","😀","😀"],"l4_utf8":["","c","aa","aa","a","ab","b","zz","a","zz","c","c","","zz","a","c","B","ab","aa","B"],"l5_utf8":["c","😀","ab","b","B","c","c","a","aa","B","b","","zz","a","c","c","B","aa","a","ab"],"l6_bool":[true,false,true,true,true,false,true,false,false,true,true,false,true,false,false,false,false,true,true,true]}}},"ops":[{"op":"sort","keys":[{"expr":{"col":"l1_i32"},"descending":true}],"nullsLast":true},{"op":"withColumns","exprs":{"x1":{"fn":"neg","args":[{"col":"l1_i32"}]}}},{"op":"filter","predicate":{"fn":"or","args":[{"fn":"gte","args":[{"col":"l5_utf8"},{"lit":"😀"}]},{"fn":"gt","args":[{"col":"x1"},{"col":"l1_i32"}]}]}},{"op":"filter","predicate":{"fn":"lte","args":[{"fn":"neg","args":[{"lit":2.5}]},{"fn":"abs","args":[{"lit":2.5}]}]}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_utf8","type":"utf8"},{"name":"l6_bool","type":"bool"},{"name":"x1","type":"i32"}],"columns":{"l0_utf8":["é","é","ab","c","aa","c","😀","aa"],"l1_i32":[-1,-1,-1,-4,-5,-5,-5,-5],"l2_f64":[14.56,29.93,-92.605,-4.667,2.0,59.544,14.517,-34.946],"l3_utf8":["😀","😀","😀","😀","aa","aa","c",""],"l4_utf8":["b","aa","B","zz","c","a","c","ab"],"l5_utf8":["c","a","ab","B","😀","B","b","aa"],"l6_bool":[true,true,true,true,false,true,true,true],"x1":[1,1,1,4,5,5,5,5]}}},{"id":"random/0039","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_utf8","type":"utf8"}],"columns":{"l0_utf8":[null,"b",null,"","😀",null,"b",null,"aa","b",null,null,"",null,"a",null,"c","aa","é","a",null,"c",null,null,null,"é","😀","é","",null,null,null,"é","B",null,"B",null,"c",null,null,"b","aa","aa","B","B","zz","é",null,"é",null,"😀","B","",null,"B",null,"b",null,null,"b"],"l1_i32":[null,null,-5,null,null,1,null,null,-4,null,-1,-2,-2,0,-2,5,null,2147483647,null,null,-1,-5,-1,4,0,null,4,3,null,2,3,-4,null,null,null,-3,755,-1,-1,null,3,5,-1,5,-3,null,null,4,null,2,1,859,2147483647,null,4,-2,3,-5,null,0],"l2_f64":[null,null,4.0,-77.618,0.0,95.472,-88.487,-53.32,-95.121,-1.0,null,88.779,null,null,-17.286,82.866,98.264,null,null,-1.0,null,"-Infinity",-85.425,4.0,null,"NaN",-27.37,-3.0,"NaN",null,null,-4.0,-1.0,null,61.758,null,-82.626,-3.0,null,27.566,null,4.0,-4.0,2.0,null,null,-4.0,94.261,-4.0,52.594,91.992,null,64.752,null,20.549,81.434,null,"NaN",-18.987,null],"l3_utf8":["é","B","b","b","zz",null,null,"B",null,"a","é",null,"","zz","ab","😀","a","😀","c",null,"ab","a","aa",null,"😀","b","😀","B","aa","","",null,null,"c","","a","😀","a",null,"c","aa","😀","aa","c","aa","aa","B",null,"B","zz","c",null,"","😀","",null,null,"","aa","B"]}}},"ops":[{"op":"filter","predicate":{"fn":"eq","args":[{"col":"l0_utf8"},{"lit":"zz"}]}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_utf8","type":"utf8"}],"columns":{"l0_utf8":["zz"],"l1_i32":[null],"l2_f64":[null],"l3_utf8":["aa"]}}},{"id":"random/0040","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_i32","type":"i32"}],"columns":{"l0_utf8":["é"],"l1_i32":[-5],"l2_utf8":["zz"],"l3_utf8":["ab"],"l4_utf8":["B"],"l5_i32":[1]}}},"ops":[{"op":"groupBy","keys":["l5_i32"],"aggs":{"n0":{"agg":"len"},"a0_0":{"agg":"max","arg":{"col":"l3_utf8"}},"a0_1":{"agg":"min","arg":{"col":"l4_utf8"}}}},{"op":"withColumns","exprs":{"x1":{"fn":"isIn","args":[{"col":"a0_0"}],"values":["zz","","aa"]}}},{"op":"head","n":7}],"expected":{"fields":[{"name":"l5_i32","type":"i32"},{"name":"n0","type":"u32"},{"name":"a0_0","type":"utf8"},{"name":"a0_1","type":"utf8"},{"name":"x1","type":"bool"}],"columns":{"l5_i32":[1],"n0":[1],"a0_0":["ab"],"a0_1":["B"],"x1":[false]}}},{"id":"random/0041","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_utf8","type":"utf8"}],"columns":{"l0_utf8":[null,null,null,"a","aa","c",null,"","c","c","😀",null,"zz","","aa","😀","aa","😀","ab","b",null,"zz","c","aa","","","b","aa","😀","",null,null,"c",null,"😀","zz","😀","c",null,"ab",null,null,null,"B","c",null,"a","😀","c",null,"é","a","aa",null,"aa","é","c","B","ab",null],"l1_i32":[-5,null,2,1,-4,-2147483648,-4,null,-2,null,null,3,null,null,-1,5,-4,null,2,null,0,null,null,null,3,null,null,-2147483648,-3,-5,null,5,4,-2,null,-2,null,null,null,-2,null,null,null,null,null,368,null,4,-2,4,3,2,0,null,1,null,339,null,4,null],"l2_i32":[-3,-3,null,null,2,-1,5,null,null,0,null,null,null,-1,-3,2,null,null,3,-2,2,3,null,2147483647,4,-5,0,0,4,null,-1,32,-2,0,null,5,-5,-3,-1,null,-3,4,null,-2,null,null,null,-2,1,null,-2,-777,2,null,-5,2,1,-4,-5,-3],"l3_utf8":["c","","😀","ab","B",null,"😀","aa","a","aa","B","B",null,"😀",null,null,null,null,null,"c","zz",null,"aa",null,null,"",null,"aa","","é",null,null,null,"aa","c",null,null,"zz",null,"b","c",null,null,"😀","b","a",null,"b",null,null,"ab","b","é",null,null,"c","c",null,"a",null]}}},"ops":[{"op":"filter","predicate":{"fn":"neq","args":[{"fn":"mul","args":[{"col":"l1_i32"},{"col":"l1_i32"}]},{"fn":"mul","args":[{"lit":-1},{"col":"l1_i32"}]}]}},{"op":"groupBy","keys":["l1_i32"],"aggs":{"n1":{"agg":"len"},"a1_0":{"agg":"last","arg":{"col":"l2_i32"}}}},{"op":"sort","keys":[{"expr":{"col":"n1"},"descending":true}],"nullsLast":false}],"expected":{"fields":[{"name":"l1_i32","type":"i32"},{"name":"n1","type":"u32"},{"name":"a1_0","type":"i32"}],"columns":{"l1_i32":[-2,4,2,-4,3,-5,1,-2147483648,5,-3,368,339],"n1":[5,4,3,3,3,2,2,2,2,1,1,1],"a1_0":[1,-5,-777,null,-2,null,-5,0,32,4,null,1]}}},{"id":"random/0042","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_f64","type":"f64"},{"name":"l5_utf8","type":"utf8"},{"name":"l6_i64","type":"i64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_utf8":[],"l3_utf8":[],"l4_f64":[],"l5_utf8":[],"l6_i64":[]}}},"ops":[{"op":"withColumns","exprs":{"x0":{"col":"l4_f64"}}},{"op":"filter","predicate":{"fn":"neq","args":[{"col":"l4_f64"},{"lit":2.5}]}},{"op":"head","n":3}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_f64","type":"f64"},{"name":"l5_utf8","type":"utf8"},{"name":"l6_i64","type":"i64"},{"name":"x0","type":"f64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_utf8":[],"l3_utf8":[],"l4_f64":[],"l5_utf8":[],"l6_i64":[],"x0":[]}}},{"id":"random/0043","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_utf8","type":"utf8"}],"columns":{"l0_utf8":[""],"l1_i32":[null],"l2_utf8":["B"],"l3_utf8":["a"],"l4_utf8":["B"],"l5_utf8":[""]}}},"ops":[{"op":"unique"},{"op":"withColumns","exprs":{"x1":{"fn":"add","args":[{"fn":"sub","args":[{"col":"l1_i32"},{"col":"l1_i32"}]},{"fn":"div","args":[{"col":"l1_i32"},{"col":"l1_i32"}]}]}}},{"op":"sort","keys":[{"expr":{"col":"l4_utf8"},"descending":false},{"expr":{"col":"l5_utf8"},"descending":false}],"nullsLast":false}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_utf8","type":"utf8"},{"name":"x1","type":"f64"}],"columns":{"l0_utf8":[""],"l1_i32":[null],"l2_utf8":["B"],"l3_utf8":["a"],"l4_utf8":["B"],"l5_utf8":[""],"x1":[null]}}},{"id":"random/0044","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_i32","type":"i32"}],"columns":{"l0_utf8":["b","aa","b","ab","a","a","ab","ab","b","ab","b","b","é","b","a","zz","zz","😀","ab","é"],"l1_i32":[3,-5,-5,-4,5,4,4,2,5,-1,-3,4,5,0,1,2,2,5,-5,-2147483648],"l2_f64":[-51.166,-16.398,-4.0,54.999,"NaN",0.0,0.0,-3.0,-5.676,3.0,4.0,-3.0,50.057,-4.0,-1.0,3.0,-2.0,-1.0,-4.0,-2.0],"l3_i32":[0,1,0,-2,-1,-1,3,2,5,-2147483648,5,-5,-2147483648,-4,3,-1,-3,-2,3,2]}}},"ops":[{"op":"head","n":2},{"op":"filter","predicate":{"fn":"lt","args":[{"fn":"abs","args":[{"col":"l3_i32"}]},{"fn":"fillNull","args":[{"lit":2},{"lit":1.5}]}]}},{"op":"filter","predicate":{"fn":"lte","args":[{"fn":"sub","args":[{"col":"l3_i32"},{"col":"l3_i32"}]},{"fn":"fillNull","args":[{"col":"l2_f64"},{"lit":1.5}]}]}},{"op":"withColumns","exprs":{"x3":{"fn":"add","args":[{"fn":"fillNull","args":[{"col":"l3_i32"},{"lit":0}]},{"fn":"mod","args":[{"col":"l2_f64"},{"col":"l3_i32"}]}]}}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_i32","type":"i32"},{"name":"x3","type":"f64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_f64":[],"l3_i32":[],"x3":[]}}},{"id":"random/0045","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_bool","type":"bool"}],"columns":{"l0_utf8":["b"],"l1_i32":[-3],"l2_i32":[null],"l3_bool":[false]}}},"ops":[{"op":"head","n":10}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_bool","type":"bool"}],"columns":{"l0_utf8":["b"],"l1_i32":[-3],"l2_i32":[null],"l3_bool":[false]}}},{"id":"random/0046","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_utf8","type":"utf8"}],"columns":{"l0_utf8":["","","zz","zz","a","","c","😀","zz","aa","zz","a","b","😀","c","é","😀","","😀",""],"l1_i32":[-5,4,5,3,1,213,-1,-3,-4,0,-3,2,4,-1,-2,5,-5,-2,0,4],"l2_f64":[2.0,71.906,9.534,0.0,86.177,1.774,-1.0,51.274,64.903,-3.0,-2.0,64.553,-3.0,-4.0,99.817,-38.348,-1.844,-56.828,23.585,35.754],"l3_utf8":["B","a","😀","ab","é","zz","B","a","😀","ab","a","c","c","zz","é","é","é","ab","B","c"]}},"right":{"fields":[{"name":"l3_utf8","type":"utf8"},{"name":"r0_utf8","type":"utf8"}],"columns":{"r0_utf8":["😀","é","B","B","aa","é","zz","😀","ab","aa","a","c","zz","😀","zz","aa","zz","","aa","","B","😀","c","aa","zz","ab","b","b","zz","c"],"l3_utf8":["ab","ab","c","é","é","B","c","é","ab","é","😀","B","a","c","B","ab","é","ab","c","c","a","ab","é","😀","c","é","B","ab","a","é"]}}},"ops":[{"op":"groupBy","keys":["l3_utf8"],"aggs":{"n0":{"agg":"len"},"a0_0":{"agg":"count","arg":{"col":"l1_i32"}}}},{"op":"sort","keys":[{"expr":{"col":"l3_utf8"},"descending":false},{"expr":{"col":"n0"},"descending":false}],"nullsLast":true},{"op":"join","right":"right","on":["l3_utf8"],"how":"semi"},{"op":"unique"}],"expected":{"fields":[{"name":"l3_utf8","type":"utf8"},{"name":"n0","type":"u32"},{"name":"a0_0","type":"u32"}],"columns":{"l3_utf8":["B","a","ab","c","é","😀"],"n0":[3,3,3,3,4,2],"a0_0":[3,3,3,3,4,2]}}},{"id":"random/0047","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_f64","type":"f64"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_bool","type":"bool"},{"name":"l6_utf8","type":"utf8"}],"columns":{"l0_utf8":["a"],"l1_i32":[-2],"l2_f64":[-12.35],"l3_f64":[-58.837],"l4_utf8":["é"],"l5_bool":[false],"l6_utf8":[null]}}},"ops":[{"op":"groupBy","keys":["l6_utf8"],"aggs":{"n0":{"agg":"len"},"a0_0":{"agg":"nUnique","arg":{"col":"l3_f64"}},"a0_1":{"agg":"median","arg":{"col":"l1_i32"}},"a0_2":{"agg":"min","arg":{"col":"l1_i32"}}}},{"op":"head","n":3},{"op":"groupBy","keys":["a0_0"],"aggs":{"n2":{"agg":"len"},"a2_0":{"agg":"last","arg":{"col":"n0"}},"a2_1":{"agg":"count","arg":{"col":"a0_2"}}}}],"expected":{"fields":[{"name":"a0_0","type":"u32"},{"name":"n2","type":"u32"},{"name":"a2_0","type":"u32"},{"name":"a2_1","type":"u32"}],"columns":{"a0_0":[1],"n2":[1],"a2_0":[1],"a2_1":[1]}}},{"id":"random/0048","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_f64","type":"f64"},{"name":"l4_i64","type":"i64"},{"name":"l5_i32","type":"i32"},{"name":"l6_f64","type":"f64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_bool":[],"l3_f64":[],"l4_i64":[],"l5_i32":[],"l6_f64":[]}}},"ops":[{"op":"withColumns","exprs":{"x0":{"fn":"add","args":[{"lit":0.5},{"fn":"neg","args":[{"col":"l5_i32"}]}]}}},{"op":"filter","predicate":{"fn":"gte","args":[{"col":"l6_f64"},{"fn":"mul","args":[{"lit":2},{"col":"l1_i32"}]}]}},{"op":"withColumns","exprs":{"x2":{"lit":2}}},{"op":"filter","predicate":{"fn":"eq","args":[{"col":"l4_i64"},{"lit":"-1","type":"i64"}]}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_f64","type":"f64"},{"name":"l4_i64","type":"i64"},{"name":"l5_i32","type":"i32"},{"name":"l6_f64","type":"f64"},{"name":"x0","type":"f64"},{"name":"x2","type":"i32"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_bool":[],"l3_f64":[],"l4_i64":[],"l5_i32":[],"l6_f64":[],"x0":[],"x2":[]}}},{"id":"random/0049","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_f64","type":"f64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_i32":[],"l3_f64":[]}},"right":{"fields":[{"name":"l1_i32","type":"i32"},{"name":"r0_bool","type":"bool"}],"columns":{"r0_bool":[],"l1_i32":[]}}},"ops":[{"op":"groupBy","keys":["l1_i32"],"aggs":{"n0":{"agg":"len"},"a0_0":{"agg":"count","arg":{"col":"l3_f64"}},"a0_1":{"agg":"first","arg":{"col":"l3_f64"}},"a0_2":{"agg":"last","arg":{"col":"l3_f64"}}}},{"op":"join","right":"right","on":["l1_i32"],"how":"left"}],"expected":{"fields":[{"name":"l1_i32","type":"i32"},{"name":"n0","type":"u32"},{"name":"a0_0","type":"u32"},{"name":"a0_1","type":"f64"},{"name":"a0_2","type":"f64"},{"name":"r0_bool","type":"bool"}],"columns":{"l1_i32":[],"n0":[],"a0_0":[],"a0_1":[],"a0_2":[],"r0_bool":[]}}},{"id":"random/0050","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_f64","type":"f64"}],"columns":{"l0_utf8":["b","B","😀","ab","é"],"l1_i32":[-2,2,-2147483648,4,3],"l2_utf8":["","","aa","zz",""],"l3_utf8":["","zz","ab","","ab"],"l4_f64":[3.0,-17.346,-4.0,43.76,3.0]}}},"ops":[{"op":"filter","predicate":{"fn":"or","args":[{"fn":"lt","args":[{"col":"l4_f64"},{"col":"l1_i32"}]},{"fn":"isNull","args":[{"col":"l1_i32"}]}]}},{"op":"sort","keys":[{"expr":{"col":"l1_i32"},"descending":false}],"nullsLast":true}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_f64","type":"f64"}],"columns":{"l0_utf8":["B"],"l1_i32":[2],"l2_utf8":[""],"l3_utf8":["zz"],"l4_f64":[-17.346]}}},{"id":"random/0051","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_f64","type":"f64"}],"columns":{"l0_utf8":["😀"],"l1_i32":[-2],"l2_f64":[57.877],"l3_f64":[0.0]}}},"ops":[{"op":"withColumns","exprs":{"x0":{"fn":"and","args":[{"fn":"strStartsWith","args":[{"col":"l0_utf8"}],"prefix":"é"},{"fn":"gte","args":[{"col":"l3_f64"},{"col":"l1_i32"}]}]}}},{"op":"sort","keys":[{"expr":{"col":"l3_f64"},"descending":false},{"expr":{"col":"l2_f64"},"descending":false}],"nullsLast":true}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_f64","type":"f64"},{"name":"x0","type":"bool"}],"columns":{"l0_utf8":["😀"],"l1_i32":[-2],"l2_f64":[57.877],"l3_f64":[0.0],"x0":[false]}}},{"id":"random/0052","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_f64","type":"f64"},{"name":"l4_i32","type":"i32"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_utf8":[],"l3_f64":[],"l4_i32":[]}}},"ops":[{"op":"filter","predicate":{"fn":"neq","args":[{"fn":"neg","args":[{"col":"l1_i32"}]},{"col":"l3_f64"}]}},{"op":"groupBy","keys":["l4_i32"],"aggs":{"n1":{"agg":"len"},"a1_0":{"agg":"nUnique","arg":{"col":"l1_i32"}},"a1_1":{"agg":"nUnique","arg":{"col":"l3_f64"}}}}],"expected":{"fields":[{"name":"l4_i32","type":"i32"},{"name":"n1","type":"u32"},{"name":"a1_0","type":"u32"},{"name":"a1_1","type":"u32"}],"columns":{"l4_i32":[],"n1":[],"a1_0":[],"a1_1":[]}}},{"id":"random/0053","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i64","type":"i64"},{"name":"l3_utf8","type":"utf8"}],"columns":{"l0_utf8":["B"],"l1_i32":[2],"l2_i64":["9007199254740993"],"l3_utf8":["zz"]}}},"ops":[{"op":"sort","keys":[{"expr":{"col":"l3_utf8"},"descending":true}],"nullsLast":true}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i64","type":"i64"},{"name":"l3_utf8","type":"utf8"}],"columns":{"l0_utf8":["B"],"l1_i32":[2],"l2_i64":["9007199254740993"],"l3_utf8":["zz"]}}},{"id":"random/0054","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_f64","type":"f64"}],"columns":{"l0_utf8":["aa","a","b","zz","B","zz","a","b","","zz","é","a","c","B","aa",null,"","aa","zz","ab"],"l1_i32":[3,1,null,null,-5,null,3,-1,-1,-4,null,3,null,4,-3,null,-5,-4,5,0],"l2_f64":[4.45,42.701,null,null,4.0,-61.538,-72.423,null,null,32.325,null,null,-1.0,null,null,-4.052,-1.0,-1.0,-1.0,-4.0],"l3_f64":[-2.0,-3.0,0.0,0.0,-2.0,null,0.0,null,null,4.0,1.0,-2.0,-1.0,-3.0,84.114,null,79.785,null,null,1.0]}},"right":{"fields":[{"name":"l1_i32","type":"i32"},{"name":"r0_bool","type":"bool"}],"columns":{"r0_bool":[false,false,true,true,false,true,false,false,true,null],"l1_i32":[-4,1,-3,3,null,-1,null,-1,-1,null]}}},"ops":[{"op":"sort","keys":[{"expr":{"col":"l3_f64"},"descending":true},{"expr":{"col":"l2_f64"},"descending":false}],"nullsLast":false},{"op":"withColumns","exprs":{"x1":{"col":"l2_f64"}}},{"op":"join","right":"right","on":["l1_i32"],"how":"full"}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_f64","type":"f64"},{"name":"x1","type":"f64"},{"name":"r0_bool","type":"bool"}],"columns":{"l0_utf8":["b","b","b","","","","zz",null,"aa","zz","aa","","zz","é","ab","b","zz","a","c","a","B","aa","B","a",null,null,null],"l1_i32":[-1,-1,-1,-1,-1,-1,null,null,-4,5,-3,-5,-4,null,0,null,null,3,null,3,-5,3,4,1,null,null,null],"l2_f64":[null,null,null,null,null,null,-61.538,-4.052,-1.0,-1.0,null,-1.0,32.325,null,-4.0,null,null,-72.423,-1.0,null,4.0,4.45,null,42.701,null,null,null],"l3_f64":[null,null,null,null,null,null,null,null,null,null,84.114,79.785,4.0,1.0,1.0,0.0,0.0,0.0,-1.0,-2.0,-2.0,-2.0,-3.0,-3.0,null,null,null],"x1":[null,null,null,null,null,null,-61.538,-4.052,-1.0,-1.0,null,-1.0,32.325,null,-4.0,null,null,-72.423,-1.0,null,4.0,4.45,null,42.701,null,null,null],"r0_bool":[true,false,true,true,false,true,null,null,false,null,true,null,false,null,null,null,null,true,null,true,null,true,null,false,false,false,null]}}},{"id":"random/0055","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_utf8","type":"utf8"}],"columns":{"l0_utf8":["zz","aa","é",null,"aa"],"l1_i32":[2,5,-1,5,5],"l2_utf8":["B","B","B","é","a"],"l3_utf8":["zz","ab","b","aa","😀"],"l4_utf8":["ab","é","é","é","ab"]}}},"ops":[{"op":"groupBy","keys":["l0_utf8"],"aggs":{"n0":{"agg":"len"},"a0_0":{"agg":"min","arg":{"col":"l3_utf8"}}}},{"op":"sort","keys":[{"expr":{"col":"n0"},"descending":true},{"expr":{"col":"l0_utf8"},"descending":true}],"nullsLast":true},{"op":"groupBy","keys":["l0_utf8"],"aggs":{"n2":{"agg":"len"},"a2_0":{"agg":"count","arg":{"col":"n0"}},"a2_1":{"agg":"min","arg":{"col":"a0_0"}},"a2_2":{"agg":"first","arg":{"col":"a0_0"}}}},{"op":"filter","predicate":{"fn":"eq","args":[{"col":"a2_1"},{"lit":""}]}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"n2","type":"u32"},{"name":"a2_0","type":"u32"},{"name":"a2_1","type":"utf8"},{"name":"a2_2","type":"utf8"}],"columns":{"l0_utf8":[],"n2":[],"a2_0":[],"a2_1":[],"a2_2":[]}}},{"id":"random/0056","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i64","type":"i64"},{"name":"l3_i32","type":"i32"},{"name":"l4_f64","type":"f64"},{"name":"l5_bool","type":"bool"},{"name":"l6_utf8","type":"utf8"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_i64":[],"l3_i32":[],"l4_f64":[],"l5_bool":[],"l6_utf8":[]}},"right":{"fields":[{"name":"l6_utf8","type":"utf8"},{"name":"r0_utf8","type":"utf8"}],"columns":{"r0_utf8":["😀","c","c","B","😀","b","zz",null,"",null,"aa","😀","b","b",null,"a","","","é","",null,"é","a","B","😀","ab","😀",null,"é","é"],"l6_utf8":["😀","ab","a","😀","😀","c","ab","c","c","a","c","ab","a","ab","😀","c","a","","a","","ab","","😀","ab","a","","😀","c","a",""]}}},"ops":[{"op":"groupBy","keys":["l6_utf8"],"aggs":{"n0":{"agg":"len"},"a0_0":{"agg":"count","arg":{"col":"l0_utf8"}},"a0_1":{"agg":"last","arg":{"col":"l4_f64"}}}},{"op":"join","right":"right","on":["l6_utf8"],"how":"anti"},{"op":"head","n":1},{"op":"head","n":3}],"expected":{"fields":[{"name":"l6_utf8","type":"utf8"},{"name":"n0","type":"u32"},{"name":"a0_0","type":"u32"},{"name":"a0_1","type":"f64"}],"columns":{"l6_utf8":[],"n0":[],"a0_0":[],"a0_1":[]}}},{"id":"random/0057","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_bool","type":"bool"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_bool","type":"bool"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_utf8":[],"l3_bool":[],"l4_utf8":[],"l5_bool":[]}}},"ops":[{"op":"head","n":2},{"op":"head","n":8}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_bool","type":"bool"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_bool","type":"bool"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_utf8":[],"l3_bool":[],"l4_utf8":[],"l5_bool":[]}}},{"id":"random/0058","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_i64","type":"i64"},{"name":"l4_utf8","type":"utf8"}],"columns":{"l0_utf8":["zz"],"l1_i32":[-2],"l2_i32":[null],"l3_i64":["-2"],"l4_utf8":["aa"]}}},"ops":[{"op":"unique"},{"op":"head","n":0}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_i64","type":"i64"},{"name":"l4_utf8","type":"utf8"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_i32":[],"l3_i64":[],"l4_utf8":[]}}},{"id":"random/0059","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_utf8","type":"utf8"}],"columns":{"l0_utf8":["a","",null,null,"ab",null,"B","é","",null,null,null,null,"c","",null,"é","zz",null,null],"l1_i32":[5,-5,null,2,null,0,0,null,-2,-1,null,383,-4,-3,5,0,4,-3,null,-4],"l2_bool":[true,false,null,true,true,null,true,null,false,false,false,false,false,null,false,false,false,false,false,false],"l3_utf8":["","aa","ab","a","aa","","é","b","","a",null,"aa","b",null,null,null,"zz","aa","b",null]}}},"ops":[{"op":"unique","columns":["l0_utf8"]}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_utf8","type":"utf8"}],"columns":{"l0_utf8":["a","",null,"ab","B","é","c","zz"],"l1_i32":[5,-5,null,null,0,null,-3,-3],"l2_bool":[true,false,null,true,true,null,null,false],"l3_utf8":["","aa","ab","aa","é","b",null,"aa"]}}},{"id":"random/0060","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_f64","type":"f64"},{"name":"l4_f64","type":"f64"},{"name":"l5_i64","type":"i64"},{"name":"l6_i32","type":"i32"}],"columns":{"l0_utf8":["aa",null,"a",null,"ab",null,null,null,"é",null,"",null,null,null,null,"ab","😀","","😀",null],"l1_i32":[4,null,null,-3,null,null,4,null,5,-3,-4,-5,null,-1,3,0,-2,4,-3,3],"l2_utf8":["😀","",null,"aa",null,null,null,"a","",null,"a","ab",null,"é",null,"é","aa","b","😀","c"],"l3_f64":[null,null,-14.078,58.329,"NaN",null,null,null,1.0,2.0,0.0,0.0,-83.448,-76.53,null,-97.317,16.945,1.0,3.0,null],"l4_f64":[-45.717,null,1.0,"NaN",2.0,43.686,"NaN",0.0,2.0,null,null,89.606,null,2.0,-27.805,-3.845,72.75,7.614,84.651,-3.0],"l5_i64":[null,"9007199254740993","2","-5","-1",null,null,"-4611686018427387904","4","9007199254740992","9007199254740993",null,null,null,"9007199254740993",null,"-1",null,null,"9007199254740992"],"l6_i32":[null,null,-1,5,2,3,3,null,5,3,3,2,null,null,null,null,-4,-1,null,null]}}},"ops":[{"op":"unique"}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_f64","type":"f64"},{"name":"l4_f64","type":"f64"},{"name":"l5_i64","type":"i64"},{"name":"l6_i32","type":"i32"}],"columns":{"l0_utf8":["aa",null,"a",null,"ab",null,null,null,"é",null,"",null,null,null,null,"ab","😀","","😀",null],"l1_i32":[4,null,null,-3,null,null,4,null,5,-3,-4,-5,null,-1,3,0,-2,4,-3,3],"l2_utf8":["😀","",null,"aa",null,null,null,"a","",null,"a","ab",null,"é",null,"é","aa","b","😀","c"],"l3_f64":[null,null,-14.078,58.329,"NaN",null,null,null,1.0,2.0,0.0,0.0,-83.448,-76.53,null,-97.317,16.945,1.0,3.0,null],"l4_f64":[-45.717,null,1.0,"NaN",2.0,43.686,"NaN",0.0,2.0,null,null,89.606,null,2.0,-27.805,-3.845,72.75,7.614,84.651,-3.0],"l5_i64":[null,"9007199254740993","2","-5","-1",null,null,"-4611686018427387904","4","9007199254740992","9007199254740993",null,null,null,"9007199254740993",null,"-1",null,null,"9007199254740992"],"l6_i32":[null,null,-1,5,2,3,3,null,5,3,3,2,null,null,null,null,-4,-1,null,null]}}},{"id":"random/0061","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_f64","type":"f64"},{"name":"l4_f64","type":"f64"},{"name":"l5_i64","type":"i64"}],"columns":{"l0_utf8":["b",null,null,"ab",""],"l1_i32":[-5,-1,null,4,-4],"l2_utf8":["a",null,null,"ab","B"],"l3_f64":[-22.867,0.0,-4.0,null,3.0],"l4_f64":[-1.0,-3.0,2.0,3.0,33.788],"l5_i64":[null,null,"-4611686018427387904","-4611686018427387904","-4611686018427387904"]}},"right":{"fields":[{"name":"l5_i64","type":"i64"},{"name":"r0_utf8","type":"utf8"}],"columns":{"r0_utf8":[],"l5_i64":[]}}},"ops":[{"op":"unique"},{"op":"filter","predicate":{"fn":"or","args":[{"fn":"lt","args":[{"col":"l3_f64"},{"col":"l1_i32"}]},{"fn":"or","args":[{"fn":"isNull","args":[{"col":"l0_utf8"}]},{"fn":"gt","args":[{"col":"l5_i64"},{"lit":"0","type":"i64"}]}]}]}},{"op":"join","right":"right","on":["l5_i64"],"how":"inner"},{"op":"groupBy","keys":["l0_utf8"],"aggs":{"n3":{"agg":"len"},"a3_0":{"agg":"max","arg":{"col":"l5_i64"}},"a3_1":{"agg":"min","arg":{"col":"l2_utf8"}},"a3_2":{"agg":"nUnique","arg":{"col":"l2_utf8"}}}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"n3","type":"u32"},{"name":"a3_0","type":"i64"},{"name":"a3_1","type":"utf8"},{"name":"a3_2","type":"u32"}],"columns":{"l0_utf8":[],"n3":[],"a3_0":[],"a3_1":[],"a3_2":[]}}},{"id":"random/0062","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_bool","type":"bool"}],"columns":{"l0_utf8":["😀",null,"é","B","😀"],"l1_i32":[2,-3,-4,4,null],"l2_bool":[true,true,true,true,false],"l3_bool":[true,null,false,null,true]}}},"ops":[{"op":"head","n":1},{"op":"filter","predicate":{"fn":"eq","args":[{"fn":"fillNull","args":[{"col":"l1_i32"},{"lit":1.5}]},{"fn":"sub","args":[{"col":"l1_i32"},{"col":"l1_i32"}]}]}},{"op":"withColumns","exprs":{"x2":{"col":"l1_i32"}}},{"op":"filter","predicate":{"fn":"isNull","args":[{"col":"l0_utf8"}]}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_bool","type":"bool"},{"name":"x2","type":"i32"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_bool":[],"l3_bool":[],"x2":[]}}},{"id":"random/0063","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_f64","type":"f64"},{"name":"l4_bool","type":"bool"}],"columns":{"l0_utf8":["c"],"l1_i32":[-2],"l2_utf8":["zz"],"l3_f64":["NaN"],"l4_bool":[true]}}},"ops":[{"op":"sort","keys":[{"expr":{"col":"l2_utf8"},"descending":true},{"expr":{"col":"l4_bool"},"descending":true}],"nullsLast":false},{"op":"filter","predicate":{"fn":"neq","args":[{"fn":"mod","args":[{"col":"l3_f64"},{"col":"l1_i32"}]},{"fn":"mul","args":[{"lit":2.5},{"col":"l3_f64"}]}]}},{"op":"sort","keys":[{"expr":{"col":"l2_utf8"},"descending":false},{"expr":{"col":"l1_i32"},"descending":true}],"nullsLast":true},{"op":"withColumns","exprs":{"x3":{"col":"l1_i32"}}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_f64","type":"f64"},{"name":"l4_bool","type":"bool"},{"name":"x3","type":"i32"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_utf8":[],"l3_f64":[],"l4_bool":[],"x3":[]}}},{"id":"random/0064","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_i64","type":"i64"},{"name":"l5_bool","type":"bool"}],"columns":{"l0_utf8":["B"],"l1_i32":[null],"l2_i32":[null],"l3_utf8":["zz"],"l4_i64":["9007199254740992"],"l5_bool":[true]}}},"ops":[{"op":"filter","predicate":{"fn":"gte","args":[{"col":"l1_i32"},{"fn":"div","args":[{"lit":2},{"col":"l1_i32"}]}]}},{"op":"filter","predicate":{"fn":"eq","args":[{"fn":"mod","args":[{"col":"l1_i32"},{"col":"l2_i32"}]},{"lit":2.5}]}},{"op":"head","n":9},{"op":"filter","predicate":{"fn":"gt","args":[{"col":"l2_i32"},{"col":"l1_i32"}]}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_i64","type":"i64"},{"name":"l5_bool","type":"bool"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_i32":[],"l3_utf8":[],"l4_i64":[],"l5_bool":[]}}},{"id":"random/0065","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_i32","type":"i32"},{"name":"l4_f64","type":"f64"},{"name":"l5_bool","type":"bool"},{"name":"l6_utf8","type":"utf8"}],"columns":{"l0_utf8":["c","","B","ab","aa"],"l1_i32":[3,3,-4,2,778],"l2_utf8":["","b",null,"é",""],"l3_i32":[null,-3,3,-2147483648,1],"l4_f64":[1.0,-2.0,"NaN",54.391,null],"l5_bool":[true,false,false,true,true],"l6_utf8":["é","B","c",null,"zz"]}}},"ops":[{"op":"withColumns","exprs":{"x0":{"fn":"mod","args":[{"fn":"sub","args":[{"col":"l4_f64"},{"lit":-1}]},{"fn":"mod","args":[{"col":"l3_i32"},{"col":"l3_i32"}]}]}}},{"op":"filter","predicate":{"fn":"isNull","args":[{"col":"l2_utf8"}]}},{"op":"unique","columns":["l4_f64"]}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_i32","type":"i32"},{"name":"l4_f64","type":"f64"},{"name":"l5_bool","type":"bool"},{"name":"l6_utf8","type":"utf8"},{"name":"x0","type":"f64"}],"columns":{"l0_utf8":["B"],"l1_i32":[-4],"l2_utf8":[null],"l3_i32":[3],"l4_f64":["NaN"],"l5_bool":[false],"l6_utf8":["c"],"x0":["NaN"]}}},{"id":"random/0066","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_i32","type":"i32"},{"name":"l4_f64","type":"f64"}],"columns":{"l0_utf8":["😀"],"l1_i32":[null],"l2_utf8":[null],"l3_i32":[1],"l4_f64":[null]}}},"ops":[{"op":"head","n":7}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_i32","type":"i32"},{"name":"l4_f64","type":"f64"}],"columns":{"l0_utf8":["😀"],"l1_i32":[null],"l2_utf8":[null],"l3_i32":[1],"l4_f64":[null]}}},{"id":"random/0067","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_i64","type":"i64"},{"name":"l4_f64","type":"f64"}],"columns":{"l0_utf8":["😀","","zz","a","c","c","aa","zz","b","B","zz","zz","c","a","","😀","zz","ab","aa","b","c","a","😀",null,null,"",null,null,"é","ab","zz","zz","b","","aa","","zz","é","","ab","a",null,"zz","😀","c","b","c","b","aa","a","ab","b","ab",null,"","ab","a","c","",""],"l1_i32":[null,0,-3,-1,1,1,-4,5,2,null,5,null,-5,null,-91,0,-3,0,-3,0,null,-4,null,-5,5,-5,-5,5,2,0,-3,2147483647,-1,null,null,null,-2,-1,-2,0,-4,-2,-2,null,2,2,5,1,5,0,-2,5,-5,-3,3,5,5,-444,4,5],"l2_bool":[true,false,false,false,true,true,false,false,false,true,false,true,false,true,false,false,true,true,true,false,true,null,true,true,false,true,true,true,true,false,true,null,false,false,false,true,true,false,false,true,true,true,false,true,false,false,false,true,true,false,true,true,false,null,false,false,null,false,null,true],"l3_i64":[null,"-2","9007199254740995","-1","-4611686018427387904","9007199254740992","9007199254740994","-3","-2","9007199254740993","-4611686018427387904","-4611686018427387904",null,"9007199254740994","9007199254740994","4",null,null,"-2","9007199254740995","1","-4611686018427387904","3","-4611686018427387904","-3",null,"-4611686018427387904","9007199254740994","4","-2","-4611686018427387904","-4611686018427387904","-4611686018427387904",null,"9007199254740992",null,"-4611686018427387904","9007199254740992","4","9007199254740995","5","1","-4611686018427387904","9007199254740994","9007199254740994",null,"-4611686018427387904","-4611686018427387904","-5","9007199254740995","4","1","-4611686018427387904","4","9007199254740992",null,"9007199254740993","9007199254740994","-4611686018427387904","9007199254740992"],"l4_f64":[-1.0,-3.0,-12.791,16.858,2.0,2.0,-77.552,-18.725,-4.0,null,3.0,-94.869,null,-1.0,96.328,-3.0,-3.0,null,2.0,-47.486,2.0,null,-2.0,-24.75,-84.96,-2.0,7.667,81.083,4.117,3.0,0.0,2.0,82.878,0.0,-2.151,2.0,30.616,0.484,-4.0,1.0,-1.0,2.0,3.0,53.676,-38.084,2.0,4.0,-55.776,-56.741,25.898,-95.433,-4.0,-77.119,-4.0,2.0,-91.497,76.809,-70.094,null,-62.333]}},"right":{"fields":[{"name":"l3_i64","type":"i64"},{"name":"r0_utf8","type":"utf8"}],"columns":{"r0_utf8":["zz","","b","a","zz","ab","c","b","B","ab"],"l3_i64":["-4611686018427387904","9007199254740992","1","1","-2",null,"9007199254740993","-4611686018427387904","3","9007199254740994"]}}},"ops":[{"op":"join","right":"right","on":["l3_i64"],"how":"anti"},{"op":"unique","columns":["l1_i32"]}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_i64","type":"i64"},{"name":"l4_f64","type":"f64"}],"columns":{"l0_utf8":["😀","zz","a","zz","c","😀","é","","a"],"l1_i32":[null,-3,-1,5,-5,0,2,-2,-4],"l2_bool":[true,false,false,false,false,false,true,false,true],"l3_i64":[null,"9007199254740995","-1","-3",null,"4","4","4","5"],"l4_f64":[-1.0,-12.791,16.858,-18.725,null,-3.0,4.117,-4.0,-1.0]}}},{"id":"random/0068","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_i64","type":"i64"},{"name":"l5_f64","type":"f64"}],"columns":{"l0_utf8":["B","c","aa","😀","zz"],"l1_i32":[-4,3,1,-3,1],"l2_f64":[73.87,null,84.426,-1.0,null],"l3_utf8":[null,"zz",null,"a",null],"l4_i64":[null,"-5","-4611686018427387904","9007199254740992","9007199254740995"],"l5_f64":[-35.732,-1.0,null,6.722,4.0]}},"right":{"fields":[{"name":"l4_i64","type":"i64"},{"name":"r0_bool","type":"bool"}],"columns":{"r0_bool":[true,false,false,null,false,true,false,true,false,false],"l4_i64":["-5","9007199254740995","-5",null,"9007199254740992","9007199254740995","9007199254740992","-5",null,"-4611686018427387904"]}}},"ops":[{"op":"join","right":"right","on":["l4_i64"],"how":"left"}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_i64","type":"i64"},{"name":"l5_f64","type":"f64"},{"name":"r0_bool","type":"bool"}],"columns":{"l0_utf8":["B","c","c","c","aa","😀","😀","zz","zz"],"l1_i32":[-4,3,3,3,1,-3,-3,1,1],"l2_f64":[73.87,null,null,null,84.426,-1.0,-1.0,null,null],"l3_utf8":[null,"zz","zz","zz",null,"a","a",null,null],"l4_i64":[null,"-5","-5","-5","-4611686018427387904","9007199254740992","9007199254740992","9007199254740995","9007199254740995"],"l5_f64":[-35.732,-1.0,-1.0,-1.0,null,6.722,6.722,4.0,4.0],"r0_bool":[null,true,false,true,false,false,false,false,true]}}},{"id":"random/0069","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_f64","type":"f64"},{"name":"l4_f64","type":"f64"}],"columns":{"l0_utf8":["ab","😀","a","ab","é","zz","😀","a","zz","😀","B","B",null,"B","é","é","B","aa",null,"ab","😀","a","B","b","b","😀","B","",null,"a","B",null,"é","b",null,"ab","b","zz","b","b","b","é","b","ab","é","B","a",null,"ab","é","B","B","zz","aa","B",null,"",null,"B",null],"l1_i32":[-2147483648,2,null,-4,null,-3,4,4,-3,2147483647,-3,null,-3,1,1,null,-1,1,-1,-2147483648,-3,1,null,-2,2147483647,-3,4,-2147483648,3,-2,-4,-3,null,-3,4,-2,5,0,4,null,-2147483648,null,5,-1,1,4,-4,-1,-2,-3,-2,0,-1,5,-2,4,-4,null,null,-2],"l2_f64":[2.0,-88.772,-42.94,-22.512,null,-2.0,-41.04,null,-88.668,3.0,null,null,4.594,-4.0,32.416,1.0,71.105,-75.714,"Infinity",null,-47.726,3.0,-16.846,71.361,52.15,null,null,94.593,-25.42,-2.0,4.0,null,"NaN",3.0,null,-92.705,-96.427,3.0,-4.0,-15.385,2.0,-16.761,null,-62.691,-37.721,2.0,-94.352,80.945,-90.385,29.012,-21.229,74.907,1.0,"NaN",88.608,-42.186,-43.28,null,-27.2,0.0],"l3_f64":[-4.0,-4.0,72.528,70.408,60.168,0.0,-93.32,-1.0,-91.246,3.0,-2.0,3.0,2.0,-98.946,4.0,0.0,-4.0,null,2.0,-3.0,-4.0,-3.0,75.506,-73.662,-3.0,4.0,-2.0,4.0,1.0,null,96.558,-4.0,null,null,"Infinity",3.0,-82.404,-1.0,-24.563,78.754,"NaN",-4.0,-4.0,-3.0,90.136,-4.0,-3.0,4.0,null,null,-24.638,-1.0,4.0,-4.0,-28.057,43.994,-3.0,4.0,-95.748,null],"l4_f64":[-1.0,"NaN",2.0,"NaN",0.0,3.0,null,55.683,32.731,0.0,-75.018,-3.0,-2.0,null,"NaN",1.0,-16.965,-2.0,-1.0,null,-3.0,82.058,4.0,0.0,"NaN",-1.0,81.169,-3.0,4.0,23.218,-8.889,null,null,-80.473,-66.253,78.524,1.0,-65.198,90.632,2.0,-51.322,-17.538,-3.0,1.0,3.0,-4.0,-3.0,3.0,"-0",-45.295,3.0,5.028,47.162,null,null,-3.0,-35.381,42.283,89.886,2.0]}}},"ops":[{"op":"filter","predicate":{"fn":"lte","args":[{"fn":"sub","args":[{"col":"l3_f64"},{"col":"l3_f64"}]},{"fn":"when","args":[{"fn":"lte","args":[{"col":"l3_f64"},{"col":"l4_f64"}]},{"col":"l2_f64"},{"col":"l3_f64"}]}]}},{"op":"sort","keys":[{"expr":{"col":"l1_i32"},"descending":false},{"expr":{"col":"l0_utf8"},"descending":true}],"nullsLast":true},{"op":"filter","predicate":{"fn":"or","args":[{"fn":"gt","args":[{"lit":2.5},{"col":"l3_f64"}]},{"fn":"eq","args":[{"lit":0.5},{"col":"l2_f64"}]}]}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_f64","type":"f64"},{"name":"l4_f64","type":"f64"}],"columns":{"l0_utf8":["ab",null,"b",null,"B","a","B","b","é"],"l1_i32":[-2147483648,-3,-2,-1,0,1,4,2147483647,null],"l2_f64":[2.0,4.594,71.361,"Infinity",74.907,3.0,2.0,52.15,1.0],"l3_f64":[-4.0,2.0,-73.662,2.0,-1.0,-3.0,-4.0,-3.0,0.0],"l4_f64":[-1.0,-2.0,0.0,-1.0,5.028,82.058,-4.0,"NaN",1.0]}}},{"id":"random/0070","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_i32","type":"i32"}],"columns":{"l0_utf8":["é","é","ab","B","ab","ab","é","a","aa","ab","a","zz","c","","zz","😀","a","ab","c","😀"],"l1_i32":[2,-1,3,-5,0,-5,-1,-1,-4,-45,3,2,-1,-3,-3,3,-3,-1,-2,2],"l2_utf8":["é","B","B","c","a","a","é","b","","","zz","😀","b","c","","a","B","c","é",""],"l3_i32":[3,-1,4,346,-3,-3,-5,-3,0,-2,-4,-4,0,3,-1,2,-2,-5,-2,-2]}}},"ops":[{"op":"unique","columns":["l0_utf8"]},{"op":"sort","keys":[{"expr":{"col":"l1_i32"},"descending":false},{"expr":{"col":"l3_i32"},"descending":false}],"nullsLast":false},{"op":"withColumns","exprs":{"x2":{"fn":"neg","args":[{"fn":"mod","args":[{"col":"l3_i32"},{"col":"l1_i32"}]}]}}},{"op":"unique","columns":["l1_i32"]}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_i32","type":"i32"},{"name":"x2","type":"i32"}],"columns":{"l0_utf8":["B","aa","","a","zz","😀"],"l1_i32":[-5,-4,-3,-1,2,3],"l2_utf8":["c","","c","b","😀","a"],"l3_i32":[346,0,3,-3,-4,2],"x2":[4,0,0,0,0,-2]}}},{"id":"random/0071","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_bool","type":"bool"},{"name":"l4_i64","type":"i64"},{"name":"l5_f64","type":"f64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_utf8":[],"l3_bool":[],"l4_i64":[],"l5_f64":[]}}},"ops":[{"op":"sort","keys":[{"expr":{"col":"l2_utf8"},"descending":true}],"nullsLast":false}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_bool","type":"bool"},{"name":"l4_i64","type":"i64"},{"name":"l5_f64","type":"f64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_utf8":[],"l3_bool":[],"l4_i64":[],"l5_f64":[]}}},{"id":"random/0072","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_f64","type":"f64"},{"name":"l5_i64","type":"i64"},{"name":"l6_f64","type":"f64"}],"columns":{"l0_utf8":["a","B","aa","c","B","é","zz","aa","B","😀","b","é","c","😀","a","aa","😀","zz","b","ab","c","b","zz","zz","b","zz","zz","a","a","B","é","é","b","é","é","zz","b","a","b","c","zz","B","zz","é","😀","B","ab","ab","b","aa","a","ab","a","a","","é","zz","zz","zz","aa"],"l1_i32":[1,0,-5,-4,5,0,1,-1,-5,0,-4,-3,1,1,5,-2,0,-1,-3,-5,-3,-1,-1,2,-2,-5,3,-5,4,-4,0,-4,-3,-2,-4,2,-2,1,-4,-1,2,-2,-2147483648,2,4,0,-4,-657,5,-3,-3,5,-5,-1,3,5,0,5,2,4],"l2_f64":[3.0,2.0,0.0,1.0,62.583,4.0,28.034,1.0,-1.0,63.127,4.0,"NaN",2.419,4.0,86.754,-3.0,84.651,-37.701,15.35,4.0,"NaN",77.174,-2.0,3.0,1.0,-80.937,-20.164,-2.0,3.0,7.038,-72.793,-2.0,"Infinity",-79.999,1.0,-3.0,2.0,-21.207,-90.84,"NaN",-3.0,-37.282,1.0,-1.0,1.0,55.814,-1.0,2.0,4.0,-1.0,-79.272,-99.055,2.0,3.0,93.0,3.0,63.422,3.0,-14.207,-3.0],"l3_utf8":["","zz","ab","😀","b","zz","aa","ab","aa","b","😀","B","é","","zz","c","zz","ab","c","😀","c","zz","ab","a","","é","B","ab","😀","ab","aa","😀","😀","zz","aa","a","zz","a","b","ab","","a","B","zz","😀","c","a","a","b","b","a","ab","zz","B","a","","B","c","é","é"],"l4_f64":[-77.483,-3.0,0.0,"Infinity",-1.0,57.049,-3.0,2.0,-1.0,75.3,34.184,4.0,0.0,25.856,-2.0,-2.0,39.886,12.863,-95.829,54.204,59.898,-3.0,4.0,1.0,"NaN",0.0,-42.762,3.0,"NaN",74.394,-55.311,-4.0,-18.89,60.932,-1.0,"-0",1.0,35.967,-92.403,75.5,0.0,1.0,51.843,1.0,14.051,-3.0,-3.0,-82.294,39.996,26.341,72.514,97.067,1.0,-69.065,-4.0,73.583,-4.0,34.37,53.009,3.0],"l5_i64":["9007199254740992","-2","9007199254740993","-4611686018427387904","9007199254740995","-4611686018427387904","9007199254740995","-4611686018427387904","9007199254740994","-5","-5","3","-4611686018427387904","1","-4611686018427387904","3","9007199254740992","9007199254740994","-4611686018427387904","-4611686018427387904","-4611686018427387904","-2","-5","9007199254740994","-4","-3","1","9007199254740994","9007199254740995","9007199254740993","-4611686018427387904","9007199254740994","9007199254740994","-4611686018427387904","-4611686018427387904","9007199254740993","-4611686018427387904","-4611686018427387904","9007199254740993","9007199254740992","-3","1","9007199254740992","-4611686018427387904","4","4","-4611686018427387904","9007199254740994","9007199254740993","9007199254740993","9007199254740992","9007199254740993","9007199254740995","-4611686018427387904","9007199254740992","9007199254740995","9007199254740995","-4611686018427387904","9007199254740995","-4611686018427387904"],"l6_f64":[-77.955,4.0,-2.0,-3.0,"Infinity",-14.667,-1.0,0.0,-5.382,-68.384,4.0,86.842,-3.0,0.0,-1.0,-75.106,2.0,-1.0,1.0,7.215,1.0,-80.72,12.674,-1.0,-1.0,0.0,3.0,-20.917,0.0,-2.0,2.0,97.303,-3.0,21.228,3.0,-4.0,30.092,25.766,4.0,2.0,-2.0,-1.0,2.0,"-Infinity",-3.0,-2.0,-4.0,18.634,64.738,-4.0,0.0,-2.0,-8.785,-1.112,-55.423,-2.0,-2.0,-36.497,53.025,-39.096]}}},"ops":[{"op":"filter","predicate":{"fn":"eq","args":[{"fn":"div","args":[{"col":"l1_i32"},{"col":"l1_i32"}]},{"col":"l1_i32"}]}},{"op":"unique"},{"op":"withColumns","exprs":{"x2":{"fn":"mod","args":[{"fn":"mul","args":[{"col":"l6_f64"},{"col":"l6_f64"}]},{"col":"l2_f64"}]}}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_f64","type":"f64"},{"name":"l5_i64","type":"i64"},{"name":"l6_f64","type":"f64"},{"name":"x2","type":"f64"}],"columns":{"l0_utf8":["a","zz","c","😀","a"],"l1_i32":[1,1,1,1,1],"l2_f64":[3.0,28.034,2.419,4.0,-21.207],"l3_utf8":["","aa","é","","a"],"l4_f64":[-77.483,-3.0,0.0,25.856,35.967],"l5_i64":["9007199254740992","9007199254740995","-4611686018427387904","1","-4611686018427387904"],"l6_f64":[-77.955,-1.0,-3.0,0.0,25.766],"x2":[1.9820249999993393,1.0,1.7430000000000003,0.0,-14.737244000000146]}}},{"id":"random/0073","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i64","type":"i64"},{"name":"l3_utf8","type":"utf8"}],"columns":{"l0_utf8":["b","b","😀","","a"],"l1_i32":[-811,0,-2,3,5],"l2_i64":["-2","-4611686018427387904","-2","-4611686018427387904","2"],"l3_utf8":["aa","aa","b","c","😀"]}}},"ops":[{"op":"head","n":2},{"op":"unique"},{"op":"withColumns","exprs":{"x2":{"fn":"when","args":[{"fn":"lte","args":[{"col":"l1_i32"},{"col":"l1_i32"}]},{"fn":"fillNull","args":[{"col":"l1_i32"},{"lit":1.5}]},{"lit":2}]}}},{"op":"unique"}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i64","type":"i64"},{"name":"l3_utf8","type":"utf8"},{"name":"x2","type":"f64"}],"columns":{"l0_utf8":["b","b"],"l1_i32":[-811,0],"l2_i64":["-2","-4611686018427387904"],"l3_utf8":["aa","aa"],"x2":[-811.0,0.0]}}},{"id":"random/0074","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_i64","type":"i64"}],"columns":{"l0_utf8":["é"],"l1_i32":[-5],"l2_i32":[-3],"l3_i64":["9007199254740994"]}}},"ops":[{"op":"head","n":2},{"op":"groupBy","keys":["l1_i32"],"aggs":{"n1":{"agg":"len"},"a1_0":{"agg":"max","arg":{"col":"l0_utf8"}}}}],"expected":{"fields":[{"name":"l1_i32","type":"i32"},{"name":"n1","type":"u32"},{"name":"a1_0","type":"utf8"}],"columns":{"l1_i32":[-5],"n1":[1],"a1_0":["é"]}}},{"id":"random/0075","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_i64","type":"i64"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_f64","type":"f64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_bool":[],"l3_i64":[],"l4_utf8":[],"l5_f64":[]}},"right":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"r0_utf8","type":"utf8"}],"columns":{"r0_utf8":["aa","é","ab"],"l0_utf8":["a","😀","b"]}}},"ops":[{"op":"join","right":"right","on":["l0_utf8"],"how":"anti"},{"op":"withColumns","exprs":{"x1":{"fn":"gt","args":[{"col":"l3_i64"},{"lit":"-2","type":"i64"}]}}},{"op":"sort","keys":[{"expr":{"col":"l1_i32"},"descending":false},{"expr":{"col":"l2_bool"},"descending":false}],"nullsLast":true}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_i64","type":"i64"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_f64","type":"f64"},{"name":"x1","type":"bool"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_bool":[],"l3_i64":[],"l4_utf8":[],"l5_f64":[],"x1":[]}}},{"id":"random/0076","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_i32","type":"i32"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_f64","type":"f64"},{"name":"l6_f64","type":"f64"}],"columns":{"l0_utf8":["a","B","c","a",""],"l1_i32":[-1,3,-2,1,3],"l2_utf8":["","B","a","é","😀"],"l3_i32":[2,4,4,2,0],"l4_utf8":["aa","ab","","aa","c"],"l5_f64":[59.826,-1.0,-53.594,60.915,-53.714],"l6_f64":[2.0,12.124,3.0,41.271,"NaN"]}},"right":{"fields":[{"name":"l3_i32","type":"i32"},{"name":"r0_bool","type":"bool"}],"columns":{"r0_bool":[],"l3_i32":[]}}},"ops":[{"op":"join","right":"right","on":["l3_i32"],"how":"semi"},{"op":"head","n":2}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_i32","type":"i32"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_f64","type":"f64"},{"name":"l6_f64","type":"f64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_utf8":[],"l3_i32":[],"l4_utf8":[],"l5_f64":[],"l6_f64":[]}}},{"id":"random/0077","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_f64","type":"f64"},{"name":"l4_i32","type":"i32"},{"name":"l5_bool","type":"bool"}],"columns":{"l0_utf8":["é","b","","😀","c","aa","é","","aa",null,"c","","","B","é",null,"a","","é","B"],"l1_i32":[5,-5,1,-4,null,1,null,0,null,-3,null,0,null,2147483647,1,-2,-5,-1,5,-3],"l2_utf8":[null,null,"B",null,"B","zz","aa","","c","é","","zz","zz",null,"c",null,"c","zz","😀","aa"],"l3_f64":[null,null,4.0,-63.695,0.0,9.115,2.0,null,-4.728,-59.593,-1.0,"NaN",30.796,1.0,null,-76.129,3.0,30.357,-56.62,-93.974],"l4_i32":[3,-3,2,-2,3,null,null,-2,-5,null,null,-3,5,2,null,-4,2147483647,null,-1,null],"l5_bool":[null,true,false,false,false,null,null,false,null,true,null,false,false,null,null,null,null,null,true,true]}},"right":{"fields":[{"name":"a0_0","type":"utf8"},{"name":"r0_f64","type":"f64"}],"columns":{"r0_f64":[],"a0_0":[]}}},"ops":[{"op":"groupBy","keys":["l1_i32"],"aggs":{"n0":{"agg":"len"},"a0_0":{"agg":"first","arg":{"col":"l2_utf8"}}}},{"op":"filter","predicate":{"fn":"gt","args":[{"col":"l1_i32"},{"col":"l1_i32"}]}},{"op":"join","right":"right","on":["a0_0"],"how":"semi"}],"expected":{"fields":[{"name":"l1_i32","type":"i32"},{"name":"n0","type":"u32"},{"name":"a0_0","type":"utf8"}],"columns":{"l1_i32":[],"n0":[],"a0_0":[]}}},{"id":"random/0078","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_f64","type":"f64"},{"name":"l4_utf8","type":"utf8"}],"columns":{"l0_utf8":[""],"l1_i32":[-1],"l2_f64":[null],"l3_f64":[-1.0],"l4_utf8":["b"]}}},"ops":[{"op":"withColumns","exprs":{"x0":{"fn":"gt","args":[{"fn":"mod","args":[{"lit":2.5},{"col":"l3_f64"}]},{"col":"l1_i32"}]}}},{"op":"head","n":10},{"op":"groupBy","keys":["l4_utf8"],"aggs":{"n2":{"agg":"len"},"a2_0":{"agg":"count","arg":{"col":"x0"}},"a2_1":{"agg":"first","arg":{"col":"l0_utf8"}}}},{"op":"filter","predicate":{"fn":"strStartsWith","args":[{"col":"a2_1"}],"prefix":"a"}}],"expected":{"fields":[{"name":"l4_utf8","type":"utf8"},{"name":"n2","type":"u32"},{"name":"a2_0","type":"u32"},{"name":"a2_1","type":"utf8"}],"columns":{"l4_utf8":[],"n2":[],"a2_0":[],"a2_1":[]}}},{"id":"random/0079","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_f64","type":"f64"},{"name":"l4_i64","type":"i64"},{"name":"l5_bool","type":"bool"},{"name":"l6_f64","type":"f64"}],"columns":{"l0_utf8":[null],"l1_i32":[null],"l2_i32":[4],"l3_f64":[null],"l4_i64":["9007199254740995"],"l5_bool":[true],"l6_f64":[21.361]}}},"ops":[{"op":"groupBy","keys":["l5_bool"],"aggs":{"n0":{"agg":"len"},"a0_0":{"agg":"last","arg":{"col":"l4_i64"}}}},{"op":"groupBy","keys":["l5_bool"],"aggs":{"n1":{"agg":"len"},"a1_0":{"agg":"nUnique","arg":{"col":"a0_0"}}}},{"op":"withColumns","exprs":{"x2":{"fn":"mul","args":[{"fn":"sub","args":[{"lit":0.5},{"lit":2.5}]},{"fn":"neg","args":[{"lit":2}]}]}}}],"expected":{"fields":[{"name":"l5_bool","type":"bool"},{"name":"n1","type":"u32"},{"name":"a1_0","type":"u32"},{"name":"x2","type":"f64"}],"columns":{"l5_bool":[true],"n1":[1],"a1_0":[1],"x2":[4.0]}}},{"id":"random/0080","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_utf8","type":"utf8"}],"columns":{"l0_utf8":["zz",null,"","😀",""],"l1_i32":[-4,-5,4,null,4],"l2_i32":[null,3,null,-2,0],"l3_utf8":["é","B","aa","a",null]}}},"ops":[{"op":"withColumns","exprs":{"x0":{"fn":"mul","args":[{"fn":"mod","args":[{"lit":0.5},{"col":"l1_i32"}]},{"fn":"fillNull","args":[{"col":"l2_i32"},{"lit":0}]}]}}},{"op":"filter","predicate":{"fn":"isIn","args":[{"col":"l3_utf8"}],"values":["é","ab","B"]}},{"op":"withColumns","exprs":{"x2":{"lit":2.5}}},{"op":"filter","predicate":{"fn":"isNotNull","args":[{"col":"x2"}]}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_utf8","type":"utf8"},{"name":"x0","type":"f64"},{"name":"x2","type":"f64"}],"columns":{"l0_utf8":["zz",null],"l1_i32":[-4,-5],"l2_i32":[null,3],"l3_utf8":["é","B"],"x0":["-0",-13.5],"x2":[2.5,2.5]}}},{"id":"random/0081","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_i32","type":"i32"}],"columns":{"l0_utf8":["😀",null,"c",null,"ab","ab","B",null,null,"c","ab","zz","aa",null,"aa","c","😀","ab",null,""],"l1_i32":[5,-3,-1,0,null,2,null,-4,null,-5,null,-3,2,3,5,-3,null,-5,4,-3],"l2_f64":[-4.802,75.504,-21.884,null,null,2.0,-21.413,-80.905,97.777,null,-1.0,55.903,1.0,-51.178,null,-85.919,-2.0,1.0,39.072,6.785],"l3_utf8":["b","zz",null,"ab","😀","a","aa","😀","zz","","é","a","b","b",null,"é","aa","","a",""],"l4_i32":[null,null,-5,-5,0,null,-5,-1,1,3,null,4,-3,null,-3,4,-2,5,0,null]}}},"ops":[{"op":"groupBy","keys":["l4_i32"],"aggs":{"n0":{"agg":"len"},"a0_0":{"agg":"median","arg":{"col":"l1_i32"}},"a0_1":{"agg":"last","arg":{"col":"l0_utf8"}},"a0_2":{"agg":"count","arg":{"col":"l1_i32"}}}}],"expected":{"fields":[{"name":"l4_i32","type":"i32"},{"name":"n0","type":"u32"},{"name":"a0_0","type":"f64"},{"name":"a0_1","type":"utf8"},{"name":"a0_2","type":"u32"}],"columns":{"l4_i32":[null,-5,0,-1,1,3,4,-3,-2,5],"n0":[6,3,2,1,1,1,2,2,1,1],"a0_0":[2.0,-0.5,4.0,-4.0,null,-5.0,-3.0,3.5,null,-5.0],"a0_1":["","B",null,null,null,"c","c","aa","😀","ab"],"a0_2":[5,2,1,1,0,1,2,2,0,1]}}},{"id":"random/0082","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_f64","type":"f64"},{"name":"l4_f64","type":"f64"}],"columns":{"l0_utf8":["b","é","b","a","a","ab","zz","","ab","B","zz","","ab","a","zz","b","B","é","zz","aa","b","c","a","ab","😀","é","zz","😀","a","ab","","é","aa","b","ab","c","ab","zz","zz","c","a","aa","aa","b","c","c","a","aa","a","c","é","é","é","é","c","😀","B","c","b","B"],"l1_i32":[2147483647,-2,0,5,0,4,-339,1,-2147483648,-2,1,-4,1,2,4,-5,0,-2,0,4,-147,0,-1,3,-3,-4,3,0,5,-5,0,-2147483648,4,-2,5,5,3,3,-4,-2,-5,4,-2,2,-4,-2147483648,-2,-2,-5,-2147483648,-1,5,2,3,-5,1,0,1,5,0],"l2_i32":[-4,-2,-870,0,2,0,-4,2,3,0,-3,-2,-1,4,-3,4,-507,-3,3,1,0,598,-2,2,-350,-1,-2,2,-3,2,5,-4,5,-5,-5,0,0,-3,-3,-5,5,-5,2,2,1,4,-2,0,-1,1,-4,5,3,4,5,3,3,-3,2,-2147483648],"l3_f64":[-3.0,33.838,-2.0,-91.595,-92.14,1.0,-3.0,-4.0,1.0,-1.0,8.86,57.037,"NaN",63.483,52.37,-1.0,4.0,-4.0,-7.472,-28.24,-18.534,-1.0,-64.607,2.0,-3.0,-42.092,25.723,69.112,0.0,2.0,4.0,2.182,4.0,"NaN",-2.0,-3.0,3.0,-2.0,-10.813,2.0,3.0,2.0,-91.656,"NaN",2.0,0.0,-3.0,1.0,1.0,-3.0,70.652,-3.0,51.01,-4.0,-3.0,-96.0,-81.575,2.0,-3.0,-2.0],"l4_f64":[1.0,99.592,-4.0,-72.414,-2.0,"NaN",-74.537,1.0,-3.0,-39.215,-7.284,2.0,-50.389,16.421,-68.618,-9.229,88.097,-18.361,97.282,-47.624,-99.102,-66.447,3.0,0.0,-24.103,2.0,4.0,4.0,78.008,1.0,45.528,-2.0,66.377,14.043,2.0,-83.24,-68.059,65.037,2.0,-2.0,"-0",-2.0,-4.0,2.0,2.0,3.0,-3.0,-12.036,-0.095,-79.071,13.979,-5.327,-43.259,-3.0,-50.46,-62.016,-27.288,"-Infinity",-1.013,0.0]}}},"ops":[{"op":"withColumns","exprs":{"x0":{"fn":"when","args":[{"fn":"isNotNull","args":[{"col":"l2_i32"}]},{"fn":"mod","args":[{"col":"l1_i32"},{"col":"l1_i32"}]},{"col":"l3_f64"}]}}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_f64","type":"f64"},{"name":"l4_f64","type":"f64"},{"name":"x0","type":"f64"}],"columns":{"l0_utf8":["b","é","b","a","a","ab","zz","","ab","B","zz","","ab","a","zz","b","B","é","zz","aa","b","c","a","ab","😀","é","zz","😀","a","ab","","é","aa","b","ab","c","ab","zz","zz","c","a","aa","aa","b","c","c","a","aa","a","c","é","é","é","é","c","😀","B","c","b","B"],"l1_i32":[2147483647,-2,0,5,0,4,-339,1,-2147483648,-2,1,-4,1,2,4,-5,0,-2,0,4,-147,0,-1,3,-3,-4,3,0,5,-5,0,-2147483648,4,-2,5,5,3,3,-4,-2,-5,4,-2,2,-4,-2147483648,-2,-2,-5,-2147483648,-1,5,2,3,-5,1,0,1,5,0],"l2_i32":[-4,-2,-870,0,2,0,-4,2,3,0,-3,-2,-1,4,-3,4,-507,-3,3,1,0,598,-2,2,-350,-1,-2,2,-3,2,5,-4,5,-5,-5,0,0,-3,-3,-5,5,-5,2,2,1,4,-2,0,-1,1,-4,5,3,4,5,3,3,-3,2,-2147483648],"l3_f64":[-3.0,33.838,-2.0,-91.595,-92.14,1.0,-3.0,-4.0,1.0,-1.0,8.86,57.037,"NaN",63.483,52.37,-1.0,4.0,-4.0,-7.472,-28.24,-18.534,-1.0,-64.607,2.0,-3.0,-42.092,25.723,69.112,0.0,2.0,4.0,2.182,4.0,"NaN",-2.0,-3.0,3.0,-2.0,-10.813,2.0,3.0,2.0,-91.656,"NaN",2.0,0.0,-3.0,1.0,1.0,-3.0,70.652,-3.0,51.01,-4.0,-3.0,-96.0,-81.575,2.0,-3.0,-2.0],"l4_f64":[1.0,99.592,-4.0,-72.414,-2.0,"NaN",-74.537,1.0,-3.0,-39.215,-7.284,2.0,-50.389,16.421,-68.618,-9.229,88.097,-18.361,97.282,-47.624,-99.102,-66.447,3.0,0.0,-24.103,2.0,4.0,4.0,78.008,1.0,45.528,-2.0,66.377,14.043,2.0,-83.24,-68.059,65.037,2.0,-2.0,"-0",-2.0,-4.0,2.0,2.0,3.0,-3.0,-12.036,-0.095,-79.071,13.979,-5.327,-43.259,-3.0,-50.46,-62.016,-27.288,"-Infinity",-1.013,0.0],"x0":[0.0,0.0,null,0.0,null,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,null,0.0,null,0.0,0.0,null,0.0,0.0,0.0,0.0,0.0,null,0.0,0.0,null,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,null,0.0,0.0,null]}}},{"id":"random/0083","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_bool","type":"bool"},{"name":"l5_f64","type":"f64"},{"name":"l6_utf8","type":"utf8"}],"columns":{"l0_utf8":["B"],"l1_i32":[-4],"l2_utf8":["b"],"l3_utf8":["b"],"l4_bool":[false],"l5_f64":[-4.0],"l6_utf8":["a"]}}},"ops":[{"op":"withColumns","exprs":{"x0":{"fn":"mul","args":[{"col":"l1_i32"},{"col":"l1_i32"}]}}},{"op":"withColumns","exprs":{"x1":{"fn":"isNull","args":[{"col":"l2_utf8"}]}}},{"op":"unique"},{"op":"head","n":3}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_bool","type":"bool"},{"name":"l5_f64","type":"f64"},{"name":"l6_utf8","type":"utf8"},{"name":"x0","type":"i32"},{"name":"x1","type":"bool"}],"columns":{"l0_utf8":["B"],"l1_i32":[-4],"l2_utf8":["b"],"l3_utf8":["b"],"l4_bool":[false],"l5_f64":[-4.0],"l6_utf8":["a"],"x0":[16],"x1":[false]}}},{"id":"random/0084","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_i32","type":"i32"},{"name":"l4_f64","type":"f64"},{"name":"l5_f64","type":"f64"},{"name":"l6_i64","type":"i64"}],"columns":{"l0_utf8":["c","é","B","","a"],"l1_i32":[null,0,null,1,null],"l2_f64":["NaN",2.318,null,1.0,null],"l3_i32":[null,-5,null,1,null],"l4_f64":[3.0,-3.0,"NaN","NaN",null],"l5_f64":[-36.112,93.106,-3.698,-1.0,-54.703],"l6_i64":[null,"9007199254740993",null,"9007199254740992","9007199254740992"]}},"right":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"r0_f64","type":"f64"}],"columns":{"r0_f64":[-1.0,"NaN",4.0,-4.0,3.0,-1.886,3.0,33.672,59.986,4.785,87.187,-97.497,-3.0,-3.0,-45.445,22.282,67.684,"NaN",-1.0,-2.0,21.07,-4.0,99.142,73.946,1.0,-21.858,-2.0,6.228,2.0,-60.12],"l0_utf8":["B","a","é","B","a","a","c","é","B","é","c","","a","B","é","B","a","","","a","","","c","","c","B","","a","a","é"]}}},"ops":[{"op":"filter","predicate":{"fn":"eq","args":[{"col":"l6_i64"},{"lit":"2","type":"i64"}]}},{"op":"join","right":"right","on":["l0_utf8"],"how":"anti"},{"op":"sort","keys":[{"expr":{"col":"l5_f64"},"descending":true},{"expr":{"col":"l0_utf8"},"descending":true}],"nullsLast":false}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_i32","type":"i32"},{"name":"l4_f64","type":"f64"},{"name":"l5_f64","type":"f64"},{"name":"l6_i64","type":"i64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_f64":[],"l3_i32":[],"l4_f64":[],"l5_f64":[],"l6_i64":[]}}},{"id":"random/0085","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_i64","type":"i64"},{"name":"l4_f64","type":"f64"},{"name":"l5_i64","type":"i64"},{"name":"l6_bool","type":"bool"}],"columns":{"l0_utf8":["a"],"l1_i32":[1],"l2_i32":[-5],"l3_i64":["-4611686018427387904"],"l4_f64":[-4.0],"l5_i64":["-4611686018427387904"],"l6_bool":[true]}}},"ops":[{"op":"sort","keys":[{"expr":{"col":"l0_utf8"},"descending":true}],"nullsLast":true}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_i64","type":"i64"},{"name":"l4_f64","type":"f64"},{"name":"l5_i64","type":"i64"},{"name":"l6_bool","type":"bool"}],"columns":{"l0_utf8":["a"],"l1_i32":[1],"l2_i32":[-5],"l3_i64":["-4611686018427387904"],"l4_f64":[-4.0],"l5_i64":["-4611686018427387904"],"l6_bool":[true]}}},{"id":"random/0086","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_bool","type":"bool"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_f64":[],"l3_utf8":[],"l4_bool":[]}},"right":{"fields":[{"name":"l3_utf8","type":"utf8"},{"name":"r0_utf8","type":"utf8"}],"columns":{"r0_utf8":["c","B",""],"l3_utf8":[null,null,null]}}},"ops":[{"op":"filter","predicate":{"fn":"strStartsWith","args":[{"col":"l3_utf8"}],"prefix":""}},{"op":"filter","predicate":{"col":"l4_bool"}},{"op":"join","right":"right","on":["l3_utf8"],"how":"anti"},{"op":"filter","predicate":{"fn":"gte","args":[{"fn":"when","args":[{"fn":"isNull","args":[{"col":"l3_utf8"}]},{"col":"l1_i32"},{"col":"l1_i32"}]},{"fn":"div","args":[{"col":"l2_f64"},{"col":"l1_i32"}]}]}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_bool","type":"bool"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_f64":[],"l3_utf8":[],"l4_bool":[]}}},{"id":"random/0087","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_f64","type":"f64"},{"name":"l5_utf8","type":"utf8"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_utf8":[],"l3_utf8":[],"l4_f64":[],"l5_utf8":[]}}},"ops":[{"op":"withColumns","exprs":{"x0":{"fn":"fillNull","args":[{"fn":"when","args":[{"fn":"isNull","args":[{"col":"l5_utf8"}]},{"col":"l4_f64"},{"col":"l4_f64"}]},{"lit":0}]}}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_f64","type":"f64"},{"name":"l5_utf8","type":"utf8"},{"name":"x0","type":"f64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_utf8":[],"l3_utf8":[],"l4_f64":[],"l5_utf8":[],"x0":[]}}},{"id":"random/0088","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_f64","type":"f64"}],"columns":{"l0_utf8":["é","B","","zz","😀"],"l1_i32":[5,-1,-4,-4,-943],"l2_utf8":["a","aa","c","aa","a"],"l3_f64":[-3.0,4.0,-28.158,"-Infinity",2.0]}}},"ops":[{"op":"withColumns","exprs":{"x0":{"fn":"mod","args":[{"col":"l3_f64"},{"col":"l3_f64"}]}}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_f64","type":"f64"},{"name":"x0","type":"f64"}],"columns":{"l0_utf8":["é","B","","zz","😀"],"l1_i32":[5,-1,-4,-4,-943],"l2_utf8":["a","aa","c","aa","a"],"l3_f64":[-3.0,4.0,-28.158,"-Infinity",2.0],"x0":[0.0,0.0,0.0,"NaN",0.0]}}},{"id":"random/0089","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_f64","type":"f64"}],"columns":{"l0_utf8":["aa"],"l1_i32":[0],"l2_i32":[1],"l3_utf8":["aa"],"l4_f64":[-2.0]}}},"ops":[{"op":"head","n":5},{"op":"filter","predicate":{"fn":"lte","args":[{"fn":"floorDiv","args":[{"col":"l2_i32"},{"col":"l2_i32"}]},{"fn":"sub","args":[{"col":"l1_i32"},{"col":"l1_i32"}]}]}},{"op":"filter","predicate":{"fn":"isNotNull","args":[{"col":"l2_i32"}]}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_f64","type":"f64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_i32":[],"l3_utf8":[],"l4_f64":[]}}},{"id":"random/0090","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_utf8","type":"utf8"}],"columns":{"l0_utf8":["B","","","ab","B","ab","c","b","a","b","😀",null,"zz","ab","😀","b","😀","b","é","a","😀",null,"c","","aa","zz","zz","ab","b","a","zz","😀","é","aa","é","😀","aa",null,"b","zz","zz",null,"a","b","a","","","b","",null,"é","zz","c","aa","b","","b","😀","","B"],"l1_i32":[-1,0,3,-3,-1,2,2,1,-3,-4,null,-4,-5,4,2147483647,3,3,-2147483648,-2,-5,2147483647,5,5,-3,-3,-4,null,0,-4,0,-3,5,-2,-4,1,2,4,0,-1,-2147483648,-3,-2,4,null,-5,-3,-5,-2,-1,-2,-2,3,0,-2,-1,3,null,1,3,2],"l2_bool":[false,false,false,null,false,true,true,true,null,false,false,false,null,false,true,true,null,false,true,false,true,true,true,true,true,null,false,false,true,true,false,false,false,null,false,false,false,false,true,false,false,false,false,true,true,false,false,false,true,true,false,null,true,false,false,false,true,true,null,false],"l3_utf8":["B",null,"","c","b","😀","","c","c","aa","aa","aa","a","b","c",null,"😀","ab","b","b","😀","zz","é","","c","b","zz","ab","a","B","é","B","é","b","a",null,"zz","B",null,"zz","😀","zz","aa","zz","c","zz","c","zz","é","c","zz","ab","a","b","zz",null,"c","ab","c","😀"]}},"right":{"fields":[{"name":"l3_utf8","type":"utf8"},{"name":"r0_f64","type":"f64"}],"columns":{"r0_f64":[-50.26,-12.185,52.145],"l3_utf8":["c","ab","c"]}}},"ops":[{"op":"filter","predicate":{"fn":"gte","args":[{"fn":"abs","args":[{"col":"l1_i32"}]},{"fn":"floorDiv","args":[{"col":"l1_i32"},{"col":"l1_i32"}]}]}},{"op":"withColumns","exprs":{"x1":{"fn":"isNull","args":[{"col":"l1_i32"}]}}},{"op":"join","right":"right","on":["l3_utf8"],"how":"left"}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_utf8","type":"utf8"},{"name":"x1","type":"bool"},{"name":"r0_f64","type":"f64"}],"columns":{"l0_utf8":["B","","ab","ab","B","ab","c","b","b","a","a","b",null,"zz","ab","😀","😀","b","😀","é","a","😀",null,"c","","aa","aa","zz","b","zz","😀","é","aa","é","😀","aa","b","zz",null,"a","a","a","","","","b","",null,null,"é","zz","aa","b","","😀","","","B"],"l1_i32":[-1,3,-3,-3,-1,2,2,1,1,-3,-3,-4,-4,-5,4,2147483647,2147483647,3,3,-2,-5,2147483647,5,5,-3,-3,-3,-4,-4,-3,5,-2,-4,1,2,4,-1,-3,-2,4,-5,-5,-3,-5,-5,-2,-1,-2,-2,-2,3,-2,-1,3,1,3,3,2],"l2_bool":[false,false,null,null,false,true,true,true,true,null,null,false,false,null,false,true,true,true,null,true,false,true,true,true,true,true,true,null,true,false,false,false,null,false,false,false,true,false,false,false,true,true,false,false,false,false,true,true,true,false,null,false,false,false,true,null,null,false],"l3_utf8":["B","","c","c","b","😀","","c","c","c","c","aa","aa","a","b","c","c",null,"😀","b","b","😀","zz","é","","c","c","b","a","é","B","é","b","a",null,"zz",null,"😀","zz","aa","c","c","zz","c","c","zz","é","c","c","zz","ab","b","zz",null,"ab","c","c","😀"],"x1":[false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false],"r0_f64":[null,null,-50.26,52.145,null,null,null,-50.26,52.145,-50.26,52.145,null,null,null,null,-50.26,52.145,null,null,null,null,null,null,null,null,-50.26,52.145,null,null,null,null,null,null,null,null,null,null,null,null,null,-50.26,52.145,null,-50.26,52.145,null,null,-50.26,52.145,null,-12.185,null,null,null,-12.185,-50.26,52.145,null]}}},{"id":"random/0091","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_bool","type":"bool"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_f64","type":"f64"},{"name":"l6_f64","type":"f64"}],"columns":{"l0_utf8":[null],"l1_i32":[-2147483648],"l2_utf8":["c"],"l3_bool":[true],"l4_utf8":["é"],"l5_f64":[38.674],"l6_f64":[-4.481]}}},"ops":[{"op":"unique"},{"op":"groupBy","keys":["l4_utf8"],"aggs":{"n1":{"agg":"len"},"a1_0":{"agg":"count","arg":{"col":"l0_utf8"}},"a1_1":{"agg":"max","arg":{"col":"l5_f64"}},"a1_2":{"agg":"mean","arg":{"col":"l1_i32"}}}},{"op":"head","n":2}],"expected":{"fields":[{"name":"l4_utf8","type":"utf8"},{"name":"n1","type":"u32"},{"name":"a1_0","type":"u32"},{"name":"a1_1","type":"f64"},{"name":"a1_2","type":"f64"}],"columns":{"l4_utf8":["é"],"n1":[1],"a1_0":[0],"a1_1":[38.674],"a1_2":[-2147483648.0]}}},{"id":"random/0092","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_i64","type":"i64"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_i32","type":"i32"},{"name":"l6_i64","type":"i64"}],"columns":{"l0_utf8":[null,"aa",null,null,null,"😀","a","a","b","aa","b",null,null,"B","😀","é","B","",null,"ab"],"l1_i32":[-2,-5,null,null,null,-3,null,1,null,null,2,-3,-1,-4,null,5,-3,null,-4,-1],"l2_i32":[2,0,null,-4,-4,5,-5,0,null,4,2,-5,-5,1,3,null,2,-5,0,null],"l3_i64":["9007199254740994","9007199254740993","9007199254740993","-4611686018427387904",null,"5","-4611686018427387904","0",null,null,null,"9007199254740994","-4611686018427387904","9007199254740995","9007199254740995","9007199254740992",null,"1","9007199254740995","9007199254740992"],"l4_utf8":[null,"a",null,"","c","",null,"é","😀","zz",null,null,"","aa",null,"B","a","é","B","aa"],"l5_i32":[-2,null,null,null,null,-319,null,1,null,1,1,null,-5,null,2,null,null,-1,3,-5],"l6_i64":["9007199254740992","1","9007199254740992","-4611686018427387904","-3","2","3","-4611686018427387904","9007199254740995","-4611686018427387904","-3","9007199254740993",null,null,"9007199254740995","9007199254740992","9007199254740992","-4611686018427387904","4","-4611686018427387904"]}}},"ops":[{"op":"head","n":1}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_i64","type":"i64"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_i32","type":"i32"},{"name":"l6_i64","type":"i64"}],"columns":{"l0_utf8":[null],"l1_i32":[-2],"l2_i32":[2],"l3_i64":["9007199254740994"],"l4_utf8":[null],"l5_i32":[-2],"l6_i64":["9007199254740992"]}}},{"id":"random/0093","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_bool","type":"bool"},{"name":"l4_f64","type":"f64"}],"columns":{"l0_utf8":["😀"],"l1_i32":[-2],"l2_bool":[true],"l3_bool":[false],"l4_f64":["NaN"]}}},"ops":[{"op":"sort","keys":[{"expr":{"col":"l1_i32"},"descending":false},{"expr":{"col":"l3_bool"},"descending":false}],"nullsLast":false},{"op":"unique","columns":["l1_i32"]},{"op":"head","n":2}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_bool","type":"bool"},{"name":"l4_f64","type":"f64"}],"columns":{"l0_utf8":["😀"],"l1_i32":[-2],"l2_bool":[true],"l3_bool":[false],"l4_f64":["NaN"]}}},{"id":"random/0094","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i64","type":"i64"},{"name":"l3_f64","type":"f64"},{"name":"l4_bool","type":"bool"},{"name":"l5_i32","type":"i32"}],"columns":{"l0_utf8":["😀","","😀","aa","b","","b","é","😀","ab","","c","c","ab","b",null,"c",null,"😀","aa"],"l1_i32":[0,2,5,null,4,null,-4,0,5,-2,null,1,2,-5,5,1,4,5,null,5],"l2_i64":["2","0","-4611686018427387904","9007199254740992","9007199254740994","9007199254740992","9007199254740995","0","9007199254740993","-4611686018427387904","9007199254740993","-4611686018427387904","5","-4611686018427387904","9007199254740995","9007199254740994","2","3","9007199254740995",null],"l3_f64":[83.107,-2.0,4.0,-1.0,2.0,null,34.601,null,26.729,4.0,-10.973,-3.0,-16.656,-4.0,null,3.0,77.078,-2.0,-12.471,"NaN"],"l4_bool":[true,true,false,false,false,true,false,true,false,false,false,false,true,false,true,false,false,false,true,false],"l5_i32":[-4,-5,1,null,2,-2,null,-1,-5,-2,-2,2,-4,-2,-3,2,-4,-2147483648,2147483647,2]}}},"ops":[{"op":"withColumns","exprs":{"x0":{"col":"l3_f64"}}},{"op":"head","n":0},{"op":"head","n":2}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i64","type":"i64"},{"name":"l3_f64","type":"f64"},{"name":"l4_bool","type":"bool"},{"name":"l5_i32","type":"i32"},{"name":"x0","type":"f64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_i64":[],"l3_f64":[],"l4_bool":[],"l5_i32":[],"x0":[]}}},{"id":"random/0095","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_i64","type":"i64"},{"name":"l4_i32","type":"i32"},{"name":"l5_i32","type":"i32"}],"columns":{"l0_utf8":["aa","c","b","é",null],"l1_i32":[0,5,5,5,3],"l2_f64":[-97.596,3.519,null,null,4.0],"l3_i64":["-4611686018427387904","9007199254740995","9007199254740994","9007199254740992","-3"],"l4_i32":[1,null,-4,3,4],"l5_i32":[2147483647,5,5,null,null]}}},"ops":[{"op":"withColumns","exprs":{"x0":{"col":"l4_i32"}}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_i64","type":"i64"},{"name":"l4_i32","type":"i32"},{"name":"l5_i32","type":"i32"},{"name":"x0","type":"i32"}],"columns":{"l0_utf8":["aa","c","b","é",null],"l1_i32":[0,5,5,5,3],"l2_f64":[-97.596,3.519,null,null,4.0],"l3_i64":["-4611686018427387904","9007199254740995","9007199254740994","9007199254740992","-3"],"l4_i32":[1,null,-4,3,4],"l5_i32":[2147483647,5,5,null,null],"x0":[1,null,-4,3,4]}}},{"id":"random/0096","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_i32","type":"i32"},{"name":"l6_i64","type":"i64"}],"columns":{"l0_utf8":["","aa","😀","B","B"],"l1_i32":[5,5,5,-1,5],"l2_f64":[28.568,93.458,96.591,22.159,3.0],"l3_utf8":["b","B","😀","😀","B"],"l4_utf8":["a","","b","a","a"],"l5_i32":[-4,-4,-5,-5,-1],"l6_i64":["-5","4","9007199254740993","-4611686018427387904","-4611686018427387904"]}}},"ops":[{"op":"head","n":7},{"op":"groupBy","keys":["l0_utf8"],"aggs":{"n1":{"agg":"len"},"a1_0":{"agg":"last","arg":{"col":"l1_i32"}},"a1_1":{"agg":"nUnique","arg":{"col":"l5_i32"}}}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"n1","type":"u32"},{"name":"a1_0","type":"i32"},{"name":"a1_1","type":"u32"}],"columns":{"l0_utf8":["","aa","😀","B"],"n1":[1,1,1,2],"a1_0":[5,5,5,5],"a1_1":[1,1,1,2]}}},{"id":"random/0097","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_f64","type":"f64"}],"columns":{"l0_utf8":["b",null,"é","a","B"],"l1_i32":[-5,null,3,null,-3],"l2_utf8":["😀","c","ab","😀","ab"],"l3_f64":[null,null,-48.169,-66.332,87.716]}},"right":{"fields":[{"name":"l2_utf8","type":"utf8"},{"name":"r0_f64","type":"f64"}],"columns":{"r0_f64":[null,-15.99,0.0,37.436,null,0.0,-4.0,-24.324,-2.0,-1.0],"l2_utf8":["c","ab","ab","😀","😀","😀","😀","😀","c","ab"]}}},"ops":[{"op":"join","right":"right","on":["l2_utf8"],"how":"full"},{"op":"sort","keys":[{"expr":{"col":"l1_i32"},"descending":false},{"expr":{"col":"l2_utf8"},"descending":true}],"nullsLast":false},{"op":"unique","columns":["l0_utf8"]}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_f64","type":"f64"},{"name":"r0_f64","type":"f64"}],"columns":{"l0_utf8":["a",null,"b","B","é"],"l1_i32":[null,null,-5,-3,3],"l2_utf8":["😀","c","😀","ab","ab"],"l3_f64":[-66.332,null,null,87.716,-48.169],"r0_f64":[37.436,null,37.436,-15.99,-15.99]}}},{"id":"random/0098","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_f64","type":"f64"},{"name":"l4_f64","type":"f64"}],"columns":{"l0_utf8":["ab","B","a","ab","😀"],"l1_i32":[null,null,4,null,-3],"l2_f64":[99.465,null,3.413,4.0,0.0],"l3_f64":[null,null,-4.0,4.0,null],"l4_f64":[-3.0,-4.0,-63.113,-4.0,-2.0]}}},"ops":[{"op":"filter","predicate":{"fn":"strStartsWith","args":[{"col":"l0_utf8"}],"prefix":"a"}},{"op":"filter","predicate":{"fn":"isNull","args":[{"col":"l3_f64"}]}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_f64","type":"f64"},{"name":"l4_f64","type":"f64"}],"columns":{"l0_utf8":["ab"],"l1_i32":[null],"l2_f64":[99.465],"l3_f64":[null],"l4_f64":[-3.0]}}},{"id":"random/0099","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_f64","type":"f64"}],"columns":{"l0_utf8":[null,null,null,null,"ab","zz","😀",null,null,null,"",null,"b","zz","ab","😀",null,"😀",null,"aa"],"l1_i32":[-4,null,null,null,4,null,null,null,3,-5,-3,-4,2,5,0,null,2,null,null,5],"l2_f64":[4.0,null,14.586,3.0,-1.0,null,3.0,-3.0,null,4.0,0.0,75.037,null,null,3.0,null,1.0,68.395,-81.302,-32.419],"l3_f64":[3.0,69.85,null,-3.0,3.0,null,null,43.911,0.0,80.338,1.0,-28.491,null,null,-14.784,null,-6.82,-2.0,3.0,null]}},"right":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"r0_f64","type":"f64"}],"columns":{"r0_f64":[],"l0_utf8":[]}}},"ops":[{"op":"unique","columns":["l0_utf8"]},{"op":"groupBy","keys":["l0_utf8"],"aggs":{"n1":{"agg":"len"},"a1_0":{"agg":"mean","arg":{"col":"l1_i32"}},"a1_1":{"agg":"max","arg":{"col":"l2_f64"}}}},{"op":"join","right":"right","on":["l0_utf8"],"how":"left"}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"n1","type":"u32"},{"name":"a1_0","type":"f64"},{"name":"a1_1","type":"f64"},{"name":"r0_f64","type":"f64"}],"columns":{"l0_utf8":[null,"ab","zz","😀","","b","aa"],"n1":[1,1,1,1,1,1,1],"a1_0":[-4.0,4.0,null,null,-3.0,2.0,5.0],"a1_1":[4.0,-1.0,null,3.0,0.0,null,-32.419],"r0_f64":[null,null,null,null,null,null,null]}}},{"id":"random/0100","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i64","type":"i64"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_utf8","type":"utf8"}],"columns":{"l0_utf8":["B","é","c",null,null],"l1_i32":[-3,5,4,-3,3],"l2_i64":["-4611686018427387904","-4611686018427387904","9007199254740992",null,"-2"],"l3_utf8":["","é","é","aa",null],"l4_utf8":["zz","😀","😀","B","b"]}}},"ops":[{"op":"sort","keys":[{"expr":{"col":"l2_i64"},"descending":true}],"nullsLast":true},{"op":"head","n":2},{"op":"withColumns","exprs":{"x2":{"fn":"isIn","args":[{"col":"l3_utf8"}],"values":["","b","ab"]}}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i64","type":"i64"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_utf8","type":"utf8"},{"name":"x2","type":"bool"}],"columns":{"l0_utf8":["c",null],"l1_i32":[4,3],"l2_i64":["9007199254740992","-2"],"l3_utf8":["é",null],"l4_utf8":["😀","b"],"x2":[false,null]}}},{"id":"random/0101","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_i32","type":"i32"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_bool","type":"bool"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_f64":[],"l3_i32":[],"l4_utf8":[],"l5_bool":[]}}},"ops":[{"op":"unique","columns":["l4_utf8"]},{"op":"filter","predicate":{"col":"l5_bool"}},{"op":"filter","predicate":{"fn":"gte","args":[{"fn":"neg","args":[{"col":"l1_i32"}]},{"fn":"floorDiv","args":[{"col":"l1_i32"},{"col":"l1_i32"}]}]}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_i32","type":"i32"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_bool","type":"bool"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_f64":[],"l3_i32":[],"l4_utf8":[],"l5_bool":[]}}},{"id":"random/0102","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_i32","type":"i32"},{"name":"l4_bool","type":"bool"}],"columns":{"l0_utf8":[""],"l1_i32":[3],"l2_bool":[true],"l3_i32":[-3],"l4_bool":[false]}},"right":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"r0_bool","type":"bool"}],"columns":{"r0_bool":[null,false,true,true,null,null,false,null,false,false],"l0_utf8":["","","","","","","","","",""]}}},"ops":[{"op":"withColumns","exprs":{"x0":{"fn":"neg","args":[{"fn":"mod","args":[{"lit":2.5},{"lit":2.5}]}]}}},{"op":"filter","predicate":{"fn":"isNull","args":[{"col":"l3_i32"}]}},{"op":"join","right":"right","on":["l0_utf8"],"how":"anti"},{"op":"unique","columns":["l1_i32"]}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_i32","type":"i32"},{"name":"l4_bool","type":"bool"},{"name":"x0","type":"f64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_bool":[],"l3_i32":[],"l4_bool":[],"x0":[]}}},{"id":"random/0103","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_f64","type":"f64"},{"name":"l4_i64","type":"i64"}],"columns":{"l0_utf8":["😀",null,"b","é",null],"l1_i32":[3,0,1,-3,0],"l2_f64":[-76.38,-2.0,-54.446,95.132,-4.0],"l3_f64":[-26.664,2.0,3.0,"NaN",-38.242],"l4_i64":["-4611686018427387904",null,"-4611686018427387904","-4611686018427387904","-4611686018427387904"]}}},"ops":[{"op":"sort","keys":[{"expr":{"col":"l1_i32"},"descending":false},{"expr":{"col":"l2_f64"},"descending":true}],"nullsLast":false},{"op":"withColumns","exprs":{"x1":{"fn":"sub","args":[{"fn":"div","args":[{"col":"l1_i32"},{"col":"l1_i32"}]},{"col":"l2_f64"}]}}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_f64","type":"f64"},{"name":"l4_i64","type":"i64"},{"name":"x1","type":"f64"}],"columns":{"l0_utf8":["é",null,null,"b","😀"],"l1_i32":[-3,0,0,1,3],"l2_f64":[95.132,-2.0,-4.0,-54.446,-76.38],"l3_f64":["NaN",2.0,-38.242,3.0,-26.664],"l4_i64":["-4611686018427387904",null,"-4611686018427387904","-4611686018427387904","-4611686018427387904"],"x1":[-94.132,"NaN","NaN",55.446,77.38]}}},{"id":"random/0104","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_f64","type":"f64"},{"name":"l4_f64","type":"f64"}],"columns":{"l0_utf8":["zz","é","ab","B","😀","","b","😀","B","b","é","","a","ab","b","B","é","B","","c"],"l1_i32":[-2,1,-3,0,-1,3,-3,-5,0,-5,1,5,1,1,1,-2,-1,-2,-2,-3],"l2_f64":[-20.036,4.0,"NaN",-1.0,0.0,-4.0,-3.0,4.0,-76.874,-4.0,0.0,-2.0,2.0,81.913,2.0,-4.0,-47.67,1.0,3.0,-26.66],"l3_f64":[93.831,-97.018,0.0,98.231,-43.895,17.645,-3.0,3.0,-53.12,2.0,2.0,"NaN",4.0,81.133,0.0,10.613,1.0,32.628,68.959,"-Infinity"],"l4_f64":[4.0,80.392,-4.0,-37.92,"-0",-3.0,2.0,-4.0,94.57,0.0,2.0,93.977,67.582,2.0,-2.0,-2.0,-1.0,2.0,87.844,1.0]}},"right":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"r0_utf8","type":"utf8"}],"columns":{"r0_utf8":["b","é","ab","ab","zz","zz","a","zz","b","zz"],"l0_utf8":["zz","B","B","c","zz","B","é","é","c","😀"]}}},"ops":[{"op":"withColumns","exprs":{"x0":{"fn":"mod","args":[{"fn":"abs","args":[{"col":"l2_f64"}]},{"fn":"when","args":[{"fn":"eq","args":[{"col":"l1_i32"},{"col":"l3_f64"}]},{"col":"l4_f64"},{"col":"l3_f64"}]}]}}},{"op":"join","right":"right","on":["l0_utf8"],"how":"left"}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_f64","type":"f64"},{"name":"l4_f64","type":"f64"},{"name":"x0","type":"f64"},{"name":"r0_utf8","type":"utf8"}],"columns":{"l0_utf8":["zz","zz","é","é","ab","B","B","B","😀","","b","😀","B","B","B","b","é","é","","a","ab","b","B","B","B","é","é","B","B","B","","c","c"],"l1_i32":[-2,-2,1,1,-3,0,0,0,-1,3,-3,-5,0,0,0,-5,1,1,5,1,1,1,-2,-2,-2,-1,-1,-2,-2,-2,-2,-3,-3],"l2_f64":[-20.036,-20.036,4.0,4.0,"NaN",-1.0,-1.0,-1.0,0.0,-4.0,-3.0,4.0,-76.874,-76.874,-76.874,-4.0,0.0,0.0,-2.0,2.0,81.913,2.0,-4.0,-4.0,-4.0,-47.67,-47.67,1.0,1.0,1.0,3.0,-26.66,-26.66],"l3_f64":[93.831,93.831,-97.018,-97.018,0.0,98.231,98.231,98.231,-43.895,17.645,-3.0,3.0,-53.12,-53.12,-53.12,2.0,2.0,2.0,"NaN",4.0,81.133,0.0,10.613,10.613,10.613,1.0,1.0,32.628,32.628,32.628,68.959,"-Infinity","-Infinity"],"l4_f64":[4.0,4.0,80.392,80.392,-4.0,-37.92,-37.92,-37.92,"-0",-3.0,2.0,-4.0,94.57,94.57,94.57,0.0,2.0,2.0,93.977,67.582,2.0,-2.0,-2.0,-2.0,-2.0,-1.0,-1.0,2.0,2.0,2.0,87.844,1.0,1.0],"x0":[20.036,20.036,-93.018,-93.018,"NaN",1.0,1.0,1.0,0.0,4.0,1.0,1.0,-29.366,-29.366,-29.366,0.0,0.0,0.0,"NaN",2.0,0.7800000000000011,"NaN",4.0,4.0,4.0,0.6700000000000017,0.6700000000000017,1.0,1.0,1.0,3.0,"NaN","NaN"],"r0_utf8":["b","zz","a","zz",null,"é","ab","zz","zz",null,null,"zz","é","ab","zz",null,"a","zz",null,null,null,null,"é","ab","zz","a","zz","é","ab","zz",null,"ab","b"]}}},{"id":"random/0105","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_i64","type":"i64"}],"columns":{"l0_utf8":[null],"l1_i32":[-4],"l2_utf8":["a"],"l3_i64":["-4611686018427387904"]}}},"ops":[{"op":"head","n":1},{"op":"unique","columns":["l0_utf8"]},{"op":"sort","keys":[{"expr":{"col":"l2_utf8"},"descending":false},{"expr":{"col":"l1_i32"},"descending":true}],"nullsLast":false},{"op":"head","n":2}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_i64","type":"i64"}],"columns":{"l0_utf8":[null],"l1_i32":[-4],"l2_utf8":["a"],"l3_i64":["-4611686018427387904"]}}},{"id":"random/0106","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_f64","type":"f64"}],"columns":{"l0_utf8":["ab","","B","é","ab"],"l1_i32":[5,-2147483648,-1,3,-1],"l2_f64":[-49.089,-58.946,-91.914,86.019,-53.887],"l3_f64":[2.0,3.0,0.0,3.0,-37.86]}}},"ops":[{"op":"groupBy","keys":["l1_i32"],"aggs":{"n0":{"agg":"len"},"a0_0":{"agg":"min","arg":{"col":"l0_utf8"}}}},{"op":"filter","predicate":{"fn":"eq","args":[{"fn":"mod","args":[{"col":"l1_i32"},{"col":"l1_i32"}]},{"fn":"mul","args":[{"col":"l1_i32"},{"lit":2}]}]}},{"op":"sort","keys":[{"expr":{"col":"a0_0"},"descending":true}],"nullsLast":false},{"op":"unique","columns":["l1_i32"]}],"expected":{"fields":[{"name":"l1_i32","type":"i32"},{"name":"n0","type":"u32"},{"name":"a0_0","type":"utf8"}],"columns":{"l1_i32":[-2147483648],"n0":[1],"a0_0":[""]}}},{"id":"random/0107","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_i64","type":"i64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_utf8":[],"l3_i64":[]}}},"ops":[{"op":"unique"}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_i64","type":"i64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_utf8":[],"l3_i64":[]}}},{"id":"random/0108","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_i32","type":"i32"},{"name":"l4_i32","type":"i32"}],"columns":{"l0_utf8":["a","a","","aa",null],"l1_i32":[597,null,-4,null,null],"l2_utf8":["b",null,null,null,"B"],"l3_i32":[5,null,5,-1,-306],"l4_i32":[null,-1,null,null,-3]}},"right":{"fields":[{"name":"l4_i32","type":"i32"},{"name":"r0_f64","type":"f64"}],"columns":{"r0_f64":["-0",44.618,41.662,null,null,-21.543,-2.0,-1.0,-2.0,-3.0,null,-50.709,3.0,12.306,0.0,null,null,-2.0,-3.0,3.0,null,"NaN",-4.0,3.0,42.326,null,0.0,null,null,-63.517],"l4_i32":[null,null,-1,-1,-1,null,null,null,null,-3,-3,null,-3,-1,-1,null,null,-3,-1,null,-3,-3,null,null,null,null,-1,null,null,null]}}},"ops":[{"op":"filter","predicate":{"fn":"isIn","args":[{"col":"l2_utf8"}],"values":["","é","😀"]}},{"op":"unique","columns":["l3_i32"]},{"op":"join","right":"right","on":["l4_i32"],"how":"semi"}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_i32","type":"i32"},{"name":"l4_i32","type":"i32"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_utf8":[],"l3_i32":[],"l4_i32":[]}}},{"id":"random/0109","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_i64","type":"i64"}],"columns":{"l0_utf8":["zz","zz","c","é","a","zz","a","B","B","😀","B","aa","b","","ab","B","ab","c","c","😀"],"l1_i32":[0,-4,0,-1,5,1,1,-5,5,0,0,-712,-824,5,-3,-4,-4,-2,-4,-2],"l2_utf8":["zz","zz","zz","","c","aa","😀","","zz","a","","é","😀","😀","aa","B","zz","aa","aa","a"],"l3_i64":["-4611686018427387904","4","9007199254740993","2","9007199254740993","-2","9007199254740995","-4611686018427387904","4","-4","9007199254740992","-4611686018427387904","2","-4611686018427387904","-4611686018427387904","-4611686018427387904","9007199254740994","9007199254740993","9007199254740992","9007199254740994"]}}},"ops":[{"op":"filter","predicate":{"fn":"eq","args":[{"col":"l3_i64"},{"lit":"3","type":"i64"}]}},{"op":"withColumns","exprs":{"x1":{"fn":"and","args":[{"fn":"lte","args":[{"col":"l1_i32"},{"col":"l1_i32"}]},{"fn":"neq","args":[{"col":"l1_i32"},{"col":"l1_i32"}]}]}}},{"op":"withColumns","exprs":{"x2":{"col":"l1_i32"}}},{"op":"withColumns","exprs":{"x3":{"fn":"mod","args":[{"fn":"when","args":[{"fn":"gte","args":[{"lit":2.5},{"col":"l1_i32"}]},{"col":"x2"},{"col":"x2"}]},{"col":"l1_i32"}]}}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_i64","type":"i64"},{"name":"x1","type":"bool"},{"name":"x2","type":"i32"},{"name":"x3","type":"i32"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_utf8":[],"l3_i64":[],"x1":[],"x2":[],"x3":[]}}},{"id":"random/0110","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_i64","type":"i64"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_bool","type":"bool"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_i32":[],"l3_i64":[],"l4_utf8":[],"l5_bool":[]}}},"ops":[{"op":"head","n":9},{"op":"filter","predicate":{"fn":"eq","args":[{"fn":"fillNull","args":[{"lit":0.5},{"lit":0}]},{"fn":"sub","args":[{"col":"l1_i32"},{"col":"l2_i32"}]}]}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_i64","type":"i64"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_bool","type":"bool"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_i32":[],"l3_i64":[],"l4_utf8":[],"l5_bool":[]}}},{"id":"random/0111","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_f64","type":"f64"},{"name":"l4_bool","type":"bool"},{"name":"l5_i64","type":"i64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_utf8":[],"l3_f64":[],"l4_bool":[],"l5_i64":[]}}},"ops":[{"op":"filter","predicate":{"fn":"lt","args":[{"fn":"sub","args":[{"lit":0.5},{"col":"l3_f64"}]},{"col":"l3_f64"}]}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_f64","type":"f64"},{"name":"l4_bool","type":"bool"},{"name":"l5_i64","type":"i64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_utf8":[],"l3_f64":[],"l4_bool":[],"l5_i64":[]}}},{"id":"random/0112","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_utf8","type":"utf8"}],"columns":{"l0_utf8":["é","aa","aa","😀","B"],"l1_i32":[-5,1,-2,3,3],"l2_f64":[97.428,0.0,-26.499,90.545,0.0],"l3_utf8":["","zz","","😀","é"]}}},"ops":[{"op":"withColumns","exprs":{"x0":{"fn":"eq","args":[{"col":"l3_utf8"},{"lit":"a"}]}}},{"op":"sort","keys":[{"expr":{"col":"l0_utf8"},"descending":false},{"expr":{"col":"l1_i32"},"descending":true}],"nullsLast":false},{"op":"withColumns","exprs":{"x2":{"col":"l2_f64"}}},{"op":"withColumns","exprs":{"x3":{"fn":"gte","args":[{"col":"l1_i32"},{"fn":"floorDiv","args":[{"col":"l1_i32"},{"col":"l2_f64"}]}]}}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_utf8","type":"utf8"},{"name":"x0","type":"bool"},{"name":"x2","type":"f64"},{"name":"x3","type":"bool"}],"columns":{"l0_utf8":["B","aa","aa","é","😀"],"l1_i32":[3,1,-2,-5,3],"l2_f64":[0.0,0.0,-26.499,97.428,90.545],"l3_utf8":["é","zz","","","😀"],"x0":[false,false,false,false,false],"x2":[0.0,0.0,-26.499,97.428,90.545],"x3":[false,false,false,false,true]}}},{"id":"random/0113","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_f64","type":"f64"},{"name":"l4_i32","type":"i32"},{"name":"l5_utf8","type":"utf8"},{"name":"l6_i64","type":"i64"}],"columns":{"l0_utf8":["😀","aa","aa","B","ab"],"l1_i32":[-3,4,2,-438,3],"l2_utf8":["zz","","a","zz","ab"],"l3_f64":[2.0,91.628,2.0,3.0,2.0],"l4_i32":[-5,2,-5,-4,-2],"l5_utf8":["c","é","c","B","b"],"l6_i64":["9007199254740995","-2","-5","-3","-4611686018427387904"]}}},"ops":[{"op":"groupBy","keys":["l6_i64"],"aggs":{"n0":{"agg":"len"},"a0_0":{"agg":"min","arg":{"col":"l4_i32"}},"a0_1":{"agg":"max","arg":{"col":"l0_utf8"}},"a0_2":{"agg":"min","arg":{"col":"l5_utf8"}}}},{"op":"head","n":9},{"op":"withColumns","exprs":{"x2":{"fn":"abs","args":[{"lit":2.5}]}}}],"expected":{"fields":[{"name":"l6_i64","type":"i64"},{"name":"n0","type":"u32"},{"name":"a0_0","type":"i32"},{"name":"a0_1","type":"utf8"},{"name":"a0_2","type":"utf8"},{"name":"x2","type":"f64"}],"columns":{"l6_i64":["9007199254740995","-2","-5","-3","-4611686018427387904"],"n0":[1,1,1,1,1],"a0_0":[-5,2,-5,-4,-2],"a0_1":["😀","aa","aa","B","ab"],"a0_2":["c","é","c","B","b"],"x2":[2.5,2.5,2.5,2.5,2.5]}}},{"id":"random/0114","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_f64","type":"f64"},{"name":"l4_i64","type":"i64"},{"name":"l5_f64","type":"f64"},{"name":"l6_i32","type":"i32"}],"columns":{"l0_utf8":["aa","😀","b","","","B","é","b","😀","zz","c","aa","é","c","😀","B","c","b","B","c","c","B","B","aa","ab","ab","aa","c","zz","ab","b","a","aa","ab","B","B","b","c","a","aa","zz","zz","c","zz","é","aa","zz","","😀","ab","c","","😀","é","","😀","😀","é","aa","zz"],"l1_i32":[-3,2147483647,-3,4,5,-2147483648,-157,-4,0,-4,3,-1,5,3,5,3,-2,3,2,-3,-2,-1,-4,3,-4,-3,-5,0,1,-4,3,-3,-1,-4,-4,-4,-3,4,5,3,4,3,-4,5,4,-1,4,2,5,2,-2147483648,5,5,4,3,2,4,-1,-3,2],"l2_i32":[2,-1,-4,5,-5,0,-2,-5,4,-1,-5,0,5,5,-1,0,-1,681,-3,2,350,3,1,2,2,-5,5,4,-2,1,3,5,-2147483648,2,2,2,1,0,4,0,-2,5,-4,-4,3,-3,3,-1,-4,-4,0,4,-5,-2147483648,3,0,-2,1,-1,-4],"l3_f64":[26.801,"NaN",-4.0,-1.0,0.0,1.0,2.0,4.0,1.0,3.0,2.0,4.0,-4.0,4.0,1.0,-1.0,3.0,-22.3,2.331,-5.4,-14.111,-94.213,-2.0,-27.078,"Infinity",46.674,60.941,23.922,-38.117,1.0,99.344,-41.243,19.693,4.0,2.0,-74.51,-65.303,3.0,-2.0,1.0,"NaN",-4.0,3.0,-2.0,4.0,3.0,3.0,-98.575,0.0,-58.964,-4.0,1.0,0.0,73.924,-1.0,2.0,3.0,-90.645,3.0,3.0],"l4_i64":["-4611686018427387904","-4","9007199254740993","9007199254740995","9007199254740995","9007199254740992","9007199254740994","-4611686018427387904","-4611686018427387904","9007199254740995","-4611686018427387904","-4611686018427387904","9007199254740992","9007199254740994","3","2","-4611686018427387904","-1","-4611686018427387904","9007199254740993","-4611686018427387904","0","-4611686018427387904","-4611686018427387904","-4611686018427387904","-4611686018427387904","-1","5","9007199254740992","-4611686018427387904","3","9007199254740992","9007199254740992","-4611686018427387904","-3","9007199254740993","9007199254740993","-3","-4611686018427387904","-1","3","9007199254740995","-4","-4611686018427387904","-4","-3","9007199254740993","9007199254740993","9007199254740993","9007199254740995","9007199254740995","2","-4611686018427387904","5","0","-4611686018427387904","-4","-4611686018427387904","0","9007199254740993"],"l5_f64":[-44.856,35.521,-31.52,-70.008,-1.0,0.0,-38.421,0.0,2.0,-4.0,"-Infinity",66.639,-78.513,12.231,-46.561,-61.45,-4.0,-41.91,-2.0,-2.0,89.202,-1.0,-91.788,-2.0,2.0,"NaN",-2.0,9.489,90.872,-76.153,-16.603,-4.0,-61.137,4.0,1.0,-47.606,3.0,-36.824,-45.113,30.015,-40.606,-77.345,46.223,2.0,90.838,-99.508,-6.783,-17.157,14.248,-3.0,88.217,69.461,-43.519,51.257,-2.0,-22.795,64.935,72.697,3.05,-3.0],"l6_i32":[1,-1,1,-1,1,1,3,2147483647,-1,-2,4,-3,4,4,2,2,-3,2,0,2,0,5,-5,3,2,-3,3,3,-4,2147483647,1,0,0,4,0,-3,928,-4,-4,-1,-3,-5,-3,-3,5,1,-3,4,4,-5,5,0,-5,5,3,-4,2,5,5,-3]}}},"ops":[{"op":"sort","keys":[{"expr":{"col":"l5_f64"},"descending":false}],"nullsLast":true}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_f64","type":"f64"},{"name":"l4_i64","type":"i64"},{"name":"l5_f64","type":"f64"},{"name":"l6_i32","type":"i32"}],"columns":{"l0_utf8":["c","aa","B","é","zz","ab","","B","aa","B","😀","a","aa","😀","b","zz","é","c","b","😀","","b","zz","zz","c","a","ab","zz","B","c","aa","aa","","","B","B","b","B","😀","ab","zz","b","aa","ab","c","c","😀","aa","😀","c","é","😀","aa","","é","c","c","é","zz","ab"],"l1_i32":[3,-1,-4,5,3,-4,4,3,-1,-4,5,5,-3,5,3,4,-157,4,-3,2,2,3,4,-4,-2,-3,2,2,2,-3,3,-5,3,5,-1,-2147483648,-4,-4,0,-4,5,-3,-3,-4,0,3,5,3,2147483647,-4,4,4,-1,5,-1,-2147483648,-2,4,1,-3],"l2_i32":[-5,-3,1,5,5,1,5,0,-2147483648,2,-1,4,2,-5,681,-2,-2,0,-4,0,-1,3,3,-1,-1,5,-4,-4,-3,2,2,5,3,-5,3,0,-5,2,4,2,-4,1,-1,2,4,5,-4,0,-1,-4,-2147483648,-2,0,4,1,0,350,3,-2,-5],"l3_f64":[2.0,3.0,-2.0,-4.0,-4.0,1.0,-1.0,-1.0,19.693,-74.51,1.0,-2.0,26.801,0.0,-22.3,"NaN",2.0,3.0,-4.0,2.0,-98.575,99.344,3.0,3.0,3.0,-41.243,-58.964,3.0,2.331,-5.4,-27.078,60.941,-1.0,0.0,-94.213,1.0,4.0,2.0,1.0,"Infinity",-2.0,-65.303,3.0,4.0,23.922,4.0,0.0,1.0,"NaN",3.0,73.924,3.0,4.0,1.0,-90.645,-4.0,-14.111,4.0,-38.117,46.674],"l4_i64":["-4611686018427387904","-3","-4611686018427387904","9007199254740992","9007199254740995","-4611686018427387904","9007199254740995","2","9007199254740992","9007199254740993","3","-4611686018427387904","-4611686018427387904","-4611686018427387904","-1","3","9007199254740994","-3","9007199254740993","-4611686018427387904","9007199254740993","3","9007199254740993","9007199254740995","-4611686018427387904","9007199254740992","9007199254740995","9007199254740993","-4611686018427387904","9007199254740993","-4611686018427387904","-1","0","9007199254740995","0","9007199254740992","-4611686018427387904","-3","-4611686018427387904","-4611686018427387904","-4611686018427387904","9007199254740993","0","-4611686018427387904","5","9007199254740994","9007199254740993","-1","-4","-4","5","-4","-4611686018427387904","2","-4611686018427387904","9007199254740995","-4611686018427387904","-4","9007199254740992","-4611686018427387904"],"l5_f64":["-Infinity",-99.508,-91.788,-78.513,-77.345,-76.153,-70.008,-61.45,-61.137,-47.606,-46.561,-45.113,-44.856,-43.519,-41.91,-40.606,-38.421,-36.824,-31.52,-22.795,-17.157,-16.603,-6.783,-4.0,-4.0,-4.0,-3.0,-3.0,-2.0,-2.0,-2.0,-2.0,-2.0,-1.0,-1.0,0.0,0.0,1.0,2.0,2.0,2.0,3.0,3.05,4.0,9.489,12.231,14.248,30.015,35.521,46.223,51.257,64.935,66.639,69.461,72.697,88.217,89.202,90.838,90.872,"NaN"],"l6_i32":[4,1,-5,4,-5,2147483647,-1,2,0,-3,2,-4,1,-5,2,-3,3,-4,1,-4,4,1,-3,-2,-3,0,-5,-3,0,2,3,3,3,1,5,1,2147483647,0,-1,2,-3,928,5,4,3,4,4,-1,-1,-3,5,2,-3,0,5,5,0,5,-4,-3]}}},{"id":"random/0115","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_utf8","type":"utf8"}],"columns":{"l0_utf8":["😀","é","ab",null,"b","a","b","B","B","aa","zz","b","c","é","zz","","é","B","ab","c","","b","zz",null,"","😀","é","ab",null,"zz","é","a","c","c","zz","aa","😀","B","é","c","b","😀","zz","aa","b","😀","b","B","c",null,"😀","ab","c","zz","b","ab",null,"zz","c","é"],"l1_i32":[5,3,1,-2147483648,-1,-3,-2,-4,0,4,1,5,4,4,-914,-2,-3,0,-5,-2,1,-700,2147483647,-5,-5,3,-1,1,4,4,-5,4,5,null,1,5,-3,4,null,0,-1,1,0,1,2,0,-2,null,3,1,-1,-2,2,null,-1,4,-5,3,1,5],"l2_i32":[-2,-5,4,-4,-2,-4,-2,618,-5,-318,-2,-1,null,1,-4,null,-4,-306,-5,0,-2,4,2,-4,3,0,-4,-3,-2147483648,-5,2147483647,3,-4,4,null,-2,4,1,2,-4,2,2,5,0,4,-5,-2,1,0,0,null,2147483647,1,2,-4,-2,-2,-3,-3,5],"l3_utf8":["aa",null,"😀","B","c","é","é","aa","aa","😀","B","ab","aa","é","é","aa",null,"aa","zz","😀","ab","é","zz","a","😀","a","a","",null,"aa","a","B","zz","c","zz","é","ab","c","a","ab","zz","zz","","a","ab","😀","zz","é",null,"aa",null,"😀","😀","c","b","","c",null,null,""]}}},"ops":[{"op":"unique","columns":["l1_i32"]},{"op":"filter","predicate":{"fn":"lte","args":[{"fn":"sub","args":[{"col":"l2_i32"},{"col":"l1_i32"}]},{"lit":2.5}]}},{"op":"withColumns","exprs":{"x2":{"col":"l1_i32"}}},{"op":"sort","keys":[{"expr":{"col":"l0_utf8"},"descending":false}],"nullsLast":true}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_utf8","type":"utf8"},{"name":"x2","type":"i32"}],"columns":{"l0_utf8":["B","a","aa","ab","b","b","b","zz","é","😀"],"l1_i32":[0,-3,4,-5,-1,-2,2,2147483647,3,5],"l2_i32":[-5,-4,-318,-5,-2,-2,4,2,-5,-2],"l3_utf8":["aa","é","😀","zz","c","é","ab","zz",null,"aa"],"x2":[0,-3,4,-5,-1,-2,2,2147483647,3,5]}}},{"id":"random/0116","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_i64","type":"i64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_f64":[],"l3_utf8":[],"l4_i64":[]}}},"ops":[{"op":"head","n":2},{"op":"filter","predicate":{"fn":"gte","args":[{"fn":"sub","args":[{"col":"l1_i32"},{"col":"l1_i32"}]},{"fn":"add","args":[{"col":"l2_f64"},{"col":"l1_i32"}]}]}},{"op":"filter","predicate":{"fn":"neq","args":[{"fn":"sub","args":[{"col":"l2_f64"},{"col":"l2_f64"}]},{"fn":"sub","args":[{"col":"l2_f64"},{"col":"l2_f64"}]}]}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_i64","type":"i64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_f64":[],"l3_utf8":[],"l4_i64":[]}}},{"id":"random/0117","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_f64","type":"f64"}],"columns":{"l0_utf8":["aa","b","","😀","B","aa","aa","zz","a","aa","a","😀","😀","zz","a","b","b","c","c","b"],"l1_i32":[-4,5,1,4,-3,3,4,5,1,2,1,3,2,-3,2,0,1,-2,2,-4],"l2_utf8":["zz","zz","c","é","é","B","é","zz","aa","😀","ab","b","é","a","zz","😀","c","ab","😀","c"],"l3_utf8":["😀","a","é","é","a","zz","zz","c","c","","😀","a","B","b","aa","a","","aa","c","aa"],"l4_f64":[4.0,2.0,"NaN",79.849,1.0,-4.0,56.742,29.192,-23.865,2.0,"-0",-68.343,1.0,-54.662,-1.0,53.017,7.166,67.556,4.0,-83.429]}}},"ops":[{"op":"filter","predicate":{"fn":"isNotNull","args":[{"col":"l0_utf8"}]}},{"op":"unique","columns":["l1_i32"]}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_f64","type":"f64"}],"columns":{"l0_utf8":["aa","b","","😀","B","aa","aa","b","c"],"l1_i32":[-4,5,1,4,-3,3,2,0,-2],"l2_utf8":["zz","zz","c","é","é","B","😀","😀","ab"],"l3_utf8":["😀","a","é","é","a","zz","","a","aa"],"l4_f64":[4.0,2.0,"NaN",79.849,1.0,-4.0,2.0,53.017,67.556]}}},{"id":"random/0118","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_i64","type":"i64"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_bool","type":"bool"},{"name":"l6_bool","type":"bool"}],"columns":{"l0_utf8":["😀","a","B","b","😀","zz","B","aa","b","zz","c","","zz","aa","é","zz","c","c","a","","a","b","a",null,"zz","c","",null,"b","ab","a","c","c","a","B",null,"b","é","😀","","b","é",null,"é",null,"",null,"a",null,null,"é","B","a","c","aa","aa","zz",null,"","é"],"l1_i32":[-1,-599,-4,-1,3,3,-5,3,4,2,-3,-4,null,4,-2,3,0,-4,-5,1,-932,1,5,2,-5,-2147483648,null,3,5,0,-1,3,0,3,-2,4,-2,-4,5,1,-2,0,5,-1,5,975,0,4,null,-4,3,-2,5,0,null,-1,-1,-3,0,4],"l2_bool":[null,false,false,false,true,true,true,true,false,true,true,true,true,true,false,false,false,false,false,null,true,null,null,false,true,false,false,true,true,null,true,true,false,false,false,false,false,false,false,false,true,true,false,true,false,true,null,true,true,null,false,null,false,true,true,true,true,null,false,false],"l3_i64":["-3","9007199254740995","9007199254740992","-2","3","-3","3",null,"9007199254740995","9007199254740993","-5","9007199254740995","-4611686018427387904",null,"5","-4611686018427387904",null,null,"9007199254740994","9007199254740995","-1","9007199254740992","9007199254740995",null,"1","-4611686018427387904","-4611686018427387904","-4611686018427387904","-4611686018427387904","-4611686018427387904","9007199254740994","-4611686018427387904","-3","3","9007199254740992","9007199254740993","-4611686018427387904","9007199254740995","9007199254740992","-4611686018427387904","9007199254740993","9007199254740995","9007199254740994","-4","-4611686018427387904","-4611686018427387904","-4","5","-4611686018427387904","-4611686018427387904","9007199254740995","3","-3","-2","0","-1",null,"4","9007199254740993","-4"],"l4_utf8":["zz","","zz","B","zz","ab","c","é","é","aa","zz","b","B","zz","b","zz","zz","zz","😀","zz","c",null,"aa","a","b",null,"ab","b",null,"b","😀","é","c","a","ab","b","ab","ab","c","ab","a",null,"B",null,"é",null,"","zz","B","B",null,"zz","B","a","B","é","b","a","","b"],"l5_bool":[null,null,true,null,false,true,true,true,false,false,false,false,false,null,false,true,true,true,false,true,true,false,true,true,false,true,true,null,false,true,true,null,false,true,true,true,false,true,true,true,true,false,true,true,false,null,true,true,false,false,false,false,false,true,true,true,true,null,false,false],"l6_bool":[false,true,false,true,true,true,true,false,true,true,true,true,true,true,null,false,null,true,true,false,false,true,false,false,true,false,false,true,false,true,false,true,true,false,true,false,true,null,true,true,false,false,true,false,true,true,true,true,true,true,false,false,false,true,false,null,true,false,true,false]}}},"ops":[{"op":"withColumns","exprs":{"x0":{"col":"l1_i32"}}},{"op":"head","n":2},{"op":"head","n":8},{"op":"filter","predicate":{"fn":"eq","args":[{"fn":"mul","args":[{"lit":2},{"col":"x0"}]},{"fn":"sub","args":[{"col":"l1_i32"},{"col":"l1_i32"}]}]}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_i64","type":"i64"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_bool","type":"bool"},{"name":"l6_bool","type":"bool"},{"name":"x0","type":"i32"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_bool":[],"l3_i64":[],"l4_utf8":[],"l5_bool":[],"l6_bool":[],"x0":[]}}},{"id":"random/0119","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_i64","type":"i64"},{"name":"l4_utf8","type":"utf8"}],"columns":{"l0_utf8":["😀","","zz","aa","aa","aa","ab","b","ab","zz","","a","B","ab","aa","😀","zz","b","ab","ab","","c","aa","B","b","zz","a","B","aa","b","B","","B","a","😀","","zz","é","B","😀","b","b","aa","zz","zz","B","zz","aa","c","a","ab","","c","b","b","B","😀","","c","zz"],"l1_i32":[3,2,1,-2,-1,1,5,2,-431,2147483647,3,-4,3,-2147483648,5,-5,-1,1,-3,4,0,5,-3,-5,5,3,0,3,-2,5,-2,2,-2147483648,4,3,-5,4,3,2,-5,-1,-2,-3,-4,-1,2,-2147483648,-3,5,3,-2,0,-5,1,-5,5,1,0,0,2147483647],"l2_f64":[64.964,-74.044,-4.0,-4.0,-55.263,-41.263,-3.0,3.0,-25.216,66.879,88.456,-4.0,-1.0,-16.014,-4.0,-38.809,-22.499,-6.775,4.0,67.084,4.0,26.549,-1.0,51.604,-2.0,2.0,2.0,-1.0,12.152,-29.916,4.0,3.0,0.0,-4.606,-70.227,-1.0,0.0,-68.512,0.0,86.079,75.548,-55.789,-36.068,-59.076,-29.762,4.0,1.0,-14.626,-87.119,-3.0,-2.0,-1.0,-3.0,19.792,82.692,0.0,-58.674,50.462,3.0,-73.246],"l3_i64":["9007199254740993","9007199254740993","9007199254740993","-4611686018427387904","-1","9007199254740995","-4611686018427387904","-4611686018427387904","-4611686018427387904","5","-4611686018427387904","9007199254740992","9007199254740994","9007199254740992","9007199254740993","-4611686018427387904","9007199254740992","-4611686018427387904","-4611686018427387904","-4611686018427387904","9007199254740994","-5","-4611686018427387904","-1","9007199254740993","9007199254740994","9007199254740992","9007199254740995","9007199254740995","-4611686018427387904","-4611686018427387904","3","-3","9007199254740992","-2","-4","-1","4","-3","-3","3","-4611686018427387904","2","-4611686018427387904","2","9007199254740995","-4611686018427387904","-4611686018427387904","-1","-4611686018427387904","9007199254740993","5","-4611686018427387904","-4611686018427387904","-4611686018427387904","3","-4611686018427387904","-4611686018427387904","9007199254740995","9007199254740994"],"l4_utf8":["b","a","ab","é","b","b","ab","zz","😀","é","","é","","B","ab","a","é","c","é","B","aa","","c","zz","","","a","a","","B","B","😀","c","aa","b","aa","é","b","B","B","","zz","é","😀","é","zz","B","é","b","aa","b","zz","B","B","","B","B","b","B","aa"]}},"right":{"fields":[{"name":"x0","type":"i32"},{"name":"r0_f64","type":"f64"}],"columns":{"r0_f64":[-1.0,-4.0,0.0],"x0":[-275,1,1]}}},"ops":[{"op":"withColumns","exprs":{"x0":{"fn":"sub","args":[{"col":"l1_i32"},{"fn":"sub","args":[{"col":"l1_i32"},{"lit":2}]}]}}},{"op":"filter","predicate":{"fn":"lt","args":[{"col":"x0"},{"fn":"floorDiv","args":[{"lit":0.5},{"lit":2}]}]}},{"op":"join","right":"right","on":["x0"],"how":"semi"}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_i64","type":"i64"},{"name":"l4_utf8","type":"utf8"},{"name":"x0","type":"i32"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_f64":[],"l3_i64":[],"l4_utf8":[],"x0":[]}}},{"id":"random/0120","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i64","type":"i64"},{"name":"l3_f64","type":"f64"}],"columns":{"l0_utf8":["c"],"l1_i32":[-2],"l2_i64":["-4611686018427387904"],"l3_f64":[-2.0]}}},"ops":[{"op":"filter","predicate":{"fn":"strStartsWith","args":[{"col":"l0_utf8"}],"prefix":"a"}},{"op":"filter","predicate":{"fn":"lte","args":[{"lit":2},{"fn":"mul","args":[{"col":"l3_f64"},{"col":"l3_f64"}]}]}},{"op":"sort","keys":[{"expr":{"col":"l1_i32"},"descending":true},{"expr":{"col":"l0_utf8"},"descending":true}],"nullsLast":true}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i64","type":"i64"},{"name":"l3_f64","type":"f64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_i64":[],"l3_f64":[]}}},{"id":"random/0121","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_f64","type":"f64"},{"name":"l5_i32","type":"i32"}],"columns":{"l0_utf8":["aa","","","b","aa"],"l1_i32":[3,2,0,2147483647,2],"l2_utf8":["","é","b","b","aa"],"l3_utf8":["a","b","","c","b"],"l4_f64":[11.189,-32.572,-97.26,-4.0,-16.338],"l5_i32":[-4,-1,-5,-5,-2]}}},"ops":[{"op":"filter","predicate":{"fn":"and","args":[{"fn":"isIn","args":[{"col":"l3_utf8"}],"values":["b","a","zz"]},{"fn":"gte","args":[{"col":"l0_utf8"},{"lit":"zz"}]}]}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_f64","type":"f64"},{"name":"l5_i32","type":"i32"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_utf8":[],"l3_utf8":[],"l4_f64":[],"l5_i32":[]}}},{"id":"random/0122","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_i32","type":"i32"},{"name":"l4_i64","type":"i64"},{"name":"l5_f64","type":"f64"}],"columns":{"l0_utf8":["zz","c","zz","é","ab"],"l1_i32":[4,-1,3,-3,5],"l2_utf8":["ab","a","zz","","b"],"l3_i32":[-2,-5,0,-4,2],"l4_i64":["-4611686018427387904","1","3","9007199254740995","-5"],"l5_f64":[34.832,1.0,-2.0,0.0,-67.208]}}},"ops":[{"op":"head","n":1},{"op":"withColumns","exprs":{"x1":{"fn":"mod","args":[{"fn":"add","args":[{"col":"l3_i32"},{"col":"l1_i32"}]},{"col":"l5_f64"}]}}},{"op":"withColumns","exprs":{"x2":{"fn":"neg","args":[{"fn":"when","args":[{"fn":"lte","args":[{"col":"l3_i32"},{"col":"x1"}]},{"col":"l3_i32"},{"col":"x1"}]}]}}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_i32","type":"i32"},{"name":"l4_i64","type":"i64"},{"name":"l5_f64","type":"f64"},{"name":"x1","type":"f64"},{"name":"x2","type":"f64"}],"columns":{"l0_utf8":["zz"],"l1_i32":[4],"l2_utf8":["ab"],"l3_i32":[-2],"l4_i64":["-4611686018427387904"],"l5_f64":[34.832],"x1":[2.0],"x2":[2.0]}}},{"id":"random/0123","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_i32","type":"i32"},{"name":"l4_f64","type":"f64"}],"columns":{"l0_utf8":["aa","😀","","B","😀","","B","c","c","b","aa","b","😀","B","c","B","😀","ab","c","B","b","c","😀","ab","😀","ab","é","aa","zz","","😀","zz","zz","c","a","é","B","B","B","b","b","b","zz","B","","ab","B","😀","é","b","a","😀","zz","😀","ab","c","😀","é","aa","é"],"l1_i32":[5,0,0,-1,1,-2147483648,-1,2,2147483647,-2,1,0,-5,-3,-1,-3,4,-5,5,-3,-341,0,-1,-4,-2,-5,1,-2,4,-4,5,-1,2,1,-4,-1,-4,3,4,-4,0,-4,-2,5,-2,-5,3,3,1,3,4,2,4,-5,-1,0,-3,0,2,4],"l2_utf8":["é","B","ab","ab","é","","zz","a","ab","a","B","c","a","","aa","zz","","é","😀","é","😀","c","a","ab","B","😀","","zz","ab","ab","b","a","c","aa","a","a","c","é","😀","c","aa","aa","c","😀","aa","aa","b","b","a","B","B","B","a","aa","😀","aa","b","","aa","B"],"l3_i32":[-1,-2,-5,-1,-4,-1,5,-5,5,2147483647,1,2,2147483647,-1,1,0,-1,759,4,-2147483648,3,-2147483648,-2,4,-2,-5,4,-2,2,5,5,1,-2,-3,-4,0,-2,4,5,-3,-3,-5,0,1,-3,615,1,-1,-4,1,4,0,-2,-4,2147483647,1,0,5,2147483647,-4],"l4_f64":[-91.18,2.0,18.149,-23.311,-26.807,2.0,"-0",1.0,-45.434,-4.0,55.245,-1.0,-80.654,-48.111,-58.468,4.0,10.762,1.0,86.399,0.0,4.0,53.839,-1.0,-2.0,-2.0,0.0,-83.807,79.569,1.0,"NaN",-3.0,2.0,2.0,8.026,4.0,71.591,41.785,-68.405,45.166,-2.0,3.0,-2.0,-3.0,"NaN",4.0,67.932,-48.252,0.0,0.0,0.0,-37.406,0.0,95.931,15.502,0.0,9.56,4.0,-2.0,-49.612,-1.0]}},"right":{"fields":[{"name":"l1_i32","type":"i32"},{"name":"r0_bool","type":"bool"}],"columns":{"r0_bool":[],"l1_i32":[]}}},"ops":[{"op":"join","right":"right","on":["l1_i32"],"how":"full"},{"op":"groupBy","keys":["l1_i32"],"aggs":{"n1":{"agg":"len"},"a1_0":{"agg":"max","arg":{"col":"l2_utf8"}},"a1_1":{"agg":"first","arg":{"col":"l3_i32"}}}},{"op":"filter","predicate":{"fn":"lte","args":[{"col":"a1_1"},{"fn":"mod","args":[{"col":"a1_1"},{"col":"a1_1"}]}]}}],"expected":{"fields":[{"name":"l1_i32","type":"i32"},{"name":"n1","type":"u32"},{"name":"a1_0","type":"utf8"},{"name":"a1_1","type":"i32"}],"columns":{"l1_i32":[5,0,-1,1,-2147483648,2,-3,4],"n1":[4,7,7,5,1,4,4,6],"a1_0":["😀","c","😀","é","","c","é","😀"],"a1_1":[-1,-2,-1,-4,-1,-5,-1,-1]}}},{"id":"random/0124","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_i64","type":"i64"},{"name":"l4_i32","type":"i32"},{"name":"l5_i64","type":"i64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_i32":[],"l3_i64":[],"l4_i32":[],"l5_i64":[]}},"right":{"fields":[{"name":"l3_i64","type":"i64"},{"name":"r0_bool","type":"bool"}],"columns":{"r0_bool":[null,true,true],"l3_i64":["9007199254740992","-4611686018427387904","-4611686018427387904"]}}},"ops":[{"op":"groupBy","keys":["l3_i64"],"aggs":{"n0":{"agg":"len"},"a0_0":{"agg":"max","arg":{"col":"l5_i64"}}}},{"op":"withColumns","exprs":{"x1":{"fn":"mul","args":[{"lit":-1},{"fn":"div","args":[{"lit":-1},{"lit":2.5}]}]}}},{"op":"join","right":"right","on":["l3_i64"],"how":"semi"}],"expected":{"fields":[{"name":"l3_i64","type":"i64"},{"name":"n0","type":"u32"},{"name":"a0_0","type":"i64"},{"name":"x1","type":"f64"}],"columns":{"l3_i64":[],"n0":[],"a0_0":[],"x1":[]}}},{"id":"random/0125","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_utf8","type":"utf8"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_utf8":[],"l3_utf8":[]}}},"ops":[{"op":"sort","keys":[{"expr":{"col":"l3_utf8"},"descending":false}],"nullsLast":false},{"op":"withColumns","exprs":{"x1":{"col":"l1_i32"}}},{"op":"withColumns","exprs":{"x2":{"fn":"add","args":[{"col":"l1_i32"},{"col":"l1_i32"}]}}},{"op":"withColumns","exprs":{"x3":{"fn":"add","args":[{"fn":"mul","args":[{"col":"l1_i32"},{"lit":2}]},{"col":"x2"}]}}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_utf8","type":"utf8"},{"name":"x1","type":"i32"},{"name":"x2","type":"i32"},{"name":"x3","type":"i32"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_utf8":[],"l3_utf8":[],"x1":[],"x2":[],"x3":[]}}},{"id":"random/0126","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_i64","type":"i64"},{"name":"l5_bool","type":"bool"}],"columns":{"l0_utf8":[null,null,"aa",null,"é","",null,null,"c",null,null,"","ab",null,null,"",null,"","b","zz",null,"zz",null,"😀","a","b","é",null,"",null,"c","aa","ab","😀","aa",null,null,"é","😀","a",null,null,null,"B","ab",null,null,"b","zz","B","B",null,"zz","aa","aa","",null,"B","B",null],"l1_i32":[-2,null,1,0,null,1,5,0,null,-1,-1,-1,0,null,0,1,548,2,2,-4,null,null,5,null,4,1,0,null,-1,-5,1,2147483647,-2147483648,null,null,null,null,1,3,null,2,-4,0,null,-3,null,null,-3,4,null,0,4,-3,null,null,1,-5,4,null,0],"l2_utf8":[null,"a","",null,"","é","zz","ab","a","B","B","","ab","😀","😀","c","b",null,"ab","b","B","zz","zz",null,"é",null,"aa","aa","😀",null,null,"ab",null,"😀","b",null,"zz","c",null,"","a","B","c",null,null,"",null,"","b",null,"B",null,null,"😀",null,"","é","😀","ab","a"],"l3_utf8":[null,null,"aa",null,"ab","aa","c","b",null,"a","b","B","B","aa","😀","aa",null,"c",null,"c","é","é","a",null,"zz","B","B","zz",null,"B","b","c","a",null,"c","c","c","",null,null,"ab","é","",null,"B",null,null,"","b","b","aa",null,"é","zz",null,"aa","zz",null,"aa",null],"l4_i64":["9007199254740993","2","-5","2","9007199254740994","9007199254740993",null,"-3",null,"9007199254740995","1",null,"-1",null,"-4611686018427387904",null,"0","9007199254740995",null,"-4611686018427387904","-4611686018427387904","9007199254740994",null,"9007199254740992","3","-4611686018427387904","-1",null,"5",null,"9007199254740995","9007199254740994",null,null,"-4611686018427387904",null,null,"3","-4611686018427387904","-4611686018427387904","-4611686018427387904","1","9007199254740995","-4611686018427387904",null,"9007199254740994","-4611686018427387904","-5",null,null,"9007199254740995","9007199254740992","-4611686018427387904",null,"-4611686018427387904",null,"9007199254740992","-2","-2","-4611686018427387904"],"l5_bool":[false,true,null,null,null,true,false,false,false,true,false,false,false,false,true,false,null,true,null,true,false,false,null,false,false,true,null,true,null,null,null,true,false,null,true,false,false,null,true,true,null,true,true,true,true,true,true,false,null,null,true,null,null,null,null,false,true,null,false,true]}},"right":{"fields":[{"name":"l1_i32","type":"i32"},{"name":"r0_utf8","type":"utf8"}],"columns":{"r0_utf8":[null,"ab","zz"],"l1_i32":[2,0,0]}}},"ops":[{"op":"withColumns","exprs":{"x0":{"col":"l1_i32"}}},{"op":"join","right":"right","on":["l1_i32"],"how":"inner"},{"op":"groupBy","keys":["l5_bool"],"aggs":{"n2":{"agg":"len"},"a2_0":{"agg":"max","arg":{"col":"l4_i64"}},"a2_1":{"agg":"std","arg":{"col":"l1_i32"}}}}],"expected":{"fields":[{"name":"l5_bool","type":"bool"},{"name":"n2","type":"u32"},{"name":"a2_0","type":"i64"},{"name":"a2_1","type":"f64"}],"columns":{"l5_bool":[null,false,true],"n2":[6,4,9],"a2_0":["2","-1","9007199254740995"],"a2_1":[1.0327955589886446,0.0,0.6666666666666667]}}},{"id":"random/0127","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_bool","type":"bool"},{"name":"l4_i32","type":"i32"},{"name":"l5_i64","type":"i64"},{"name":"l6_bool","type":"bool"}],"columns":{"l0_utf8":["a","a",null,"😀",null,null,"a","c","c","é","","B",null,"aa","c",null,"aa",null,null,"aa","aa","ab","a",null,"c","aa","","😀",null,"c",null,null,null,"ab","","","ab","zz",null,"ab","ab","a",null,"é","ab","zz","ab",null,"","a","c","zz",null,"","",null,null,null,"","aa"],"l1_i32":[null,-3,1,1,-4,null,null,null,0,null,1,5,-1,-4,null,-2,0,null,-3,-2,2,1,-3,null,-4,3,null,-5,-3,-2147483648,0,-2,-3,null,5,-2,null,-5,0,-3,-2,null,2147483647,-3,null,null,null,-2,null,0,-2,null,1,-2,null,1,3,1,null,null],"l2_utf8":[null,null,null,null,"aa","a",null,"c",null,"ab","B","b","b","é","B",null,"ab","aa","é","é",null,null,"aa","b","😀","b","",null,"B","","B",null,"aa",null,"😀",null,null,"aa",null,"😀","a","a","zz","a",null,"B","c",null,"😀",null,null,"b",null,"zz","é","c",null,null,"😀","é"],"l3_bool":[null,null,true,false,true,false,false,null,true,false,true,true,null,true,true,null,null,true,null,false,false,true,true,true,true,null,false,false,false,false,true,null,null,false,false,false,null,true,true,true,false,null,false,null,false,false,false,false,null,false,true,true,false,false,true,null,null,true,null,null],"l4_i32":[4,-5,3,3,null,null,2,null,-3,null,-1,-3,3,-2,null,-2,-3,3,-2,-5,-4,-3,0,null,-5,0,5,-2,-4,-3,-1,1,null,-2,4,-1,0,3,3,null,0,2147483647,-4,-5,-4,null,-2,null,0,2,1,null,4,3,null,-1,-5,0,null,-2],"l5_i64":[null,null,"9007199254740993",null,null,"-4611686018427387904",null,null,"5","9007199254740995",null,"-2","9007199254740992",null,"-3","-1","-4611686018427387904","1",null,"9007199254740994","9007199254740993",null,"-4611686018427387904",null,"9007199254740995","9007199254740993","9007199254740993","-3","9007199254740993","-5",null,"0",null,"9007199254740993",null,null,"-2",null,"0","-2","-4611686018427387904","-4611686018427387904","9007199254740995",null,"5","9007199254740994",null,"9007199254740993","9007199254740993","5",null,"-3","-4611686018427387904","-4611686018427387904","-4611686018427387904","-4","-3","9007199254740993","-4611686018427387904","3"],"l6_bool":[true,true,null,false,true,null,true,true,null,null,true,true,false,true,null,null,true,true,false,null,null,false,false,true,null,null,false,true,null,null,false,null,null,false,null,true,null,null,false,null,null,true,false,false,true,null,false,false,false,null,false,false,false,true,false,true,true,false,true,false]}},"right":{"fields":[{"name":"l1_i32","type":"i32"},{"name":"r0_f64","type":"f64"}],"columns":{"r0_f64":[3.0,4.0,-43.28,-72.491,2.0,29.017,4.0,null,-57.398,-3.0],"l1_i32":[-1,5,null,null,-2,-3,1,-3,-3,5]}}},"ops":[{"op":"join","right":"right","on":["l1_i32"],"how":"left"},{"op":"filter","predicate":{"fn":"eq","args":[{"fn":"div","args":[{"col":"r0_f64"},{"col":"r0_f64"}]},{"col":"l1_i32"}]}},{"op":"withColumns","exprs":{"x2":{"fn":"lt","args":[{"fn":"floorDiv","args":[{"col":"l4_i32"},{"lit":-1}]},{"fn":"div","args":[{"lit":-1},{"col":"r0_f64"}]}]}}},{"op":"withColumns","exprs":{"x3":{"lit":2}}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_bool","type":"bool"},{"name":"l4_i32","type":"i32"},{"name":"l5_i64","type":"i64"},{"name":"l6_bool","type":"bool"},{"name":"r0_f64","type":"f64"},{"name":"x2","type":"bool"},{"name":"x3","type":"i32"}],"columns":{"l0_utf8":[null,"😀","","ab",null,null,null],"l1_i32":[1,1,1,1,1,1,1],"l2_utf8":[null,null,"B",null,null,"c",null],"l3_bool":[true,false,true,true,false,null,true],"l4_i32":[3,3,-1,-3,4,-1,0],"l5_i64":["9007199254740993",null,null,null,"-4611686018427387904","-4","9007199254740993"],"l6_bool":[null,false,true,false,false,true,false],"r0_f64":[4.0,4.0,4.0,4.0,4.0,4.0,4.0],"x2":[true,true,false,false,true,false,false],"x3":[2,2,2,2,2,2,2]}}},{"id":"random/0128","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_utf8","type":"utf8"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_utf8":[],"l3_utf8":[]}}},"ops":[{"op":"sort","keys":[{"expr":{"col":"l3_utf8"},"descending":false},{"expr":{"col":"l2_utf8"},"descending":true}],"nullsLast":false},{"op":"unique","columns":["l2_utf8"]},{"op":"head","n":7},{"op":"sort","keys":[{"expr":{"col":"l1_i32"},"descending":false},{"expr":{"col":"l2_utf8"},"descending":false}],"nullsLast":true}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_utf8","type":"utf8"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_utf8":[],"l3_utf8":[]}}},{"id":"random/0129","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_f64","type":"f64"},{"name":"l5_f64","type":"f64"},{"name":"l6_utf8","type":"utf8"}],"columns":{"l0_utf8":["","a","B","b","ab"],"l1_i32":[null,2147483647,-2,1,-3],"l2_f64":["-0",73.898,4.0,-26.96,-19.2],"l3_utf8":["ab","ab","a","c","ab"],"l4_f64":[-91.115,3.0,-47.084,-77.853,4.0],"l5_f64":[2.0,-4.0,95.601,-2.0,74.686],"l6_utf8":["b",null,"😀","a","ab"]}}},"ops":[{"op":"filter","predicate":{"fn":"isNotNull","args":[{"col":"l4_f64"}]}},{"op":"withColumns","exprs":{"x1":{"fn":"isNotNull","args":[{"col":"l5_f64"}]}}},{"op":"withColumns","exprs":{"x2":{"fn":"floorDiv","args":[{"fn":"div","args":[{"col":"l4_f64"},{"col":"l2_f64"}]},{"fn":"mul","args":[{"col":"l1_i32"},{"col":"l5_f64"}]}]}}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_f64","type":"f64"},{"name":"l5_f64","type":"f64"},{"name":"l6_utf8","type":"utf8"},{"name":"x1","type":"bool"},{"name":"x2","type":"f64"}],"columns":{"l0_utf8":["","a","B","b","ab"],"l1_i32":[null,2147483647,-2,1,-3],"l2_f64":["-0",73.898,4.0,-26.96,-19.2],"l3_utf8":["ab","ab","a","c","ab"],"l4_f64":[-91.115,3.0,-47.084,-77.853,4.0],"l5_f64":[2.0,-4.0,95.601,-2.0,74.686],"l6_utf8":["b",null,"😀","a","ab"],"x1":[true,true,true,true,true],"x2":[null,-1.0,0.0,-2.0,0.0]}}},{"id":"random/0130","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_i64","type":"i64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_f64":[],"l3_utf8":[],"l4_utf8":[],"l5_i64":[]}}},"ops":[{"op":"withColumns","exprs":{"x0":{"fn":"floorDiv","args":[{"fn":"neg","args":[{"col":"l2_f64"}]},{"fn":"when","args":[{"fn":"lte","args":[{"col":"l1_i32"},{"lit":0.5}]},{"col":"l1_i32"},{"col":"l2_f64"}]}]}}},{"op":"unique","columns":["l4_utf8"]}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_i64","type":"i64"},{"name":"x0","type":"f64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_f64":[],"l3_utf8":[],"l4_utf8":[],"l5_i64":[],"x0":[]}}},{"id":"random/0131","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_f64","type":"f64"},{"name":"l4_f64","type":"f64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_f64":[],"l3_f64":[],"l4_f64":[]}}},"ops":[{"op":"unique"},{"op":"head","n":5}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_f64","type":"f64"},{"name":"l4_f64","type":"f64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_f64":[],"l3_f64":[],"l4_f64":[]}}},{"id":"random/0132","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_i64","type":"i64"},{"name":"l4_i32","type":"i32"}],"columns":{"l0_utf8":["b","","","a","b"],"l1_i32":[null,4,0,-4,-1],"l2_i32":[-3,4,-4,1,5],"l3_i64":["-4611686018427387904","9007199254740993","-4","9007199254740993","9007199254740994"],"l4_i32":[2147483647,1,-1,1,null]}}},"ops":[{"op":"filter","predicate":{"fn":"lt","args":[{"lit":0.5},{"fn":"abs","args":[{"col":"l4_i32"}]}]}},{"op":"withColumns","exprs":{"x1":{"fn":"isIn","args":[{"col":"l0_utf8"}],"values":["B","a","é"]}}},{"op":"filter","predicate":{"fn":"eq","args":[{"lit":-1},{"col":"l2_i32"}]}},{"op":"head","n":7}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_i64","type":"i64"},{"name":"l4_i32","type":"i32"},{"name":"x1","type":"bool"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_i32":[],"l3_i64":[],"l4_i32":[],"x1":[]}}},{"id":"random/0133","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_i32","type":"i32"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_utf8":[],"l3_i32":[]}}},"ops":[{"op":"sort","keys":[{"expr":{"col":"l2_utf8"},"descending":true},{"expr":{"col":"l0_utf8"},"descending":true}],"nullsLast":true},{"op":"withColumns","exprs":{"x1":{"col":"l1_i32"}}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_i32","type":"i32"},{"name":"x1","type":"i32"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_utf8":[],"l3_i32":[],"x1":[]}}},{"id":"random/0134","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_bool","type":"bool"}],"columns":{"l0_utf8":["c","c","é","a","zz","c","","😀","é","a","zz","B","c","b","B","a","B","zz","c","B"],"l1_i32":[3,-557,3,0,-2,1,-1,3,0,-3,-5,1,-4,-3,3,-1,4,2147483647,-2,4],"l2_utf8":["a","zz","b","aa","c","c","ab","ab","ab","b","b","zz","zz","a","b","B","","é","zz","a"],"l3_bool":[true,false,false,false,true,true,false,false,false,false,true,false,true,false,true,false,false,false,false,false]}}},"ops":[{"op":"filter","predicate":{"fn":"neq","args":[{"fn":"abs","args":[{"col":"l1_i32"}]},{"col":"l1_i32"}]}},{"op":"filter","predicate":{"fn":"isNull","args":[{"col":"l1_i32"}]}},{"op":"withColumns","exprs":{"x2":{"fn":"and","args":[{"fn":"and","args":[{"fn":"gt","args":[{"col":"l1_i32"},{"col":"l1_i32"}]},{"fn":"isIn","args":[{"col":"l2_utf8"}],"values":["ab","c","😀"]}]},{"fn":"or","args":[{"fn":"isNotNull","args":[{"col":"l3_bool"}]},{"fn":"eq","args":[{"col":"l1_i32"},{"lit":-1}]}]}]}}},{"op":"filter","predicate":{"fn":"lt","args":[{"lit":0.5},{"col":"l1_i32"}]}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_bool","type":"bool"},{"name":"x2","type":"bool"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_utf8":[],"l3_bool":[],"x2":[]}}},{"id":"random/0135","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_i64","type":"i64"}],"columns":{"l0_utf8":["a","a","😀","ab","aa","a","B",null,null,"ab","ab",null,"a","","zz","b","B","é","B","B","b","é","B","ab","b","😀","ab","aa","",null,"a","😀","ab","b",null,"😀","B","é","c","b","a","a","zz","😀","zz","é","é",null,"ab","a",null,"aa",null,"b","zz","zz","b","ab","a","😀"],"l1_i32":[-1,3,3,0,null,null,-3,1,1,-4,119,1,4,-4,2,-4,2,null,2,2,2,3,null,3,-4,null,-1,5,-4,2147483647,0,-1,-3,4,4,-3,-1,1,-5,2,-1,1,3,955,4,-1,-1,-1,5,-4,null,5,-3,null,1,2147483647,-2,5,-4,5],"l2_i32":[-2,-2,0,2,-4,-5,3,2,-4,null,3,5,-4,2,4,1,-3,3,789,-5,3,2147483647,4,-811,null,-5,5,1,-3,-1,4,-5,null,-2,-4,1,1,5,null,-1,-2,-3,5,1,1,0,1,3,3,4,1,5,2,-137,-608,1,-2,4,null,1],"l3_i64":["-4611686018427387904","2","-4611686018427387904","9007199254740992","-4611686018427387904","0","5","-4611686018427387904","5","-4611686018427387904","9007199254740995","-4611686018427387904","9007199254740992","-4611686018427387904","9007199254740993",null,"-4611686018427387904","-4611686018427387904","-4611686018427387904","-4611686018427387904","9007199254740995",null,"-4611686018427387904",null,"-2","-4611686018427387904","9007199254740995","9007199254740992","5","9007199254740995","-4611686018427387904","-4611686018427387904","-4611686018427387904","9007199254740992","9007199254740995","9007199254740992","0","-4611686018427387904","-4611686018427387904","-2","-4611686018427387904","9007199254740994","5","9007199254740993","-4","-4611686018427387904","-4611686018427387904","-4611686018427387904","-1",null,"2","-4611686018427387904","9007199254740993","-5","9007199254740994","-4611686018427387904","9007199254740992","4","-5","-4611686018427387904"]}}},"ops":[{"op":"withColumns","exprs":{"x0":{"fn":"fillNull","args":[{"fn":"sub","args":[{"col":"l2_i32"},{"col":"l1_i32"}]},{"lit":0}]}}},{"op":"filter","predicate":{"fn":"neq","args":[{"fn":"div","args":[{"col":"l2_i32"},{"col":"l2_i32"}]},{"fn":"mul","args":[{"col":"l1_i32"},{"col":"x0"}]}]}},{"op":"sort","keys":[{"expr":{"col":"l0_utf8"},"descending":false}],"nullsLast":false}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_i64","type":"i64"},{"name":"x0","type":"i32"}],"columns":{"l0_utf8":[null,null,null,null,null,null,"","","B","B","B","B","B","a","a","a","a","a","aa","aa","ab","ab","ab","ab","ab","ab","b","b","b","b","b","zz","zz","zz","zz","zz","é","é","é","é","😀","😀","😀","😀","😀"],"l1_i32":[1,1,2147483647,4,-1,-3,-4,-4,-3,2,2,2,-1,3,4,0,1,-4,5,5,0,119,3,-1,5,5,-4,2,4,2,-2,2,3,4,1,2147483647,3,1,-1,-1,3,-1,-3,955,5],"l2_i32":[-4,5,-1,-4,3,2,2,-3,3,-3,789,-5,1,-2,-4,4,-3,4,1,5,2,3,-811,5,3,4,1,3,-2,-1,-2,4,5,1,-608,1,2147483647,5,0,1,0,-5,1,1,1],"l3_i64":["5","-4611686018427387904","9007199254740995","9007199254740995","-4611686018427387904","9007199254740993","-4611686018427387904","5","5","-4611686018427387904","-4611686018427387904","-4611686018427387904","0","2","9007199254740992","-4611686018427387904","9007199254740994",null,"9007199254740992","-4611686018427387904","9007199254740992","9007199254740995",null,"9007199254740995","-1","4",null,"9007199254740995","9007199254740992","-2","9007199254740992","9007199254740993","5","-4","9007199254740994","-4611686018427387904",null,"-4611686018427387904","-4611686018427387904","-4611686018427387904","-4611686018427387904","-4611686018427387904","9007199254740992","9007199254740993","-4611686018427387904"],"x0":[-5,4,-2147483648,-8,4,5,6,1,6,-5,787,-7,2,-5,-8,4,-4,8,-4,0,2,-116,-814,6,-2,-1,5,1,-6,-3,0,2,2,-3,-609,-2147483646,2147483644,4,1,2,-3,-4,4,-954,-4]}}},{"id":"random/0136","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_i32","type":"i32"},{"name":"l6_i64","type":"i64"}],"columns":{"l0_utf8":["a","😀","","ab","a"],"l1_i32":[-5,1,4,4,-1],"l2_i32":[-2,2,-1,3,2],"l3_utf8":["B","c","B","c","é"],"l4_utf8":["B","a","ab","b","aa"],"l5_i32":[1,3,0,-3,-5],"l6_i64":["-4611686018427387904","3","4","-4","9007199254740994"]}}},"ops":[{"op":"filter","predicate":{"fn":"and","args":[{"fn":"gt","args":[{"lit":0.5},{"lit":2.5}]},{"fn":"gt","args":[{"col":"l2_i32"},{"col":"l5_i32"}]}]}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_i32","type":"i32"},{"name":"l6_i64","type":"i64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_i32":[],"l3_utf8":[],"l4_utf8":[],"l5_i32":[],"l6_i64":[]}}},{"id":"random/0137","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_i32","type":"i32"},{"name":"l4_i32","type":"i32"}],"columns":{"l0_utf8":["é",null,"aa","a",null],"l1_i32":[4,1,-1,1,-1],"l2_f64":[-53.175,3.0,-2.0,3.0,null],"l3_i32":[-5,3,-3,4,-5],"l4_i32":[1,1,-3,null,-1]}}},"ops":[{"op":"head","n":3},{"op":"filter","predicate":{"fn":"eq","args":[{"fn":"add","args":[{"col":"l1_i32"},{"col":"l2_f64"}]},{"fn":"mod","args":[{"lit":0.5},{"col":"l1_i32"}]}]}},{"op":"withColumns","exprs":{"x2":{"fn":"abs","args":[{"col":"l4_i32"}]}}},{"op":"filter","predicate":{"fn":"eq","args":[{"fn":"when","args":[{"fn":"isNotNull","args":[{"col":"l0_utf8"}]},{"lit":2.5},{"col":"l3_i32"}]},{"col":"l4_i32"}]}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_i32","type":"i32"},{"name":"l4_i32","type":"i32"},{"name":"x2","type":"i32"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_f64":[],"l3_i32":[],"l4_i32":[],"x2":[]}}},{"id":"random/0138","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_utf8","type":"utf8"}],"columns":{"l0_utf8":["b","é","a","","😀","a","","zz","é","aa","b","ab","zz","aa","aa","aa","zz","zz","B","b"],"l1_i32":[5,1,2,-2,-3,-4,3,5,-4,-2,-2147483648,0,-1,0,5,-4,2147483647,0,4,0],"l2_utf8":["c","","c","a","aa","B","😀","B","😀","aa","é","zz","😀","a","é","ab","aa","😀","B",""],"l3_utf8":["ab","c","c","c","b","é","ab","","","B","ab","ab","B","a","ab","zz","a","😀","a","ab"],"l4_utf8":["a","aa","zz","ab","c","B","","aa","b","zz","c","b","","b","c","b","zz","","zz","b"]}}},"ops":[{"op":"withColumns","exprs":{"x0":{"fn":"isIn","args":[{"col":"l4_utf8"}],"values":["ab","B","😀"]}}},{"op":"groupBy","keys":["l2_utf8"],"aggs":{"n1":{"agg":"len"},"a1_0":{"agg":"max","arg":{"col":"l0_utf8"}}}},{"op":"withColumns","exprs":{"x2":{"fn":"abs","args":[{"lit":-1}]}}},{"op":"sort","keys":[{"expr":{"col":"n1"},"descending":false}],"nullsLast":true}],"expected":{"fields":[{"name":"l2_utf8","type":"utf8"},{"name":"n1","type":"u32"},{"name":"a1_0","type":"utf8"},{"name":"x2","type":"i32"}],"columns":{"l2_utf8":["zz","ab","c","","a","é","aa","B","😀"],"n1":[1,1,2,2,2,2,3,3,4],"a1_0":["ab","aa","b","é","aa","b","😀","zz","é"],"x2":[1,1,1,1,1,1,1,1,1]}}},{"id":"random/0139","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_i32","type":"i32"},{"name":"l4_bool","type":"bool"},{"name":"l5_bool","type":"bool"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_f64":[],"l3_i32":[],"l4_bool":[],"l5_bool":[]}}},"ops":[{"op":"groupBy","keys":["l4_bool"],"aggs":{"n0":{"agg":"len"},"a0_0":{"agg":"max","arg":{"col":"l2_f64"}}}}],"expected":{"fields":[{"name":"l4_bool","type":"bool"},{"name":"n0","type":"u32"},{"name":"a0_0","type":"f64"}],"columns":{"l4_bool":[],"n0":[],"a0_0":[]}}},{"id":"random/0140","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i64","type":"i64"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_f64","type":"f64"}],"columns":{"l0_utf8":["😀"],"l1_i32":[4],"l2_i64":["9007199254740993"],"l3_utf8":["b"],"l4_f64":[-42.28]}}},"ops":[{"op":"sort","keys":[{"expr":{"col":"l3_utf8"},"descending":false},{"expr":{"col":"l4_f64"},"descending":false}],"nullsLast":false},{"op":"unique"}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i64","type":"i64"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_f64","type":"f64"}],"columns":{"l0_utf8":["😀"],"l1_i32":[4],"l2_i64":["9007199254740993"],"l3_utf8":["b"],"l4_f64":[-42.28]}}},{"id":"random/0141","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_bool","type":"bool"},{"name":"l4_utf8","type":"utf8"}],"columns":{"l0_utf8":[null,null,null,"é","c","zz","aa","a","aa","aa","é","B","B","c","b","😀","😀","","","aa",null,"é","😀","","é","B","aa","ab",null,"c","aa",null,"B",null,"é",null,"","a","zz",null,"zz",null,null,null,"B","é","zz",null,"ab","ab",null,null,"zz","B","c","c",null,null,null,"😀"],"l1_i32":[-1,5,4,null,4,3,0,1,null,-3,0,null,-4,5,0,null,5,4,992,null,-2,1,-1,2,-2,-4,2,124,null,-5,null,-3,-3,null,-5,-4,null,-3,4,-2,-1,4,null,2,null,null,0,5,0,null,-2147483648,null,-1,2,-1,null,3,-5,-5,1],"l2_f64":[null,2.0,95.44,80.659,55.69,-3.0,null,null,-3.767,-4.0,-3.0,-4.0,-1.0,3.0,null,-4.0,null,4.0,1.0,null,-59.737,-46.968,-50.905,null,-81.427,null,null,-2.0,3.0,-14.915,null,-82.116,0.593,97.652,51.958,1.0,null,67.806,-9.207,3.0,-81.453,"NaN",null,null,null,0.0,3.0,null,-4.0,23.348,null,null,null,-53.527,null,0.0,-94.58,2.0,-83.113,-55.022],"l3_bool":[true,true,null,true,true,null,true,true,false,false,false,false,false,true,false,true,true,null,null,null,true,false,null,false,true,false,true,false,true,false,true,false,true,false,null,true,null,true,true,false,false,false,true,true,null,false,null,false,null,false,true,true,null,false,null,null,false,true,false,null],"l4_utf8":[null,"b","","😀",null,"😀",null,"",null,"aa","B","a",null,"a",null,"😀","zz","zz","aa","😀",null,"","c","aa","B","a","a",null,"ab","",null,null,"ab","aa","b",null,"ab",null,"a","a","c","ab","aa",null,"ab","c",null,"a",null,"B",null,"b","b",null,"aa","b","é","zz",null,null]}}},"ops":[{"op":"withColumns","exprs":{"x0":{"col":"l1_i32"}}},{"op":"filter","predicate":{"fn":"neq","args":[{"fn":"add","args":[{"col":"l1_i32"},{"col":"l1_i32"}]},{"fn":"floorDiv","args":[{"col":"l2_f64"},{"col":"x0"}]}]}},{"op":"filter","predicate":{"fn":"strStartsWith","args":[{"col":"l0_utf8"}],"prefix":"é"}},{"op":"sort","keys":[{"expr":{"col":"l4_utf8"},"descending":false},{"expr":{"col":"l0_utf8"},"descending":false}],"nullsLast":true}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_bool","type":"bool"},{"name":"l4_utf8","type":"utf8"},{"name":"x0","type":"i32"}],"columns":{"l0_utf8":["é","é","é","é"],"l1_i32":[1,0,-2,-5],"l2_f64":[-46.968,-3.0,-81.427,51.958],"l3_bool":[false,false,true,null],"l4_utf8":["","B","B","b"],"x0":[1,0,-2,-5]}}},{"id":"random/0142","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_bool","type":"bool"}],"columns":{"l0_utf8":["aa",null,"aa","B",null,"ab","B",null,"B","a","é","B",null,"a","B","a","é","zz","aa","zz"],"l1_i32":[2,5,1,3,-2,null,0,-1,2,5,2,0,2,4,-77,-855,1,-238,-1,2147483647],"l2_utf8":["c","b","B","zz","😀","zz","é",null,"zz","b","aa","","","aa","ab","c","ab","😀","😀","zz"],"l3_utf8":["B","B","aa","c",null,"zz","ab","😀","c","é","aa","","a",null,"aa","B","é","ab",null,"a"],"l4_bool":[true,true,false,true,false,true,true,null,false,true,false,false,true,true,false,false,true,false,null,true]}}},"ops":[{"op":"filter","predicate":{"fn":"and","args":[{"fn":"isNull","args":[{"col":"l4_bool"}]},{"fn":"gt","args":[{"lit":-1},{"col":"l1_i32"}]}]}},{"op":"unique"}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_bool","type":"bool"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_utf8":[],"l3_utf8":[],"l4_bool":[]}}},{"id":"random/0143","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i64","type":"i64"},{"name":"l3_i32","type":"i32"}],"columns":{"l0_utf8":["😀","aa",null,null,null,"B",null,"a","","b","B","b","",null,"b","😀","é",null,"aa","c",null,"ab","a","aa","c",null,"aa","aa","",null,null,"B","c","b","b","ab","","zz","😀",null,"B",null,null,null,"c",null,"aa",null,"c",null,"ab","c",null,"ab","zz","é","","","ab",""],"l1_i32":[-4,null,-4,null,3,5,3,-3,-1,null,4,null,-1,5,5,1,2,-5,null,1,1,-4,2,2,0,4,-2,null,4,null,-2,3,4,5,-1,null,null,5,-4,1,-2,-1,2147483647,-3,3,2147483647,3,2,null,null,null,-1,-4,null,3,1,null,-4,3,1],"l2_i64":["9007199254740995","-4611686018427387904","-4611686018427387904","-4611686018427387904",null,null,"9007199254740992",null,null,null,"9007199254740993","-4611686018427387904","-5",null,"4","-4611686018427387904","-4611686018427387904","-4611686018427387904",null,"-4611686018427387904","-4","-4611686018427387904",null,"5","-4611686018427387904","-4611686018427387904","-4611686018427387904",null,"-4611686018427387904","-4611686018427387904","-4611686018427387904",null,"9007199254740992","-4611686018427387904","-5","-3",null,null,"-4611686018427387904","-4611686018427387904","-4611686018427387904","9007199254740993","-4611686018427387904","2",null,"-4611686018427387904",null,"9007199254740994","-4","-4611686018427387904",null,"5","9007199254740992",null,"-4611686018427387904","0","-4611686018427387904","9007199254740992","9007199254740994","9007199254740994"],"l3_i32":[-2,-5,-3,-2,2,4,3,-5,null,0,-5,null,4,3,-5,null,-1,0,-4,1,null,-3,1,null,0,-3,-2,null,-2147483648,-4,4,1,null,-3,null,-1,-2,4,2,null,-2,1,1,null,0,null,-5,null,3,null,2,null,-2,-2,-1,5,1,0,null,1]}},"right":{"fields":[{"name":"l3_i32","type":"i32"},{"name":"r0_f64","type":"f64"}],"columns":{"r0_f64":[1.0,1.0,-2.0,-1.0,-2.0,10.676,96.286,45.979,-40.883,-1.0,1.0,-16.329,-17.196,0.0,-4.0,-4.0,"NaN",3.0,"-0",-3.0,-99.901,13.946,0.0,34.747,-4.0,-77.636,3.0,93.732,3.0,14.443],"l3_i32":[null,-1,-5,null,null,-2,-2,-3,3,1,-5,2,-5,4,-2,-5,1,-4,4,-2,-2,-2,0,4,-2,0,-3,1,-3,null]}}},"ops":[{"op":"filter","predicate":{"fn":"gt","args":[{"col":"l1_i32"},{"col":"l1_i32"}]}},{"op":"withColumns","exprs":{"x1":{"fn":"fillNull","args":[{"lit":2.5},{"lit":1.5}]}}},{"op":"join","right":"right","on":["l3_i32"],"how":"inner"}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i64","type":"i64"},{"name":"l3_i32","type":"i32"},{"name":"x1","type":"f64"},{"name":"r0_f64","type":"f64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_i64":[],"l3_i32":[],"x1":[],"r0_f64":[]}}},{"id":"random/0144","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_f64","type":"f64"}],"columns":{"l0_utf8":["zz","b","","","aa"],"l1_i32":[1,4,2,null,2],"l2_f64":[null,95.323,97.124,2.0,0.338],"l3_f64":["-0",4.0,-77.463,-1.0,-2.0]}}},"ops":[{"op":"sort","keys":[{"expr":{"col":"l3_f64"},"descending":false},{"expr":{"col":"l0_utf8"},"descending":true}],"nullsLast":true},{"op":"filter","predicate":{"fn":"lt","args":[{"col":"l1_i32"},{"col":"l1_i32"}]}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_f64","type":"f64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_f64":[],"l3_f64":[]}}},{"id":"random/0145","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_i64","type":"i64"},{"name":"l5_i32","type":"i32"},{"name":"l6_utf8","type":"utf8"}],"columns":{"l0_utf8":["zz"],"l1_i32":[-3],"l2_utf8":["B"],"l3_utf8":["B"],"l4_i64":["-4611686018427387904"],"l5_i32":[0],"l6_utf8":["zz"]}},"right":{"fields":[{"name":"l6_utf8","type":"utf8"},{"name":"r0_utf8","type":"utf8"}],"columns":{"r0_utf8":[null,"😀","a","a","ab","b","ab","😀","","😀","😀","é","b","","ab","aa","c",null,"é","c",null,"é","aa","b","aa","é","","B","aa","😀"],"l6_utf8":["zz","zz","zz","zz","zz","zz","zz","zz","zz","zz","zz","zz","zz","zz","zz","zz","zz","zz","zz","zz","zz","zz","zz","zz","zz","zz","zz","zz","zz","zz"]}}},"ops":[{"op":"unique"},{"op":"join","right":"right","on":["l6_utf8"],"how":"anti"},{"op":"groupBy","keys":["l0_utf8"],"aggs":{"n2":{"agg":"len"},"a2_0":{"agg":"first","arg":{"col":"l5_i32"}},"a2_1":{"agg":"count","arg":{"col":"l1_i32"}},"a2_2":{"agg":"count","arg":{"col":"l2_utf8"}}}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"n2","type":"u32"},{"name":"a2_0","type":"i32"},{"name":"a2_1","type":"u32"},{"name":"a2_2","type":"u32"}],"columns":{"l0_utf8":[],"n2":[],"a2_0":[],"a2_1":[],"a2_2":[]}}},{"id":"random/0146","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_bool","type":"bool"},{"name":"l4_utf8","type":"utf8"}],"columns":{"l0_utf8":["aa"],"l1_i32":[0],"l2_f64":[-68.128],"l3_bool":[false],"l4_utf8":["b"]}}},"ops":[{"op":"head","n":1}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_bool","type":"bool"},{"name":"l4_utf8","type":"utf8"}],"columns":{"l0_utf8":["aa"],"l1_i32":[0],"l2_f64":[-68.128],"l3_bool":[false],"l4_utf8":["b"]}}},{"id":"random/0147","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_bool","type":"bool"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_bool","type":"bool"}],"columns":{"l0_utf8":["aa"],"l1_i32":[-5],"l2_bool":[true],"l3_bool":[true],"l4_utf8":[null],"l5_bool":[null]}}},"ops":[{"op":"withColumns","exprs":{"x0":{"col":"l1_i32"}}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_bool","type":"bool"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_bool","type":"bool"},{"name":"x0","type":"i32"}],"columns":{"l0_utf8":["aa"],"l1_i32":[-5],"l2_bool":[true],"l3_bool":[true],"l4_utf8":[null],"l5_bool":[null],"x0":[-5]}}},{"id":"random/0148","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_f64","type":"f64"},{"name":"l5_f64","type":"f64"},{"name":"l6_utf8","type":"utf8"}],"columns":{"l0_utf8":["ab","c",null,null,null,"c","B","aa","ab","b",null,null,"B","ab",null,null,"zz",null,"b","B"],"l1_i32":[-4,-2,1,null,-4,2,null,-5,2,null,null,null,4,2,-1,-4,-1,-2147483648,null,null],"l2_f64":[86.534,-53.607,-66.98,15.709,0.0,90.824,-3.0,null,-14.19,-80.437,null,null,3.778,null,-47.743,-62.376,-1.0,null,2.0,3.0],"l3_utf8":["B",null,null,null,"c","aa","",null,"","B","zz","",null,null,null,"B",null,"a",null,""],"l4_f64":[null,-2.0,-1.0,"NaN",-13.437,-97.496,57.442,null,36.855,69.561,3.0,null,-4.0,3.0,-4.0,-15.522,5.934,null,91.203,2.0],"l5_f64":[null,97.873,0.0,null,4.0,1.0,-1.0,-82.395,55.378,1.467,-46.733,-93.682,null,71.854,0.0,-1.0,-2.0,null,null,1.0],"l6_utf8":["c","","é","b","😀","B",null,"",null,null,"ab",null,"a",null,"😀","😀","","B","😀",null]}}},"ops":[{"op":"withColumns","exprs":{"x0":{"col":"l1_i32"}}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_f64","type":"f64"},{"name":"l5_f64","type":"f64"},{"name":"l6_utf8","type":"utf8"},{"name":"x0","type":"i32"}],"columns":{"l0_utf8":["ab","c",null,null,null,"c","B","aa","ab","b",null,null,"B","ab",null,null,"zz",null,"b","B"],"l1_i32":[-4,-2,1,null,-4,2,null,-5,2,null,null,null,4,2,-1,-4,-1,-2147483648,null,null],"l2_f64":[86.534,-53.607,-66.98,15.709,0.0,90.824,-3.0,null,-14.19,-80.437,null,null,3.778,null,-47.743,-62.376,-1.0,null,2.0,3.0],"l3_utf8":["B",null,null,null,"c","aa","",null,"","B","zz","",null,null,null,"B",null,"a",null,""],"l4_f64":[null,-2.0,-1.0,"NaN",-13.437,-97.496,57.442,null,36.855,69.561,3.0,null,-4.0,3.0,-4.0,-15.522,5.934,null,91.203,2.0],"l5_f64":[null,97.873,0.0,null,4.0,1.0,-1.0,-82.395,55.378,1.467,-46.733,-93.682,null,71.854,0.0,-1.0,-2.0,null,null,1.0],"l6_utf8":["c","","é","b","😀","B",null,"",null,null,"ab",null,"a",null,"😀","😀","","B","😀",null],"x0":[-4,-2,1,null,-4,2,null,-5,2,null,null,null,4,2,-1,-4,-1,-2147483648,null,null]}}},{"id":"random/0149","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_i64","type":"i64"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_bool","type":"bool"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_bool":[],"l3_i64":[],"l4_utf8":[],"l5_bool":[]}}},"ops":[{"op":"withColumns","exprs":{"x0":{"fn":"gte","args":[{"fn":"neg","args":[{"lit":2}]},{"fn":"add","args":[{"col":"l1_i32"},{"lit":2.5}]}]}}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_i64","type":"i64"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_bool","type":"bool"},{"name":"x0","type":"bool"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_bool":[],"l3_i64":[],"l4_utf8":[],"l5_bool":[],"x0":[]}}},{"id":"random/0150","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_i32","type":"i32"},{"name":"l4_f64","type":"f64"},{"name":"l5_f64","type":"f64"}],"columns":{"l0_utf8":["ab","😀",null,"B","é"],"l1_i32":[-4,3,2,1,3],"l2_f64":[-4.0,4.0,null,29.092,2.0],"l3_i32":[1,null,null,-2,null],"l4_f64":[null,48.813,89.755,-8.91,null],"l5_f64":[87.465,null,-4.0,null,null]}},"right":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"r0_f64","type":"f64"}],"columns":{"r0_f64":[-85.126,60.31,-5.522,-4.0,75.31,4.0,0.0,null,-1.0,null],"l0_utf8":[null,"B","B","B","ab","B","B","😀","😀","é"]}}},"ops":[{"op":"sort","keys":[{"expr":{"col":"l1_i32"},"descending":false}],"nullsLast":false},{"op":"unique","columns":["l2_f64"]},{"op":"filter","predicate":{"fn":"and","args":[{"fn":"isNotNull","args":[{"col":"l2_f64"}]},{"fn":"neq","args":[{"col":"l4_f64"},{"col":"l3_i32"}]}]}},{"op":"join","right":"right","on":["l0_utf8"],"how":"inner"}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_i32","type":"i32"},{"name":"l4_f64","type":"f64"},{"name":"l5_f64","type":"f64"},{"name":"r0_f64","type":"f64"}],"columns":{"l0_utf8":["B","B","B","B","B"],"l1_i32":[1,1,1,1,1],"l2_f64":[29.092,29.092,29.092,29.092,29.092],"l3_i32":[-2,-2,-2,-2,-2],"l4_f64":[-8.91,-8.91,-8.91,-8.91,-8.91],"l5_f64":[null,null,null,null,null],"r0_f64":[60.31,-5.522,-4.0,4.0,0.0]}}},{"id":"random/0151","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_i32","type":"i32"},{"name":"l4_f64","type":"f64"},{"name":"l5_i64","type":"i64"}],"columns":{"l0_utf8":["b","😀","😀","","ab",null,"","a","c",null,"😀","é","c","ab","😀","","a","é","B",null],"l1_i32":[1,1,-1,-3,1,-1,1,-1,-1,0,null,null,-5,1,5,null,-5,1,2147483647,null],"l2_utf8":["a",null,"B","ab","c","","😀","c","zz","B","😀","aa","zz","zz","zz","c",null,"😀","b",null],"l3_i32":[-4,0,-4,-3,-1,-4,0,4,null,-932,1,3,-89,null,-3,1,-2,-4,0,3],"l4_f64":[-2.0,-96.715,64.633,-2.0,34.835,-2.126,null,"NaN",-28.808,4.0,null,3.0,78.393,15.949,-2.0,-78.44,-4.0,2.0,72.641,-4.0],"l5_i64":["-4611686018427387904","9007199254740995","-4611686018427387904",null,"-4611686018427387904","1",null,"-4611686018427387904","4","9007199254740995","9007199254740994","9007199254740993",null,"9007199254740994","-4611686018427387904","-4611686018427387904","9007199254740993","-4611686018427387904","-2",null]}}},"ops":[{"op":"unique","columns":["l3_i32"]},{"op":"sort","keys":[{"expr":{"col":"l4_f64"},"descending":false},{"expr":{"col":"l1_i32"},"descending":false}],"nullsLast":false},{"op":"filter","predicate":{"fn":"gt","args":[{"col":"l5_i64"},{"lit":"0","type":"i64"}]}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_i32","type":"i32"},{"name":"l4_f64","type":"f64"},{"name":"l5_i64","type":"i64"}],"columns":{"l0_utf8":["😀","😀","c","a","é",null],"l1_i32":[null,1,-1,-5,null,0],"l2_utf8":["😀",null,"zz",null,"aa","B"],"l3_i32":[1,0,null,-2,3,-932],"l4_f64":[null,-96.715,-28.808,-4.0,3.0,4.0],"l5_i64":["9007199254740994","9007199254740995","4","9007199254740993","9007199254740993","9007199254740995"]}}},{"id":"random/0152","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_bool","type":"bool"},{"name":"l4_i32","type":"i32"},{"name":"l5_bool","type":"bool"}],"columns":{"l0_utf8":["","","B","a","ab","zz","a","é","b","c","c","","","B","aa","c","é","😀","😀","😀","zz","B","","a","c","B","aa","B","zz","B","é","é","zz","ab","aa","é","aa","zz","c","aa","a","B","a","c","zz","B","ab","zz","b","c","ab","zz","B","c","c","aa","","é","ab","zz"],"l1_i32":[0,-2147483648,2,-4,3,0,5,-3,1,-5,1,-5,-2,-3,2,0,-2,0,-4,-5,131,-4,-3,-2,-2,-2147483648,5,-2,5,3,-3,1,0,2,-2,-651,0,-1,0,2,-2,-3,-3,-2,4,0,-1,-1,0,3,2,5,-272,-4,1,-5,-5,2,1,-3],"l2_f64":[-3.0,-3.0,-1.0,"-Infinity",4.0,-1.0,0.0,2.0,84.359,98.241,"NaN",2.0,-4.0,3.0,-65.684,64.024,-59.022,-66.915,-18.931,-2.0,-3.0,76.356,-25.823,1.0,-4.0,4.0,-4.0,0.0,-65.241,2.0,2.0,-3.0,3.872,2.0,64.983,2.0,2.0,2.0,3.0,4.0,59.986,-44.445,"-Infinity",2.0,15.858,-3.0,-13.691,-61.578,-2.0,-1.0,-58.97,4.0,-92.025,4.0,3.0,3.0,3.0,-4.0,2.0,"NaN"],"l3_bool":[false,false,true,true,false,false,true,true,true,true,true,false,false,false,true,false,true,false,false,false,false,true,false,false,true,false,true,false,true,true,false,true,true,true,false,true,false,false,false,false,false,false,true,false,true,false,false,true,false,true,true,true,true,false,true,true,false,false,false,false],"l4_i32":[2,-1,-3,-5,5,-3,-1,-1,-2,5,-4,0,-2,-2,-4,1,0,-4,-4,-2,5,5,5,-4,2,-3,4,-4,4,-4,0,-4,2,2,4,-1,-2147483648,-2,-1,0,-3,3,-5,-5,-3,2,3,-4,5,0,5,-5,4,-2147483648,2147483647,-4,5,1,-5,5],"l5_bool":[true,true,true,false,false,true,false,true,false,false,true,true,false,false,true,true,false,true,true,false,false,false,false,true,false,true,false,false,false,false,true,false,true,false,false,true,false,false,false,true,false,false,false,false,false,false,false,false,true,false,false,true,true,false,false,true,true,true,false,false]}}},"ops":[{"op":"sort","keys":[{"expr":{"col":"l1_i32"},"descending":false}],"nullsLast":false},{"op":"filter","predicate":{"fn":"or","args":[{"fn":"gt","args":[{"col":"l2_f64"},{"col":"l2_f64"}]},{"fn":"isIn","args":[{"col":"l0_utf8"}],"values":["b","B","ab"]}]}},{"op":"groupBy","keys":["l5_bool"],"aggs":{"n2":{"agg":"len"},"a2_0":{"agg":"first","arg":{"col":"l1_i32"}},"a2_1":{"agg":"count","arg":{"col":"l3_bool"}},"a2_2":{"agg":"count","arg":{"col":"l4_i32"}}}}],"expected":{"fields":[{"name":"l5_bool","type":"bool"},{"name":"n2","type":"u32"},{"name":"a2_0","type":"i32"},{"name":"a2_1","type":"u32"},{"name":"a2_2","type":"u32"}],"columns":{"l5_bool":[true,false],"n2":[4,12],"a2_0":[-2147483648,-4],"a2_1":[4,12],"a2_2":[4,12]}}},{"id":"random/0153","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_i32","type":"i32"},{"name":"l5_i32","type":"i32"}],"columns":{"l0_utf8":["ab","é","b","aa",null,"a","😀","é","B",null,null,null,null,"é","aa","zz","B",null,null,"",null,null,"é","","B","é","zz",null,null,"é","😀","ab","c","zz",null,"c","ab","b",null,"aa",null,"é","😀",null,null,null,null,null,"😀","a","😀","aa","a",null,"é",null,"zz",null,"b",""],"l1_i32":[-4,1,4,-2,0,-2147483648,-1,-2,-4,-3,3,0,4,3,-2,3,5,-5,-2,5,-5,4,-4,5,null,1,3,-4,-4,null,-3,430,1,null,-2,3,5,null,5,3,null,4,null,null,-4,null,null,null,null,null,-4,-3,null,0,null,1,-1,null,-3,-1],"l2_i32":[2,5,5,null,2,null,-4,464,1,3,-4,-62,-4,4,-2,null,1,null,null,null,-5,0,-3,null,4,1,null,null,null,1,2,null,-3,null,0,null,null,5,-1,null,null,-2,null,3,-2,-4,5,null,null,-5,-5,null,3,null,2,null,null,null,null,-2147483648],"l3_utf8":[null,"B","ab",null,"ab","é",null,"ab","aa",null,"B","",null,null,"ab","aa","é","😀","a","é","😀",null,"b","a","😀",null,null,null,null,null,"😀","😀",null,"ab",null,"😀",null,null,"B","ab",null,"b",null,null,"c",null,"B","😀","é","aa","😀",null,"ab","aa","B","aa","😀",null,"","aa"],"l4_i32":[4,3,-4,-672,null,5,null,2147483647,null,-5,4,0,null,null,null,-1,4,null,2147483647,0,null,null,null,4,0,-2,null,null,3,0,2147483647,5,null,2,2,-1,null,-1,-5,2,1,null,5,-3,-2147483648,-4,1,-4,3,null,null,3,-3,4,null,null,null,null,-4,null],"l5_i32":[-4,null,-4,-5,-3,-4,2,2147483647,null,null,0,4,null,null,null,2,null,null,-5,-2,-2,null,null,null,null,-3,180,-4,null,-1,null,0,null,0,4,-2,0,-2,null,-2147483648,-2147483648,null,-2,4,4,-1,-1,1,null,null,2,-5,3,0,489,null,3,4,-5,-1]}}},"ops":[{"op":"filter","predicate":{"fn":"and","args":[{"fn":"or","args":[{"fn":"lte","args":[{"col":"l5_i32"},{"lit":0.5}]},{"fn":"lt","args":[{"lit":0.5},{"col":"l5_i32"}]}]},{"fn":"isNull","args":[{"col":"l5_i32"}]}]}},{"op":"filter","predicate":{"fn":"gte","args":[{"col":"l2_i32"},{"col":"l2_i32"}]}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_i32","type":"i32"},{"name":"l5_i32","type":"i32"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_i32":[],"l3_utf8":[],"l4_i32":[],"l5_i32":[]}}},{"id":"random/0154","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_bool","type":"bool"},{"name":"l4_utf8","type":"utf8"}],"columns":{"l0_utf8":["ab","ab","aa","aa","c","😀","","aa","c","zz","ab","zz","é","zz","😀","","😀","a","b","a"],"l1_i32":[-3,5,5,-4,-3,0,-2,3,-4,-3,-3,2,0,1,-2,4,5,-5,-3,-3],"l2_f64":[-78.172,2.0,-31.358,"-Infinity",54.072,-95.226,-4.0,54.577,-43.75,4.0,-1.0,-4.0,-47.045,2.0,86.183,98.903,-3.0,-86.161,0.0,-3.0],"l3_bool":[false,true,false,true,false,false,false,true,true,true,false,false,false,true,true,true,true,false,true,false],"l4_utf8":["ab","aa","ab","aa","zz","é","aa","","c","é","ab","c","a","é","b","aa","ab","aa","zz","b"]}},"right":{"fields":[{"name":"l4_utf8","type":"utf8"},{"name":"r0_utf8","type":"utf8"}],"columns":{"r0_utf8":[],"l4_utf8":[]}}},"ops":[{"op":"sort","keys":[{"expr":{"col":"l3_bool"},"descending":false},{"expr":{"col":"l1_i32"},"descending":true}],"nullsLast":false},{"op":"join","right":"right","on":["l4_utf8"],"how":"anti"},{"op":"filter","predicate":{"fn":"neq","args":[{"fn":"mul","args":[{"col":"l1_i32"},{"col":"l2_f64"}]},{"fn":"when","args":[{"fn":"isNull","args":[{"col":"l3_bool"}]},{"col":"l1_i32"},{"lit":0.5}]}]}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_bool","type":"bool"},{"name":"l4_utf8","type":"utf8"}],"columns":{"l0_utf8":["aa","zz","😀","é","","ab","c","ab","a","a","ab","😀","","aa","zz","😀","zz","b","aa","c"],"l1_i32":[5,2,0,0,-2,-3,-3,-3,-3,-5,5,5,4,3,1,-2,-3,-3,-4,-4],"l2_f64":[-31.358,-4.0,-95.226,-47.045,-4.0,-78.172,54.072,-1.0,-3.0,-86.161,2.0,-3.0,98.903,54.577,2.0,86.183,4.0,0.0,"-Infinity",-43.75],"l3_bool":[false,false,false,false,false,false,false,false,false,false,true,true,true,true,true,true,true,true,true,true],"l4_utf8":["ab","c","é","a","aa","ab","zz","ab","b","aa","aa","ab","aa","","é","b","é","zz","aa","c"]}}},{"id":"random/0155","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i64","type":"i64"},{"name":"l3_f64","type":"f64"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_utf8","type":"utf8"},{"name":"l6_i64","type":"i64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_i64":[],"l3_f64":[],"l4_utf8":[],"l5_utf8":[],"l6_i64":[]}},"right":{"fields":[{"name":"l6_i64","type":"i64"},{"name":"r0_utf8","type":"utf8"}],"columns":{"r0_utf8":["a","zz","c","zz",null,null,null,null,null,null],"l6_i64":["9007199254740995",null,"9007199254740993","-4611686018427387904",null,"-4611686018427387904","9007199254740995","9007199254740995","9007199254740995","-4611686018427387904"]}}},"ops":[{"op":"join","right":"right","on":["l6_i64"],"how":"semi"},{"op":"filter","predicate":{"fn":"eq","args":[{"fn":"fillNull","args":[{"col":"l1_i32"},{"lit":1.5}]},{"lit":2.5}]}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i64","type":"i64"},{"name":"l3_f64","type":"f64"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_utf8","type":"utf8"},{"name":"l6_i64","type":"i64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_i64":[],"l3_f64":[],"l4_utf8":[],"l5_utf8":[],"l6_i64":[]}}},{"id":"random/0156","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_f64","type":"f64"},{"name":"l4_i32","type":"i32"},{"name":"l5_utf8","type":"utf8"}],"columns":{"l0_utf8":["","b","ab","a","b"],"l1_i32":[3,1,-4,-2,0],"l2_f64":[2.0,-3.0,50.839,-1.0,-2.0],"l3_f64":[3.0,-8.132,2.0,"NaN",-43.937],"l4_i32":[-2,-5,2,2147483647,-2],"l5_utf8":["a","b","B","b",""]}},"right":{"fields":[{"name":"l1_i32","type":"i32"},{"name":"r0_utf8","type":"utf8"}],"columns":{"r0_utf8":[],"l1_i32":[]}}},"ops":[{"op":"join","right":"right","on":["l1_i32"],"how":"left"},{"op":"unique","columns":["r0_utf8"]},{"op":"filter","predicate":{"fn":"isNull","args":[{"col":"l0_utf8"}]}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_f64","type":"f64"},{"name":"l4_i32","type":"i32"},{"name":"l5_utf8","type":"utf8"},{"name":"r0_utf8","type":"utf8"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_f64":[],"l3_f64":[],"l4_i32":[],"l5_utf8":[],"r0_utf8":[]}}},{"id":"random/0157","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_f64","type":"f64"},{"name":"l4_f64","type":"f64"},{"name":"l5_utf8","type":"utf8"},{"name":"l6_i64","type":"i64"}],"columns":{"l0_utf8":["c",null,"B",null,null,null,null,null,"aa","😀","zz","aa",null,"aa","","ab",null,"b","b",null],"l1_i32":[-3,null,1,null,-4,5,2147483647,null,1,null,2,null,null,null,212,-4,3,0,-1,null],"l2_i32":[null,null,5,2,null,3,2,1,-2,1,3,2,-5,null,-4,null,2,0,-2147483648,null],"l3_f64":[88.229,null,3.0,null,null,null,43.13,-48.14,-1.0,-36.69,-41.651,54.971,3.0,-26.911,4.0,-4.0,"-Infinity",0.0,60.721,-27.958],"l4_f64":[1.0,-4.653,-3.0,2.0,80.555,null,null,"NaN",null,-1.0,"NaN",2.0,3.0,null,-50.776,null,-74.384,-4.0,58.135,"NaN"],"l5_utf8":[null,null,"é","a","aa",null,"ab",null,"",null,"c","😀",null,"😀","b","B","😀",null,"zz","é"],"l6_i64":[null,"-5",null,"9007199254740995",null,"9007199254740994","9007199254740992","2","-4611686018427387904","9007199254740995","-4611686018427387904","9007199254740993",null,"-4611686018427387904","9007199254740992","-1","-5","-4611686018427387904","-4611686018427387904","-4611686018427387904"]}}},"ops":[{"op":"sort","keys":[{"expr":{"col":"l1_i32"},"descending":true},{"expr":{"col":"l0_utf8"},"descending":true}],"nullsLast":true},{"op":"withColumns","exprs":{"x1":{"fn":"gte","args":[{"fn":"abs","args":[{"col":"l2_i32"}]},{"col":"l2_i32"}]}}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_f64","type":"f64"},{"name":"l4_f64","type":"f64"},{"name":"l5_utf8","type":"utf8"},{"name":"l6_i64","type":"i64"},{"name":"x1","type":"bool"}],"columns":{"l0_utf8":[null,"",null,null,"zz","aa","B","b","b","c","ab",null,"😀","aa","aa",null,null,null,null,null],"l1_i32":[2147483647,212,5,3,2,1,1,0,-1,-3,-4,-4,null,null,null,null,null,null,null,null],"l2_i32":[2,-4,3,2,3,-2,5,0,-2147483648,null,null,null,1,2,null,null,2,1,-5,null],"l3_f64":[43.13,4.0,null,"-Infinity",-41.651,-1.0,3.0,0.0,60.721,88.229,-4.0,null,-36.69,54.971,-26.911,null,null,-48.14,3.0,-27.958],"l4_f64":[null,-50.776,null,-74.384,"NaN",null,-3.0,-4.0,58.135,1.0,null,80.555,-1.0,2.0,null,-4.653,2.0,"NaN",3.0,"NaN"],"l5_utf8":["ab","b",null,"😀","c","","é",null,"zz",null,"B","aa",null,"😀","😀",null,"a",null,null,"é"],"l6_i64":["9007199254740992","9007199254740992","9007199254740994","-5","-4611686018427387904","-4611686018427387904",null,"-4611686018427387904","-4611686018427387904",null,"-1",null,"9007199254740995","9007199254740993","-4611686018427387904","-5","9007199254740995","2",null,"-4611686018427387904"],"x1":[true,true,true,true,true,true,true,true,true,null,null,null,true,true,null,null,true,true,true,null]}}},{"id":"random/0158","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_i64","type":"i64"},{"name":"l4_utf8","type":"utf8"}],"columns":{"l0_utf8":["a","","aa","","zz","aa","B","a","b","ab","","é","c","a","","a","zz","ab","aa","aa","a","é","b","ab","c","c","aa","😀","b","","c","zz","😀","😀","a","b","zz","","é","😀","😀","aa","zz","b","c","zz","😀","aa","é","c","zz","zz","aa","c","","zz","é","a","B","B"],"l1_i32":[-5,2,-2,-5,0,5,-1,4,4,3,-3,3,3,-2,-1,2,-4,1,-3,2147483647,1,-5,-1,2,2,2,-3,-1,-4,-79,-1,3,4,5,-4,0,5,5,-5,-4,-2,4,-2,1,-3,1,-5,0,3,-3,4,-2147483648,-5,1,0,-4,201,-2,5,-2],"l2_i32":[3,2,-4,2,5,1,2,4,-5,-1,3,-4,1,-4,1,-5,-1,0,1,2,-43,1,4,3,-1,3,-4,-3,2147483647,-2147483648,-1,0,-3,1,-2,-4,2,5,191,-5,4,-2,4,-3,-2,-4,3,4,-4,-1,-2,2,3,0,1,5,-5,-5,-3,2147483647],"l3_i64":["-4611686018427387904","2","-4611686018427387904","-2","-3","-2","1","9007199254740992","-1","9007199254740995","-5","-4611686018427387904","9007199254740995","-4611686018427387904","-4611686018427387904","9007199254740992","9007199254740993","9007199254740993","-4611686018427387904","-3","9007199254740993","-4611686018427387904","-4611686018427387904","0","1","9007199254740992","9007199254740994","-4611686018427387904","-4611686018427387904","9007199254740994","-4611686018427387904","-4611686018427387904","2","9007199254740992","9007199254740992","2","9007199254740995","-4611686018427387904","2","-4611686018427387904","9007199254740992","9007199254740995","-2","-4611686018427387904","9007199254740992","-5","9007199254740993","-4611686018427387904","9007199254740993","-4611686018427387904","-2","-4611686018427387904","2","9007199254740992","2","-4611686018427387904","4","-4","3","-4611686018427387904"],"l4_utf8":["zz","b","é","c","c","aa","c","B","aa","a","c","a","b","😀","a","é","c","a","a","é","b","b","aa","B","","aa","a","é","b","aa","😀","","zz","ab","","😀","B","c","B","B","","","","ab","😀","aa","","","aa","aa","b","c","c","aa","B","b","aa","b","é",""]}}},"ops":[{"op":"head","n":10},{"op":"unique"},{"op":"groupBy","keys":["l1_i32"],"aggs":{"n2":{"agg":"len"},"a2_0":{"agg":"max","arg":{"col":"l2_i32"}},"a2_1":{"agg":"nUnique","arg":{"col":"l3_i64"}},"a2_2":{"agg":"sum","arg":{"col":"l2_i32"}}}},{"op":"filter","predicate":{"fn":"or","args":[{"fn":"and","args":[{"fn":"lt","args":[{"lit":-1},{"col":"a2_2"}]},{"fn":"gt","args":[{"col":"a2_2"},{"col":"a2_2"}]}]},{"fn":"neq","args":[{"lit":2},{"col":"a2_0"}]}]}}],"expected":{"fields":[{"name":"l1_i32","type":"i32"},{"name":"n2","type":"u32"},{"name":"a2_0","type":"i32"},{"name":"a2_1","type":"u32"},{"name":"a2_2","type":"i32"}],"columns":{"l1_i32":[-5,-2,0,5,4,3],"n2":[2,1,1,1,2,1],"a2_0":[3,-4,5,1,4,-1],"a2_1":[2,1,1,1,2,1],"a2_2":[5,-4,5,1,-1,-1]}}},{"id":"random/0159","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_f64","type":"f64"},{"name":"l4_i32","type":"i32"},{"name":"l5_utf8","type":"utf8"},{"name":"l6_i64","type":"i64"}],"columns":{"l0_utf8":["zz"],"l1_i32":[2147483647],"l2_f64":[0.0],"l3_f64":[97.147],"l4_i32":[null],"l5_utf8":[null],"l6_i64":["-4611686018427387904"]}},"right":{"fields":[{"name":"l1_i32","type":"i32"},{"name":"r0_f64","type":"f64"}],"columns":{"r0_f64":[],"l1_i32":[]}}},"ops":[{"op":"withColumns","exprs":{"x0":{"fn":"mul","args":[{"fn":"fillNull","args":[{"col":"l3_f64"},{"lit":1.5}]},{"fn":"abs","args":[{"col":"l4_i32"}]}]}}},{"op":"filter","predicate":{"fn":"gte","args":[{"fn":"abs","args":[{"col":"l3_f64"}]},{"col":"l3_f64"}]}},{"op":"join","right":"right","on":["l1_i32"],"how":"inner"},{"op":"sort","keys":[{"expr":{"col":"l4_i32"},"descending":false},{"expr":{"col":"l3_f64"},"descending":false}],"nullsLast":true}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_f64","type":"f64"},{"name":"l4_i32","type":"i32"},{"name":"l5_utf8","type":"utf8"},{"name":"l6_i64","type":"i64"},{"name":"x0","type":"f64"},{"name":"r0_f64","type":"f64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_f64":[],"l3_f64":[],"l4_i32":[],"l5_utf8":[],"l6_i64":[],"x0":[],"r0_f64":[]}}},{"id":"random/0160","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_bool","type":"bool"},{"name":"l4_i64","type":"i64"},{"name":"l5_i32","type":"i32"},{"name":"l6_i64","type":"i64"}],"columns":{"l0_utf8":["B"],"l1_i32":[-1],"l2_bool":[null],"l3_bool":[null],"l4_i64":["5"],"l5_i32":[4],"l6_i64":["4"]}}},"ops":[{"op":"withColumns","exprs":{"x0":{"col":"l1_i32"}}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_bool","type":"bool"},{"name":"l4_i64","type":"i64"},{"name":"l5_i32","type":"i32"},{"name":"l6_i64","type":"i64"},{"name":"x0","type":"i32"}],"columns":{"l0_utf8":["B"],"l1_i32":[-1],"l2_bool":[null],"l3_bool":[null],"l4_i64":["5"],"l5_i32":[4],"l6_i64":["4"],"x0":[-1]}}},{"id":"random/0161","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_f64","type":"f64"},{"name":"l4_utf8","type":"utf8"}],"columns":{"l0_utf8":["é","ab",null,"c",null],"l1_i32":[5,null,5,5,null],"l2_f64":[null,1.0,54.199,2.0,3.206],"l3_f64":[-37.154,0.0,null,-52.505,1.0],"l4_utf8":["","zz",null,"é","c"]}}},"ops":[{"op":"head","n":10},{"op":"groupBy","keys":["l4_utf8"],"aggs":{"n1":{"agg":"len"},"a1_0":{"agg":"max","arg":{"col":"l3_f64"}}}},{"op":"sort","keys":[{"expr":{"col":"n1"},"descending":false},{"expr":{"col":"a1_0"},"descending":false}],"nullsLast":false}],"expected":{"fields":[{"name":"l4_utf8","type":"utf8"},{"name":"n1","type":"u32"},{"name":"a1_0","type":"f64"}],"columns":{"l4_utf8":[null,"é","","zz","c"],"n1":[1,1,1,1,1],"a1_0":[null,-52.505,-37.154,0.0,1.0]}}},{"id":"random/0162","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_f64","type":"f64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_f64":[],"l3_f64":[]}},"right":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"r0_utf8","type":"utf8"}],"columns":{"r0_utf8":["ab","c","zz","é","😀","é","a","b","b","","a","ab","a",null,"c","aa","aa","aa","é","zz","😀","a","ab","é","😀","B","é","ab","zz","aa"],"l0_utf8":["","ab",null,"a","😀","a","a","😀","","a",null,null,"ab","a","😀",null,"😀","😀",null,"a","","a","",null,"a",null,"ab","a","",null]}}},"ops":[{"op":"join","right":"right","on":["l0_utf8"],"how":"left"},{"op":"sort","keys":[{"expr":{"col":"l2_f64"},"descending":true},{"expr":{"col":"l1_i32"},"descending":true}],"nullsLast":true},{"op":"filter","predicate":{"fn":"gte","args":[{"fn":"floorDiv","args":[{"col":"l1_i32"},{"col":"l1_i32"}]},{"col":"l2_f64"}]}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_f64","type":"f64"},{"name":"r0_utf8","type":"utf8"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_f64":[],"l3_f64":[],"r0_utf8":[]}}},{"id":"random/0163","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_i32","type":"i32"},{"name":"l4_i64","type":"i64"},{"name":"l5_bool","type":"bool"}],"columns":{"l0_utf8":["zz","a","b","é","c"],"l1_i32":[null,null,2,-5,null],"l2_utf8":["ab",null,"b",null,"aa"],"l3_i32":[-4,-1,-1,null,5],"l4_i64":[null,null,null,null,null],"l5_bool":[null,false,true,null,false]}}},"ops":[{"op":"filter","predicate":{"fn":"or","args":[{"fn":"eq","args":[{"col":"l3_i32"},{"col":"l1_i32"}]},{"fn":"eq","args":[{"col":"l1_i32"},{"col":"l1_i32"}]}]}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_i32","type":"i32"},{"name":"l4_i64","type":"i64"},{"name":"l5_bool","type":"bool"}],"columns":{"l0_utf8":["b","é"],"l1_i32":[2,-5],"l2_utf8":["b",null],"l3_i32":[-1,null],"l4_i64":[null,null],"l5_bool":[true,null]}}},{"id":"random/0164","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_f64","type":"f64"}],"columns":{"l0_utf8":[null,"c",null,"c","😀","B",null,"aa","b","a",null,"zz","ab","é",null,"","","😀","",""],"l1_i32":[2,3,1,null,-2,null,null,3,-4,2,-3,-2,null,0,null,null,null,-1,null,-5],"l2_f64":[-25.4,null,59.931,-95.535,null,0.0,-28.035,null,null,null,null,75.582,null,null,89.818,-46.36,null,-4.0,50.941,21.846],"l3_utf8":["B","a",null,"B","B",null,null,null,"","é","😀","é","b",null,null,"ab","c","a","é","b"],"l4_f64":[83.783,-1.0,96.957,4.0,1.0,null,-1.0,-47.318,null,null,-52.309,1.0,0.0,-3.0,4.0,70.654,null,-4.0,-1.0,"NaN"]}}},"ops":[{"op":"sort","keys":[{"expr":{"col":"l1_i32"},"descending":false}],"nullsLast":true}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_f64","type":"f64"}],"columns":{"l0_utf8":["","b",null,"😀","zz","😀","é",null,null,"a","c","aa","c","B",null,"ab",null,"","",""],"l1_i32":[-5,-4,-3,-2,-2,-1,0,1,2,2,3,3,null,null,null,null,null,null,null,null],"l2_f64":[21.846,null,null,null,75.582,-4.0,null,59.931,-25.4,null,null,null,-95.535,0.0,-28.035,null,89.818,-46.36,null,50.941],"l3_utf8":["b","","😀","B","é","a",null,null,"B","é","a",null,"B",null,null,"b",null,"ab","c","é"],"l4_f64":["NaN",null,-52.309,1.0,1.0,-4.0,-3.0,96.957,83.783,null,-1.0,-47.318,4.0,null,-1.0,0.0,4.0,70.654,null,-1.0]}}},{"id":"random/0165","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_i64","type":"i64"}],"columns":{"l0_utf8":["c"],"l1_i32":[0],"l2_utf8":["aa"],"l3_i64":["-4611686018427387904"]}}},"ops":[{"op":"groupBy","keys":["l2_utf8"],"aggs":{"n0":{"agg":"len"},"a0_0":{"agg":"count","arg":{"col":"l3_i64"}},"a0_1":{"agg":"nUnique","arg":{"col":"l0_utf8"}},"a0_2":{"agg":"count","arg":{"col":"l0_utf8"}}}},{"op":"filter","predicate":{"fn":"strStartsWith","args":[{"col":"l2_utf8"}],"prefix":"a"}}],"expected":{"fields":[{"name":"l2_utf8","type":"utf8"},{"name":"n0","type":"u32"},{"name":"a0_0","type":"u32"},{"name":"a0_1","type":"u32"},{"name":"a0_2","type":"u32"}],"columns":{"l2_utf8":["aa"],"n0":[1],"a0_0":[1],"a0_1":[1],"a0_2":[1]}}},{"id":"random/0166","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_i32","type":"i32"},{"name":"l4_i64","type":"i64"},{"name":"l5_bool","type":"bool"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_utf8":[],"l3_i32":[],"l4_i64":[],"l5_bool":[]}}},"ops":[{"op":"head","n":1},{"op":"groupBy","keys":["l5_bool"],"aggs":{"n1":{"agg":"len"},"a1_0":{"agg":"last","arg":{"col":"l0_utf8"}}}},{"op":"withColumns","exprs":{"x2":{"fn":"add","args":[{"lit":2.5},{"lit":2}]}}}],"expected":{"fields":[{"name":"l5_bool","type":"bool"},{"name":"n1","type":"u32"},{"name":"a1_0","type":"utf8"},{"name":"x2","type":"f64"}],"columns":{"l5_bool":[],"n1":[],"a1_0":[],"x2":[]}}},{"id":"random/0167","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_utf8","type":"utf8"}],"columns":{"l0_utf8":["b","ab","😀",null,"","","","😀","c","","a","b",null,"ab","c","","zz","","zz",null],"l1_i32":[-1,-3,0,0,5,-5,0,-2147483648,-3,-5,683,1,5,null,2,-2,-5,-5,3,2147483647],"l2_f64":[76.958,-0.784,3.0,null,90.124,4.0,-4.0,"NaN",-86.322,-90.133,-2.0,null,"NaN",89.251,52.872,-3.0,-1.0,0.0,-2.0,null],"l3_utf8":["b","","😀","😀","😀","😀","é","c","aa","ab","b","b","aa","zz","é",null,"zz","b","😀","zz"]}}},"ops":[{"op":"sort","keys":[{"expr":{"col":"l2_f64"},"descending":true}],"nullsLast":false},{"op":"groupBy","keys":["l1_i32"],"aggs":{"n1":{"agg":"len"},"a1_0":{"agg":"min","arg":{"col":"l0_utf8"}}}}],"expected":{"fields":[{"name":"l1_i32","type":"i32"},{"name":"n1","type":"u32"},{"name":"a1_0","type":"utf8"}],"columns":{"l1_i32":[0,1,2147483647,-2147483648,5,null,-1,2,-5,-3,683,3,-2],"n1":[3,1,1,1,2,1,1,1,4,2,1,1,1],"a1_0":["","b",null,"😀","","ab","b","c","","ab","a","zz",""]}}},{"id":"random/0168","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_i64","type":"i64"}],"columns":{"l0_utf8":["a"],"l1_i32":[1],"l2_i32":[4],"l3_i64":["-2"]}}},"ops":[{"op":"filter","predicate":{"fn":"eq","args":[{"col":"l3_i64"},{"lit":"-1","type":"i64"}]}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_i64","type":"i64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_i32":[],"l3_i64":[]}}},{"id":"random/0169","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_i64","type":"i64"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_i64","type":"i64"}],"columns":{"l0_utf8":["B"],"l1_i32":[-1],"l2_f64":[32.563],"l3_i64":["9007199254740992"],"l4_utf8":["é"],"l5_i64":["-4611686018427387904"]}},"right":{"fields":[{"name":"l4_utf8","type":"utf8"},{"name":"r0_utf8","type":"utf8"}],"columns":{"r0_utf8":["aa","c","ab"],"l4_utf8":["é","é","é"]}}},"ops":[{"op":"filter","predicate":{"fn":"lte","args":[{"fn":"mul","args":[{"col":"l2_f64"},{"col":"l2_f64"}]},{"col":"l2_f64"}]}},{"op":"join","right":"right","on":["l4_utf8"],"how":"full"}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_i64","type":"i64"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_i64","type":"i64"},{"name":"r0_utf8","type":"utf8"}],"columns":{"l0_utf8":[null,null,null],"l1_i32":[null,null,null],"l2_f64":[null,null,null],"l3_i64":[null,null,null],"l4_utf8":["é","é","é"],"l5_i64":[null,null,null],"r0_utf8":["aa","c","ab"]}}},{"id":"random/0170","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_f64","type":"f64"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_utf8","type":"utf8"}],"columns":{"l0_utf8":["c","","c","ab","b","c","ab","B","😀","","zz",null,"é","b","ab","ab","b","ab",null,null],"l1_i32":[0,3,null,-2,-850,null,null,null,2147483647,null,1,null,511,null,null,-1,-4,-1,null,2],"l2_i32":[-2,3,null,0,-2,-3,-4,3,-3,1,2,4,-783,null,null,null,null,5,-3,0],"l3_f64":[-49.32,"NaN",null,null,71.79,-4.0,null,-2.0,1.0,-3.794,37.924,-4.0,4.0,4.0,-2.0,97.559,-47.037,49.061,-28.537,-28.574],"l4_utf8":[null,"c","ab","ab","a",null,"ab","B",null,"a","B","😀",null,null,"b","aa",null,"ab","é","zz"],"l5_utf8":["","ab","b",null,"","ab","B",null,null,"B",null,"b","ab","ab","aa","","zz","B","é",""]}}},"ops":[{"op":"groupBy","keys":["l4_utf8"],"aggs":{"n0":{"agg":"len"},"a0_0":{"agg":"min","arg":{"col":"l2_i32"}},"a0_1":{"agg":"max","arg":{"col":"l0_utf8"}}}},{"op":"withColumns","exprs":{"x1":{"fn":"when","args":[{"fn":"or","args":[{"fn":"lte","args":[{"col":"a0_0"},{"col":"a0_0"}]},{"fn":"isNull","args":[{"col":"l4_utf8"}]}]},{"col":"a0_0"},{"fn":"fillNull","args":[{"col":"a0_0"},{"lit":1.5}]}]}}},{"op":"filter","predicate":{"fn":"gt","args":[{"col":"a0_0"},{"fn":"sub","args":[{"col":"a0_0"},{"col":"a0_0"}]}]}},{"op":"sort","keys":[{"expr":{"col":"n0"},"descending":false}],"nullsLast":false}],"expected":{"fields":[{"name":"l4_utf8","type":"utf8"},{"name":"n0","type":"u32"},{"name":"a0_0","type":"i32"},{"name":"a0_1","type":"utf8"},{"name":"x1","type":"f64"}],"columns":{"l4_utf8":["c","😀","B"],"n0":[1,1,2],"a0_0":[3,4,2],"a0_1":["",null,"zz"],"x1":[3.0,4.0,2.0]}}},{"id":"random/0171","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_utf8","type":"utf8"}],"columns":{"l0_utf8":["b","aa","b","b","b"],"l1_i32":[-1,-3,2,1,-3],"l2_utf8":[null,"😀","ab","b","aa"],"l3_utf8":["aa","a","B","zz","zz"]}},"right":{"fields":[{"name":"l1_i32","type":"i32"},{"name":"r0_bool","type":"bool"}],"columns":{"r0_bool":[false,false,false],"l1_i32":[1,-1,-3]}}},"ops":[{"op":"withColumns","exprs":{"x0":{"fn":"gt","args":[{"col":"l1_i32"},{"col":"l1_i32"}]}}},{"op":"join","right":"right","on":["l1_i32"],"how":"left"},{"op":"sort","keys":[{"expr":{"col":"l2_utf8"},"descending":false}],"nullsLast":false},{"op":"sort","keys":[{"expr":{"col":"l1_i32"},"descending":true},{"expr":{"col":"l2_utf8"},"descending":false}],"nullsLast":false}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_utf8","type":"utf8"},{"name":"x0","type":"bool"},{"name":"r0_bool","type":"bool"}],"columns":{"l0_utf8":["b","b","b","b","aa"],"l1_i32":[2,1,-1,-3,-3],"l2_utf8":["ab","b",null,"aa","😀"],"l3_utf8":["B","zz","aa","zz","a"],"x0":[false,false,false,false,false],"r0_bool":[null,false,false,false,false]}}},{"id":"random/0172","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_i32","type":"i32"},{"name":"l5_utf8","type":"utf8"},{"name":"l6_utf8","type":"utf8"}],"columns":{"l0_utf8":["zz","ab","é","c","c"],"l1_i32":[-2147483648,-4,2,-5,1],"l2_f64":[2.0,-42.236,"NaN",4.0,20.388],"l3_utf8":["B","a","é","","zz"],"l4_i32":[-5,-2147483648,1,-1,3],"l5_utf8":["zz","","ab","aa","aa"],"l6_utf8":["","ab","c","","😀"]}}},"ops":[{"op":"sort","keys":[{"expr":{"col":"l6_utf8"},"descending":true},{"expr":{"col":"l0_utf8"},"descending":true}],"nullsLast":true},{"op":"groupBy","keys":["l0_utf8"],"aggs":{"n1":{"agg":"len"},"a1_0":{"agg":"first","arg":{"col":"l6_utf8"}},"a1_1":{"agg":"min","arg":{"col":"l2_f64"}},"a1_2":{"agg":"last","arg":{"col":"l1_i32"}}}},{"op":"sort","keys":[{"expr":{"col":"a1_2"},"descending":false},{"expr":{"col":"a1_1"},"descending":false}],"nullsLast":false},{"op":"withColumns","exprs":{"x3":{"fn":"lte","args":[{"col":"a1_2"},{"fn":"div","args":[{"lit":2},{"col":"a1_1"}]}]}}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"n1","type":"u32"},{"name":"a1_0","type":"utf8"},{"name":"a1_1","type":"f64"},{"name":"a1_2","type":"i32"},{"name":"x3","type":"bool"}],"columns":{"l0_utf8":["zz","c","ab","é"],"n1":[1,2,1,1],"a1_0":["","😀","ab","c"],"a1_1":[2.0,4.0,-42.236,"NaN"],"a1_2":[-2147483648,-5,-4,2],"x3":[true,true,true,true]}}},{"id":"random/0173","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_utf8","type":"utf8"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_f64":[],"l3_utf8":[],"l4_utf8":[]}}},"ops":[{"op":"withColumns","exprs":{"x0":{"lit":0.5}}},{"op":"unique","columns":["x0"]}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_utf8","type":"utf8"},{"name":"x0","type":"f64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_f64":[],"l3_utf8":[],"l4_utf8":[],"x0":[]}}},{"id":"random/0174","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_i32","type":"i32"}],"columns":{"l0_utf8":["a","a","B","a","é","c","b","ab","c","é","😀","B","aa",null,"","b","aa","😀","a","b"],"l1_i32":[3,1,-5,-3,-4,3,0,-1,-4,-3,-4,2,-2147483648,-974,-4,-5,2147483647,2,5,null],"l2_f64":[-1.0,null,0.593,null,18.255,70.064,4.0,42.864,-2.0,-72.168,"NaN",-4.0,39.957,-90.685,-32.366,4.0,-3.0,2.0,"-0",-22.617],"l3_i32":[4,-5,-4,-5,3,-1,2,-5,-4,-947,-5,-4,2,0,-4,-1,null,null,5,3]}}},"ops":[{"op":"filter","predicate":{"fn":"gt","args":[{"col":"l3_i32"},{"fn":"floorDiv","args":[{"col":"l2_f64"},{"col":"l3_i32"}]}]}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_i32","type":"i32"}],"columns":{"l0_utf8":["a","c","ab",null,"b","a","b"],"l1_i32":[3,3,-1,-974,-5,5,null],"l2_f64":[-1.0,70.064,42.864,-90.685,4.0,"-0",-22.617],"l3_i32":[4,-1,-5,0,-1,5,3]}}},{"id":"random/0175","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_bool","type":"bool"},{"name":"l4_f64","type":"f64"}],"columns":{"l0_utf8":[null,null,"zz",null,null,"ab","😀","ab","a","aa","zz",null,null,"a","ab","ab","","😀","a","ab"],"l1_i32":[0,null,null,-3,null,-4,-1,1,null,null,-3,-5,-1,3,3,-464,-3,3,-4,null],"l2_utf8":[null,"B","B","b","b","aa","a",null,"a","c","😀","B","zz","ab","b","a","c","c","B","é"],"l3_bool":[true,true,null,true,true,false,true,true,true,null,null,false,null,true,null,true,false,true,false,false],"l4_f64":[null,-4.0,null,null,-0.679,-4.0,null,35.308,4.0,"NaN",87.138,35.986,-15.632,-48.286,-44.037,null,-75.71,1.0,4.0,null]}}},"ops":[{"op":"withColumns","exprs":{"x0":{"fn":"when","args":[{"fn":"neq","args":[{"col":"l4_f64"},{"col":"l1_i32"}]},{"col":"l1_i32"},{"col":"l1_i32"}]}}},{"op":"withColumns","exprs":{"x1":{"fn":"isNull","args":[{"col":"l3_bool"}]}}},{"op":"filter","predicate":{"fn":"strStartsWith","args":[{"col":"l0_utf8"}],"prefix":"a"}},{"op":"filter","predicate":{"fn":"lt","args":[{"fn":"sub","args":[{"lit":0.5},{"col":"l1_i32"}]},{"fn":"mul","args":[{"col":"l4_f64"},{"col":"l4_f64"}]}]}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_bool","type":"bool"},{"name":"l4_f64","type":"f64"},{"name":"x0","type":"i32"},{"name":"x1","type":"bool"}],"columns":{"l0_utf8":["ab","ab","a","ab","a"],"l1_i32":[-4,1,3,3,-4],"l2_utf8":["aa",null,"ab","b","B"],"l3_bool":[false,true,true,null,false],"l4_f64":[-4.0,35.308,-48.286,-44.037,4.0],"x0":[-4,1,3,3,-4],"x1":[false,false,false,true,false]}}},{"id":"random/0176","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_bool","type":"bool"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_f64","type":"f64"},{"name":"l6_i64","type":"i64"}],"columns":{"l0_utf8":["",null,"zz",null,"","B",null,null,"😀",null,null,null,"😀",null,"a","ab",null,"ab","ab","ab","","c","","😀","aa",null,null,"😀",null,null,"a",null,"B","a",null,null,"ab","zz","B","ab","B","😀","😀",null,"a","zz","aa","aa","b",null,"zz",null,null,null,"😀",null,null,"😀",null,"B"],"l1_i32":[-3,null,-5,null,null,null,5,null,null,null,-3,2,null,4,null,3,-5,4,4,-5,null,null,null,null,2147483647,-2147483648,null,-2,null,null,-3,-5,0,-3,4,-3,5,null,-3,0,1,1,null,-5,-2,null,-1,-5,-5,-3,0,null,4,3,0,1,-5,-5,4,-2],"l2_f64":[41.511,-3.0,"NaN",null,null,74.465,null,-16.684,4.0,24.067,null,-4.0,50.247,2.0,26.984,49.046,-3.0,30.488,-36.623,-2.0,null,-43.541,-4.0,null,-11.235,null,null,4.0,-87.602,null,null,4.0,-77.958,3.0,null,null,-2.0,3.0,-61.422,27.453,3.0,1.0,null,3.0,2.0,null,1.0,-4.0,-3.0,null,null,41.178,null,-4.0,null,4.0,36.737,62.647,-98.38,-4.0],"l3_bool":[null,null,true,true,null,true,null,true,false,true,true,true,true,null,false,true,false,true,null,false,false,false,true,null,false,null,null,null,null,false,true,true,null,true,true,true,true,false,false,false,null,true,null,true,null,null,true,null,true,false,null,false,false,false,false,true,true,null,false,null],"l4_utf8":["zz","aa","","a",null,"b","a","😀","a","zz","aa",null,"B","zz","aa",null,null,null,"zz","é",null,"c",null,null,"b","B","c","a","b",null,"ab","a","a","ab","","😀","aa","c","c","😀",null,"zz","aa",null,"a","ab","😀",null,null,"😀","b",null,"ab","c","zz","é","B","B","a","B"],"l5_f64":[-1.0,-1.0,null,null,4.0,null,"-Infinity",45.817,null,-2.0,-1.0,3.0,-2.977,3.0,-72.494,null,-1.0,null,-35.721,-1.0,"-Infinity",-3.0,-94.137,0.0,null,-1.0,null,null,-84.754,-1.0,"NaN",71.179,null,3.0,4.193,78.118,-19.278,null,58.217,-45.336,58.631,2.0,4.0,-67.984,null,null,null,1.0,null,-4.0,3.0,-1.0,null,null,null,null,3.0,-15.707,-4.0,90.511],"l6_i64":["-4611686018427387904",null,"-2","-4611686018427387904","-4611686018427387904","9007199254740994","-2",null,null,"-4611686018427387904",null,"-2","-4611686018427387904","-4611686018427387904","9007199254740995",null,null,"-3","-4611686018427387904","4","-5","-4611686018427387904","9007199254740992","9007199254740993","9007199254740995",null,"9007199254740994","9007199254740995",null,null,"-4611686018427387904","9007199254740993","9007199254740993","-4611686018427387904","9007199254740994","9007199254740995","9007199254740993","9007199254740992","-2","0","2","-4611686018427387904","9007199254740994","4","-5","4","-4611686018427387904","5",null,"-4611686018427387904","9007199254740992","-4611686018427387904","9007199254740994","-4611686018427387904","-4611686018427387904","-4611686018427387904","9007199254740994",null,"1","-4611686018427387904"]}}},"ops":[{"op":"filter","predicate":{"col":"l3_bool"}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_bool","type":"bool"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_f64","type":"f64"},{"name":"l6_i64","type":"i64"}],"columns":{"l0_utf8":["zz",null,"B",null,null,null,null,"😀","ab","ab","","a",null,"a",null,null,"ab","😀",null,"aa","b",null,null],"l1_i32":[-5,null,null,null,null,-3,2,null,3,4,null,-3,-5,-3,4,-3,5,1,-5,-1,-5,1,-5],"l2_f64":["NaN",null,74.465,-16.684,24.067,null,-4.0,50.247,49.046,30.488,-4.0,null,4.0,3.0,null,null,-2.0,1.0,3.0,1.0,-3.0,4.0,36.737],"l3_bool":[true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true],"l4_utf8":["","a","b","😀","zz","aa",null,"B",null,null,null,"ab","a","ab","","😀","aa","zz",null,"😀",null,"é","B"],"l5_f64":[null,null,null,45.817,-2.0,-1.0,3.0,-2.977,null,null,-94.137,"NaN",71.179,3.0,4.193,78.118,-19.278,2.0,-67.984,null,null,null,3.0],"l6_i64":["-2","-4611686018427387904","9007199254740994",null,"-4611686018427387904",null,"-2","-4611686018427387904",null,"-3","9007199254740992","-4611686018427387904","9007199254740993","-4611686018427387904","9007199254740994","9007199254740995","9007199254740993","-4611686018427387904","4","-4611686018427387904",null,"-4611686018427387904","9007199254740994"]}}},{"id":"random/0177","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_i64","type":"i64"}],"columns":{"l0_utf8":["é","b",null,"ab",null,"ab",null,"aa","a","c",null,"a","B","aa","a",null,"aa","c",null,"b"],"l1_i32":[-3,-2147483648,-1,-4,null,0,2,-1,5,3,2147483647,1,3,1,-1,-2,-2,0,-2,-5],"l2_f64":[0.0,"NaN",96.439,2.0,null,null,63.372,31.718,-53.943,31.912,2.0,"NaN","-Infinity",-97.364,2.0,null,1.0,0.0,-33.8,0.0],"l3_utf8":["c","c",null,"B","zz",null,"a","é","b","ab","c","ab","aa","é","","a","c","aa","ab",null],"l4_utf8":["",null,null,null,"c",null,"c","aa","b","😀",null,"aa",null,"c","c","aa",null,"😀","B","aa"],"l5_i64":[null,"9007199254740993","-4611686018427387904","9007199254740992",null,"4","9007199254740995","9007199254740994","-4611686018427387904",null,"2","-4611686018427387904",null,null,null,"-4611686018427387904",null,"9007199254740992","9007199254740995",null]}}},"ops":[{"op":"filter","predicate":{"fn":"eq","args":[{"col":"l5_i64"},{"lit":"-3","type":"i64"}]}},{"op":"unique","columns":["l2_f64"]},{"op":"filter","predicate":{"fn":"gte","args":[{"fn":"div","args":[{"lit":-1},{"lit":2}]},{"fn":"abs","args":[{"lit":0.5}]}]}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_i64","type":"i64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_f64":[],"l3_utf8":[],"l4_utf8":[],"l5_i64":[]}}},{"id":"random/0178","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_i64","type":"i64"}],"columns":{"l0_utf8":["b","c",null,"aa","é","b",null,"ab","zz","😀","😀",null,null,"aa","a",null,"aa","aa",null,"ab","b","b","b","a",null,"ab","","ab","ab","aa","é","","","B","zz","😀","ab","a",null,null,"a",null,"ab","😀",null,"aa",null,null,"","",null,null,"zz",null,"","😀","é","b","B","zz"],"l1_i32":[0,-2,null,null,null,5,null,null,3,-2,5,-5,4,null,null,5,null,-3,null,null,2,null,null,-4,null,3,2,2147483647,null,0,null,5,5,-2,null,2,null,null,-5,null,1,-2147483648,null,null,null,3,null,-1,3,null,-5,null,4,1,1,null,-4,5,-3,4],"l2_i32":[-237,-3,318,null,null,4,null,1,5,null,null,0,-3,5,4,2,-3,4,null,1,null,-4,null,3,3,3,3,5,null,4,null,null,-5,-2,null,-4,5,-3,1,-5,-1,null,0,-4,-1,2,5,-1,null,-2,-3,null,null,4,0,null,null,null,-1,0],"l3_i64":["-3","9007199254740992",null,null,"9007199254740995","-4611686018427387904","9007199254740995","2","9007199254740994","1","-4611686018427387904","-4611686018427387904","-4","-4611686018427387904",null,"-4611686018427387904","3",null,"3","4","-5",null,"-4","-2","-4611686018427387904","-4","3",null,null,"-4611686018427387904",null,"-4611686018427387904","5",null,null,null,"5",null,"-3","2","9007199254740994",null,"5","-3","-4611686018427387904","9007199254740994","-4611686018427387904","9007199254740993","9007199254740995","-4611686018427387904",null,"-4611686018427387904","9007199254740992",null,null,null,"9007199254740994","-1","-4611686018427387904",null]}}},"ops":[{"op":"groupBy","keys":["l3_i64"],"aggs":{"n0":{"agg":"len"},"a0_0":{"agg":"last","arg":{"col":"l0_utf8"}}}},{"op":"head","n":9}],"expected":{"fields":[{"name":"l3_i64","type":"i64"},{"name":"n0","type":"u32"},{"name":"a0_0","type":"utf8"}],"columns":{"l3_i64":["-3","9007199254740992",null,"9007199254740995","-4611686018427387904","2","9007199254740994","1","-4"],"n0":[3,2,18,3,13,2,4,1,3],"a0_0":["😀","zz","zz","","B",null,"é","😀","ab"]}}},{"id":"random/0179","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_f64","type":"f64"},{"name":"l5_bool","type":"bool"},{"name":"l6_f64","type":"f64"}],"columns":{"l0_utf8":["","b","aa",null,"B","ab","aa","ab","😀",null,null,"é","B","B","c","","😀","B","zz","😀","zz",null,"",null,"B","é","😀","zz",null,"",null,"ab",null,"c","ab","😀","c","c","😀","b","c",null,"","a","B",null,"B",null,"c","","é","😀","B","é","é","B","😀",null,"c","😀"],"l1_i32":[-5,0,-4,-1,null,2,null,0,null,0,-5,-5,null,-4,null,-4,-2147483648,-4,-4,-3,0,-5,null,null,null,1,2,-1,null,0,null,null,-2147483648,null,null,2147483647,-2,-2,-3,null,null,0,null,-1,-4,3,null,5,null,-5,4,null,1,5,null,-1,-217,null,null,5],"l2_utf8":[null,"B","zz","","aa",null,"😀","é","c",null,"B",null,null,null,"ab","a","a","a","ab","","","c",null,"😀","a",null,null,"aa",null,"ab","é","é",null,null,"a","","😀","zz",null,"c","",null,null,"B","c",null,"zz","c",null,"é","zz",null,"é",null,null,"c","😀","a",null,null],"l3_utf8":[null,"c","zz",null,"zz","aa","ab","zz","aa","😀","b","c","c",null,null,"",null,null,null,"zz","zz",null,"c",null,"é","c","a","aa","😀","","é","é","a",null,null,"B","b","ab","c","zz",null,null,"c","aa","a",null,"B","aa","é","ab","zz","","😀","",null,"é","a","zz","b",null],"l4_f64":[-3.0,null,null,2.0,null,-3.0,"NaN",11.817,null,-63.499,null,null,4.0,null,-4.0,76.822,3.0,-71.436,null,55.604,null,-1.0,"NaN",-5.044,null,4.0,50.228,null,null,-5.686,1.0,4.0,-8.518,"Infinity",93.661,-66.411,-1.0,null,57.63,null,-5.153,31.942,38.776,null,0.0,-3.0,-34.805,0.0,69.303,-91.914,-18.616,-15.251,null,-29.761,1.0,3.0,-4.0,-82.802,4.0,-1.0],"l5_bool":[true,true,true,false,false,true,null,false,false,true,null,true,true,true,null,null,true,null,true,true,null,false,true,true,false,true,null,null,false,true,false,true,false,null,false,null,true,false,null,true,true,null,false,true,null,null,false,true,true,null,true,true,true,true,true,true,true,false,false,false],"l6_f64":[null,8.691,null,null,3.0,24.87,-3.0,21.096,null,null,3.0,null,null,2.0,null,null,1.0,null,4.0,-9.151,21.632,null,-1.0,-77.064,94.815,null,-54.955,3.0,29.347,1.0,null,93.951,-1.0,1.0,94.065,4.0,27.558,null,null,-15.281,1.0,3.0,-16.829,-11.592,null,-17.289,null,-1.0,2.0,-4.0,null,-1.0,0.0,null,-74.999,-3.0,null,null,-1.0,4.0]}}},"ops":[{"op":"filter","predicate":{"fn":"lt","args":[{"fn":"when","args":[{"fn":"strStartsWith","args":[{"col":"l0_utf8"}],"prefix":"a"},{"col":"l1_i32"},{"col":"l6_f64"}]},{"fn":"floorDiv","args":[{"col":"l1_i32"},{"col":"l4_f64"}]}]}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_f64","type":"f64"},{"name":"l5_bool","type":"bool"},{"name":"l6_f64","type":"f64"}],"columns":{"l0_utf8":["😀","😀",null,null,null,"","B"],"l1_i32":[-3,2,-2147483648,3,5,-5,-1],"l2_utf8":["",null,null,null,"c","é","c"],"l3_utf8":["zz","a","a",null,"aa","ab","é"],"l4_f64":[55.604,50.228,-8.518,-3.0,0.0,-91.914,3.0],"l5_bool":[true,null,false,null,true,null,true],"l6_f64":[-9.151,-54.955,-1.0,-17.289,-1.0,-4.0,-3.0]}}},{"id":"random/0180","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_i64","type":"i64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_i32":[],"l3_i64":[]}}},"ops":[{"op":"head","n":4},{"op":"unique","columns":["l2_i32"]},{"op":"filter","predicate":{"fn":"and","args":[{"fn":"neq","args":[{"col":"l2_i32"},{"col":"l1_i32"}]},{"fn":"lte","args":[{"col":"l2_i32"},{"col":"l1_i32"}]}]}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_i64","type":"i64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_i32":[],"l3_i64":[]}}},{"id":"random/0181","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_i64","type":"i64"}],"columns":{"l0_utf8":[null],"l1_i32":[4],"l2_f64":[-2.0],"l3_i64":["-3"]}},"right":{"fields":[{"name":"l3_i64","type":"i64"},{"name":"r0_bool","type":"bool"}],"columns":{"r0_bool":[false,false,true,true,false,false,true,true,false,false,false,true,true,false,null,true,true,false,true,true,true,false,false,true,true,true,true,true,true,true],"l3_i64":["-3","-3","-3","-3","-3","-3","-3","-3","-3","-3","-3","-3","-3","-3","-3","-3","-3","-3","-3","-3","-3","-3","-3","-3","-3","-3","-3","-3","-3","-3"]}}},"ops":[{"op":"join","right":"right","on":["l3_i64"],"how":"full"},{"op":"sort","keys":[{"expr":{"col":"l1_i32"},"descending":false},{"expr":{"col":"l2_f64"},"descending":true}],"nullsLast":true},{"op":"unique","columns":["l2_f64"]},{"op":"head","n":4}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_i64","type":"i64"},{"name":"r0_bool","type":"bool"}],"columns":{"l0_utf8":[null],"l1_i32":[4],"l2_f64":[-2.0],"l3_i64":["-3"],"r0_bool":[false]}}},{"id":"random/0182","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_i32","type":"i32"},{"name":"l5_utf8","type":"utf8"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_utf8":[],"l3_utf8":[],"l4_i32":[],"l5_utf8":[]}}},"ops":[{"op":"unique"},{"op":"unique","columns":["l1_i32"]},{"op":"filter","predicate":{"fn":"isNull","args":[{"col":"l2_utf8"}]}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_i32","type":"i32"},{"name":"l5_utf8","type":"utf8"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_utf8":[],"l3_utf8":[],"l4_i32":[],"l5_utf8":[]}}},{"id":"random/0183","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_bool","type":"bool"},{"name":"l4_bool","type":"bool"},{"name":"l5_utf8","type":"utf8"},{"name":"l6_f64","type":"f64"}],"columns":{"l0_utf8":["ab","zz","ab",null,"aa","aa","zz",null,"😀",null,"","😀","aa","😀","zz","b","é","é","zz","😀"],"l1_i32":[-3,0,4,-2,1,-4,0,5,4,1,2,-5,-3,-3,-3,5,-1,null,null,4],"l2_i32":[4,3,1,-1,4,-2,-497,-3,-74,null,-5,-5,-4,-3,1,-2,2147483647,null,1,-2],"l3_bool":[true,false,false,false,false,null,false,true,false,true,false,true,false,false,false,false,false,true,null,true],"l4_bool":[true,false,true,true,true,true,true,false,false,true,false,null,true,false,false,false,true,false,true,false],"l5_utf8":["","aa","aa","B","ab","é",null,"zz","b","é","","zz","b",null,"😀","ab","a","","b","c"],"l6_f64":[-4.0,2.0,2.109,3.0,-1.0,96.202,1.0,null,-1.0,74.777,1.0,71.745,-4.0,"NaN",-50.369,16.428,-3.0,null,-2.0,-56.69]}},"right":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"r0_f64","type":"f64"}],"columns":{"r0_f64":[],"l0_utf8":[]}}},"ops":[{"op":"unique","columns":["l6_f64"]},{"op":"join","right":"right","on":["l0_utf8"],"how":"left"},{"op":"head","n":9}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_bool","type":"bool"},{"name":"l4_bool","type":"bool"},{"name":"l5_utf8","type":"utf8"},{"name":"l6_f64","type":"f64"},{"name":"r0_f64","type":"f64"}],"columns":{"l0_utf8":["ab","zz","ab",null,"aa","aa","zz",null,null],"l1_i32":[-3,0,4,-2,1,-4,0,5,1],"l2_i32":[4,3,1,-1,4,-2,-497,-3,null],"l3_bool":[true,false,false,false,false,null,false,true,true],"l4_bool":[true,false,true,true,true,true,true,false,true],"l5_utf8":["","aa","aa","B","ab","é",null,"zz","é"],"l6_f64":[-4.0,2.0,2.109,3.0,-1.0,96.202,1.0,null,74.777],"r0_f64":[null,null,null,null,null,null,null,null,null]}}},{"id":"random/0184","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_f64","type":"f64"},{"name":"l4_i32","type":"i32"},{"name":"l5_i64","type":"i64"}],"columns":{"l0_utf8":["a",null,"é",null,"😀",null,null,"é","a","é",null,"ab","B","B","é","c","é","😀","a","B","b",null,"c","B","ab",null,"aa","b","c","é","ab","ab","zz","😀","😀","","zz","a","","aa","B","😀","😀","B","aa","","b","😀","","zz","b","aa","B",null,"b","","aa","b","é",""],"l1_i32":[1,0,2,3,null,3,5,-4,-5,-3,-5,-3,0,-1,-1,-2147483648,-1,1,4,5,-2147483648,-1,0,-2147483648,-2,2,0,-3,0,2,0,1,-3,-1,2147483647,-4,3,null,-2,4,2,-4,-5,-1,-2,-1,-1,5,-3,-2,-2147483648,-1,-1,-2147483648,-3,4,-1,-3,-1,-5],"l2_f64":[4.0,-3.0,-65.19,-92.118,-15.573,-64.016,null,0.0,0.0,"NaN",4.0,87.605,60.195,"NaN",-3.0,80.614,null,6.365,4.0,"Infinity",-4.0,-32.301,20.736,-3.0,-0.294,null,2.0,0.0,-45.75,2.0,"NaN",-92.801,null,"NaN",-18.803,-9.658,69.782,56.547,-3.0,86.524,"NaN",1.0,-47.966,2.0,1.0,-1.0,94.149,-2.0,1.0,-62.636,3.0,-83.386,3.0,-64.836,-84.221,"NaN",-4.0,0.0,58.95,-84.119],"l3_f64":[3.0,59.876,1.0,21.773,-4.0,3.0,-23.168,-24.849,79.674,-15.735,-8.364,63.536,-77.007,-6.574,1.0,6.346,"Infinity",44.342,-49.961,3.0,-3.0,-65.053,-65.201,83.539,-4.0,-3.0,-21.815,-4.0,-2.0,-17.633,98.676,-23.7,null,31.561,-3.0,0.0,29.272,3.0,-2.0,"NaN",-3.0,null,null,1.0,42.086,1.0,-13.623,-35.905,"NaN",-65.353,-87.631,null,-49.222,-2.0,-6.288,3.0,-1.0,-87.624,-4.0,null],"l4_i32":[-1,0,5,4,1,-5,-2,-1,-1,3,-2,null,2147483647,69,1,0,-3,5,4,1,-2,-436,-3,-5,5,-4,-4,null,null,-1,872,4,-3,0,0,2,5,null,-1,-1,-2147483648,2,2,-3,-2,1,-1,3,null,-5,-4,-3,-1,5,-2,2147483647,-3,1,null,1],"l5_i64":["9007199254740993","-4611686018427387904","-4611686018427387904","0","9007199254740992","0",null,"9007199254740992","-3","9007199254740992","9007199254740992","4","-4611686018427387904","2","9007199254740992","-3","-4611686018427387904","4","-4","9007199254740992","-4611686018427387904","-4611686018427387904","-4611686018427387904","-4611686018427387904","4",null,"0","-4611686018427387904","-4611686018427387904","9007199254740994","0","9007199254740994","5","5",null,"0",null,"-4611686018427387904",null,null,"-4611686018427387904","-4611686018427387904","5","9007199254740992","9007199254740992","-4611686018427387904","9007199254740995","5","-3","-4611686018427387904","9007199254740993","3","-4611686018427387904","9007199254740995",null,"0",null,"-4611686018427387904","-4611686018427387904","-4611686018427387904"]}},"right":{"fields":[{"name":"l5_i64","type":"i64"},{"name":"r0_bool","type":"bool"}],"columns":{"r0_bool":[],"l5_i64":[]}}},"ops":[{"op":"join","right":"right","on":["l5_i64"],"how":"full"},{"op":"filter","predicate":{"fn":"isIn","args":[{"col":"l0_utf8"}],"values":["b","ab","a"]}},{"op":"withColumns","exprs":{"x2":{"fn":"eq","args":[{"fn":"mod","args":[{"col":"l2_f64"},{"lit":2}]},{"fn":"mod","args":[{"col":"l4_i32"},{"lit":2.5}]}]}}},{"op":"head","n":9}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_f64","type":"f64"},{"name":"l4_i32","type":"i32"},{"name":"l5_i64","type":"i64"},{"name":"r0_bool","type":"bool"},{"name":"x2","type":"bool"}],"columns":{"l0_utf8":["a","a","ab","a","b","ab","b","ab","ab"],"l1_i32":[1,-5,-3,4,-2147483648,-2,-3,0,1],"l2_f64":[4.0,0.0,87.605,4.0,-4.0,-0.294,0.0,"NaN",-92.801],"l3_f64":[3.0,79.674,63.536,-49.961,-3.0,-4.0,-4.0,98.676,-23.7],"l4_i32":[-1,-1,null,4,-2,5,null,872,4],"l5_i64":["9007199254740993","-3","4","-4","-4611686018427387904","4","-4611686018427387904","0","9007199254740994"],"r0_bool":[null,null,null,null,null,null,null,null,null],"x2":[false,false,null,false,false,false,null,false,false]}}},{"id":"random/0185","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_f64","type":"f64"},{"name":"l4_bool","type":"bool"},{"name":"l5_f64","type":"f64"},{"name":"l6_utf8","type":"utf8"}],"columns":{"l0_utf8":["B"],"l1_i32":[1],"l2_utf8":["ab"],"l3_f64":[3.0],"l4_bool":[false],"l5_f64":[15.563],"l6_utf8":["c"]}},"right":{"fields":[{"name":"l2_utf8","type":"utf8"},{"name":"r0_utf8","type":"utf8"}],"columns":{"r0_utf8":["zz","é","a","a",null,null,"c","é","",null],"l2_utf8":["ab","ab","ab","ab","ab","ab","ab","ab","ab","ab"]}}},"ops":[{"op":"join","right":"right","on":["l2_utf8"],"how":"inner"},{"op":"withColumns","exprs":{"x1":{"fn":"abs","args":[{"fn":"sub","args":[{"col":"l3_f64"},{"col":"l1_i32"}]}]}}},{"op":"filter","predicate":{"fn":"eq","args":[{"col":"l6_utf8"},{"lit":"aa"}]}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_f64","type":"f64"},{"name":"l4_bool","type":"bool"},{"name":"l5_f64","type":"f64"},{"name":"l6_utf8","type":"utf8"},{"name":"r0_utf8","type":"utf8"},{"name":"x1","type":"f64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_utf8":[],"l3_f64":[],"l4_bool":[],"l5_f64":[],"l6_utf8":[],"r0_utf8":[],"x1":[]}}},{"id":"random/0186","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_f64","type":"f64"}],"columns":{"l0_utf8":[null,null,null,"B",null,null,null,"ab","B",null,"zz","zz",null,"B",null,"a","😀","ab","😀","ab","é","aa","aa",null,null,null,"é","B","B","B",null,null,"c","é","zz","B","aa","a",null,"ab","c",null,"aa","ab",null,null,"zz",null,"zz","c",null,"zz",null,null,"B","",null,null,null,"ab"],"l1_i32":[5,-2,-4,null,-1,1,5,3,-5,4,2,1,-1,0,-1,-3,null,null,1,-1,2147483647,null,2,null,5,3,null,1,5,null,-1,2,2,0,0,0,-2,4,null,0,3,0,3,0,-4,-2,1,5,null,85,-4,0,null,0,-1,-3,null,3,5,2],"l2_utf8":[null,"a","é",null,"","b","😀","a","a","b",null,"c","b","b","a","zz",null,null,"aa","😀","é","é","c","c","B","aa","aa",null,"ab",null,"","b",null,"B","aa","B","c","B",null,null,"a",null,"é","c","c","😀",null,null,"B","aa",null,"😀","a",null,null,null,null,"zz","é","zz"],"l3_f64":[null,-80.618,46.704,-4.0,4.0,47.307,null,"-0",-96.618,null,null,-3.254,62.393,null,4.0,null,-4.0,91.465,null,-3.0,-87.054,null,-3.0,null,14.261,-1.0,53.966,3.0,null,null,-4.0,-3.0,null,null,-48.155,-20.805,null,null,1.056,null,95.428,14.488,null,null,null,-4.0,92.499,2.0,-1.621,null,4.0,null,-16.197,null,40.93,-82.865,77.192,4.0,-95.184,-1.0]}}},"ops":[{"op":"withColumns","exprs":{"x0":{"fn":"gt","args":[{"fn":"floorDiv","args":[{"col":"l1_i32"},{"lit":2}]},{"col":"l3_f64"}]}}},{"op":"filter","predicate":{"fn":"eq","args":[{"fn":"div","args":[{"col":"l3_f64"},{"col":"l1_i32"}]},{"fn":"div","args":[{"lit":0.5},{"lit":2.5}]}]}},{"op":"filter","predicate":{"fn":"gte","args":[{"fn":"div","args":[{"col":"l3_f64"},{"col":"l1_i32"}]},{"col":"l1_i32"}]}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_f64","type":"f64"},{"name":"x0","type":"bool"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_utf8":[],"l3_f64":[],"x0":[]}}},{"id":"random/0187","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_utf8","type":"utf8"}],"columns":{"l0_utf8":["B","😀","B","","B","😀",null,"é",null,"","aa","é",null,"zz","ab",null,"c","ab","B","é"],"l1_i32":[0,4,null,0,5,-2,-5,-4,5,0,-2,1,2,null,-3,-4,-3,3,-5,1],"l2_f64":[2.0,-26.144,-61.449,2.0,1.0,-3.0,-80.039,-92.818,-54.194,-1.0,4.0,-1.0,3.0,2.0,2.0,38.867,-3.0,null,4.0,-59.41],"l3_utf8":["c","😀","aa",null,"B","aa","","","zz",null,"zz","ab","","zz","","aa","ab","b","a","B"],"l4_utf8":["ab","zz","b","b","a","c","aa","a","😀",null,"ab","zz",null,null,"b","a","c","B","B","aa"]}}},"ops":[{"op":"filter","predicate":{"fn":"lt","args":[{"fn":"fillNull","args":[{"col":"l2_f64"},{"lit":1.5}]},{"col":"l2_f64"}]}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_utf8","type":"utf8"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_f64":[],"l3_utf8":[],"l4_utf8":[]}}},{"id":"random/0188","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_bool","type":"bool"},{"name":"l4_i32","type":"i32"}],"columns":{"l0_utf8":["c"],"l1_i32":[null],"l2_f64":["NaN"],"l3_bool":[null],"l4_i32":[-5]}},"right":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"r0_utf8","type":"utf8"}],"columns":{"r0_utf8":["aa","ab","é"],"l0_utf8":["c","c","c"]}}},"ops":[{"op":"join","right":"right","on":["l0_utf8"],"how":"inner"},{"op":"groupBy","keys":["l0_utf8"],"aggs":{"n1":{"agg":"len"},"a1_0":{"agg":"mean","arg":{"col":"l4_i32"}}}},{"op":"filter","predicate":{"fn":"lt","args":[{"fn":"fillNull","args":[{"col":"a1_0"},{"lit":0}]},{"fn":"mod","args":[{"col":"a1_0"},{"col":"a1_0"}]}]}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"n1","type":"u32"},{"name":"a1_0","type":"f64"}],"columns":{"l0_utf8":["c"],"n1":[3],"a1_0":[-5.0]}}},{"id":"random/0189","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_bool","type":"bool"},{"name":"l4_f64","type":"f64"},{"name":"l5_utf8","type":"utf8"},{"name":"l6_f64","type":"f64"}],"columns":{"l0_utf8":["😀"],"l1_i32":[2],"l2_i32":[-2],"l3_bool":[false],"l4_f64":["-0"],"l5_utf8":["é"],"l6_f64":["NaN"]}}},"ops":[{"op":"head","n":9},{"op":"filter","predicate":{"fn":"and","args":[{"col":"l3_bool"},{"fn":"gt","args":[{"col":"l4_f64"},{"col":"l4_f64"}]}]}},{"op":"sort","keys":[{"expr":{"col":"l0_utf8"},"descending":true},{"expr":{"col":"l6_f64"},"descending":true}],"nullsLast":true},{"op":"withColumns","exprs":{"x3":{"fn":"sub","args":[{"fn":"fillNull","args":[{"col":"l2_i32"},{"lit":1.5}]},{"fn":"neg","args":[{"col":"l1_i32"}]}]}}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_bool","type":"bool"},{"name":"l4_f64","type":"f64"},{"name":"l5_utf8","type":"utf8"},{"name":"l6_f64","type":"f64"},{"name":"x3","type":"f64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_i32":[],"l3_bool":[],"l4_f64":[],"l5_utf8":[],"l6_f64":[],"x3":[]}}},{"id":"random/0190","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_f64","type":"f64"}],"columns":{"l0_utf8":["ab"],"l1_i32":[2],"l2_utf8":["zz"],"l3_f64":[0.0]}}},"ops":[{"op":"unique"}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_f64","type":"f64"}],"columns":{"l0_utf8":["ab"],"l1_i32":[2],"l2_utf8":["zz"],"l3_f64":[0.0]}}},{"id":"random/0191","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_i64","type":"i64"},{"name":"l4_bool","type":"bool"},{"name":"l5_f64","type":"f64"},{"name":"l6_f64","type":"f64"}],"columns":{"l0_utf8":[null,null,"é",null,"b"],"l1_i32":[null,-2,-1,-3,564],"l2_f64":[null,4.0,94.073,null,51.67],"l3_i64":["-4611686018427387904","-4611686018427387904","-3","-4611686018427387904","-4"],"l4_bool":[false,null,false,false,true],"l5_f64":[3.0,31.502,4.0,3.0,null],"l6_f64":[null,null,3.0,1.0,-1.0]}}},"ops":[{"op":"sort","keys":[{"expr":{"col":"l6_f64"},"descending":true}],"nullsLast":false}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_i64","type":"i64"},{"name":"l4_bool","type":"bool"},{"name":"l5_f64","type":"f64"},{"name":"l6_f64","type":"f64"}],"columns":{"l0_utf8":[null,null,"é",null,"b"],"l1_i32":[null,-2,-1,-3,564],"l2_f64":[null,4.0,94.073,null,51.67],"l3_i64":["-4611686018427387904","-4611686018427387904","-3","-4611686018427387904","-4"],"l4_bool":[false,null,false,false,true],"l5_f64":[3.0,31.502,4.0,3.0,null],"l6_f64":[null,null,3.0,1.0,-1.0]}}},{"id":"random/0192","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_f64","type":"f64"},{"name":"l4_bool","type":"bool"},{"name":"l5_bool","type":"bool"},{"name":"l6_i32","type":"i32"}],"columns":{"l0_utf8":["😀","a","ab","aa","😀","","","aa","B","zz","😀","é","ab","é","","a","😀","aa","B","a"],"l1_i32":[5,-4,-4,-2,0,-5,5,2147483647,2,-3,-2,3,3,-2,-4,-3,2,-804,-3,-3],"l2_utf8":["ab","c","b","B","zz","a","B","b","zz","B","😀","B","aa","a","B","é","c","","zz","ab"],"l3_f64":[2.0,"NaN",2.0,4.0,-79.802,-42.574,-16.129,-2.0,2.0,20.741,-1.771,-1.0,-65.11,1.0,-1.0,-22.167,79.233,-2.0,4.0,76.8],"l4_bool":[false,false,true,true,false,true,false,false,true,false,true,false,false,true,false,false,true,false,false,false],"l5_bool":[true,false,true,false,true,false,true,false,false,false,false,false,false,false,true,true,false,false,false,true],"l6_i32":[0,0,5,0,2,1,-1,3,3,1,0,-3,-5,-4,1,-2147483648,-1,0,3,-2147483648]}}},"ops":[{"op":"sort","keys":[{"expr":{"col":"l4_bool"},"descending":true}],"nullsLast":false},{"op":"sort","keys":[{"expr":{"col":"l5_bool"},"descending":false}],"nullsLast":true}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_f64","type":"f64"},{"name":"l4_bool","type":"bool"},{"name":"l5_bool","type":"bool"},{"name":"l6_i32","type":"i32"}],"columns":{"l0_utf8":["aa","","B","😀","é","😀","a","aa","zz","é","ab","aa","B","ab","😀","😀","","","a","a"],"l1_i32":[-2,-5,2,-2,-2,2,-4,2147483647,-3,3,3,-804,-3,-4,5,0,5,-4,-3,-3],"l2_utf8":["B","a","zz","😀","a","c","c","b","B","B","aa","","zz","b","ab","zz","B","B","é","ab"],"l3_f64":[4.0,-42.574,2.0,-1.771,1.0,79.233,"NaN",-2.0,20.741,-1.0,-65.11,-2.0,4.0,2.0,2.0,-79.802,-16.129,-1.0,-22.167,76.8],"l4_bool":[true,true,true,true,true,true,false,false,false,false,false,false,false,true,false,false,false,false,false,false],"l5_bool":[false,false,false,false,false,false,false,false,false,false,false,false,false,true,true,true,true,true,true,true],"l6_i32":[0,1,3,0,-4,-1,0,3,1,-3,-5,0,3,5,0,2,-1,1,-2147483648,-2147483648]}}},{"id":"random/0193","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_i64","type":"i64"},{"name":"l4_bool","type":"bool"}],"columns":{"l0_utf8":["😀","😀","a","c","c"],"l1_i32":[-4,-3,-4,-2,2147483647],"l2_bool":[true,false,false,false,false],"l3_i64":["0","9007199254740995","-4611686018427387904",null,"9007199254740995"],"l4_bool":[false,true,false,false,false]}}},"ops":[{"op":"unique","columns":["l3_i64"]}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_i64","type":"i64"},{"name":"l4_bool","type":"bool"}],"columns":{"l0_utf8":["😀","😀","a","c"],"l1_i32":[-4,-3,-4,-2],"l2_bool":[true,false,false,false],"l3_i64":["0","9007199254740995","-4611686018427387904",null],"l4_bool":[false,true,false,false]}}},{"id":"random/0194","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i64","type":"i64"},{"name":"l3_f64","type":"f64"},{"name":"l4_f64","type":"f64"},{"name":"l5_f64","type":"f64"},{"name":"l6_f64","type":"f64"}],"columns":{"l0_utf8":["é"],"l1_i32":[1],"l2_i64":["-4611686018427387904"],"l3_f64":[-4.0],"l4_f64":[-67.348],"l5_f64":[3.0],"l6_f64":[2.0]}},"right":{"fields":[{"name":"l1_i32","type":"i32"},{"name":"r0_bool","type":"bool"}],"columns":{"r0_bool":[],"l1_i32":[]}}},"ops":[{"op":"withColumns","exprs":{"x0":{"fn":"fillNull","args":[{"fn":"neg","args":[{"col":"l5_f64"}]},{"lit":0}]}}},{"op":"join","right":"right","on":["l1_i32"],"how":"left"}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i64","type":"i64"},{"name":"l3_f64","type":"f64"},{"name":"l4_f64","type":"f64"},{"name":"l5_f64","type":"f64"},{"name":"l6_f64","type":"f64"},{"name":"x0","type":"f64"},{"name":"r0_bool","type":"bool"}],"columns":{"l0_utf8":["é"],"l1_i32":[1],"l2_i64":["-4611686018427387904"],"l3_f64":[-4.0],"l4_f64":[-67.348],"l5_f64":[3.0],"l6_f64":[2.0],"x0":[-3.0],"r0_bool":[null]}}},{"id":"random/0195","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i64","type":"i64"},{"name":"l3_f64","type":"f64"},{"name":"l4_utf8","type":"utf8"}],"columns":{"l0_utf8":["é",null,"aa","c",null,null,"ab","aa",null,null,"","c","aa","c","",null,null,"b",null,"B","😀","é","b",null,"é","zz","😀","ab","zz","zz","","a","c","","c","B","ab","b",null,"c","a","😀",null,"😀","zz","zz",null,null,"aa",null,"ab","é",null,null,"b","",null,"","b","😀"],"l1_i32":[1,-4,null,null,-2,null,420,4,-1,null,0,null,-3,5,-3,-740,null,-4,-4,2,3,null,0,-2,-2,null,-2147483648,null,-1,null,null,-5,1,-3,2147483647,3,-1,-3,0,null,3,1,-1,null,1,-1,-2,-5,-5,-2,5,-1,-2,4,null,null,4,-3,-2147483648,-2],"l2_i64":["-1",null,"-4611686018427387904","-4","-4611686018427387904",null,"-4611686018427387904",null,null,null,"-3","9007199254740994",null,"-4","9007199254740992","9007199254740993","9007199254740994","-4611686018427387904","-4611686018427387904","-2","9007199254740992","9007199254740993",null,"-4611686018427387904","1","9007199254740995",null,null,"9007199254740995",null,"-4611686018427387904","9007199254740995","-4611686018427387904","-1",null,null,null,"9007199254740992","9007199254740993","4",null,"9007199254740994","9007199254740994","9007199254740995","5","-4611686018427387904",null,null,"9007199254740994","1","-4611686018427387904",null,null,"9007199254740994","9007199254740992","9007199254740995","-4611686018427387904","-4611686018427387904","-4611686018427387904",null],"l3_f64":[null,1.0,-1.0,2.0,23.609,-14.493,4.0,"-0","NaN",-77.55,4.0,-1.0,0.684,null,null,null,null,11.974,4.0,null,null,1.0,86.202,-32.586,null,null,0.0,"-0",null,null,null,3.0,0.0,null,null,null,-3.0,2.0,-43.209,null,null,null,null,-3.0,null,-18.291,"NaN",-54.214,70.865,-1.0,null,null,4.0,2.0,4.0,-2.0,1.0,-27.053,2.0,null],"l4_utf8":["b","aa","B","a","b",null,null,null,"b","B",null,"a",null,null,"aa","zz",null,"é","aa","a","😀",null,"aa","aa",null,null,"😀","a","ab","c","é","😀","c",null,null,"aa","a",null,null,null,null,"😀","aa","b","aa","aa","zz",null,"B","","b","B","é","ab","ab",null,null,"é",null,null]}}},"ops":[{"op":"filter","predicate":{"fn":"and","args":[{"fn":"lte","args":[{"lit":0.5},{"col":"l1_i32"}]},{"fn":"isIn","args":[{"col":"l0_utf8"}],"values":["a","aa","c"]}]}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i64","type":"i64"},{"name":"l3_f64","type":"f64"},{"name":"l4_utf8","type":"utf8"}],"columns":{"l0_utf8":["aa","c","c","c","a"],"l1_i32":[4,5,1,2147483647,3],"l2_i64":[null,"-4","-4611686018427387904",null,null],"l3_f64":["-0",null,0.0,null,null],"l4_utf8":[null,null,"c",null,null]}}},{"id":"random/0196","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_f64","type":"f64"},{"name":"l5_f64","type":"f64"},{"name":"l6_f64","type":"f64"}],"columns":{"l0_utf8":["a","é","😀","é","aa"],"l1_i32":[null,null,-1,-3,2],"l2_bool":[true,false,false,false,true],"l3_utf8":["é","b","ab","b","ab"],"l4_f64":[null,null,null,null,0.0],"l5_f64":[53.478,null,-25.778,1.0,65.7],"l6_f64":[2.0,1.0,-43.169,"-0","-0"]}}},"ops":[{"op":"sort","keys":[{"expr":{"col":"l6_f64"},"descending":true},{"expr":{"col":"l2_bool"},"descending":true}],"nullsLast":false},{"op":"withColumns","exprs":{"x1":{"lit":-1}}},{"op":"unique","columns":["l6_f64"]},{"op":"head","n":7}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_f64","type":"f64"},{"name":"l5_f64","type":"f64"},{"name":"l6_f64","type":"f64"},{"name":"x1","type":"i32"}],"columns":{"l0_utf8":["a","é","aa","😀"],"l1_i32":[null,null,2,-1],"l2_bool":[true,false,true,false],"l3_utf8":["é","b","ab","ab"],"l4_f64":[null,null,0.0,null],"l5_f64":[53.478,null,65.7,-25.778],"l6_f64":[2.0,1.0,"-0",-43.169],"x1":[-1,-1,-1,-1]}}},{"id":"random/0197","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_i32","type":"i32"}],"columns":{"l0_utf8":[null,"aa","ab","b","","","😀","c","aa","aa","ab","a","","zz","B","é","é","a","a","😀","","😀","é","é","ab","é","","ab","é","zz","zz","aa","","",null,"b","a","c","aa","ab","é","b","ab","B","",null,"c","b","aa","zz","","c","c","zz","ab","aa","c","zz","a","zz"],"l1_i32":[2,-2,5,null,-2147483648,2,null,-5,null,3,-1,5,3,4,-4,-4,-5,2,-4,2147483647,1,1,-3,4,null,4,null,-5,0,-5,-1,-4,2147483647,-2,null,2,5,2,-671,null,2,null,-2,1,4,-2147483648,0,4,-1,5,3,-2,-5,1,1,4,0,-2147483648,-2,-4],"l2_f64":[1.0,2.0,-85.911,1.0,47.608,21.428,null,null,-27.594,40.902,1.0,-95.911,-95.194,88.774,-4.0,32.199,1.0,-30.514,-15.871,10.805,42.693,null,43.274,4.0,-96.566,94.018,3.0,4.0,-3.0,-4.0,"NaN",-2.0,-61.991,4.0,2.0,-3.0,71.736,-1.0,null,2.0,-1.0,-27.609,3.0,-1.0,-53.676,-91.928,2.0,38.243,2.0,4.0,"-Infinity",82.87,null,0.0,-87.721,96.855,93.878,-3.0,29.723,null],"l3_utf8":["a","b","b","c","b","c","é","😀","c","B","B","",null,"B",null,"aa","B","ab",null,"aa","b","ab","","aa",null,"B","c","a","b","a","zz","B","zz","😀","😀","aa","😀","aa","a","é","B","a","zz","é","a","zz","b","","ab",null,"aa","B","B","c","","c","é","","😀","c"],"l4_i32":[2147483647,4,-3,2147483647,5,5,-1,1,-3,3,null,-2,3,-1,5,-2,-1,-1,null,-1,3,0,3,-4,-4,-2,2,-2,-3,-1,-2,-8,null,280,-5,1,0,-4,0,1,-1,null,2147483647,4,5,-5,5,2147483647,2,5,3,4,-5,-5,-3,3,-2,5,-2,5]}}},"ops":[{"op":"sort","keys":[{"expr":{"col":"l4_i32"},"descending":false},{"expr":{"col":"l2_f64"},"descending":true}],"nullsLast":true},{"op":"sort","keys":[{"expr":{"col":"l0_utf8"},"descending":true},{"expr":{"col":"l2_f64"},"descending":false}],"nullsLast":false},{"op":"filter","predicate":{"fn":"and","args":[{"fn":"isNotNull","args":[{"col":"l0_utf8"}]},{"fn":"eq","args":[{"col":"l1_i32"},{"col":"l1_i32"}]}]}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_i32","type":"i32"}],"columns":{"l0_utf8":["😀","😀","é","é","é","é","é","é","é","zz","zz","zz","zz","zz","zz","zz","c","c","c","c","c","c","b","b","ab","ab","ab","ab","ab","aa","aa","aa","aa","aa","aa","a","a","a","a","a","B","B","","","","","","","",""],"l1_i32":[1,2147483647,0,2,-5,4,-4,-3,4,-4,-5,-2147483648,1,5,4,-1,-5,-5,2,0,-2,0,2,4,1,5,-1,-2,-5,-671,-4,-1,-2,3,4,5,2,-4,-2,5,-4,1,3,3,2147483647,4,-2,2,1,-2147483648],"l2_f64":[null,10.805,-3.0,-1.0,1.0,4.0,32.199,43.274,94.018,null,-4.0,-3.0,0.0,4.0,88.774,"NaN",null,null,-1.0,2.0,82.87,93.878,-3.0,38.243,-87.721,-85.911,1.0,3.0,4.0,null,-2.0,2.0,2.0,40.902,96.855,-95.911,-30.514,-15.871,29.723,71.736,-4.0,-1.0,"-Infinity",-95.194,-61.991,-53.676,4.0,21.428,42.693,47.608],"l3_utf8":["ab","aa","b","B","B","aa","aa","","B","c","a","","c",null,"B","zz","B","😀","aa","b","B","é","aa","","","b","B","zz","a","a","B","ab","b","B","c","","ab",null,"😀","😀",null,"é","aa",null,"zz","a","😀","c","b","b"],"l4_i32":[0,-1,-3,-1,-1,-4,-2,3,-2,5,-1,5,-5,5,-1,-2,-5,1,-4,5,4,-2,1,2147483647,-3,-3,null,2147483647,-2,0,-8,2,4,3,3,-2,-1,null,-2,0,5,4,3,3,null,5,280,5,3,5]}}},{"id":"random/0198","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_utf8","type":"utf8"},{"name":"l6_utf8","type":"utf8"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_bool":[],"l3_utf8":[],"l4_utf8":[],"l5_utf8":[],"l6_utf8":[]}}},"ops":[{"op":"filter","predicate":{"col":"l2_bool"}},{"op":"groupBy","keys":["l5_utf8"],"aggs":{"n1":{"agg":"len"},"a1_0":{"agg":"std","arg":{"col":"l1_i32"}},"a1_1":{"agg":"first","arg":{"col":"l4_utf8"}}}},{"op":"unique"},{"op":"withColumns","exprs":{"x3":{"fn":"mul","args":[{"fn":"add","args":[{"col":"a1_0"},{"col":"a1_0"}]},{"fn":"sub","args":[{"col":"a1_0"},{"lit":2}]}]}}}],"expected":{"fields":[{"name":"l5_utf8","type":"utf8"},{"name":"n1","type":"u32"},{"name":"a1_0","type":"f64"},{"name":"a1_1","type":"utf8"},{"name":"x3","type":"f64"}],"columns":{"l5_utf8":[],"n1":[],"a1_0":[],"a1_1":[],"x3":[]}}},{"id":"random/0199","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_utf8","type":"utf8"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_i32":[],"l3_utf8":[]}},"right":{"fields":[{"name":"l2_i32","type":"i32"},{"name":"r0_utf8","type":"utf8"}],"columns":{"r0_utf8":["é","😀","c","B","b",null,"b","a","aa","😀","zz","ab","a",null,"B","é","é","ab","","é","a",null,"a","b","b","é","é","b","","a"],"l2_i32":[-4,1,null,null,null,-4,-4,-4,null,4,-4,-4,-4,null,null,-4,null,null,null,4,4,1,-4,4,-4,null,-4,-3,1,null]}}},"ops":[{"op":"head","n":10},{"op":"join","right":"right","on":["l2_i32"],"how":"left"},{"op":"filter","predicate":{"fn":"isNull","args":[{"col":"l2_i32"}]}},{"op":"filter","predicate":{"fn":"eq","args":[{"fn":"mul","args":[{"col":"l2_i32"},{"col":"l2_i32"}]},{"fn":"neg","args":[{"col":"l2_i32"}]}]}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_utf8","type":"utf8"},{"name":"r0_utf8","type":"utf8"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_i32":[],"l3_utf8":[],"r0_utf8":[]}}},{"id":"random/0200","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_bool","type":"bool"},{"name":"l4_bool","type":"bool"},{"name":"l5_utf8","type":"utf8"},{"name":"l6_bool","type":"bool"}],"columns":{"l0_utf8":["😀"],"l1_i32":[1],"l2_f64":[-2.0],"l3_bool":[true],"l4_bool":[false],"l5_utf8":["a"],"l6_bool":[null]}}},"ops":[{"op":"unique","columns":["l0_utf8"]},{"op":"withColumns","exprs":{"x1":{"lit":2}}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_bool","type":"bool"},{"name":"l4_bool","type":"bool"},{"name":"l5_utf8","type":"utf8"},{"name":"l6_bool","type":"bool"},{"name":"x1","type":"i32"}],"columns":{"l0_utf8":["😀"],"l1_i32":[1],"l2_f64":[-2.0],"l3_bool":[true],"l4_bool":[false],"l5_utf8":["a"],"l6_bool":[null],"x1":[2]}}},{"id":"random/0201","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i64","type":"i64"},{"name":"l3_f64","type":"f64"},{"name":"l4_f64","type":"f64"}],"columns":{"l0_utf8":["é","zz","é","b","aa",null,"ab","😀","c",null,"😀","ab","a","a",null,"b",null,"aa","a","😀",null,"zz",null,"😀","","😀","ab","B","é","a","ab","b","aa",null,"é",null,"",null,"ab","😀",null,null,"zz",null,"ab",null,"B","B","","c",null,null,null,null,null,"ab",null,"","ab",""],"l1_i32":[null,-5,null,-5,null,-1,2,null,null,null,null,null,null,-5,-1,-4,-4,3,-2,-2147483648,-3,null,null,0,null,1,null,null,null,1,-5,0,-4,-3,-4,null,0,null,0,null,1,null,-3,-1,3,-2,null,-3,-1,null,0,623,5,2147483647,null,0,null,2147483647,-5,-1],"l2_i64":["-4611686018427387904","-4611686018427387904","-1","-4611686018427387904","-4611686018427387904","9007199254740995","9007199254740993",null,"9007199254740995","-4611686018427387904","4","9007199254740995","-4611686018427387904","9007199254740995",null,"9007199254740995","9007199254740995",null,null,"9007199254740992","-4611686018427387904","9007199254740994","-4611686018427387904","9007199254740992","-4611686018427387904","-4611686018427387904",null,"-5","-4611686018427387904",null,"9007199254740992",null,null,"-4611686018427387904",null,"-2","-4611686018427387904","0","-4","9007199254740992",null,"9007199254740994","-4611686018427387904",null,null,"9007199254740994","-2","-4611686018427387904","-5","1",null,"-4611686018427387904",null,"1","9007199254740994",null,"-2",null,null,"-4611686018427387904"],"l3_f64":[97.832,-33.702,61.607,-4.0,-0.679,-90.259,-3.0,47.6,53.769,null,4.0,26.312,3.0,4.0,24.95,0.0,null,null,-2.0,-2.0,null,null,-27.435,null,3.0,null,null,-5.829,null,-89.209,null,"Infinity",null,1.0,null,-84.841,51.677,-2.0,null,-48.557,-79.214,3.0,93.112,null,-64.79,null,"NaN",null,null,-42.789,93.731,33.596,null,-3.0,2.0,-3.0,-52.018,4.0,33.687,null],"l4_f64":[-43.244,-1.0,null,4.0,null,null,-3.376,74.328,0.0,null,null,-33.249,2.0,null,null,null,null,"NaN",4.0,53.329,80.045,-4.0,4.0,92.42,-4.0,24.234,14.611,-3.0,-3.0,null,-4.0,null,7.815,33.421,null,0.0,null,-16.38,-3.0,62.388,-1.0,0.0,null,null,65.676,null,null,-2.0,25.51,null,-52.689,43.536,null,69.697,null,null,86.29,-23.279,null,1.0]}}},"ops":[{"op":"filter","predicate":{"fn":"lt","args":[{"fn":"floorDiv","args":[{"col":"l3_f64"},{"col":"l3_f64"}]},{"fn":"abs","args":[{"col":"l1_i32"}]}]}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i64","type":"i64"},{"name":"l3_f64","type":"f64"},{"name":"l4_f64","type":"f64"}],"columns":{"l0_utf8":["zz","b","ab","a","a",null,"zz","ab",null,null,"","ab"],"l1_i32":[-5,-5,2,-5,-2,-3,-3,3,623,2147483647,2147483647,-5],"l2_i64":["-4611686018427387904","-4611686018427387904","9007199254740993","9007199254740995",null,"-4611686018427387904","-4611686018427387904",null,"-4611686018427387904","1",null,null],"l3_f64":[-33.702,-4.0,-3.0,4.0,-2.0,1.0,93.112,-64.79,33.596,-3.0,4.0,33.687],"l4_f64":[-1.0,4.0,-3.376,null,4.0,33.421,null,65.676,43.536,69.697,-23.279,null]}}},{"id":"random/0202","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_i64","type":"i64"},{"name":"l4_f64","type":"f64"},{"name":"l5_i64","type":"i64"},{"name":"l6_utf8","type":"utf8"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_bool":[],"l3_i64":[],"l4_f64":[],"l5_i64":[],"l6_utf8":[]}}},"ops":[{"op":"groupBy","keys":["l2_bool"],"aggs":{"n0":{"agg":"len"},"a0_0":{"agg":"mean","arg":{"col":"l1_i32"}}}}],"expected":{"fields":[{"name":"l2_bool","type":"bool"},{"name":"n0","type":"u32"},{"name":"a0_0","type":"f64"}],"columns":{"l2_bool":[],"n0":[],"a0_0":[]}}},{"id":"random/0203","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_i32","type":"i32"},{"name":"l4_i64","type":"i64"},{"name":"l5_i64","type":"i64"},{"name":"l6_bool","type":"bool"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_f64":[],"l3_i32":[],"l4_i64":[],"l5_i64":[],"l6_bool":[]}},"right":{"fields":[{"name":"l4_i64","type":"i64"},{"name":"r0_utf8","type":"utf8"}],"columns":{"r0_utf8":["é","B","ab","ab","a","aa","zz","é","a","a"],"l4_i64":["4","9007199254740995","9007199254740995","9007199254740995","9007199254740995","9007199254740995","9007199254740992","9007199254740995","1","1"]}}},"ops":[{"op":"filter","predicate":{"fn":"lt","args":[{"fn":"div","args":[{"col":"l3_i32"},{"lit":0.5}]},{"lit":2.5}]}},{"op":"unique"},{"op":"join","right":"right","on":["l4_i64"],"how":"semi"}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_i32","type":"i32"},{"name":"l4_i64","type":"i64"},{"name":"l5_i64","type":"i64"},{"name":"l6_bool","type":"bool"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_f64":[],"l3_i32":[],"l4_i64":[],"l5_i64":[],"l6_bool":[]}}},{"id":"random/0204","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_bool","type":"bool"},{"name":"l5_utf8","type":"utf8"}],"columns":{"l0_utf8":["aa","c","b",null,"c","😀","é",null,null,"b",null,null,"😀","aa",null,"B","zz",null,"aa",null,"B","zz","😀","aa","é","b","","","😀","ab",null,"b","b","ab",null,"é",null,"😀",null,null,"c","zz",null,"b",null,null,null,"😀","é","aa","😀","b","","ab","a","",null,"😀",null,"c"],"l1_i32":[null,-3,null,-4,1,-5,1,-5,2,4,null,5,-3,4,-3,-4,0,null,4,-5,-3,1,4,-2,0,-5,4,3,5,-4,null,null,1,0,-4,null,1,0,-1,-5,-4,null,-5,-4,-2,2,5,-3,-1,1,null,2147483647,null,null,4,null,-3,3,null,null],"l2_i32":[5,-5,-4,3,null,-3,1,0,0,4,null,-4,3,-3,-2,4,4,459,null,4,null,4,5,-5,-4,-2,2,-710,null,null,5,null,-2,2,1,2,null,null,2,-2147483648,-4,2,-4,2,-2,5,2,3,5,2147483647,-4,0,null,1,4,-4,0,-1,3,-2],"l3_utf8":[null,"é","😀","b",null,"",null,"c","é","",null,"aa",null,"b","B","b","é","B","","ab",null,"","B","","a","B","aa","ab","😀","c","b",null,"","","é","","ab",null,"😀","aa",null,"B","zz","c","ab","c","","aa",null,"",null,null,null,"😀","aa","b","ab",null,"b",null],"l4_bool":[null,true,null,false,null,true,null,true,null,false,null,true,true,false,true,true,null,true,false,true,false,true,false,true,false,false,true,true,null,null,false,false,true,true,null,true,null,false,null,null,true,null,true,false,null,false,false,null,true,false,true,null,true,false,true,null,true,true,true,true],"l5_utf8":["a","ab","c",null,"b",null,"","zz","é","","é","ab","",null,null,"é","a","","😀","aa","B","b","","","B","😀",null,"ab","b","ab","b",null,"aa","","aa",null,"é","😀","c","B",null,"B",null,"😀",null,"ab","b","😀",null,null,"a","ab",null,null,"c","b",null,"a","B","b"]}}},"ops":[{"op":"head","n":0}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_bool","type":"bool"},{"name":"l5_utf8","type":"utf8"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_i32":[],"l3_utf8":[],"l4_bool":[],"l5_utf8":[]}}},{"id":"random/0205","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i64","type":"i64"},{"name":"l3_f64","type":"f64"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_i64","type":"i64"},{"name":"l6_utf8","type":"utf8"}],"columns":{"l0_utf8":["c","ab","c","zz","zz"],"l1_i32":[0,1,-4,0,-2147483648],"l2_i64":["9007199254740992","-4611686018427387904","-1",null,"-4611686018427387904"],"l3_f64":[0.558,-43.336,-57.192,null,-3.0],"l4_utf8":["a","B","ab","é","c"],"l5_i64":["0","9007199254740994","2","-1","-4611686018427387904"],"l6_utf8":["zz","a","zz",null,null]}}},"ops":[{"op":"withColumns","exprs":{"x0":{"fn":"lte","args":[{"fn":"div","args":[{"col":"l1_i32"},{"col":"l3_f64"}]},{"fn":"mod","args":[{"col":"l1_i32"},{"col":"l3_f64"}]}]}}},{"op":"unique"}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i64","type":"i64"},{"name":"l3_f64","type":"f64"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_i64","type":"i64"},{"name":"l6_utf8","type":"utf8"},{"name":"x0","type":"bool"}],"columns":{"l0_utf8":["c","ab","c","zz","zz"],"l1_i32":[0,1,-4,0,-2147483648],"l2_i64":["9007199254740992","-4611686018427387904","-1",null,"-4611686018427387904"],"l3_f64":[0.558,-43.336,-57.192,null,-3.0],"l4_utf8":["a","B","ab","é","c"],"l5_i64":["0","9007199254740994","2","-1","-4611686018427387904"],"l6_utf8":["zz","a","zz",null,null],"x0":[true,false,false,null,false]}}},{"id":"random/0206","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_f64","type":"f64"},{"name":"l4_bool","type":"bool"},{"name":"l5_f64","type":"f64"},{"name":"l6_bool","type":"bool"}],"columns":{"l0_utf8":["zz","b","a","zz","zz","B","","aa","a","B","😀","a","zz","B","B","","ab","b","ab","","é","a","c","ab","b","ab","a","a","ab","é","c","aa","😀","é","c","","aa","ab","é","aa","c","B","","zz","aa","c","a","a","é","c","é","aa","B","ab","","b","a","é","ab","zz"],"l1_i32":[-3,-1,-1,1,-1,2,-5,25,3,4,-5,0,3,0,1,-2147483648,-4,-1,1,5,4,3,-1,0,-5,-5,1,1,-5,-5,5,-1,-5,-3,-1,4,2,-2,3,4,12,5,3,2,-2,4,0,3,2,2,-4,2,-5,-4,1,-2,0,1,5,-3],"l2_f64":[1.0,38.593,-63.191,-99.704,0.0,-78.5,3.0,-2.0,-2.0,-4.0,-96.736,58.778,37.631,1.0,-93.254,1.0,-0.868,-0.826,0.0,1.0,-3.0,21.399,"NaN",2.0,-1.0,48.38,-70.187,0.0,85.86,"-Infinity",67.757,-7.196,13.937,"NaN",50.461,73.241,-81.186,-3.0,-4.0,-1.867,-67.255,71.539,-79.551,45.863,15.527,0.0,78.287,"-Infinity","NaN",-2.0,-16.858,-23.292,11.734,-21.906,2.0,4.0,-36.747,4.0,-1.0,0.0],"l3_f64":[68.252,1.0,-4.0,-3.0,39.731,-8.76,17.226,-2.0,-82.681,-1.0,5.043,-6.784,68.285,49.183,-85.1,-39.214,2.0,37.264,-37.637,0.0,0.0,52.363,-18.341,-1.0,23.267,-47.354,1.0,2.0,-2.0,-4.0,-4.0,-3.0,24.086,-62.928,1.0,-60.783,89.114,-69.633,-1.0,-2.0,96.015,-65.122,-4.784,-4.0,97.493,3.0,-4.0,"NaN",74.193,"-Infinity",0.0,59.531,-3.0,0.513,"NaN",71.304,"NaN",-2.0,1.0,-1.0],"l4_bool":[false,false,true,false,true,false,false,false,true,true,true,true,false,true,false,false,true,false,true,false,true,false,false,true,false,false,false,false,false,false,true,false,true,true,true,false,true,true,true,false,false,false,false,false,false,false,true,true,false,true,false,false,true,false,true,false,false,true,true,true],"l5_f64":["Infinity",-2.0,-43.573,4.0,70.412,-2.0,1.0,2.0,-9.775,-81.927,-2.0,2.0,-4.0,3.0,-79.73,-42.157,4.0,"NaN",-30.62,0.0,-90.737,54.892,-76.408,17.278,-40.126,-4.0,60.679,1.0,-2.0,-1.0,60.671,-1.0,15.392,-9.168,87.451,-21.446,2.0,-1.0,4.0,-75.187,54.033,-37.172,-99.921,-20.228,1.0,-98.529,3.0,2.0,1.0,-4.0,-3.0,-75.127,-30.512,43.807,20.774,0.0,9.318,11.938,-67.214,-56.725],"l6_bool":[false,true,false,false,true,true,false,false,false,false,false,false,true,false,true,false,false,true,true,false,true,false,true,true,false,true,true,false,false,false,false,true,false,false,true,false,true,false,true,false,false,false,true,false,false,true,true,false,true,true,true,false,true,true,true,true,true,true,false,true]}}},"ops":[{"op":"head","n":7},{"op":"head","n":7}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_f64","type":"f64"},{"name":"l4_bool","type":"bool"},{"name":"l5_f64","type":"f64"},{"name":"l6_bool","type":"bool"}],"columns":{"l0_utf8":["zz","b","a","zz","zz","B",""],"l1_i32":[-3,-1,-1,1,-1,2,-5],"l2_f64":[1.0,38.593,-63.191,-99.704,0.0,-78.5,3.0],"l3_f64":[68.252,1.0,-4.0,-3.0,39.731,-8.76,17.226],"l4_bool":[false,false,true,false,true,false,false],"l5_f64":["Infinity",-2.0,-43.573,4.0,70.412,-2.0,1.0],"l6_bool":[false,true,false,false,true,true,false]}}},{"id":"random/0207","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_i64","type":"i64"},{"name":"l4_i64","type":"i64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_f64":[],"l3_i64":[],"l4_i64":[]}}},"ops":[{"op":"sort","keys":[{"expr":{"col":"l0_utf8"},"descending":true}],"nullsLast":true},{"op":"sort","keys":[{"expr":{"col":"l0_utf8"},"descending":false}],"nullsLast":false},{"op":"filter","predicate":{"fn":"isNull","args":[{"col":"l1_i32"}]}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_i64","type":"i64"},{"name":"l4_i64","type":"i64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_f64":[],"l3_i64":[],"l4_i64":[]}}},{"id":"random/0208","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_i32","type":"i32"},{"name":"l4_i64","type":"i64"},{"name":"l5_bool","type":"bool"}],"columns":{"l0_utf8":["c"],"l1_i32":[-1],"l2_i32":[4],"l3_i32":[3],"l4_i64":["9007199254740995"],"l5_bool":[true]}},"right":{"fields":[{"name":"l4_i64","type":"i64"},{"name":"r0_utf8","type":"utf8"}],"columns":{"r0_utf8":["zz","","B","ab","aa","ab","ab","ab","a","c"],"l4_i64":["9007199254740995","9007199254740995","9007199254740995","9007199254740995","9007199254740995","9007199254740995","9007199254740995","9007199254740995","9007199254740995","9007199254740995"]}}},"ops":[{"op":"join","right":"right","on":["l4_i64"],"how":"anti"},{"op":"groupBy","keys":["l3_i32"],"aggs":{"n1":{"agg":"len"},"a1_0":{"agg":"count","arg":{"col":"l4_i64"}},"a1_1":{"agg":"min","arg":{"col":"l2_i32"}}}},{"op":"sort","keys":[{"expr":{"col":"n1"},"descending":true}],"nullsLast":true}],"expected":{"fields":[{"name":"l3_i32","type":"i32"},{"name":"n1","type":"u32"},{"name":"a1_0","type":"u32"},{"name":"a1_1","type":"i32"}],"columns":{"l3_i32":[],"n1":[],"a1_0":[],"a1_1":[]}}},{"id":"random/0209","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i64","type":"i64"},{"name":"l3_f64","type":"f64"},{"name":"l4_i64","type":"i64"},{"name":"l5_i32","type":"i32"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_i64":[],"l3_f64":[],"l4_i64":[],"l5_i32":[]}}},"ops":[{"op":"filter","predicate":{"fn":"and","args":[{"fn":"and","args":[{"fn":"lt","args":[{"col":"l3_f64"},{"col":"l1_i32"}]},{"fn":"eq","args":[{"col":"l1_i32"},{"col":"l1_i32"}]}]},{"fn":"lte","args":[{"col":"l5_i32"},{"col":"l5_i32"}]}]}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i64","type":"i64"},{"name":"l3_f64","type":"f64"},{"name":"l4_i64","type":"i64"},{"name":"l5_i32","type":"i32"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_i64":[],"l3_f64":[],"l4_i64":[],"l5_i32":[]}}},{"id":"random/0210","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_i64","type":"i64"}],"columns":{"l0_utf8":["","ab",null,"b",null,"","c","c","ab","zz",null,"c","é","😀","😀","ab","","a","zz","c"],"l1_i32":[3,2,-1,-4,1,-94,null,-5,3,-2147483648,-4,4,5,-4,-5,0,3,2,0,3],"l2_i32":[5,0,3,-3,3,-5,2,5,4,1,2,4,4,-3,5,4,-5,null,-4,-5],"l3_i64":["9007199254740994","9007199254740993","4","-3","-4611686018427387904","3","4","-4611686018427387904","2","-4611686018427387904","-2","3","1","-4611686018427387904",null,null,"9007199254740994","-4611686018427387904","9007199254740995","9007199254740994"]}}},"ops":[{"op":"unique"},{"op":"filter","predicate":{"fn":"lt","args":[{"col":"l0_utf8"},{"lit":"b"}]}},{"op":"groupBy","keys":["l3_i64"],"aggs":{"n2":{"agg":"len"},"a2_0":{"agg":"nUnique","arg":{"col":"l1_i32"}}}}],"expected":{"fields":[{"name":"l3_i64","type":"i64"},{"name":"n2","type":"u32"},{"name":"a2_0","type":"u32"}],"columns":{"l3_i64":["9007199254740994","9007199254740993","3","2",null,"-4611686018427387904"],"n2":[2,1,1,1,1,1],"a2_0":[1,1,1,1,1,1]}}},{"id":"random/0211","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_i64","type":"i64"}],"columns":{"l0_utf8":["B","é","a","a","aa"],"l1_i32":[2,3,-4,-4,null],"l2_f64":[-0.071,4.836,-73.244,10.929,-16.182],"l3_i64":["9007199254740995","0","-4611686018427387904","9007199254740992","-4611686018427387904"]}}},"ops":[{"op":"head","n":5},{"op":"filter","predicate":{"fn":"and","args":[{"fn":"isIn","args":[{"col":"l0_utf8"}],"values":["B","c","zz"]},{"fn":"isNotNull","args":[{"col":"l1_i32"}]}]}},{"op":"groupBy","keys":["l3_i64"],"aggs":{"n2":{"agg":"len"},"a2_0":{"agg":"std","arg":{"col":"l2_f64"}},"a2_1":{"agg":"mean","arg":{"col":"l1_i32"}}}}],"expected":{"fields":[{"name":"l3_i64","type":"i64"},{"name":"n2","type":"u32"},{"name":"a2_0","type":"f64"},{"name":"a2_1","type":"f64"}],"columns":{"l3_i64":["9007199254740995"],"n2":[1],"a2_0":[null],"a2_1":[2.0]}}},{"id":"random/0212","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_i64","type":"i64"},{"name":"l4_i32","type":"i32"},{"name":"l5_i64","type":"i64"},{"name":"l6_f64","type":"f64"}],"columns":{"l0_utf8":["😀"],"l1_i32":[-4],"l2_utf8":["😀"],"l3_i64":["5"],"l4_i32":[-5],"l5_i64":["3"],"l6_f64":[-4.0]}},"right":{"fields":[{"name":"l1_i32","type":"i32"},{"name":"r0_utf8","type":"utf8"}],"columns":{"r0_utf8":["a",null,"zz"],"l1_i32":[-4,-4,-4]}}},"ops":[{"op":"join","right":"right","on":["l1_i32"],"how":"full"},{"op":"sort","keys":[{"expr":{"col":"l0_utf8"},"descending":true},{"expr":{"col":"l4_i32"},"descending":false}],"nullsLast":false}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_i64","type":"i64"},{"name":"l4_i32","type":"i32"},{"name":"l5_i64","type":"i64"},{"name":"l6_f64","type":"f64"},{"name":"r0_utf8","type":"utf8"}],"columns":{"l0_utf8":["😀","😀","😀"],"l1_i32":[-4,-4,-4],"l2_utf8":["😀","😀","😀"],"l3_i64":["5","5","5"],"l4_i32":[-5,-5,-5],"l5_i64":["3","3","3"],"l6_f64":[-4.0,-4.0,-4.0],"r0_utf8":["a",null,"zz"]}}},{"id":"random/0213","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_f64","type":"f64"},{"name":"l4_f64","type":"f64"},{"name":"l5_f64","type":"f64"},{"name":"l6_f64","type":"f64"}],"columns":{"l0_utf8":["😀",null,"a",null,null,"","a","c",null,"a","é","c",null,null,null,"b",null,"c","aa","😀",null,"é","",null,null,null,"c","aa","b","é",null,"é","é",null,null,"a",null,"c","😀",null,"aa","ab",null,"b",null,"ab","é","c","aa","c","b",null,null,null,null,"","B",null,"b",null],"l1_i32":[4,-2147483648,null,2,-1,308,null,-4,1,-4,5,null,null,-4,5,null,4,2,-5,5,1,-2,-3,3,-1,2,-1,-2,-5,0,null,0,null,null,5,null,5,4,1,-4,null,-3,5,-3,-2,2,2,null,0,-2,null,3,null,null,null,2,null,5,-2,1],"l2_utf8":["aa","c","a","b","ab","ab",null,"é","😀","😀","😀","a","b","b","b","",null,"a","aa","","aa","ab",null,"aa","zz","a",null,"é","é",null,"😀",null,"B","aa","é","c",null,null,"a","b","a","ab",null,"b",null,"a","ab","c",null,null,null,"B","ab",null,"a","zz","é","",null,null],"l3_f64":[null,null,25.895,null,2.839,4.0,-4.0,3.0,-88.774,-21.16,-91.207,null,76.192,4.0,null,null,3.0,null,null,-80.575,null,-1.0,76.616,null,1.0,2.0,null,3.0,null,null,null,null,-4.0,null,86.765,null,-3.0,null,-1.0,-3.0,null,-3.0,null,4.0,-46.78,74.941,-32.155,4.0,4.0,-2.732,-95.332,"NaN",-56.878,null,null,-3.0,-2.0,1.0,-4.32,67.783],"l4_f64":[-1.0,4.0,3.0,null,null,63.374,1.0,50.043,-22.708,-1.0,-3.0,33.92,-1.0,-14.174,null,-4.0,28.34,null,-26.677,53.186,-1.0,null,null,4.0,-57.442,null,1.0,-82.252,null,0.0,-1.0,null,82.802,99.932,null,null,-2.0,27.107,-58.632,2.0,null,0.0,-2.0,null,88.035,null,"NaN",null,null,null,null,3.0,-67.932,-96.907,null,-84.683,-33.927,null,-67.959,-75.226],"l5_f64":[null,1.0,-4.0,null,-2.0,16.909,-3.0,null,-18.741,null,-3.0,88.097,-1.0,-4.0,88.849,76.421,-3.042,31.853,1.0,18.612,-39.35,null,null,0.0,-3.0,-10.269,77.401,-3.0,null,null,null,-27.671,null,-91.003,3.0,4.0,65.696,-2.0,12.171,5.978,null,4.0,-1.0,-47.148,-5.348,null,-58.924,-60.944,35.267,1.0,null,1.0,-95.781,19.696,null,-3.0,34.837,32.993,null,null],"l6_f64":[-44.109,-4.0,97.54,null,null,null,null,null,61.356,-86.173,-3.976,-61.305,54.262,1.0,null,46.635,-33.824,null,null,null,null,null,2.0,92.121,80.197,-3.0,-23.282,null,62.116,77.404,-70.053,-4.0,-2.0,8.181,null,-89.572,4.0,null,1.0,-36.077,-79.747,39.497,-4.0,1.0,null,3.0,"NaN",-4.0,null,98.458,-11.873,2.0,55.533,null,0.0,null,null,1.0,3.0,-14.624]}},"right":{"fields":[{"name":"l2_utf8","type":"utf8"},{"name":"r0_f64","type":"f64"}],"columns":{"r0_f64":[58.001,-2.0,29.667,4.0,"-0",88.62,-72.689,-2.0,45.252,-40.089],"l2_utf8":["ab","a","aa","😀","ab","B","aa",null,"😀",null]}}},"ops":[{"op":"unique"},{"op":"join","right":"right","on":["l2_utf8"],"how":"semi"}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_f64","type":"f64"},{"name":"l4_f64","type":"f64"},{"name":"l5_f64","type":"f64"},{"name":"l6_f64","type":"f64"}],"columns":{"l0_utf8":["😀","a",null,"",null,"a","é","c","c","aa",null,"é",null,null,null,"é",null,"😀","aa","ab","ab","é",null,null,null],"l1_i32":[4,null,-1,308,1,-4,5,null,2,-5,1,-2,3,2,null,null,null,1,null,-3,2,2,3,null,null],"l2_utf8":["aa","a","ab","ab","😀","😀","😀","a","a","aa","aa","ab","aa","a","😀","B","aa","a","a","ab","a","ab","B","ab","a"],"l3_f64":[null,25.895,2.839,4.0,-88.774,-21.16,-91.207,null,null,null,null,-1.0,null,2.0,null,-4.0,null,-1.0,null,-3.0,74.941,-32.155,"NaN",-56.878,null],"l4_f64":[-1.0,3.0,null,63.374,-22.708,-1.0,-3.0,33.92,null,-26.677,-1.0,null,4.0,null,-1.0,82.802,99.932,-58.632,null,0.0,null,"NaN",3.0,-67.932,null],"l5_f64":[null,-4.0,-2.0,16.909,-18.741,null,-3.0,88.097,31.853,1.0,-39.35,null,0.0,-10.269,null,null,-91.003,12.171,null,4.0,null,-58.924,1.0,-95.781,null],"l6_f64":[-44.109,97.54,null,null,61.356,-86.173,-3.976,-61.305,null,null,null,null,92.121,-3.0,-70.053,-2.0,8.181,1.0,-79.747,39.497,3.0,"NaN",2.0,55.533,0.0]}}},{"id":"random/0214","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_bool","type":"bool"},{"name":"l4_i64","type":"i64"}],"columns":{"l0_utf8":[null,"aa","ab","é",""],"l1_i32":[0,3,-2147483648,-4,2],"l2_i32":[-4,-2,1,-4,-5],"l3_bool":[true,false,true,true,false],"l4_i64":["-4","9007199254740992","-4611686018427387904","9007199254740995","-5"]}}},"ops":[{"op":"head","n":10}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_bool","type":"bool"},{"name":"l4_i64","type":"i64"}],"columns":{"l0_utf8":[null,"aa","ab","é",""],"l1_i32":[0,3,-2147483648,-4,2],"l2_i32":[-4,-2,1,-4,-5],"l3_bool":[true,false,true,true,false],"l4_i64":["-4","9007199254740992","-4611686018427387904","9007199254740995","-5"]}}},{"id":"random/0215","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_f64","type":"f64"},{"name":"l4_f64","type":"f64"},{"name":"l5_i64","type":"i64"}],"columns":{"l0_utf8":["aa","é","",null,null,"zz","c",null,"ab",null,"😀","b",null,"aa",null,"😀","B","b",null,"😀","c",null,null,"a","aa","é","B","é","b","zz","b","ab","a","B",null,null,"",null,"zz","b","B","a",null,null,null,null,null,"b",null,"c",null,"ab",null,"é","b","",null,"",null,null],"l1_i32":[null,4,null,-3,-1,null,2147483647,3,-4,-5,-4,153,-4,3,0,0,null,-2,null,null,295,2147483647,-2,3,null,null,3,5,null,null,-2,2,-2,null,null,5,4,1,5,-2147483648,-4,3,null,-2147483648,4,4,5,0,0,null,1,-3,null,null,null,5,null,null,1,4],"l2_utf8":["ab","B",null,null,null,"aa","ab","b","ab",null,"aa","aa","b",null,null,"a","aa","ab",null,null,"é","aa","zz","é","c","c",null,"","aa",null,"ab",null,"aa","😀",null,null,"aa","a","b","b","aa","aa","B",null,"","B",null,null,"aa",null,"😀","😀","zz",null,null,"b","","B","zz",null],"l3_f64":[null,52.748,99.642,null,34.731,null,null,66.186,null,93.923,null,null,60.79,-2.0,4.0,null,"-Infinity",1.0,null,96.593,null,3.0,-98.716,73.762,-98.233,-1.0,null,2.0,null,4.0,null,null,42.111,2.505,null,-80.865,2.0,null,null,-4.0,null,-3.0,null,1.0,64.877,null,-83.345,-4.0,-5.338,-46.042,-2.0,-2.0,37.442,-70.122,-52.481,null,72.292,53.522,"NaN",null],"l4_f64":[3.062,56.649,29.412,-4.0,-1.0,null,-4.0,3.0,-92.394,null,-3.0,-33.367,2.0,-1.0,86.688,null,53.964,null,-2.0,-47.448,66.908,72.515,4.0,-32.809,null,-5.598,null,-4.0,1.0,2.0,-53.313,0.0,2.0,null,24.006,null,-2.0,1.0,2.0,null,"-Infinity",-2.0,-75.967,null,-2.0,null,14.605,-3.0,-2.0,null,null,98.536,-26.445,59.173,null,-16.507,null,"-Infinity",57.006,2.0],"l5_i64":[null,"-4611686018427387904",null,"-5","9007199254740995","9007199254740993","9007199254740995",null,"4","9007199254740992","-5",null,null,"-4611686018427387904","9007199254740992","-4611686018427387904","9007199254740994","-4","9007199254740995",null,"-4611686018427387904",null,null,"-4611686018427387904","-4611686018427387904","5","9007199254740993","-4611686018427387904","2","-5",null,null,"-4611686018427387904","-4611686018427387904","1","-4611686018427387904","-3","0","-4611686018427387904","4","9007199254740992","9007199254740992","9007199254740995","-4611686018427387904","9007199254740993","-5","9007199254740993",null,"-4611686018427387904","-5",null,"-4611686018427387904","9007199254740993","3","1","-5","9007199254740995","-5","-4611686018427387904",null]}}},"ops":[{"op":"unique","columns":["l1_i32"]},{"op":"filter","predicate":{"fn":"eq","args":[{"fn":"add","args":[{"col":"l3_f64"},{"col":"l3_f64"}]},{"fn":"add","args":[{"col":"l1_i32"},{"col":"l3_f64"}]}]}},{"op":"withColumns","exprs":{"x2":{"col":"l3_f64"}}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_f64","type":"f64"},{"name":"l4_f64","type":"f64"},{"name":"l5_i64","type":"i64"},{"name":"x2","type":"f64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_utf8":[],"l3_f64":[],"l4_f64":[],"l5_i64":[],"x2":[]}}},{"id":"random/0216","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_f64","type":"f64"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_f64","type":"f64"},{"name":"l6_i32","type":"i32"}],"columns":{"l0_utf8":["😀",null,"a","b","a","zz","a","aa","ab","c","ab",null,null,"b","B","aa",null,"zz","c","B"],"l1_i32":[-1,4,1,-2,null,-4,-2,0,1,0,5,2147483647,2,-5,-3,1,0,2,-425,null],"l2_utf8":["😀","aa","😀","B","zz","a","aa","a","ab","B","ab","zz","😀",null,null,"é","b","b","b",null],"l3_f64":[3.0,1.0,-7.796,2.0,-2.0,4.0,-3.0,48.888,3.0,3.0,null,-53.183,-2.0,56.105,-42.137,-4.0,-72.442,-1.0,null,null],"l4_utf8":["é","a","aa","é","c","é",null,"aa",null,"","😀","ab","","aa","ab","a","zz","B","ab",""],"l5_f64":[18.131,0.0,-2.0,2.0,-21.778,53.097,-3.0,-52.456,3.0,1.0,-2.0,-4.0,-1.0,43.727,3.977,null,91.858,-29.44,56.761,-31.553],"l6_i32":[-3,5,null,-2147483648,-5,4,4,-2147483648,0,-5,-3,0,3,2,1,-5,-4,0,-1,null]}},"right":{"fields":[{"name":"l2_utf8","type":"utf8"},{"name":"r0_bool","type":"bool"}],"columns":{"r0_bool":[null,false,null],"l2_utf8":["a","b","😀"]}}},"ops":[{"op":"join","right":"right","on":["l2_utf8"],"how":"left"},{"op":"withColumns","exprs":{"x1":{"col":"l3_f64"}}},{"op":"head","n":2}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_f64","type":"f64"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_f64","type":"f64"},{"name":"l6_i32","type":"i32"},{"name":"r0_bool","type":"bool"},{"name":"x1","type":"f64"}],"columns":{"l0_utf8":["😀",null],"l1_i32":[-1,4],"l2_utf8":["😀","aa"],"l3_f64":[3.0,1.0],"l4_utf8":["é","a"],"l5_f64":[18.131,0.0],"l6_i32":[-3,5],"r0_bool":[null,null],"x1":[3.0,1.0]}}},{"id":"random/0217","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i64","type":"i64"},{"name":"l3_i32","type":"i32"},{"name":"l4_f64","type":"f64"},{"name":"l5_i64","type":"i64"},{"name":"l6_utf8","type":"utf8"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_i64":[],"l3_i32":[],"l4_f64":[],"l5_i64":[],"l6_utf8":[]}}},"ops":[{"op":"head","n":7},{"op":"filter","predicate":{"fn":"isNull","args":[{"col":"l0_utf8"}]}},{"op":"sort","keys":[{"expr":{"col":"l5_i64"},"descending":true},{"expr":{"col":"l6_utf8"},"descending":false}],"nullsLast":false}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i64","type":"i64"},{"name":"l3_i32","type":"i32"},{"name":"l4_f64","type":"f64"},{"name":"l5_i64","type":"i64"},{"name":"l6_utf8","type":"utf8"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_i64":[],"l3_i32":[],"l4_f64":[],"l5_i64":[],"l6_utf8":[]}}},{"id":"random/0218","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_bool","type":"bool"},{"name":"l4_i32","type":"i32"},{"name":"l5_i32","type":"i32"}],"columns":{"l0_utf8":["zz","é","b","é","","c","😀","é","aa","ab","é","zz","zz","😀","a","a","a","","aa","B","zz","b","B","b","é","ab","b","B","","B","B","zz","zz","aa","é","c","😀","zz","B","é","b","ab","zz","zz","c","b","b","B","😀","zz","B","a","a","ab","a","é","😀","😀","","é"],"l1_i32":[-3,-3,5,-1,-2,2,3,-4,-4,3,-4,2,-2147483648,0,-4,-2,3,-2147483648,-4,-2,-2,3,2,-1,1,-4,-5,5,4,3,-1,4,-3,5,4,-2,4,-1,-2,3,2,0,0,-5,5,0,5,4,-4,1,-3,2,-4,-5,-5,-4,0,2,1,-3],"l2_i32":[2,4,1,-5,5,4,-3,0,-1,-2,1,-1,-1,4,-4,-2,2147483647,1,545,-2,-3,-666,2147483647,3,-3,-1,4,-5,-5,0,1,2,3,2,3,710,3,0,4,-1,-3,4,-4,-5,4,-3,5,2147483647,2,2147483647,3,2,2,4,-4,0,-1,2,2,-3],"l3_bool":[true,true,false,true,true,true,true,false,false,true,false,true,false,true,false,false,false,true,true,true,false,true,true,false,false,true,true,true,true,true,false,true,false,false,false,false,false,false,true,true,true,false,true,true,true,true,false,true,true,true,true,false,false,false,false,false,false,true,false,false],"l4_i32":[1,-1,2,4,0,-3,3,1,-3,-4,1,-2147483648,-3,-2,1,4,-5,312,-4,0,0,-5,3,2,0,3,1,5,2,4,0,5,-208,0,-5,-3,-3,0,3,1,3,0,-3,4,4,3,-3,2,-5,-1,-1,0,-4,0,-5,-5,-1,1,0,3],"l5_i32":[4,-5,-2,1,2,-4,-4,-5,-614,-5,0,2,3,5,3,-4,5,2,-1,379,5,4,-4,4,-1,5,1,-4,-3,-3,2,4,4,5,-4,-3,4,5,-4,0,-401,2,-3,3,-3,-5,5,-4,2,-3,4,3,2,-757,2,-886,-2147483648,-3,0,0]}},"right":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"r0_f64","type":"f64"}],"columns":{"r0_f64":[0.0,-1.0,3.0,4.0,3.0,"NaN",1.0,1.0,-91.739,-1.0],"l0_utf8":["c","ab","c","","","zz","b","é","zz","B"]}}},"ops":[{"op":"head","n":2},{"op":"join","right":"right","on":["l0_utf8"],"how":"full"}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_bool","type":"bool"},{"name":"l4_i32","type":"i32"},{"name":"l5_i32","type":"i32"},{"name":"r0_f64","type":"f64"}],"columns":{"l0_utf8":["zz","zz","é","c","ab","c","","","b","B"],"l1_i32":[-3,-3,-3,null,null,null,null,null,null,null],"l2_i32":[2,2,4,null,null,null,null,null,null,null],"l3_bool":[true,true,true,null,null,null,null,null,null,null],"l4_i32":[1,1,-1,null,null,null,null,null,null,null],"l5_i32":[4,4,-5,null,null,null,null,null,null,null],"r0_f64":["NaN",-91.739,1.0,0.0,-1.0,3.0,4.0,3.0,1.0,-1.0]}}},{"id":"random/0219","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_i32","type":"i32"}],"columns":{"l0_utf8":["ab",null,null,"😀","aa"],"l1_i32":[4,1,null,null,null],"l2_utf8":["é","c","b","ab",null],"l3_i32":[-2,4,-3,-5,-4]}}},"ops":[{"op":"groupBy","keys":["l2_utf8"],"aggs":{"n0":{"agg":"len"},"a0_0":{"agg":"mean","arg":{"col":"l1_i32"}}}}],"expected":{"fields":[{"name":"l2_utf8","type":"utf8"},{"name":"n0","type":"u32"},{"name":"a0_0","type":"f64"}],"columns":{"l2_utf8":["é","c","b","ab",null],"n0":[1,1,1,1,1],"a0_0":[4.0,1.0,null,null,null]}}},{"id":"random/0220","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_utf8","type":"utf8"}],"columns":{"l0_utf8":["B"],"l1_i32":[3],"l2_f64":[null],"l3_utf8":[null],"l4_utf8":["ab"]}},"right":{"fields":[{"name":"l4_utf8","type":"utf8"},{"name":"r0_f64","type":"f64"}],"columns":{"r0_f64":[],"l4_utf8":[]}}},"ops":[{"op":"join","right":"right","on":["l4_utf8"],"how":"left"},{"op":"filter","predicate":{"fn":"isNull","args":[{"col":"l0_utf8"}]}},{"op":"unique","columns":["l1_i32"]}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_utf8","type":"utf8"},{"name":"r0_f64","type":"f64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_f64":[],"l3_utf8":[],"l4_utf8":[],"r0_f64":[]}}},{"id":"random/0221","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_f64","type":"f64"},{"name":"l4_i32","type":"i32"}],"columns":{"l0_utf8":["😀","zz","c",null,"B","c","😀","c","ab","c",null,"ab","a","ab","b","aa","😀","é","😀","",null,null,"B","B","B","😀","","zz","B","😀",null,null,"ab","é","B","zz",null,"zz","zz","😀","ab","a","😀","😀","é","😀","","c","é","","b","é","zz","é","é","aa",null,"a","c","ab"],"l1_i32":[-2,-5,-1,1,1,1,-1,-3,4,-4,-4,3,-2,-2,4,1,1,-5,-4,-3,-1,3,null,-4,-1,-4,-4,-5,5,null,-1,-3,1,null,2147483647,null,-3,2,0,4,1,3,-5,-3,-3,0,null,2,null,2,4,-5,3,3,-3,2,-1,2,-2,-3],"l2_f64":[0.0,42.491,-1.0,-4.0,-26.008,3.0,0.0,42.481,78.047,"NaN",-4.0,3.0,12.125,-1.0,null,70.992,null,null,-91.794,-4.0,94.123,37.878,"NaN",null,null,-1.0,4.0,-1.315,null,-1.0,-63.981,35.741,-1.0,45.796,-3.0,-3.0,null,4.0,72.764,2.0,-3.0,null,-18.37,3.0,-16.12,"NaN",0.0,4.0,3.0,null,-11.094,54.595,3.0,-88.373,78.396,47.438,null,-47.668,51.065,-14.731],"l3_f64":[94.582,3.0,-3.0,-59.457,1.0,4.0,"NaN","-Infinity",-4.0,-11.029,68.956,-46.159,-42.456,-56.346,-4.0,null,0.0,null,"NaN",62.046,null,-35.436,-4.0,-3.0,-65.786,3.0,null,53.279,4.0,null,-34.303,88.949,1.0,-1.0,-66.53,-92.453,-81.095,-9.721,-27.32,0.0,-1.0,-98.068,17.653,-4.0,-4.104,81.062,-2.0,58.832,-2.0,null,null,4.0,-95.306,-59.747,"Infinity",-92.423,0.0,75.806,"-0",34.707],"l4_i32":[0,-3,-5,-4,1,-1,3,4,null,2,0,-5,-2,1,3,-2,-3,-1,5,2,4,0,2,2,535,4,4,0,0,0,-2,-1,-2,-3,-4,null,5,-4,-2,4,0,null,null,2,2147483647,4,2,2147483647,-4,4,4,-1,null,0,-4,-5,-5,4,0,null]}}},"ops":[{"op":"withColumns","exprs":{"x0":{"fn":"lte","args":[{"fn":"add","args":[{"col":"l2_f64"},{"col":"l4_i32"}]},{"fn":"abs","args":[{"col":"l1_i32"}]}]}}},{"op":"head","n":2}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_f64","type":"f64"},{"name":"l4_i32","type":"i32"},{"name":"x0","type":"bool"}],"columns":{"l0_utf8":["😀","zz"],"l1_i32":[-2,-5],"l2_f64":[0.0,42.491],"l3_f64":[94.582,3.0],"l4_i32":[0,-3],"x0":[true,false]}}},{"id":"random/0222","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i64","type":"i64"},{"name":"l3_i32","type":"i32"},{"name":"l4_i64","type":"i64"},{"name":"l5_i32","type":"i32"},{"name":"l6_f64","type":"f64"}],"columns":{"l0_utf8":["é","😀","zz","c","😀","c","a","B","c","zz","c","b","","b","aa","","zz","😀","a","😀"],"l1_i32":[-3,-3,-2,0,-3,-3,-3,-5,3,0,3,-5,-2,2147483647,891,-3,-5,4,-3,-717],"l2_i64":["-4611686018427387904","9007199254740995","-4611686018427387904","9007199254740993","-4611686018427387904","-4611686018427387904","3","1","2","-5","2","-4611686018427387904","-3","0","3","9007199254740995","-3","-2","9007199254740993","9007199254740995"],"l3_i32":[2,5,5,0,4,-1,2,-4,3,-1,1,3,-4,-3,-3,5,-5,-1,2,-2],"l4_i64":["-4611686018427387904","-4","9007199254740993","-4611686018427387904","-4611686018427387904","-4611686018427387904","5","9007199254740992","-3","0","-1","9007199254740994","4","9007199254740995","-4611686018427387904","-4611686018427387904","9007199254740994","-4611686018427387904","9007199254740994","-4"],"l5_i32":[4,4,2,0,3,-5,0,0,-3,3,-2,-653,-1,1,5,1,-5,5,-4,1],"l6_f64":[4.0,2.0,4.0,42.667,0.0,-1.0,51.507,2.0,-87.014,-19.101,-11.413,-19.665,-28.32,4.0,"-0",-1.0,-21.511,-1.0,-2.0,0.0]}}},"ops":[{"op":"withColumns","exprs":{"x0":{"fn":"neq","args":[{"col":"l5_i32"},{"fn":"when","args":[{"fn":"isNull","args":[{"col":"l6_f64"}]},{"col":"l5_i32"},{"col":"l5_i32"}]}]}}},{"op":"withColumns","exprs":{"x1":{"fn":"abs","args":[{"col":"l5_i32"}]}}},{"op":"groupBy","keys":["l5_i32"],"aggs":{"n2":{"agg":"len"},"a2_0":{"agg":"count","arg":{"col":"l4_i64"}},"a2_1":{"agg":"max","arg":{"col":"l2_i64"}},"a2_2":{"agg":"count","arg":{"col":"x0"}}}}],"expected":{"fields":[{"name":"l5_i32","type":"i32"},{"name":"n2","type":"u32"},{"name":"a2_0","type":"u32"},{"name":"a2_1","type":"i64"},{"name":"a2_2","type":"u32"}],"columns":{"l5_i32":[4,2,0,3,-5,-3,-2,-653,-1,1,5,-4],"n2":[2,1,3,2,2,1,1,1,1,3,2,1],"a2_0":[2,1,3,2,2,1,1,1,1,3,2,1],"a2_1":["9007199254740995","-4611686018427387904","9007199254740993","-5","-3","2","2","-4611686018427387904","-3","9007199254740995","3","9007199254740993"],"a2_2":[2,1,3,2,2,1,1,1,1,3,2,1]}}},{"id":"random/0223","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_bool","type":"bool"},{"name":"l4_f64","type":"f64"}],"columns":{"l0_utf8":["c"],"l1_i32":[2147483647],"l2_f64":[93.957],"l3_bool":[true],"l4_f64":[-44.174]}}},"ops":[{"op":"withColumns","exprs":{"x0":{"fn":"isNull","args":[{"col":"l2_f64"}]}}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_bool","type":"bool"},{"name":"l4_f64","type":"f64"},{"name":"x0","type":"bool"}],"columns":{"l0_utf8":["c"],"l1_i32":[2147483647],"l2_f64":[93.957],"l3_bool":[true],"l4_f64":[-44.174],"x0":[false]}}},{"id":"random/0224","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_f64","type":"f64"},{"name":"l4_i32","type":"i32"}],"columns":{"l0_utf8":["c",null,"B",null,"😀","😀","ab","B",null,"B",null,"c",null,"b","b",null,"a",null,null,"zz","B",null,"B","😀","é","😀","b","B","😀","😀","😀","aa","c","a",null,"zz","a",null,"a","é","😀","zz","aa","aa","b",null,"ab","b","c","","","é","a",null,"aa",null,"é",null,null,"ab"],"l1_i32":[3,-5,-4,null,3,4,-1,4,4,null,-2,null,null,4,-2,null,4,null,-4,5,5,2,-5,5,-3,-5,0,null,-4,null,4,5,null,5,null,null,1,0,null,null,null,2,-4,-3,-4,31,-5,-3,1000,3,null,null,-4,null,-3,0,null,-1,-5,3],"l2_f64":[null,2.0,1.0,3.0,-29.533,null,68.366,-12.378,-4.0,null,null,-1.0,3.0,4.0,null,-2.0,1.0,98.549,null,26.348,17.38,null,45.508,-11.136,null,null,2.0,-4.0,2.0,0.0,null,-68.745,4.0,22.039,-50.479,null,-20.232,43.639,null,38.77,-2.0,-4.0,4.0,2.0,-2.0,-52.457,-50.314,37.991,-1.0,null,-58.155,99.711,null,null,4.0,3.0,63.397,1.0,-1.0,-2.0],"l3_f64":[null,-2.0,null,2.0,13.707,-1.0,null,-1.0,48.235,24.045,null,0.0,2.0,-46.87,2.0,null,3.0,-43.354,-2.0,null,65.82,-29.526,4.0,38.266,4.0,-27.323,null,1.0,null,null,38.197,"-Infinity",-58.434,71.398,33.567,null,24.188,-1.0,0.0,null,-4.0,null,4.0,1.0,63.88,-2.0,-20.828,null,-4.478,null,null,null,null,-4.0,-72.593,null,null,56.85,-2.0,-1.0],"l4_i32":[3,-5,-2,4,null,0,-3,-2,null,-3,null,0,null,0,-2147483648,-5,0,-4,null,5,2,0,1,-2147483648,5,-5,4,-1,null,2,0,null,2,5,-3,-3,-5,1,null,5,-3,null,3,4,1,4,null,null,-5,-2,null,-1,null,4,-4,2,null,-1,-110,null]}}},"ops":[{"op":"groupBy","keys":["l4_i32"],"aggs":{"n0":{"agg":"len"},"a0_0":{"agg":"min","arg":{"col":"l0_utf8"}}}}],"expected":{"fields":[{"name":"l4_i32","type":"i32"},{"name":"n0","type":"u32"},{"name":"a0_0","type":"utf8"}],"columns":{"l4_i32":[3,-5,-2,4,null,0,-3,-2147483648,-4,5,2,1,-1,-110],"n0":[2,5,3,5,15,6,5,2,2,4,4,3,3,1],"a0_0":["aa","a","","aa","","a","B","b","aa","a","B","B","B",null]}}},{"id":"random/0225","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_i64","type":"i64"}],"columns":{"l0_utf8":["aa","ab","B","b","é"],"l1_i32":[5,273,5,-5,-2147483648],"l2_bool":[true,true,true,false,true],"l3_i64":["2","-4611686018427387904","9007199254740995","-2","9007199254740994"]}},"right":{"fields":[{"name":"l1_i32","type":"i32"},{"name":"r0_utf8","type":"utf8"}],"columns":{"r0_utf8":["b","B","b","B","aa","B","a","ab","aa","a"],"l1_i32":[-2147483648,5,-5,273,5,-2147483648,5,-5,-5,273]}}},"ops":[{"op":"join","right":"right","on":["l1_i32"],"how":"left"},{"op":"unique","columns":["l2_bool"]},{"op":"withColumns","exprs":{"x2":{"fn":"add","args":[{"col":"l1_i32"},{"col":"l1_i32"}]}}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_i64","type":"i64"},{"name":"r0_utf8","type":"utf8"},{"name":"x2","type":"i32"}],"columns":{"l0_utf8":["aa","b"],"l1_i32":[5,-5],"l2_bool":[true,false],"l3_i64":["2","-2"],"r0_utf8":["B","b"],"x2":[10,-10]}}},{"id":"random/0226","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_bool","type":"bool"}],"columns":{"l0_utf8":["a",null,"B","zz","aa"],"l1_i32":[-2,0,0,null,-5],"l2_bool":[false,null,true,true,true],"l3_utf8":["c","c","","b",null],"l4_utf8":["ab","é","c","c",null],"l5_bool":[true,false,null,false,true]}},"right":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"r0_bool","type":"bool"}],"columns":{"r0_bool":[null,true,true,true,true,false,true,true,null,false,false,false,true,true,true,false,null,null,true,false,false,false,true,false,true,true,false,false,true,true],"l0_utf8":["aa","a","a",null,"aa","B","aa","B","B","zz","zz","B","B",null,"zz","zz","zz","aa","zz",null,"a","B",null,"B","zz","a","zz","aa","aa","zz"]}}},"ops":[{"op":"unique","columns":["l2_bool"]},{"op":"head","n":4},{"op":"join","right":"right","on":["l0_utf8"],"how":"anti"}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_bool","type":"bool"}],"columns":{"l0_utf8":[null],"l1_i32":[0],"l2_bool":[null],"l3_utf8":["c"],"l4_utf8":["é"],"l5_bool":[false]}}},{"id":"random/0227","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_i32","type":"i32"}],"columns":{"l0_utf8":["😀"],"l1_i32":[0],"l2_bool":[true],"l3_i32":[-3]}}},"ops":[{"op":"unique","columns":["l0_utf8"]}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_i32","type":"i32"}],"columns":{"l0_utf8":["😀"],"l1_i32":[0],"l2_bool":[true],"l3_i32":[-3]}}},{"id":"random/0228","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_f64","type":"f64"},{"name":"l4_bool","type":"bool"},{"name":"l5_i32","type":"i32"},{"name":"l6_utf8","type":"utf8"}],"columns":{"l0_utf8":["aa",null,null,"aa",null],"l1_i32":[null,3,1,-1,-2],"l2_utf8":["ab",null,null,"😀","B"],"l3_f64":[null,null,null,null,94.131],"l4_bool":[true,true,null,null,false],"l5_i32":[1,null,3,-4,-1],"l6_utf8":["ab","é",null,"B","zz"]}}},"ops":[{"op":"head","n":1},{"op":"filter","predicate":{"fn":"neq","args":[{"fn":"floorDiv","args":[{"col":"l1_i32"},{"col":"l5_i32"}]},{"fn":"neg","args":[{"lit":2.5}]}]}},{"op":"withColumns","exprs":{"x2":{"fn":"gte","args":[{"col":"l6_utf8"},{"lit":"c"}]}}},{"op":"unique","columns":["x2"]}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_f64","type":"f64"},{"name":"l4_bool","type":"bool"},{"name":"l5_i32","type":"i32"},{"name":"l6_utf8","type":"utf8"},{"name":"x2","type":"bool"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_utf8":[],"l3_f64":[],"l4_bool":[],"l5_i32":[],"l6_utf8":[],"x2":[]}}},{"id":"random/0229","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_f64","type":"f64"}],"columns":{"l0_utf8":[null,"aa","aa","","a",null,"zz","aa",null,"B","B",null,"a",null,"b","zz",null,"",null,"","a","é",null,"aa",null,"a","😀",null,null,"","aa",null,null,"b","zz","ab","a",null,"ab","c","é",null,null,null,"é",null,"B",null,"é","b","c",null,"b","ab","ab","zz","ab","B","é",null],"l1_i32":[-1,null,-341,-2,-1,null,-2,5,2147483647,-2,-5,null,-3,3,null,-2,637,2147483647,null,null,5,958,3,null,-5,3,null,-4,3,-1,1,4,null,5,null,0,-5,null,-3,4,3,null,null,3,1,-2,5,null,null,null,0,1,1,2,null,2,2,null,0,0],"l2_bool":[false,null,true,false,true,true,null,false,true,null,null,null,false,null,null,true,false,null,null,null,true,false,null,false,true,null,true,false,false,false,false,true,true,true,null,true,true,true,true,false,null,null,false,true,null,true,null,true,true,null,false,null,true,false,true,true,null,false,false,false],"l3_f64":[-3.0,null,-4.0,3.0,-98.122,null,null,-16.845,16.978,null,null,-76.432,null,null,null,"NaN",null,-29.52,null,null,3.0,2.0,47.342,null,-45.524,null,"NaN",-1.0,null,1.0,null,null,2.0,null,-3.0,2.0,50.551,-4.0,56.546,-2.0,89.544,1.0,null,-65.012,null,null,2.0,-2.0,-1.0,1.0,-1.0,-4.0,2.0,33.84,3.0,"-0",null,-4.0,null,-4.0]}},"right":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"r0_f64","type":"f64"}],"columns":{"r0_f64":[],"l0_utf8":[]}}},"ops":[{"op":"join","right":"right","on":["l0_utf8"],"how":"semi"},{"op":"filter","predicate":{"fn":"neq","args":[{"col":"l1_i32"},{"fn":"mod","args":[{"col":"l3_f64"},{"col":"l1_i32"}]}]}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_f64","type":"f64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_bool":[],"l3_f64":[]}}},{"id":"random/0230","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i64","type":"i64"},{"name":"l3_bool","type":"bool"}],"columns":{"l0_utf8":["a","","b","a","ab","aa","B","b","b","😀","c","b","aa","c","B","a","b","B","é","B"],"l1_i32":[5,-2147483648,-4,3,2147483647,2,-2147483648,3,5,3,0,2147483647,5,4,-2,4,3,-2,3,-5],"l2_i64":["-4","-1","9007199254740993","9007199254740992","3","9007199254740995","9007199254740993","9007199254740992","0","-1","5","9007199254740993","9007199254740992","9007199254740993","9007199254740993","9007199254740994","9007199254740994","9007199254740995","-4611686018427387904","-5"],"l3_bool":[true,true,false,false,false,false,true,false,true,false,true,false,false,false,true,false,false,false,false,true]}}},"ops":[{"op":"head","n":6},{"op":"head","n":7},{"op":"filter","predicate":{"fn":"and","args":[{"fn":"gte","args":[{"col":"l0_utf8"},{"lit":"é"}]},{"fn":"neq","args":[{"col":"l1_i32"},{"col":"l1_i32"}]}]}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i64","type":"i64"},{"name":"l3_bool","type":"bool"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_i64":[],"l3_bool":[]}}},{"id":"random/0231","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_f64","type":"f64"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_f64","type":"f64"},{"name":"l6_f64","type":"f64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_f64":[],"l3_f64":[],"l4_utf8":[],"l5_f64":[],"l6_f64":[]}}},"ops":[{"op":"head","n":9}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_f64","type":"f64"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_f64","type":"f64"},{"name":"l6_f64","type":"f64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_f64":[],"l3_f64":[],"l4_utf8":[],"l5_f64":[],"l6_f64":[]}}},{"id":"random/0232","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_i64","type":"i64"},{"name":"l4_i32","type":"i32"}],"columns":{"l0_utf8":["aa","😀","a","aa","😀"],"l1_i32":[5,1,2,-3,-2],"l2_f64":[4.0,1.0,-98.713,-8.581,-3.0],"l3_i64":["-4611686018427387904","-3","-5","9007199254740994","-4611686018427387904"],"l4_i32":[3,3,-2,-1,3]}}},"ops":[{"op":"withColumns","exprs":{"x0":{"col":"l4_i32"}}},{"op":"withColumns","exprs":{"x1":{"col":"l2_f64"}}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_i64","type":"i64"},{"name":"l4_i32","type":"i32"},{"name":"x0","type":"i32"},{"name":"x1","type":"f64"}],"columns":{"l0_utf8":["aa","😀","a","aa","😀"],"l1_i32":[5,1,2,-3,-2],"l2_f64":[4.0,1.0,-98.713,-8.581,-3.0],"l3_i64":["-4611686018427387904","-3","-5","9007199254740994","-4611686018427387904"],"l4_i32":[3,3,-2,-1,3],"x0":[3,3,-2,-1,3],"x1":[4.0,1.0,-98.713,-8.581,-3.0]}}},{"id":"random/0233","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_i32","type":"i32"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_utf8","type":"utf8"},{"name":"l6_utf8","type":"utf8"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_utf8":[],"l3_i32":[],"l4_utf8":[],"l5_utf8":[],"l6_utf8":[]}},"right":{"fields":[{"name":"l6_utf8","type":"utf8"},{"name":"r0_bool","type":"bool"}],"columns":{"r0_bool":[false,true,null],"l6_utf8":["😀","ab",""]}}},"ops":[{"op":"sort","keys":[{"expr":{"col":"l4_utf8"},"descending":true}],"nullsLast":true},{"op":"join","right":"right","on":["l6_utf8"],"how":"inner"}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_i32","type":"i32"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_utf8","type":"utf8"},{"name":"l6_utf8","type":"utf8"},{"name":"r0_bool","type":"bool"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_utf8":[],"l3_i32":[],"l4_utf8":[],"l5_utf8":[],"l6_utf8":[],"r0_bool":[]}}},{"id":"random/0234","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_bool","type":"bool"},{"name":"l4_f64","type":"f64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_utf8":[],"l3_bool":[],"l4_f64":[]}}},"ops":[{"op":"head","n":2},{"op":"groupBy","keys":["l0_utf8"],"aggs":{"n1":{"agg":"len"},"a1_0":{"agg":"median","arg":{"col":"l4_f64"}}}},{"op":"withColumns","exprs":{"x2":{"fn":"and","args":[{"fn":"gt","args":[{"col":"a1_0"},{"lit":0.5}]},{"fn":"isNotNull","args":[{"col":"n1"}]}]}}},{"op":"filter","predicate":{"fn":"eq","args":[{"col":"a1_0"},{"col":"a1_0"}]}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"n1","type":"u32"},{"name":"a1_0","type":"f64"},{"name":"x2","type":"bool"}],"columns":{"l0_utf8":[],"n1":[],"a1_0":[],"x2":[]}}},{"id":"random/0235","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_f64","type":"f64"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_utf8","type":"utf8"},{"name":"l6_utf8","type":"utf8"}],"columns":{"l0_utf8":["","a","zz","b","zz","aa","B","ab","😀","b","b","c","","c",null,"é","B","c","B","","","😀","ab","é","c","B","é","B","c","","c","a","😀","c","b",null,null,"a","zz","B","é","a","c","ab","","aa","é","c","ab","zz","é","a","😀","","zz","zz","ab",null,"😀",""],"l1_i32":[0,-4,5,3,null,3,2,0,3,2,-3,-1,2,2,-5,4,2,-3,-3,5,-2,2,-3,3,-4,1,-1,2147483647,-3,5,2,2,-2147483648,5,-5,4,-5,2,null,3,5,-2,null,null,0,1,-2,5,-3,0,1,1,-4,-1,5,5,3,2,-3,-2],"l2_f64":[null,-4.0,-54.756,2.0,1.0,0.729,4.0,null,-33.134,-80.384,35.102,0.0,3.0,2.0,2.0,4.0,99.354,-13.696,-18.416,-2.0,-3.0,-8.874,14.178,null,76.35,"NaN",97.722,-2.28,"-0",-4.0,-3.0,85.173,8.127,4.0,86.32,-47.955,-4.0,-4.0,-3.0,null,-80.717,-0.989,null,4.0,1.0,null,16.717,-3.0,-21.783,-3.0,-20.893,88.62,42.84,null,3.0,2.0,-1.0,-4.0,0.0,43.989],"l3_f64":[-80.292,-1.0,-45.918,-10.035,"-Infinity",0.0,61.056,-37.111,null,-4.0,28.574,86.734,-32.981,-94.458,-3.0,0.0,-79.813,"NaN",88.501,-1.0,-59.624,0.0,-2.0,3.0,93.276,-51.682,23.688,null,1.0,null,-58.625,-1.0,-15.889,-1.0,-84.714,41.402,4.0,85.1,-34.633,75.944,4.0,47.116,"-0",0.0,19.453,2.0,-1.0,94.906,-62.922,80.246,-3.0,-4.0,46.232,-2.0,null,73.862,-49.725,-1.0,-50.358,-92.881],"l4_utf8":["é","a","ab","😀",null,"b","ab","",null,"😀","ab","😀","a","c","c","B","","😀","B","B","ab","c","ab",null,"zz","zz","é","b","a","B","","zz","B","b","c","","B","zz","b","zz",null,"","zz","é",null,"B","b","c","c","zz","zz","b","c","ab","aa","b","aa","😀","😀","ab"],"l5_utf8":["aa","a","","😀","c","aa","","ab","zz","","aa","B","B",null,"","c","zz","aa","c","B","B",null,"aa","a","B","B","c",null,"a","zz","","ab","ab","b",null,"c","zz","c","é","b","b","a","b","B","ab","é","a","aa","b","B","é",null,"a","aa","B","b","a","","😀","zz"],"l6_utf8":["","B","😀","c","","c","aa",null,"zz","c",null,"😀","ab","😀","😀",null,"😀",null,"é","aa",null,"zz","aa","b","","c","zz","b","😀","é","ab","😀","zz","😀","","aa","é","a","aa","é","b","aa",null,"b",null,"é","c","c","c",null,"c","B","c",null,"a","😀","aa",null,"é","c"]}}},"ops":[{"op":"withColumns","exprs":{"x0":{"col":"l3_f64"}}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_f64","type":"f64"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_utf8","type":"utf8"},{"name":"l6_utf8","type":"utf8"},{"name":"x0","type":"f64"}],"columns":{"l0_utf8":["","a","zz","b","zz","aa","B","ab","😀","b","b","c","","c",null,"é","B","c","B","","","😀","ab","é","c","B","é","B","c","","c","a","😀","c","b",null,null,"a","zz","B","é","a","c","ab","","aa","é","c","ab","zz","é","a","😀","","zz","zz","ab",null,"😀",""],"l1_i32":[0,-4,5,3,null,3,2,0,3,2,-3,-1,2,2,-5,4,2,-3,-3,5,-2,2,-3,3,-4,1,-1,2147483647,-3,5,2,2,-2147483648,5,-5,4,-5,2,null,3,5,-2,null,null,0,1,-2,5,-3,0,1,1,-4,-1,5,5,3,2,-3,-2],"l2_f64":[null,-4.0,-54.756,2.0,1.0,0.729,4.0,null,-33.134,-80.384,35.102,0.0,3.0,2.0,2.0,4.0,99.354,-13.696,-18.416,-2.0,-3.0,-8.874,14.178,null,76.35,"NaN",97.722,-2.28,"-0",-4.0,-3.0,85.173,8.127,4.0,86.32,-47.955,-4.0,-4.0,-3.0,null,-80.717,-0.989,null,4.0,1.0,null,16.717,-3.0,-21.783,-3.0,-20.893,88.62,42.84,null,3.0,2.0,-1.0,-4.0,0.0,43.989],"l3_f64":[-80.292,-1.0,-45.918,-10.035,"-Infinity",0.0,61.056,-37.111,null,-4.0,28.574,86.734,-32.981,-94.458,-3.0,0.0,-79.813,"NaN",88.501,-1.0,-59.624,0.0,-2.0,3.0,93.276,-51.682,23.688,null,1.0,null,-58.625,-1.0,-15.889,-1.0,-84.714,41.402,4.0,85.1,-34.633,75.944,4.0,47.116,"-0",0.0,19.453,2.0,-1.0,94.906,-62.922,80.246,-3.0,-4.0,46.232,-2.0,null,73.862,-49.725,-1.0,-50.358,-92.881],"l4_utf8":["é","a","ab","😀",null,"b","ab","",null,"😀","ab","😀","a","c","c","B","","😀","B","B","ab","c","ab",null,"zz","zz","é","b","a","B","","zz","B","b","c","","B","zz","b","zz",null,"","zz","é",null,"B","b","c","c","zz","zz","b","c","ab","aa","b","aa","😀","😀","ab"],"l5_utf8":["aa","a","","😀","c","aa","","ab","zz","","aa","B","B",null,"","c","zz","aa","c","B","B",null,"aa","a","B","B","c",null,"a","zz","","ab","ab","b",null,"c","zz","c","é","b","b","a","b","B","ab","é","a","aa","b","B","é",null,"a","aa","B","b","a","","😀","zz"],"l6_utf8":["","B","😀","c","","c","aa",null,"zz","c",null,"😀","ab","😀","😀",null,"😀",null,"é","aa",null,"zz","aa","b","","c","zz","b","😀","é","ab","😀","zz","😀","","aa","é","a","aa","é","b","aa",null,"b",null,"é","c","c","c",null,"c","B","c",null,"a","😀","aa",null,"é","c"],"x0":[-80.292,-1.0,-45.918,-10.035,"-Infinity",0.0,61.056,-37.111,null,-4.0,28.574,86.734,-32.981,-94.458,-3.0,0.0,-79.813,"NaN",88.501,-1.0,-59.624,0.0,-2.0,3.0,93.276,-51.682,23.688,null,1.0,null,-58.625,-1.0,-15.889,-1.0,-84.714,41.402,4.0,85.1,-34.633,75.944,4.0,47.116,"-0",0.0,19.453,2.0,-1.0,94.906,-62.922,80.246,-3.0,-4.0,46.232,-2.0,null,73.862,-49.725,-1.0,-50.358,-92.881]}}},{"id":"random/0236","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_bool","type":"bool"},{"name":"l4_f64","type":"f64"}],"columns":{"l0_utf8":["ab","zz","😀","ab","é","a","B","b","B","aa","b","ab","é","a","b","zz","zz","é","a","c","aa","a","aa","aa","","zz","B","😀","b","zz","aa","zz","zz","a","ab","zz","😀","B","aa","zz","","ab","c","","😀","aa","c","B","a","zz","zz","aa","ab","B","é","aa","a","aa","","aa"],"l1_i32":[0,-2,-1,4,3,2,-3,4,3,0,5,-3,-3,1,5,-5,4,4,2,1,-5,2147483647,-5,3,-1,1,3,3,5,2147483647,3,4,0,-3,-3,3,-2,0,-5,-4,-1,4,-4,3,1,-1,-3,4,0,0,4,-5,3,-4,1,2,-2,-2,-1,2],"l2_f64":[0.0,"Infinity",-3.0,91.966,"NaN",-1.0,-48.666,0.0,-4.0,-78.985,55.042,-63.404,-2.0,-57.506,-2.0,3.0,"NaN",4.0,4.0,-99.896,-2.0,4.0,"-0",-2.0,-48.387,4.0,-1.0,2.414,-82.947,15.064,-2.0,-4.0,-8.441,-1.0,-1.0,-7.999,-4.0,2.0,"-Infinity",2.0,-3.0,"-Infinity",0.0,4.0,1.0,"NaN",-65.336,-3.0,26.026,94.73,-89.729,-71.779,11.368,0.0,0.0,-90.587,-82.643,"NaN",0.0,54.153],"l3_bool":[false,false,true,false,false,false,true,true,true,false,false,false,false,true,true,false,false,true,false,false,false,true,true,true,true,true,false,true,false,true,false,false,true,false,false,true,true,true,true,false,true,false,true,true,false,true,true,false,true,true,false,false,true,true,true,true,true,false,true,true],"l4_f64":[3.0,-4.0,-48.295,-1.0,43.491,-15.89,43.34,-4.0,35.856,0.05,-99.277,-86.003,21.311,-3.0,-3.0,2.0,-71.88,0.0,-7.59,99.821,-1.0,"Infinity",24.225,4.0,-2.0,-4.0,-2.0,4.0,1.0,-96.062,4.0,-2.0,-3.0,1.0,-3.0,65.661,-3.0,63.601,4.0,0.0,0.0,57.254,72.563,-1.0,-4.0,-1.0,2.0,4.0,-2.0,-2.0,-1.0,-1.0,67.378,-39.861,-98.145,0.0,-74.499,"NaN",-69.03,0.0]}}},"ops":[{"op":"groupBy","keys":["l3_bool"],"aggs":{"n0":{"agg":"len"},"a0_0":{"agg":"first","arg":{"col":"l0_utf8"}},"a0_1":{"agg":"min","arg":{"col":"l0_utf8"}}}},{"op":"filter","predicate":{"fn":"strStartsWith","args":[{"col":"a0_1"}],"prefix":"a"}},{"op":"unique","columns":["n0"]},{"op":"filter","predicate":{"fn":"isNotNull","args":[{"col":"n0"}]}}],"expected":{"fields":[{"name":"l3_bool","type":"bool"},{"name":"n0","type":"u32"},{"name":"a0_0","type":"utf8"},{"name":"a0_1","type":"utf8"}],"columns":{"l3_bool":[],"n0":[],"a0_0":[],"a0_1":[]}}},{"id":"random/0237","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_i64","type":"i64"},{"name":"l4_utf8","type":"utf8"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_f64":[],"l3_i64":[],"l4_utf8":[]}}},"ops":[{"op":"withColumns","exprs":{"x0":{"lit":-1}}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_i64","type":"i64"},{"name":"l4_utf8","type":"utf8"},{"name":"x0","type":"i32"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_f64":[],"l3_i64":[],"l4_utf8":[],"x0":[]}}},{"id":"random/0238","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_i64","type":"i64"},{"name":"l5_utf8","type":"utf8"},{"name":"l6_bool","type":"bool"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_i32":[],"l3_utf8":[],"l4_i64":[],"l5_utf8":[],"l6_bool":[]}}},"ops":[{"op":"unique"}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_i64","type":"i64"},{"name":"l5_utf8","type":"utf8"},{"name":"l6_bool","type":"bool"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_i32":[],"l3_utf8":[],"l4_i64":[],"l5_utf8":[],"l6_bool":[]}}},{"id":"random/0239","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_f64","type":"f64"},{"name":"l4_f64","type":"f64"}],"columns":{"l0_utf8":["😀","c","b",null,"zz",null,null,null,null,"😀",null,null,"zz",null,"zz",null,null,"","","b","aa","😀",null,null,"a","é","c","aa","B","aa","ab","aa","é","a","ab",null,null,"é","é",null,"zz","aa","é","","a","a","😀","a","c","a","c","aa","ab",null,"","é","😀","aa","ab",""],"l1_i32":[1,-2,1,-2,null,3,null,-4,null,3,-3,4,4,2,null,-5,null,5,798,2,null,5,0,null,2147483647,-2,1,5,3,0,1,null,null,2,null,0,null,-1,null,0,null,-1,null,4,5,5,-4,3,-5,null,-4,-4,2,5,1,-1,4,null,3,null],"l2_utf8":["a","é","b",null,null,"é","é","é","é","b","B","aa","aa","B","b","a","",null,"","B","B","B",null,"","B",null,null,null,null,"b","a","","b","a","é","B",null,"B","a",null,"B",null,"ab",null,"ab","c","c","a","é","a",null,"😀",null,"😀",null,"B","a",null,"zz","c"],"l3_f64":[-99.232,-9.422,34.061,-69.56,null,null,50.713,0.0,-3.0,-4.0,2.0,-39.637,4.0,81.495,2.0,-2.0,0.0,-35.471,-4.0,2.0,28.241,-1.0,17.905,null,"-Infinity",null,47.089,31.218,null,"NaN",90.508,null,67.411,-1.0,null,null,null,-97.001,2.0,null,48.292,98.7,null,null,0.0,9.362,-2.0,-40.237,-1.0,"-Infinity","NaN",-83.774,-51.063,0.0,-96.234,-51.284,null,1.0,-6.491,-3.0],"l4_f64":[9.056,-3.0,null,-1.0,20.059,null,55.155,-1.0,-2.0,-37.396,1.604,-1.0,null,-35.521,27.489,null,4.0,null,3.0,-1.0,null,null,0.0,82.397,null,-3.0,60.581,23.866,null,-84.213,null,null,null,20.805,null,1.0,null,null,-2.0,null,3.0,null,-1.0,69.185,-3.0,null,-73.862,35.531,-3.376,-36.58,28.205,null,null,48.231,0.0,-4.0,"NaN",null,-76.614,null]}}},"ops":[{"op":"filter","predicate":{"fn":"isNull","args":[{"col":"l0_utf8"}]}},{"op":"head","n":1},{"op":"sort","keys":[{"expr":{"col":"l1_i32"},"descending":false}],"nullsLast":false},{"op":"unique"}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_f64","type":"f64"},{"name":"l4_f64","type":"f64"}],"columns":{"l0_utf8":[null],"l1_i32":[-2],"l2_utf8":[null],"l3_f64":[-69.56],"l4_f64":[-1.0]}}},{"id":"random/0240","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_f64","type":"f64"},{"name":"l4_i64","type":"i64"},{"name":"l5_f64","type":"f64"},{"name":"l6_i32","type":"i32"}],"columns":{"l0_utf8":["c","ab","a","c","aa","b","B","é","é","a","ab","b","ab","a","b","c","😀","aa","😀","é"],"l1_i32":[-1,0,-4,4,-4,-2147483648,4,-1,-5,1,5,1,4,-3,-1,929,-5,-5,2,3],"l2_i32":[0,-1,4,-4,-5,870,5,2,5,-2,-2,4,1,4,5,4,-4,-1,4,-2],"l3_f64":[4.0,-40.572,"NaN",-1.869,-3.0,17.415,"NaN",-4.0,7.265,4.0,-3.0,4.0,4.0,2.0,-1.0,-60.249,-1.0,0.0,-1.0,1.0],"l4_i64":["5","5","9007199254740993","9007199254740993","9007199254740994","4","-5","-4611686018427387904","5","9007199254740992","-4611686018427387904","3","1","-4611686018427387904","9007199254740995","-4611686018427387904","-4611686018427387904","0","-4611686018427387904","4"],"l5_f64":[25.856,"NaN",4.0,4.0,82.473,-52.108,15.155,2.0,0.0,-26.534,4.0,51.677,2.0,4.0,1.0,2.81,1.0,-49.718,2.0,0.0],"l6_i32":[-5,-3,-4,0,0,5,2147483647,-502,-1,-2,5,-3,4,-5,-1,4,-2147483648,0,-2,-1]}}},"ops":[{"op":"withColumns","exprs":{"x0":{"fn":"gte","args":[{"col":"l0_utf8"},{"lit":"aa"}]}}},{"op":"head","n":8},{"op":"filter","predicate":{"fn":"eq","args":[{"col":"l4_i64"},{"lit":"0","type":"i64"}]}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_f64","type":"f64"},{"name":"l4_i64","type":"i64"},{"name":"l5_f64","type":"f64"},{"name":"l6_i32","type":"i32"},{"name":"x0","type":"bool"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_i32":[],"l3_f64":[],"l4_i64":[],"l5_f64":[],"l6_i32":[],"x0":[]}}},{"id":"random/0241","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_i64","type":"i64"}],"columns":{"l0_utf8":["é","c","zz","b","ab","b",null,"B","aa","b","c","ab","😀","é","a","zz",null,"é","c","ab","😀","😀","ab","😀","","c","ab","B","aa","","é","c","ab","ab","😀","aa","😀","a","zz","😀","aa","aa","","b","ab","ab","","c",null,null,"aa","B","zz","c","B","","aa","B","ab","b"],"l1_i32":[3,661,1,-3,5,-4,2147483647,-1,3,null,4,2,-3,-2,2,-3,3,-1,2147483647,-2,-3,2147483647,-5,null,-2,1,0,0,4,-5,null,5,1,-4,3,-2147483648,-4,0,0,5,-2,-1,-5,-2,1,-4,1,-331,null,-2,3,-2,2,-1,-3,0,-4,0,4,4],"l2_bool":[false,true,false,false,true,true,false,false,false,false,true,true,false,false,false,true,null,false,true,false,true,true,null,true,false,false,false,true,false,true,true,null,true,false,false,false,false,true,true,true,false,null,false,true,true,true,true,false,true,false,true,false,true,null,false,false,true,false,true,false],"l3_i64":["-4611686018427387904","-2","5","9007199254740992","-4611686018427387904","-4611686018427387904","-3","9007199254740992","-4611686018427387904","-4611686018427387904","9007199254740994","-4611686018427387904","9007199254740994","9007199254740994","9007199254740995","-5","9007199254740995","-4611686018427387904","9007199254740995","9007199254740993","0","-1",null,"-4611686018427387904","-4611686018427387904","9007199254740993","-1","-4611686018427387904","-4611686018427387904",null,"0",null,"9007199254740995","0","-4611686018427387904","-4611686018427387904","-4611686018427387904","9007199254740995",null,null,"9007199254740993","2","5","9007199254740995","-4611686018427387904","-4611686018427387904",null,"-1","9007199254740994","3",null,null,"2","-4611686018427387904","-4611686018427387904","-4611686018427387904","0","-4611686018427387904","2","9007199254740995"]}}},"ops":[{"op":"unique"},{"op":"filter","predicate":{"fn":"isNull","args":[{"col":"l0_utf8"}]}},{"op":"withColumns","exprs":{"x2":{"fn":"neq","args":[{"col":"l1_i32"},{"fn":"abs","args":[{"col":"l1_i32"}]}]}}},{"op":"groupBy","keys":["x2"],"aggs":{"n3":{"agg":"len"},"a3_0":{"agg":"nUnique","arg":{"col":"l0_utf8"}},"a3_1":{"agg":"max","arg":{"col":"l3_i64"}}}}],"expected":{"fields":[{"name":"x2","type":"bool"},{"name":"n3","type":"u32"},{"name":"a3_0","type":"u32"},{"name":"a3_1","type":"i64"}],"columns":{"x2":[false,null,true],"n3":[2,1,1],"a3_0":[1,1,1],"a3_1":["9007199254740995","9007199254740994","3"]}}},{"id":"random/0242","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_bool","type":"bool"},{"name":"l5_bool","type":"bool"},{"name":"l6_utf8","type":"utf8"}],"columns":{"l0_utf8":["😀","ab","c","zz","😀"],"l1_i32":[0,-1,4,5,5],"l2_bool":[true,false,false,false,false],"l3_utf8":["","B","😀","é","é"],"l4_bool":[false,false,true,false,false],"l5_bool":[false,false,false,true,false],"l6_utf8":["zz","B","😀","ab","zz"]}}},"ops":[{"op":"unique"},{"op":"groupBy","keys":["l4_bool"],"aggs":{"n1":{"agg":"len"},"a1_0":{"agg":"last","arg":{"col":"l6_utf8"}},"a1_1":{"agg":"first","arg":{"col":"l6_utf8"}},"a1_2":{"agg":"count","arg":{"col":"l2_bool"}}}},{"op":"unique","columns":["l4_bool"]},{"op":"sort","keys":[{"expr":{"col":"l4_bool"},"descending":true}],"nullsLast":true}],"expected":{"fields":[{"name":"l4_bool","type":"bool"},{"name":"n1","type":"u32"},{"name":"a1_0","type":"utf8"},{"name":"a1_1","type":"utf8"},{"name":"a1_2","type":"u32"}],"columns":{"l4_bool":[true,false],"n1":[1,4],"a1_0":["😀","zz"],"a1_1":["😀","zz"],"a1_2":[1,4]}}},{"id":"random/0243","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_bool","type":"bool"}],"columns":{"l0_utf8":["","",null,"b","😀"],"l1_i32":[5,4,null,-4,null],"l2_bool":[false,true,true,true,false],"l3_bool":[false,false,true,true,true]}},"right":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"r0_utf8","type":"utf8"}],"columns":{"r0_utf8":["","B","B","","a","b","ab","",null,null,null,"B","aa","aa","",null,"a",null,"B","a","😀",null,"a",null,"B","B","ab","é",null,null],"l0_utf8":["b","b",null,"😀","","","😀","😀","","",null,"b","","","b","","b","","😀","","😀","b","😀","","😀","","","b","",null]}}},"ops":[{"op":"join","right":"right","on":["l0_utf8"],"how":"left"},{"op":"withColumns","exprs":{"x1":{"fn":"neq","args":[{"fn":"sub","args":[{"lit":2.5},{"lit":2.5}]},{"fn":"abs","args":[{"col":"l1_i32"}]}]}}},{"op":"filter","predicate":{"col":"l3_bool"}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_bool","type":"bool"},{"name":"r0_utf8","type":"utf8"},{"name":"x1","type":"bool"}],"columns":{"l0_utf8":[null,"b","b","b","b","b","b","b","😀","😀","😀","😀","😀","😀","😀"],"l1_i32":[null,-4,-4,-4,-4,-4,-4,-4,null,null,null,null,null,null,null],"l2_bool":[true,true,true,true,true,true,true,true,false,false,false,false,false,false,false],"l3_bool":[true,true,true,true,true,true,true,true,true,true,true,true,true,true,true],"r0_utf8":[null,"","B","B","","a",null,"é","","ab","","B","😀","a","B"],"x1":[null,true,true,true,true,true,true,true,null,null,null,null,null,null,null]}}},{"id":"random/0244","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_f64","type":"f64"},{"name":"l4_bool","type":"bool"},{"name":"l5_utf8","type":"utf8"}],"columns":{"l0_utf8":["c","ab",null,"c","é","","","a",null,null,"B","","","aa","é",null,null,null,null,"b","zz",null,null,null,"b","aa","","aa","😀","aa",null,"😀","","é",null,"😀","😀","b","a","ab",null,"😀","b",null,"zz","aa","é",null,"aa",null,"aa",null,"zz","b",null,"a","",null,"zz","é"],"l1_i32":[null,1,null,1,5,5,-2147483648,2147483647,null,3,null,-4,null,-5,1,-4,null,5,3,-4,null,null,0,null,1,3,-2147483648,-3,2,null,null,-5,null,-3,5,null,-1,null,null,null,3,-3,4,2,3,-5,null,-1,5,5,null,-5,-4,2,-4,null,null,463,3,-4],"l2_utf8":[null,"😀",null,"","","aa","aa","","é","é",null,"ab","b","ab","b","",null,"ab","😀","ab","",null,null,null,null,null,"aa",null,"aa","B",null,null,null,"aa","ab","aa","ab","é","é",null,"c",null,"B","é","b","aa",null,"c",null,"c",null,"c",null,"",null,"B","a","a","zz","zz"],"l3_f64":[4.0,null,3.0,4.0,2.0,4.0,39.481,null,null,2.0,3.0,-3.0,-12.291,-84.05,null,null,null,-58.788,-2.0,-15.772,-0.204,null,null,3.0,72.981,1.0,4.0,4.0,1.0,1.0,null,-1.0,"NaN",-3.0,null,75.078,0.0,null,2.0,45.946,3.0,4.0,68.869,-12.299,-3.0,-2.0,11.648,null,-61.455,null,0.0,76.771,-2.0,1.0,null,-93.902,4.0,null,99.297,"Infinity"],"l4_bool":[true,false,false,false,false,false,false,null,null,null,false,null,true,null,true,false,false,true,false,null,null,true,true,true,true,null,true,true,true,true,false,true,null,null,null,true,false,null,true,true,true,false,false,true,true,null,true,true,false,true,true,true,false,true,null,false,true,false,true,null],"l5_utf8":["ab","B","ab","B","c",null,"😀","","","","aa","é","a",null,null,"ab",null,"a","","B","b","b","B","b",null,"a","zz","a",null,"zz","ab","é","zz","B",null,"ab","😀",null,"a","é",null,null,"zz",null,"aa","a","B","",null,"a","","a","zz",null,null,"aa","zz","zz",null,"zz"]}}},"ops":[{"op":"filter","predicate":{"fn":"strStartsWith","args":[{"col":"l0_utf8"}],"prefix":""}},{"op":"withColumns","exprs":{"x1":{"fn":"div","args":[{"col":"l1_i32"},{"fn":"mul","args":[{"col":"l3_f64"},{"col":"l3_f64"}]}]}}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_f64","type":"f64"},{"name":"l4_bool","type":"bool"},{"name":"l5_utf8","type":"utf8"},{"name":"x1","type":"f64"}],"columns":{"l0_utf8":["c","ab","c","é","","","a","B","","","aa","é","b","zz","b","aa","","aa","😀","aa","😀","","é","😀","😀","b","a","ab","😀","b","zz","aa","é","aa","aa","zz","b","a","","zz","é"],"l1_i32":[null,1,1,5,5,-2147483648,2147483647,null,-4,null,-5,1,-4,null,1,3,-2147483648,-3,2,null,-5,null,-3,null,-1,null,null,null,-3,4,3,-5,null,5,null,-4,2,null,null,3,-4],"l2_utf8":[null,"😀","","","aa","aa","",null,"ab","b","ab","b","ab","",null,null,"aa",null,"aa","B",null,null,"aa","aa","ab","é","é",null,null,"B","b","aa",null,null,null,null,"","B","a","zz","zz"],"l3_f64":[4.0,null,4.0,2.0,4.0,39.481,null,3.0,-3.0,-12.291,-84.05,null,-15.772,-0.204,72.981,1.0,4.0,4.0,1.0,1.0,-1.0,"NaN",-3.0,75.078,0.0,null,2.0,45.946,4.0,68.869,-3.0,-2.0,11.648,-61.455,0.0,-2.0,1.0,-93.902,4.0,99.297,"Infinity"],"l4_bool":[true,false,false,false,false,false,null,false,null,true,null,true,null,null,true,null,true,true,true,true,true,null,null,true,false,null,true,true,false,false,true,null,true,false,true,false,true,false,true,true,null],"l5_utf8":["ab","B","B","c",null,"😀","","aa","é","a",null,null,"B","b",null,"a","zz","a",null,"zz","é","zz","B","ab","😀",null,"a","é",null,"zz","aa","a","B",null,"","zz",null,"aa","zz",null,"zz"],"x1":[null,null,0.0625,1.25,0.3125,-1377696.5699105745,null,null,-0.4444444444444444,null,-0.0007077739412498085,null,-0.016080015184679938,null,0.00018775018788579049,3.0,-134217728.0,-0.1875,2.0,null,-5.0,null,-0.3333333333333333,null,"-Infinity",null,null,null,-0.1875,0.0008433589097855181,0.3333333333333333,-1.25,null,0.0013239011466579232,null,-1.0,2.0,null,null,0.0003042628994195124,"-0"]}}},{"id":"random/0245","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_i32","type":"i32"},{"name":"l4_bool","type":"bool"},{"name":"l5_f64","type":"f64"},{"name":"l6_f64","type":"f64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_utf8":[],"l3_i32":[],"l4_bool":[],"l5_f64":[],"l6_f64":[]}}},"ops":[{"op":"groupBy","keys":["l3_i32"],"aggs":{"n0":{"agg":"len"},"a0_0":{"agg":"min","arg":{"col":"l0_utf8"}},"a0_1":{"agg":"min","arg":{"col":"l5_f64"}}}},{"op":"head","n":7}],"expected":{"fields":[{"name":"l3_i32","type":"i32"},{"name":"n0","type":"u32"},{"name":"a0_0","type":"utf8"},{"name":"a0_1","type":"f64"}],"columns":{"l3_i32":[],"n0":[],"a0_0":[],"a0_1":[]}}},{"id":"random/0246","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_f64","type":"f64"},{"name":"l6_i64","type":"i64"}],"columns":{"l0_utf8":["B","b","😀","B","zz"],"l1_i32":[null,-2147483648,-2,-2,1],"l2_utf8":["B","c","a","b","😀"],"l3_utf8":[null,"aa","c","zz","B"],"l4_utf8":["ab","b","😀","😀","c"],"l5_f64":[null,0.0,2.0,-45.254,18.6],"l6_i64":["-4611686018427387904",null,null,"2","9007199254740995"]}},"right":{"fields":[{"name":"l1_i32","type":"i32"},{"name":"r0_utf8","type":"utf8"}],"columns":{"r0_utf8":[],"l1_i32":[]}}},"ops":[{"op":"join","right":"right","on":["l1_i32"],"how":"anti"}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_f64","type":"f64"},{"name":"l6_i64","type":"i64"}],"columns":{"l0_utf8":["B","b","😀","B","zz"],"l1_i32":[null,-2147483648,-2,-2,1],"l2_utf8":["B","c","a","b","😀"],"l3_utf8":[null,"aa","c","zz","B"],"l4_utf8":["ab","b","😀","😀","c"],"l5_f64":[null,0.0,2.0,-45.254,18.6],"l6_i64":["-4611686018427387904",null,null,"2","9007199254740995"]}}},{"id":"random/0247","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_i32","type":"i32"},{"name":"l4_f64","type":"f64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_bool":[],"l3_i32":[],"l4_f64":[]}}},"ops":[{"op":"withColumns","exprs":{"x0":{"fn":"gte","args":[{"fn":"neg","args":[{"col":"l1_i32"}]},{"fn":"abs","args":[{"col":"l1_i32"}]}]}}},{"op":"filter","predicate":{"fn":"eq","args":[{"col":"l0_utf8"},{"lit":"aa"}]}},{"op":"head","n":0},{"op":"sort","keys":[{"expr":{"col":"l2_bool"},"descending":true}],"nullsLast":false}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_i32","type":"i32"},{"name":"l4_f64","type":"f64"},{"name":"x0","type":"bool"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_bool":[],"l3_i32":[],"l4_f64":[],"x0":[]}}},{"id":"random/0248","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_i64","type":"i64"}],"columns":{"l0_utf8":["aa","b","zz","B","a",null,"a","B","a","",null,"😀","ab","B","ab","a","é","ab",null,"ab","zz","a","aa",null,"é","b",null,"é",null,"c","aa","c","B","é","zz","c","ab",null,"ab","",null,"a","aa","zz",null,"ab","zz","B","zz","aa","ab","","a","ab","aa","zz","a","b","B","zz"],"l1_i32":[3,5,-2147483648,3,2,0,5,3,5,4,0,1,2,3,-3,5,1,-1,-3,-2,0,-1,-4,-2,-4,-2,5,null,0,4,3,-3,-4,4,null,-3,4,4,-3,-2,-4,-2147483648,5,-3,1,-1,-4,-5,1,0,4,1,4,4,0,1,3,0,-281,-4],"l2_f64":[-3.0,15.128,null,64.526,62.551,89.957,-77.7,48.174,-1.0,null,-19.471,43.356,-32.53,null,75.568,-2.0,34.511,80.155,-95.603,2.0,-4.0,4.0,2.0,"-0",-1.0,4.0,-89.485,-2.0,4.0,0.0,-75.144,1.595,null,90.841,20.587,-86.741,4.0,81.398,2.0,-2.0,-42.336,-31.887,null,-23.467,-2.0,"Infinity",-64.381,2.0,-31.847,4.0,-1.655,-3.0,null,1.0,null,-4.0,-3.0,0.0,5.333,73.821],"l3_i64":[null,"-4611686018427387904","-4611686018427387904",null,"-4611686018427387904","9007199254740993","9007199254740993","-1","9007199254740995","9007199254740994","9007199254740993","9007199254740994","-3","9007199254740993","5","-1","2","3","-3","-4611686018427387904","-4611686018427387904","9007199254740993","1","-4611686018427387904","9007199254740995","0","5","-4","-1",null,"9007199254740992","-5","9007199254740992","3","-2","9007199254740995","9007199254740994","9007199254740995","-4611686018427387904","-4611686018427387904","-5","2","9007199254740993","-4611686018427387904","9007199254740993","9007199254740992","4","9007199254740992","-1","9007199254740995","9007199254740994","2",null,"9007199254740994","9007199254740995","9007199254740995","3","-4","-4611686018427387904",null]}}},"ops":[{"op":"groupBy","keys":["l0_utf8"],"aggs":{"n0":{"agg":"len"},"a0_0":{"agg":"nUnique","arg":{"col":"l3_i64"}}}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"n0","type":"u32"},{"name":"a0_0","type":"u32"}],"columns":{"l0_utf8":["aa","b","zz","B","a",null,"","😀","ab","é","c"],"n0":[6,3,8,6,8,9,3,1,9,4,3],"a0_0":[5,3,6,5,7,7,3,1,6,4,3]}}},{"id":"random/0249","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_f64","type":"f64"},{"name":"l4_bool","type":"bool"},{"name":"l5_f64","type":"f64"},{"name":"l6_f64","type":"f64"}],"columns":{"l0_utf8":["a","aa","c","aa","","c","b","zz","a","a","é","c","ab","c","zz","B","zz","zz","B","zz"],"l1_i32":[-4,5,5,2,-2,5,4,-2,-1,244,4,1,4,1,2147483647,-4,-5,3,0,3],"l2_utf8":["b","B","aa","ab","b","zz","","b","c","😀","aa","B","a","😀","😀","😀","c","aa","zz","ab"],"l3_f64":[57.945,82.487,44.9,8.595,-1.0,47.409,2.0,-1.0,9.477,0.0,-3.0,3.0,4.0,96.591,-2.0,"NaN",-1.0,20.051,31.566,1.0],"l4_bool":[true,false,false,true,true,true,false,false,true,true,false,false,true,false,true,false,false,false,false,true],"l5_f64":[1.0,78.699,-64.51,-2.0,3.0,33.522,97.317,2.0,-99.269,82.531,2.0,-1.0,-66.606,2.0,-4.0,-4.0,"Infinity",-4.0,-4.0,-85.09],"l6_f64":[-4.0,93.604,-4.0,7.561,-54.185,3.0,1.0,2.0,-1.0,0.0,-61.743,-2.0,99.685,-4.334,-10.862,40.795,1.0,-75.639,-1.0,-2.0]}}},"ops":[{"op":"head","n":3}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_f64","type":"f64"},{"name":"l4_bool","type":"bool"},{"name":"l5_f64","type":"f64"},{"name":"l6_f64","type":"f64"}],"columns":{"l0_utf8":["a","aa","c"],"l1_i32":[-4,5,5],"l2_utf8":["b","B","aa"],"l3_f64":[57.945,82.487,44.9],"l4_bool":[true,false,false],"l5_f64":[1.0,78.699,-64.51],"l6_f64":[-4.0,93.604,-4.0]}}},{"id":"random/0250","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_f64","type":"f64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_i32":[],"l3_f64":[]}}},"ops":[{"op":"filter","predicate":{"fn":"isNull","args":[{"col":"l0_utf8"}]}},{"op":"sort","keys":[{"expr":{"col":"l2_i32"},"descending":true}],"nullsLast":true},{"op":"withColumns","exprs":{"x2":{"fn":"fillNull","args":[{"fn":"fillNull","args":[{"col":"l2_i32"},{"lit":1.5}]},{"lit":0}]}}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_f64","type":"f64"},{"name":"x2","type":"f64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_i32":[],"l3_f64":[],"x2":[]}}},{"id":"random/0251","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i64","type":"i64"},{"name":"l3_f64","type":"f64"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_i32","type":"i32"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_i64":[],"l3_f64":[],"l4_utf8":[],"l5_i32":[]}}},"ops":[{"op":"unique","columns":["l2_i64"]},{"op":"groupBy","keys":["l2_i64"],"aggs":{"n1":{"agg":"len"},"a1_0":{"agg":"min","arg":{"col":"l5_i32"}}}}],"expected":{"fields":[{"name":"l2_i64","type":"i64"},{"name":"n1","type":"u32"},{"name":"a1_0","type":"i32"}],"columns":{"l2_i64":[],"n1":[],"a1_0":[]}}},{"id":"random/0252","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_i64","type":"i64"},{"name":"l4_f64","type":"f64"},{"name":"l5_bool","type":"bool"}],"columns":{"l0_utf8":["ab"],"l1_i32":[-2],"l2_f64":[98.978],"l3_i64":["-4611686018427387904"],"l4_f64":[-18.877],"l5_bool":[true]}}},"ops":[{"op":"withColumns","exprs":{"x0":{"fn":"sub","args":[{"fn":"neg","args":[{"col":"l4_f64"}]},{"fn":"abs","args":[{"lit":-1}]}]}}},{"op":"withColumns","exprs":{"x1":{"fn":"gt","args":[{"fn":"add","args":[{"col":"l2_f64"},{"col":"l4_f64"}]},{"col":"l4_f64"}]}}},{"op":"withColumns","exprs":{"x2":{"fn":"gte","args":[{"fn":"sub","args":[{"col":"l4_f64"},{"col":"x0"}]},{"col":"x0"}]}}},{"op":"groupBy","keys":["l0_utf8"],"aggs":{"n3":{"agg":"len"},"a3_0":{"agg":"first","arg":{"col":"x2"}},"a3_1":{"agg":"first","arg":{"col":"l2_f64"}}}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"n3","type":"u32"},{"name":"a3_0","type":"bool"},{"name":"a3_1","type":"f64"}],"columns":{"l0_utf8":["ab"],"n3":[1],"a3_0":[false],"a3_1":[98.978]}}},{"id":"random/0253","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_utf8","type":"utf8"}],"columns":{"l0_utf8":["c"],"l1_i32":[4],"l2_utf8":["b"],"l3_utf8":["a"]}}},"ops":[{"op":"groupBy","keys":["l0_utf8"],"aggs":{"n0":{"agg":"len"},"a0_0":{"agg":"count","arg":{"col":"l3_utf8"}},"a0_1":{"agg":"max","arg":{"col":"l2_utf8"}},"a0_2":{"agg":"min","arg":{"col":"l3_utf8"}}}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"n0","type":"u32"},{"name":"a0_0","type":"u32"},{"name":"a0_1","type":"utf8"},{"name":"a0_2","type":"utf8"}],"columns":{"l0_utf8":["c"],"n0":[1],"a0_0":[1],"a0_1":["b"],"a0_2":["a"]}}},{"id":"random/0254","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_utf8","type":"utf8"}],"columns":{"l0_utf8":["😀","c","ab","a","ab"],"l1_i32":[-3,-2,0,2,-4],"l2_f64":[-75.476,2.0,2.0,-83.161,1.0],"l3_utf8":["a","aa","B","aa","😀"]}}},"ops":[{"op":"head","n":0},{"op":"sort","keys":[{"expr":{"col":"l2_f64"},"descending":false},{"expr":{"col":"l1_i32"},"descending":true}],"nullsLast":true},{"op":"head","n":4},{"op":"head","n":3}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_utf8","type":"utf8"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_f64":[],"l3_utf8":[]}}},{"id":"random/0255","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_f64","type":"f64"},{"name":"l4_i32","type":"i32"},{"name":"l5_utf8","type":"utf8"},{"name":"l6_bool","type":"bool"}],"columns":{"l0_utf8":["😀","é",null,"aa","B","ab","😀","a","B",null,"é","c","c","","é","",null,"c","😀","zz"],"l1_i32":[2,null,-1,-1,5,-4,null,-1,-2,-1,-4,null,null,null,-1,null,null,4,1,0],"l2_bool":[null,false,true,true,false,null,false,false,true,false,true,null,null,false,true,false,true,null,true,true],"l3_f64":[-69.949,-1.0,null,-13.598,83.66,65.863,-3.0,null,-2.0,"NaN",null,null,null,63.318,-79.414,null,4.0,3.0,60.217,-4.0],"l4_i32":[5,-3,-4,-5,4,-2,5,-2,null,4,-3,2147483647,-2,null,2147483647,null,-3,4,-2,-2],"l5_utf8":["😀","ab","a","😀","ab","aa","b",null,null,null,null,"é","","zz","aa","😀","zz","","😀",null],"l6_bool":[null,true,false,true,true,false,true,false,false,false,true,true,false,null,true,null,false,null,false,false]}}},"ops":[{"op":"sort","keys":[{"expr":{"col":"l2_bool"},"descending":true},{"expr":{"col":"l5_utf8"},"descending":true}],"nullsLast":false},{"op":"groupBy","keys":["l0_utf8"],"aggs":{"n1":{"agg":"len"},"a1_0":{"agg":"nUnique","arg":{"col":"l1_i32"}}}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"n1","type":"u32"},{"name":"a1_0","type":"u32"}],"columns":{"l0_utf8":["😀","c","ab","B","é","zz","aa",null,"a",""],"n1":[3,3,1,2,3,1,1,3,1,2],"a1_0":[3,2,1,2,3,1,1,2,1,1]}}},{"id":"random/0256","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_i64","type":"i64"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_utf8","type":"utf8"}],"columns":{"l0_utf8":["ab"],"l1_i32":[-1],"l2_utf8":[null],"l3_i64":["3"],"l4_utf8":["aa"],"l5_utf8":["aa"]}},"right":{"fields":[{"name":"l4_utf8","type":"utf8"},{"name":"r0_bool","type":"bool"}],"columns":{"r0_bool":[],"l4_utf8":[]}}},"ops":[{"op":"join","right":"right","on":["l4_utf8"],"how":"inner"}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_i64","type":"i64"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_utf8","type":"utf8"},{"name":"r0_bool","type":"bool"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_utf8":[],"l3_i64":[],"l4_utf8":[],"l5_utf8":[],"r0_bool":[]}}},{"id":"random/0257","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_f64","type":"f64"},{"name":"l4_i64","type":"i64"},{"name":"l5_f64","type":"f64"},{"name":"l6_f64","type":"f64"}],"columns":{"l0_utf8":["B",null,"B","ab","zz","ab","é","😀",null,"B","zz","é",null,"B","aa","aa","aa","ab","c","a"],"l1_i32":[5,-1,5,-2147483648,-4,-2,-1,5,0,-1,-2,4,4,-2147483648,-1,1,1,3,3,5],"l2_i32":[2,4,3,3,-4,5,null,-2,1,0,null,-4,2,null,-3,0,5,3,0,-795],"l3_f64":[-83.741,"NaN",0.0,-2.0,0.0,0.0,-5.782,1.0,61.608,-1.0,"Infinity",-1.0,-32.064,-4.0,1.0,0.0,-82.663,-84.411,-2.0,31.84],"l4_i64":["-3","9007199254740993","-4611686018427387904","-4",null,null,null,"-4611686018427387904","-4611686018427387904","9007199254740994",null,"-4611686018427387904","-4611686018427387904","-4611686018427387904","9007199254740994","9007199254740995","3",null,"9007199254740993","0"],"l5_f64":[38.438,3.0,-39.995,-71.935,-1.0,-72.475,null,-83.312,"NaN",-15.703,52.806,8.177,3.0,-69.348,null,null,-80.234,-2.0,62.93,null],"l6_f64":["NaN",71.247,6.244,49.364,-4.0,-43.941,null,-4.909,30.102,4.0,null,-3.0,-2.0,51.341,-2.0,-47.749,3.0,-99.466,14.958,66.319]}}},"ops":[{"op":"sort","keys":[{"expr":{"col":"l4_i64"},"descending":false},{"expr":{"col":"l0_utf8"},"descending":false}],"nullsLast":false},{"op":"unique"},{"op":"withColumns","exprs":{"x2":{"lit":-1}}},{"op":"filter","predicate":{"fn":"neq","args":[{"col":"l5_f64"},{"col":"l3_f64"}]}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_f64","type":"f64"},{"name":"l4_i64","type":"i64"},{"name":"l5_f64","type":"f64"},{"name":"l6_f64","type":"f64"},{"name":"x2","type":"i32"}],"columns":{"l0_utf8":["ab","ab","zz","zz",null,null,"B","B","é","😀","ab","B","aa",null,"c","B"],"l1_i32":[-2,3,-4,-2,0,4,5,-2147483648,4,5,-2147483648,5,1,-1,3,-1],"l2_i32":[5,3,-4,null,1,2,3,null,-4,-2,3,2,5,4,0,0],"l3_f64":[0.0,-84.411,0.0,"Infinity",61.608,-32.064,0.0,-4.0,-1.0,1.0,-2.0,-83.741,-82.663,"NaN",-2.0,-1.0],"l4_i64":[null,null,null,null,"-4611686018427387904","-4611686018427387904","-4611686018427387904","-4611686018427387904","-4611686018427387904","-4611686018427387904","-4","-3","3","9007199254740993","9007199254740993","9007199254740994"],"l5_f64":[-72.475,-2.0,-1.0,52.806,"NaN",3.0,-39.995,-69.348,8.177,-83.312,-71.935,38.438,-80.234,3.0,62.93,-15.703],"l6_f64":[-43.941,-99.466,-4.0,null,30.102,-2.0,6.244,51.341,-3.0,-4.909,49.364,"NaN",3.0,71.247,14.958,4.0],"x2":[-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1]}}},{"id":"random/0258","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_utf8","type":"utf8"}],"columns":{"l0_utf8":[null],"l1_i32":[3],"l2_f64":["NaN"],"l3_utf8":["B"]}}},"ops":[{"op":"withColumns","exprs":{"x0":{"col":"l2_f64"}}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_utf8","type":"utf8"},{"name":"x0","type":"f64"}],"columns":{"l0_utf8":[null],"l1_i32":[3],"l2_f64":["NaN"],"l3_utf8":["B"],"x0":["NaN"]}}},{"id":"random/0259","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_i64","type":"i64"}],"columns":{"l0_utf8":["a","c","b","ab","a"],"l1_i32":[-5,3,3,4,2],"l2_utf8":["c","aa","b","aa",""],"l3_i64":["-5","-1","-2","-4611686018427387904","1"]}}},"ops":[{"op":"head","n":10},{"op":"groupBy","keys":["l2_utf8"],"aggs":{"n1":{"agg":"len"},"a1_0":{"agg":"min","arg":{"col":"l0_utf8"}},"a1_1":{"agg":"max","arg":{"col":"l1_i32"}},"a1_2":{"agg":"median","arg":{"col":"l1_i32"}}}},{"op":"sort","keys":[{"expr":{"col":"l2_utf8"},"descending":false}],"nullsLast":true},{"op":"withColumns","exprs":{"x3":{"fn":"neg","args":[{"fn":"sub","args":[{"col":"a1_1"},{"col":"a1_1"}]}]}}}],"expected":{"fields":[{"name":"l2_utf8","type":"utf8"},{"name":"n1","type":"u32"},{"name":"a1_0","type":"utf8"},{"name":"a1_1","type":"i32"},{"name":"a1_2","type":"f64"},{"name":"x3","type":"i32"}],"columns":{"l2_utf8":["","aa","b","c"],"n1":[1,2,1,1],"a1_0":["a","ab","b","a"],"a1_1":[2,4,3,-5],"a1_2":[2.0,3.5,3.0,-5.0],"x3":[0,0,0,0]}}},{"id":"random/0260","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i64","type":"i64"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_utf8","type":"utf8"}],"columns":{"l0_utf8":["c"],"l1_i32":[-417],"l2_i64":["9007199254740995"],"l3_utf8":["B"],"l4_utf8":["B"],"l5_utf8":["aa"]}},"right":{"fields":[{"name":"l4_utf8","type":"utf8"},{"name":"r0_f64","type":"f64"}],"columns":{"r0_f64":[84.015,-30.607,1.0],"l4_utf8":["B","B","B"]}}},"ops":[{"op":"join","right":"right","on":["l4_utf8"],"how":"semi"},{"op":"head","n":5},{"op":"groupBy","keys":["l4_utf8"],"aggs":{"n2":{"agg":"len"},"a2_0":{"agg":"count","arg":{"col":"l1_i32"}}}},{"op":"groupBy","keys":["n2"],"aggs":{"n3":{"agg":"len"},"a3_0":{"agg":"min","arg":{"col":"l4_utf8"}},"a3_1":{"agg":"last","arg":{"col":"a2_0"}}}}],"expected":{"fields":[{"name":"n2","type":"u32"},{"name":"n3","type":"u32"},{"name":"a3_0","type":"utf8"},{"name":"a3_1","type":"u32"}],"columns":{"n2":[1],"n3":[1],"a3_0":["B"],"a3_1":[1]}}},{"id":"random/0261","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_bool","type":"bool"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_i64","type":"i64"}],"columns":{"l0_utf8":["ab","a","b",null,"b","😀","zz","zz","é","aa","ab","b","ab","a","aa","b","é",null,"é","😀"],"l1_i32":[-3,-4,-1,4,387,-4,-2,-2,4,2,1,-5,-4,5,-4,2,4,-2,1,224],"l2_f64":[67.664,45.897,0.0,null,-1.0,3.0,-1.0,-2.0,2.0,-2.0,58.538,3.0,-81.989,50.055,null,-13.973,44.091,"NaN",2.0,-47.32],"l3_bool":[false,false,true,false,true,true,false,true,false,false,null,false,true,true,false,true,null,true,null,true],"l4_utf8":["B","","c","😀","é","aa",null,"é",null,"é","zz","b","ab","b","zz",null,null,"","b","a"],"l5_i64":["9007199254740992","2",null,"-4611686018427387904","1","-4611686018427387904",null,"9007199254740992","-1","9007199254740992","-2","-2","9007199254740993","-4611686018427387904","9007199254740992","-4611686018427387904","9007199254740995","-4611686018427387904","2",null]}}},"ops":[{"op":"head","n":0},{"op":"sort","keys":[{"expr":{"col":"l1_i32"},"descending":false},{"expr":{"col":"l5_i64"},"descending":false}],"nullsLast":false}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_bool","type":"bool"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_i64","type":"i64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_f64":[],"l3_bool":[],"l4_utf8":[],"l5_i64":[]}}},{"id":"random/0262","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_f64","type":"f64"}],"columns":{"l0_utf8":[null,"c","b","ab",""],"l1_i32":[-5,261,-4,3,2],"l2_utf8":["zz","é","B","é",""],"l3_f64":[-99.638,-73.602,"NaN",-1.0,-1.0]}},"right":{"fields":[{"name":"l2_utf8","type":"utf8"},{"name":"r0_bool","type":"bool"}],"columns":{"r0_bool":[],"l2_utf8":[]}}},"ops":[{"op":"sort","keys":[{"expr":{"col":"l0_utf8"},"descending":false}],"nullsLast":true},{"op":"join","right":"right","on":["l2_utf8"],"how":"full"},{"op":"unique","columns":["l1_i32"]},{"op":"filter","predicate":{"fn":"or","args":[{"fn":"lte","args":[{"col":"l3_f64"},{"col":"l1_i32"}]},{"fn":"isNotNull","args":[{"col":"l0_utf8"}]}]}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_f64","type":"f64"},{"name":"r0_bool","type":"bool"}],"columns":{"l0_utf8":["","ab","b","c",null],"l1_i32":[2,3,-4,261,-5],"l2_utf8":["","é","B","é","zz"],"l3_f64":[-1.0,-1.0,"NaN",-73.602,-99.638],"r0_bool":[null,null,null,null,null]}}},{"id":"random/0263","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_f64","type":"f64"},{"name":"l4_f64","type":"f64"},{"name":"l5_utf8","type":"utf8"}],"columns":{"l0_utf8":["😀","é","aa","","B"],"l1_i32":[0,2,1,-5,-3],"l2_bool":[true,true,null,false,null],"l3_f64":[-72.163,-0.197,-1.0,31.045,3.0],"l4_f64":[28.357,71.659,"Infinity",-22.611,33.707],"l5_utf8":["",null,"B","é","c"]}}},"ops":[{"op":"groupBy","keys":["l0_utf8"],"aggs":{"n0":{"agg":"len"},"a0_0":{"agg":"median","arg":{"col":"l4_f64"}},"a0_1":{"agg":"min","arg":{"col":"l5_utf8"}},"a0_2":{"agg":"min","arg":{"col":"l5_utf8"}}}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"n0","type":"u32"},{"name":"a0_0","type":"f64"},{"name":"a0_1","type":"utf8"},{"name":"a0_2","type":"utf8"}],"columns":{"l0_utf8":["😀","é","aa","","B"],"n0":[1,1,1,1,1],"a0_0":[28.357,71.659,"Infinity",-22.611,33.707],"a0_1":["",null,"B","é","c"],"a0_2":["",null,"B","é","c"]}}},{"id":"random/0264","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_utf8","type":"utf8"}],"columns":{"l0_utf8":[null,null,"ab","B","zz"],"l1_i32":[-879,4,null,-4,-2],"l2_utf8":[null,null,"aa","ab","c"],"l3_utf8":[null,"c","B","a","B"],"l4_utf8":[null,null,null,"zz","aa"]}}},"ops":[{"op":"unique"},{"op":"filter","predicate":{"fn":"lt","args":[{"col":"l2_utf8"},{"lit":"ab"}]}},{"op":"filter","predicate":{"fn":"isNotNull","args":[{"col":"l0_utf8"}]}},{"op":"unique","columns":["l0_utf8"]}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_utf8","type":"utf8"}],"columns":{"l0_utf8":["ab"],"l1_i32":[null],"l2_utf8":["aa"],"l3_utf8":["B"],"l4_utf8":[null]}}},{"id":"random/0265","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_i64","type":"i64"},{"name":"l5_bool","type":"bool"},{"name":"l6_bool","type":"bool"}],"columns":{"l0_utf8":["","é","a","B","😀"],"l1_i32":[-2147483648,4,-3,-2,1],"l2_f64":[-94.735,-4.0,-93.718,-5.125,-6.849],"l3_utf8":["aa","😀","aa","aa","zz"],"l4_i64":["-4611686018427387904","9007199254740993","9007199254740993","4","9007199254740992"],"l5_bool":[true,false,false,false,true],"l6_bool":[false,false,true,true,false]}}},"ops":[{"op":"filter","predicate":{"fn":"and","args":[{"fn":"eq","args":[{"lit":2},{"col":"l1_i32"}]},{"fn":"isNull","args":[{"col":"l1_i32"}]}]}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_i64","type":"i64"},{"name":"l5_bool","type":"bool"},{"name":"l6_bool","type":"bool"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_f64":[],"l3_utf8":[],"l4_i64":[],"l5_bool":[],"l6_bool":[]}}},{"id":"random/0266","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_i64","type":"i64"},{"name":"l4_i64","type":"i64"},{"name":"l5_utf8","type":"utf8"},{"name":"l6_i64","type":"i64"}],"columns":{"l0_utf8":["c",null,null,null,"b"],"l1_i32":[null,1,-5,-2147483648,-2],"l2_bool":[true,false,true,true,false],"l3_i64":["-4611686018427387904",null,"3","-4611686018427387904","-4611686018427387904"],"l4_i64":["-1","-4611686018427387904","9007199254740995","2","-4"],"l5_utf8":["😀","ab","","é","zz"],"l6_i64":["-1",null,"-4611686018427387904",null,"-4611686018427387904"]}}},"ops":[{"op":"filter","predicate":{"fn":"lte","args":[{"fn":"div","args":[{"col":"l1_i32"},{"col":"l1_i32"}]},{"fn":"sub","args":[{"col":"l1_i32"},{"col":"l1_i32"}]}]}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_i64","type":"i64"},{"name":"l4_i64","type":"i64"},{"name":"l5_utf8","type":"utf8"},{"name":"l6_i64","type":"i64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_bool":[],"l3_i64":[],"l4_i64":[],"l5_utf8":[],"l6_i64":[]}}},{"id":"random/0267","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i64","type":"i64"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_i64","type":"i64"},{"name":"l6_utf8","type":"utf8"}],"columns":{"l0_utf8":["a","a","é","ab","B","c","","zz","c","a","b","b","","b","","a","B",null,"","c",null,"zz","aa","","","b","B","b",null,"b","a","aa","zz","😀","","ab","é","zz","zz","ab","zz","c","B","a","zz","aa","","b","b","ab",null,"b","","aa","é","😀","zz","a","","B"],"l1_i32":[1,0,-2147483648,1,3,-4,2,-5,-5,4,5,-98,-3,0,-3,3,4,5,-2147483648,4,3,-1,-1,2,-1,-4,-4,-5,0,null,5,2,0,2,4,-2,null,2,4,5,5,-2,-3,5,2,0,-4,-3,-5,5,1,-2,2,-5,5,5,2,-4,-1,2147483647],"l2_i64":["3","-4611686018427387904","-3","-4611686018427387904","-4611686018427387904","9007199254740993","2","9007199254740994","9007199254740994","-4611686018427387904","9007199254740993","-4611686018427387904","9007199254740994","9007199254740993","9007199254740993","9007199254740992","-5",null,"-2","-2","9007199254740993","-4611686018427387904","9007199254740993","-4","-4611686018427387904","-4611686018427387904",null,"-4611686018427387904","-4611686018427387904","9007199254740992","9007199254740995","9007199254740993","-4611686018427387904","-4611686018427387904","-4611686018427387904","-5","-4611686018427387904","9007199254740993","9007199254740995","-4611686018427387904","9007199254740994","4",null,"9007199254740995","9007199254740995","-3","9007199254740993","9007199254740994","9007199254740992","-4611686018427387904","9007199254740993",null,"9007199254740992","9007199254740993","-1","-4","-1","-4611686018427387904","9007199254740995","-4611686018427387904"],"l3_utf8":["aa","😀","c","😀",null,"b","aa","😀","b","c","","😀","zz","a","B","b","B","é","b","ab","ab","b","ab","B","aa","B","B",null,"",null,null,"b","😀","ab","zz","b","B","","ab",null,"ab","a",null,"B","b","zz","é","a","b",null,"😀","😀","c","a","aa","B","zz","","B",null],"l4_utf8":["ab",null,"b","","zz","a","B","😀",null,"","aa","",null,"zz",null,"a",null,"zz","B",null,null,"ab","zz","ab","😀","😀","b","ab","ab","c","c",null,"a","B",null,"B","ab","zz","é","B","a","c","zz",null,"b","c",null,"é","B","ab","aa","zz","aa","é","aa","ab","a","B","c","a"],"l5_i64":["3","2","4","-5","-4611686018427387904","-5","9007199254740993","-2","-1","-4611686018427387904","9007199254740995","-4611686018427387904","0","9007199254740994","-4611686018427387904","3","9007199254740993","1",null,"-4611686018427387904","-4611686018427387904","9007199254740995","9007199254740995","-4611686018427387904","-4611686018427387904",null,"-4611686018427387904","-4611686018427387904","-5","-4611686018427387904","-4611686018427387904","-3","-1","9007199254740993","-4611686018427387904","-5","-2","-4611686018427387904","-2","4","-4611686018427387904","9007199254740995","-4611686018427387904","9007199254740995","9007199254740994","9007199254740995","9007199254740993","-5","-1","-4611686018427387904","-2","9007199254740993","4",null,"9007199254740992","9007199254740993","1","-4611686018427387904","-2","-4611686018427387904"],"l6_utf8":["😀",null,"zz","é","é","zz","c","aa","c","zz",null,"c",null,"c",null,"","aa","ab","b","","😀","zz","😀",null,"a","zz",null,"é","ab",null,"c","zz","é","","B","c","c","","ab","",null,"b","aa","b","é","b","aa","","","ab","aa","é","ab",null,"ab","zz","😀","aa","é","B"]}}},"ops":[{"op":"groupBy","keys":["l0_utf8"],"aggs":{"n0":{"agg":"len"},"a0_0":{"agg":"nUnique","arg":{"col":"l2_i64"}}}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"n0","type":"u32"},{"name":"a0_0","type":"u32"}],"columns":{"l0_utf8":["a","é","ab","B","c","","zz","b",null,"aa","😀"],"n0":[7,3,4,5,4,10,8,9,4,4,2],"a0_0":[4,3,2,3,4,8,5,5,3,2,2]}}},{"id":"random/0268","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i64","type":"i64"},{"name":"l3_i32","type":"i32"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_i64":[],"l3_i32":[]}}},"ops":[{"op":"withColumns","exprs":{"x0":{"col":"l3_i32"}}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i64","type":"i64"},{"name":"l3_i32","type":"i32"},{"name":"x0","type":"i32"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_i64":[],"l3_i32":[],"x0":[]}}},{"id":"random/0269","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_i64","type":"i64"},{"name":"l4_bool","type":"bool"}],"columns":{"l0_utf8":[null,"ab",null,"","ab"],"l1_i32":[-5,null,4,2,2],"l2_utf8":["😀","😀",null,null,"B"],"l3_i64":[null,"-4611686018427387904","9007199254740995","2","9007199254740993"],"l4_bool":[true,null,false,false,false]}}},"ops":[{"op":"head","n":1},{"op":"filter","predicate":{"fn":"lte","args":[{"lit":-1},{"col":"l1_i32"}]}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_i64","type":"i64"},{"name":"l4_bool","type":"bool"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_utf8":[],"l3_i64":[],"l4_bool":[]}}},{"id":"random/0270","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_bool","type":"bool"},{"name":"l4_f64","type":"f64"},{"name":"l5_utf8","type":"utf8"},{"name":"l6_f64","type":"f64"}],"columns":{"l0_utf8":[null,"😀","é","zz","é"],"l1_i32":[null,-3,3,3,2],"l2_utf8":[null,"ab","ab","é","😀"],"l3_bool":[false,false,true,false,false],"l4_f64":[null,-62.83,"NaN",0.0,null],"l5_utf8":["aa","é",null,"c",""],"l6_f64":["NaN",-4.0,-30.748,-4.0,2.0]}}},"ops":[{"op":"filter","predicate":{"fn":"strStartsWith","args":[{"col":"l0_utf8"}],"prefix":""}},{"op":"withColumns","exprs":{"x1":{"lit":2}}},{"op":"withColumns","exprs":{"x2":{"col":"l1_i32"}}},{"op":"groupBy","keys":["l2_utf8"],"aggs":{"n3":{"agg":"len"},"a3_0":{"agg":"count","arg":{"col":"l5_utf8"}}}}],"expected":{"fields":[{"name":"l2_utf8","type":"utf8"},{"name":"n3","type":"u32"},{"name":"a3_0","type":"u32"}],"columns":{"l2_utf8":["ab","é","😀"],"n3":[2,1,1],"a3_0":[1,1,1]}}},{"id":"random/0271","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_bool","type":"bool"},{"name":"l4_i64","type":"i64"}],"columns":{"l0_utf8":["aa",null,"😀","é","ab","aa","ab","ab",null,"a",null,"a","b","B","","ab",null,"a",null,"zz"],"l1_i32":[null,-1,null,2,null,2,-5,null,null,4,-1,null,null,-1,-5,-4,-3,null,-5,0],"l2_f64":[null,88.906,0.0,1.131,29.86,24.466,null,-74.316,2.0,-61.404,null,-50.65,null,-33.186,-99.577,0.0,-2.0,null,-97.617,-50.398],"l3_bool":[true,false,true,true,true,false,true,null,false,false,false,true,false,false,false,null,true,true,null,true],"l4_i64":[null,null,null,"9007199254740994","9007199254740993",null,"1","-4611686018427387904","0",null,"9007199254740994","9007199254740992","-4611686018427387904","9007199254740994","9007199254740995","9007199254740994","-4611686018427387904","-4611686018427387904",null,"9007199254740992"]}}},"ops":[{"op":"sort","keys":[{"expr":{"col":"l3_bool"},"descending":false},{"expr":{"col":"l2_f64"},"descending":false}],"nullsLast":false}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_bool","type":"bool"},{"name":"l4_i64","type":"i64"}],"columns":{"l0_utf8":[null,"ab","ab",null,"b","","a","B",null,"aa",null,"aa","ab","a","a","zz",null,"😀","é","ab"],"l1_i32":[-5,null,-4,-1,null,-5,4,-1,null,2,-1,null,-5,null,null,0,-3,null,2,null],"l2_f64":[-97.617,-74.316,0.0,null,null,-99.577,-61.404,-33.186,2.0,24.466,88.906,null,null,null,-50.65,-50.398,-2.0,0.0,1.131,29.86],"l3_bool":[null,null,null,false,false,false,false,false,false,false,false,true,true,true,true,true,true,true,true,true],"l4_i64":[null,"-4611686018427387904","9007199254740994","9007199254740994","-4611686018427387904","9007199254740995",null,"9007199254740994","0",null,null,null,"1","-4611686018427387904","9007199254740992","9007199254740992","-4611686018427387904",null,"9007199254740994","9007199254740993"]}}},{"id":"random/0272","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_utf8","type":"utf8"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_i32":[],"l3_utf8":[]}}},"ops":[{"op":"filter","predicate":{"fn":"and","args":[{"fn":"lte","args":[{"col":"l1_i32"},{"col":"l1_i32"}]},{"fn":"gte","args":[{"col":"l1_i32"},{"col":"l1_i32"}]}]}},{"op":"filter","predicate":{"fn":"neq","args":[{"fn":"mod","args":[{"col":"l2_i32"},{"col":"l1_i32"}]},{"fn":"mul","args":[{"col":"l1_i32"},{"lit":0.5}]}]}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_utf8","type":"utf8"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_i32":[],"l3_utf8":[]}}},{"id":"random/0273","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_i64","type":"i64"},{"name":"l4_f64","type":"f64"},{"name":"l5_i32","type":"i32"}],"columns":{"l0_utf8":["c","","zz","B",null],"l1_i32":[-2,2,null,5,4],"l2_utf8":["ab","ab","b","c","😀"],"l3_i64":["-4611686018427387904","-4","-4611686018427387904","-4611686018427387904","9007199254740995"],"l4_f64":[26.061,1.0,80.89,80.11,-93.81],"l5_i32":[-1,null,null,0,null]}},"right":{"fields":[{"name":"l1_i32","type":"i32"},{"name":"r0_utf8","type":"utf8"}],"columns":{"r0_utf8":[],"l1_i32":[]}}},"ops":[{"op":"join","right":"right","on":["l1_i32"],"how":"full"},{"op":"head","n":10}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_i64","type":"i64"},{"name":"l4_f64","type":"f64"},{"name":"l5_i32","type":"i32"},{"name":"r0_utf8","type":"utf8"}],"columns":{"l0_utf8":["c","","zz","B",null],"l1_i32":[-2,2,null,5,4],"l2_utf8":["ab","ab","b","c","😀"],"l3_i64":["-4611686018427387904","-4","-4611686018427387904","-4611686018427387904","9007199254740995"],"l4_f64":[26.061,1.0,80.89,80.11,-93.81],"l5_i32":[-1,null,null,0,null],"r0_utf8":[null,null,null,null,null]}}},{"id":"random/0274","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_bool","type":"bool"}],"columns":{"l0_utf8":["c"],"l1_i32":[-3],"l2_f64":[null],"l3_bool":[null]}}},"ops":[{"op":"groupBy","keys":["l1_i32"],"aggs":{"n0":{"agg":"len"},"a0_0":{"agg":"last","arg":{"col":"l2_f64"}}}}],"expected":{"fields":[{"name":"l1_i32","type":"i32"},{"name":"n0","type":"u32"},{"name":"a0_0","type":"f64"}],"columns":{"l1_i32":[-3],"n0":[1],"a0_0":[null]}}},{"id":"random/0275","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i64","type":"i64"},{"name":"l3_i64","type":"i64"},{"name":"l4_i32","type":"i32"},{"name":"l5_i32","type":"i32"},{"name":"l6_f64","type":"f64"}],"columns":{"l0_utf8":["","ab","b","a","b"],"l1_i32":[1,0,-5,-4,-5],"l2_i64":["9007199254740995","-4611686018427387904","-1","-4611686018427387904","9007199254740992"],"l3_i64":["9007199254740993","-4611686018427387904","9007199254740994","-2","9007199254740992"],"l4_i32":[3,-2147483648,2,3,-3],"l5_i32":[5,-5,-2147483648,1,5],"l6_f64":[3.0,-3.0,-77.309,-64.987,-2.0]}},"right":{"fields":[{"name":"l4_i32","type":"i32"},{"name":"r0_f64","type":"f64"}],"columns":{"r0_f64":[-18.789,-68.118,4.0],"l4_i32":[3,-2147483648,2]}}},"ops":[{"op":"join","right":"right","on":["l4_i32"],"how":"left"}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i64","type":"i64"},{"name":"l3_i64","type":"i64"},{"name":"l4_i32","type":"i32"},{"name":"l5_i32","type":"i32"},{"name":"l6_f64","type":"f64"},{"name":"r0_f64","type":"f64"}],"columns":{"l0_utf8":["","ab","b","a","b"],"l1_i32":[1,0,-5,-4,-5],"l2_i64":["9007199254740995","-4611686018427387904","-1","-4611686018427387904","9007199254740992"],"l3_i64":["9007199254740993","-4611686018427387904","9007199254740994","-2","9007199254740992"],"l4_i32":[3,-2147483648,2,3,-3],"l5_i32":[5,-5,-2147483648,1,5],"l6_f64":[3.0,-3.0,-77.309,-64.987,-2.0],"r0_f64":[-18.789,-68.118,4.0,-18.789,null]}}},{"id":"random/0276","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i64","type":"i64"},{"name":"l3_i32","type":"i32"},{"name":"l4_f64","type":"f64"},{"name":"l5_i32","type":"i32"},{"name":"l6_i32","type":"i32"}],"columns":{"l0_utf8":["😀","c","aa","😀","","c","ab","ab","","c","","aa","aa","zz","zz","B","ab","aa","B","aa","zz","zz","zz","a","aa","ab","ab","c","ab","zz","b","aa","zz","zz","ab","B","zz","b","a","","B","a","b","b","😀","b","","é","😀","aa","c","b","é","c","b","aa","ab","é","c","é"],"l1_i32":[3,-4,0,-2,0,423,-1,-3,4,4,3,5,-5,3,4,-5,-2147483648,-2,5,-2,2,-5,5,-2147483648,-4,-2,4,4,0,0,3,-2147483648,1,5,-1,-3,2,-4,1,-3,-5,4,-1,-2,3,1,4,0,-4,3,4,-2,0,-2,5,473,0,611,-4,5],"l2_i64":["9007199254740995","-4611686018427387904","9007199254740992","-4611686018427387904","-4611686018427387904","9007199254740993","9007199254740992","9007199254740992","9007199254740995","4","-4611686018427387904","-4611686018427387904","-4611686018427387904","-4611686018427387904","-4611686018427387904","9007199254740994","-4611686018427387904","-4611686018427387904","-2","9007199254740992","3","9007199254740992","-1","0","0","-4611686018427387904","-5","0","9007199254740995","-4611686018427387904","9007199254740994","-4611686018427387904","9007199254740995","-4611686018427387904","-4611686018427387904","-4611686018427387904","-3","5","9007199254740992","3","-4611686018427387904","9007199254740995","-4611686018427387904","5","9007199254740992","5","-1","9007199254740994","-4611686018427387904","-4611686018427387904","-5","-4611686018427387904","9007199254740992","3","9007199254740994","9007199254740994","-4611686018427387904","-4611686018427387904","9007199254740992","9007199254740992"],"l3_i32":[5,-4,0,-2,-3,0,-5,-5,5,-2147483648,-3,2,-5,-1,-3,1,-5,-3,-2147483648,-4,3,2,-4,-2,4,4,2,3,-5,-4,1,-1,-1,-2,-4,2,-1,5,-4,-3,2147483647,-1,0,-3,3,2,2,2,-2,3,-4,2,5,-3,2,-5,2147483647,3,-3,-3],"l4_f64":[1.0,-58.91,2.0,0.0,-3.0,2.0,1.0,45.507,-45.335,0.0,4.0,2.0,"NaN",-1.0,46.833,-70.909,75.758,-2.0,4.0,1.0,69.296,-15.606,42.874,-1.0,-6.354,50.366,-8.082,9.719,-10.494,-54.182,48.055,4.0,"Infinity",59.099,-2.0,2.0,-2.0,30.238,74.281,34.887,-51.918,79.905,0.0,26.671,-48.914,84.759,0.0,-3.0,2.0,4.086,-24.467,-48.825,80.735,-3.0,-28.107,14.719,71.93,91.071,2.0,2.0],"l5_i32":[-3,0,-3,4,-3,3,-1,-1,3,-3,4,5,1,2,-1,-5,4,2,5,-1,1,-1,913,4,2,-1,-4,-5,3,-781,-3,0,4,-2,-4,-3,5,-3,-5,-1,-2,4,4,4,-338,-3,-1,2,3,1,1,1,-4,2147483647,5,3,1,3,2,3],"l6_i32":[2,-3,1,-2147483648,2147483647,4,1,2,0,-1,3,-2,-609,-2,-1,3,1,0,4,-2,-3,3,2147483647,-5,5,-5,5,-2,-2,5,1,-1,2147483647,-5,-2,5,3,3,-4,5,1,1,-1,4,2,2,-790,3,0,1,-1,-1,-3,1,2,2,4,-4,5,1]}}},"ops":[{"op":"filter","predicate":{"fn":"gte","args":[{"fn":"when","args":[{"fn":"lt","args":[{"col":"l5_i32"},{"lit":-1}]},{"col":"l6_i32"},{"col":"l6_i32"}]},{"col":"l6_i32"}]}},{"op":"head","n":3}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i64","type":"i64"},{"name":"l3_i32","type":"i32"},{"name":"l4_f64","type":"f64"},{"name":"l5_i32","type":"i32"},{"name":"l6_i32","type":"i32"}],"columns":{"l0_utf8":["😀","c","aa"],"l1_i32":[3,-4,0],"l2_i64":["9007199254740995","-4611686018427387904","9007199254740992"],"l3_i32":[5,-4,0],"l4_f64":[1.0,-58.91,2.0],"l5_i32":[-3,0,-3],"l6_i32":[2,-3,1]}}},{"id":"random/0277","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i64","type":"i64"},{"name":"l3_bool","type":"bool"}],"columns":{"l0_utf8":["","aa","aa","aa","a","c","ab","c","aa","zz","zz","aa","ab","c","","aa","a","c","é","zz"],"l1_i32":[0,-2,2,5,-5,5,1,0,2,1,-5,5,2,4,3,0,-4,2,-4,1],"l2_i64":["-3","9007199254740994","9007199254740995","-4611686018427387904","2","9007199254740993","-4611686018427387904","9007199254740992","2","-4611686018427387904","-4611686018427387904","-4611686018427387904","3","-2","-4611686018427387904","3","-4","9007199254740995","9007199254740992","-4611686018427387904"],"l3_bool":[true,true,true,true,true,true,false,false,false,false,false,true,false,false,true,false,false,true,true,true]}}},"ops":[{"op":"head","n":1},{"op":"filter","predicate":{"fn":"isNotNull","args":[{"col":"l0_utf8"}]}},{"op":"withColumns","exprs":{"x2":{"fn":"add","args":[{"col":"l1_i32"},{"fn":"abs","args":[{"lit":2.5}]}]}}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i64","type":"i64"},{"name":"l3_bool","type":"bool"},{"name":"x2","type":"f64"}],"columns":{"l0_utf8":[""],"l1_i32":[0],"l2_i64":["-3"],"l3_bool":[true],"x2":[2.5]}}},{"id":"random/0278","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_i32","type":"i32"},{"name":"l4_i64","type":"i64"},{"name":"l5_utf8","type":"utf8"},{"name":"l6_f64","type":"f64"}],"columns":{"l0_utf8":["zz",null,null,"B","b",null,"ab","a","c","a","ab","zz","a","😀",null,"ab",null,"é","b",null,"B","zz","","","😀","zz","😀",null,null,"c","a","aa","😀","B","a","😀","zz","","ab","","aa","😀","B","😀","é","😀","a","c","ab","c",null,"é","c","ab","b","é","b","zz",null,"😀"],"l1_i32":[3,0,-1,-3,5,-5,0,-5,null,3,3,null,4,0,3,-4,-2,4,0,-4,0,-5,1,1,5,5,2147483647,3,-4,1,3,2,-5,3,-5,-2,0,-3,3,4,-4,0,-1,-5,2,-2,-5,-2147483648,1,null,3,-3,-5,1,-3,5,0,-2,1,-4],"l2_utf8":["","b","aa","","😀","😀",null,"é","c","ab","","😀","zz","😀","aa","é","zz","B","","","",null,"B","ab","","a","é","B","c","","zz","a","a","ab","é","","B","aa","ab","é",null,"é","é","😀","","ab","",null,"b","","aa","ab","é",null,"B",null,"B","","😀","c"],"l3_i32":[3,2,-3,-2147483648,3,-1,2,3,-3,-3,-1,5,null,3,-3,-2,-4,0,1,2,2,1,null,-1,5,0,3,null,-5,-4,-1,2,-5,-3,-2,0,117,-4,-5,2,-2,-36,3,1,0,-5,4,4,-3,1,5,2,718,-4,-1,-5,5,0,-4,-3],"l4_i64":["-5","9007199254740994","-4611686018427387904",null,"-4611686018427387904","-5","9007199254740993","-4611686018427387904",null,"9007199254740993","9007199254740993","9007199254740994","-4611686018427387904",null,"-4611686018427387904","-4611686018427387904","-5","0","9007199254740994","-4611686018427387904",null,"-4611686018427387904","9007199254740993","-4611686018427387904","9007199254740992","-4611686018427387904","9007199254740993","9007199254740992","9007199254740995","-2","9007199254740994","-4611686018427387904","9007199254740994",null,"-4611686018427387904","9007199254740995","-4611686018427387904","9007199254740992","5","-4","-1","9007199254740994","9007199254740993","9007199254740994","-2","-4611686018427387904","5","4","9007199254740992","9007199254740995","-3","9007199254740994","3","-4611686018427387904","0","2","-4611686018427387904","-4611686018427387904","-3","9007199254740992"],"l5_utf8":[null,null,"é","zz","ab","😀","é","B","zz","zz",null,"ab","ab","","c","aa","😀","a","a",null,"b","c","c","b","aa",null,"aa","B",null,"a","B","é","B","é","B","a","😀","c","😀","","zz","ab","B","aa","a","c","aa",null,"","c","😀","b","",null,"zz","ab","é","ab","c",null],"l6_f64":[2.0,3.0,-1.0,0.0,39.381,1.0,-3.0,null,4.0,-4.0,2.0,-3.0,3.0,20.604,84.733,-99.626,-4.0,"NaN",24.842,74.001,-53.649,3.0,4.0,-11.963,61.192,null,"NaN",-3.0,4.0,-30.612,3.0,null,76.677,null,-2.0,3.0,-53.767,-3.0,92.667,80.557,null,-2.0,-3.0,-1.0,-4.0,5.069,-75.01,-3.0,"Infinity",3.0,-4.0,-25.584,0.0,3.0,-2.0,-2.0,-4.0,null,27.947,1.0]}}},"ops":[{"op":"filter","predicate":{"fn":"isNull","args":[{"col":"l1_i32"}]}},{"op":"filter","predicate":{"fn":"or","args":[{"fn":"gt","args":[{"lit":2.5},{"col":"l6_f64"}]},{"fn":"neq","args":[{"lit":-1},{"col":"l3_i32"}]}]}},{"op":"head","n":3}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_i32","type":"i32"},{"name":"l4_i64","type":"i64"},{"name":"l5_utf8","type":"utf8"},{"name":"l6_f64","type":"f64"}],"columns":{"l0_utf8":["c","zz","c"],"l1_i32":[null,null,null],"l2_utf8":["c","😀",""],"l3_i32":[-3,5,1],"l4_i64":[null,"9007199254740994","9007199254740995"],"l5_utf8":["zz","ab","c"],"l6_f64":[4.0,-3.0,3.0]}}},{"id":"random/0279","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_i32","type":"i32"},{"name":"l4_f64","type":"f64"},{"name":"l5_f64","type":"f64"},{"name":"l6_i32","type":"i32"}],"columns":{"l0_utf8":["","B","😀","zz","😀"],"l1_i32":[4,-5,-4,1,1],"l2_utf8":["B","c","é","","B"],"l3_i32":[-1,5,1,-1,3],"l4_f64":[-3.0,-37.236,"NaN",-3.0,3.0],"l5_f64":[-2.0,-1.0,2.0,3.0,50.502],"l6_i32":[1,-3,3,-4,5]}},"right":{"fields":[{"name":"l6_i32","type":"i32"},{"name":"r0_utf8","type":"utf8"}],"columns":{"r0_utf8":["zz","😀","","a","zz","B",null,"😀","é","😀"],"l6_i32":[-4,-3,3,3,3,-4,-3,1,5,1]}}},"ops":[{"op":"join","right":"right","on":["l6_i32"],"how":"inner"},{"op":"filter","predicate":{"fn":"isIn","args":[{"col":"l0_utf8"}],"values":["b","é","ab"]}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_i32","type":"i32"},{"name":"l4_f64","type":"f64"},{"name":"l5_f64","type":"f64"},{"name":"l6_i32","type":"i32"},{"name":"r0_utf8","type":"utf8"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_utf8":[],"l3_i32":[],"l4_f64":[],"l5_f64":[],"l6_i32":[],"r0_utf8":[]}}},{"id":"random/0280","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i64","type":"i64"},{"name":"l3_f64","type":"f64"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_utf8","type":"utf8"},{"name":"l6_i32","type":"i32"}],"columns":{"l0_utf8":["c","ab","😀",null,null,"a",null,null,"a","é","B","",null,"B",null,null,"ab","ab","aa","c",null,null,"b","ab","","aa",null,null,"😀","B","B",null,null,null,"",null,null,"b",null,"ab","a","ab",null,"b","b","b","😀","😀","B",null,"zz","😀",null,"a","aa","","B","b","",null],"l1_i32":[3,null,null,2,4,0,null,-2,4,null,4,-5,0,0,null,2,5,null,3,-1,3,-3,-3,-3,2,0,1,-4,3,2,-4,-4,3,1,null,4,1,-2147483648,1,2,1,null,4,-5,-1,4,2,3,null,5,null,-4,-1,-2,5,2147483647,5,-3,-3,0],"l2_i64":[null,"-5","0",null,"-1","-4611686018427387904",null,null,"9007199254740993","-4611686018427387904",null,null,null,null,"4","-4611686018427387904","-4611686018427387904","-4611686018427387904","2","-4","-4611686018427387904","-4611686018427387904","9007199254740993",null,null,"5","4",null,"-4611686018427387904","0","4","9007199254740994","9007199254740995","-4611686018427387904",null,null,"9007199254740994","-4611686018427387904","-4611686018427387904","-4611686018427387904","9007199254740994",null,"9007199254740994",null,null,"-4611686018427387904","-4611686018427387904","-4611686018427387904","3",null,"-4611686018427387904","9007199254740993",null,null,"9007199254740992","4",null,"-4","5","9007199254740992"],"l3_f64":[2.0,-2.0,0.0,null,42.04,null,-58.976,80.696,null,-44.734,0.0,null,0.0,-2.0,-36.532,null,44.922,null,0.0,1.0,1.0,-2.0,-4.0,null,14.109,-4.0,-86.251,null,-1.0,15.799,-56.317,-4.0,-2.0,null,null,-1.0,null,null,-3.0,-50.054,0.0,2.0,null,null,3.0,"Infinity",-86.636,"NaN",null,-96.171,null,null,-1.0,null,null,1.0,null,null,-31.124,15.469],"l4_utf8":["B","B",null,null,"é","c",null,"é","b","b","c","a","zz","B",null,"é","c",null,"é","B",null,"c",null,null,"zz","é","zz","c","c","","zz","é","zz","a","b",null,"é","zz","😀",null,"😀","c","a",null,"ab",null,"c","ab","zz","é",null,"ab","é","B","","😀","é",null,null,null],"l5_utf8":[null,"ab","é",null,null,"aa","b",null,"zz",null,"é","b",null,"c",null,"a","😀","😀",null,"😀","aa","zz",null,"c","aa","a","B",null,"ab",null,null,"",null,"zz",null,"a","é","a",null,null,"B","",null,null,"😀",null,null,"zz",null,"é","a","b",null,"😀","b","😀","😀","zz","é","c"],"l6_i32":[-4,-5,null,1,null,null,null,5,null,-5,-2147483648,-4,2,-4,3,-2,null,-2,null,-4,null,5,0,null,2,-2147483648,2,-4,null,1,null,5,null,-1,-1,null,-3,null,-2,null,null,2,-4,null,0,-5,-3,2,-4,null,2147483647,null,1,3,-1,null,null,-2,3,null]}},"right":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"r0_utf8","type":"utf8"}],"columns":{"r0_utf8":["ab","B","B","😀","aa","B","zz","aa","ab","a","b","B","😀","c","b","c","ab","ab","é","zz","ab","a","","é","ab","aa","","","aa",""],"l0_utf8":["c",null,"b","",null,"b","b",null,null,null,"😀","b","aa","B",null,null,null,null,"ab","a",null,null,"b",null,null,"","B","é",null,"😀"]}}},"ops":[{"op":"join","right":"right","on":["l0_utf8"],"how":"inner"},{"op":"head","n":5}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i64","type":"i64"},{"name":"l3_f64","type":"f64"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_utf8","type":"utf8"},{"name":"l6_i32","type":"i32"},{"name":"r0_utf8","type":"utf8"}],"columns":{"l0_utf8":["c","ab","😀","😀","a"],"l1_i32":[3,null,null,null,0],"l2_i64":[null,"-5","0","0","-4611686018427387904"],"l3_f64":[2.0,-2.0,0.0,0.0,null],"l4_utf8":["B","B",null,null,"c"],"l5_utf8":[null,"ab","é","é","aa"],"l6_i32":[-4,-5,null,null,null],"r0_utf8":["ab","é","b","","zz"]}}},{"id":"random/0281","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_i64","type":"i64"},{"name":"l4_bool","type":"bool"}],"columns":{"l0_utf8":["é","aa","c","😀","aa","ab","zz","ab","😀","zz","zz","B","zz","aa","😀","b","a","B","aa","😀"],"l1_i32":[4,5,-3,-3,2,-2,-2,3,-3,1,1,5,767,2,4,-4,2,-2,-3,1],"l2_f64":[95.949,-96.362,1.0,70.335,-4.0,-39.164,-35.858,-71.696,"-0",-4.778,3.0,29.286,-31.932,-18.787,-28.814,3.0,-4.0,-20.156,53.813,77.436],"l3_i64":["-1","9007199254740995","-5","9007199254740994","-4611686018427387904","4","-4611686018427387904","-1","2","-4","9007199254740992","9007199254740992","9007199254740994","-4611686018427387904","9007199254740992","-4611686018427387904","-3","-4611686018427387904","-4611686018427387904","9007199254740995"],"l4_bool":[false,true,true,true,false,true,true,false,false,false,true,false,false,true,false,true,false,false,false,false]}}},"ops":[{"op":"filter","predicate":{"fn":"isNull","args":[{"col":"l0_utf8"}]}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_i64","type":"i64"},{"name":"l4_bool","type":"bool"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_f64":[],"l3_i64":[],"l4_bool":[]}}},{"id":"random/0282","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_bool","type":"bool"},{"name":"l4_f64","type":"f64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_f64":[],"l3_bool":[],"l4_f64":[]}}},"ops":[{"op":"sort","keys":[{"expr":{"col":"l3_bool"},"descending":false}],"nullsLast":true},{"op":"sort","keys":[{"expr":{"col":"l0_utf8"},"descending":false},{"expr":{"col":"l2_f64"},"descending":false}],"nullsLast":false},{"op":"groupBy","keys":["l0_utf8"],"aggs":{"n2":{"agg":"len"},"a2_0":{"agg":"min","arg":{"col":"l2_f64"}}}},{"op":"filter","predicate":{"fn":"or","args":[{"fn":"isNull","args":[{"col":"a2_0"}]},{"fn":"and","args":[{"fn":"isNull","args":[{"col":"l0_utf8"}]},{"fn":"isNotNull","args":[{"col":"n2"}]}]}]}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"n2","type":"u32"},{"name":"a2_0","type":"f64"}],"columns":{"l0_utf8":[],"n2":[],"a2_0":[]}}},{"id":"random/0283","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_i32","type":"i32"},{"name":"l4_i32","type":"i32"},{"name":"l5_f64","type":"f64"},{"name":"l6_utf8","type":"utf8"}],"columns":{"l0_utf8":["😀","aa","aa","aa","😀","😀","c","zz","c","B","","b","zz","b","é","b","","é","zz","😀"],"l1_i32":[3,0,4,-5,-3,2,5,3,-5,4,-2147483648,-1,1,-1,5,2,-3,-3,-3,5],"l2_f64":[-37.142,3.0,64.921,6.902,87.139,95.625,4.0,4.0,-2.0,75.522,1.0,4.0,3.0,2.0,0.0,1.0,97.376,31.017,1.0,-71.814],"l3_i32":[-4,5,4,-2147483648,3,-3,-2147483648,-1,-5,2147483647,-1,-3,1,0,5,-4,0,5,2,-2147483648],"l4_i32":[-5,5,-2,3,1,-4,-3,5,-2147483648,-5,1,-3,2,-3,5,-3,-5,355,5,-5],"l5_f64":["NaN",23.923,0.0,-85.042,-87.429,0.0,-12.488,-4.0,-57.977,4.0,59.073,4.0,-2.0,47.171,0.0,3.0,-31.97,-3.0,16.292,34.942],"l6_utf8":["a","B","aa","😀","c","a","aa","B","zz","","c","c","a","c","😀","b","b","c","b","😀"]}},"right":{"fields":[{"name":"l6_utf8","type":"utf8"},{"name":"r0_bool","type":"bool"}],"columns":{"r0_bool":[false,true,false,false,true,false,null,true,null,false,null,null,false,true,true,true,false,false,false,true,true,false,null,null,false,true,null,false,null,false],"l6_utf8":["a","a","aa","a","b","c","b","aa","aa","B","c","😀","b","a","c","a","b","zz","a","c","b","c","b","c","😀","b","b","😀","B","zz"]}}},"ops":[{"op":"unique"},{"op":"join","right":"right","on":["l6_utf8"],"how":"inner"},{"op":"withColumns","exprs":{"x2":{"lit":-1}}},{"op":"withColumns","exprs":{"x3":{"fn":"mod","args":[{"col":"l2_f64"},{"fn":"neg","args":[{"col":"l5_f64"}]}]}}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_i32","type":"i32"},{"name":"l4_i32","type":"i32"},{"name":"l5_f64","type":"f64"},{"name":"l6_utf8","type":"utf8"},{"name":"r0_bool","type":"bool"},{"name":"x2","type":"i32"},{"name":"x3","type":"f64"}],"columns":{"l0_utf8":["😀","😀","😀","😀","😀","😀","aa","aa","aa","aa","aa","aa","aa","aa","😀","😀","😀","😀","😀","😀","😀","😀","😀","😀","😀","😀","c","c","c","zz","zz","c","c","","","","","","","b","b","b","b","b","b","zz","zz","zz","zz","zz","zz","b","b","b","b","b","b","é","é","é","b","b","b","b","b","b","b","b","","","","","","","","","é","é","é","é","é","é","zz","zz","zz","zz","zz","zz","zz","zz","😀","😀","😀"],"l1_i32":[3,3,3,3,3,3,0,0,4,4,4,-5,-5,-5,-3,-3,-3,-3,-3,-3,2,2,2,2,2,2,5,5,5,3,3,-5,-5,-2147483648,-2147483648,-2147483648,-2147483648,-2147483648,-2147483648,-1,-1,-1,-1,-1,-1,1,1,1,1,1,1,-1,-1,-1,-1,-1,-1,5,5,5,2,2,2,2,2,2,2,2,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,-3,5,5,5],"l2_f64":[-37.142,-37.142,-37.142,-37.142,-37.142,-37.142,3.0,3.0,64.921,64.921,64.921,6.902,6.902,6.902,87.139,87.139,87.139,87.139,87.139,87.139,95.625,95.625,95.625,95.625,95.625,95.625,4.0,4.0,4.0,4.0,4.0,-2.0,-2.0,1.0,1.0,1.0,1.0,1.0,1.0,4.0,4.0,4.0,4.0,4.0,4.0,3.0,3.0,3.0,3.0,3.0,3.0,2.0,2.0,2.0,2.0,2.0,2.0,0.0,0.0,0.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,97.376,97.376,97.376,97.376,97.376,97.376,97.376,97.376,31.017,31.017,31.017,31.017,31.017,31.017,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,-71.814,-71.814,-71.814],"l3_i32":[-4,-4,-4,-4,-4,-4,5,5,4,4,4,-2147483648,-2147483648,-2147483648,3,3,3,3,3,3,-3,-3,-3,-3,-3,-3,-2147483648,-2147483648,-2147483648,-1,-1,-5,-5,-1,-1,-1,-1,-1,-1,-3,-3,-3,-3,-3,-3,1,1,1,1,1,1,0,0,0,0,0,0,5,5,5,-4,-4,-4,-4,-4,-4,-4,-4,0,0,0,0,0,0,0,0,5,5,5,5,5,5,2,2,2,2,2,2,2,2,-2147483648,-2147483648,-2147483648],"l4_i32":[-5,-5,-5,-5,-5,-5,5,5,-2,-2,-2,3,3,3,1,1,1,1,1,1,-4,-4,-4,-4,-4,-4,-3,-3,-3,5,5,-2147483648,-2147483648,1,1,1,1,1,1,-3,-3,-3,-3,-3,-3,2,2,2,2,2,2,-3,-3,-3,-3,-3,-3,5,5,5,-3,-3,-3,-3,-3,-3,-3,-3,-5,-5,-5,-5,-5,-5,-5,-5,355,355,355,355,355,355,5,5,5,5,5,5,5,5,-5,-5,-5],"l5_f64":["NaN","NaN","NaN","NaN","NaN","NaN",23.923,23.923,0.0,0.0,0.0,-85.042,-85.042,-85.042,-87.429,-87.429,-87.429,-87.429,-87.429,-87.429,0.0,0.0,0.0,0.0,0.0,0.0,-12.488,-12.488,-12.488,-4.0,-4.0,-57.977,-57.977,59.073,59.073,59.073,59.073,59.073,59.073,4.0,4.0,4.0,4.0,4.0,4.0,-2.0,-2.0,-2.0,-2.0,-2.0,-2.0,47.171,47.171,47.171,47.171,47.171,47.171,0.0,0.0,0.0,3.0,3.0,3.0,3.0,3.0,3.0,3.0,3.0,-31.97,-31.97,-31.97,-31.97,-31.97,-31.97,-31.97,-31.97,-3.0,-3.0,-3.0,-3.0,-3.0,-3.0,16.292,16.292,16.292,16.292,16.292,16.292,16.292,16.292,34.942,34.942,34.942],"l6_utf8":["a","a","a","a","a","a","B","B","aa","aa","aa","😀","😀","😀","c","c","c","c","c","c","a","a","a","a","a","a","aa","aa","aa","B","B","zz","zz","c","c","c","c","c","c","c","c","c","c","c","c","a","a","a","a","a","a","c","c","c","c","c","c","😀","😀","😀","b","b","b","b","b","b","b","b","b","b","b","b","b","b","b","b","c","c","c","c","c","c","b","b","b","b","b","b","b","b","😀","😀","😀"],"r0_bool":[false,true,false,true,true,false,false,null,false,true,null,null,false,false,false,null,true,true,false,null,false,true,false,true,true,false,false,true,null,false,null,false,false,false,null,true,true,false,null,false,null,true,true,false,null,false,true,false,true,true,false,false,null,true,true,false,null,null,false,false,true,null,false,false,true,null,true,null,true,null,false,false,true,null,true,null,false,null,true,true,false,null,true,null,false,false,true,null,true,null,null,false,false],"x2":[-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1],"x3":["NaN","NaN","NaN","NaN","NaN","NaN",-20.923,-20.923,"NaN","NaN","NaN",6.902,6.902,6.902,87.139,87.139,87.139,87.139,87.139,87.139,"NaN","NaN","NaN","NaN","NaN","NaN",4.0,4.0,4.0,0.0,0.0,55.977,55.977,-58.073,-58.073,-58.073,-58.073,-58.073,-58.073,0.0,0.0,0.0,0.0,0.0,0.0,1.0,1.0,1.0,1.0,1.0,1.0,-45.171,-45.171,-45.171,-45.171,-45.171,-45.171,"NaN","NaN","NaN",-2.0,-2.0,-2.0,-2.0,-2.0,-2.0,-2.0,-2.0,1.4660000000000082,1.4660000000000082,1.4660000000000082,1.4660000000000082,1.4660000000000082,1.4660000000000082,1.4660000000000082,1.4660000000000082,1.0169999999999995,1.0169999999999995,1.0169999999999995,1.0169999999999995,1.0169999999999995,1.0169999999999995,-15.292000000000002,-15.292000000000002,-15.292000000000002,-15.292000000000002,-15.292000000000002,-15.292000000000002,-15.292000000000002,-15.292000000000002,-1.9299999999999926,-1.9299999999999926,-1.9299999999999926]}}},{"id":"random/0284","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_i64","type":"i64"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_i32","type":"i32"},{"name":"l6_bool","type":"bool"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_i32":[],"l3_i64":[],"l4_utf8":[],"l5_i32":[],"l6_bool":[]}},"right":{"fields":[{"name":"l1_i32","type":"i32"},{"name":"r0_utf8","type":"utf8"}],"columns":{"r0_utf8":["é","😀","zz","é","aa","b","ab","B","aa","é"],"l1_i32":[null,-3,null,null,-3,-3,null,null,null,null]}}},"ops":[{"op":"unique","columns":["l5_i32"]},{"op":"join","right":"right","on":["l1_i32"],"how":"semi"},{"op":"sort","keys":[{"expr":{"col":"l4_utf8"},"descending":false},{"expr":{"col":"l0_utf8"},"descending":true}],"nullsLast":false}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_i64","type":"i64"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_i32","type":"i32"},{"name":"l6_bool","type":"bool"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_i32":[],"l3_i64":[],"l4_utf8":[],"l5_i32":[],"l6_bool":[]}}},{"id":"random/0285","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i64","type":"i64"},{"name":"l3_bool","type":"bool"},{"name":"l4_i32","type":"i32"}],"columns":{"l0_utf8":["a","b","","zz","B"],"l1_i32":[3,5,1,-4,-2],"l2_i64":["9007199254740994","9007199254740994","-4611686018427387904","0","-4611686018427387904"],"l3_bool":[true,true,false,true,true],"l4_i32":[0,-3,3,-2147483648,-3]}},"right":{"fields":[{"name":"a0_0","type":"i64"},{"name":"r0_f64","type":"f64"}],"columns":{"r0_f64":[1.0,-11.481,-3.0,-82.76,null,66.535,-1.0,null,4.0,null],"a0_0":["-1","9007199254740992","-5","9007199254740992","9007199254740992","-1","-1","5","9007199254740995","9007199254740995"]}}},"ops":[{"op":"groupBy","keys":["l1_i32"],"aggs":{"n0":{"agg":"len"},"a0_0":{"agg":"last","arg":{"col":"l2_i64"}}}},{"op":"groupBy","keys":["a0_0"],"aggs":{"n1":{"agg":"len"},"a1_0":{"agg":"std","arg":{"col":"l1_i32"}},"a1_1":{"agg":"last","arg":{"col":"n0"}}}},{"op":"join","right":"right","on":["a0_0"],"how":"left"},{"op":"filter","predicate":{"fn":"gte","args":[{"lit":-1},{"col":"a1_0"}]}}],"expected":{"fields":[{"name":"a0_0","type":"i64"},{"name":"n1","type":"u32"},{"name":"a1_0","type":"f64"},{"name":"a1_1","type":"u32"},{"name":"r0_f64","type":"f64"}],"columns":{"a0_0":[],"n1":[],"a1_0":[],"a1_1":[],"r0_f64":[]}}},{"id":"random/0286","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_f64","type":"f64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_f64":[],"l3_f64":[]}}},"ops":[{"op":"withColumns","exprs":{"x0":{"fn":"abs","args":[{"col":"l3_f64"}]}}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_f64","type":"f64"},{"name":"x0","type":"f64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_f64":[],"l3_f64":[],"x0":[]}}},{"id":"random/0287","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_bool","type":"bool"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_i32","type":"i32"}],"columns":{"l0_utf8":["é"],"l1_i32":[null],"l2_bool":[false],"l3_bool":[true],"l4_utf8":["😀"],"l5_i32":[null]}}},"ops":[{"op":"head","n":7},{"op":"head","n":4}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_bool","type":"bool"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_i32","type":"i32"}],"columns":{"l0_utf8":["é"],"l1_i32":[null],"l2_bool":[false],"l3_bool":[true],"l4_utf8":["😀"],"l5_i32":[null]}}},{"id":"random/0288","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_i32","type":"i32"},{"name":"l4_f64","type":"f64"}],"columns":{"l0_utf8":[null],"l1_i32":[null],"l2_utf8":[""],"l3_i32":[-3],"l4_f64":[null]}}},"ops":[{"op":"unique"},{"op":"sort","keys":[{"expr":{"col":"l0_utf8"},"descending":true},{"expr":{"col":"l2_utf8"},"descending":true}],"nullsLast":false}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_i32","type":"i32"},{"name":"l4_f64","type":"f64"}],"columns":{"l0_utf8":[null],"l1_i32":[null],"l2_utf8":[""],"l3_i32":[-3],"l4_f64":[null]}}},{"id":"random/0289","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_f64","type":"f64"}],"columns":{"l0_utf8":["😀",null,"a","B","b","B","aa","ab","B","ab","c","é","B",null,"","aa","a","aa",null,"","B","aa","a","c","é","é","B","zz","aa","a","c","😀","ab","é","ab","zz","a","ab",null,"é","aa","aa",null,"ab","c","a","B","B","aa","😀","a","aa","zz","","😀","","B","b","B",""],"l1_i32":[-1,-4,null,-3,1,1,null,-3,4,0,-2147483648,null,null,-5,3,null,-2,-5,2,2,0,-2147483648,4,5,3,4,0,4,4,1,5,2147483647,1,null,-4,-4,null,null,-4,3,4,2,-5,0,5,null,null,1,null,-5,3,-2,-3,-2,3,-1,-2,-2,-5,4],"l2_f64":[78.614,2.0,81.278,54.942,4.0,1.0,-43.712,-3.0,63.337,59.685,-1.0,4.0,-4.0,0.0,-84.52,-21.837,-62.322,48.897,1.0,-4.0,45.238,-4.0,74.091,-2.0,-51.584,"NaN",-57.912,-48.624,null,null,null,null,-68.349,-10.5,-1.0,0.0,1.0,31.772,null,-89.082,56.136,1.0,-4.0,0.469,38.959,-2.0,27.899,-1.0,-2.0,44.575,null,-4.0,"NaN",32.785,-70.022,1.0,3.0,null,14.619,14.95],"l3_f64":[-42.527,83.106,-4.0,null,null,-19.767,0.0,3.0,-89.853,2.0,58.895,-59.428,1.0,-99.055,null,"NaN",9.787,-4.0,-2.0,55.341,-24.85,33.511,-1.0,null,"NaN",-83.142,-4.0,-66.574,-2.0,null,74.024,-2.0,1.0,4.0,45.524,16.591,4.0,null,86.067,85.196,null,-98.946,71.407,-3.597,98.871,-68.416,2.0,-37.482,-66.238,24.771,91.751,4.0,94.149,"NaN",null,80.776,-3.0,null,-1.0,1.0]}},"right":{"fields":[{"name":"l1_i32","type":"i32"},{"name":"r0_utf8","type":"utf8"}],"columns":{"r0_utf8":["aa","","é","B","aa","a","c","ab","é","zz","","😀","b","","😀","zz","ab","é","a","B","b","a","a","aa","zz","b","aa","a","b","aa"],"l1_i32":[-2,4,2,null,null,5,null,2,4,1,-5,null,5,null,null,2,0,-2,-2,3,-3,-5,null,-5,1,1,2,3,4,1]}}},"ops":[{"op":"join","right":"right","on":["l1_i32"],"how":"semi"}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_f64","type":"f64"}],"columns":{"l0_utf8":["B","b","B","ab","B","ab",null,"","a","aa",null,"","B","a","c","é","é","B","zz","aa","a","c","ab","é","aa","aa",null,"ab","c","B","😀","a","aa","zz","","😀","B","b","B",""],"l1_i32":[-3,1,1,-3,4,0,-5,3,-2,-5,2,2,0,4,5,3,4,0,4,4,1,5,1,3,4,2,-5,0,5,1,-5,3,-2,-3,-2,3,-2,-2,-5,4],"l2_f64":[54.942,4.0,1.0,-3.0,63.337,59.685,0.0,-84.52,-62.322,48.897,1.0,-4.0,45.238,74.091,-2.0,-51.584,"NaN",-57.912,-48.624,null,null,null,-68.349,-89.082,56.136,1.0,-4.0,0.469,38.959,-1.0,44.575,null,-4.0,"NaN",32.785,-70.022,3.0,null,14.619,14.95],"l3_f64":[null,null,-19.767,3.0,-89.853,2.0,-99.055,null,9.787,-4.0,-2.0,55.341,-24.85,-1.0,null,"NaN",-83.142,-4.0,-66.574,-2.0,null,74.024,1.0,85.196,null,-98.946,71.407,-3.597,98.871,-37.482,24.771,91.751,4.0,94.149,"NaN",null,-3.0,null,-1.0,1.0]}}},{"id":"random/0290","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_i64","type":"i64"},{"name":"l4_bool","type":"bool"},{"name":"l5_utf8","type":"utf8"},{"name":"l6_i32","type":"i32"}],"columns":{"l0_utf8":["zz",null,"b","aa","a","","😀","aa",null,null,"é","a","zz","é",null,"B","zz",null,"B",null,"c","😀","zz","zz",null,"ab",null,"c","a","zz","zz","B",null,null,null,"😀",null,null,"B","b",null,"b","a","a","a",null,"a","ab","c",null,"c","zz","","B","b","aa",null,null,"a","zz"],"l1_i32":[-3,-3,2,-1,2,null,null,5,null,4,null,-2147483648,-4,-2,2,3,3,4,-2,-2147483648,-5,-2147483648,-4,-1,-5,-1,null,null,1,5,4,-3,0,0,-3,-4,null,2,null,5,0,null,4,1,2,-2,-2,0,null,-4,-1,-4,3,-1,4,1,null,-1,-1,-5],"l2_i32":[null,-4,-5,2,-4,null,0,null,4,0,3,5,null,null,1,-3,2,null,-1,2147483647,3,3,-2,null,-3,-1,null,0,null,0,-1,4,0,-2,-3,null,-2,1,null,-4,-5,-2,null,-4,5,3,3,2,1,0,0,null,null,4,1,5,0,null,null,4],"l3_i64":["-4",null,"9007199254740992","-4611686018427387904","5",null,"5",null,"-4611686018427387904","-4611686018427387904","-4","9007199254740995","0","1","9007199254740995",null,null,"9007199254740993","9007199254740995","-4","9007199254740992",null,"-4611686018427387904",null,"5",null,"-4611686018427387904","9007199254740992","2","0","9007199254740992","-4",null,"9007199254740992",null,null,"-4611686018427387904","-2","-4611686018427387904",null,"-5","-4611686018427387904",null,null,null,"3","0","9007199254740994","9007199254740993","9007199254740994","9007199254740995","-4611686018427387904","9007199254740994","9007199254740993",null,"9007199254740994","9007199254740994","-4611686018427387904","-4611686018427387904",null],"l4_bool":[null,null,true,null,null,null,false,null,false,false,null,false,true,false,false,null,false,true,false,false,false,null,false,false,true,false,true,null,null,null,true,false,false,false,true,false,null,null,null,true,null,true,null,false,true,true,null,false,true,false,null,true,false,null,false,true,false,null,null,true],"l5_utf8":["a","b","",null,"😀",null,"é",null,"c","B","é","😀","ab","zz","B","é",null,"","a","aa",null,null,null,"aa",null,"","😀",null,"zz","B","aa","😀",null,"é","c",null,"a","é","é","aa","é","aa","😀","c","zz","aa","ab","c",null,null,null,"c","😀","B","😀","b","a","é","a",null],"l6_i32":[null,0,-3,null,null,null,-5,null,null,2,5,-2,2,1,762,null,null,2147483647,2,null,-1,null,-5,-2147483648,null,-2,null,0,null,-5,1,-1,3,525,null,0,-5,-3,null,null,null,-5,-1,3,2,2,5,-3,5,null,-5,null,-1,null,-5,-1,-4,0,null,null]}}},"ops":[{"op":"withColumns","exprs":{"x0":{"col":"l1_i32"}}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_i64","type":"i64"},{"name":"l4_bool","type":"bool"},{"name":"l5_utf8","type":"utf8"},{"name":"l6_i32","type":"i32"},{"name":"x0","type":"i32"}],"columns":{"l0_utf8":["zz",null,"b","aa","a","","😀","aa",null,null,"é","a","zz","é",null,"B","zz",null,"B",null,"c","😀","zz","zz",null,"ab",null,"c","a","zz","zz","B",null,null,null,"😀",null,null,"B","b",null,"b","a","a","a",null,"a","ab","c",null,"c","zz","","B","b","aa",null,null,"a","zz"],"l1_i32":[-3,-3,2,-1,2,null,null,5,null,4,null,-2147483648,-4,-2,2,3,3,4,-2,-2147483648,-5,-2147483648,-4,-1,-5,-1,null,null,1,5,4,-3,0,0,-3,-4,null,2,null,5,0,null,4,1,2,-2,-2,0,null,-4,-1,-4,3,-1,4,1,null,-1,-1,-5],"l2_i32":[null,-4,-5,2,-4,null,0,null,4,0,3,5,null,null,1,-3,2,null,-1,2147483647,3,3,-2,null,-3,-1,null,0,null,0,-1,4,0,-2,-3,null,-2,1,null,-4,-5,-2,null,-4,5,3,3,2,1,0,0,null,null,4,1,5,0,null,null,4],"l3_i64":["-4",null,"9007199254740992","-4611686018427387904","5",null,"5",null,"-4611686018427387904","-4611686018427387904","-4","9007199254740995","0","1","9007199254740995",null,null,"9007199254740993","9007199254740995","-4","9007199254740992",null,"-4611686018427387904",null,"5",null,"-4611686018427387904","9007199254740992","2","0","9007199254740992","-4",null,"9007199254740992",null,null,"-4611686018427387904","-2","-4611686018427387904",null,"-5","-4611686018427387904",null,null,null,"3","0","9007199254740994","9007199254740993","9007199254740994","9007199254740995","-4611686018427387904","9007199254740994","9007199254740993",null,"9007199254740994","9007199254740994","-4611686018427387904","-4611686018427387904",null],"l4_bool":[null,null,true,null,null,null,false,null,false,false,null,false,true,false,false,null,false,true,false,false,false,null,false,false,true,false,true,null,null,null,true,false,false,false,true,false,null,null,null,true,null,true,null,false,true,true,null,false,true,false,null,true,false,null,false,true,false,null,null,true],"l5_utf8":["a","b","",null,"😀",null,"é",null,"c","B","é","😀","ab","zz","B","é",null,"","a","aa",null,null,null,"aa",null,"","😀",null,"zz","B","aa","😀",null,"é","c",null,"a","é","é","aa","é","aa","😀","c","zz","aa","ab","c",null,null,null,"c","😀","B","😀","b","a","é","a",null],"l6_i32":[null,0,-3,null,null,null,-5,null,null,2,5,-2,2,1,762,null,null,2147483647,2,null,-1,null,-5,-2147483648,null,-2,null,0,null,-5,1,-1,3,525,null,0,-5,-3,null,null,null,-5,-1,3,2,2,5,-3,5,null,-5,null,-1,null,-5,-1,-4,0,null,null],"x0":[-3,-3,2,-1,2,null,null,5,null,4,null,-2147483648,-4,-2,2,3,3,4,-2,-2147483648,-5,-2147483648,-4,-1,-5,-1,null,null,1,5,4,-3,0,0,-3,-4,null,2,null,5,0,null,4,1,2,-2,-2,0,null,-4,-1,-4,3,-1,4,1,null,-1,-1,-5]}}},{"id":"random/0291","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_i32","type":"i32"},{"name":"l4_i32","type":"i32"}],"columns":{"l0_utf8":["","zz","b","c","c","","😀","aa","zz","a","a","zz","ab","é","zz","é","b","B","c","b"],"l1_i32":[5,-5,2,-5,5,-4,-2,2,2147483647,-4,-4,2147483647,-2147483648,3,5,-2,-1,-1,0,4],"l2_f64":[0.0,2.0,-18.013,23.503,4.0,78.396,"NaN","Infinity",60.004,97.443,-2.0,-1.0,-1.0,-3.0,-4.0,2.0,0.0,24.082,-89.978,-2.0],"l3_i32":[-3,0,-3,2147483647,0,1,2,2,3,-2,-1,-2,1,2147483647,-2,2,0,-1,5,-1],"l4_i32":[2,1,4,3,1,0,1,-3,0,0,338,-4,2,5,-5,-2,-3,3,1,2147483647]}}},"ops":[{"op":"groupBy","keys":["l0_utf8"],"aggs":{"n0":{"agg":"len"},"a0_0":{"agg":"max","arg":{"col":"l3_i32"}},"a0_1":{"agg":"count","arg":{"col":"l2_f64"}},"a0_2":{"agg":"sum","arg":{"col":"l1_i32"}}}},{"op":"filter","predicate":{"fn":"eq","args":[{"fn":"abs","args":[{"col":"a0_2"}]},{"fn":"abs","args":[{"col":"a0_0"}]}]}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"n0","type":"u32"},{"name":"a0_0","type":"i32"},{"name":"a0_1","type":"u32"},{"name":"a0_2","type":"i32"}],"columns":{"l0_utf8":["","😀","aa","B"],"n0":[2,1,1,1],"a0_0":[1,2,2,-1],"a0_1":[2,1,1,1],"a0_2":[1,-2,2,-1]}}},{"id":"random/0292","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_f64","type":"f64"},{"name":"l4_i64","type":"i64"},{"name":"l5_utf8","type":"utf8"},{"name":"l6_f64","type":"f64"}],"columns":{"l0_utf8":["b"],"l1_i32":[null],"l2_f64":["Infinity"],"l3_f64":[-4.0],"l4_i64":["-1"],"l5_utf8":["é"],"l6_f64":[82.584]}}},"ops":[{"op":"filter","predicate":{"fn":"lt","args":[{"col":"l3_f64"},{"fn":"mul","args":[{"lit":2},{"col":"l1_i32"}]}]}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_f64","type":"f64"},{"name":"l4_i64","type":"i64"},{"name":"l5_utf8","type":"utf8"},{"name":"l6_f64","type":"f64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_f64":[],"l3_f64":[],"l4_i64":[],"l5_utf8":[],"l6_f64":[]}}},{"id":"random/0293","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_f64","type":"f64"},{"name":"l4_f64","type":"f64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_i32":[],"l3_f64":[],"l4_f64":[]}},"right":{"fields":[{"name":"l2_i32","type":"i32"},{"name":"r0_f64","type":"f64"}],"columns":{"r0_f64":[0.0,null,-21.881,-3.0,90.634,2.0,null,70.23,"NaN",null],"l2_i32":[2,1,null,1,1,2,null,-4,null,1]}}},"ops":[{"op":"join","right":"right","on":["l2_i32"],"how":"anti"},{"op":"head","n":4},{"op":"withColumns","exprs":{"x2":{"fn":"mod","args":[{"col":"l2_i32"},{"lit":2}]}}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_f64","type":"f64"},{"name":"l4_f64","type":"f64"},{"name":"x2","type":"i32"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_i32":[],"l3_f64":[],"l4_f64":[],"x2":[]}}},{"id":"random/0294","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i64","type":"i64"},{"name":"l3_i32","type":"i32"},{"name":"l4_f64","type":"f64"},{"name":"l5_i32","type":"i32"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_i64":[],"l3_i32":[],"l4_f64":[],"l5_i32":[]}}},"ops":[{"op":"withColumns","exprs":{"x0":{"fn":"div","args":[{"fn":"when","args":[{"fn":"gte","args":[{"col":"l4_f64"},{"col":"l5_i32"}]},{"lit":2},{"lit":2.5}]},{"fn":"when","args":[{"fn":"lte","args":[{"col":"l5_i32"},{"col":"l1_i32"}]},{"col":"l3_i32"},{"col":"l1_i32"}]}]}}},{"op":"unique"}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i64","type":"i64"},{"name":"l3_i32","type":"i32"},{"name":"l4_f64","type":"f64"},{"name":"l5_i32","type":"i32"},{"name":"x0","type":"f64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_i64":[],"l3_i32":[],"l4_f64":[],"l5_i32":[],"x0":[]}}},{"id":"random/0295","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_f64","type":"f64"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_f64","type":"f64"},{"name":"l6_utf8","type":"utf8"}],"columns":{"l0_utf8":["é",null,"a","é","","zz","é",null,"zz","a",null,"c","B",null,"","é",null,"B","B","b"],"l1_i32":[-5,-5,5,-5,null,3,3,null,-2147483648,null,-3,-2,-4,4,null,3,null,3,2,-5],"l2_utf8":["a","😀","😀","c","😀",null,null,"a",null,"ab","",null,"","aa","B","a","ab","aa","b",null],"l3_f64":[-3.0,-26.751,null,-1.0,-17.015,79.401,null,"-Infinity",null,null,4.0,null,0.0,-48.615,3.0,0.0,0.0,1.864,-74.026,null],"l4_utf8":["B",null,null,"b","😀",null,"é",null,"aa","😀",null,null,"c",null,"ab","aa","zz",null,"é","a"],"l5_f64":["-Infinity",-44.075,-66.765,60.654,4.0,null,4.0,-66.148,3.0,-71.796,3.0,4.0,-2.0,50.446,-1.303,3.0,3.0,1.0,22.653,74.43],"l6_utf8":[null,"aa",null,"a","zz",null,"B","b","aa",null,"zz","aa","ab","","aa","b","😀","ab","é",null]}},"right":{"fields":[{"name":"l4_utf8","type":"utf8"},{"name":"r0_bool","type":"bool"}],"columns":{"r0_bool":[true,true,true],"l4_utf8":["aa","zz","b"]}}},"ops":[{"op":"head","n":6},{"op":"join","right":"right","on":["l4_utf8"],"how":"anti"},{"op":"groupBy","keys":["l2_utf8"],"aggs":{"n2":{"agg":"len"},"a2_0":{"agg":"count","arg":{"col":"l5_f64"}},"a2_1":{"agg":"std","arg":{"col":"l5_f64"}},"a2_2":{"agg":"nUnique","arg":{"col":"l6_utf8"}}}},{"op":"sort","keys":[{"expr":{"col":"a2_0"},"descending":false}],"nullsLast":true}],"expected":{"fields":[{"name":"l2_utf8","type":"utf8"},{"name":"n2","type":"u32"},{"name":"a2_0","type":"u32"},{"name":"a2_1","type":"f64"},{"name":"a2_2","type":"u32"}],"columns":{"l2_utf8":[null,"a","😀"],"n2":[1,1,3],"a2_0":[0,1,3],"a2_1":[null,null,36.13338011220834],"a2_2":[1,1,3]}}},{"id":"random/0296","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_utf8","type":"utf8"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_utf8":[],"l3_utf8":[]}}},"ops":[{"op":"filter","predicate":{"fn":"gt","args":[{"fn":"mod","args":[{"col":"l1_i32"},{"col":"l1_i32"}]},{"col":"l1_i32"}]}},{"op":"unique","columns":["l2_utf8"]},{"op":"sort","keys":[{"expr":{"col":"l0_utf8"},"descending":false},{"expr":{"col":"l3_utf8"},"descending":false}],"nullsLast":true}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_utf8","type":"utf8"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_utf8":[],"l3_utf8":[]}}},{"id":"random/0297","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_f64","type":"f64"},{"name":"l4_i32","type":"i32"},{"name":"l5_utf8","type":"utf8"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_f64":[],"l3_f64":[],"l4_i32":[],"l5_utf8":[]}}},"ops":[{"op":"sort","keys":[{"expr":{"col":"l5_utf8"},"descending":false},{"expr":{"col":"l4_i32"},"descending":true}],"nullsLast":true}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_f64","type":"f64"},{"name":"l4_i32","type":"i32"},{"name":"l5_utf8","type":"utf8"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_f64":[],"l3_f64":[],"l4_i32":[],"l5_utf8":[]}}},{"id":"random/0298","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_f64","type":"f64"},{"name":"l4_i32","type":"i32"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_i32":[],"l3_f64":[],"l4_i32":[]}},"right":{"fields":[{"name":"l1_i32","type":"i32"},{"name":"r0_utf8","type":"utf8"}],"columns":{"r0_utf8":["b",null,null,"B",null,null,null,"a",null,"a",null,null,"B",null,null,"aa","😀",null,"","c","b","b","a",null,null,null,"c",null,null,"b"],"l1_i32":[4,-1,-2,-1,4,null,-2,-1,-1,-2,null,null,-1,null,-2,-1,4,-2,4,-1,4,-2,4,-1,-1,-1,-2,-1,-2,-1]}}},"ops":[{"op":"join","right":"right","on":["l1_i32"],"how":"left"}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_f64","type":"f64"},{"name":"l4_i32","type":"i32"},{"name":"r0_utf8","type":"utf8"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_i32":[],"l3_f64":[],"l4_i32":[],"r0_utf8":[]}}},{"id":"random/0299","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_i32","type":"i32"},{"name":"l4_f64","type":"f64"}],"columns":{"l0_utf8":["😀","é","a","zz","",null,"","ab","b","b","b","","é","zz","aa","ab","a","ab","a","b",null,"é","","😀","B","B","😀","é",null,"a","a","ab","é",null,null,"é","","","aa","b","é","a","é","a","é","zz","ab","aa","b","é","B","a","aa","é","c","b","zz","c",null,"b"],"l1_i32":[4,-1,-3,-3,1,5,-5,4,1,3,-5,1,-4,-1,-5,-1,-3,-2,3,-624,3,5,5,0,-1,5,-4,4,5,-4,1,2,4,-1,-5,4,2,3,null,2147483647,null,0,-1,3,5,-2,2,4,-2,null,0,5,5,4,-4,5,-3,-4,1,3],"l2_f64":[3.0,-89.222,-3.0,1.0,-13.516,4.0,-91.489,null,"-0",-1.0,3.0,-92.838,null,68.566,1.0,94.688,-47.088,1.0,34.821,-2.0,-0.594,3.926,-53.736,58.087,null,88.539,77.812,3.0,75.16,88.596,null,-2.0,-1.0,3.0,0.699,-69.835,-2.0,-2.0,-2.0,1.0,11.063,79.633,-2.0,-74.547,1.0,96.52,45.294,53.723,2.0,-4.0,-2.0,5.525,-96.454,3.0,null,4.0,2.0,4.78,"Infinity",4.0],"l3_i32":[-4,2,-2,-5,-3,1,-4,null,-4,-2147483648,2147483647,-1,-2,4,-4,2,3,-5,0,3,null,3,-1,4,-4,-5,1,5,1,1,-5,null,-1,-2147483648,-1,475,3,4,0,-4,-4,-2147483648,2,4,4,0,1,-2,5,0,4,4,-3,4,null,-1,-4,-4,-5,3],"l4_f64":[-81.288,1.0,-4.0,-40.736,-27.874,null,"-Infinity",73.25,-3.0,-87.428,1.0,-91.868,1.0,"NaN",3.0,1.0,-82.594,null,-1.0,81.486,-6.757,null,1.0,4.0,1.0,34.593,0.0,-25.71,4.0,null,-1.0,"NaN",4.0,-2.0,44.859,32.274,-2.0,null,6.368,-1.0,null,-4.0,-40.202,"NaN",null,null,0.0,-9.888,50.082,-83.599,2.0,null,"NaN",-3.0,83.74,-60.385,-2.0,-77.321,-65.482,-83.571]}}},"ops":[{"op":"withColumns","exprs":{"x0":{"fn":"add","args":[{"lit":2.5},{"fn":"sub","args":[{"col":"l4_f64"},{"lit":2.5}]}]}}},{"op":"groupBy","keys":["l0_utf8"],"aggs":{"n1":{"agg":"len"},"a1_0":{"agg":"median","arg":{"col":"l4_f64"}},"a1_1":{"agg":"nUnique","arg":{"col":"x0"}}}},{"op":"unique","columns":["l0_utf8"]}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"n1","type":"u32"},{"name":"a1_0","type":"f64"},{"name":"a1_1","type":"u32"}],"columns":{"l0_utf8":["😀","é","a","zz","",null,"ab","b","aa","B","c"],"n1":[3,11,8,4,6,6,5,8,4,3,2],"a1_0":[0.0,-1.0,-2.5,-2.0,-27.874,-2.0,37.125,-2.0,4.684,2.0,3.2094999999999914],"a1_1":[3,8,5,4,6,6,5,8,4,3,2]}}},{"id":"random/0300","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_f64","type":"f64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_utf8":[],"l3_f64":[]}}},"ops":[{"op":"sort","keys":[{"expr":{"col":"l3_f64"},"descending":false},{"expr":{"col":"l0_utf8"},"descending":true}],"nullsLast":false},{"op":"filter","predicate":{"fn":"eq","args":[{"col":"l2_utf8"},{"lit":"a"}]}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_f64","type":"f64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_utf8":[],"l3_f64":[]}}},{"id":"random/0301","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_f64","type":"f64"}],"columns":{"l0_utf8":["a"],"l1_i32":[-736],"l2_f64":[-2.0],"l3_f64":[9.993]}}},"ops":[{"op":"filter","predicate":{"fn":"gt","args":[{"col":"l2_f64"},{"lit":0.5}]}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_f64","type":"f64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_f64":[],"l3_f64":[]}}},{"id":"random/0302","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_f64","type":"f64"},{"name":"l4_i64","type":"i64"},{"name":"l5_utf8","type":"utf8"}],"columns":{"l0_utf8":["zz","B","zz","B","","","é","aa","","B","aa","é",null,"zz","aa",null,"a",null,"zz","ab","zz","zz","😀","😀","a","c",null,"zz","😀","B","a","a","c",null,null,"é","é","aa","B","😀","zz","b","zz","zz","zz","aa","ab","😀","ab","aa","aa","b","a","a","b","ab","😀","","ab","b"],"l1_i32":[null,5,-3,-5,null,2,-5,-2,3,3,-3,-2147483648,-3,-4,-1,-2147483648,-2147483648,3,1,-4,-4,2,-2,-3,2,-2147483648,4,3,-3,-5,4,null,3,-2,5,-2,null,5,0,-5,null,-3,4,-3,-1,4,-4,-3,3,0,4,4,-327,-1,-3,-5,4,0,-3,2],"l2_f64":[24.279,2.0,33.381,-1.0,71.719,1.0,-3.0,null,1.0,64.25,1.0,"NaN",-3.0,null,3.0,-45.003,36.915,-42.858,"NaN",4.0,3.0,16.196,53.754,50.722,-99.994,67.264,3.0,4.0,-3.0,94.468,3.0,-63.206,-4.0,3.0,-37.287,1.0,4.0,0.0,-3.0,null,4.0,71.534,-1.0,-87.435,1.0,4.0,3.0,-19.91,null,"-0",6.79,94.772,3.0,-3.0,3.0,"NaN",null,4.0,-2.0,-75.0],"l3_f64":[97.925,-1.0,2.0,32.875,4.0,4.0,-1.0,1.0,2.0,"-Infinity",4.773,71.286,-1.0,-81.403,-96.953,50.214,82.771,"NaN",-2.0,12.154,"NaN",-81.006,2.0,-37.136,-3.0,2.0,88.384,-19.163,-35.765,-30.188,1.0,null,4.0,4.0,null,3.0,-1.0,-93.227,-33.405,null,-28.687,-43.035,2.0,77.431,null,73.607,-4.0,null,0.0,74.969,"NaN",-1.0,"NaN",null,-43.08,-2.0,36.227,1.312,3.0,-48.59],"l4_i64":["9007199254740992","9007199254740994","9007199254740995","-4","-4611686018427387904","4","-4611686018427387904","-4611686018427387904","1","1","9007199254740992","-4611686018427387904",null,"-4611686018427387904","-4611686018427387904","5","9007199254740992","5","9007199254740993","-1","9007199254740995","4","2","0","2","5","9007199254740992","9007199254740993","0","3","-2","-5","-3","-4611686018427387904","-2","2","-1","9007199254740994","9007199254740992","9007199254740994","9007199254740995","-4611686018427387904","1","9007199254740993","-1","9007199254740995","9007199254740993","-5",null,"1","4","-4611686018427387904","1","-4611686018427387904","-4611686018427387904","3","9007199254740992","-4611686018427387904","-4611686018427387904","5"],"l5_utf8":["b","😀","b","b","b",null,"é","b","é","b","c",null,"aa","","","😀","aa","ab","a","c","aa","c","😀","zz","","ab",null,"😀","a",null,"a","b","zz","B","b","a","aa","","aa","b","😀","zz","ab","ab","aa","a","c","B","b","é","zz","b","😀","a","a","zz","a",null,"é",""]}},"right":{"fields":[{"name":"l4_i64","type":"i64"},{"name":"r0_utf8","type":"utf8"}],"columns":{"r0_utf8":[],"l4_i64":[]}}},"ops":[{"op":"head","n":10},{"op":"join","right":"right","on":["l4_i64"],"how":"inner"}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_f64","type":"f64"},{"name":"l4_i64","type":"i64"},{"name":"l5_utf8","type":"utf8"},{"name":"r0_utf8","type":"utf8"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_f64":[],"l3_f64":[],"l4_i64":[],"l5_utf8":[],"r0_utf8":[]}}},{"id":"random/0303","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_i64","type":"i64"}],"columns":{"l0_utf8":["B",null,"c",null,null,"aa",null,"B","aa","c",null,null,null,"ab","zz","aa",null,"c","zz","B"],"l1_i32":[-1,2,1,5,null,2147483647,-2,0,null,-5,5,-5,-3,1,null,-4,-4,0,-1,-4],"l2_f64":[null,3.0,80.379,-1.0,4.0,null,"-0",null,8.109,92.95,90.369,null,0.0,-84.745,null,72.89,"NaN",null,null,-26.295],"l3_i64":["-4","-4611686018427387904",null,"0","2",null,"-4611686018427387904","-4611686018427387904","9007199254740993","-4611686018427387904","-4611686018427387904",null,"9007199254740994","-2",null,null,null,"-4611686018427387904",null,null]}}},"ops":[{"op":"sort","keys":[{"expr":{"col":"l1_i32"},"descending":false}],"nullsLast":true},{"op":"groupBy","keys":["l3_i64"],"aggs":{"n1":{"agg":"len"},"a1_0":{"agg":"mean","arg":{"col":"l2_f64"}}}},{"op":"withColumns","exprs":{"x2":{"fn":"when","args":[{"fn":"neq","args":[{"col":"a1_0"},{"col":"a1_0"}]},{"col":"a1_0"},{"fn":"div","args":[{"col":"a1_0"},{"lit":-1}]}]}}}],"expected":{"fields":[{"name":"l3_i64","type":"i64"},{"name":"n1","type":"u32"},{"name":"a1_0","type":"f64"},{"name":"x2","type":"f64"}],"columns":{"l3_i64":["-4611686018427387904",null,"9007199254740994","-4","-2","0","2","9007199254740993"],"n1":[6,8,1,1,1,1,1,1],"a1_0":[46.579750000000004,"NaN",0.0,null,-84.745,-1.0,4.0,8.109],"x2":[-46.579750000000004,"NaN","-0",null,84.745,1.0,-4.0,-8.109]}}},{"id":"random/0304","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_i32","type":"i32"},{"name":"l5_i64","type":"i64"},{"name":"l6_f64","type":"f64"}],"columns":{"l0_utf8":["c",null,"😀","",""],"l1_i32":[null,-1,5,-3,-2],"l2_f64":[null,1.0,77.277,null,7.574],"l3_utf8":[null,null,null,"","B"],"l4_i32":[null,5,2,-2,-5],"l5_i64":["9007199254740994",null,"0","-4",null],"l6_f64":[2.0,-66.123,0.0,null,99.73]}}},"ops":[{"op":"unique","columns":["l4_i32"]},{"op":"filter","predicate":{"fn":"neq","args":[{"fn":"neg","args":[{"lit":2}]},{"lit":2.5}]}},{"op":"head","n":2},{"op":"head","n":8}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_i32","type":"i32"},{"name":"l5_i64","type":"i64"},{"name":"l6_f64","type":"f64"}],"columns":{"l0_utf8":["c",null],"l1_i32":[null,-1],"l2_f64":[null,1.0],"l3_utf8":[null,null],"l4_i32":[null,5],"l5_i64":["9007199254740994",null],"l6_f64":[2.0,-66.123]}}},{"id":"random/0305","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_i32","type":"i32"},{"name":"l5_i64","type":"i64"},{"name":"l6_f64","type":"f64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_utf8":[],"l3_utf8":[],"l4_i32":[],"l5_i64":[],"l6_f64":[]}}},"ops":[{"op":"filter","predicate":{"fn":"isNotNull","args":[{"col":"l6_f64"}]}},{"op":"sort","keys":[{"expr":{"col":"l0_utf8"},"descending":false}],"nullsLast":false}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_i32","type":"i32"},{"name":"l5_i64","type":"i64"},{"name":"l6_f64","type":"f64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_utf8":[],"l3_utf8":[],"l4_i32":[],"l5_i64":[],"l6_f64":[]}}},{"id":"random/0306","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_f64","type":"f64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_f64":[],"l3_utf8":[],"l4_f64":[]}}},"ops":[{"op":"filter","predicate":{"fn":"and","args":[{"fn":"isNull","args":[{"col":"l0_utf8"}]},{"fn":"eq","args":[{"col":"l2_f64"},{"col":"l2_f64"}]}]}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_f64","type":"f64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_f64":[],"l3_utf8":[],"l4_f64":[]}}},{"id":"random/0307","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_i64","type":"i64"},{"name":"l4_bool","type":"bool"},{"name":"l5_utf8","type":"utf8"}],"columns":{"l0_utf8":["aa","aa","c","ab","","","B","","a","a","","B","c","","é","zz","c","c","B","zz"],"l1_i32":[-2,-1,-186,3,-2,2,0,-1,-1,-3,4,0,5,-4,1,-2,-5,-5,4,4],"l2_utf8":["ab","c","ab","ab","ab","","zz","B","c","","b","zz","aa","b","B","zz","😀","zz","zz","zz"],"l3_i64":["-3","-4611686018427387904","9007199254740994","9007199254740994","-4611686018427387904","-3","4","0","9007199254740992","1","-4611686018427387904","-4611686018427387904","9007199254740993","0","3","9007199254740995","9007199254740993","-4611686018427387904","9007199254740995","9007199254740995"],"l4_bool":[true,false,true,false,true,true,false,false,false,false,true,false,true,true,false,false,false,false,false,true],"l5_utf8":["é","c","c","aa","aa","c","é","é","é","c","aa","ab","B","c","","B","😀","a","zz","a"]}}},"ops":[{"op":"groupBy","keys":["l5_utf8"],"aggs":{"n0":{"agg":"len"},"a0_0":{"agg":"first","arg":{"col":"l4_bool"}},"a0_1":{"agg":"last","arg":{"col":"l2_utf8"}}}},{"op":"withColumns","exprs":{"x1":{"fn":"floorDiv","args":[{"fn":"neg","args":[{"lit":-1}]},{"lit":-1}]}}},{"op":"withColumns","exprs":{"x2":{"col":"x1"}}},{"op":"head","n":9}],"expected":{"fields":[{"name":"l5_utf8","type":"utf8"},{"name":"n0","type":"u32"},{"name":"a0_0","type":"bool"},{"name":"a0_1","type":"utf8"},{"name":"x1","type":"i32"},{"name":"x2","type":"i32"}],"columns":{"l5_utf8":["é","c","aa","ab","B","","😀","a","zz"],"n0":[4,5,3,1,2,1,1,2,1],"a0_0":[true,false,false,false,true,false,false,false,false],"a0_1":["c","b","b","zz","zz","B","😀","zz","zz"],"x1":[-1,-1,-1,-1,-1,-1,-1,-1,-1],"x2":[-1,-1,-1,-1,-1,-1,-1,-1,-1]}}},{"id":"random/0308","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i64","type":"i64"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_utf8","type":"utf8"}],"columns":{"l0_utf8":["ab",null,"😀","aa","zz","aa","😀","ab","ab","a","",null,"😀","B","é","b","é","","ab","b","a","zz",null,"zz","B","c","b","",null,"ab","B","aa","zz","aa","aa","c","c","ab",null,null,null,"é","b","aa","a",null,null,null,"B","","B","zz","aa",null,"zz",null,"é","zz",null,"B"],"l1_i32":[3,null,null,null,null,4,null,null,null,1,-5,4,2,null,-5,3,null,-5,1,null,5,2,0,2147483647,2,null,1,5,0,5,3,4,-2,-3,-5,0,-1,null,2,5,1,null,1,-1,3,3,0,4,4,1,null,4,-2147483648,-4,-2,5,null,null,-4,2],"l2_i64":["1","-4611686018427387904",null,"9007199254740992","-4611686018427387904","2","9007199254740994","9007199254740993",null,null,"-4611686018427387904","-4611686018427387904","9007199254740992","-4611686018427387904",null,"-4611686018427387904","9007199254740992",null,"-1",null,"-4611686018427387904",null,"9007199254740992",null,"1","-3","-4611686018427387904","5","9007199254740994","9007199254740993","9007199254740993","-4611686018427387904","-5",null,null,"-4611686018427387904","9007199254740992","-4611686018427387904","5",null,"9007199254740995","0","-4611686018427387904","-4611686018427387904","-4611686018427387904","9007199254740993",null,"9007199254740994","9007199254740993","9007199254740993","9007199254740992","3",null,"4",null,"-4","-4611686018427387904","-5","-2","9007199254740992"],"l3_utf8":["a","aa","","zz","ab","B","B","","é","aa","zz","aa","zz","😀","B","aa",null,null,"zz","c","ab","é","b","B","é","B",null,"","",null,"é","b","ab","😀","c","aa","ab","a","zz","zz","b",null,"😀","","a","aa","b","é","ab","","b","zz",null,"a",null,null,null,"zz",null,"B"],"l4_utf8":["a","B",null,"zz","B",null,"zz","é","b","zz","😀","aa","aa","zz","zz","é","ab","é",null,"b","",null,null,"é",null,"b","a",null,"zz",null,null,"ab",null,null,"B","aa","b","😀",null,"zz","B","c","c",null,"b",null,"é","b","c",null,"",null,null,null,"é","b","","ab","é",null],"l5_utf8":["é","B",null,null,null,"aa",null,null,null,"ab",null,"b","c","é","😀","c",null,"B","ab",null,"",null,null,null,"a",null,"","",null,"B","zz","","a","😀","ab",null,null,null,"B","B","😀","aa","c","a","zz",null,"b",null,null,"c",null,"a",null,"B","b","a",null,"ab","ab",""]}}},"ops":[{"op":"filter","predicate":{"fn":"neq","args":[{"col":"l1_i32"},{"lit":-1}]}},{"op":"withColumns","exprs":{"x1":{"col":"l1_i32"}}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i64","type":"i64"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_utf8","type":"utf8"},{"name":"x1","type":"i32"}],"columns":{"l0_utf8":["ab","aa","a","",null,"😀","é","b","","ab","a","zz",null,"zz","B","b","",null,"ab","B","aa","zz","aa","aa","c",null,null,null,"b","a",null,null,null,"B","","zz","aa",null,"zz",null,null,"B"],"l1_i32":[3,4,1,-5,4,2,-5,3,-5,1,5,2,0,2147483647,2,1,5,0,5,3,4,-2,-3,-5,0,2,5,1,1,3,3,0,4,4,1,4,-2147483648,-4,-2,5,-4,2],"l2_i64":["1","2",null,"-4611686018427387904","-4611686018427387904","9007199254740992",null,"-4611686018427387904",null,"-1","-4611686018427387904",null,"9007199254740992",null,"1","-4611686018427387904","5","9007199254740994","9007199254740993","9007199254740993","-4611686018427387904","-5",null,null,"-4611686018427387904","5",null,"9007199254740995","-4611686018427387904","-4611686018427387904","9007199254740993",null,"9007199254740994","9007199254740993","9007199254740993","3",null,"4",null,"-4","-2","9007199254740992"],"l3_utf8":["a","B","aa","zz","aa","zz","B","aa",null,"zz","ab","é","b","B","é",null,"","",null,"é","b","ab","😀","c","aa","zz","zz","b","😀","a","aa","b","é","ab","","zz",null,"a",null,null,null,"B"],"l4_utf8":["a",null,"zz","😀","aa","aa","zz","é","é",null,"",null,null,"é",null,"a",null,"zz",null,null,"ab",null,null,"B","aa",null,"zz","B","c","b",null,"é","b","c",null,null,null,null,"é","b","é",null],"l5_utf8":["é","aa","ab",null,"b","c","😀","c","B","ab","",null,null,null,"a","","",null,"B","zz","","a","😀","ab",null,"B","B","😀","c","zz",null,"b",null,null,"c","a",null,"B","b","a","ab",""],"x1":[3,4,1,-5,4,2,-5,3,-5,1,5,2,0,2147483647,2,1,5,0,5,3,4,-2,-3,-5,0,2,5,1,1,3,3,0,4,4,1,4,-2147483648,-4,-2,5,-4,2]}}},{"id":"random/0309","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_i32","type":"i32"},{"name":"l5_f64","type":"f64"},{"name":"l6_f64","type":"f64"}],"columns":{"l0_utf8":[null,"a","B","a","zz","","","b","é","b","😀","😀",null,"é","B",null,"","😀",null,"B"],"l1_i32":[3,-1,0,2147483647,1,4,null,0,null,-4,-3,0,-2147483648,null,null,-3,3,-4,4,-3],"l2_i32":[null,null,-136,-2,0,4,-1,2,0,null,-1,null,2,0,0,4,4,-2,5,-3],"l3_utf8":[null,"é","zz","B","b","ab","zz",null,"","aa","ab","😀",null,"",null,null,"😀","","ab",""],"l4_i32":[-4,4,null,2147483647,null,2,3,-1,-3,1,null,null,-5,null,null,-4,null,5,-3,null],"l5_f64":[1.0,-4.0,2.0,null,null,-7.276,-32.276,0.0,0.0,3.0,-2.0,-90.833,-1.0,null,null,-1.0,2.0,2.0,2.0,null],"l6_f64":[null,62.979,null,null,null,"-0","Infinity",null,43.278,null,3.0,40.34,45.49,1.0,-2.0,-1.0,null,null,-3.0,0.0]}}},"ops":[{"op":"filter","predicate":{"fn":"strStartsWith","args":[{"col":"l0_utf8"}],"prefix":"a"}},{"op":"withColumns","exprs":{"x1":{"fn":"fillNull","args":[{"lit":2},{"lit":1.5}]}}},{"op":"withColumns","exprs":{"x2":{"fn":"add","args":[{"col":"l4_i32"},{"fn":"floorDiv","args":[{"col":"l5_f64"},{"col":"l2_i32"}]}]}}},{"op":"withColumns","exprs":{"x3":{"fn":"fillNull","args":[{"fn":"abs","args":[{"col":"l2_i32"}]},{"lit":0}]}}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_i32","type":"i32"},{"name":"l5_f64","type":"f64"},{"name":"l6_f64","type":"f64"},{"name":"x1","type":"f64"},{"name":"x2","type":"f64"},{"name":"x3","type":"i32"}],"columns":{"l0_utf8":["a","a"],"l1_i32":[-1,2147483647],"l2_i32":[null,-2],"l3_utf8":["é","B"],"l4_i32":[4,2147483647],"l5_f64":[-4.0,null],"l6_f64":[62.979,null],"x1":[2.0,2.0],"x2":[null,null],"x3":[0,2]}}},{"id":"random/0310","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_bool","type":"bool"}],"columns":{"l0_utf8":["ab","zz",null,"B","a","ab","B","B",null,null,null,"","",null,null,"😀","aa","",null,null,"c","b","😀","é","aa","zz","ab","b",null,"é",null,"é","ab","é","ab",null,"aa","","",null,"",null,null,"a",null,null,"aa","b","",null,"c","","c",null,"","c","😀","c","b","zz"],"l1_i32":[null,1,-5,-4,null,-2,null,1,2147483647,null,2,null,-3,-4,3,5,null,5,5,null,-4,1,-3,3,-4,0,null,null,0,-4,-2147483648,-2,-5,-3,2,-3,2,null,4,645,null,-1,5,null,null,null,-4,null,4,2,null,null,null,null,0,null,2,-2,-1,-2],"l2_f64":[-2.0,null,98.071,89.214,4.0,-50.322,95.241,null,-4.0,null,null,null,null,null,null,-2.0,null,29.67,70.479,-49.82,-14.163,1.0,null,null,null,null,-2.0,-4.0,4.0,-1.0,"NaN",null,33.093,2.0,2.0,-52.046,-4.0,"NaN",-61.733,null,-40.678,null,-17.927,0.0,3.0,2.0,99.723,72.53,3.0,1.0,-65.399,95.754,null,null,62.832,-75.369,44.201,null,null,99.544],"l3_bool":[null,null,null,true,null,true,false,false,false,true,true,false,null,null,true,false,false,true,true,false,false,true,true,false,null,null,false,false,false,true,null,false,null,true,null,false,null,true,false,false,false,true,false,null,true,true,true,null,null,true,null,null,false,false,false,null,null,null,false,false]}}},"ops":[{"op":"withColumns","exprs":{"x0":{"col":"l1_i32"}}},{"op":"groupBy","keys":["l3_bool"],"aggs":{"n1":{"agg":"len"},"a1_0":{"agg":"count","arg":{"col":"l1_i32"}},"a1_1":{"agg":"mean","arg":{"col":"x0"}}}},{"op":"filter","predicate":{"fn":"isNotNull","args":[{"col":"a1_0"}]}}],"expected":{"fields":[{"name":"l3_bool","type":"bool"},{"name":"n1","type":"u32"},{"name":"a1_0","type":"u32"},{"name":"a1_1","type":"f64"}],"columns":{"l3_bool":[null,true,false],"n1":[20,17,23],"a1_0":[13,13,14],"a1_1":[-165191050.76923078,-0.23076923076923078,153391735.57142857]}}},{"id":"random/0311","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_bool","type":"bool"},{"name":"l4_f64","type":"f64"}],"columns":{"l0_utf8":["c"],"l1_i32":[-5],"l2_f64":[68.974],"l3_bool":[true],"l4_f64":[2.0]}}},"ops":[{"op":"head","n":5},{"op":"sort","keys":[{"expr":{"col":"l2_f64"},"descending":false}],"nullsLast":false},{"op":"filter","predicate":{"fn":"isIn","args":[{"col":"l0_utf8"}],"values":["B","b","é"]}},{"op":"sort","keys":[{"expr":{"col":"l3_bool"},"descending":true}],"nullsLast":false}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_bool","type":"bool"},{"name":"l4_f64","type":"f64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_f64":[],"l3_bool":[],"l4_f64":[]}}},{"id":"random/0312","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_f64","type":"f64"},{"name":"l4_bool","type":"bool"}],"columns":{"l0_utf8":["zz","é","B","zz","😀","é","😀","ab",null,null,"a","","B",null,null,null,null,"😀",null,null],"l1_i32":[0,0,-4,5,5,null,1,-4,-2,-1,4,-2,-4,-3,-5,3,null,null,-2,4],"l2_f64":[null,null,0.0,1.0,-55.526,-63.151,40.087,-4.0,-44.212,85.092,67.789,3.0,-4.0,-84.354,null,49.33,-20.72,19.609,-48.678,null],"l3_f64":[-1.0,3.0,null,null,null,null,null,52.088,-4.0,-3.0,91.669,-67.019,-4.0,-4.0,null,-4.0,null,-48.751,0.0,-77.087],"l4_bool":[false,true,true,null,false,true,true,false,null,false,true,true,false,true,false,true,false,true,false,true]}}},"ops":[{"op":"unique"},{"op":"filter","predicate":{"fn":"or","args":[{"fn":"isIn","args":[{"col":"l0_utf8"}],"values":["B","ab","c"]},{"col":"l4_bool"}]}},{"op":"withColumns","exprs":{"x2":{"lit":0.5}}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_f64","type":"f64"},{"name":"l4_bool","type":"bool"},{"name":"x2","type":"f64"}],"columns":{"l0_utf8":["é","B","é","😀","ab","a","","B",null,null,"😀",null],"l1_i32":[0,-4,null,1,-4,4,-2,-4,-3,3,null,4],"l2_f64":[null,0.0,-63.151,40.087,-4.0,67.789,3.0,-4.0,-84.354,49.33,19.609,null],"l3_f64":[3.0,null,null,null,52.088,91.669,-67.019,-4.0,-4.0,-4.0,-48.751,-77.087],"l4_bool":[true,true,true,true,false,true,true,false,true,true,true,true],"x2":[0.5,0.5,0.5,0.5,0.5,0.5,0.5,0.5,0.5,0.5,0.5,0.5]}}},{"id":"random/0313","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_f64","type":"f64"},{"name":"l4_f64","type":"f64"},{"name":"l5_f64","type":"f64"},{"name":"l6_utf8","type":"utf8"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_i32":[],"l3_f64":[],"l4_f64":[],"l5_f64":[],"l6_utf8":[]}}},"ops":[{"op":"sort","keys":[{"expr":{"col":"l3_f64"},"descending":false},{"expr":{"col":"l0_utf8"},"descending":true}],"nullsLast":false}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_f64","type":"f64"},{"name":"l4_f64","type":"f64"},{"name":"l5_f64","type":"f64"},{"name":"l6_utf8","type":"utf8"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_i32":[],"l3_f64":[],"l4_f64":[],"l5_f64":[],"l6_utf8":[]}}},{"id":"random/0314","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i64","type":"i64"},{"name":"l3_i32","type":"i32"},{"name":"l4_f64","type":"f64"},{"name":"l5_i32","type":"i32"},{"name":"l6_f64","type":"f64"}],"columns":{"l0_utf8":["😀","é","B","B","b","aa","c","B","a","","B","a","zz","zz","c","ab","b","","é","aa"],"l1_i32":[4,-2,5,-5,-1,-1,0,4,-5,-4,5,2,-5,-2,-1,-1,4,640,4,2],"l2_i64":["9007199254740993","-5","3","9007199254740992","4","9007199254740995","1","-4611686018427387904","1","9007199254740993","1","-5","9007199254740994","-4611686018427387904","-4611686018427387904","9007199254740994","-4","9007199254740995","-4611686018427387904","9007199254740993"],"l3_i32":[-5,-3,5,3,4,0,-4,-5,1,-4,0,-4,-2147483648,-4,5,-4,-2,-4,3,-5],"l4_f64":[85.662,2.0,36.4,3.0,-1.0,4.0,-29.808,-50.279,-17.544,"-0",0.0,-3.0,"NaN",4.0,36.122,4.0,89.992,-63.832,-86.765,10.751],"l5_i32":[-2,5,5,-3,5,-5,5,2,-4,1,2,3,-4,3,0,2147483647,0,-4,1,1],"l6_f64":[1.0,-44.722,1.0,3.0,8.396,-92.965,4.0,4.0,76.008,0.0,89.842,-99.56,10.14,-1.0,1.0,-4.0,-43.084,4.0,82.845,4.0]}},"right":{"fields":[{"name":"l2_i64","type":"i64"},{"name":"r0_f64","type":"f64"}],"columns":{"r0_f64":[-30.34,1.0,null],"l2_i64":["-4611686018427387904","-4611686018427387904","9007199254740992"]}}},"ops":[{"op":"groupBy","keys":["l2_i64"],"aggs":{"n0":{"agg":"len"},"a0_0":{"agg":"mean","arg":{"col":"l4_f64"}}}},{"op":"filter","predicate":{"fn":"eq","args":[{"fn":"div","args":[{"col":"a0_0"},{"lit":2.5}]},{"fn":"neg","args":[{"col":"a0_0"}]}]}},{"op":"join","right":"right","on":["l2_i64"],"how":"full"}],"expected":{"fields":[{"name":"l2_i64","type":"i64"},{"name":"n0","type":"u32"},{"name":"a0_0","type":"f64"},{"name":"r0_f64","type":"f64"}],"columns":{"l2_i64":["9007199254740994","-4611686018427387904","-4611686018427387904","9007199254740992"],"n0":[2,null,null,null],"a0_0":["NaN",null,null,null],"r0_f64":[null,-30.34,1.0,null]}}},{"id":"random/0315","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_bool","type":"bool"}],"columns":{"l0_utf8":["aa","zz","","zz","","ab","B",null,"zz","ab","ab","",null,"aa","B","😀","","zz","é","😀"],"l1_i32":[1,-2147483648,0,null,2,-3,0,null,-4,2,null,-2147483648,-5,1,0,null,2,1,null,0],"l2_utf8":["c","b","ab","ab","😀","ab","b","😀","","é","B","ab","","b","B","a","é","a","b","é"],"l3_bool":[true,false,false,true,true,false,false,true,false,false,true,false,true,null,null,false,null,true,true,false]}}},"ops":[{"op":"head","n":4}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_bool","type":"bool"}],"columns":{"l0_utf8":["aa","zz","","zz"],"l1_i32":[1,-2147483648,0,null],"l2_utf8":["c","b","ab","ab"],"l3_bool":[true,false,false,true]}}},{"id":"random/0316","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_bool","type":"bool"}],"columns":{"l0_utf8":["","aa","","B","ab"],"l1_i32":[-3,3,4,1,-1],"l2_f64":[-4.0,"NaN",-41.699,-4.0,45.194],"l3_bool":[false,true,true,null,false]}}},"ops":[{"op":"groupBy","keys":["l0_utf8"],"aggs":{"n0":{"agg":"len"},"a0_0":{"agg":"std","arg":{"col":"l2_f64"}},"a0_1":{"agg":"max","arg":{"col":"l1_i32"}},"a0_2":{"agg":"nUnique","arg":{"col":"l2_f64"}}}},{"op":"withColumns","exprs":{"x1":{"fn":"neg","args":[{"fn":"div","args":[{"col":"a0_0"},{"col":"a0_0"}]}]}}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"n0","type":"u32"},{"name":"a0_0","type":"f64"},{"name":"a0_1","type":"i32"},{"name":"a0_2","type":"u32"},{"name":"x1","type":"f64"}],"columns":{"l0_utf8":["","aa","B","ab"],"n0":[2,1,1,1],"a0_0":[26.657218543951654,null,null,null],"a0_1":[4,3,1,-1],"a0_2":[2,1,1,1],"x1":[-1.0,null,null,null]}}},{"id":"random/0317","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_i32","type":"i32"}],"columns":{"l0_utf8":["a","😀","ab","é","c","zz","aa","aa","ab","b","B","c","","aa","😀","c","b","b","b","aa","B","b","B","ab","","é","ab","a","","aa","B","a","zz","zz","zz","c","😀","😀","é","zz","zz","😀","B","a","aa","é","zz","b","zz","aa","😀","😀","😀","b","zz","é","😀","c","c","😀"],"l1_i32":[5,0,5,4,-4,-4,-2,0,2,4,0,3,-3,-2147483648,-2,-4,1,1,-4,0,0,5,-1,5,-5,-4,-2,1,1,0,-1,-4,2,-1,-1,4,-4,2147483647,-5,5,-5,4,-2147483648,-2,-3,0,-5,5,5,2,-3,5,-2,1,-4,-1,2,5,-2,1],"l2_i32":[5,5,3,5,-1,4,3,2,-5,0,5,-3,4,3,5,-4,5,-4,-2,0,1,3,0,-3,1,4,-1,1,-5,2147483647,0,-2,5,3,-4,-3,2,4,-1,0,-4,-2147483648,1,0,-5,1,1,0,-3,3,-5,1,0,-4,0,-3,-1,-5,2147483647,0],"l3_i32":[2,3,3,2,2,2,0,0,-5,-4,-4,1,0,-1,5,-3,-3,1,-4,-5,0,0,5,-2,1,-2,4,0,4,2,-108,-1,-2,-4,1,-4,-3,5,-2,-5,2,-2147483648,2147483647,-1,1,5,-4,-1,4,-5,2,3,-3,-4,4,-4,-1,1,-1,4]}}},"ops":[{"op":"withColumns","exprs":{"x0":{"col":"l3_i32"}}},{"op":"withColumns","exprs":{"x1":{"fn":"fillNull","args":[{"fn":"sub","args":[{"col":"x0"},{"col":"l2_i32"}]},{"lit":1.5}]}}},{"op":"head","n":5}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_i32","type":"i32"},{"name":"x0","type":"i32"},{"name":"x1","type":"f64"}],"columns":{"l0_utf8":["a","😀","ab","é","c"],"l1_i32":[5,0,5,4,-4],"l2_i32":[5,5,3,5,-1],"l3_i32":[2,3,3,2,2],"x0":[2,3,3,2,2],"x1":[-3.0,-2.0,0.0,-3.0,3.0]}}},{"id":"random/0318","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_bool","type":"bool"},{"name":"l4_i64","type":"i64"},{"name":"l5_bool","type":"bool"},{"name":"l6_utf8","type":"utf8"}],"columns":{"l0_utf8":["aa"],"l1_i32":[2],"l2_utf8":["aa"],"l3_bool":[true],"l4_i64":["4"],"l5_bool":[false],"l6_utf8":["B"]}}},"ops":[{"op":"withColumns","exprs":{"x0":{"lit":2.5}}},{"op":"groupBy","keys":["l6_utf8"],"aggs":{"n1":{"agg":"len"},"a1_0":{"agg":"first","arg":{"col":"l1_i32"}},"a1_1":{"agg":"sum","arg":{"col":"x0"}},"a1_2":{"agg":"count","arg":{"col":"l5_bool"}}}},{"op":"unique"},{"op":"filter","predicate":{"fn":"eq","args":[{"fn":"mul","args":[{"col":"a1_0"},{"col":"a1_1"}]},{"fn":"fillNull","args":[{"lit":2.5},{"lit":1.5}]}]}}],"expected":{"fields":[{"name":"l6_utf8","type":"utf8"},{"name":"n1","type":"u32"},{"name":"a1_0","type":"i32"},{"name":"a1_1","type":"f64"},{"name":"a1_2","type":"u32"}],"columns":{"l6_utf8":[],"n1":[],"a1_0":[],"a1_1":[],"a1_2":[]}}},{"id":"random/0319","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_i32","type":"i32"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_f64":[],"l3_i32":[]}}},"ops":[{"op":"unique","columns":["l0_utf8"]},{"op":"withColumns","exprs":{"x1":{"col":"l1_i32"}}},{"op":"unique"},{"op":"withColumns","exprs":{"x3":{"fn":"isNotNull","args":[{"col":"l1_i32"}]}}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_i32","type":"i32"},{"name":"x1","type":"i32"},{"name":"x3","type":"bool"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_f64":[],"l3_i32":[],"x1":[],"x3":[]}}},{"id":"random/0320","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_i32","type":"i32"},{"name":"l4_f64","type":"f64"},{"name":"l5_bool","type":"bool"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_f64":[],"l3_i32":[],"l4_f64":[],"l5_bool":[]}}},"ops":[{"op":"head","n":5},{"op":"unique"}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_i32","type":"i32"},{"name":"l4_f64","type":"f64"},{"name":"l5_bool","type":"bool"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_f64":[],"l3_i32":[],"l4_f64":[],"l5_bool":[]}}},{"id":"random/0321","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i64","type":"i64"},{"name":"l3_f64","type":"f64"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_utf8","type":"utf8"}],"columns":{"l0_utf8":["ab","b",null,"aa","b","zz","ab","😀",null,"😀","é","B","b","aa","ab",null,"zz","c","b","c"],"l1_i32":[4,5,-2,null,2147483647,4,1,-4,null,-5,4,-3,null,3,null,4,null,-4,-4,-4],"l2_i64":["9007199254740995","-4","9007199254740994","-4611686018427387904","-4611686018427387904","-4611686018427387904","-4611686018427387904","-4611686018427387904","-4611686018427387904","2","-4611686018427387904","4","-4611686018427387904","-4611686018427387904","-4611686018427387904","-4611686018427387904","9007199254740992","9007199254740994","-1","-4611686018427387904"],"l3_f64":[-94.266,4.0,74.442,0.0,null,98.62,49.703,-91.639,7.616,-98.368,-17.88,1.962,52.256,-35.736,0.0,null,7.986,-2.0,null,"-0"],"l4_utf8":[null,null,"b","é","😀","c","aa","b","aa","B","c","B","😀","a","a","aa",null,"ab","é","a"],"l5_utf8":["a","B","c","zz","a","zz","c","zz","",null,"zz","zz","aa","c","ab","b","B","c","zz","B"]}}},"ops":[{"op":"sort","keys":[{"expr":{"col":"l5_utf8"},"descending":true}],"nullsLast":false}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i64","type":"i64"},{"name":"l3_f64","type":"f64"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_utf8","type":"utf8"}],"columns":{"l0_utf8":["😀","aa","zz","😀","é","B","b",null,"ab","aa","c",null,"ab","b","ab","b","b","zz","c",null],"l1_i32":[-5,null,4,-4,4,-3,-4,-2,1,3,-4,4,null,null,4,2147483647,5,null,-4,null],"l2_i64":["2","-4611686018427387904","-4611686018427387904","-4611686018427387904","-4611686018427387904","4","-1","9007199254740994","-4611686018427387904","-4611686018427387904","9007199254740994","-4611686018427387904","-4611686018427387904","-4611686018427387904","9007199254740995","-4611686018427387904","-4","9007199254740992","-4611686018427387904","-4611686018427387904"],"l3_f64":[-98.368,0.0,98.62,-91.639,-17.88,1.962,null,74.442,49.703,-35.736,-2.0,null,0.0,52.256,-94.266,null,4.0,7.986,"-0",7.616],"l4_utf8":["B","é","c","b","c","B","é","b","aa","a","ab","aa","a","😀",null,"😀",null,null,"a","aa"],"l5_utf8":[null,"zz","zz","zz","zz","zz","zz","c","c","c","c","b","ab","aa","a","a","B","B","B",""]}}},{"id":"random/0322","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_f64","type":"f64"}],"columns":{"l0_utf8":["B","B","",null,"😀",null,null,"b","c","c","aa","","aa","c","ab","aa",null,"😀","","","","a","ab","b","zz","é","é","😀","b","",null,"","c","aa","aa","a","B","c","😀","a","zz","b","B","B","zz","B","aa","ab","a","é","","é","b","aa","c","ab","b",null,"","a"],"l1_i32":[3,-2147483648,0,2147483647,4,null,4,5,null,2,-4,1,-3,-4,-4,-3,-3,-2,-3,-4,-5,null,-4,0,-2,3,48,null,-1,5,4,4,-4,-5,-1,-4,-2,null,-2,3,null,1,-3,null,1,-2147483648,null,5,-1,-5,1,3,-3,-5,-2,1,-4,3,-1,0],"l2_f64":[-4.0,-4.0,-3.0,-2.0,44.075,-17.403,0.0,-53.745,1.0,-42.752,89.707,-4.0,null,86.835,null,-1.0,-98.328,64.515,-3.0,59.602,-2.0,59.059,1.0,-3.0,-30.275,-74.895,-1.0,33.457,0.0,"NaN",-26.772,19.748,87.905,null,-97.472,null,null,null,-59.401,26.346,-82.706,-1.0,-34.487,-74.857,-36.659,-79.387,2.0,0.331,-12.907,-3.0,-61.585,"NaN",36.07,3.0,null,null,2.0,1.0,-4.0,-80.884],"l3_utf8":["",null,"aa","b","zz","é",null,"ab","a","aa","c","aa","","é","b","c","","aa",null,"c","aa","","B","c","ab","c","c","","c","b",null,"é","B","zz",null,"é","ab","ab","B","zz","aa","B","ab","c","b","a","b","","aa","a","aa",null,null,"a","😀","zz",null,"zz","b","é"],"l4_f64":[-56.966,0.0,-3.0,-10.992,-63.911,-0.351,null,4.0,2.0,1.0,-47.133,1.0,-5.18,-3.0,null,0.0,3.0,-96.854,0.0,-2.0,-74.906,30.148,-52.973,0.0,null,0.0,28.388,-2.0,"NaN",25.487,-4.0,null,3.234,-57.805,1.0,3.0,null,-8.562,-97.114,1.0,2.0,3.0,4.0,3.0,0.0,-4.0,3.0,6.789,null,2.0,2.0,22.428,-3.0,-2.0,2.0,4.0,-3.0,98.651,null,-3.0]}}},"ops":[{"op":"sort","keys":[{"expr":{"col":"l0_utf8"},"descending":true}],"nullsLast":false},{"op":"filter","predicate":{"fn":"gte","args":[{"col":"l4_f64"},{"fn":"neg","args":[{"col":"l1_i32"}]}]}},{"op":"sort","keys":[{"expr":{"col":"l4_f64"},"descending":true}],"nullsLast":false}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_f64","type":"f64"}],"columns":{"l0_utf8":["b",null,"é","","é","ab","b","ab","B",null,"b","c","","c","aa","a","","é","zz","b","B",null,"B",null],"l1_i32":[-1,3,48,5,3,5,5,1,-3,-3,1,-2,1,2,-1,3,1,3,1,0,-2147483648,4,-2147483648,2147483647],"l2_f64":[0.0,1.0,-1.0,"NaN","NaN",0.331,-53.745,null,-34.487,-98.328,-1.0,null,-61.585,-42.752,-97.472,26.346,-4.0,-74.895,-36.659,-3.0,-4.0,-26.772,-79.387,-2.0],"l3_utf8":["c","zz","c","b",null,"","ab","zz","ab","","B","😀","aa","aa",null,"zz","aa","c","b","c",null,null,"a","b"],"l4_f64":["NaN",98.651,28.388,25.487,22.428,6.789,4.0,4.0,4.0,3.0,3.0,2.0,2.0,1.0,1.0,1.0,1.0,0.0,0.0,0.0,0.0,-4.0,-4.0,-10.992]}}},{"id":"random/0323","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_f64","type":"f64"},{"name":"l4_f64","type":"f64"},{"name":"l5_utf8","type":"utf8"},{"name":"l6_f64","type":"f64"}],"columns":{"l0_utf8":["","a","B","c","c","c","a","😀","c","é","aa","ab","b","zz","B","😀","aa","ab","😀","a","aa","b","aa","B","zz","é","B","ab","B","ab","c","c","a","a","ab","b","B","aa","","😀","b","é","ab","c","zz","zz","😀","zz","aa","ab","b","zz","c","ab","ab","","😀","a","ab","zz"],"l1_i32":[4,-5,-1,5,4,3,2,-1,3,-4,0,3,4,1,-4,-2147483648,1,2,-1,-4,2,-3,-1,0,-3,4,2,-3,-2,4,0,-4,-4,0,4,-2,5,0,-5,4,-3,4,-5,-2147483648,-3,-4,5,3,4,-4,2,1,3,5,3,-2,2,-3,1,2147483647],"l2_f64":[-3.0,-2.0,-95.644,3.0,3.0,-31.119,54.996,65.428,22.954,-86.944,1.0,-1.0,"NaN",3.0,"NaN",1.0,"Infinity",76.302,2.0,-24.355,21.887,-95.501,-11.173,11.639,"-Infinity",-1.0,-83.71,4.0,63.768,-72.85,-19.932,3.0,"NaN",-3.0,-3.0,-3.0,"NaN",33.018,2.0,3.0,-2.0,-1.0,0.0,-2.0,65.578,-2.0,0.0,80.569,3.0,0.0,-2.0,1.0,-4.0,-2.693,-2.0,-3.0,-44.09,-26.687,36.001,2.0],"l3_f64":[3.0,-3.0,-14.914,13.753,-1.0,56.707,2.263,3.0,-67.117,-52.789,37.412,-2.0,0.0,-66.29,-1.0,1.0,-95.195,4.0,2.0,56.817,-1.0,-1.0,2.095,-1.0,16.128,0.0,92.065,1.0,3.0,-4.0,-36.127,-1.0,-4.0,-83.266,38.497,19.83,-4.0,3.0,1.0,-4.0,-64.866,-1.0,-56.858,-99.89,3.0,90.15,-4.0,-60.825,-3.0,2.0,-4.0,"NaN",-14.154,-82.85,21.796,-4.0,65.401,1.0,-4.0,"NaN"],"l4_f64":[-40.006,-1.0,80.817,-77.932,-37.408,4.0,64.523,10.347,2.0,82.35,58.229,86.016,-89.32,2.0,"-0",-1.0,3.0,-86.911,-1.0,24.049,51.948,-1.0,-1.0,4.0,41.107,-69.321,59.812,2.0,-1.0,48.809,71.321,-4.0,4.0,6.531,-35.026,-1.0,4.0,3.0,65.371,30.792,0.0,-65.286,30.487,1.0,0.0,-4.0,4.0,-91.068,"NaN",-4.0,4.0,1.116,2.0,58.102,2.0,3.0,14.635,-52.814,-73.368,-49.882],"l5_utf8":["a","é","ab","😀","","B","b","B","","b","é","b","é","é","c","aa","😀","aa","é","zz","😀","é","","😀","aa","B","zz","","b","a","é","a","","😀","","ab","é","b","","a","zz","a","zz","B","b","😀","😀","ab","zz","c","B","","","","😀","c","","","é","c"],"l6_f64":[-15.244,-1.0,50.592,0.0,-1.223,-3.0,-1.0,52.706,-41.383,4.0,-4.0,2.0,62.195,54.008,95.143,89.529,-50.65,-2.0,8.663,1.0,-4.0,"NaN",-76.464,-1.0,-24.512,-1.0,-71.786,-3.0,-43.843,-10.063,-4.0,3.0,-3.0,-2.0,94.356,-3.0,-1.0,-4.0,-40.981,73.162,"Infinity",-4.0,47.47,0.0,-57.74,3.0,4.0,1.828,4.0,-3.0,-1.0,-4.0,-3.0,3.0,-3.0,-96.122,49.208,29.534,2.0,-1.0]}},"right":{"fields":[{"name":"l5_utf8","type":"utf8"},{"name":"r0_f64","type":"f64"}],"columns":{"r0_f64":[58.317,"Infinity",2.0,null,1.832,null,1.0,null,54.71,4.0,46.141,-2.0,1.0,null,null,4.0,1.0,null,-15.16,"NaN",null,73.585,null,-1.0,-1.0,2.0,80.628,null,-56.186,-36.764],"l5_utf8":["😀","😀","","😀","é","b","😀","","é","é","aa","B","😀","😀","zz","b","zz","😀","","","b","B","😀","B","😀","","😀","zz","c","b"]}}},"ops":[{"op":"sort","keys":[{"expr":{"col":"l3_f64"},"descending":false},{"expr":{"col":"l6_f64"},"descending":true}],"nullsLast":false},{"op":"filter","predicate":{"fn":"eq","args":[{"fn":"mod","args":[{"col":"l6_f64"},{"col":"l6_f64"}]},{"fn":"abs","args":[{"col":"l3_f64"}]}]}},{"op":"join","right":"right","on":["l5_utf8"],"how":"semi"}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_f64","type":"f64"},{"name":"l4_f64","type":"f64"},{"name":"l5_utf8","type":"utf8"},{"name":"l6_f64","type":"f64"}],"columns":{"l0_utf8":["b","é"],"l1_i32":[4,4],"l2_f64":["NaN",-1.0],"l3_f64":[0.0,0.0],"l4_f64":[-89.32,-69.321],"l5_utf8":["é","B"],"l6_f64":[62.195,-1.0]}}},{"id":"random/0324","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_i32","type":"i32"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_bool","type":"bool"}],"columns":{"l0_utf8":["B","c","B","aa","c","c","aa","é","aa","a","é","😀","c","a","","😀","zz","a","😀","😀","c","😀","","aa","B","a","😀","ab","😀","😀","c","a","c","😀","","zz","aa","zz","a","B","é","😀","B","aa","B","zz","B","","aa","aa","aa","b","zz","b","ab","B","B","a","B","zz"],"l1_i32":[3,-3,-5,2,3,-1,-3,1,0,-2147483648,-2,-3,-2,3,-5,0,-2,-5,4,-5,4,-4,-1,3,-4,-5,-5,3,-4,-1,3,-2,3,3,-2,-1,2,4,5,-2147483648,0,-4,2,-1,4,0,-5,4,2147483647,4,-3,-5,0,3,-2147483648,2147483647,3,-1,0,-5],"l2_utf8":["b","é","aa","😀","aa","a","😀","ab","aa","😀","aa","é","B","B","é","c","zz","b","B","aa","ab","😀","B","b","","b","b","a","b","aa","ab","b","a","a","😀","😀","B","c","é","c","c","é","é","c","a","c","aa","a","😀","","é","","a","b","😀","é","ab","c","😀","😀"],"l3_i32":[5,-4,2,2,4,4,-5,0,-2,5,0,-5,0,5,-1,1,1,2,3,2,0,-939,-3,3,-3,-2,4,-2,-5,2,-4,0,-2147483648,1,4,-3,4,4,-1,1,2147483647,2147483647,2,-5,5,-786,5,-4,-1,2147483647,-5,-2,0,-2147483648,3,-1,5,-4,-4,4],"l4_utf8":["😀","zz","aa","é","a","b","c","B","zz","aa","c","é","ab","B","B","b","","zz","","c","a","","aa","a","aa","","b","a","B","B","ab","zz","b","ab","zz","zz","b","","😀","zz","a","c","B","B","b","é","😀","","b","é","b","b","b","é","","😀","zz","ab","c","aa"],"l5_bool":[false,true,true,true,false,true,true,false,false,true,true,false,false,true,true,false,true,false,false,true,true,true,false,true,true,false,false,false,true,false,false,true,false,true,true,false,false,false,true,true,true,false,true,false,false,true,true,true,false,true,false,true,true,false,true,true,true,false,true,true]}},"right":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"r0_utf8","type":"utf8"}],"columns":{"r0_utf8":[],"l0_utf8":[]}}},"ops":[{"op":"unique"},{"op":"join","right":"right","on":["l0_utf8"],"how":"inner"},{"op":"withColumns","exprs":{"x2":{"fn":"when","args":[{"fn":"neq","args":[{"col":"l3_i32"},{"col":"l3_i32"}]},{"col":"l3_i32"},{"fn":"abs","args":[{"lit":2.5}]}]}}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_i32","type":"i32"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_bool","type":"bool"},{"name":"r0_utf8","type":"utf8"},{"name":"x2","type":"f64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_utf8":[],"l3_i32":[],"l4_utf8":[],"l5_bool":[],"r0_utf8":[],"x2":[]}}},{"id":"random/0325","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_i32","type":"i32"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_f64","type":"f64"}],"columns":{"l0_utf8":["B","c","é","ab","zz","ab","B","b","zz",null,"b","","c","zz","b","ab","ab","c","a","c","b","B","c","é","😀","c","zz",null,"ab","","zz","zz","B","b","😀","","","c","aa","ab","aa","ab",null,"B",null,"zz","ab","ab",null,"a","zz","aa","",null,"ab","c",null,"a","ab","aa"],"l1_i32":[5,-1,-1,-1,4,5,1,2,4,-4,-98,2147483647,2147483647,-5,-2,-4,-1,5,0,1,1,null,2147483647,-3,1,-2147483648,-5,-1,-4,1,-4,0,5,null,2,4,5,2,0,-5,-1,5,3,-5,-4,-3,4,-3,3,0,-2147483648,2,-2147483648,2,4,2,792,-2,-2,1],"l2_utf8":["ab","aa","","aa","aa","a",null,"zz","😀",null,"é","😀","b",null,"c","ab","ab","B",null,"😀","aa","ab","B","a","","ab","a","c","c","é","b","é","","","B","b","b","é","b","c","zz",null,"zz","c","ab","é","zz","aa","ab","","é","B","ab","B","B","B","aa","b","ab","b"],"l3_i32":[2,4,-5,0,-3,1,5,1,-2,4,2,4,0,-4,-4,1,null,1,-5,0,-1,-2147483648,null,829,-1,0,1,-4,3,-2,-5,-5,null,-1,3,2,1,0,3,1,-5,3,-1,-3,2147483647,-1,4,-1,-5,null,-3,-5,1,-2,-4,1,2,-4,-5,-1],"l4_utf8":["ab","aa","😀","é","zz","ab","a","B","é","😀","b","B","c","b","zz","c","b","B","😀",null,null,"é","zz","a","","","c","é","b","b","😀","😀","zz","B","zz","c","b","","aa","","B","ab","","aa","aa","😀","","😀","b","😀",null,"zz",null,"c","c","c","😀","😀","aa","😀"],"l5_f64":[-1.0,-4.0,-0.785,86.736,-51.792,1.0,22.674,-1.0,3.0,"NaN",null,-40.495,-3.0,-1.0,-5.507,0.0,null,1.0,"Infinity",4.0,1.0,1.029,77.125,80.272,2.0,null,-97.61,null,0.162,2.0,-54.182,-4.0,-4.0,30.133,-25.351,0.0,-95.774,-21.763,50.805,-20.86,24.348,3.0,-35.245,-82.184,null,"NaN",-86.909,60.393,-2.0,-2.0,22.212,4.0,2.0,1.0,81.519,0.0,2.0,"NaN",3.0,2.0]}},"right":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"r0_bool","type":"bool"}],"columns":{"r0_bool":[],"l0_utf8":[]}}},"ops":[{"op":"withColumns","exprs":{"x0":{"fn":"when","args":[{"fn":"eq","args":[{"lit":2},{"col":"l3_i32"}]},{"fn":"mod","args":[{"col":"l1_i32"},{"col":"l3_i32"}]},{"lit":0.5}]}}},{"op":"join","right":"right","on":["l0_utf8"],"how":"full"},{"op":"unique"}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_i32","type":"i32"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_f64","type":"f64"},{"name":"x0","type":"f64"},{"name":"r0_bool","type":"bool"}],"columns":{"l0_utf8":["B","c","é","ab","zz","ab","B","b","zz",null,"b","","c","zz","b","ab","ab","c","a","c","b","B","c","é","😀","c","zz",null,"ab","","zz","zz","B","b","😀","","","c","aa","ab","aa","ab",null,"B",null,"zz","ab","ab",null,"a","zz","aa","",null,"ab","c",null,"a","ab","aa"],"l1_i32":[5,-1,-1,-1,4,5,1,2,4,-4,-98,2147483647,2147483647,-5,-2,-4,-1,5,0,1,1,null,2147483647,-3,1,-2147483648,-5,-1,-4,1,-4,0,5,null,2,4,5,2,0,-5,-1,5,3,-5,-4,-3,4,-3,3,0,-2147483648,2,-2147483648,2,4,2,792,-2,-2,1],"l2_utf8":["ab","aa","","aa","aa","a",null,"zz","😀",null,"é","😀","b",null,"c","ab","ab","B",null,"😀","aa","ab","B","a","","ab","a","c","c","é","b","é","","","B","b","b","é","b","c","zz",null,"zz","c","ab","é","zz","aa","ab","","é","B","ab","B","B","B","aa","b","ab","b"],"l3_i32":[2,4,-5,0,-3,1,5,1,-2,4,2,4,0,-4,-4,1,null,1,-5,0,-1,-2147483648,null,829,-1,0,1,-4,3,-2,-5,-5,null,-1,3,2,1,0,3,1,-5,3,-1,-3,2147483647,-1,4,-1,-5,null,-3,-5,1,-2,-4,1,2,-4,-5,-1],"l4_utf8":["ab","aa","😀","é","zz","ab","a","B","é","😀","b","B","c","b","zz","c","b","B","😀",null,null,"é","zz","a","","","c","é","b","b","😀","😀","zz","B","zz","c","b","","aa","","B","ab","","aa","aa","😀","","😀","b","😀",null,"zz",null,"c","c","c","😀","😀","aa","😀"],"l5_f64":[-1.0,-4.0,-0.785,86.736,-51.792,1.0,22.674,-1.0,3.0,"NaN",null,-40.495,-3.0,-1.0,-5.507,0.0,null,1.0,"Infinity",4.0,1.0,1.029,77.125,80.272,2.0,null,-97.61,null,0.162,2.0,-54.182,-4.0,-4.0,30.133,-25.351,0.0,-95.774,-21.763,50.805,-20.86,24.348,3.0,-35.245,-82.184,null,"NaN",-86.909,60.393,-2.0,-2.0,22.212,4.0,2.0,1.0,81.519,0.0,2.0,"NaN",3.0,2.0],"x0":[1.0,0.5,0.5,0.5,0.5,0.5,0.5,0.5,0.5,0.5,0.0,0.5,0.5,0.5,0.5,0.5,0.5,0.5,0.5,0.5,0.5,0.5,0.5,0.5,0.5,0.5,0.5,0.5,0.5,0.5,0.5,0.5,0.5,0.5,0.5,0.0,0.5,0.5,0.5,0.5,0.5,0.5,0.5,0.5,0.5,0.5,0.5,0.5,0.5,0.5,0.5,0.5,0.5,0.5,0.5,0.5,0.0,0.5,0.5,0.5],"r0_bool":[null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null]}}},{"id":"random/0326","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_i32","type":"i32"}],"columns":{"l0_utf8":["é","😀","é","aa","aa","B","😀","a","ab","aa","b","aa","aa","a","aa","b","é","😀","c","😀"],"l1_i32":[4,4,-4,-4,-5,-2,5,-4,-5,1,1,4,-5,-1,3,5,-4,0,-2,2],"l2_f64":[-2.0,3.0,93.997,-3.0,-3.0,-1.0,2.0,-2.0,-1.0,85.929,-51.236,2.731,0.0,88.98,-12.564,74.445,-3.0,0.0,-2.0,3.0],"l3_i32":[-2,-1,3,5,-1,4,0,-3,-2,1,4,0,-5,-5,-906,2147483647,-1,4,1,5]}},"right":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"r0_f64","type":"f64"}],"columns":{"r0_f64":[null,21.515,null,25.888,-51.528,null,5.198,-90.889,-4.0,"NaN",null,-1.0,-33.017,2.0,0.0,1.0,-10.947,-69.676,-3.0,-56.61,0.0,14.478,3.0,-48.5,-2.0,null,-3.0,null,-4.0,-82.267],"l0_utf8":["é","é","B","é","aa","B","c","é","aa","aa","aa","😀","a","é","a","😀","é","é","😀","a","b","a","aa","é","aa","é","aa","aa","b","aa"]}}},"ops":[{"op":"join","right":"right","on":["l0_utf8"],"how":"inner"},{"op":"sort","keys":[{"expr":{"col":"l3_i32"},"descending":false}],"nullsLast":false},{"op":"filter","predicate":{"fn":"isNotNull","args":[{"col":"l1_i32"}]}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_i32","type":"i32"},{"name":"r0_f64","type":"f64"}],"columns":{"l0_utf8":["aa","aa","aa","aa","aa","aa","aa","aa","aa","aa","aa","aa","aa","aa","aa","aa","aa","aa","a","a","a","a","a","a","a","a","é","é","é","é","é","é","é","é","é","😀","😀","😀","aa","aa","aa","aa","aa","aa","aa","aa","aa","é","é","é","é","é","é","é","é","é","😀","😀","😀","aa","aa","aa","aa","aa","aa","aa","aa","aa","aa","aa","aa","aa","aa","aa","aa","aa","aa","c","é","é","é","é","é","é","é","é","é","B","B","b","b","😀","😀","😀","aa","aa","aa","aa","aa","aa","aa","aa","aa","😀","😀","😀","b","b"],"l1_i32":[3,3,3,3,3,3,3,3,3,-5,-5,-5,-5,-5,-5,-5,-5,-5,-1,-1,-1,-1,-4,-4,-4,-4,4,4,4,4,4,4,4,4,4,4,4,4,-5,-5,-5,-5,-5,-5,-5,-5,-5,-4,-4,-4,-4,-4,-4,-4,-4,-4,5,5,5,4,4,4,4,4,4,4,4,4,1,1,1,1,1,1,1,1,1,-2,-4,-4,-4,-4,-4,-4,-4,-4,-4,-2,-2,1,1,0,0,0,-4,-4,-4,-4,-4,-4,-4,-4,-4,2,2,2,5,5],"l2_f64":[-12.564,-12.564,-12.564,-12.564,-12.564,-12.564,-12.564,-12.564,-12.564,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,88.98,88.98,88.98,88.98,-2.0,-2.0,-2.0,-2.0,-2.0,-2.0,-2.0,-2.0,-2.0,-2.0,-2.0,-2.0,-2.0,3.0,3.0,3.0,-3.0,-3.0,-3.0,-3.0,-3.0,-3.0,-3.0,-3.0,-3.0,-3.0,-3.0,-3.0,-3.0,-3.0,-3.0,-3.0,-3.0,-3.0,2.0,2.0,2.0,2.731,2.731,2.731,2.731,2.731,2.731,2.731,2.731,2.731,85.929,85.929,85.929,85.929,85.929,85.929,85.929,85.929,85.929,-2.0,93.997,93.997,93.997,93.997,93.997,93.997,93.997,93.997,93.997,-1.0,-1.0,-51.236,-51.236,0.0,0.0,0.0,-3.0,-3.0,-3.0,-3.0,-3.0,-3.0,-3.0,-3.0,-3.0,3.0,3.0,3.0,74.445,74.445],"l3_i32":[-906,-906,-906,-906,-906,-906,-906,-906,-906,-5,-5,-5,-5,-5,-5,-5,-5,-5,-5,-5,-5,-5,-3,-3,-3,-3,-2,-2,-2,-2,-2,-2,-2,-2,-2,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,0,0,0,0,0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1,1,1,3,3,3,3,3,3,3,3,3,4,4,4,4,4,4,4,5,5,5,5,5,5,5,5,5,5,5,5,2147483647,2147483647],"r0_f64":[-51.528,-4.0,"NaN",null,3.0,-2.0,-3.0,null,-82.267,-51.528,-4.0,"NaN",null,3.0,-2.0,-3.0,null,-82.267,-33.017,0.0,-56.61,14.478,-33.017,0.0,-56.61,14.478,null,21.515,25.888,-90.889,2.0,-10.947,-69.676,-48.5,null,-1.0,1.0,-3.0,-51.528,-4.0,"NaN",null,3.0,-2.0,-3.0,null,-82.267,null,21.515,25.888,-90.889,2.0,-10.947,-69.676,-48.5,null,-1.0,1.0,-3.0,-51.528,-4.0,"NaN",null,3.0,-2.0,-3.0,null,-82.267,-51.528,-4.0,"NaN",null,3.0,-2.0,-3.0,null,-82.267,5.198,null,21.515,25.888,-90.889,2.0,-10.947,-69.676,-48.5,null,null,null,0.0,-4.0,-1.0,1.0,-3.0,-51.528,-4.0,"NaN",null,3.0,-2.0,-3.0,null,-82.267,-1.0,1.0,-3.0,0.0,-4.0]}}},{"id":"random/0327","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i64","type":"i64"},{"name":"l3_f64","type":"f64"},{"name":"l4_f64","type":"f64"},{"name":"l5_utf8","type":"utf8"},{"name":"l6_i64","type":"i64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_i64":[],"l3_f64":[],"l4_f64":[],"l5_utf8":[],"l6_i64":[]}},"right":{"fields":[{"name":"l5_utf8","type":"utf8"},{"name":"r0_f64","type":"f64"}],"columns":{"r0_f64":[-48.354,null,-4.0,-15.727,null,30.4,null,null,2.0,null,null,1.0,null,-3.0,44.911,"NaN",null,3.0,2.0,null,4.0,47.398,58.981,null,-1.0,-1.0,-2.0,null,null,null],"l5_utf8":["ab","a","a","😀","a","zz",null,"ab",null,"zz","zz","ab","a","a",null,"ab","ab","😀","zz","zz",null,null,"zz","ab","😀",null,"😀","ab","ab",null]}}},"ops":[{"op":"unique","columns":["l3_f64"]},{"op":"head","n":5},{"op":"filter","predicate":{"fn":"isNull","args":[{"col":"l6_i64"}]}},{"op":"join","right":"right","on":["l5_utf8"],"how":"semi"}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i64","type":"i64"},{"name":"l3_f64","type":"f64"},{"name":"l4_f64","type":"f64"},{"name":"l5_utf8","type":"utf8"},{"name":"l6_i64","type":"i64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_i64":[],"l3_f64":[],"l4_f64":[],"l5_utf8":[],"l6_i64":[]}}},{"id":"random/0328","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_utf8","type":"utf8"},{"name":"l6_utf8","type":"utf8"}],"columns":{"l0_utf8":[null],"l1_i32":[-5],"l2_utf8":["ab"],"l3_utf8":["zz"],"l4_utf8":[""],"l5_utf8":["é"],"l6_utf8":[""]}}},"ops":[{"op":"withColumns","exprs":{"x0":{"fn":"sub","args":[{"fn":"neg","args":[{"col":"l1_i32"}]},{"col":"l1_i32"}]}}},{"op":"withColumns","exprs":{"x1":{"col":"x0"}}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_utf8","type":"utf8"},{"name":"l6_utf8","type":"utf8"},{"name":"x0","type":"i32"},{"name":"x1","type":"i32"}],"columns":{"l0_utf8":[null],"l1_i32":[-5],"l2_utf8":["ab"],"l3_utf8":["zz"],"l4_utf8":[""],"l5_utf8":["é"],"l6_utf8":[""],"x0":[10],"x1":[10]}}},{"id":"random/0329","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_f64","type":"f64"},{"name":"l4_bool","type":"bool"},{"name":"l5_bool","type":"bool"},{"name":"l6_i32","type":"i32"}],"columns":{"l0_utf8":["é","c","aa","c","B"],"l1_i32":[3,3,-4,null,-5],"l2_f64":[-13.792,null,null,"NaN",null],"l3_f64":[null,-1.0,-3.0,null,1.0],"l4_bool":[false,true,null,false,false],"l5_bool":[true,false,null,true,true],"l6_i32":[2,3,1,2,-1]}}},"ops":[{"op":"filter","predicate":{"fn":"isNull","args":[{"col":"l4_bool"}]}},{"op":"sort","keys":[{"expr":{"col":"l1_i32"},"descending":true},{"expr":{"col":"l6_i32"},"descending":true}],"nullsLast":true},{"op":"withColumns","exprs":{"x2":{"fn":"fillNull","args":[{"fn":"when","args":[{"fn":"isNotNull","args":[{"col":"l0_utf8"}]},{"col":"l6_i32"},{"col":"l6_i32"}]},{"lit":0}]}}},{"op":"sort","keys":[{"expr":{"col":"x2"},"descending":false},{"expr":{"col":"l3_f64"},"descending":true}],"nullsLast":true}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_f64","type":"f64"},{"name":"l4_bool","type":"bool"},{"name":"l5_bool","type":"bool"},{"name":"l6_i32","type":"i32"},{"name":"x2","type":"i32"}],"columns":{"l0_utf8":["aa"],"l1_i32":[-4],"l2_f64":[null],"l3_f64":[-3.0],"l4_bool":[null],"l5_bool":[null],"l6_i32":[1],"x2":[1]}}},{"id":"random/0330","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i64","type":"i64"},{"name":"l3_f64","type":"f64"},{"name":"l4_f64","type":"f64"}],"columns":{"l0_utf8":["c","aa",null,"b","a"],"l1_i32":[5,-4,5,5,null],"l2_i64":["-4611686018427387904","9007199254740995","1",null,"9007199254740995"],"l3_f64":[3.0,-63.171,-29.968,null,-51.323],"l4_f64":[7.766,null,1.0,null,null]}},"right":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"r0_utf8","type":"utf8"}],"columns":{"r0_utf8":["ab","ab",null,"zz","😀","a","c","c","😀","B"],"l0_utf8":["a",null,"b","b","a","aa","aa","a","b","c"]}}},"ops":[{"op":"join","right":"right","on":["l0_utf8"],"how":"left"}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i64","type":"i64"},{"name":"l3_f64","type":"f64"},{"name":"l4_f64","type":"f64"},{"name":"r0_utf8","type":"utf8"}],"columns":{"l0_utf8":["c","aa","aa",null,"b","b","b","a","a","a"],"l1_i32":[5,-4,-4,5,5,5,5,null,null,null],"l2_i64":["-4611686018427387904","9007199254740995","9007199254740995","1",null,null,null,"9007199254740995","9007199254740995","9007199254740995"],"l3_f64":[3.0,-63.171,-63.171,-29.968,null,null,null,-51.323,-51.323,-51.323],"l4_f64":[7.766,null,null,1.0,null,null,null,null,null,null],"r0_utf8":["B","a","c",null,null,"zz","😀","ab","😀","c"]}}},{"id":"random/0331","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_f64","type":"f64"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_f64","type":"f64"}],"columns":{"l0_utf8":["a","aa","zz","é","aa"],"l1_i32":[-2,-1,2,-1,-5],"l2_f64":[4.0,-90.04,0.0,3.0,-63.794],"l3_f64":[49.181,93.965,3.0,-2.0,8.462],"l4_utf8":["aa","a","😀","😀","B"],"l5_f64":[70.517,26.583,-11.654,34.858,2.0]}}},"ops":[{"op":"filter","predicate":{"fn":"gte","args":[{"col":"l4_utf8"},{"lit":""}]}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_f64","type":"f64"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_f64","type":"f64"}],"columns":{"l0_utf8":["a","aa","zz","é","aa"],"l1_i32":[-2,-1,2,-1,-5],"l2_f64":[4.0,-90.04,0.0,3.0,-63.794],"l3_f64":[49.181,93.965,3.0,-2.0,8.462],"l4_utf8":["aa","a","😀","😀","B"],"l5_f64":[70.517,26.583,-11.654,34.858,2.0]}}},{"id":"random/0332","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_bool","type":"bool"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_bool","type":"bool"},{"name":"l6_f64","type":"f64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_f64":[],"l3_bool":[],"l4_utf8":[],"l5_bool":[],"l6_f64":[]}},"right":{"fields":[{"name":"l1_i32","type":"i32"},{"name":"r0_f64","type":"f64"}],"columns":{"r0_f64":[-3.0,-1.0,24.233,null,null,30.994,-11.362,13.536,1.0,3.0,-44.785,63.041,-2.0,1.0,2.0,-2.0,42.463,null,2.0,null,0.0,-32.384,0.0,null,null,-1.0,-62.665,-40.386,"Infinity",-3.0],"l1_i32":[-5,null,137,-5,-5,-5,-2147483648,null,137,null,137,-2147483648,137,null,-2147483648,-5,null,-5,null,137,-5,-2147483648,137,null,null,-2147483648,137,-5,null,137]}}},"ops":[{"op":"join","right":"right","on":["l1_i32"],"how":"anti"},{"op":"unique","columns":["l3_bool"]}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_bool","type":"bool"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_bool","type":"bool"},{"name":"l6_f64","type":"f64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_f64":[],"l3_bool":[],"l4_utf8":[],"l5_bool":[],"l6_f64":[]}}},{"id":"random/0333","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_bool","type":"bool"},{"name":"l5_utf8","type":"utf8"}],"columns":{"l0_utf8":["","B","b","","aa","ab","a","zz","é","é","ab","ab","c","😀","b","😀","c","é","","aa"],"l1_i32":[-2,3,3,-5,1,0,5,0,4,-5,-1,1,2,1,3,-5,4,-3,-4,4],"l2_i32":[-2,1,1,1,-137,1,-4,-5,5,-5,1,-1,-4,-2,-3,0,4,-1,-1,5],"l3_utf8":["c","zz","a","aa","é","😀","a","a","😀","B","zz","","zz","😀","aa","","b","ab","aa",""],"l4_bool":[true,false,false,false,false,false,true,true,false,true,true,false,true,true,false,false,false,true,false,true],"l5_utf8":["zz","ab","","c","é","ab","B","zz","c","aa","","","","a","","zz","aa","b","😀","a"]}}},"ops":[{"op":"head","n":9},{"op":"head","n":2}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_bool","type":"bool"},{"name":"l5_utf8","type":"utf8"}],"columns":{"l0_utf8":["","B"],"l1_i32":[-2,3],"l2_i32":[-2,1],"l3_utf8":["c","zz"],"l4_bool":[true,false],"l5_utf8":["zz","ab"]}}},{"id":"random/0334","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_i32","type":"i32"}],"columns":{"l0_utf8":[""],"l1_i32":[-1],"l2_i32":[-2],"l3_i32":[1]}}},"ops":[{"op":"head","n":5}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_i32","type":"i32"}],"columns":{"l0_utf8":[""],"l1_i32":[-1],"l2_i32":[-2],"l3_i32":[1]}}},{"id":"random/0335","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_f64","type":"f64"},{"name":"l5_f64","type":"f64"}],"columns":{"l0_utf8":["aa","c","a","","b","","😀","","B","c","é","b","ab","é","a","ab","","","b","aa"],"l1_i32":[1,5,-2,-3,-5,-2,2,1,1,2,1,-3,2147483647,1,-4,4,3,-3,2,-4],"l2_utf8":["B","","a","😀","B","😀","😀","a","a","a","c","b","b","c","é","B","ab","😀","aa",""],"l3_utf8":["a","b","b","","B","","😀","é","ab","aa","aa","ab","😀","c","ab","é","ab","zz","zz","zz"],"l4_f64":[-1.0,-4.0,-4.0,4.0,68.766,3.0,-2.0,-79.718,2.0,-2.0,3.0,0.0,57.309,4.0,-2.0,2.0,4.0,80.324,-57.7,1.0],"l5_f64":[3.0,-83.617,-4.0,-44.595,92.618,-35.172,-29.783,0.0,4.0,2.0,1.0,53.314,-4.0,57.942,1.0,68.779,27.002,3.0,-4.0,-20.644]}}},"ops":[{"op":"sort","keys":[{"expr":{"col":"l5_f64"},"descending":false},{"expr":{"col":"l0_utf8"},"descending":true}],"nullsLast":false},{"op":"filter","predicate":{"fn":"isNull","args":[{"col":"l2_utf8"}]}},{"op":"sort","keys":[{"expr":{"col":"l3_utf8"},"descending":false}],"nullsLast":false}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_f64","type":"f64"},{"name":"l5_f64","type":"f64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_utf8":[],"l3_utf8":[],"l4_f64":[],"l5_f64":[]}}},{"id":"random/0336","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_utf8","type":"utf8"}],"columns":{"l0_utf8":["😀","é","ab","é",null],"l1_i32":[0,null,4,null,-5],"l2_utf8":[null,null,"zz","a","😀"],"l3_utf8":["b","aa","é",null,null]}}},"ops":[{"op":"filter","predicate":{"fn":"isNotNull","args":[{"col":"l3_utf8"}]}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_utf8","type":"utf8"}],"columns":{"l0_utf8":["😀","é","ab"],"l1_i32":[0,null,4],"l2_utf8":[null,null,"zz"],"l3_utf8":["b","aa","é"]}}},{"id":"random/0337","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_f64","type":"f64"},{"name":"l4_i32","type":"i32"},{"name":"l5_i64","type":"i64"},{"name":"l6_bool","type":"bool"}],"columns":{"l0_utf8":["é"],"l1_i32":[2],"l2_utf8":["c"],"l3_f64":[-72.194],"l4_i32":[-2],"l5_i64":["-4611686018427387904"],"l6_bool":[true]}},"right":{"fields":[{"name":"l5_i64","type":"i64"},{"name":"r0_utf8","type":"utf8"}],"columns":{"r0_utf8":["é","é","ab"],"l5_i64":["-4611686018427387904","-4611686018427387904","-4611686018427387904"]}}},"ops":[{"op":"filter","predicate":{"fn":"isNull","args":[{"col":"l1_i32"}]}},{"op":"join","right":"right","on":["l5_i64"],"how":"left"},{"op":"withColumns","exprs":{"x2":{"fn":"strStartsWith","args":[{"col":"r0_utf8"}],"prefix":""}}},{"op":"groupBy","keys":["r0_utf8"],"aggs":{"n3":{"agg":"len"},"a3_0":{"agg":"nUnique","arg":{"col":"l4_i32"}},"a3_1":{"agg":"min","arg":{"col":"l2_utf8"}},"a3_2":{"agg":"max","arg":{"col":"l2_utf8"}}}}],"expected":{"fields":[{"name":"r0_utf8","type":"utf8"},{"name":"n3","type":"u32"},{"name":"a3_0","type":"u32"},{"name":"a3_1","type":"utf8"},{"name":"a3_2","type":"utf8"}],"columns":{"r0_utf8":[],"n3":[],"a3_0":[],"a3_1":[],"a3_2":[]}}},{"id":"random/0338","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_bool","type":"bool"},{"name":"l4_f64","type":"f64"},{"name":"l5_utf8","type":"utf8"}],"columns":{"l0_utf8":["é","B","zz","😀","ab","","zz","","é","zz","zz","c","a","a","zz","zz","zz","a","zz","c","","","zz","B","B","","a","c","ab","zz","é","","B","😀","a","é","","B","c","é","😀","B","B","ab","c","é","","😀","aa","a","","aa","B","ab","c","a","","B","a","c"],"l1_i32":[-3,0,3,4,2147483647,0,3,-4,0,4,4,-2,2147483647,3,-4,2,-4,-4,4,-3,2,-1,1,0,4,-3,1,5,1,-4,-2,5,-1,-5,0,2,-2,0,1,2,4,1,-1,-1,-1,-4,-4,-186,0,1,0,-5,-2,1,457,-2,-4,0,4,0],"l2_bool":[true,true,true,true,true,false,false,false,false,true,false,true,true,true,true,true,true,true,true,true,false,false,false,false,false,false,true,true,true,false,false,true,false,false,false,true,true,false,false,false,true,false,true,true,true,true,false,true,false,true,false,true,true,true,false,false,true,false,false,false],"l3_bool":[true,false,false,true,false,true,false,true,true,true,true,true,true,true,true,false,false,true,false,false,true,false,true,false,true,true,true,true,false,true,false,false,true,false,true,false,true,false,false,false,true,true,false,true,true,false,true,false,true,true,false,true,true,false,true,true,true,false,true,true],"l4_f64":[-3.0,3.0,-2.0,1.0,74.616,-27.784,-39.18,0.0,3.0,"Infinity","-0",-3.0,-70.052,3.0,4.0,0.0,-0.823,-28.264,-2.0,"NaN",4.0,-3.0,61.994,2.0,4.0,-2.0,-4.0,-82.517,89.721,3.0,0.0,-72.568,3.0,"NaN",2.0,"-Infinity",4.0,1.0,-4.0,56.028,-2.0,0.0,-2.0,99.684,-54.377,-2.0,-2.0,3.0,-81.435,-63.381,-60.605,-92.898,3.0,-76.533,3.0,3.0,-1.0,-4.0,-1.0,14.964],"l5_utf8":["","","B","B","a","","c","😀","é","c","c","c","B","c","é","é","c","B","B","","b","aa","aa","😀","aa","B","B","é","","b","é","ab","é","😀","😀","aa","zz","","zz","aa","zz","c","😀","a","aa","","zz","😀","a","B","é","ab","B","zz","a","","","b","B","zz"]}},"right":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"r0_bool","type":"bool"}],"columns":{"r0_bool":[true,null,true,true,true,null,true,false,false,false,true,true,false,true,true,true,false,true,true,null,false,true,true,null,true,true,true,true,null,true],"l0_utf8":["B","zz","zz","zz","zz","zz","a","","B","c","c","c","zz","😀","","é","😀","a","","","a","","","é","","zz","","B","B",""]}}},"ops":[{"op":"join","right":"right","on":["l0_utf8"],"how":"inner"}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_bool","type":"bool"},{"name":"l4_f64","type":"f64"},{"name":"l5_utf8","type":"utf8"},{"name":"r0_bool","type":"bool"}],"columns":{"l0_utf8":["é","é","B","B","B","B","zz","zz","zz","zz","zz","zz","zz","😀","😀","","","","","","","","","","zz","zz","zz","zz","zz","zz","zz","","","","","","","","","","é","é","zz","zz","zz","zz","zz","zz","zz","zz","zz","zz","zz","zz","zz","zz","c","c","c","a","a","a","a","a","a","zz","zz","zz","zz","zz","zz","zz","zz","zz","zz","zz","zz","zz","zz","zz","zz","zz","zz","zz","zz","zz","a","a","a","zz","zz","zz","zz","zz","zz","zz","c","c","c","","","","","","","","","","","","","","","","","","","zz","zz","zz","zz","zz","zz","zz","B","B","B","B","B","B","B","B","","","","","","","","","","a","a","a","c","c","c","zz","zz","zz","zz","zz","zz","zz","é","é","","","","","","","","","","B","B","B","B","😀","😀","a","a","a","é","é","","","","","","","","","","B","B","B","B","c","c","c","é","é","😀","😀","B","B","B","B","B","B","B","B","c","c","c","é","é","","","","","","","","","","😀","😀","a","a","a","","","","","","","","","","B","B","B","B","c","c","c","a","a","a","","","","","","","","","","B","B","B","B","a","a","a","c","c","c"],"l1_i32":[-3,-3,0,0,0,0,3,3,3,3,3,3,3,4,4,0,0,0,0,0,0,0,0,0,3,3,3,3,3,3,3,-4,-4,-4,-4,-4,-4,-4,-4,-4,0,0,4,4,4,4,4,4,4,4,4,4,4,4,4,4,-2,-2,-2,2147483647,2147483647,2147483647,3,3,3,-4,-4,-4,-4,-4,-4,-4,2,2,2,2,2,2,2,-4,-4,-4,-4,-4,-4,-4,-4,-4,-4,4,4,4,4,4,4,4,-3,-3,-3,2,2,2,2,2,2,2,2,2,-1,-1,-1,-1,-1,-1,-1,-1,-1,1,1,1,1,1,1,1,0,0,0,0,4,4,4,4,-3,-3,-3,-3,-3,-3,-3,-3,-3,1,1,1,5,5,5,-4,-4,-4,-4,-4,-4,-4,-2,-2,5,5,5,5,5,5,5,5,5,-1,-1,-1,-1,-5,-5,0,0,0,2,2,-2,-2,-2,-2,-2,-2,-2,-2,-2,0,0,0,0,1,1,1,2,2,4,4,1,1,1,1,-1,-1,-1,-1,-1,-1,-1,-4,-4,-4,-4,-4,-4,-4,-4,-4,-4,-4,-186,-186,1,1,1,0,0,0,0,0,0,0,0,0,-2,-2,-2,-2,457,457,457,-2,-2,-2,-4,-4,-4,-4,-4,-4,-4,-4,-4,0,0,0,0,4,4,4,0,0,0],"l2_bool":[true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,true,true,true,true,true,true,true,false,false,false,false,false,false,false,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,true,true,true,true,true,true,false,false,false,false,false,false,false,false,false,true,true,true,true,true,true,true,true,true,false,false,false,false,false,false,false,false,false,true,true,true,true,true,true,true,true,true,true,true,false,false,false,false,false,false,false,false,false,true,true,false,false,false,false,true,true,true,true,true,true,true,true,true,false,false,false,false,false,false,false,false,false,true,true,true,true,true,false,false,false,false,false,false,false,false,false,true,true,true,true,false,false,false,false,false,false,true,true,true,true,true,true,true,true,true,false,false,false,false,false,false,false,false,false,false],"l3_bool":[true,true,false,false,false,false,false,false,false,false,false,false,false,true,true,true,true,true,true,true,true,true,true,true,false,false,false,false,false,false,false,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,false,false,false,false,false,false,false,false,false,false,false,false,false,false,true,true,true,false,false,false,false,false,false,false,false,false,false,true,true,true,true,true,true,true,true,true,false,false,false,false,false,false,false,false,false,true,true,true,true,true,true,true,false,false,false,false,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,false,false,false,false,false,false,false,false,false,false,false,true,true,true,true,false,false,true,true,true,false,false,true,true,true,true,true,true,true,true,true,false,false,false,false,false,false,false,false,false,true,true,true,true,true,true,false,false,false,false,true,true,true,false,false,true,true,true,true,true,true,true,true,true,false,false,true,true,true,false,false,false,false,false,false,false,false,false,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,false,false,false,false,true,true,true,true,true,true],"l4_f64":[-3.0,-3.0,3.0,3.0,3.0,3.0,-2.0,-2.0,-2.0,-2.0,-2.0,-2.0,-2.0,1.0,1.0,-27.784,-27.784,-27.784,-27.784,-27.784,-27.784,-27.784,-27.784,-27.784,-39.18,-39.18,-39.18,-39.18,-39.18,-39.18,-39.18,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,3.0,3.0,"Infinity","Infinity","Infinity","Infinity","Infinity","Infinity","Infinity","-0","-0","-0","-0","-0","-0","-0",-3.0,-3.0,-3.0,-70.052,-70.052,-70.052,3.0,3.0,3.0,4.0,4.0,4.0,4.0,4.0,4.0,4.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.823,-0.823,-0.823,-0.823,-0.823,-0.823,-0.823,-28.264,-28.264,-28.264,-2.0,-2.0,-2.0,-2.0,-2.0,-2.0,-2.0,"NaN","NaN","NaN",4.0,4.0,4.0,4.0,4.0,4.0,4.0,4.0,4.0,-3.0,-3.0,-3.0,-3.0,-3.0,-3.0,-3.0,-3.0,-3.0,61.994,61.994,61.994,61.994,61.994,61.994,61.994,2.0,2.0,2.0,2.0,4.0,4.0,4.0,4.0,-2.0,-2.0,-2.0,-2.0,-2.0,-2.0,-2.0,-2.0,-2.0,-4.0,-4.0,-4.0,-82.517,-82.517,-82.517,3.0,3.0,3.0,3.0,3.0,3.0,3.0,0.0,0.0,-72.568,-72.568,-72.568,-72.568,-72.568,-72.568,-72.568,-72.568,-72.568,3.0,3.0,3.0,3.0,"NaN","NaN",2.0,2.0,2.0,"-Infinity","-Infinity",4.0,4.0,4.0,4.0,4.0,4.0,4.0,4.0,4.0,1.0,1.0,1.0,1.0,-4.0,-4.0,-4.0,56.028,56.028,-2.0,-2.0,0.0,0.0,0.0,0.0,-2.0,-2.0,-2.0,-2.0,-54.377,-54.377,-54.377,-2.0,-2.0,-2.0,-2.0,-2.0,-2.0,-2.0,-2.0,-2.0,-2.0,-2.0,3.0,3.0,-63.381,-63.381,-63.381,-60.605,-60.605,-60.605,-60.605,-60.605,-60.605,-60.605,-60.605,-60.605,3.0,3.0,3.0,3.0,3.0,3.0,3.0,3.0,3.0,3.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-4.0,-4.0,-4.0,-4.0,-1.0,-1.0,-1.0,14.964,14.964,14.964],"l5_utf8":["","","","","","","B","B","B","B","B","B","B","B","B","","","","","","","","","","c","c","c","c","c","c","c","😀","😀","😀","😀","😀","😀","😀","😀","😀","é","é","c","c","c","c","c","c","c","c","c","c","c","c","c","c","c","c","c","B","B","B","c","c","c","é","é","é","é","é","é","é","é","é","é","é","é","é","é","c","c","c","c","c","c","c","B","B","B","B","B","B","B","B","B","B","","","","b","b","b","b","b","b","b","b","b","aa","aa","aa","aa","aa","aa","aa","aa","aa","aa","aa","aa","aa","aa","aa","aa","😀","😀","😀","😀","aa","aa","aa","aa","B","B","B","B","B","B","B","B","B","B","B","B","é","é","é","b","b","b","b","b","b","b","é","é","ab","ab","ab","ab","ab","ab","ab","ab","ab","é","é","é","é","😀","😀","😀","😀","😀","aa","aa","zz","zz","zz","zz","zz","zz","zz","zz","zz","","","","","zz","zz","zz","aa","aa","zz","zz","c","c","c","c","😀","😀","😀","😀","aa","aa","aa","","","zz","zz","zz","zz","zz","zz","zz","zz","zz","😀","😀","B","B","B","é","é","é","é","é","é","é","é","é","B","B","B","B","a","a","a","","","","","","","","","","","","","b","b","b","b","B","B","B","zz","zz","zz"],"r0_bool":[true,null,true,false,true,null,null,true,true,true,null,false,true,true,false,false,true,true,null,true,true,true,true,true,null,true,true,true,null,false,true,false,true,true,null,true,true,true,true,true,true,null,null,true,true,true,null,false,true,null,true,true,true,null,false,true,false,true,true,true,true,false,true,true,false,null,true,true,true,null,false,true,null,true,true,true,null,false,true,null,true,true,true,null,false,true,true,true,false,null,true,true,true,null,false,true,false,true,true,false,true,true,null,true,true,true,true,true,false,true,true,null,true,true,true,true,true,null,true,true,true,null,false,true,true,false,true,null,true,false,true,null,false,true,true,null,true,true,true,true,true,true,true,false,false,true,true,null,true,true,true,null,false,true,true,null,false,true,true,null,true,true,true,true,true,true,false,true,null,true,false,true,true,false,true,null,false,true,true,null,true,true,true,true,true,true,false,true,null,false,true,true,true,null,true,false,true,false,true,null,true,false,true,null,false,true,true,true,null,false,true,true,null,true,true,true,true,true,true,false,true,true,false,false,true,true,null,true,true,true,true,true,true,false,true,null,false,true,true,true,true,false,false,true,true,null,true,true,true,true,true,true,false,true,null,true,true,false,false,true,true]}}},{"id":"random/0339","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_i64","type":"i64"},{"name":"l4_i32","type":"i32"}],"columns":{"l0_utf8":[null,null,"a",null,"ab","","aa",null,null,"ab","B",null,null,"ab","é","é","zz",null,"c",null],"l1_i32":[2,null,5,1,null,2147483647,-4,-3,-5,2,null,null,null,-3,2,1,2,null,null,null],"l2_i32":[null,null,0,-5,3,3,0,1,-4,-5,null,0,-4,null,null,null,null,null,-4,3],"l3_i64":["9007199254740993","-4611686018427387904","-1",null,"4","-4611686018427387904","9007199254740992","9007199254740994","-1",null,null,"-4611686018427387904",null,"-4611686018427387904","3","-4611686018427387904","1",null,"-5","-4611686018427387904"],"l4_i32":[0,null,-1,null,-1,-1,-3,-1,null,1,null,null,3,-1,-2147483648,0,null,2,-2147483648,0]}}},"ops":[{"op":"unique"},{"op":"filter","predicate":{"fn":"neq","args":[{"fn":"sub","args":[{"col":"l2_i32"},{"col":"l2_i32"}]},{"col":"l2_i32"}]}},{"op":"unique","columns":["l4_i32"]},{"op":"groupBy","keys":["l2_i32"],"aggs":{"n3":{"agg":"len"},"a3_0":{"agg":"last","arg":{"col":"l0_utf8"}},"a3_1":{"agg":"mean","arg":{"col":"l4_i32"}},"a3_2":{"agg":"first","arg":{"col":"l3_i64"}}}}],"expected":{"fields":[{"name":"l2_i32","type":"i32"},{"name":"n3","type":"u32"},{"name":"a3_0","type":"utf8"},{"name":"a3_1","type":"f64"},{"name":"a3_2","type":"i64"}],"columns":{"l2_i32":[-5,3,-4],"n3":[2,2,2],"a3_0":["ab",null,"c"],"a3_1":[1.0,-0.5,-1073741822.5],"a3_2":[null,"4",null]}}},{"id":"random/0340","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_utf8","type":"utf8"}],"columns":{"l0_utf8":["😀","😀","B","b","b","aa",null,"b",null,null,"","é",null,"",null,"😀","","","B","","aa","a","b",null,"B","zz",null,"a","a",null,"c",null,"",null,"b","😀","","😀",null,"B","aa",null,"aa","b","😀","é","aa","a",null,"a","aa","😀","😀",null,"ab","a","c",null,"aa","aa"],"l1_i32":[null,null,null,null,-5,4,-2,-2,null,2,-1,4,5,5,5,-2,-4,551,-4,-1,4,2147483647,null,null,4,-3,null,null,4,1,null,null,4,null,null,5,0,5,3,null,5,-1,-5,37,null,2147483647,-1,null,2,0,2,-4,null,-2147483648,2,2,-4,0,-2,-3],"l2_bool":[null,null,true,true,null,false,null,true,null,true,false,false,null,false,false,true,false,false,false,false,false,false,true,false,null,false,false,false,true,null,null,null,false,false,false,false,false,true,true,null,true,null,false,true,false,true,null,null,true,true,false,null,true,null,true,true,true,true,false,false],"l3_utf8":["zz",null,"b",null,"c","B","a",null,"","aa",null,"😀","a",null,null,null,null,"b","aa","","é","","b","é","ab","c",null,"b","é","ab","é","b","B","zz","B",null,"ab","B",null,"","zz","c",null,"c","c","é","",null,"é",null,"ab","B","",null,null,null,"","b","ab","c"],"l4_utf8":[null,"a",null,"zz","aa",null,null,"😀",null,"ab","B","aa","B","ab",null,"a",null,null,"B","c","😀",null,"",null,null,"😀",null,"B","é","","zz","😀",null,"a",null,"a","b","",null,null,"","a","é",null,"","ab",null,"zz",null,"","c","é",null,null,"","c","aa",null,"b","😀"],"l5_utf8":["é",null,null,"ab","😀","B",null,null,"","a",null,null,"😀","é","aa",null,null,"","","zz","ab",null,"a",null,"c",null,null,null,null,null,"aa",null,null,"a",null,"b","a","aa","a","😀","ab","aa",null,null,"c",null,null,"b","B",null,null,"c","zz",null,null,"",null,null,null,null]}}},"ops":[{"op":"withColumns","exprs":{"x0":{"col":"l1_i32"}}},{"op":"filter","predicate":{"fn":"isNotNull","args":[{"col":"l3_utf8"}]}},{"op":"head","n":2}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_utf8","type":"utf8"},{"name":"x0","type":"i32"}],"columns":{"l0_utf8":["😀","B"],"l1_i32":[null,null],"l2_bool":[null,true],"l3_utf8":["zz","b"],"l4_utf8":[null,null],"l5_utf8":["é",null],"x0":[null,null]}}},{"id":"random/0341","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_i32","type":"i32"},{"name":"l4_utf8","type":"utf8"}],"columns":{"l0_utf8":["aa","","c","a","é","","B","aa","aa","b","zz","a","b","a","aa","ab","c","b","a","c","aa","b","","a","c","😀","b","b","ab","B","a","zz","aa","aa","","B","é","","ab","aa","😀","B","b","","é","ab","😀","","b","zz","B","b","é","é","b","aa","a","aa","aa","a"],"l1_i32":[-2,0,-1,2147483647,-2,-5,2,-2147483648,2,-983,5,4,0,-1,1,3,2,2,-2,-3,1,3,-1,-1,2147483647,-2,5,-1,4,1,-5,1,-2147483648,-2,2,2,3,-3,-5,3,-3,1,-3,-3,0,5,5,5,2,5,0,2,1,1,4,-1,-2147483648,0,-2,-1],"l2_bool":[true,true,false,true,false,true,false,false,true,true,true,true,false,false,false,false,false,false,true,true,true,true,true,false,false,false,false,false,true,true,true,false,false,false,false,true,true,false,false,true,true,true,true,false,false,true,false,true,false,true,true,false,false,true,true,true,true,false,false,false],"l3_i32":[-4,0,-1,3,-2,-1,0,3,0,2147483647,4,-4,-4,4,0,5,3,-3,0,-3,-4,-4,3,-5,5,1,-3,4,-5,3,0,4,-1,4,0,4,5,-3,-2147483648,1,4,1,-4,1,3,-2,-4,-2147483648,-2,-3,-2,-3,-2,5,-4,2,2147483647,1,1,-5],"l4_utf8":["c","a","ab","a","ab","😀","😀","","B","😀","é","B","c","é","a","c","ab","b","ab","B","😀","B","","😀","B","é","😀","","b","b","😀","aa","c","c","ab","","zz","a","a","b","ab","B","zz","a","B","zz","zz","aa","c","ab","😀","a","😀","é","aa","é","a","a","","c"]}},"right":{"fields":[{"name":"l1_i32","type":"i32"},{"name":"r0_utf8","type":"utf8"}],"columns":{"r0_utf8":["é","zz","c","😀","é","","zz","c","zz","ab"],"l1_i32":[-3,5,4,-1,-2,5,-2,2,2,0]}}},"ops":[{"op":"withColumns","exprs":{"x0":{"col":"l2_bool"}}},{"op":"sort","keys":[{"expr":{"col":"l4_utf8"},"descending":true},{"expr":{"col":"l2_bool"},"descending":false}],"nullsLast":false},{"op":"unique"},{"op":"join","right":"right","on":["l1_i32"],"how":"semi"}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_i32","type":"i32"},{"name":"l4_utf8","type":"utf8"},{"name":"x0","type":"bool"}],"columns":{"l0_utf8":["B","a","b","B","a","😀","zz","aa","😀","b","ab","b","aa","b","a","aa","b","ab","c","é","c","","a","😀","zz","","b","","","b","aa","","é","aa","a","c","b","aa","","B"],"l1_i32":[2,-1,5,0,-1,-2,5,-1,5,-3,5,0,-2,2,-1,-2,2,4,-1,-2,2,2,-2,-3,5,5,4,-3,-3,2,0,0,0,2,4,-3,-1,-2,-1,2],"l2_bool":[false,false,false,true,false,false,true,true,false,true,true,false,false,false,false,true,false,true,false,false,false,false,true,true,true,true,true,false,false,false,false,true,false,true,true,true,false,false,true,true],"l3_i32":[0,-5,-3,-2,4,1,4,2,-4,-4,-2,-4,4,-2,-5,-4,-3,-5,-1,-2,3,0,0,4,-3,-2147483648,-4,-3,1,-3,1,0,3,0,-4,-3,4,1,3,4],"l4_utf8":["😀","😀","😀","😀","é","é","é","é","zz","zz","zz","c","c","c","c","c","b","b","ab","ab","ab","ab","ab","ab","ab","aa","aa","a","a","a","a","a","B","B","B","B","","","",""],"x0":[false,false,false,true,false,false,true,true,false,true,true,false,false,false,false,true,false,true,false,false,false,false,true,true,true,true,true,false,false,false,false,true,false,true,true,true,false,false,true,true]}}},{"id":"random/0342","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_f64","type":"f64"}],"columns":{"l0_utf8":["c","é","é","ab","zz","aa","😀","é","B","B","b","😀","ab","a","c","😀","B","a","c","","","aa","b","b","B","zz","😀","a","c","é","é","zz","é",null,"b","ab","","B","b","é","b","b","zz","B","","a","😀","😀","ab","b","B","ab","😀","c","B",null,"c","b",null,null],"l1_i32":[0,-2,null,3,3,-2,4,null,4,-4,-4,1,-952,5,4,-2,0,-2,0,-4,5,null,null,-5,1,1,-1,3,1,-1,-1,4,-1,null,4,-2,0,-1,5,-5,2,4,-2,-4,5,4,-2,-2147483648,4,null,1,-1,2,-4,3,-4,4,5,-2,-5],"l2_f64":[-1.0,-88.042,null,-4.0,1.0,-1.0,99.455,-98.725,-2.0,-3.0,-4.0,-1.0,-95.145,1.0,-59.943,54.562,82.518,3.0,3.0,null,-1.0,46.246,-20.842,-30.231,-48.379,-1.0,22.54,38.5,94.032,null,-8.288,-39.017,77.237,-1.0,-92.143,null,3.0,null,4.0,22.119,-2.0,0.0,null,-3.0,-1.0,-3.0,1.0,-88.093,38.305,null,-53.892,4.0,0.0,3.0,0.0,null,3.0,44.688,-2.148,-42.835],"l3_f64":[null,2.0,-4.0,72.066,4.0,-3.0,-20.263,-69.479,2.0,-77.91,-57.52,-1.0,97.641,7.176,3.0,-57.867,-2.0,-4.0,81.071,-1.0,-4.0,-40.62,2.0,-3.0,37.187,-62.227,1.0,-20.351,-4.0,-3.0,-1.0,1.0,3.0,-4.0,-3.0,-71.383,-35.722,null,-4.0,null,3.0,1.0,-35.699,79.75,-32.474,1.0,-99.216,4.389,null,null,-53.364,3.0,0.0,"-0","NaN",-82.899,2.0,null,null,61.323]}}},"ops":[{"op":"filter","predicate":{"fn":"lt","args":[{"col":"l3_f64"},{"fn":"fillNull","args":[{"col":"l1_i32"},{"lit":1.5}]}]}},{"op":"filter","predicate":{"fn":"and","args":[{"fn":"lte","args":[{"col":"l1_i32"},{"col":"l2_f64"}]},{"fn":"and","args":[{"fn":"isNull","args":[{"col":"l2_f64"}]},{"fn":"gte","args":[{"lit":0.5},{"col":"l1_i32"}]}]}]}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_f64","type":"f64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_f64":[],"l3_f64":[]}}},{"id":"random/0343","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_utf8","type":"utf8"}],"columns":{"l0_utf8":["c","zz","c","é","c","ab","B","b","ab","é","",null,"😀","ab","aa","B","b","","😀","B","","b","","b","zz","B","c","a","c","😀",null,"ab","c","ab","é","zz","",null,null,"a","ab","a","ab","aa","😀","c",null,"a","aa","a","","c","a","a","B","a","",null,"","zz"],"l1_i32":[4,-2,-5,-4,1,-1,2,-2,-2,-2,-2147483648,3,-1,-3,2147483647,-4,-2147483648,-2,null,5,-1,0,null,5,1,3,-3,-2,2,null,-2,0,5,4,0,-2,-5,-5,-5,2,2,-1,-3,-4,-4,-2147483648,1,-2,-2,-1,3,3,-4,-5,-3,3,1,2147483647,0,2],"l2_f64":[1.0,-3.0,null,85.842,14.504,-86.062,0.0,1.0,-2.0,-2.0,0.0,-34.704,null,-3.0,-38.271,-2.0,-26.281,84.39,-20.42,61.652,5.481,-24.706,38.776,"-0",2.0,58.354,1.0,null,3.0,64.007,"NaN",null,2.0,null,1.0,4.0,-1.0,-73.277,-65.266,54.824,-68.6,4.0,-4.0,-1.0,83.67,89.417,45.086,76.31,9.927,null,2.0,-68.415,57.815,-4.0,-1.0,-1.0,7.538,4.928,"NaN","-Infinity"],"l3_utf8":["c","😀",null,"zz","c","é","b",null,null,"","a","a","zz","a","zz","c","c","a","é","ab","zz","b","B","","c","c","😀",null,"B","a","b","a","a",null,null,"c","zz","b","zz","b","","ab","ab","","c","b","😀",null,"aa","c","zz",null,"😀","😀","ab","","ab","aa","ab",""]}},"right":{"fields":[{"name":"l1_i32","type":"i32"},{"name":"r0_f64","type":"f64"}],"columns":{"r0_f64":[],"l1_i32":[]}}},"ops":[{"op":"head","n":0},{"op":"join","right":"right","on":["l1_i32"],"how":"semi"}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_utf8","type":"utf8"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_f64":[],"l3_utf8":[]}}},{"id":"random/0344","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_i32","type":"i32"},{"name":"l4_bool","type":"bool"},{"name":"l5_bool","type":"bool"},{"name":"l6_i64","type":"i64"}],"columns":{"l0_utf8":["c"],"l1_i32":[-4],"l2_f64":[-66.266],"l3_i32":[2],"l4_bool":[true],"l5_bool":[false],"l6_i64":["-4611686018427387904"]}},"right":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"r0_bool","type":"bool"}],"columns":{"r0_bool":[true,true,false,false,false,true,false,true,true,true,false,true,false,false,false,true,false,true,true,true,false,true,true,true,false,true,true,false,false,false],"l0_utf8":["c","c","c","c","c","c","c","c","c","c","c","c","c","c","c","c","c","c","c","c","c","c","c","c","c","c","c","c","c","c"]}}},"ops":[{"op":"join","right":"right","on":["l0_utf8"],"how":"left"},{"op":"filter","predicate":{"fn":"gt","args":[{"col":"l3_i32"},{"fn":"fillNull","args":[{"lit":0.5},{"lit":1.5}]}]}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_i32","type":"i32"},{"name":"l4_bool","type":"bool"},{"name":"l5_bool","type":"bool"},{"name":"l6_i64","type":"i64"},{"name":"r0_bool","type":"bool"}],"columns":{"l0_utf8":["c","c","c","c","c","c","c","c","c","c","c","c","c","c","c","c","c","c","c","c","c","c","c","c","c","c","c","c","c","c"],"l1_i32":[-4,-4,-4,-4,-4,-4,-4,-4,-4,-4,-4,-4,-4,-4,-4,-4,-4,-4,-4,-4,-4,-4,-4,-4,-4,-4,-4,-4,-4,-4],"l2_f64":[-66.266,-66.266,-66.266,-66.266,-66.266,-66.266,-66.266,-66.266,-66.266,-66.266,-66.266,-66.266,-66.266,-66.266,-66.266,-66.266,-66.266,-66.266,-66.266,-66.266,-66.266,-66.266,-66.266,-66.266,-66.266,-66.266,-66.266,-66.266,-66.266,-66.266],"l3_i32":[2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2],"l4_bool":[true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true],"l5_bool":[false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false],"l6_i64":["-4611686018427387904","-4611686018427387904","-4611686018427387904","-4611686018427387904","-4611686018427387904","-4611686018427387904","-4611686018427387904","-4611686018427387904","-4611686018427387904","-4611686018427387904","-4611686018427387904","-4611686018427387904","-4611686018427387904","-4611686018427387904","-4611686018427387904","-4611686018427387904","-4611686018427387904","-4611686018427387904","-4611686018427387904","-4611686018427387904","-4611686018427387904","-4611686018427387904","-4611686018427387904","-4611686018427387904","-4611686018427387904","-4611686018427387904","-4611686018427387904","-4611686018427387904","-4611686018427387904","-4611686018427387904"],"r0_bool":[true,true,false,false,false,true,false,true,true,true,false,true,false,false,false,true,false,true,true,true,false,true,true,true,false,true,true,false,false,false]}}},{"id":"random/0345","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_f64","type":"f64"}],"columns":{"l0_utf8":["zz","é","é","B","aa","","","zz","ab","B","c","","aa","zz","aa","ab","aa","ab","ab","a","c","b","aa","","ab","😀","😀","B","aa","aa","","é","","ab","B","a","B","zz","ab","zz","ab","b","😀","aa","c","B","ab","B","","aa","a","é","b","aa","ab","zz","zz","é","aa","é"],"l1_i32":[-324,-3,4,-3,-4,5,2,-5,-4,3,-3,0,-4,2,3,5,-4,0,3,0,-2,-5,4,-1,5,2147483647,-2,-551,2,2,-1,-1,3,-2,-5,1,2,5,1,4,4,5,2,-4,0,-3,4,-3,-3,-3,4,3,2,2,-2147483648,4,1,5,2,-4],"l2_bool":[true,false,true,false,false,true,true,true,false,false,true,true,false,true,true,true,false,false,false,false,false,true,true,false,false,true,true,true,true,true,false,true,true,true,false,false,true,true,false,false,false,true,true,false,true,true,false,true,true,true,false,true,false,true,true,false,true,true,false,false],"l3_utf8":["B","ab","zz","ab","a","zz","é","zz","zz","","ab","é","B","c","zz","ab","c","c","","","","ab","aa","zz","zz","é","a","B","zz","ab","c","é","a","zz","a","","c","aa","é","zz","😀","b","😀","ab","c","b","c","ab","ab","","B","B","b","😀","😀","b","aa","😀","é","é"],"l4_utf8":["","aa","zz","😀","c","😀","b","aa","zz","zz","B","","a","","zz","","c","😀","ab","é","","a","zz","😀","B","é","aa","","c","c","é","c","zz","é","zz","zz","B","😀","","b","😀","aa","aa","c","c","aa","😀","zz","é","c","","a","b","c","","ab","é","😀","B","c"],"l5_f64":[82.058,-2.0,-1.0,-46.863,-78.829,-55.809,95.667,-21.715,-24.689,-1.0,-1.0,1.0,50.176,-18.988,2.0,"NaN",0.0,28.655,-2.0,0.0,94.161,1.0,-14.313,-70.898,97.98,0.0,-82.732,1.0,75.229,-2.0,-1.0,3.0,70.41,-2.0,-2.0,1.0,37.354,1.0,-10.941,-4.0,-84.086,-4.0,-93.37,-88.595,86.611,3.0,61.699,4.0,1.0,48.982,65.049,2.0,-97.157,20.197,3.0,2.0,0.0,-3.0,-3.0,42.472]}}},"ops":[{"op":"unique"},{"op":"withColumns","exprs":{"x1":{"col":"l1_i32"}}},{"op":"withColumns","exprs":{"x2":{"lit":0.5}}},{"op":"unique","columns":["x1"]}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_f64","type":"f64"},{"name":"x1","type":"i32"},{"name":"x2","type":"f64"}],"columns":{"l0_utf8":["zz","é","é","aa","","","zz","B","","c","","😀","B","a","ab"],"l1_i32":[-324,-3,4,-4,5,2,-5,3,0,-2,-1,2147483647,-551,1,-2147483648],"l2_bool":[true,false,true,false,true,true,true,false,true,false,false,true,true,false,true],"l3_utf8":["B","ab","zz","a","zz","é","zz","","é","","zz","é","B","","😀"],"l4_utf8":["","aa","zz","c","😀","b","aa","zz","","","😀","é","","zz",""],"l5_f64":[82.058,-2.0,-1.0,-78.829,-55.809,95.667,-21.715,-1.0,1.0,94.161,-70.898,0.0,1.0,1.0,3.0],"x1":[-324,-3,4,-4,5,2,-5,3,0,-2,-1,2147483647,-551,1,-2147483648],"x2":[0.5,0.5,0.5,0.5,0.5,0.5,0.5,0.5,0.5,0.5,0.5,0.5,0.5,0.5,0.5]}}},{"id":"random/0346","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_bool","type":"bool"},{"name":"l4_f64","type":"f64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_bool":[],"l3_bool":[],"l4_f64":[]}}},"ops":[{"op":"filter","predicate":{"fn":"isIn","args":[{"col":"l0_utf8"}],"values":["ab","zz","😀"]}},{"op":"withColumns","exprs":{"x1":{"fn":"mod","args":[{"fn":"mul","args":[{"col":"l4_f64"},{"col":"l4_f64"}]},{"fn":"neg","args":[{"col":"l1_i32"}]}]}}},{"op":"filter","predicate":{"fn":"lte","args":[{"fn":"div","args":[{"col":"l4_f64"},{"col":"l1_i32"}]},{"fn":"abs","args":[{"lit":2.5}]}]}},{"op":"filter","predicate":{"col":"l3_bool"}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_bool","type":"bool"},{"name":"l4_f64","type":"f64"},{"name":"x1","type":"f64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_bool":[],"l3_bool":[],"l4_f64":[],"x1":[]}}},{"id":"random/0347","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i64","type":"i64"},{"name":"l3_i64","type":"i64"}],"columns":{"l0_utf8":["c","é","😀","aa",null,"b","aa","B","c","ab","é",null,"a","zz","a","é","😀","ab","c","","c","B","c","aa","c","ab","aa","b","B","é","b","ab","b","B","é","😀","","a","a","B","B","aa","c","b","zz","b",null,"aa","é","B","ab","zz","","ab","😀","zz","😀","a","b","B"],"l1_i32":[-5,4,null,5,-2,0,4,-2,-4,1,-2,0,-5,-194,-2147483648,2,-1,4,1,0,1,-3,0,-3,1,3,null,null,4,-3,2,2,-5,-5,-5,-5,5,3,-1,1,-4,1,-3,3,4,-1,5,null,5,0,-1,-2,0,2147483647,4,-1,-2,0,0,0],"l2_i64":["-4611686018427387904","9007199254740995","-4611686018427387904","9007199254740993","3",null,null,"-4611686018427387904","-4611686018427387904","-2","9007199254740994","-3","-4611686018427387904","5",null,"-4611686018427387904","-4","2","-4611686018427387904","-4","9007199254740994","-4","-2","-4611686018427387904","5","-4611686018427387904","5","9007199254740995","-4611686018427387904","9007199254740992","-4611686018427387904",null,null,"9007199254740995","9007199254740994","-5","-4611686018427387904","-2","-4611686018427387904","-4611686018427387904","-3","4","9007199254740992","5","9007199254740992","9007199254740992","9007199254740992","9007199254740993","9007199254740994","0","-4611686018427387904","-4611686018427387904","9007199254740995",null,"9007199254740992","-4611686018427387904","-4611686018427387904","-4611686018427387904","9007199254740995","-4611686018427387904"],"l3_i64":["-4","-4611686018427387904","9007199254740994","-4611686018427387904","0","2","9007199254740994","-4611686018427387904","-4611686018427387904","-4","-4611686018427387904",null,"-5",null,"9007199254740993","9007199254740992","4","0","-4","0","4",null,"9007199254740995","9007199254740993","-5","-4611686018427387904","-4611686018427387904","-4611686018427387904","-2","9007199254740993","9007199254740993","-4611686018427387904","9007199254740994","9007199254740995","9007199254740995","-2","5","9007199254740993","-1","-4611686018427387904","9007199254740993","-2","-4611686018427387904","-4611686018427387904","0",null,"-2","3","9007199254740994","-4611686018427387904","-4611686018427387904","-1","-4611686018427387904","-5","9007199254740993","-4611686018427387904","3","-4611686018427387904","-4611686018427387904","0"]}}},"ops":[{"op":"withColumns","exprs":{"x0":{"col":"l1_i32"}}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i64","type":"i64"},{"name":"l3_i64","type":"i64"},{"name":"x0","type":"i32"}],"columns":{"l0_utf8":["c","é","😀","aa",null,"b","aa","B","c","ab","é",null,"a","zz","a","é","😀","ab","c","","c","B","c","aa","c","ab","aa","b","B","é","b","ab","b","B","é","😀","","a","a","B","B","aa","c","b","zz","b",null,"aa","é","B","ab","zz","","ab","😀","zz","😀","a","b","B"],"l1_i32":[-5,4,null,5,-2,0,4,-2,-4,1,-2,0,-5,-194,-2147483648,2,-1,4,1,0,1,-3,0,-3,1,3,null,null,4,-3,2,2,-5,-5,-5,-5,5,3,-1,1,-4,1,-3,3,4,-1,5,null,5,0,-1,-2,0,2147483647,4,-1,-2,0,0,0],"l2_i64":["-4611686018427387904","9007199254740995","-4611686018427387904","9007199254740993","3",null,null,"-4611686018427387904","-4611686018427387904","-2","9007199254740994","-3","-4611686018427387904","5",null,"-4611686018427387904","-4","2","-4611686018427387904","-4","9007199254740994","-4","-2","-4611686018427387904","5","-4611686018427387904","5","9007199254740995","-4611686018427387904","9007199254740992","-4611686018427387904",null,null,"9007199254740995","9007199254740994","-5","-4611686018427387904","-2","-4611686018427387904","-4611686018427387904","-3","4","9007199254740992","5","9007199254740992","9007199254740992","9007199254740992","9007199254740993","9007199254740994","0","-4611686018427387904","-4611686018427387904","9007199254740995",null,"9007199254740992","-4611686018427387904","-4611686018427387904","-4611686018427387904","9007199254740995","-4611686018427387904"],"l3_i64":["-4","-4611686018427387904","9007199254740994","-4611686018427387904","0","2","9007199254740994","-4611686018427387904","-4611686018427387904","-4","-4611686018427387904",null,"-5",null,"9007199254740993","9007199254740992","4","0","-4","0","4",null,"9007199254740995","9007199254740993","-5","-4611686018427387904","-4611686018427387904","-4611686018427387904","-2","9007199254740993","9007199254740993","-4611686018427387904","9007199254740994","9007199254740995","9007199254740995","-2","5","9007199254740993","-1","-4611686018427387904","9007199254740993","-2","-4611686018427387904","-4611686018427387904","0",null,"-2","3","9007199254740994","-4611686018427387904","-4611686018427387904","-1","-4611686018427387904","-5","9007199254740993","-4611686018427387904","3","-4611686018427387904","-4611686018427387904","0"],"x0":[-5,4,null,5,-2,0,4,-2,-4,1,-2,0,-5,-194,-2147483648,2,-1,4,1,0,1,-3,0,-3,1,3,null,null,4,-3,2,2,-5,-5,-5,-5,5,3,-1,1,-4,1,-3,3,4,-1,5,null,5,0,-1,-2,0,2147483647,4,-1,-2,0,0,0]}}},{"id":"random/0348","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_utf8","type":"utf8"}],"columns":{"l0_utf8":["a","b","😀","","é",null,null,"zz","ab","aa",null,"zz","a","a","ab","c","é","","zz","aa"],"l1_i32":[-2,-4,0,1,2,-1,0,-5,5,0,null,3,null,-4,2,-3,-2,2147483647,5,0],"l2_i32":[2,0,3,-3,3,2,-1,4,2,null,-4,-2,0,-3,null,0,2,0,0,null],"l3_utf8":["B","b","B",null,"B","a","B",null,"ab","aa","é","ab",null,"ab","aa","a","B","ab",null,"😀"]}}},"ops":[{"op":"filter","predicate":{"fn":"lt","args":[{"fn":"mod","args":[{"col":"l2_i32"},{"col":"l2_i32"}]},{"fn":"mul","args":[{"col":"l1_i32"},{"col":"l2_i32"}]}]}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_utf8","type":"utf8"}],"columns":{"l0_utf8":["é","ab","a"],"l1_i32":[2,5,-4],"l2_i32":[3,2,-3],"l3_utf8":["B","ab","ab"]}}},{"id":"random/0349","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_f64","type":"f64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_f64":[],"l3_f64":[]}}},"ops":[{"op":"unique"},{"op":"withColumns","exprs":{"x1":{"fn":"mod","args":[{"fn":"mod","args":[{"col":"l1_i32"},{"col":"l2_f64"}]},{"col":"l3_f64"}]}}},{"op":"withColumns","exprs":{"x2":{"fn":"when","args":[{"fn":"isIn","args":[{"col":"l0_utf8"}],"values":["😀","ab","b"]},{"col":"l1_i32"},{"fn":"mod","args":[{"lit":2.5},{"col":"l2_f64"}]}]}}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_f64","type":"f64"},{"name":"x1","type":"f64"},{"name":"x2","type":"f64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_f64":[],"l3_f64":[],"x1":[],"x2":[]}}},{"id":"random/0350","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i64","type":"i64"},{"name":"l3_i32","type":"i32"}],"columns":{"l0_utf8":["zz","a","ab",null,null,"zz",null,"😀","ab",null,null,null,null,"ab","B","a","","é","b","ab","b","c","c",null,"a","😀",null,null,"c",null,"b","a","aa","aa",null,null,null,"é","ab","😀","","c","ab",null,"ab","a","aa",null,"é","é",null,"c","",null,"ab","ab",null,"a","b","B"],"l1_i32":[null,null,null,null,1,null,1,-4,-4,-5,null,-5,5,null,5,3,1,null,-1,2147483647,5,null,null,2147483647,null,null,null,0,null,3,-5,2147483647,1,-4,null,null,0,4,4,5,2147483647,null,null,-1,5,null,-5,5,null,null,null,2,-5,-5,-5,-4,null,-1,-5,-3],"l2_i64":["-4611686018427387904","-3","-5",null,"9007199254740993","3","-4611686018427387904","-4611686018427387904","-4611686018427387904",null,null,"-4611686018427387904","9007199254740993","-1","-2","-4611686018427387904",null,null,"9007199254740993","-4",null,"-4611686018427387904",null,null,"0","9007199254740993","9007199254740992",null,"9007199254740994","9007199254740994",null,"9007199254740993","9007199254740993","-4611686018427387904",null,"-1","-1","3",null,"-4611686018427387904","4","9007199254740992","9007199254740995",null,"-4611686018427387904",null,null,"-4611686018427387904",null,"-4611686018427387904","9007199254740993","-4611686018427387904","-4611686018427387904","-4611686018427387904","9007199254740992",null,"-4611686018427387904","-1","-4611686018427387904","2"],"l3_i32":[0,4,null,-4,4,-2,2,null,-1,-3,3,null,0,-2147483648,null,1,4,-3,5,null,-4,null,3,null,0,5,-2,0,null,null,2,5,null,-5,null,null,0,null,-2,3,5,null,4,-5,null,-5,1,null,null,-4,null,-4,-2,null,-3,1,1,3,1,null]}}},"ops":[{"op":"groupBy","keys":["l1_i32"],"aggs":{"n0":{"agg":"len"},"a0_0":{"agg":"last","arg":{"col":"l0_utf8"}},"a0_1":{"agg":"first","arg":{"col":"l3_i32"}}}}],"expected":{"fields":[{"name":"l1_i32","type":"i32"},{"name":"n0","type":"u32"},{"name":"a0_0","type":"utf8"},{"name":"a0_1","type":"i32"}],"columns":{"l1_i32":[null,1,-4,-5,5,3,-1,2147483647,0,4,2,-3],"n0":[23,4,4,8,6,2,3,4,2,2,1,1],"a0_0":[null,"aa","ab","b",null,null,"a","",null,"ab","c","B"],"a0_1":[0,4,null,-3,0,1,5,null,0,null,-4,null]}}},{"id":"random/0351","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_i32","type":"i32"},{"name":"l5_utf8","type":"utf8"}],"columns":{"l0_utf8":["b","aa","aa","","é"],"l1_i32":[3,2,5,2,1],"l2_i32":[-3,-2,3,-2,2147483647],"l3_utf8":["a","aa","B","é","é"],"l4_i32":[-4,2,3,-4,3],"l5_utf8":["zz","b","aa","a","é"]}},"right":{"fields":[{"name":"l5_utf8","type":"utf8"},{"name":"r0_bool","type":"bool"}],"columns":{"r0_bool":[true,false,null,null,false,false,false,false,false,true],"l5_utf8":["a","zz","aa","zz","é","aa","zz","b","b","é"]}}},"ops":[{"op":"unique","columns":["l2_i32"]},{"op":"join","right":"right","on":["l5_utf8"],"how":"left"}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_i32","type":"i32"},{"name":"l5_utf8","type":"utf8"},{"name":"r0_bool","type":"bool"}],"columns":{"l0_utf8":["b","b","b","aa","aa","aa","aa","é","é"],"l1_i32":[3,3,3,2,2,5,5,1,1],"l2_i32":[-3,-3,-3,-2,-2,3,3,2147483647,2147483647],"l3_utf8":["a","a","a","aa","aa","B","B","é","é"],"l4_i32":[-4,-4,-4,2,2,3,3,3,3],"l5_utf8":["zz","zz","zz","b","b","aa","aa","é","é"],"r0_bool":[false,null,false,false,false,null,false,false,true]}}},{"id":"random/0352","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_i64","type":"i64"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_utf8","type":"utf8"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_f64":[],"l3_i64":[],"l4_utf8":[],"l5_utf8":[]}},"right":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"r0_f64","type":"f64"}],"columns":{"r0_f64":[-2.0,72.578,-1.0,2.0,-71.148,59.274,-34.088,null,null,6.753,-1.0,null,4.0,65.255,"-Infinity",4.0,0.0,86.774,-3.0,78.656,null,2.0,4.0,-1.0,3.0,-24.017,19.725,15.561,-93.072,1.0],"l0_utf8":["zz",null,null,null,"c","zz","c","zz",null,null,null,"zz","c",null,"c",null,null,null,"zz",null,null,null,null,"c","c",null,null,"zz",null,null]}}},"ops":[{"op":"join","right":"right","on":["l0_utf8"],"how":"semi"},{"op":"withColumns","exprs":{"x1":{"fn":"gt","args":[{"fn":"abs","args":[{"lit":0.5}]},{"fn":"fillNull","args":[{"col":"l2_f64"},{"lit":0}]}]}}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_i64","type":"i64"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_utf8","type":"utf8"},{"name":"x1","type":"bool"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_f64":[],"l3_i64":[],"l4_utf8":[],"l5_utf8":[],"x1":[]}}},{"id":"random/0353","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_i64","type":"i64"},{"name":"l4_bool","type":"bool"}],"columns":{"l0_utf8":["","ab","B",null,"b",null,"a",null,"","a",null,null,"😀","b","a","a",null,null,"b","zz"],"l1_i32":[null,null,2,-1,5,null,null,-1,null,3,-2,4,-3,null,null,3,null,null,0,null],"l2_f64":[2.0,4.0,-1.0,null,-2.0,null,null,-76.337,2.0,null,1.0,-4.0,-1.0,-3.0,null,-4.915,null,null,null,null],"l3_i64":["-4611686018427387904","9007199254740993",null,"-2","9007199254740995",null,"1","-4611686018427387904","-4611686018427387904","9007199254740994","9007199254740992","-5",null,null,null,null,"9007199254740994","9007199254740994","-4611686018427387904","9007199254740993"],"l4_bool":[false,true,null,false,null,null,false,true,null,false,null,null,true,false,null,null,null,false,null,null]}}},"ops":[{"op":"filter","predicate":{"fn":"or","args":[{"fn":"lt","args":[{"lit":2.5},{"lit":2.5}]},{"fn":"lte","args":[{"col":"l1_i32"},{"col":"l2_f64"}]}]}},{"op":"head","n":8},{"op":"head","n":2},{"op":"withColumns","exprs":{"x3":{"fn":"mul","args":[{"col":"l1_i32"},{"fn":"div","args":[{"col":"l2_f64"},{"col":"l1_i32"}]}]}}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_i64","type":"i64"},{"name":"l4_bool","type":"bool"},{"name":"x3","type":"f64"}],"columns":{"l0_utf8":[null,"😀"],"l1_i32":[-2,-3],"l2_f64":[1.0,-1.0],"l3_i64":["9007199254740992",null],"l4_bool":[null,true],"x3":[1.0,-1.0]}}},{"id":"random/0354","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_f64","type":"f64"}],"columns":{"l0_utf8":["c","B","aa","zz","c","B","zz","b","","c","aa","aa","zz","b","ab","aa","zz","c","b","a"],"l1_i32":[-1,5,-177,-2147483648,-1,-2,4,-4,2,0,-2,4,5,0,-5,-3,3,2147483647,2,-2],"l2_utf8":["a","ab","😀","B","b","a","b","","a","","aa","😀","c","","c","B","é","b","","ab"],"l3_f64":[1.0,-83.144,4.0,-2.0,-2.0,-37.133,3.0,3.0,1.0,-4.0,2.0,3.243,-1.0,"Infinity",85.625,-9.517,53.079,1.0,88.34,3.0]}}},"ops":[{"op":"unique","columns":["l0_utf8"]}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_f64","type":"f64"}],"columns":{"l0_utf8":["c","B","aa","zz","b","","ab","a"],"l1_i32":[-1,5,-177,-2147483648,-4,2,-5,-2],"l2_utf8":["a","ab","😀","B","","a","c","ab"],"l3_f64":[1.0,-83.144,4.0,-2.0,3.0,1.0,85.625,3.0]}}},{"id":"random/0355","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_f64","type":"f64"},{"name":"l4_utf8","type":"utf8"}],"columns":{"l0_utf8":["é","aa","ab","",null,"é","ab","b","😀","b","B",null,"é","😀","zz",null,"B","aa","zz","aa","zz","😀",null,null,"zz","ab","aa","a","aa","a",null,"B","aa",null,null,null,"ab","😀","ab","b",null,"b",null,null,"c","a","c","a","c","c","b",null,null,"b","",null,"a","zz","zz",null],"l1_i32":[-4,-5,5,5,null,-1,-1,-3,0,-1,-3,-5,null,-2,null,-3,-3,null,-2147483648,2,3,null,-4,-1,null,1,2,null,2,2,5,5,0,2,-33,-3,null,-1,3,0,null,-3,-2,-1,2,-1,4,-4,null,-1,3,5,null,null,-1,-3,null,2,-4,2],"l2_i32":[2147483647,5,373,null,-3,2,null,1,230,-5,-2,null,-1,-4,-2,-1,null,null,-4,1,-1,null,2,1,-1,584,-1,null,1,null,0,0,-2,2,null,1,-386,1,4,null,null,null,2,-5,0,null,null,null,null,null,5,null,0,2,1,3,-1,null,3,-1],"l3_f64":[null,-3.0,-1.0,-1.0,70.582,-74.329,34.995,null,null,null,4.0,null,-80.667,"-Infinity",null,null,85.949,16.927,-15.204,null,-27.389,null,2.0,3.0,64.335,null,-2.0,null,45.655,-3.0,52.655,null,2.0,80.234,null,null,4.0,null,-3.0,null,1.0,null,null,81.853,3.0,-55.605,-1.0,2.0,null,80.967,15.024,-1.0,null,-4.0,2.0,null,-50.76,-1.0,null,-2.0],"l4_utf8":[null,"zz","c","B","c","b","é","c","b","B",null,"zz",null,"é",null,"B","é","é","é","a","c","b","a","ab",null,"aa","c","a",null,"B","c","aa","aa","é","","é",null,null,"ab","a",null,"b","B","ab","a",null,"aa",null,null,"",null,"😀","B","B",null,"zz","é","é",null,"é"]}}},"ops":[{"op":"unique","columns":["l1_i32"]}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_f64","type":"f64"},{"name":"l4_utf8","type":"utf8"}],"columns":{"l0_utf8":["é","aa","ab",null,"é","b","😀","😀","zz","aa","zz","ab",null,"c"],"l1_i32":[-4,-5,5,null,-1,-3,0,-2,-2147483648,2,3,1,-33,4],"l2_i32":[2147483647,5,373,-3,2,1,230,-4,-4,1,-1,584,null,null],"l3_f64":[null,-3.0,-1.0,70.582,-74.329,null,null,"-Infinity",-15.204,null,-27.389,null,null,-1.0],"l4_utf8":[null,"zz","c","c","b","c","b","é","é","a","c","aa","","aa"]}}},{"id":"random/0356","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_f64","type":"f64"},{"name":"l4_f64","type":"f64"},{"name":"l5_utf8","type":"utf8"},{"name":"l6_bool","type":"bool"}],"columns":{"l0_utf8":["a","ab","b","B","a","B","😀","ab","é","zz","a","c","ab","😀","ab","aa","B","😀","c","B","😀","c","B","ab","é","B","B","","B","é","c","é","c","c","B","é","c","é","","B","b","é","😀","c","c","é","b","zz","ab","B","é","c","B","","","ab","a","é","","zz"],"l1_i32":[2,-719,-1,-2,4,5,-3,-1,-3,2,-1,2147483647,3,0,-1,-3,3,-4,3,-5,-2,-5,-1,4,-4,-3,0,0,2,-1,5,-3,0,5,-2,-5,-2,-3,3,-3,4,1,0,-1,-4,2147483647,-4,-3,2,1,3,4,4,-2,4,-2,-1,-2,2147483647,-5],"l2_bool":[true,true,false,true,true,true,false,true,false,false,false,true,true,false,true,false,true,false,true,true,false,true,false,true,true,false,true,true,false,true,true,true,true,true,true,true,true,true,true,true,false,true,false,true,true,true,false,false,true,false,false,false,true,false,true,true,true,false,false,true],"l3_f64":[2.0,87.653,3.0,-4.0,76.157,-1.0,57.499,3.0,-8.773,1.0,70.916,2.0,73.256,-11.823,3.0,-4.0,4.0,4.0,-6.069,-1.0,49.332,4.0,94.435,56.141,-45.918,-1.0,2.0,46.248,4.0,2.0,-4.0,1.0,0.0,"NaN",4.0,-30.443,-3.0,-89.29,-3.0,-10.199,-93.638,-25.26,"NaN",47.655,3.0,-3.0,-36.12,-2.0,-4.0,-2.0,4.0,-51.796,1.0,3.0,-3.0,"Infinity",-13.928,4.0,-32.928,24.555],"l4_f64":[22.359,-3.0,-11.005,-81.129,18.19,-10.841,88.416,-89.606,"Infinity",6.099,1.0,-2.0,-44.209,"NaN","Infinity",-1.0,76.081,1.0,1.0,68.449,-1.0,2.0,4.0,69.834,0.0,73.307,-3.0,38.218,"-Infinity",68.903,-4.0,-2.0,2.0,-4.0,69.513,-4.0,"NaN",1.0,-71.237,-27.491,61.794,-1.0,3.0,-90.193,36.29,-24.872,1.0,-2.0,-43.485,-3.0,82.224,25.985,4.0,-7.983,2.0,2.0,"Infinity",-1.0,-57.222,"-Infinity"],"l5_utf8":["zz","é","é","aa","é","","","c","zz","ab","zz","a","","aa","c","zz","a","a","é","a","B","b","","ab","c","😀","é","😀","a","c","c","b","c","B","zz","😀","B","é","b","é","B","😀","😀","ab","B","aa","ab","a","B","B","b","c","é","zz","a","B","aa","😀","b","zz"],"l6_bool":[true,true,true,true,false,false,true,true,false,true,true,false,true,true,false,true,true,false,true,false,true,true,false,true,false,true,true,false,false,false,true,false,true,true,true,true,true,true,true,true,false,false,false,false,true,false,false,true,false,true,true,true,false,true,false,true,true,true,true,false]}}},"ops":[{"op":"filter","predicate":{"fn":"gte","args":[{"col":"l3_f64"},{"col":"l4_f64"}]}},{"op":"withColumns","exprs":{"x1":{"fn":"gte","args":[{"col":"l1_i32"},{"col":"l4_f64"}]}}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_f64","type":"f64"},{"name":"l4_f64","type":"f64"},{"name":"l5_utf8","type":"utf8"},{"name":"l6_bool","type":"bool"},{"name":"x1","type":"bool"}],"columns":{"l0_utf8":["ab","b","B","a","B","ab","a","c","ab","😀","😀","c","B","B","","B","c","é","c","","B","😀","c","é","zz","ab","B","","ab","é","","zz"],"l1_i32":[-719,-1,-2,4,5,-1,-1,2147483647,3,-4,-2,-5,-1,0,0,2,5,-3,5,3,-3,0,-1,2147483647,-3,2,1,-2,-2,-2,2147483647,-5],"l2_bool":[true,false,true,true,true,true,false,true,true,false,false,true,false,true,true,false,true,true,true,true,true,false,true,true,false,true,false,false,true,false,false,true],"l3_f64":[87.653,3.0,-4.0,76.157,-1.0,3.0,70.916,2.0,73.256,4.0,49.332,4.0,94.435,2.0,46.248,4.0,-4.0,1.0,"NaN",-3.0,-10.199,"NaN",47.655,-3.0,-2.0,-4.0,-2.0,3.0,"Infinity",4.0,-32.928,24.555],"l4_f64":[-3.0,-11.005,-81.129,18.19,-10.841,-89.606,1.0,-2.0,-44.209,1.0,-1.0,2.0,4.0,-3.0,38.218,"-Infinity",-4.0,-2.0,-4.0,-71.237,-27.491,3.0,-90.193,-24.872,-2.0,-43.485,-3.0,-7.983,2.0,-1.0,-57.222,"-Infinity"],"l5_utf8":["é","é","aa","é","","c","zz","a","","a","B","b","","é","😀","a","c","b","B","b","é","😀","ab","aa","a","B","B","zz","B","😀","b","zz"],"l6_bool":[true,true,true,false,false,true,true,false,true,false,true,true,false,true,false,false,true,false,true,true,true,false,false,false,true,false,true,true,true,true,true,false],"x1":[false,true,true,false,true,true,false,true,true,false,false,false,false,true,false,true,true,false,true,true,true,false,true,true,false,true,true,true,false,false,true,true]}}},{"id":"random/0357","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_f64","type":"f64"}],"columns":{"l0_utf8":["a"],"l1_i32":[-2],"l2_f64":[-63.258],"l3_f64":[-3.0]}}},"ops":[{"op":"head","n":1},{"op":"filter","predicate":{"fn":"neq","args":[{"col":"l1_i32"},{"col":"l1_i32"}]}},{"op":"sort","keys":[{"expr":{"col":"l1_i32"},"descending":true}],"nullsLast":false},{"op":"head","n":4}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_f64","type":"f64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_f64":[],"l3_f64":[]}}},{"id":"random/0358","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_f64","type":"f64"},{"name":"l4_f64","type":"f64"},{"name":"l5_i32","type":"i32"}],"columns":{"l0_utf8":["aa"],"l1_i32":[2],"l2_f64":[82.429],"l3_f64":[9.052],"l4_f64":[-1.0],"l5_i32":[null]}},"right":{"fields":[{"name":"l1_i32","type":"i32"},{"name":"r0_bool","type":"bool"}],"columns":{"r0_bool":[false,true,true],"l1_i32":[2,2,2]}}},"ops":[{"op":"filter","predicate":{"fn":"or","args":[{"fn":"gte","args":[{"col":"l3_f64"},{"col":"l2_f64"}]},{"fn":"or","args":[{"fn":"isNotNull","args":[{"col":"l0_utf8"}]},{"fn":"isNull","args":[{"col":"l1_i32"}]}]}]}},{"op":"withColumns","exprs":{"x1":{"fn":"abs","args":[{"fn":"fillNull","args":[{"lit":-1},{"lit":1.5}]}]}}},{"op":"unique","columns":["l3_f64"]},{"op":"join","right":"right","on":["l1_i32"],"how":"full"}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_f64","type":"f64"},{"name":"l4_f64","type":"f64"},{"name":"l5_i32","type":"i32"},{"name":"x1","type":"f64"},{"name":"r0_bool","type":"bool"}],"columns":{"l0_utf8":["aa","aa","aa"],"l1_i32":[2,2,2],"l2_f64":[82.429,82.429,82.429],"l3_f64":[9.052,9.052,9.052],"l4_f64":[-1.0,-1.0,-1.0],"l5_i32":[null,null,null],"x1":[1.0,1.0,1.0],"r0_bool":[false,true,true]}}},{"id":"random/0359","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i64","type":"i64"},{"name":"l3_bool","type":"bool"},{"name":"l4_f64","type":"f64"},{"name":"l5_i64","type":"i64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_i64":[],"l3_bool":[],"l4_f64":[],"l5_i64":[]}}},"ops":[{"op":"unique","columns":["l5_i64"]},{"op":"withColumns","exprs":{"x1":{"col":"l1_i32"}}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i64","type":"i64"},{"name":"l3_bool","type":"bool"},{"name":"l4_f64","type":"f64"},{"name":"l5_i64","type":"i64"},{"name":"x1","type":"i32"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_i64":[],"l3_bool":[],"l4_f64":[],"l5_i64":[],"x1":[]}}},{"id":"random/0360","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_bool","type":"bool"},{"name":"l5_f64","type":"f64"}],"columns":{"l0_utf8":["c","zz","","zz","a","b","a","aa",null,"zz",null,"B","é","😀","b","a","a","😀","zz","zz","B","B","é","é","b","zz","a","aa","😀","é","b","B","c","a","","c","😀","c","😀","a","😀",null,"","b",null,"b",null,null,"b","a",null,"a",null,"",null,"aa","b","B","",null],"l1_i32":[-2,-3,-821,0,0,4,-1,3,2,-4,-1,-197,null,5,-4,-2,null,1,-2147483648,-3,null,-1,0,null,0,2147483647,-2147483648,-3,-3,-2,null,2147483647,1,3,-3,5,2,-5,-2,-5,5,-5,null,-4,-1,0,4,5,-3,1,-5,0,4,4,null,3,-1,1,-2,null],"l2_f64":[null,0.0,null,3.0,0.0,-3.0,3.0,5.91,4.0,2.379,3.0,-4.0,-3.0,"NaN",3.0,-26.447,-3.0,3.0,null,-86.247,0.0,78.903,-3.0,23.67,-55.282,-61.11,4.0,null,-78.798,1.0,-4.0,0.0,82.054,"-Infinity",-85.757,19.351,-29.656,-59.259,-2.0,46.709,-1.0,3.0,-2.0,-49.252,-97.942,null,0.0,-2.0,83.354,1.0,-45.569,-58.389,-85.061,-85.979,-24.486,0.0,-7.18,-2.0,1.0,null],"l3_utf8":["😀","😀","😀","aa","B","","é","b","zz","aa","aa","c","a","zz","zz","c",null,"",null,"é","","zz","a",null,null,"zz","é","b","b","ab","ab","aa","é","zz",null,"B","b","zz","aa","ab","ab","b","é","😀","é","B",null,"zz","c","é","a","b","😀","😀","😀",null,"c","B","b","B"],"l4_bool":[true,false,false,false,false,true,true,null,true,false,true,true,false,false,true,false,true,false,null,false,false,true,false,false,false,true,true,true,true,true,false,true,true,true,false,false,true,false,true,true,true,null,false,false,false,false,true,true,true,null,null,true,true,null,true,true,false,null,false,true],"l5_f64":[null,-77.123,-86.068,3.0,4.0,-44.472,3.0,3.0,2.0,2.0,-3.0,-4.0,-67.576,-82.718,1.0,3.0,null,-3.0,4.0,0.0,null,88.863,-2.0,-40.152,-4.0,-18.522,0.0,1.0,53.61,"NaN",-1.0,2.0,-14.238,53.69,3.0,4.0,76.145,null,-3.0,55.596,-4.0,37.33,0.0,null,46.626,3.0,-73.747,-4.407,-16.886,-1.0,-0.8,"-0",-30.764,25.544,0.0,-17.526,null,null,-76.044,null]}}},"ops":[{"op":"unique","columns":["l3_utf8"]},{"op":"head","n":3},{"op":"groupBy","keys":["l4_bool"],"aggs":{"n2":{"agg":"len"},"a2_0":{"agg":"count","arg":{"col":"l3_utf8"}},"a2_1":{"agg":"std","arg":{"col":"l5_f64"}}}}],"expected":{"fields":[{"name":"l4_bool","type":"bool"},{"name":"n2","type":"u32"},{"name":"a2_0","type":"u32"},{"name":"a2_1","type":"f64"}],"columns":{"l4_bool":[true,false],"n2":[1,2],"a2_0":[1,2],"a2_1":[null,0.7071067811865476]}}},{"id":"random/0361","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i64","type":"i64"},{"name":"l3_bool","type":"bool"},{"name":"l4_f64","type":"f64"}],"columns":{"l0_utf8":["a"],"l1_i32":[-2],"l2_i64":["9007199254740992"],"l3_bool":[true],"l4_f64":[-1.0]}}},"ops":[{"op":"withColumns","exprs":{"x0":{"fn":"mul","args":[{"col":"l4_f64"},{"lit":-1}]}}},{"op":"filter","predicate":{"fn":"lte","args":[{"col":"x0"},{"fn":"mul","args":[{"lit":2},{"col":"x0"}]}]}},{"op":"groupBy","keys":["l0_utf8"],"aggs":{"n2":{"agg":"len"},"a2_0":{"agg":"count","arg":{"col":"l2_i64"}}}},{"op":"unique"}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"n2","type":"u32"},{"name":"a2_0","type":"u32"}],"columns":{"l0_utf8":["a"],"n2":[1],"a2_0":[1]}}},{"id":"random/0362","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_f64","type":"f64"}],"columns":{"l0_utf8":["ab",null,"",null,"😀",null,"ab",null,"a",null,null,"zz",null,null,"é",null,"ab","b","😀",""],"l1_i32":[3,5,null,0,-5,1,0,null,-3,-1,null,-2,null,-3,-3,-5,-2,1,-1,null],"l2_i32":[null,null,4,-3,-5,null,null,3,null,null,-3,null,-5,null,-1,2,null,-3,-1,0],"l3_f64":[null,null,-1.182,-3.0,null,-21.432,1.0,-70.023,-1.0,-41.194,62.733,null,-82.339,null,74.982,null,null,70.931,"NaN",null]}},"right":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"r0_bool","type":"bool"}],"columns":{"r0_bool":[true,false,true,true,true,false,true,true,false,true,false,true,true,true,false,true,true,true,false,false,true,false,false,false,true,false,true,true,false,true],"l0_utf8":["😀","é","zz","é","é","é",null,"😀",null,"a","ab",null,null,"a",null,"b","ab",null,null,"b","b",null,"😀",null,null,null,"a","a","a",null]}}},"ops":[{"op":"unique"},{"op":"join","right":"right","on":["l0_utf8"],"how":"inner"}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_f64","type":"f64"},{"name":"r0_bool","type":"bool"}],"columns":{"l0_utf8":["ab","ab","😀","😀","😀","ab","ab","a","a","a","a","a","zz","é","é","é","é","ab","ab","b","b","b","😀","😀","😀"],"l1_i32":[3,3,-5,-5,-5,0,0,-3,-3,-3,-3,-3,-2,-3,-3,-3,-3,-2,-2,1,1,1,-1,-1,-1],"l2_i32":[null,null,-5,-5,-5,null,null,null,null,null,null,null,null,-1,-1,-1,-1,null,null,-3,-3,-3,-1,-1,-1],"l3_f64":[null,null,null,null,null,1.0,1.0,-1.0,-1.0,-1.0,-1.0,-1.0,null,74.982,74.982,74.982,74.982,null,null,70.931,70.931,70.931,"NaN","NaN","NaN"],"r0_bool":[false,true,true,true,false,false,true,true,true,true,true,false,true,false,true,true,false,false,true,true,false,true,true,true,false]}}},{"id":"random/0363","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_bool","type":"bool"},{"name":"l4_f64","type":"f64"},{"name":"l5_bool","type":"bool"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_utf8":[],"l3_bool":[],"l4_f64":[],"l5_bool":[]}},"right":{"fields":[{"name":"l2_utf8","type":"utf8"},{"name":"r0_f64","type":"f64"}],"columns":{"r0_f64":[-2.0,2.0,3.0,81.286,-3.0,null,-3.0,-51.322,-3.105,-64.29,4.0,1.0,49.196,78.498,-4.0,null,null,75.322,null,11.381,-2.0,null,-1.0,null,-2.0,-4.0,44.238,-3.0,-2.0,78.262],"l2_utf8":["a","B","a",null,null,null,"B",null,"a","a","a",null,"a","a","a","a","B","B","B","B","B","a","B","B","B",null,"B",null,"a",null]}}},"ops":[{"op":"join","right":"right","on":["l2_utf8"],"how":"anti"},{"op":"unique"}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_bool","type":"bool"},{"name":"l4_f64","type":"f64"},{"name":"l5_bool","type":"bool"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_utf8":[],"l3_bool":[],"l4_f64":[],"l5_bool":[]}}},{"id":"random/0364","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i64","type":"i64"},{"name":"l3_f64","type":"f64"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_f64","type":"f64"},{"name":"l6_utf8","type":"utf8"}],"columns":{"l0_utf8":["😀",null,"b","a","😀",null,null,null,"c",null,"é",null,"zz","ab","B","",null,"ab",null,null],"l1_i32":[null,-4,null,4,5,-3,-3,-3,null,-5,null,4,5,null,-1,-4,null,-1,2147483647,null],"l2_i64":["3",null,"9007199254740995","-4611686018427387904","9007199254740994",null,"-4611686018427387904","5","-4611686018427387904","9007199254740995",null,null,"-4611686018427387904",null,null,"9007199254740994","-4","-1",null,"9007199254740994"],"l3_f64":[-50.694,-23.692,null,-3.0,null,84.871,3.0,null,4.0,-3.0,null,1.0,-3.0,null,-1.0,57.757,null,49.863,4.0,null],"l4_utf8":[null,null,"ab","😀","c","c","b","aa",null,"B",null,"é","a","ab","ab","c","ab","b",null,null],"l5_f64":[73.907,0.0,1.0,null,null,-48.306,null,null,27.27,null,-1.0,null,null,null,1.0,-30.214,null,-48.025,4.0,-3.0],"l6_utf8":["é","aa","ab","é","b","ab","😀","b","é","c",null,"ab","zz","B","😀","é",null,"B","ab","zz"]}},"right":{"fields":[{"name":"l6_utf8","type":"utf8"},{"name":"r0_f64","type":"f64"}],"columns":{"r0_f64":[-22.732,null,91.352,3.0,-2.0,null,null,null,-32.074,-22.024,2.0,null,2.0,3.0,4.0,-70.781,29.995,"NaN",null,4.0,null,-79.081,null,null,null,-2.0,"NaN",null,-1.0,97.862],"l6_utf8":["ab","é","c","zz","ab","é","b","é","ab","é","c","é","é","B",null,"é","b","B","é","ab","B","😀","😀","é","ab","é","ab","😀",null,"é"]}}},"ops":[{"op":"groupBy","keys":["l6_utf8"],"aggs":{"n0":{"agg":"len"},"a0_0":{"agg":"count","arg":{"col":"l2_i64"}}}},{"op":"join","right":"right","on":["l6_utf8"],"how":"anti"}],"expected":{"fields":[{"name":"l6_utf8","type":"utf8"},{"name":"n0","type":"u32"},{"name":"a0_0","type":"u32"}],"columns":{"l6_utf8":["aa",null],"n0":[1,2],"a0_0":[0,1]}}},{"id":"random/0365","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_bool","type":"bool"},{"name":"l4_f64","type":"f64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_f64":[],"l3_bool":[],"l4_f64":[]}}},"ops":[{"op":"withColumns","exprs":{"x0":{"fn":"isIn","args":[{"col":"l0_utf8"}],"values":["c","a","B"]}}},{"op":"unique"},{"op":"withColumns","exprs":{"x2":{"fn":"and","args":[{"fn":"eq","args":[{"col":"l0_utf8"},{"lit":"c"}]},{"fn":"lt","args":[{"lit":2},{"col":"l4_f64"}]}]}}},{"op":"withColumns","exprs":{"x3":{"col":"l4_f64"}}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_bool","type":"bool"},{"name":"l4_f64","type":"f64"},{"name":"x0","type":"bool"},{"name":"x2","type":"bool"},{"name":"x3","type":"f64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_f64":[],"l3_bool":[],"l4_f64":[],"x0":[],"x2":[],"x3":[]}}},{"id":"random/0366","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_f64","type":"f64"},{"name":"l5_i64","type":"i64"},{"name":"l6_utf8","type":"utf8"}],"columns":{"l0_utf8":[null,"","a",null,"c",null,"é","B",null,"ab","é","zz",null,null,null,null,null,"b","","",null,"b","c","","ab","é","B","ab","B","","a","😀","😀","aa","😀","😀","b","","😀","a","ab",null,"","c","😀","é","zz","aa","😀","zz","","ab","é","ab","aa","zz",null,null,null,"b"],"l1_i32":[-5,5,2,0,1,null,3,3,null,-2147483648,1,-3,5,null,-5,5,null,-3,5,null,5,null,-4,-1,-1,-5,-254,2147483647,null,5,3,1,4,0,2,null,2,null,null,3,1,null,null,-3,null,null,0,-1,null,2,null,5,0,-1,-5,0,0,-5,1,1],"l2_bool":[null,null,null,null,null,true,false,null,true,false,null,false,true,null,true,true,true,false,true,false,false,null,false,null,true,true,true,true,false,null,true,null,false,null,null,true,true,false,null,false,false,true,null,true,true,true,false,null,false,null,false,false,false,true,true,null,null,null,true,true],"l3_utf8":[null,"aa","ab","aa","ab","a","aa","ab",null,null,"aa","B",null,"aa",null,"b",null,null,null,"ab","","b","é",null,"zz","zz","b","a","aa","ab","ab",null,null,"a","😀","😀","b","ab","😀","zz","b","b",null,"a","ab","😀",null,"é","😀","aa","😀","😀",null,"😀",null,"ab","é","","ab",null],"l4_f64":["NaN",null,null,"NaN",87.24,"NaN",1.0,2.0,null,2.0,null,61.212,36.952,-2.0,0.0,"-0",null,57.019,null,null,-4.0,2.0,69.638,-4.0,null,-14.064,78.188,null,-3.0,null,4.0,null,55.408,0.0,"NaN",null,null,-1.0,-70.486,62.488,null,-9.905,41.272,-4.0,1.0,"NaN",null,0.0,-2.0,null,4.0,63.367,-15.665,-22.358,null,-4.0,-2.0,-39.799,-58.209,-15.887],"l5_i64":["-4","9007199254740992","-4611686018427387904",null,"9007199254740993","-3",null,"9007199254740995","-4611686018427387904","-4611686018427387904","9007199254740995","-4611686018427387904",null,null,"-4611686018427387904","0","0","9007199254740994","-5","-5",null,"9007199254740995","9007199254740993","0","-3","-4611686018427387904","9007199254740994","0",null,"-4611686018427387904","9007199254740994","0",null,"3","9007199254740993","9007199254740992","-1",null,null,"9007199254740992",null,"9007199254740994",null,"9007199254740992","-4611686018427387904",null,null,"3","3","3","-4611686018427387904",null,null,"-4611686018427387904",null,"-2","9007199254740994",null,"-2",null],"l6_utf8":["c","é",null,"",null,"aa","ab",null,null,"é","ab","c","é","B","aa","c","aa","zz","c","aa",null,"c","aa","",null,"aa",null,null,null,"a","é","a",null,"","aa","","😀",null,"c","é","a","ab",null,"B","","c","aa",null,null,null,"é",null,null,null,null,null,"c","b",null,""]}}},"ops":[{"op":"withColumns","exprs":{"x0":{"fn":"mul","args":[{"lit":0.5},{"fn":"abs","args":[{"col":"l4_f64"}]}]}}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_f64","type":"f64"},{"name":"l5_i64","type":"i64"},{"name":"l6_utf8","type":"utf8"},{"name":"x0","type":"f64"}],"columns":{"l0_utf8":[null,"","a",null,"c",null,"é","B",null,"ab","é","zz",null,null,null,null,null,"b","","",null,"b","c","","ab","é","B","ab","B","","a","😀","😀","aa","😀","😀","b","","😀","a","ab",null,"","c","😀","é","zz","aa","😀","zz","","ab","é","ab","aa","zz",null,null,null,"b"],"l1_i32":[-5,5,2,0,1,null,3,3,null,-2147483648,1,-3,5,null,-5,5,null,-3,5,null,5,null,-4,-1,-1,-5,-254,2147483647,null,5,3,1,4,0,2,null,2,null,null,3,1,null,null,-3,null,null,0,-1,null,2,null,5,0,-1,-5,0,0,-5,1,1],"l2_bool":[null,null,null,null,null,true,false,null,true,false,null,false,true,null,true,true,true,false,true,false,false,null,false,null,true,true,true,true,false,null,true,null,false,null,null,true,true,false,null,false,false,true,null,true,true,true,false,null,false,null,false,false,false,true,true,null,null,null,true,true],"l3_utf8":[null,"aa","ab","aa","ab","a","aa","ab",null,null,"aa","B",null,"aa",null,"b",null,null,null,"ab","","b","é",null,"zz","zz","b","a","aa","ab","ab",null,null,"a","😀","😀","b","ab","😀","zz","b","b",null,"a","ab","😀",null,"é","😀","aa","😀","😀",null,"😀",null,"ab","é","","ab",null],"l4_f64":["NaN",null,null,"NaN",87.24,"NaN",1.0,2.0,null,2.0,null,61.212,36.952,-2.0,0.0,"-0",null,57.019,null,null,-4.0,2.0,69.638,-4.0,null,-14.064,78.188,null,-3.0,null,4.0,null,55.408,0.0,"NaN",null,null,-1.0,-70.486,62.488,null,-9.905,41.272,-4.0,1.0,"NaN",null,0.0,-2.0,null,4.0,63.367,-15.665,-22.358,null,-4.0,-2.0,-39.799,-58.209,-15.887],"l5_i64":["-4","9007199254740992","-4611686018427387904",null,"9007199254740993","-3",null,"9007199254740995","-4611686018427387904","-4611686018427387904","9007199254740995","-4611686018427387904",null,null,"-4611686018427387904","0","0","9007199254740994","-5","-5",null,"9007199254740995","9007199254740993","0","-3","-4611686018427387904","9007199254740994","0",null,"-4611686018427387904","9007199254740994","0",null,"3","9007199254740993","9007199254740992","-1",null,null,"9007199254740992",null,"9007199254740994",null,"9007199254740992","-4611686018427387904",null,null,"3","3","3","-4611686018427387904",null,null,"-4611686018427387904",null,"-2","9007199254740994",null,"-2",null],"l6_utf8":["c","é",null,"",null,"aa","ab",null,null,"é","ab","c","é","B","aa","c","aa","zz","c","aa",null,"c","aa","",null,"aa",null,null,null,"a","é","a",null,"","aa","","😀",null,"c","é","a","ab",null,"B","","c","aa",null,null,null,"é",null,null,null,null,null,"c","b",null,""],"x0":["NaN",null,null,"NaN",43.62,"NaN",0.5,1.0,null,1.0,null,30.606,18.476,1.0,0.0,0.0,null,28.5095,null,null,2.0,1.0,34.819,2.0,null,7.032,39.094,null,1.5,null,2.0,null,27.704,0.0,"NaN",null,null,0.5,35.243,31.244,null,4.9525,20.636,2.0,0.5,"NaN",null,0.0,1.0,null,2.0,31.6835,7.8325,11.179,null,2.0,1.0,19.8995,29.1045,7.9435]}}},{"id":"random/0367","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_i64","type":"i64"},{"name":"l4_f64","type":"f64"},{"name":"l5_i64","type":"i64"},{"name":"l6_f64","type":"f64"}],"columns":{"l0_utf8":[null,"B","b",null,"zz","aa","b","b",null,null,null,"b",null,"é","ab","B","aa","b","","zz"],"l1_i32":[null,null,null,null,4,-1,3,-2,2,-983,null,2,2,3,0,2,2,-5,null,2],"l2_utf8":[null,null,"c","aa","c","b",null,"😀",null,null,null,"😀","zz","é","B","é","B","a","a","a"],"l3_i64":["-5","9007199254740994","9007199254740993","9007199254740994","5","3",null,"-4611686018427387904",null,"9007199254740992",null,"-1","-4611686018427387904",null,"4",null,null,"-4611686018427387904","-4611686018427387904",null],"l4_f64":[null,-2.0,58.184,null,null,90.503,-2.0,null,null,71.104,3.0,-1.0,-1.0,null,null,1.634,2.0,1.0,null,-33.622],"l5_i64":["-1",null,null,"-4611686018427387904","9007199254740993",null,"4","-4611686018427387904","-4611686018427387904","9007199254740995",null,null,"-4611686018427387904","-4611686018427387904","9007199254740995","9007199254740995","9007199254740994","-4",null,"-4"],"l6_f64":[-17.201,3.0,-21.711,-65.025,null,0.0,17.356,null,-2.0,1.0,1.0,2.0,null,null,27.118,4.0,4.0,"-0",51.47,-92.759]}}},"ops":[{"op":"unique","columns":["l0_utf8"]},{"op":"withColumns","exprs":{"x1":{"fn":"mod","args":[{"fn":"floorDiv","args":[{"col":"l4_f64"},{"col":"l1_i32"}]},{"col":"l1_i32"}]}}},{"op":"sort","keys":[{"expr":{"col":"l2_utf8"},"descending":true}],"nullsLast":true},{"op":"filter","predicate":{"fn":"lte","args":[{"col":"x1"},{"col":"l1_i32"}]}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_i64","type":"i64"},{"name":"l4_f64","type":"f64"},{"name":"l5_i64","type":"i64"},{"name":"l6_f64","type":"f64"},{"name":"x1","type":"f64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_utf8":[],"l3_i64":[],"l4_f64":[],"l5_i64":[],"l6_f64":[],"x1":[]}}},{"id":"random/0368","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_f64","type":"f64"},{"name":"l4_f64","type":"f64"},{"name":"l5_utf8","type":"utf8"}],"columns":{"l0_utf8":["zz","😀","a","","😀"],"l1_i32":[null,2,2147483647,2147483647,-4],"l2_i32":[2,4,-2,-1,-1],"l3_f64":[21.339,-34.706,14.51,-48.685,3.0],"l4_f64":[3.0,1.0,-31.516,null,null],"l5_utf8":[null,"é",null,"a","ab"]}},"right":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"r0_utf8","type":"utf8"}],"columns":{"r0_utf8":["😀","😀",null,"b","zz","B","a","b","😀","é","b","aa","a","ab","é","c","B","é","ab",null,"a",null,"é","😀","a","é","a","ab",null,null],"l0_utf8":["a","😀","😀","a","😀","a","😀","","a","a","","zz","😀","a","zz","zz","😀","😀","😀","😀","😀","😀","a","😀","zz","😀","a","zz","zz","zz"]}}},"ops":[{"op":"join","right":"right","on":["l0_utf8"],"how":"inner"},{"op":"sort","keys":[{"expr":{"col":"r0_utf8"},"descending":true},{"expr":{"col":"l1_i32"},"descending":false}],"nullsLast":false},{"op":"sort","keys":[{"expr":{"col":"l1_i32"},"descending":false},{"expr":{"col":"l5_utf8"},"descending":true}],"nullsLast":true}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_f64","type":"f64"},{"name":"l4_f64","type":"f64"},{"name":"l5_utf8","type":"utf8"},{"name":"r0_utf8","type":"utf8"}],"columns":{"l0_utf8":["😀","😀","😀","😀","😀","😀","😀","😀","😀","😀","😀","😀","😀","😀","😀","😀","😀","😀","😀","😀","😀","😀","😀","😀","😀","😀","","","a","a","a","a","a","a","a","a","zz","zz","zz","zz","zz","zz","zz"],"l1_i32":[-4,-4,-4,-4,-4,-4,-4,-4,-4,-4,-4,-4,-4,2,2,2,2,2,2,2,2,2,2,2,2,2,2147483647,2147483647,2147483647,2147483647,2147483647,2147483647,2147483647,2147483647,2147483647,2147483647,null,null,null,null,null,null,null],"l2_i32":[-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,4,4,4,4,4,4,4,4,4,4,4,4,4,-1,-1,-2,-2,-2,-2,-2,-2,-2,-2,2,2,2,2,2,2,2],"l3_f64":[3.0,3.0,3.0,3.0,3.0,3.0,3.0,3.0,3.0,3.0,3.0,3.0,3.0,-34.706,-34.706,-34.706,-34.706,-34.706,-34.706,-34.706,-34.706,-34.706,-34.706,-34.706,-34.706,-34.706,-48.685,-48.685,14.51,14.51,14.51,14.51,14.51,14.51,14.51,14.51,21.339,21.339,21.339,21.339,21.339,21.339,21.339],"l4_f64":[null,null,null,null,null,null,null,null,null,null,null,null,null,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,null,null,-31.516,-31.516,-31.516,-31.516,-31.516,-31.516,-31.516,-31.516,3.0,3.0,3.0,3.0,3.0,3.0,3.0],"l5_utf8":["ab","ab","ab","ab","ab","ab","ab","ab","ab","ab","ab","ab","ab","é","é","é","é","é","é","é","é","é","é","é","é","é","a","a",null,null,null,null,null,null,null,null,null,null,null,null,null,null,null],"r0_utf8":[null,null,null,"😀","😀","é","é","zz","ab","a","a","a","B",null,null,null,"😀","😀","é","é","zz","ab","a","a","a","B","b","b","😀","😀","é","é","b","ab","a","B",null,null,"é","c","ab","aa","a"]}}},{"id":"random/0369","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_i32","type":"i32"}],"columns":{"l0_utf8":["c","aa","é","b","c"],"l1_i32":[-3,-2,-4,3,5],"l2_f64":[80.017,-1.0,-26.669,0.0,70.019],"l3_utf8":["a","😀","a","aa","c"],"l4_utf8":["ab","a","B","c","c"],"l5_i32":[-3,4,-3,0,4]}}},"ops":[{"op":"unique","columns":["l1_i32"]}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_i32","type":"i32"}],"columns":{"l0_utf8":["c","aa","é","b","c"],"l1_i32":[-3,-2,-4,3,5],"l2_f64":[80.017,-1.0,-26.669,0.0,70.019],"l3_utf8":["a","😀","a","aa","c"],"l4_utf8":["ab","a","B","c","c"],"l5_i32":[-3,4,-3,0,4]}}},{"id":"random/0370","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_f64","type":"f64"},{"name":"l4_f64","type":"f64"}],"columns":{"l0_utf8":["😀","zz","b","é","aa","","a","c","zz","aa","é","ab","😀","😀","c","b","é","B","zz","B"],"l1_i32":[-3,-2,-2,-1,-3,5,5,158,3,2,5,-3,-3,-1,4,5,-3,2,2,-4],"l2_utf8":["c","zz","aa","zz","b","b","B","b","","c","aa","é","zz","zz","B","zz","b","😀","zz","a"],"l3_f64":[62.662,-32.088,2.0,-3.0,4.0,"-0",4.0,86.394,-3.0,22.222,-34.013,63.166,-68.276,0.0,-3.0,3.0,-1.0,0.0,-91.737,-69.461],"l4_f64":[0.0,-50.463,-1.0,-57.016,3.43,31.591,0.0,22.693,39.996,-29.512,4.0,6.292,4.0,15.69,-3.0,43.847,28.804,94.01,-51.104,16.502]}}},"ops":[{"op":"groupBy","keys":["l2_utf8"],"aggs":{"n0":{"agg":"len"},"a0_0":{"agg":"nUnique","arg":{"col":"l1_i32"}},"a0_1":{"agg":"first","arg":{"col":"l0_utf8"}},"a0_2":{"agg":"nUnique","arg":{"col":"l0_utf8"}}}}],"expected":{"fields":[{"name":"l2_utf8","type":"utf8"},{"name":"n0","type":"u32"},{"name":"a0_0","type":"u32"},{"name":"a0_1","type":"utf8"},{"name":"a0_2","type":"u32"}],"columns":{"l2_utf8":["c","zz","aa","b","B","","é","😀","a"],"n0":[2,6,2,4,2,1,1,1,1],"a0_0":[2,5,2,3,2,1,1,1,1],"a0_1":["😀","zz","b","aa","a","zz","ab","B","B"],"a0_2":[2,4,2,4,2,1,1,1,1]}}},{"id":"random/0371","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_i32","type":"i32"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_f64":[],"l3_i32":[]}}},"ops":[{"op":"withColumns","exprs":{"x0":{"fn":"add","args":[{"col":"l1_i32"},{"fn":"div","args":[{"col":"l2_f64"},{"col":"l1_i32"}]}]}}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_i32","type":"i32"},{"name":"x0","type":"f64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_f64":[],"l3_i32":[],"x0":[]}}},{"id":"random/0372","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_utf8","type":"utf8"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_f64":[],"l3_utf8":[]}}},"ops":[{"op":"unique","columns":["l3_utf8"]},{"op":"withColumns","exprs":{"x1":{"fn":"gte","args":[{"fn":"abs","args":[{"col":"l2_f64"}]},{"col":"l2_f64"}]}}},{"op":"unique"},{"op":"withColumns","exprs":{"x3":{"col":"l1_i32"}}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_utf8","type":"utf8"},{"name":"x1","type":"bool"},{"name":"x3","type":"i32"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_f64":[],"l3_utf8":[],"x1":[],"x3":[]}}},{"id":"random/0373","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_bool","type":"bool"},{"name":"l4_i64","type":"i64"},{"name":"l5_i32","type":"i32"}],"columns":{"l0_utf8":["","a","","ab",""],"l1_i32":[-3,5,-2,-2147483648,5],"l2_f64":[-3.224,97.391,81.367,69.525,-17.591],"l3_bool":[true,true,false,false,false],"l4_i64":["-3","-4611686018427387904","-3","-4611686018427387904","-5"],"l5_i32":[3,1,2,-3,-5]}}},"ops":[{"op":"unique"},{"op":"groupBy","keys":["l0_utf8"],"aggs":{"n1":{"agg":"len"},"a1_0":{"agg":"nUnique","arg":{"col":"l5_i32"}},"a1_1":{"agg":"max","arg":{"col":"l4_i64"}}}},{"op":"withColumns","exprs":{"x2":{"fn":"neg","args":[{"lit":0.5}]}}},{"op":"unique"}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"n1","type":"u32"},{"name":"a1_0","type":"u32"},{"name":"a1_1","type":"i64"},{"name":"x2","type":"f64"}],"columns":{"l0_utf8":["","a","ab"],"n1":[3,1,1],"a1_0":[3,1,1],"a1_1":["-3","-4611686018427387904","-4611686018427387904"],"x2":[-0.5,-0.5,-0.5]}}},{"id":"random/0374","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_bool","type":"bool"}],"columns":{"l0_utf8":["","b","c","é","c","b",null,"b","zz","c","é","b","","😀","😀","aa","é",null,"ab","aa",null,"c",null,"c","","zz","😀","","aa","zz","a","a","aa","B",null,"a","zz","B","B","😀","","😀","b","b","ab","é","é","","B","","a","a","zz","aa","B",null,"ab","","ab",null],"l1_i32":[0,3,-2,5,-1,5,null,2147483647,5,-4,3,1,0,0,null,4,-5,-4,2,4,-4,1,5,-5,2147483647,-4,null,4,3,-1,2,3,0,4,-3,null,3,null,5,-4,5,-3,-4,null,-1,-4,580,3,-5,-5,-5,0,3,-2,-124,5,-3,2147483647,2,-4],"l2_bool":[false,true,false,null,false,false,true,false,false,true,false,false,true,true,false,true,null,false,true,false,true,false,false,true,true,true,false,true,true,true,null,true,true,true,false,null,true,false,false,true,true,false,true,false,false,false,false,false,true,false,false,true,true,true,true,false,false,true,false,false],"l3_bool":[false,true,false,null,true,true,false,false,true,true,false,false,true,null,true,true,false,true,false,false,true,false,false,true,true,null,false,null,false,true,null,null,false,false,false,true,false,true,false,false,true,null,false,false,false,true,true,true,false,false,false,true,false,true,true,true,false,false,false,false]}}},"ops":[{"op":"head","n":8},{"op":"filter","predicate":{"fn":"gte","args":[{"col":"l1_i32"},{"lit":0.5}]}},{"op":"unique","columns":["l3_bool"]}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_bool","type":"bool"}],"columns":{"l0_utf8":["b","é","b"],"l1_i32":[3,5,2147483647],"l2_bool":[true,null,false],"l3_bool":[true,null,false]}}},{"id":"random/0375","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_f64","type":"f64"}],"columns":{"l0_utf8":["ab","c","😀",null,null,null,"b","ab",null,"aa","😀",null,null,null,null,"zz","aa",null,"c","é","B","aa","é",null,"B","é","B","😀","aa","a","ab","zz","b",null,"ab","😀",null,"b",null,null,"aa","aa",null,"ab",null,"aa","B","b","aa","b",null,null,null,"B",null,"😀","zz","c",null,"c"],"l1_i32":[0,4,-2,-5,null,4,509,-2,-5,-1,null,4,3,null,4,null,2,3,3,-4,-3,-1,-4,-5,null,4,-4,null,-3,1,-5,-5,null,null,3,null,1,null,4,1,-4,1,null,5,-3,5,-2,null,0,null,-3,null,0,-3,5,4,-5,-2,-5,null],"l2_utf8":["zz","a","zz","c",null,"ab","b",null,"b",null,"😀",null,"aa","a","","b",null,"zz",null,"é","😀","","zz","B",null,"B","ab","zz","😀","c","aa",null,"😀","c",null,"😀",null,"😀",null,"aa","é","é","aa","b","a","","b","c","aa",null,null,null,"aa",null,"a","a","😀","B","zz","ab"],"l3_f64":[-82.624,null,69.766,1.0,null,"Infinity",21.82,null,-23.211,43.465,null,"Infinity",-35.363,-4.0,3.0,4.0,null,-3.0,null,null,4.0,1.0,1.0,-19.733,-96.705,null,21.653,null,4.0,null,0.0,null,null,null,-0.128,-24.846,70.008,null,97.433,-4.0,-2.0,null,91.059,-59.189,null,4.0,0.0,-79.568,null,31.318,-76.594,null,2.0,null,14.886,33.306,-78.883,null,null,87.406]}},"right":{"fields":[{"name":"l2_utf8","type":"utf8"},{"name":"r0_f64","type":"f64"}],"columns":{"r0_f64":[null,-3.0,90.588,2.0,-2.0,"Infinity",0.0,90.013,-70.274,4.0,null,-1.0,-1.0,4.0,-2.0,-86.404,null,-25.606,null,null,-4.0,null,-2.0,0.0,null,80.253,-47.288,-3.0,null,3.0],"l2_utf8":["zz","aa","",null,"zz","B","ab","😀",null,"zz",null,null,"B",null,"ab",null,"B","c",null,"aa",null,"ab",null,"c","aa","aa","é","c","B",null]}}},"ops":[{"op":"join","right":"right","on":["l2_utf8"],"how":"left"},{"op":"unique"}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_f64","type":"f64"},{"name":"r0_f64","type":"f64"}],"columns":{"l0_utf8":["ab","ab","ab","c","😀","😀","😀",null,null,null,null,null,null,null,"b","ab",null,"aa","😀",null,null,null,null,null,null,"zz","aa",null,null,null,"c","é","B","aa","é","é","é",null,null,null,"B","é","é","é","B","B","B","😀","😀","😀","aa","a","a","a","ab","ab","ab","zz","b",null,null,null,"ab","😀",null,null,null,null,null,"aa","aa",null,null,null,"ab",null,"aa","B","b","b","b","aa","aa","aa","b",null,null,null,null,"B",null,"😀","zz","c","c","c",null,null,null,"c","c","c"],"l1_i32":[0,0,0,4,-2,-2,-2,-5,-5,-5,null,4,4,4,509,-2,-5,-1,null,4,3,3,3,null,4,null,2,3,3,3,3,-4,-3,-1,-4,-4,-4,-5,-5,-5,null,4,4,4,-4,-4,-4,null,null,null,-3,1,1,1,-5,-5,-5,-5,null,null,null,null,3,null,1,4,1,1,1,-4,1,null,null,null,5,-3,5,-2,null,null,null,0,0,0,null,-3,0,0,0,-3,5,4,-5,-2,-2,-2,-5,-5,-5,null,null,null],"l2_utf8":["zz","zz","zz","a","zz","zz","zz","c","c","c",null,"ab","ab","ab","b",null,"b",null,"😀",null,"aa","aa","aa","a","","b",null,"zz","zz","zz",null,"é","😀","","zz","zz","zz","B","B","B",null,"B","B","B","ab","ab","ab","zz","zz","zz","😀","c","c","c","aa","aa","aa",null,"😀","c","c","c",null,"😀",null,null,"aa","aa","aa","é","é","aa","aa","aa","b","a","","b","c","c","c","aa","aa","aa",null,null,"aa","aa","aa",null,"a","a","😀","B","B","B","zz","zz","zz","ab","ab","ab"],"l3_f64":[-82.624,-82.624,-82.624,null,69.766,69.766,69.766,1.0,1.0,1.0,null,"Infinity","Infinity","Infinity",21.82,null,-23.211,43.465,null,"Infinity",-35.363,-35.363,-35.363,-4.0,3.0,4.0,null,-3.0,-3.0,-3.0,null,null,4.0,1.0,1.0,1.0,1.0,-19.733,-19.733,-19.733,-96.705,null,null,null,21.653,21.653,21.653,null,null,null,4.0,null,null,null,0.0,0.0,0.0,null,null,null,null,null,-0.128,-24.846,70.008,97.433,-4.0,-4.0,-4.0,-2.0,null,91.059,91.059,91.059,-59.189,null,4.0,0.0,-79.568,-79.568,-79.568,null,null,null,31.318,-76.594,2.0,2.0,2.0,null,14.886,33.306,-78.883,null,null,null,null,null,null,87.406,87.406,87.406],"r0_f64":[null,-2.0,4.0,null,null,-2.0,4.0,-25.606,0.0,-3.0,null,0.0,-2.0,null,null,null,null,null,90.013,null,-3.0,null,80.253,null,90.588,null,null,null,-2.0,4.0,null,-47.288,90.013,90.588,null,-2.0,4.0,"Infinity",-1.0,null,null,"Infinity",-1.0,null,0.0,-2.0,null,null,-2.0,4.0,90.013,-25.606,0.0,-3.0,-3.0,null,80.253,null,90.013,-25.606,0.0,-3.0,null,90.013,null,null,-3.0,null,80.253,-47.288,-47.288,-3.0,null,80.253,null,null,90.588,null,-25.606,0.0,-3.0,-3.0,null,80.253,null,null,-3.0,null,80.253,null,null,null,90.013,"Infinity",-1.0,null,null,-2.0,4.0,0.0,-2.0,null]}}},{"id":"random/0376","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_i64","type":"i64"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_i32","type":"i32"}],"columns":{"l0_utf8":["B","c",null,"a","😀","B",null,null,"","c","é","B","b","aa","a","é","😀","😀","aa",null,"é","zz",null,"aa","","c","é","b","a","zz","a","b","B","a","c","","b",null,"a","b","a","","c","","B","c","é","","a","c","aa","aa","aa","B","é","aa","ab","😀","c","a"],"l1_i32":[-5,-3,null,2,3,-3,-1,-1,1,163,-3,4,-5,-2147483648,-3,0,null,4,5,-2,1,1,-5,-5,-1,1,4,-1,null,5,-3,5,2,null,-5,-3,1,3,-2147483648,4,-3,-1,5,-2,null,-3,135,-2,-5,137,null,null,4,-3,2,-2,-5,-3,4,-2],"l2_f64":[null,39.892,97.164,-1.0,-70.666,-4.0,4.0,"Infinity",-2.0,77.843,-16.664,29.839,"NaN",85.912,-1.0,-44.827,3.0,null,-30.099,-4.0,81.847,-62.871,4.0,"-Infinity",-38.092,-83.794,-4.0,2.0,-1.0,1.0,-4.0,"-0",3.0,"NaN",null,2.253,43.466,-98.923,28.898,4.0,null,37.342,-2.0,-76.652,"-0",47.819,3.0,91.661,-3.0,93.174,1.0,-13.203,0.0,17.27,10.722,1.0,null,2.0,67.061,4.0],"l3_i64":["0","2","9007199254740993","9007199254740992","-4611686018427387904","1","2","-2","-4611686018427387904","-2","-4611686018427387904","9007199254740995","-2","-4611686018427387904","9007199254740995","-4611686018427387904",null,"9007199254740993","9007199254740993","-4611686018427387904","-5","-4611686018427387904","-4611686018427387904","9007199254740995","9007199254740995","4",null,"-4","-2","9007199254740995","9007199254740995","5","-4611686018427387904","-4611686018427387904",null,"5","-4611686018427387904",null,"0","-4611686018427387904","-2","9007199254740992",null,"-4611686018427387904","-4611686018427387904","9007199254740994","9007199254740995","-4611686018427387904","-2","9007199254740994",null,"-4611686018427387904","-2",null,"-4611686018427387904","-4611686018427387904","-1",null,"3","9007199254740993"],"l4_utf8":["zz","é","c","😀",null,"aa",null,"zz",null,null,"c","😀","zz","b","c","zz","a","c","😀","B","é","c","","B","","ab","b","a","zz","😀","B","a","c","c","ab","c","😀","é","","a","b","aa",null,"b","c","a","zz",null,"aa","aa","aa","c","😀","c","é","c","B","aa",null,"c"],"l5_i32":[null,3,3,4,1,1,0,1,0,0,769,-1,3,2,5,-5,-2,4,3,4,-3,5,2,-5,null,-4,4,1,-2,-2,4,-3,5,4,2,-5,5,1,3,5,3,-4,-3,-1,null,5,-3,2147483647,1,-3,3,null,2147483647,-5,2,-1,0,3,3,1]}}},"ops":[{"op":"head","n":9},{"op":"withColumns","exprs":{"x1":{"fn":"lt","args":[{"col":"l5_i32"},{"col":"l2_f64"}]}}},{"op":"groupBy","keys":["l0_utf8"],"aggs":{"n2":{"agg":"len"},"a2_0":{"agg":"max","arg":{"col":"l5_i32"}},"a2_1":{"agg":"median","arg":{"col":"l2_f64"}},"a2_2":{"agg":"last","arg":{"col":"l5_i32"}}}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"n2","type":"u32"},{"name":"a2_0","type":"i32"},{"name":"a2_1","type":"f64"},{"name":"a2_2","type":"i32"}],"columns":{"l0_utf8":["B","c",null,"a","😀",""],"n2":[2,1,3,1,1,1],"a2_0":[1,3,3,4,1,0],"a2_1":[-4.0,39.892,97.164,-1.0,-70.666,-2.0],"a2_2":[1,3,1,4,1,0]}}},{"id":"random/0377","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_bool","type":"bool"},{"name":"l4_f64","type":"f64"}],"columns":{"l0_utf8":["aa","😀","é","aa","c"],"l1_i32":[-4,-2,0,5,-2],"l2_utf8":["","B","é","😀","aa"],"l3_bool":[false,true,false,true,true],"l4_f64":[74.859,6.729,3.0,-2.0,2.0]}},"right":{"fields":[{"name":"l2_utf8","type":"utf8"},{"name":"r0_bool","type":"bool"}],"columns":{"r0_bool":[false,true,true],"l2_utf8":["B","","é"]}}},"ops":[{"op":"sort","keys":[{"expr":{"col":"l2_utf8"},"descending":false},{"expr":{"col":"l1_i32"},"descending":false}],"nullsLast":false},{"op":"join","right":"right","on":["l2_utf8"],"how":"anti"},{"op":"withColumns","exprs":{"x2":{"fn":"mul","args":[{"fn":"sub","args":[{"lit":2},{"col":"l1_i32"}]},{"col":"l1_i32"}]}}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_bool","type":"bool"},{"name":"l4_f64","type":"f64"},{"name":"x2","type":"i32"}],"columns":{"l0_utf8":["c","aa"],"l1_i32":[-2,5],"l2_utf8":["aa","😀"],"l3_bool":[true,true],"l4_f64":[2.0,-2.0],"x2":[-8,-15]}}},{"id":"random/0378","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i64","type":"i64"},{"name":"l3_bool","type":"bool"}],"columns":{"l0_utf8":["😀","B","","a","zz"],"l1_i32":[-2,-1,3,5,-5],"l2_i64":["9007199254740995","-1","-4611686018427387904","-4611686018427387904","-4611686018427387904"],"l3_bool":[true,false,false,true,false]}}},"ops":[{"op":"withColumns","exprs":{"x0":{"fn":"mod","args":[{"fn":"neg","args":[{"col":"l1_i32"}]},{"lit":2}]}}},{"op":"filter","predicate":{"fn":"gt","args":[{"fn":"mul","args":[{"col":"l1_i32"},{"lit":2}]},{"fn":"floorDiv","args":[{"col":"x0"},{"lit":0.5}]}]}},{"op":"withColumns","exprs":{"x2":{"fn":"mul","args":[{"col":"l1_i32"},{"fn":"fillNull","args":[{"col":"x0"},{"lit":0}]}]}}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i64","type":"i64"},{"name":"l3_bool","type":"bool"},{"name":"x0","type":"i32"},{"name":"x2","type":"i32"}],"columns":{"l0_utf8":["","a"],"l1_i32":[3,5],"l2_i64":["-4611686018427387904","-4611686018427387904"],"l3_bool":[false,true],"x0":[1,1],"x2":[3,5]}}},{"id":"random/0379","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i64","type":"i64"},{"name":"l3_f64","type":"f64"},{"name":"l4_utf8","type":"utf8"}],"columns":{"l0_utf8":["aa","zz","aa",null,"b","b","b",null,"a","zz","a","é","ab",null,"zz","é","B","zz","aa","b","é","aa","c","","B","B","😀","zz","a","b","zz","aa","é","b","ab","","a","😀","","zz","😀","zz","ab","a","","zz","😀","B","ab","","zz","é","😀","a","😀","ab",null,"b","zz","zz"],"l1_i32":[-3,-4,-5,2,-4,-1,5,-2,-4,1,1,178,-5,0,5,-3,null,0,4,-1,0,5,-5,2147483647,null,2147483647,-5,null,1,-5,4,4,5,-1,-1,-2147483648,-1,2,5,null,2,-4,5,5,-4,-4,1,3,-2,1,-1,834,-3,4,-3,4,null,3,null,null],"l2_i64":["9007199254740992",null,null,"2","3","-1",null,"-4611686018427387904",null,"-4611686018427387904","4","-3","-4611686018427387904","4","-4611686018427387904","9007199254740994",null,"0",null,"-4611686018427387904",null,null,"-4611686018427387904","-3",null,"-4611686018427387904",null,"-4611686018427387904","3","-4611686018427387904","4","9007199254740993","9007199254740995","3",null,"9007199254740992","1","5","-4611686018427387904","0","9007199254740993","-5","9007199254740995","-4611686018427387904","-4611686018427387904","9007199254740994","9007199254740992","-3","9007199254740993","9007199254740992","-4611686018427387904","-5","-2","-4611686018427387904","9007199254740994","9007199254740994","-4611686018427387904","5","9007199254740994","9007199254740992"],"l3_f64":[-15.151,3.0,-4.0,-3.0,"NaN",65.997,-4.0,38.324,0.0,-86.048,36.861,null,null,47.053,-4.0,-69.156,null,-1.0,-2.0,-48.991,0.0,96.449,-27.568,20.163,-4.0,34.694,-2.0,-1.0,0.0,-96.488,2.0,1.0,-86.385,"NaN",-97.619,-68.609,82.633,4.0,null,-2.0,41.63,1.0,-21.976,0.0,-3.0,38.903,"-0",-1.0,-1.0,44.678,1.0,-43.061,58.309,-4.0,-99.753,null,-3.0,-4.0,-47.155,"NaN"],"l4_utf8":["aa","é","ab",null,"zz","b","zz","","😀",null,"a",null,"B","a","","é","","aa","😀","aa","😀","b","B","zz","é","ab","a","ab","","b","c","ab","","B","b","ab","b","ab","aa","B","😀","b","","aa","zz","ab","😀","é","é","😀","","a","B","😀",null,"c","B","B","c","ab"]}}},"ops":[{"op":"filter","predicate":{"fn":"isNull","args":[{"col":"l4_utf8"}]}},{"op":"filter","predicate":{"fn":"gte","args":[{"fn":"sub","args":[{"col":"l3_f64"},{"col":"l3_f64"}]},{"col":"l3_f64"}]}},{"op":"unique"},{"op":"groupBy","keys":["l2_i64"],"aggs":{"n3":{"agg":"len"},"a3_0":{"agg":"min","arg":{"col":"l0_utf8"}},"a3_1":{"agg":"max","arg":{"col":"l1_i32"}},"a3_2":{"agg":"max","arg":{"col":"l1_i32"}}}}],"expected":{"fields":[{"name":"l2_i64","type":"i64"},{"name":"n3","type":"u32"},{"name":"a3_0","type":"utf8"},{"name":"a3_1","type":"i32"},{"name":"a3_2","type":"i32"}],"columns":{"l2_i64":["2","-4611686018427387904","9007199254740994"],"n3":[1,1,1],"a3_0":[null,"zz","😀"],"a3_1":[2,1,-3],"a3_2":[2,1,-3]}}},{"id":"random/0380","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_f64","type":"f64"},{"name":"l4_f64","type":"f64"}],"columns":{"l0_utf8":["😀"],"l1_i32":[-5],"l2_bool":[true],"l3_f64":[-3.0],"l4_f64":[83.042]}}},"ops":[{"op":"head","n":2},{"op":"filter","predicate":{"fn":"isNotNull","args":[{"col":"l4_f64"}]}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_f64","type":"f64"},{"name":"l4_f64","type":"f64"}],"columns":{"l0_utf8":["😀"],"l1_i32":[-5],"l2_bool":[true],"l3_f64":[-3.0],"l4_f64":[83.042]}}},{"id":"random/0381","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_f64","type":"f64"}],"columns":{"l0_utf8":["a","😀","é","","ab"],"l1_i32":[1,5,3,1,2147483647],"l2_bool":[true,false,true,true,false],"l3_f64":[-2.0,-53.744,-86.46,-62.362,"-0"]}}},"ops":[{"op":"sort","keys":[{"expr":{"col":"l2_bool"},"descending":false},{"expr":{"col":"l3_f64"},"descending":true}],"nullsLast":true},{"op":"head","n":5},{"op":"withColumns","exprs":{"x2":{"fn":"lt","args":[{"fn":"when","args":[{"fn":"lt","args":[{"col":"l1_i32"},{"col":"l1_i32"}]},{"lit":0.5},{"col":"l1_i32"}]},{"fn":"floorDiv","args":[{"col":"l1_i32"},{"col":"l3_f64"}]}]}}},{"op":"sort","keys":[{"expr":{"col":"l0_utf8"},"descending":true}],"nullsLast":false}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_f64","type":"f64"},{"name":"x2","type":"bool"}],"columns":{"l0_utf8":["😀","é","ab","a",""],"l1_i32":[5,3,2147483647,1,1],"l2_bool":[false,true,false,true,true],"l3_f64":[-53.744,-86.46,"-0",-2.0,-62.362],"x2":[false,false,false,false,false]}}},{"id":"random/0382","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_i64","type":"i64"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_bool","type":"bool"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_f64":[],"l3_i64":[],"l4_utf8":[],"l5_bool":[]}}},"ops":[{"op":"sort","keys":[{"expr":{"col":"l4_utf8"},"descending":true},{"expr":{"col":"l0_utf8"},"descending":false}],"nullsLast":false},{"op":"withColumns","exprs":{"x1":{"col":"l5_bool"}}},{"op":"unique","columns":["l5_bool"]},{"op":"sort","keys":[{"expr":{"col":"l1_i32"},"descending":false},{"expr":{"col":"l0_utf8"},"descending":true}],"nullsLast":false}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_i64","type":"i64"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_bool","type":"bool"},{"name":"x1","type":"bool"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_f64":[],"l3_i64":[],"l4_utf8":[],"l5_bool":[],"x1":[]}}},{"id":"random/0383","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_i64","type":"i64"},{"name":"l4_i64","type":"i64"}],"columns":{"l0_utf8":["zz","😀","😀","c","a"],"l1_i32":[4,-5,-1,-4,3],"l2_f64":[-43.781,3.0,-20.504,-3.0,98.645],"l3_i64":["9007199254740992","9007199254740995","9007199254740995","-4","-4611686018427387904"],"l4_i64":["0","-4611686018427387904","-4611686018427387904","-3","-4611686018427387904"]}},"right":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"r0_utf8","type":"utf8"}],"columns":{"r0_utf8":["c","a","B","",null,"b","","c",null,"B","ab","B","c","B","","c","😀","zz","😀","b","B","zz","B","b","b","c","😀","a","b","😀"],"l0_utf8":["😀","zz","😀","😀","zz","zz","c","c","c","😀","zz","zz","c","😀","😀","😀","c","a","😀","😀","a","😀","c","c","a","c","c","a","zz","c"]}}},"ops":[{"op":"join","right":"right","on":["l0_utf8"],"how":"anti"},{"op":"head","n":2},{"op":"sort","keys":[{"expr":{"col":"l3_i64"},"descending":false}],"nullsLast":true}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_i64","type":"i64"},{"name":"l4_i64","type":"i64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_f64":[],"l3_i64":[],"l4_i64":[]}}},{"id":"random/0384","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_bool","type":"bool"},{"name":"l4_bool","type":"bool"},{"name":"l5_bool","type":"bool"},{"name":"l6_f64","type":"f64"}],"columns":{"l0_utf8":["aa"],"l1_i32":[-5],"l2_utf8":["B"],"l3_bool":[false],"l4_bool":[true],"l5_bool":[false],"l6_f64":[-9.728]}}},"ops":[{"op":"groupBy","keys":["l2_utf8"],"aggs":{"n0":{"agg":"len"},"a0_0":{"agg":"nUnique","arg":{"col":"l3_bool"}},"a0_1":{"agg":"nUnique","arg":{"col":"l6_f64"}}}},{"op":"unique"}],"expected":{"fields":[{"name":"l2_utf8","type":"utf8"},{"name":"n0","type":"u32"},{"name":"a0_0","type":"u32"},{"name":"a0_1","type":"u32"}],"columns":{"l2_utf8":["B"],"n0":[1],"a0_0":[1],"a0_1":[1]}}},{"id":"random/0385","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_f64","type":"f64"},{"name":"l4_bool","type":"bool"},{"name":"l5_i64","type":"i64"}],"columns":{"l0_utf8":["c","a","c","a","","c","zz","","ab","b","b","zz","zz","c","zz","aa","😀","ab","aa","zz"],"l1_i32":[-3,1,-4,5,4,1,-2,-3,-1,-4,3,1,1,-2,5,-4,3,-4,5,3],"l2_bool":[true,true,false,true,true,true,false,true,false,false,false,false,true,true,true,true,true,true,false,false],"l3_f64":[98.008,-56.997,-67.646,44.162,8.09,4.0,2.0,"NaN",1.0,-44.407,-4.0,"NaN",86.318,-78.083,3.0,-43.878,0.0,1.0,-2.0,1.0],"l4_bool":[false,false,true,true,false,true,true,true,true,false,false,true,true,false,true,true,false,false,false,false],"l5_i64":["-4","9007199254740994","9007199254740994","3","4","5","-4611686018427387904","4","9007199254740994","9007199254740995","2","-4611686018427387904","9007199254740995","9007199254740995","-4611686018427387904","9007199254740994","-4611686018427387904","-3","2","5"]}}},"ops":[{"op":"withColumns","exprs":{"x0":{"fn":"lte","args":[{"fn":"add","args":[{"col":"l3_f64"},{"col":"l3_f64"}]},{"col":"l3_f64"}]}}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_f64","type":"f64"},{"name":"l4_bool","type":"bool"},{"name":"l5_i64","type":"i64"},{"name":"x0","type":"bool"}],"columns":{"l0_utf8":["c","a","c","a","","c","zz","","ab","b","b","zz","zz","c","zz","aa","😀","ab","aa","zz"],"l1_i32":[-3,1,-4,5,4,1,-2,-3,-1,-4,3,1,1,-2,5,-4,3,-4,5,3],"l2_bool":[true,true,false,true,true,true,false,true,false,false,false,false,true,true,true,true,true,true,false,false],"l3_f64":[98.008,-56.997,-67.646,44.162,8.09,4.0,2.0,"NaN",1.0,-44.407,-4.0,"NaN",86.318,-78.083,3.0,-43.878,0.0,1.0,-2.0,1.0],"l4_bool":[false,false,true,true,false,true,true,true,true,false,false,true,true,false,true,true,false,false,false,false],"l5_i64":["-4","9007199254740994","9007199254740994","3","4","5","-4611686018427387904","4","9007199254740994","9007199254740995","2","-4611686018427387904","9007199254740995","9007199254740995","-4611686018427387904","9007199254740994","-4611686018427387904","-3","2","5"],"x0":[false,true,true,false,false,false,false,true,false,true,true,true,false,true,false,true,true,false,true,false]}}},{"id":"random/0386","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_bool","type":"bool"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_bool","type":"bool"},{"name":"l6_utf8","type":"utf8"}],"columns":{"l0_utf8":["é"],"l1_i32":[-5],"l2_bool":[false],"l3_bool":[false],"l4_utf8":["ab"],"l5_bool":[false],"l6_utf8":[""]}},"right":{"fields":[{"name":"l4_utf8","type":"utf8"},{"name":"r0_utf8","type":"utf8"}],"columns":{"r0_utf8":["ab","aa","","B","zz","b","zz",null,"c","zz","B","é","b","zz","c","","B","","ab","ab","é","zz","aa","b","é","b","ab","","😀","a"],"l4_utf8":["ab","ab","ab","ab","ab","ab","ab","ab","ab","ab","ab","ab","ab","ab","ab","ab","ab","ab","ab","ab","ab","ab","ab","ab","ab","ab","ab","ab","ab","ab"]}}},"ops":[{"op":"join","right":"right","on":["l4_utf8"],"how":"inner"},{"op":"unique"},{"op":"unique","columns":["l5_bool"]}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_bool","type":"bool"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_bool","type":"bool"},{"name":"l6_utf8","type":"utf8"},{"name":"r0_utf8","type":"utf8"}],"columns":{"l0_utf8":["é"],"l1_i32":[-5],"l2_bool":[false],"l3_bool":[false],"l4_utf8":["ab"],"l5_bool":[false],"l6_utf8":[""],"r0_utf8":["ab"]}}},{"id":"random/0387","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_i64","type":"i64"},{"name":"l6_f64","type":"f64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_f64":[],"l3_utf8":[],"l4_utf8":[],"l5_i64":[],"l6_f64":[]}}},"ops":[{"op":"sort","keys":[{"expr":{"col":"l1_i32"},"descending":false},{"expr":{"col":"l6_f64"},"descending":true}],"nullsLast":false},{"op":"head","n":5},{"op":"filter","predicate":{"fn":"gt","args":[{"col":"l2_f64"},{"fn":"add","args":[{"col":"l2_f64"},{"col":"l2_f64"}]}]}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_i64","type":"i64"},{"name":"l6_f64","type":"f64"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_f64":[],"l3_utf8":[],"l4_utf8":[],"l5_i64":[],"l6_f64":[]}}},{"id":"random/0388","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_f64","type":"f64"},{"name":"l4_i32","type":"i32"}],"columns":{"l0_utf8":["b"],"l1_i32":[-1],"l2_bool":[null],"l3_f64":[-3.0],"l4_i32":[4]}}},"ops":[{"op":"sort","keys":[{"expr":{"col":"l0_utf8"},"descending":false}],"nullsLast":false},{"op":"unique","columns":["l3_f64"]},{"op":"sort","keys":[{"expr":{"col":"l3_f64"},"descending":true},{"expr":{"col":"l1_i32"},"descending":true}],"nullsLast":false},{"op":"withColumns","exprs":{"x3":{"fn":"neg","args":[{"fn":"floorDiv","args":[{"col":"l4_i32"},{"col":"l4_i32"}]}]}}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_f64","type":"f64"},{"name":"l4_i32","type":"i32"},{"name":"x3","type":"i32"}],"columns":{"l0_utf8":["b"],"l1_i32":[-1],"l2_bool":[null],"l3_f64":[-3.0],"l4_i32":[4],"x3":[-1]}}},{"id":"random/0389","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i64","type":"i64"},{"name":"l3_f64","type":"f64"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_utf8","type":"utf8"}],"columns":{"l0_utf8":[null,"zz","😀","é","c"],"l1_i32":[4,5,null,null,2],"l2_i64":["3","9007199254740992",null,"-4611686018427387904",null],"l3_f64":[1.0,null,-3.0,-79.931,-56.128],"l4_utf8":["","😀",null,null,"zz"],"l5_utf8":["aa","c","B",null,"a"]}},"right":{"fields":[{"name":"l2_i64","type":"i64"},{"name":"r0_f64","type":"f64"}],"columns":{"r0_f64":[],"l2_i64":[]}}},"ops":[{"op":"join","right":"right","on":["l2_i64"],"how":"left"},{"op":"unique"},{"op":"sort","keys":[{"expr":{"col":"r0_f64"},"descending":true}],"nullsLast":false}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i64","type":"i64"},{"name":"l3_f64","type":"f64"},{"name":"l4_utf8","type":"utf8"},{"name":"l5_utf8","type":"utf8"},{"name":"r0_f64","type":"f64"}],"columns":{"l0_utf8":[null,"zz","😀","é","c"],"l1_i32":[4,5,null,null,2],"l2_i64":["3","9007199254740992",null,"-4611686018427387904",null],"l3_f64":[1.0,null,-3.0,-79.931,-56.128],"l4_utf8":["","😀",null,null,"zz"],"l5_utf8":["aa","c","B",null,"a"],"r0_f64":[null,null,null,null,null]}}},{"id":"random/0390","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_i64","type":"i64"},{"name":"l4_f64","type":"f64"},{"name":"l5_utf8","type":"utf8"},{"name":"l6_utf8","type":"utf8"}],"columns":{"l0_utf8":["B","c","é","ab","",null,"","aa","b","c","a",null,"zz","ab","😀","B","",null,"","😀","é","😀","zz","","😀","a","ab","","ab","B","B",null,"aa","b","B","é",null,"B",null,"é","ab","é","b",null,"😀","B","","😀","aa",null,"B",null,"b","",null,"😀","😀","😀",null,"B"],"l1_i32":[-4,-4,2,2,-4,null,-2,-2147483648,-4,1,-2,2,4,null,null,0,4,null,5,0,null,4,null,0,null,2,5,1,-2,-3,4,2,null,4,-1,null,null,null,1,null,5,2,-5,null,2,null,5,-2,2,1,null,null,-3,2,-2,-4,-2,-3,1,3],"l2_bool":[false,null,false,true,null,true,null,null,true,false,true,null,null,null,false,false,true,false,true,true,null,false,false,true,null,false,false,true,null,true,false,false,true,null,null,true,null,false,true,null,true,null,false,true,true,true,true,false,true,true,false,null,false,false,false,false,false,null,true,false],"l3_i64":["9007199254740994","-4611686018427387904","0","-3","-2",null,"-4611686018427387904","2","-3","9007199254740995","9007199254740992",null,"0",null,"-4611686018427387904","0","1",null,"-4611686018427387904",null,"9007199254740994","1","-4611686018427387904","-4611686018427387904","2","3",null,null,"9007199254740992","-4611686018427387904","-4611686018427387904","-4611686018427387904","9007199254740994","-3","3","9007199254740993",null,"9007199254740993","-5","-4611686018427387904",null,"-4611686018427387904","-5",null,null,null,"-5","3","-4611686018427387904",null,"-4611686018427387904","-4611686018427387904","-2","-4611686018427387904","-4611686018427387904","5","9007199254740992","2",null,"-4611686018427387904"],"l4_f64":[3.0,1.0,-3.0,-2.358,"NaN",45.769,-53.974,23.623,null,2.0,2.0,null,0.0,null,4.0,null,null,-2.0,-96.639,-2.0,-3.0,1.0,-1.0,-4.0,-3.0,-56.952,null,null,10.31,-3.0,94.615,-35.494,-26.213,-2.0,-57.762,null,null,33.267,null,3.0,-4.0,89.094,null,-1.0,-3.0,-63.585,3.0,1.0,23.643,-76.82,1.0,"Infinity",-1.0,null,null,-46.717,-2.0,65.951,null,null],"l5_utf8":["",null,"b","a","😀",null,"😀","ab","zz",null,null,"aa","c","aa",null,"a","ab","zz","😀",null,"b",null,"ab",null,"a",null,"a","aa",null,"c","b",null,null,"zz","ab",null,"b",null,"zz","",null,"é","zz","a",null,"zz","","zz","B","B",null,"zz","zz","ab","B","b","ab",null,"aa","ab"],"l6_utf8":["B","c","c","é",null,null,"é",null,"aa",null,"é",null,null,"c",null,"zz","b","b",null,null,"a","zz","é","B","","😀","é","B","aa","aa","B","c","aa",null,null,"aa","","b","zz","ab","é",null,"B",null,"aa","b","B","zz",null,"é",null,null,"zz",null,null,null,null,"","zz","é"]}}},"ops":[{"op":"head","n":9}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_bool","type":"bool"},{"name":"l3_i64","type":"i64"},{"name":"l4_f64","type":"f64"},{"name":"l5_utf8","type":"utf8"},{"name":"l6_utf8","type":"utf8"}],"columns":{"l0_utf8":["B","c","é","ab","",null,"","aa","b"],"l1_i32":[-4,-4,2,2,-4,null,-2,-2147483648,-4],"l2_bool":[false,null,false,true,null,true,null,null,true],"l3_i64":["9007199254740994","-4611686018427387904","0","-3","-2",null,"-4611686018427387904","2","-3"],"l4_f64":[3.0,1.0,-3.0,-2.358,"NaN",45.769,-53.974,23.623,null],"l5_utf8":["",null,"b","a","😀",null,"😀","ab","zz"],"l6_utf8":["B","c","c","é",null,null,"é",null,"aa"]}}},{"id":"random/0391","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_bool","type":"bool"},{"name":"l5_utf8","type":"utf8"},{"name":"l6_f64","type":"f64"}],"columns":{"l0_utf8":["a","zz","ab","é","😀",null,"zz",null,"b","é","c","zz","",null,"a","b","ab","é","zz","b"],"l1_i32":[-4,-5,-2147483648,4,3,-3,4,-2,-2,-3,132,1,1,0,5,3,0,5,-3,1],"l2_utf8":["",null,"b","B","zz","é","zz","b",null,"","a","a","aa","ab","B","a","","é","B","é"],"l3_utf8":["😀","zz","ab","😀","b","b","b","ab","ab","ab","é","a","B","aa","a","ab","c","é","a",null],"l4_bool":[false,true,true,true,true,true,null,false,false,false,true,true,true,null,false,true,true,false,true,true],"l5_utf8":["aa","a","😀","aa","a",null,"ab","zz","é","c","zz","ab","b","é","c","zz","B","é","😀","c"],"l6_f64":[-2.0,0.0,-1.0,-4.0,null,-2.0,-33.402,-67.905,"NaN",null,0.0,80.515,1.0,-88.39,"Infinity",-3.0,0.0,4.0,-49.545,3.0]}}},"ops":[{"op":"unique"},{"op":"unique","columns":["l1_i32"]}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_bool","type":"bool"},{"name":"l5_utf8","type":"utf8"},{"name":"l6_f64","type":"f64"}],"columns":{"l0_utf8":["a","zz","ab","é","😀",null,null,"c","zz",null,"a"],"l1_i32":[-4,-5,-2147483648,4,3,-3,-2,132,1,0,5],"l2_utf8":["",null,"b","B","zz","é","b","a","a","ab","B"],"l3_utf8":["😀","zz","ab","😀","b","b","ab","é","a","aa","a"],"l4_bool":[false,true,true,true,true,true,false,true,true,null,false],"l5_utf8":["aa","a","😀","aa","a",null,"zz","zz","ab","é","c"],"l6_f64":[-2.0,0.0,-1.0,-4.0,null,-2.0,-67.905,0.0,80.515,-88.39,"Infinity"]}}},{"id":"random/0392","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i64","type":"i64"},{"name":"l3_i64","type":"i64"}],"columns":{"l0_utf8":[""],"l1_i32":[2],"l2_i64":["9007199254740995"],"l3_i64":["2"]}}},"ops":[{"op":"withColumns","exprs":{"x0":{"fn":"isNotNull","args":[{"col":"l2_i64"}]}}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i64","type":"i64"},{"name":"l3_i64","type":"i64"},{"name":"x0","type":"bool"}],"columns":{"l0_utf8":[""],"l1_i32":[2],"l2_i64":["9007199254740995"],"l3_i64":["2"],"x0":[true]}}},{"id":"random/0393","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_i32","type":"i32"},{"name":"l4_i32","type":"i32"}],"columns":{"l0_utf8":["B","B",null,"ab","","a","","","B","c","c","é","B","aa",null,null,null,"😀","aa","a"],"l1_i32":[-3,-3,4,null,-4,null,-527,null,1,-4,5,3,null,3,1,-4,null,null,-3,5],"l2_i32":[-1,0,-3,null,null,null,-4,null,null,0,-2,2,-4,null,5,-5,4,0,null,2147483647],"l3_i32":[5,-2,null,-3,null,-539,-3,null,2,-2147483648,-5,null,null,4,null,1,2,-2,2,-1],"l4_i32":[3,3,-2,null,2,-5,-1,1,null,null,2,-337,3,1,4,null,0,-3,null,-1]}}},"ops":[{"op":"withColumns","exprs":{"x0":{"fn":"mul","args":[{"col":"l4_i32"},{"col":"l3_i32"}]}}},{"op":"head","n":7}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_i32","type":"i32"},{"name":"l4_i32","type":"i32"},{"name":"x0","type":"i32"}],"columns":{"l0_utf8":["B","B",null,"ab","","a",""],"l1_i32":[-3,-3,4,null,-4,null,-527],"l2_i32":[-1,0,-3,null,null,null,-4],"l3_i32":[5,-2,null,-3,null,-539,-3],"l4_i32":[3,3,-2,null,2,-5,-1],"x0":[15,-6,null,null,null,2695,3]}}},{"id":"random/0394","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_utf8","type":"utf8"},{"name":"l3_utf8","type":"utf8"},{"name":"l4_bool","type":"bool"}],"columns":{"l0_utf8":["a","","c","b",""],"l1_i32":[1,-976,-4,-1,3],"l2_utf8":["aa",null,"😀","zz","c"],"l3_utf8":["b","zz","zz","B","a"],"l4_bool":[null,null,true,true,true]}},"right":{"fields":[{"name":"a0_1","type":"utf8"},{"name":"r0_utf8","type":"utf8"}],"columns":{"r0_utf8":[null,"b","😀"],"a0_1":["😀","ab","ab"]}}},"ops":[{"op":"groupBy","keys":["l3_utf8"],"aggs":{"n0":{"agg":"len"},"a0_0":{"agg":"nUnique","arg":{"col":"l0_utf8"}},"a0_1":{"agg":"min","arg":{"col":"l0_utf8"}},"a0_2":{"agg":"first","arg":{"col":"l4_bool"}}}},{"op":"join","right":"right","on":["a0_1"],"how":"left"}],"expected":{"fields":[{"name":"l3_utf8","type":"utf8"},{"name":"n0","type":"u32"},{"name":"a0_0","type":"u32"},{"name":"a0_1","type":"utf8"},{"name":"a0_2","type":"bool"},{"name":"r0_utf8","type":"utf8"}],"columns":{"l3_utf8":["b","zz","B","a"],"n0":[1,2,1,1],"a0_0":[1,2,1,1],"a0_1":["a","","b",""],"a0_2":[null,null,true,true],"r0_utf8":[null,null,null,null]}}},{"id":"random/0395","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_f64","type":"f64"},{"name":"l4_i64","type":"i64"},{"name":"l5_i32","type":"i32"}],"columns":{"l0_utf8":["","😀","zz","a","B"],"l1_i32":[2,-2147483648,2147483647,1,-4],"l2_f64":[-2.0,-82.606,-0.035,-99.118,0.555],"l3_f64":[2.584,-2.0,91.867,69.91,27.93],"l4_i64":["9007199254740994","3","-4","9007199254740992","-3"],"l5_i32":[-5,-5,-3,1,-3]}}},"ops":[{"op":"unique","columns":["l2_f64"]},{"op":"filter","predicate":{"fn":"lte","args":[{"fn":"sub","args":[{"col":"l5_i32"},{"col":"l3_f64"}]},{"fn":"mod","args":[{"col":"l3_f64"},{"col":"l3_f64"}]}]}},{"op":"sort","keys":[{"expr":{"col":"l0_utf8"},"descending":true}],"nullsLast":false},{"op":"filter","predicate":{"fn":"gte","args":[{"fn":"when","args":[{"fn":"eq","args":[{"col":"l4_i64"},{"lit":"1","type":"i64"}]},{"col":"l5_i32"},{"col":"l2_f64"}]},{"fn":"mod","args":[{"lit":0.5},{"lit":0.5}]}]}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_f64","type":"f64"},{"name":"l4_i64","type":"i64"},{"name":"l5_i32","type":"i32"}],"columns":{"l0_utf8":["B"],"l1_i32":[-4],"l2_f64":[0.555],"l3_f64":[27.93],"l4_i64":["-3"],"l5_i32":[-3]}}},{"id":"random/0396","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i64","type":"i64"},{"name":"l3_utf8","type":"utf8"}],"columns":{"l0_utf8":["","a","c","ab","😀"],"l1_i32":[3,4,-3,-1,5],"l2_i64":["-4611686018427387904","-4611686018427387904","9007199254740992","-4611686018427387904","9007199254740992"],"l3_utf8":["aa","zz","c","","ab"]}},"right":{"fields":[{"name":"l2_i64","type":"i64"},{"name":"r0_f64","type":"f64"}],"columns":{"r0_f64":[-2.0,99.674,null,4.0,23.982,"-Infinity",76.323,1.0,-1.0,-95.789],"l2_i64":["-4611686018427387904","9007199254740992","-4611686018427387904","-4611686018427387904","-4611686018427387904","-4611686018427387904","-4611686018427387904","9007199254740992","-4611686018427387904","9007199254740992"]}}},"ops":[{"op":"unique"},{"op":"unique","columns":["l0_utf8"]},{"op":"withColumns","exprs":{"x2":{"fn":"isIn","args":[{"col":"l0_utf8"}],"values":["😀","ab",""]}}},{"op":"join","right":"right","on":["l2_i64"],"how":"inner"}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i64","type":"i64"},{"name":"l3_utf8","type":"utf8"},{"name":"x2","type":"bool"},{"name":"r0_f64","type":"f64"}],"columns":{"l0_utf8":["","","","","","","","a","a","a","a","a","a","a","c","c","c","ab","ab","ab","ab","ab","ab","ab","😀","😀","😀"],"l1_i32":[3,3,3,3,3,3,3,4,4,4,4,4,4,4,-3,-3,-3,-1,-1,-1,-1,-1,-1,-1,5,5,5],"l2_i64":["-4611686018427387904","-4611686018427387904","-4611686018427387904","-4611686018427387904","-4611686018427387904","-4611686018427387904","-4611686018427387904","-4611686018427387904","-4611686018427387904","-4611686018427387904","-4611686018427387904","-4611686018427387904","-4611686018427387904","-4611686018427387904","9007199254740992","9007199254740992","9007199254740992","-4611686018427387904","-4611686018427387904","-4611686018427387904","-4611686018427387904","-4611686018427387904","-4611686018427387904","-4611686018427387904","9007199254740992","9007199254740992","9007199254740992"],"l3_utf8":["aa","aa","aa","aa","aa","aa","aa","zz","zz","zz","zz","zz","zz","zz","c","c","c","","","","","","","","ab","ab","ab"],"x2":[true,true,true,true,true,true,true,false,false,false,false,false,false,false,false,false,false,true,true,true,true,true,true,true,true,true,true],"r0_f64":[-2.0,null,4.0,23.982,"-Infinity",76.323,-1.0,-2.0,null,4.0,23.982,"-Infinity",76.323,-1.0,99.674,1.0,-95.789,-2.0,null,4.0,23.982,"-Infinity",76.323,-1.0,99.674,1.0,-95.789]}}},{"id":"random/0397","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i64","type":"i64"},{"name":"l3_f64","type":"f64"}],"columns":{"l0_utf8":["b"],"l1_i32":[2147483647],"l2_i64":["9007199254740992"],"l3_f64":[-2.0]}}},"ops":[{"op":"sort","keys":[{"expr":{"col":"l0_utf8"},"descending":false},{"expr":{"col":"l2_i64"},"descending":false}],"nullsLast":true},{"op":"sort","keys":[{"expr":{"col":"l3_f64"},"descending":false},{"expr":{"col":"l1_i32"},"descending":true}],"nullsLast":false},{"op":"head","n":7}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i64","type":"i64"},{"name":"l3_f64","type":"f64"}],"columns":{"l0_utf8":["b"],"l1_i32":[2147483647],"l2_i64":["9007199254740992"],"l3_f64":[-2.0]}}},{"id":"random/0398","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_i64","type":"i64"},{"name":"l4_utf8","type":"utf8"}],"columns":{"l0_utf8":["é","ab","zz","ab","é","ab","é","a","é","ab","c","é","c","aa","zz","B","a","b","c","é","","aa",null,"c","aa","zz","B","c","é","é","c",null,"",null,"","😀","B","a","b",null,"c","ab","","zz","","ab","a","é","aa",null,"zz","zz","B","aa","😀","b","c","a","c","B"],"l1_i32":[-4,-4,null,5,2147483647,null,-1,0,-3,-4,-4,-1,0,-4,-3,3,null,-1,-3,2,1,2,2,1,null,793,-3,1,-1,0,5,-1,-1,0,-4,4,5,4,2,1,-1,-3,2,-2,0,-2,-4,null,5,null,-4,-3,-2,-2,-2,5,5,null,null,null],"l2_i32":[3,-5,2,-5,1,2,-2,-4,-3,-5,4,3,4,-2147483648,3,2,4,5,1,1,1,null,2147483647,null,4,5,3,4,-3,1,3,-2,2,null,5,5,4,2147483647,0,0,null,-2,-4,0,null,3,-2147483648,4,2,-2,-5,0,null,4,1,-1,5,-1,5,-1],"l3_i64":["4","2","-4611686018427387904","-4611686018427387904","9007199254740994","9007199254740992","4","2","-4611686018427387904","1","-4611686018427387904","-4611686018427387904","9007199254740995","9007199254740995","9007199254740994","9007199254740992","-4611686018427387904","-1","2","-4611686018427387904","-4611686018427387904","-4611686018427387904","3","-4611686018427387904","-4611686018427387904","9007199254740993","-3","-2","9007199254740995","-4611686018427387904","2","-4611686018427387904","0","9007199254740995","3","-4611686018427387904","-4611686018427387904","0","-4611686018427387904","-4","-4611686018427387904","5","9007199254740993","9007199254740994","2","5","-4611686018427387904","0","-4611686018427387904","-2","-4611686018427387904",null,"-5","9007199254740993","0","-4611686018427387904",null,"-4611686018427387904","-4","-1"],"l4_utf8":["a","é","a","",null,"ab","a","zz","","b","😀",null,"😀","B","B","ab","😀","B","ab","B","a",null,"b","B","a","b","b","b","b","b","a","zz","é","é","ab","b","","é","😀","b","aa","b","é","zz","é","B","ab","b","c","c","","b","c","ab","c","ab","😀","","","aa"]}}},"ops":[{"op":"filter","predicate":{"fn":"lt","args":[{"fn":"mod","args":[{"col":"l2_i32"},{"col":"l2_i32"}]},{"fn":"div","args":[{"lit":-1},{"col":"l1_i32"}]}]}},{"op":"head","n":5}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_i32","type":"i32"},{"name":"l3_i64","type":"i64"},{"name":"l4_utf8","type":"utf8"}],"columns":{"l0_utf8":["é","ab","é","é","ab"],"l1_i32":[-4,-4,-1,-3,-4],"l2_i32":[3,-5,-2,-3,-5],"l3_i64":["4","2","4","-4611686018427387904","1"],"l4_utf8":["a","é","a","","b"]}}},{"id":"random/0399","description":"seeded random pipeline","inputs":{"left":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_utf8","type":"utf8"}],"columns":{"l0_utf8":["a",null,"ab",null,null,"a",null,"b",null,"😀","zz","aa",null,"zz","ab","c",null,"aa",null,null],"l1_i32":[null,null,null,-3,null,4,3,null,null,-4,null,null,null,null,null,1,null,4,-3,null],"l2_f64":[48.242,null,null,-39.956,4.0,73.933,null,82.848,-4.0,null,-30.608,-4.0,-3.0,null,null,null,null,1.0,-33.925,null],"l3_utf8":["ab",null,"","a","aa",null,null,"é",null,"😀","ab","zz","","ab",null,"zz",null,"é","😀","B"]}}},"ops":[{"op":"head","n":6},{"op":"filter","predicate":{"fn":"eq","args":[{"col":"l2_f64"},{"fn":"sub","args":[{"col":"l1_i32"},{"col":"l2_f64"}]}]}}],"expected":{"fields":[{"name":"l0_utf8","type":"utf8"},{"name":"l1_i32","type":"i32"},{"name":"l2_f64","type":"f64"},{"name":"l3_utf8","type":"utf8"}],"columns":{"l0_utf8":[],"l1_i32":[],"l2_f64":[],"l3_utf8":[]}}}]} diff --git a/packages/effect/test/fixtures/dataframe/sql_oracle.py b/packages/effect/test/fixtures/dataframe/sql_oracle.py deleted file mode 100644 index e98544408c0..00000000000 --- a/packages/effect/test/fixtures/dataframe/sql_oracle.py +++ /dev/null @@ -1,133 +0,0 @@ -""" -Cross-checks the SQL engine: runs every gold plan, compiled to DuckDB SQL by -compile_sql.ts, in DuckDB over the gold inputs, and compares the rows with -the Polars result. - - node test/fixtures/dataframe/compile_sql.ts duckdb - python test/fixtures/dataframe/sql_oracle.py - -Types are not compared (SQL engines widen integers and counts); values are, -in row order, floats with a relative tolerance. Prints a summary and writes -.sql/sql-duckdb-report.json (git-ignored) with every difference. -""" - -import json -import math -import sys -from decimal import Decimal -from pathlib import Path - -import duckdb - -sys.path.insert(0, str(Path(__file__).parent)) -import generate_gold as g # noqa: E402 - -HERE = Path(__file__).parent - - -def load(name): - return json.loads((HERE / "gold" / f"{name}.json").read_text())["cases"] - - -def same(a, b): - if a is None or b is None: - return a is None and b is None - if isinstance(b, list): - return isinstance(a, list) and len(a) == len(b) and all(same(x, y) for x, y in zip(a, b)) - if isinstance(b, dict): - return isinstance(a, dict) and all(same(a.get(k), v) for k, v in b.items()) - if isinstance(b, bool) or isinstance(a, bool): - return bool(a) == bool(b) and type(a) is type(b) - if isinstance(b, (int, float, Decimal)) and isinstance(a, (int, float, Decimal)): - a, b = float(a), float(b) - if math.isnan(a) or math.isnan(b): - return math.isnan(a) and math.isnan(b) - if a == b: - return True - return math.isfinite(a) and math.isfinite(b) and abs(a - b) <= 1e-9 * max(abs(a), abs(b)) - return a == b - - -def expected_rows(table): - names = [f["name"] for f in table["fields"]] - types = {f["name"]: f["type"] for f in table["fields"]} - n = len(table["columns"][names[0]]) if names else 0 - rows = [] - for i in range(n): - row = {} - for name in names: - v = table["columns"][name][i] - t = types[name] - row[name] = g.decode_value(v, t) if t not in g.TEMPORAL else v - rows.append(row) - return names, types, rows - - -def temporal_int(v, t): - """Gold temporal values are epoch integers; DuckDB returns Python objects.""" - import datetime as dt - if v is None: - return None - if t == "date": - return (v - dt.date(1970, 1, 1)).days - if t.startswith("datetime"): - us = int((v - dt.datetime(1970, 1, 1)) / dt.timedelta(microseconds=1)) - return us if t.endswith("[us]") else us // 1000 - if t.startswith("duration"): - us = int(v / dt.timedelta(microseconds=1)) - return us if t.endswith("[us]") else us // 1000 - return v - - -def run_case(case, compiled): - con = duckdb.connect() - con.execute("SET threads = 1") - con.execute("SET preserve_insertion_order = true") - for name, table in case["inputs"].items(): - df = g.to_polars(table) - con.register(f"_{name}", df) - con.execute(f'CREATE TABLE "{name}" AS SELECT * FROM "_{name}"') - result = con.execute(compiled["sql"]) - columns = [d[0] for d in result.description] - got = [dict(zip(columns, row)) for row in result.fetchall()] - names, types, want = expected_rows(case["expected"]) - if columns != names: - return f"columns {columns}, expected {names}" - if len(got) != len(want): - return f"{len(got)} rows, expected {len(want)}" - for i, (a, b) in enumerate(zip(got, want)): - for name in names: - x = a[name] - if types[name] in g.TEMPORAL: - x = temporal_int(x, types[name]) - if not same(x, b[name]): - return f"row {i} column {name}: got {x!r}, expected {b[name]!r}" - return None - - -def main(): - cases = {c["id"]: c for name in ("spec", "random", "extended") for c in load(name)} - compiled = json.loads((HERE / ".sql" / "sql-duckdb.json").read_text()) - report = {"pass": [], "fail": {}, "unsupported": {}, "error": {}} - for c in compiled: - if "error" in c: - report["unsupported"][c["id"]] = c["error"] - continue - try: - diff = run_case(cases[c["id"]], c) - except Exception as e: # DuckDB rejected the SQL or raised at run time - report["error"][c["id"]] = f"{type(e).__name__}: {str(e)[:300]}" - continue - if diff is None: - report["pass"].append(c["id"]) - else: - report["fail"][c["id"]] = diff - (HERE / ".sql" / "sql-duckdb-report.json").write_text(json.dumps(report, indent=1) + "\n") - print( - f"duckdb {duckdb.__version__}: {len(report['pass'])} pass, {len(report['fail'])} differ, " - f"{len(report['error'])} errors, {len(report['unsupported'])} unsupported" - ) - - -if __name__ == "__main__": - main() From 971ace212fe8c81de40ddda1220fc30bb0ccb33a Mon Sep 17 00:00:00 2001 From: Effect Bot Date: Sun, 15 Nov 2026 11:47:13 +0000 Subject: [PATCH 03/88] Check table columns against their schema structurally --- packages/effect/src/dataframe/DataFrame.ts | 57 ++- packages/effect/src/dataframe/Plan.ts | 42 ++- packages/effect/src/dataframe/Query.ts | 12 +- .../effect/src/dataframe/internal/query.ts | 2 +- .../effect/src/dataframe/internal/schema.ts | 326 ++++++++++++++---- .../effect/src/dataframe/internal/stream.ts | 2 +- .../effect/test/dataframe/DataFrame.test.ts | 55 ++- packages/effect/test/dataframe/Query.test.ts | 6 + packages/effect/test/dataframe/oracle/Plan.ts | 24 +- packages/effect/typetest/DataFrame.tst.ts | 21 +- 10 files changed, 423 insertions(+), 124 deletions(-) diff --git a/packages/effect/src/dataframe/DataFrame.ts b/packages/effect/src/dataframe/DataFrame.ts index 6d46b9628d2..0ec791a653f 100644 --- a/packages/effect/src/dataframe/DataFrame.ts +++ b/packages/effect/src/dataframe/DataFrame.ts @@ -41,6 +41,7 @@ import * as ArrowIpc from "../arrow/ArrowIpc.ts" import * as ArrowSchema from "../arrow/ArrowSchema.ts" import * as ArrowTable from "../arrow/ArrowTable.ts" +import type * as BigDecimal from "../BigDecimal.ts" import * as Data from "../Data.ts" import * as DateTime from "../DateTime.ts" import * as Duration from "../Duration.ts" @@ -95,8 +96,28 @@ export interface DataFrame extends Pipeable, Inspectable, Equal.Equal, */ export type Row = F extends DataFrame ? R : never +/** + * The rows a frame holds for rows of type `T`: optional fields become + * required and `undefined` becomes `null`, in nested structs and lists + * too, as frames store a missing value as `null`. + * + * @stability experimental + * @category type-level + * @since 4.1.5 + */ +export type Normalize = { + readonly [K in keyof T]-?: NormalizeValue | ({} extends Pick ? null : never) +} + type Simplify = { readonly [K in keyof A]: A[K] } extends infer B ? B : never +type NormalizeValue = A extends undefined ? null + : A extends BigDecimal.BigDecimal | DateTime.DateTime | Duration.Duration | Uint8Array ? A + : A extends Array ? Array> + : A extends ReadonlyArray ? ReadonlyArray> + : A extends object ? Normalize + : A + /** * Raised for invalid input that the types can't rule out: a row that doesn't * match its schema, an Arrow table whose columns don't match a schema, or a @@ -142,11 +163,15 @@ export const isDataFrame = (u: unknown): u is DataFrame => F.isFrame(u) * from the schema's type side: `String` is Utf8, `Number` is Float64, * `Boolean` is Bool, `BigInt` is Int64, and `NullOr`/optional fields are * nullable. `options.types` overrides the Arrow type of individual columns - * (for example `ArrowSchema.int32`). + * (for example `ArrowSchema.int32`); `BigDecimal` fields need one, with + * the decimal's precision and scale. * * **Details** * - * Throws a {@link DataFrameError} when a value doesn't fit its column. + * Missing values read back as `null` (see {@link Normalize}). + * + * Throws a {@link DataFrameError} when a value doesn't fit its column, or an + * override can't hold its field's values. * * @stability experimental * @category constructors @@ -156,7 +181,7 @@ export const fromRows = schema: S, rows: Iterable, options?: { readonly types?: { readonly [K in keyof S["Type"]]?: ArrowSchema.DataType } | undefined } -): DataFrame> => +): DataFrame> => unsafe(() => { const arrow = Schema_.toArrowSchema(schema, options?.types as any) const table = Result.getOrElse( @@ -333,20 +358,31 @@ export const decodeArrowStream = ( ) /** - * An empty frame with the columns of a struct schema. + * An empty frame with the columns of a struct schema, typed as in + * {@link fromRows}, which also describes `options.types`. * * @stability experimental * @category constructors * @since 4.1.5 */ export const empty = ( - schema: S -): DataFrame> => fromRows(schema, []) + schema: S, + options?: { readonly types?: { readonly [K in keyof S["Type"]]?: ArrowSchema.DataType } | undefined } +): DataFrame> => fromRows(schema, [], options) /** * Checks that a frame has the columns of a struct schema, with compatible * types, and returns it typed (with only the schema's columns, in its - * order). Integer columns are accepted for `Number` fields. + * order). + * + * **Details** + * + * A column is compatible when its values read back as the field's: `Number` + * takes floats and integers below 64 bits, `BigInt` takes 64-bit integers, + * `BigDecimal` takes decimals of any precision and scale, `DateTimeUtc` and + * `Duration` take timestamps and durations in any unit, and structs and + * arrays need compatible fields and items. Values of fields the schema + * constrains further (literals, enums, refinements) are checked against it. * * @stability experimental * @category constructors @@ -355,15 +391,14 @@ export const empty = ( export const validate: { ( schema: S - ): (self: DataFrame) => Result.Result>, DataFrameError> + ): (self: DataFrame) => Result.Result>, DataFrameError> ( self: DataFrame, schema: S - ): Result.Result>, DataFrameError> + ): Result.Result>, DataFrameError> } = dual(2, (self: DataFrame, schema: { readonly ast: any }) => { try { - const columns = Schema_.tableColumns(Schema_.toArrowSchema(schema, undefined)) - return Result.succeed(wrap(Schema_.conform(frame(self), columns))) + return Result.succeed(wrap(Schema_.conform(frame(self), Schema_.columns(schema), schema))) } catch (cause) { return Result.fail(fromDefect(cause)) } diff --git a/packages/effect/src/dataframe/Plan.ts b/packages/effect/src/dataframe/Plan.ts index 09cbeeef283..df20b2c324c 100644 --- a/packages/effect/src/dataframe/Plan.ts +++ b/packages/effect/src/dataframe/Plan.ts @@ -667,12 +667,20 @@ export const TypeName = Schema.Literals([ "Time", "Timestamp", "Duration", - "Decimal" + "Decimal", + "List", + "Struct" ]) /** * A column a {@link Table} is expected to have. * + * **Details** + * + * `fields` describes the children of a nested column: the single `item` of + * a `List`, or the fields of a `Struct`. Without it, a nested column's + * children are not checked. + * * @stability experimental * @category models * @since 4.1.5 @@ -681,6 +689,7 @@ export interface TableColumn { readonly name: string readonly type: TypeName readonly nullable: boolean + readonly fields?: ReadonlyArray | undefined } /** @@ -688,10 +697,21 @@ export interface TableColumn { * * **Details** * - * When `columns` is given, the table must have these columns, with types of - * the same kind (any integer width for an integer column, and integers below - * 64 bits for a float column), and no nulls in non-nullable ones; the input - * is narrowed to these columns, in this order. + * When `columns` is given, the table must have these columns and no nulls + * in non-nullable ones; the input is narrowed to these columns, in this + * order. A column's type must hold the declared values: integers below 64 + * bits for floats and narrower integers, any 64-bit integer for `Int64` + * and `UInt64`, any decimal for `Decimal`, the same + * temporal type in any unit, and lists and structs with compatible children. + * + * `schema`, set by `Query.table`, is the struct schema the rows must + * satisfy: values of fields it constrains beyond their column type (literals, + * enums, refinements) are checked against it. + * + * **Gotchas** + * + * `schema` is not part of the JSON form: a plan decoded from JSON checks + * column types only. * * @stability experimental * @category models @@ -701,6 +721,7 @@ export interface Table { readonly _tag: "Table" readonly name: string readonly columns?: ReadonlyArray | undefined + readonly schema?: Schema.Top | undefined } /** @@ -1002,6 +1023,13 @@ export interface Unnest { readonly column: string } +const TableColumnSchema: Schema.Codec = Schema.Struct({ + name: Schema.String, + type: TypeName, + nullable: Schema.Boolean, + fields: Schema.optional(Schema.Array(Schema.suspend(() => TableColumnSchema))) +}) + const NodeSchema = Schema.suspend((): Schema.Codec => NodeUnion) const Columns = Schema.Array(Schema.String) const NonNegativeInt = Schema.Int.check(Schema.isGreaterThanOrEqualTo(0)) @@ -1011,9 +1039,7 @@ const NodeUnion = Schema.Union([ Schema.Struct({ _tag: Schema.tag("Table"), name: Schema.String, - columns: Schema.optional( - Schema.Array(Schema.Struct({ name: Schema.String, type: TypeName, nullable: Schema.Boolean })) - ) + columns: Schema.optional(Schema.Array(TableColumnSchema)) }), Schema.Struct({ _tag: Schema.tag("Filter"), input: NodeSchema, predicate: ExpressionSchema }), Schema.Struct({ _tag: Schema.tag("Select"), input: NodeSchema, columns: Schema.Array(NamedSchema) }), diff --git a/packages/effect/src/dataframe/Query.ts b/packages/effect/src/dataframe/Query.ts index b67867e9b15..4e9d2b8717c 100644 --- a/packages/effect/src/dataframe/Query.ts +++ b/packages/effect/src/dataframe/Query.ts @@ -73,9 +73,10 @@ import * as Effect from "../Effect.ts" import * as Effectable from "../Effectable.ts" import { dual } from "../Function.ts" import { NodeInspectSymbol } from "../Inspectable.ts" +import type * as Schema from "../Schema.ts" import * as Stream from "../Stream.ts" import type * as Types from "../Types.ts" -import type { DataFrame, DataFrameError } from "./DataFrame.ts" +import type { DataFrame, DataFrameError, Normalize } from "./DataFrame.ts" import { Engine } from "./Engine.ts" import type * as Expr from "./Expr.ts" import * as E from "./internal/expr.ts" @@ -232,8 +233,9 @@ export const from = (frame: DataFrame): Query => make({ _tag: "Values", /** * A query over the table called `name`, provided by the `Catalog` when - * the query runs. The table must have the columns of the struct schema (see - * `DataFrame.validate`); the query sees only those, in the schema's order. + * the query runs. The table must have the columns of the struct schema, and + * its values must satisfy it, as with `DataFrame.validate`; the query sees + * only those columns, in the schema's order. * * @stability experimental * @category constructors @@ -242,8 +244,8 @@ export const from = (frame: DataFrame): Query => make({ _tag: "Values", export const table = ( name: string, schema: S -): Query> => - make({ _tag: "Table", name, columns: Schema_.tableColumns(Schema_.toArrowSchema(schema, undefined)) }) +): Query> => + make({ _tag: "Table", name, columns: Schema_.columns(schema), schema: schema as unknown as Schema.Top }) /** * A query from its plan, with untyped rows. Decode untrusted plans with diff --git a/packages/effect/src/dataframe/internal/query.ts b/packages/effect/src/dataframe/internal/query.ts index 30e3fdcabe6..fd814ee0420 100644 --- a/packages/effect/src/dataframe/internal/query.ts +++ b/packages/effect/src/dataframe/internal/query.ts @@ -98,7 +98,7 @@ export const resolve = (plan: Plan.Node, catalog: Catalog.Service): Effect.Effec switch (node._tag) { case "Table": { const frame = tables.get(node.name)! - return values(node.columns === undefined ? frame : conform(frame, node.columns)) + return values(node.columns === undefined ? frame : conform(frame, node.columns, node.schema)) } case "Values": return node diff --git a/packages/effect/src/dataframe/internal/schema.ts b/packages/effect/src/dataframe/internal/schema.ts index 6d7b7c55853..7d496728bb7 100644 --- a/packages/effect/src/dataframe/internal/schema.ts +++ b/packages/effect/src/dataframe/internal/schema.ts @@ -1,102 +1,94 @@ /** - * Arrow types for the fields of an Effect Schema struct. + * Table columns for the fields of an Effect Schema struct: their Arrow types, + * and the checks a frame must pass to be read as rows of the schema. * * @internal */ import * as S from "../../arrow/ArrowSchema.ts" import * as AST from "../../SchemaAST.ts" +import * as Parser from "../../SchemaParser.ts" import type * as Plan from "../Plan.ts" import * as D from "./dtype.ts" import * as F from "./frame.ts" import * as V from "./vector.ts" -interface Mapped { - readonly type: S.DataType - readonly nullable: boolean -} +type Column = Omit + +const leaf = (type: Plan.TypeName, nullable = false): Column => ({ type, nullable }) -const literalType = (literal: unknown): S.DataType | undefined => { +const literalType = (literal: unknown): Plan.TypeName | undefined => { switch (typeof literal) { case "string": - return S.utf8 + return "Utf8" case "number": - return S.float64 + return "Float64" case "boolean": - return S.bool + return "Bool" case "bigint": - return S.int64 + return "Int64" default: return undefined } } -const map = (ast: AST.AST, path: string): Mapped => { +const sameShape = (a: Column, b: Column): boolean => + a.type === b.type && JSON.stringify(a.fields ?? []) === JSON.stringify(b.fields ?? []) + +const columnOf = (ast: AST.AST, path: string): Column => { switch (ast._tag) { case "String": case "TemplateLiteral": - return { type: S.utf8, nullable: false } + return leaf("Utf8") case "Number": - return { type: S.float64, nullable: false } + return leaf("Float64") case "Boolean": - return { type: S.bool, nullable: false } + return leaf("Bool") case "BigInt": - return { type: S.int64, nullable: false } + return leaf("Int64") case "Null": case "Undefined": - return { type: S.nullType, nullable: true } + return leaf("Null", true) case "Literal": { const type = literalType(ast.literal) - return type === undefined ? unsupported(ast, path) : { type, nullable: false } + return type === undefined ? unsupported(ast, path) : leaf(type) } case "Enum": { const kinds = new Set(ast.enums.map(([, v]) => typeof v)) return kinds.size === 1 && kinds.has("string") - ? { type: S.utf8, nullable: false } + ? leaf("Utf8") : kinds.size === 1 - ? { type: S.float64, nullable: false } + ? leaf("Float64") : unsupported(ast, path) } case "Union": { - let type: S.DataType | undefined + let column: Column | undefined let nullable = false for (const member of ast.types) { - const m = map(member, path) + const m = columnOf(member, path) nullable ||= m.nullable - if (m.type._tag === "Null") continue - if (type === undefined) type = m.type - else if (type._tag !== m.type._tag) return unsupported(ast, path) + if (m.type === "Null") continue + if (column === undefined) column = m + else if (!sameShape(column, m)) return unsupported(ast, path) } - return { type: type ?? S.nullType, nullable } + return { ...(column ?? leaf("Null")), nullable } } case "Suspend": - return map(ast.thunk(), path) + return columnOf(ast.thunk(), path) case "Arrays": { const element = ast.rest[0] if (ast.elements.length > 0 || element === undefined || ast.rest.length > 1) return unsupported(ast, path) - const m = map(element, `${path}[]`) - return { type: S.list(S.field("item", m.type, { nullable: m.nullable })), nullable: false } - } - case "Objects": { - if (ast.indexSignatures.length > 0) return unsupported(ast, path) - const fields = ast.propertySignatures.map((ps) => { - const name = String(ps.name) - const m = map(ps.type, `${path}.${name}`) - return S.field(name, m.type, { nullable: m.nullable || AST.isOptional(ps.type) }) - }) - return { type: S.struct(fields), nullable: false } + return { type: "List", nullable: false, fields: [{ name: "item", ...columnOf(element, `${path}[]`) }] } } + case "Objects": + return ast.indexSignatures.length > 0 + ? unsupported(ast, path) + : { type: "Struct", nullable: false, fields: fieldsOf(ast, path) } case "Declaration": { const id = (ast.annotations as { readonly representation?: { readonly id?: unknown } } | undefined) ?.representation?.id - if (id === "effect/schema/DateTimeUtc") return { type: S.timestamp("microsecond"), nullable: false } - if (id === "effect/schema/Duration") return { type: S.duration("microsecond"), nullable: false } - if (id === "effect/schema/BigDecimal") { - return V.fail( - `field ${ - JSON.stringify(path) - }: decimal columns need a precision and scale: pass options.types (for example ArrowSchema.decimal(18, 2))` - ) - } + if (id === "effect/schema/DateTimeUtc") return leaf("Timestamp") + if (id === "effect/schema/Duration") return leaf("Duration") + if (id === "effect/schema/BigDecimal") return leaf("Decimal") return unsupported(ast, path) } default: @@ -107,23 +99,83 @@ const map = (ast: AST.AST, path: string): Mapped => { const unsupported = (ast: AST.AST, path: string): never => V.fail(`field ${JSON.stringify(path)}: schemas of kind ${ast._tag} have no DataFrame column type yet`) -/** The Arrow schema of a struct schema's type side. @internal */ +const fieldsOf = (ast: AST.Objects, prefix: string | undefined): Array => + ast.propertySignatures.map((ps) => { + const name = String(ps.name) + const column = columnOf(ps.type, prefix === undefined ? name : `${prefix}.${name}`) + return { name, ...column, nullable: column.nullable || AST.isOptional(ps.type) } + }) + +const structOf = (schema: { readonly ast: AST.AST }): AST.Objects => { + const ast = AST.toType(schema.ast) + return ast._tag === "Objects" && ast.indexSignatures.length === 0 + ? ast + : V.fail("DataFrame rows must be described by a struct schema") +} + +/** The table columns of a struct schema's type side. @internal */ +export const columns = (schema: { readonly ast: AST.AST }): Array => + fieldsOf(structOf(schema), undefined) + +const defaultType = (column: Plan.TableColumn, path: string): S.DataType => { + switch (column.type) { + case "List": { + const item = column.fields![0] + return S.list(S.field("item", defaultType(item, `${path}[]`), { nullable: item.nullable })) + } + case "Struct": + return S.struct( + column.fields!.map((f) => S.field(f.name, defaultType(f, `${path}.${f.name}`), { nullable: f.nullable })) + ) + case "Utf8": + return S.utf8 + case "Float64": + return S.float64 + case "Bool": + return S.bool + case "Int64": + return S.int64 + case "Null": + return S.nullType + case "Timestamp": + return S.timestamp("microsecond") + case "Duration": + return S.duration("microsecond") + case "Decimal": + return V.fail( + `field ${ + JSON.stringify(path) + }: decimal columns need a precision and scale: pass options.types (for example ArrowSchema.decimal(18, 2))` + ) + default: + return V.fail(`field ${JSON.stringify(path)}: no default Arrow type for ${column.type}`) + } +} + +/** + * The Arrow schema of a struct schema's type side, with `overrides` for the + * types of some of its columns. + * + * @internal + */ export const toArrowSchema = ( schema: { readonly ast: AST.AST }, overrides: { readonly [name: string]: S.DataType | undefined } | undefined -): S.Schema => { - const ast = AST.toType(schema.ast) - if (ast._tag !== "Objects") return V.fail("DataFrame rows must be described by a struct schema") - return S.make(ast.propertySignatures.map((ps) => { - const name = String(ps.name) - const m = map(ps.type, name) - const type = overrides?.[name] ?? m.type - return S.field(name, type, { nullable: m.nullable || AST.isOptional(ps.type) }) - })) -} +): S.Schema => + S.make( + columns(schema).map((column) => { + const override = overrides?.[column.name] + if (override !== undefined && !compatible(override, column)) { + return V.fail( + `field ${JSON.stringify(column.name)}: ${describe(override)} can't hold values of ${describeColumn(column)}` + ) + } + return S.field(column.name, override ?? defaultType(column, column.name), { nullable: column.nullable }) + }) + ) // ----------------------------------------------------------------------------- -// Types of table columns +// Checking frames // ----------------------------------------------------------------------------- /** The plan type name of an Arrow type. @internal */ @@ -141,37 +193,161 @@ export const typeName = (t: S.DataType): Plan.TypeName => { case "BinaryView": case "FixedSizeBinary": return "Binary" + case "LargeList": + return "List" default: return t._tag as Plan.TypeName } } -const kind = (name: string): string => - name.startsWith("Int") || name.startsWith("UInt") ? "Int" : name.startsWith("Float") ? "Float" : name +const describe = (t: S.DataType): string => { + const name = typeName(t) + if (t._tag === "List" || t._tag === "LargeList") return `List<${describe(t.child.type)}>` + return name +} + +const describeColumn = (c: Column): string => c.type === "List" ? `List<${describeColumn(c.fields![0])}>` : c.type + +/** + * Whether a column of Arrow type `t` reads as values of the declared + * column: integers below 64 bits for floats and narrower integers, 64-bit + * integers (bigints) for 64-bit ones, any decimal for a decimal, temporal + * types of the same kind in any unit, and lists and structs whose children + * are compatible (unless `shallow`). + */ +const compatible = (t: S.DataType, expected: Column, shallow = false): boolean => { + const actual = typeName(t) + switch (expected.type) { + case "Float16": + case "Float32": + case "Float64": + return t._tag === "Float" || (t._tag === "Int" && t.bitWidth < 64) + case "Int8": + case "Int16": + case "Int32": + case "UInt8": + case "UInt16": + case "UInt32": + return t._tag === "Int" && t.bitWidth < 64 + case "Int64": + case "UInt64": + return t._tag === "Int" && t.bitWidth === 64 + case "List": + return (t._tag === "List" || t._tag === "LargeList") && + (shallow || expected.fields === undefined || compatible(t.child.type, expected.fields[0])) + case "Struct": + return t._tag === "Struct" && + (shallow || (expected.fields ?? []).every((f) => { + const field = t.fields.find((g) => g.name === f.name) + return field !== undefined && compatible(field.type, f) + })) + default: + return actual === expected.type + } +} + +const allNull = (v: V.Vector, rows: Uint8Array | undefined): boolean => { + for (let i = 0; i < v.length; i++) if ((rows === undefined || rows[i] === 1) && V.isValid(v, i)) return false + return true +} + +const check = (path: string, v: V.Vector, expected: Column, rows: Uint8Array | undefined): void => { + if (!compatible(v.type, expected, true) && !(expected.type === "Null" && allNull(v, rows))) { + return V.fail(`column ${JSON.stringify(path)} is ${describe(v.type)}, expected ${describeColumn(expected)}`) + } + if (!expected.nullable) { + for (let i = 0; i < v.length; i++) { + if ((rows === undefined || rows[i] === 1) && !V.isValid(v, i)) { + return V.fail(`column ${JSON.stringify(path)} has missing values`) + } + } + } + if (expected.fields === undefined) return + if (v.kind === "struct" && v.type._tag === "Struct") { + const present = new Uint8Array(v.length) + for (let i = 0; i < v.length; i++) present[i] = (rows === undefined || rows[i] === 1) && V.isValid(v, i) ? 1 : 0 + for (const f of expected.fields) { + const k = v.type.fields.findIndex((g) => g.name === f.name) + if (k === -1) return V.fail(`missing column ${JSON.stringify(`${path}.${f.name}`)}`) + check(`${path}.${f.name}`, v.children[k], f, present) + } + } else if (v.kind === "list") { + const present = new Uint8Array(v.child.length) + for (let i = 0; i < v.length; i++) { + if ((rows === undefined || rows[i] === 1) && V.isValid(v, i)) present.fill(1, v.offsets[i], v.offsets[i + 1]) + } + check(`${path}[]`, v.child, expected.fields[0], present) + } +} + +const refined = (ast: AST.AST): boolean => + ast.checks !== undefined || ast._tag === "Literal" || ast._tag === "Enum" || ast._tag === "TemplateLiteral" -/** Columns of an Effect-Schema-derived Arrow schema, as table columns. @internal */ -export const tableColumns = (schema: S.Schema): Array => - schema.fields.map((f) => ({ name: f.name, type: typeName(f.type), nullable: f.nullable })) +/** + * A test for the values of a field whose schema constrains them beyond its + * column type (literals, enums, template literals and refinements), or + * `undefined` when the column type says it all. Nulls pass: nullability is + * checked with the column. + */ +const valuesCheck = (ast: AST.AST): ((value: unknown) => boolean) | undefined => { + if (refined(ast)) { + const is = Parser._is(ast) + return (value) => value === null || is(value) + } + switch (ast._tag) { + case "Suspend": + return valuesCheck(ast.thunk()) + case "Union": { + const members = ast.types.filter((t) => t._tag !== "Null" && t._tag !== "Undefined") + if (members.length === 1) return valuesCheck(members[0]) + if (!members.some(refined)) return undefined + const is = Parser._is(ast) + return (value) => value === null || is(value) + } + case "Arrays": { + const item = valuesCheck(ast.rest[0]) + return item && ((value) => value === null || (value as ReadonlyArray).every(item)) + } + case "Objects": { + const fields = ast.propertySignatures.flatMap((ps) => { + const f = valuesCheck(ps.type) + return f === undefined ? [] : [[ps.name, f] as const] + }) + return fields.length === 0 + ? undefined + : (value) => value === null || fields.every(([name, f]) => f((value as any)[name])) + } + default: + return undefined + } +} /** - * Checks that a frame has the expected columns and narrows it to them. + * Checks that a frame has the expected columns and narrows it to them. With + * a `schema`, the values of fields it constrains beyond their column type are + * checked against it too. * * @internal */ -export const conform = (frame: F.Frame, columns: ReadonlyArray): F.Frame => { +export const conform = ( + frame: F.Frame, + columns: ReadonlyArray, + schema?: { readonly ast: AST.AST } | undefined +): F.Frame => { for (const expected of columns) { const c = frame.columns.find((c) => c.name === expected.name) if (c === undefined) return V.fail(`missing column ${JSON.stringify(expected.name)}`) - const actual = typeName(c.type) - const compatible = kind(actual) === kind(expected.type) || - (kind(expected.type) === "Float" && c.type._tag === "Int" && c.type.bitWidth < 64) || - (expected.type === "Null" && expected.nullable) - if (!compatible) { - return V.fail(`column ${JSON.stringify(expected.name)} is ${actual}, expected ${expected.type}`) - } - if (!expected.nullable && V.nullCount(c.values()) > 0) { - return V.fail(`column ${JSON.stringify(expected.name)} has missing values`) + check(expected.name, c.values(), expected, undefined) + } + const narrowed = F.pick(frame, columns.map((c) => c.name)) + if (schema !== undefined) { + for (const ps of structOf(schema).propertySignatures) { + const test = valuesCheck(ps.type) + const name = String(ps.name) + if (test !== undefined && !F.toArray(narrowed, name).every(test)) { + return V.fail(`column ${JSON.stringify(name)} has values its schema rejects`) + } } } - return F.pick(frame, columns.map((c) => c.name)) + return narrowed } diff --git a/packages/effect/src/dataframe/internal/stream.ts b/packages/effect/src/dataframe/internal/stream.ts index 62cb0467f5c..f8e3c2a6f03 100644 --- a/packages/effect/src/dataframe/internal/stream.ts +++ b/packages/effect/src/dataframe/internal/stream.ts @@ -190,7 +190,7 @@ export const batches = (node: Q.Node, catalog: Catalog.Service): Batches => { return Stream.suspend(() => catalog.stream(node.name)).pipe( Stream.mapEffect((frame) => { const batch = frame as unknown as F.Frame - return Q.attempt(() => (node.columns === undefined ? batch : conform(batch, node.columns))) + return Q.attempt(() => (node.columns === undefined ? batch : conform(batch, node.columns, node.schema))) }) ) case "Values": diff --git a/packages/effect/test/dataframe/DataFrame.test.ts b/packages/effect/test/dataframe/DataFrame.test.ts index b17ce543086..386b53a36b3 100644 --- a/packages/effect/test/dataframe/DataFrame.test.ts +++ b/packages/effect/test/dataframe/DataFrame.test.ts @@ -1,5 +1,5 @@ import { assert, describe, it } from "@effect/vitest" -import { Effect, Equal, Result, Schema, Stream } from "effect" +import { BigDecimal, Effect, Equal, Result, Schema, Stream } from "effect" import { ArrowIpc, ArrowSchema, ArrowTable } from "effect/arrow" import { DataFrame, Expr, Query } from "effect/dataframe" @@ -38,6 +38,24 @@ describe("DataFrame", () => { assert.deepStrictEqual(df.schema.fields[0].type, ArrowSchema.int32) }) + it("fromRows builds decimal columns from options.types", () => { + const Price = Schema.Struct({ n: Schema.BigDecimal }) + const df = DataFrame.fromRows(Price, [{ n: BigDecimal.make(123n, 2) }], { + types: { n: ArrowSchema.decimal(18, 2) } + }) + assert.deepStrictEqual(df.schema.fields[0].type, ArrowSchema.decimal(18, 2)) + assert.isTrue(BigDecimal.equals(DataFrame.toRows(df)[0].n, BigDecimal.make(123n, 2))) + assert.strictEqual(DataFrame.empty(Price, { types: { n: ArrowSchema.decimal(9, 0) } }).length, 0) + assert.isTrue(Result.isSuccess(DataFrame.validate(df, Price))) + }) + + it("fromRows rejects overrides that can't hold the field's values", () => { + assert.throws( + () => DataFrame.fromRows(Schema.Struct({ n: Schema.Number }), [], { types: { n: ArrowSchema.int64 } }), + `field "n": Int64 can't hold values of Float64` + ) + }) + it("fromRows rejects schemas without a column type", () => { try { DataFrame.fromRows(Schema.Struct({ xs: Schema.Record(Schema.String, Schema.Number) }), []) @@ -73,14 +91,35 @@ describe("DataFrame", () => { assert.deepStrictEqual(DataFrame.columnNames(Result.getOrThrow(result)), ["temp", "city"]) }) - it("fails on missing columns, wrong types and unexpected nulls", () => { - const cases: ReadonlyArray, string]> = [ - [Schema.Struct({ country: Schema.String }), `missing column "country"`], - [Schema.Struct({ city: Schema.Number }), `column "city" is Utf8, expected Float64`], - [Schema.Struct({ rain: Schema.Number }), `column "rain" has missing values`] + it("fails on missing columns, wrong types, unexpected nulls and values the schema rejects", () => { + const nested = DataFrame.fromRows( + Schema.Struct({ p: Schema.Struct({ x: Schema.String }), xs: Schema.Array(Schema.NullOr(Schema.Number)) }), + [{ p: { x: "a" }, xs: [1, null] }] + ) + const cases: ReadonlyArray, Schema.Struct, string]> = [ + [untyped, Schema.Struct({ country: Schema.String }), `missing column "country"`], + [untyped, Schema.Struct({ city: Schema.Number }), `column "city" is Utf8, expected Float64`], + [untyped, Schema.Struct({ rain: Schema.Number }), `column "rain" has missing values`], + [ + DataFrame.fromColumns({ n: Int32Array.of(1) }), + Schema.Struct({ n: Schema.BigInt }), + `column "n" is Int32, expected Int64` + ], + [nested, Schema.Struct({ p: Schema.Struct({ x: Schema.Number }) }), `column "p.x" is Utf8, expected Float64`], + [nested, Schema.Struct({ xs: Schema.Array(Schema.Number) }), `column "xs[]" has missing values`], + [ + DataFrame.fromColumns({ s: ["allowed", "nope"] }), + Schema.Struct({ s: Schema.Literal("allowed") }), + `column "s" has values its schema rejects` + ], + [ + DataFrame.fromColumns({ n: [1, 1.5] }), + Schema.Struct({ n: Schema.Int }), + `column "n" has values its schema rejects` + ] ] - for (const [schema, message] of cases) { - const result = DataFrame.validate(untyped, schema) + for (const [df, schema, message] of cases) { + const result = DataFrame.validate(df, schema) assert.isTrue(Result.isFailure(result)) assert.strictEqual(Result.isFailure(result) && result.failure.message, message) } diff --git a/packages/effect/test/dataframe/Query.test.ts b/packages/effect/test/dataframe/Query.test.ts index 01f4836ff1f..1723c74accf 100644 --- a/packages/effect/test/dataframe/Query.test.ts +++ b/packages/effect/test/dataframe/Query.test.ts @@ -72,6 +72,12 @@ describe("Query", () => { const bad = Stream.make(DataFrame.fromColumns({ city: [1] })) const streamed = yield* Effect.flip(Query.table("t", City).pipe(Effect.provide(Catalog.layer({ t: bad })))) assert.strictEqual(streamed.message, `column "city" is Float64, expected Utf8`) + const Kind = Schema.Struct({ kind: Schema.Literals(["a", "b"]) }) + const kinds = DataFrame.fromColumns({ kind: ["a", "c"] }) + for (const t of [kinds, Stream.make(kinds)]) { + const error = yield* Effect.flip(Query.table("t", Kind).pipe(Effect.provide(Catalog.layer({ t })))) + assert.strictEqual(error.message, `column "kind" has values its schema rejects`) + } })) it.effect("narrows tables to their declared columns", () => diff --git a/packages/effect/test/dataframe/oracle/Plan.ts b/packages/effect/test/dataframe/oracle/Plan.ts index 51f57b975b7..bf021db866b 100644 --- a/packages/effect/test/dataframe/oracle/Plan.ts +++ b/packages/effect/test/dataframe/oracle/Plan.ts @@ -67,23 +67,19 @@ export const typeName = (type: ArrowSchema.DataType): string => { return type._tag } -/** The table column type name of an Arrow type (nested types as Null: not checked). */ -const tableType = (t: ArrowSchema.DataType): Plan.TypeName => { +/** The table column declaring a gold column of Arrow type `t`. */ +const tableColumn = (name: string, t: ArrowSchema.DataType): Plan.TableColumn => { switch (t._tag) { case "Int": - return `${t.signed ? "Int" : "UInt"}${t.bitWidth}` as Plan.TypeName + return { name, type: `${t.signed ? "Int" : "UInt"}${t.bitWidth}` as Plan.TypeName, nullable: true } case "Float": - return `Float${t.bitWidth}` as Plan.TypeName - case "Utf8": - case "Bool": - case "Date": - case "Timestamp": - case "Duration": - case "Null": - case "Decimal": - return t._tag + return { name, type: `Float${t.bitWidth}` as Plan.TypeName, nullable: true } + case "List": + return { name, type: "List", nullable: true, fields: [tableColumn("item", t.child.type)] } + case "Struct": + return { name, type: "Struct", nullable: true, fields: t.fields.map((f) => tableColumn(f.name, f.type)) } default: - return "Null" + return { name, type: t._tag as Plan.TypeName, nullable: true } } } @@ -241,7 +237,7 @@ const named = (record: { readonly [name: string]: Gold.Expr }): Array { const columns = (name: string): Array => - gold.inputs[name].fields.map((f) => ({ name: f.name, type: tableType(arrowType(f.type)), nullable: true })) + gold.inputs[name].fields.map((f) => tableColumn(f.name, arrowType(f.type))) const input = (name: string): Plan.Node => inline ? { _tag: "Values", frame: frameOf(gold.inputs[name]) } diff --git a/packages/effect/typetest/DataFrame.tst.ts b/packages/effect/typetest/DataFrame.tst.ts index 1a6c3435abb..605f5fbcc7e 100644 --- a/packages/effect/typetest/DataFrame.tst.ts +++ b/packages/effect/typetest/DataFrame.tst.ts @@ -1,4 +1,4 @@ -import type { BigDecimal, Stream } from "effect" +import type { BigDecimal, Result, Stream } from "effect" import { DateTime, Duration, Effect, Schema } from "effect" import { DataFrame, Expr, Query } from "effect/dataframe" import { describe, expect, it } from "tstyche" @@ -22,6 +22,25 @@ describe("DataFrame", () => { >() }) + it("fromRows, empty, validate and Query.table read missing values back as null", () => { + const S = Schema.Struct({ + s: Schema.optional(Schema.String), + u: Schema.UndefinedOr(Schema.String), + p: Schema.Struct({ x: Schema.optional(Schema.Number), xs: Schema.Array(Schema.UndefinedOr(Schema.Number)) }) + }) + type Row = { + readonly s: string | null + readonly u: string | null + readonly p: { readonly x: number | null; readonly xs: ReadonlyArray } + } + expect(DataFrame.fromRows(S, [{ u: undefined, p: { xs: [undefined] } }])).type.toBe>() + expect(DataFrame.empty(S)).type.toBe>() + expect(DataFrame.validate(DataFrame.empty(S), S)).type.toBe< + Result.Result, DataFrame.DataFrameError> + >() + expect(Query.table("t", S)).type.toBe>() + }) + it("fromColumns infers column types", () => { expect(DataFrame.fromColumns({ a: new Float64Array(1), b: ["x", null], c: new BigInt64Array(1) })).type.toBe< DataFrame.DataFrame<{ readonly a: number; readonly b: string | null; readonly c: bigint }> From ebd2e24538ee53bae679e9f80fa3600a8d2d1341 Mon Sep 17 00:00:00 2001 From: Effect Bot Date: Sun, 15 Nov 2026 11:53:06 +0000 Subject: [PATCH 04/88] Run windows over the rows they see in filters and grouped aggregates --- packages/effect/src/dataframe/Query.ts | 5 +++++ packages/effect/src/dataframe/internal/expr.ts | 15 +++++++++------ .../effect/src/dataframe/internal/frame.ts | 2 +- .../effect/src/dataframe/internal/query.ts | 14 ++++++++++---- .../effect/src/dataframe/internal/stream.ts | 12 ++++++++---- .../test/fixtures/dataframe/generate_gold.py | 18 ++++++++++++++++++ .../test/fixtures/dataframe/gold/extended.json | 2 +- 7 files changed, 52 insertions(+), 16 deletions(-) diff --git a/packages/effect/src/dataframe/Query.ts b/packages/effect/src/dataframe/Query.ts index 4e9d2b8717c..85efe83fd6d 100644 --- a/packages/effect/src/dataframe/Query.ts +++ b/packages/effect/src/dataframe/Query.ts @@ -584,6 +584,11 @@ export const groupBy: { * Computes aggregates per group: one row per distinct key, in order of first * appearance, with the key columns first. * + * **Details** + * + * An aggregate's argument is evaluated per group: window functions in it + * (`Expr.cumSum`, `Expr.over`, …) see only the group's rows, in row order. + * * @stability experimental * @category grouping * @since 4.1.5 diff --git a/packages/effect/src/dataframe/internal/expr.ts b/packages/effect/src/dataframe/internal/expr.ts index 9cc548ecbe2..49a7ccdca28 100644 --- a/packages/effect/src/dataframe/internal/expr.ts +++ b/packages/effect/src/dataframe/internal/expr.ts @@ -143,7 +143,10 @@ export const show = (node: Node): string => { export interface Context { readonly length: number readonly column: (name: string) => V.Vector - /** The partitions of an enclosing `Over`: window functions run per partition. */ + /** + * The partitions of an enclosing `Over` or grouped aggregate: window + * functions run per partition, and aggregates broadcast over it. + */ readonly partition?: W.Partition | undefined } @@ -188,17 +191,18 @@ export const evaluate = (node: Node, ctx: Context): K.Value => { case "Literal": return node.type === undefined || node.value === null ? literal(node.value) : literalOfType(node.value, node.type) case "AggregateCall": { - // Inside `over`: the partition's aggregate on each of its rows. + // Inside `over` or an aggregate: the partition's aggregate on each of its rows. if (ctx.partition === undefined) return V.fail(`aggregate ${show(node)} can only be used in agg or over`) if (node.arg === undefined && node.fn !== "count") return V.fail(`${node.fn} needs an argument`) - const arg = node.arg === undefined ? undefined : evaluateColumn(node.arg, { ...ctx, partition: undefined }) + const arg = node.arg === undefined ? undefined : evaluateColumn(node.arg, ctx) const values = Agg.aggregate(node.arg === undefined ? "len" : node.fn, arg, ctx.partition) return K.column(V.take(values, ctx.partition.ids)) } case "Call": return evaluateCall(node, ctx) case "Over": { - const keys = node.partitionBy.map((k) => evaluateColumn(k, { ...ctx, partition: undefined })) + const keys = node.partitionBy.map((k) => evaluateColumn(k, ctx)) + if (ctx.partition !== undefined) keys.unshift(V.i32(S.int32, ctx.partition.ids)) const g = Hash.groups(keys, ctx.length) return evaluate(node.expression, { ...ctx, partition: { ids: g.ids, count: g.count } }) } @@ -456,8 +460,7 @@ const evaluateCall = (node: Plan.Call, ctx: Context): K.Value => { case "rollingMean": case "rollingMin": case "rollingMax": { - // The argument is evaluated over all rows, the window per partition. - const x = K.materialize(evaluate(node.args[0], { ...ctx, partition: undefined }), n) + const x = K.materialize(evaluate(node.args[0], ctx), n) const p = ctx.partition ?? W.whole(n) switch (node.fn) { case "cumSum": diff --git a/packages/effect/src/dataframe/internal/frame.ts b/packages/effect/src/dataframe/internal/frame.ts index e98966250bf..5355d698e99 100644 --- a/packages/effect/src/dataframe/internal/frame.ts +++ b/packages/effect/src/dataframe/internal/frame.ts @@ -344,7 +344,7 @@ const aggregateColumns = ( aggs: { readonly [name: string]: E.Node }, grouping: Agg.Grouping ): Array => { - const ctx = context(self) + const ctx: E.Context = { ...context(self), partition: grouping } return Object.keys(aggs).map((name) => { const node = aggs[name] if (node._tag !== "AggregateCall") return V.fail(`${name} is not an aggregate expression`) diff --git a/packages/effect/src/dataframe/internal/query.ts b/packages/effect/src/dataframe/internal/query.ts index fd814ee0420..fd91b36e8b8 100644 --- a/packages/effect/src/dataframe/internal/query.ts +++ b/packages/effect/src/dataframe/internal/query.ts @@ -4,9 +4,10 @@ * Execution is a post-order walk that calls the frame operations. Before it, * `optimize` rewrites the plan so less work happens: * - * - predicates move below the operations they commute with (sorts, - * projections, derived columns they don't read, group keys, either side of - * a join, every input of a concat), and filters that meet merge; + * - row-wise predicates move below the operations they commute with + * (sorts, projections, derived columns they don't read, group keys, either + * side of a join, every input of a concat), and filters that meet merge; + * predicates with windows stay where they are; * - slices move below row-wise operations, and a slice of a sort becomes a * top-k selection; * - columns nothing reads are dropped at the inputs, and derived columns and @@ -232,9 +233,14 @@ const pushFilters = (node: Node): Node => { const filtered = (input: Node, predicates: ReadonlyArray): Node => predicates.length === 0 ? input : { _tag: "Filter", input, predicate: and(predicates) } -/** Places each predicate as deep as it can go below `node`. */ +/** + * Places each predicate as deep as it can go below `node`. Predicates that + * are not row-wise (windows) depend on exactly the rows they see, so a set + * containing one stays where it is. + */ const push = (node: Node, predicates: ReadonlyArray): Node => { if (predicates.length === 0) return node + if (!predicates.every(E.isRowWise)) return filtered(node, predicates) const split = (canPush: (refs: Set, p: Plan.Expression) => boolean) => { const down: Array = [] const here: Array = [] diff --git a/packages/effect/src/dataframe/internal/stream.ts b/packages/effect/src/dataframe/internal/stream.ts index f8e3c2a6f03..bf36e41c490 100644 --- a/packages/effect/src/dataframe/internal/stream.ts +++ b/packages/effect/src/dataframe/internal/stream.ts @@ -1,14 +1,15 @@ /** * Running plans batch by batch over streamed tables. * - * Row-wise operations (filters, projections, derived columns without + * Row-wise operations (filters, projections and derived columns without * windows, renames, `dropNulls`, `explode`, `unnest`) run on every batch as * it arrives, a slice from the start stops pulling batches once it has its * rows, joins stream their left side against the collected right side, and * concatenations stream their inputs one after the other. Grouping and whole- * table aggregation fold every batch into partial aggregates, so memory grows - * with the number of groups, not rows (medians and distinct counts collect - * instead). Every other operation collects its inputs first. + * with the number of groups, not rows (medians, distinct counts and + * aggregates of windows collect instead). Every other operation collects its + * inputs first. * * @internal */ @@ -86,6 +87,7 @@ const collectFrames = (batches: Batches, node: Q.Node): Effect.Effect { switch (node._tag) { case "Filter": + return E.isRowWise(node.predicate) case "Pick": case "Omit": case "Rename": @@ -121,7 +123,9 @@ const aggregate = ( const fn: Agg.AggFn = expression.arg === undefined ? "len" : expression.fn return { name, fn, arg: expression.arg } }) - if (!specs.every((s) => Partial.combinable(s.fn))) return collected(node, catalog) + if (!specs.every((s) => Partial.combinable(s.fn) && (s.arg === undefined || E.isRowWise(s.arg)))) { + return collected(node, catalog) + } const fns = specs.map((s) => s.fn) // The first batch's input vectors and key columns, for result types. let template: { readonly inputs: ReadonlyArray; readonly frame: F.Frame } | undefined diff --git a/packages/effect/test/fixtures/dataframe/generate_gold.py b/packages/effect/test/fixtures/dataframe/generate_gold.py index 0eeac75fb8c..2d34d84951e 100644 --- a/packages/effect/test/fixtures/dataframe/generate_gold.py +++ b/packages/effect/test/fixtures/dataframe/generate_gold.py @@ -890,6 +890,24 @@ def extended_cases(): add("decimal/sort", "sort by a decimal", D, [{"op": "sort", "keys": [{"expr": col("price"), "descending": True}], "nullsLast": True}]) add("decimal/round-ties", "narrowing the scale rounds exact ties", {"left": table([("t", "decimal[6,3]")], {"t": ["0.250", "0.350", "-0.250", "-0.350", "1.050", "2.500", "0.249"]})}, [{"op": "select", "exprs": {"one": fn("cast", col("t"), to="decimal[6,1]"), "zero": fn("cast", col("t"), to="decimal[6,0]")}}]) + # windows see exactly the rows of their input, whatever runs before or after them + W = {"left": table([("g", "i32"), ("h", "i32"), ("x", "f64")], {"g": [1, 2, 1, 2], "h": [1, 1, 2, 2], "x": [3.0, 1.0, 2.0, 4.0]}), + "right": table([("g", "i32"), ("name", "utf8")], {"g": [1], "name": ["one"]})} + add("window/filter-after-sort", "a window predicate sees the sorted rows", W, + [{"op": "sort", "keys": [{"expr": col("x")}]}, {"op": "filter", "predicate": fn("gt", fn("cumSum", col("x")), lit(3.0))}]) + add("window/filter-after-filter", "a window predicate sees only the rows an earlier filter kept", W, + [{"op": "filter", "predicate": fn("gt", col("x"), lit(1.0))}, {"op": "filter", "predicate": fn("gt", fn("shift", col("x")), lit(2.0))}]) + add("window/filter-after-concat", "a window predicate sees every input of a concat", W, + [{"op": "concat", "with": "left"}, {"op": "filter", "predicate": fn("gt", fn("cumSum", col("x")), lit(15.0))}]) + add("window/filter-after-join", "a window predicate sees only the joined rows", W, + [{"op": "join", "right": "right", "on": ["g"], "how": "inner"}, {"op": "filter", "predicate": fn("gt", fn("shift", col("x")), lit(2.0))}]) + add("window/filter-whole", "a window predicate sees the whole input", W, + [{"op": "filter", "predicate": fn("gt", fn("cumSum", col("x")), lit(5.0))}]) + add("window/agg-of-window", "an aggregate of a window sees the whole input", W, + [{"op": "agg", "aggs": {"s": agg("sum", fn("cumSum", col("x"))), "n": agg("len")}}]) + add("window/grouped-agg-of-window", "windows inside grouped aggregates run per group", W, + [{"op": "groupBy", "keys": ["g"], "aggs": {"s": agg("sum", fn("cumSum", col("x"))), "l": agg("list", fn("cumSum", col("x"))), + "o": agg("sum", over(agg("sum", col("x")), "h"))}}]) return out diff --git a/packages/effect/test/fixtures/dataframe/gold/extended.json b/packages/effect/test/fixtures/dataframe/gold/extended.json index 5528f1600e1..3832464118b 100644 --- a/packages/effect/test/fixtures/dataframe/gold/extended.json +++ b/packages/effect/test/fixtures/dataframe/gold/extended.json @@ -1 +1 @@ -{"generator":"generate_gold.py","polars":"2.1.0","deviations":["JavaScript has no float literal distinct from an integer: an integral number literal behaves like a Polars integer literal (it takes the integer type of the column it meets), so cases never combine an integer column with an integral float literal.","Float to string casts use JavaScript number formatting (Polars prints 5.0 as '5.0', JavaScript as '5').","Joins never add '_right' suffixes: overlapping non-key columns are a type error.","Arithmetic between 64-bit integers (bigint) and other numeric types requires an explicit cast."],"cases":[{"id":"window/cum","description":"cumulative sum, min, max and count; nulls stay null","inputs":{"left":{"fields":[{"name":"g","type":"utf8"},{"name":"t","type":"i32"},{"name":"x","type":"f64"},{"name":"i","type":"i32"},{"name":"b","type":"bool"}],"columns":{"g":["a","b","a","a","b",null,"a","b","b","a"],"t":[1,1,2,3,2,1,4,3,4,5],"x":[1.5,2.0,null,4.0,"NaN",3.0,-1.0,2.0,0.5,2.0],"i":[3,1,4,1,5,9,null,6,5,3],"b":[true,false,null,true,true,false,true,null,false,true]}}},"ops":[{"op":"select","exprs":{"sx":{"fn":"cumSum","args":[{"col":"x"}]},"si":{"fn":"cumSum","args":[{"col":"i"}]},"mn":{"fn":"cumMin","args":[{"col":"i"}]},"mx":{"fn":"cumMax","args":[{"col":"x"}]},"n":{"fn":"cumCount","args":[{"col":"x"}]},"sb":{"fn":"cumSum","args":[{"col":"b"}]}}}],"expected":{"fields":[{"name":"sx","type":"f64"},{"name":"si","type":"i32"},{"name":"mn","type":"i32"},{"name":"mx","type":"f64"},{"name":"n","type":"u32"},{"name":"sb","type":"u32"}],"columns":{"sx":[1.5,3.5,null,7.5,"NaN","NaN","NaN","NaN","NaN","NaN"],"si":[3,4,8,9,14,23,null,29,34,37],"mn":[3,1,1,1,1,1,null,1,1,1],"mx":[1.5,2.0,null,4.0,4.0,4.0,4.0,4.0,4.0,4.0],"n":[1,2,2,3,4,5,6,7,8,9],"sb":[1,1,null,2,3,3,4,null,4,5]}}},{"id":"window/shift-diff","description":"shift (lag and lead) and diff","inputs":{"left":{"fields":[{"name":"g","type":"utf8"},{"name":"t","type":"i32"},{"name":"x","type":"f64"},{"name":"i","type":"i32"},{"name":"b","type":"bool"}],"columns":{"g":["a","b","a","a","b",null,"a","b","b","a"],"t":[1,1,2,3,2,1,4,3,4,5],"x":[1.5,2.0,null,4.0,"NaN",3.0,-1.0,2.0,0.5,2.0],"i":[3,1,4,1,5,9,null,6,5,3],"b":[true,false,null,true,true,false,true,null,false,true]}}},"ops":[{"op":"select","exprs":{"lag":{"fn":"shift","args":[{"col":"i"}]},"lead":{"fn":"shift","args":[{"col":"x"}],"n":-2},"d":{"fn":"diff","args":[{"col":"i"}]},"d2":{"fn":"diff","args":[{"col":"x"}],"n":2},"ls":{"fn":"shift","args":[{"col":"g"}]}}}],"expected":{"fields":[{"name":"lag","type":"i32"},{"name":"lead","type":"f64"},{"name":"d","type":"i32"},{"name":"d2","type":"f64"},{"name":"ls","type":"utf8"}],"columns":{"lag":[null,3,1,4,1,5,9,null,6,5],"lead":[null,4.0,"NaN",3.0,-1.0,2.0,0.5,2.0,null,null],"d":[null,-2,3,-3,4,4,null,null,-1,-2],"d2":[null,null,null,2.0,null,-1.0,"NaN",-1.0,1.5,0.0],"ls":[null,"a","b","a","a","b",null,"a","b","b"]}}},{"id":"window/rank","description":"rank methods, nulls and NaN","inputs":{"left":{"fields":[{"name":"g","type":"utf8"},{"name":"t","type":"i32"},{"name":"x","type":"f64"},{"name":"i","type":"i32"},{"name":"b","type":"bool"}],"columns":{"g":["a","b","a","a","b",null,"a","b","b","a"],"t":[1,1,2,3,2,1,4,3,4,5],"x":[1.5,2.0,null,4.0,"NaN",3.0,-1.0,2.0,0.5,2.0],"i":[3,1,4,1,5,9,null,6,5,3],"b":[true,false,null,true,true,false,true,null,false,true]}}},"ops":[{"op":"select","exprs":{"avg":{"fn":"rank","args":[{"col":"x"}]},"min":{"fn":"rank","args":[{"col":"i"}],"method":"min"},"max":{"fn":"rank","args":[{"col":"i"}],"method":"max"},"dense":{"fn":"rank","args":[{"col":"i"}],"method":"dense","descending":true},"ord":{"fn":"rank","args":[{"col":"x"}],"method":"ordinal"},"s":{"fn":"rank","args":[{"col":"g"}],"method":"dense"}}}],"expected":{"fields":[{"name":"avg","type":"f64"},{"name":"min","type":"u32"},{"name":"max","type":"u32"},{"name":"dense","type":"u32"},{"name":"ord","type":"u32"},{"name":"s","type":"u32"}],"columns":{"avg":[3.0,5.0,null,8.0,9.0,7.0,1.0,5.0,2.0,5.0],"min":[3,1,5,1,6,9,null,8,6,3],"max":[4,2,5,2,7,9,null,8,7,4],"dense":[5,6,4,6,3,1,null,2,3,5],"ord":[3,4,null,8,9,7,1,5,2,6],"s":[1,2,1,1,2,null,1,2,2,1]}}},{"id":"window/rolling","description":"rolling sum, mean, min and max","inputs":{"left":{"fields":[{"name":"g","type":"utf8"},{"name":"t","type":"i32"},{"name":"x","type":"f64"},{"name":"i","type":"i32"},{"name":"b","type":"bool"}],"columns":{"g":["a","b","a","a","b",null,"a","b","b","a"],"t":[1,1,2,3,2,1,4,3,4,5],"x":[1.5,2.0,null,4.0,"NaN",3.0,-1.0,2.0,0.5,2.0],"i":[3,1,4,1,5,9,null,6,5,3],"b":[true,false,null,true,true,false,true,null,false,true]}}},"ops":[{"op":"select","exprs":{"s":{"fn":"rollingSum","args":[{"col":"x"}],"window":3},"m":{"fn":"rollingMean","args":[{"col":"i"}],"window":2},"mn":{"fn":"rollingMin","args":[{"col":"x"}],"window":3,"minPeriods":1},"mx":{"fn":"rollingMax","args":[{"col":"i"}],"window":4,"minPeriods":2}}}],"expected":{"fields":[{"name":"s","type":"f64"},{"name":"m","type":"f64"},{"name":"mn","type":"f64"},{"name":"mx","type":"i32"}],"columns":{"s":[null,null,null,null,null,"NaN","NaN",4.0,1.5,4.5],"m":[null,2.0,2.5,2.5,3.0,7.0,null,null,5.5,4.0],"mn":[1.5,1.5,1.5,2.0,"NaN","NaN","NaN",-1.0,-1.0,0.5],"mx":[null,3,4,4,5,9,9,9,9,6]}}},{"id":"window/over-agg","description":"aggregates broadcast over partitions","inputs":{"left":{"fields":[{"name":"g","type":"utf8"},{"name":"t","type":"i32"},{"name":"x","type":"f64"},{"name":"i","type":"i32"},{"name":"b","type":"bool"}],"columns":{"g":["a","b","a","a","b",null,"a","b","b","a"],"t":[1,1,2,3,2,1,4,3,4,5],"x":[1.5,2.0,null,4.0,"NaN",3.0,-1.0,2.0,0.5,2.0],"i":[3,1,4,1,5,9,null,6,5,3],"b":[true,false,null,true,true,false,true,null,false,true]}}},"ops":[{"op":"withColumns","exprs":{"total":{"over":{"agg":"sum","arg":{"col":"x"}},"partitionBy":[{"col":"g"}]},"n":{"over":{"agg":"len"},"partitionBy":[{"col":"g"}]},"mean":{"over":{"agg":"mean","arg":{"col":"i"}},"partitionBy":[{"col":"g"}]},"first":{"over":{"agg":"first","arg":{"col":"t"}},"partitionBy":[{"col":"g"},{"col":"b"}]}}}],"expected":{"fields":[{"name":"g","type":"utf8"},{"name":"t","type":"i32"},{"name":"x","type":"f64"},{"name":"i","type":"i32"},{"name":"b","type":"bool"},{"name":"total","type":"f64"},{"name":"n","type":"u32"},{"name":"mean","type":"f64"},{"name":"first","type":"i32"}],"columns":{"g":["a","b","a","a","b",null,"a","b","b","a"],"t":[1,1,2,3,2,1,4,3,4,5],"x":[1.5,2.0,null,4.0,"NaN",3.0,-1.0,2.0,0.5,2.0],"i":[3,1,4,1,5,9,null,6,5,3],"b":[true,false,null,true,true,false,true,null,false,true],"total":[6.5,"NaN",6.5,6.5,"NaN",3.0,6.5,"NaN","NaN",6.5],"n":[5,4,5,5,4,1,5,4,4,5],"mean":[2.75,4.25,2.75,2.75,4.25,9.0,2.75,4.25,4.25,2.75],"first":[1,1,2,1,2,1,1,3,1,1]}}},{"id":"window/over-cum","description":"cumulative and shifted values per partition, in row order","inputs":{"left":{"fields":[{"name":"g","type":"utf8"},{"name":"t","type":"i32"},{"name":"x","type":"f64"},{"name":"i","type":"i32"},{"name":"b","type":"bool"}],"columns":{"g":["a","b","a","a","b",null,"a","b","b","a"],"t":[1,1,2,3,2,1,4,3,4,5],"x":[1.5,2.0,null,4.0,"NaN",3.0,-1.0,2.0,0.5,2.0],"i":[3,1,4,1,5,9,null,6,5,3],"b":[true,false,null,true,true,false,true,null,false,true]}}},"ops":[{"op":"withColumns","exprs":{"cs":{"over":{"fn":"cumSum","args":[{"col":"i"}]},"partitionBy":[{"col":"g"}]},"prev":{"over":{"fn":"shift","args":[{"col":"x"}]},"partitionBy":[{"col":"g"}]},"d":{"over":{"fn":"diff","args":[{"col":"t"}]},"partitionBy":[{"col":"g"}]},"r":{"over":{"fn":"rank","args":[{"col":"x"}],"method":"min"},"partitionBy":[{"col":"g"}]},"roll":{"over":{"fn":"rollingSum","args":[{"col":"i"}],"window":2,"minPeriods":1},"partitionBy":[{"col":"g"}]}}}],"expected":{"fields":[{"name":"g","type":"utf8"},{"name":"t","type":"i32"},{"name":"x","type":"f64"},{"name":"i","type":"i32"},{"name":"b","type":"bool"},{"name":"cs","type":"i32"},{"name":"prev","type":"f64"},{"name":"d","type":"i32"},{"name":"r","type":"u32"},{"name":"roll","type":"i32"}],"columns":{"g":["a","b","a","a","b",null,"a","b","b","a"],"t":[1,1,2,3,2,1,4,3,4,5],"x":[1.5,2.0,null,4.0,"NaN",3.0,-1.0,2.0,0.5,2.0],"i":[3,1,4,1,5,9,null,6,5,3],"b":[true,false,null,true,true,false,true,null,false,true],"cs":[3,1,7,8,6,9,null,12,17,11],"prev":[null,null,1.5,null,2.0,null,4.0,"NaN",2.0,-1.0],"d":[null,null,1,1,1,null,1,1,1,1],"r":[2,2,null,4,4,1,1,2,1,3],"roll":[3,1,7,5,6,9,1,11,11,3]}}},{"id":"window/over-filter","description":"filters after windows see the windowed values","inputs":{"left":{"fields":[{"name":"g","type":"utf8"},{"name":"t","type":"i32"},{"name":"x","type":"f64"},{"name":"i","type":"i32"},{"name":"b","type":"bool"}],"columns":{"g":["a","b","a","a","b",null,"a","b","b","a"],"t":[1,1,2,3,2,1,4,3,4,5],"x":[1.5,2.0,null,4.0,"NaN",3.0,-1.0,2.0,0.5,2.0],"i":[3,1,4,1,5,9,null,6,5,3],"b":[true,false,null,true,true,false,true,null,false,true]}}},"ops":[{"op":"withColumns","exprs":{"cs":{"over":{"fn":"cumSum","args":[{"col":"t"}]},"partitionBy":[{"col":"g"}]}}},{"op":"filter","predicate":{"fn":"gt","args":[{"col":"t"},{"lit":2}]}}],"expected":{"fields":[{"name":"g","type":"utf8"},{"name":"t","type":"i32"},{"name":"x","type":"f64"},{"name":"i","type":"i32"},{"name":"b","type":"bool"},{"name":"cs","type":"i32"}],"columns":{"g":["a","a","b","b","a"],"t":[3,4,3,4,5],"x":[4.0,-1.0,2.0,0.5,2.0],"i":[1,null,6,5,3],"b":[true,true,null,false,true],"cs":[6,10,6,10,15]}}},{"id":"window/filter-first","description":"windows after a filter only see the remaining rows","inputs":{"left":{"fields":[{"name":"g","type":"utf8"},{"name":"t","type":"i32"},{"name":"x","type":"f64"},{"name":"i","type":"i32"},{"name":"b","type":"bool"}],"columns":{"g":["a","b","a","a","b",null,"a","b","b","a"],"t":[1,1,2,3,2,1,4,3,4,5],"x":[1.5,2.0,null,4.0,"NaN",3.0,-1.0,2.0,0.5,2.0],"i":[3,1,4,1,5,9,null,6,5,3],"b":[true,false,null,true,true,false,true,null,false,true]}}},"ops":[{"op":"filter","predicate":{"fn":"isNotNull","args":[{"col":"x"}]}},{"op":"withColumns","exprs":{"cs":{"fn":"cumSum","args":[{"col":"x"}]},"r":{"over":{"fn":"rank","args":[{"col":"x"}]},"partitionBy":[{"col":"g"}]}}},{"op":"head","n":4}],"expected":{"fields":[{"name":"g","type":"utf8"},{"name":"t","type":"i32"},{"name":"x","type":"f64"},{"name":"i","type":"i32"},{"name":"b","type":"bool"},{"name":"cs","type":"f64"},{"name":"r","type":"f64"}],"columns":{"g":["a","b","a","b"],"t":[1,1,3,2],"x":[1.5,2.0,4.0,"NaN"],"i":[3,1,1,5],"b":[true,false,true,true],"cs":[1.5,3.5,7.5,"NaN"],"r":[2.0,2.5,4.0,4.0]}}},{"id":"pivot/sum","description":"pivot with a sum per cell; missing cells are null","inputs":{"left":{"fields":[{"name":"region","type":"utf8"},{"name":"quarter","type":"utf8"},{"name":"units","type":"i32"},{"name":"price","type":"f64"}],"columns":{"region":["north","north","south","south","north","east","south","north"],"quarter":["q1","q2","q1","q3","q1","q2","q1",null],"units":[10,20,5,null,7,3,8,1],"price":[1.5,2.0,1.0,3.0,null,2.5,1.0,9.0]}}},"ops":[{"op":"pivot","on":"quarter","index":["region"],"values":"units","aggregate":"sum"}],"expected":{"fields":[{"name":"region","type":"utf8"},{"name":"q1","type":"i32"},{"name":"q2","type":"i32"},{"name":"q3","type":"i32"},{"name":"null","type":"i32"}],"columns":{"region":["north","south","east"],"q1":[17,13,0],"q2":[20,0,3],"q3":[0,0,0],"null":[1,0,0]}}},{"id":"pivot/first","description":"pivot with the first value per cell","inputs":{"left":{"fields":[{"name":"region","type":"utf8"},{"name":"quarter","type":"utf8"},{"name":"units","type":"i32"},{"name":"price","type":"f64"}],"columns":{"region":["north","north","south","south","north","east","south","north"],"quarter":["q1","q2","q1","q3","q1","q2","q1",null],"units":[10,20,5,null,7,3,8,1],"price":[1.5,2.0,1.0,3.0,null,2.5,1.0,9.0]}}},"ops":[{"op":"pivot","on":"region","index":["quarter"],"values":"price","aggregate":"first"}],"expected":{"fields":[{"name":"quarter","type":"utf8"},{"name":"north","type":"f64"},{"name":"south","type":"f64"},{"name":"east","type":"f64"}],"columns":{"quarter":["q1","q2","q3",null],"north":[1.5,2.0,null,9.0],"south":[1.0,null,3.0,null],"east":[null,2.5,null,null]}}},{"id":"pivot/len","description":"pivot counting rows per cell","inputs":{"left":{"fields":[{"name":"region","type":"utf8"},{"name":"quarter","type":"utf8"},{"name":"units","type":"i32"},{"name":"price","type":"f64"}],"columns":{"region":["north","north","south","south","north","east","south","north"],"quarter":["q1","q2","q1","q3","q1","q2","q1",null],"units":[10,20,5,null,7,3,8,1],"price":[1.5,2.0,1.0,3.0,null,2.5,1.0,9.0]}}},"ops":[{"op":"pivot","on":"quarter","index":["region"],"values":"units","aggregate":"len"}],"expected":{"fields":[{"name":"region","type":"utf8"},{"name":"q1","type":"u32"},{"name":"q2","type":"u32"},{"name":"q3","type":"u32"},{"name":"null","type":"u32"}],"columns":{"region":["north","south","east"],"q1":[2,2,0],"q2":[1,0,1],"q3":[0,1,0],"null":[1,0,0]}}},{"id":"pivot/columns","description":"pivot with declared output columns","inputs":{"left":{"fields":[{"name":"region","type":"utf8"},{"name":"quarter","type":"utf8"},{"name":"units","type":"i32"},{"name":"price","type":"f64"}],"columns":{"region":["north","north","south","south","north","east","south","north"],"quarter":["q1","q2","q1","q3","q1","q2","q1",null],"units":[10,20,5,null,7,3,8,1],"price":[1.5,2.0,1.0,3.0,null,2.5,1.0,9.0]}}},"ops":[{"op":"pivot","on":"quarter","index":["region"],"values":"price","aggregate":"max","columns":["q1","q2","q4"]}],"expected":{"fields":[{"name":"region","type":"utf8"},{"name":"q1","type":"f64"},{"name":"q2","type":"f64"},{"name":"q4","type":"f64"}],"columns":{"region":["north","south","east"],"q1":[1.5,1.0,null],"q2":[2.0,null,2.5],"q4":[null,null,null]}}},{"id":"pivot/numeric-on","description":"pivot on a numeric column names columns by value","inputs":{"left":{"fields":[{"name":"g","type":"utf8"},{"name":"t","type":"i32"},{"name":"x","type":"f64"},{"name":"i","type":"i32"},{"name":"b","type":"bool"}],"columns":{"g":["a","b","a","a","b",null,"a","b","b","a"],"t":[1,1,2,3,2,1,4,3,4,5],"x":[1.5,2.0,null,4.0,"NaN",3.0,-1.0,2.0,0.5,2.0],"i":[3,1,4,1,5,9,null,6,5,3],"b":[true,false,null,true,true,false,true,null,false,true]}}},"ops":[{"op":"filter","predicate":{"fn":"isNotNull","args":[{"col":"g"}]}},{"op":"pivot","on":"t","index":["g"],"values":"i","aggregate":"min"}],"expected":{"fields":[{"name":"g","type":"utf8"},{"name":"1","type":"i32"},{"name":"2","type":"i32"},{"name":"3","type":"i32"},{"name":"4","type":"i32"},{"name":"5","type":"i32"}],"columns":{"g":["a","b"],"1":[3,1],"2":[4,5],"3":[1,6],"4":[null,5],"5":[3,null]}}},{"id":"unpivot/basic","description":"unpivot stacks columns, column by column","inputs":{"left":{"fields":[{"name":"region","type":"utf8"},{"name":"quarter","type":"utf8"},{"name":"units","type":"i32"},{"name":"price","type":"f64"}],"columns":{"region":["north","north","south","south","north","east","south","north"],"quarter":["q1","q2","q1","q3","q1","q2","q1",null],"units":[10,20,5,null,7,3,8,1],"price":[1.5,2.0,1.0,3.0,null,2.5,1.0,9.0]}}},"ops":[{"op":"unpivot","index":["region"],"on":["units","price"]}],"expected":{"fields":[{"name":"region","type":"utf8"},{"name":"variable","type":"utf8"},{"name":"value","type":"f64"}],"columns":{"region":["north","north","south","south","north","east","south","north","north","north","south","south","north","east","south","north"],"variable":["units","units","units","units","units","units","units","units","price","price","price","price","price","price","price","price"],"value":[10.0,20.0,5.0,null,7.0,3.0,8.0,1.0,1.5,2.0,1.0,3.0,null,2.5,1.0,9.0]}}},{"id":"unpivot/names","description":"unpivot with named variable and value columns","inputs":{"left":{"fields":[{"name":"region","type":"utf8"},{"name":"quarter","type":"utf8"},{"name":"units","type":"i32"},{"name":"price","type":"f64"}],"columns":{"region":["north","north","south","south","north","east","south","north"],"quarter":["q1","q2","q1","q3","q1","q2","q1",null],"units":[10,20,5,null,7,3,8,1],"price":[1.5,2.0,1.0,3.0,null,2.5,1.0,9.0]}}},"ops":[{"op":"pick","columns":["region","quarter"]},{"op":"unpivot","index":[],"on":["region","quarter"],"variable":"field","value":"text"}],"expected":{"fields":[{"name":"field","type":"utf8"},{"name":"text","type":"utf8"}],"columns":{"field":["region","region","region","region","region","region","region","region","quarter","quarter","quarter","quarter","quarter","quarter","quarter","quarter"],"text":["north","north","south","south","north","east","south","north","q1","q2","q1","q3","q1","q2","q1",null]}}},{"id":"temporal/extract","description":"calendar fields of timestamps and dates","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"ts","type":"datetime[us]"},{"name":"ms","type":"datetime[ms]"},{"name":"d","type":"date"},{"name":"dur","type":"duration[us]"}],"columns":{"k":["a","b","c","d","e","f"],"ts":[1709214307123456,1704067199999000,1709510400000000,null,-14182940000000,1710052200000500],"ms":[1709214307123,1704067200000,null,1709251200000,0,1709942400000],"d":[19782,19722,19786,19782,null,19792],"dur":[5400000000,-1500000,null,259200000001,0,90000000000]}},"right":{"fields":[{"name":"d","type":"date"},{"name":"name","type":"utf8"}],"columns":{"d":[19782,19792,20089],"name":["leap","sunday","new year"]}}},"ops":[{"op":"select","exprs":{"y":{"fn":"year","args":[{"col":"ts"}]},"mo":{"fn":"month","args":[{"col":"ts"}]},"d":{"fn":"day","args":[{"col":"ts"}]},"h":{"fn":"hour","args":[{"col":"ts"}]},"mi":{"fn":"minute","args":[{"col":"ts"}]},"s":{"fn":"second","args":[{"col":"ts"}]},"ms":{"fn":"millisecond","args":[{"col":"ts"}]},"wd":{"fn":"weekday","args":[{"col":"ts"}]},"od":{"fn":"ordinalDay","args":[{"col":"ts"}]},"dy":{"fn":"year","args":[{"col":"d"}]},"dwd":{"fn":"weekday","args":[{"col":"d"}]},"dod":{"fn":"ordinalDay","args":[{"col":"d"}]}}}],"expected":{"fields":[{"name":"y","type":"i32"},{"name":"mo","type":"i8"},{"name":"d","type":"i8"},{"name":"h","type":"i8"},{"name":"mi","type":"i8"},{"name":"s","type":"i8"},{"name":"ms","type":"i32"},{"name":"wd","type":"i8"},{"name":"od","type":"i16"},{"name":"dy","type":"i32"},{"name":"dwd","type":"i8"},{"name":"dod","type":"i16"}],"columns":{"y":[2024,2023,2024,null,1969,2024],"mo":[2,12,3,null,7,3],"d":[29,31,4,null,20,10],"h":[13,23,0,null,20,6],"mi":[45,59,0,null,17,30],"s":[7,59,0,null,40,0],"ms":[123,999,0,null,0,0],"wd":[4,7,1,null,7,7],"od":[60,365,64,null,201,70],"dy":[2024,2023,2024,2024,null,2024],"dwd":[4,7,1,4,null,7],"dod":[60,365,64,60,null,70]}}},{"id":"temporal/arith","description":"timestamp differences are durations; durations shift timestamps","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"ts","type":"datetime[us]"},{"name":"ms","type":"datetime[ms]"},{"name":"d","type":"date"},{"name":"dur","type":"duration[us]"}],"columns":{"k":["a","b","c","d","e","f"],"ts":[1709214307123456,1704067199999000,1709510400000000,null,-14182940000000,1710052200000500],"ms":[1709214307123,1704067200000,null,1709251200000,0,1709942400000],"d":[19782,19722,19786,19782,null,19792],"dur":[5400000000,-1500000,null,259200000001,0,90000000000]}},"right":{"fields":[{"name":"d","type":"date"},{"name":"name","type":"utf8"}],"columns":{"d":[19782,19792,20089],"name":["leap","sunday","new year"]}}},"ops":[{"op":"select","exprs":{"gap":{"fn":"sub","args":[{"col":"ts"},{"col":"ms"}]},"later":{"fn":"add","args":[{"col":"ts"},{"col":"dur"}]},"earlier":{"fn":"sub","args":[{"col":"ts"},{"col":"dur"}]},"days":{"fn":"sub","args":[{"col":"d"},{"lit":18000,"type":"date"}]},"twice":{"fn":"add","args":[{"col":"dur"},{"col":"dur"}]}}}],"expected":{"fields":[{"name":"gap","type":"duration[ms]"},{"name":"later","type":"datetime[us]"},{"name":"earlier","type":"datetime[us]"},{"name":"days","type":"duration[us]"},{"name":"twice","type":"duration[us]"}],"columns":{"gap":[0,-1,null,null,-14182940000,109800000],"later":[1709219707123456,1704067198499000,null,null,-14182940000000,1710142200000500],"earlier":[1709208907123456,1704067201499000,null,null,-14182940000000,1709962200000500],"days":[153964800000000,148780800000000,154310400000000,153964800000000,null,154828800000000],"twice":[10800000000,-3000000,null,518400000002,0,180000000000]}}},{"id":"temporal/truncate","description":"truncate to a calendar unit","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"ts","type":"datetime[us]"},{"name":"ms","type":"datetime[ms]"},{"name":"d","type":"date"},{"name":"dur","type":"duration[us]"}],"columns":{"k":["a","b","c","d","e","f"],"ts":[1709214307123456,1704067199999000,1709510400000000,null,-14182940000000,1710052200000500],"ms":[1709214307123,1704067200000,null,1709251200000,0,1709942400000],"d":[19782,19722,19786,19782,null,19792],"dur":[5400000000,-1500000,null,259200000001,0,90000000000]}},"right":{"fields":[{"name":"d","type":"date"},{"name":"name","type":"utf8"}],"columns":{"d":[19782,19792,20089],"name":["leap","sunday","new year"]}}},"ops":[{"op":"select","exprs":{"h":{"fn":"truncate","args":[{"col":"ts"}],"every":"1h"},"d":{"fn":"truncate","args":[{"col":"ts"}],"every":"1d"},"w":{"fn":"truncate","args":[{"col":"ts"}],"every":"1w"},"mo":{"fn":"truncate","args":[{"col":"ts"}],"every":"1mo"},"y":{"fn":"truncate","args":[{"col":"ts"}],"every":"1y"},"dmo":{"fn":"truncate","args":[{"col":"d"}],"every":"1mo"}}}],"expected":{"fields":[{"name":"h","type":"datetime[us]"},{"name":"d","type":"datetime[us]"},{"name":"w","type":"datetime[us]"},{"name":"mo","type":"datetime[us]"},{"name":"y","type":"datetime[us]"},{"name":"dmo","type":"date"}],"columns":{"h":[1709211600000000,1704063600000000,1709510400000000,null,-14184000000000,1710050400000000],"d":[1709164800000000,1703980800000000,1709510400000000,null,-14256000000000,1710028800000000],"w":[1708905600000000,1703462400000000,1709510400000000,null,-14774400000000,1709510400000000],"mo":[1706745600000000,1701388800000000,1709251200000000,null,-15897600000000,1709251200000000],"y":[1704067200000000,1672531200000000,1704067200000000,null,-31536000000000,1704067200000000],"dmo":[19754,19692,19783,19754,null,19783]}}},{"id":"temporal/duration","description":"durations in whole units","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"ts","type":"datetime[us]"},{"name":"ms","type":"datetime[ms]"},{"name":"d","type":"date"},{"name":"dur","type":"duration[us]"}],"columns":{"k":["a","b","c","d","e","f"],"ts":[1709214307123456,1704067199999000,1709510400000000,null,-14182940000000,1710052200000500],"ms":[1709214307123,1704067200000,null,1709251200000,0,1709942400000],"d":[19782,19722,19786,19782,null,19792],"dur":[5400000000,-1500000,null,259200000001,0,90000000000]}},"right":{"fields":[{"name":"d","type":"date"},{"name":"name","type":"utf8"}],"columns":{"d":[19782,19792,20089],"name":["leap","sunday","new year"]}}},"ops":[{"op":"select","exprs":{"d":{"fn":"totalDays","args":[{"col":"dur"}]},"h":{"fn":"totalHours","args":[{"col":"dur"}]},"m":{"fn":"totalMinutes","args":[{"col":"dur"}]},"s":{"fn":"totalSeconds","args":[{"col":"dur"}]},"ms":{"fn":"totalMilliseconds","args":[{"col":"dur"}]}}}],"expected":{"fields":[{"name":"d","type":"i64"},{"name":"h","type":"i64"},{"name":"m","type":"i64"},{"name":"s","type":"i64"},{"name":"ms","type":"i64"}],"columns":{"d":["0","0",null,"3","0","1"],"h":["1","0",null,"72","0","25"],"m":["90","0",null,"4320","0","1500"],"s":["5400","-1",null,"259200","0","90000"],"ms":["5400000","-1500",null,"259200000","0","90000000"]}}},{"id":"temporal/cast","description":"casts between dates, timestamps and integers","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"ts","type":"datetime[us]"},{"name":"ms","type":"datetime[ms]"},{"name":"d","type":"date"},{"name":"dur","type":"duration[us]"}],"columns":{"k":["a","b","c","d","e","f"],"ts":[1709214307123456,1704067199999000,1709510400000000,null,-14182940000000,1710052200000500],"ms":[1709214307123,1704067200000,null,1709251200000,0,1709942400000],"d":[19782,19722,19786,19782,null,19792],"dur":[5400000000,-1500000,null,259200000001,0,90000000000]}},"right":{"fields":[{"name":"d","type":"date"},{"name":"name","type":"utf8"}],"columns":{"d":[19782,19792,20089],"name":["leap","sunday","new year"]}}},"ops":[{"op":"select","exprs":{"date":{"fn":"cast","args":[{"col":"ts"}],"to":"date"},"ts":{"fn":"cast","args":[{"col":"d"}],"to":"datetime[us]"},"us":{"fn":"cast","args":[{"col":"ts"}],"to":"i64"},"days":{"fn":"cast","args":[{"col":"d"}],"to":"i32"},"ms":{"fn":"cast","args":[{"col":"ts"}],"to":"datetime[ms]"}}}],"expected":{"fields":[{"name":"date","type":"date"},{"name":"ts","type":"datetime[us]"},{"name":"us","type":"i64"},{"name":"days","type":"i32"},{"name":"ms","type":"datetime[ms]"}],"columns":{"date":[19782,19722,19786,null,-165,19792],"ts":[1709164800000000,1703980800000000,1709510400000000,1709164800000000,null,1710028800000000],"us":["1709214307123456","1704067199999000","1709510400000000",null,"-14182940000000","1710052200000500"],"days":[19782,19722,19786,19782,null,19792],"ms":[1709214307123,1704067199999,1709510400000,null,-14182940000,1710052200000]}}},{"id":"temporal/compare","description":"comparisons with timestamp literals and between units","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"ts","type":"datetime[us]"},{"name":"ms","type":"datetime[ms]"},{"name":"d","type":"date"},{"name":"dur","type":"duration[us]"}],"columns":{"k":["a","b","c","d","e","f"],"ts":[1709214307123456,1704067199999000,1709510400000000,null,-14182940000000,1710052200000500],"ms":[1709214307123,1704067200000,null,1709251200000,0,1709942400000],"d":[19782,19722,19786,19782,null,19792],"dur":[5400000000,-1500000,null,259200000001,0,90000000000]}},"right":{"fields":[{"name":"d","type":"date"},{"name":"name","type":"utf8"}],"columns":{"d":[19782,19792,20089],"name":["leap","sunday","new year"]}}},"ops":[{"op":"filter","predicate":{"fn":"and","args":[{"fn":"gte","args":[{"col":"ts"},{"lit":1709251200000000,"type":"datetime[us]"}]},{"fn":"lt","args":[{"col":"ms"},{"col":"ts"}]}]}}],"expected":{"fields":[{"name":"k","type":"utf8"},{"name":"ts","type":"datetime[us]"},{"name":"ms","type":"datetime[ms]"},{"name":"d","type":"date"},{"name":"dur","type":"duration[us]"}],"columns":{"k":["f"],"ts":[1710052200000500],"ms":[1709942400000],"d":[19792],"dur":[90000000000]}}},{"id":"temporal/group-sort","description":"group by date, aggregate timestamps, sort","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"ts","type":"datetime[us]"},{"name":"ms","type":"datetime[ms]"},{"name":"d","type":"date"},{"name":"dur","type":"duration[us]"}],"columns":{"k":["a","b","c","d","e","f"],"ts":[1709214307123456,1704067199999000,1709510400000000,null,-14182940000000,1710052200000500],"ms":[1709214307123,1704067200000,null,1709251200000,0,1709942400000],"d":[19782,19722,19786,19782,null,19792],"dur":[5400000000,-1500000,null,259200000001,0,90000000000]}},"right":{"fields":[{"name":"d","type":"date"},{"name":"name","type":"utf8"}],"columns":{"d":[19782,19792,20089],"name":["leap","sunday","new year"]}}},"ops":[{"op":"groupBy","keys":["d"],"aggs":{"first":{"agg":"min","arg":{"col":"ts"}},"last":{"agg":"max","arg":{"col":"ts"}},"n":{"agg":"len"},"total":{"agg":"sum","arg":{"col":"dur"}}}},{"op":"sort","keys":[{"expr":{"col":"last"},"descending":true}]}],"expected":{"fields":[{"name":"d","type":"date"},{"name":"first","type":"datetime[us]"},{"name":"last","type":"datetime[us]"},{"name":"n","type":"u32"},{"name":"total","type":"duration[us]"}],"columns":{"d":[19792,19786,19782,19722,null],"first":[1710052200000500,1709510400000000,1709214307123456,1704067199999000,-14182940000000],"last":[1710052200000500,1709510400000000,1709214307123456,1704067199999000,-14182940000000],"n":[1,1,2,1,1],"total":[90000000000,0,264600000001,-1500000,0]}}},{"id":"temporal/join","description":"join on a date","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"ts","type":"datetime[us]"},{"name":"ms","type":"datetime[ms]"},{"name":"d","type":"date"},{"name":"dur","type":"duration[us]"}],"columns":{"k":["a","b","c","d","e","f"],"ts":[1709214307123456,1704067199999000,1709510400000000,null,-14182940000000,1710052200000500],"ms":[1709214307123,1704067200000,null,1709251200000,0,1709942400000],"d":[19782,19722,19786,19782,null,19792],"dur":[5400000000,-1500000,null,259200000001,0,90000000000]}},"right":{"fields":[{"name":"d","type":"date"},{"name":"name","type":"utf8"}],"columns":{"d":[19782,19792,20089],"name":["leap","sunday","new year"]}}},"ops":[{"op":"join","right":"right","on":["d"],"how":"left"}],"expected":{"fields":[{"name":"k","type":"utf8"},{"name":"ts","type":"datetime[us]"},{"name":"ms","type":"datetime[ms]"},{"name":"d","type":"date"},{"name":"dur","type":"duration[us]"},{"name":"name","type":"utf8"}],"columns":{"k":["a","b","c","d","e","f"],"ts":[1709214307123456,1704067199999000,1709510400000000,null,-14182940000000,1710052200000500],"ms":[1709214307123,1704067200000,null,1709251200000,0,1709942400000],"d":[19782,19722,19786,19782,null,19792],"dur":[5400000000,-1500000,null,259200000001,0,90000000000],"name":["leap",null,null,"leap",null,"sunday"]}}},{"id":"nested/list-ops","description":"list length, element access and aggregates","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"g","type":"i32"},{"name":"xs","type":"list"},{"name":"ws","type":"list"},{"name":"s","type":"struct"}],"columns":{"k":["a","b","c","d","e"],"g":[1,2,1,2,1],"xs":[[1,2,3],[],null,[4,null,2],[7]],"ws":[["a","b"],["c"],[],null,["z","a"]],"s":[{"a":1,"b":"x"},{"a":null,"b":"y"},null,{"a":4,"b":null},{"a":5,"b":"z"}]}},"right":{"fields":[{"name":"k","type":"utf8"},{"name":"tags","type":"list"}],"columns":{"k":["a","d","x"],"tags":[["t1"],["t2","t3"],[]]}}},"ops":[{"op":"select","exprs":{"n":{"fn":"listLength","args":[{"col":"xs"}]},"first":{"fn":"listGet","args":[{"col":"xs"}],"index":0},"last":{"fn":"listGet","args":[{"col":"xs"}],"index":-1},"oob":{"fn":"listGet","args":[{"col":"xs"}],"index":5},"sum":{"fn":"listSum","args":[{"col":"xs"}]},"mean":{"fn":"listMean","args":[{"col":"xs"}]},"min":{"fn":"listMin","args":[{"col":"xs"}]},"max":{"fn":"listMax","args":[{"col":"ws"}]},"has2":{"fn":"listContains","args":[{"col":"xs"},{"lit":2}]},"hasA":{"fn":"listContains","args":[{"col":"ws"},{"lit":"a"}]}}}],"expected":{"fields":[{"name":"n","type":"u32"},{"name":"first","type":"i32"},{"name":"last","type":"i32"},{"name":"oob","type":"i32"},{"name":"sum","type":"i32"},{"name":"mean","type":"f64"},{"name":"min","type":"i32"},{"name":"max","type":"utf8"},{"name":"has2","type":"bool"},{"name":"hasA","type":"bool"}],"columns":{"n":[3,0,null,3,1],"first":[1,null,null,4,7],"last":[3,null,null,2,7],"oob":[null,null,null,null,null],"sum":[6,0,null,6,7],"mean":[2.0,null,null,3.0,7.0],"min":[1,null,null,2,7],"max":["b","c",null,null,"z"],"has2":[true,false,null,true,false],"hasA":[true,false,false,null,true]}}},{"id":"nested/struct","description":"struct fields and struct construction","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"g","type":"i32"},{"name":"xs","type":"list"},{"name":"ws","type":"list"},{"name":"s","type":"struct"}],"columns":{"k":["a","b","c","d","e"],"g":[1,2,1,2,1],"xs":[[1,2,3],[],null,[4,null,2],[7]],"ws":[["a","b"],["c"],[],null,["z","a"]],"s":[{"a":1,"b":"x"},{"a":null,"b":"y"},null,{"a":4,"b":null},{"a":5,"b":"z"}]}},"right":{"fields":[{"name":"k","type":"utf8"},{"name":"tags","type":"list"}],"columns":{"k":["a","d","x"],"tags":[["t1"],["t2","t3"],[]]}}},"ops":[{"op":"select","exprs":{"a":{"fn":"field","args":[{"col":"s"}],"field":"a"},"b":{"fn":"field","args":[{"col":"s"}],"field":"b"},"pair":{"fn":"struct","args":[{"col":"k"},{"fn":"listLength","args":[{"col":"xs"}]}],"names":["key","size"]}}}],"expected":{"fields":[{"name":"a","type":"i32"},{"name":"b","type":"utf8"},{"name":"pair","type":"struct"}],"columns":{"a":[1,null,null,4,5],"b":["x","y",null,null,"z"],"pair":[{"key":"a","size":3},{"key":"b","size":0},{"key":"c","size":null},{"key":"d","size":3},{"key":"e","size":1}]}}},{"id":"nested/explode","description":"explode lists into rows; a null list gives a null row, an empty list no rows","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"g","type":"i32"},{"name":"xs","type":"list"},{"name":"ws","type":"list"},{"name":"s","type":"struct"}],"columns":{"k":["a","b","c","d","e"],"g":[1,2,1,2,1],"xs":[[1,2,3],[],null,[4,null,2],[7]],"ws":[["a","b"],["c"],[],null,["z","a"]],"s":[{"a":1,"b":"x"},{"a":null,"b":"y"},null,{"a":4,"b":null},{"a":5,"b":"z"}]}},"right":{"fields":[{"name":"k","type":"utf8"},{"name":"tags","type":"list"}],"columns":{"k":["a","d","x"],"tags":[["t1"],["t2","t3"],[]]}}},"ops":[{"op":"pick","columns":["k","xs"]},{"op":"explode","column":"xs"}],"expected":{"fields":[{"name":"k","type":"utf8"},{"name":"xs","type":"i32"}],"columns":{"k":["a","a","a","c","d","d","d","e"],"xs":[1,2,3,null,4,null,2,7]}}},{"id":"nested/unnest","description":"unnest a struct into columns","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"g","type":"i32"},{"name":"xs","type":"list"},{"name":"ws","type":"list"},{"name":"s","type":"struct"}],"columns":{"k":["a","b","c","d","e"],"g":[1,2,1,2,1],"xs":[[1,2,3],[],null,[4,null,2],[7]],"ws":[["a","b"],["c"],[],null,["z","a"]],"s":[{"a":1,"b":"x"},{"a":null,"b":"y"},null,{"a":4,"b":null},{"a":5,"b":"z"}]}},"right":{"fields":[{"name":"k","type":"utf8"},{"name":"tags","type":"list"}],"columns":{"k":["a","d","x"],"tags":[["t1"],["t2","t3"],[]]}}},"ops":[{"op":"pick","columns":["k","s"]},{"op":"unnest","column":"s"}],"expected":{"fields":[{"name":"k","type":"utf8"},{"name":"a","type":"i32"},{"name":"b","type":"utf8"}],"columns":{"k":["a","b","c","d","e"],"a":[1,null,null,4,5],"b":["x","y",null,null,"z"]}}},{"id":"nested/agg-list","description":"collect values into lists per group","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"g","type":"i32"},{"name":"xs","type":"list"},{"name":"ws","type":"list"},{"name":"s","type":"struct"}],"columns":{"k":["a","b","c","d","e"],"g":[1,2,1,2,1],"xs":[[1,2,3],[],null,[4,null,2],[7]],"ws":[["a","b"],["c"],[],null,["z","a"]],"s":[{"a":1,"b":"x"},{"a":null,"b":"y"},null,{"a":4,"b":null},{"a":5,"b":"z"}]}},"right":{"fields":[{"name":"k","type":"utf8"},{"name":"tags","type":"list"}],"columns":{"k":["a","d","x"],"tags":[["t1"],["t2","t3"],[]]}}},"ops":[{"op":"groupBy","keys":["g"],"aggs":{"ks":{"agg":"list","arg":{"col":"k"}},"firsts":{"agg":"first","arg":{"col":"xs"}},"n":{"agg":"len"}}}],"expected":{"fields":[{"name":"g","type":"i32"},{"name":"ks","type":"list"},{"name":"firsts","type":"list"},{"name":"n","type":"u32"}],"columns":{"g":[1,2],"ks":[["a","c","e"],["b","d"]],"firsts":[[1,2,3],[]],"n":[3,2]}}},{"id":"nested/carry","description":"lists and structs survive filters, sorts, joins and slices","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"g","type":"i32"},{"name":"xs","type":"list"},{"name":"ws","type":"list"},{"name":"s","type":"struct"}],"columns":{"k":["a","b","c","d","e"],"g":[1,2,1,2,1],"xs":[[1,2,3],[],null,[4,null,2],[7]],"ws":[["a","b"],["c"],[],null,["z","a"]],"s":[{"a":1,"b":"x"},{"a":null,"b":"y"},null,{"a":4,"b":null},{"a":5,"b":"z"}]}},"right":{"fields":[{"name":"k","type":"utf8"},{"name":"tags","type":"list"}],"columns":{"k":["a","d","x"],"tags":[["t1"],["t2","t3"],[]]}}},"ops":[{"op":"filter","predicate":{"fn":"neq","args":[{"col":"k"},{"lit":"c"}]}},{"op":"sort","keys":[{"expr":{"col":"g"},"descending":true},{"expr":{"col":"k"}}]},{"op":"join","right":"right","on":["k"],"how":"left"},{"op":"head","n":4}],"expected":{"fields":[{"name":"k","type":"utf8"},{"name":"g","type":"i32"},{"name":"xs","type":"list"},{"name":"ws","type":"list"},{"name":"s","type":"struct"},{"name":"tags","type":"list"}],"columns":{"k":["b","d","a","e"],"g":[2,2,1,1],"xs":[[],[4,null,2],[1,2,3],[7]],"ws":[["c"],null,["a","b"],["z","a"]],"s":[{"a":null,"b":"y"},{"a":4,"b":null},{"a":1,"b":"x"},{"a":5,"b":"z"}],"tags":[null,["t2","t3"],["t1"],null]}}},{"id":"decimal/arith","description":"decimal addition, subtraction and multiplication","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"price","type":"decimal[10,2]"},{"name":"fee","type":"decimal[6,3]"},{"name":"qty","type":"decimal[4,0]"},{"name":"x","type":"f64"}],"columns":{"k":["a","b","a","c","b","a"],"price":["12.34","0.10","-5.00",null,"99999999.99","10.01"],"fee":["0.125","1.000",null,"2.500","-0.001","10.010"],"qty":["3","10","2","1",null,"7"],"x":[1.005,-2.5,123.456,null,1000000.0,0.0]}}},"ops":[{"op":"select","exprs":{"sum":{"fn":"add","args":[{"col":"price"},{"col":"fee"}]},"diff":{"fn":"sub","args":[{"col":"price"},{"col":"fee"}]},"prod":{"fn":"mul","args":[{"col":"price"},{"col":"qty"}]},"plus":{"fn":"add","args":[{"col":"price"},{"lit":"0.5","type":"decimal[3,1]"}]}}}],"expected":{"fields":[{"name":"sum","type":"decimal[38,3]"},{"name":"diff","type":"decimal[38,3]"},{"name":"prod","type":"decimal[38,2]"},{"name":"plus","type":"decimal[38,2]"}],"columns":{"sum":["12.465","1.100",null,null,"99999999.989","20.020"],"diff":["12.215","-0.900",null,null,"99999999.991","0.000"],"prod":["37.02","1.00","-10.00",null,null,"70.07"],"plus":["12.84","0.60","-4.50",null,"100000000.49","10.51"]}}},{"id":"decimal/compare","description":"decimal comparisons, also across scales","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"price","type":"decimal[10,2]"},{"name":"fee","type":"decimal[6,3]"},{"name":"qty","type":"decimal[4,0]"},{"name":"x","type":"f64"}],"columns":{"k":["a","b","a","c","b","a"],"price":["12.34","0.10","-5.00",null,"99999999.99","10.01"],"fee":["0.125","1.000",null,"2.500","-0.001","10.010"],"qty":["3","10","2","1",null,"7"],"x":[1.005,-2.5,123.456,null,1000000.0,0.0]}}},"ops":[{"op":"filter","predicate":{"fn":"and","args":[{"fn":"gt","args":[{"col":"price"},{"lit":"10.00","type":"decimal[10,2]"}]},{"fn":"gte","args":[{"col":"price"},{"col":"fee"}]}]}}],"expected":{"fields":[{"name":"k","type":"utf8"},{"name":"price","type":"decimal[10,2]"},{"name":"fee","type":"decimal[6,3]"},{"name":"qty","type":"decimal[4,0]"},{"name":"x","type":"f64"}],"columns":{"k":["a","b","a"],"price":["12.34","99999999.99","10.01"],"fee":["0.125","-0.001","10.010"],"qty":["3",null,"7"],"x":[1.005,1000000.0,0.0]}}},{"id":"decimal/aggregate","description":"sums, extremes and means of decimals per group","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"price","type":"decimal[10,2]"},{"name":"fee","type":"decimal[6,3]"},{"name":"qty","type":"decimal[4,0]"},{"name":"x","type":"f64"}],"columns":{"k":["a","b","a","c","b","a"],"price":["12.34","0.10","-5.00",null,"99999999.99","10.01"],"fee":["0.125","1.000",null,"2.500","-0.001","10.010"],"qty":["3","10","2","1",null,"7"],"x":[1.005,-2.5,123.456,null,1000000.0,0.0]}}},"ops":[{"op":"groupBy","keys":["k"],"aggs":{"total":{"agg":"sum","arg":{"col":"price"}},"min":{"agg":"min","arg":{"col":"price"}},"max":{"agg":"max","arg":{"col":"fee"}},"mean":{"agg":"mean","arg":{"col":"price"}},"n":{"agg":"count","arg":{"col":"price"}}}}],"expected":{"fields":[{"name":"k","type":"utf8"},{"name":"total","type":"decimal[38,2]"},{"name":"min","type":"decimal[10,2]"},{"name":"max","type":"decimal[6,3]"},{"name":"mean","type":"f64"},{"name":"n","type":"u32"}],"columns":{"k":["a","b","c"],"total":["17.35","100000000.09","0.00"],"min":["-5.00","0.10",null],"max":["10.010","1.000","2.500"],"mean":[5.783333333333334,50000000.044999994,null],"n":[3,2,0]}}},{"id":"decimal/cast","description":"casts between decimals, floats, integers and strings","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"price","type":"decimal[10,2]"},{"name":"fee","type":"decimal[6,3]"},{"name":"qty","type":"decimal[4,0]"},{"name":"x","type":"f64"}],"columns":{"k":["a","b","a","c","b","a"],"price":["12.34","0.10","-5.00",null,"99999999.99","10.01"],"fee":["0.125","1.000",null,"2.500","-0.001","10.010"],"qty":["3","10","2","1",null,"7"],"x":[1.005,-2.5,123.456,null,1000000.0,0.0]}}},"ops":[{"op":"select","exprs":{"f":{"fn":"cast","args":[{"col":"price"}],"to":"f64"},"i":{"fn":"cast","args":[{"col":"price"}],"to":"i32"},"s":{"fn":"cast","args":[{"col":"fee"}],"to":"utf8"},"d":{"fn":"cast","args":[{"col":"x"}],"to":"decimal[10,2]"},"r":{"fn":"cast","args":[{"col":"price"}],"to":"decimal[12,1]"}}}],"expected":{"fields":[{"name":"f","type":"f64"},{"name":"i","type":"i32"},{"name":"s","type":"utf8"},{"name":"d","type":"decimal[10,2]"},{"name":"r","type":"decimal[12,1]"}],"columns":{"f":[12.34,0.1,-5.0,null,99999999.99,10.01],"i":[12,0,-5,null,99999999,10],"s":["0.125","1.000",null,"2.500","-0.001","10.010"],"d":["1.00","-2.50","123.46",null,"1000000.00","0.00"],"r":["12.3","0.1","-5.0",null,"100000000.0","10.0"]}}},{"id":"decimal/sort","description":"sort by a decimal","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"price","type":"decimal[10,2]"},{"name":"fee","type":"decimal[6,3]"},{"name":"qty","type":"decimal[4,0]"},{"name":"x","type":"f64"}],"columns":{"k":["a","b","a","c","b","a"],"price":["12.34","0.10","-5.00",null,"99999999.99","10.01"],"fee":["0.125","1.000",null,"2.500","-0.001","10.010"],"qty":["3","10","2","1",null,"7"],"x":[1.005,-2.5,123.456,null,1000000.0,0.0]}}},"ops":[{"op":"sort","keys":[{"expr":{"col":"price"},"descending":true}],"nullsLast":true}],"expected":{"fields":[{"name":"k","type":"utf8"},{"name":"price","type":"decimal[10,2]"},{"name":"fee","type":"decimal[6,3]"},{"name":"qty","type":"decimal[4,0]"},{"name":"x","type":"f64"}],"columns":{"k":["b","a","a","b","a","c"],"price":["99999999.99","12.34","10.01","0.10","-5.00",null],"fee":["-0.001","0.125","10.010","1.000",null,"2.500"],"qty":[null,"3","7","10","2","1"],"x":[1000000.0,1.005,0.0,-2.5,123.456,null]}}},{"id":"decimal/round-ties","description":"narrowing the scale rounds exact ties","inputs":{"left":{"fields":[{"name":"t","type":"decimal[6,3]"}],"columns":{"t":["0.250","0.350","-0.250","-0.350","1.050","2.500","0.249"]}}},"ops":[{"op":"select","exprs":{"one":{"fn":"cast","args":[{"col":"t"}],"to":"decimal[6,1]"},"zero":{"fn":"cast","args":[{"col":"t"}],"to":"decimal[6,0]"}}}],"expected":{"fields":[{"name":"one","type":"decimal[6,1]"},{"name":"zero","type":"decimal[6,0]"}],"columns":{"one":["0.2","0.4","-0.2","-0.4","1.0","2.5","0.2"],"zero":["0","0","0","0","1","2","0"]}}}]} +{"generator":"generate_gold.py","polars":"2.1.0","deviations":["JavaScript has no float literal distinct from an integer: an integral number literal behaves like a Polars integer literal (it takes the integer type of the column it meets), so cases never combine an integer column with an integral float literal.","Float to string casts use JavaScript number formatting (Polars prints 5.0 as '5.0', JavaScript as '5').","Joins never add '_right' suffixes: overlapping non-key columns are a type error.","Arithmetic between 64-bit integers (bigint) and other numeric types requires an explicit cast."],"cases":[{"id":"window/cum","description":"cumulative sum, min, max and count; nulls stay null","inputs":{"left":{"fields":[{"name":"g","type":"utf8"},{"name":"t","type":"i32"},{"name":"x","type":"f64"},{"name":"i","type":"i32"},{"name":"b","type":"bool"}],"columns":{"g":["a","b","a","a","b",null,"a","b","b","a"],"t":[1,1,2,3,2,1,4,3,4,5],"x":[1.5,2.0,null,4.0,"NaN",3.0,-1.0,2.0,0.5,2.0],"i":[3,1,4,1,5,9,null,6,5,3],"b":[true,false,null,true,true,false,true,null,false,true]}}},"ops":[{"op":"select","exprs":{"sx":{"fn":"cumSum","args":[{"col":"x"}]},"si":{"fn":"cumSum","args":[{"col":"i"}]},"mn":{"fn":"cumMin","args":[{"col":"i"}]},"mx":{"fn":"cumMax","args":[{"col":"x"}]},"n":{"fn":"cumCount","args":[{"col":"x"}]},"sb":{"fn":"cumSum","args":[{"col":"b"}]}}}],"expected":{"fields":[{"name":"sx","type":"f64"},{"name":"si","type":"i32"},{"name":"mn","type":"i32"},{"name":"mx","type":"f64"},{"name":"n","type":"u32"},{"name":"sb","type":"u32"}],"columns":{"sx":[1.5,3.5,null,7.5,"NaN","NaN","NaN","NaN","NaN","NaN"],"si":[3,4,8,9,14,23,null,29,34,37],"mn":[3,1,1,1,1,1,null,1,1,1],"mx":[1.5,2.0,null,4.0,4.0,4.0,4.0,4.0,4.0,4.0],"n":[1,2,2,3,4,5,6,7,8,9],"sb":[1,1,null,2,3,3,4,null,4,5]}}},{"id":"window/shift-diff","description":"shift (lag and lead) and diff","inputs":{"left":{"fields":[{"name":"g","type":"utf8"},{"name":"t","type":"i32"},{"name":"x","type":"f64"},{"name":"i","type":"i32"},{"name":"b","type":"bool"}],"columns":{"g":["a","b","a","a","b",null,"a","b","b","a"],"t":[1,1,2,3,2,1,4,3,4,5],"x":[1.5,2.0,null,4.0,"NaN",3.0,-1.0,2.0,0.5,2.0],"i":[3,1,4,1,5,9,null,6,5,3],"b":[true,false,null,true,true,false,true,null,false,true]}}},"ops":[{"op":"select","exprs":{"lag":{"fn":"shift","args":[{"col":"i"}]},"lead":{"fn":"shift","args":[{"col":"x"}],"n":-2},"d":{"fn":"diff","args":[{"col":"i"}]},"d2":{"fn":"diff","args":[{"col":"x"}],"n":2},"ls":{"fn":"shift","args":[{"col":"g"}]}}}],"expected":{"fields":[{"name":"lag","type":"i32"},{"name":"lead","type":"f64"},{"name":"d","type":"i32"},{"name":"d2","type":"f64"},{"name":"ls","type":"utf8"}],"columns":{"lag":[null,3,1,4,1,5,9,null,6,5],"lead":[null,4.0,"NaN",3.0,-1.0,2.0,0.5,2.0,null,null],"d":[null,-2,3,-3,4,4,null,null,-1,-2],"d2":[null,null,null,2.0,null,-1.0,"NaN",-1.0,1.5,0.0],"ls":[null,"a","b","a","a","b",null,"a","b","b"]}}},{"id":"window/rank","description":"rank methods, nulls and NaN","inputs":{"left":{"fields":[{"name":"g","type":"utf8"},{"name":"t","type":"i32"},{"name":"x","type":"f64"},{"name":"i","type":"i32"},{"name":"b","type":"bool"}],"columns":{"g":["a","b","a","a","b",null,"a","b","b","a"],"t":[1,1,2,3,2,1,4,3,4,5],"x":[1.5,2.0,null,4.0,"NaN",3.0,-1.0,2.0,0.5,2.0],"i":[3,1,4,1,5,9,null,6,5,3],"b":[true,false,null,true,true,false,true,null,false,true]}}},"ops":[{"op":"select","exprs":{"avg":{"fn":"rank","args":[{"col":"x"}]},"min":{"fn":"rank","args":[{"col":"i"}],"method":"min"},"max":{"fn":"rank","args":[{"col":"i"}],"method":"max"},"dense":{"fn":"rank","args":[{"col":"i"}],"method":"dense","descending":true},"ord":{"fn":"rank","args":[{"col":"x"}],"method":"ordinal"},"s":{"fn":"rank","args":[{"col":"g"}],"method":"dense"}}}],"expected":{"fields":[{"name":"avg","type":"f64"},{"name":"min","type":"u32"},{"name":"max","type":"u32"},{"name":"dense","type":"u32"},{"name":"ord","type":"u32"},{"name":"s","type":"u32"}],"columns":{"avg":[3.0,5.0,null,8.0,9.0,7.0,1.0,5.0,2.0,5.0],"min":[3,1,5,1,6,9,null,8,6,3],"max":[4,2,5,2,7,9,null,8,7,4],"dense":[5,6,4,6,3,1,null,2,3,5],"ord":[3,4,null,8,9,7,1,5,2,6],"s":[1,2,1,1,2,null,1,2,2,1]}}},{"id":"window/rolling","description":"rolling sum, mean, min and max","inputs":{"left":{"fields":[{"name":"g","type":"utf8"},{"name":"t","type":"i32"},{"name":"x","type":"f64"},{"name":"i","type":"i32"},{"name":"b","type":"bool"}],"columns":{"g":["a","b","a","a","b",null,"a","b","b","a"],"t":[1,1,2,3,2,1,4,3,4,5],"x":[1.5,2.0,null,4.0,"NaN",3.0,-1.0,2.0,0.5,2.0],"i":[3,1,4,1,5,9,null,6,5,3],"b":[true,false,null,true,true,false,true,null,false,true]}}},"ops":[{"op":"select","exprs":{"s":{"fn":"rollingSum","args":[{"col":"x"}],"window":3},"m":{"fn":"rollingMean","args":[{"col":"i"}],"window":2},"mn":{"fn":"rollingMin","args":[{"col":"x"}],"window":3,"minPeriods":1},"mx":{"fn":"rollingMax","args":[{"col":"i"}],"window":4,"minPeriods":2}}}],"expected":{"fields":[{"name":"s","type":"f64"},{"name":"m","type":"f64"},{"name":"mn","type":"f64"},{"name":"mx","type":"i32"}],"columns":{"s":[null,null,null,null,null,"NaN","NaN",4.0,1.5,4.5],"m":[null,2.0,2.5,2.5,3.0,7.0,null,null,5.5,4.0],"mn":[1.5,1.5,1.5,2.0,"NaN","NaN","NaN",-1.0,-1.0,0.5],"mx":[null,3,4,4,5,9,9,9,9,6]}}},{"id":"window/over-agg","description":"aggregates broadcast over partitions","inputs":{"left":{"fields":[{"name":"g","type":"utf8"},{"name":"t","type":"i32"},{"name":"x","type":"f64"},{"name":"i","type":"i32"},{"name":"b","type":"bool"}],"columns":{"g":["a","b","a","a","b",null,"a","b","b","a"],"t":[1,1,2,3,2,1,4,3,4,5],"x":[1.5,2.0,null,4.0,"NaN",3.0,-1.0,2.0,0.5,2.0],"i":[3,1,4,1,5,9,null,6,5,3],"b":[true,false,null,true,true,false,true,null,false,true]}}},"ops":[{"op":"withColumns","exprs":{"total":{"over":{"agg":"sum","arg":{"col":"x"}},"partitionBy":[{"col":"g"}]},"n":{"over":{"agg":"len"},"partitionBy":[{"col":"g"}]},"mean":{"over":{"agg":"mean","arg":{"col":"i"}},"partitionBy":[{"col":"g"}]},"first":{"over":{"agg":"first","arg":{"col":"t"}},"partitionBy":[{"col":"g"},{"col":"b"}]}}}],"expected":{"fields":[{"name":"g","type":"utf8"},{"name":"t","type":"i32"},{"name":"x","type":"f64"},{"name":"i","type":"i32"},{"name":"b","type":"bool"},{"name":"total","type":"f64"},{"name":"n","type":"u32"},{"name":"mean","type":"f64"},{"name":"first","type":"i32"}],"columns":{"g":["a","b","a","a","b",null,"a","b","b","a"],"t":[1,1,2,3,2,1,4,3,4,5],"x":[1.5,2.0,null,4.0,"NaN",3.0,-1.0,2.0,0.5,2.0],"i":[3,1,4,1,5,9,null,6,5,3],"b":[true,false,null,true,true,false,true,null,false,true],"total":[6.5,"NaN",6.5,6.5,"NaN",3.0,6.5,"NaN","NaN",6.5],"n":[5,4,5,5,4,1,5,4,4,5],"mean":[2.75,4.25,2.75,2.75,4.25,9.0,2.75,4.25,4.25,2.75],"first":[1,1,2,1,2,1,1,3,1,1]}}},{"id":"window/over-cum","description":"cumulative and shifted values per partition, in row order","inputs":{"left":{"fields":[{"name":"g","type":"utf8"},{"name":"t","type":"i32"},{"name":"x","type":"f64"},{"name":"i","type":"i32"},{"name":"b","type":"bool"}],"columns":{"g":["a","b","a","a","b",null,"a","b","b","a"],"t":[1,1,2,3,2,1,4,3,4,5],"x":[1.5,2.0,null,4.0,"NaN",3.0,-1.0,2.0,0.5,2.0],"i":[3,1,4,1,5,9,null,6,5,3],"b":[true,false,null,true,true,false,true,null,false,true]}}},"ops":[{"op":"withColumns","exprs":{"cs":{"over":{"fn":"cumSum","args":[{"col":"i"}]},"partitionBy":[{"col":"g"}]},"prev":{"over":{"fn":"shift","args":[{"col":"x"}]},"partitionBy":[{"col":"g"}]},"d":{"over":{"fn":"diff","args":[{"col":"t"}]},"partitionBy":[{"col":"g"}]},"r":{"over":{"fn":"rank","args":[{"col":"x"}],"method":"min"},"partitionBy":[{"col":"g"}]},"roll":{"over":{"fn":"rollingSum","args":[{"col":"i"}],"window":2,"minPeriods":1},"partitionBy":[{"col":"g"}]}}}],"expected":{"fields":[{"name":"g","type":"utf8"},{"name":"t","type":"i32"},{"name":"x","type":"f64"},{"name":"i","type":"i32"},{"name":"b","type":"bool"},{"name":"cs","type":"i32"},{"name":"prev","type":"f64"},{"name":"d","type":"i32"},{"name":"r","type":"u32"},{"name":"roll","type":"i32"}],"columns":{"g":["a","b","a","a","b",null,"a","b","b","a"],"t":[1,1,2,3,2,1,4,3,4,5],"x":[1.5,2.0,null,4.0,"NaN",3.0,-1.0,2.0,0.5,2.0],"i":[3,1,4,1,5,9,null,6,5,3],"b":[true,false,null,true,true,false,true,null,false,true],"cs":[3,1,7,8,6,9,null,12,17,11],"prev":[null,null,1.5,null,2.0,null,4.0,"NaN",2.0,-1.0],"d":[null,null,1,1,1,null,1,1,1,1],"r":[2,2,null,4,4,1,1,2,1,3],"roll":[3,1,7,5,6,9,1,11,11,3]}}},{"id":"window/over-filter","description":"filters after windows see the windowed values","inputs":{"left":{"fields":[{"name":"g","type":"utf8"},{"name":"t","type":"i32"},{"name":"x","type":"f64"},{"name":"i","type":"i32"},{"name":"b","type":"bool"}],"columns":{"g":["a","b","a","a","b",null,"a","b","b","a"],"t":[1,1,2,3,2,1,4,3,4,5],"x":[1.5,2.0,null,4.0,"NaN",3.0,-1.0,2.0,0.5,2.0],"i":[3,1,4,1,5,9,null,6,5,3],"b":[true,false,null,true,true,false,true,null,false,true]}}},"ops":[{"op":"withColumns","exprs":{"cs":{"over":{"fn":"cumSum","args":[{"col":"t"}]},"partitionBy":[{"col":"g"}]}}},{"op":"filter","predicate":{"fn":"gt","args":[{"col":"t"},{"lit":2}]}}],"expected":{"fields":[{"name":"g","type":"utf8"},{"name":"t","type":"i32"},{"name":"x","type":"f64"},{"name":"i","type":"i32"},{"name":"b","type":"bool"},{"name":"cs","type":"i32"}],"columns":{"g":["a","a","b","b","a"],"t":[3,4,3,4,5],"x":[4.0,-1.0,2.0,0.5,2.0],"i":[1,null,6,5,3],"b":[true,true,null,false,true],"cs":[6,10,6,10,15]}}},{"id":"window/filter-first","description":"windows after a filter only see the remaining rows","inputs":{"left":{"fields":[{"name":"g","type":"utf8"},{"name":"t","type":"i32"},{"name":"x","type":"f64"},{"name":"i","type":"i32"},{"name":"b","type":"bool"}],"columns":{"g":["a","b","a","a","b",null,"a","b","b","a"],"t":[1,1,2,3,2,1,4,3,4,5],"x":[1.5,2.0,null,4.0,"NaN",3.0,-1.0,2.0,0.5,2.0],"i":[3,1,4,1,5,9,null,6,5,3],"b":[true,false,null,true,true,false,true,null,false,true]}}},"ops":[{"op":"filter","predicate":{"fn":"isNotNull","args":[{"col":"x"}]}},{"op":"withColumns","exprs":{"cs":{"fn":"cumSum","args":[{"col":"x"}]},"r":{"over":{"fn":"rank","args":[{"col":"x"}]},"partitionBy":[{"col":"g"}]}}},{"op":"head","n":4}],"expected":{"fields":[{"name":"g","type":"utf8"},{"name":"t","type":"i32"},{"name":"x","type":"f64"},{"name":"i","type":"i32"},{"name":"b","type":"bool"},{"name":"cs","type":"f64"},{"name":"r","type":"f64"}],"columns":{"g":["a","b","a","b"],"t":[1,1,3,2],"x":[1.5,2.0,4.0,"NaN"],"i":[3,1,1,5],"b":[true,false,true,true],"cs":[1.5,3.5,7.5,"NaN"],"r":[2.0,2.5,4.0,4.0]}}},{"id":"pivot/sum","description":"pivot with a sum per cell; missing cells are null","inputs":{"left":{"fields":[{"name":"region","type":"utf8"},{"name":"quarter","type":"utf8"},{"name":"units","type":"i32"},{"name":"price","type":"f64"}],"columns":{"region":["north","north","south","south","north","east","south","north"],"quarter":["q1","q2","q1","q3","q1","q2","q1",null],"units":[10,20,5,null,7,3,8,1],"price":[1.5,2.0,1.0,3.0,null,2.5,1.0,9.0]}}},"ops":[{"op":"pivot","on":"quarter","index":["region"],"values":"units","aggregate":"sum"}],"expected":{"fields":[{"name":"region","type":"utf8"},{"name":"q1","type":"i32"},{"name":"q2","type":"i32"},{"name":"q3","type":"i32"},{"name":"null","type":"i32"}],"columns":{"region":["north","south","east"],"q1":[17,13,0],"q2":[20,0,3],"q3":[0,0,0],"null":[1,0,0]}}},{"id":"pivot/first","description":"pivot with the first value per cell","inputs":{"left":{"fields":[{"name":"region","type":"utf8"},{"name":"quarter","type":"utf8"},{"name":"units","type":"i32"},{"name":"price","type":"f64"}],"columns":{"region":["north","north","south","south","north","east","south","north"],"quarter":["q1","q2","q1","q3","q1","q2","q1",null],"units":[10,20,5,null,7,3,8,1],"price":[1.5,2.0,1.0,3.0,null,2.5,1.0,9.0]}}},"ops":[{"op":"pivot","on":"region","index":["quarter"],"values":"price","aggregate":"first"}],"expected":{"fields":[{"name":"quarter","type":"utf8"},{"name":"north","type":"f64"},{"name":"south","type":"f64"},{"name":"east","type":"f64"}],"columns":{"quarter":["q1","q2","q3",null],"north":[1.5,2.0,null,9.0],"south":[1.0,null,3.0,null],"east":[null,2.5,null,null]}}},{"id":"pivot/len","description":"pivot counting rows per cell","inputs":{"left":{"fields":[{"name":"region","type":"utf8"},{"name":"quarter","type":"utf8"},{"name":"units","type":"i32"},{"name":"price","type":"f64"}],"columns":{"region":["north","north","south","south","north","east","south","north"],"quarter":["q1","q2","q1","q3","q1","q2","q1",null],"units":[10,20,5,null,7,3,8,1],"price":[1.5,2.0,1.0,3.0,null,2.5,1.0,9.0]}}},"ops":[{"op":"pivot","on":"quarter","index":["region"],"values":"units","aggregate":"len"}],"expected":{"fields":[{"name":"region","type":"utf8"},{"name":"q1","type":"u32"},{"name":"q2","type":"u32"},{"name":"q3","type":"u32"},{"name":"null","type":"u32"}],"columns":{"region":["north","south","east"],"q1":[2,2,0],"q2":[1,0,1],"q3":[0,1,0],"null":[1,0,0]}}},{"id":"pivot/columns","description":"pivot with declared output columns","inputs":{"left":{"fields":[{"name":"region","type":"utf8"},{"name":"quarter","type":"utf8"},{"name":"units","type":"i32"},{"name":"price","type":"f64"}],"columns":{"region":["north","north","south","south","north","east","south","north"],"quarter":["q1","q2","q1","q3","q1","q2","q1",null],"units":[10,20,5,null,7,3,8,1],"price":[1.5,2.0,1.0,3.0,null,2.5,1.0,9.0]}}},"ops":[{"op":"pivot","on":"quarter","index":["region"],"values":"price","aggregate":"max","columns":["q1","q2","q4"]}],"expected":{"fields":[{"name":"region","type":"utf8"},{"name":"q1","type":"f64"},{"name":"q2","type":"f64"},{"name":"q4","type":"f64"}],"columns":{"region":["north","south","east"],"q1":[1.5,1.0,null],"q2":[2.0,null,2.5],"q4":[null,null,null]}}},{"id":"pivot/numeric-on","description":"pivot on a numeric column names columns by value","inputs":{"left":{"fields":[{"name":"g","type":"utf8"},{"name":"t","type":"i32"},{"name":"x","type":"f64"},{"name":"i","type":"i32"},{"name":"b","type":"bool"}],"columns":{"g":["a","b","a","a","b",null,"a","b","b","a"],"t":[1,1,2,3,2,1,4,3,4,5],"x":[1.5,2.0,null,4.0,"NaN",3.0,-1.0,2.0,0.5,2.0],"i":[3,1,4,1,5,9,null,6,5,3],"b":[true,false,null,true,true,false,true,null,false,true]}}},"ops":[{"op":"filter","predicate":{"fn":"isNotNull","args":[{"col":"g"}]}},{"op":"pivot","on":"t","index":["g"],"values":"i","aggregate":"min"}],"expected":{"fields":[{"name":"g","type":"utf8"},{"name":"1","type":"i32"},{"name":"2","type":"i32"},{"name":"3","type":"i32"},{"name":"4","type":"i32"},{"name":"5","type":"i32"}],"columns":{"g":["a","b"],"1":[3,1],"2":[4,5],"3":[1,6],"4":[null,5],"5":[3,null]}}},{"id":"unpivot/basic","description":"unpivot stacks columns, column by column","inputs":{"left":{"fields":[{"name":"region","type":"utf8"},{"name":"quarter","type":"utf8"},{"name":"units","type":"i32"},{"name":"price","type":"f64"}],"columns":{"region":["north","north","south","south","north","east","south","north"],"quarter":["q1","q2","q1","q3","q1","q2","q1",null],"units":[10,20,5,null,7,3,8,1],"price":[1.5,2.0,1.0,3.0,null,2.5,1.0,9.0]}}},"ops":[{"op":"unpivot","index":["region"],"on":["units","price"]}],"expected":{"fields":[{"name":"region","type":"utf8"},{"name":"variable","type":"utf8"},{"name":"value","type":"f64"}],"columns":{"region":["north","north","south","south","north","east","south","north","north","north","south","south","north","east","south","north"],"variable":["units","units","units","units","units","units","units","units","price","price","price","price","price","price","price","price"],"value":[10.0,20.0,5.0,null,7.0,3.0,8.0,1.0,1.5,2.0,1.0,3.0,null,2.5,1.0,9.0]}}},{"id":"unpivot/names","description":"unpivot with named variable and value columns","inputs":{"left":{"fields":[{"name":"region","type":"utf8"},{"name":"quarter","type":"utf8"},{"name":"units","type":"i32"},{"name":"price","type":"f64"}],"columns":{"region":["north","north","south","south","north","east","south","north"],"quarter":["q1","q2","q1","q3","q1","q2","q1",null],"units":[10,20,5,null,7,3,8,1],"price":[1.5,2.0,1.0,3.0,null,2.5,1.0,9.0]}}},"ops":[{"op":"pick","columns":["region","quarter"]},{"op":"unpivot","index":[],"on":["region","quarter"],"variable":"field","value":"text"}],"expected":{"fields":[{"name":"field","type":"utf8"},{"name":"text","type":"utf8"}],"columns":{"field":["region","region","region","region","region","region","region","region","quarter","quarter","quarter","quarter","quarter","quarter","quarter","quarter"],"text":["north","north","south","south","north","east","south","north","q1","q2","q1","q3","q1","q2","q1",null]}}},{"id":"temporal/extract","description":"calendar fields of timestamps and dates","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"ts","type":"datetime[us]"},{"name":"ms","type":"datetime[ms]"},{"name":"d","type":"date"},{"name":"dur","type":"duration[us]"}],"columns":{"k":["a","b","c","d","e","f"],"ts":[1709214307123456,1704067199999000,1709510400000000,null,-14182940000000,1710052200000500],"ms":[1709214307123,1704067200000,null,1709251200000,0,1709942400000],"d":[19782,19722,19786,19782,null,19792],"dur":[5400000000,-1500000,null,259200000001,0,90000000000]}},"right":{"fields":[{"name":"d","type":"date"},{"name":"name","type":"utf8"}],"columns":{"d":[19782,19792,20089],"name":["leap","sunday","new year"]}}},"ops":[{"op":"select","exprs":{"y":{"fn":"year","args":[{"col":"ts"}]},"mo":{"fn":"month","args":[{"col":"ts"}]},"d":{"fn":"day","args":[{"col":"ts"}]},"h":{"fn":"hour","args":[{"col":"ts"}]},"mi":{"fn":"minute","args":[{"col":"ts"}]},"s":{"fn":"second","args":[{"col":"ts"}]},"ms":{"fn":"millisecond","args":[{"col":"ts"}]},"wd":{"fn":"weekday","args":[{"col":"ts"}]},"od":{"fn":"ordinalDay","args":[{"col":"ts"}]},"dy":{"fn":"year","args":[{"col":"d"}]},"dwd":{"fn":"weekday","args":[{"col":"d"}]},"dod":{"fn":"ordinalDay","args":[{"col":"d"}]}}}],"expected":{"fields":[{"name":"y","type":"i32"},{"name":"mo","type":"i8"},{"name":"d","type":"i8"},{"name":"h","type":"i8"},{"name":"mi","type":"i8"},{"name":"s","type":"i8"},{"name":"ms","type":"i32"},{"name":"wd","type":"i8"},{"name":"od","type":"i16"},{"name":"dy","type":"i32"},{"name":"dwd","type":"i8"},{"name":"dod","type":"i16"}],"columns":{"y":[2024,2023,2024,null,1969,2024],"mo":[2,12,3,null,7,3],"d":[29,31,4,null,20,10],"h":[13,23,0,null,20,6],"mi":[45,59,0,null,17,30],"s":[7,59,0,null,40,0],"ms":[123,999,0,null,0,0],"wd":[4,7,1,null,7,7],"od":[60,365,64,null,201,70],"dy":[2024,2023,2024,2024,null,2024],"dwd":[4,7,1,4,null,7],"dod":[60,365,64,60,null,70]}}},{"id":"temporal/arith","description":"timestamp differences are durations; durations shift timestamps","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"ts","type":"datetime[us]"},{"name":"ms","type":"datetime[ms]"},{"name":"d","type":"date"},{"name":"dur","type":"duration[us]"}],"columns":{"k":["a","b","c","d","e","f"],"ts":[1709214307123456,1704067199999000,1709510400000000,null,-14182940000000,1710052200000500],"ms":[1709214307123,1704067200000,null,1709251200000,0,1709942400000],"d":[19782,19722,19786,19782,null,19792],"dur":[5400000000,-1500000,null,259200000001,0,90000000000]}},"right":{"fields":[{"name":"d","type":"date"},{"name":"name","type":"utf8"}],"columns":{"d":[19782,19792,20089],"name":["leap","sunday","new year"]}}},"ops":[{"op":"select","exprs":{"gap":{"fn":"sub","args":[{"col":"ts"},{"col":"ms"}]},"later":{"fn":"add","args":[{"col":"ts"},{"col":"dur"}]},"earlier":{"fn":"sub","args":[{"col":"ts"},{"col":"dur"}]},"days":{"fn":"sub","args":[{"col":"d"},{"lit":18000,"type":"date"}]},"twice":{"fn":"add","args":[{"col":"dur"},{"col":"dur"}]}}}],"expected":{"fields":[{"name":"gap","type":"duration[ms]"},{"name":"later","type":"datetime[us]"},{"name":"earlier","type":"datetime[us]"},{"name":"days","type":"duration[us]"},{"name":"twice","type":"duration[us]"}],"columns":{"gap":[0,-1,null,null,-14182940000,109800000],"later":[1709219707123456,1704067198499000,null,null,-14182940000000,1710142200000500],"earlier":[1709208907123456,1704067201499000,null,null,-14182940000000,1709962200000500],"days":[153964800000000,148780800000000,154310400000000,153964800000000,null,154828800000000],"twice":[10800000000,-3000000,null,518400000002,0,180000000000]}}},{"id":"temporal/truncate","description":"truncate to a calendar unit","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"ts","type":"datetime[us]"},{"name":"ms","type":"datetime[ms]"},{"name":"d","type":"date"},{"name":"dur","type":"duration[us]"}],"columns":{"k":["a","b","c","d","e","f"],"ts":[1709214307123456,1704067199999000,1709510400000000,null,-14182940000000,1710052200000500],"ms":[1709214307123,1704067200000,null,1709251200000,0,1709942400000],"d":[19782,19722,19786,19782,null,19792],"dur":[5400000000,-1500000,null,259200000001,0,90000000000]}},"right":{"fields":[{"name":"d","type":"date"},{"name":"name","type":"utf8"}],"columns":{"d":[19782,19792,20089],"name":["leap","sunday","new year"]}}},"ops":[{"op":"select","exprs":{"h":{"fn":"truncate","args":[{"col":"ts"}],"every":"1h"},"d":{"fn":"truncate","args":[{"col":"ts"}],"every":"1d"},"w":{"fn":"truncate","args":[{"col":"ts"}],"every":"1w"},"mo":{"fn":"truncate","args":[{"col":"ts"}],"every":"1mo"},"y":{"fn":"truncate","args":[{"col":"ts"}],"every":"1y"},"dmo":{"fn":"truncate","args":[{"col":"d"}],"every":"1mo"}}}],"expected":{"fields":[{"name":"h","type":"datetime[us]"},{"name":"d","type":"datetime[us]"},{"name":"w","type":"datetime[us]"},{"name":"mo","type":"datetime[us]"},{"name":"y","type":"datetime[us]"},{"name":"dmo","type":"date"}],"columns":{"h":[1709211600000000,1704063600000000,1709510400000000,null,-14184000000000,1710050400000000],"d":[1709164800000000,1703980800000000,1709510400000000,null,-14256000000000,1710028800000000],"w":[1708905600000000,1703462400000000,1709510400000000,null,-14774400000000,1709510400000000],"mo":[1706745600000000,1701388800000000,1709251200000000,null,-15897600000000,1709251200000000],"y":[1704067200000000,1672531200000000,1704067200000000,null,-31536000000000,1704067200000000],"dmo":[19754,19692,19783,19754,null,19783]}}},{"id":"temporal/duration","description":"durations in whole units","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"ts","type":"datetime[us]"},{"name":"ms","type":"datetime[ms]"},{"name":"d","type":"date"},{"name":"dur","type":"duration[us]"}],"columns":{"k":["a","b","c","d","e","f"],"ts":[1709214307123456,1704067199999000,1709510400000000,null,-14182940000000,1710052200000500],"ms":[1709214307123,1704067200000,null,1709251200000,0,1709942400000],"d":[19782,19722,19786,19782,null,19792],"dur":[5400000000,-1500000,null,259200000001,0,90000000000]}},"right":{"fields":[{"name":"d","type":"date"},{"name":"name","type":"utf8"}],"columns":{"d":[19782,19792,20089],"name":["leap","sunday","new year"]}}},"ops":[{"op":"select","exprs":{"d":{"fn":"totalDays","args":[{"col":"dur"}]},"h":{"fn":"totalHours","args":[{"col":"dur"}]},"m":{"fn":"totalMinutes","args":[{"col":"dur"}]},"s":{"fn":"totalSeconds","args":[{"col":"dur"}]},"ms":{"fn":"totalMilliseconds","args":[{"col":"dur"}]}}}],"expected":{"fields":[{"name":"d","type":"i64"},{"name":"h","type":"i64"},{"name":"m","type":"i64"},{"name":"s","type":"i64"},{"name":"ms","type":"i64"}],"columns":{"d":["0","0",null,"3","0","1"],"h":["1","0",null,"72","0","25"],"m":["90","0",null,"4320","0","1500"],"s":["5400","-1",null,"259200","0","90000"],"ms":["5400000","-1500",null,"259200000","0","90000000"]}}},{"id":"temporal/cast","description":"casts between dates, timestamps and integers","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"ts","type":"datetime[us]"},{"name":"ms","type":"datetime[ms]"},{"name":"d","type":"date"},{"name":"dur","type":"duration[us]"}],"columns":{"k":["a","b","c","d","e","f"],"ts":[1709214307123456,1704067199999000,1709510400000000,null,-14182940000000,1710052200000500],"ms":[1709214307123,1704067200000,null,1709251200000,0,1709942400000],"d":[19782,19722,19786,19782,null,19792],"dur":[5400000000,-1500000,null,259200000001,0,90000000000]}},"right":{"fields":[{"name":"d","type":"date"},{"name":"name","type":"utf8"}],"columns":{"d":[19782,19792,20089],"name":["leap","sunday","new year"]}}},"ops":[{"op":"select","exprs":{"date":{"fn":"cast","args":[{"col":"ts"}],"to":"date"},"ts":{"fn":"cast","args":[{"col":"d"}],"to":"datetime[us]"},"us":{"fn":"cast","args":[{"col":"ts"}],"to":"i64"},"days":{"fn":"cast","args":[{"col":"d"}],"to":"i32"},"ms":{"fn":"cast","args":[{"col":"ts"}],"to":"datetime[ms]"}}}],"expected":{"fields":[{"name":"date","type":"date"},{"name":"ts","type":"datetime[us]"},{"name":"us","type":"i64"},{"name":"days","type":"i32"},{"name":"ms","type":"datetime[ms]"}],"columns":{"date":[19782,19722,19786,null,-165,19792],"ts":[1709164800000000,1703980800000000,1709510400000000,1709164800000000,null,1710028800000000],"us":["1709214307123456","1704067199999000","1709510400000000",null,"-14182940000000","1710052200000500"],"days":[19782,19722,19786,19782,null,19792],"ms":[1709214307123,1704067199999,1709510400000,null,-14182940000,1710052200000]}}},{"id":"temporal/compare","description":"comparisons with timestamp literals and between units","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"ts","type":"datetime[us]"},{"name":"ms","type":"datetime[ms]"},{"name":"d","type":"date"},{"name":"dur","type":"duration[us]"}],"columns":{"k":["a","b","c","d","e","f"],"ts":[1709214307123456,1704067199999000,1709510400000000,null,-14182940000000,1710052200000500],"ms":[1709214307123,1704067200000,null,1709251200000,0,1709942400000],"d":[19782,19722,19786,19782,null,19792],"dur":[5400000000,-1500000,null,259200000001,0,90000000000]}},"right":{"fields":[{"name":"d","type":"date"},{"name":"name","type":"utf8"}],"columns":{"d":[19782,19792,20089],"name":["leap","sunday","new year"]}}},"ops":[{"op":"filter","predicate":{"fn":"and","args":[{"fn":"gte","args":[{"col":"ts"},{"lit":1709251200000000,"type":"datetime[us]"}]},{"fn":"lt","args":[{"col":"ms"},{"col":"ts"}]}]}}],"expected":{"fields":[{"name":"k","type":"utf8"},{"name":"ts","type":"datetime[us]"},{"name":"ms","type":"datetime[ms]"},{"name":"d","type":"date"},{"name":"dur","type":"duration[us]"}],"columns":{"k":["f"],"ts":[1710052200000500],"ms":[1709942400000],"d":[19792],"dur":[90000000000]}}},{"id":"temporal/group-sort","description":"group by date, aggregate timestamps, sort","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"ts","type":"datetime[us]"},{"name":"ms","type":"datetime[ms]"},{"name":"d","type":"date"},{"name":"dur","type":"duration[us]"}],"columns":{"k":["a","b","c","d","e","f"],"ts":[1709214307123456,1704067199999000,1709510400000000,null,-14182940000000,1710052200000500],"ms":[1709214307123,1704067200000,null,1709251200000,0,1709942400000],"d":[19782,19722,19786,19782,null,19792],"dur":[5400000000,-1500000,null,259200000001,0,90000000000]}},"right":{"fields":[{"name":"d","type":"date"},{"name":"name","type":"utf8"}],"columns":{"d":[19782,19792,20089],"name":["leap","sunday","new year"]}}},"ops":[{"op":"groupBy","keys":["d"],"aggs":{"first":{"agg":"min","arg":{"col":"ts"}},"last":{"agg":"max","arg":{"col":"ts"}},"n":{"agg":"len"},"total":{"agg":"sum","arg":{"col":"dur"}}}},{"op":"sort","keys":[{"expr":{"col":"last"},"descending":true}]}],"expected":{"fields":[{"name":"d","type":"date"},{"name":"first","type":"datetime[us]"},{"name":"last","type":"datetime[us]"},{"name":"n","type":"u32"},{"name":"total","type":"duration[us]"}],"columns":{"d":[19792,19786,19782,19722,null],"first":[1710052200000500,1709510400000000,1709214307123456,1704067199999000,-14182940000000],"last":[1710052200000500,1709510400000000,1709214307123456,1704067199999000,-14182940000000],"n":[1,1,2,1,1],"total":[90000000000,0,264600000001,-1500000,0]}}},{"id":"temporal/join","description":"join on a date","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"ts","type":"datetime[us]"},{"name":"ms","type":"datetime[ms]"},{"name":"d","type":"date"},{"name":"dur","type":"duration[us]"}],"columns":{"k":["a","b","c","d","e","f"],"ts":[1709214307123456,1704067199999000,1709510400000000,null,-14182940000000,1710052200000500],"ms":[1709214307123,1704067200000,null,1709251200000,0,1709942400000],"d":[19782,19722,19786,19782,null,19792],"dur":[5400000000,-1500000,null,259200000001,0,90000000000]}},"right":{"fields":[{"name":"d","type":"date"},{"name":"name","type":"utf8"}],"columns":{"d":[19782,19792,20089],"name":["leap","sunday","new year"]}}},"ops":[{"op":"join","right":"right","on":["d"],"how":"left"}],"expected":{"fields":[{"name":"k","type":"utf8"},{"name":"ts","type":"datetime[us]"},{"name":"ms","type":"datetime[ms]"},{"name":"d","type":"date"},{"name":"dur","type":"duration[us]"},{"name":"name","type":"utf8"}],"columns":{"k":["a","b","c","d","e","f"],"ts":[1709214307123456,1704067199999000,1709510400000000,null,-14182940000000,1710052200000500],"ms":[1709214307123,1704067200000,null,1709251200000,0,1709942400000],"d":[19782,19722,19786,19782,null,19792],"dur":[5400000000,-1500000,null,259200000001,0,90000000000],"name":["leap",null,null,"leap",null,"sunday"]}}},{"id":"nested/list-ops","description":"list length, element access and aggregates","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"g","type":"i32"},{"name":"xs","type":"list"},{"name":"ws","type":"list"},{"name":"s","type":"struct"}],"columns":{"k":["a","b","c","d","e"],"g":[1,2,1,2,1],"xs":[[1,2,3],[],null,[4,null,2],[7]],"ws":[["a","b"],["c"],[],null,["z","a"]],"s":[{"a":1,"b":"x"},{"a":null,"b":"y"},null,{"a":4,"b":null},{"a":5,"b":"z"}]}},"right":{"fields":[{"name":"k","type":"utf8"},{"name":"tags","type":"list"}],"columns":{"k":["a","d","x"],"tags":[["t1"],["t2","t3"],[]]}}},"ops":[{"op":"select","exprs":{"n":{"fn":"listLength","args":[{"col":"xs"}]},"first":{"fn":"listGet","args":[{"col":"xs"}],"index":0},"last":{"fn":"listGet","args":[{"col":"xs"}],"index":-1},"oob":{"fn":"listGet","args":[{"col":"xs"}],"index":5},"sum":{"fn":"listSum","args":[{"col":"xs"}]},"mean":{"fn":"listMean","args":[{"col":"xs"}]},"min":{"fn":"listMin","args":[{"col":"xs"}]},"max":{"fn":"listMax","args":[{"col":"ws"}]},"has2":{"fn":"listContains","args":[{"col":"xs"},{"lit":2}]},"hasA":{"fn":"listContains","args":[{"col":"ws"},{"lit":"a"}]}}}],"expected":{"fields":[{"name":"n","type":"u32"},{"name":"first","type":"i32"},{"name":"last","type":"i32"},{"name":"oob","type":"i32"},{"name":"sum","type":"i32"},{"name":"mean","type":"f64"},{"name":"min","type":"i32"},{"name":"max","type":"utf8"},{"name":"has2","type":"bool"},{"name":"hasA","type":"bool"}],"columns":{"n":[3,0,null,3,1],"first":[1,null,null,4,7],"last":[3,null,null,2,7],"oob":[null,null,null,null,null],"sum":[6,0,null,6,7],"mean":[2.0,null,null,3.0,7.0],"min":[1,null,null,2,7],"max":["b","c",null,null,"z"],"has2":[true,false,null,true,false],"hasA":[true,false,false,null,true]}}},{"id":"nested/struct","description":"struct fields and struct construction","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"g","type":"i32"},{"name":"xs","type":"list"},{"name":"ws","type":"list"},{"name":"s","type":"struct"}],"columns":{"k":["a","b","c","d","e"],"g":[1,2,1,2,1],"xs":[[1,2,3],[],null,[4,null,2],[7]],"ws":[["a","b"],["c"],[],null,["z","a"]],"s":[{"a":1,"b":"x"},{"a":null,"b":"y"},null,{"a":4,"b":null},{"a":5,"b":"z"}]}},"right":{"fields":[{"name":"k","type":"utf8"},{"name":"tags","type":"list"}],"columns":{"k":["a","d","x"],"tags":[["t1"],["t2","t3"],[]]}}},"ops":[{"op":"select","exprs":{"a":{"fn":"field","args":[{"col":"s"}],"field":"a"},"b":{"fn":"field","args":[{"col":"s"}],"field":"b"},"pair":{"fn":"struct","args":[{"col":"k"},{"fn":"listLength","args":[{"col":"xs"}]}],"names":["key","size"]}}}],"expected":{"fields":[{"name":"a","type":"i32"},{"name":"b","type":"utf8"},{"name":"pair","type":"struct"}],"columns":{"a":[1,null,null,4,5],"b":["x","y",null,null,"z"],"pair":[{"key":"a","size":3},{"key":"b","size":0},{"key":"c","size":null},{"key":"d","size":3},{"key":"e","size":1}]}}},{"id":"nested/explode","description":"explode lists into rows; a null list gives a null row, an empty list no rows","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"g","type":"i32"},{"name":"xs","type":"list"},{"name":"ws","type":"list"},{"name":"s","type":"struct"}],"columns":{"k":["a","b","c","d","e"],"g":[1,2,1,2,1],"xs":[[1,2,3],[],null,[4,null,2],[7]],"ws":[["a","b"],["c"],[],null,["z","a"]],"s":[{"a":1,"b":"x"},{"a":null,"b":"y"},null,{"a":4,"b":null},{"a":5,"b":"z"}]}},"right":{"fields":[{"name":"k","type":"utf8"},{"name":"tags","type":"list"}],"columns":{"k":["a","d","x"],"tags":[["t1"],["t2","t3"],[]]}}},"ops":[{"op":"pick","columns":["k","xs"]},{"op":"explode","column":"xs"}],"expected":{"fields":[{"name":"k","type":"utf8"},{"name":"xs","type":"i32"}],"columns":{"k":["a","a","a","c","d","d","d","e"],"xs":[1,2,3,null,4,null,2,7]}}},{"id":"nested/unnest","description":"unnest a struct into columns","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"g","type":"i32"},{"name":"xs","type":"list"},{"name":"ws","type":"list"},{"name":"s","type":"struct"}],"columns":{"k":["a","b","c","d","e"],"g":[1,2,1,2,1],"xs":[[1,2,3],[],null,[4,null,2],[7]],"ws":[["a","b"],["c"],[],null,["z","a"]],"s":[{"a":1,"b":"x"},{"a":null,"b":"y"},null,{"a":4,"b":null},{"a":5,"b":"z"}]}},"right":{"fields":[{"name":"k","type":"utf8"},{"name":"tags","type":"list"}],"columns":{"k":["a","d","x"],"tags":[["t1"],["t2","t3"],[]]}}},"ops":[{"op":"pick","columns":["k","s"]},{"op":"unnest","column":"s"}],"expected":{"fields":[{"name":"k","type":"utf8"},{"name":"a","type":"i32"},{"name":"b","type":"utf8"}],"columns":{"k":["a","b","c","d","e"],"a":[1,null,null,4,5],"b":["x","y",null,null,"z"]}}},{"id":"nested/agg-list","description":"collect values into lists per group","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"g","type":"i32"},{"name":"xs","type":"list"},{"name":"ws","type":"list"},{"name":"s","type":"struct"}],"columns":{"k":["a","b","c","d","e"],"g":[1,2,1,2,1],"xs":[[1,2,3],[],null,[4,null,2],[7]],"ws":[["a","b"],["c"],[],null,["z","a"]],"s":[{"a":1,"b":"x"},{"a":null,"b":"y"},null,{"a":4,"b":null},{"a":5,"b":"z"}]}},"right":{"fields":[{"name":"k","type":"utf8"},{"name":"tags","type":"list"}],"columns":{"k":["a","d","x"],"tags":[["t1"],["t2","t3"],[]]}}},"ops":[{"op":"groupBy","keys":["g"],"aggs":{"ks":{"agg":"list","arg":{"col":"k"}},"firsts":{"agg":"first","arg":{"col":"xs"}},"n":{"agg":"len"}}}],"expected":{"fields":[{"name":"g","type":"i32"},{"name":"ks","type":"list"},{"name":"firsts","type":"list"},{"name":"n","type":"u32"}],"columns":{"g":[1,2],"ks":[["a","c","e"],["b","d"]],"firsts":[[1,2,3],[]],"n":[3,2]}}},{"id":"nested/carry","description":"lists and structs survive filters, sorts, joins and slices","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"g","type":"i32"},{"name":"xs","type":"list"},{"name":"ws","type":"list"},{"name":"s","type":"struct"}],"columns":{"k":["a","b","c","d","e"],"g":[1,2,1,2,1],"xs":[[1,2,3],[],null,[4,null,2],[7]],"ws":[["a","b"],["c"],[],null,["z","a"]],"s":[{"a":1,"b":"x"},{"a":null,"b":"y"},null,{"a":4,"b":null},{"a":5,"b":"z"}]}},"right":{"fields":[{"name":"k","type":"utf8"},{"name":"tags","type":"list"}],"columns":{"k":["a","d","x"],"tags":[["t1"],["t2","t3"],[]]}}},"ops":[{"op":"filter","predicate":{"fn":"neq","args":[{"col":"k"},{"lit":"c"}]}},{"op":"sort","keys":[{"expr":{"col":"g"},"descending":true},{"expr":{"col":"k"}}]},{"op":"join","right":"right","on":["k"],"how":"left"},{"op":"head","n":4}],"expected":{"fields":[{"name":"k","type":"utf8"},{"name":"g","type":"i32"},{"name":"xs","type":"list"},{"name":"ws","type":"list"},{"name":"s","type":"struct"},{"name":"tags","type":"list"}],"columns":{"k":["b","d","a","e"],"g":[2,2,1,1],"xs":[[],[4,null,2],[1,2,3],[7]],"ws":[["c"],null,["a","b"],["z","a"]],"s":[{"a":null,"b":"y"},{"a":4,"b":null},{"a":1,"b":"x"},{"a":5,"b":"z"}],"tags":[null,["t2","t3"],["t1"],null]}}},{"id":"decimal/arith","description":"decimal addition, subtraction and multiplication","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"price","type":"decimal[10,2]"},{"name":"fee","type":"decimal[6,3]"},{"name":"qty","type":"decimal[4,0]"},{"name":"x","type":"f64"}],"columns":{"k":["a","b","a","c","b","a"],"price":["12.34","0.10","-5.00",null,"99999999.99","10.01"],"fee":["0.125","1.000",null,"2.500","-0.001","10.010"],"qty":["3","10","2","1",null,"7"],"x":[1.005,-2.5,123.456,null,1000000.0,0.0]}}},"ops":[{"op":"select","exprs":{"sum":{"fn":"add","args":[{"col":"price"},{"col":"fee"}]},"diff":{"fn":"sub","args":[{"col":"price"},{"col":"fee"}]},"prod":{"fn":"mul","args":[{"col":"price"},{"col":"qty"}]},"plus":{"fn":"add","args":[{"col":"price"},{"lit":"0.5","type":"decimal[3,1]"}]}}}],"expected":{"fields":[{"name":"sum","type":"decimal[38,3]"},{"name":"diff","type":"decimal[38,3]"},{"name":"prod","type":"decimal[38,2]"},{"name":"plus","type":"decimal[38,2]"}],"columns":{"sum":["12.465","1.100",null,null,"99999999.989","20.020"],"diff":["12.215","-0.900",null,null,"99999999.991","0.000"],"prod":["37.02","1.00","-10.00",null,null,"70.07"],"plus":["12.84","0.60","-4.50",null,"100000000.49","10.51"]}}},{"id":"decimal/compare","description":"decimal comparisons, also across scales","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"price","type":"decimal[10,2]"},{"name":"fee","type":"decimal[6,3]"},{"name":"qty","type":"decimal[4,0]"},{"name":"x","type":"f64"}],"columns":{"k":["a","b","a","c","b","a"],"price":["12.34","0.10","-5.00",null,"99999999.99","10.01"],"fee":["0.125","1.000",null,"2.500","-0.001","10.010"],"qty":["3","10","2","1",null,"7"],"x":[1.005,-2.5,123.456,null,1000000.0,0.0]}}},"ops":[{"op":"filter","predicate":{"fn":"and","args":[{"fn":"gt","args":[{"col":"price"},{"lit":"10.00","type":"decimal[10,2]"}]},{"fn":"gte","args":[{"col":"price"},{"col":"fee"}]}]}}],"expected":{"fields":[{"name":"k","type":"utf8"},{"name":"price","type":"decimal[10,2]"},{"name":"fee","type":"decimal[6,3]"},{"name":"qty","type":"decimal[4,0]"},{"name":"x","type":"f64"}],"columns":{"k":["a","b","a"],"price":["12.34","99999999.99","10.01"],"fee":["0.125","-0.001","10.010"],"qty":["3",null,"7"],"x":[1.005,1000000.0,0.0]}}},{"id":"decimal/aggregate","description":"sums, extremes and means of decimals per group","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"price","type":"decimal[10,2]"},{"name":"fee","type":"decimal[6,3]"},{"name":"qty","type":"decimal[4,0]"},{"name":"x","type":"f64"}],"columns":{"k":["a","b","a","c","b","a"],"price":["12.34","0.10","-5.00",null,"99999999.99","10.01"],"fee":["0.125","1.000",null,"2.500","-0.001","10.010"],"qty":["3","10","2","1",null,"7"],"x":[1.005,-2.5,123.456,null,1000000.0,0.0]}}},"ops":[{"op":"groupBy","keys":["k"],"aggs":{"total":{"agg":"sum","arg":{"col":"price"}},"min":{"agg":"min","arg":{"col":"price"}},"max":{"agg":"max","arg":{"col":"fee"}},"mean":{"agg":"mean","arg":{"col":"price"}},"n":{"agg":"count","arg":{"col":"price"}}}}],"expected":{"fields":[{"name":"k","type":"utf8"},{"name":"total","type":"decimal[38,2]"},{"name":"min","type":"decimal[10,2]"},{"name":"max","type":"decimal[6,3]"},{"name":"mean","type":"f64"},{"name":"n","type":"u32"}],"columns":{"k":["a","b","c"],"total":["17.35","100000000.09","0.00"],"min":["-5.00","0.10",null],"max":["10.010","1.000","2.500"],"mean":[5.783333333333334,50000000.044999994,null],"n":[3,2,0]}}},{"id":"decimal/cast","description":"casts between decimals, floats, integers and strings","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"price","type":"decimal[10,2]"},{"name":"fee","type":"decimal[6,3]"},{"name":"qty","type":"decimal[4,0]"},{"name":"x","type":"f64"}],"columns":{"k":["a","b","a","c","b","a"],"price":["12.34","0.10","-5.00",null,"99999999.99","10.01"],"fee":["0.125","1.000",null,"2.500","-0.001","10.010"],"qty":["3","10","2","1",null,"7"],"x":[1.005,-2.5,123.456,null,1000000.0,0.0]}}},"ops":[{"op":"select","exprs":{"f":{"fn":"cast","args":[{"col":"price"}],"to":"f64"},"i":{"fn":"cast","args":[{"col":"price"}],"to":"i32"},"s":{"fn":"cast","args":[{"col":"fee"}],"to":"utf8"},"d":{"fn":"cast","args":[{"col":"x"}],"to":"decimal[10,2]"},"r":{"fn":"cast","args":[{"col":"price"}],"to":"decimal[12,1]"}}}],"expected":{"fields":[{"name":"f","type":"f64"},{"name":"i","type":"i32"},{"name":"s","type":"utf8"},{"name":"d","type":"decimal[10,2]"},{"name":"r","type":"decimal[12,1]"}],"columns":{"f":[12.34,0.1,-5.0,null,99999999.99,10.01],"i":[12,0,-5,null,99999999,10],"s":["0.125","1.000",null,"2.500","-0.001","10.010"],"d":["1.00","-2.50","123.46",null,"1000000.00","0.00"],"r":["12.3","0.1","-5.0",null,"100000000.0","10.0"]}}},{"id":"decimal/sort","description":"sort by a decimal","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"price","type":"decimal[10,2]"},{"name":"fee","type":"decimal[6,3]"},{"name":"qty","type":"decimal[4,0]"},{"name":"x","type":"f64"}],"columns":{"k":["a","b","a","c","b","a"],"price":["12.34","0.10","-5.00",null,"99999999.99","10.01"],"fee":["0.125","1.000",null,"2.500","-0.001","10.010"],"qty":["3","10","2","1",null,"7"],"x":[1.005,-2.5,123.456,null,1000000.0,0.0]}}},"ops":[{"op":"sort","keys":[{"expr":{"col":"price"},"descending":true}],"nullsLast":true}],"expected":{"fields":[{"name":"k","type":"utf8"},{"name":"price","type":"decimal[10,2]"},{"name":"fee","type":"decimal[6,3]"},{"name":"qty","type":"decimal[4,0]"},{"name":"x","type":"f64"}],"columns":{"k":["b","a","a","b","a","c"],"price":["99999999.99","12.34","10.01","0.10","-5.00",null],"fee":["-0.001","0.125","10.010","1.000",null,"2.500"],"qty":[null,"3","7","10","2","1"],"x":[1000000.0,1.005,0.0,-2.5,123.456,null]}}},{"id":"decimal/round-ties","description":"narrowing the scale rounds exact ties","inputs":{"left":{"fields":[{"name":"t","type":"decimal[6,3]"}],"columns":{"t":["0.250","0.350","-0.250","-0.350","1.050","2.500","0.249"]}}},"ops":[{"op":"select","exprs":{"one":{"fn":"cast","args":[{"col":"t"}],"to":"decimal[6,1]"},"zero":{"fn":"cast","args":[{"col":"t"}],"to":"decimal[6,0]"}}}],"expected":{"fields":[{"name":"one","type":"decimal[6,1]"},{"name":"zero","type":"decimal[6,0]"}],"columns":{"one":["0.2","0.4","-0.2","-0.4","1.0","2.5","0.2"],"zero":["0","0","0","0","1","2","0"]}}},{"id":"window/filter-after-sort","description":"a window predicate sees the sorted rows","inputs":{"left":{"fields":[{"name":"g","type":"i32"},{"name":"h","type":"i32"},{"name":"x","type":"f64"}],"columns":{"g":[1,2,1,2],"h":[1,1,2,2],"x":[3.0,1.0,2.0,4.0]}},"right":{"fields":[{"name":"g","type":"i32"},{"name":"name","type":"utf8"}],"columns":{"g":[1],"name":["one"]}}},"ops":[{"op":"sort","keys":[{"expr":{"col":"x"}}]},{"op":"filter","predicate":{"fn":"gt","args":[{"fn":"cumSum","args":[{"col":"x"}]},{"lit":3.0}]}}],"expected":{"fields":[{"name":"g","type":"i32"},{"name":"h","type":"i32"},{"name":"x","type":"f64"}],"columns":{"g":[1,2],"h":[1,2],"x":[3.0,4.0]}}},{"id":"window/filter-after-filter","description":"a window predicate sees only the rows an earlier filter kept","inputs":{"left":{"fields":[{"name":"g","type":"i32"},{"name":"h","type":"i32"},{"name":"x","type":"f64"}],"columns":{"g":[1,2,1,2],"h":[1,1,2,2],"x":[3.0,1.0,2.0,4.0]}},"right":{"fields":[{"name":"g","type":"i32"},{"name":"name","type":"utf8"}],"columns":{"g":[1],"name":["one"]}}},"ops":[{"op":"filter","predicate":{"fn":"gt","args":[{"col":"x"},{"lit":1.0}]}},{"op":"filter","predicate":{"fn":"gt","args":[{"fn":"shift","args":[{"col":"x"}]},{"lit":2.0}]}}],"expected":{"fields":[{"name":"g","type":"i32"},{"name":"h","type":"i32"},{"name":"x","type":"f64"}],"columns":{"g":[1],"h":[2],"x":[2.0]}}},{"id":"window/filter-after-concat","description":"a window predicate sees every input of a concat","inputs":{"left":{"fields":[{"name":"g","type":"i32"},{"name":"h","type":"i32"},{"name":"x","type":"f64"}],"columns":{"g":[1,2,1,2],"h":[1,1,2,2],"x":[3.0,1.0,2.0,4.0]}},"right":{"fields":[{"name":"g","type":"i32"},{"name":"name","type":"utf8"}],"columns":{"g":[1],"name":["one"]}}},"ops":[{"op":"concat","with":"left"},{"op":"filter","predicate":{"fn":"gt","args":[{"fn":"cumSum","args":[{"col":"x"}]},{"lit":15.0}]}}],"expected":{"fields":[{"name":"g","type":"i32"},{"name":"h","type":"i32"},{"name":"x","type":"f64"}],"columns":{"g":[1,2],"h":[2,2],"x":[2.0,4.0]}}},{"id":"window/filter-after-join","description":"a window predicate sees only the joined rows","inputs":{"left":{"fields":[{"name":"g","type":"i32"},{"name":"h","type":"i32"},{"name":"x","type":"f64"}],"columns":{"g":[1,2,1,2],"h":[1,1,2,2],"x":[3.0,1.0,2.0,4.0]}},"right":{"fields":[{"name":"g","type":"i32"},{"name":"name","type":"utf8"}],"columns":{"g":[1],"name":["one"]}}},"ops":[{"op":"join","right":"right","on":["g"],"how":"inner"},{"op":"filter","predicate":{"fn":"gt","args":[{"fn":"shift","args":[{"col":"x"}]},{"lit":2.0}]}}],"expected":{"fields":[{"name":"g","type":"i32"},{"name":"h","type":"i32"},{"name":"x","type":"f64"},{"name":"name","type":"utf8"}],"columns":{"g":[1],"h":[2],"x":[2.0],"name":["one"]}}},{"id":"window/filter-whole","description":"a window predicate sees the whole input","inputs":{"left":{"fields":[{"name":"g","type":"i32"},{"name":"h","type":"i32"},{"name":"x","type":"f64"}],"columns":{"g":[1,2,1,2],"h":[1,1,2,2],"x":[3.0,1.0,2.0,4.0]}},"right":{"fields":[{"name":"g","type":"i32"},{"name":"name","type":"utf8"}],"columns":{"g":[1],"name":["one"]}}},"ops":[{"op":"filter","predicate":{"fn":"gt","args":[{"fn":"cumSum","args":[{"col":"x"}]},{"lit":5.0}]}}],"expected":{"fields":[{"name":"g","type":"i32"},{"name":"h","type":"i32"},{"name":"x","type":"f64"}],"columns":{"g":[1,2],"h":[2,2],"x":[2.0,4.0]}}},{"id":"window/agg-of-window","description":"an aggregate of a window sees the whole input","inputs":{"left":{"fields":[{"name":"g","type":"i32"},{"name":"h","type":"i32"},{"name":"x","type":"f64"}],"columns":{"g":[1,2,1,2],"h":[1,1,2,2],"x":[3.0,1.0,2.0,4.0]}},"right":{"fields":[{"name":"g","type":"i32"},{"name":"name","type":"utf8"}],"columns":{"g":[1],"name":["one"]}}},"ops":[{"op":"agg","aggs":{"s":{"agg":"sum","arg":{"fn":"cumSum","args":[{"col":"x"}]}},"n":{"agg":"len"}}}],"expected":{"fields":[{"name":"s","type":"f64"},{"name":"n","type":"u32"}],"columns":{"s":[23.0],"n":[4]}}},{"id":"window/grouped-agg-of-window","description":"windows inside grouped aggregates run per group","inputs":{"left":{"fields":[{"name":"g","type":"i32"},{"name":"h","type":"i32"},{"name":"x","type":"f64"}],"columns":{"g":[1,2,1,2],"h":[1,1,2,2],"x":[3.0,1.0,2.0,4.0]}},"right":{"fields":[{"name":"g","type":"i32"},{"name":"name","type":"utf8"}],"columns":{"g":[1],"name":["one"]}}},"ops":[{"op":"groupBy","keys":["g"],"aggs":{"s":{"agg":"sum","arg":{"fn":"cumSum","args":[{"col":"x"}]}},"l":{"agg":"list","arg":{"fn":"cumSum","args":[{"col":"x"}]}},"o":{"agg":"sum","arg":{"over":{"agg":"sum","arg":{"col":"x"}},"partitionBy":[{"col":"h"}]}}}}],"expected":{"fields":[{"name":"g","type":"i32"},{"name":"s","type":"f64"},{"name":"l","type":"list"},{"name":"o","type":"f64"}],"columns":{"g":[1,2],"s":[8.0,6.0],"l":[[3.0,5.0],[1.0,5.0]],"o":[5.0,5.0]}}}]} From ae7445985fce92e6c00a9c978644800c7df68c49 Mon Sep 17 00:00:00 2001 From: Effect Bot Date: Sun, 15 Nov 2026 11:53:42 +0000 Subject: [PATCH 05/88] Fix streamed reads, slices, empty batches and explain --- .../effect/src/dataframe/internal/query.ts | 104 +++++++++--------- .../effect/src/dataframe/internal/stream.ts | 50 +++++++-- packages/effect/test/dataframe/Query.test.ts | 40 +++++++ 3 files changed, 131 insertions(+), 63 deletions(-) diff --git a/packages/effect/src/dataframe/internal/query.ts b/packages/effect/src/dataframe/internal/query.ts index fd91b36e8b8..1bed04985b6 100644 --- a/packages/effect/src/dataframe/internal/query.ts +++ b/packages/effect/src/dataframe/internal/query.ts @@ -63,55 +63,56 @@ export const attempt = (f: () => A): Effect.Effect => { // Resolving tables // ----------------------------------------------------------------------------- -const values = (frame: F.Frame): Node => ({ _tag: "Values", frame: frame as any }) +/** A frame as a plan node. @internal */ +export const values = (frame: F.Frame): Node => ({ _tag: "Values", frame: frame as any }) const frameOf = (node: { readonly frame: unknown }): F.Frame => node.frame as F.Frame -/** Replaces every `Table` with the catalog's frame. @internal */ -export const resolve = (plan: Plan.Node, catalog: Catalog.Service): Effect.Effect => { - const names = new Set() - const collect = (node: Plan.Node): void => { +/** How many times a plan names each table. @internal */ +export const tables = (plan: Plan.Node): Map => { + const out = new Map() + const walk = (node: Node): void => { + if (node._tag === "Table") out.set(node.name, (out.get(node.name) ?? 0) + 1) + for (const c of children(node)) walk(c) + } + walk(plan as Node) + return out +} + +/** A plan with each `Table` replaced by `f` of it. @internal */ +export const mapTables = (plan: Plan.Node, f: (table: Plan.Table) => Node): Node => { + const replace = (node: Plan.Node): Node => { switch (node._tag) { case "Table": - names.add(node.name) - return + return f(node) case "Values": - return + return node case "Join": - collect(node.left) - collect(node.right) - return + return { ...node, left: replace(node.left), right: replace(node.right) } case "Concat": - for (const n of node.inputs) collect(n) - return + return { ...node, inputs: node.inputs.map(replace) } default: - collect(node.input) + return { ...node, input: replace(node.input) } as Node } } - collect(plan) - if (names.size === 0) return Effect.succeed(plan as Node) - return Effect.forEach([...names], (name) => - catalog.table(name).pipe( - Effect.map((df) => [name, df as unknown as F.Frame] as const) - )).pipe( + return replace(plan) +} + +/** A table's frame, checked against and narrowed to its declared columns. @internal */ +export const tableValues = (table: Plan.Table, frame: F.Frame): Node => + values(table.columns === undefined ? frame : conform(frame, table.columns, table.schema)) + +/** Replaces every `Table` with the catalog's frame, reading each table once. @internal */ +export const resolve = (plan: Plan.Node, catalog: Catalog.Service): Effect.Effect => { + const names = [...tables(plan).keys()] + if (names.length === 0) return Effect.succeed(plan as Node) + return Effect.forEach( + names, + (name) => Effect.map(catalog.table(name), (df) => [name, df as unknown as F.Frame] as const) + ) + .pipe( Effect.flatMap((entries) => { - const tables = new Map(entries) - const replace = (node: Plan.Node): Node => { - switch (node._tag) { - case "Table": { - const frame = tables.get(node.name)! - return values(node.columns === undefined ? frame : conform(frame, node.columns, node.schema)) - } - case "Values": - return node - case "Join": - return { ...node, left: replace(node.left), right: replace(node.right) } - case "Concat": - return { ...node, inputs: node.inputs.map(replace) } - default: - return { ...node, input: replace(node.input) } as Node - } - } - return attempt(() => replace(plan)) + const frames = new Map(entries) + return attempt(() => mapTables(plan, (table) => tableValues(table, frames.get(table.name)!))) }) ) } @@ -619,17 +620,6 @@ export const execute = (plan: Node): Effect.Effect => { return run(plan) } -/** The names of the tables a plan reads. @internal */ -export const tables = (plan: Plan.Node): Set => { - const out = new Set() - const walk = (node: Node): void => { - if (node._tag === "Table") out.add(node.name) - for (const c of children(node)) walk(c) - } - walk(plan as Node) - return out -} - /** * Resolves, optimises and runs a plan. Plans that read streamed tables run * batch by batch (see `stream.ts`). @@ -637,7 +627,7 @@ export const tables = (plan: Plan.Node): Set => { * @internal */ export const collect = (plan: Plan.Node, catalog: Catalog.Service): Effect.Effect => - [...tables(plan)].some(catalog.streaming) + isStreaming(plan, catalog) ? Streaming.collect(plan, catalog).pipe( Effect.withSpan("dataframe.Query", { attributes: { "dataframe.operations": count(plan), "dataframe.streaming": true } @@ -649,8 +639,18 @@ export const collect = (plan: Plan.Node, catalog: Catalog.Service): Effect.Effec Effect.withSpan("dataframe.Query", { attributes: { "dataframe.operations": count(plan) } }) ) -/** The optimised plan, as text. @internal */ +/** + * The optimised plan that runs, as text. Plans over streamed tables show + * them as `Table` nodes, without reading them. + * + * @internal + */ export const explain = (plan: Plan.Node, catalog: Catalog.Service): Effect.Effect => - resolve(plan, catalog).pipe(Effect.flatMap((resolved) => attempt(() => show(optimize(resolved))))) + isStreaming(plan, catalog) + ? attempt(() => show(optimize(plan as Node))) + : resolve(plan, catalog).pipe(Effect.flatMap((resolved) => attempt(() => show(optimize(resolved))))) + +const isStreaming = (plan: Plan.Node, catalog: Catalog.Service): boolean => + [...tables(plan).keys()].some(catalog.streaming) const count = (node: Plan.Node): number => 1 + children(node as Node).reduce((n, c) => n + count(c as Plan.Node), 0) diff --git a/packages/effect/src/dataframe/internal/stream.ts b/packages/effect/src/dataframe/internal/stream.ts index bf36e41c490..fad16b1e863 100644 --- a/packages/effect/src/dataframe/internal/stream.ts +++ b/packages/effect/src/dataframe/internal/stream.ts @@ -64,9 +64,9 @@ const runEmpty = (node: Q.Node): Effect.Effect => { const empty = (n: Q.Node): Q.Node => { switch (n._tag) { case "Table": - return { _tag: "Values", frame: emptyTable(n) as any } + return Q.values(emptyTable(n)) case "Values": - return { _tag: "Values", frame: F.slice(n.frame as unknown as F.Frame, 0, 0) as any } + return Q.values(F.slice(n.frame as unknown as F.Frame, 0, 0)) case "Join": return { ...n, left: empty(n.left), right: empty(n.right) } case "Concat": @@ -127,14 +127,18 @@ const aggregate = ( return collected(node, catalog) } const fns = specs.map((s) => s.fn) - // The first batch's input vectors and key columns, for result types. + // The first non-empty batch's input vectors and key columns, for result types. let template: { readonly inputs: ReadonlyArray; readonly frame: F.Frame } | undefined + // The first batch, to aggregate over no rows when every batch is empty. + let first: F.Frame | undefined return Stream.unwrap( Stream.runFoldEffect( batches(node.input, catalog), (): Partial.State | undefined => undefined, (state, batch) => Q.attempt(() => { + first ??= batch + if (batch.length === 0) return state const ctx = F.context(batch) const inputs = specs.map((s) => (s.arg === undefined ? undefined : E.evaluateColumn(s.arg, ctx))) template ??= { inputs, frame: batch } @@ -147,7 +151,10 @@ const aggregate = ( }) ).pipe( Effect.flatMap((state) => { - if (state === undefined || template === undefined) return runEmpty(node) + if (state === undefined || template === undefined) { + const empty = first + return empty === undefined ? runEmpty(node) : Q.attempt(() => Q.step(node, [empty])) + } const t = template return Q.attempt(() => { const keyColumns = keys.map((k, i) => @@ -165,11 +172,10 @@ const aggregate = ( } const slice = (input: Batches, offset: number, length: number | undefined): Batches => - Stream.suspend(() => { + length === 0 ? Stream.empty : Stream.suspend(() => { let skip = offset let left = length ?? Infinity return input.pipe( - Stream.takeWhile(() => left > 0), Stream.map((batch) => { const start = Math.min(skip, batch.length) skip -= start @@ -177,6 +183,7 @@ const slice = (input: Batches, offset: number, length: number | undefined): Batc left -= end - start return F.slice(batch, start, end) }), + Stream.takeUntil(() => left === 0), Stream.filter((batch) => batch.length > 0) ) }) @@ -222,13 +229,34 @@ export const batches = (node: Q.Node, catalog: Catalog.Service): Batches => { } } +/** + * The optimised plan, with the tables it names more than once read once, + * as frames: each run reads each table once. + */ +const prepare = (plan: Plan.Node, catalog: Catalog.Service): Effect.Effect => { + const repeated = [...Q.tables(plan)].flatMap(([name, n]) => n > 1 ? [name] : []) + return Effect.forEach( + repeated, + (name) => Effect.map(Stream.runCollect(catalog.stream(name)), (frames) => [name, frames] as const) + ).pipe( + Effect.flatMap((entries) => { + const read = new Map(entries) + return Q.attempt(() => + Q.optimize(Q.mapTables(plan, (table) => { + const frames = read.get(table.name) as ReadonlyArray | undefined + if (frames === undefined) return table + if (frames.length === 0) return Q.values(emptyTable(table)) + return Q.tableValues(table, frames.length === 1 ? frames[0] : F.concat(frames)) + })) + ) + }) + ) +} + /** The batches of a plan's result, after optimisation. @internal */ export const stream = (plan: Plan.Node, catalog: Catalog.Service): Batches => - Stream.unwrap(Effect.map(Q.attempt(() => Q.optimize(plan as Q.Node)), (optimized) => batches(optimized, catalog))) + Stream.unwrap(Effect.map(prepare(plan, catalog), (optimized) => batches(optimized, catalog))) /** A plan's result as one frame, computed batch by batch. @internal */ export const collect = (plan: Plan.Node, catalog: Catalog.Service): Effect.Effect => - Effect.flatMap( - Q.attempt(() => Q.optimize(plan as Q.Node)), - (optimized) => collectFrames(batches(optimized, catalog), optimized) - ) + Effect.flatMap(prepare(plan, catalog), (optimized) => collectFrames(batches(optimized, catalog), optimized)) diff --git a/packages/effect/test/dataframe/Query.test.ts b/packages/effect/test/dataframe/Query.test.ts index 1723c74accf..1a66e5c2a38 100644 --- a/packages/effect/test/dataframe/Query.test.ts +++ b/packages/effect/test/dataframe/Query.test.ts @@ -321,6 +321,46 @@ describe("Query", () => { describe("streamed tables", () => { const Row = Schema.Struct({ x: Schema.Number }) const batches = Array.from({ length: 8 }, (_, b) => DataFrame.fromRows(Row, [{ x: 2 * b }, { x: 2 * b + 1 }])) + const t = Query.table("t", Row) + const streamed = (query: Query.Query, source: Catalog.Source) => + rows(query).pipe(Effect.provide(Catalog.layer({ t: source }))) + + it.effect("a slice that has its rows pulls no further batch", () => + Effect.gen(function*() { + const one = Stream.concat(Stream.succeed(DataFrame.fromColumns({ x: [1] })), Stream.die("pulled again")) + assert.deepStrictEqual(yield* streamed(Query.head(t, 1), one), [{ x: 1 }]) + assert.deepStrictEqual(yield* streamed(Query.head(t, 0), Stream.die("pulled")), []) + })) + + it.effect("zero-row batches aggregate like an empty table", () => + Effect.gen(function*() { + const query = Query.aggregate(t, (c) => ({ n: Expr.count(), total: Expr.sum(c.x), mean: Expr.mean(c.x) })) + const empty = Stream.make(DataFrame.fromColumns({ x: Float64Array.of() })) + assert.deepStrictEqual(yield* streamed(query, empty), [{ n: 0, total: 0, mean: null }]) + })) + + it.effect("reads a table named twice once per run", () => + Effect.gen(function*() { + let loads = 0 + const load = Effect.sync(() => DataFrame.fromColumns({ x: [++loads] })) + assert.deepStrictEqual(yield* streamed(Query.concat([t, t]), Stream.fromEffect(load)), [{ x: 1 }, { x: 1 }]) + assert.strictEqual(loads, 1) + const frames = yield* Query.concat([t, t]).pipe( + Query.stream, + Stream.runCollect, + Effect.provide(Catalog.layer({ t: load })) + ) + assert.deepStrictEqual(frames.flatMap((frame) => DataFrame.toArray(frame, "x")), [2, 2]) + assert.strictEqual(loads, 2) + })) + + it.effect("explain shows the plan without reading the stream", () => + Effect.gen(function*() { + const plan = yield* Query.explain(Query.head(t, 1)).pipe( + Effect.provide(Catalog.layer({ t: Stream.die("read") })) + ) + assert.strictEqual(plan, [`Slice offset 0 length 1`, ` Table t [x]`].join("\n")) + })) it.effect("a head stops reading the stream", () => Effect.gen(function*() { From 627171650d11121e046c1d9b68b3c2f375eef866 Mon Sep 17 00:00:00 2001 From: Effect Bot Date: Sun, 15 Nov 2026 11:56:03 +0000 Subject: [PATCH 06/88] Return null for UInt32 floor division and modulo by zero --- .../effect/src/dataframe/internal/kernels.ts | 61 +++++++------------ 1 file changed, 23 insertions(+), 38 deletions(-) diff --git a/packages/effect/src/dataframe/internal/kernels.ts b/packages/effect/src/dataframe/internal/kernels.ts index 10c6b70b11f..6c856baeb68 100644 --- a/packages/effect/src/dataframe/internal/kernels.ts +++ b/packages/effect/src/dataframe/internal/kernels.ts @@ -387,9 +387,12 @@ export const arith = (op: ArithOp, a: Value, b: Value, length: number): Value => case "f64": { const x = lv.values const o = new Float64Array(n) + const y = (rv as V.F64).values const unsigned32 = type._tag === "Int" - if (rs) arithF64Scalar(op, x, (rv as V.F64).values[0], o, n, unsigned32) - else arithF64(op, x, (rv as V.F64).values, o, n, unsigned32) + arithF64(op, x, y, rs ? 0 : 1, o, n, unsigned32) + if (unsigned32 && (op === "floorDiv" || op === "mod")) { + nullOut = Uint8Array.from({ length: n }, (_, i) => y[rs ? 0 : i] === 0 ? 0 : 1) + } V.roundFloat32InPlace(type, o) out = V.f64(type, o, valid) break @@ -461,54 +464,36 @@ const floorMod = (a: number, b: number): number => { return r !== 0 && (r < 0) !== (b < 0) ? r + b : r } -const arithF64 = (op: ArithOp, x: Float64Array, y: Float64Array, o: Float64Array, n: number, u32: boolean) => { - switch (op) { - case "add": - for (let i = 0; i < n; i++) o[i] = x[i] + y[i] - break - case "sub": - for (let i = 0; i < n; i++) o[i] = x[i] - y[i] - break - case "mul": - if (u32) { for (let i = 0; i < n; i++) o[i] = Math.imul(x[i], y[i]) >>> 0 } - else for (let i = 0; i < n; i++) o[i] = x[i] * y[i] - break - case "div": - for (let i = 0; i < n; i++) o[i] = x[i] / y[i] - break - case "floorDiv": - if (u32) { for (let i = 0; i < n; i++) o[i] = Math.floor(x[i] / y[i]) } - else for (let i = 0; i < n; i++) o[i] = Math.floor(x[i] / y[i]) - break - case "mod": - if (u32) { for (let i = 0; i < n; i++) o[i] = floorMod(x[i], y[i]) } - else for (let i = 0; i < n; i++) o[i] = floatMod(x[i], y[i]) - break - } - if (u32) { for (let i = 0; i < n; i++) o[i] = o[i] >>> 0 } -} - -const arithF64Scalar = (op: ArithOp, x: Float64Array, y: number, o: Float64Array, n: number, u32: boolean) => { +/** Float and UInt32 kernels; `step` 0 reads `y[0]` for every row (a scalar). */ +const arithF64 = ( + op: ArithOp, + x: Float64Array, + y: Float64Array, + step: 0 | 1, + o: Float64Array, + n: number, + u32: boolean +) => { switch (op) { case "add": - for (let i = 0; i < n; i++) o[i] = x[i] + y + for (let i = 0; i < n; i++) o[i] = x[i] + y[i * step] break case "sub": - for (let i = 0; i < n; i++) o[i] = x[i] - y + for (let i = 0; i < n; i++) o[i] = x[i] - y[i * step] break case "mul": - if (u32) { for (let i = 0; i < n; i++) o[i] = Math.imul(x[i], y) >>> 0 } - else for (let i = 0; i < n; i++) o[i] = x[i] * y + if (u32) { for (let i = 0; i < n; i++) o[i] = Math.imul(x[i], y[i * step]) >>> 0 } + else for (let i = 0; i < n; i++) o[i] = x[i] * y[i * step] break case "div": - for (let i = 0; i < n; i++) o[i] = x[i] / y + for (let i = 0; i < n; i++) o[i] = x[i] / y[i * step] break case "floorDiv": - for (let i = 0; i < n; i++) o[i] = Math.floor(x[i] / y) + for (let i = 0; i < n; i++) o[i] = Math.floor(x[i] / y[i * step]) break case "mod": - if (u32) { for (let i = 0; i < n; i++) o[i] = floorMod(x[i], y) } - else for (let i = 0; i < n; i++) o[i] = floatMod(x[i], y) + if (u32) { for (let i = 0; i < n; i++) o[i] = floorMod(x[i], y[i * step]) } + else for (let i = 0; i < n; i++) o[i] = floatMod(x[i], y[i * step]) break } if (u32) { for (let i = 0; i < n; i++) o[i] = o[i] >>> 0 } From 3263c37ae4e79680257904071267a75444b48ea4 Mon Sep 17 00:00:00 2001 From: Effect Bot Date: Sun, 15 Nov 2026 11:56:03 +0000 Subject: [PATCH 07/88] Round decimals half to even, also when casting negative floats --- packages/effect/src/dataframe/Expr.ts | 17 +++++++++++-- .../effect/src/dataframe/internal/decimal.ts | 24 +++++++++++++++---- .../effect/src/dataframe/internal/kernels.ts | 6 +++++ packages/effect/test/dataframe/Query.test.ts | 10 ++++++++ .../test/fixtures/dataframe/generate_gold.py | 11 +++++++++ .../fixtures/dataframe/gold/extended.json | 2 +- packages/effect/typetest/DataFrame.tst.ts | 1 + 7 files changed, 64 insertions(+), 7 deletions(-) diff --git a/packages/effect/src/dataframe/Expr.ts b/packages/effect/src/dataframe/Expr.ts index 6db051ff747..a011e20f43d 100644 --- a/packages/effect/src/dataframe/Expr.ts +++ b/packages/effect/src/dataframe/Expr.ts @@ -420,13 +420,26 @@ export const sqrt = (self: Expr): Expr(self: Expr) => Expr - (self: Expr, decimals?: number): Expr + (decimals?: number): (self: Expr) => Expr + (self: Expr, decimals?: number): Expr } = dual( (args) => isExpr(args[0]), (self: Expr, decimals = 0) => internal.call("round", [self], { decimals }) diff --git a/packages/effect/src/dataframe/internal/decimal.ts b/packages/effect/src/dataframe/internal/decimal.ts index dd944426afd..6798f10186e 100644 --- a/packages/effect/src/dataframe/internal/decimal.ts +++ b/packages/effect/src/dataframe/internal/decimal.ts @@ -108,10 +108,9 @@ export const cast = (v: V.Vector, target: S.DataType): V.Vector | undefined => { const x = v.values[i] if (Number.isFinite(x)) { const scaled = x * 10 ** target.scale - const r = Math.round(scaled) - // round half to even - const even = Math.abs(scaled - Math.trunc(scaled)) === 0.5 && r % 2 !== 0 ? r - Math.sign(scaled) : r - put(i, BigInt(even)) + const floor = Math.floor(scaled) + const rest = scaled - floor + put(i, BigInt(rest > 0.5 || (rest === 0.5 && floor % 2 !== 0) ? floor + 1 : floor)) } break } @@ -189,6 +188,23 @@ export const arith = ( return { values: out, type: decimal(scale) } } +/** + * Rounds to `places` decimal places, halves to even, keeping the scale. As + * in Polars, the precision grows by one digit when values can round up. + * + * @internal + */ +export const round = (v: V.I64, type: S.Decimal, places: number): V.Vector => { + if (places >= type.scale) return v + const k = type.scale - places + const factor = pow10(k) + return V.i64( + S.decimal(Math.min(type.precision + 1, PRECISION), type.scale), + BigInt64Array.from(v.values, (x) => checked(divRound(x, k) * factor)), + v.valid + ) +} + /** Unscaled values as floats. @internal */ export const toFloat = (values: BigInt64Array, scale: number): Float64Array => Float64Array.from(values, (x) => Number(x) / 10 ** scale) diff --git a/packages/effect/src/dataframe/internal/kernels.ts b/packages/effect/src/dataframe/internal/kernels.ts index 6c856baeb68..ae52dc5f624 100644 --- a/packages/effect/src/dataframe/internal/kernels.ts +++ b/packages/effect/src/dataframe/internal/kernels.ts @@ -554,6 +554,12 @@ export const unaryNumeric = (op: "neg" | "abs" | "sqrt" | "round", a: Value, dec const n = v.length if (v.kind === "null") return a if (op === "sqrt" && v.kind !== "f64") return unaryNumeric(op, castValue(a, S.float64)) + if (op === "round" && !(Number.isInteger(decimals) && decimals >= 0)) { + return V.fail(`round needs a non-negative whole number of decimals, got ${decimals}`) + } + if (op === "round" && v.kind === "i64" && Dec.isDecimal(v.type)) { + return { vector: Dec.round(v, v.type, decimals), scalar: a.scalar, dynamic: a.dynamic } + } switch (v.kind) { case "f64": { const x = v.values diff --git a/packages/effect/test/dataframe/Query.test.ts b/packages/effect/test/dataframe/Query.test.ts index 1a66e5c2a38..b406f8db688 100644 --- a/packages/effect/test/dataframe/Query.test.ts +++ b/packages/effect/test/dataframe/Query.test.ts @@ -96,6 +96,16 @@ describe("Query", () => { assert.instanceOf(error, DataFrame.DataFrameError) assert.strictEqual(error.message, "filter predicate must be boolean, got Utf8") })) + + it.effect("round rejects negative or fractional places", () => + Effect.gen(function*() { + for (const places of [-1, 0.5]) { + const error = yield* Effect.flip( + Query.from(readings).pipe(Query.select((c) => ({ t: Expr.round(c.temp, places) }))) + ) + assert.strictEqual(error.message, `round needs a non-negative whole number of decimals, got ${places}`) + } + })) }) describe("operations", () => { diff --git a/packages/effect/test/fixtures/dataframe/generate_gold.py b/packages/effect/test/fixtures/dataframe/generate_gold.py index 2d34d84951e..45f1c1b636e 100644 --- a/packages/effect/test/fixtures/dataframe/generate_gold.py +++ b/packages/effect/test/fixtures/dataframe/generate_gold.py @@ -890,6 +890,17 @@ def extended_cases(): add("decimal/sort", "sort by a decimal", D, [{"op": "sort", "keys": [{"expr": col("price"), "descending": True}], "nullsLast": True}]) add("decimal/round-ties", "narrowing the scale rounds exact ties", {"left": table([("t", "decimal[6,3]")], {"t": ["0.250", "0.350", "-0.250", "-0.350", "1.050", "2.500", "0.249"]})}, [{"op": "select", "exprs": {"one": fn("cast", col("t"), to="decimal[6,1]"), "zero": fn("cast", col("t"), to="decimal[6,0]")}}]) + add("decimal/cast-float-ties", "floats cast to decimals round exact ties half to even, also below zero", + {"left": table([("x", "f64")], {"x": [-1.5, -2.5, 1.5, 2.5, -0.75]})}, + [{"op": "select", "exprs": {"zero": fn("cast", col("x"), to="decimal[18,0]"), "one": fn("cast", col("x"), to="decimal[18,1]")}}]) + add("decimal/round", "rounding decimals keeps the scale and rounds half to even", + {"left": table([("t", "decimal[6,3]")], {"t": ["0.250", "0.350", "-0.250", "-0.350", "1.050", "2.500", "0.249", None]})}, + [{"op": "select", "exprs": {"r0": fn("round", col("t")), "r1": fn("round", col("t"), decimals=1), + "r2": fn("round", col("t"), decimals=2), "r3": fn("round", col("t"), decimals=3)}}]) + add("arith/u32-divide-by-zero", "unsigned 32-bit floor division and modulo by zero give null", + {"left": table([("n", "u32"), ("d", "u32")], {"n": [7, 0, 5], "d": [0, 0, 2]})}, + [{"op": "select", "exprs": {"q": fn("floorDiv", col("n"), col("d")), "m": fn("mod", col("n"), col("d")), + "qs": fn("floorDiv", col("n"), lit(0, "u32")), "ms": fn("mod", col("n"), lit(0, "u32"))}}]) # windows see exactly the rows of their input, whatever runs before or after them W = {"left": table([("g", "i32"), ("h", "i32"), ("x", "f64")], {"g": [1, 2, 1, 2], "h": [1, 1, 2, 2], "x": [3.0, 1.0, 2.0, 4.0]}), "right": table([("g", "i32"), ("name", "utf8")], {"g": [1], "name": ["one"]})} diff --git a/packages/effect/test/fixtures/dataframe/gold/extended.json b/packages/effect/test/fixtures/dataframe/gold/extended.json index 3832464118b..e3fbd166bb6 100644 --- a/packages/effect/test/fixtures/dataframe/gold/extended.json +++ b/packages/effect/test/fixtures/dataframe/gold/extended.json @@ -1 +1 @@ -{"generator":"generate_gold.py","polars":"2.1.0","deviations":["JavaScript has no float literal distinct from an integer: an integral number literal behaves like a Polars integer literal (it takes the integer type of the column it meets), so cases never combine an integer column with an integral float literal.","Float to string casts use JavaScript number formatting (Polars prints 5.0 as '5.0', JavaScript as '5').","Joins never add '_right' suffixes: overlapping non-key columns are a type error.","Arithmetic between 64-bit integers (bigint) and other numeric types requires an explicit cast."],"cases":[{"id":"window/cum","description":"cumulative sum, min, max and count; nulls stay null","inputs":{"left":{"fields":[{"name":"g","type":"utf8"},{"name":"t","type":"i32"},{"name":"x","type":"f64"},{"name":"i","type":"i32"},{"name":"b","type":"bool"}],"columns":{"g":["a","b","a","a","b",null,"a","b","b","a"],"t":[1,1,2,3,2,1,4,3,4,5],"x":[1.5,2.0,null,4.0,"NaN",3.0,-1.0,2.0,0.5,2.0],"i":[3,1,4,1,5,9,null,6,5,3],"b":[true,false,null,true,true,false,true,null,false,true]}}},"ops":[{"op":"select","exprs":{"sx":{"fn":"cumSum","args":[{"col":"x"}]},"si":{"fn":"cumSum","args":[{"col":"i"}]},"mn":{"fn":"cumMin","args":[{"col":"i"}]},"mx":{"fn":"cumMax","args":[{"col":"x"}]},"n":{"fn":"cumCount","args":[{"col":"x"}]},"sb":{"fn":"cumSum","args":[{"col":"b"}]}}}],"expected":{"fields":[{"name":"sx","type":"f64"},{"name":"si","type":"i32"},{"name":"mn","type":"i32"},{"name":"mx","type":"f64"},{"name":"n","type":"u32"},{"name":"sb","type":"u32"}],"columns":{"sx":[1.5,3.5,null,7.5,"NaN","NaN","NaN","NaN","NaN","NaN"],"si":[3,4,8,9,14,23,null,29,34,37],"mn":[3,1,1,1,1,1,null,1,1,1],"mx":[1.5,2.0,null,4.0,4.0,4.0,4.0,4.0,4.0,4.0],"n":[1,2,2,3,4,5,6,7,8,9],"sb":[1,1,null,2,3,3,4,null,4,5]}}},{"id":"window/shift-diff","description":"shift (lag and lead) and diff","inputs":{"left":{"fields":[{"name":"g","type":"utf8"},{"name":"t","type":"i32"},{"name":"x","type":"f64"},{"name":"i","type":"i32"},{"name":"b","type":"bool"}],"columns":{"g":["a","b","a","a","b",null,"a","b","b","a"],"t":[1,1,2,3,2,1,4,3,4,5],"x":[1.5,2.0,null,4.0,"NaN",3.0,-1.0,2.0,0.5,2.0],"i":[3,1,4,1,5,9,null,6,5,3],"b":[true,false,null,true,true,false,true,null,false,true]}}},"ops":[{"op":"select","exprs":{"lag":{"fn":"shift","args":[{"col":"i"}]},"lead":{"fn":"shift","args":[{"col":"x"}],"n":-2},"d":{"fn":"diff","args":[{"col":"i"}]},"d2":{"fn":"diff","args":[{"col":"x"}],"n":2},"ls":{"fn":"shift","args":[{"col":"g"}]}}}],"expected":{"fields":[{"name":"lag","type":"i32"},{"name":"lead","type":"f64"},{"name":"d","type":"i32"},{"name":"d2","type":"f64"},{"name":"ls","type":"utf8"}],"columns":{"lag":[null,3,1,4,1,5,9,null,6,5],"lead":[null,4.0,"NaN",3.0,-1.0,2.0,0.5,2.0,null,null],"d":[null,-2,3,-3,4,4,null,null,-1,-2],"d2":[null,null,null,2.0,null,-1.0,"NaN",-1.0,1.5,0.0],"ls":[null,"a","b","a","a","b",null,"a","b","b"]}}},{"id":"window/rank","description":"rank methods, nulls and NaN","inputs":{"left":{"fields":[{"name":"g","type":"utf8"},{"name":"t","type":"i32"},{"name":"x","type":"f64"},{"name":"i","type":"i32"},{"name":"b","type":"bool"}],"columns":{"g":["a","b","a","a","b",null,"a","b","b","a"],"t":[1,1,2,3,2,1,4,3,4,5],"x":[1.5,2.0,null,4.0,"NaN",3.0,-1.0,2.0,0.5,2.0],"i":[3,1,4,1,5,9,null,6,5,3],"b":[true,false,null,true,true,false,true,null,false,true]}}},"ops":[{"op":"select","exprs":{"avg":{"fn":"rank","args":[{"col":"x"}]},"min":{"fn":"rank","args":[{"col":"i"}],"method":"min"},"max":{"fn":"rank","args":[{"col":"i"}],"method":"max"},"dense":{"fn":"rank","args":[{"col":"i"}],"method":"dense","descending":true},"ord":{"fn":"rank","args":[{"col":"x"}],"method":"ordinal"},"s":{"fn":"rank","args":[{"col":"g"}],"method":"dense"}}}],"expected":{"fields":[{"name":"avg","type":"f64"},{"name":"min","type":"u32"},{"name":"max","type":"u32"},{"name":"dense","type":"u32"},{"name":"ord","type":"u32"},{"name":"s","type":"u32"}],"columns":{"avg":[3.0,5.0,null,8.0,9.0,7.0,1.0,5.0,2.0,5.0],"min":[3,1,5,1,6,9,null,8,6,3],"max":[4,2,5,2,7,9,null,8,7,4],"dense":[5,6,4,6,3,1,null,2,3,5],"ord":[3,4,null,8,9,7,1,5,2,6],"s":[1,2,1,1,2,null,1,2,2,1]}}},{"id":"window/rolling","description":"rolling sum, mean, min and max","inputs":{"left":{"fields":[{"name":"g","type":"utf8"},{"name":"t","type":"i32"},{"name":"x","type":"f64"},{"name":"i","type":"i32"},{"name":"b","type":"bool"}],"columns":{"g":["a","b","a","a","b",null,"a","b","b","a"],"t":[1,1,2,3,2,1,4,3,4,5],"x":[1.5,2.0,null,4.0,"NaN",3.0,-1.0,2.0,0.5,2.0],"i":[3,1,4,1,5,9,null,6,5,3],"b":[true,false,null,true,true,false,true,null,false,true]}}},"ops":[{"op":"select","exprs":{"s":{"fn":"rollingSum","args":[{"col":"x"}],"window":3},"m":{"fn":"rollingMean","args":[{"col":"i"}],"window":2},"mn":{"fn":"rollingMin","args":[{"col":"x"}],"window":3,"minPeriods":1},"mx":{"fn":"rollingMax","args":[{"col":"i"}],"window":4,"minPeriods":2}}}],"expected":{"fields":[{"name":"s","type":"f64"},{"name":"m","type":"f64"},{"name":"mn","type":"f64"},{"name":"mx","type":"i32"}],"columns":{"s":[null,null,null,null,null,"NaN","NaN",4.0,1.5,4.5],"m":[null,2.0,2.5,2.5,3.0,7.0,null,null,5.5,4.0],"mn":[1.5,1.5,1.5,2.0,"NaN","NaN","NaN",-1.0,-1.0,0.5],"mx":[null,3,4,4,5,9,9,9,9,6]}}},{"id":"window/over-agg","description":"aggregates broadcast over partitions","inputs":{"left":{"fields":[{"name":"g","type":"utf8"},{"name":"t","type":"i32"},{"name":"x","type":"f64"},{"name":"i","type":"i32"},{"name":"b","type":"bool"}],"columns":{"g":["a","b","a","a","b",null,"a","b","b","a"],"t":[1,1,2,3,2,1,4,3,4,5],"x":[1.5,2.0,null,4.0,"NaN",3.0,-1.0,2.0,0.5,2.0],"i":[3,1,4,1,5,9,null,6,5,3],"b":[true,false,null,true,true,false,true,null,false,true]}}},"ops":[{"op":"withColumns","exprs":{"total":{"over":{"agg":"sum","arg":{"col":"x"}},"partitionBy":[{"col":"g"}]},"n":{"over":{"agg":"len"},"partitionBy":[{"col":"g"}]},"mean":{"over":{"agg":"mean","arg":{"col":"i"}},"partitionBy":[{"col":"g"}]},"first":{"over":{"agg":"first","arg":{"col":"t"}},"partitionBy":[{"col":"g"},{"col":"b"}]}}}],"expected":{"fields":[{"name":"g","type":"utf8"},{"name":"t","type":"i32"},{"name":"x","type":"f64"},{"name":"i","type":"i32"},{"name":"b","type":"bool"},{"name":"total","type":"f64"},{"name":"n","type":"u32"},{"name":"mean","type":"f64"},{"name":"first","type":"i32"}],"columns":{"g":["a","b","a","a","b",null,"a","b","b","a"],"t":[1,1,2,3,2,1,4,3,4,5],"x":[1.5,2.0,null,4.0,"NaN",3.0,-1.0,2.0,0.5,2.0],"i":[3,1,4,1,5,9,null,6,5,3],"b":[true,false,null,true,true,false,true,null,false,true],"total":[6.5,"NaN",6.5,6.5,"NaN",3.0,6.5,"NaN","NaN",6.5],"n":[5,4,5,5,4,1,5,4,4,5],"mean":[2.75,4.25,2.75,2.75,4.25,9.0,2.75,4.25,4.25,2.75],"first":[1,1,2,1,2,1,1,3,1,1]}}},{"id":"window/over-cum","description":"cumulative and shifted values per partition, in row order","inputs":{"left":{"fields":[{"name":"g","type":"utf8"},{"name":"t","type":"i32"},{"name":"x","type":"f64"},{"name":"i","type":"i32"},{"name":"b","type":"bool"}],"columns":{"g":["a","b","a","a","b",null,"a","b","b","a"],"t":[1,1,2,3,2,1,4,3,4,5],"x":[1.5,2.0,null,4.0,"NaN",3.0,-1.0,2.0,0.5,2.0],"i":[3,1,4,1,5,9,null,6,5,3],"b":[true,false,null,true,true,false,true,null,false,true]}}},"ops":[{"op":"withColumns","exprs":{"cs":{"over":{"fn":"cumSum","args":[{"col":"i"}]},"partitionBy":[{"col":"g"}]},"prev":{"over":{"fn":"shift","args":[{"col":"x"}]},"partitionBy":[{"col":"g"}]},"d":{"over":{"fn":"diff","args":[{"col":"t"}]},"partitionBy":[{"col":"g"}]},"r":{"over":{"fn":"rank","args":[{"col":"x"}],"method":"min"},"partitionBy":[{"col":"g"}]},"roll":{"over":{"fn":"rollingSum","args":[{"col":"i"}],"window":2,"minPeriods":1},"partitionBy":[{"col":"g"}]}}}],"expected":{"fields":[{"name":"g","type":"utf8"},{"name":"t","type":"i32"},{"name":"x","type":"f64"},{"name":"i","type":"i32"},{"name":"b","type":"bool"},{"name":"cs","type":"i32"},{"name":"prev","type":"f64"},{"name":"d","type":"i32"},{"name":"r","type":"u32"},{"name":"roll","type":"i32"}],"columns":{"g":["a","b","a","a","b",null,"a","b","b","a"],"t":[1,1,2,3,2,1,4,3,4,5],"x":[1.5,2.0,null,4.0,"NaN",3.0,-1.0,2.0,0.5,2.0],"i":[3,1,4,1,5,9,null,6,5,3],"b":[true,false,null,true,true,false,true,null,false,true],"cs":[3,1,7,8,6,9,null,12,17,11],"prev":[null,null,1.5,null,2.0,null,4.0,"NaN",2.0,-1.0],"d":[null,null,1,1,1,null,1,1,1,1],"r":[2,2,null,4,4,1,1,2,1,3],"roll":[3,1,7,5,6,9,1,11,11,3]}}},{"id":"window/over-filter","description":"filters after windows see the windowed values","inputs":{"left":{"fields":[{"name":"g","type":"utf8"},{"name":"t","type":"i32"},{"name":"x","type":"f64"},{"name":"i","type":"i32"},{"name":"b","type":"bool"}],"columns":{"g":["a","b","a","a","b",null,"a","b","b","a"],"t":[1,1,2,3,2,1,4,3,4,5],"x":[1.5,2.0,null,4.0,"NaN",3.0,-1.0,2.0,0.5,2.0],"i":[3,1,4,1,5,9,null,6,5,3],"b":[true,false,null,true,true,false,true,null,false,true]}}},"ops":[{"op":"withColumns","exprs":{"cs":{"over":{"fn":"cumSum","args":[{"col":"t"}]},"partitionBy":[{"col":"g"}]}}},{"op":"filter","predicate":{"fn":"gt","args":[{"col":"t"},{"lit":2}]}}],"expected":{"fields":[{"name":"g","type":"utf8"},{"name":"t","type":"i32"},{"name":"x","type":"f64"},{"name":"i","type":"i32"},{"name":"b","type":"bool"},{"name":"cs","type":"i32"}],"columns":{"g":["a","a","b","b","a"],"t":[3,4,3,4,5],"x":[4.0,-1.0,2.0,0.5,2.0],"i":[1,null,6,5,3],"b":[true,true,null,false,true],"cs":[6,10,6,10,15]}}},{"id":"window/filter-first","description":"windows after a filter only see the remaining rows","inputs":{"left":{"fields":[{"name":"g","type":"utf8"},{"name":"t","type":"i32"},{"name":"x","type":"f64"},{"name":"i","type":"i32"},{"name":"b","type":"bool"}],"columns":{"g":["a","b","a","a","b",null,"a","b","b","a"],"t":[1,1,2,3,2,1,4,3,4,5],"x":[1.5,2.0,null,4.0,"NaN",3.0,-1.0,2.0,0.5,2.0],"i":[3,1,4,1,5,9,null,6,5,3],"b":[true,false,null,true,true,false,true,null,false,true]}}},"ops":[{"op":"filter","predicate":{"fn":"isNotNull","args":[{"col":"x"}]}},{"op":"withColumns","exprs":{"cs":{"fn":"cumSum","args":[{"col":"x"}]},"r":{"over":{"fn":"rank","args":[{"col":"x"}]},"partitionBy":[{"col":"g"}]}}},{"op":"head","n":4}],"expected":{"fields":[{"name":"g","type":"utf8"},{"name":"t","type":"i32"},{"name":"x","type":"f64"},{"name":"i","type":"i32"},{"name":"b","type":"bool"},{"name":"cs","type":"f64"},{"name":"r","type":"f64"}],"columns":{"g":["a","b","a","b"],"t":[1,1,3,2],"x":[1.5,2.0,4.0,"NaN"],"i":[3,1,1,5],"b":[true,false,true,true],"cs":[1.5,3.5,7.5,"NaN"],"r":[2.0,2.5,4.0,4.0]}}},{"id":"pivot/sum","description":"pivot with a sum per cell; missing cells are null","inputs":{"left":{"fields":[{"name":"region","type":"utf8"},{"name":"quarter","type":"utf8"},{"name":"units","type":"i32"},{"name":"price","type":"f64"}],"columns":{"region":["north","north","south","south","north","east","south","north"],"quarter":["q1","q2","q1","q3","q1","q2","q1",null],"units":[10,20,5,null,7,3,8,1],"price":[1.5,2.0,1.0,3.0,null,2.5,1.0,9.0]}}},"ops":[{"op":"pivot","on":"quarter","index":["region"],"values":"units","aggregate":"sum"}],"expected":{"fields":[{"name":"region","type":"utf8"},{"name":"q1","type":"i32"},{"name":"q2","type":"i32"},{"name":"q3","type":"i32"},{"name":"null","type":"i32"}],"columns":{"region":["north","south","east"],"q1":[17,13,0],"q2":[20,0,3],"q3":[0,0,0],"null":[1,0,0]}}},{"id":"pivot/first","description":"pivot with the first value per cell","inputs":{"left":{"fields":[{"name":"region","type":"utf8"},{"name":"quarter","type":"utf8"},{"name":"units","type":"i32"},{"name":"price","type":"f64"}],"columns":{"region":["north","north","south","south","north","east","south","north"],"quarter":["q1","q2","q1","q3","q1","q2","q1",null],"units":[10,20,5,null,7,3,8,1],"price":[1.5,2.0,1.0,3.0,null,2.5,1.0,9.0]}}},"ops":[{"op":"pivot","on":"region","index":["quarter"],"values":"price","aggregate":"first"}],"expected":{"fields":[{"name":"quarter","type":"utf8"},{"name":"north","type":"f64"},{"name":"south","type":"f64"},{"name":"east","type":"f64"}],"columns":{"quarter":["q1","q2","q3",null],"north":[1.5,2.0,null,9.0],"south":[1.0,null,3.0,null],"east":[null,2.5,null,null]}}},{"id":"pivot/len","description":"pivot counting rows per cell","inputs":{"left":{"fields":[{"name":"region","type":"utf8"},{"name":"quarter","type":"utf8"},{"name":"units","type":"i32"},{"name":"price","type":"f64"}],"columns":{"region":["north","north","south","south","north","east","south","north"],"quarter":["q1","q2","q1","q3","q1","q2","q1",null],"units":[10,20,5,null,7,3,8,1],"price":[1.5,2.0,1.0,3.0,null,2.5,1.0,9.0]}}},"ops":[{"op":"pivot","on":"quarter","index":["region"],"values":"units","aggregate":"len"}],"expected":{"fields":[{"name":"region","type":"utf8"},{"name":"q1","type":"u32"},{"name":"q2","type":"u32"},{"name":"q3","type":"u32"},{"name":"null","type":"u32"}],"columns":{"region":["north","south","east"],"q1":[2,2,0],"q2":[1,0,1],"q3":[0,1,0],"null":[1,0,0]}}},{"id":"pivot/columns","description":"pivot with declared output columns","inputs":{"left":{"fields":[{"name":"region","type":"utf8"},{"name":"quarter","type":"utf8"},{"name":"units","type":"i32"},{"name":"price","type":"f64"}],"columns":{"region":["north","north","south","south","north","east","south","north"],"quarter":["q1","q2","q1","q3","q1","q2","q1",null],"units":[10,20,5,null,7,3,8,1],"price":[1.5,2.0,1.0,3.0,null,2.5,1.0,9.0]}}},"ops":[{"op":"pivot","on":"quarter","index":["region"],"values":"price","aggregate":"max","columns":["q1","q2","q4"]}],"expected":{"fields":[{"name":"region","type":"utf8"},{"name":"q1","type":"f64"},{"name":"q2","type":"f64"},{"name":"q4","type":"f64"}],"columns":{"region":["north","south","east"],"q1":[1.5,1.0,null],"q2":[2.0,null,2.5],"q4":[null,null,null]}}},{"id":"pivot/numeric-on","description":"pivot on a numeric column names columns by value","inputs":{"left":{"fields":[{"name":"g","type":"utf8"},{"name":"t","type":"i32"},{"name":"x","type":"f64"},{"name":"i","type":"i32"},{"name":"b","type":"bool"}],"columns":{"g":["a","b","a","a","b",null,"a","b","b","a"],"t":[1,1,2,3,2,1,4,3,4,5],"x":[1.5,2.0,null,4.0,"NaN",3.0,-1.0,2.0,0.5,2.0],"i":[3,1,4,1,5,9,null,6,5,3],"b":[true,false,null,true,true,false,true,null,false,true]}}},"ops":[{"op":"filter","predicate":{"fn":"isNotNull","args":[{"col":"g"}]}},{"op":"pivot","on":"t","index":["g"],"values":"i","aggregate":"min"}],"expected":{"fields":[{"name":"g","type":"utf8"},{"name":"1","type":"i32"},{"name":"2","type":"i32"},{"name":"3","type":"i32"},{"name":"4","type":"i32"},{"name":"5","type":"i32"}],"columns":{"g":["a","b"],"1":[3,1],"2":[4,5],"3":[1,6],"4":[null,5],"5":[3,null]}}},{"id":"unpivot/basic","description":"unpivot stacks columns, column by column","inputs":{"left":{"fields":[{"name":"region","type":"utf8"},{"name":"quarter","type":"utf8"},{"name":"units","type":"i32"},{"name":"price","type":"f64"}],"columns":{"region":["north","north","south","south","north","east","south","north"],"quarter":["q1","q2","q1","q3","q1","q2","q1",null],"units":[10,20,5,null,7,3,8,1],"price":[1.5,2.0,1.0,3.0,null,2.5,1.0,9.0]}}},"ops":[{"op":"unpivot","index":["region"],"on":["units","price"]}],"expected":{"fields":[{"name":"region","type":"utf8"},{"name":"variable","type":"utf8"},{"name":"value","type":"f64"}],"columns":{"region":["north","north","south","south","north","east","south","north","north","north","south","south","north","east","south","north"],"variable":["units","units","units","units","units","units","units","units","price","price","price","price","price","price","price","price"],"value":[10.0,20.0,5.0,null,7.0,3.0,8.0,1.0,1.5,2.0,1.0,3.0,null,2.5,1.0,9.0]}}},{"id":"unpivot/names","description":"unpivot with named variable and value columns","inputs":{"left":{"fields":[{"name":"region","type":"utf8"},{"name":"quarter","type":"utf8"},{"name":"units","type":"i32"},{"name":"price","type":"f64"}],"columns":{"region":["north","north","south","south","north","east","south","north"],"quarter":["q1","q2","q1","q3","q1","q2","q1",null],"units":[10,20,5,null,7,3,8,1],"price":[1.5,2.0,1.0,3.0,null,2.5,1.0,9.0]}}},"ops":[{"op":"pick","columns":["region","quarter"]},{"op":"unpivot","index":[],"on":["region","quarter"],"variable":"field","value":"text"}],"expected":{"fields":[{"name":"field","type":"utf8"},{"name":"text","type":"utf8"}],"columns":{"field":["region","region","region","region","region","region","region","region","quarter","quarter","quarter","quarter","quarter","quarter","quarter","quarter"],"text":["north","north","south","south","north","east","south","north","q1","q2","q1","q3","q1","q2","q1",null]}}},{"id":"temporal/extract","description":"calendar fields of timestamps and dates","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"ts","type":"datetime[us]"},{"name":"ms","type":"datetime[ms]"},{"name":"d","type":"date"},{"name":"dur","type":"duration[us]"}],"columns":{"k":["a","b","c","d","e","f"],"ts":[1709214307123456,1704067199999000,1709510400000000,null,-14182940000000,1710052200000500],"ms":[1709214307123,1704067200000,null,1709251200000,0,1709942400000],"d":[19782,19722,19786,19782,null,19792],"dur":[5400000000,-1500000,null,259200000001,0,90000000000]}},"right":{"fields":[{"name":"d","type":"date"},{"name":"name","type":"utf8"}],"columns":{"d":[19782,19792,20089],"name":["leap","sunday","new year"]}}},"ops":[{"op":"select","exprs":{"y":{"fn":"year","args":[{"col":"ts"}]},"mo":{"fn":"month","args":[{"col":"ts"}]},"d":{"fn":"day","args":[{"col":"ts"}]},"h":{"fn":"hour","args":[{"col":"ts"}]},"mi":{"fn":"minute","args":[{"col":"ts"}]},"s":{"fn":"second","args":[{"col":"ts"}]},"ms":{"fn":"millisecond","args":[{"col":"ts"}]},"wd":{"fn":"weekday","args":[{"col":"ts"}]},"od":{"fn":"ordinalDay","args":[{"col":"ts"}]},"dy":{"fn":"year","args":[{"col":"d"}]},"dwd":{"fn":"weekday","args":[{"col":"d"}]},"dod":{"fn":"ordinalDay","args":[{"col":"d"}]}}}],"expected":{"fields":[{"name":"y","type":"i32"},{"name":"mo","type":"i8"},{"name":"d","type":"i8"},{"name":"h","type":"i8"},{"name":"mi","type":"i8"},{"name":"s","type":"i8"},{"name":"ms","type":"i32"},{"name":"wd","type":"i8"},{"name":"od","type":"i16"},{"name":"dy","type":"i32"},{"name":"dwd","type":"i8"},{"name":"dod","type":"i16"}],"columns":{"y":[2024,2023,2024,null,1969,2024],"mo":[2,12,3,null,7,3],"d":[29,31,4,null,20,10],"h":[13,23,0,null,20,6],"mi":[45,59,0,null,17,30],"s":[7,59,0,null,40,0],"ms":[123,999,0,null,0,0],"wd":[4,7,1,null,7,7],"od":[60,365,64,null,201,70],"dy":[2024,2023,2024,2024,null,2024],"dwd":[4,7,1,4,null,7],"dod":[60,365,64,60,null,70]}}},{"id":"temporal/arith","description":"timestamp differences are durations; durations shift timestamps","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"ts","type":"datetime[us]"},{"name":"ms","type":"datetime[ms]"},{"name":"d","type":"date"},{"name":"dur","type":"duration[us]"}],"columns":{"k":["a","b","c","d","e","f"],"ts":[1709214307123456,1704067199999000,1709510400000000,null,-14182940000000,1710052200000500],"ms":[1709214307123,1704067200000,null,1709251200000,0,1709942400000],"d":[19782,19722,19786,19782,null,19792],"dur":[5400000000,-1500000,null,259200000001,0,90000000000]}},"right":{"fields":[{"name":"d","type":"date"},{"name":"name","type":"utf8"}],"columns":{"d":[19782,19792,20089],"name":["leap","sunday","new year"]}}},"ops":[{"op":"select","exprs":{"gap":{"fn":"sub","args":[{"col":"ts"},{"col":"ms"}]},"later":{"fn":"add","args":[{"col":"ts"},{"col":"dur"}]},"earlier":{"fn":"sub","args":[{"col":"ts"},{"col":"dur"}]},"days":{"fn":"sub","args":[{"col":"d"},{"lit":18000,"type":"date"}]},"twice":{"fn":"add","args":[{"col":"dur"},{"col":"dur"}]}}}],"expected":{"fields":[{"name":"gap","type":"duration[ms]"},{"name":"later","type":"datetime[us]"},{"name":"earlier","type":"datetime[us]"},{"name":"days","type":"duration[us]"},{"name":"twice","type":"duration[us]"}],"columns":{"gap":[0,-1,null,null,-14182940000,109800000],"later":[1709219707123456,1704067198499000,null,null,-14182940000000,1710142200000500],"earlier":[1709208907123456,1704067201499000,null,null,-14182940000000,1709962200000500],"days":[153964800000000,148780800000000,154310400000000,153964800000000,null,154828800000000],"twice":[10800000000,-3000000,null,518400000002,0,180000000000]}}},{"id":"temporal/truncate","description":"truncate to a calendar unit","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"ts","type":"datetime[us]"},{"name":"ms","type":"datetime[ms]"},{"name":"d","type":"date"},{"name":"dur","type":"duration[us]"}],"columns":{"k":["a","b","c","d","e","f"],"ts":[1709214307123456,1704067199999000,1709510400000000,null,-14182940000000,1710052200000500],"ms":[1709214307123,1704067200000,null,1709251200000,0,1709942400000],"d":[19782,19722,19786,19782,null,19792],"dur":[5400000000,-1500000,null,259200000001,0,90000000000]}},"right":{"fields":[{"name":"d","type":"date"},{"name":"name","type":"utf8"}],"columns":{"d":[19782,19792,20089],"name":["leap","sunday","new year"]}}},"ops":[{"op":"select","exprs":{"h":{"fn":"truncate","args":[{"col":"ts"}],"every":"1h"},"d":{"fn":"truncate","args":[{"col":"ts"}],"every":"1d"},"w":{"fn":"truncate","args":[{"col":"ts"}],"every":"1w"},"mo":{"fn":"truncate","args":[{"col":"ts"}],"every":"1mo"},"y":{"fn":"truncate","args":[{"col":"ts"}],"every":"1y"},"dmo":{"fn":"truncate","args":[{"col":"d"}],"every":"1mo"}}}],"expected":{"fields":[{"name":"h","type":"datetime[us]"},{"name":"d","type":"datetime[us]"},{"name":"w","type":"datetime[us]"},{"name":"mo","type":"datetime[us]"},{"name":"y","type":"datetime[us]"},{"name":"dmo","type":"date"}],"columns":{"h":[1709211600000000,1704063600000000,1709510400000000,null,-14184000000000,1710050400000000],"d":[1709164800000000,1703980800000000,1709510400000000,null,-14256000000000,1710028800000000],"w":[1708905600000000,1703462400000000,1709510400000000,null,-14774400000000,1709510400000000],"mo":[1706745600000000,1701388800000000,1709251200000000,null,-15897600000000,1709251200000000],"y":[1704067200000000,1672531200000000,1704067200000000,null,-31536000000000,1704067200000000],"dmo":[19754,19692,19783,19754,null,19783]}}},{"id":"temporal/duration","description":"durations in whole units","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"ts","type":"datetime[us]"},{"name":"ms","type":"datetime[ms]"},{"name":"d","type":"date"},{"name":"dur","type":"duration[us]"}],"columns":{"k":["a","b","c","d","e","f"],"ts":[1709214307123456,1704067199999000,1709510400000000,null,-14182940000000,1710052200000500],"ms":[1709214307123,1704067200000,null,1709251200000,0,1709942400000],"d":[19782,19722,19786,19782,null,19792],"dur":[5400000000,-1500000,null,259200000001,0,90000000000]}},"right":{"fields":[{"name":"d","type":"date"},{"name":"name","type":"utf8"}],"columns":{"d":[19782,19792,20089],"name":["leap","sunday","new year"]}}},"ops":[{"op":"select","exprs":{"d":{"fn":"totalDays","args":[{"col":"dur"}]},"h":{"fn":"totalHours","args":[{"col":"dur"}]},"m":{"fn":"totalMinutes","args":[{"col":"dur"}]},"s":{"fn":"totalSeconds","args":[{"col":"dur"}]},"ms":{"fn":"totalMilliseconds","args":[{"col":"dur"}]}}}],"expected":{"fields":[{"name":"d","type":"i64"},{"name":"h","type":"i64"},{"name":"m","type":"i64"},{"name":"s","type":"i64"},{"name":"ms","type":"i64"}],"columns":{"d":["0","0",null,"3","0","1"],"h":["1","0",null,"72","0","25"],"m":["90","0",null,"4320","0","1500"],"s":["5400","-1",null,"259200","0","90000"],"ms":["5400000","-1500",null,"259200000","0","90000000"]}}},{"id":"temporal/cast","description":"casts between dates, timestamps and integers","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"ts","type":"datetime[us]"},{"name":"ms","type":"datetime[ms]"},{"name":"d","type":"date"},{"name":"dur","type":"duration[us]"}],"columns":{"k":["a","b","c","d","e","f"],"ts":[1709214307123456,1704067199999000,1709510400000000,null,-14182940000000,1710052200000500],"ms":[1709214307123,1704067200000,null,1709251200000,0,1709942400000],"d":[19782,19722,19786,19782,null,19792],"dur":[5400000000,-1500000,null,259200000001,0,90000000000]}},"right":{"fields":[{"name":"d","type":"date"},{"name":"name","type":"utf8"}],"columns":{"d":[19782,19792,20089],"name":["leap","sunday","new year"]}}},"ops":[{"op":"select","exprs":{"date":{"fn":"cast","args":[{"col":"ts"}],"to":"date"},"ts":{"fn":"cast","args":[{"col":"d"}],"to":"datetime[us]"},"us":{"fn":"cast","args":[{"col":"ts"}],"to":"i64"},"days":{"fn":"cast","args":[{"col":"d"}],"to":"i32"},"ms":{"fn":"cast","args":[{"col":"ts"}],"to":"datetime[ms]"}}}],"expected":{"fields":[{"name":"date","type":"date"},{"name":"ts","type":"datetime[us]"},{"name":"us","type":"i64"},{"name":"days","type":"i32"},{"name":"ms","type":"datetime[ms]"}],"columns":{"date":[19782,19722,19786,null,-165,19792],"ts":[1709164800000000,1703980800000000,1709510400000000,1709164800000000,null,1710028800000000],"us":["1709214307123456","1704067199999000","1709510400000000",null,"-14182940000000","1710052200000500"],"days":[19782,19722,19786,19782,null,19792],"ms":[1709214307123,1704067199999,1709510400000,null,-14182940000,1710052200000]}}},{"id":"temporal/compare","description":"comparisons with timestamp literals and between units","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"ts","type":"datetime[us]"},{"name":"ms","type":"datetime[ms]"},{"name":"d","type":"date"},{"name":"dur","type":"duration[us]"}],"columns":{"k":["a","b","c","d","e","f"],"ts":[1709214307123456,1704067199999000,1709510400000000,null,-14182940000000,1710052200000500],"ms":[1709214307123,1704067200000,null,1709251200000,0,1709942400000],"d":[19782,19722,19786,19782,null,19792],"dur":[5400000000,-1500000,null,259200000001,0,90000000000]}},"right":{"fields":[{"name":"d","type":"date"},{"name":"name","type":"utf8"}],"columns":{"d":[19782,19792,20089],"name":["leap","sunday","new year"]}}},"ops":[{"op":"filter","predicate":{"fn":"and","args":[{"fn":"gte","args":[{"col":"ts"},{"lit":1709251200000000,"type":"datetime[us]"}]},{"fn":"lt","args":[{"col":"ms"},{"col":"ts"}]}]}}],"expected":{"fields":[{"name":"k","type":"utf8"},{"name":"ts","type":"datetime[us]"},{"name":"ms","type":"datetime[ms]"},{"name":"d","type":"date"},{"name":"dur","type":"duration[us]"}],"columns":{"k":["f"],"ts":[1710052200000500],"ms":[1709942400000],"d":[19792],"dur":[90000000000]}}},{"id":"temporal/group-sort","description":"group by date, aggregate timestamps, sort","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"ts","type":"datetime[us]"},{"name":"ms","type":"datetime[ms]"},{"name":"d","type":"date"},{"name":"dur","type":"duration[us]"}],"columns":{"k":["a","b","c","d","e","f"],"ts":[1709214307123456,1704067199999000,1709510400000000,null,-14182940000000,1710052200000500],"ms":[1709214307123,1704067200000,null,1709251200000,0,1709942400000],"d":[19782,19722,19786,19782,null,19792],"dur":[5400000000,-1500000,null,259200000001,0,90000000000]}},"right":{"fields":[{"name":"d","type":"date"},{"name":"name","type":"utf8"}],"columns":{"d":[19782,19792,20089],"name":["leap","sunday","new year"]}}},"ops":[{"op":"groupBy","keys":["d"],"aggs":{"first":{"agg":"min","arg":{"col":"ts"}},"last":{"agg":"max","arg":{"col":"ts"}},"n":{"agg":"len"},"total":{"agg":"sum","arg":{"col":"dur"}}}},{"op":"sort","keys":[{"expr":{"col":"last"},"descending":true}]}],"expected":{"fields":[{"name":"d","type":"date"},{"name":"first","type":"datetime[us]"},{"name":"last","type":"datetime[us]"},{"name":"n","type":"u32"},{"name":"total","type":"duration[us]"}],"columns":{"d":[19792,19786,19782,19722,null],"first":[1710052200000500,1709510400000000,1709214307123456,1704067199999000,-14182940000000],"last":[1710052200000500,1709510400000000,1709214307123456,1704067199999000,-14182940000000],"n":[1,1,2,1,1],"total":[90000000000,0,264600000001,-1500000,0]}}},{"id":"temporal/join","description":"join on a date","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"ts","type":"datetime[us]"},{"name":"ms","type":"datetime[ms]"},{"name":"d","type":"date"},{"name":"dur","type":"duration[us]"}],"columns":{"k":["a","b","c","d","e","f"],"ts":[1709214307123456,1704067199999000,1709510400000000,null,-14182940000000,1710052200000500],"ms":[1709214307123,1704067200000,null,1709251200000,0,1709942400000],"d":[19782,19722,19786,19782,null,19792],"dur":[5400000000,-1500000,null,259200000001,0,90000000000]}},"right":{"fields":[{"name":"d","type":"date"},{"name":"name","type":"utf8"}],"columns":{"d":[19782,19792,20089],"name":["leap","sunday","new year"]}}},"ops":[{"op":"join","right":"right","on":["d"],"how":"left"}],"expected":{"fields":[{"name":"k","type":"utf8"},{"name":"ts","type":"datetime[us]"},{"name":"ms","type":"datetime[ms]"},{"name":"d","type":"date"},{"name":"dur","type":"duration[us]"},{"name":"name","type":"utf8"}],"columns":{"k":["a","b","c","d","e","f"],"ts":[1709214307123456,1704067199999000,1709510400000000,null,-14182940000000,1710052200000500],"ms":[1709214307123,1704067200000,null,1709251200000,0,1709942400000],"d":[19782,19722,19786,19782,null,19792],"dur":[5400000000,-1500000,null,259200000001,0,90000000000],"name":["leap",null,null,"leap",null,"sunday"]}}},{"id":"nested/list-ops","description":"list length, element access and aggregates","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"g","type":"i32"},{"name":"xs","type":"list"},{"name":"ws","type":"list"},{"name":"s","type":"struct"}],"columns":{"k":["a","b","c","d","e"],"g":[1,2,1,2,1],"xs":[[1,2,3],[],null,[4,null,2],[7]],"ws":[["a","b"],["c"],[],null,["z","a"]],"s":[{"a":1,"b":"x"},{"a":null,"b":"y"},null,{"a":4,"b":null},{"a":5,"b":"z"}]}},"right":{"fields":[{"name":"k","type":"utf8"},{"name":"tags","type":"list"}],"columns":{"k":["a","d","x"],"tags":[["t1"],["t2","t3"],[]]}}},"ops":[{"op":"select","exprs":{"n":{"fn":"listLength","args":[{"col":"xs"}]},"first":{"fn":"listGet","args":[{"col":"xs"}],"index":0},"last":{"fn":"listGet","args":[{"col":"xs"}],"index":-1},"oob":{"fn":"listGet","args":[{"col":"xs"}],"index":5},"sum":{"fn":"listSum","args":[{"col":"xs"}]},"mean":{"fn":"listMean","args":[{"col":"xs"}]},"min":{"fn":"listMin","args":[{"col":"xs"}]},"max":{"fn":"listMax","args":[{"col":"ws"}]},"has2":{"fn":"listContains","args":[{"col":"xs"},{"lit":2}]},"hasA":{"fn":"listContains","args":[{"col":"ws"},{"lit":"a"}]}}}],"expected":{"fields":[{"name":"n","type":"u32"},{"name":"first","type":"i32"},{"name":"last","type":"i32"},{"name":"oob","type":"i32"},{"name":"sum","type":"i32"},{"name":"mean","type":"f64"},{"name":"min","type":"i32"},{"name":"max","type":"utf8"},{"name":"has2","type":"bool"},{"name":"hasA","type":"bool"}],"columns":{"n":[3,0,null,3,1],"first":[1,null,null,4,7],"last":[3,null,null,2,7],"oob":[null,null,null,null,null],"sum":[6,0,null,6,7],"mean":[2.0,null,null,3.0,7.0],"min":[1,null,null,2,7],"max":["b","c",null,null,"z"],"has2":[true,false,null,true,false],"hasA":[true,false,false,null,true]}}},{"id":"nested/struct","description":"struct fields and struct construction","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"g","type":"i32"},{"name":"xs","type":"list"},{"name":"ws","type":"list"},{"name":"s","type":"struct"}],"columns":{"k":["a","b","c","d","e"],"g":[1,2,1,2,1],"xs":[[1,2,3],[],null,[4,null,2],[7]],"ws":[["a","b"],["c"],[],null,["z","a"]],"s":[{"a":1,"b":"x"},{"a":null,"b":"y"},null,{"a":4,"b":null},{"a":5,"b":"z"}]}},"right":{"fields":[{"name":"k","type":"utf8"},{"name":"tags","type":"list"}],"columns":{"k":["a","d","x"],"tags":[["t1"],["t2","t3"],[]]}}},"ops":[{"op":"select","exprs":{"a":{"fn":"field","args":[{"col":"s"}],"field":"a"},"b":{"fn":"field","args":[{"col":"s"}],"field":"b"},"pair":{"fn":"struct","args":[{"col":"k"},{"fn":"listLength","args":[{"col":"xs"}]}],"names":["key","size"]}}}],"expected":{"fields":[{"name":"a","type":"i32"},{"name":"b","type":"utf8"},{"name":"pair","type":"struct"}],"columns":{"a":[1,null,null,4,5],"b":["x","y",null,null,"z"],"pair":[{"key":"a","size":3},{"key":"b","size":0},{"key":"c","size":null},{"key":"d","size":3},{"key":"e","size":1}]}}},{"id":"nested/explode","description":"explode lists into rows; a null list gives a null row, an empty list no rows","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"g","type":"i32"},{"name":"xs","type":"list"},{"name":"ws","type":"list"},{"name":"s","type":"struct"}],"columns":{"k":["a","b","c","d","e"],"g":[1,2,1,2,1],"xs":[[1,2,3],[],null,[4,null,2],[7]],"ws":[["a","b"],["c"],[],null,["z","a"]],"s":[{"a":1,"b":"x"},{"a":null,"b":"y"},null,{"a":4,"b":null},{"a":5,"b":"z"}]}},"right":{"fields":[{"name":"k","type":"utf8"},{"name":"tags","type":"list"}],"columns":{"k":["a","d","x"],"tags":[["t1"],["t2","t3"],[]]}}},"ops":[{"op":"pick","columns":["k","xs"]},{"op":"explode","column":"xs"}],"expected":{"fields":[{"name":"k","type":"utf8"},{"name":"xs","type":"i32"}],"columns":{"k":["a","a","a","c","d","d","d","e"],"xs":[1,2,3,null,4,null,2,7]}}},{"id":"nested/unnest","description":"unnest a struct into columns","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"g","type":"i32"},{"name":"xs","type":"list"},{"name":"ws","type":"list"},{"name":"s","type":"struct"}],"columns":{"k":["a","b","c","d","e"],"g":[1,2,1,2,1],"xs":[[1,2,3],[],null,[4,null,2],[7]],"ws":[["a","b"],["c"],[],null,["z","a"]],"s":[{"a":1,"b":"x"},{"a":null,"b":"y"},null,{"a":4,"b":null},{"a":5,"b":"z"}]}},"right":{"fields":[{"name":"k","type":"utf8"},{"name":"tags","type":"list"}],"columns":{"k":["a","d","x"],"tags":[["t1"],["t2","t3"],[]]}}},"ops":[{"op":"pick","columns":["k","s"]},{"op":"unnest","column":"s"}],"expected":{"fields":[{"name":"k","type":"utf8"},{"name":"a","type":"i32"},{"name":"b","type":"utf8"}],"columns":{"k":["a","b","c","d","e"],"a":[1,null,null,4,5],"b":["x","y",null,null,"z"]}}},{"id":"nested/agg-list","description":"collect values into lists per group","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"g","type":"i32"},{"name":"xs","type":"list"},{"name":"ws","type":"list"},{"name":"s","type":"struct"}],"columns":{"k":["a","b","c","d","e"],"g":[1,2,1,2,1],"xs":[[1,2,3],[],null,[4,null,2],[7]],"ws":[["a","b"],["c"],[],null,["z","a"]],"s":[{"a":1,"b":"x"},{"a":null,"b":"y"},null,{"a":4,"b":null},{"a":5,"b":"z"}]}},"right":{"fields":[{"name":"k","type":"utf8"},{"name":"tags","type":"list"}],"columns":{"k":["a","d","x"],"tags":[["t1"],["t2","t3"],[]]}}},"ops":[{"op":"groupBy","keys":["g"],"aggs":{"ks":{"agg":"list","arg":{"col":"k"}},"firsts":{"agg":"first","arg":{"col":"xs"}},"n":{"agg":"len"}}}],"expected":{"fields":[{"name":"g","type":"i32"},{"name":"ks","type":"list"},{"name":"firsts","type":"list"},{"name":"n","type":"u32"}],"columns":{"g":[1,2],"ks":[["a","c","e"],["b","d"]],"firsts":[[1,2,3],[]],"n":[3,2]}}},{"id":"nested/carry","description":"lists and structs survive filters, sorts, joins and slices","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"g","type":"i32"},{"name":"xs","type":"list"},{"name":"ws","type":"list"},{"name":"s","type":"struct"}],"columns":{"k":["a","b","c","d","e"],"g":[1,2,1,2,1],"xs":[[1,2,3],[],null,[4,null,2],[7]],"ws":[["a","b"],["c"],[],null,["z","a"]],"s":[{"a":1,"b":"x"},{"a":null,"b":"y"},null,{"a":4,"b":null},{"a":5,"b":"z"}]}},"right":{"fields":[{"name":"k","type":"utf8"},{"name":"tags","type":"list"}],"columns":{"k":["a","d","x"],"tags":[["t1"],["t2","t3"],[]]}}},"ops":[{"op":"filter","predicate":{"fn":"neq","args":[{"col":"k"},{"lit":"c"}]}},{"op":"sort","keys":[{"expr":{"col":"g"},"descending":true},{"expr":{"col":"k"}}]},{"op":"join","right":"right","on":["k"],"how":"left"},{"op":"head","n":4}],"expected":{"fields":[{"name":"k","type":"utf8"},{"name":"g","type":"i32"},{"name":"xs","type":"list"},{"name":"ws","type":"list"},{"name":"s","type":"struct"},{"name":"tags","type":"list"}],"columns":{"k":["b","d","a","e"],"g":[2,2,1,1],"xs":[[],[4,null,2],[1,2,3],[7]],"ws":[["c"],null,["a","b"],["z","a"]],"s":[{"a":null,"b":"y"},{"a":4,"b":null},{"a":1,"b":"x"},{"a":5,"b":"z"}],"tags":[null,["t2","t3"],["t1"],null]}}},{"id":"decimal/arith","description":"decimal addition, subtraction and multiplication","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"price","type":"decimal[10,2]"},{"name":"fee","type":"decimal[6,3]"},{"name":"qty","type":"decimal[4,0]"},{"name":"x","type":"f64"}],"columns":{"k":["a","b","a","c","b","a"],"price":["12.34","0.10","-5.00",null,"99999999.99","10.01"],"fee":["0.125","1.000",null,"2.500","-0.001","10.010"],"qty":["3","10","2","1",null,"7"],"x":[1.005,-2.5,123.456,null,1000000.0,0.0]}}},"ops":[{"op":"select","exprs":{"sum":{"fn":"add","args":[{"col":"price"},{"col":"fee"}]},"diff":{"fn":"sub","args":[{"col":"price"},{"col":"fee"}]},"prod":{"fn":"mul","args":[{"col":"price"},{"col":"qty"}]},"plus":{"fn":"add","args":[{"col":"price"},{"lit":"0.5","type":"decimal[3,1]"}]}}}],"expected":{"fields":[{"name":"sum","type":"decimal[38,3]"},{"name":"diff","type":"decimal[38,3]"},{"name":"prod","type":"decimal[38,2]"},{"name":"plus","type":"decimal[38,2]"}],"columns":{"sum":["12.465","1.100",null,null,"99999999.989","20.020"],"diff":["12.215","-0.900",null,null,"99999999.991","0.000"],"prod":["37.02","1.00","-10.00",null,null,"70.07"],"plus":["12.84","0.60","-4.50",null,"100000000.49","10.51"]}}},{"id":"decimal/compare","description":"decimal comparisons, also across scales","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"price","type":"decimal[10,2]"},{"name":"fee","type":"decimal[6,3]"},{"name":"qty","type":"decimal[4,0]"},{"name":"x","type":"f64"}],"columns":{"k":["a","b","a","c","b","a"],"price":["12.34","0.10","-5.00",null,"99999999.99","10.01"],"fee":["0.125","1.000",null,"2.500","-0.001","10.010"],"qty":["3","10","2","1",null,"7"],"x":[1.005,-2.5,123.456,null,1000000.0,0.0]}}},"ops":[{"op":"filter","predicate":{"fn":"and","args":[{"fn":"gt","args":[{"col":"price"},{"lit":"10.00","type":"decimal[10,2]"}]},{"fn":"gte","args":[{"col":"price"},{"col":"fee"}]}]}}],"expected":{"fields":[{"name":"k","type":"utf8"},{"name":"price","type":"decimal[10,2]"},{"name":"fee","type":"decimal[6,3]"},{"name":"qty","type":"decimal[4,0]"},{"name":"x","type":"f64"}],"columns":{"k":["a","b","a"],"price":["12.34","99999999.99","10.01"],"fee":["0.125","-0.001","10.010"],"qty":["3",null,"7"],"x":[1.005,1000000.0,0.0]}}},{"id":"decimal/aggregate","description":"sums, extremes and means of decimals per group","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"price","type":"decimal[10,2]"},{"name":"fee","type":"decimal[6,3]"},{"name":"qty","type":"decimal[4,0]"},{"name":"x","type":"f64"}],"columns":{"k":["a","b","a","c","b","a"],"price":["12.34","0.10","-5.00",null,"99999999.99","10.01"],"fee":["0.125","1.000",null,"2.500","-0.001","10.010"],"qty":["3","10","2","1",null,"7"],"x":[1.005,-2.5,123.456,null,1000000.0,0.0]}}},"ops":[{"op":"groupBy","keys":["k"],"aggs":{"total":{"agg":"sum","arg":{"col":"price"}},"min":{"agg":"min","arg":{"col":"price"}},"max":{"agg":"max","arg":{"col":"fee"}},"mean":{"agg":"mean","arg":{"col":"price"}},"n":{"agg":"count","arg":{"col":"price"}}}}],"expected":{"fields":[{"name":"k","type":"utf8"},{"name":"total","type":"decimal[38,2]"},{"name":"min","type":"decimal[10,2]"},{"name":"max","type":"decimal[6,3]"},{"name":"mean","type":"f64"},{"name":"n","type":"u32"}],"columns":{"k":["a","b","c"],"total":["17.35","100000000.09","0.00"],"min":["-5.00","0.10",null],"max":["10.010","1.000","2.500"],"mean":[5.783333333333334,50000000.044999994,null],"n":[3,2,0]}}},{"id":"decimal/cast","description":"casts between decimals, floats, integers and strings","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"price","type":"decimal[10,2]"},{"name":"fee","type":"decimal[6,3]"},{"name":"qty","type":"decimal[4,0]"},{"name":"x","type":"f64"}],"columns":{"k":["a","b","a","c","b","a"],"price":["12.34","0.10","-5.00",null,"99999999.99","10.01"],"fee":["0.125","1.000",null,"2.500","-0.001","10.010"],"qty":["3","10","2","1",null,"7"],"x":[1.005,-2.5,123.456,null,1000000.0,0.0]}}},"ops":[{"op":"select","exprs":{"f":{"fn":"cast","args":[{"col":"price"}],"to":"f64"},"i":{"fn":"cast","args":[{"col":"price"}],"to":"i32"},"s":{"fn":"cast","args":[{"col":"fee"}],"to":"utf8"},"d":{"fn":"cast","args":[{"col":"x"}],"to":"decimal[10,2]"},"r":{"fn":"cast","args":[{"col":"price"}],"to":"decimal[12,1]"}}}],"expected":{"fields":[{"name":"f","type":"f64"},{"name":"i","type":"i32"},{"name":"s","type":"utf8"},{"name":"d","type":"decimal[10,2]"},{"name":"r","type":"decimal[12,1]"}],"columns":{"f":[12.34,0.1,-5.0,null,99999999.99,10.01],"i":[12,0,-5,null,99999999,10],"s":["0.125","1.000",null,"2.500","-0.001","10.010"],"d":["1.00","-2.50","123.46",null,"1000000.00","0.00"],"r":["12.3","0.1","-5.0",null,"100000000.0","10.0"]}}},{"id":"decimal/sort","description":"sort by a decimal","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"price","type":"decimal[10,2]"},{"name":"fee","type":"decimal[6,3]"},{"name":"qty","type":"decimal[4,0]"},{"name":"x","type":"f64"}],"columns":{"k":["a","b","a","c","b","a"],"price":["12.34","0.10","-5.00",null,"99999999.99","10.01"],"fee":["0.125","1.000",null,"2.500","-0.001","10.010"],"qty":["3","10","2","1",null,"7"],"x":[1.005,-2.5,123.456,null,1000000.0,0.0]}}},"ops":[{"op":"sort","keys":[{"expr":{"col":"price"},"descending":true}],"nullsLast":true}],"expected":{"fields":[{"name":"k","type":"utf8"},{"name":"price","type":"decimal[10,2]"},{"name":"fee","type":"decimal[6,3]"},{"name":"qty","type":"decimal[4,0]"},{"name":"x","type":"f64"}],"columns":{"k":["b","a","a","b","a","c"],"price":["99999999.99","12.34","10.01","0.10","-5.00",null],"fee":["-0.001","0.125","10.010","1.000",null,"2.500"],"qty":[null,"3","7","10","2","1"],"x":[1000000.0,1.005,0.0,-2.5,123.456,null]}}},{"id":"decimal/round-ties","description":"narrowing the scale rounds exact ties","inputs":{"left":{"fields":[{"name":"t","type":"decimal[6,3]"}],"columns":{"t":["0.250","0.350","-0.250","-0.350","1.050","2.500","0.249"]}}},"ops":[{"op":"select","exprs":{"one":{"fn":"cast","args":[{"col":"t"}],"to":"decimal[6,1]"},"zero":{"fn":"cast","args":[{"col":"t"}],"to":"decimal[6,0]"}}}],"expected":{"fields":[{"name":"one","type":"decimal[6,1]"},{"name":"zero","type":"decimal[6,0]"}],"columns":{"one":["0.2","0.4","-0.2","-0.4","1.0","2.5","0.2"],"zero":["0","0","0","0","1","2","0"]}}},{"id":"window/filter-after-sort","description":"a window predicate sees the sorted rows","inputs":{"left":{"fields":[{"name":"g","type":"i32"},{"name":"h","type":"i32"},{"name":"x","type":"f64"}],"columns":{"g":[1,2,1,2],"h":[1,1,2,2],"x":[3.0,1.0,2.0,4.0]}},"right":{"fields":[{"name":"g","type":"i32"},{"name":"name","type":"utf8"}],"columns":{"g":[1],"name":["one"]}}},"ops":[{"op":"sort","keys":[{"expr":{"col":"x"}}]},{"op":"filter","predicate":{"fn":"gt","args":[{"fn":"cumSum","args":[{"col":"x"}]},{"lit":3.0}]}}],"expected":{"fields":[{"name":"g","type":"i32"},{"name":"h","type":"i32"},{"name":"x","type":"f64"}],"columns":{"g":[1,2],"h":[1,2],"x":[3.0,4.0]}}},{"id":"window/filter-after-filter","description":"a window predicate sees only the rows an earlier filter kept","inputs":{"left":{"fields":[{"name":"g","type":"i32"},{"name":"h","type":"i32"},{"name":"x","type":"f64"}],"columns":{"g":[1,2,1,2],"h":[1,1,2,2],"x":[3.0,1.0,2.0,4.0]}},"right":{"fields":[{"name":"g","type":"i32"},{"name":"name","type":"utf8"}],"columns":{"g":[1],"name":["one"]}}},"ops":[{"op":"filter","predicate":{"fn":"gt","args":[{"col":"x"},{"lit":1.0}]}},{"op":"filter","predicate":{"fn":"gt","args":[{"fn":"shift","args":[{"col":"x"}]},{"lit":2.0}]}}],"expected":{"fields":[{"name":"g","type":"i32"},{"name":"h","type":"i32"},{"name":"x","type":"f64"}],"columns":{"g":[1],"h":[2],"x":[2.0]}}},{"id":"window/filter-after-concat","description":"a window predicate sees every input of a concat","inputs":{"left":{"fields":[{"name":"g","type":"i32"},{"name":"h","type":"i32"},{"name":"x","type":"f64"}],"columns":{"g":[1,2,1,2],"h":[1,1,2,2],"x":[3.0,1.0,2.0,4.0]}},"right":{"fields":[{"name":"g","type":"i32"},{"name":"name","type":"utf8"}],"columns":{"g":[1],"name":["one"]}}},"ops":[{"op":"concat","with":"left"},{"op":"filter","predicate":{"fn":"gt","args":[{"fn":"cumSum","args":[{"col":"x"}]},{"lit":15.0}]}}],"expected":{"fields":[{"name":"g","type":"i32"},{"name":"h","type":"i32"},{"name":"x","type":"f64"}],"columns":{"g":[1,2],"h":[2,2],"x":[2.0,4.0]}}},{"id":"window/filter-after-join","description":"a window predicate sees only the joined rows","inputs":{"left":{"fields":[{"name":"g","type":"i32"},{"name":"h","type":"i32"},{"name":"x","type":"f64"}],"columns":{"g":[1,2,1,2],"h":[1,1,2,2],"x":[3.0,1.0,2.0,4.0]}},"right":{"fields":[{"name":"g","type":"i32"},{"name":"name","type":"utf8"}],"columns":{"g":[1],"name":["one"]}}},"ops":[{"op":"join","right":"right","on":["g"],"how":"inner"},{"op":"filter","predicate":{"fn":"gt","args":[{"fn":"shift","args":[{"col":"x"}]},{"lit":2.0}]}}],"expected":{"fields":[{"name":"g","type":"i32"},{"name":"h","type":"i32"},{"name":"x","type":"f64"},{"name":"name","type":"utf8"}],"columns":{"g":[1],"h":[2],"x":[2.0],"name":["one"]}}},{"id":"window/filter-whole","description":"a window predicate sees the whole input","inputs":{"left":{"fields":[{"name":"g","type":"i32"},{"name":"h","type":"i32"},{"name":"x","type":"f64"}],"columns":{"g":[1,2,1,2],"h":[1,1,2,2],"x":[3.0,1.0,2.0,4.0]}},"right":{"fields":[{"name":"g","type":"i32"},{"name":"name","type":"utf8"}],"columns":{"g":[1],"name":["one"]}}},"ops":[{"op":"filter","predicate":{"fn":"gt","args":[{"fn":"cumSum","args":[{"col":"x"}]},{"lit":5.0}]}}],"expected":{"fields":[{"name":"g","type":"i32"},{"name":"h","type":"i32"},{"name":"x","type":"f64"}],"columns":{"g":[1,2],"h":[2,2],"x":[2.0,4.0]}}},{"id":"window/agg-of-window","description":"an aggregate of a window sees the whole input","inputs":{"left":{"fields":[{"name":"g","type":"i32"},{"name":"h","type":"i32"},{"name":"x","type":"f64"}],"columns":{"g":[1,2,1,2],"h":[1,1,2,2],"x":[3.0,1.0,2.0,4.0]}},"right":{"fields":[{"name":"g","type":"i32"},{"name":"name","type":"utf8"}],"columns":{"g":[1],"name":["one"]}}},"ops":[{"op":"agg","aggs":{"s":{"agg":"sum","arg":{"fn":"cumSum","args":[{"col":"x"}]}},"n":{"agg":"len"}}}],"expected":{"fields":[{"name":"s","type":"f64"},{"name":"n","type":"u32"}],"columns":{"s":[23.0],"n":[4]}}},{"id":"window/grouped-agg-of-window","description":"windows inside grouped aggregates run per group","inputs":{"left":{"fields":[{"name":"g","type":"i32"},{"name":"h","type":"i32"},{"name":"x","type":"f64"}],"columns":{"g":[1,2,1,2],"h":[1,1,2,2],"x":[3.0,1.0,2.0,4.0]}},"right":{"fields":[{"name":"g","type":"i32"},{"name":"name","type":"utf8"}],"columns":{"g":[1],"name":["one"]}}},"ops":[{"op":"groupBy","keys":["g"],"aggs":{"s":{"agg":"sum","arg":{"fn":"cumSum","args":[{"col":"x"}]}},"l":{"agg":"list","arg":{"fn":"cumSum","args":[{"col":"x"}]}},"o":{"agg":"sum","arg":{"over":{"agg":"sum","arg":{"col":"x"}},"partitionBy":[{"col":"h"}]}}}}],"expected":{"fields":[{"name":"g","type":"i32"},{"name":"s","type":"f64"},{"name":"l","type":"list"},{"name":"o","type":"f64"}],"columns":{"g":[1,2],"s":[8.0,6.0],"l":[[3.0,5.0],[1.0,5.0]],"o":[5.0,5.0]}}}]} +{"generator":"generate_gold.py","polars":"2.1.0","deviations":["JavaScript has no float literal distinct from an integer: an integral number literal behaves like a Polars integer literal (it takes the integer type of the column it meets), so cases never combine an integer column with an integral float literal.","Float to string casts use JavaScript number formatting (Polars prints 5.0 as '5.0', JavaScript as '5').","Joins never add '_right' suffixes: overlapping non-key columns are a type error.","Arithmetic between 64-bit integers (bigint) and other numeric types requires an explicit cast."],"cases":[{"id":"window/cum","description":"cumulative sum, min, max and count; nulls stay null","inputs":{"left":{"fields":[{"name":"g","type":"utf8"},{"name":"t","type":"i32"},{"name":"x","type":"f64"},{"name":"i","type":"i32"},{"name":"b","type":"bool"}],"columns":{"g":["a","b","a","a","b",null,"a","b","b","a"],"t":[1,1,2,3,2,1,4,3,4,5],"x":[1.5,2.0,null,4.0,"NaN",3.0,-1.0,2.0,0.5,2.0],"i":[3,1,4,1,5,9,null,6,5,3],"b":[true,false,null,true,true,false,true,null,false,true]}}},"ops":[{"op":"select","exprs":{"sx":{"fn":"cumSum","args":[{"col":"x"}]},"si":{"fn":"cumSum","args":[{"col":"i"}]},"mn":{"fn":"cumMin","args":[{"col":"i"}]},"mx":{"fn":"cumMax","args":[{"col":"x"}]},"n":{"fn":"cumCount","args":[{"col":"x"}]},"sb":{"fn":"cumSum","args":[{"col":"b"}]}}}],"expected":{"fields":[{"name":"sx","type":"f64"},{"name":"si","type":"i32"},{"name":"mn","type":"i32"},{"name":"mx","type":"f64"},{"name":"n","type":"u32"},{"name":"sb","type":"u32"}],"columns":{"sx":[1.5,3.5,null,7.5,"NaN","NaN","NaN","NaN","NaN","NaN"],"si":[3,4,8,9,14,23,null,29,34,37],"mn":[3,1,1,1,1,1,null,1,1,1],"mx":[1.5,2.0,null,4.0,4.0,4.0,4.0,4.0,4.0,4.0],"n":[1,2,2,3,4,5,6,7,8,9],"sb":[1,1,null,2,3,3,4,null,4,5]}}},{"id":"window/shift-diff","description":"shift (lag and lead) and diff","inputs":{"left":{"fields":[{"name":"g","type":"utf8"},{"name":"t","type":"i32"},{"name":"x","type":"f64"},{"name":"i","type":"i32"},{"name":"b","type":"bool"}],"columns":{"g":["a","b","a","a","b",null,"a","b","b","a"],"t":[1,1,2,3,2,1,4,3,4,5],"x":[1.5,2.0,null,4.0,"NaN",3.0,-1.0,2.0,0.5,2.0],"i":[3,1,4,1,5,9,null,6,5,3],"b":[true,false,null,true,true,false,true,null,false,true]}}},"ops":[{"op":"select","exprs":{"lag":{"fn":"shift","args":[{"col":"i"}]},"lead":{"fn":"shift","args":[{"col":"x"}],"n":-2},"d":{"fn":"diff","args":[{"col":"i"}]},"d2":{"fn":"diff","args":[{"col":"x"}],"n":2},"ls":{"fn":"shift","args":[{"col":"g"}]}}}],"expected":{"fields":[{"name":"lag","type":"i32"},{"name":"lead","type":"f64"},{"name":"d","type":"i32"},{"name":"d2","type":"f64"},{"name":"ls","type":"utf8"}],"columns":{"lag":[null,3,1,4,1,5,9,null,6,5],"lead":[null,4.0,"NaN",3.0,-1.0,2.0,0.5,2.0,null,null],"d":[null,-2,3,-3,4,4,null,null,-1,-2],"d2":[null,null,null,2.0,null,-1.0,"NaN",-1.0,1.5,0.0],"ls":[null,"a","b","a","a","b",null,"a","b","b"]}}},{"id":"window/rank","description":"rank methods, nulls and NaN","inputs":{"left":{"fields":[{"name":"g","type":"utf8"},{"name":"t","type":"i32"},{"name":"x","type":"f64"},{"name":"i","type":"i32"},{"name":"b","type":"bool"}],"columns":{"g":["a","b","a","a","b",null,"a","b","b","a"],"t":[1,1,2,3,2,1,4,3,4,5],"x":[1.5,2.0,null,4.0,"NaN",3.0,-1.0,2.0,0.5,2.0],"i":[3,1,4,1,5,9,null,6,5,3],"b":[true,false,null,true,true,false,true,null,false,true]}}},"ops":[{"op":"select","exprs":{"avg":{"fn":"rank","args":[{"col":"x"}]},"min":{"fn":"rank","args":[{"col":"i"}],"method":"min"},"max":{"fn":"rank","args":[{"col":"i"}],"method":"max"},"dense":{"fn":"rank","args":[{"col":"i"}],"method":"dense","descending":true},"ord":{"fn":"rank","args":[{"col":"x"}],"method":"ordinal"},"s":{"fn":"rank","args":[{"col":"g"}],"method":"dense"}}}],"expected":{"fields":[{"name":"avg","type":"f64"},{"name":"min","type":"u32"},{"name":"max","type":"u32"},{"name":"dense","type":"u32"},{"name":"ord","type":"u32"},{"name":"s","type":"u32"}],"columns":{"avg":[3.0,5.0,null,8.0,9.0,7.0,1.0,5.0,2.0,5.0],"min":[3,1,5,1,6,9,null,8,6,3],"max":[4,2,5,2,7,9,null,8,7,4],"dense":[5,6,4,6,3,1,null,2,3,5],"ord":[3,4,null,8,9,7,1,5,2,6],"s":[1,2,1,1,2,null,1,2,2,1]}}},{"id":"window/rolling","description":"rolling sum, mean, min and max","inputs":{"left":{"fields":[{"name":"g","type":"utf8"},{"name":"t","type":"i32"},{"name":"x","type":"f64"},{"name":"i","type":"i32"},{"name":"b","type":"bool"}],"columns":{"g":["a","b","a","a","b",null,"a","b","b","a"],"t":[1,1,2,3,2,1,4,3,4,5],"x":[1.5,2.0,null,4.0,"NaN",3.0,-1.0,2.0,0.5,2.0],"i":[3,1,4,1,5,9,null,6,5,3],"b":[true,false,null,true,true,false,true,null,false,true]}}},"ops":[{"op":"select","exprs":{"s":{"fn":"rollingSum","args":[{"col":"x"}],"window":3},"m":{"fn":"rollingMean","args":[{"col":"i"}],"window":2},"mn":{"fn":"rollingMin","args":[{"col":"x"}],"window":3,"minPeriods":1},"mx":{"fn":"rollingMax","args":[{"col":"i"}],"window":4,"minPeriods":2}}}],"expected":{"fields":[{"name":"s","type":"f64"},{"name":"m","type":"f64"},{"name":"mn","type":"f64"},{"name":"mx","type":"i32"}],"columns":{"s":[null,null,null,null,null,"NaN","NaN",4.0,1.5,4.5],"m":[null,2.0,2.5,2.5,3.0,7.0,null,null,5.5,4.0],"mn":[1.5,1.5,1.5,2.0,"NaN","NaN","NaN",-1.0,-1.0,0.5],"mx":[null,3,4,4,5,9,9,9,9,6]}}},{"id":"window/over-agg","description":"aggregates broadcast over partitions","inputs":{"left":{"fields":[{"name":"g","type":"utf8"},{"name":"t","type":"i32"},{"name":"x","type":"f64"},{"name":"i","type":"i32"},{"name":"b","type":"bool"}],"columns":{"g":["a","b","a","a","b",null,"a","b","b","a"],"t":[1,1,2,3,2,1,4,3,4,5],"x":[1.5,2.0,null,4.0,"NaN",3.0,-1.0,2.0,0.5,2.0],"i":[3,1,4,1,5,9,null,6,5,3],"b":[true,false,null,true,true,false,true,null,false,true]}}},"ops":[{"op":"withColumns","exprs":{"total":{"over":{"agg":"sum","arg":{"col":"x"}},"partitionBy":[{"col":"g"}]},"n":{"over":{"agg":"len"},"partitionBy":[{"col":"g"}]},"mean":{"over":{"agg":"mean","arg":{"col":"i"}},"partitionBy":[{"col":"g"}]},"first":{"over":{"agg":"first","arg":{"col":"t"}},"partitionBy":[{"col":"g"},{"col":"b"}]}}}],"expected":{"fields":[{"name":"g","type":"utf8"},{"name":"t","type":"i32"},{"name":"x","type":"f64"},{"name":"i","type":"i32"},{"name":"b","type":"bool"},{"name":"total","type":"f64"},{"name":"n","type":"u32"},{"name":"mean","type":"f64"},{"name":"first","type":"i32"}],"columns":{"g":["a","b","a","a","b",null,"a","b","b","a"],"t":[1,1,2,3,2,1,4,3,4,5],"x":[1.5,2.0,null,4.0,"NaN",3.0,-1.0,2.0,0.5,2.0],"i":[3,1,4,1,5,9,null,6,5,3],"b":[true,false,null,true,true,false,true,null,false,true],"total":[6.5,"NaN",6.5,6.5,"NaN",3.0,6.5,"NaN","NaN",6.5],"n":[5,4,5,5,4,1,5,4,4,5],"mean":[2.75,4.25,2.75,2.75,4.25,9.0,2.75,4.25,4.25,2.75],"first":[1,1,2,1,2,1,1,3,1,1]}}},{"id":"window/over-cum","description":"cumulative and shifted values per partition, in row order","inputs":{"left":{"fields":[{"name":"g","type":"utf8"},{"name":"t","type":"i32"},{"name":"x","type":"f64"},{"name":"i","type":"i32"},{"name":"b","type":"bool"}],"columns":{"g":["a","b","a","a","b",null,"a","b","b","a"],"t":[1,1,2,3,2,1,4,3,4,5],"x":[1.5,2.0,null,4.0,"NaN",3.0,-1.0,2.0,0.5,2.0],"i":[3,1,4,1,5,9,null,6,5,3],"b":[true,false,null,true,true,false,true,null,false,true]}}},"ops":[{"op":"withColumns","exprs":{"cs":{"over":{"fn":"cumSum","args":[{"col":"i"}]},"partitionBy":[{"col":"g"}]},"prev":{"over":{"fn":"shift","args":[{"col":"x"}]},"partitionBy":[{"col":"g"}]},"d":{"over":{"fn":"diff","args":[{"col":"t"}]},"partitionBy":[{"col":"g"}]},"r":{"over":{"fn":"rank","args":[{"col":"x"}],"method":"min"},"partitionBy":[{"col":"g"}]},"roll":{"over":{"fn":"rollingSum","args":[{"col":"i"}],"window":2,"minPeriods":1},"partitionBy":[{"col":"g"}]}}}],"expected":{"fields":[{"name":"g","type":"utf8"},{"name":"t","type":"i32"},{"name":"x","type":"f64"},{"name":"i","type":"i32"},{"name":"b","type":"bool"},{"name":"cs","type":"i32"},{"name":"prev","type":"f64"},{"name":"d","type":"i32"},{"name":"r","type":"u32"},{"name":"roll","type":"i32"}],"columns":{"g":["a","b","a","a","b",null,"a","b","b","a"],"t":[1,1,2,3,2,1,4,3,4,5],"x":[1.5,2.0,null,4.0,"NaN",3.0,-1.0,2.0,0.5,2.0],"i":[3,1,4,1,5,9,null,6,5,3],"b":[true,false,null,true,true,false,true,null,false,true],"cs":[3,1,7,8,6,9,null,12,17,11],"prev":[null,null,1.5,null,2.0,null,4.0,"NaN",2.0,-1.0],"d":[null,null,1,1,1,null,1,1,1,1],"r":[2,2,null,4,4,1,1,2,1,3],"roll":[3,1,7,5,6,9,1,11,11,3]}}},{"id":"window/over-filter","description":"filters after windows see the windowed values","inputs":{"left":{"fields":[{"name":"g","type":"utf8"},{"name":"t","type":"i32"},{"name":"x","type":"f64"},{"name":"i","type":"i32"},{"name":"b","type":"bool"}],"columns":{"g":["a","b","a","a","b",null,"a","b","b","a"],"t":[1,1,2,3,2,1,4,3,4,5],"x":[1.5,2.0,null,4.0,"NaN",3.0,-1.0,2.0,0.5,2.0],"i":[3,1,4,1,5,9,null,6,5,3],"b":[true,false,null,true,true,false,true,null,false,true]}}},"ops":[{"op":"withColumns","exprs":{"cs":{"over":{"fn":"cumSum","args":[{"col":"t"}]},"partitionBy":[{"col":"g"}]}}},{"op":"filter","predicate":{"fn":"gt","args":[{"col":"t"},{"lit":2}]}}],"expected":{"fields":[{"name":"g","type":"utf8"},{"name":"t","type":"i32"},{"name":"x","type":"f64"},{"name":"i","type":"i32"},{"name":"b","type":"bool"},{"name":"cs","type":"i32"}],"columns":{"g":["a","a","b","b","a"],"t":[3,4,3,4,5],"x":[4.0,-1.0,2.0,0.5,2.0],"i":[1,null,6,5,3],"b":[true,true,null,false,true],"cs":[6,10,6,10,15]}}},{"id":"window/filter-first","description":"windows after a filter only see the remaining rows","inputs":{"left":{"fields":[{"name":"g","type":"utf8"},{"name":"t","type":"i32"},{"name":"x","type":"f64"},{"name":"i","type":"i32"},{"name":"b","type":"bool"}],"columns":{"g":["a","b","a","a","b",null,"a","b","b","a"],"t":[1,1,2,3,2,1,4,3,4,5],"x":[1.5,2.0,null,4.0,"NaN",3.0,-1.0,2.0,0.5,2.0],"i":[3,1,4,1,5,9,null,6,5,3],"b":[true,false,null,true,true,false,true,null,false,true]}}},"ops":[{"op":"filter","predicate":{"fn":"isNotNull","args":[{"col":"x"}]}},{"op":"withColumns","exprs":{"cs":{"fn":"cumSum","args":[{"col":"x"}]},"r":{"over":{"fn":"rank","args":[{"col":"x"}]},"partitionBy":[{"col":"g"}]}}},{"op":"head","n":4}],"expected":{"fields":[{"name":"g","type":"utf8"},{"name":"t","type":"i32"},{"name":"x","type":"f64"},{"name":"i","type":"i32"},{"name":"b","type":"bool"},{"name":"cs","type":"f64"},{"name":"r","type":"f64"}],"columns":{"g":["a","b","a","b"],"t":[1,1,3,2],"x":[1.5,2.0,4.0,"NaN"],"i":[3,1,1,5],"b":[true,false,true,true],"cs":[1.5,3.5,7.5,"NaN"],"r":[2.0,2.5,4.0,4.0]}}},{"id":"pivot/sum","description":"pivot with a sum per cell; missing cells are null","inputs":{"left":{"fields":[{"name":"region","type":"utf8"},{"name":"quarter","type":"utf8"},{"name":"units","type":"i32"},{"name":"price","type":"f64"}],"columns":{"region":["north","north","south","south","north","east","south","north"],"quarter":["q1","q2","q1","q3","q1","q2","q1",null],"units":[10,20,5,null,7,3,8,1],"price":[1.5,2.0,1.0,3.0,null,2.5,1.0,9.0]}}},"ops":[{"op":"pivot","on":"quarter","index":["region"],"values":"units","aggregate":"sum"}],"expected":{"fields":[{"name":"region","type":"utf8"},{"name":"q1","type":"i32"},{"name":"q2","type":"i32"},{"name":"q3","type":"i32"},{"name":"null","type":"i32"}],"columns":{"region":["north","south","east"],"q1":[17,13,0],"q2":[20,0,3],"q3":[0,0,0],"null":[1,0,0]}}},{"id":"pivot/first","description":"pivot with the first value per cell","inputs":{"left":{"fields":[{"name":"region","type":"utf8"},{"name":"quarter","type":"utf8"},{"name":"units","type":"i32"},{"name":"price","type":"f64"}],"columns":{"region":["north","north","south","south","north","east","south","north"],"quarter":["q1","q2","q1","q3","q1","q2","q1",null],"units":[10,20,5,null,7,3,8,1],"price":[1.5,2.0,1.0,3.0,null,2.5,1.0,9.0]}}},"ops":[{"op":"pivot","on":"region","index":["quarter"],"values":"price","aggregate":"first"}],"expected":{"fields":[{"name":"quarter","type":"utf8"},{"name":"north","type":"f64"},{"name":"south","type":"f64"},{"name":"east","type":"f64"}],"columns":{"quarter":["q1","q2","q3",null],"north":[1.5,2.0,null,9.0],"south":[1.0,null,3.0,null],"east":[null,2.5,null,null]}}},{"id":"pivot/len","description":"pivot counting rows per cell","inputs":{"left":{"fields":[{"name":"region","type":"utf8"},{"name":"quarter","type":"utf8"},{"name":"units","type":"i32"},{"name":"price","type":"f64"}],"columns":{"region":["north","north","south","south","north","east","south","north"],"quarter":["q1","q2","q1","q3","q1","q2","q1",null],"units":[10,20,5,null,7,3,8,1],"price":[1.5,2.0,1.0,3.0,null,2.5,1.0,9.0]}}},"ops":[{"op":"pivot","on":"quarter","index":["region"],"values":"units","aggregate":"len"}],"expected":{"fields":[{"name":"region","type":"utf8"},{"name":"q1","type":"u32"},{"name":"q2","type":"u32"},{"name":"q3","type":"u32"},{"name":"null","type":"u32"}],"columns":{"region":["north","south","east"],"q1":[2,2,0],"q2":[1,0,1],"q3":[0,1,0],"null":[1,0,0]}}},{"id":"pivot/columns","description":"pivot with declared output columns","inputs":{"left":{"fields":[{"name":"region","type":"utf8"},{"name":"quarter","type":"utf8"},{"name":"units","type":"i32"},{"name":"price","type":"f64"}],"columns":{"region":["north","north","south","south","north","east","south","north"],"quarter":["q1","q2","q1","q3","q1","q2","q1",null],"units":[10,20,5,null,7,3,8,1],"price":[1.5,2.0,1.0,3.0,null,2.5,1.0,9.0]}}},"ops":[{"op":"pivot","on":"quarter","index":["region"],"values":"price","aggregate":"max","columns":["q1","q2","q4"]}],"expected":{"fields":[{"name":"region","type":"utf8"},{"name":"q1","type":"f64"},{"name":"q2","type":"f64"},{"name":"q4","type":"f64"}],"columns":{"region":["north","south","east"],"q1":[1.5,1.0,null],"q2":[2.0,null,2.5],"q4":[null,null,null]}}},{"id":"pivot/numeric-on","description":"pivot on a numeric column names columns by value","inputs":{"left":{"fields":[{"name":"g","type":"utf8"},{"name":"t","type":"i32"},{"name":"x","type":"f64"},{"name":"i","type":"i32"},{"name":"b","type":"bool"}],"columns":{"g":["a","b","a","a","b",null,"a","b","b","a"],"t":[1,1,2,3,2,1,4,3,4,5],"x":[1.5,2.0,null,4.0,"NaN",3.0,-1.0,2.0,0.5,2.0],"i":[3,1,4,1,5,9,null,6,5,3],"b":[true,false,null,true,true,false,true,null,false,true]}}},"ops":[{"op":"filter","predicate":{"fn":"isNotNull","args":[{"col":"g"}]}},{"op":"pivot","on":"t","index":["g"],"values":"i","aggregate":"min"}],"expected":{"fields":[{"name":"g","type":"utf8"},{"name":"1","type":"i32"},{"name":"2","type":"i32"},{"name":"3","type":"i32"},{"name":"4","type":"i32"},{"name":"5","type":"i32"}],"columns":{"g":["a","b"],"1":[3,1],"2":[4,5],"3":[1,6],"4":[null,5],"5":[3,null]}}},{"id":"unpivot/basic","description":"unpivot stacks columns, column by column","inputs":{"left":{"fields":[{"name":"region","type":"utf8"},{"name":"quarter","type":"utf8"},{"name":"units","type":"i32"},{"name":"price","type":"f64"}],"columns":{"region":["north","north","south","south","north","east","south","north"],"quarter":["q1","q2","q1","q3","q1","q2","q1",null],"units":[10,20,5,null,7,3,8,1],"price":[1.5,2.0,1.0,3.0,null,2.5,1.0,9.0]}}},"ops":[{"op":"unpivot","index":["region"],"on":["units","price"]}],"expected":{"fields":[{"name":"region","type":"utf8"},{"name":"variable","type":"utf8"},{"name":"value","type":"f64"}],"columns":{"region":["north","north","south","south","north","east","south","north","north","north","south","south","north","east","south","north"],"variable":["units","units","units","units","units","units","units","units","price","price","price","price","price","price","price","price"],"value":[10.0,20.0,5.0,null,7.0,3.0,8.0,1.0,1.5,2.0,1.0,3.0,null,2.5,1.0,9.0]}}},{"id":"unpivot/names","description":"unpivot with named variable and value columns","inputs":{"left":{"fields":[{"name":"region","type":"utf8"},{"name":"quarter","type":"utf8"},{"name":"units","type":"i32"},{"name":"price","type":"f64"}],"columns":{"region":["north","north","south","south","north","east","south","north"],"quarter":["q1","q2","q1","q3","q1","q2","q1",null],"units":[10,20,5,null,7,3,8,1],"price":[1.5,2.0,1.0,3.0,null,2.5,1.0,9.0]}}},"ops":[{"op":"pick","columns":["region","quarter"]},{"op":"unpivot","index":[],"on":["region","quarter"],"variable":"field","value":"text"}],"expected":{"fields":[{"name":"field","type":"utf8"},{"name":"text","type":"utf8"}],"columns":{"field":["region","region","region","region","region","region","region","region","quarter","quarter","quarter","quarter","quarter","quarter","quarter","quarter"],"text":["north","north","south","south","north","east","south","north","q1","q2","q1","q3","q1","q2","q1",null]}}},{"id":"temporal/extract","description":"calendar fields of timestamps and dates","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"ts","type":"datetime[us]"},{"name":"ms","type":"datetime[ms]"},{"name":"d","type":"date"},{"name":"dur","type":"duration[us]"}],"columns":{"k":["a","b","c","d","e","f"],"ts":[1709214307123456,1704067199999000,1709510400000000,null,-14182940000000,1710052200000500],"ms":[1709214307123,1704067200000,null,1709251200000,0,1709942400000],"d":[19782,19722,19786,19782,null,19792],"dur":[5400000000,-1500000,null,259200000001,0,90000000000]}},"right":{"fields":[{"name":"d","type":"date"},{"name":"name","type":"utf8"}],"columns":{"d":[19782,19792,20089],"name":["leap","sunday","new year"]}}},"ops":[{"op":"select","exprs":{"y":{"fn":"year","args":[{"col":"ts"}]},"mo":{"fn":"month","args":[{"col":"ts"}]},"d":{"fn":"day","args":[{"col":"ts"}]},"h":{"fn":"hour","args":[{"col":"ts"}]},"mi":{"fn":"minute","args":[{"col":"ts"}]},"s":{"fn":"second","args":[{"col":"ts"}]},"ms":{"fn":"millisecond","args":[{"col":"ts"}]},"wd":{"fn":"weekday","args":[{"col":"ts"}]},"od":{"fn":"ordinalDay","args":[{"col":"ts"}]},"dy":{"fn":"year","args":[{"col":"d"}]},"dwd":{"fn":"weekday","args":[{"col":"d"}]},"dod":{"fn":"ordinalDay","args":[{"col":"d"}]}}}],"expected":{"fields":[{"name":"y","type":"i32"},{"name":"mo","type":"i8"},{"name":"d","type":"i8"},{"name":"h","type":"i8"},{"name":"mi","type":"i8"},{"name":"s","type":"i8"},{"name":"ms","type":"i32"},{"name":"wd","type":"i8"},{"name":"od","type":"i16"},{"name":"dy","type":"i32"},{"name":"dwd","type":"i8"},{"name":"dod","type":"i16"}],"columns":{"y":[2024,2023,2024,null,1969,2024],"mo":[2,12,3,null,7,3],"d":[29,31,4,null,20,10],"h":[13,23,0,null,20,6],"mi":[45,59,0,null,17,30],"s":[7,59,0,null,40,0],"ms":[123,999,0,null,0,0],"wd":[4,7,1,null,7,7],"od":[60,365,64,null,201,70],"dy":[2024,2023,2024,2024,null,2024],"dwd":[4,7,1,4,null,7],"dod":[60,365,64,60,null,70]}}},{"id":"temporal/arith","description":"timestamp differences are durations; durations shift timestamps","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"ts","type":"datetime[us]"},{"name":"ms","type":"datetime[ms]"},{"name":"d","type":"date"},{"name":"dur","type":"duration[us]"}],"columns":{"k":["a","b","c","d","e","f"],"ts":[1709214307123456,1704067199999000,1709510400000000,null,-14182940000000,1710052200000500],"ms":[1709214307123,1704067200000,null,1709251200000,0,1709942400000],"d":[19782,19722,19786,19782,null,19792],"dur":[5400000000,-1500000,null,259200000001,0,90000000000]}},"right":{"fields":[{"name":"d","type":"date"},{"name":"name","type":"utf8"}],"columns":{"d":[19782,19792,20089],"name":["leap","sunday","new year"]}}},"ops":[{"op":"select","exprs":{"gap":{"fn":"sub","args":[{"col":"ts"},{"col":"ms"}]},"later":{"fn":"add","args":[{"col":"ts"},{"col":"dur"}]},"earlier":{"fn":"sub","args":[{"col":"ts"},{"col":"dur"}]},"days":{"fn":"sub","args":[{"col":"d"},{"lit":18000,"type":"date"}]},"twice":{"fn":"add","args":[{"col":"dur"},{"col":"dur"}]}}}],"expected":{"fields":[{"name":"gap","type":"duration[ms]"},{"name":"later","type":"datetime[us]"},{"name":"earlier","type":"datetime[us]"},{"name":"days","type":"duration[us]"},{"name":"twice","type":"duration[us]"}],"columns":{"gap":[0,-1,null,null,-14182940000,109800000],"later":[1709219707123456,1704067198499000,null,null,-14182940000000,1710142200000500],"earlier":[1709208907123456,1704067201499000,null,null,-14182940000000,1709962200000500],"days":[153964800000000,148780800000000,154310400000000,153964800000000,null,154828800000000],"twice":[10800000000,-3000000,null,518400000002,0,180000000000]}}},{"id":"temporal/truncate","description":"truncate to a calendar unit","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"ts","type":"datetime[us]"},{"name":"ms","type":"datetime[ms]"},{"name":"d","type":"date"},{"name":"dur","type":"duration[us]"}],"columns":{"k":["a","b","c","d","e","f"],"ts":[1709214307123456,1704067199999000,1709510400000000,null,-14182940000000,1710052200000500],"ms":[1709214307123,1704067200000,null,1709251200000,0,1709942400000],"d":[19782,19722,19786,19782,null,19792],"dur":[5400000000,-1500000,null,259200000001,0,90000000000]}},"right":{"fields":[{"name":"d","type":"date"},{"name":"name","type":"utf8"}],"columns":{"d":[19782,19792,20089],"name":["leap","sunday","new year"]}}},"ops":[{"op":"select","exprs":{"h":{"fn":"truncate","args":[{"col":"ts"}],"every":"1h"},"d":{"fn":"truncate","args":[{"col":"ts"}],"every":"1d"},"w":{"fn":"truncate","args":[{"col":"ts"}],"every":"1w"},"mo":{"fn":"truncate","args":[{"col":"ts"}],"every":"1mo"},"y":{"fn":"truncate","args":[{"col":"ts"}],"every":"1y"},"dmo":{"fn":"truncate","args":[{"col":"d"}],"every":"1mo"}}}],"expected":{"fields":[{"name":"h","type":"datetime[us]"},{"name":"d","type":"datetime[us]"},{"name":"w","type":"datetime[us]"},{"name":"mo","type":"datetime[us]"},{"name":"y","type":"datetime[us]"},{"name":"dmo","type":"date"}],"columns":{"h":[1709211600000000,1704063600000000,1709510400000000,null,-14184000000000,1710050400000000],"d":[1709164800000000,1703980800000000,1709510400000000,null,-14256000000000,1710028800000000],"w":[1708905600000000,1703462400000000,1709510400000000,null,-14774400000000,1709510400000000],"mo":[1706745600000000,1701388800000000,1709251200000000,null,-15897600000000,1709251200000000],"y":[1704067200000000,1672531200000000,1704067200000000,null,-31536000000000,1704067200000000],"dmo":[19754,19692,19783,19754,null,19783]}}},{"id":"temporal/duration","description":"durations in whole units","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"ts","type":"datetime[us]"},{"name":"ms","type":"datetime[ms]"},{"name":"d","type":"date"},{"name":"dur","type":"duration[us]"}],"columns":{"k":["a","b","c","d","e","f"],"ts":[1709214307123456,1704067199999000,1709510400000000,null,-14182940000000,1710052200000500],"ms":[1709214307123,1704067200000,null,1709251200000,0,1709942400000],"d":[19782,19722,19786,19782,null,19792],"dur":[5400000000,-1500000,null,259200000001,0,90000000000]}},"right":{"fields":[{"name":"d","type":"date"},{"name":"name","type":"utf8"}],"columns":{"d":[19782,19792,20089],"name":["leap","sunday","new year"]}}},"ops":[{"op":"select","exprs":{"d":{"fn":"totalDays","args":[{"col":"dur"}]},"h":{"fn":"totalHours","args":[{"col":"dur"}]},"m":{"fn":"totalMinutes","args":[{"col":"dur"}]},"s":{"fn":"totalSeconds","args":[{"col":"dur"}]},"ms":{"fn":"totalMilliseconds","args":[{"col":"dur"}]}}}],"expected":{"fields":[{"name":"d","type":"i64"},{"name":"h","type":"i64"},{"name":"m","type":"i64"},{"name":"s","type":"i64"},{"name":"ms","type":"i64"}],"columns":{"d":["0","0",null,"3","0","1"],"h":["1","0",null,"72","0","25"],"m":["90","0",null,"4320","0","1500"],"s":["5400","-1",null,"259200","0","90000"],"ms":["5400000","-1500",null,"259200000","0","90000000"]}}},{"id":"temporal/cast","description":"casts between dates, timestamps and integers","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"ts","type":"datetime[us]"},{"name":"ms","type":"datetime[ms]"},{"name":"d","type":"date"},{"name":"dur","type":"duration[us]"}],"columns":{"k":["a","b","c","d","e","f"],"ts":[1709214307123456,1704067199999000,1709510400000000,null,-14182940000000,1710052200000500],"ms":[1709214307123,1704067200000,null,1709251200000,0,1709942400000],"d":[19782,19722,19786,19782,null,19792],"dur":[5400000000,-1500000,null,259200000001,0,90000000000]}},"right":{"fields":[{"name":"d","type":"date"},{"name":"name","type":"utf8"}],"columns":{"d":[19782,19792,20089],"name":["leap","sunday","new year"]}}},"ops":[{"op":"select","exprs":{"date":{"fn":"cast","args":[{"col":"ts"}],"to":"date"},"ts":{"fn":"cast","args":[{"col":"d"}],"to":"datetime[us]"},"us":{"fn":"cast","args":[{"col":"ts"}],"to":"i64"},"days":{"fn":"cast","args":[{"col":"d"}],"to":"i32"},"ms":{"fn":"cast","args":[{"col":"ts"}],"to":"datetime[ms]"}}}],"expected":{"fields":[{"name":"date","type":"date"},{"name":"ts","type":"datetime[us]"},{"name":"us","type":"i64"},{"name":"days","type":"i32"},{"name":"ms","type":"datetime[ms]"}],"columns":{"date":[19782,19722,19786,null,-165,19792],"ts":[1709164800000000,1703980800000000,1709510400000000,1709164800000000,null,1710028800000000],"us":["1709214307123456","1704067199999000","1709510400000000",null,"-14182940000000","1710052200000500"],"days":[19782,19722,19786,19782,null,19792],"ms":[1709214307123,1704067199999,1709510400000,null,-14182940000,1710052200000]}}},{"id":"temporal/compare","description":"comparisons with timestamp literals and between units","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"ts","type":"datetime[us]"},{"name":"ms","type":"datetime[ms]"},{"name":"d","type":"date"},{"name":"dur","type":"duration[us]"}],"columns":{"k":["a","b","c","d","e","f"],"ts":[1709214307123456,1704067199999000,1709510400000000,null,-14182940000000,1710052200000500],"ms":[1709214307123,1704067200000,null,1709251200000,0,1709942400000],"d":[19782,19722,19786,19782,null,19792],"dur":[5400000000,-1500000,null,259200000001,0,90000000000]}},"right":{"fields":[{"name":"d","type":"date"},{"name":"name","type":"utf8"}],"columns":{"d":[19782,19792,20089],"name":["leap","sunday","new year"]}}},"ops":[{"op":"filter","predicate":{"fn":"and","args":[{"fn":"gte","args":[{"col":"ts"},{"lit":1709251200000000,"type":"datetime[us]"}]},{"fn":"lt","args":[{"col":"ms"},{"col":"ts"}]}]}}],"expected":{"fields":[{"name":"k","type":"utf8"},{"name":"ts","type":"datetime[us]"},{"name":"ms","type":"datetime[ms]"},{"name":"d","type":"date"},{"name":"dur","type":"duration[us]"}],"columns":{"k":["f"],"ts":[1710052200000500],"ms":[1709942400000],"d":[19792],"dur":[90000000000]}}},{"id":"temporal/group-sort","description":"group by date, aggregate timestamps, sort","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"ts","type":"datetime[us]"},{"name":"ms","type":"datetime[ms]"},{"name":"d","type":"date"},{"name":"dur","type":"duration[us]"}],"columns":{"k":["a","b","c","d","e","f"],"ts":[1709214307123456,1704067199999000,1709510400000000,null,-14182940000000,1710052200000500],"ms":[1709214307123,1704067200000,null,1709251200000,0,1709942400000],"d":[19782,19722,19786,19782,null,19792],"dur":[5400000000,-1500000,null,259200000001,0,90000000000]}},"right":{"fields":[{"name":"d","type":"date"},{"name":"name","type":"utf8"}],"columns":{"d":[19782,19792,20089],"name":["leap","sunday","new year"]}}},"ops":[{"op":"groupBy","keys":["d"],"aggs":{"first":{"agg":"min","arg":{"col":"ts"}},"last":{"agg":"max","arg":{"col":"ts"}},"n":{"agg":"len"},"total":{"agg":"sum","arg":{"col":"dur"}}}},{"op":"sort","keys":[{"expr":{"col":"last"},"descending":true}]}],"expected":{"fields":[{"name":"d","type":"date"},{"name":"first","type":"datetime[us]"},{"name":"last","type":"datetime[us]"},{"name":"n","type":"u32"},{"name":"total","type":"duration[us]"}],"columns":{"d":[19792,19786,19782,19722,null],"first":[1710052200000500,1709510400000000,1709214307123456,1704067199999000,-14182940000000],"last":[1710052200000500,1709510400000000,1709214307123456,1704067199999000,-14182940000000],"n":[1,1,2,1,1],"total":[90000000000,0,264600000001,-1500000,0]}}},{"id":"temporal/join","description":"join on a date","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"ts","type":"datetime[us]"},{"name":"ms","type":"datetime[ms]"},{"name":"d","type":"date"},{"name":"dur","type":"duration[us]"}],"columns":{"k":["a","b","c","d","e","f"],"ts":[1709214307123456,1704067199999000,1709510400000000,null,-14182940000000,1710052200000500],"ms":[1709214307123,1704067200000,null,1709251200000,0,1709942400000],"d":[19782,19722,19786,19782,null,19792],"dur":[5400000000,-1500000,null,259200000001,0,90000000000]}},"right":{"fields":[{"name":"d","type":"date"},{"name":"name","type":"utf8"}],"columns":{"d":[19782,19792,20089],"name":["leap","sunday","new year"]}}},"ops":[{"op":"join","right":"right","on":["d"],"how":"left"}],"expected":{"fields":[{"name":"k","type":"utf8"},{"name":"ts","type":"datetime[us]"},{"name":"ms","type":"datetime[ms]"},{"name":"d","type":"date"},{"name":"dur","type":"duration[us]"},{"name":"name","type":"utf8"}],"columns":{"k":["a","b","c","d","e","f"],"ts":[1709214307123456,1704067199999000,1709510400000000,null,-14182940000000,1710052200000500],"ms":[1709214307123,1704067200000,null,1709251200000,0,1709942400000],"d":[19782,19722,19786,19782,null,19792],"dur":[5400000000,-1500000,null,259200000001,0,90000000000],"name":["leap",null,null,"leap",null,"sunday"]}}},{"id":"nested/list-ops","description":"list length, element access and aggregates","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"g","type":"i32"},{"name":"xs","type":"list"},{"name":"ws","type":"list"},{"name":"s","type":"struct"}],"columns":{"k":["a","b","c","d","e"],"g":[1,2,1,2,1],"xs":[[1,2,3],[],null,[4,null,2],[7]],"ws":[["a","b"],["c"],[],null,["z","a"]],"s":[{"a":1,"b":"x"},{"a":null,"b":"y"},null,{"a":4,"b":null},{"a":5,"b":"z"}]}},"right":{"fields":[{"name":"k","type":"utf8"},{"name":"tags","type":"list"}],"columns":{"k":["a","d","x"],"tags":[["t1"],["t2","t3"],[]]}}},"ops":[{"op":"select","exprs":{"n":{"fn":"listLength","args":[{"col":"xs"}]},"first":{"fn":"listGet","args":[{"col":"xs"}],"index":0},"last":{"fn":"listGet","args":[{"col":"xs"}],"index":-1},"oob":{"fn":"listGet","args":[{"col":"xs"}],"index":5},"sum":{"fn":"listSum","args":[{"col":"xs"}]},"mean":{"fn":"listMean","args":[{"col":"xs"}]},"min":{"fn":"listMin","args":[{"col":"xs"}]},"max":{"fn":"listMax","args":[{"col":"ws"}]},"has2":{"fn":"listContains","args":[{"col":"xs"},{"lit":2}]},"hasA":{"fn":"listContains","args":[{"col":"ws"},{"lit":"a"}]}}}],"expected":{"fields":[{"name":"n","type":"u32"},{"name":"first","type":"i32"},{"name":"last","type":"i32"},{"name":"oob","type":"i32"},{"name":"sum","type":"i32"},{"name":"mean","type":"f64"},{"name":"min","type":"i32"},{"name":"max","type":"utf8"},{"name":"has2","type":"bool"},{"name":"hasA","type":"bool"}],"columns":{"n":[3,0,null,3,1],"first":[1,null,null,4,7],"last":[3,null,null,2,7],"oob":[null,null,null,null,null],"sum":[6,0,null,6,7],"mean":[2.0,null,null,3.0,7.0],"min":[1,null,null,2,7],"max":["b","c",null,null,"z"],"has2":[true,false,null,true,false],"hasA":[true,false,false,null,true]}}},{"id":"nested/struct","description":"struct fields and struct construction","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"g","type":"i32"},{"name":"xs","type":"list"},{"name":"ws","type":"list"},{"name":"s","type":"struct"}],"columns":{"k":["a","b","c","d","e"],"g":[1,2,1,2,1],"xs":[[1,2,3],[],null,[4,null,2],[7]],"ws":[["a","b"],["c"],[],null,["z","a"]],"s":[{"a":1,"b":"x"},{"a":null,"b":"y"},null,{"a":4,"b":null},{"a":5,"b":"z"}]}},"right":{"fields":[{"name":"k","type":"utf8"},{"name":"tags","type":"list"}],"columns":{"k":["a","d","x"],"tags":[["t1"],["t2","t3"],[]]}}},"ops":[{"op":"select","exprs":{"a":{"fn":"field","args":[{"col":"s"}],"field":"a"},"b":{"fn":"field","args":[{"col":"s"}],"field":"b"},"pair":{"fn":"struct","args":[{"col":"k"},{"fn":"listLength","args":[{"col":"xs"}]}],"names":["key","size"]}}}],"expected":{"fields":[{"name":"a","type":"i32"},{"name":"b","type":"utf8"},{"name":"pair","type":"struct"}],"columns":{"a":[1,null,null,4,5],"b":["x","y",null,null,"z"],"pair":[{"key":"a","size":3},{"key":"b","size":0},{"key":"c","size":null},{"key":"d","size":3},{"key":"e","size":1}]}}},{"id":"nested/explode","description":"explode lists into rows; a null list gives a null row, an empty list no rows","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"g","type":"i32"},{"name":"xs","type":"list"},{"name":"ws","type":"list"},{"name":"s","type":"struct"}],"columns":{"k":["a","b","c","d","e"],"g":[1,2,1,2,1],"xs":[[1,2,3],[],null,[4,null,2],[7]],"ws":[["a","b"],["c"],[],null,["z","a"]],"s":[{"a":1,"b":"x"},{"a":null,"b":"y"},null,{"a":4,"b":null},{"a":5,"b":"z"}]}},"right":{"fields":[{"name":"k","type":"utf8"},{"name":"tags","type":"list"}],"columns":{"k":["a","d","x"],"tags":[["t1"],["t2","t3"],[]]}}},"ops":[{"op":"pick","columns":["k","xs"]},{"op":"explode","column":"xs"}],"expected":{"fields":[{"name":"k","type":"utf8"},{"name":"xs","type":"i32"}],"columns":{"k":["a","a","a","c","d","d","d","e"],"xs":[1,2,3,null,4,null,2,7]}}},{"id":"nested/unnest","description":"unnest a struct into columns","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"g","type":"i32"},{"name":"xs","type":"list"},{"name":"ws","type":"list"},{"name":"s","type":"struct"}],"columns":{"k":["a","b","c","d","e"],"g":[1,2,1,2,1],"xs":[[1,2,3],[],null,[4,null,2],[7]],"ws":[["a","b"],["c"],[],null,["z","a"]],"s":[{"a":1,"b":"x"},{"a":null,"b":"y"},null,{"a":4,"b":null},{"a":5,"b":"z"}]}},"right":{"fields":[{"name":"k","type":"utf8"},{"name":"tags","type":"list"}],"columns":{"k":["a","d","x"],"tags":[["t1"],["t2","t3"],[]]}}},"ops":[{"op":"pick","columns":["k","s"]},{"op":"unnest","column":"s"}],"expected":{"fields":[{"name":"k","type":"utf8"},{"name":"a","type":"i32"},{"name":"b","type":"utf8"}],"columns":{"k":["a","b","c","d","e"],"a":[1,null,null,4,5],"b":["x","y",null,null,"z"]}}},{"id":"nested/agg-list","description":"collect values into lists per group","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"g","type":"i32"},{"name":"xs","type":"list"},{"name":"ws","type":"list"},{"name":"s","type":"struct"}],"columns":{"k":["a","b","c","d","e"],"g":[1,2,1,2,1],"xs":[[1,2,3],[],null,[4,null,2],[7]],"ws":[["a","b"],["c"],[],null,["z","a"]],"s":[{"a":1,"b":"x"},{"a":null,"b":"y"},null,{"a":4,"b":null},{"a":5,"b":"z"}]}},"right":{"fields":[{"name":"k","type":"utf8"},{"name":"tags","type":"list"}],"columns":{"k":["a","d","x"],"tags":[["t1"],["t2","t3"],[]]}}},"ops":[{"op":"groupBy","keys":["g"],"aggs":{"ks":{"agg":"list","arg":{"col":"k"}},"firsts":{"agg":"first","arg":{"col":"xs"}},"n":{"agg":"len"}}}],"expected":{"fields":[{"name":"g","type":"i32"},{"name":"ks","type":"list"},{"name":"firsts","type":"list"},{"name":"n","type":"u32"}],"columns":{"g":[1,2],"ks":[["a","c","e"],["b","d"]],"firsts":[[1,2,3],[]],"n":[3,2]}}},{"id":"nested/carry","description":"lists and structs survive filters, sorts, joins and slices","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"g","type":"i32"},{"name":"xs","type":"list"},{"name":"ws","type":"list"},{"name":"s","type":"struct"}],"columns":{"k":["a","b","c","d","e"],"g":[1,2,1,2,1],"xs":[[1,2,3],[],null,[4,null,2],[7]],"ws":[["a","b"],["c"],[],null,["z","a"]],"s":[{"a":1,"b":"x"},{"a":null,"b":"y"},null,{"a":4,"b":null},{"a":5,"b":"z"}]}},"right":{"fields":[{"name":"k","type":"utf8"},{"name":"tags","type":"list"}],"columns":{"k":["a","d","x"],"tags":[["t1"],["t2","t3"],[]]}}},"ops":[{"op":"filter","predicate":{"fn":"neq","args":[{"col":"k"},{"lit":"c"}]}},{"op":"sort","keys":[{"expr":{"col":"g"},"descending":true},{"expr":{"col":"k"}}]},{"op":"join","right":"right","on":["k"],"how":"left"},{"op":"head","n":4}],"expected":{"fields":[{"name":"k","type":"utf8"},{"name":"g","type":"i32"},{"name":"xs","type":"list"},{"name":"ws","type":"list"},{"name":"s","type":"struct"},{"name":"tags","type":"list"}],"columns":{"k":["b","d","a","e"],"g":[2,2,1,1],"xs":[[],[4,null,2],[1,2,3],[7]],"ws":[["c"],null,["a","b"],["z","a"]],"s":[{"a":null,"b":"y"},{"a":4,"b":null},{"a":1,"b":"x"},{"a":5,"b":"z"}],"tags":[null,["t2","t3"],["t1"],null]}}},{"id":"decimal/arith","description":"decimal addition, subtraction and multiplication","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"price","type":"decimal[10,2]"},{"name":"fee","type":"decimal[6,3]"},{"name":"qty","type":"decimal[4,0]"},{"name":"x","type":"f64"}],"columns":{"k":["a","b","a","c","b","a"],"price":["12.34","0.10","-5.00",null,"99999999.99","10.01"],"fee":["0.125","1.000",null,"2.500","-0.001","10.010"],"qty":["3","10","2","1",null,"7"],"x":[1.005,-2.5,123.456,null,1000000.0,0.0]}}},"ops":[{"op":"select","exprs":{"sum":{"fn":"add","args":[{"col":"price"},{"col":"fee"}]},"diff":{"fn":"sub","args":[{"col":"price"},{"col":"fee"}]},"prod":{"fn":"mul","args":[{"col":"price"},{"col":"qty"}]},"plus":{"fn":"add","args":[{"col":"price"},{"lit":"0.5","type":"decimal[3,1]"}]}}}],"expected":{"fields":[{"name":"sum","type":"decimal[38,3]"},{"name":"diff","type":"decimal[38,3]"},{"name":"prod","type":"decimal[38,2]"},{"name":"plus","type":"decimal[38,2]"}],"columns":{"sum":["12.465","1.100",null,null,"99999999.989","20.020"],"diff":["12.215","-0.900",null,null,"99999999.991","0.000"],"prod":["37.02","1.00","-10.00",null,null,"70.07"],"plus":["12.84","0.60","-4.50",null,"100000000.49","10.51"]}}},{"id":"decimal/compare","description":"decimal comparisons, also across scales","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"price","type":"decimal[10,2]"},{"name":"fee","type":"decimal[6,3]"},{"name":"qty","type":"decimal[4,0]"},{"name":"x","type":"f64"}],"columns":{"k":["a","b","a","c","b","a"],"price":["12.34","0.10","-5.00",null,"99999999.99","10.01"],"fee":["0.125","1.000",null,"2.500","-0.001","10.010"],"qty":["3","10","2","1",null,"7"],"x":[1.005,-2.5,123.456,null,1000000.0,0.0]}}},"ops":[{"op":"filter","predicate":{"fn":"and","args":[{"fn":"gt","args":[{"col":"price"},{"lit":"10.00","type":"decimal[10,2]"}]},{"fn":"gte","args":[{"col":"price"},{"col":"fee"}]}]}}],"expected":{"fields":[{"name":"k","type":"utf8"},{"name":"price","type":"decimal[10,2]"},{"name":"fee","type":"decimal[6,3]"},{"name":"qty","type":"decimal[4,0]"},{"name":"x","type":"f64"}],"columns":{"k":["a","b","a"],"price":["12.34","99999999.99","10.01"],"fee":["0.125","-0.001","10.010"],"qty":["3",null,"7"],"x":[1.005,1000000.0,0.0]}}},{"id":"decimal/aggregate","description":"sums, extremes and means of decimals per group","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"price","type":"decimal[10,2]"},{"name":"fee","type":"decimal[6,3]"},{"name":"qty","type":"decimal[4,0]"},{"name":"x","type":"f64"}],"columns":{"k":["a","b","a","c","b","a"],"price":["12.34","0.10","-5.00",null,"99999999.99","10.01"],"fee":["0.125","1.000",null,"2.500","-0.001","10.010"],"qty":["3","10","2","1",null,"7"],"x":[1.005,-2.5,123.456,null,1000000.0,0.0]}}},"ops":[{"op":"groupBy","keys":["k"],"aggs":{"total":{"agg":"sum","arg":{"col":"price"}},"min":{"agg":"min","arg":{"col":"price"}},"max":{"agg":"max","arg":{"col":"fee"}},"mean":{"agg":"mean","arg":{"col":"price"}},"n":{"agg":"count","arg":{"col":"price"}}}}],"expected":{"fields":[{"name":"k","type":"utf8"},{"name":"total","type":"decimal[38,2]"},{"name":"min","type":"decimal[10,2]"},{"name":"max","type":"decimal[6,3]"},{"name":"mean","type":"f64"},{"name":"n","type":"u32"}],"columns":{"k":["a","b","c"],"total":["17.35","100000000.09","0.00"],"min":["-5.00","0.10",null],"max":["10.010","1.000","2.500"],"mean":[5.783333333333334,50000000.044999994,null],"n":[3,2,0]}}},{"id":"decimal/cast","description":"casts between decimals, floats, integers and strings","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"price","type":"decimal[10,2]"},{"name":"fee","type":"decimal[6,3]"},{"name":"qty","type":"decimal[4,0]"},{"name":"x","type":"f64"}],"columns":{"k":["a","b","a","c","b","a"],"price":["12.34","0.10","-5.00",null,"99999999.99","10.01"],"fee":["0.125","1.000",null,"2.500","-0.001","10.010"],"qty":["3","10","2","1",null,"7"],"x":[1.005,-2.5,123.456,null,1000000.0,0.0]}}},"ops":[{"op":"select","exprs":{"f":{"fn":"cast","args":[{"col":"price"}],"to":"f64"},"i":{"fn":"cast","args":[{"col":"price"}],"to":"i32"},"s":{"fn":"cast","args":[{"col":"fee"}],"to":"utf8"},"d":{"fn":"cast","args":[{"col":"x"}],"to":"decimal[10,2]"},"r":{"fn":"cast","args":[{"col":"price"}],"to":"decimal[12,1]"}}}],"expected":{"fields":[{"name":"f","type":"f64"},{"name":"i","type":"i32"},{"name":"s","type":"utf8"},{"name":"d","type":"decimal[10,2]"},{"name":"r","type":"decimal[12,1]"}],"columns":{"f":[12.34,0.1,-5.0,null,99999999.99,10.01],"i":[12,0,-5,null,99999999,10],"s":["0.125","1.000",null,"2.500","-0.001","10.010"],"d":["1.00","-2.50","123.46",null,"1000000.00","0.00"],"r":["12.3","0.1","-5.0",null,"100000000.0","10.0"]}}},{"id":"decimal/sort","description":"sort by a decimal","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"price","type":"decimal[10,2]"},{"name":"fee","type":"decimal[6,3]"},{"name":"qty","type":"decimal[4,0]"},{"name":"x","type":"f64"}],"columns":{"k":["a","b","a","c","b","a"],"price":["12.34","0.10","-5.00",null,"99999999.99","10.01"],"fee":["0.125","1.000",null,"2.500","-0.001","10.010"],"qty":["3","10","2","1",null,"7"],"x":[1.005,-2.5,123.456,null,1000000.0,0.0]}}},"ops":[{"op":"sort","keys":[{"expr":{"col":"price"},"descending":true}],"nullsLast":true}],"expected":{"fields":[{"name":"k","type":"utf8"},{"name":"price","type":"decimal[10,2]"},{"name":"fee","type":"decimal[6,3]"},{"name":"qty","type":"decimal[4,0]"},{"name":"x","type":"f64"}],"columns":{"k":["b","a","a","b","a","c"],"price":["99999999.99","12.34","10.01","0.10","-5.00",null],"fee":["-0.001","0.125","10.010","1.000",null,"2.500"],"qty":[null,"3","7","10","2","1"],"x":[1000000.0,1.005,0.0,-2.5,123.456,null]}}},{"id":"decimal/round-ties","description":"narrowing the scale rounds exact ties","inputs":{"left":{"fields":[{"name":"t","type":"decimal[6,3]"}],"columns":{"t":["0.250","0.350","-0.250","-0.350","1.050","2.500","0.249"]}}},"ops":[{"op":"select","exprs":{"one":{"fn":"cast","args":[{"col":"t"}],"to":"decimal[6,1]"},"zero":{"fn":"cast","args":[{"col":"t"}],"to":"decimal[6,0]"}}}],"expected":{"fields":[{"name":"one","type":"decimal[6,1]"},{"name":"zero","type":"decimal[6,0]"}],"columns":{"one":["0.2","0.4","-0.2","-0.4","1.0","2.5","0.2"],"zero":["0","0","0","0","1","2","0"]}}},{"id":"decimal/cast-float-ties","description":"floats cast to decimals round exact ties half to even, also below zero","inputs":{"left":{"fields":[{"name":"x","type":"f64"}],"columns":{"x":[-1.5,-2.5,1.5,2.5,-0.75]}}},"ops":[{"op":"select","exprs":{"zero":{"fn":"cast","args":[{"col":"x"}],"to":"decimal[18,0]"},"one":{"fn":"cast","args":[{"col":"x"}],"to":"decimal[18,1]"}}}],"expected":{"fields":[{"name":"zero","type":"decimal[18,0]"},{"name":"one","type":"decimal[18,1]"}],"columns":{"zero":["-2","-2","2","2","-1"],"one":["-1.5","-2.5","1.5","2.5","-0.8"]}}},{"id":"decimal/round","description":"rounding decimals keeps the scale and rounds half to even","inputs":{"left":{"fields":[{"name":"t","type":"decimal[6,3]"}],"columns":{"t":["0.250","0.350","-0.250","-0.350","1.050","2.500","0.249",null]}}},"ops":[{"op":"select","exprs":{"r0":{"fn":"round","args":[{"col":"t"}]},"r1":{"fn":"round","args":[{"col":"t"}],"decimals":1},"r2":{"fn":"round","args":[{"col":"t"}],"decimals":2},"r3":{"fn":"round","args":[{"col":"t"}],"decimals":3}}}],"expected":{"fields":[{"name":"r0","type":"decimal[7,3]"},{"name":"r1","type":"decimal[7,3]"},{"name":"r2","type":"decimal[7,3]"},{"name":"r3","type":"decimal[6,3]"}],"columns":{"r0":["0.000","0.000","0.000","0.000","1.000","2.000","0.000",null],"r1":["0.200","0.400","-0.200","-0.400","1.000","2.500","0.200",null],"r2":["0.250","0.350","-0.250","-0.350","1.050","2.500","0.250",null],"r3":["0.250","0.350","-0.250","-0.350","1.050","2.500","0.249",null]}}},{"id":"arith/u32-divide-by-zero","description":"unsigned 32-bit floor division and modulo by zero give null","inputs":{"left":{"fields":[{"name":"n","type":"u32"},{"name":"d","type":"u32"}],"columns":{"n":[7,0,5],"d":[0,0,2]}}},"ops":[{"op":"select","exprs":{"q":{"fn":"floorDiv","args":[{"col":"n"},{"col":"d"}]},"m":{"fn":"mod","args":[{"col":"n"},{"col":"d"}]},"qs":{"fn":"floorDiv","args":[{"col":"n"},{"lit":0,"type":"u32"}]},"ms":{"fn":"mod","args":[{"col":"n"},{"lit":0,"type":"u32"}]}}}],"expected":{"fields":[{"name":"q","type":"u32"},{"name":"m","type":"u32"},{"name":"qs","type":"u32"},{"name":"ms","type":"u32"}],"columns":{"q":[null,null,2],"m":[null,null,1],"qs":[null,null,null],"ms":[null,null,null]}}},{"id":"window/filter-after-sort","description":"a window predicate sees the sorted rows","inputs":{"left":{"fields":[{"name":"g","type":"i32"},{"name":"h","type":"i32"},{"name":"x","type":"f64"}],"columns":{"g":[1,2,1,2],"h":[1,1,2,2],"x":[3.0,1.0,2.0,4.0]}},"right":{"fields":[{"name":"g","type":"i32"},{"name":"name","type":"utf8"}],"columns":{"g":[1],"name":["one"]}}},"ops":[{"op":"sort","keys":[{"expr":{"col":"x"}}]},{"op":"filter","predicate":{"fn":"gt","args":[{"fn":"cumSum","args":[{"col":"x"}]},{"lit":3.0}]}}],"expected":{"fields":[{"name":"g","type":"i32"},{"name":"h","type":"i32"},{"name":"x","type":"f64"}],"columns":{"g":[1,2],"h":[1,2],"x":[3.0,4.0]}}},{"id":"window/filter-after-filter","description":"a window predicate sees only the rows an earlier filter kept","inputs":{"left":{"fields":[{"name":"g","type":"i32"},{"name":"h","type":"i32"},{"name":"x","type":"f64"}],"columns":{"g":[1,2,1,2],"h":[1,1,2,2],"x":[3.0,1.0,2.0,4.0]}},"right":{"fields":[{"name":"g","type":"i32"},{"name":"name","type":"utf8"}],"columns":{"g":[1],"name":["one"]}}},"ops":[{"op":"filter","predicate":{"fn":"gt","args":[{"col":"x"},{"lit":1.0}]}},{"op":"filter","predicate":{"fn":"gt","args":[{"fn":"shift","args":[{"col":"x"}]},{"lit":2.0}]}}],"expected":{"fields":[{"name":"g","type":"i32"},{"name":"h","type":"i32"},{"name":"x","type":"f64"}],"columns":{"g":[1],"h":[2],"x":[2.0]}}},{"id":"window/filter-after-concat","description":"a window predicate sees every input of a concat","inputs":{"left":{"fields":[{"name":"g","type":"i32"},{"name":"h","type":"i32"},{"name":"x","type":"f64"}],"columns":{"g":[1,2,1,2],"h":[1,1,2,2],"x":[3.0,1.0,2.0,4.0]}},"right":{"fields":[{"name":"g","type":"i32"},{"name":"name","type":"utf8"}],"columns":{"g":[1],"name":["one"]}}},"ops":[{"op":"concat","with":"left"},{"op":"filter","predicate":{"fn":"gt","args":[{"fn":"cumSum","args":[{"col":"x"}]},{"lit":15.0}]}}],"expected":{"fields":[{"name":"g","type":"i32"},{"name":"h","type":"i32"},{"name":"x","type":"f64"}],"columns":{"g":[1,2],"h":[2,2],"x":[2.0,4.0]}}},{"id":"window/filter-after-join","description":"a window predicate sees only the joined rows","inputs":{"left":{"fields":[{"name":"g","type":"i32"},{"name":"h","type":"i32"},{"name":"x","type":"f64"}],"columns":{"g":[1,2,1,2],"h":[1,1,2,2],"x":[3.0,1.0,2.0,4.0]}},"right":{"fields":[{"name":"g","type":"i32"},{"name":"name","type":"utf8"}],"columns":{"g":[1],"name":["one"]}}},"ops":[{"op":"join","right":"right","on":["g"],"how":"inner"},{"op":"filter","predicate":{"fn":"gt","args":[{"fn":"shift","args":[{"col":"x"}]},{"lit":2.0}]}}],"expected":{"fields":[{"name":"g","type":"i32"},{"name":"h","type":"i32"},{"name":"x","type":"f64"},{"name":"name","type":"utf8"}],"columns":{"g":[1],"h":[2],"x":[2.0],"name":["one"]}}},{"id":"window/filter-whole","description":"a window predicate sees the whole input","inputs":{"left":{"fields":[{"name":"g","type":"i32"},{"name":"h","type":"i32"},{"name":"x","type":"f64"}],"columns":{"g":[1,2,1,2],"h":[1,1,2,2],"x":[3.0,1.0,2.0,4.0]}},"right":{"fields":[{"name":"g","type":"i32"},{"name":"name","type":"utf8"}],"columns":{"g":[1],"name":["one"]}}},"ops":[{"op":"filter","predicate":{"fn":"gt","args":[{"fn":"cumSum","args":[{"col":"x"}]},{"lit":5.0}]}}],"expected":{"fields":[{"name":"g","type":"i32"},{"name":"h","type":"i32"},{"name":"x","type":"f64"}],"columns":{"g":[1,2],"h":[2,2],"x":[2.0,4.0]}}},{"id":"window/agg-of-window","description":"an aggregate of a window sees the whole input","inputs":{"left":{"fields":[{"name":"g","type":"i32"},{"name":"h","type":"i32"},{"name":"x","type":"f64"}],"columns":{"g":[1,2,1,2],"h":[1,1,2,2],"x":[3.0,1.0,2.0,4.0]}},"right":{"fields":[{"name":"g","type":"i32"},{"name":"name","type":"utf8"}],"columns":{"g":[1],"name":["one"]}}},"ops":[{"op":"agg","aggs":{"s":{"agg":"sum","arg":{"fn":"cumSum","args":[{"col":"x"}]}},"n":{"agg":"len"}}}],"expected":{"fields":[{"name":"s","type":"f64"},{"name":"n","type":"u32"}],"columns":{"s":[23.0],"n":[4]}}},{"id":"window/grouped-agg-of-window","description":"windows inside grouped aggregates run per group","inputs":{"left":{"fields":[{"name":"g","type":"i32"},{"name":"h","type":"i32"},{"name":"x","type":"f64"}],"columns":{"g":[1,2,1,2],"h":[1,1,2,2],"x":[3.0,1.0,2.0,4.0]}},"right":{"fields":[{"name":"g","type":"i32"},{"name":"name","type":"utf8"}],"columns":{"g":[1],"name":["one"]}}},"ops":[{"op":"groupBy","keys":["g"],"aggs":{"s":{"agg":"sum","arg":{"fn":"cumSum","args":[{"col":"x"}]}},"l":{"agg":"list","arg":{"fn":"cumSum","args":[{"col":"x"}]}},"o":{"agg":"sum","arg":{"over":{"agg":"sum","arg":{"col":"x"}},"partitionBy":[{"col":"h"}]}}}}],"expected":{"fields":[{"name":"g","type":"i32"},{"name":"s","type":"f64"},{"name":"l","type":"list"},{"name":"o","type":"f64"}],"columns":{"g":[1,2],"s":[8.0,6.0],"l":[[3.0,5.0],[1.0,5.0]],"o":[5.0,5.0]}}}]} diff --git a/packages/effect/typetest/DataFrame.tst.ts b/packages/effect/typetest/DataFrame.tst.ts index 605f5fbcc7e..25cdce2b5a1 100644 --- a/packages/effect/typetest/DataFrame.tst.ts +++ b/packages/effect/typetest/DataFrame.tst.ts @@ -286,6 +286,7 @@ describe("Types", () => { it("decimals", () => { expect(Expr.add(price, price)).type.toBe>() expect(Expr.sum(price)).type.toBe>() + expect(Expr.round(price, 1)).type.toBe>() expect(Expr.castDecimal(Expr.lit(1.5), 10, 2)).type.toBe>() }) }) From fb715f2590b102bb72038d49d797b0555cb48be5 Mon Sep 17 00:00:00 2001 From: Effect Bot Date: Sun, 15 Nov 2026 11:56:59 +0000 Subject: [PATCH 08/88] Fail decimal sums and running sums on 64-bit overflow --- .../src/dataframe/internal/aggregate.ts | 3 ++- .../effect/src/dataframe/internal/decimal.ts | 7 +++--- .../effect/src/dataframe/internal/window.ts | 4 +++- packages/effect/test/dataframe/Query.test.ts | 23 +++++++++++++++++++ 4 files changed, 32 insertions(+), 5 deletions(-) diff --git a/packages/effect/src/dataframe/internal/aggregate.ts b/packages/effect/src/dataframe/internal/aggregate.ts index b62677342dd..6b7c0ecb397 100644 --- a/packages/effect/src/dataframe/internal/aggregate.ts +++ b/packages/effect/src/dataframe/internal/aggregate.ts @@ -156,7 +156,8 @@ const sum = (v: V.Vector, g: Grouping): V.Vector => { const valid = v.valid for (let i = 0; i < n; i++) if (valid === undefined || valid[i] === 1) acc[ids[i]] += x[i] const out = new BigInt64Array(groups) - for (let k = 0; k < groups; k++) out[k] = BigInt.asIntN(64, acc[k]) + const fit = Dec.isDecimal(type) ? Dec.checked : (x: bigint) => BigInt.asIntN(64, x) + for (let k = 0; k < groups; k++) out[k] = fit(acc[k]) return V.i64(type, out) } default: diff --git a/packages/effect/src/dataframe/internal/decimal.ts b/packages/effect/src/dataframe/internal/decimal.ts index 6798f10186e..9a0658ec6a5 100644 --- a/packages/effect/src/dataframe/internal/decimal.ts +++ b/packages/effect/src/dataframe/internal/decimal.ts @@ -1,8 +1,8 @@ /** * Decimals: unscaled 64-bit integers with a scale (`value / 10^scale`), so * up to 18 significant digits. Arithmetic follows Polars' result types - * (precision 38, scales added for products, the larger scale for sums) and - * fails instead of overflowing 64 bits. Rescaling down rounds half to even. + * (precision 38, scales added for products, the larger scale for sums); + * arithmetic, sums and running sums fail instead of overflowing 64 bits. Rescaling down rounds half to even. * * @internal */ @@ -39,7 +39,8 @@ export const supertype = (a: S.DataType, b: S.DataType): S.DataType | undefined return undefined } -const checked = (x: bigint): bigint => (x > MAX || x < MIN ? V.fail("decimal value does not fit 18 digits") : x) +/** `x`, or a failure when it overflows 64-bit decimal storage. @internal */ +export const checked = (x: bigint): bigint => (x > MAX || x < MIN ? V.fail("decimal value overflows 64 bits") : x) /** Rounds `x / 10^k` half to even (`BigDecimal.round`). */ const divRound = (x: bigint, k: number): bigint => diff --git a/packages/effect/src/dataframe/internal/window.ts b/packages/effect/src/dataframe/internal/window.ts index 8e0851ddaf5..dd0c3cf9ee1 100644 --- a/packages/effect/src/dataframe/internal/window.ts +++ b/packages/effect/src/dataframe/internal/window.ts @@ -7,6 +7,7 @@ */ import * as S from "../../arrow/ArrowSchema.ts" import type * as Plan from "../Plan.ts" +import * as Dec from "./decimal.ts" import * as D from "./dtype.ts" import { compareBytes } from "./kernels.ts" import * as Sort from "./sort.ts" @@ -78,8 +79,9 @@ export const cumSum = (v: V.Vector, p: Partition): V.Vector => { const x = v.values const acc = new BigInt64Array(p.count) const out = new BigInt64Array(n) + const fit = Dec.isDecimal(type) ? Dec.checked : (x: bigint) => BigInt.asIntN(64, x) for (let i = 0; i < n; i++) { - if (valid === undefined || valid[i] === 1) out[i] = acc[ids[i]] = BigInt.asIntN(64, acc[ids[i]] + x[i]) + if (valid === undefined || valid[i] === 1) out[i] = acc[ids[i]] = fit(acc[ids[i]] + x[i]) } return V.i64(type, out, valid) } diff --git a/packages/effect/test/dataframe/Query.test.ts b/packages/effect/test/dataframe/Query.test.ts index b406f8db688..d642a190bfb 100644 --- a/packages/effect/test/dataframe/Query.test.ts +++ b/packages/effect/test/dataframe/Query.test.ts @@ -97,6 +97,29 @@ describe("Query", () => { assert.strictEqual(error.message, "filter predicate must be boolean, got Utf8") })) + it.effect("decimal sums fail instead of overflowing 64 bits", () => + Effect.gen(function*() { + const nines = (rows: number) => + DataFrame.fromColumns({ s: Array.from({ length: rows }, () => "999999999999999999") }) + const d = (c: Query.Columns<{ readonly s: string }>) => Expr.castDecimal(c.s, 18, 0) + const overflow = (query: Effect.Effect) => + Effect.map(Effect.flip(query), (error) => error.message) + const ten = Query.from(nines(10)) + assert.strictEqual( + yield* overflow(Query.aggregate(ten, (c) => ({ total: Expr.sum(d(c)) }))), + "decimal value overflows 64 bits" + ) + assert.strictEqual( + yield* overflow(Query.select(ten, (c) => ({ running: Expr.cumSum(d(c)) }))), + "decimal value overflows 64 bits" + ) + const streamed = Query.fromPlan({ _tag: "Table", name: "t" }).pipe( + Query.aggregate((c) => ({ total: Expr.sum(Expr.castDecimal(c.s as Expr.Expr, 18, 0)) })), + Effect.provide(Catalog.layer({ t: Stream.make(nines(5), nines(5)) })) + ) + assert.strictEqual(yield* overflow(streamed), "decimal value overflows 64 bits") + })) + it.effect("round rejects negative or fractional places", () => Effect.gen(function*() { for (const places of [-1, 0.5]) { From 8aa5020114dffac9790d846705d5b43c7fdd5a39 Mon Sep 17 00:00:00 2001 From: Effect Bot Date: Sun, 15 Nov 2026 11:57:45 +0000 Subject: [PATCH 09/88] Trim list children when importing sliced Arrow data --- .../effect/src/dataframe/internal/vector.ts | 9 ++++++--- .../effect/test/dataframe/DataFrame.test.ts | 17 +++++++++++++++++ 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/packages/effect/src/dataframe/internal/vector.ts b/packages/effect/src/dataframe/internal/vector.ts index 760446363e0..ccfc769f788 100644 --- a/packages/effect/src/dataframe/internal/vector.ts +++ b/packages/effect/src/dataframe/internal/vector.ts @@ -84,7 +84,10 @@ export interface List { readonly kind: "list" readonly type: S.DataType readonly length: number - /** `offsets[i]..offsets[i + 1]` are slot `i`'s elements in `child`. */ + /** + * `offsets[i]..offsets[i + 1]` are slot `i`'s elements in `child`; + * `offsets[0]` is 0 and `child` holds exactly `offsets[length]` elements. + */ readonly offsets: Int32Array readonly child: Vector readonly valid: Uint8Array | undefined @@ -578,8 +581,8 @@ export const fromArrowData = (data: ArrowData.ArrowData): Vector => { : typed(Int32Array, data.buffers[0]).subarray(data.offset, data.offset + n + 1) const child = fromArrowData(data.children[0]) const first = raw[0] - if (first === 0) return list(storage, raw, child, valid) - const offsets = Int32Array.from(raw, (o) => o - first) + if (first === 0 && raw[n] === child.length) return list(storage, raw, child, valid) + const offsets = first === 0 ? raw : Int32Array.from(raw, (o) => o - first) return list(storage, offsets, slice(child, first, raw[n]), valid) } case "Struct": { diff --git a/packages/effect/test/dataframe/DataFrame.test.ts b/packages/effect/test/dataframe/DataFrame.test.ts index 386b53a36b3..aab6112fa43 100644 --- a/packages/effect/test/dataframe/DataFrame.test.ts +++ b/packages/effect/test/dataframe/DataFrame.test.ts @@ -156,6 +156,23 @@ describe("DataFrame", () => { assert.strictEqual(table.batches[0].length, 2) assert.deepStrictEqual(DataFrame.toRows(DataFrame.fromArrowTable(table)), DataFrame.toRows(df)) }) + + it("imports sliced list columns with only their own elements", () => { + const Lists = Schema.Struct({ xs: Schema.Array(Schema.Number) }) + const df = DataFrame.fromRows(Lists, [{ xs: [1] }, { xs: [2] }, { xs: [3] }]) + const table = DataFrame.toArrowTable(df) + const sliced = DataFrame.fromArrowTable(ArrowTable.slice(table, 0, 1)) + const sums = Query.from(sliced).pipe( + Query.select((c) => ({ s: Expr.listSum(c.xs as Expr.Expr>) })), + Effect.runSync + ) + assert.deepStrictEqual(DataFrame.toRows(sums), [{ s: 1 }]) + const two = ArrowTable.make(table.schema, [ + ...ArrowTable.slice(table, 0, 1).batches, + ...ArrowTable.slice(table, 2, 3).batches + ]) + assert.deepStrictEqual(DataFrame.toRows(DataFrame.fromArrowTable(two)), [{ xs: [1] }, { xs: [3] }]) + }) }) describe("values", () => { From 2a6cb37488bcbad512c6f9a9b1ea936762ff478e Mon Sep 17 00:00:00 2001 From: Effect Bot Date: Sun, 15 Nov 2026 11:58:55 +0000 Subject: [PATCH 10/88] Clip slices from their unclamped start, as Polars does --- packages/effect/src/dataframe/Query.ts | 6 ++++++ packages/effect/src/dataframe/internal/query.ts | 6 ++++-- packages/effect/test/fixtures/dataframe/generate_gold.py | 3 +++ packages/effect/test/fixtures/dataframe/gold/extended.json | 2 +- 4 files changed, 14 insertions(+), 3 deletions(-) diff --git a/packages/effect/src/dataframe/Query.ts b/packages/effect/src/dataframe/Query.ts index 85efe83fd6d..0564c775b15 100644 --- a/packages/effect/src/dataframe/Query.ts +++ b/packages/effect/src/dataframe/Query.ts @@ -366,6 +366,12 @@ export const sort: { * `length` rows (all remaining when omitted) starting at `offset`; a negative * offset counts from the end. * + * **Details** + * + * As in Polars, the slice covers `offset` up to `offset + length` and is then + * clipped to the rows there are: over 3 rows, `slice(-5, 3)` keeps the first + * row and `slice(-5, 1)` none. + * * @stability experimental * @category slicing * @since 4.1.5 diff --git a/packages/effect/src/dataframe/internal/query.ts b/packages/effect/src/dataframe/internal/query.ts index 1bed04985b6..72e22ca1357 100644 --- a/packages/effect/src/dataframe/internal/query.ts +++ b/packages/effect/src/dataframe/internal/query.ts @@ -568,8 +568,10 @@ export const step = (node: Node, inputs: ReadonlyArray): F.Frame => { return F.sort(input, sortKeys(node.keys), node.nullsLast, node.limit) case "Slice": { const n = input.length - const start = node.offset < 0 ? Math.max(0, n + node.offset) : Math.min(node.offset, n) - return F.slice(input, start, node.length === undefined ? n : start + node.length) + const from = node.offset < 0 ? n + node.offset : node.offset + const start = Math.min(Math.max(from, 0), n) + const end = node.length === undefined ? n : Math.min(Math.max(from + node.length, start), n) + return F.slice(input, start, end) } case "Unique": return F.unique(input, node.columns) diff --git a/packages/effect/test/fixtures/dataframe/generate_gold.py b/packages/effect/test/fixtures/dataframe/generate_gold.py index 45f1c1b636e..e59059260e3 100644 --- a/packages/effect/test/fixtures/dataframe/generate_gold.py +++ b/packages/effect/test/fixtures/dataframe/generate_gold.py @@ -901,6 +901,9 @@ def extended_cases(): {"left": table([("n", "u32"), ("d", "u32")], {"n": [7, 0, 5], "d": [0, 0, 2]})}, [{"op": "select", "exprs": {"q": fn("floorDiv", col("n"), col("d")), "m": fn("mod", col("n"), col("d")), "qs": fn("floorDiv", col("n"), lit(0, "u32")), "ms": fn("mod", col("n"), lit(0, "u32"))}}]) + for offset, length in [(-5, 1), (-5, 3), (-5, 4), (-2, 5), (-5, None), (1, 0)]: + add(f"slice/clip{offset}{'' if length is None else f'+{length}'}", "slices reaching before the first row clip from the unclamped start", + {"left": table([("x", "i32")], {"x": [1, 2, 3]})}, [{"op": "slice", "offset": offset, **({} if length is None else {"length": length})}]) # windows see exactly the rows of their input, whatever runs before or after them W = {"left": table([("g", "i32"), ("h", "i32"), ("x", "f64")], {"g": [1, 2, 1, 2], "h": [1, 1, 2, 2], "x": [3.0, 1.0, 2.0, 4.0]}), "right": table([("g", "i32"), ("name", "utf8")], {"g": [1], "name": ["one"]})} diff --git a/packages/effect/test/fixtures/dataframe/gold/extended.json b/packages/effect/test/fixtures/dataframe/gold/extended.json index e3fbd166bb6..f7adffd327f 100644 --- a/packages/effect/test/fixtures/dataframe/gold/extended.json +++ b/packages/effect/test/fixtures/dataframe/gold/extended.json @@ -1 +1 @@ -{"generator":"generate_gold.py","polars":"2.1.0","deviations":["JavaScript has no float literal distinct from an integer: an integral number literal behaves like a Polars integer literal (it takes the integer type of the column it meets), so cases never combine an integer column with an integral float literal.","Float to string casts use JavaScript number formatting (Polars prints 5.0 as '5.0', JavaScript as '5').","Joins never add '_right' suffixes: overlapping non-key columns are a type error.","Arithmetic between 64-bit integers (bigint) and other numeric types requires an explicit cast."],"cases":[{"id":"window/cum","description":"cumulative sum, min, max and count; nulls stay null","inputs":{"left":{"fields":[{"name":"g","type":"utf8"},{"name":"t","type":"i32"},{"name":"x","type":"f64"},{"name":"i","type":"i32"},{"name":"b","type":"bool"}],"columns":{"g":["a","b","a","a","b",null,"a","b","b","a"],"t":[1,1,2,3,2,1,4,3,4,5],"x":[1.5,2.0,null,4.0,"NaN",3.0,-1.0,2.0,0.5,2.0],"i":[3,1,4,1,5,9,null,6,5,3],"b":[true,false,null,true,true,false,true,null,false,true]}}},"ops":[{"op":"select","exprs":{"sx":{"fn":"cumSum","args":[{"col":"x"}]},"si":{"fn":"cumSum","args":[{"col":"i"}]},"mn":{"fn":"cumMin","args":[{"col":"i"}]},"mx":{"fn":"cumMax","args":[{"col":"x"}]},"n":{"fn":"cumCount","args":[{"col":"x"}]},"sb":{"fn":"cumSum","args":[{"col":"b"}]}}}],"expected":{"fields":[{"name":"sx","type":"f64"},{"name":"si","type":"i32"},{"name":"mn","type":"i32"},{"name":"mx","type":"f64"},{"name":"n","type":"u32"},{"name":"sb","type":"u32"}],"columns":{"sx":[1.5,3.5,null,7.5,"NaN","NaN","NaN","NaN","NaN","NaN"],"si":[3,4,8,9,14,23,null,29,34,37],"mn":[3,1,1,1,1,1,null,1,1,1],"mx":[1.5,2.0,null,4.0,4.0,4.0,4.0,4.0,4.0,4.0],"n":[1,2,2,3,4,5,6,7,8,9],"sb":[1,1,null,2,3,3,4,null,4,5]}}},{"id":"window/shift-diff","description":"shift (lag and lead) and diff","inputs":{"left":{"fields":[{"name":"g","type":"utf8"},{"name":"t","type":"i32"},{"name":"x","type":"f64"},{"name":"i","type":"i32"},{"name":"b","type":"bool"}],"columns":{"g":["a","b","a","a","b",null,"a","b","b","a"],"t":[1,1,2,3,2,1,4,3,4,5],"x":[1.5,2.0,null,4.0,"NaN",3.0,-1.0,2.0,0.5,2.0],"i":[3,1,4,1,5,9,null,6,5,3],"b":[true,false,null,true,true,false,true,null,false,true]}}},"ops":[{"op":"select","exprs":{"lag":{"fn":"shift","args":[{"col":"i"}]},"lead":{"fn":"shift","args":[{"col":"x"}],"n":-2},"d":{"fn":"diff","args":[{"col":"i"}]},"d2":{"fn":"diff","args":[{"col":"x"}],"n":2},"ls":{"fn":"shift","args":[{"col":"g"}]}}}],"expected":{"fields":[{"name":"lag","type":"i32"},{"name":"lead","type":"f64"},{"name":"d","type":"i32"},{"name":"d2","type":"f64"},{"name":"ls","type":"utf8"}],"columns":{"lag":[null,3,1,4,1,5,9,null,6,5],"lead":[null,4.0,"NaN",3.0,-1.0,2.0,0.5,2.0,null,null],"d":[null,-2,3,-3,4,4,null,null,-1,-2],"d2":[null,null,null,2.0,null,-1.0,"NaN",-1.0,1.5,0.0],"ls":[null,"a","b","a","a","b",null,"a","b","b"]}}},{"id":"window/rank","description":"rank methods, nulls and NaN","inputs":{"left":{"fields":[{"name":"g","type":"utf8"},{"name":"t","type":"i32"},{"name":"x","type":"f64"},{"name":"i","type":"i32"},{"name":"b","type":"bool"}],"columns":{"g":["a","b","a","a","b",null,"a","b","b","a"],"t":[1,1,2,3,2,1,4,3,4,5],"x":[1.5,2.0,null,4.0,"NaN",3.0,-1.0,2.0,0.5,2.0],"i":[3,1,4,1,5,9,null,6,5,3],"b":[true,false,null,true,true,false,true,null,false,true]}}},"ops":[{"op":"select","exprs":{"avg":{"fn":"rank","args":[{"col":"x"}]},"min":{"fn":"rank","args":[{"col":"i"}],"method":"min"},"max":{"fn":"rank","args":[{"col":"i"}],"method":"max"},"dense":{"fn":"rank","args":[{"col":"i"}],"method":"dense","descending":true},"ord":{"fn":"rank","args":[{"col":"x"}],"method":"ordinal"},"s":{"fn":"rank","args":[{"col":"g"}],"method":"dense"}}}],"expected":{"fields":[{"name":"avg","type":"f64"},{"name":"min","type":"u32"},{"name":"max","type":"u32"},{"name":"dense","type":"u32"},{"name":"ord","type":"u32"},{"name":"s","type":"u32"}],"columns":{"avg":[3.0,5.0,null,8.0,9.0,7.0,1.0,5.0,2.0,5.0],"min":[3,1,5,1,6,9,null,8,6,3],"max":[4,2,5,2,7,9,null,8,7,4],"dense":[5,6,4,6,3,1,null,2,3,5],"ord":[3,4,null,8,9,7,1,5,2,6],"s":[1,2,1,1,2,null,1,2,2,1]}}},{"id":"window/rolling","description":"rolling sum, mean, min and max","inputs":{"left":{"fields":[{"name":"g","type":"utf8"},{"name":"t","type":"i32"},{"name":"x","type":"f64"},{"name":"i","type":"i32"},{"name":"b","type":"bool"}],"columns":{"g":["a","b","a","a","b",null,"a","b","b","a"],"t":[1,1,2,3,2,1,4,3,4,5],"x":[1.5,2.0,null,4.0,"NaN",3.0,-1.0,2.0,0.5,2.0],"i":[3,1,4,1,5,9,null,6,5,3],"b":[true,false,null,true,true,false,true,null,false,true]}}},"ops":[{"op":"select","exprs":{"s":{"fn":"rollingSum","args":[{"col":"x"}],"window":3},"m":{"fn":"rollingMean","args":[{"col":"i"}],"window":2},"mn":{"fn":"rollingMin","args":[{"col":"x"}],"window":3,"minPeriods":1},"mx":{"fn":"rollingMax","args":[{"col":"i"}],"window":4,"minPeriods":2}}}],"expected":{"fields":[{"name":"s","type":"f64"},{"name":"m","type":"f64"},{"name":"mn","type":"f64"},{"name":"mx","type":"i32"}],"columns":{"s":[null,null,null,null,null,"NaN","NaN",4.0,1.5,4.5],"m":[null,2.0,2.5,2.5,3.0,7.0,null,null,5.5,4.0],"mn":[1.5,1.5,1.5,2.0,"NaN","NaN","NaN",-1.0,-1.0,0.5],"mx":[null,3,4,4,5,9,9,9,9,6]}}},{"id":"window/over-agg","description":"aggregates broadcast over partitions","inputs":{"left":{"fields":[{"name":"g","type":"utf8"},{"name":"t","type":"i32"},{"name":"x","type":"f64"},{"name":"i","type":"i32"},{"name":"b","type":"bool"}],"columns":{"g":["a","b","a","a","b",null,"a","b","b","a"],"t":[1,1,2,3,2,1,4,3,4,5],"x":[1.5,2.0,null,4.0,"NaN",3.0,-1.0,2.0,0.5,2.0],"i":[3,1,4,1,5,9,null,6,5,3],"b":[true,false,null,true,true,false,true,null,false,true]}}},"ops":[{"op":"withColumns","exprs":{"total":{"over":{"agg":"sum","arg":{"col":"x"}},"partitionBy":[{"col":"g"}]},"n":{"over":{"agg":"len"},"partitionBy":[{"col":"g"}]},"mean":{"over":{"agg":"mean","arg":{"col":"i"}},"partitionBy":[{"col":"g"}]},"first":{"over":{"agg":"first","arg":{"col":"t"}},"partitionBy":[{"col":"g"},{"col":"b"}]}}}],"expected":{"fields":[{"name":"g","type":"utf8"},{"name":"t","type":"i32"},{"name":"x","type":"f64"},{"name":"i","type":"i32"},{"name":"b","type":"bool"},{"name":"total","type":"f64"},{"name":"n","type":"u32"},{"name":"mean","type":"f64"},{"name":"first","type":"i32"}],"columns":{"g":["a","b","a","a","b",null,"a","b","b","a"],"t":[1,1,2,3,2,1,4,3,4,5],"x":[1.5,2.0,null,4.0,"NaN",3.0,-1.0,2.0,0.5,2.0],"i":[3,1,4,1,5,9,null,6,5,3],"b":[true,false,null,true,true,false,true,null,false,true],"total":[6.5,"NaN",6.5,6.5,"NaN",3.0,6.5,"NaN","NaN",6.5],"n":[5,4,5,5,4,1,5,4,4,5],"mean":[2.75,4.25,2.75,2.75,4.25,9.0,2.75,4.25,4.25,2.75],"first":[1,1,2,1,2,1,1,3,1,1]}}},{"id":"window/over-cum","description":"cumulative and shifted values per partition, in row order","inputs":{"left":{"fields":[{"name":"g","type":"utf8"},{"name":"t","type":"i32"},{"name":"x","type":"f64"},{"name":"i","type":"i32"},{"name":"b","type":"bool"}],"columns":{"g":["a","b","a","a","b",null,"a","b","b","a"],"t":[1,1,2,3,2,1,4,3,4,5],"x":[1.5,2.0,null,4.0,"NaN",3.0,-1.0,2.0,0.5,2.0],"i":[3,1,4,1,5,9,null,6,5,3],"b":[true,false,null,true,true,false,true,null,false,true]}}},"ops":[{"op":"withColumns","exprs":{"cs":{"over":{"fn":"cumSum","args":[{"col":"i"}]},"partitionBy":[{"col":"g"}]},"prev":{"over":{"fn":"shift","args":[{"col":"x"}]},"partitionBy":[{"col":"g"}]},"d":{"over":{"fn":"diff","args":[{"col":"t"}]},"partitionBy":[{"col":"g"}]},"r":{"over":{"fn":"rank","args":[{"col":"x"}],"method":"min"},"partitionBy":[{"col":"g"}]},"roll":{"over":{"fn":"rollingSum","args":[{"col":"i"}],"window":2,"minPeriods":1},"partitionBy":[{"col":"g"}]}}}],"expected":{"fields":[{"name":"g","type":"utf8"},{"name":"t","type":"i32"},{"name":"x","type":"f64"},{"name":"i","type":"i32"},{"name":"b","type":"bool"},{"name":"cs","type":"i32"},{"name":"prev","type":"f64"},{"name":"d","type":"i32"},{"name":"r","type":"u32"},{"name":"roll","type":"i32"}],"columns":{"g":["a","b","a","a","b",null,"a","b","b","a"],"t":[1,1,2,3,2,1,4,3,4,5],"x":[1.5,2.0,null,4.0,"NaN",3.0,-1.0,2.0,0.5,2.0],"i":[3,1,4,1,5,9,null,6,5,3],"b":[true,false,null,true,true,false,true,null,false,true],"cs":[3,1,7,8,6,9,null,12,17,11],"prev":[null,null,1.5,null,2.0,null,4.0,"NaN",2.0,-1.0],"d":[null,null,1,1,1,null,1,1,1,1],"r":[2,2,null,4,4,1,1,2,1,3],"roll":[3,1,7,5,6,9,1,11,11,3]}}},{"id":"window/over-filter","description":"filters after windows see the windowed values","inputs":{"left":{"fields":[{"name":"g","type":"utf8"},{"name":"t","type":"i32"},{"name":"x","type":"f64"},{"name":"i","type":"i32"},{"name":"b","type":"bool"}],"columns":{"g":["a","b","a","a","b",null,"a","b","b","a"],"t":[1,1,2,3,2,1,4,3,4,5],"x":[1.5,2.0,null,4.0,"NaN",3.0,-1.0,2.0,0.5,2.0],"i":[3,1,4,1,5,9,null,6,5,3],"b":[true,false,null,true,true,false,true,null,false,true]}}},"ops":[{"op":"withColumns","exprs":{"cs":{"over":{"fn":"cumSum","args":[{"col":"t"}]},"partitionBy":[{"col":"g"}]}}},{"op":"filter","predicate":{"fn":"gt","args":[{"col":"t"},{"lit":2}]}}],"expected":{"fields":[{"name":"g","type":"utf8"},{"name":"t","type":"i32"},{"name":"x","type":"f64"},{"name":"i","type":"i32"},{"name":"b","type":"bool"},{"name":"cs","type":"i32"}],"columns":{"g":["a","a","b","b","a"],"t":[3,4,3,4,5],"x":[4.0,-1.0,2.0,0.5,2.0],"i":[1,null,6,5,3],"b":[true,true,null,false,true],"cs":[6,10,6,10,15]}}},{"id":"window/filter-first","description":"windows after a filter only see the remaining rows","inputs":{"left":{"fields":[{"name":"g","type":"utf8"},{"name":"t","type":"i32"},{"name":"x","type":"f64"},{"name":"i","type":"i32"},{"name":"b","type":"bool"}],"columns":{"g":["a","b","a","a","b",null,"a","b","b","a"],"t":[1,1,2,3,2,1,4,3,4,5],"x":[1.5,2.0,null,4.0,"NaN",3.0,-1.0,2.0,0.5,2.0],"i":[3,1,4,1,5,9,null,6,5,3],"b":[true,false,null,true,true,false,true,null,false,true]}}},"ops":[{"op":"filter","predicate":{"fn":"isNotNull","args":[{"col":"x"}]}},{"op":"withColumns","exprs":{"cs":{"fn":"cumSum","args":[{"col":"x"}]},"r":{"over":{"fn":"rank","args":[{"col":"x"}]},"partitionBy":[{"col":"g"}]}}},{"op":"head","n":4}],"expected":{"fields":[{"name":"g","type":"utf8"},{"name":"t","type":"i32"},{"name":"x","type":"f64"},{"name":"i","type":"i32"},{"name":"b","type":"bool"},{"name":"cs","type":"f64"},{"name":"r","type":"f64"}],"columns":{"g":["a","b","a","b"],"t":[1,1,3,2],"x":[1.5,2.0,4.0,"NaN"],"i":[3,1,1,5],"b":[true,false,true,true],"cs":[1.5,3.5,7.5,"NaN"],"r":[2.0,2.5,4.0,4.0]}}},{"id":"pivot/sum","description":"pivot with a sum per cell; missing cells are null","inputs":{"left":{"fields":[{"name":"region","type":"utf8"},{"name":"quarter","type":"utf8"},{"name":"units","type":"i32"},{"name":"price","type":"f64"}],"columns":{"region":["north","north","south","south","north","east","south","north"],"quarter":["q1","q2","q1","q3","q1","q2","q1",null],"units":[10,20,5,null,7,3,8,1],"price":[1.5,2.0,1.0,3.0,null,2.5,1.0,9.0]}}},"ops":[{"op":"pivot","on":"quarter","index":["region"],"values":"units","aggregate":"sum"}],"expected":{"fields":[{"name":"region","type":"utf8"},{"name":"q1","type":"i32"},{"name":"q2","type":"i32"},{"name":"q3","type":"i32"},{"name":"null","type":"i32"}],"columns":{"region":["north","south","east"],"q1":[17,13,0],"q2":[20,0,3],"q3":[0,0,0],"null":[1,0,0]}}},{"id":"pivot/first","description":"pivot with the first value per cell","inputs":{"left":{"fields":[{"name":"region","type":"utf8"},{"name":"quarter","type":"utf8"},{"name":"units","type":"i32"},{"name":"price","type":"f64"}],"columns":{"region":["north","north","south","south","north","east","south","north"],"quarter":["q1","q2","q1","q3","q1","q2","q1",null],"units":[10,20,5,null,7,3,8,1],"price":[1.5,2.0,1.0,3.0,null,2.5,1.0,9.0]}}},"ops":[{"op":"pivot","on":"region","index":["quarter"],"values":"price","aggregate":"first"}],"expected":{"fields":[{"name":"quarter","type":"utf8"},{"name":"north","type":"f64"},{"name":"south","type":"f64"},{"name":"east","type":"f64"}],"columns":{"quarter":["q1","q2","q3",null],"north":[1.5,2.0,null,9.0],"south":[1.0,null,3.0,null],"east":[null,2.5,null,null]}}},{"id":"pivot/len","description":"pivot counting rows per cell","inputs":{"left":{"fields":[{"name":"region","type":"utf8"},{"name":"quarter","type":"utf8"},{"name":"units","type":"i32"},{"name":"price","type":"f64"}],"columns":{"region":["north","north","south","south","north","east","south","north"],"quarter":["q1","q2","q1","q3","q1","q2","q1",null],"units":[10,20,5,null,7,3,8,1],"price":[1.5,2.0,1.0,3.0,null,2.5,1.0,9.0]}}},"ops":[{"op":"pivot","on":"quarter","index":["region"],"values":"units","aggregate":"len"}],"expected":{"fields":[{"name":"region","type":"utf8"},{"name":"q1","type":"u32"},{"name":"q2","type":"u32"},{"name":"q3","type":"u32"},{"name":"null","type":"u32"}],"columns":{"region":["north","south","east"],"q1":[2,2,0],"q2":[1,0,1],"q3":[0,1,0],"null":[1,0,0]}}},{"id":"pivot/columns","description":"pivot with declared output columns","inputs":{"left":{"fields":[{"name":"region","type":"utf8"},{"name":"quarter","type":"utf8"},{"name":"units","type":"i32"},{"name":"price","type":"f64"}],"columns":{"region":["north","north","south","south","north","east","south","north"],"quarter":["q1","q2","q1","q3","q1","q2","q1",null],"units":[10,20,5,null,7,3,8,1],"price":[1.5,2.0,1.0,3.0,null,2.5,1.0,9.0]}}},"ops":[{"op":"pivot","on":"quarter","index":["region"],"values":"price","aggregate":"max","columns":["q1","q2","q4"]}],"expected":{"fields":[{"name":"region","type":"utf8"},{"name":"q1","type":"f64"},{"name":"q2","type":"f64"},{"name":"q4","type":"f64"}],"columns":{"region":["north","south","east"],"q1":[1.5,1.0,null],"q2":[2.0,null,2.5],"q4":[null,null,null]}}},{"id":"pivot/numeric-on","description":"pivot on a numeric column names columns by value","inputs":{"left":{"fields":[{"name":"g","type":"utf8"},{"name":"t","type":"i32"},{"name":"x","type":"f64"},{"name":"i","type":"i32"},{"name":"b","type":"bool"}],"columns":{"g":["a","b","a","a","b",null,"a","b","b","a"],"t":[1,1,2,3,2,1,4,3,4,5],"x":[1.5,2.0,null,4.0,"NaN",3.0,-1.0,2.0,0.5,2.0],"i":[3,1,4,1,5,9,null,6,5,3],"b":[true,false,null,true,true,false,true,null,false,true]}}},"ops":[{"op":"filter","predicate":{"fn":"isNotNull","args":[{"col":"g"}]}},{"op":"pivot","on":"t","index":["g"],"values":"i","aggregate":"min"}],"expected":{"fields":[{"name":"g","type":"utf8"},{"name":"1","type":"i32"},{"name":"2","type":"i32"},{"name":"3","type":"i32"},{"name":"4","type":"i32"},{"name":"5","type":"i32"}],"columns":{"g":["a","b"],"1":[3,1],"2":[4,5],"3":[1,6],"4":[null,5],"5":[3,null]}}},{"id":"unpivot/basic","description":"unpivot stacks columns, column by column","inputs":{"left":{"fields":[{"name":"region","type":"utf8"},{"name":"quarter","type":"utf8"},{"name":"units","type":"i32"},{"name":"price","type":"f64"}],"columns":{"region":["north","north","south","south","north","east","south","north"],"quarter":["q1","q2","q1","q3","q1","q2","q1",null],"units":[10,20,5,null,7,3,8,1],"price":[1.5,2.0,1.0,3.0,null,2.5,1.0,9.0]}}},"ops":[{"op":"unpivot","index":["region"],"on":["units","price"]}],"expected":{"fields":[{"name":"region","type":"utf8"},{"name":"variable","type":"utf8"},{"name":"value","type":"f64"}],"columns":{"region":["north","north","south","south","north","east","south","north","north","north","south","south","north","east","south","north"],"variable":["units","units","units","units","units","units","units","units","price","price","price","price","price","price","price","price"],"value":[10.0,20.0,5.0,null,7.0,3.0,8.0,1.0,1.5,2.0,1.0,3.0,null,2.5,1.0,9.0]}}},{"id":"unpivot/names","description":"unpivot with named variable and value columns","inputs":{"left":{"fields":[{"name":"region","type":"utf8"},{"name":"quarter","type":"utf8"},{"name":"units","type":"i32"},{"name":"price","type":"f64"}],"columns":{"region":["north","north","south","south","north","east","south","north"],"quarter":["q1","q2","q1","q3","q1","q2","q1",null],"units":[10,20,5,null,7,3,8,1],"price":[1.5,2.0,1.0,3.0,null,2.5,1.0,9.0]}}},"ops":[{"op":"pick","columns":["region","quarter"]},{"op":"unpivot","index":[],"on":["region","quarter"],"variable":"field","value":"text"}],"expected":{"fields":[{"name":"field","type":"utf8"},{"name":"text","type":"utf8"}],"columns":{"field":["region","region","region","region","region","region","region","region","quarter","quarter","quarter","quarter","quarter","quarter","quarter","quarter"],"text":["north","north","south","south","north","east","south","north","q1","q2","q1","q3","q1","q2","q1",null]}}},{"id":"temporal/extract","description":"calendar fields of timestamps and dates","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"ts","type":"datetime[us]"},{"name":"ms","type":"datetime[ms]"},{"name":"d","type":"date"},{"name":"dur","type":"duration[us]"}],"columns":{"k":["a","b","c","d","e","f"],"ts":[1709214307123456,1704067199999000,1709510400000000,null,-14182940000000,1710052200000500],"ms":[1709214307123,1704067200000,null,1709251200000,0,1709942400000],"d":[19782,19722,19786,19782,null,19792],"dur":[5400000000,-1500000,null,259200000001,0,90000000000]}},"right":{"fields":[{"name":"d","type":"date"},{"name":"name","type":"utf8"}],"columns":{"d":[19782,19792,20089],"name":["leap","sunday","new year"]}}},"ops":[{"op":"select","exprs":{"y":{"fn":"year","args":[{"col":"ts"}]},"mo":{"fn":"month","args":[{"col":"ts"}]},"d":{"fn":"day","args":[{"col":"ts"}]},"h":{"fn":"hour","args":[{"col":"ts"}]},"mi":{"fn":"minute","args":[{"col":"ts"}]},"s":{"fn":"second","args":[{"col":"ts"}]},"ms":{"fn":"millisecond","args":[{"col":"ts"}]},"wd":{"fn":"weekday","args":[{"col":"ts"}]},"od":{"fn":"ordinalDay","args":[{"col":"ts"}]},"dy":{"fn":"year","args":[{"col":"d"}]},"dwd":{"fn":"weekday","args":[{"col":"d"}]},"dod":{"fn":"ordinalDay","args":[{"col":"d"}]}}}],"expected":{"fields":[{"name":"y","type":"i32"},{"name":"mo","type":"i8"},{"name":"d","type":"i8"},{"name":"h","type":"i8"},{"name":"mi","type":"i8"},{"name":"s","type":"i8"},{"name":"ms","type":"i32"},{"name":"wd","type":"i8"},{"name":"od","type":"i16"},{"name":"dy","type":"i32"},{"name":"dwd","type":"i8"},{"name":"dod","type":"i16"}],"columns":{"y":[2024,2023,2024,null,1969,2024],"mo":[2,12,3,null,7,3],"d":[29,31,4,null,20,10],"h":[13,23,0,null,20,6],"mi":[45,59,0,null,17,30],"s":[7,59,0,null,40,0],"ms":[123,999,0,null,0,0],"wd":[4,7,1,null,7,7],"od":[60,365,64,null,201,70],"dy":[2024,2023,2024,2024,null,2024],"dwd":[4,7,1,4,null,7],"dod":[60,365,64,60,null,70]}}},{"id":"temporal/arith","description":"timestamp differences are durations; durations shift timestamps","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"ts","type":"datetime[us]"},{"name":"ms","type":"datetime[ms]"},{"name":"d","type":"date"},{"name":"dur","type":"duration[us]"}],"columns":{"k":["a","b","c","d","e","f"],"ts":[1709214307123456,1704067199999000,1709510400000000,null,-14182940000000,1710052200000500],"ms":[1709214307123,1704067200000,null,1709251200000,0,1709942400000],"d":[19782,19722,19786,19782,null,19792],"dur":[5400000000,-1500000,null,259200000001,0,90000000000]}},"right":{"fields":[{"name":"d","type":"date"},{"name":"name","type":"utf8"}],"columns":{"d":[19782,19792,20089],"name":["leap","sunday","new year"]}}},"ops":[{"op":"select","exprs":{"gap":{"fn":"sub","args":[{"col":"ts"},{"col":"ms"}]},"later":{"fn":"add","args":[{"col":"ts"},{"col":"dur"}]},"earlier":{"fn":"sub","args":[{"col":"ts"},{"col":"dur"}]},"days":{"fn":"sub","args":[{"col":"d"},{"lit":18000,"type":"date"}]},"twice":{"fn":"add","args":[{"col":"dur"},{"col":"dur"}]}}}],"expected":{"fields":[{"name":"gap","type":"duration[ms]"},{"name":"later","type":"datetime[us]"},{"name":"earlier","type":"datetime[us]"},{"name":"days","type":"duration[us]"},{"name":"twice","type":"duration[us]"}],"columns":{"gap":[0,-1,null,null,-14182940000,109800000],"later":[1709219707123456,1704067198499000,null,null,-14182940000000,1710142200000500],"earlier":[1709208907123456,1704067201499000,null,null,-14182940000000,1709962200000500],"days":[153964800000000,148780800000000,154310400000000,153964800000000,null,154828800000000],"twice":[10800000000,-3000000,null,518400000002,0,180000000000]}}},{"id":"temporal/truncate","description":"truncate to a calendar unit","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"ts","type":"datetime[us]"},{"name":"ms","type":"datetime[ms]"},{"name":"d","type":"date"},{"name":"dur","type":"duration[us]"}],"columns":{"k":["a","b","c","d","e","f"],"ts":[1709214307123456,1704067199999000,1709510400000000,null,-14182940000000,1710052200000500],"ms":[1709214307123,1704067200000,null,1709251200000,0,1709942400000],"d":[19782,19722,19786,19782,null,19792],"dur":[5400000000,-1500000,null,259200000001,0,90000000000]}},"right":{"fields":[{"name":"d","type":"date"},{"name":"name","type":"utf8"}],"columns":{"d":[19782,19792,20089],"name":["leap","sunday","new year"]}}},"ops":[{"op":"select","exprs":{"h":{"fn":"truncate","args":[{"col":"ts"}],"every":"1h"},"d":{"fn":"truncate","args":[{"col":"ts"}],"every":"1d"},"w":{"fn":"truncate","args":[{"col":"ts"}],"every":"1w"},"mo":{"fn":"truncate","args":[{"col":"ts"}],"every":"1mo"},"y":{"fn":"truncate","args":[{"col":"ts"}],"every":"1y"},"dmo":{"fn":"truncate","args":[{"col":"d"}],"every":"1mo"}}}],"expected":{"fields":[{"name":"h","type":"datetime[us]"},{"name":"d","type":"datetime[us]"},{"name":"w","type":"datetime[us]"},{"name":"mo","type":"datetime[us]"},{"name":"y","type":"datetime[us]"},{"name":"dmo","type":"date"}],"columns":{"h":[1709211600000000,1704063600000000,1709510400000000,null,-14184000000000,1710050400000000],"d":[1709164800000000,1703980800000000,1709510400000000,null,-14256000000000,1710028800000000],"w":[1708905600000000,1703462400000000,1709510400000000,null,-14774400000000,1709510400000000],"mo":[1706745600000000,1701388800000000,1709251200000000,null,-15897600000000,1709251200000000],"y":[1704067200000000,1672531200000000,1704067200000000,null,-31536000000000,1704067200000000],"dmo":[19754,19692,19783,19754,null,19783]}}},{"id":"temporal/duration","description":"durations in whole units","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"ts","type":"datetime[us]"},{"name":"ms","type":"datetime[ms]"},{"name":"d","type":"date"},{"name":"dur","type":"duration[us]"}],"columns":{"k":["a","b","c","d","e","f"],"ts":[1709214307123456,1704067199999000,1709510400000000,null,-14182940000000,1710052200000500],"ms":[1709214307123,1704067200000,null,1709251200000,0,1709942400000],"d":[19782,19722,19786,19782,null,19792],"dur":[5400000000,-1500000,null,259200000001,0,90000000000]}},"right":{"fields":[{"name":"d","type":"date"},{"name":"name","type":"utf8"}],"columns":{"d":[19782,19792,20089],"name":["leap","sunday","new year"]}}},"ops":[{"op":"select","exprs":{"d":{"fn":"totalDays","args":[{"col":"dur"}]},"h":{"fn":"totalHours","args":[{"col":"dur"}]},"m":{"fn":"totalMinutes","args":[{"col":"dur"}]},"s":{"fn":"totalSeconds","args":[{"col":"dur"}]},"ms":{"fn":"totalMilliseconds","args":[{"col":"dur"}]}}}],"expected":{"fields":[{"name":"d","type":"i64"},{"name":"h","type":"i64"},{"name":"m","type":"i64"},{"name":"s","type":"i64"},{"name":"ms","type":"i64"}],"columns":{"d":["0","0",null,"3","0","1"],"h":["1","0",null,"72","0","25"],"m":["90","0",null,"4320","0","1500"],"s":["5400","-1",null,"259200","0","90000"],"ms":["5400000","-1500",null,"259200000","0","90000000"]}}},{"id":"temporal/cast","description":"casts between dates, timestamps and integers","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"ts","type":"datetime[us]"},{"name":"ms","type":"datetime[ms]"},{"name":"d","type":"date"},{"name":"dur","type":"duration[us]"}],"columns":{"k":["a","b","c","d","e","f"],"ts":[1709214307123456,1704067199999000,1709510400000000,null,-14182940000000,1710052200000500],"ms":[1709214307123,1704067200000,null,1709251200000,0,1709942400000],"d":[19782,19722,19786,19782,null,19792],"dur":[5400000000,-1500000,null,259200000001,0,90000000000]}},"right":{"fields":[{"name":"d","type":"date"},{"name":"name","type":"utf8"}],"columns":{"d":[19782,19792,20089],"name":["leap","sunday","new year"]}}},"ops":[{"op":"select","exprs":{"date":{"fn":"cast","args":[{"col":"ts"}],"to":"date"},"ts":{"fn":"cast","args":[{"col":"d"}],"to":"datetime[us]"},"us":{"fn":"cast","args":[{"col":"ts"}],"to":"i64"},"days":{"fn":"cast","args":[{"col":"d"}],"to":"i32"},"ms":{"fn":"cast","args":[{"col":"ts"}],"to":"datetime[ms]"}}}],"expected":{"fields":[{"name":"date","type":"date"},{"name":"ts","type":"datetime[us]"},{"name":"us","type":"i64"},{"name":"days","type":"i32"},{"name":"ms","type":"datetime[ms]"}],"columns":{"date":[19782,19722,19786,null,-165,19792],"ts":[1709164800000000,1703980800000000,1709510400000000,1709164800000000,null,1710028800000000],"us":["1709214307123456","1704067199999000","1709510400000000",null,"-14182940000000","1710052200000500"],"days":[19782,19722,19786,19782,null,19792],"ms":[1709214307123,1704067199999,1709510400000,null,-14182940000,1710052200000]}}},{"id":"temporal/compare","description":"comparisons with timestamp literals and between units","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"ts","type":"datetime[us]"},{"name":"ms","type":"datetime[ms]"},{"name":"d","type":"date"},{"name":"dur","type":"duration[us]"}],"columns":{"k":["a","b","c","d","e","f"],"ts":[1709214307123456,1704067199999000,1709510400000000,null,-14182940000000,1710052200000500],"ms":[1709214307123,1704067200000,null,1709251200000,0,1709942400000],"d":[19782,19722,19786,19782,null,19792],"dur":[5400000000,-1500000,null,259200000001,0,90000000000]}},"right":{"fields":[{"name":"d","type":"date"},{"name":"name","type":"utf8"}],"columns":{"d":[19782,19792,20089],"name":["leap","sunday","new year"]}}},"ops":[{"op":"filter","predicate":{"fn":"and","args":[{"fn":"gte","args":[{"col":"ts"},{"lit":1709251200000000,"type":"datetime[us]"}]},{"fn":"lt","args":[{"col":"ms"},{"col":"ts"}]}]}}],"expected":{"fields":[{"name":"k","type":"utf8"},{"name":"ts","type":"datetime[us]"},{"name":"ms","type":"datetime[ms]"},{"name":"d","type":"date"},{"name":"dur","type":"duration[us]"}],"columns":{"k":["f"],"ts":[1710052200000500],"ms":[1709942400000],"d":[19792],"dur":[90000000000]}}},{"id":"temporal/group-sort","description":"group by date, aggregate timestamps, sort","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"ts","type":"datetime[us]"},{"name":"ms","type":"datetime[ms]"},{"name":"d","type":"date"},{"name":"dur","type":"duration[us]"}],"columns":{"k":["a","b","c","d","e","f"],"ts":[1709214307123456,1704067199999000,1709510400000000,null,-14182940000000,1710052200000500],"ms":[1709214307123,1704067200000,null,1709251200000,0,1709942400000],"d":[19782,19722,19786,19782,null,19792],"dur":[5400000000,-1500000,null,259200000001,0,90000000000]}},"right":{"fields":[{"name":"d","type":"date"},{"name":"name","type":"utf8"}],"columns":{"d":[19782,19792,20089],"name":["leap","sunday","new year"]}}},"ops":[{"op":"groupBy","keys":["d"],"aggs":{"first":{"agg":"min","arg":{"col":"ts"}},"last":{"agg":"max","arg":{"col":"ts"}},"n":{"agg":"len"},"total":{"agg":"sum","arg":{"col":"dur"}}}},{"op":"sort","keys":[{"expr":{"col":"last"},"descending":true}]}],"expected":{"fields":[{"name":"d","type":"date"},{"name":"first","type":"datetime[us]"},{"name":"last","type":"datetime[us]"},{"name":"n","type":"u32"},{"name":"total","type":"duration[us]"}],"columns":{"d":[19792,19786,19782,19722,null],"first":[1710052200000500,1709510400000000,1709214307123456,1704067199999000,-14182940000000],"last":[1710052200000500,1709510400000000,1709214307123456,1704067199999000,-14182940000000],"n":[1,1,2,1,1],"total":[90000000000,0,264600000001,-1500000,0]}}},{"id":"temporal/join","description":"join on a date","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"ts","type":"datetime[us]"},{"name":"ms","type":"datetime[ms]"},{"name":"d","type":"date"},{"name":"dur","type":"duration[us]"}],"columns":{"k":["a","b","c","d","e","f"],"ts":[1709214307123456,1704067199999000,1709510400000000,null,-14182940000000,1710052200000500],"ms":[1709214307123,1704067200000,null,1709251200000,0,1709942400000],"d":[19782,19722,19786,19782,null,19792],"dur":[5400000000,-1500000,null,259200000001,0,90000000000]}},"right":{"fields":[{"name":"d","type":"date"},{"name":"name","type":"utf8"}],"columns":{"d":[19782,19792,20089],"name":["leap","sunday","new year"]}}},"ops":[{"op":"join","right":"right","on":["d"],"how":"left"}],"expected":{"fields":[{"name":"k","type":"utf8"},{"name":"ts","type":"datetime[us]"},{"name":"ms","type":"datetime[ms]"},{"name":"d","type":"date"},{"name":"dur","type":"duration[us]"},{"name":"name","type":"utf8"}],"columns":{"k":["a","b","c","d","e","f"],"ts":[1709214307123456,1704067199999000,1709510400000000,null,-14182940000000,1710052200000500],"ms":[1709214307123,1704067200000,null,1709251200000,0,1709942400000],"d":[19782,19722,19786,19782,null,19792],"dur":[5400000000,-1500000,null,259200000001,0,90000000000],"name":["leap",null,null,"leap",null,"sunday"]}}},{"id":"nested/list-ops","description":"list length, element access and aggregates","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"g","type":"i32"},{"name":"xs","type":"list"},{"name":"ws","type":"list"},{"name":"s","type":"struct"}],"columns":{"k":["a","b","c","d","e"],"g":[1,2,1,2,1],"xs":[[1,2,3],[],null,[4,null,2],[7]],"ws":[["a","b"],["c"],[],null,["z","a"]],"s":[{"a":1,"b":"x"},{"a":null,"b":"y"},null,{"a":4,"b":null},{"a":5,"b":"z"}]}},"right":{"fields":[{"name":"k","type":"utf8"},{"name":"tags","type":"list"}],"columns":{"k":["a","d","x"],"tags":[["t1"],["t2","t3"],[]]}}},"ops":[{"op":"select","exprs":{"n":{"fn":"listLength","args":[{"col":"xs"}]},"first":{"fn":"listGet","args":[{"col":"xs"}],"index":0},"last":{"fn":"listGet","args":[{"col":"xs"}],"index":-1},"oob":{"fn":"listGet","args":[{"col":"xs"}],"index":5},"sum":{"fn":"listSum","args":[{"col":"xs"}]},"mean":{"fn":"listMean","args":[{"col":"xs"}]},"min":{"fn":"listMin","args":[{"col":"xs"}]},"max":{"fn":"listMax","args":[{"col":"ws"}]},"has2":{"fn":"listContains","args":[{"col":"xs"},{"lit":2}]},"hasA":{"fn":"listContains","args":[{"col":"ws"},{"lit":"a"}]}}}],"expected":{"fields":[{"name":"n","type":"u32"},{"name":"first","type":"i32"},{"name":"last","type":"i32"},{"name":"oob","type":"i32"},{"name":"sum","type":"i32"},{"name":"mean","type":"f64"},{"name":"min","type":"i32"},{"name":"max","type":"utf8"},{"name":"has2","type":"bool"},{"name":"hasA","type":"bool"}],"columns":{"n":[3,0,null,3,1],"first":[1,null,null,4,7],"last":[3,null,null,2,7],"oob":[null,null,null,null,null],"sum":[6,0,null,6,7],"mean":[2.0,null,null,3.0,7.0],"min":[1,null,null,2,7],"max":["b","c",null,null,"z"],"has2":[true,false,null,true,false],"hasA":[true,false,false,null,true]}}},{"id":"nested/struct","description":"struct fields and struct construction","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"g","type":"i32"},{"name":"xs","type":"list"},{"name":"ws","type":"list"},{"name":"s","type":"struct"}],"columns":{"k":["a","b","c","d","e"],"g":[1,2,1,2,1],"xs":[[1,2,3],[],null,[4,null,2],[7]],"ws":[["a","b"],["c"],[],null,["z","a"]],"s":[{"a":1,"b":"x"},{"a":null,"b":"y"},null,{"a":4,"b":null},{"a":5,"b":"z"}]}},"right":{"fields":[{"name":"k","type":"utf8"},{"name":"tags","type":"list"}],"columns":{"k":["a","d","x"],"tags":[["t1"],["t2","t3"],[]]}}},"ops":[{"op":"select","exprs":{"a":{"fn":"field","args":[{"col":"s"}],"field":"a"},"b":{"fn":"field","args":[{"col":"s"}],"field":"b"},"pair":{"fn":"struct","args":[{"col":"k"},{"fn":"listLength","args":[{"col":"xs"}]}],"names":["key","size"]}}}],"expected":{"fields":[{"name":"a","type":"i32"},{"name":"b","type":"utf8"},{"name":"pair","type":"struct"}],"columns":{"a":[1,null,null,4,5],"b":["x","y",null,null,"z"],"pair":[{"key":"a","size":3},{"key":"b","size":0},{"key":"c","size":null},{"key":"d","size":3},{"key":"e","size":1}]}}},{"id":"nested/explode","description":"explode lists into rows; a null list gives a null row, an empty list no rows","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"g","type":"i32"},{"name":"xs","type":"list"},{"name":"ws","type":"list"},{"name":"s","type":"struct"}],"columns":{"k":["a","b","c","d","e"],"g":[1,2,1,2,1],"xs":[[1,2,3],[],null,[4,null,2],[7]],"ws":[["a","b"],["c"],[],null,["z","a"]],"s":[{"a":1,"b":"x"},{"a":null,"b":"y"},null,{"a":4,"b":null},{"a":5,"b":"z"}]}},"right":{"fields":[{"name":"k","type":"utf8"},{"name":"tags","type":"list"}],"columns":{"k":["a","d","x"],"tags":[["t1"],["t2","t3"],[]]}}},"ops":[{"op":"pick","columns":["k","xs"]},{"op":"explode","column":"xs"}],"expected":{"fields":[{"name":"k","type":"utf8"},{"name":"xs","type":"i32"}],"columns":{"k":["a","a","a","c","d","d","d","e"],"xs":[1,2,3,null,4,null,2,7]}}},{"id":"nested/unnest","description":"unnest a struct into columns","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"g","type":"i32"},{"name":"xs","type":"list"},{"name":"ws","type":"list"},{"name":"s","type":"struct"}],"columns":{"k":["a","b","c","d","e"],"g":[1,2,1,2,1],"xs":[[1,2,3],[],null,[4,null,2],[7]],"ws":[["a","b"],["c"],[],null,["z","a"]],"s":[{"a":1,"b":"x"},{"a":null,"b":"y"},null,{"a":4,"b":null},{"a":5,"b":"z"}]}},"right":{"fields":[{"name":"k","type":"utf8"},{"name":"tags","type":"list"}],"columns":{"k":["a","d","x"],"tags":[["t1"],["t2","t3"],[]]}}},"ops":[{"op":"pick","columns":["k","s"]},{"op":"unnest","column":"s"}],"expected":{"fields":[{"name":"k","type":"utf8"},{"name":"a","type":"i32"},{"name":"b","type":"utf8"}],"columns":{"k":["a","b","c","d","e"],"a":[1,null,null,4,5],"b":["x","y",null,null,"z"]}}},{"id":"nested/agg-list","description":"collect values into lists per group","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"g","type":"i32"},{"name":"xs","type":"list"},{"name":"ws","type":"list"},{"name":"s","type":"struct"}],"columns":{"k":["a","b","c","d","e"],"g":[1,2,1,2,1],"xs":[[1,2,3],[],null,[4,null,2],[7]],"ws":[["a","b"],["c"],[],null,["z","a"]],"s":[{"a":1,"b":"x"},{"a":null,"b":"y"},null,{"a":4,"b":null},{"a":5,"b":"z"}]}},"right":{"fields":[{"name":"k","type":"utf8"},{"name":"tags","type":"list"}],"columns":{"k":["a","d","x"],"tags":[["t1"],["t2","t3"],[]]}}},"ops":[{"op":"groupBy","keys":["g"],"aggs":{"ks":{"agg":"list","arg":{"col":"k"}},"firsts":{"agg":"first","arg":{"col":"xs"}},"n":{"agg":"len"}}}],"expected":{"fields":[{"name":"g","type":"i32"},{"name":"ks","type":"list"},{"name":"firsts","type":"list"},{"name":"n","type":"u32"}],"columns":{"g":[1,2],"ks":[["a","c","e"],["b","d"]],"firsts":[[1,2,3],[]],"n":[3,2]}}},{"id":"nested/carry","description":"lists and structs survive filters, sorts, joins and slices","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"g","type":"i32"},{"name":"xs","type":"list"},{"name":"ws","type":"list"},{"name":"s","type":"struct"}],"columns":{"k":["a","b","c","d","e"],"g":[1,2,1,2,1],"xs":[[1,2,3],[],null,[4,null,2],[7]],"ws":[["a","b"],["c"],[],null,["z","a"]],"s":[{"a":1,"b":"x"},{"a":null,"b":"y"},null,{"a":4,"b":null},{"a":5,"b":"z"}]}},"right":{"fields":[{"name":"k","type":"utf8"},{"name":"tags","type":"list"}],"columns":{"k":["a","d","x"],"tags":[["t1"],["t2","t3"],[]]}}},"ops":[{"op":"filter","predicate":{"fn":"neq","args":[{"col":"k"},{"lit":"c"}]}},{"op":"sort","keys":[{"expr":{"col":"g"},"descending":true},{"expr":{"col":"k"}}]},{"op":"join","right":"right","on":["k"],"how":"left"},{"op":"head","n":4}],"expected":{"fields":[{"name":"k","type":"utf8"},{"name":"g","type":"i32"},{"name":"xs","type":"list"},{"name":"ws","type":"list"},{"name":"s","type":"struct"},{"name":"tags","type":"list"}],"columns":{"k":["b","d","a","e"],"g":[2,2,1,1],"xs":[[],[4,null,2],[1,2,3],[7]],"ws":[["c"],null,["a","b"],["z","a"]],"s":[{"a":null,"b":"y"},{"a":4,"b":null},{"a":1,"b":"x"},{"a":5,"b":"z"}],"tags":[null,["t2","t3"],["t1"],null]}}},{"id":"decimal/arith","description":"decimal addition, subtraction and multiplication","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"price","type":"decimal[10,2]"},{"name":"fee","type":"decimal[6,3]"},{"name":"qty","type":"decimal[4,0]"},{"name":"x","type":"f64"}],"columns":{"k":["a","b","a","c","b","a"],"price":["12.34","0.10","-5.00",null,"99999999.99","10.01"],"fee":["0.125","1.000",null,"2.500","-0.001","10.010"],"qty":["3","10","2","1",null,"7"],"x":[1.005,-2.5,123.456,null,1000000.0,0.0]}}},"ops":[{"op":"select","exprs":{"sum":{"fn":"add","args":[{"col":"price"},{"col":"fee"}]},"diff":{"fn":"sub","args":[{"col":"price"},{"col":"fee"}]},"prod":{"fn":"mul","args":[{"col":"price"},{"col":"qty"}]},"plus":{"fn":"add","args":[{"col":"price"},{"lit":"0.5","type":"decimal[3,1]"}]}}}],"expected":{"fields":[{"name":"sum","type":"decimal[38,3]"},{"name":"diff","type":"decimal[38,3]"},{"name":"prod","type":"decimal[38,2]"},{"name":"plus","type":"decimal[38,2]"}],"columns":{"sum":["12.465","1.100",null,null,"99999999.989","20.020"],"diff":["12.215","-0.900",null,null,"99999999.991","0.000"],"prod":["37.02","1.00","-10.00",null,null,"70.07"],"plus":["12.84","0.60","-4.50",null,"100000000.49","10.51"]}}},{"id":"decimal/compare","description":"decimal comparisons, also across scales","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"price","type":"decimal[10,2]"},{"name":"fee","type":"decimal[6,3]"},{"name":"qty","type":"decimal[4,0]"},{"name":"x","type":"f64"}],"columns":{"k":["a","b","a","c","b","a"],"price":["12.34","0.10","-5.00",null,"99999999.99","10.01"],"fee":["0.125","1.000",null,"2.500","-0.001","10.010"],"qty":["3","10","2","1",null,"7"],"x":[1.005,-2.5,123.456,null,1000000.0,0.0]}}},"ops":[{"op":"filter","predicate":{"fn":"and","args":[{"fn":"gt","args":[{"col":"price"},{"lit":"10.00","type":"decimal[10,2]"}]},{"fn":"gte","args":[{"col":"price"},{"col":"fee"}]}]}}],"expected":{"fields":[{"name":"k","type":"utf8"},{"name":"price","type":"decimal[10,2]"},{"name":"fee","type":"decimal[6,3]"},{"name":"qty","type":"decimal[4,0]"},{"name":"x","type":"f64"}],"columns":{"k":["a","b","a"],"price":["12.34","99999999.99","10.01"],"fee":["0.125","-0.001","10.010"],"qty":["3",null,"7"],"x":[1.005,1000000.0,0.0]}}},{"id":"decimal/aggregate","description":"sums, extremes and means of decimals per group","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"price","type":"decimal[10,2]"},{"name":"fee","type":"decimal[6,3]"},{"name":"qty","type":"decimal[4,0]"},{"name":"x","type":"f64"}],"columns":{"k":["a","b","a","c","b","a"],"price":["12.34","0.10","-5.00",null,"99999999.99","10.01"],"fee":["0.125","1.000",null,"2.500","-0.001","10.010"],"qty":["3","10","2","1",null,"7"],"x":[1.005,-2.5,123.456,null,1000000.0,0.0]}}},"ops":[{"op":"groupBy","keys":["k"],"aggs":{"total":{"agg":"sum","arg":{"col":"price"}},"min":{"agg":"min","arg":{"col":"price"}},"max":{"agg":"max","arg":{"col":"fee"}},"mean":{"agg":"mean","arg":{"col":"price"}},"n":{"agg":"count","arg":{"col":"price"}}}}],"expected":{"fields":[{"name":"k","type":"utf8"},{"name":"total","type":"decimal[38,2]"},{"name":"min","type":"decimal[10,2]"},{"name":"max","type":"decimal[6,3]"},{"name":"mean","type":"f64"},{"name":"n","type":"u32"}],"columns":{"k":["a","b","c"],"total":["17.35","100000000.09","0.00"],"min":["-5.00","0.10",null],"max":["10.010","1.000","2.500"],"mean":[5.783333333333334,50000000.044999994,null],"n":[3,2,0]}}},{"id":"decimal/cast","description":"casts between decimals, floats, integers and strings","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"price","type":"decimal[10,2]"},{"name":"fee","type":"decimal[6,3]"},{"name":"qty","type":"decimal[4,0]"},{"name":"x","type":"f64"}],"columns":{"k":["a","b","a","c","b","a"],"price":["12.34","0.10","-5.00",null,"99999999.99","10.01"],"fee":["0.125","1.000",null,"2.500","-0.001","10.010"],"qty":["3","10","2","1",null,"7"],"x":[1.005,-2.5,123.456,null,1000000.0,0.0]}}},"ops":[{"op":"select","exprs":{"f":{"fn":"cast","args":[{"col":"price"}],"to":"f64"},"i":{"fn":"cast","args":[{"col":"price"}],"to":"i32"},"s":{"fn":"cast","args":[{"col":"fee"}],"to":"utf8"},"d":{"fn":"cast","args":[{"col":"x"}],"to":"decimal[10,2]"},"r":{"fn":"cast","args":[{"col":"price"}],"to":"decimal[12,1]"}}}],"expected":{"fields":[{"name":"f","type":"f64"},{"name":"i","type":"i32"},{"name":"s","type":"utf8"},{"name":"d","type":"decimal[10,2]"},{"name":"r","type":"decimal[12,1]"}],"columns":{"f":[12.34,0.1,-5.0,null,99999999.99,10.01],"i":[12,0,-5,null,99999999,10],"s":["0.125","1.000",null,"2.500","-0.001","10.010"],"d":["1.00","-2.50","123.46",null,"1000000.00","0.00"],"r":["12.3","0.1","-5.0",null,"100000000.0","10.0"]}}},{"id":"decimal/sort","description":"sort by a decimal","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"price","type":"decimal[10,2]"},{"name":"fee","type":"decimal[6,3]"},{"name":"qty","type":"decimal[4,0]"},{"name":"x","type":"f64"}],"columns":{"k":["a","b","a","c","b","a"],"price":["12.34","0.10","-5.00",null,"99999999.99","10.01"],"fee":["0.125","1.000",null,"2.500","-0.001","10.010"],"qty":["3","10","2","1",null,"7"],"x":[1.005,-2.5,123.456,null,1000000.0,0.0]}}},"ops":[{"op":"sort","keys":[{"expr":{"col":"price"},"descending":true}],"nullsLast":true}],"expected":{"fields":[{"name":"k","type":"utf8"},{"name":"price","type":"decimal[10,2]"},{"name":"fee","type":"decimal[6,3]"},{"name":"qty","type":"decimal[4,0]"},{"name":"x","type":"f64"}],"columns":{"k":["b","a","a","b","a","c"],"price":["99999999.99","12.34","10.01","0.10","-5.00",null],"fee":["-0.001","0.125","10.010","1.000",null,"2.500"],"qty":[null,"3","7","10","2","1"],"x":[1000000.0,1.005,0.0,-2.5,123.456,null]}}},{"id":"decimal/round-ties","description":"narrowing the scale rounds exact ties","inputs":{"left":{"fields":[{"name":"t","type":"decimal[6,3]"}],"columns":{"t":["0.250","0.350","-0.250","-0.350","1.050","2.500","0.249"]}}},"ops":[{"op":"select","exprs":{"one":{"fn":"cast","args":[{"col":"t"}],"to":"decimal[6,1]"},"zero":{"fn":"cast","args":[{"col":"t"}],"to":"decimal[6,0]"}}}],"expected":{"fields":[{"name":"one","type":"decimal[6,1]"},{"name":"zero","type":"decimal[6,0]"}],"columns":{"one":["0.2","0.4","-0.2","-0.4","1.0","2.5","0.2"],"zero":["0","0","0","0","1","2","0"]}}},{"id":"decimal/cast-float-ties","description":"floats cast to decimals round exact ties half to even, also below zero","inputs":{"left":{"fields":[{"name":"x","type":"f64"}],"columns":{"x":[-1.5,-2.5,1.5,2.5,-0.75]}}},"ops":[{"op":"select","exprs":{"zero":{"fn":"cast","args":[{"col":"x"}],"to":"decimal[18,0]"},"one":{"fn":"cast","args":[{"col":"x"}],"to":"decimal[18,1]"}}}],"expected":{"fields":[{"name":"zero","type":"decimal[18,0]"},{"name":"one","type":"decimal[18,1]"}],"columns":{"zero":["-2","-2","2","2","-1"],"one":["-1.5","-2.5","1.5","2.5","-0.8"]}}},{"id":"decimal/round","description":"rounding decimals keeps the scale and rounds half to even","inputs":{"left":{"fields":[{"name":"t","type":"decimal[6,3]"}],"columns":{"t":["0.250","0.350","-0.250","-0.350","1.050","2.500","0.249",null]}}},"ops":[{"op":"select","exprs":{"r0":{"fn":"round","args":[{"col":"t"}]},"r1":{"fn":"round","args":[{"col":"t"}],"decimals":1},"r2":{"fn":"round","args":[{"col":"t"}],"decimals":2},"r3":{"fn":"round","args":[{"col":"t"}],"decimals":3}}}],"expected":{"fields":[{"name":"r0","type":"decimal[7,3]"},{"name":"r1","type":"decimal[7,3]"},{"name":"r2","type":"decimal[7,3]"},{"name":"r3","type":"decimal[6,3]"}],"columns":{"r0":["0.000","0.000","0.000","0.000","1.000","2.000","0.000",null],"r1":["0.200","0.400","-0.200","-0.400","1.000","2.500","0.200",null],"r2":["0.250","0.350","-0.250","-0.350","1.050","2.500","0.250",null],"r3":["0.250","0.350","-0.250","-0.350","1.050","2.500","0.249",null]}}},{"id":"arith/u32-divide-by-zero","description":"unsigned 32-bit floor division and modulo by zero give null","inputs":{"left":{"fields":[{"name":"n","type":"u32"},{"name":"d","type":"u32"}],"columns":{"n":[7,0,5],"d":[0,0,2]}}},"ops":[{"op":"select","exprs":{"q":{"fn":"floorDiv","args":[{"col":"n"},{"col":"d"}]},"m":{"fn":"mod","args":[{"col":"n"},{"col":"d"}]},"qs":{"fn":"floorDiv","args":[{"col":"n"},{"lit":0,"type":"u32"}]},"ms":{"fn":"mod","args":[{"col":"n"},{"lit":0,"type":"u32"}]}}}],"expected":{"fields":[{"name":"q","type":"u32"},{"name":"m","type":"u32"},{"name":"qs","type":"u32"},{"name":"ms","type":"u32"}],"columns":{"q":[null,null,2],"m":[null,null,1],"qs":[null,null,null],"ms":[null,null,null]}}},{"id":"window/filter-after-sort","description":"a window predicate sees the sorted rows","inputs":{"left":{"fields":[{"name":"g","type":"i32"},{"name":"h","type":"i32"},{"name":"x","type":"f64"}],"columns":{"g":[1,2,1,2],"h":[1,1,2,2],"x":[3.0,1.0,2.0,4.0]}},"right":{"fields":[{"name":"g","type":"i32"},{"name":"name","type":"utf8"}],"columns":{"g":[1],"name":["one"]}}},"ops":[{"op":"sort","keys":[{"expr":{"col":"x"}}]},{"op":"filter","predicate":{"fn":"gt","args":[{"fn":"cumSum","args":[{"col":"x"}]},{"lit":3.0}]}}],"expected":{"fields":[{"name":"g","type":"i32"},{"name":"h","type":"i32"},{"name":"x","type":"f64"}],"columns":{"g":[1,2],"h":[1,2],"x":[3.0,4.0]}}},{"id":"window/filter-after-filter","description":"a window predicate sees only the rows an earlier filter kept","inputs":{"left":{"fields":[{"name":"g","type":"i32"},{"name":"h","type":"i32"},{"name":"x","type":"f64"}],"columns":{"g":[1,2,1,2],"h":[1,1,2,2],"x":[3.0,1.0,2.0,4.0]}},"right":{"fields":[{"name":"g","type":"i32"},{"name":"name","type":"utf8"}],"columns":{"g":[1],"name":["one"]}}},"ops":[{"op":"filter","predicate":{"fn":"gt","args":[{"col":"x"},{"lit":1.0}]}},{"op":"filter","predicate":{"fn":"gt","args":[{"fn":"shift","args":[{"col":"x"}]},{"lit":2.0}]}}],"expected":{"fields":[{"name":"g","type":"i32"},{"name":"h","type":"i32"},{"name":"x","type":"f64"}],"columns":{"g":[1],"h":[2],"x":[2.0]}}},{"id":"window/filter-after-concat","description":"a window predicate sees every input of a concat","inputs":{"left":{"fields":[{"name":"g","type":"i32"},{"name":"h","type":"i32"},{"name":"x","type":"f64"}],"columns":{"g":[1,2,1,2],"h":[1,1,2,2],"x":[3.0,1.0,2.0,4.0]}},"right":{"fields":[{"name":"g","type":"i32"},{"name":"name","type":"utf8"}],"columns":{"g":[1],"name":["one"]}}},"ops":[{"op":"concat","with":"left"},{"op":"filter","predicate":{"fn":"gt","args":[{"fn":"cumSum","args":[{"col":"x"}]},{"lit":15.0}]}}],"expected":{"fields":[{"name":"g","type":"i32"},{"name":"h","type":"i32"},{"name":"x","type":"f64"}],"columns":{"g":[1,2],"h":[2,2],"x":[2.0,4.0]}}},{"id":"window/filter-after-join","description":"a window predicate sees only the joined rows","inputs":{"left":{"fields":[{"name":"g","type":"i32"},{"name":"h","type":"i32"},{"name":"x","type":"f64"}],"columns":{"g":[1,2,1,2],"h":[1,1,2,2],"x":[3.0,1.0,2.0,4.0]}},"right":{"fields":[{"name":"g","type":"i32"},{"name":"name","type":"utf8"}],"columns":{"g":[1],"name":["one"]}}},"ops":[{"op":"join","right":"right","on":["g"],"how":"inner"},{"op":"filter","predicate":{"fn":"gt","args":[{"fn":"shift","args":[{"col":"x"}]},{"lit":2.0}]}}],"expected":{"fields":[{"name":"g","type":"i32"},{"name":"h","type":"i32"},{"name":"x","type":"f64"},{"name":"name","type":"utf8"}],"columns":{"g":[1],"h":[2],"x":[2.0],"name":["one"]}}},{"id":"window/filter-whole","description":"a window predicate sees the whole input","inputs":{"left":{"fields":[{"name":"g","type":"i32"},{"name":"h","type":"i32"},{"name":"x","type":"f64"}],"columns":{"g":[1,2,1,2],"h":[1,1,2,2],"x":[3.0,1.0,2.0,4.0]}},"right":{"fields":[{"name":"g","type":"i32"},{"name":"name","type":"utf8"}],"columns":{"g":[1],"name":["one"]}}},"ops":[{"op":"filter","predicate":{"fn":"gt","args":[{"fn":"cumSum","args":[{"col":"x"}]},{"lit":5.0}]}}],"expected":{"fields":[{"name":"g","type":"i32"},{"name":"h","type":"i32"},{"name":"x","type":"f64"}],"columns":{"g":[1,2],"h":[2,2],"x":[2.0,4.0]}}},{"id":"window/agg-of-window","description":"an aggregate of a window sees the whole input","inputs":{"left":{"fields":[{"name":"g","type":"i32"},{"name":"h","type":"i32"},{"name":"x","type":"f64"}],"columns":{"g":[1,2,1,2],"h":[1,1,2,2],"x":[3.0,1.0,2.0,4.0]}},"right":{"fields":[{"name":"g","type":"i32"},{"name":"name","type":"utf8"}],"columns":{"g":[1],"name":["one"]}}},"ops":[{"op":"agg","aggs":{"s":{"agg":"sum","arg":{"fn":"cumSum","args":[{"col":"x"}]}},"n":{"agg":"len"}}}],"expected":{"fields":[{"name":"s","type":"f64"},{"name":"n","type":"u32"}],"columns":{"s":[23.0],"n":[4]}}},{"id":"window/grouped-agg-of-window","description":"windows inside grouped aggregates run per group","inputs":{"left":{"fields":[{"name":"g","type":"i32"},{"name":"h","type":"i32"},{"name":"x","type":"f64"}],"columns":{"g":[1,2,1,2],"h":[1,1,2,2],"x":[3.0,1.0,2.0,4.0]}},"right":{"fields":[{"name":"g","type":"i32"},{"name":"name","type":"utf8"}],"columns":{"g":[1],"name":["one"]}}},"ops":[{"op":"groupBy","keys":["g"],"aggs":{"s":{"agg":"sum","arg":{"fn":"cumSum","args":[{"col":"x"}]}},"l":{"agg":"list","arg":{"fn":"cumSum","args":[{"col":"x"}]}},"o":{"agg":"sum","arg":{"over":{"agg":"sum","arg":{"col":"x"}},"partitionBy":[{"col":"h"}]}}}}],"expected":{"fields":[{"name":"g","type":"i32"},{"name":"s","type":"f64"},{"name":"l","type":"list"},{"name":"o","type":"f64"}],"columns":{"g":[1,2],"s":[8.0,6.0],"l":[[3.0,5.0],[1.0,5.0]],"o":[5.0,5.0]}}}]} +{"generator":"generate_gold.py","polars":"2.1.0","deviations":["JavaScript has no float literal distinct from an integer: an integral number literal behaves like a Polars integer literal (it takes the integer type of the column it meets), so cases never combine an integer column with an integral float literal.","Float to string casts use JavaScript number formatting (Polars prints 5.0 as '5.0', JavaScript as '5').","Joins never add '_right' suffixes: overlapping non-key columns are a type error.","Arithmetic between 64-bit integers (bigint) and other numeric types requires an explicit cast."],"cases":[{"id":"window/cum","description":"cumulative sum, min, max and count; nulls stay null","inputs":{"left":{"fields":[{"name":"g","type":"utf8"},{"name":"t","type":"i32"},{"name":"x","type":"f64"},{"name":"i","type":"i32"},{"name":"b","type":"bool"}],"columns":{"g":["a","b","a","a","b",null,"a","b","b","a"],"t":[1,1,2,3,2,1,4,3,4,5],"x":[1.5,2.0,null,4.0,"NaN",3.0,-1.0,2.0,0.5,2.0],"i":[3,1,4,1,5,9,null,6,5,3],"b":[true,false,null,true,true,false,true,null,false,true]}}},"ops":[{"op":"select","exprs":{"sx":{"fn":"cumSum","args":[{"col":"x"}]},"si":{"fn":"cumSum","args":[{"col":"i"}]},"mn":{"fn":"cumMin","args":[{"col":"i"}]},"mx":{"fn":"cumMax","args":[{"col":"x"}]},"n":{"fn":"cumCount","args":[{"col":"x"}]},"sb":{"fn":"cumSum","args":[{"col":"b"}]}}}],"expected":{"fields":[{"name":"sx","type":"f64"},{"name":"si","type":"i32"},{"name":"mn","type":"i32"},{"name":"mx","type":"f64"},{"name":"n","type":"u32"},{"name":"sb","type":"u32"}],"columns":{"sx":[1.5,3.5,null,7.5,"NaN","NaN","NaN","NaN","NaN","NaN"],"si":[3,4,8,9,14,23,null,29,34,37],"mn":[3,1,1,1,1,1,null,1,1,1],"mx":[1.5,2.0,null,4.0,4.0,4.0,4.0,4.0,4.0,4.0],"n":[1,2,2,3,4,5,6,7,8,9],"sb":[1,1,null,2,3,3,4,null,4,5]}}},{"id":"window/shift-diff","description":"shift (lag and lead) and diff","inputs":{"left":{"fields":[{"name":"g","type":"utf8"},{"name":"t","type":"i32"},{"name":"x","type":"f64"},{"name":"i","type":"i32"},{"name":"b","type":"bool"}],"columns":{"g":["a","b","a","a","b",null,"a","b","b","a"],"t":[1,1,2,3,2,1,4,3,4,5],"x":[1.5,2.0,null,4.0,"NaN",3.0,-1.0,2.0,0.5,2.0],"i":[3,1,4,1,5,9,null,6,5,3],"b":[true,false,null,true,true,false,true,null,false,true]}}},"ops":[{"op":"select","exprs":{"lag":{"fn":"shift","args":[{"col":"i"}]},"lead":{"fn":"shift","args":[{"col":"x"}],"n":-2},"d":{"fn":"diff","args":[{"col":"i"}]},"d2":{"fn":"diff","args":[{"col":"x"}],"n":2},"ls":{"fn":"shift","args":[{"col":"g"}]}}}],"expected":{"fields":[{"name":"lag","type":"i32"},{"name":"lead","type":"f64"},{"name":"d","type":"i32"},{"name":"d2","type":"f64"},{"name":"ls","type":"utf8"}],"columns":{"lag":[null,3,1,4,1,5,9,null,6,5],"lead":[null,4.0,"NaN",3.0,-1.0,2.0,0.5,2.0,null,null],"d":[null,-2,3,-3,4,4,null,null,-1,-2],"d2":[null,null,null,2.0,null,-1.0,"NaN",-1.0,1.5,0.0],"ls":[null,"a","b","a","a","b",null,"a","b","b"]}}},{"id":"window/rank","description":"rank methods, nulls and NaN","inputs":{"left":{"fields":[{"name":"g","type":"utf8"},{"name":"t","type":"i32"},{"name":"x","type":"f64"},{"name":"i","type":"i32"},{"name":"b","type":"bool"}],"columns":{"g":["a","b","a","a","b",null,"a","b","b","a"],"t":[1,1,2,3,2,1,4,3,4,5],"x":[1.5,2.0,null,4.0,"NaN",3.0,-1.0,2.0,0.5,2.0],"i":[3,1,4,1,5,9,null,6,5,3],"b":[true,false,null,true,true,false,true,null,false,true]}}},"ops":[{"op":"select","exprs":{"avg":{"fn":"rank","args":[{"col":"x"}]},"min":{"fn":"rank","args":[{"col":"i"}],"method":"min"},"max":{"fn":"rank","args":[{"col":"i"}],"method":"max"},"dense":{"fn":"rank","args":[{"col":"i"}],"method":"dense","descending":true},"ord":{"fn":"rank","args":[{"col":"x"}],"method":"ordinal"},"s":{"fn":"rank","args":[{"col":"g"}],"method":"dense"}}}],"expected":{"fields":[{"name":"avg","type":"f64"},{"name":"min","type":"u32"},{"name":"max","type":"u32"},{"name":"dense","type":"u32"},{"name":"ord","type":"u32"},{"name":"s","type":"u32"}],"columns":{"avg":[3.0,5.0,null,8.0,9.0,7.0,1.0,5.0,2.0,5.0],"min":[3,1,5,1,6,9,null,8,6,3],"max":[4,2,5,2,7,9,null,8,7,4],"dense":[5,6,4,6,3,1,null,2,3,5],"ord":[3,4,null,8,9,7,1,5,2,6],"s":[1,2,1,1,2,null,1,2,2,1]}}},{"id":"window/rolling","description":"rolling sum, mean, min and max","inputs":{"left":{"fields":[{"name":"g","type":"utf8"},{"name":"t","type":"i32"},{"name":"x","type":"f64"},{"name":"i","type":"i32"},{"name":"b","type":"bool"}],"columns":{"g":["a","b","a","a","b",null,"a","b","b","a"],"t":[1,1,2,3,2,1,4,3,4,5],"x":[1.5,2.0,null,4.0,"NaN",3.0,-1.0,2.0,0.5,2.0],"i":[3,1,4,1,5,9,null,6,5,3],"b":[true,false,null,true,true,false,true,null,false,true]}}},"ops":[{"op":"select","exprs":{"s":{"fn":"rollingSum","args":[{"col":"x"}],"window":3},"m":{"fn":"rollingMean","args":[{"col":"i"}],"window":2},"mn":{"fn":"rollingMin","args":[{"col":"x"}],"window":3,"minPeriods":1},"mx":{"fn":"rollingMax","args":[{"col":"i"}],"window":4,"minPeriods":2}}}],"expected":{"fields":[{"name":"s","type":"f64"},{"name":"m","type":"f64"},{"name":"mn","type":"f64"},{"name":"mx","type":"i32"}],"columns":{"s":[null,null,null,null,null,"NaN","NaN",4.0,1.5,4.5],"m":[null,2.0,2.5,2.5,3.0,7.0,null,null,5.5,4.0],"mn":[1.5,1.5,1.5,2.0,"NaN","NaN","NaN",-1.0,-1.0,0.5],"mx":[null,3,4,4,5,9,9,9,9,6]}}},{"id":"window/over-agg","description":"aggregates broadcast over partitions","inputs":{"left":{"fields":[{"name":"g","type":"utf8"},{"name":"t","type":"i32"},{"name":"x","type":"f64"},{"name":"i","type":"i32"},{"name":"b","type":"bool"}],"columns":{"g":["a","b","a","a","b",null,"a","b","b","a"],"t":[1,1,2,3,2,1,4,3,4,5],"x":[1.5,2.0,null,4.0,"NaN",3.0,-1.0,2.0,0.5,2.0],"i":[3,1,4,1,5,9,null,6,5,3],"b":[true,false,null,true,true,false,true,null,false,true]}}},"ops":[{"op":"withColumns","exprs":{"total":{"over":{"agg":"sum","arg":{"col":"x"}},"partitionBy":[{"col":"g"}]},"n":{"over":{"agg":"len"},"partitionBy":[{"col":"g"}]},"mean":{"over":{"agg":"mean","arg":{"col":"i"}},"partitionBy":[{"col":"g"}]},"first":{"over":{"agg":"first","arg":{"col":"t"}},"partitionBy":[{"col":"g"},{"col":"b"}]}}}],"expected":{"fields":[{"name":"g","type":"utf8"},{"name":"t","type":"i32"},{"name":"x","type":"f64"},{"name":"i","type":"i32"},{"name":"b","type":"bool"},{"name":"total","type":"f64"},{"name":"n","type":"u32"},{"name":"mean","type":"f64"},{"name":"first","type":"i32"}],"columns":{"g":["a","b","a","a","b",null,"a","b","b","a"],"t":[1,1,2,3,2,1,4,3,4,5],"x":[1.5,2.0,null,4.0,"NaN",3.0,-1.0,2.0,0.5,2.0],"i":[3,1,4,1,5,9,null,6,5,3],"b":[true,false,null,true,true,false,true,null,false,true],"total":[6.5,"NaN",6.5,6.5,"NaN",3.0,6.5,"NaN","NaN",6.5],"n":[5,4,5,5,4,1,5,4,4,5],"mean":[2.75,4.25,2.75,2.75,4.25,9.0,2.75,4.25,4.25,2.75],"first":[1,1,2,1,2,1,1,3,1,1]}}},{"id":"window/over-cum","description":"cumulative and shifted values per partition, in row order","inputs":{"left":{"fields":[{"name":"g","type":"utf8"},{"name":"t","type":"i32"},{"name":"x","type":"f64"},{"name":"i","type":"i32"},{"name":"b","type":"bool"}],"columns":{"g":["a","b","a","a","b",null,"a","b","b","a"],"t":[1,1,2,3,2,1,4,3,4,5],"x":[1.5,2.0,null,4.0,"NaN",3.0,-1.0,2.0,0.5,2.0],"i":[3,1,4,1,5,9,null,6,5,3],"b":[true,false,null,true,true,false,true,null,false,true]}}},"ops":[{"op":"withColumns","exprs":{"cs":{"over":{"fn":"cumSum","args":[{"col":"i"}]},"partitionBy":[{"col":"g"}]},"prev":{"over":{"fn":"shift","args":[{"col":"x"}]},"partitionBy":[{"col":"g"}]},"d":{"over":{"fn":"diff","args":[{"col":"t"}]},"partitionBy":[{"col":"g"}]},"r":{"over":{"fn":"rank","args":[{"col":"x"}],"method":"min"},"partitionBy":[{"col":"g"}]},"roll":{"over":{"fn":"rollingSum","args":[{"col":"i"}],"window":2,"minPeriods":1},"partitionBy":[{"col":"g"}]}}}],"expected":{"fields":[{"name":"g","type":"utf8"},{"name":"t","type":"i32"},{"name":"x","type":"f64"},{"name":"i","type":"i32"},{"name":"b","type":"bool"},{"name":"cs","type":"i32"},{"name":"prev","type":"f64"},{"name":"d","type":"i32"},{"name":"r","type":"u32"},{"name":"roll","type":"i32"}],"columns":{"g":["a","b","a","a","b",null,"a","b","b","a"],"t":[1,1,2,3,2,1,4,3,4,5],"x":[1.5,2.0,null,4.0,"NaN",3.0,-1.0,2.0,0.5,2.0],"i":[3,1,4,1,5,9,null,6,5,3],"b":[true,false,null,true,true,false,true,null,false,true],"cs":[3,1,7,8,6,9,null,12,17,11],"prev":[null,null,1.5,null,2.0,null,4.0,"NaN",2.0,-1.0],"d":[null,null,1,1,1,null,1,1,1,1],"r":[2,2,null,4,4,1,1,2,1,3],"roll":[3,1,7,5,6,9,1,11,11,3]}}},{"id":"window/over-filter","description":"filters after windows see the windowed values","inputs":{"left":{"fields":[{"name":"g","type":"utf8"},{"name":"t","type":"i32"},{"name":"x","type":"f64"},{"name":"i","type":"i32"},{"name":"b","type":"bool"}],"columns":{"g":["a","b","a","a","b",null,"a","b","b","a"],"t":[1,1,2,3,2,1,4,3,4,5],"x":[1.5,2.0,null,4.0,"NaN",3.0,-1.0,2.0,0.5,2.0],"i":[3,1,4,1,5,9,null,6,5,3],"b":[true,false,null,true,true,false,true,null,false,true]}}},"ops":[{"op":"withColumns","exprs":{"cs":{"over":{"fn":"cumSum","args":[{"col":"t"}]},"partitionBy":[{"col":"g"}]}}},{"op":"filter","predicate":{"fn":"gt","args":[{"col":"t"},{"lit":2}]}}],"expected":{"fields":[{"name":"g","type":"utf8"},{"name":"t","type":"i32"},{"name":"x","type":"f64"},{"name":"i","type":"i32"},{"name":"b","type":"bool"},{"name":"cs","type":"i32"}],"columns":{"g":["a","a","b","b","a"],"t":[3,4,3,4,5],"x":[4.0,-1.0,2.0,0.5,2.0],"i":[1,null,6,5,3],"b":[true,true,null,false,true],"cs":[6,10,6,10,15]}}},{"id":"window/filter-first","description":"windows after a filter only see the remaining rows","inputs":{"left":{"fields":[{"name":"g","type":"utf8"},{"name":"t","type":"i32"},{"name":"x","type":"f64"},{"name":"i","type":"i32"},{"name":"b","type":"bool"}],"columns":{"g":["a","b","a","a","b",null,"a","b","b","a"],"t":[1,1,2,3,2,1,4,3,4,5],"x":[1.5,2.0,null,4.0,"NaN",3.0,-1.0,2.0,0.5,2.0],"i":[3,1,4,1,5,9,null,6,5,3],"b":[true,false,null,true,true,false,true,null,false,true]}}},"ops":[{"op":"filter","predicate":{"fn":"isNotNull","args":[{"col":"x"}]}},{"op":"withColumns","exprs":{"cs":{"fn":"cumSum","args":[{"col":"x"}]},"r":{"over":{"fn":"rank","args":[{"col":"x"}]},"partitionBy":[{"col":"g"}]}}},{"op":"head","n":4}],"expected":{"fields":[{"name":"g","type":"utf8"},{"name":"t","type":"i32"},{"name":"x","type":"f64"},{"name":"i","type":"i32"},{"name":"b","type":"bool"},{"name":"cs","type":"f64"},{"name":"r","type":"f64"}],"columns":{"g":["a","b","a","b"],"t":[1,1,3,2],"x":[1.5,2.0,4.0,"NaN"],"i":[3,1,1,5],"b":[true,false,true,true],"cs":[1.5,3.5,7.5,"NaN"],"r":[2.0,2.5,4.0,4.0]}}},{"id":"pivot/sum","description":"pivot with a sum per cell; missing cells are null","inputs":{"left":{"fields":[{"name":"region","type":"utf8"},{"name":"quarter","type":"utf8"},{"name":"units","type":"i32"},{"name":"price","type":"f64"}],"columns":{"region":["north","north","south","south","north","east","south","north"],"quarter":["q1","q2","q1","q3","q1","q2","q1",null],"units":[10,20,5,null,7,3,8,1],"price":[1.5,2.0,1.0,3.0,null,2.5,1.0,9.0]}}},"ops":[{"op":"pivot","on":"quarter","index":["region"],"values":"units","aggregate":"sum"}],"expected":{"fields":[{"name":"region","type":"utf8"},{"name":"q1","type":"i32"},{"name":"q2","type":"i32"},{"name":"q3","type":"i32"},{"name":"null","type":"i32"}],"columns":{"region":["north","south","east"],"q1":[17,13,0],"q2":[20,0,3],"q3":[0,0,0],"null":[1,0,0]}}},{"id":"pivot/first","description":"pivot with the first value per cell","inputs":{"left":{"fields":[{"name":"region","type":"utf8"},{"name":"quarter","type":"utf8"},{"name":"units","type":"i32"},{"name":"price","type":"f64"}],"columns":{"region":["north","north","south","south","north","east","south","north"],"quarter":["q1","q2","q1","q3","q1","q2","q1",null],"units":[10,20,5,null,7,3,8,1],"price":[1.5,2.0,1.0,3.0,null,2.5,1.0,9.0]}}},"ops":[{"op":"pivot","on":"region","index":["quarter"],"values":"price","aggregate":"first"}],"expected":{"fields":[{"name":"quarter","type":"utf8"},{"name":"north","type":"f64"},{"name":"south","type":"f64"},{"name":"east","type":"f64"}],"columns":{"quarter":["q1","q2","q3",null],"north":[1.5,2.0,null,9.0],"south":[1.0,null,3.0,null],"east":[null,2.5,null,null]}}},{"id":"pivot/len","description":"pivot counting rows per cell","inputs":{"left":{"fields":[{"name":"region","type":"utf8"},{"name":"quarter","type":"utf8"},{"name":"units","type":"i32"},{"name":"price","type":"f64"}],"columns":{"region":["north","north","south","south","north","east","south","north"],"quarter":["q1","q2","q1","q3","q1","q2","q1",null],"units":[10,20,5,null,7,3,8,1],"price":[1.5,2.0,1.0,3.0,null,2.5,1.0,9.0]}}},"ops":[{"op":"pivot","on":"quarter","index":["region"],"values":"units","aggregate":"len"}],"expected":{"fields":[{"name":"region","type":"utf8"},{"name":"q1","type":"u32"},{"name":"q2","type":"u32"},{"name":"q3","type":"u32"},{"name":"null","type":"u32"}],"columns":{"region":["north","south","east"],"q1":[2,2,0],"q2":[1,0,1],"q3":[0,1,0],"null":[1,0,0]}}},{"id":"pivot/columns","description":"pivot with declared output columns","inputs":{"left":{"fields":[{"name":"region","type":"utf8"},{"name":"quarter","type":"utf8"},{"name":"units","type":"i32"},{"name":"price","type":"f64"}],"columns":{"region":["north","north","south","south","north","east","south","north"],"quarter":["q1","q2","q1","q3","q1","q2","q1",null],"units":[10,20,5,null,7,3,8,1],"price":[1.5,2.0,1.0,3.0,null,2.5,1.0,9.0]}}},"ops":[{"op":"pivot","on":"quarter","index":["region"],"values":"price","aggregate":"max","columns":["q1","q2","q4"]}],"expected":{"fields":[{"name":"region","type":"utf8"},{"name":"q1","type":"f64"},{"name":"q2","type":"f64"},{"name":"q4","type":"f64"}],"columns":{"region":["north","south","east"],"q1":[1.5,1.0,null],"q2":[2.0,null,2.5],"q4":[null,null,null]}}},{"id":"pivot/numeric-on","description":"pivot on a numeric column names columns by value","inputs":{"left":{"fields":[{"name":"g","type":"utf8"},{"name":"t","type":"i32"},{"name":"x","type":"f64"},{"name":"i","type":"i32"},{"name":"b","type":"bool"}],"columns":{"g":["a","b","a","a","b",null,"a","b","b","a"],"t":[1,1,2,3,2,1,4,3,4,5],"x":[1.5,2.0,null,4.0,"NaN",3.0,-1.0,2.0,0.5,2.0],"i":[3,1,4,1,5,9,null,6,5,3],"b":[true,false,null,true,true,false,true,null,false,true]}}},"ops":[{"op":"filter","predicate":{"fn":"isNotNull","args":[{"col":"g"}]}},{"op":"pivot","on":"t","index":["g"],"values":"i","aggregate":"min"}],"expected":{"fields":[{"name":"g","type":"utf8"},{"name":"1","type":"i32"},{"name":"2","type":"i32"},{"name":"3","type":"i32"},{"name":"4","type":"i32"},{"name":"5","type":"i32"}],"columns":{"g":["a","b"],"1":[3,1],"2":[4,5],"3":[1,6],"4":[null,5],"5":[3,null]}}},{"id":"unpivot/basic","description":"unpivot stacks columns, column by column","inputs":{"left":{"fields":[{"name":"region","type":"utf8"},{"name":"quarter","type":"utf8"},{"name":"units","type":"i32"},{"name":"price","type":"f64"}],"columns":{"region":["north","north","south","south","north","east","south","north"],"quarter":["q1","q2","q1","q3","q1","q2","q1",null],"units":[10,20,5,null,7,3,8,1],"price":[1.5,2.0,1.0,3.0,null,2.5,1.0,9.0]}}},"ops":[{"op":"unpivot","index":["region"],"on":["units","price"]}],"expected":{"fields":[{"name":"region","type":"utf8"},{"name":"variable","type":"utf8"},{"name":"value","type":"f64"}],"columns":{"region":["north","north","south","south","north","east","south","north","north","north","south","south","north","east","south","north"],"variable":["units","units","units","units","units","units","units","units","price","price","price","price","price","price","price","price"],"value":[10.0,20.0,5.0,null,7.0,3.0,8.0,1.0,1.5,2.0,1.0,3.0,null,2.5,1.0,9.0]}}},{"id":"unpivot/names","description":"unpivot with named variable and value columns","inputs":{"left":{"fields":[{"name":"region","type":"utf8"},{"name":"quarter","type":"utf8"},{"name":"units","type":"i32"},{"name":"price","type":"f64"}],"columns":{"region":["north","north","south","south","north","east","south","north"],"quarter":["q1","q2","q1","q3","q1","q2","q1",null],"units":[10,20,5,null,7,3,8,1],"price":[1.5,2.0,1.0,3.0,null,2.5,1.0,9.0]}}},"ops":[{"op":"pick","columns":["region","quarter"]},{"op":"unpivot","index":[],"on":["region","quarter"],"variable":"field","value":"text"}],"expected":{"fields":[{"name":"field","type":"utf8"},{"name":"text","type":"utf8"}],"columns":{"field":["region","region","region","region","region","region","region","region","quarter","quarter","quarter","quarter","quarter","quarter","quarter","quarter"],"text":["north","north","south","south","north","east","south","north","q1","q2","q1","q3","q1","q2","q1",null]}}},{"id":"temporal/extract","description":"calendar fields of timestamps and dates","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"ts","type":"datetime[us]"},{"name":"ms","type":"datetime[ms]"},{"name":"d","type":"date"},{"name":"dur","type":"duration[us]"}],"columns":{"k":["a","b","c","d","e","f"],"ts":[1709214307123456,1704067199999000,1709510400000000,null,-14182940000000,1710052200000500],"ms":[1709214307123,1704067200000,null,1709251200000,0,1709942400000],"d":[19782,19722,19786,19782,null,19792],"dur":[5400000000,-1500000,null,259200000001,0,90000000000]}},"right":{"fields":[{"name":"d","type":"date"},{"name":"name","type":"utf8"}],"columns":{"d":[19782,19792,20089],"name":["leap","sunday","new year"]}}},"ops":[{"op":"select","exprs":{"y":{"fn":"year","args":[{"col":"ts"}]},"mo":{"fn":"month","args":[{"col":"ts"}]},"d":{"fn":"day","args":[{"col":"ts"}]},"h":{"fn":"hour","args":[{"col":"ts"}]},"mi":{"fn":"minute","args":[{"col":"ts"}]},"s":{"fn":"second","args":[{"col":"ts"}]},"ms":{"fn":"millisecond","args":[{"col":"ts"}]},"wd":{"fn":"weekday","args":[{"col":"ts"}]},"od":{"fn":"ordinalDay","args":[{"col":"ts"}]},"dy":{"fn":"year","args":[{"col":"d"}]},"dwd":{"fn":"weekday","args":[{"col":"d"}]},"dod":{"fn":"ordinalDay","args":[{"col":"d"}]}}}],"expected":{"fields":[{"name":"y","type":"i32"},{"name":"mo","type":"i8"},{"name":"d","type":"i8"},{"name":"h","type":"i8"},{"name":"mi","type":"i8"},{"name":"s","type":"i8"},{"name":"ms","type":"i32"},{"name":"wd","type":"i8"},{"name":"od","type":"i16"},{"name":"dy","type":"i32"},{"name":"dwd","type":"i8"},{"name":"dod","type":"i16"}],"columns":{"y":[2024,2023,2024,null,1969,2024],"mo":[2,12,3,null,7,3],"d":[29,31,4,null,20,10],"h":[13,23,0,null,20,6],"mi":[45,59,0,null,17,30],"s":[7,59,0,null,40,0],"ms":[123,999,0,null,0,0],"wd":[4,7,1,null,7,7],"od":[60,365,64,null,201,70],"dy":[2024,2023,2024,2024,null,2024],"dwd":[4,7,1,4,null,7],"dod":[60,365,64,60,null,70]}}},{"id":"temporal/arith","description":"timestamp differences are durations; durations shift timestamps","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"ts","type":"datetime[us]"},{"name":"ms","type":"datetime[ms]"},{"name":"d","type":"date"},{"name":"dur","type":"duration[us]"}],"columns":{"k":["a","b","c","d","e","f"],"ts":[1709214307123456,1704067199999000,1709510400000000,null,-14182940000000,1710052200000500],"ms":[1709214307123,1704067200000,null,1709251200000,0,1709942400000],"d":[19782,19722,19786,19782,null,19792],"dur":[5400000000,-1500000,null,259200000001,0,90000000000]}},"right":{"fields":[{"name":"d","type":"date"},{"name":"name","type":"utf8"}],"columns":{"d":[19782,19792,20089],"name":["leap","sunday","new year"]}}},"ops":[{"op":"select","exprs":{"gap":{"fn":"sub","args":[{"col":"ts"},{"col":"ms"}]},"later":{"fn":"add","args":[{"col":"ts"},{"col":"dur"}]},"earlier":{"fn":"sub","args":[{"col":"ts"},{"col":"dur"}]},"days":{"fn":"sub","args":[{"col":"d"},{"lit":18000,"type":"date"}]},"twice":{"fn":"add","args":[{"col":"dur"},{"col":"dur"}]}}}],"expected":{"fields":[{"name":"gap","type":"duration[ms]"},{"name":"later","type":"datetime[us]"},{"name":"earlier","type":"datetime[us]"},{"name":"days","type":"duration[us]"},{"name":"twice","type":"duration[us]"}],"columns":{"gap":[0,-1,null,null,-14182940000,109800000],"later":[1709219707123456,1704067198499000,null,null,-14182940000000,1710142200000500],"earlier":[1709208907123456,1704067201499000,null,null,-14182940000000,1709962200000500],"days":[153964800000000,148780800000000,154310400000000,153964800000000,null,154828800000000],"twice":[10800000000,-3000000,null,518400000002,0,180000000000]}}},{"id":"temporal/truncate","description":"truncate to a calendar unit","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"ts","type":"datetime[us]"},{"name":"ms","type":"datetime[ms]"},{"name":"d","type":"date"},{"name":"dur","type":"duration[us]"}],"columns":{"k":["a","b","c","d","e","f"],"ts":[1709214307123456,1704067199999000,1709510400000000,null,-14182940000000,1710052200000500],"ms":[1709214307123,1704067200000,null,1709251200000,0,1709942400000],"d":[19782,19722,19786,19782,null,19792],"dur":[5400000000,-1500000,null,259200000001,0,90000000000]}},"right":{"fields":[{"name":"d","type":"date"},{"name":"name","type":"utf8"}],"columns":{"d":[19782,19792,20089],"name":["leap","sunday","new year"]}}},"ops":[{"op":"select","exprs":{"h":{"fn":"truncate","args":[{"col":"ts"}],"every":"1h"},"d":{"fn":"truncate","args":[{"col":"ts"}],"every":"1d"},"w":{"fn":"truncate","args":[{"col":"ts"}],"every":"1w"},"mo":{"fn":"truncate","args":[{"col":"ts"}],"every":"1mo"},"y":{"fn":"truncate","args":[{"col":"ts"}],"every":"1y"},"dmo":{"fn":"truncate","args":[{"col":"d"}],"every":"1mo"}}}],"expected":{"fields":[{"name":"h","type":"datetime[us]"},{"name":"d","type":"datetime[us]"},{"name":"w","type":"datetime[us]"},{"name":"mo","type":"datetime[us]"},{"name":"y","type":"datetime[us]"},{"name":"dmo","type":"date"}],"columns":{"h":[1709211600000000,1704063600000000,1709510400000000,null,-14184000000000,1710050400000000],"d":[1709164800000000,1703980800000000,1709510400000000,null,-14256000000000,1710028800000000],"w":[1708905600000000,1703462400000000,1709510400000000,null,-14774400000000,1709510400000000],"mo":[1706745600000000,1701388800000000,1709251200000000,null,-15897600000000,1709251200000000],"y":[1704067200000000,1672531200000000,1704067200000000,null,-31536000000000,1704067200000000],"dmo":[19754,19692,19783,19754,null,19783]}}},{"id":"temporal/duration","description":"durations in whole units","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"ts","type":"datetime[us]"},{"name":"ms","type":"datetime[ms]"},{"name":"d","type":"date"},{"name":"dur","type":"duration[us]"}],"columns":{"k":["a","b","c","d","e","f"],"ts":[1709214307123456,1704067199999000,1709510400000000,null,-14182940000000,1710052200000500],"ms":[1709214307123,1704067200000,null,1709251200000,0,1709942400000],"d":[19782,19722,19786,19782,null,19792],"dur":[5400000000,-1500000,null,259200000001,0,90000000000]}},"right":{"fields":[{"name":"d","type":"date"},{"name":"name","type":"utf8"}],"columns":{"d":[19782,19792,20089],"name":["leap","sunday","new year"]}}},"ops":[{"op":"select","exprs":{"d":{"fn":"totalDays","args":[{"col":"dur"}]},"h":{"fn":"totalHours","args":[{"col":"dur"}]},"m":{"fn":"totalMinutes","args":[{"col":"dur"}]},"s":{"fn":"totalSeconds","args":[{"col":"dur"}]},"ms":{"fn":"totalMilliseconds","args":[{"col":"dur"}]}}}],"expected":{"fields":[{"name":"d","type":"i64"},{"name":"h","type":"i64"},{"name":"m","type":"i64"},{"name":"s","type":"i64"},{"name":"ms","type":"i64"}],"columns":{"d":["0","0",null,"3","0","1"],"h":["1","0",null,"72","0","25"],"m":["90","0",null,"4320","0","1500"],"s":["5400","-1",null,"259200","0","90000"],"ms":["5400000","-1500",null,"259200000","0","90000000"]}}},{"id":"temporal/cast","description":"casts between dates, timestamps and integers","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"ts","type":"datetime[us]"},{"name":"ms","type":"datetime[ms]"},{"name":"d","type":"date"},{"name":"dur","type":"duration[us]"}],"columns":{"k":["a","b","c","d","e","f"],"ts":[1709214307123456,1704067199999000,1709510400000000,null,-14182940000000,1710052200000500],"ms":[1709214307123,1704067200000,null,1709251200000,0,1709942400000],"d":[19782,19722,19786,19782,null,19792],"dur":[5400000000,-1500000,null,259200000001,0,90000000000]}},"right":{"fields":[{"name":"d","type":"date"},{"name":"name","type":"utf8"}],"columns":{"d":[19782,19792,20089],"name":["leap","sunday","new year"]}}},"ops":[{"op":"select","exprs":{"date":{"fn":"cast","args":[{"col":"ts"}],"to":"date"},"ts":{"fn":"cast","args":[{"col":"d"}],"to":"datetime[us]"},"us":{"fn":"cast","args":[{"col":"ts"}],"to":"i64"},"days":{"fn":"cast","args":[{"col":"d"}],"to":"i32"},"ms":{"fn":"cast","args":[{"col":"ts"}],"to":"datetime[ms]"}}}],"expected":{"fields":[{"name":"date","type":"date"},{"name":"ts","type":"datetime[us]"},{"name":"us","type":"i64"},{"name":"days","type":"i32"},{"name":"ms","type":"datetime[ms]"}],"columns":{"date":[19782,19722,19786,null,-165,19792],"ts":[1709164800000000,1703980800000000,1709510400000000,1709164800000000,null,1710028800000000],"us":["1709214307123456","1704067199999000","1709510400000000",null,"-14182940000000","1710052200000500"],"days":[19782,19722,19786,19782,null,19792],"ms":[1709214307123,1704067199999,1709510400000,null,-14182940000,1710052200000]}}},{"id":"temporal/compare","description":"comparisons with timestamp literals and between units","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"ts","type":"datetime[us]"},{"name":"ms","type":"datetime[ms]"},{"name":"d","type":"date"},{"name":"dur","type":"duration[us]"}],"columns":{"k":["a","b","c","d","e","f"],"ts":[1709214307123456,1704067199999000,1709510400000000,null,-14182940000000,1710052200000500],"ms":[1709214307123,1704067200000,null,1709251200000,0,1709942400000],"d":[19782,19722,19786,19782,null,19792],"dur":[5400000000,-1500000,null,259200000001,0,90000000000]}},"right":{"fields":[{"name":"d","type":"date"},{"name":"name","type":"utf8"}],"columns":{"d":[19782,19792,20089],"name":["leap","sunday","new year"]}}},"ops":[{"op":"filter","predicate":{"fn":"and","args":[{"fn":"gte","args":[{"col":"ts"},{"lit":1709251200000000,"type":"datetime[us]"}]},{"fn":"lt","args":[{"col":"ms"},{"col":"ts"}]}]}}],"expected":{"fields":[{"name":"k","type":"utf8"},{"name":"ts","type":"datetime[us]"},{"name":"ms","type":"datetime[ms]"},{"name":"d","type":"date"},{"name":"dur","type":"duration[us]"}],"columns":{"k":["f"],"ts":[1710052200000500],"ms":[1709942400000],"d":[19792],"dur":[90000000000]}}},{"id":"temporal/group-sort","description":"group by date, aggregate timestamps, sort","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"ts","type":"datetime[us]"},{"name":"ms","type":"datetime[ms]"},{"name":"d","type":"date"},{"name":"dur","type":"duration[us]"}],"columns":{"k":["a","b","c","d","e","f"],"ts":[1709214307123456,1704067199999000,1709510400000000,null,-14182940000000,1710052200000500],"ms":[1709214307123,1704067200000,null,1709251200000,0,1709942400000],"d":[19782,19722,19786,19782,null,19792],"dur":[5400000000,-1500000,null,259200000001,0,90000000000]}},"right":{"fields":[{"name":"d","type":"date"},{"name":"name","type":"utf8"}],"columns":{"d":[19782,19792,20089],"name":["leap","sunday","new year"]}}},"ops":[{"op":"groupBy","keys":["d"],"aggs":{"first":{"agg":"min","arg":{"col":"ts"}},"last":{"agg":"max","arg":{"col":"ts"}},"n":{"agg":"len"},"total":{"agg":"sum","arg":{"col":"dur"}}}},{"op":"sort","keys":[{"expr":{"col":"last"},"descending":true}]}],"expected":{"fields":[{"name":"d","type":"date"},{"name":"first","type":"datetime[us]"},{"name":"last","type":"datetime[us]"},{"name":"n","type":"u32"},{"name":"total","type":"duration[us]"}],"columns":{"d":[19792,19786,19782,19722,null],"first":[1710052200000500,1709510400000000,1709214307123456,1704067199999000,-14182940000000],"last":[1710052200000500,1709510400000000,1709214307123456,1704067199999000,-14182940000000],"n":[1,1,2,1,1],"total":[90000000000,0,264600000001,-1500000,0]}}},{"id":"temporal/join","description":"join on a date","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"ts","type":"datetime[us]"},{"name":"ms","type":"datetime[ms]"},{"name":"d","type":"date"},{"name":"dur","type":"duration[us]"}],"columns":{"k":["a","b","c","d","e","f"],"ts":[1709214307123456,1704067199999000,1709510400000000,null,-14182940000000,1710052200000500],"ms":[1709214307123,1704067200000,null,1709251200000,0,1709942400000],"d":[19782,19722,19786,19782,null,19792],"dur":[5400000000,-1500000,null,259200000001,0,90000000000]}},"right":{"fields":[{"name":"d","type":"date"},{"name":"name","type":"utf8"}],"columns":{"d":[19782,19792,20089],"name":["leap","sunday","new year"]}}},"ops":[{"op":"join","right":"right","on":["d"],"how":"left"}],"expected":{"fields":[{"name":"k","type":"utf8"},{"name":"ts","type":"datetime[us]"},{"name":"ms","type":"datetime[ms]"},{"name":"d","type":"date"},{"name":"dur","type":"duration[us]"},{"name":"name","type":"utf8"}],"columns":{"k":["a","b","c","d","e","f"],"ts":[1709214307123456,1704067199999000,1709510400000000,null,-14182940000000,1710052200000500],"ms":[1709214307123,1704067200000,null,1709251200000,0,1709942400000],"d":[19782,19722,19786,19782,null,19792],"dur":[5400000000,-1500000,null,259200000001,0,90000000000],"name":["leap",null,null,"leap",null,"sunday"]}}},{"id":"nested/list-ops","description":"list length, element access and aggregates","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"g","type":"i32"},{"name":"xs","type":"list"},{"name":"ws","type":"list"},{"name":"s","type":"struct"}],"columns":{"k":["a","b","c","d","e"],"g":[1,2,1,2,1],"xs":[[1,2,3],[],null,[4,null,2],[7]],"ws":[["a","b"],["c"],[],null,["z","a"]],"s":[{"a":1,"b":"x"},{"a":null,"b":"y"},null,{"a":4,"b":null},{"a":5,"b":"z"}]}},"right":{"fields":[{"name":"k","type":"utf8"},{"name":"tags","type":"list"}],"columns":{"k":["a","d","x"],"tags":[["t1"],["t2","t3"],[]]}}},"ops":[{"op":"select","exprs":{"n":{"fn":"listLength","args":[{"col":"xs"}]},"first":{"fn":"listGet","args":[{"col":"xs"}],"index":0},"last":{"fn":"listGet","args":[{"col":"xs"}],"index":-1},"oob":{"fn":"listGet","args":[{"col":"xs"}],"index":5},"sum":{"fn":"listSum","args":[{"col":"xs"}]},"mean":{"fn":"listMean","args":[{"col":"xs"}]},"min":{"fn":"listMin","args":[{"col":"xs"}]},"max":{"fn":"listMax","args":[{"col":"ws"}]},"has2":{"fn":"listContains","args":[{"col":"xs"},{"lit":2}]},"hasA":{"fn":"listContains","args":[{"col":"ws"},{"lit":"a"}]}}}],"expected":{"fields":[{"name":"n","type":"u32"},{"name":"first","type":"i32"},{"name":"last","type":"i32"},{"name":"oob","type":"i32"},{"name":"sum","type":"i32"},{"name":"mean","type":"f64"},{"name":"min","type":"i32"},{"name":"max","type":"utf8"},{"name":"has2","type":"bool"},{"name":"hasA","type":"bool"}],"columns":{"n":[3,0,null,3,1],"first":[1,null,null,4,7],"last":[3,null,null,2,7],"oob":[null,null,null,null,null],"sum":[6,0,null,6,7],"mean":[2.0,null,null,3.0,7.0],"min":[1,null,null,2,7],"max":["b","c",null,null,"z"],"has2":[true,false,null,true,false],"hasA":[true,false,false,null,true]}}},{"id":"nested/struct","description":"struct fields and struct construction","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"g","type":"i32"},{"name":"xs","type":"list"},{"name":"ws","type":"list"},{"name":"s","type":"struct"}],"columns":{"k":["a","b","c","d","e"],"g":[1,2,1,2,1],"xs":[[1,2,3],[],null,[4,null,2],[7]],"ws":[["a","b"],["c"],[],null,["z","a"]],"s":[{"a":1,"b":"x"},{"a":null,"b":"y"},null,{"a":4,"b":null},{"a":5,"b":"z"}]}},"right":{"fields":[{"name":"k","type":"utf8"},{"name":"tags","type":"list"}],"columns":{"k":["a","d","x"],"tags":[["t1"],["t2","t3"],[]]}}},"ops":[{"op":"select","exprs":{"a":{"fn":"field","args":[{"col":"s"}],"field":"a"},"b":{"fn":"field","args":[{"col":"s"}],"field":"b"},"pair":{"fn":"struct","args":[{"col":"k"},{"fn":"listLength","args":[{"col":"xs"}]}],"names":["key","size"]}}}],"expected":{"fields":[{"name":"a","type":"i32"},{"name":"b","type":"utf8"},{"name":"pair","type":"struct"}],"columns":{"a":[1,null,null,4,5],"b":["x","y",null,null,"z"],"pair":[{"key":"a","size":3},{"key":"b","size":0},{"key":"c","size":null},{"key":"d","size":3},{"key":"e","size":1}]}}},{"id":"nested/explode","description":"explode lists into rows; a null list gives a null row, an empty list no rows","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"g","type":"i32"},{"name":"xs","type":"list"},{"name":"ws","type":"list"},{"name":"s","type":"struct"}],"columns":{"k":["a","b","c","d","e"],"g":[1,2,1,2,1],"xs":[[1,2,3],[],null,[4,null,2],[7]],"ws":[["a","b"],["c"],[],null,["z","a"]],"s":[{"a":1,"b":"x"},{"a":null,"b":"y"},null,{"a":4,"b":null},{"a":5,"b":"z"}]}},"right":{"fields":[{"name":"k","type":"utf8"},{"name":"tags","type":"list"}],"columns":{"k":["a","d","x"],"tags":[["t1"],["t2","t3"],[]]}}},"ops":[{"op":"pick","columns":["k","xs"]},{"op":"explode","column":"xs"}],"expected":{"fields":[{"name":"k","type":"utf8"},{"name":"xs","type":"i32"}],"columns":{"k":["a","a","a","c","d","d","d","e"],"xs":[1,2,3,null,4,null,2,7]}}},{"id":"nested/unnest","description":"unnest a struct into columns","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"g","type":"i32"},{"name":"xs","type":"list"},{"name":"ws","type":"list"},{"name":"s","type":"struct"}],"columns":{"k":["a","b","c","d","e"],"g":[1,2,1,2,1],"xs":[[1,2,3],[],null,[4,null,2],[7]],"ws":[["a","b"],["c"],[],null,["z","a"]],"s":[{"a":1,"b":"x"},{"a":null,"b":"y"},null,{"a":4,"b":null},{"a":5,"b":"z"}]}},"right":{"fields":[{"name":"k","type":"utf8"},{"name":"tags","type":"list"}],"columns":{"k":["a","d","x"],"tags":[["t1"],["t2","t3"],[]]}}},"ops":[{"op":"pick","columns":["k","s"]},{"op":"unnest","column":"s"}],"expected":{"fields":[{"name":"k","type":"utf8"},{"name":"a","type":"i32"},{"name":"b","type":"utf8"}],"columns":{"k":["a","b","c","d","e"],"a":[1,null,null,4,5],"b":["x","y",null,null,"z"]}}},{"id":"nested/agg-list","description":"collect values into lists per group","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"g","type":"i32"},{"name":"xs","type":"list"},{"name":"ws","type":"list"},{"name":"s","type":"struct"}],"columns":{"k":["a","b","c","d","e"],"g":[1,2,1,2,1],"xs":[[1,2,3],[],null,[4,null,2],[7]],"ws":[["a","b"],["c"],[],null,["z","a"]],"s":[{"a":1,"b":"x"},{"a":null,"b":"y"},null,{"a":4,"b":null},{"a":5,"b":"z"}]}},"right":{"fields":[{"name":"k","type":"utf8"},{"name":"tags","type":"list"}],"columns":{"k":["a","d","x"],"tags":[["t1"],["t2","t3"],[]]}}},"ops":[{"op":"groupBy","keys":["g"],"aggs":{"ks":{"agg":"list","arg":{"col":"k"}},"firsts":{"agg":"first","arg":{"col":"xs"}},"n":{"agg":"len"}}}],"expected":{"fields":[{"name":"g","type":"i32"},{"name":"ks","type":"list"},{"name":"firsts","type":"list"},{"name":"n","type":"u32"}],"columns":{"g":[1,2],"ks":[["a","c","e"],["b","d"]],"firsts":[[1,2,3],[]],"n":[3,2]}}},{"id":"nested/carry","description":"lists and structs survive filters, sorts, joins and slices","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"g","type":"i32"},{"name":"xs","type":"list"},{"name":"ws","type":"list"},{"name":"s","type":"struct"}],"columns":{"k":["a","b","c","d","e"],"g":[1,2,1,2,1],"xs":[[1,2,3],[],null,[4,null,2],[7]],"ws":[["a","b"],["c"],[],null,["z","a"]],"s":[{"a":1,"b":"x"},{"a":null,"b":"y"},null,{"a":4,"b":null},{"a":5,"b":"z"}]}},"right":{"fields":[{"name":"k","type":"utf8"},{"name":"tags","type":"list"}],"columns":{"k":["a","d","x"],"tags":[["t1"],["t2","t3"],[]]}}},"ops":[{"op":"filter","predicate":{"fn":"neq","args":[{"col":"k"},{"lit":"c"}]}},{"op":"sort","keys":[{"expr":{"col":"g"},"descending":true},{"expr":{"col":"k"}}]},{"op":"join","right":"right","on":["k"],"how":"left"},{"op":"head","n":4}],"expected":{"fields":[{"name":"k","type":"utf8"},{"name":"g","type":"i32"},{"name":"xs","type":"list"},{"name":"ws","type":"list"},{"name":"s","type":"struct"},{"name":"tags","type":"list"}],"columns":{"k":["b","d","a","e"],"g":[2,2,1,1],"xs":[[],[4,null,2],[1,2,3],[7]],"ws":[["c"],null,["a","b"],["z","a"]],"s":[{"a":null,"b":"y"},{"a":4,"b":null},{"a":1,"b":"x"},{"a":5,"b":"z"}],"tags":[null,["t2","t3"],["t1"],null]}}},{"id":"decimal/arith","description":"decimal addition, subtraction and multiplication","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"price","type":"decimal[10,2]"},{"name":"fee","type":"decimal[6,3]"},{"name":"qty","type":"decimal[4,0]"},{"name":"x","type":"f64"}],"columns":{"k":["a","b","a","c","b","a"],"price":["12.34","0.10","-5.00",null,"99999999.99","10.01"],"fee":["0.125","1.000",null,"2.500","-0.001","10.010"],"qty":["3","10","2","1",null,"7"],"x":[1.005,-2.5,123.456,null,1000000.0,0.0]}}},"ops":[{"op":"select","exprs":{"sum":{"fn":"add","args":[{"col":"price"},{"col":"fee"}]},"diff":{"fn":"sub","args":[{"col":"price"},{"col":"fee"}]},"prod":{"fn":"mul","args":[{"col":"price"},{"col":"qty"}]},"plus":{"fn":"add","args":[{"col":"price"},{"lit":"0.5","type":"decimal[3,1]"}]}}}],"expected":{"fields":[{"name":"sum","type":"decimal[38,3]"},{"name":"diff","type":"decimal[38,3]"},{"name":"prod","type":"decimal[38,2]"},{"name":"plus","type":"decimal[38,2]"}],"columns":{"sum":["12.465","1.100",null,null,"99999999.989","20.020"],"diff":["12.215","-0.900",null,null,"99999999.991","0.000"],"prod":["37.02","1.00","-10.00",null,null,"70.07"],"plus":["12.84","0.60","-4.50",null,"100000000.49","10.51"]}}},{"id":"decimal/compare","description":"decimal comparisons, also across scales","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"price","type":"decimal[10,2]"},{"name":"fee","type":"decimal[6,3]"},{"name":"qty","type":"decimal[4,0]"},{"name":"x","type":"f64"}],"columns":{"k":["a","b","a","c","b","a"],"price":["12.34","0.10","-5.00",null,"99999999.99","10.01"],"fee":["0.125","1.000",null,"2.500","-0.001","10.010"],"qty":["3","10","2","1",null,"7"],"x":[1.005,-2.5,123.456,null,1000000.0,0.0]}}},"ops":[{"op":"filter","predicate":{"fn":"and","args":[{"fn":"gt","args":[{"col":"price"},{"lit":"10.00","type":"decimal[10,2]"}]},{"fn":"gte","args":[{"col":"price"},{"col":"fee"}]}]}}],"expected":{"fields":[{"name":"k","type":"utf8"},{"name":"price","type":"decimal[10,2]"},{"name":"fee","type":"decimal[6,3]"},{"name":"qty","type":"decimal[4,0]"},{"name":"x","type":"f64"}],"columns":{"k":["a","b","a"],"price":["12.34","99999999.99","10.01"],"fee":["0.125","-0.001","10.010"],"qty":["3",null,"7"],"x":[1.005,1000000.0,0.0]}}},{"id":"decimal/aggregate","description":"sums, extremes and means of decimals per group","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"price","type":"decimal[10,2]"},{"name":"fee","type":"decimal[6,3]"},{"name":"qty","type":"decimal[4,0]"},{"name":"x","type":"f64"}],"columns":{"k":["a","b","a","c","b","a"],"price":["12.34","0.10","-5.00",null,"99999999.99","10.01"],"fee":["0.125","1.000",null,"2.500","-0.001","10.010"],"qty":["3","10","2","1",null,"7"],"x":[1.005,-2.5,123.456,null,1000000.0,0.0]}}},"ops":[{"op":"groupBy","keys":["k"],"aggs":{"total":{"agg":"sum","arg":{"col":"price"}},"min":{"agg":"min","arg":{"col":"price"}},"max":{"agg":"max","arg":{"col":"fee"}},"mean":{"agg":"mean","arg":{"col":"price"}},"n":{"agg":"count","arg":{"col":"price"}}}}],"expected":{"fields":[{"name":"k","type":"utf8"},{"name":"total","type":"decimal[38,2]"},{"name":"min","type":"decimal[10,2]"},{"name":"max","type":"decimal[6,3]"},{"name":"mean","type":"f64"},{"name":"n","type":"u32"}],"columns":{"k":["a","b","c"],"total":["17.35","100000000.09","0.00"],"min":["-5.00","0.10",null],"max":["10.010","1.000","2.500"],"mean":[5.783333333333334,50000000.044999994,null],"n":[3,2,0]}}},{"id":"decimal/cast","description":"casts between decimals, floats, integers and strings","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"price","type":"decimal[10,2]"},{"name":"fee","type":"decimal[6,3]"},{"name":"qty","type":"decimal[4,0]"},{"name":"x","type":"f64"}],"columns":{"k":["a","b","a","c","b","a"],"price":["12.34","0.10","-5.00",null,"99999999.99","10.01"],"fee":["0.125","1.000",null,"2.500","-0.001","10.010"],"qty":["3","10","2","1",null,"7"],"x":[1.005,-2.5,123.456,null,1000000.0,0.0]}}},"ops":[{"op":"select","exprs":{"f":{"fn":"cast","args":[{"col":"price"}],"to":"f64"},"i":{"fn":"cast","args":[{"col":"price"}],"to":"i32"},"s":{"fn":"cast","args":[{"col":"fee"}],"to":"utf8"},"d":{"fn":"cast","args":[{"col":"x"}],"to":"decimal[10,2]"},"r":{"fn":"cast","args":[{"col":"price"}],"to":"decimal[12,1]"}}}],"expected":{"fields":[{"name":"f","type":"f64"},{"name":"i","type":"i32"},{"name":"s","type":"utf8"},{"name":"d","type":"decimal[10,2]"},{"name":"r","type":"decimal[12,1]"}],"columns":{"f":[12.34,0.1,-5.0,null,99999999.99,10.01],"i":[12,0,-5,null,99999999,10],"s":["0.125","1.000",null,"2.500","-0.001","10.010"],"d":["1.00","-2.50","123.46",null,"1000000.00","0.00"],"r":["12.3","0.1","-5.0",null,"100000000.0","10.0"]}}},{"id":"decimal/sort","description":"sort by a decimal","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"price","type":"decimal[10,2]"},{"name":"fee","type":"decimal[6,3]"},{"name":"qty","type":"decimal[4,0]"},{"name":"x","type":"f64"}],"columns":{"k":["a","b","a","c","b","a"],"price":["12.34","0.10","-5.00",null,"99999999.99","10.01"],"fee":["0.125","1.000",null,"2.500","-0.001","10.010"],"qty":["3","10","2","1",null,"7"],"x":[1.005,-2.5,123.456,null,1000000.0,0.0]}}},"ops":[{"op":"sort","keys":[{"expr":{"col":"price"},"descending":true}],"nullsLast":true}],"expected":{"fields":[{"name":"k","type":"utf8"},{"name":"price","type":"decimal[10,2]"},{"name":"fee","type":"decimal[6,3]"},{"name":"qty","type":"decimal[4,0]"},{"name":"x","type":"f64"}],"columns":{"k":["b","a","a","b","a","c"],"price":["99999999.99","12.34","10.01","0.10","-5.00",null],"fee":["-0.001","0.125","10.010","1.000",null,"2.500"],"qty":[null,"3","7","10","2","1"],"x":[1000000.0,1.005,0.0,-2.5,123.456,null]}}},{"id":"decimal/round-ties","description":"narrowing the scale rounds exact ties","inputs":{"left":{"fields":[{"name":"t","type":"decimal[6,3]"}],"columns":{"t":["0.250","0.350","-0.250","-0.350","1.050","2.500","0.249"]}}},"ops":[{"op":"select","exprs":{"one":{"fn":"cast","args":[{"col":"t"}],"to":"decimal[6,1]"},"zero":{"fn":"cast","args":[{"col":"t"}],"to":"decimal[6,0]"}}}],"expected":{"fields":[{"name":"one","type":"decimal[6,1]"},{"name":"zero","type":"decimal[6,0]"}],"columns":{"one":["0.2","0.4","-0.2","-0.4","1.0","2.5","0.2"],"zero":["0","0","0","0","1","2","0"]}}},{"id":"decimal/cast-float-ties","description":"floats cast to decimals round exact ties half to even, also below zero","inputs":{"left":{"fields":[{"name":"x","type":"f64"}],"columns":{"x":[-1.5,-2.5,1.5,2.5,-0.75]}}},"ops":[{"op":"select","exprs":{"zero":{"fn":"cast","args":[{"col":"x"}],"to":"decimal[18,0]"},"one":{"fn":"cast","args":[{"col":"x"}],"to":"decimal[18,1]"}}}],"expected":{"fields":[{"name":"zero","type":"decimal[18,0]"},{"name":"one","type":"decimal[18,1]"}],"columns":{"zero":["-2","-2","2","2","-1"],"one":["-1.5","-2.5","1.5","2.5","-0.8"]}}},{"id":"decimal/round","description":"rounding decimals keeps the scale and rounds half to even","inputs":{"left":{"fields":[{"name":"t","type":"decimal[6,3]"}],"columns":{"t":["0.250","0.350","-0.250","-0.350","1.050","2.500","0.249",null]}}},"ops":[{"op":"select","exprs":{"r0":{"fn":"round","args":[{"col":"t"}]},"r1":{"fn":"round","args":[{"col":"t"}],"decimals":1},"r2":{"fn":"round","args":[{"col":"t"}],"decimals":2},"r3":{"fn":"round","args":[{"col":"t"}],"decimals":3}}}],"expected":{"fields":[{"name":"r0","type":"decimal[7,3]"},{"name":"r1","type":"decimal[7,3]"},{"name":"r2","type":"decimal[7,3]"},{"name":"r3","type":"decimal[6,3]"}],"columns":{"r0":["0.000","0.000","0.000","0.000","1.000","2.000","0.000",null],"r1":["0.200","0.400","-0.200","-0.400","1.000","2.500","0.200",null],"r2":["0.250","0.350","-0.250","-0.350","1.050","2.500","0.250",null],"r3":["0.250","0.350","-0.250","-0.350","1.050","2.500","0.249",null]}}},{"id":"arith/u32-divide-by-zero","description":"unsigned 32-bit floor division and modulo by zero give null","inputs":{"left":{"fields":[{"name":"n","type":"u32"},{"name":"d","type":"u32"}],"columns":{"n":[7,0,5],"d":[0,0,2]}}},"ops":[{"op":"select","exprs":{"q":{"fn":"floorDiv","args":[{"col":"n"},{"col":"d"}]},"m":{"fn":"mod","args":[{"col":"n"},{"col":"d"}]},"qs":{"fn":"floorDiv","args":[{"col":"n"},{"lit":0,"type":"u32"}]},"ms":{"fn":"mod","args":[{"col":"n"},{"lit":0,"type":"u32"}]}}}],"expected":{"fields":[{"name":"q","type":"u32"},{"name":"m","type":"u32"},{"name":"qs","type":"u32"},{"name":"ms","type":"u32"}],"columns":{"q":[null,null,2],"m":[null,null,1],"qs":[null,null,null],"ms":[null,null,null]}}},{"id":"slice/clip-5+1","description":"slices reaching before the first row clip from the unclamped start","inputs":{"left":{"fields":[{"name":"x","type":"i32"}],"columns":{"x":[1,2,3]}}},"ops":[{"op":"slice","offset":-5,"length":1}],"expected":{"fields":[{"name":"x","type":"i32"}],"columns":{"x":[]}}},{"id":"slice/clip-5+3","description":"slices reaching before the first row clip from the unclamped start","inputs":{"left":{"fields":[{"name":"x","type":"i32"}],"columns":{"x":[1,2,3]}}},"ops":[{"op":"slice","offset":-5,"length":3}],"expected":{"fields":[{"name":"x","type":"i32"}],"columns":{"x":[1]}}},{"id":"slice/clip-5+4","description":"slices reaching before the first row clip from the unclamped start","inputs":{"left":{"fields":[{"name":"x","type":"i32"}],"columns":{"x":[1,2,3]}}},"ops":[{"op":"slice","offset":-5,"length":4}],"expected":{"fields":[{"name":"x","type":"i32"}],"columns":{"x":[1,2]}}},{"id":"slice/clip-2+5","description":"slices reaching before the first row clip from the unclamped start","inputs":{"left":{"fields":[{"name":"x","type":"i32"}],"columns":{"x":[1,2,3]}}},"ops":[{"op":"slice","offset":-2,"length":5}],"expected":{"fields":[{"name":"x","type":"i32"}],"columns":{"x":[2,3]}}},{"id":"slice/clip-5","description":"slices reaching before the first row clip from the unclamped start","inputs":{"left":{"fields":[{"name":"x","type":"i32"}],"columns":{"x":[1,2,3]}}},"ops":[{"op":"slice","offset":-5}],"expected":{"fields":[{"name":"x","type":"i32"}],"columns":{"x":[1,2,3]}}},{"id":"slice/clip1+0","description":"slices reaching before the first row clip from the unclamped start","inputs":{"left":{"fields":[{"name":"x","type":"i32"}],"columns":{"x":[1,2,3]}}},"ops":[{"op":"slice","offset":1,"length":0}],"expected":{"fields":[{"name":"x","type":"i32"}],"columns":{"x":[]}}},{"id":"window/filter-after-sort","description":"a window predicate sees the sorted rows","inputs":{"left":{"fields":[{"name":"g","type":"i32"},{"name":"h","type":"i32"},{"name":"x","type":"f64"}],"columns":{"g":[1,2,1,2],"h":[1,1,2,2],"x":[3.0,1.0,2.0,4.0]}},"right":{"fields":[{"name":"g","type":"i32"},{"name":"name","type":"utf8"}],"columns":{"g":[1],"name":["one"]}}},"ops":[{"op":"sort","keys":[{"expr":{"col":"x"}}]},{"op":"filter","predicate":{"fn":"gt","args":[{"fn":"cumSum","args":[{"col":"x"}]},{"lit":3.0}]}}],"expected":{"fields":[{"name":"g","type":"i32"},{"name":"h","type":"i32"},{"name":"x","type":"f64"}],"columns":{"g":[1,2],"h":[1,2],"x":[3.0,4.0]}}},{"id":"window/filter-after-filter","description":"a window predicate sees only the rows an earlier filter kept","inputs":{"left":{"fields":[{"name":"g","type":"i32"},{"name":"h","type":"i32"},{"name":"x","type":"f64"}],"columns":{"g":[1,2,1,2],"h":[1,1,2,2],"x":[3.0,1.0,2.0,4.0]}},"right":{"fields":[{"name":"g","type":"i32"},{"name":"name","type":"utf8"}],"columns":{"g":[1],"name":["one"]}}},"ops":[{"op":"filter","predicate":{"fn":"gt","args":[{"col":"x"},{"lit":1.0}]}},{"op":"filter","predicate":{"fn":"gt","args":[{"fn":"shift","args":[{"col":"x"}]},{"lit":2.0}]}}],"expected":{"fields":[{"name":"g","type":"i32"},{"name":"h","type":"i32"},{"name":"x","type":"f64"}],"columns":{"g":[1],"h":[2],"x":[2.0]}}},{"id":"window/filter-after-concat","description":"a window predicate sees every input of a concat","inputs":{"left":{"fields":[{"name":"g","type":"i32"},{"name":"h","type":"i32"},{"name":"x","type":"f64"}],"columns":{"g":[1,2,1,2],"h":[1,1,2,2],"x":[3.0,1.0,2.0,4.0]}},"right":{"fields":[{"name":"g","type":"i32"},{"name":"name","type":"utf8"}],"columns":{"g":[1],"name":["one"]}}},"ops":[{"op":"concat","with":"left"},{"op":"filter","predicate":{"fn":"gt","args":[{"fn":"cumSum","args":[{"col":"x"}]},{"lit":15.0}]}}],"expected":{"fields":[{"name":"g","type":"i32"},{"name":"h","type":"i32"},{"name":"x","type":"f64"}],"columns":{"g":[1,2],"h":[2,2],"x":[2.0,4.0]}}},{"id":"window/filter-after-join","description":"a window predicate sees only the joined rows","inputs":{"left":{"fields":[{"name":"g","type":"i32"},{"name":"h","type":"i32"},{"name":"x","type":"f64"}],"columns":{"g":[1,2,1,2],"h":[1,1,2,2],"x":[3.0,1.0,2.0,4.0]}},"right":{"fields":[{"name":"g","type":"i32"},{"name":"name","type":"utf8"}],"columns":{"g":[1],"name":["one"]}}},"ops":[{"op":"join","right":"right","on":["g"],"how":"inner"},{"op":"filter","predicate":{"fn":"gt","args":[{"fn":"shift","args":[{"col":"x"}]},{"lit":2.0}]}}],"expected":{"fields":[{"name":"g","type":"i32"},{"name":"h","type":"i32"},{"name":"x","type":"f64"},{"name":"name","type":"utf8"}],"columns":{"g":[1],"h":[2],"x":[2.0],"name":["one"]}}},{"id":"window/filter-whole","description":"a window predicate sees the whole input","inputs":{"left":{"fields":[{"name":"g","type":"i32"},{"name":"h","type":"i32"},{"name":"x","type":"f64"}],"columns":{"g":[1,2,1,2],"h":[1,1,2,2],"x":[3.0,1.0,2.0,4.0]}},"right":{"fields":[{"name":"g","type":"i32"},{"name":"name","type":"utf8"}],"columns":{"g":[1],"name":["one"]}}},"ops":[{"op":"filter","predicate":{"fn":"gt","args":[{"fn":"cumSum","args":[{"col":"x"}]},{"lit":5.0}]}}],"expected":{"fields":[{"name":"g","type":"i32"},{"name":"h","type":"i32"},{"name":"x","type":"f64"}],"columns":{"g":[1,2],"h":[2,2],"x":[2.0,4.0]}}},{"id":"window/agg-of-window","description":"an aggregate of a window sees the whole input","inputs":{"left":{"fields":[{"name":"g","type":"i32"},{"name":"h","type":"i32"},{"name":"x","type":"f64"}],"columns":{"g":[1,2,1,2],"h":[1,1,2,2],"x":[3.0,1.0,2.0,4.0]}},"right":{"fields":[{"name":"g","type":"i32"},{"name":"name","type":"utf8"}],"columns":{"g":[1],"name":["one"]}}},"ops":[{"op":"agg","aggs":{"s":{"agg":"sum","arg":{"fn":"cumSum","args":[{"col":"x"}]}},"n":{"agg":"len"}}}],"expected":{"fields":[{"name":"s","type":"f64"},{"name":"n","type":"u32"}],"columns":{"s":[23.0],"n":[4]}}},{"id":"window/grouped-agg-of-window","description":"windows inside grouped aggregates run per group","inputs":{"left":{"fields":[{"name":"g","type":"i32"},{"name":"h","type":"i32"},{"name":"x","type":"f64"}],"columns":{"g":[1,2,1,2],"h":[1,1,2,2],"x":[3.0,1.0,2.0,4.0]}},"right":{"fields":[{"name":"g","type":"i32"},{"name":"name","type":"utf8"}],"columns":{"g":[1],"name":["one"]}}},"ops":[{"op":"groupBy","keys":["g"],"aggs":{"s":{"agg":"sum","arg":{"fn":"cumSum","args":[{"col":"x"}]}},"l":{"agg":"list","arg":{"fn":"cumSum","args":[{"col":"x"}]}},"o":{"agg":"sum","arg":{"over":{"agg":"sum","arg":{"col":"x"}},"partitionBy":[{"col":"h"}]}}}}],"expected":{"fields":[{"name":"g","type":"i32"},{"name":"s","type":"f64"},{"name":"l","type":"list"},{"name":"o","type":"f64"}],"columns":{"g":[1,2],"s":[8.0,6.0],"l":[[3.0,5.0],[1.0,5.0]],"o":[5.0,5.0]}}}]} From 465db761a3c1db47ba768c6fcf79a26bf3f61bdc Mon Sep 17 00:00:00 2001 From: Effect Bot Date: Sun, 15 Nov 2026 11:51:17 +0000 Subject: [PATCH 11/88] Type SQL engine results from the in-memory engine Compile runs the plan through the in-memory engine on zero-row inputs (values emptied, tables from their declared columns) and uses the frames for every type decision. Compiled now carries the result schema; the final query renders each column as text (temporal values as integers in their unit) and the engine decodes rows against that schema. Also fixes round (float, integer and decimal semantics of the engine), boolean sums, window functions in filters, temporal and decimal values at full precision, OVER () for empty partition lists, countDistinct of empty input, helper names that clash with columns, sqrt of negative numbers, and keeps the driver's message in errors. --- packages/effect/src/dataframe/Sql.ts | 53 +- packages/effect/src/dataframe/internal/sql.ts | 884 +++++++++--------- packages/sql/pglite/test/DataFrameSql.test.ts | 177 +++- 3 files changed, 609 insertions(+), 505 deletions(-) diff --git a/packages/effect/src/dataframe/Sql.ts b/packages/effect/src/dataframe/Sql.ts index 7d8a929f49a..2803dbd4c19 100644 --- a/packages/effect/src/dataframe/Sql.ts +++ b/packages/effect/src/dataframe/Sql.ts @@ -26,29 +26,29 @@ * ) * * const compiled = Result.getOrThrow(Sql.compile(query.plan, { dialect: "postgres" })) - * compiled.columns // => ["city", "total"] + * compiled.schema.fields.map((f) => f.name) // => ["city", "total"] * ``` * * **Details** * - * Values follow the database's semantics where they differ from the - * in-memory engine's: integer overflow raises an error instead of wrapping, - * division by zero follows the database, string order follows its collation - * (binary in DuckDB), `round` rounds half away from zero in PostgreSQL, and - * `min`/`max` of floats treat NaN as the largest value. Pivots, unpivots, + * Results have the column types the in-memory engine gives for the same + * plan (see {@link Compiled}). Values follow the database's semantics where + * they differ from the in-memory engine's: integer overflow raises an error + * instead of wrapping, division by zero follows the database, and string + * order follows its collation (binary in DuckDB). Pivots, unpivots, * `explode` and `unnest` are not supported yet; tables need declared * columns (`Query.table` with a schema). * * @stability experimental * @since 4.1.5 */ -import * as DateTime from "../DateTime.ts" +import type * as ArrowSchema from "../arrow/ArrowSchema.ts" import * as Effect from "../Effect.ts" import * as Layer from "../Layer.ts" import * as Result from "../Result.ts" import { SqlClient } from "../sql/SqlClient.ts" import * as Stream from "../Stream.ts" -import { DataFrameError, fromColumns } from "./DataFrame.ts" +import { DataFrameError } from "./DataFrame.ts" import type { DataFrame } from "./DataFrame.ts" import { Engine } from "./Engine.ts" import * as internal from "./internal/sql.ts" @@ -65,7 +65,15 @@ import type * as Plan from "./Plan.ts" export type Dialect = "duckdb" | "postgres" /** - * A compiled plan: the query, and the names of its result columns in order. + * A compiled plan: the query, and the schema of its result. + * + * **Details** + * + * The schema has the column types the in-memory engine gives for the same + * plan, whatever the database's own types are; every field is nullable. The + * query renders each column in a form {@link engine} reads back as that + * type, so its raw rows are meant for that decoding rather than for direct + * use. * * @stability experimental * @category models @@ -73,11 +81,17 @@ export type Dialect = "duckdb" | "postgres" */ export interface Compiled { readonly sql: string - readonly columns: ReadonlyArray + readonly schema: ArrowSchema.Schema } -const toError = (cause: unknown) => - new DataFrameError({ message: cause instanceof Error ? cause.message : String(cause), cause }) +/** The messages of an error and its causes, outermost first, without repeats. */ +const toError = (cause: unknown) => { + const messages: Array = [] + for (let c: unknown = cause; c instanceof Error && messages.length < 8; c = c.cause) { + if (c.message !== "" && !messages.includes(c.message)) messages.push(c.message) + } + return new DataFrameError({ message: messages.length === 0 ? String(cause) : messages.join(": "), cause }) +} /** * Compiles a plan to one SQL query (dialect `postgres` by default). Fails @@ -99,14 +113,6 @@ export const compile = ( } } -const frameOf = (columns: ReadonlyArray, rows: ReadonlyArray>): DataFrame => { - const data: Record> = {} - // Drivers return JS Dates for timestamps and dates. - const value = (v: unknown) => (v === undefined ? null : v instanceof Date ? DateTime.makeUnsafe(v) : v) - for (const name of columns) data[name] = rows.map((r) => value(r[name])) - return fromColumns(data as any) as DataFrame -} - /** * An engine that compiles queries to SQL and runs them with the * `SqlClient` in the context. @@ -127,8 +133,11 @@ export const engine = (options?: { readonly dialect?: Dialect | undefined }): Ef const collect = (plan: Plan.Node): Effect.Effect, DataFrameError> => Effect.gen(function*() { const query = yield* compiled(plan) - const rows = yield* sql.unsafe>(query.sql).pipe(Effect.mapError(toError)) - return yield* Effect.try({ try: () => frameOf(query.columns, rows), catch: toError }) + const rows = yield* sql.unsafe(query.sql).values.pipe(Effect.mapError(toError)) + return yield* Effect.try({ + try: () => internal.decode(query.schema, rows) as unknown as DataFrame, + catch: toError + }) }) return { name: `sql (${dialect})`, diff --git a/packages/effect/src/dataframe/internal/sql.ts b/packages/effect/src/dataframe/internal/sql.ts index cc5e51cf926..225a245d8c2 100644 --- a/packages/effect/src/dataframe/internal/sql.ts +++ b/packages/effect/src/dataframe/internal/sql.ts @@ -1,81 +1,51 @@ /** * Compiling plans to SQL. * - * Every relation the compiler builds carries a hidden `__row` column: a key + * Every relation the compiler builds carries a hidden row column: a key * that increases in row order. Operations keep it (filters, projections), * redefine it (sorts, joins, concatenations: `ROW_NUMBER()` over the new - * order) or derive it (groups: the first row's `__row`), and the final - * query orders by it. This reproduces the row order of the in-memory - * engine: stable sorts, groups in order of first appearance, joins in left - * row order, `first`/`last` by row order, windows over rows in order. + * order) or derive it (groups: the first row's key), and the final query + * orders by it. This reproduces the row order of the in-memory engine: + * stable sorts, groups in order of first appearance, joins in left row + * order, `first`/`last` by row order, windows over rows in order. + * + * Column types come from the in-memory engine: every relation carries the + * frame the engine gives for its plan on zero-row inputs, so the compiler + * knows the engine's type of every column and expression. The final query + * renders every column as text (temporal values as integers in their unit), + * and `decode` reads the rows back as columns of those types. + * + * Helper columns (the row key and the like) share one prefix, chosen per + * plan so that no column of the plan starts with it. * * @internal */ +import * as ArrowColumn from "../../arrow/ArrowColumn.ts" import * as S from "../../arrow/ArrowSchema.ts" +import * as BigDecimal from "../../BigDecimal.ts" +import * as Result from "../../Result.ts" import type * as Plan from "../Plan.ts" +import * as D from "./dtype.ts" +import * as E from "./expr.ts" import * as F from "./frame.ts" +import * as K from "./kernels.ts" +import * as Q from "./query.ts" import { civil } from "./temporal.ts" import * as V from "./vector.ts" /** @internal */ export type Dialect = "duckdb" | "postgres" -/** What the compiler knows about a column's type (enough to pick SQL that keeps the engine's semantics). */ -type Kind = "int" | "float" | "string" | "bool" | "date" | "timestamp" | "duration" | "decimal" | undefined - -interface Relation { - /** A SELECT whose columns are `columns` plus `__row`. */ - readonly sql: string - readonly columns: ReadonlyArray - readonly kinds: ReadonlyArray -} - -const kindOfType = (t: S.DataType): Kind => { - switch (t._tag) { - case "Int": - return "int" - case "Float": - return "float" - case "Utf8": - case "LargeUtf8": - case "Utf8View": - return "string" - case "Bool": - return "bool" - case "Date": - return "date" - case "Timestamp": - return "timestamp" - case "Duration": - return "duration" - case "Decimal": - return "decimal" - default: - return undefined - } +interface Compiler { + readonly dialect: Dialect + /** The engine's frame for every node of the plan, on zero-row inputs. */ + readonly frames: ReadonlyMap + /** A quoted helper column name, disjoint from the plan's column names. */ + readonly hidden: (name: string) => string + /** The quoted row key. */ + readonly row: string } -const kindOfName = (name: Plan.TypeName): Kind => - name.startsWith("Int") || name.startsWith("UInt") - ? "int" - : name.startsWith("Float") - ? "float" - : name === "Utf8" - ? "string" - : name === "Bool" - ? "bool" - : name === "Date" - ? "date" - : name === "Timestamp" - ? "timestamp" - : name === "Duration" - ? "duration" - : name === "Decimal" - ? "decimal" - : undefined - -const ROW = `"__row"` - /** @internal */ export const quote = (name: string): string => `"${name.replaceAll(`"`, `""`)}"` @@ -130,28 +100,23 @@ const sqlType = (t: S.DataType, dialect: Dialect): string => { } } -const castTypes: Record = { - Int8: S.int8, - Int16: S.int16, - Int32: S.int32, - Int64: S.int64, - UInt8: S.uint8, - UInt16: S.uint16, - UInt32: S.uint32, - UInt64: S.uint64, - Float32: S.float32, - Float64: S.float64, - Utf8: S.utf8, - Bool: S.bool, - Date: S.date("day"), - "Timestamp[s]": S.timestamp("second"), - "Timestamp[ms]": S.timestamp("millisecond"), - "Timestamp[us]": S.timestamp("microsecond"), - "Timestamp[ns]": S.timestamp("nanosecond"), - "Duration[s]": S.duration("second"), - "Duration[ms]": S.duration("millisecond"), - "Duration[us]": S.duration("microsecond"), - "Duration[ns]": S.duration("nanosecond") +/** The type the SQL engine reads a declared table column as. */ +const tableType = (table: string, column: Plan.TableColumn, dialect: Dialect): S.DataType => { + switch (column.type) { + case "Timestamp": + return S.timestamp("microsecond") + case "Duration": + return S.duration("microsecond") + case "Binary": + return S.binary + case "Null": + return S.nullType + default: + return D.castTargets[column.type] ?? unsupported( + `column ${JSON.stringify(column.name)} of table ${JSON.stringify(table)}: ${column.type} table columns`, + dialect + ) + } } const string = (s: string): string => `'${s.replaceAll("'", "''")}'` @@ -180,6 +145,8 @@ const timestampText = (value: bigint, unit: Plan.TimeUnit): string => { }.${pad(micros, 6)}` } +const NAN = "CAST('NaN' AS DOUBLE PRECISION)" + const literal = (value: unknown, type: Plan.LiteralType | undefined, dialect: Dialect): string => { if (value === null || value === undefined) return "NULL" if (type !== undefined) { @@ -190,7 +157,9 @@ const literal = (value: unknown, type: Plan.LiteralType | undefined, dialect: Di return `DATE '${pad(y, 4)}-${pad(m)}-${pad(d)}'` } case "Timestamp": - return `TIMESTAMP '${timestampText(big, type.unit)}'` + return type.timezone === undefined + ? `TIMESTAMP '${timestampText(big, type.unit)}'` + : `TIMESTAMPTZ '${timestampText(big, type.unit)}+00'` case "Duration": { const micros = big * BigInt(1_000_000) / perSecond[type.unit] return dialect === "duckdb" @@ -207,7 +176,7 @@ const literal = (value: unknown, type: Plan.LiteralType | undefined, dialect: Di } switch (typeof value) { case "number": - if (Number.isNaN(value)) return `CAST('NaN' AS DOUBLE PRECISION)` + if (Number.isNaN(value)) return NAN if (!Number.isFinite(value)) return `CAST('${value > 0 ? "" : "-"}Infinity' AS DOUBLE PRECISION)` if (Object.is(value, -0)) return `CAST('-0' AS DOUBLE PRECISION)` return Number.isInteger(value) ? String(value) : `CAST(${value} AS DOUBLE PRECISION)` @@ -227,169 +196,91 @@ const literal = (value: unknown, type: Plan.LiteralType | undefined, dialect: Di // ----------------------------------------------------------------------------- interface Scope { - readonly dialect: Dialect - /** The PARTITION BY of an enclosing `over`. */ - readonly partition: string | undefined - /** Kinds of the input's columns. */ - readonly kinds: ReadonlyMap + readonly compiler: Compiler + /** The input relation's frame (zero rows). */ + readonly frame: F.Frame + /** The PARTITION BY expressions of an enclosing `over`. */ + readonly partition: ReadonlyArray | undefined } -const castKinds: Record = { - Utf8: "string", - Bool: "bool", - Date: "date", - Decimal: "decimal" -} +/** The engine's type of an expression's result. */ +const typeOf = (node: Plan.Expression, scope: Scope): S.DataType => + E.evaluate( + scope.partition === undefined ? node : { _tag: "Over", expression: node, partitionBy: [] }, + F.context(scope.frame) + ).vector.type -/** The kind of an expression's result, when the compiler can tell. */ -const kindOf = (node: Plan.Expression, scope: Scope): Kind => { - switch (node._tag) { - case "Column": - return scope.kinds.get(node.name) - case "Literal": - if (node.type !== undefined) return node.type._tag.toLowerCase() as Kind - return typeof node.value === "number" - ? Number.isInteger(node.value) && !Object.is(node.value, -0) ? "int" : "float" - : typeof node.value === "bigint" - ? "int" - : typeof node.value === "string" - ? "string" - : typeof node.value === "boolean" - ? "bool" - : undefined - case "Over": - return kindOf(node.expression, scope) - case "AggregateCall": - return node.fn === "count" || node.fn === "countDistinct" - ? "int" - : node.fn === "mean" || node.fn === "median" || node.fn === "std" || node.fn === "variance" - ? "float" - : node.arg === undefined - ? undefined - : kindOf(node.arg, scope) - case "Call": { - const a = node.args.map((x) => kindOf(x, scope)) - switch (node.fn) { - case "add": - case "subtract": - case "multiply": - case "floorDivide": - case "mod": - if (a[0] === "timestamp" || a[1] === "timestamp") { - return node.fn === "subtract" && a[0] === a[1] - ? "duration" - : "timestamp" - } - if (a[0] === "date" && a[1] === "date") return "duration" - return a[0] === "int" && a[1] === "int" - ? "int" - : a[0] === "decimal" || a[1] === "decimal" - ? "decimal" - : a[0] && a[1] - ? "float" - : undefined - case "divide": - case "sqrt": - case "rollingMean": - return "float" - case "negate": - case "abs": - case "round": - case "cumSum": - case "cumMin": - case "cumMax": - case "shift": - case "diff": - case "rollingSum": - case "rollingMin": - case "rollingMax": - case "fillNull": - case "coalesce": - case "truncate": - return a[0] - case "when": - return a[1] ?? a[2] - case "cast": { - const to = node.options?.to ?? "" - return castKinds[to] ?? - (to.startsWith("Int") || to.startsWith("UInt") - ? "int" - : to.startsWith("Float") - ? "float" - : to.startsWith("Timestamp") - ? "timestamp" - : to.startsWith("Duration") - ? "duration" - : undefined) - } - case "length": - case "cumCount": - case "rank": - case "year": - case "month": - case "day": - case "hour": - case "minute": - case "second": - case "millisecond": - case "weekday": - case "ordinalDay": - case "totalDays": - case "totalHours": - case "totalMinutes": - case "totalSeconds": - case "totalMilliseconds": - case "listLength": - return "int" - case "toLowerCase": - case "toUpperCase": - case "concat": - return "string" - default: - return node.fn === "eq" || node.fn === "neq" || node.fn === "lt" || node.fn === "lte" || node.fn === "gt" || - node.fn === "gte" || node.fn === "and" || node.fn === "or" || node.fn === "not" || node.fn === "isNull" || - node.fn === "isNotNull" || node.fn === "isNaN" || node.fn === "isIn" || node.fn === "between" - ? "bool" - : undefined - } - } - } -} +const partitionBy = (scope: Scope, separator = ""): string => + scope.partition === undefined || scope.partition.length === 0 + ? "" + : `PARTITION BY ${scope.partition.join(", ")}${separator}` const isNaNSql = ( x: string, d: Dialect -) => (d === "duckdb" ? `ISNAN(${x})` : `(${x} = CAST('NaN' AS DOUBLE PRECISION))`) +) => (d === "duckdb" ? `ISNAN(${x})` : `(${x} = ${NAN})`) const window = (scope: Scope, frame?: string): string => - `OVER (${scope.partition === undefined ? "" : `PARTITION BY ${scope.partition} `}ORDER BY ${ROW}${ - frame === undefined ? "" : ` ${frame}` - })` + `OVER (${partitionBy(scope, " ")}ORDER BY ${scope.compiler.row}${frame === undefined ? "" : ` ${frame}`})` const running = "ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW" const float = (x: string) => `CAST(${x} AS DOUBLE PRECISION)` +/** A power of ten as an exact SQL literal. */ +const power = (k: number) => k >= 0 ? `1${"0".repeat(k)}` : `0.${"0".repeat(-k - 1)}1` + +/** + * `round` as the engine does it: floats as `rint(x * 10^d) / 10^d` in double + * precision, decimals half to even at `d` places into the engine's result + * type, integers unchanged. + */ +const round = (x: string, type: S.DataType, result: S.DataType, decimals: number, d: Dialect): string => { + switch (type._tag) { + case "Int": + case "Null": + return x + case "Decimal": { + if (decimals >= type.scale) return x + const scaled = decimals === 0 ? x : `(${x} * ${power(decimals)})` + const floor = `FLOOR(${scaled})` + const up = `(${scaled} - ${floor} > 0.5 OR (${scaled} - ${floor} = 0.5 AND MOD(${floor}, 2) <> 0))` + const rounded = `(CASE WHEN ${up} THEN ${floor} + 1 ELSE ${floor} END)` + return `CAST(${decimals === 0 ? rounded : `(${rounded} / ${power(decimals)})`} AS ${sqlType(result, d)})` + } + default: { + const rint = (y: string) => d === "duckdb" ? `ROUND_EVEN(${y}, 0)` : `ROUND(${y})` + if (decimals === 0) return rint(float(x)) + const scale = literal(10 ** decimals, undefined, d) + return `(${rint(`(${float(x)} * ${float(scale)})`)} / ${float(scale)})` + } + } +} + const aggregate = (node: Plan.AggregateCall, scope: Scope, over: string | undefined): string => { - const d = scope.dialect - const x = node.arg === undefined ? "" : expression(node.arg, { ...scope, partition: undefined }) - const kind = node.arg === undefined ? undefined : kindOf(node.arg, scope) + const d = scope.compiler.dialect + const row = scope.compiler.row + const inner = { ...scope, partition: undefined } + const x = node.arg === undefined ? "" : expression(node.arg, inner) + const type = node.arg === undefined ? undefined : typeOf(node.arg, inner)._tag const o = over === undefined ? "" : ` ${over}` switch (node.fn) { case "count": return node.arg === undefined ? `COUNT(*)${o}` : `COUNT(${x})${o}` case "sum": - if (kind === "duration" && d === "duckdb") { - return `TO_MICROSECONDS(CAST(COALESCE(SUM(EXTRACT(EPOCH FROM ${x}) * 1000000)${o}, 0) AS BIGINT))` + if (type === "Duration") { + return d === "duckdb" + ? `TO_MICROSECONDS(CAST(COALESCE(SUM(EXTRACT(EPOCH FROM ${x}) * 1000000)${o}, 0) AS BIGINT))` + : `COALESCE(SUM(${x})${o}, INTERVAL '0 seconds')` } - return `COALESCE(SUM(${x})${o}, 0)` + return `COALESCE(SUM(${type === "Bool" ? `CAST(${x} AS INTEGER)` : x})${o}, 0)` case "mean": return `AVG(${x})${o}` case "min": case "max": { const fn = node.fn.toUpperCase() // Floats: NaN is skipped unless every value is NaN. - return kind === "float" + return type === "Float" ? `COALESCE(${fn}(CASE WHEN ${isNaNSql(x, d)} THEN NULL ELSE ${x} END)${o}, ${fn}(${x})${o})` : `${fn}(${x})${o}` } @@ -402,21 +293,22 @@ const aggregate = (node: Plan.AggregateCall, scope: Scope, over: string | undefi return d === "duckdb" ? `MEDIAN(${x})` : `PERCENTILE_CONT(0.5) WITHIN GROUP (ORDER BY ${x})` case "countDistinct": if (over !== undefined) return unsupported("countDistinct over partitions", d) - return `(COUNT(DISTINCT ${x}) + MAX(CASE WHEN ${x} IS NULL THEN 1 ELSE 0 END))` + // Nulls count as one value. + return `(COUNT(DISTINCT ${x}) + CASE WHEN COUNT(*) > COUNT(${x}) THEN 1 ELSE 0 END)` case "first": case "last": { if (over !== undefined) { return `${node.fn === "first" ? "FIRST_VALUE" : "LAST_VALUE"}(${x}) OVER (${ - scope.partition === undefined ? "" : `PARTITION BY ${scope.partition} ` - }ORDER BY ${ROW} ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING)` + partitionBy(scope, " ") + }ORDER BY ${row} ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING)` } return d === "duckdb" - ? `${node.fn.toUpperCase()}(${x} ORDER BY ${ROW})` - : `(ARRAY_AGG(${x} ORDER BY ${ROW} ${node.fn === "first" ? "ASC" : "DESC"}))[1]` + ? `${node.fn.toUpperCase()}(${x} ORDER BY ${row})` + : `(ARRAY_AGG(${x} ORDER BY ${row} ${node.fn === "first" ? "ASC" : "DESC"}))[1]` } case "list": if (over !== undefined) return unsupported("list over partitions", d) - return d === "duckdb" ? `LIST(${x} ORDER BY ${ROW})` : `ARRAY_AGG(${x} ORDER BY ${ROW})` + return d === "duckdb" ? `LIST(${x} ORDER BY ${row})` : `ARRAY_AGG(${x} ORDER BY ${row})` } } @@ -430,10 +322,11 @@ const totalSeconds: Record = { } const call = (node: Plan.Call, scope: Scope): string => { - const d = scope.dialect + const d = scope.compiler.dialect + const row = scope.compiler.row const a = node.args.map((x) => expression(x, scope)) - const k = node.args.map((x) => kindOf(x, scope)) - const ints = k[0] === "int" && k[1] === "int" + const k = node.args.map((x) => typeOf(x, scope)._tag) + const ints = k[0] === "Int" && k[1] === "Int" const o = node.options ?? {} // Floored modulo of integers, exact, null for a zero divisor. const intMod = () => `((${a[0]} % NULLIF(${a[1]}, 0)) + ${a[1]}) % NULLIF(${a[1]}, 0)` @@ -441,7 +334,7 @@ const call = (node: Plan.Call, scope: Scope): string => { case "add": return `(${a[0]} + ${a[1]})` case "subtract": - if (k[0] === "date" && k[1] === "date") return `((${a[0]} - ${a[1]}) * INTERVAL '1 day')` + if (k[0] === "Date" && k[1] === "Date") return `((${a[0]} - ${a[1]}) * INTERVAL '1 day')` return `(${a[0]} - ${a[1]})` case "multiply": return `(${a[0]} * ${a[1]})` @@ -458,9 +351,9 @@ const call = (node: Plan.Call, scope: Scope): string => { case "abs": return `ABS(${a[0]})` case "sqrt": - return `SQRT(${a[0]})` + return `(CASE WHEN ${float(a[0])} < 0 THEN ${NAN} ELSE SQRT(${float(a[0])}) END)` case "round": - return d === "duckdb" ? `ROUND_EVEN(${a[0]}, ${o.decimals ?? 0})` : `ROUND(${a[0]}, ${o.decimals ?? 0})` + return round(a[0], typeOf(node.args[0], scope), typeOf(node, scope), o.decimals ?? 0, d) case "eq": return `(${a[0]} = ${a[1]})` case "neq": @@ -492,7 +385,7 @@ const call = (node: Plan.Call, scope: Scope): string => { case "isNotNull": return `(${a[0]} IS NOT NULL)` case "isNaN": - return d === "duckdb" ? `ISNAN(${a[0]})` : `(${a[0]} = CAST('NaN' AS DOUBLE PRECISION))` + return isNaNSql(a[0], d) case "fillNull": case "coalesce": return `COALESCE(${a.join(", ")})` @@ -502,17 +395,17 @@ const call = (node: Plan.Call, scope: Scope): string => { const to = o.to ?? "" const target = to === "Decimal" ? `DECIMAL(${o.precision}, ${o.scale})` - : sqlType(castTypes[to] ?? V.fail(`unknown cast target ${to}`), d) + : sqlType(D.castTargets[to] ?? V.fail(`unknown cast target ${to}`), d) const integer = to.startsWith("Int") || to.startsWith("UInt") let x = a[0] - // Temporal values become their integers; decimals truncate (as the engine does). - if (integer && k[0] === "timestamp") { + // Temporal values become their integers; floats and decimals truncate (as the engine does). + if (integer && k[0] === "Timestamp") { x = d === "duckdb" ? `EPOCH_US(${x})` : `CAST(EXTRACT(EPOCH FROM ${x}) * 1000000 AS BIGINT)` - } else if (integer && k[0] === "date") x = `(${x} - DATE '1970-01-01')` - else if (integer && k[0] === "duration") x = `CAST(EXTRACT(EPOCH FROM ${x}) * 1000000 AS BIGINT)` - else if (integer && k[0] === "decimal") x = `TRUNC(${x})` + } else if (integer && k[0] === "Date") x = `(${x} - DATE '1970-01-01')` + else if (integer && k[0] === "Duration") x = `CAST(EXTRACT(EPOCH FROM ${x}) * 1000000 AS BIGINT)` + else if (integer && (k[0] === "Decimal" || k[0] === "Float")) x = `TRUNC(${x})` return d === "duckdb" ? `TRY_CAST(${x} AS ${target})` : `CAST(${x} AS ${target})` } case "length": @@ -530,12 +423,14 @@ const call = (node: Plan.Call, scope: Scope): string => { case "concat": return `(${a.join(" || ")})` case "cumSum": - return `(CASE WHEN ${a[0]} IS NULL THEN NULL ELSE SUM(${a[0]}) ${window(scope, running)} END)` + return `(CASE WHEN ${a[0]} IS NULL THEN NULL ELSE SUM(${k[0] === "Bool" ? `CAST(${a[0]} AS INTEGER)` : a[0]}) ${ + window(scope, running) + } END)` case "cumMin": case "cumMax": { const fn = node.fn === "cumMin" ? "MIN" : "MAX" // Floats: NaN is skipped (it stays only while no other value was seen). - const value = k[0] === "float" + const value = k[0] === "Float" ? `COALESCE(${fn}(CASE WHEN ${isNaNSql(a[0], d)} THEN NULL ELSE ${a[0]} END) ${window(scope, running)}, ${ a[0] })` @@ -552,8 +447,8 @@ const call = (node: Plan.Call, scope: Scope): string => { return `(${a[0]} - LAG(${a[0]}, ${o.n ?? 1}) ${window(scope)})` case "rank": { const order = `${a[0]} ${o.descending ? "DESC" : "ASC"} NULLS LAST` - const part = scope.partition === undefined ? "" : `PARTITION BY ${scope.partition} ` - const ties = `COUNT(*) OVER (PARTITION BY ${scope.partition === undefined ? "" : `${scope.partition}, `}${a[0]})` + const part = partitionBy(scope, " ") + const ties = `COUNT(*) OVER (PARTITION BY ${[...(scope.partition ?? []), a[0]].join(", ")})` const ranked = (() => { switch (o.method ?? "average") { case "min": @@ -563,7 +458,7 @@ const call = (node: Plan.Call, scope: Scope): string => { case "dense": return `DENSE_RANK() OVER (${part}ORDER BY ${order})` case "ordinal": - return `ROW_NUMBER() OVER (${part}ORDER BY ${order}, ${ROW})` + return `ROW_NUMBER() OVER (${part}ORDER BY ${order}, ${row})` case "average": return `(RANK() OVER (${part}ORDER BY ${order}) + (${ties} - 1) / 2.0)` } @@ -578,10 +473,10 @@ const call = (node: Plan.Call, scope: Scope): string => { const fn = { rollingSum: "SUM", rollingMean: "AVG", rollingMin: "MIN", rollingMax: "MAX" }[node.fn] const enough = `COUNT(${a[0]}) ${window(scope, frame)} >= ${o.minPeriods ?? o.window ?? 1}` // Floats: a NaN in the window makes the minimum or maximum NaN. - const value = k[0] === "float" && (fn === "MIN" || fn === "MAX") + const value = k[0] === "Float" && (fn === "MIN" || fn === "MAX") ? `CASE WHEN MAX(CASE WHEN ${isNaNSql(a[0], d)} THEN 1 ELSE 0 END) ${ window(scope, frame) - } = 1 THEN CAST('NaN' AS DOUBLE PRECISION) ELSE ${fn}(${a[0]}) ${window(scope, frame)} END` + } = 1 THEN ${NAN} ELSE ${fn}(${a[0]}) ${window(scope, frame)} END` : `${fn}(${a[0]}) ${window(scope, frame)}` return `(CASE WHEN ${enough} THEN ${value} END)` } @@ -601,7 +496,7 @@ const call = (node: Plan.Call, scope: Scope): string => { return `CAST(EXTRACT(DOY FROM ${a[0]}) AS INTEGER)` case "truncate": { const unit = truncateUnits[o.every ?? ""] ?? unsupported(`truncate to ${o.every}`, d) - return k[0] === "date" ? `CAST(DATE_TRUNC('${unit}', ${a[0]}) AS DATE)` : `DATE_TRUNC('${unit}', ${a[0]})` + return k[0] === "Date" ? `CAST(DATE_TRUNC('${unit}', ${a[0]}) AS DATE)` : `DATE_TRUNC('${unit}', ${a[0]})` } case "totalDays": case "totalHours": @@ -648,283 +543,354 @@ const expression = (node: Plan.Expression, scope: Scope): string => { case "Column": return quote(node.name) case "Literal": - return literal(node.value, node.type, scope.dialect) + return literal(node.value, node.type, scope.compiler.dialect) case "Call": return call(node, scope) case "AggregateCall": return scope.partition === undefined ? V.fail(`aggregate ${node.fn} can only be used in agg or over`) - : aggregate(node, scope, `OVER (PARTITION BY ${scope.partition})`) - case "Over": { - const partition = node.partitionBy.map((k) => expression(k, { ...scope, partition: undefined })).join(", ") - return expression(node.expression, { ...scope, partition: partition === "" ? undefined : partition }) - } + : aggregate(node, scope, `OVER (${partitionBy(scope)})`) + case "Over": + return expression(node.expression, { + ...scope, + partition: node.partitionBy.map((k) => expression(k, { ...scope, partition: undefined })) + }) } } +// ----------------------------------------------------------------------------- +// Types of relations +// ----------------------------------------------------------------------------- + +const unsupportedNodes: ReadonlySet = new Set(["Pivot", "Unpivot", "Explode", "Unnest"]) + +/** + * Runs the plan through the in-memory engine on zero-row inputs (tables as + * their declared columns), recording every node's frame. + */ +const typed = (node: Plan.Node, frames: Map, dialect: Dialect): F.Frame => { + const known = frames.get(node) + if (known !== undefined) return known + let frame: F.Frame + if (unsupportedNodes.has(node._tag)) return unsupported(node._tag, dialect) + if (node._tag === "Table") { + const columns = node.columns ?? V.fail( + `the SQL engine needs the columns of table ${JSON.stringify(node.name)}: use Query.table with a schema` + ) + frame = F.fromVectors( + columns.map((c) => c.name), + columns.map((c) => K.nullsOf(tableType(node.name, c, dialect), 0)), + 0 + ) + } else if (node._tag === "Values") { + frame = F.slice(node.frame as unknown as F.Frame, 0, 0) + } else { + const inputs = Q.children(node as Q.Node).map((n) => typed(n as Plan.Node, frames, dialect)) + frame = Q.step(node as Q.Node, inputs) + } + frames.set(node, frame) + return frame +} + // ----------------------------------------------------------------------------- // Relations // ----------------------------------------------------------------------------- -const from = (r: Relation, alias = "t") => `(${r.sql}) AS ${alias}` - -const rowOrdered = ( - columns: ReadonlyArray, - kinds: ReadonlyArray, - input: Relation, - order: string -): Relation => ({ - sql: `SELECT ${list(columns)}${columns.length > 0 ? ", " : ""}ROW_NUMBER() OVER (ORDER BY ${order}) AS ${ROW} FROM ${ - from(input) - }`, - columns, - kinds +const columnsOf = (node: Plan.Node, c: Compiler): ReadonlyArray => + c.frames.get(node)!.columns.map((col) => col.name) + +const scopeOf = (node: Plan.Node, compiler: Compiler): Scope => ({ + compiler, + frame: compiler.frames.get(node)!, + partition: undefined }) -const values = (frame: F.Frame, dialect: Dialect): Relation => { - const columns = frame.columns.map((c) => c.name) - const kinds = frame.columns.map((c) => kindOfType(c.type)) - const types = frame.columns.map((c) => sqlType(c.type, dialect)) - const rows = F.toRows(frame) - const cell = (v: unknown, i: number): string => { - const t = frame.columns[i].type - if (v === null || v === undefined) return `CAST(NULL AS ${types[i]})` - if (t._tag === "Timestamp" || t._tag === "Date" || t._tag === "Duration" || t._tag === "Decimal") { - return `CAST(${string(String(v instanceof Object && "toJSON" in v ? (v as any).toJSON() : v))} AS ${types[i]})` - } - return `CAST(${literal(v, undefined, dialect)} AS ${types[i]})` - } - if (rows.length === 0) { - return { - sql: `SELECT ${columns.map((c, i) => `CAST(NULL AS ${types[i]}) AS ${quote(c)}`).join(", ")}${ - columns.length > 0 ? ", " : "" - }CAST(0 AS BIGINT) AS ${ROW} WHERE FALSE`, - columns, - kinds +const values = (frame: F.Frame, c: Compiler): string => { + const d = c.dialect + const columns = frame.columns.map((col) => { + const type = col.type + const vector = col.values() + const raw = (vector as V.I32 | V.I64).values + const values = type._tag === "Date" || type._tag === "Timestamp" || type._tag === "Duration" || + type._tag === "Decimal" + ? undefined + : F.toArray(frame, col.name) + // Temporal and decimal values are written from their integers, at full precision. + const value = (i: number): string => { + switch (type._tag) { + case "Date": + return literal(type.unit === "day" ? raw[i] : floorDiv(raw[i] as bigint, BigInt(86_400_000)), { + _tag: "Date" + }, d) + case "Timestamp": + return literal(raw[i], { _tag: "Timestamp", unit: type.unit, timezone: type.timezone }, d) + case "Duration": + return literal(raw[i], { _tag: "Duration", unit: type.unit }, d) + case "Decimal": + return literal(raw[i], { _tag: "Decimal", precision: type.precision, scale: type.scale }, d) + default: { + const v = values![i] + return typeof v === "bigint" ? String(v) : literal(v, undefined, d) + } + } } + const sql = sqlType(type, d) + return { name: quote(col.name), cell: (i: number) => `CAST(${V.isValid(vector, i) ? value(i) : "NULL"} AS ${sql})` } + }) + if (frame.length === 0) { + const nulls = frame.columns.map((col, i) => `CAST(NULL AS ${sqlType(col.type, d)}) AS ${columns[i].name}`) + return `SELECT ${[...nulls, `CAST(0 AS BIGINT) AS ${c.row}`].join(", ")} WHERE FALSE` } - const tuples = rows.map((row, r) => `(${[...columns.map((c, i) => cell(row[c], i)), String(r + 1)].join(", ")})`) - return { - sql: `SELECT * FROM (VALUES ${tuples.join(", ")}) AS v(${list([...columns, "__row"])})`, - columns, - kinds - } + const tuples = Array.from( + { length: frame.length }, + (_, r) => `(${[...columns.map((col) => col.cell(r)), String(r + 1)].join(", ")})` + ) + return `SELECT * FROM (VALUES ${tuples.join(", ")}) AS v(${[...columns.map((col) => col.name), c.row].join(", ")})` } const named = (items: ReadonlyArray, scope: Scope) => items.map((c) => `${expression(c.expression, scope)} AS ${quote(c.name)}`) -const same = (r: Relation, sql: string): Relation => ({ sql, columns: r.columns, kinds: r.kinds }) - -const pick = (r: Relation, columns: ReadonlyArray): ReadonlyArray => - columns.map((c) => r.kinds[r.columns.indexOf(c)]) - -const scopeOf = (r: Relation, dialect: Dialect): Scope => ({ - dialect, - partition: undefined, - kinds: new Map(r.columns.map((c, i) => [c, r.kinds[i]])) -}) - -/** @internal */ -export const relation = (node: Plan.Node, dialect: Dialect): Relation => { - const input = () => relation((node as { readonly input: Plan.Node }).input, dialect) +/** A SELECT of the node's columns plus the row key. */ +const relation = (node: Plan.Node, c: Compiler): string => { + const row = c.row + const columns = columnsOf(node, c) + const keep = [...columns.map(quote), row].join(", ") + const from = (alias = "t") => `(${relation((node as { readonly input: Plan.Node }).input, c)}) AS ${alias}` + const scope = () => scopeOf((node as { readonly input: Plan.Node }).input, c) + /** The rows of `input` renumbered in `order`. */ + const renumbered = (input: string, order: string) => + `SELECT ${ + [...columns.map(quote), `ROW_NUMBER() OVER (ORDER BY ${order}) AS ${row}`].join(", ") + } FROM (${input}) AS t` switch (node._tag) { - case "Table": { - if (node.columns === undefined) { - return V.fail( - `the SQL engine needs the columns of table ${JSON.stringify(node.name)}: use Query.table with a schema` - ) - } - const columns = node.columns.map((c) => c.name) - return { - sql: `SELECT ${list(columns)}, ROW_NUMBER() OVER () AS ${ROW} FROM ${quote(node.name)}`, - columns, - kinds: node.columns.map((c) => kindOfName(c.type)) - } - } + case "Table": + return `SELECT ${[...columns.map(quote), `ROW_NUMBER() OVER () AS ${row}`].join(", ")} FROM ${quote(node.name)}` case "Values": - return values(node.frame as unknown as F.Frame, dialect) + return values(node.frame as unknown as F.Frame, c) case "Filter": { - const r = input() - return same(r, `SELECT * FROM ${from(r)} WHERE ${expression(node.predicate, scopeOf(r, dialect))}`) - } - case "Select": { - const r = input() - const scope = scopeOf(r, dialect) - return { - sql: `SELECT ${[...named(node.columns, scope), ROW].join(", ")} FROM ${from(r)}`, - columns: node.columns.map((c) => c.name), - kinds: node.columns.map((c) => kindOf(c.expression, scope)) - } + // The predicate is computed over the whole input first: it may hold window functions. + const flag = c.hidden("keep") + return `SELECT ${keep} FROM (SELECT *, ${ + expression(node.predicate, scope()) + } AS ${flag} FROM ${from()}) AS t WHERE ${flag}` } + case "Select": + return `SELECT ${[...named(node.columns, scope()), row].join(", ")} FROM ${from()}` case "WithColumns": { - const r = input() - const scope = scopeOf(r, dialect) - const added = new Map(node.columns.map((c) => [c.name, c.expression])) - const kept = r.columns.map((c) => { - const e = added.get(c) - return e === undefined ? quote(c) : `${expression(e, scope)} AS ${quote(c)}` + const s = scope() + const added = new Map(node.columns.map((col) => [col.name, col.expression])) + const inputs = columnsOf(node.input, c) + const kept = inputs.map((name) => { + const e = added.get(name) + return e === undefined ? quote(name) : `${expression(e, s)} AS ${quote(name)}` }) - const extra = node.columns.filter((c) => !r.columns.includes(c.name)) - return { - sql: `SELECT ${[...kept, ...named(extra, scope), ROW].join(", ")} FROM ${from(r)}`, - columns: [...r.columns, ...extra.map((c) => c.name)], - kinds: [ - ...r.columns.map((c, i) => { - const e = added.get(c) - return e === undefined ? r.kinds[i] : kindOf(e, scope) - }), - ...extra.map((c) => kindOf(c.expression, scope)) - ] - } - } - case "Pick": { - const r = input() - return { - sql: `SELECT ${list([...node.columns, "__row"])} FROM ${from(r)}`, - columns: node.columns, - kinds: pick(r, node.columns) - } - } - case "Omit": { - const r = input() - const columns = r.columns.filter((c) => !node.columns.includes(c)) - return { sql: `SELECT ${list([...columns, "__row"])} FROM ${from(r)}`, columns, kinds: pick(r, columns) } - } - case "Rename": { - const r = input() - const columns = r.columns.map((c) => node.mapping[c] ?? c) - return { - sql: `SELECT ${[...r.columns.map((c, i) => `${quote(c)} AS ${quote(columns[i])}`), ROW].join(", ")} FROM ${ - from(r) - }`, - columns, - kinds: r.kinds - } + const extra = node.columns.filter((col) => !inputs.includes(col.name)) + return `SELECT ${[...kept, ...named(extra, s), row].join(", ")} FROM ${from()}` } + case "Pick": + case "Omit": + return `SELECT ${keep} FROM ${from()}` + case "Rename": + return `SELECT ${ + [...columnsOf(node.input, c).map((name) => `${quote(name)} AS ${quote(node.mapping[name] ?? name)}`), row].join( + ", " + ) + } FROM ${from()}` case "Sort": { - const r = input() - const scope = scopeOf(r, dialect) + const s = scope() const keys = node.keys.map((k) => - `${expression(k.expression, scope)} ${k.descending ? "DESC" : "ASC"} NULLS ${node.nullsLast ? "LAST" : "FIRST"}` + `${expression(k.expression, s)} ${k.descending ? "DESC" : "ASC"} NULLS ${node.nullsLast ? "LAST" : "FIRST"}` ) - return rowOrdered(r.columns, r.kinds, r, [...keys, ROW].join(", ")) + return renumbered(`SELECT * FROM ${from("s")}`, [...keys, row].join(", ")) } case "Slice": { - const r = input() if (node.offset >= 0) { - return same( - r, - `SELECT * FROM ${from(r)} ORDER BY ${ROW}${ - node.length === undefined ? "" : ` LIMIT ${node.length}` - } OFFSET ${node.offset}` - ) + return `SELECT * FROM ${from()} ORDER BY ${row}${ + node.length === undefined ? "" : ` LIMIT ${node.length}` + } OFFSET ${node.offset}` } - const upper = node.length === undefined ? "" : ` AND "__i" <= "__n" + ${node.offset} + ${node.length}` - return same( - r, - `SELECT ${ - list([...r.columns, "__row"]) - } FROM (SELECT *, COUNT(*) OVER () AS "__n", ROW_NUMBER() OVER (ORDER BY ${ROW}) AS "__i" FROM ${ - from(r, "s") - }) AS t WHERE "__i" > "__n" + ${node.offset}${upper}` - ) + const n = c.hidden("n") + const i = c.hidden("i") + const upper = node.length === undefined ? "" : ` AND ${i} <= ${n} + ${node.offset} + ${node.length}` + return `SELECT ${keep} FROM (SELECT *, COUNT(*) OVER () AS ${n}, ROW_NUMBER() OVER (ORDER BY ${row}) AS ${i} FROM ${ + from("s") + }) AS t WHERE ${i} > ${n} + ${node.offset}${upper}` } case "Unique": { - const r = input() - const keys = node.columns ?? r.columns - return same( - r, - `SELECT ${list([...r.columns, "__row"])} FROM (SELECT *, ROW_NUMBER() OVER (PARTITION BY ${ - list(keys) - } ORDER BY ${ROW}) AS "__k" FROM ${from(r, "s")}) AS t WHERE "__k" = 1` - ) + const k = c.hidden("k") + return `SELECT ${keep} FROM (SELECT *, ROW_NUMBER() OVER (PARTITION BY ${ + list(node.columns ?? columns) + } ORDER BY ${row}) AS ${k} FROM ${from("s")}) AS t WHERE ${k} = 1` } case "DropNulls": { - const r = input() - const keys = node.columns ?? r.columns - return same( - r, - `SELECT * FROM ${from(r)}${ - keys.length === 0 ? "" : ` WHERE ${keys.map((k) => `${quote(k)} IS NOT NULL`).join(" AND ")}` - }` - ) + const keys = node.columns ?? columns + return `SELECT * FROM ${from()}${ + keys.length === 0 ? "" : ` WHERE ${keys.map((k) => `${quote(k)} IS NOT NULL`).join(" AND ")}` + }` } case "GroupBy": case "Aggregate": { - const r = input() - const scope = scopeOf(r, dialect) + const s = scope() const keys = node._tag === "GroupBy" ? node.keys : [] const aggs = node.aggregates.map((a) => - a.expression._tag === "AggregateCall" - ? `${aggregate(a.expression, scope, undefined)} AS ${quote(a.name)}` - : V.fail(`${a.name} is not an aggregate expression`) + `${aggregate(a.expression as Plan.AggregateCall, s, undefined)} AS ${quote(a.name)}` ) - const row = keys.length === 0 ? `1 AS ${ROW}` : `MIN(${ROW}) AS ${ROW}` - return { - sql: `SELECT ${[...keys.map(quote), ...aggs, row].join(", ")} FROM ${from(r)}${ - keys.length === 0 ? "" : ` GROUP BY ${list(keys)}` - }`, - columns: [...keys, ...node.aggregates.map((a) => a.name)], - kinds: [...pick(r, keys), ...node.aggregates.map((a) => kindOf(a.expression, scope))] - } + const key = keys.length === 0 ? `1 AS ${row}` : `MIN(${row}) AS ${row}` + return `SELECT ${[...keys.map(quote), ...aggs, key].join(", ")} FROM ${from()}${ + keys.length === 0 ? "" : ` GROUP BY ${list(keys)}` + }` } case "Join": { - const l = relation(node.left, dialect) - const rr = relation(node.right, dialect) + const l = relation(node.left, c) + const r = relation(node.right, c) const on = node.on.map((k) => `l.${quote(k)} = r.${quote(k)}`).join(" AND ") if (node.how === "semi" || node.how === "anti") { - return same( - l, - `SELECT * FROM (${l.sql}) AS l WHERE ${ - node.how === "anti" ? "NOT " : "" - }EXISTS (SELECT 1 FROM (${rr.sql}) AS r WHERE ${on})` - ) + return `SELECT * FROM (${l}) AS l WHERE ${ + node.how === "anti" ? "NOT " : "" + }EXISTS (SELECT 1 FROM (${r}) AS r WHERE ${on})` } - const rightOnly = rr.columns.filter((c) => !node.on.includes(c)) - const clash = rightOnly.find((c) => l.columns.includes(c)) - if (clash !== undefined) return V.fail(`both sides of the join have a column ${JSON.stringify(clash)}`) - const left = l.columns.map((c) => - node.how === "full" && node.on.includes(c) - ? `COALESCE(l.${quote(c)}, r.${quote(c)}) AS ${quote(c)}` - : `l.${quote(c)}` + const rightOnly = columnsOf(node.right, c).filter((name) => !node.on.includes(name)) + const left = columnsOf(node.left, c).map((name) => + node.how === "full" && node.on.includes(name) + ? `COALESCE(l.${quote(name)}, r.${quote(name)}) AS ${quote(name)}` + : `l.${quote(name)}` ) const join = node.how === "inner" ? "INNER JOIN" : node.how === "left" ? "LEFT JOIN" : "FULL JOIN" - return { - sql: `SELECT ${ - [...left, ...rightOnly.map((c) => `r.${quote(c)}`)].join(", ") - }, ROW_NUMBER() OVER (ORDER BY l.${ROW}${ - node.how === "full" ? " NULLS LAST" : "" - }, r.${ROW}) AS ${ROW} FROM (${l.sql}) AS l ${join} (${rr.sql}) AS r ON ${on}`, - columns: [...l.columns, ...rightOnly], - kinds: [...l.kinds, ...pick(rr, rightOnly)] - } + return `SELECT ${ + [...left, ...rightOnly.map((name) => `r.${quote(name)}`)].join(", ") + }, ROW_NUMBER() OVER (ORDER BY l.${row}${ + node.how === "full" ? " NULLS LAST" : "" + }, r.${row}) AS ${row} FROM (${l}) AS l ${join} (${r}) AS r ON ${on}` } case "Concat": { - const inputs = node.inputs.map((n) => relation(n, dialect)) - if (inputs.length === 0) return V.fail("concat needs at least one input") - const columns = inputs[0].columns - const union = inputs - .map((r, i) => `SELECT ${list([...columns, "__row"])}, ${i} AS "__source" FROM (${r.sql}) AS t${i}`) + const source = c.hidden("source") + const union = node.inputs + .map((n, i) => `SELECT ${keep}, ${i} AS ${source} FROM (${relation(n, c)}) AS t${i}`) .join(" UNION ALL ") - return rowOrdered(columns, inputs[0].kinds, { sql: union, columns, kinds: inputs[0].kinds }, `"__source", ${ROW}`) + return renumbered(union, `${source}, ${row}`) } case "Pivot": case "Unpivot": case "Explode": case "Unnest": - return unsupported(node._tag, dialect) + return unsupported(node._tag, c.dialect) + } +} + +// ----------------------------------------------------------------------------- +// Results +// ----------------------------------------------------------------------------- + +/** A temporal value as an integer in `unit` since the epoch (or as a length). */ +const epoch = (x: string, type: S.Timestamp | S.Duration, d: Dialect): string => { + if (d === "duckdb" && type._tag === "Timestamp") { + return type.unit === "second" ? + `CAST(FLOOR(EPOCH_MS(${x}) / 1000.0) AS BIGINT)` : + `EPOCH_${({ millisecond: "MS", microsecond: "US", nanosecond: "NS" } as const)[type.unit]}(${x})` + } + return `CAST(FLOOR(EXTRACT(EPOCH FROM ${x}) * ${perSecond[type.unit]}) AS BIGINT)` +} + +const scalars: ReadonlySet = new Set(["Null", "Bool", "Int", "Float", "Decimal", "Utf8"]) + +/** SQL that renders a result column of `type` in the form `reader` reads. */ +const output = (x: string, type: S.DataType, d: Dialect): string => { + const text = (y: string) => `CAST(${y} AS ${d === "duckdb" ? "VARCHAR" : "TEXT"})` + switch (type._tag) { + case "Null": + return "NULL" + case "Bool": + case "Int": + case "Float": + case "Decimal": + case "Utf8": + return text(`CAST(${x} AS ${sqlType(type, d)})`) + case "Date": + return text(`(CAST(${x} AS DATE) - DATE '1970-01-01')${type.unit === "day" ? "" : " * 86400000"}`) + case "Timestamp": + case "Duration": + return text(epoch(x, type, d)) + case "Binary": + return x + case "List": + if (scalars.has(type.child.type._tag)) return `CAST(${x} AS ${d === "duckdb" ? "VARCHAR" : "TEXT"}[])` + } + return unsupported(`results of type ${D.describe(type)}`, d) +} + +const number = (s: string): number => { + const n = Number(s) + if (!Number.isNaN(n)) return n + const lower = s.toLowerCase() + return lower.endsWith("inf") ? (lower.startsWith("-") ? -Infinity : Infinity) : NaN +} + +/** Reads one non-null value of a result column of `type`. */ +const reader = (type: S.DataType): (value: unknown) => unknown => { + switch (type._tag) { + case "Int": + return type.bitWidth === 64 ? (v) => BigInt(String(v)) : (v) => Number(v) + case "Float": + return (v) => number(String(v)) + case "Bool": + return (v) => v === true || v === "true" || v === "t" + case "Decimal": + return (v) => BigDecimal.fromStringUnsafe(String(v)) + case "Date": + case "Timestamp": + case "Duration": + return (v) => BigInt(String(v)) + case "List": { + const item = reader(type.child.type) + return (v) => (v as ReadonlyArray).map((x) => (x === null || x === undefined ? null : item(x))) + } + default: + return (v) => v } } +/** @internal */ +export interface Compiled { + readonly sql: string + readonly schema: S.Schema +} + /** A plan as one SQL query, its rows in the plan's row order. @internal */ -export const compile = ( - plan: Plan.Node, - dialect: Dialect -): { readonly sql: string; readonly columns: ReadonlyArray } => { - const r = relation(plan, dialect) +export const compile = (plan: Plan.Node, dialect: Dialect): Compiled => { + const frames = new Map() + const result = typed(plan, frames, dialect) + const used = new Set() + for (const frame of frames.values()) for (const col of frame.columns) used.add(col.name) + let prefix = "__" + while ([...used].some((name) => name.startsWith(prefix))) prefix += "_" + const hidden = (name: string) => quote(prefix + name) + const compiler: Compiler = { dialect, frames, hidden, row: hidden("row") } + const sql = relation(plan, compiler) + const columns = result.columns.map((col) => `${output(quote(col.name), col.type, dialect)} AS ${quote(col.name)}`) return { - sql: `SELECT ${r.columns.length === 0 ? "1" : list(r.columns)} FROM (${r.sql}) AS t ORDER BY ${ROW}`, - columns: r.columns + sql: `SELECT ${columns.length === 0 ? "1" : columns.join(", ")} FROM (${sql}) AS t ORDER BY ${compiler.row}`, + schema: S.make(result.columns.map((col) => S.field(col.name, col.type, { nullable: true }))) } } + +/** The rows of a compiled query's result, one array of values per row, as a frame. @internal */ +export const decode = (schema: S.Schema, rows: ReadonlyArray>): F.Frame => + F.make( + schema.fields.map((f, i) => { + const read = reader(f.type) + let nullable = false + const values = rows.map((row) => { + const v = row[i] + if (v !== null && v !== undefined) return read(v) + nullable = true + return null + }) + const column = Result.getOrElse( + ArrowColumn.fromValues(S.field(f.name, f.type, { nullable }), values), + (error) => V.fail(`column ${JSON.stringify(f.name)}: ${error.message}`) + ) + return new F.Column(f.name, V.fromArrowData(column.chunks[0]), undefined, nullable) + }), + rows.length + ) diff --git a/packages/sql/pglite/test/DataFrameSql.test.ts b/packages/sql/pglite/test/DataFrameSql.test.ts index f0730515ee1..8a627877aa5 100644 --- a/packages/sql/pglite/test/DataFrameSql.test.ts +++ b/packages/sql/pglite/test/DataFrameSql.test.ts @@ -4,7 +4,8 @@ */ import { PgliteClient } from "@effect/sql-pglite" import { assert, describe, layer } from "@effect/vitest" -import { DateTime, Effect, Layer, Schema } from "effect" +import { BigDecimal, DateTime, Duration, Effect, Layer, Result, Schema } from "effect" +import { ArrowSchema, ArrowTable } from "effect/arrow" import { Catalog, DataFrame, Expr, Query, Sql } from "effect/dataframe" import { SqlClient } from "effect/sql/SqlClient" @@ -53,38 +54,166 @@ const memory = Catalog.layer({ const table = Query.table("sales", Sale) -/** Values as plain JS for comparison: numbers from numeric strings, dates as ISO strings. */ -const plain = (u: unknown): unknown => - DateTime.isDateTime(u) - ? DateTime.formatIso(u) - : u instanceof Date - ? u.toISOString() - : typeof u === "bigint" - ? Number(u) - : typeof u === "string" && /^-?\d+(\.\d+)?$/.test(u) - ? Number(u) - : u - const close = (a: unknown, b: unknown) => - typeof a === "number" && typeof b === "number" ? Math.abs(a - b) <= 1e-9 * Math.max(1, Math.abs(a)) : a === b + typeof a === "number" && typeof b === "number" + ? Object.is(a, b) || Math.abs(a - b) <= 1e-9 * Math.max(1, Math.abs(a)) + : false + +const types = (df: DataFrame.DataFrame) => + DataFrame.toArrowTable(df).schema.fields.map((f) => ({ name: f.name, type: f.type })) + +const sql = Sql.layer({ dialect: "postgres" }) +const onSql = (query: Query.Query) => query.pipe(Effect.provide(sql)) +const rowsOnSql = (query: Query.Query) => Effect.map(onSql(query), DataFrame.toRows) + +const xs = Query.from(DataFrame.fromColumns({ x: [1, 2, 3] })) +const ints = Query.from(DataFrame.fromColumns({ x: Int32Array.of(1, 2, 3) })) +const none = ints.pipe(Query.filter((c) => Expr.gt(c.x, 9))) +/** A query over a frame with exactly the given Arrow columns. */ +const arrow = (fields: ReadonlyArray, columns: ReadonlyArray>) => + Query.from( + DataFrame.fromArrowTable(Result.getOrThrow(ArrowTable.fromColumns(ArrowSchema.make(fields), columns))) + ) as Query.Query as Query.Query +const ties = [0.5, 1.5, 2.5, -0.5, -1.5, -2.5, 2.675, 1.005, 0.125, 0.375, 1.25, 1.35] +const decimals = arrow<{ d: BigDecimal.BigDecimal | null }>( + [ArrowSchema.field("d", ArrowSchema.decimal(18, 3))], + [["0.5", "1.5", "2.5", "-0.5", "-1.5", "-2.5", "2.675", "1.005", "0.125", null].map((s) => + s === null ? null : BigDecimal.fromStringUnsafe(s) + )] +) + +/** Queries whose SQL result must equal the in-memory result, column types included. */ +const cases: ReadonlyArray]> = [ + ["int64 columns", Query.from(DataFrame.fromColumns({ d: BigInt64Array.of(1n, 9007199254740993n, -3n) }))], + [ + "aggregate types", + ints.pipe( + Query.aggregate((c) => ({ n: Expr.count(), s: Expr.sum(c.x), m: Expr.mean(c.x), d: Expr.countDistinct(c.x) })) + ) + ], + ["empty results", none.pipe(Query.withColumns((c) => ({ y: Expr.divide(c.x, 2), s: Expr.concat("a", "b") })))], + ["aggregates of empty input", none.pipe(Query.aggregate((c) => ({ n: Expr.countDistinct(c.x), s: Expr.sum(c.x) })))], + [ + "boolean sums", + Query.from(DataFrame.fromColumns({ b: [true, false, true] })).pipe( + Query.select((c) => ({ s: Expr.over(Expr.sum(c.b), []), r: Expr.cumSum(c.b) })) + ) + ], + [ + "boolean sum aggregate", + Query.from(DataFrame.fromColumns({ b: [true, false, true] })).pipe(Query.aggregate((c) => ({ s: Expr.sum(c.b) }))) + ], + ["filters on a window function", xs.pipe(Query.filter((c) => Expr.gt(Expr.cumSum(c.x), 3)))], + ["filters on an aggregate over the input", xs.pipe(Query.filter((c) => Expr.gt(c.x, Expr.over(Expr.mean(c.x), []))))], + [ + "over an empty partition list", + xs.pipe(Query.select((c) => ({ n: Expr.over(Expr.sum(c.x), []), m: Expr.over(Expr.first(c.x), []) }))) + ], + [ + "temporal and decimal inputs", + arrow<{ d: Duration.Duration | null; t: DateTime.Utc | null }>([ + ArrowSchema.field("d", ArrowSchema.duration("microsecond")), + ArrowSchema.field("t", ArrowSchema.timestamp("microsecond")), + ArrowSchema.field("day", ArrowSchema.date("day")), + ArrowSchema.field("n", ArrowSchema.decimal(18, 2)) + ], [ + [1500n, -90_000_001n, null], + [1_700_000_000_123_456n, -1n, null], + [19_000, -1, null], + [BigDecimal.fromStringUnsafe("12.34"), BigDecimal.fromStringUnsafe("-0.05"), null] + ]).pipe(Query.withColumns((c) => ({ s: Expr.totalSeconds(c.d), year: Expr.year(c.t) }))) + ], + [ + "durations", + Query.from(DataFrame.fromColumns({ d: [Duration.seconds(30), Duration.seconds(90)] })).pipe( + Query.filter((c) => Expr.gt(Expr.totalSeconds(c.d), 60n)) + ) + ], + [ + "columns named like helpers", + Query.from(DataFrame.fromColumns({ __row: [1, 2], __n: [3, 4], __source: ["a", "b"] })) + ], + ["tail of a column named __n", Query.from(DataFrame.fromColumns({ __n: [1, 2] })).pipe(Query.tail(1))], + [ + "concat of a column named __source", + Query.concat([ + Query.from(DataFrame.fromColumns({ __source: [1] })), + Query.from(DataFrame.fromColumns({ __source: [2] })) + ]) + ], + [ + "sqrt of negative numbers", + Query.from(DataFrame.fromColumns({ x: [-1, 4, null, -0] })).pipe(Query.select((c) => ({ r: Expr.sqrt(c.x) }))) + ], + [ + "rounding floats", + Query.from(DataFrame.fromColumns({ x: ties })).pipe( + Query.select((c) => ({ r0: Expr.round(c.x), r1: Expr.round(c.x, 1), r2: Expr.round(c.x, 2) })) + ) + ], + [ + "rounding integers", + Query.from(DataFrame.fromColumns({ x: Int32Array.of(5, -15, 25) })).pipe( + Query.select((c) => ({ r0: Expr.round(c.x), r1: Expr.round(c.x, 1) })) + ) + ] +] describe("effect/dataframe on PostgreSQL (PGlite)", { concurrent: false }, () => { layer(Layer.provideMerge(Setup, PgliteClient.layer()), { timeout: "60 seconds" })((it) => { const both = Effect.fnUntraced(function*(query: Query.Query) { - const expected = DataFrame.toRows(yield* query.pipe(Effect.provide(memory))) as Array> - const actual = DataFrame.toRows(yield* query.pipe(Effect.provide(Sql.layer({ dialect: "postgres" })))) as Array< - Record - > - assert.strictEqual(actual.length, expected.length) - for (let r = 0; r < expected.length; r++) { - for (const k of Object.keys(expected[r])) { - const a = plain(actual[r][k]) - const b = plain(expected[r][k]) - if (!close(a, b)) assert.deepStrictEqual({ row: r, column: k, value: a }, { row: r, column: k, value: b }) + const expected = yield* query.pipe(Effect.provide(memory)) + const actual = yield* onSql(query) + assert.deepStrictEqual(types(actual), types(expected)) + const a = DataFrame.toRows(actual) as Array> + const b = DataFrame.toRows(expected) as Array> + assert.strictEqual(a.length, b.length) + for (let r = 0; r < b.length; r++) { + for (const k of Object.keys(b[r])) { + if (!close(a[r][k], b[r][k])) { + assert.deepStrictEqual({ row: r, column: k, value: a[r][k] }, { row: r, column: k, value: b[r][k] }) + } } } }) + for (const [name, query] of cases) it.effect(name, () => both(query)) + + it.effect("negative slices clamp both ends", () => + Effect.gen(function*() { + assert.deepStrictEqual(yield* rowsOnSql(xs.pipe(Query.slice(-5, 1))), []) + assert.deepStrictEqual(yield* rowsOnSql(xs.pipe(Query.slice(-5, 3))), [{ x: 1 }]) + })) + + it.effect("rounds decimals half to even, keeping the scale", () => + Effect.gen(function*() { + const df = yield* onSql( + decimals.pipe(Query.select((c) => ({ r0: Expr.round(c.d, 0), r1: Expr.round(c.d, 1) }))) + ) + assert.deepStrictEqual(types(df).map((f) => f.type), [ArrowSchema.decimal(19, 3), ArrowSchema.decimal(19, 3)]) + assert.deepStrictEqual( + DataFrame.toRows(df).map((r: any) => [r.r0 && BigDecimal.format(r.r0), r.r1 && BigDecimal.format(r.r1)]), + [ + ["0", "0.5"], + ["2", "1.5"], + ["2", "2.5"], + ["0", "-0.5"], + ["-2", "-1.5"], + ["-2", "-2.5"], + ["3", "2.7"], + ["1", "1"], + ["0", "0.1"], + [null, null] + ] + ) + })) + + it.effect("errors carry the database's message", () => + Effect.gen(function*() { + const error = yield* Effect.flip(onSql(ints.pipe(Query.select((c) => ({ y: Expr.add(c.x, 2147483647) }))))) + assert.include(error.message, "integer out of range") + })) + it.effect("filters, derives and sorts", () => both(table.pipe( Query.filter((c) => Expr.and(Expr.isNotNull(c.units), Expr.gt(c.price, 20))), From 4985a7e31d60532ce7016c51bad1a5e7b7a387dc Mon Sep 17 00:00:00 2001 From: Effect Bot Date: Sun, 15 Nov 2026 12:09:23 +0000 Subject: [PATCH 12/88] Name the plan-level row rules after what they guarantee --- packages/effect/src/dataframe/internal/query.ts | 5 +++-- packages/effect/src/dataframe/internal/stream.ts | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/packages/effect/src/dataframe/internal/query.ts b/packages/effect/src/dataframe/internal/query.ts index 72e22ca1357..01eddcc7d6a 100644 --- a/packages/effect/src/dataframe/internal/query.ts +++ b/packages/effect/src/dataframe/internal/query.ts @@ -312,7 +312,8 @@ const push = (node: Node, predicates: ReadonlyArray): Node => { } } -const isRowWise = (node: Node): boolean => +/** One output row per input row, in order, so a slice can move below it. */ +const keepsRows = (node: Node): boolean => ((node._tag === "Select" || node._tag === "WithColumns") && node.columns.every((c) => E.isRowWise(c.expression))) || node._tag === "Rename" || node._tag === "Pick" || node._tag === "Omit" @@ -322,7 +323,7 @@ const pushSlices = (node: Node): Node => { } const slice = (input: Node, offset: number, length: number | undefined): Node => { - if (isRowWise(input)) { + if (keepsRows(input)) { return { ...input, input: slice((input as { readonly input: Node }).input, offset, length) } as Node } if (input._tag === "Slice" && input.offset >= 0 && offset >= 0) { diff --git a/packages/effect/src/dataframe/internal/stream.ts b/packages/effect/src/dataframe/internal/stream.ts index fad16b1e863..42e7475abc5 100644 --- a/packages/effect/src/dataframe/internal/stream.ts +++ b/packages/effect/src/dataframe/internal/stream.ts @@ -84,7 +84,8 @@ const collectFrames = (batches: Batches, node: Q.Node): Effect.Effect frames.length === 1 ? frames[0] : F.concat(frames))) -const isRowWise = (node: Q.Node): boolean => { +/** Each output row depends only on its input row, so the node runs per batch. */ +const perBatch = (node: Q.Node): boolean => { switch (node._tag) { case "Filter": return E.isRowWise(node.predicate) @@ -190,7 +191,7 @@ const slice = (input: Batches, offset: number, length: number | undefined): Batc /** The batches of a node's result. @internal */ export const batches = (node: Q.Node, catalog: Catalog.Service): Batches => { - if (isRowWise(node)) { + if (perBatch(node)) { return Stream.mapEffect( batches((node as { readonly input: Q.Node }).input, catalog), (batch) => Q.attempt(() => Q.step(node, [batch])) From 5038d8e9ebc37886f2a619fe2e17e18e9a22131d Mon Sep 17 00:00:00 2001 From: Effect Bot Date: Sun, 15 Nov 2026 12:31:35 +0000 Subject: [PATCH 13/88] Move filters and slices only below row-wise operations A sort by a window (or any operation whose expressions read other rows) sees every row of its input, so a filter moved below it changed its result. Filter pushdown, slice pushdown and per-batch streaming now share one operation-level rule: every expression the operation evaluates is row-wise. --- .../effect/src/dataframe/internal/query.ts | 58 +++++++++++++++---- .../effect/src/dataframe/internal/stream.ts | 10 ++-- .../test/fixtures/dataframe/generate_gold.py | 3 + .../fixtures/dataframe/gold/extended.json | 2 +- 4 files changed, 54 insertions(+), 19 deletions(-) diff --git a/packages/effect/src/dataframe/internal/query.ts b/packages/effect/src/dataframe/internal/query.ts index 01eddcc7d6a..ba7ca9a10b0 100644 --- a/packages/effect/src/dataframe/internal/query.ts +++ b/packages/effect/src/dataframe/internal/query.ts @@ -4,12 +4,17 @@ * Execution is a post-order walk that calls the frame operations. Before it, * `optimize` rewrites the plan so less work happens: * - * - row-wise predicates move below the operations they commute with - * (sorts, projections, derived columns they don't read, group keys, either - * side of a join, every input of a concat), and filters that meet merge; - * predicates with windows stay where they are; - * - slices move below row-wise operations, and a slice of a sort becomes a - * top-k selection; + * - row-wise predicates move below the row-wise operations they commute + * with (sorts, projections, derived columns they don't read, group keys, + * either side of a join, every input of a concat), and filters that meet + * merge; + * - slices move below row-wise operations that keep every row in place, and + * a slice of a sort becomes a top-k selection. + * + * An operation is row-wise when its expressions are (see `isRowWise`): a + * window in a predicate, a sort key or a derived column sees every row of its + * input, so moving a filter or slice below it changes its result. + * * - columns nothing reads are dropped at the inputs, and derived columns and * aggregates nothing reads are not computed. * @@ -195,6 +200,36 @@ const computeNames = (node: Node): ReadonlyArray | undefined => { // Optimisation // ----------------------------------------------------------------------------- +/** The expressions an operation evaluates over its input rows. */ +const expressions = (node: Node): ReadonlyArray => { + switch (node._tag) { + case "Filter": + return [node.predicate] + case "Select": + case "WithColumns": + return node.columns.map((c) => c.expression) + case "Sort": + case "TopK": + return node.keys.map((k) => k.expression) + case "GroupBy": + case "Aggregate": + return node.aggregates.map((a) => a.expression._tag === "AggregateCall" ? a.expression.arg : a.expression) + .filter((e) => e !== undefined) + default: + return [] + } +} + +/** + * Whether an operation's expressions are row-wise (`E.isRowWise`; for + * aggregations, their arguments), so that each input row contributes the + * same values whichever other rows are present. Only row-wise operations + * let filters and slices move below them, and run batch by batch. + * + * @internal + */ +export const isRowWise = (node: Node): boolean => expressions(node).every(E.isRowWise) + /** @internal */ export const optimize = (plan: Node): Node => prune(pushSlices(pushFilters(plan)), undefined) @@ -237,11 +272,12 @@ const filtered = (input: Node, predicates: ReadonlyArray): Node /** * Places each predicate as deep as it can go below `node`. Predicates that * are not row-wise (windows) depend on exactly the rows they see, so a set - * containing one stays where it is. + * containing one stays where it is, as do predicates above an operation that + * is not row-wise. */ const push = (node: Node, predicates: ReadonlyArray): Node => { if (predicates.length === 0) return node - if (!predicates.every(E.isRowWise)) return filtered(node, predicates) + if (!predicates.every(E.isRowWise) || !isRowWise(node)) return filtered(node, predicates) const split = (canPush: (refs: Set, p: Plan.Expression) => boolean) => { const down: Array = [] const here: Array = [] @@ -264,13 +300,11 @@ const push = (node: Node, predicates: ReadonlyArray): Node => { return { ...node, input: push(node.input, renamed) } } case "WithColumns": { - if (!node.columns.every((c) => E.isRowWise(c.expression))) return filtered(node, predicates) const added = new Set(node.columns.map((c) => c.name)) const { down, here } = split((refs) => [...refs].every((r) => !added.has(r))) return filtered({ ...node, input: push(node.input, down) }, here) } case "Select": { - if (!node.columns.every((c) => E.isRowWise(c.expression))) return filtered(node, predicates) const byName = new Map(node.columns.map((c) => [c.name, c.expression])) const { down, here } = split((refs) => [...refs].every((r) => byName.get(r)?._tag === "Column")) const inlined = down.map((p) => E.mapColumns(p, (name) => byName.get(name)!)) @@ -314,8 +348,8 @@ const push = (node: Node, predicates: ReadonlyArray): Node => { /** One output row per input row, in order, so a slice can move below it. */ const keepsRows = (node: Node): boolean => - ((node._tag === "Select" || node._tag === "WithColumns") && node.columns.every((c) => E.isRowWise(c.expression))) || - node._tag === "Rename" || node._tag === "Pick" || node._tag === "Omit" + (node._tag === "Select" || node._tag === "WithColumns" || node._tag === "Rename" || node._tag === "Pick" || + node._tag === "Omit") && isRowWise(node) const pushSlices = (node: Node): Node => { const optimized = mapChildren(node, pushSlices) diff --git a/packages/effect/src/dataframe/internal/stream.ts b/packages/effect/src/dataframe/internal/stream.ts index 42e7475abc5..6c1301b2507 100644 --- a/packages/effect/src/dataframe/internal/stream.ts +++ b/packages/effect/src/dataframe/internal/stream.ts @@ -88,17 +88,15 @@ const collectFrames = (batches: Batches, node: Q.Node): Effect.Effect { switch (node._tag) { case "Filter": - return E.isRowWise(node.predicate) + case "Select": + case "WithColumns": case "Pick": case "Omit": case "Rename": case "DropNulls": case "Explode": case "Unnest": - return true - case "Select": - case "WithColumns": - return node.columns.every((c) => E.isRowWise(c.expression)) + return Q.isRowWise(node) default: return false } @@ -124,7 +122,7 @@ const aggregate = ( const fn: Agg.AggFn = expression.arg === undefined ? "len" : expression.fn return { name, fn, arg: expression.arg } }) - if (!specs.every((s) => Partial.combinable(s.fn) && (s.arg === undefined || E.isRowWise(s.arg)))) { + if (!Q.isRowWise(node) || !specs.every((s) => Partial.combinable(s.fn))) { return collected(node, catalog) } const fns = specs.map((s) => s.fn) diff --git a/packages/effect/test/fixtures/dataframe/generate_gold.py b/packages/effect/test/fixtures/dataframe/generate_gold.py index e59059260e3..7f2d785e1ad 100644 --- a/packages/effect/test/fixtures/dataframe/generate_gold.py +++ b/packages/effect/test/fixtures/dataframe/generate_gold.py @@ -909,6 +909,9 @@ def extended_cases(): "right": table([("g", "i32"), ("name", "utf8")], {"g": [1], "name": ["one"]})} add("window/filter-after-sort", "a window predicate sees the sorted rows", W, [{"op": "sort", "keys": [{"expr": col("x")}]}, {"op": "filter", "predicate": fn("gt", fn("cumSum", col("x")), lit(3.0))}]) + add("window/filter-after-sort-by-window", "a filter after a sort by a window leaves the sort's order alone", + {"left": table([("x", "f64")], {"x": [-3.0, 5.0, -1.0]})}, + [{"op": "sort", "keys": [{"expr": fn("cumSum", col("x"))}]}, {"op": "filter", "predicate": fn("neq", col("x"), lit(5.0))}]) add("window/filter-after-filter", "a window predicate sees only the rows an earlier filter kept", W, [{"op": "filter", "predicate": fn("gt", col("x"), lit(1.0))}, {"op": "filter", "predicate": fn("gt", fn("shift", col("x")), lit(2.0))}]) add("window/filter-after-concat", "a window predicate sees every input of a concat", W, diff --git a/packages/effect/test/fixtures/dataframe/gold/extended.json b/packages/effect/test/fixtures/dataframe/gold/extended.json index f7adffd327f..84b7eb80731 100644 --- a/packages/effect/test/fixtures/dataframe/gold/extended.json +++ b/packages/effect/test/fixtures/dataframe/gold/extended.json @@ -1 +1 @@ -{"generator":"generate_gold.py","polars":"2.1.0","deviations":["JavaScript has no float literal distinct from an integer: an integral number literal behaves like a Polars integer literal (it takes the integer type of the column it meets), so cases never combine an integer column with an integral float literal.","Float to string casts use JavaScript number formatting (Polars prints 5.0 as '5.0', JavaScript as '5').","Joins never add '_right' suffixes: overlapping non-key columns are a type error.","Arithmetic between 64-bit integers (bigint) and other numeric types requires an explicit cast."],"cases":[{"id":"window/cum","description":"cumulative sum, min, max and count; nulls stay null","inputs":{"left":{"fields":[{"name":"g","type":"utf8"},{"name":"t","type":"i32"},{"name":"x","type":"f64"},{"name":"i","type":"i32"},{"name":"b","type":"bool"}],"columns":{"g":["a","b","a","a","b",null,"a","b","b","a"],"t":[1,1,2,3,2,1,4,3,4,5],"x":[1.5,2.0,null,4.0,"NaN",3.0,-1.0,2.0,0.5,2.0],"i":[3,1,4,1,5,9,null,6,5,3],"b":[true,false,null,true,true,false,true,null,false,true]}}},"ops":[{"op":"select","exprs":{"sx":{"fn":"cumSum","args":[{"col":"x"}]},"si":{"fn":"cumSum","args":[{"col":"i"}]},"mn":{"fn":"cumMin","args":[{"col":"i"}]},"mx":{"fn":"cumMax","args":[{"col":"x"}]},"n":{"fn":"cumCount","args":[{"col":"x"}]},"sb":{"fn":"cumSum","args":[{"col":"b"}]}}}],"expected":{"fields":[{"name":"sx","type":"f64"},{"name":"si","type":"i32"},{"name":"mn","type":"i32"},{"name":"mx","type":"f64"},{"name":"n","type":"u32"},{"name":"sb","type":"u32"}],"columns":{"sx":[1.5,3.5,null,7.5,"NaN","NaN","NaN","NaN","NaN","NaN"],"si":[3,4,8,9,14,23,null,29,34,37],"mn":[3,1,1,1,1,1,null,1,1,1],"mx":[1.5,2.0,null,4.0,4.0,4.0,4.0,4.0,4.0,4.0],"n":[1,2,2,3,4,5,6,7,8,9],"sb":[1,1,null,2,3,3,4,null,4,5]}}},{"id":"window/shift-diff","description":"shift (lag and lead) and diff","inputs":{"left":{"fields":[{"name":"g","type":"utf8"},{"name":"t","type":"i32"},{"name":"x","type":"f64"},{"name":"i","type":"i32"},{"name":"b","type":"bool"}],"columns":{"g":["a","b","a","a","b",null,"a","b","b","a"],"t":[1,1,2,3,2,1,4,3,4,5],"x":[1.5,2.0,null,4.0,"NaN",3.0,-1.0,2.0,0.5,2.0],"i":[3,1,4,1,5,9,null,6,5,3],"b":[true,false,null,true,true,false,true,null,false,true]}}},"ops":[{"op":"select","exprs":{"lag":{"fn":"shift","args":[{"col":"i"}]},"lead":{"fn":"shift","args":[{"col":"x"}],"n":-2},"d":{"fn":"diff","args":[{"col":"i"}]},"d2":{"fn":"diff","args":[{"col":"x"}],"n":2},"ls":{"fn":"shift","args":[{"col":"g"}]}}}],"expected":{"fields":[{"name":"lag","type":"i32"},{"name":"lead","type":"f64"},{"name":"d","type":"i32"},{"name":"d2","type":"f64"},{"name":"ls","type":"utf8"}],"columns":{"lag":[null,3,1,4,1,5,9,null,6,5],"lead":[null,4.0,"NaN",3.0,-1.0,2.0,0.5,2.0,null,null],"d":[null,-2,3,-3,4,4,null,null,-1,-2],"d2":[null,null,null,2.0,null,-1.0,"NaN",-1.0,1.5,0.0],"ls":[null,"a","b","a","a","b",null,"a","b","b"]}}},{"id":"window/rank","description":"rank methods, nulls and NaN","inputs":{"left":{"fields":[{"name":"g","type":"utf8"},{"name":"t","type":"i32"},{"name":"x","type":"f64"},{"name":"i","type":"i32"},{"name":"b","type":"bool"}],"columns":{"g":["a","b","a","a","b",null,"a","b","b","a"],"t":[1,1,2,3,2,1,4,3,4,5],"x":[1.5,2.0,null,4.0,"NaN",3.0,-1.0,2.0,0.5,2.0],"i":[3,1,4,1,5,9,null,6,5,3],"b":[true,false,null,true,true,false,true,null,false,true]}}},"ops":[{"op":"select","exprs":{"avg":{"fn":"rank","args":[{"col":"x"}]},"min":{"fn":"rank","args":[{"col":"i"}],"method":"min"},"max":{"fn":"rank","args":[{"col":"i"}],"method":"max"},"dense":{"fn":"rank","args":[{"col":"i"}],"method":"dense","descending":true},"ord":{"fn":"rank","args":[{"col":"x"}],"method":"ordinal"},"s":{"fn":"rank","args":[{"col":"g"}],"method":"dense"}}}],"expected":{"fields":[{"name":"avg","type":"f64"},{"name":"min","type":"u32"},{"name":"max","type":"u32"},{"name":"dense","type":"u32"},{"name":"ord","type":"u32"},{"name":"s","type":"u32"}],"columns":{"avg":[3.0,5.0,null,8.0,9.0,7.0,1.0,5.0,2.0,5.0],"min":[3,1,5,1,6,9,null,8,6,3],"max":[4,2,5,2,7,9,null,8,7,4],"dense":[5,6,4,6,3,1,null,2,3,5],"ord":[3,4,null,8,9,7,1,5,2,6],"s":[1,2,1,1,2,null,1,2,2,1]}}},{"id":"window/rolling","description":"rolling sum, mean, min and max","inputs":{"left":{"fields":[{"name":"g","type":"utf8"},{"name":"t","type":"i32"},{"name":"x","type":"f64"},{"name":"i","type":"i32"},{"name":"b","type":"bool"}],"columns":{"g":["a","b","a","a","b",null,"a","b","b","a"],"t":[1,1,2,3,2,1,4,3,4,5],"x":[1.5,2.0,null,4.0,"NaN",3.0,-1.0,2.0,0.5,2.0],"i":[3,1,4,1,5,9,null,6,5,3],"b":[true,false,null,true,true,false,true,null,false,true]}}},"ops":[{"op":"select","exprs":{"s":{"fn":"rollingSum","args":[{"col":"x"}],"window":3},"m":{"fn":"rollingMean","args":[{"col":"i"}],"window":2},"mn":{"fn":"rollingMin","args":[{"col":"x"}],"window":3,"minPeriods":1},"mx":{"fn":"rollingMax","args":[{"col":"i"}],"window":4,"minPeriods":2}}}],"expected":{"fields":[{"name":"s","type":"f64"},{"name":"m","type":"f64"},{"name":"mn","type":"f64"},{"name":"mx","type":"i32"}],"columns":{"s":[null,null,null,null,null,"NaN","NaN",4.0,1.5,4.5],"m":[null,2.0,2.5,2.5,3.0,7.0,null,null,5.5,4.0],"mn":[1.5,1.5,1.5,2.0,"NaN","NaN","NaN",-1.0,-1.0,0.5],"mx":[null,3,4,4,5,9,9,9,9,6]}}},{"id":"window/over-agg","description":"aggregates broadcast over partitions","inputs":{"left":{"fields":[{"name":"g","type":"utf8"},{"name":"t","type":"i32"},{"name":"x","type":"f64"},{"name":"i","type":"i32"},{"name":"b","type":"bool"}],"columns":{"g":["a","b","a","a","b",null,"a","b","b","a"],"t":[1,1,2,3,2,1,4,3,4,5],"x":[1.5,2.0,null,4.0,"NaN",3.0,-1.0,2.0,0.5,2.0],"i":[3,1,4,1,5,9,null,6,5,3],"b":[true,false,null,true,true,false,true,null,false,true]}}},"ops":[{"op":"withColumns","exprs":{"total":{"over":{"agg":"sum","arg":{"col":"x"}},"partitionBy":[{"col":"g"}]},"n":{"over":{"agg":"len"},"partitionBy":[{"col":"g"}]},"mean":{"over":{"agg":"mean","arg":{"col":"i"}},"partitionBy":[{"col":"g"}]},"first":{"over":{"agg":"first","arg":{"col":"t"}},"partitionBy":[{"col":"g"},{"col":"b"}]}}}],"expected":{"fields":[{"name":"g","type":"utf8"},{"name":"t","type":"i32"},{"name":"x","type":"f64"},{"name":"i","type":"i32"},{"name":"b","type":"bool"},{"name":"total","type":"f64"},{"name":"n","type":"u32"},{"name":"mean","type":"f64"},{"name":"first","type":"i32"}],"columns":{"g":["a","b","a","a","b",null,"a","b","b","a"],"t":[1,1,2,3,2,1,4,3,4,5],"x":[1.5,2.0,null,4.0,"NaN",3.0,-1.0,2.0,0.5,2.0],"i":[3,1,4,1,5,9,null,6,5,3],"b":[true,false,null,true,true,false,true,null,false,true],"total":[6.5,"NaN",6.5,6.5,"NaN",3.0,6.5,"NaN","NaN",6.5],"n":[5,4,5,5,4,1,5,4,4,5],"mean":[2.75,4.25,2.75,2.75,4.25,9.0,2.75,4.25,4.25,2.75],"first":[1,1,2,1,2,1,1,3,1,1]}}},{"id":"window/over-cum","description":"cumulative and shifted values per partition, in row order","inputs":{"left":{"fields":[{"name":"g","type":"utf8"},{"name":"t","type":"i32"},{"name":"x","type":"f64"},{"name":"i","type":"i32"},{"name":"b","type":"bool"}],"columns":{"g":["a","b","a","a","b",null,"a","b","b","a"],"t":[1,1,2,3,2,1,4,3,4,5],"x":[1.5,2.0,null,4.0,"NaN",3.0,-1.0,2.0,0.5,2.0],"i":[3,1,4,1,5,9,null,6,5,3],"b":[true,false,null,true,true,false,true,null,false,true]}}},"ops":[{"op":"withColumns","exprs":{"cs":{"over":{"fn":"cumSum","args":[{"col":"i"}]},"partitionBy":[{"col":"g"}]},"prev":{"over":{"fn":"shift","args":[{"col":"x"}]},"partitionBy":[{"col":"g"}]},"d":{"over":{"fn":"diff","args":[{"col":"t"}]},"partitionBy":[{"col":"g"}]},"r":{"over":{"fn":"rank","args":[{"col":"x"}],"method":"min"},"partitionBy":[{"col":"g"}]},"roll":{"over":{"fn":"rollingSum","args":[{"col":"i"}],"window":2,"minPeriods":1},"partitionBy":[{"col":"g"}]}}}],"expected":{"fields":[{"name":"g","type":"utf8"},{"name":"t","type":"i32"},{"name":"x","type":"f64"},{"name":"i","type":"i32"},{"name":"b","type":"bool"},{"name":"cs","type":"i32"},{"name":"prev","type":"f64"},{"name":"d","type":"i32"},{"name":"r","type":"u32"},{"name":"roll","type":"i32"}],"columns":{"g":["a","b","a","a","b",null,"a","b","b","a"],"t":[1,1,2,3,2,1,4,3,4,5],"x":[1.5,2.0,null,4.0,"NaN",3.0,-1.0,2.0,0.5,2.0],"i":[3,1,4,1,5,9,null,6,5,3],"b":[true,false,null,true,true,false,true,null,false,true],"cs":[3,1,7,8,6,9,null,12,17,11],"prev":[null,null,1.5,null,2.0,null,4.0,"NaN",2.0,-1.0],"d":[null,null,1,1,1,null,1,1,1,1],"r":[2,2,null,4,4,1,1,2,1,3],"roll":[3,1,7,5,6,9,1,11,11,3]}}},{"id":"window/over-filter","description":"filters after windows see the windowed values","inputs":{"left":{"fields":[{"name":"g","type":"utf8"},{"name":"t","type":"i32"},{"name":"x","type":"f64"},{"name":"i","type":"i32"},{"name":"b","type":"bool"}],"columns":{"g":["a","b","a","a","b",null,"a","b","b","a"],"t":[1,1,2,3,2,1,4,3,4,5],"x":[1.5,2.0,null,4.0,"NaN",3.0,-1.0,2.0,0.5,2.0],"i":[3,1,4,1,5,9,null,6,5,3],"b":[true,false,null,true,true,false,true,null,false,true]}}},"ops":[{"op":"withColumns","exprs":{"cs":{"over":{"fn":"cumSum","args":[{"col":"t"}]},"partitionBy":[{"col":"g"}]}}},{"op":"filter","predicate":{"fn":"gt","args":[{"col":"t"},{"lit":2}]}}],"expected":{"fields":[{"name":"g","type":"utf8"},{"name":"t","type":"i32"},{"name":"x","type":"f64"},{"name":"i","type":"i32"},{"name":"b","type":"bool"},{"name":"cs","type":"i32"}],"columns":{"g":["a","a","b","b","a"],"t":[3,4,3,4,5],"x":[4.0,-1.0,2.0,0.5,2.0],"i":[1,null,6,5,3],"b":[true,true,null,false,true],"cs":[6,10,6,10,15]}}},{"id":"window/filter-first","description":"windows after a filter only see the remaining rows","inputs":{"left":{"fields":[{"name":"g","type":"utf8"},{"name":"t","type":"i32"},{"name":"x","type":"f64"},{"name":"i","type":"i32"},{"name":"b","type":"bool"}],"columns":{"g":["a","b","a","a","b",null,"a","b","b","a"],"t":[1,1,2,3,2,1,4,3,4,5],"x":[1.5,2.0,null,4.0,"NaN",3.0,-1.0,2.0,0.5,2.0],"i":[3,1,4,1,5,9,null,6,5,3],"b":[true,false,null,true,true,false,true,null,false,true]}}},"ops":[{"op":"filter","predicate":{"fn":"isNotNull","args":[{"col":"x"}]}},{"op":"withColumns","exprs":{"cs":{"fn":"cumSum","args":[{"col":"x"}]},"r":{"over":{"fn":"rank","args":[{"col":"x"}]},"partitionBy":[{"col":"g"}]}}},{"op":"head","n":4}],"expected":{"fields":[{"name":"g","type":"utf8"},{"name":"t","type":"i32"},{"name":"x","type":"f64"},{"name":"i","type":"i32"},{"name":"b","type":"bool"},{"name":"cs","type":"f64"},{"name":"r","type":"f64"}],"columns":{"g":["a","b","a","b"],"t":[1,1,3,2],"x":[1.5,2.0,4.0,"NaN"],"i":[3,1,1,5],"b":[true,false,true,true],"cs":[1.5,3.5,7.5,"NaN"],"r":[2.0,2.5,4.0,4.0]}}},{"id":"pivot/sum","description":"pivot with a sum per cell; missing cells are null","inputs":{"left":{"fields":[{"name":"region","type":"utf8"},{"name":"quarter","type":"utf8"},{"name":"units","type":"i32"},{"name":"price","type":"f64"}],"columns":{"region":["north","north","south","south","north","east","south","north"],"quarter":["q1","q2","q1","q3","q1","q2","q1",null],"units":[10,20,5,null,7,3,8,1],"price":[1.5,2.0,1.0,3.0,null,2.5,1.0,9.0]}}},"ops":[{"op":"pivot","on":"quarter","index":["region"],"values":"units","aggregate":"sum"}],"expected":{"fields":[{"name":"region","type":"utf8"},{"name":"q1","type":"i32"},{"name":"q2","type":"i32"},{"name":"q3","type":"i32"},{"name":"null","type":"i32"}],"columns":{"region":["north","south","east"],"q1":[17,13,0],"q2":[20,0,3],"q3":[0,0,0],"null":[1,0,0]}}},{"id":"pivot/first","description":"pivot with the first value per cell","inputs":{"left":{"fields":[{"name":"region","type":"utf8"},{"name":"quarter","type":"utf8"},{"name":"units","type":"i32"},{"name":"price","type":"f64"}],"columns":{"region":["north","north","south","south","north","east","south","north"],"quarter":["q1","q2","q1","q3","q1","q2","q1",null],"units":[10,20,5,null,7,3,8,1],"price":[1.5,2.0,1.0,3.0,null,2.5,1.0,9.0]}}},"ops":[{"op":"pivot","on":"region","index":["quarter"],"values":"price","aggregate":"first"}],"expected":{"fields":[{"name":"quarter","type":"utf8"},{"name":"north","type":"f64"},{"name":"south","type":"f64"},{"name":"east","type":"f64"}],"columns":{"quarter":["q1","q2","q3",null],"north":[1.5,2.0,null,9.0],"south":[1.0,null,3.0,null],"east":[null,2.5,null,null]}}},{"id":"pivot/len","description":"pivot counting rows per cell","inputs":{"left":{"fields":[{"name":"region","type":"utf8"},{"name":"quarter","type":"utf8"},{"name":"units","type":"i32"},{"name":"price","type":"f64"}],"columns":{"region":["north","north","south","south","north","east","south","north"],"quarter":["q1","q2","q1","q3","q1","q2","q1",null],"units":[10,20,5,null,7,3,8,1],"price":[1.5,2.0,1.0,3.0,null,2.5,1.0,9.0]}}},"ops":[{"op":"pivot","on":"quarter","index":["region"],"values":"units","aggregate":"len"}],"expected":{"fields":[{"name":"region","type":"utf8"},{"name":"q1","type":"u32"},{"name":"q2","type":"u32"},{"name":"q3","type":"u32"},{"name":"null","type":"u32"}],"columns":{"region":["north","south","east"],"q1":[2,2,0],"q2":[1,0,1],"q3":[0,1,0],"null":[1,0,0]}}},{"id":"pivot/columns","description":"pivot with declared output columns","inputs":{"left":{"fields":[{"name":"region","type":"utf8"},{"name":"quarter","type":"utf8"},{"name":"units","type":"i32"},{"name":"price","type":"f64"}],"columns":{"region":["north","north","south","south","north","east","south","north"],"quarter":["q1","q2","q1","q3","q1","q2","q1",null],"units":[10,20,5,null,7,3,8,1],"price":[1.5,2.0,1.0,3.0,null,2.5,1.0,9.0]}}},"ops":[{"op":"pivot","on":"quarter","index":["region"],"values":"price","aggregate":"max","columns":["q1","q2","q4"]}],"expected":{"fields":[{"name":"region","type":"utf8"},{"name":"q1","type":"f64"},{"name":"q2","type":"f64"},{"name":"q4","type":"f64"}],"columns":{"region":["north","south","east"],"q1":[1.5,1.0,null],"q2":[2.0,null,2.5],"q4":[null,null,null]}}},{"id":"pivot/numeric-on","description":"pivot on a numeric column names columns by value","inputs":{"left":{"fields":[{"name":"g","type":"utf8"},{"name":"t","type":"i32"},{"name":"x","type":"f64"},{"name":"i","type":"i32"},{"name":"b","type":"bool"}],"columns":{"g":["a","b","a","a","b",null,"a","b","b","a"],"t":[1,1,2,3,2,1,4,3,4,5],"x":[1.5,2.0,null,4.0,"NaN",3.0,-1.0,2.0,0.5,2.0],"i":[3,1,4,1,5,9,null,6,5,3],"b":[true,false,null,true,true,false,true,null,false,true]}}},"ops":[{"op":"filter","predicate":{"fn":"isNotNull","args":[{"col":"g"}]}},{"op":"pivot","on":"t","index":["g"],"values":"i","aggregate":"min"}],"expected":{"fields":[{"name":"g","type":"utf8"},{"name":"1","type":"i32"},{"name":"2","type":"i32"},{"name":"3","type":"i32"},{"name":"4","type":"i32"},{"name":"5","type":"i32"}],"columns":{"g":["a","b"],"1":[3,1],"2":[4,5],"3":[1,6],"4":[null,5],"5":[3,null]}}},{"id":"unpivot/basic","description":"unpivot stacks columns, column by column","inputs":{"left":{"fields":[{"name":"region","type":"utf8"},{"name":"quarter","type":"utf8"},{"name":"units","type":"i32"},{"name":"price","type":"f64"}],"columns":{"region":["north","north","south","south","north","east","south","north"],"quarter":["q1","q2","q1","q3","q1","q2","q1",null],"units":[10,20,5,null,7,3,8,1],"price":[1.5,2.0,1.0,3.0,null,2.5,1.0,9.0]}}},"ops":[{"op":"unpivot","index":["region"],"on":["units","price"]}],"expected":{"fields":[{"name":"region","type":"utf8"},{"name":"variable","type":"utf8"},{"name":"value","type":"f64"}],"columns":{"region":["north","north","south","south","north","east","south","north","north","north","south","south","north","east","south","north"],"variable":["units","units","units","units","units","units","units","units","price","price","price","price","price","price","price","price"],"value":[10.0,20.0,5.0,null,7.0,3.0,8.0,1.0,1.5,2.0,1.0,3.0,null,2.5,1.0,9.0]}}},{"id":"unpivot/names","description":"unpivot with named variable and value columns","inputs":{"left":{"fields":[{"name":"region","type":"utf8"},{"name":"quarter","type":"utf8"},{"name":"units","type":"i32"},{"name":"price","type":"f64"}],"columns":{"region":["north","north","south","south","north","east","south","north"],"quarter":["q1","q2","q1","q3","q1","q2","q1",null],"units":[10,20,5,null,7,3,8,1],"price":[1.5,2.0,1.0,3.0,null,2.5,1.0,9.0]}}},"ops":[{"op":"pick","columns":["region","quarter"]},{"op":"unpivot","index":[],"on":["region","quarter"],"variable":"field","value":"text"}],"expected":{"fields":[{"name":"field","type":"utf8"},{"name":"text","type":"utf8"}],"columns":{"field":["region","region","region","region","region","region","region","region","quarter","quarter","quarter","quarter","quarter","quarter","quarter","quarter"],"text":["north","north","south","south","north","east","south","north","q1","q2","q1","q3","q1","q2","q1",null]}}},{"id":"temporal/extract","description":"calendar fields of timestamps and dates","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"ts","type":"datetime[us]"},{"name":"ms","type":"datetime[ms]"},{"name":"d","type":"date"},{"name":"dur","type":"duration[us]"}],"columns":{"k":["a","b","c","d","e","f"],"ts":[1709214307123456,1704067199999000,1709510400000000,null,-14182940000000,1710052200000500],"ms":[1709214307123,1704067200000,null,1709251200000,0,1709942400000],"d":[19782,19722,19786,19782,null,19792],"dur":[5400000000,-1500000,null,259200000001,0,90000000000]}},"right":{"fields":[{"name":"d","type":"date"},{"name":"name","type":"utf8"}],"columns":{"d":[19782,19792,20089],"name":["leap","sunday","new year"]}}},"ops":[{"op":"select","exprs":{"y":{"fn":"year","args":[{"col":"ts"}]},"mo":{"fn":"month","args":[{"col":"ts"}]},"d":{"fn":"day","args":[{"col":"ts"}]},"h":{"fn":"hour","args":[{"col":"ts"}]},"mi":{"fn":"minute","args":[{"col":"ts"}]},"s":{"fn":"second","args":[{"col":"ts"}]},"ms":{"fn":"millisecond","args":[{"col":"ts"}]},"wd":{"fn":"weekday","args":[{"col":"ts"}]},"od":{"fn":"ordinalDay","args":[{"col":"ts"}]},"dy":{"fn":"year","args":[{"col":"d"}]},"dwd":{"fn":"weekday","args":[{"col":"d"}]},"dod":{"fn":"ordinalDay","args":[{"col":"d"}]}}}],"expected":{"fields":[{"name":"y","type":"i32"},{"name":"mo","type":"i8"},{"name":"d","type":"i8"},{"name":"h","type":"i8"},{"name":"mi","type":"i8"},{"name":"s","type":"i8"},{"name":"ms","type":"i32"},{"name":"wd","type":"i8"},{"name":"od","type":"i16"},{"name":"dy","type":"i32"},{"name":"dwd","type":"i8"},{"name":"dod","type":"i16"}],"columns":{"y":[2024,2023,2024,null,1969,2024],"mo":[2,12,3,null,7,3],"d":[29,31,4,null,20,10],"h":[13,23,0,null,20,6],"mi":[45,59,0,null,17,30],"s":[7,59,0,null,40,0],"ms":[123,999,0,null,0,0],"wd":[4,7,1,null,7,7],"od":[60,365,64,null,201,70],"dy":[2024,2023,2024,2024,null,2024],"dwd":[4,7,1,4,null,7],"dod":[60,365,64,60,null,70]}}},{"id":"temporal/arith","description":"timestamp differences are durations; durations shift timestamps","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"ts","type":"datetime[us]"},{"name":"ms","type":"datetime[ms]"},{"name":"d","type":"date"},{"name":"dur","type":"duration[us]"}],"columns":{"k":["a","b","c","d","e","f"],"ts":[1709214307123456,1704067199999000,1709510400000000,null,-14182940000000,1710052200000500],"ms":[1709214307123,1704067200000,null,1709251200000,0,1709942400000],"d":[19782,19722,19786,19782,null,19792],"dur":[5400000000,-1500000,null,259200000001,0,90000000000]}},"right":{"fields":[{"name":"d","type":"date"},{"name":"name","type":"utf8"}],"columns":{"d":[19782,19792,20089],"name":["leap","sunday","new year"]}}},"ops":[{"op":"select","exprs":{"gap":{"fn":"sub","args":[{"col":"ts"},{"col":"ms"}]},"later":{"fn":"add","args":[{"col":"ts"},{"col":"dur"}]},"earlier":{"fn":"sub","args":[{"col":"ts"},{"col":"dur"}]},"days":{"fn":"sub","args":[{"col":"d"},{"lit":18000,"type":"date"}]},"twice":{"fn":"add","args":[{"col":"dur"},{"col":"dur"}]}}}],"expected":{"fields":[{"name":"gap","type":"duration[ms]"},{"name":"later","type":"datetime[us]"},{"name":"earlier","type":"datetime[us]"},{"name":"days","type":"duration[us]"},{"name":"twice","type":"duration[us]"}],"columns":{"gap":[0,-1,null,null,-14182940000,109800000],"later":[1709219707123456,1704067198499000,null,null,-14182940000000,1710142200000500],"earlier":[1709208907123456,1704067201499000,null,null,-14182940000000,1709962200000500],"days":[153964800000000,148780800000000,154310400000000,153964800000000,null,154828800000000],"twice":[10800000000,-3000000,null,518400000002,0,180000000000]}}},{"id":"temporal/truncate","description":"truncate to a calendar unit","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"ts","type":"datetime[us]"},{"name":"ms","type":"datetime[ms]"},{"name":"d","type":"date"},{"name":"dur","type":"duration[us]"}],"columns":{"k":["a","b","c","d","e","f"],"ts":[1709214307123456,1704067199999000,1709510400000000,null,-14182940000000,1710052200000500],"ms":[1709214307123,1704067200000,null,1709251200000,0,1709942400000],"d":[19782,19722,19786,19782,null,19792],"dur":[5400000000,-1500000,null,259200000001,0,90000000000]}},"right":{"fields":[{"name":"d","type":"date"},{"name":"name","type":"utf8"}],"columns":{"d":[19782,19792,20089],"name":["leap","sunday","new year"]}}},"ops":[{"op":"select","exprs":{"h":{"fn":"truncate","args":[{"col":"ts"}],"every":"1h"},"d":{"fn":"truncate","args":[{"col":"ts"}],"every":"1d"},"w":{"fn":"truncate","args":[{"col":"ts"}],"every":"1w"},"mo":{"fn":"truncate","args":[{"col":"ts"}],"every":"1mo"},"y":{"fn":"truncate","args":[{"col":"ts"}],"every":"1y"},"dmo":{"fn":"truncate","args":[{"col":"d"}],"every":"1mo"}}}],"expected":{"fields":[{"name":"h","type":"datetime[us]"},{"name":"d","type":"datetime[us]"},{"name":"w","type":"datetime[us]"},{"name":"mo","type":"datetime[us]"},{"name":"y","type":"datetime[us]"},{"name":"dmo","type":"date"}],"columns":{"h":[1709211600000000,1704063600000000,1709510400000000,null,-14184000000000,1710050400000000],"d":[1709164800000000,1703980800000000,1709510400000000,null,-14256000000000,1710028800000000],"w":[1708905600000000,1703462400000000,1709510400000000,null,-14774400000000,1709510400000000],"mo":[1706745600000000,1701388800000000,1709251200000000,null,-15897600000000,1709251200000000],"y":[1704067200000000,1672531200000000,1704067200000000,null,-31536000000000,1704067200000000],"dmo":[19754,19692,19783,19754,null,19783]}}},{"id":"temporal/duration","description":"durations in whole units","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"ts","type":"datetime[us]"},{"name":"ms","type":"datetime[ms]"},{"name":"d","type":"date"},{"name":"dur","type":"duration[us]"}],"columns":{"k":["a","b","c","d","e","f"],"ts":[1709214307123456,1704067199999000,1709510400000000,null,-14182940000000,1710052200000500],"ms":[1709214307123,1704067200000,null,1709251200000,0,1709942400000],"d":[19782,19722,19786,19782,null,19792],"dur":[5400000000,-1500000,null,259200000001,0,90000000000]}},"right":{"fields":[{"name":"d","type":"date"},{"name":"name","type":"utf8"}],"columns":{"d":[19782,19792,20089],"name":["leap","sunday","new year"]}}},"ops":[{"op":"select","exprs":{"d":{"fn":"totalDays","args":[{"col":"dur"}]},"h":{"fn":"totalHours","args":[{"col":"dur"}]},"m":{"fn":"totalMinutes","args":[{"col":"dur"}]},"s":{"fn":"totalSeconds","args":[{"col":"dur"}]},"ms":{"fn":"totalMilliseconds","args":[{"col":"dur"}]}}}],"expected":{"fields":[{"name":"d","type":"i64"},{"name":"h","type":"i64"},{"name":"m","type":"i64"},{"name":"s","type":"i64"},{"name":"ms","type":"i64"}],"columns":{"d":["0","0",null,"3","0","1"],"h":["1","0",null,"72","0","25"],"m":["90","0",null,"4320","0","1500"],"s":["5400","-1",null,"259200","0","90000"],"ms":["5400000","-1500",null,"259200000","0","90000000"]}}},{"id":"temporal/cast","description":"casts between dates, timestamps and integers","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"ts","type":"datetime[us]"},{"name":"ms","type":"datetime[ms]"},{"name":"d","type":"date"},{"name":"dur","type":"duration[us]"}],"columns":{"k":["a","b","c","d","e","f"],"ts":[1709214307123456,1704067199999000,1709510400000000,null,-14182940000000,1710052200000500],"ms":[1709214307123,1704067200000,null,1709251200000,0,1709942400000],"d":[19782,19722,19786,19782,null,19792],"dur":[5400000000,-1500000,null,259200000001,0,90000000000]}},"right":{"fields":[{"name":"d","type":"date"},{"name":"name","type":"utf8"}],"columns":{"d":[19782,19792,20089],"name":["leap","sunday","new year"]}}},"ops":[{"op":"select","exprs":{"date":{"fn":"cast","args":[{"col":"ts"}],"to":"date"},"ts":{"fn":"cast","args":[{"col":"d"}],"to":"datetime[us]"},"us":{"fn":"cast","args":[{"col":"ts"}],"to":"i64"},"days":{"fn":"cast","args":[{"col":"d"}],"to":"i32"},"ms":{"fn":"cast","args":[{"col":"ts"}],"to":"datetime[ms]"}}}],"expected":{"fields":[{"name":"date","type":"date"},{"name":"ts","type":"datetime[us]"},{"name":"us","type":"i64"},{"name":"days","type":"i32"},{"name":"ms","type":"datetime[ms]"}],"columns":{"date":[19782,19722,19786,null,-165,19792],"ts":[1709164800000000,1703980800000000,1709510400000000,1709164800000000,null,1710028800000000],"us":["1709214307123456","1704067199999000","1709510400000000",null,"-14182940000000","1710052200000500"],"days":[19782,19722,19786,19782,null,19792],"ms":[1709214307123,1704067199999,1709510400000,null,-14182940000,1710052200000]}}},{"id":"temporal/compare","description":"comparisons with timestamp literals and between units","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"ts","type":"datetime[us]"},{"name":"ms","type":"datetime[ms]"},{"name":"d","type":"date"},{"name":"dur","type":"duration[us]"}],"columns":{"k":["a","b","c","d","e","f"],"ts":[1709214307123456,1704067199999000,1709510400000000,null,-14182940000000,1710052200000500],"ms":[1709214307123,1704067200000,null,1709251200000,0,1709942400000],"d":[19782,19722,19786,19782,null,19792],"dur":[5400000000,-1500000,null,259200000001,0,90000000000]}},"right":{"fields":[{"name":"d","type":"date"},{"name":"name","type":"utf8"}],"columns":{"d":[19782,19792,20089],"name":["leap","sunday","new year"]}}},"ops":[{"op":"filter","predicate":{"fn":"and","args":[{"fn":"gte","args":[{"col":"ts"},{"lit":1709251200000000,"type":"datetime[us]"}]},{"fn":"lt","args":[{"col":"ms"},{"col":"ts"}]}]}}],"expected":{"fields":[{"name":"k","type":"utf8"},{"name":"ts","type":"datetime[us]"},{"name":"ms","type":"datetime[ms]"},{"name":"d","type":"date"},{"name":"dur","type":"duration[us]"}],"columns":{"k":["f"],"ts":[1710052200000500],"ms":[1709942400000],"d":[19792],"dur":[90000000000]}}},{"id":"temporal/group-sort","description":"group by date, aggregate timestamps, sort","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"ts","type":"datetime[us]"},{"name":"ms","type":"datetime[ms]"},{"name":"d","type":"date"},{"name":"dur","type":"duration[us]"}],"columns":{"k":["a","b","c","d","e","f"],"ts":[1709214307123456,1704067199999000,1709510400000000,null,-14182940000000,1710052200000500],"ms":[1709214307123,1704067200000,null,1709251200000,0,1709942400000],"d":[19782,19722,19786,19782,null,19792],"dur":[5400000000,-1500000,null,259200000001,0,90000000000]}},"right":{"fields":[{"name":"d","type":"date"},{"name":"name","type":"utf8"}],"columns":{"d":[19782,19792,20089],"name":["leap","sunday","new year"]}}},"ops":[{"op":"groupBy","keys":["d"],"aggs":{"first":{"agg":"min","arg":{"col":"ts"}},"last":{"agg":"max","arg":{"col":"ts"}},"n":{"agg":"len"},"total":{"agg":"sum","arg":{"col":"dur"}}}},{"op":"sort","keys":[{"expr":{"col":"last"},"descending":true}]}],"expected":{"fields":[{"name":"d","type":"date"},{"name":"first","type":"datetime[us]"},{"name":"last","type":"datetime[us]"},{"name":"n","type":"u32"},{"name":"total","type":"duration[us]"}],"columns":{"d":[19792,19786,19782,19722,null],"first":[1710052200000500,1709510400000000,1709214307123456,1704067199999000,-14182940000000],"last":[1710052200000500,1709510400000000,1709214307123456,1704067199999000,-14182940000000],"n":[1,1,2,1,1],"total":[90000000000,0,264600000001,-1500000,0]}}},{"id":"temporal/join","description":"join on a date","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"ts","type":"datetime[us]"},{"name":"ms","type":"datetime[ms]"},{"name":"d","type":"date"},{"name":"dur","type":"duration[us]"}],"columns":{"k":["a","b","c","d","e","f"],"ts":[1709214307123456,1704067199999000,1709510400000000,null,-14182940000000,1710052200000500],"ms":[1709214307123,1704067200000,null,1709251200000,0,1709942400000],"d":[19782,19722,19786,19782,null,19792],"dur":[5400000000,-1500000,null,259200000001,0,90000000000]}},"right":{"fields":[{"name":"d","type":"date"},{"name":"name","type":"utf8"}],"columns":{"d":[19782,19792,20089],"name":["leap","sunday","new year"]}}},"ops":[{"op":"join","right":"right","on":["d"],"how":"left"}],"expected":{"fields":[{"name":"k","type":"utf8"},{"name":"ts","type":"datetime[us]"},{"name":"ms","type":"datetime[ms]"},{"name":"d","type":"date"},{"name":"dur","type":"duration[us]"},{"name":"name","type":"utf8"}],"columns":{"k":["a","b","c","d","e","f"],"ts":[1709214307123456,1704067199999000,1709510400000000,null,-14182940000000,1710052200000500],"ms":[1709214307123,1704067200000,null,1709251200000,0,1709942400000],"d":[19782,19722,19786,19782,null,19792],"dur":[5400000000,-1500000,null,259200000001,0,90000000000],"name":["leap",null,null,"leap",null,"sunday"]}}},{"id":"nested/list-ops","description":"list length, element access and aggregates","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"g","type":"i32"},{"name":"xs","type":"list"},{"name":"ws","type":"list"},{"name":"s","type":"struct"}],"columns":{"k":["a","b","c","d","e"],"g":[1,2,1,2,1],"xs":[[1,2,3],[],null,[4,null,2],[7]],"ws":[["a","b"],["c"],[],null,["z","a"]],"s":[{"a":1,"b":"x"},{"a":null,"b":"y"},null,{"a":4,"b":null},{"a":5,"b":"z"}]}},"right":{"fields":[{"name":"k","type":"utf8"},{"name":"tags","type":"list"}],"columns":{"k":["a","d","x"],"tags":[["t1"],["t2","t3"],[]]}}},"ops":[{"op":"select","exprs":{"n":{"fn":"listLength","args":[{"col":"xs"}]},"first":{"fn":"listGet","args":[{"col":"xs"}],"index":0},"last":{"fn":"listGet","args":[{"col":"xs"}],"index":-1},"oob":{"fn":"listGet","args":[{"col":"xs"}],"index":5},"sum":{"fn":"listSum","args":[{"col":"xs"}]},"mean":{"fn":"listMean","args":[{"col":"xs"}]},"min":{"fn":"listMin","args":[{"col":"xs"}]},"max":{"fn":"listMax","args":[{"col":"ws"}]},"has2":{"fn":"listContains","args":[{"col":"xs"},{"lit":2}]},"hasA":{"fn":"listContains","args":[{"col":"ws"},{"lit":"a"}]}}}],"expected":{"fields":[{"name":"n","type":"u32"},{"name":"first","type":"i32"},{"name":"last","type":"i32"},{"name":"oob","type":"i32"},{"name":"sum","type":"i32"},{"name":"mean","type":"f64"},{"name":"min","type":"i32"},{"name":"max","type":"utf8"},{"name":"has2","type":"bool"},{"name":"hasA","type":"bool"}],"columns":{"n":[3,0,null,3,1],"first":[1,null,null,4,7],"last":[3,null,null,2,7],"oob":[null,null,null,null,null],"sum":[6,0,null,6,7],"mean":[2.0,null,null,3.0,7.0],"min":[1,null,null,2,7],"max":["b","c",null,null,"z"],"has2":[true,false,null,true,false],"hasA":[true,false,false,null,true]}}},{"id":"nested/struct","description":"struct fields and struct construction","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"g","type":"i32"},{"name":"xs","type":"list"},{"name":"ws","type":"list"},{"name":"s","type":"struct"}],"columns":{"k":["a","b","c","d","e"],"g":[1,2,1,2,1],"xs":[[1,2,3],[],null,[4,null,2],[7]],"ws":[["a","b"],["c"],[],null,["z","a"]],"s":[{"a":1,"b":"x"},{"a":null,"b":"y"},null,{"a":4,"b":null},{"a":5,"b":"z"}]}},"right":{"fields":[{"name":"k","type":"utf8"},{"name":"tags","type":"list"}],"columns":{"k":["a","d","x"],"tags":[["t1"],["t2","t3"],[]]}}},"ops":[{"op":"select","exprs":{"a":{"fn":"field","args":[{"col":"s"}],"field":"a"},"b":{"fn":"field","args":[{"col":"s"}],"field":"b"},"pair":{"fn":"struct","args":[{"col":"k"},{"fn":"listLength","args":[{"col":"xs"}]}],"names":["key","size"]}}}],"expected":{"fields":[{"name":"a","type":"i32"},{"name":"b","type":"utf8"},{"name":"pair","type":"struct"}],"columns":{"a":[1,null,null,4,5],"b":["x","y",null,null,"z"],"pair":[{"key":"a","size":3},{"key":"b","size":0},{"key":"c","size":null},{"key":"d","size":3},{"key":"e","size":1}]}}},{"id":"nested/explode","description":"explode lists into rows; a null list gives a null row, an empty list no rows","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"g","type":"i32"},{"name":"xs","type":"list"},{"name":"ws","type":"list"},{"name":"s","type":"struct"}],"columns":{"k":["a","b","c","d","e"],"g":[1,2,1,2,1],"xs":[[1,2,3],[],null,[4,null,2],[7]],"ws":[["a","b"],["c"],[],null,["z","a"]],"s":[{"a":1,"b":"x"},{"a":null,"b":"y"},null,{"a":4,"b":null},{"a":5,"b":"z"}]}},"right":{"fields":[{"name":"k","type":"utf8"},{"name":"tags","type":"list"}],"columns":{"k":["a","d","x"],"tags":[["t1"],["t2","t3"],[]]}}},"ops":[{"op":"pick","columns":["k","xs"]},{"op":"explode","column":"xs"}],"expected":{"fields":[{"name":"k","type":"utf8"},{"name":"xs","type":"i32"}],"columns":{"k":["a","a","a","c","d","d","d","e"],"xs":[1,2,3,null,4,null,2,7]}}},{"id":"nested/unnest","description":"unnest a struct into columns","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"g","type":"i32"},{"name":"xs","type":"list"},{"name":"ws","type":"list"},{"name":"s","type":"struct"}],"columns":{"k":["a","b","c","d","e"],"g":[1,2,1,2,1],"xs":[[1,2,3],[],null,[4,null,2],[7]],"ws":[["a","b"],["c"],[],null,["z","a"]],"s":[{"a":1,"b":"x"},{"a":null,"b":"y"},null,{"a":4,"b":null},{"a":5,"b":"z"}]}},"right":{"fields":[{"name":"k","type":"utf8"},{"name":"tags","type":"list"}],"columns":{"k":["a","d","x"],"tags":[["t1"],["t2","t3"],[]]}}},"ops":[{"op":"pick","columns":["k","s"]},{"op":"unnest","column":"s"}],"expected":{"fields":[{"name":"k","type":"utf8"},{"name":"a","type":"i32"},{"name":"b","type":"utf8"}],"columns":{"k":["a","b","c","d","e"],"a":[1,null,null,4,5],"b":["x","y",null,null,"z"]}}},{"id":"nested/agg-list","description":"collect values into lists per group","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"g","type":"i32"},{"name":"xs","type":"list"},{"name":"ws","type":"list"},{"name":"s","type":"struct"}],"columns":{"k":["a","b","c","d","e"],"g":[1,2,1,2,1],"xs":[[1,2,3],[],null,[4,null,2],[7]],"ws":[["a","b"],["c"],[],null,["z","a"]],"s":[{"a":1,"b":"x"},{"a":null,"b":"y"},null,{"a":4,"b":null},{"a":5,"b":"z"}]}},"right":{"fields":[{"name":"k","type":"utf8"},{"name":"tags","type":"list"}],"columns":{"k":["a","d","x"],"tags":[["t1"],["t2","t3"],[]]}}},"ops":[{"op":"groupBy","keys":["g"],"aggs":{"ks":{"agg":"list","arg":{"col":"k"}},"firsts":{"agg":"first","arg":{"col":"xs"}},"n":{"agg":"len"}}}],"expected":{"fields":[{"name":"g","type":"i32"},{"name":"ks","type":"list"},{"name":"firsts","type":"list"},{"name":"n","type":"u32"}],"columns":{"g":[1,2],"ks":[["a","c","e"],["b","d"]],"firsts":[[1,2,3],[]],"n":[3,2]}}},{"id":"nested/carry","description":"lists and structs survive filters, sorts, joins and slices","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"g","type":"i32"},{"name":"xs","type":"list"},{"name":"ws","type":"list"},{"name":"s","type":"struct"}],"columns":{"k":["a","b","c","d","e"],"g":[1,2,1,2,1],"xs":[[1,2,3],[],null,[4,null,2],[7]],"ws":[["a","b"],["c"],[],null,["z","a"]],"s":[{"a":1,"b":"x"},{"a":null,"b":"y"},null,{"a":4,"b":null},{"a":5,"b":"z"}]}},"right":{"fields":[{"name":"k","type":"utf8"},{"name":"tags","type":"list"}],"columns":{"k":["a","d","x"],"tags":[["t1"],["t2","t3"],[]]}}},"ops":[{"op":"filter","predicate":{"fn":"neq","args":[{"col":"k"},{"lit":"c"}]}},{"op":"sort","keys":[{"expr":{"col":"g"},"descending":true},{"expr":{"col":"k"}}]},{"op":"join","right":"right","on":["k"],"how":"left"},{"op":"head","n":4}],"expected":{"fields":[{"name":"k","type":"utf8"},{"name":"g","type":"i32"},{"name":"xs","type":"list"},{"name":"ws","type":"list"},{"name":"s","type":"struct"},{"name":"tags","type":"list"}],"columns":{"k":["b","d","a","e"],"g":[2,2,1,1],"xs":[[],[4,null,2],[1,2,3],[7]],"ws":[["c"],null,["a","b"],["z","a"]],"s":[{"a":null,"b":"y"},{"a":4,"b":null},{"a":1,"b":"x"},{"a":5,"b":"z"}],"tags":[null,["t2","t3"],["t1"],null]}}},{"id":"decimal/arith","description":"decimal addition, subtraction and multiplication","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"price","type":"decimal[10,2]"},{"name":"fee","type":"decimal[6,3]"},{"name":"qty","type":"decimal[4,0]"},{"name":"x","type":"f64"}],"columns":{"k":["a","b","a","c","b","a"],"price":["12.34","0.10","-5.00",null,"99999999.99","10.01"],"fee":["0.125","1.000",null,"2.500","-0.001","10.010"],"qty":["3","10","2","1",null,"7"],"x":[1.005,-2.5,123.456,null,1000000.0,0.0]}}},"ops":[{"op":"select","exprs":{"sum":{"fn":"add","args":[{"col":"price"},{"col":"fee"}]},"diff":{"fn":"sub","args":[{"col":"price"},{"col":"fee"}]},"prod":{"fn":"mul","args":[{"col":"price"},{"col":"qty"}]},"plus":{"fn":"add","args":[{"col":"price"},{"lit":"0.5","type":"decimal[3,1]"}]}}}],"expected":{"fields":[{"name":"sum","type":"decimal[38,3]"},{"name":"diff","type":"decimal[38,3]"},{"name":"prod","type":"decimal[38,2]"},{"name":"plus","type":"decimal[38,2]"}],"columns":{"sum":["12.465","1.100",null,null,"99999999.989","20.020"],"diff":["12.215","-0.900",null,null,"99999999.991","0.000"],"prod":["37.02","1.00","-10.00",null,null,"70.07"],"plus":["12.84","0.60","-4.50",null,"100000000.49","10.51"]}}},{"id":"decimal/compare","description":"decimal comparisons, also across scales","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"price","type":"decimal[10,2]"},{"name":"fee","type":"decimal[6,3]"},{"name":"qty","type":"decimal[4,0]"},{"name":"x","type":"f64"}],"columns":{"k":["a","b","a","c","b","a"],"price":["12.34","0.10","-5.00",null,"99999999.99","10.01"],"fee":["0.125","1.000",null,"2.500","-0.001","10.010"],"qty":["3","10","2","1",null,"7"],"x":[1.005,-2.5,123.456,null,1000000.0,0.0]}}},"ops":[{"op":"filter","predicate":{"fn":"and","args":[{"fn":"gt","args":[{"col":"price"},{"lit":"10.00","type":"decimal[10,2]"}]},{"fn":"gte","args":[{"col":"price"},{"col":"fee"}]}]}}],"expected":{"fields":[{"name":"k","type":"utf8"},{"name":"price","type":"decimal[10,2]"},{"name":"fee","type":"decimal[6,3]"},{"name":"qty","type":"decimal[4,0]"},{"name":"x","type":"f64"}],"columns":{"k":["a","b","a"],"price":["12.34","99999999.99","10.01"],"fee":["0.125","-0.001","10.010"],"qty":["3",null,"7"],"x":[1.005,1000000.0,0.0]}}},{"id":"decimal/aggregate","description":"sums, extremes and means of decimals per group","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"price","type":"decimal[10,2]"},{"name":"fee","type":"decimal[6,3]"},{"name":"qty","type":"decimal[4,0]"},{"name":"x","type":"f64"}],"columns":{"k":["a","b","a","c","b","a"],"price":["12.34","0.10","-5.00",null,"99999999.99","10.01"],"fee":["0.125","1.000",null,"2.500","-0.001","10.010"],"qty":["3","10","2","1",null,"7"],"x":[1.005,-2.5,123.456,null,1000000.0,0.0]}}},"ops":[{"op":"groupBy","keys":["k"],"aggs":{"total":{"agg":"sum","arg":{"col":"price"}},"min":{"agg":"min","arg":{"col":"price"}},"max":{"agg":"max","arg":{"col":"fee"}},"mean":{"agg":"mean","arg":{"col":"price"}},"n":{"agg":"count","arg":{"col":"price"}}}}],"expected":{"fields":[{"name":"k","type":"utf8"},{"name":"total","type":"decimal[38,2]"},{"name":"min","type":"decimal[10,2]"},{"name":"max","type":"decimal[6,3]"},{"name":"mean","type":"f64"},{"name":"n","type":"u32"}],"columns":{"k":["a","b","c"],"total":["17.35","100000000.09","0.00"],"min":["-5.00","0.10",null],"max":["10.010","1.000","2.500"],"mean":[5.783333333333334,50000000.044999994,null],"n":[3,2,0]}}},{"id":"decimal/cast","description":"casts between decimals, floats, integers and strings","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"price","type":"decimal[10,2]"},{"name":"fee","type":"decimal[6,3]"},{"name":"qty","type":"decimal[4,0]"},{"name":"x","type":"f64"}],"columns":{"k":["a","b","a","c","b","a"],"price":["12.34","0.10","-5.00",null,"99999999.99","10.01"],"fee":["0.125","1.000",null,"2.500","-0.001","10.010"],"qty":["3","10","2","1",null,"7"],"x":[1.005,-2.5,123.456,null,1000000.0,0.0]}}},"ops":[{"op":"select","exprs":{"f":{"fn":"cast","args":[{"col":"price"}],"to":"f64"},"i":{"fn":"cast","args":[{"col":"price"}],"to":"i32"},"s":{"fn":"cast","args":[{"col":"fee"}],"to":"utf8"},"d":{"fn":"cast","args":[{"col":"x"}],"to":"decimal[10,2]"},"r":{"fn":"cast","args":[{"col":"price"}],"to":"decimal[12,1]"}}}],"expected":{"fields":[{"name":"f","type":"f64"},{"name":"i","type":"i32"},{"name":"s","type":"utf8"},{"name":"d","type":"decimal[10,2]"},{"name":"r","type":"decimal[12,1]"}],"columns":{"f":[12.34,0.1,-5.0,null,99999999.99,10.01],"i":[12,0,-5,null,99999999,10],"s":["0.125","1.000",null,"2.500","-0.001","10.010"],"d":["1.00","-2.50","123.46",null,"1000000.00","0.00"],"r":["12.3","0.1","-5.0",null,"100000000.0","10.0"]}}},{"id":"decimal/sort","description":"sort by a decimal","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"price","type":"decimal[10,2]"},{"name":"fee","type":"decimal[6,3]"},{"name":"qty","type":"decimal[4,0]"},{"name":"x","type":"f64"}],"columns":{"k":["a","b","a","c","b","a"],"price":["12.34","0.10","-5.00",null,"99999999.99","10.01"],"fee":["0.125","1.000",null,"2.500","-0.001","10.010"],"qty":["3","10","2","1",null,"7"],"x":[1.005,-2.5,123.456,null,1000000.0,0.0]}}},"ops":[{"op":"sort","keys":[{"expr":{"col":"price"},"descending":true}],"nullsLast":true}],"expected":{"fields":[{"name":"k","type":"utf8"},{"name":"price","type":"decimal[10,2]"},{"name":"fee","type":"decimal[6,3]"},{"name":"qty","type":"decimal[4,0]"},{"name":"x","type":"f64"}],"columns":{"k":["b","a","a","b","a","c"],"price":["99999999.99","12.34","10.01","0.10","-5.00",null],"fee":["-0.001","0.125","10.010","1.000",null,"2.500"],"qty":[null,"3","7","10","2","1"],"x":[1000000.0,1.005,0.0,-2.5,123.456,null]}}},{"id":"decimal/round-ties","description":"narrowing the scale rounds exact ties","inputs":{"left":{"fields":[{"name":"t","type":"decimal[6,3]"}],"columns":{"t":["0.250","0.350","-0.250","-0.350","1.050","2.500","0.249"]}}},"ops":[{"op":"select","exprs":{"one":{"fn":"cast","args":[{"col":"t"}],"to":"decimal[6,1]"},"zero":{"fn":"cast","args":[{"col":"t"}],"to":"decimal[6,0]"}}}],"expected":{"fields":[{"name":"one","type":"decimal[6,1]"},{"name":"zero","type":"decimal[6,0]"}],"columns":{"one":["0.2","0.4","-0.2","-0.4","1.0","2.5","0.2"],"zero":["0","0","0","0","1","2","0"]}}},{"id":"decimal/cast-float-ties","description":"floats cast to decimals round exact ties half to even, also below zero","inputs":{"left":{"fields":[{"name":"x","type":"f64"}],"columns":{"x":[-1.5,-2.5,1.5,2.5,-0.75]}}},"ops":[{"op":"select","exprs":{"zero":{"fn":"cast","args":[{"col":"x"}],"to":"decimal[18,0]"},"one":{"fn":"cast","args":[{"col":"x"}],"to":"decimal[18,1]"}}}],"expected":{"fields":[{"name":"zero","type":"decimal[18,0]"},{"name":"one","type":"decimal[18,1]"}],"columns":{"zero":["-2","-2","2","2","-1"],"one":["-1.5","-2.5","1.5","2.5","-0.8"]}}},{"id":"decimal/round","description":"rounding decimals keeps the scale and rounds half to even","inputs":{"left":{"fields":[{"name":"t","type":"decimal[6,3]"}],"columns":{"t":["0.250","0.350","-0.250","-0.350","1.050","2.500","0.249",null]}}},"ops":[{"op":"select","exprs":{"r0":{"fn":"round","args":[{"col":"t"}]},"r1":{"fn":"round","args":[{"col":"t"}],"decimals":1},"r2":{"fn":"round","args":[{"col":"t"}],"decimals":2},"r3":{"fn":"round","args":[{"col":"t"}],"decimals":3}}}],"expected":{"fields":[{"name":"r0","type":"decimal[7,3]"},{"name":"r1","type":"decimal[7,3]"},{"name":"r2","type":"decimal[7,3]"},{"name":"r3","type":"decimal[6,3]"}],"columns":{"r0":["0.000","0.000","0.000","0.000","1.000","2.000","0.000",null],"r1":["0.200","0.400","-0.200","-0.400","1.000","2.500","0.200",null],"r2":["0.250","0.350","-0.250","-0.350","1.050","2.500","0.250",null],"r3":["0.250","0.350","-0.250","-0.350","1.050","2.500","0.249",null]}}},{"id":"arith/u32-divide-by-zero","description":"unsigned 32-bit floor division and modulo by zero give null","inputs":{"left":{"fields":[{"name":"n","type":"u32"},{"name":"d","type":"u32"}],"columns":{"n":[7,0,5],"d":[0,0,2]}}},"ops":[{"op":"select","exprs":{"q":{"fn":"floorDiv","args":[{"col":"n"},{"col":"d"}]},"m":{"fn":"mod","args":[{"col":"n"},{"col":"d"}]},"qs":{"fn":"floorDiv","args":[{"col":"n"},{"lit":0,"type":"u32"}]},"ms":{"fn":"mod","args":[{"col":"n"},{"lit":0,"type":"u32"}]}}}],"expected":{"fields":[{"name":"q","type":"u32"},{"name":"m","type":"u32"},{"name":"qs","type":"u32"},{"name":"ms","type":"u32"}],"columns":{"q":[null,null,2],"m":[null,null,1],"qs":[null,null,null],"ms":[null,null,null]}}},{"id":"slice/clip-5+1","description":"slices reaching before the first row clip from the unclamped start","inputs":{"left":{"fields":[{"name":"x","type":"i32"}],"columns":{"x":[1,2,3]}}},"ops":[{"op":"slice","offset":-5,"length":1}],"expected":{"fields":[{"name":"x","type":"i32"}],"columns":{"x":[]}}},{"id":"slice/clip-5+3","description":"slices reaching before the first row clip from the unclamped start","inputs":{"left":{"fields":[{"name":"x","type":"i32"}],"columns":{"x":[1,2,3]}}},"ops":[{"op":"slice","offset":-5,"length":3}],"expected":{"fields":[{"name":"x","type":"i32"}],"columns":{"x":[1]}}},{"id":"slice/clip-5+4","description":"slices reaching before the first row clip from the unclamped start","inputs":{"left":{"fields":[{"name":"x","type":"i32"}],"columns":{"x":[1,2,3]}}},"ops":[{"op":"slice","offset":-5,"length":4}],"expected":{"fields":[{"name":"x","type":"i32"}],"columns":{"x":[1,2]}}},{"id":"slice/clip-2+5","description":"slices reaching before the first row clip from the unclamped start","inputs":{"left":{"fields":[{"name":"x","type":"i32"}],"columns":{"x":[1,2,3]}}},"ops":[{"op":"slice","offset":-2,"length":5}],"expected":{"fields":[{"name":"x","type":"i32"}],"columns":{"x":[2,3]}}},{"id":"slice/clip-5","description":"slices reaching before the first row clip from the unclamped start","inputs":{"left":{"fields":[{"name":"x","type":"i32"}],"columns":{"x":[1,2,3]}}},"ops":[{"op":"slice","offset":-5}],"expected":{"fields":[{"name":"x","type":"i32"}],"columns":{"x":[1,2,3]}}},{"id":"slice/clip1+0","description":"slices reaching before the first row clip from the unclamped start","inputs":{"left":{"fields":[{"name":"x","type":"i32"}],"columns":{"x":[1,2,3]}}},"ops":[{"op":"slice","offset":1,"length":0}],"expected":{"fields":[{"name":"x","type":"i32"}],"columns":{"x":[]}}},{"id":"window/filter-after-sort","description":"a window predicate sees the sorted rows","inputs":{"left":{"fields":[{"name":"g","type":"i32"},{"name":"h","type":"i32"},{"name":"x","type":"f64"}],"columns":{"g":[1,2,1,2],"h":[1,1,2,2],"x":[3.0,1.0,2.0,4.0]}},"right":{"fields":[{"name":"g","type":"i32"},{"name":"name","type":"utf8"}],"columns":{"g":[1],"name":["one"]}}},"ops":[{"op":"sort","keys":[{"expr":{"col":"x"}}]},{"op":"filter","predicate":{"fn":"gt","args":[{"fn":"cumSum","args":[{"col":"x"}]},{"lit":3.0}]}}],"expected":{"fields":[{"name":"g","type":"i32"},{"name":"h","type":"i32"},{"name":"x","type":"f64"}],"columns":{"g":[1,2],"h":[1,2],"x":[3.0,4.0]}}},{"id":"window/filter-after-filter","description":"a window predicate sees only the rows an earlier filter kept","inputs":{"left":{"fields":[{"name":"g","type":"i32"},{"name":"h","type":"i32"},{"name":"x","type":"f64"}],"columns":{"g":[1,2,1,2],"h":[1,1,2,2],"x":[3.0,1.0,2.0,4.0]}},"right":{"fields":[{"name":"g","type":"i32"},{"name":"name","type":"utf8"}],"columns":{"g":[1],"name":["one"]}}},"ops":[{"op":"filter","predicate":{"fn":"gt","args":[{"col":"x"},{"lit":1.0}]}},{"op":"filter","predicate":{"fn":"gt","args":[{"fn":"shift","args":[{"col":"x"}]},{"lit":2.0}]}}],"expected":{"fields":[{"name":"g","type":"i32"},{"name":"h","type":"i32"},{"name":"x","type":"f64"}],"columns":{"g":[1],"h":[2],"x":[2.0]}}},{"id":"window/filter-after-concat","description":"a window predicate sees every input of a concat","inputs":{"left":{"fields":[{"name":"g","type":"i32"},{"name":"h","type":"i32"},{"name":"x","type":"f64"}],"columns":{"g":[1,2,1,2],"h":[1,1,2,2],"x":[3.0,1.0,2.0,4.0]}},"right":{"fields":[{"name":"g","type":"i32"},{"name":"name","type":"utf8"}],"columns":{"g":[1],"name":["one"]}}},"ops":[{"op":"concat","with":"left"},{"op":"filter","predicate":{"fn":"gt","args":[{"fn":"cumSum","args":[{"col":"x"}]},{"lit":15.0}]}}],"expected":{"fields":[{"name":"g","type":"i32"},{"name":"h","type":"i32"},{"name":"x","type":"f64"}],"columns":{"g":[1,2],"h":[2,2],"x":[2.0,4.0]}}},{"id":"window/filter-after-join","description":"a window predicate sees only the joined rows","inputs":{"left":{"fields":[{"name":"g","type":"i32"},{"name":"h","type":"i32"},{"name":"x","type":"f64"}],"columns":{"g":[1,2,1,2],"h":[1,1,2,2],"x":[3.0,1.0,2.0,4.0]}},"right":{"fields":[{"name":"g","type":"i32"},{"name":"name","type":"utf8"}],"columns":{"g":[1],"name":["one"]}}},"ops":[{"op":"join","right":"right","on":["g"],"how":"inner"},{"op":"filter","predicate":{"fn":"gt","args":[{"fn":"shift","args":[{"col":"x"}]},{"lit":2.0}]}}],"expected":{"fields":[{"name":"g","type":"i32"},{"name":"h","type":"i32"},{"name":"x","type":"f64"},{"name":"name","type":"utf8"}],"columns":{"g":[1],"h":[2],"x":[2.0],"name":["one"]}}},{"id":"window/filter-whole","description":"a window predicate sees the whole input","inputs":{"left":{"fields":[{"name":"g","type":"i32"},{"name":"h","type":"i32"},{"name":"x","type":"f64"}],"columns":{"g":[1,2,1,2],"h":[1,1,2,2],"x":[3.0,1.0,2.0,4.0]}},"right":{"fields":[{"name":"g","type":"i32"},{"name":"name","type":"utf8"}],"columns":{"g":[1],"name":["one"]}}},"ops":[{"op":"filter","predicate":{"fn":"gt","args":[{"fn":"cumSum","args":[{"col":"x"}]},{"lit":5.0}]}}],"expected":{"fields":[{"name":"g","type":"i32"},{"name":"h","type":"i32"},{"name":"x","type":"f64"}],"columns":{"g":[1,2],"h":[2,2],"x":[2.0,4.0]}}},{"id":"window/agg-of-window","description":"an aggregate of a window sees the whole input","inputs":{"left":{"fields":[{"name":"g","type":"i32"},{"name":"h","type":"i32"},{"name":"x","type":"f64"}],"columns":{"g":[1,2,1,2],"h":[1,1,2,2],"x":[3.0,1.0,2.0,4.0]}},"right":{"fields":[{"name":"g","type":"i32"},{"name":"name","type":"utf8"}],"columns":{"g":[1],"name":["one"]}}},"ops":[{"op":"agg","aggs":{"s":{"agg":"sum","arg":{"fn":"cumSum","args":[{"col":"x"}]}},"n":{"agg":"len"}}}],"expected":{"fields":[{"name":"s","type":"f64"},{"name":"n","type":"u32"}],"columns":{"s":[23.0],"n":[4]}}},{"id":"window/grouped-agg-of-window","description":"windows inside grouped aggregates run per group","inputs":{"left":{"fields":[{"name":"g","type":"i32"},{"name":"h","type":"i32"},{"name":"x","type":"f64"}],"columns":{"g":[1,2,1,2],"h":[1,1,2,2],"x":[3.0,1.0,2.0,4.0]}},"right":{"fields":[{"name":"g","type":"i32"},{"name":"name","type":"utf8"}],"columns":{"g":[1],"name":["one"]}}},"ops":[{"op":"groupBy","keys":["g"],"aggs":{"s":{"agg":"sum","arg":{"fn":"cumSum","args":[{"col":"x"}]}},"l":{"agg":"list","arg":{"fn":"cumSum","args":[{"col":"x"}]}},"o":{"agg":"sum","arg":{"over":{"agg":"sum","arg":{"col":"x"}},"partitionBy":[{"col":"h"}]}}}}],"expected":{"fields":[{"name":"g","type":"i32"},{"name":"s","type":"f64"},{"name":"l","type":"list"},{"name":"o","type":"f64"}],"columns":{"g":[1,2],"s":[8.0,6.0],"l":[[3.0,5.0],[1.0,5.0]],"o":[5.0,5.0]}}}]} +{"generator":"generate_gold.py","polars":"2.1.0","deviations":["JavaScript has no float literal distinct from an integer: an integral number literal behaves like a Polars integer literal (it takes the integer type of the column it meets), so cases never combine an integer column with an integral float literal.","Float to string casts use JavaScript number formatting (Polars prints 5.0 as '5.0', JavaScript as '5').","Joins never add '_right' suffixes: overlapping non-key columns are a type error.","Arithmetic between 64-bit integers (bigint) and other numeric types requires an explicit cast."],"cases":[{"id":"window/cum","description":"cumulative sum, min, max and count; nulls stay null","inputs":{"left":{"fields":[{"name":"g","type":"utf8"},{"name":"t","type":"i32"},{"name":"x","type":"f64"},{"name":"i","type":"i32"},{"name":"b","type":"bool"}],"columns":{"g":["a","b","a","a","b",null,"a","b","b","a"],"t":[1,1,2,3,2,1,4,3,4,5],"x":[1.5,2.0,null,4.0,"NaN",3.0,-1.0,2.0,0.5,2.0],"i":[3,1,4,1,5,9,null,6,5,3],"b":[true,false,null,true,true,false,true,null,false,true]}}},"ops":[{"op":"select","exprs":{"sx":{"fn":"cumSum","args":[{"col":"x"}]},"si":{"fn":"cumSum","args":[{"col":"i"}]},"mn":{"fn":"cumMin","args":[{"col":"i"}]},"mx":{"fn":"cumMax","args":[{"col":"x"}]},"n":{"fn":"cumCount","args":[{"col":"x"}]},"sb":{"fn":"cumSum","args":[{"col":"b"}]}}}],"expected":{"fields":[{"name":"sx","type":"f64"},{"name":"si","type":"i32"},{"name":"mn","type":"i32"},{"name":"mx","type":"f64"},{"name":"n","type":"u32"},{"name":"sb","type":"u32"}],"columns":{"sx":[1.5,3.5,null,7.5,"NaN","NaN","NaN","NaN","NaN","NaN"],"si":[3,4,8,9,14,23,null,29,34,37],"mn":[3,1,1,1,1,1,null,1,1,1],"mx":[1.5,2.0,null,4.0,4.0,4.0,4.0,4.0,4.0,4.0],"n":[1,2,2,3,4,5,6,7,8,9],"sb":[1,1,null,2,3,3,4,null,4,5]}}},{"id":"window/shift-diff","description":"shift (lag and lead) and diff","inputs":{"left":{"fields":[{"name":"g","type":"utf8"},{"name":"t","type":"i32"},{"name":"x","type":"f64"},{"name":"i","type":"i32"},{"name":"b","type":"bool"}],"columns":{"g":["a","b","a","a","b",null,"a","b","b","a"],"t":[1,1,2,3,2,1,4,3,4,5],"x":[1.5,2.0,null,4.0,"NaN",3.0,-1.0,2.0,0.5,2.0],"i":[3,1,4,1,5,9,null,6,5,3],"b":[true,false,null,true,true,false,true,null,false,true]}}},"ops":[{"op":"select","exprs":{"lag":{"fn":"shift","args":[{"col":"i"}]},"lead":{"fn":"shift","args":[{"col":"x"}],"n":-2},"d":{"fn":"diff","args":[{"col":"i"}]},"d2":{"fn":"diff","args":[{"col":"x"}],"n":2},"ls":{"fn":"shift","args":[{"col":"g"}]}}}],"expected":{"fields":[{"name":"lag","type":"i32"},{"name":"lead","type":"f64"},{"name":"d","type":"i32"},{"name":"d2","type":"f64"},{"name":"ls","type":"utf8"}],"columns":{"lag":[null,3,1,4,1,5,9,null,6,5],"lead":[null,4.0,"NaN",3.0,-1.0,2.0,0.5,2.0,null,null],"d":[null,-2,3,-3,4,4,null,null,-1,-2],"d2":[null,null,null,2.0,null,-1.0,"NaN",-1.0,1.5,0.0],"ls":[null,"a","b","a","a","b",null,"a","b","b"]}}},{"id":"window/rank","description":"rank methods, nulls and NaN","inputs":{"left":{"fields":[{"name":"g","type":"utf8"},{"name":"t","type":"i32"},{"name":"x","type":"f64"},{"name":"i","type":"i32"},{"name":"b","type":"bool"}],"columns":{"g":["a","b","a","a","b",null,"a","b","b","a"],"t":[1,1,2,3,2,1,4,3,4,5],"x":[1.5,2.0,null,4.0,"NaN",3.0,-1.0,2.0,0.5,2.0],"i":[3,1,4,1,5,9,null,6,5,3],"b":[true,false,null,true,true,false,true,null,false,true]}}},"ops":[{"op":"select","exprs":{"avg":{"fn":"rank","args":[{"col":"x"}]},"min":{"fn":"rank","args":[{"col":"i"}],"method":"min"},"max":{"fn":"rank","args":[{"col":"i"}],"method":"max"},"dense":{"fn":"rank","args":[{"col":"i"}],"method":"dense","descending":true},"ord":{"fn":"rank","args":[{"col":"x"}],"method":"ordinal"},"s":{"fn":"rank","args":[{"col":"g"}],"method":"dense"}}}],"expected":{"fields":[{"name":"avg","type":"f64"},{"name":"min","type":"u32"},{"name":"max","type":"u32"},{"name":"dense","type":"u32"},{"name":"ord","type":"u32"},{"name":"s","type":"u32"}],"columns":{"avg":[3.0,5.0,null,8.0,9.0,7.0,1.0,5.0,2.0,5.0],"min":[3,1,5,1,6,9,null,8,6,3],"max":[4,2,5,2,7,9,null,8,7,4],"dense":[5,6,4,6,3,1,null,2,3,5],"ord":[3,4,null,8,9,7,1,5,2,6],"s":[1,2,1,1,2,null,1,2,2,1]}}},{"id":"window/rolling","description":"rolling sum, mean, min and max","inputs":{"left":{"fields":[{"name":"g","type":"utf8"},{"name":"t","type":"i32"},{"name":"x","type":"f64"},{"name":"i","type":"i32"},{"name":"b","type":"bool"}],"columns":{"g":["a","b","a","a","b",null,"a","b","b","a"],"t":[1,1,2,3,2,1,4,3,4,5],"x":[1.5,2.0,null,4.0,"NaN",3.0,-1.0,2.0,0.5,2.0],"i":[3,1,4,1,5,9,null,6,5,3],"b":[true,false,null,true,true,false,true,null,false,true]}}},"ops":[{"op":"select","exprs":{"s":{"fn":"rollingSum","args":[{"col":"x"}],"window":3},"m":{"fn":"rollingMean","args":[{"col":"i"}],"window":2},"mn":{"fn":"rollingMin","args":[{"col":"x"}],"window":3,"minPeriods":1},"mx":{"fn":"rollingMax","args":[{"col":"i"}],"window":4,"minPeriods":2}}}],"expected":{"fields":[{"name":"s","type":"f64"},{"name":"m","type":"f64"},{"name":"mn","type":"f64"},{"name":"mx","type":"i32"}],"columns":{"s":[null,null,null,null,null,"NaN","NaN",4.0,1.5,4.5],"m":[null,2.0,2.5,2.5,3.0,7.0,null,null,5.5,4.0],"mn":[1.5,1.5,1.5,2.0,"NaN","NaN","NaN",-1.0,-1.0,0.5],"mx":[null,3,4,4,5,9,9,9,9,6]}}},{"id":"window/over-agg","description":"aggregates broadcast over partitions","inputs":{"left":{"fields":[{"name":"g","type":"utf8"},{"name":"t","type":"i32"},{"name":"x","type":"f64"},{"name":"i","type":"i32"},{"name":"b","type":"bool"}],"columns":{"g":["a","b","a","a","b",null,"a","b","b","a"],"t":[1,1,2,3,2,1,4,3,4,5],"x":[1.5,2.0,null,4.0,"NaN",3.0,-1.0,2.0,0.5,2.0],"i":[3,1,4,1,5,9,null,6,5,3],"b":[true,false,null,true,true,false,true,null,false,true]}}},"ops":[{"op":"withColumns","exprs":{"total":{"over":{"agg":"sum","arg":{"col":"x"}},"partitionBy":[{"col":"g"}]},"n":{"over":{"agg":"len"},"partitionBy":[{"col":"g"}]},"mean":{"over":{"agg":"mean","arg":{"col":"i"}},"partitionBy":[{"col":"g"}]},"first":{"over":{"agg":"first","arg":{"col":"t"}},"partitionBy":[{"col":"g"},{"col":"b"}]}}}],"expected":{"fields":[{"name":"g","type":"utf8"},{"name":"t","type":"i32"},{"name":"x","type":"f64"},{"name":"i","type":"i32"},{"name":"b","type":"bool"},{"name":"total","type":"f64"},{"name":"n","type":"u32"},{"name":"mean","type":"f64"},{"name":"first","type":"i32"}],"columns":{"g":["a","b","a","a","b",null,"a","b","b","a"],"t":[1,1,2,3,2,1,4,3,4,5],"x":[1.5,2.0,null,4.0,"NaN",3.0,-1.0,2.0,0.5,2.0],"i":[3,1,4,1,5,9,null,6,5,3],"b":[true,false,null,true,true,false,true,null,false,true],"total":[6.5,"NaN",6.5,6.5,"NaN",3.0,6.5,"NaN","NaN",6.5],"n":[5,4,5,5,4,1,5,4,4,5],"mean":[2.75,4.25,2.75,2.75,4.25,9.0,2.75,4.25,4.25,2.75],"first":[1,1,2,1,2,1,1,3,1,1]}}},{"id":"window/over-cum","description":"cumulative and shifted values per partition, in row order","inputs":{"left":{"fields":[{"name":"g","type":"utf8"},{"name":"t","type":"i32"},{"name":"x","type":"f64"},{"name":"i","type":"i32"},{"name":"b","type":"bool"}],"columns":{"g":["a","b","a","a","b",null,"a","b","b","a"],"t":[1,1,2,3,2,1,4,3,4,5],"x":[1.5,2.0,null,4.0,"NaN",3.0,-1.0,2.0,0.5,2.0],"i":[3,1,4,1,5,9,null,6,5,3],"b":[true,false,null,true,true,false,true,null,false,true]}}},"ops":[{"op":"withColumns","exprs":{"cs":{"over":{"fn":"cumSum","args":[{"col":"i"}]},"partitionBy":[{"col":"g"}]},"prev":{"over":{"fn":"shift","args":[{"col":"x"}]},"partitionBy":[{"col":"g"}]},"d":{"over":{"fn":"diff","args":[{"col":"t"}]},"partitionBy":[{"col":"g"}]},"r":{"over":{"fn":"rank","args":[{"col":"x"}],"method":"min"},"partitionBy":[{"col":"g"}]},"roll":{"over":{"fn":"rollingSum","args":[{"col":"i"}],"window":2,"minPeriods":1},"partitionBy":[{"col":"g"}]}}}],"expected":{"fields":[{"name":"g","type":"utf8"},{"name":"t","type":"i32"},{"name":"x","type":"f64"},{"name":"i","type":"i32"},{"name":"b","type":"bool"},{"name":"cs","type":"i32"},{"name":"prev","type":"f64"},{"name":"d","type":"i32"},{"name":"r","type":"u32"},{"name":"roll","type":"i32"}],"columns":{"g":["a","b","a","a","b",null,"a","b","b","a"],"t":[1,1,2,3,2,1,4,3,4,5],"x":[1.5,2.0,null,4.0,"NaN",3.0,-1.0,2.0,0.5,2.0],"i":[3,1,4,1,5,9,null,6,5,3],"b":[true,false,null,true,true,false,true,null,false,true],"cs":[3,1,7,8,6,9,null,12,17,11],"prev":[null,null,1.5,null,2.0,null,4.0,"NaN",2.0,-1.0],"d":[null,null,1,1,1,null,1,1,1,1],"r":[2,2,null,4,4,1,1,2,1,3],"roll":[3,1,7,5,6,9,1,11,11,3]}}},{"id":"window/over-filter","description":"filters after windows see the windowed values","inputs":{"left":{"fields":[{"name":"g","type":"utf8"},{"name":"t","type":"i32"},{"name":"x","type":"f64"},{"name":"i","type":"i32"},{"name":"b","type":"bool"}],"columns":{"g":["a","b","a","a","b",null,"a","b","b","a"],"t":[1,1,2,3,2,1,4,3,4,5],"x":[1.5,2.0,null,4.0,"NaN",3.0,-1.0,2.0,0.5,2.0],"i":[3,1,4,1,5,9,null,6,5,3],"b":[true,false,null,true,true,false,true,null,false,true]}}},"ops":[{"op":"withColumns","exprs":{"cs":{"over":{"fn":"cumSum","args":[{"col":"t"}]},"partitionBy":[{"col":"g"}]}}},{"op":"filter","predicate":{"fn":"gt","args":[{"col":"t"},{"lit":2}]}}],"expected":{"fields":[{"name":"g","type":"utf8"},{"name":"t","type":"i32"},{"name":"x","type":"f64"},{"name":"i","type":"i32"},{"name":"b","type":"bool"},{"name":"cs","type":"i32"}],"columns":{"g":["a","a","b","b","a"],"t":[3,4,3,4,5],"x":[4.0,-1.0,2.0,0.5,2.0],"i":[1,null,6,5,3],"b":[true,true,null,false,true],"cs":[6,10,6,10,15]}}},{"id":"window/filter-first","description":"windows after a filter only see the remaining rows","inputs":{"left":{"fields":[{"name":"g","type":"utf8"},{"name":"t","type":"i32"},{"name":"x","type":"f64"},{"name":"i","type":"i32"},{"name":"b","type":"bool"}],"columns":{"g":["a","b","a","a","b",null,"a","b","b","a"],"t":[1,1,2,3,2,1,4,3,4,5],"x":[1.5,2.0,null,4.0,"NaN",3.0,-1.0,2.0,0.5,2.0],"i":[3,1,4,1,5,9,null,6,5,3],"b":[true,false,null,true,true,false,true,null,false,true]}}},"ops":[{"op":"filter","predicate":{"fn":"isNotNull","args":[{"col":"x"}]}},{"op":"withColumns","exprs":{"cs":{"fn":"cumSum","args":[{"col":"x"}]},"r":{"over":{"fn":"rank","args":[{"col":"x"}]},"partitionBy":[{"col":"g"}]}}},{"op":"head","n":4}],"expected":{"fields":[{"name":"g","type":"utf8"},{"name":"t","type":"i32"},{"name":"x","type":"f64"},{"name":"i","type":"i32"},{"name":"b","type":"bool"},{"name":"cs","type":"f64"},{"name":"r","type":"f64"}],"columns":{"g":["a","b","a","b"],"t":[1,1,3,2],"x":[1.5,2.0,4.0,"NaN"],"i":[3,1,1,5],"b":[true,false,true,true],"cs":[1.5,3.5,7.5,"NaN"],"r":[2.0,2.5,4.0,4.0]}}},{"id":"pivot/sum","description":"pivot with a sum per cell; missing cells are null","inputs":{"left":{"fields":[{"name":"region","type":"utf8"},{"name":"quarter","type":"utf8"},{"name":"units","type":"i32"},{"name":"price","type":"f64"}],"columns":{"region":["north","north","south","south","north","east","south","north"],"quarter":["q1","q2","q1","q3","q1","q2","q1",null],"units":[10,20,5,null,7,3,8,1],"price":[1.5,2.0,1.0,3.0,null,2.5,1.0,9.0]}}},"ops":[{"op":"pivot","on":"quarter","index":["region"],"values":"units","aggregate":"sum"}],"expected":{"fields":[{"name":"region","type":"utf8"},{"name":"q1","type":"i32"},{"name":"q2","type":"i32"},{"name":"q3","type":"i32"},{"name":"null","type":"i32"}],"columns":{"region":["north","south","east"],"q1":[17,13,0],"q2":[20,0,3],"q3":[0,0,0],"null":[1,0,0]}}},{"id":"pivot/first","description":"pivot with the first value per cell","inputs":{"left":{"fields":[{"name":"region","type":"utf8"},{"name":"quarter","type":"utf8"},{"name":"units","type":"i32"},{"name":"price","type":"f64"}],"columns":{"region":["north","north","south","south","north","east","south","north"],"quarter":["q1","q2","q1","q3","q1","q2","q1",null],"units":[10,20,5,null,7,3,8,1],"price":[1.5,2.0,1.0,3.0,null,2.5,1.0,9.0]}}},"ops":[{"op":"pivot","on":"region","index":["quarter"],"values":"price","aggregate":"first"}],"expected":{"fields":[{"name":"quarter","type":"utf8"},{"name":"north","type":"f64"},{"name":"south","type":"f64"},{"name":"east","type":"f64"}],"columns":{"quarter":["q1","q2","q3",null],"north":[1.5,2.0,null,9.0],"south":[1.0,null,3.0,null],"east":[null,2.5,null,null]}}},{"id":"pivot/len","description":"pivot counting rows per cell","inputs":{"left":{"fields":[{"name":"region","type":"utf8"},{"name":"quarter","type":"utf8"},{"name":"units","type":"i32"},{"name":"price","type":"f64"}],"columns":{"region":["north","north","south","south","north","east","south","north"],"quarter":["q1","q2","q1","q3","q1","q2","q1",null],"units":[10,20,5,null,7,3,8,1],"price":[1.5,2.0,1.0,3.0,null,2.5,1.0,9.0]}}},"ops":[{"op":"pivot","on":"quarter","index":["region"],"values":"units","aggregate":"len"}],"expected":{"fields":[{"name":"region","type":"utf8"},{"name":"q1","type":"u32"},{"name":"q2","type":"u32"},{"name":"q3","type":"u32"},{"name":"null","type":"u32"}],"columns":{"region":["north","south","east"],"q1":[2,2,0],"q2":[1,0,1],"q3":[0,1,0],"null":[1,0,0]}}},{"id":"pivot/columns","description":"pivot with declared output columns","inputs":{"left":{"fields":[{"name":"region","type":"utf8"},{"name":"quarter","type":"utf8"},{"name":"units","type":"i32"},{"name":"price","type":"f64"}],"columns":{"region":["north","north","south","south","north","east","south","north"],"quarter":["q1","q2","q1","q3","q1","q2","q1",null],"units":[10,20,5,null,7,3,8,1],"price":[1.5,2.0,1.0,3.0,null,2.5,1.0,9.0]}}},"ops":[{"op":"pivot","on":"quarter","index":["region"],"values":"price","aggregate":"max","columns":["q1","q2","q4"]}],"expected":{"fields":[{"name":"region","type":"utf8"},{"name":"q1","type":"f64"},{"name":"q2","type":"f64"},{"name":"q4","type":"f64"}],"columns":{"region":["north","south","east"],"q1":[1.5,1.0,null],"q2":[2.0,null,2.5],"q4":[null,null,null]}}},{"id":"pivot/numeric-on","description":"pivot on a numeric column names columns by value","inputs":{"left":{"fields":[{"name":"g","type":"utf8"},{"name":"t","type":"i32"},{"name":"x","type":"f64"},{"name":"i","type":"i32"},{"name":"b","type":"bool"}],"columns":{"g":["a","b","a","a","b",null,"a","b","b","a"],"t":[1,1,2,3,2,1,4,3,4,5],"x":[1.5,2.0,null,4.0,"NaN",3.0,-1.0,2.0,0.5,2.0],"i":[3,1,4,1,5,9,null,6,5,3],"b":[true,false,null,true,true,false,true,null,false,true]}}},"ops":[{"op":"filter","predicate":{"fn":"isNotNull","args":[{"col":"g"}]}},{"op":"pivot","on":"t","index":["g"],"values":"i","aggregate":"min"}],"expected":{"fields":[{"name":"g","type":"utf8"},{"name":"1","type":"i32"},{"name":"2","type":"i32"},{"name":"3","type":"i32"},{"name":"4","type":"i32"},{"name":"5","type":"i32"}],"columns":{"g":["a","b"],"1":[3,1],"2":[4,5],"3":[1,6],"4":[null,5],"5":[3,null]}}},{"id":"unpivot/basic","description":"unpivot stacks columns, column by column","inputs":{"left":{"fields":[{"name":"region","type":"utf8"},{"name":"quarter","type":"utf8"},{"name":"units","type":"i32"},{"name":"price","type":"f64"}],"columns":{"region":["north","north","south","south","north","east","south","north"],"quarter":["q1","q2","q1","q3","q1","q2","q1",null],"units":[10,20,5,null,7,3,8,1],"price":[1.5,2.0,1.0,3.0,null,2.5,1.0,9.0]}}},"ops":[{"op":"unpivot","index":["region"],"on":["units","price"]}],"expected":{"fields":[{"name":"region","type":"utf8"},{"name":"variable","type":"utf8"},{"name":"value","type":"f64"}],"columns":{"region":["north","north","south","south","north","east","south","north","north","north","south","south","north","east","south","north"],"variable":["units","units","units","units","units","units","units","units","price","price","price","price","price","price","price","price"],"value":[10.0,20.0,5.0,null,7.0,3.0,8.0,1.0,1.5,2.0,1.0,3.0,null,2.5,1.0,9.0]}}},{"id":"unpivot/names","description":"unpivot with named variable and value columns","inputs":{"left":{"fields":[{"name":"region","type":"utf8"},{"name":"quarter","type":"utf8"},{"name":"units","type":"i32"},{"name":"price","type":"f64"}],"columns":{"region":["north","north","south","south","north","east","south","north"],"quarter":["q1","q2","q1","q3","q1","q2","q1",null],"units":[10,20,5,null,7,3,8,1],"price":[1.5,2.0,1.0,3.0,null,2.5,1.0,9.0]}}},"ops":[{"op":"pick","columns":["region","quarter"]},{"op":"unpivot","index":[],"on":["region","quarter"],"variable":"field","value":"text"}],"expected":{"fields":[{"name":"field","type":"utf8"},{"name":"text","type":"utf8"}],"columns":{"field":["region","region","region","region","region","region","region","region","quarter","quarter","quarter","quarter","quarter","quarter","quarter","quarter"],"text":["north","north","south","south","north","east","south","north","q1","q2","q1","q3","q1","q2","q1",null]}}},{"id":"temporal/extract","description":"calendar fields of timestamps and dates","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"ts","type":"datetime[us]"},{"name":"ms","type":"datetime[ms]"},{"name":"d","type":"date"},{"name":"dur","type":"duration[us]"}],"columns":{"k":["a","b","c","d","e","f"],"ts":[1709214307123456,1704067199999000,1709510400000000,null,-14182940000000,1710052200000500],"ms":[1709214307123,1704067200000,null,1709251200000,0,1709942400000],"d":[19782,19722,19786,19782,null,19792],"dur":[5400000000,-1500000,null,259200000001,0,90000000000]}},"right":{"fields":[{"name":"d","type":"date"},{"name":"name","type":"utf8"}],"columns":{"d":[19782,19792,20089],"name":["leap","sunday","new year"]}}},"ops":[{"op":"select","exprs":{"y":{"fn":"year","args":[{"col":"ts"}]},"mo":{"fn":"month","args":[{"col":"ts"}]},"d":{"fn":"day","args":[{"col":"ts"}]},"h":{"fn":"hour","args":[{"col":"ts"}]},"mi":{"fn":"minute","args":[{"col":"ts"}]},"s":{"fn":"second","args":[{"col":"ts"}]},"ms":{"fn":"millisecond","args":[{"col":"ts"}]},"wd":{"fn":"weekday","args":[{"col":"ts"}]},"od":{"fn":"ordinalDay","args":[{"col":"ts"}]},"dy":{"fn":"year","args":[{"col":"d"}]},"dwd":{"fn":"weekday","args":[{"col":"d"}]},"dod":{"fn":"ordinalDay","args":[{"col":"d"}]}}}],"expected":{"fields":[{"name":"y","type":"i32"},{"name":"mo","type":"i8"},{"name":"d","type":"i8"},{"name":"h","type":"i8"},{"name":"mi","type":"i8"},{"name":"s","type":"i8"},{"name":"ms","type":"i32"},{"name":"wd","type":"i8"},{"name":"od","type":"i16"},{"name":"dy","type":"i32"},{"name":"dwd","type":"i8"},{"name":"dod","type":"i16"}],"columns":{"y":[2024,2023,2024,null,1969,2024],"mo":[2,12,3,null,7,3],"d":[29,31,4,null,20,10],"h":[13,23,0,null,20,6],"mi":[45,59,0,null,17,30],"s":[7,59,0,null,40,0],"ms":[123,999,0,null,0,0],"wd":[4,7,1,null,7,7],"od":[60,365,64,null,201,70],"dy":[2024,2023,2024,2024,null,2024],"dwd":[4,7,1,4,null,7],"dod":[60,365,64,60,null,70]}}},{"id":"temporal/arith","description":"timestamp differences are durations; durations shift timestamps","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"ts","type":"datetime[us]"},{"name":"ms","type":"datetime[ms]"},{"name":"d","type":"date"},{"name":"dur","type":"duration[us]"}],"columns":{"k":["a","b","c","d","e","f"],"ts":[1709214307123456,1704067199999000,1709510400000000,null,-14182940000000,1710052200000500],"ms":[1709214307123,1704067200000,null,1709251200000,0,1709942400000],"d":[19782,19722,19786,19782,null,19792],"dur":[5400000000,-1500000,null,259200000001,0,90000000000]}},"right":{"fields":[{"name":"d","type":"date"},{"name":"name","type":"utf8"}],"columns":{"d":[19782,19792,20089],"name":["leap","sunday","new year"]}}},"ops":[{"op":"select","exprs":{"gap":{"fn":"sub","args":[{"col":"ts"},{"col":"ms"}]},"later":{"fn":"add","args":[{"col":"ts"},{"col":"dur"}]},"earlier":{"fn":"sub","args":[{"col":"ts"},{"col":"dur"}]},"days":{"fn":"sub","args":[{"col":"d"},{"lit":18000,"type":"date"}]},"twice":{"fn":"add","args":[{"col":"dur"},{"col":"dur"}]}}}],"expected":{"fields":[{"name":"gap","type":"duration[ms]"},{"name":"later","type":"datetime[us]"},{"name":"earlier","type":"datetime[us]"},{"name":"days","type":"duration[us]"},{"name":"twice","type":"duration[us]"}],"columns":{"gap":[0,-1,null,null,-14182940000,109800000],"later":[1709219707123456,1704067198499000,null,null,-14182940000000,1710142200000500],"earlier":[1709208907123456,1704067201499000,null,null,-14182940000000,1709962200000500],"days":[153964800000000,148780800000000,154310400000000,153964800000000,null,154828800000000],"twice":[10800000000,-3000000,null,518400000002,0,180000000000]}}},{"id":"temporal/truncate","description":"truncate to a calendar unit","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"ts","type":"datetime[us]"},{"name":"ms","type":"datetime[ms]"},{"name":"d","type":"date"},{"name":"dur","type":"duration[us]"}],"columns":{"k":["a","b","c","d","e","f"],"ts":[1709214307123456,1704067199999000,1709510400000000,null,-14182940000000,1710052200000500],"ms":[1709214307123,1704067200000,null,1709251200000,0,1709942400000],"d":[19782,19722,19786,19782,null,19792],"dur":[5400000000,-1500000,null,259200000001,0,90000000000]}},"right":{"fields":[{"name":"d","type":"date"},{"name":"name","type":"utf8"}],"columns":{"d":[19782,19792,20089],"name":["leap","sunday","new year"]}}},"ops":[{"op":"select","exprs":{"h":{"fn":"truncate","args":[{"col":"ts"}],"every":"1h"},"d":{"fn":"truncate","args":[{"col":"ts"}],"every":"1d"},"w":{"fn":"truncate","args":[{"col":"ts"}],"every":"1w"},"mo":{"fn":"truncate","args":[{"col":"ts"}],"every":"1mo"},"y":{"fn":"truncate","args":[{"col":"ts"}],"every":"1y"},"dmo":{"fn":"truncate","args":[{"col":"d"}],"every":"1mo"}}}],"expected":{"fields":[{"name":"h","type":"datetime[us]"},{"name":"d","type":"datetime[us]"},{"name":"w","type":"datetime[us]"},{"name":"mo","type":"datetime[us]"},{"name":"y","type":"datetime[us]"},{"name":"dmo","type":"date"}],"columns":{"h":[1709211600000000,1704063600000000,1709510400000000,null,-14184000000000,1710050400000000],"d":[1709164800000000,1703980800000000,1709510400000000,null,-14256000000000,1710028800000000],"w":[1708905600000000,1703462400000000,1709510400000000,null,-14774400000000,1709510400000000],"mo":[1706745600000000,1701388800000000,1709251200000000,null,-15897600000000,1709251200000000],"y":[1704067200000000,1672531200000000,1704067200000000,null,-31536000000000,1704067200000000],"dmo":[19754,19692,19783,19754,null,19783]}}},{"id":"temporal/duration","description":"durations in whole units","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"ts","type":"datetime[us]"},{"name":"ms","type":"datetime[ms]"},{"name":"d","type":"date"},{"name":"dur","type":"duration[us]"}],"columns":{"k":["a","b","c","d","e","f"],"ts":[1709214307123456,1704067199999000,1709510400000000,null,-14182940000000,1710052200000500],"ms":[1709214307123,1704067200000,null,1709251200000,0,1709942400000],"d":[19782,19722,19786,19782,null,19792],"dur":[5400000000,-1500000,null,259200000001,0,90000000000]}},"right":{"fields":[{"name":"d","type":"date"},{"name":"name","type":"utf8"}],"columns":{"d":[19782,19792,20089],"name":["leap","sunday","new year"]}}},"ops":[{"op":"select","exprs":{"d":{"fn":"totalDays","args":[{"col":"dur"}]},"h":{"fn":"totalHours","args":[{"col":"dur"}]},"m":{"fn":"totalMinutes","args":[{"col":"dur"}]},"s":{"fn":"totalSeconds","args":[{"col":"dur"}]},"ms":{"fn":"totalMilliseconds","args":[{"col":"dur"}]}}}],"expected":{"fields":[{"name":"d","type":"i64"},{"name":"h","type":"i64"},{"name":"m","type":"i64"},{"name":"s","type":"i64"},{"name":"ms","type":"i64"}],"columns":{"d":["0","0",null,"3","0","1"],"h":["1","0",null,"72","0","25"],"m":["90","0",null,"4320","0","1500"],"s":["5400","-1",null,"259200","0","90000"],"ms":["5400000","-1500",null,"259200000","0","90000000"]}}},{"id":"temporal/cast","description":"casts between dates, timestamps and integers","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"ts","type":"datetime[us]"},{"name":"ms","type":"datetime[ms]"},{"name":"d","type":"date"},{"name":"dur","type":"duration[us]"}],"columns":{"k":["a","b","c","d","e","f"],"ts":[1709214307123456,1704067199999000,1709510400000000,null,-14182940000000,1710052200000500],"ms":[1709214307123,1704067200000,null,1709251200000,0,1709942400000],"d":[19782,19722,19786,19782,null,19792],"dur":[5400000000,-1500000,null,259200000001,0,90000000000]}},"right":{"fields":[{"name":"d","type":"date"},{"name":"name","type":"utf8"}],"columns":{"d":[19782,19792,20089],"name":["leap","sunday","new year"]}}},"ops":[{"op":"select","exprs":{"date":{"fn":"cast","args":[{"col":"ts"}],"to":"date"},"ts":{"fn":"cast","args":[{"col":"d"}],"to":"datetime[us]"},"us":{"fn":"cast","args":[{"col":"ts"}],"to":"i64"},"days":{"fn":"cast","args":[{"col":"d"}],"to":"i32"},"ms":{"fn":"cast","args":[{"col":"ts"}],"to":"datetime[ms]"}}}],"expected":{"fields":[{"name":"date","type":"date"},{"name":"ts","type":"datetime[us]"},{"name":"us","type":"i64"},{"name":"days","type":"i32"},{"name":"ms","type":"datetime[ms]"}],"columns":{"date":[19782,19722,19786,null,-165,19792],"ts":[1709164800000000,1703980800000000,1709510400000000,1709164800000000,null,1710028800000000],"us":["1709214307123456","1704067199999000","1709510400000000",null,"-14182940000000","1710052200000500"],"days":[19782,19722,19786,19782,null,19792],"ms":[1709214307123,1704067199999,1709510400000,null,-14182940000,1710052200000]}}},{"id":"temporal/compare","description":"comparisons with timestamp literals and between units","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"ts","type":"datetime[us]"},{"name":"ms","type":"datetime[ms]"},{"name":"d","type":"date"},{"name":"dur","type":"duration[us]"}],"columns":{"k":["a","b","c","d","e","f"],"ts":[1709214307123456,1704067199999000,1709510400000000,null,-14182940000000,1710052200000500],"ms":[1709214307123,1704067200000,null,1709251200000,0,1709942400000],"d":[19782,19722,19786,19782,null,19792],"dur":[5400000000,-1500000,null,259200000001,0,90000000000]}},"right":{"fields":[{"name":"d","type":"date"},{"name":"name","type":"utf8"}],"columns":{"d":[19782,19792,20089],"name":["leap","sunday","new year"]}}},"ops":[{"op":"filter","predicate":{"fn":"and","args":[{"fn":"gte","args":[{"col":"ts"},{"lit":1709251200000000,"type":"datetime[us]"}]},{"fn":"lt","args":[{"col":"ms"},{"col":"ts"}]}]}}],"expected":{"fields":[{"name":"k","type":"utf8"},{"name":"ts","type":"datetime[us]"},{"name":"ms","type":"datetime[ms]"},{"name":"d","type":"date"},{"name":"dur","type":"duration[us]"}],"columns":{"k":["f"],"ts":[1710052200000500],"ms":[1709942400000],"d":[19792],"dur":[90000000000]}}},{"id":"temporal/group-sort","description":"group by date, aggregate timestamps, sort","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"ts","type":"datetime[us]"},{"name":"ms","type":"datetime[ms]"},{"name":"d","type":"date"},{"name":"dur","type":"duration[us]"}],"columns":{"k":["a","b","c","d","e","f"],"ts":[1709214307123456,1704067199999000,1709510400000000,null,-14182940000000,1710052200000500],"ms":[1709214307123,1704067200000,null,1709251200000,0,1709942400000],"d":[19782,19722,19786,19782,null,19792],"dur":[5400000000,-1500000,null,259200000001,0,90000000000]}},"right":{"fields":[{"name":"d","type":"date"},{"name":"name","type":"utf8"}],"columns":{"d":[19782,19792,20089],"name":["leap","sunday","new year"]}}},"ops":[{"op":"groupBy","keys":["d"],"aggs":{"first":{"agg":"min","arg":{"col":"ts"}},"last":{"agg":"max","arg":{"col":"ts"}},"n":{"agg":"len"},"total":{"agg":"sum","arg":{"col":"dur"}}}},{"op":"sort","keys":[{"expr":{"col":"last"},"descending":true}]}],"expected":{"fields":[{"name":"d","type":"date"},{"name":"first","type":"datetime[us]"},{"name":"last","type":"datetime[us]"},{"name":"n","type":"u32"},{"name":"total","type":"duration[us]"}],"columns":{"d":[19792,19786,19782,19722,null],"first":[1710052200000500,1709510400000000,1709214307123456,1704067199999000,-14182940000000],"last":[1710052200000500,1709510400000000,1709214307123456,1704067199999000,-14182940000000],"n":[1,1,2,1,1],"total":[90000000000,0,264600000001,-1500000,0]}}},{"id":"temporal/join","description":"join on a date","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"ts","type":"datetime[us]"},{"name":"ms","type":"datetime[ms]"},{"name":"d","type":"date"},{"name":"dur","type":"duration[us]"}],"columns":{"k":["a","b","c","d","e","f"],"ts":[1709214307123456,1704067199999000,1709510400000000,null,-14182940000000,1710052200000500],"ms":[1709214307123,1704067200000,null,1709251200000,0,1709942400000],"d":[19782,19722,19786,19782,null,19792],"dur":[5400000000,-1500000,null,259200000001,0,90000000000]}},"right":{"fields":[{"name":"d","type":"date"},{"name":"name","type":"utf8"}],"columns":{"d":[19782,19792,20089],"name":["leap","sunday","new year"]}}},"ops":[{"op":"join","right":"right","on":["d"],"how":"left"}],"expected":{"fields":[{"name":"k","type":"utf8"},{"name":"ts","type":"datetime[us]"},{"name":"ms","type":"datetime[ms]"},{"name":"d","type":"date"},{"name":"dur","type":"duration[us]"},{"name":"name","type":"utf8"}],"columns":{"k":["a","b","c","d","e","f"],"ts":[1709214307123456,1704067199999000,1709510400000000,null,-14182940000000,1710052200000500],"ms":[1709214307123,1704067200000,null,1709251200000,0,1709942400000],"d":[19782,19722,19786,19782,null,19792],"dur":[5400000000,-1500000,null,259200000001,0,90000000000],"name":["leap",null,null,"leap",null,"sunday"]}}},{"id":"nested/list-ops","description":"list length, element access and aggregates","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"g","type":"i32"},{"name":"xs","type":"list"},{"name":"ws","type":"list"},{"name":"s","type":"struct"}],"columns":{"k":["a","b","c","d","e"],"g":[1,2,1,2,1],"xs":[[1,2,3],[],null,[4,null,2],[7]],"ws":[["a","b"],["c"],[],null,["z","a"]],"s":[{"a":1,"b":"x"},{"a":null,"b":"y"},null,{"a":4,"b":null},{"a":5,"b":"z"}]}},"right":{"fields":[{"name":"k","type":"utf8"},{"name":"tags","type":"list"}],"columns":{"k":["a","d","x"],"tags":[["t1"],["t2","t3"],[]]}}},"ops":[{"op":"select","exprs":{"n":{"fn":"listLength","args":[{"col":"xs"}]},"first":{"fn":"listGet","args":[{"col":"xs"}],"index":0},"last":{"fn":"listGet","args":[{"col":"xs"}],"index":-1},"oob":{"fn":"listGet","args":[{"col":"xs"}],"index":5},"sum":{"fn":"listSum","args":[{"col":"xs"}]},"mean":{"fn":"listMean","args":[{"col":"xs"}]},"min":{"fn":"listMin","args":[{"col":"xs"}]},"max":{"fn":"listMax","args":[{"col":"ws"}]},"has2":{"fn":"listContains","args":[{"col":"xs"},{"lit":2}]},"hasA":{"fn":"listContains","args":[{"col":"ws"},{"lit":"a"}]}}}],"expected":{"fields":[{"name":"n","type":"u32"},{"name":"first","type":"i32"},{"name":"last","type":"i32"},{"name":"oob","type":"i32"},{"name":"sum","type":"i32"},{"name":"mean","type":"f64"},{"name":"min","type":"i32"},{"name":"max","type":"utf8"},{"name":"has2","type":"bool"},{"name":"hasA","type":"bool"}],"columns":{"n":[3,0,null,3,1],"first":[1,null,null,4,7],"last":[3,null,null,2,7],"oob":[null,null,null,null,null],"sum":[6,0,null,6,7],"mean":[2.0,null,null,3.0,7.0],"min":[1,null,null,2,7],"max":["b","c",null,null,"z"],"has2":[true,false,null,true,false],"hasA":[true,false,false,null,true]}}},{"id":"nested/struct","description":"struct fields and struct construction","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"g","type":"i32"},{"name":"xs","type":"list"},{"name":"ws","type":"list"},{"name":"s","type":"struct"}],"columns":{"k":["a","b","c","d","e"],"g":[1,2,1,2,1],"xs":[[1,2,3],[],null,[4,null,2],[7]],"ws":[["a","b"],["c"],[],null,["z","a"]],"s":[{"a":1,"b":"x"},{"a":null,"b":"y"},null,{"a":4,"b":null},{"a":5,"b":"z"}]}},"right":{"fields":[{"name":"k","type":"utf8"},{"name":"tags","type":"list"}],"columns":{"k":["a","d","x"],"tags":[["t1"],["t2","t3"],[]]}}},"ops":[{"op":"select","exprs":{"a":{"fn":"field","args":[{"col":"s"}],"field":"a"},"b":{"fn":"field","args":[{"col":"s"}],"field":"b"},"pair":{"fn":"struct","args":[{"col":"k"},{"fn":"listLength","args":[{"col":"xs"}]}],"names":["key","size"]}}}],"expected":{"fields":[{"name":"a","type":"i32"},{"name":"b","type":"utf8"},{"name":"pair","type":"struct"}],"columns":{"a":[1,null,null,4,5],"b":["x","y",null,null,"z"],"pair":[{"key":"a","size":3},{"key":"b","size":0},{"key":"c","size":null},{"key":"d","size":3},{"key":"e","size":1}]}}},{"id":"nested/explode","description":"explode lists into rows; a null list gives a null row, an empty list no rows","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"g","type":"i32"},{"name":"xs","type":"list"},{"name":"ws","type":"list"},{"name":"s","type":"struct"}],"columns":{"k":["a","b","c","d","e"],"g":[1,2,1,2,1],"xs":[[1,2,3],[],null,[4,null,2],[7]],"ws":[["a","b"],["c"],[],null,["z","a"]],"s":[{"a":1,"b":"x"},{"a":null,"b":"y"},null,{"a":4,"b":null},{"a":5,"b":"z"}]}},"right":{"fields":[{"name":"k","type":"utf8"},{"name":"tags","type":"list"}],"columns":{"k":["a","d","x"],"tags":[["t1"],["t2","t3"],[]]}}},"ops":[{"op":"pick","columns":["k","xs"]},{"op":"explode","column":"xs"}],"expected":{"fields":[{"name":"k","type":"utf8"},{"name":"xs","type":"i32"}],"columns":{"k":["a","a","a","c","d","d","d","e"],"xs":[1,2,3,null,4,null,2,7]}}},{"id":"nested/unnest","description":"unnest a struct into columns","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"g","type":"i32"},{"name":"xs","type":"list"},{"name":"ws","type":"list"},{"name":"s","type":"struct"}],"columns":{"k":["a","b","c","d","e"],"g":[1,2,1,2,1],"xs":[[1,2,3],[],null,[4,null,2],[7]],"ws":[["a","b"],["c"],[],null,["z","a"]],"s":[{"a":1,"b":"x"},{"a":null,"b":"y"},null,{"a":4,"b":null},{"a":5,"b":"z"}]}},"right":{"fields":[{"name":"k","type":"utf8"},{"name":"tags","type":"list"}],"columns":{"k":["a","d","x"],"tags":[["t1"],["t2","t3"],[]]}}},"ops":[{"op":"pick","columns":["k","s"]},{"op":"unnest","column":"s"}],"expected":{"fields":[{"name":"k","type":"utf8"},{"name":"a","type":"i32"},{"name":"b","type":"utf8"}],"columns":{"k":["a","b","c","d","e"],"a":[1,null,null,4,5],"b":["x","y",null,null,"z"]}}},{"id":"nested/agg-list","description":"collect values into lists per group","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"g","type":"i32"},{"name":"xs","type":"list"},{"name":"ws","type":"list"},{"name":"s","type":"struct"}],"columns":{"k":["a","b","c","d","e"],"g":[1,2,1,2,1],"xs":[[1,2,3],[],null,[4,null,2],[7]],"ws":[["a","b"],["c"],[],null,["z","a"]],"s":[{"a":1,"b":"x"},{"a":null,"b":"y"},null,{"a":4,"b":null},{"a":5,"b":"z"}]}},"right":{"fields":[{"name":"k","type":"utf8"},{"name":"tags","type":"list"}],"columns":{"k":["a","d","x"],"tags":[["t1"],["t2","t3"],[]]}}},"ops":[{"op":"groupBy","keys":["g"],"aggs":{"ks":{"agg":"list","arg":{"col":"k"}},"firsts":{"agg":"first","arg":{"col":"xs"}},"n":{"agg":"len"}}}],"expected":{"fields":[{"name":"g","type":"i32"},{"name":"ks","type":"list"},{"name":"firsts","type":"list"},{"name":"n","type":"u32"}],"columns":{"g":[1,2],"ks":[["a","c","e"],["b","d"]],"firsts":[[1,2,3],[]],"n":[3,2]}}},{"id":"nested/carry","description":"lists and structs survive filters, sorts, joins and slices","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"g","type":"i32"},{"name":"xs","type":"list"},{"name":"ws","type":"list"},{"name":"s","type":"struct"}],"columns":{"k":["a","b","c","d","e"],"g":[1,2,1,2,1],"xs":[[1,2,3],[],null,[4,null,2],[7]],"ws":[["a","b"],["c"],[],null,["z","a"]],"s":[{"a":1,"b":"x"},{"a":null,"b":"y"},null,{"a":4,"b":null},{"a":5,"b":"z"}]}},"right":{"fields":[{"name":"k","type":"utf8"},{"name":"tags","type":"list"}],"columns":{"k":["a","d","x"],"tags":[["t1"],["t2","t3"],[]]}}},"ops":[{"op":"filter","predicate":{"fn":"neq","args":[{"col":"k"},{"lit":"c"}]}},{"op":"sort","keys":[{"expr":{"col":"g"},"descending":true},{"expr":{"col":"k"}}]},{"op":"join","right":"right","on":["k"],"how":"left"},{"op":"head","n":4}],"expected":{"fields":[{"name":"k","type":"utf8"},{"name":"g","type":"i32"},{"name":"xs","type":"list"},{"name":"ws","type":"list"},{"name":"s","type":"struct"},{"name":"tags","type":"list"}],"columns":{"k":["b","d","a","e"],"g":[2,2,1,1],"xs":[[],[4,null,2],[1,2,3],[7]],"ws":[["c"],null,["a","b"],["z","a"]],"s":[{"a":null,"b":"y"},{"a":4,"b":null},{"a":1,"b":"x"},{"a":5,"b":"z"}],"tags":[null,["t2","t3"],["t1"],null]}}},{"id":"decimal/arith","description":"decimal addition, subtraction and multiplication","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"price","type":"decimal[10,2]"},{"name":"fee","type":"decimal[6,3]"},{"name":"qty","type":"decimal[4,0]"},{"name":"x","type":"f64"}],"columns":{"k":["a","b","a","c","b","a"],"price":["12.34","0.10","-5.00",null,"99999999.99","10.01"],"fee":["0.125","1.000",null,"2.500","-0.001","10.010"],"qty":["3","10","2","1",null,"7"],"x":[1.005,-2.5,123.456,null,1000000.0,0.0]}}},"ops":[{"op":"select","exprs":{"sum":{"fn":"add","args":[{"col":"price"},{"col":"fee"}]},"diff":{"fn":"sub","args":[{"col":"price"},{"col":"fee"}]},"prod":{"fn":"mul","args":[{"col":"price"},{"col":"qty"}]},"plus":{"fn":"add","args":[{"col":"price"},{"lit":"0.5","type":"decimal[3,1]"}]}}}],"expected":{"fields":[{"name":"sum","type":"decimal[38,3]"},{"name":"diff","type":"decimal[38,3]"},{"name":"prod","type":"decimal[38,2]"},{"name":"plus","type":"decimal[38,2]"}],"columns":{"sum":["12.465","1.100",null,null,"99999999.989","20.020"],"diff":["12.215","-0.900",null,null,"99999999.991","0.000"],"prod":["37.02","1.00","-10.00",null,null,"70.07"],"plus":["12.84","0.60","-4.50",null,"100000000.49","10.51"]}}},{"id":"decimal/compare","description":"decimal comparisons, also across scales","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"price","type":"decimal[10,2]"},{"name":"fee","type":"decimal[6,3]"},{"name":"qty","type":"decimal[4,0]"},{"name":"x","type":"f64"}],"columns":{"k":["a","b","a","c","b","a"],"price":["12.34","0.10","-5.00",null,"99999999.99","10.01"],"fee":["0.125","1.000",null,"2.500","-0.001","10.010"],"qty":["3","10","2","1",null,"7"],"x":[1.005,-2.5,123.456,null,1000000.0,0.0]}}},"ops":[{"op":"filter","predicate":{"fn":"and","args":[{"fn":"gt","args":[{"col":"price"},{"lit":"10.00","type":"decimal[10,2]"}]},{"fn":"gte","args":[{"col":"price"},{"col":"fee"}]}]}}],"expected":{"fields":[{"name":"k","type":"utf8"},{"name":"price","type":"decimal[10,2]"},{"name":"fee","type":"decimal[6,3]"},{"name":"qty","type":"decimal[4,0]"},{"name":"x","type":"f64"}],"columns":{"k":["a","b","a"],"price":["12.34","99999999.99","10.01"],"fee":["0.125","-0.001","10.010"],"qty":["3",null,"7"],"x":[1.005,1000000.0,0.0]}}},{"id":"decimal/aggregate","description":"sums, extremes and means of decimals per group","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"price","type":"decimal[10,2]"},{"name":"fee","type":"decimal[6,3]"},{"name":"qty","type":"decimal[4,0]"},{"name":"x","type":"f64"}],"columns":{"k":["a","b","a","c","b","a"],"price":["12.34","0.10","-5.00",null,"99999999.99","10.01"],"fee":["0.125","1.000",null,"2.500","-0.001","10.010"],"qty":["3","10","2","1",null,"7"],"x":[1.005,-2.5,123.456,null,1000000.0,0.0]}}},"ops":[{"op":"groupBy","keys":["k"],"aggs":{"total":{"agg":"sum","arg":{"col":"price"}},"min":{"agg":"min","arg":{"col":"price"}},"max":{"agg":"max","arg":{"col":"fee"}},"mean":{"agg":"mean","arg":{"col":"price"}},"n":{"agg":"count","arg":{"col":"price"}}}}],"expected":{"fields":[{"name":"k","type":"utf8"},{"name":"total","type":"decimal[38,2]"},{"name":"min","type":"decimal[10,2]"},{"name":"max","type":"decimal[6,3]"},{"name":"mean","type":"f64"},{"name":"n","type":"u32"}],"columns":{"k":["a","b","c"],"total":["17.35","100000000.09","0.00"],"min":["-5.00","0.10",null],"max":["10.010","1.000","2.500"],"mean":[5.783333333333334,50000000.044999994,null],"n":[3,2,0]}}},{"id":"decimal/cast","description":"casts between decimals, floats, integers and strings","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"price","type":"decimal[10,2]"},{"name":"fee","type":"decimal[6,3]"},{"name":"qty","type":"decimal[4,0]"},{"name":"x","type":"f64"}],"columns":{"k":["a","b","a","c","b","a"],"price":["12.34","0.10","-5.00",null,"99999999.99","10.01"],"fee":["0.125","1.000",null,"2.500","-0.001","10.010"],"qty":["3","10","2","1",null,"7"],"x":[1.005,-2.5,123.456,null,1000000.0,0.0]}}},"ops":[{"op":"select","exprs":{"f":{"fn":"cast","args":[{"col":"price"}],"to":"f64"},"i":{"fn":"cast","args":[{"col":"price"}],"to":"i32"},"s":{"fn":"cast","args":[{"col":"fee"}],"to":"utf8"},"d":{"fn":"cast","args":[{"col":"x"}],"to":"decimal[10,2]"},"r":{"fn":"cast","args":[{"col":"price"}],"to":"decimal[12,1]"}}}],"expected":{"fields":[{"name":"f","type":"f64"},{"name":"i","type":"i32"},{"name":"s","type":"utf8"},{"name":"d","type":"decimal[10,2]"},{"name":"r","type":"decimal[12,1]"}],"columns":{"f":[12.34,0.1,-5.0,null,99999999.99,10.01],"i":[12,0,-5,null,99999999,10],"s":["0.125","1.000",null,"2.500","-0.001","10.010"],"d":["1.00","-2.50","123.46",null,"1000000.00","0.00"],"r":["12.3","0.1","-5.0",null,"100000000.0","10.0"]}}},{"id":"decimal/sort","description":"sort by a decimal","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"price","type":"decimal[10,2]"},{"name":"fee","type":"decimal[6,3]"},{"name":"qty","type":"decimal[4,0]"},{"name":"x","type":"f64"}],"columns":{"k":["a","b","a","c","b","a"],"price":["12.34","0.10","-5.00",null,"99999999.99","10.01"],"fee":["0.125","1.000",null,"2.500","-0.001","10.010"],"qty":["3","10","2","1",null,"7"],"x":[1.005,-2.5,123.456,null,1000000.0,0.0]}}},"ops":[{"op":"sort","keys":[{"expr":{"col":"price"},"descending":true}],"nullsLast":true}],"expected":{"fields":[{"name":"k","type":"utf8"},{"name":"price","type":"decimal[10,2]"},{"name":"fee","type":"decimal[6,3]"},{"name":"qty","type":"decimal[4,0]"},{"name":"x","type":"f64"}],"columns":{"k":["b","a","a","b","a","c"],"price":["99999999.99","12.34","10.01","0.10","-5.00",null],"fee":["-0.001","0.125","10.010","1.000",null,"2.500"],"qty":[null,"3","7","10","2","1"],"x":[1000000.0,1.005,0.0,-2.5,123.456,null]}}},{"id":"decimal/round-ties","description":"narrowing the scale rounds exact ties","inputs":{"left":{"fields":[{"name":"t","type":"decimal[6,3]"}],"columns":{"t":["0.250","0.350","-0.250","-0.350","1.050","2.500","0.249"]}}},"ops":[{"op":"select","exprs":{"one":{"fn":"cast","args":[{"col":"t"}],"to":"decimal[6,1]"},"zero":{"fn":"cast","args":[{"col":"t"}],"to":"decimal[6,0]"}}}],"expected":{"fields":[{"name":"one","type":"decimal[6,1]"},{"name":"zero","type":"decimal[6,0]"}],"columns":{"one":["0.2","0.4","-0.2","-0.4","1.0","2.5","0.2"],"zero":["0","0","0","0","1","2","0"]}}},{"id":"decimal/cast-float-ties","description":"floats cast to decimals round exact ties half to even, also below zero","inputs":{"left":{"fields":[{"name":"x","type":"f64"}],"columns":{"x":[-1.5,-2.5,1.5,2.5,-0.75]}}},"ops":[{"op":"select","exprs":{"zero":{"fn":"cast","args":[{"col":"x"}],"to":"decimal[18,0]"},"one":{"fn":"cast","args":[{"col":"x"}],"to":"decimal[18,1]"}}}],"expected":{"fields":[{"name":"zero","type":"decimal[18,0]"},{"name":"one","type":"decimal[18,1]"}],"columns":{"zero":["-2","-2","2","2","-1"],"one":["-1.5","-2.5","1.5","2.5","-0.8"]}}},{"id":"decimal/round","description":"rounding decimals keeps the scale and rounds half to even","inputs":{"left":{"fields":[{"name":"t","type":"decimal[6,3]"}],"columns":{"t":["0.250","0.350","-0.250","-0.350","1.050","2.500","0.249",null]}}},"ops":[{"op":"select","exprs":{"r0":{"fn":"round","args":[{"col":"t"}]},"r1":{"fn":"round","args":[{"col":"t"}],"decimals":1},"r2":{"fn":"round","args":[{"col":"t"}],"decimals":2},"r3":{"fn":"round","args":[{"col":"t"}],"decimals":3}}}],"expected":{"fields":[{"name":"r0","type":"decimal[7,3]"},{"name":"r1","type":"decimal[7,3]"},{"name":"r2","type":"decimal[7,3]"},{"name":"r3","type":"decimal[6,3]"}],"columns":{"r0":["0.000","0.000","0.000","0.000","1.000","2.000","0.000",null],"r1":["0.200","0.400","-0.200","-0.400","1.000","2.500","0.200",null],"r2":["0.250","0.350","-0.250","-0.350","1.050","2.500","0.250",null],"r3":["0.250","0.350","-0.250","-0.350","1.050","2.500","0.249",null]}}},{"id":"arith/u32-divide-by-zero","description":"unsigned 32-bit floor division and modulo by zero give null","inputs":{"left":{"fields":[{"name":"n","type":"u32"},{"name":"d","type":"u32"}],"columns":{"n":[7,0,5],"d":[0,0,2]}}},"ops":[{"op":"select","exprs":{"q":{"fn":"floorDiv","args":[{"col":"n"},{"col":"d"}]},"m":{"fn":"mod","args":[{"col":"n"},{"col":"d"}]},"qs":{"fn":"floorDiv","args":[{"col":"n"},{"lit":0,"type":"u32"}]},"ms":{"fn":"mod","args":[{"col":"n"},{"lit":0,"type":"u32"}]}}}],"expected":{"fields":[{"name":"q","type":"u32"},{"name":"m","type":"u32"},{"name":"qs","type":"u32"},{"name":"ms","type":"u32"}],"columns":{"q":[null,null,2],"m":[null,null,1],"qs":[null,null,null],"ms":[null,null,null]}}},{"id":"slice/clip-5+1","description":"slices reaching before the first row clip from the unclamped start","inputs":{"left":{"fields":[{"name":"x","type":"i32"}],"columns":{"x":[1,2,3]}}},"ops":[{"op":"slice","offset":-5,"length":1}],"expected":{"fields":[{"name":"x","type":"i32"}],"columns":{"x":[]}}},{"id":"slice/clip-5+3","description":"slices reaching before the first row clip from the unclamped start","inputs":{"left":{"fields":[{"name":"x","type":"i32"}],"columns":{"x":[1,2,3]}}},"ops":[{"op":"slice","offset":-5,"length":3}],"expected":{"fields":[{"name":"x","type":"i32"}],"columns":{"x":[1]}}},{"id":"slice/clip-5+4","description":"slices reaching before the first row clip from the unclamped start","inputs":{"left":{"fields":[{"name":"x","type":"i32"}],"columns":{"x":[1,2,3]}}},"ops":[{"op":"slice","offset":-5,"length":4}],"expected":{"fields":[{"name":"x","type":"i32"}],"columns":{"x":[1,2]}}},{"id":"slice/clip-2+5","description":"slices reaching before the first row clip from the unclamped start","inputs":{"left":{"fields":[{"name":"x","type":"i32"}],"columns":{"x":[1,2,3]}}},"ops":[{"op":"slice","offset":-2,"length":5}],"expected":{"fields":[{"name":"x","type":"i32"}],"columns":{"x":[2,3]}}},{"id":"slice/clip-5","description":"slices reaching before the first row clip from the unclamped start","inputs":{"left":{"fields":[{"name":"x","type":"i32"}],"columns":{"x":[1,2,3]}}},"ops":[{"op":"slice","offset":-5}],"expected":{"fields":[{"name":"x","type":"i32"}],"columns":{"x":[1,2,3]}}},{"id":"slice/clip1+0","description":"slices reaching before the first row clip from the unclamped start","inputs":{"left":{"fields":[{"name":"x","type":"i32"}],"columns":{"x":[1,2,3]}}},"ops":[{"op":"slice","offset":1,"length":0}],"expected":{"fields":[{"name":"x","type":"i32"}],"columns":{"x":[]}}},{"id":"window/filter-after-sort","description":"a window predicate sees the sorted rows","inputs":{"left":{"fields":[{"name":"g","type":"i32"},{"name":"h","type":"i32"},{"name":"x","type":"f64"}],"columns":{"g":[1,2,1,2],"h":[1,1,2,2],"x":[3.0,1.0,2.0,4.0]}},"right":{"fields":[{"name":"g","type":"i32"},{"name":"name","type":"utf8"}],"columns":{"g":[1],"name":["one"]}}},"ops":[{"op":"sort","keys":[{"expr":{"col":"x"}}]},{"op":"filter","predicate":{"fn":"gt","args":[{"fn":"cumSum","args":[{"col":"x"}]},{"lit":3.0}]}}],"expected":{"fields":[{"name":"g","type":"i32"},{"name":"h","type":"i32"},{"name":"x","type":"f64"}],"columns":{"g":[1,2],"h":[1,2],"x":[3.0,4.0]}}},{"id":"window/filter-after-sort-by-window","description":"a filter after a sort by a window leaves the sort's order alone","inputs":{"left":{"fields":[{"name":"x","type":"f64"}],"columns":{"x":[-3.0,5.0,-1.0]}}},"ops":[{"op":"sort","keys":[{"expr":{"fn":"cumSum","args":[{"col":"x"}]}}]},{"op":"filter","predicate":{"fn":"neq","args":[{"col":"x"},{"lit":5.0}]}}],"expected":{"fields":[{"name":"x","type":"f64"}],"columns":{"x":[-3.0,-1.0]}}},{"id":"window/filter-after-filter","description":"a window predicate sees only the rows an earlier filter kept","inputs":{"left":{"fields":[{"name":"g","type":"i32"},{"name":"h","type":"i32"},{"name":"x","type":"f64"}],"columns":{"g":[1,2,1,2],"h":[1,1,2,2],"x":[3.0,1.0,2.0,4.0]}},"right":{"fields":[{"name":"g","type":"i32"},{"name":"name","type":"utf8"}],"columns":{"g":[1],"name":["one"]}}},"ops":[{"op":"filter","predicate":{"fn":"gt","args":[{"col":"x"},{"lit":1.0}]}},{"op":"filter","predicate":{"fn":"gt","args":[{"fn":"shift","args":[{"col":"x"}]},{"lit":2.0}]}}],"expected":{"fields":[{"name":"g","type":"i32"},{"name":"h","type":"i32"},{"name":"x","type":"f64"}],"columns":{"g":[1],"h":[2],"x":[2.0]}}},{"id":"window/filter-after-concat","description":"a window predicate sees every input of a concat","inputs":{"left":{"fields":[{"name":"g","type":"i32"},{"name":"h","type":"i32"},{"name":"x","type":"f64"}],"columns":{"g":[1,2,1,2],"h":[1,1,2,2],"x":[3.0,1.0,2.0,4.0]}},"right":{"fields":[{"name":"g","type":"i32"},{"name":"name","type":"utf8"}],"columns":{"g":[1],"name":["one"]}}},"ops":[{"op":"concat","with":"left"},{"op":"filter","predicate":{"fn":"gt","args":[{"fn":"cumSum","args":[{"col":"x"}]},{"lit":15.0}]}}],"expected":{"fields":[{"name":"g","type":"i32"},{"name":"h","type":"i32"},{"name":"x","type":"f64"}],"columns":{"g":[1,2],"h":[2,2],"x":[2.0,4.0]}}},{"id":"window/filter-after-join","description":"a window predicate sees only the joined rows","inputs":{"left":{"fields":[{"name":"g","type":"i32"},{"name":"h","type":"i32"},{"name":"x","type":"f64"}],"columns":{"g":[1,2,1,2],"h":[1,1,2,2],"x":[3.0,1.0,2.0,4.0]}},"right":{"fields":[{"name":"g","type":"i32"},{"name":"name","type":"utf8"}],"columns":{"g":[1],"name":["one"]}}},"ops":[{"op":"join","right":"right","on":["g"],"how":"inner"},{"op":"filter","predicate":{"fn":"gt","args":[{"fn":"shift","args":[{"col":"x"}]},{"lit":2.0}]}}],"expected":{"fields":[{"name":"g","type":"i32"},{"name":"h","type":"i32"},{"name":"x","type":"f64"},{"name":"name","type":"utf8"}],"columns":{"g":[1],"h":[2],"x":[2.0],"name":["one"]}}},{"id":"window/filter-whole","description":"a window predicate sees the whole input","inputs":{"left":{"fields":[{"name":"g","type":"i32"},{"name":"h","type":"i32"},{"name":"x","type":"f64"}],"columns":{"g":[1,2,1,2],"h":[1,1,2,2],"x":[3.0,1.0,2.0,4.0]}},"right":{"fields":[{"name":"g","type":"i32"},{"name":"name","type":"utf8"}],"columns":{"g":[1],"name":["one"]}}},"ops":[{"op":"filter","predicate":{"fn":"gt","args":[{"fn":"cumSum","args":[{"col":"x"}]},{"lit":5.0}]}}],"expected":{"fields":[{"name":"g","type":"i32"},{"name":"h","type":"i32"},{"name":"x","type":"f64"}],"columns":{"g":[1,2],"h":[2,2],"x":[2.0,4.0]}}},{"id":"window/agg-of-window","description":"an aggregate of a window sees the whole input","inputs":{"left":{"fields":[{"name":"g","type":"i32"},{"name":"h","type":"i32"},{"name":"x","type":"f64"}],"columns":{"g":[1,2,1,2],"h":[1,1,2,2],"x":[3.0,1.0,2.0,4.0]}},"right":{"fields":[{"name":"g","type":"i32"},{"name":"name","type":"utf8"}],"columns":{"g":[1],"name":["one"]}}},"ops":[{"op":"agg","aggs":{"s":{"agg":"sum","arg":{"fn":"cumSum","args":[{"col":"x"}]}},"n":{"agg":"len"}}}],"expected":{"fields":[{"name":"s","type":"f64"},{"name":"n","type":"u32"}],"columns":{"s":[23.0],"n":[4]}}},{"id":"window/grouped-agg-of-window","description":"windows inside grouped aggregates run per group","inputs":{"left":{"fields":[{"name":"g","type":"i32"},{"name":"h","type":"i32"},{"name":"x","type":"f64"}],"columns":{"g":[1,2,1,2],"h":[1,1,2,2],"x":[3.0,1.0,2.0,4.0]}},"right":{"fields":[{"name":"g","type":"i32"},{"name":"name","type":"utf8"}],"columns":{"g":[1],"name":["one"]}}},"ops":[{"op":"groupBy","keys":["g"],"aggs":{"s":{"agg":"sum","arg":{"fn":"cumSum","args":[{"col":"x"}]}},"l":{"agg":"list","arg":{"fn":"cumSum","args":[{"col":"x"}]}},"o":{"agg":"sum","arg":{"over":{"agg":"sum","arg":{"col":"x"}},"partitionBy":[{"col":"h"}]}}}}],"expected":{"fields":[{"name":"g","type":"i32"},{"name":"s","type":"f64"},{"name":"l","type":"list"},{"name":"o","type":"f64"}],"columns":{"g":[1,2],"s":[8.0,6.0],"l":[[3.0,5.0],[1.0,5.0]],"o":[5.0,5.0]}}}]} From 8687484068b5e5978a815594bff18f624b7fd7cd Mon Sep 17 00:00:00 2001 From: Effect Bot Date: Sun, 15 Nov 2026 12:34:33 +0000 Subject: [PATCH 14/88] Store every computed decimal through one 64-bit check Decimal literals, abs and negate of the minimum, and rolling sums wrote unchecked values to 64-bit storage and silently wrapped. They now fail with DataFrameError like sums and arithmetic do. Casts make values that don't fit null as documented: floats whose scaled value is infinite no longer throw a RangeError, UInt64 sources are read unsigned, and negative decimals cast to UInt64 are null. Rolling sums and means of 64-bit integers and decimals are exact, and decimal rolling means are no longer in unscaled units. --- packages/effect/src/dataframe/Expr.ts | 8 +- .../src/dataframe/internal/aggregate.ts | 26 +++--- .../effect/src/dataframe/internal/decimal.ts | 47 +++++++--- .../effect/src/dataframe/internal/expr.ts | 7 +- .../effect/src/dataframe/internal/kernels.ts | 5 +- .../effect/src/dataframe/internal/window.ts | 86 +++++++++++-------- packages/effect/test/dataframe/Query.test.ts | 39 ++++++--- .../test/fixtures/dataframe/generate_gold.py | 2 + .../fixtures/dataframe/gold/extended.json | 2 +- 9 files changed, 146 insertions(+), 76 deletions(-) diff --git a/packages/effect/src/dataframe/Expr.ts b/packages/effect/src/dataframe/Expr.ts index a011e20f43d..21efa1354a2 100644 --- a/packages/effect/src/dataframe/Expr.ts +++ b/packages/effect/src/dataframe/Expr.ts @@ -31,6 +31,11 @@ * and an integer division by zero gives `null`. * - An integer literal takes the integer type of the column it meets * (`c.int8Column + 1` stays Int8). + * - Decimals are stored as 64-bit unscaled integers, so they hold up to 18 + * digits. Unlike integers they never wrap: a decimal result outside that + * storage (from arithmetic, sums, `abs`, `negate`, `round` or a + * literal) fails the query with a `DataFrameError`, and a cast makes it + * `null`. * * **Example** (Building expressions) * @@ -680,7 +685,8 @@ export const cast: { /** * Converts to a decimal with `precision` digits, `scale` of them after the - * point. Values that don't fit become null; extra digits round half to even. + * point. Values that don't fit (more than `precision` digits, or outside + * 64-bit decimal storage) become null; extra digits round half to even. * * @stability experimental * @category casts diff --git a/packages/effect/src/dataframe/internal/aggregate.ts b/packages/effect/src/dataframe/internal/aggregate.ts index 6b7c0ecb397..0bdcbd27f1a 100644 --- a/packages/effect/src/dataframe/internal/aggregate.ts +++ b/packages/effect/src/dataframe/internal/aggregate.ts @@ -150,21 +150,27 @@ const sum = (v: V.Vector, g: Grouping): V.Vector => { for (let i = 0; i < n; i++) if (V.isValid(v, i)) acc[ids[i]] += x[i] return V.f64(S.uint32, acc) } - case "i64": { - const x = v.values - const acc = new Array(groups).fill(I0) - const valid = v.valid - for (let i = 0; i < n; i++) if (valid === undefined || valid[i] === 1) acc[ids[i]] += x[i] - const out = new BigInt64Array(groups) - const fit = Dec.isDecimal(type) ? Dec.checked : (x: bigint) => BigInt.asIntN(64, x) - for (let k = 0; k < groups; k++) out[k] = fit(acc[k]) - return V.i64(type, out) - } + case "i64": + return fromSums(type, exactSums(v, g)) default: return V.fail(`sum is not defined for ${D.describe(v.type)}`) } } +/** Per-group sums of 64-bit integers or decimals, exact (unbounded). @internal */ +export const exactSums = (v: V.I64, g: Grouping): Array => { + const x = v.values + const ids = g.ids + const acc = new Array(g.count).fill(I0) + const valid = v.valid + for (let i = 0; i < ids.length; i++) if (valid === undefined || valid[i] === 1) acc[ids[i]] += x[i] + return acc +} + +/** Exact sums as a column of `type`, stored as `Dec.fit` says. @internal */ +export const fromSums = (type: S.DataType, sums: ReadonlyArray): V.Vector => + V.i64(type, BigInt64Array.from(sums, Dec.fit(type))) + /** Every value as a float, with validity. */ const asFloat = (v: V.Vector): { readonly x: Float64Array; readonly valid: Uint8Array | undefined } => { switch (v.kind) { diff --git a/packages/effect/src/dataframe/internal/decimal.ts b/packages/effect/src/dataframe/internal/decimal.ts index 9a0658ec6a5..59180a663a9 100644 --- a/packages/effect/src/dataframe/internal/decimal.ts +++ b/packages/effect/src/dataframe/internal/decimal.ts @@ -1,8 +1,10 @@ /** * Decimals: unscaled 64-bit integers with a scale (`value / 10^scale`), so * up to 18 significant digits. Arithmetic follows Polars' result types - * (precision 38, scales added for products, the larger scale for sums); - * arithmetic, sums and running sums fail instead of overflowing 64 bits. Rescaling down rounds half to even. + * (precision 38, scales added for products, the larger scale for sums). + * Every computed decimal is stored through `checked` (or `fit`), so a + * value outside 64 bits fails instead of wrapping; casts make such values + * null instead. Rescaling down rounds half to even. * * @internal */ @@ -39,8 +41,20 @@ export const supertype = (a: S.DataType, b: S.DataType): S.DataType | undefined return undefined } +const inRange = (x: bigint): boolean => x <= MAX && x >= MIN + /** `x`, or a failure when it overflows 64-bit decimal storage. @internal */ -export const checked = (x: bigint): bigint => (x > MAX || x < MIN ? V.fail("decimal value overflows 64 bits") : x) +export const checked = (x: bigint): bigint => (inRange(x) ? x : V.fail("decimal value overflows 64 bits")) + +const wrap = (x: bigint): bigint => BigInt.asIntN(64, x) + +/** + * How an exact result of `type` is stored in 64 bits: decimals are + * `checked`, 64-bit integers wrap. + * + * @internal + */ +export const fit = (type: S.DataType): (x: bigint) => bigint => isDecimal(type) ? checked : wrap /** Rounds `x / 10^k` half to even (`BigDecimal.round`). */ const divRound = (x: bigint, k: number): bigint => @@ -68,7 +82,7 @@ export const parse = (s: string, scale: number): bigint | undefined => { const digitsOf = (x: bigint): number => (x < BigInt(0) ? -x : x).toString().length -const fits = (x: bigint, t: S.Decimal): boolean => x <= MAX && x >= MIN && digitsOf(x) <= t.precision +const fits = (x: bigint, t: S.Decimal): boolean => inRange(x) && digitsOf(x) <= t.precision /** * Casts to and from decimals; `undefined` when neither side is a decimal. @@ -92,23 +106,24 @@ export const cast = (v: V.Vector, target: S.DataType): V.Vector | undefined => { for (let i = 0; i < n; i++) { if (!V.isValid(v, i)) continue switch (v.kind) { - case "i64": + case "i64": { + const x = V.isUnsigned64(from) ? BigInt.asUintN(64, v.values[i]) : v.values[i] put( i, isDecimal(from) ? from.scale >= target.scale - ? divRound(v.values[i], from.scale - target.scale) - : v.values[i] * pow10(target.scale - from.scale) - : v.values[i] * pow10(target.scale) + ? divRound(x, from.scale - target.scale) + : x * pow10(target.scale - from.scale) + : x * pow10(target.scale) ) break + } case "i32": put(i, BigInt(v.values[i]) * pow10(target.scale)) break case "f64": { - const x = v.values[i] - if (Number.isFinite(x)) { - const scaled = x * 10 ** target.scale + const scaled = v.values[i] * 10 ** target.scale + if (Number.isFinite(scaled)) { const floor = Math.floor(scaled) const rest = scaled - floor put(i, BigInt(rest > 0.5 || (rest === 0.5 && floor % 2 !== 0) ? floor + 1 : floor)) @@ -136,7 +151,15 @@ export const cast = (v: V.Vector, target: S.DataType): V.Vector | undefined => { } case "Int": { if (target.bitWidth === 64) { - return V.i64(target, BigInt64Array.from(v.values, (x) => x / scale), v.valid) + const out = new BigInt64Array(n) + for (let i = 0; i < n; i++) { + const x = v.values[i] / scale + if (V.isValid(v, i) && (target.signed || x >= BigInt(0))) { + out[i] = x + valid[i] = 1 + } + } + return V.i64(target, out, valid) } const lo = target.signed ? -(2 ** (target.bitWidth - 1)) : 0 const hi = target.signed ? 2 ** (target.bitWidth - 1) - 1 : 2 ** target.bitWidth - 1 diff --git a/packages/effect/src/dataframe/internal/expr.ts b/packages/effect/src/dataframe/internal/expr.ts index 49a7ccdca28..65b552ca7f0 100644 --- a/packages/effect/src/dataframe/internal/expr.ts +++ b/packages/effect/src/dataframe/internal/expr.ts @@ -10,6 +10,7 @@ import * as Duration from "../../Duration.ts" import { pipeArguments } from "../../Pipeable.ts" import type * as Plan from "../Plan.ts" import * as Agg from "./aggregate.ts" +import * as Dec from "./decimal.ts" import * as D from "./dtype.ts" import * as Hash from "./hash.ts" import * as K from "./kernels.ts" @@ -159,8 +160,10 @@ const literalOfType = (value: unknown, type: Plan.LiteralType): K.Value => { return K.scalar(V.i64(S.timestamp(type.unit, type.timezone), BigInt64Array.of(big))) case "Duration": return K.scalar(V.i64(S.duration(type.unit), BigInt64Array.of(big))) - case "Decimal": - return K.scalar(V.i64(S.decimal(type.precision, type.scale), BigInt64Array.of(big))) + case "Decimal": { + const unscaled = BigInt64Array.of(Dec.checked(big)) + return K.scalar(V.i64(S.decimal(type.precision, type.scale), unscaled)) + } } } diff --git a/packages/effect/src/dataframe/internal/kernels.ts b/packages/effect/src/dataframe/internal/kernels.ts index ae52dc5f624..e761ae15e0f 100644 --- a/packages/effect/src/dataframe/internal/kernels.ts +++ b/packages/effect/src/dataframe/internal/kernels.ts @@ -585,8 +585,9 @@ export const unaryNumeric = (op: "neg" | "abs" | "sqrt" | "round", a: Value, dec case "i64": { const x = v.values const o = new BigInt64Array(n) - if (op === "neg") { for (let i = 0; i < n; i++) o[i] = BigInt.asIntN(64, -x[i]) } - else if (op === "abs") { for (let i = 0; i < n; i++) o[i] = BigInt.asIntN(64, x[i] < I0 ? -x[i] : x[i]) } + const fit = Dec.fit(v.type) + if (op === "neg") { for (let i = 0; i < n; i++) o[i] = fit(-x[i]) } + else if (op === "abs") { for (let i = 0; i < n; i++) o[i] = fit(x[i] < I0 ? -x[i] : x[i]) } else o.set(x) return { vector: V.i64(v.type, o, v.valid), scalar: a.scalar, dynamic: a.dynamic } } diff --git a/packages/effect/src/dataframe/internal/window.ts b/packages/effect/src/dataframe/internal/window.ts index dd0c3cf9ee1..0278346ed0b 100644 --- a/packages/effect/src/dataframe/internal/window.ts +++ b/packages/effect/src/dataframe/internal/window.ts @@ -79,7 +79,7 @@ export const cumSum = (v: V.Vector, p: Partition): V.Vector => { const x = v.values const acc = new BigInt64Array(p.count) const out = new BigInt64Array(n) - const fit = Dec.isDecimal(type) ? Dec.checked : (x: bigint) => BigInt.asIntN(64, x) + const fit = Dec.fit(type) for (let i = 0; i < n; i++) { if (valid === undefined || valid[i] === 1) out[i] = acc[ids[i]] = fit(acc[ids[i]] + x[i]) } @@ -253,59 +253,77 @@ export const rolling = ( ): V.Vector => { const n = v.length const { rows, starts } = rowsByPartition(p) + /** Calls `f` with each row and the window `rows[first..last]` ending at it. */ + const windows = (f: (row: number, first: number, last: number) => void) => { + for (let g = 0; g < p.count; g++) { + for (let j = starts[g]; j < starts[g + 1]; j++) f(rows[j], Math.max(starts[g], j - window + 1), j) + } + } const ok = new Uint8Array(n) if (fn === "rollingMin" || fn === "rollingMax") { const order = rowOrder(v) const sign = fn === "rollingMin" ? -1 : 1 const idx = new Int32Array(n).fill(-1) - for (let g = 0; g < p.count; g++) { - for (let j = starts[g]; j < starts[g + 1]; j++) { - let best = -1 - let count = 0 - for (let k = Math.max(starts[g], j - window + 1); k <= j; k++) { - const r = rows[k] - if (!V.isValid(v, r)) continue - count++ - if (best >= 0 && isNaNAt(v, best)) continue - if (isNaNAt(v, r) || best < 0 || order(r, best) * sign > 0) best = r - } - if (count >= minPeriods) idx[rows[j]] = best + windows((row, first, last) => { + let best = -1 + let count = 0 + for (let k = first; k <= last; k++) { + const r = rows[k] + if (!V.isValid(v, r)) continue + count++ + if (best >= 0 && isNaNAt(v, best)) continue + if (isNaNAt(v, r) || best < 0 || order(r, best) * sign > 0) best = r } - } + if (count >= minPeriods) idx[row] = best + }) return V.take(v, idx) } if (v.kind === "str" || v.kind === "null" || v.kind === "list" || v.kind === "struct") { return V.fail(`${fn} is not defined for ${D.describe(v.type)}`) } - const x = v.kind === "i64" - ? Float64Array.from(v.values, Number) - : v.kind === "f64" - ? v.values - : Float64Array.from(v.values as Int32Array | Uint8Array) - const out = new Float64Array(n) - for (let g = 0; g < p.count; g++) { - for (let j = starts[g]; j < starts[g + 1]; j++) { - let sum = 0 + const type = fn === "rollingMean" ? D.meanType(v.type) : D.sumType(v.type) + if (v.kind === "i64") { + // Exact sums: 64-bit integers wrap and decimals fail outside 64 bits. + const fit = Dec.fit(type) + const unit = Dec.isDecimal(v.type) ? 10 ** v.type.scale : 1 + const sums = new BigInt64Array(n) + const means = new Float64Array(n) + windows((row, first, last) => { + let sum = BigInt(0) let count = 0 - for (let k = Math.max(starts[g], j - window + 1); k <= j; k++) { + for (let k = first; k <= last; k++) { const r = rows[k] if (!V.isValid(v, r)) continue - sum += x[r] + sum += v.values[r] count++ } - if (count >= minPeriods) { - ok[rows[j]] = 1 - out[rows[j]] = fn === "rollingMean" ? sum / count : sum - } - } + if (count < minPeriods) return + ok[row] = 1 + if (fn === "rollingMean") means[row] = Number(sum) / unit / count + else sums[row] = fit(sum) + }) + return fn === "rollingMean" ? V.f64(type, means, ok) : V.i64(type, sums, ok) } - if (fn === "rollingMean") return V.f64(D.meanType(v.type), out, ok) - const type = D.sumType(v.type) + const x = v.kind === "f64" ? v.values : Float64Array.from(v.values) + const out = new Float64Array(n) + windows((row, first, last) => { + let sum = 0 + let count = 0 + for (let k = first; k <= last; k++) { + const r = rows[k] + if (!V.isValid(v, r)) continue + sum += x[r] + count++ + } + if (count >= minPeriods) { + ok[row] = 1 + out[row] = fn === "rollingMean" ? sum / count : sum + } + }) + if (fn === "rollingMean") return V.f64(type, out, ok) switch (v.kind) { case "i32": return V.i32(type, Int32Array.from(out, (a) => a | 0), ok) - case "i64": - return V.i64(type, BigInt64Array.from(out, (a) => BigInt.asIntN(64, BigInt(a))), ok) case "bool": return V.f64(S.uint32, out, ok) default: diff --git a/packages/effect/test/dataframe/Query.test.ts b/packages/effect/test/dataframe/Query.test.ts index d642a190bfb..d49ba4f9ab3 100644 --- a/packages/effect/test/dataframe/Query.test.ts +++ b/packages/effect/test/dataframe/Query.test.ts @@ -1,5 +1,5 @@ import { assert, describe, it } from "@effect/vitest" -import { Effect, Exit, Result, Schema, Stream } from "effect" +import { BigDecimal, Effect, Exit, Result, Schema, Stream } from "effect" import { ArrowSchema, ArrowTable } from "effect/arrow" import { Catalog, DataFrame, Expr, Plan, Query } from "effect/dataframe" import * as Compare from "./oracle/Compare.ts" @@ -97,27 +97,29 @@ describe("Query", () => { assert.strictEqual(error.message, "filter predicate must be boolean, got Utf8") })) - it.effect("decimal sums fail instead of overflowing 64 bits", () => + it.effect("decimal results fail instead of overflowing 64 bits", () => Effect.gen(function*() { const nines = (rows: number) => DataFrame.fromColumns({ s: Array.from({ length: rows }, () => "999999999999999999") }) - const d = (c: Query.Columns<{ readonly s: string }>) => Expr.castDecimal(c.s, 18, 0) - const overflow = (query: Effect.Effect) => - Effect.map(Effect.flip(query), (error) => error.message) + const d = (c: Query.Columns<{ readonly s: string }>) => Expr.castDecimal(c.s, 19, 0) const ten = Query.from(nines(10)) - assert.strictEqual( - yield* overflow(Query.aggregate(ten, (c) => ({ total: Expr.sum(d(c)) }))), - "decimal value overflows 64 bits" - ) - assert.strictEqual( - yield* overflow(Query.select(ten, (c) => ({ running: Expr.cumSum(d(c)) }))), - "decimal value overflows 64 bits" - ) + const min = Query.from(DataFrame.fromColumns({ s: ["-9223372036854775808"] })) const streamed = Query.fromPlan({ _tag: "Table", name: "t" }).pipe( Query.aggregate((c) => ({ total: Expr.sum(Expr.castDecimal(c.s as Expr.Expr, 18, 0)) })), Effect.provide(Catalog.layer({ t: Stream.make(nines(5), nines(5)) })) ) - assert.strictEqual(yield* overflow(streamed), "decimal value overflows 64 bits") + const queries: ReadonlyArray> = [ + Query.aggregate(ten, (c) => ({ total: Expr.sum(d(c)) })), + Query.select(ten, (c) => ({ running: Expr.cumSum(d(c)) })), + Query.select(ten, (c) => ({ rolling: Expr.rollingSum(d(c), 10) })), + Query.select(min, (c) => ({ abs: Expr.abs(d(c)) })), + Query.select(min, (c) => ({ negated: Expr.negate(d(c)) })), + Query.select(min, () => ({ literal: Expr.lit(BigDecimal.fromBigInt(10n ** 20n)) })), + streamed + ] + for (const query of queries) { + assert.strictEqual((yield* Effect.flip(query)).message, "decimal value overflows 64 bits") + } })) it.effect("round rejects negative or fractional places", () => @@ -175,6 +177,15 @@ describe("Query", () => { { i: null, s: "30000000000" }, { i: null, s: "NaN" } ]) + const decimals = DataFrame.fromColumns({ f: [1e308], u: BigUint64Array.of(2n ** 64n - 1n), d: ["-1.5"] }) + const casts = yield* rows( + Query.from(decimals).pipe(Query.select((c) => ({ + fromFloat: Expr.castDecimal(c.f, 18, 18), + fromUInt64: Expr.castDecimal(c.u, 20, 0), + toUInt64: Expr.cast(Expr.castDecimal(c.d, 2, 1), "UInt64") + }))) + ) + assert.deepStrictEqual(casts, [{ fromFloat: null, fromUInt64: null, toUInt64: null }]) })) }) diff --git a/packages/effect/test/fixtures/dataframe/generate_gold.py b/packages/effect/test/fixtures/dataframe/generate_gold.py index 7f2d785e1ad..56dc3190d24 100644 --- a/packages/effect/test/fixtures/dataframe/generate_gold.py +++ b/packages/effect/test/fixtures/dataframe/generate_gold.py @@ -897,6 +897,8 @@ def extended_cases(): {"left": table([("t", "decimal[6,3]")], {"t": ["0.250", "0.350", "-0.250", "-0.350", "1.050", "2.500", "0.249", None]})}, [{"op": "select", "exprs": {"r0": fn("round", col("t")), "r1": fn("round", col("t"), decimals=1), "r2": fn("round", col("t"), decimals=2), "r3": fn("round", col("t"), decimals=3)}}]) + add("decimal/rolling-mean", "rolling means of decimals are floats of their values", D, + [{"op": "select", "exprs": {"m": fn("rollingMean", col("price"), window=2), "mf": fn("rollingMean", col("fee"), window=3, minPeriods=1)}}]) add("arith/u32-divide-by-zero", "unsigned 32-bit floor division and modulo by zero give null", {"left": table([("n", "u32"), ("d", "u32")], {"n": [7, 0, 5], "d": [0, 0, 2]})}, [{"op": "select", "exprs": {"q": fn("floorDiv", col("n"), col("d")), "m": fn("mod", col("n"), col("d")), diff --git a/packages/effect/test/fixtures/dataframe/gold/extended.json b/packages/effect/test/fixtures/dataframe/gold/extended.json index 84b7eb80731..e79e2bffec4 100644 --- a/packages/effect/test/fixtures/dataframe/gold/extended.json +++ b/packages/effect/test/fixtures/dataframe/gold/extended.json @@ -1 +1 @@ -{"generator":"generate_gold.py","polars":"2.1.0","deviations":["JavaScript has no float literal distinct from an integer: an integral number literal behaves like a Polars integer literal (it takes the integer type of the column it meets), so cases never combine an integer column with an integral float literal.","Float to string casts use JavaScript number formatting (Polars prints 5.0 as '5.0', JavaScript as '5').","Joins never add '_right' suffixes: overlapping non-key columns are a type error.","Arithmetic between 64-bit integers (bigint) and other numeric types requires an explicit cast."],"cases":[{"id":"window/cum","description":"cumulative sum, min, max and count; nulls stay null","inputs":{"left":{"fields":[{"name":"g","type":"utf8"},{"name":"t","type":"i32"},{"name":"x","type":"f64"},{"name":"i","type":"i32"},{"name":"b","type":"bool"}],"columns":{"g":["a","b","a","a","b",null,"a","b","b","a"],"t":[1,1,2,3,2,1,4,3,4,5],"x":[1.5,2.0,null,4.0,"NaN",3.0,-1.0,2.0,0.5,2.0],"i":[3,1,4,1,5,9,null,6,5,3],"b":[true,false,null,true,true,false,true,null,false,true]}}},"ops":[{"op":"select","exprs":{"sx":{"fn":"cumSum","args":[{"col":"x"}]},"si":{"fn":"cumSum","args":[{"col":"i"}]},"mn":{"fn":"cumMin","args":[{"col":"i"}]},"mx":{"fn":"cumMax","args":[{"col":"x"}]},"n":{"fn":"cumCount","args":[{"col":"x"}]},"sb":{"fn":"cumSum","args":[{"col":"b"}]}}}],"expected":{"fields":[{"name":"sx","type":"f64"},{"name":"si","type":"i32"},{"name":"mn","type":"i32"},{"name":"mx","type":"f64"},{"name":"n","type":"u32"},{"name":"sb","type":"u32"}],"columns":{"sx":[1.5,3.5,null,7.5,"NaN","NaN","NaN","NaN","NaN","NaN"],"si":[3,4,8,9,14,23,null,29,34,37],"mn":[3,1,1,1,1,1,null,1,1,1],"mx":[1.5,2.0,null,4.0,4.0,4.0,4.0,4.0,4.0,4.0],"n":[1,2,2,3,4,5,6,7,8,9],"sb":[1,1,null,2,3,3,4,null,4,5]}}},{"id":"window/shift-diff","description":"shift (lag and lead) and diff","inputs":{"left":{"fields":[{"name":"g","type":"utf8"},{"name":"t","type":"i32"},{"name":"x","type":"f64"},{"name":"i","type":"i32"},{"name":"b","type":"bool"}],"columns":{"g":["a","b","a","a","b",null,"a","b","b","a"],"t":[1,1,2,3,2,1,4,3,4,5],"x":[1.5,2.0,null,4.0,"NaN",3.0,-1.0,2.0,0.5,2.0],"i":[3,1,4,1,5,9,null,6,5,3],"b":[true,false,null,true,true,false,true,null,false,true]}}},"ops":[{"op":"select","exprs":{"lag":{"fn":"shift","args":[{"col":"i"}]},"lead":{"fn":"shift","args":[{"col":"x"}],"n":-2},"d":{"fn":"diff","args":[{"col":"i"}]},"d2":{"fn":"diff","args":[{"col":"x"}],"n":2},"ls":{"fn":"shift","args":[{"col":"g"}]}}}],"expected":{"fields":[{"name":"lag","type":"i32"},{"name":"lead","type":"f64"},{"name":"d","type":"i32"},{"name":"d2","type":"f64"},{"name":"ls","type":"utf8"}],"columns":{"lag":[null,3,1,4,1,5,9,null,6,5],"lead":[null,4.0,"NaN",3.0,-1.0,2.0,0.5,2.0,null,null],"d":[null,-2,3,-3,4,4,null,null,-1,-2],"d2":[null,null,null,2.0,null,-1.0,"NaN",-1.0,1.5,0.0],"ls":[null,"a","b","a","a","b",null,"a","b","b"]}}},{"id":"window/rank","description":"rank methods, nulls and NaN","inputs":{"left":{"fields":[{"name":"g","type":"utf8"},{"name":"t","type":"i32"},{"name":"x","type":"f64"},{"name":"i","type":"i32"},{"name":"b","type":"bool"}],"columns":{"g":["a","b","a","a","b",null,"a","b","b","a"],"t":[1,1,2,3,2,1,4,3,4,5],"x":[1.5,2.0,null,4.0,"NaN",3.0,-1.0,2.0,0.5,2.0],"i":[3,1,4,1,5,9,null,6,5,3],"b":[true,false,null,true,true,false,true,null,false,true]}}},"ops":[{"op":"select","exprs":{"avg":{"fn":"rank","args":[{"col":"x"}]},"min":{"fn":"rank","args":[{"col":"i"}],"method":"min"},"max":{"fn":"rank","args":[{"col":"i"}],"method":"max"},"dense":{"fn":"rank","args":[{"col":"i"}],"method":"dense","descending":true},"ord":{"fn":"rank","args":[{"col":"x"}],"method":"ordinal"},"s":{"fn":"rank","args":[{"col":"g"}],"method":"dense"}}}],"expected":{"fields":[{"name":"avg","type":"f64"},{"name":"min","type":"u32"},{"name":"max","type":"u32"},{"name":"dense","type":"u32"},{"name":"ord","type":"u32"},{"name":"s","type":"u32"}],"columns":{"avg":[3.0,5.0,null,8.0,9.0,7.0,1.0,5.0,2.0,5.0],"min":[3,1,5,1,6,9,null,8,6,3],"max":[4,2,5,2,7,9,null,8,7,4],"dense":[5,6,4,6,3,1,null,2,3,5],"ord":[3,4,null,8,9,7,1,5,2,6],"s":[1,2,1,1,2,null,1,2,2,1]}}},{"id":"window/rolling","description":"rolling sum, mean, min and max","inputs":{"left":{"fields":[{"name":"g","type":"utf8"},{"name":"t","type":"i32"},{"name":"x","type":"f64"},{"name":"i","type":"i32"},{"name":"b","type":"bool"}],"columns":{"g":["a","b","a","a","b",null,"a","b","b","a"],"t":[1,1,2,3,2,1,4,3,4,5],"x":[1.5,2.0,null,4.0,"NaN",3.0,-1.0,2.0,0.5,2.0],"i":[3,1,4,1,5,9,null,6,5,3],"b":[true,false,null,true,true,false,true,null,false,true]}}},"ops":[{"op":"select","exprs":{"s":{"fn":"rollingSum","args":[{"col":"x"}],"window":3},"m":{"fn":"rollingMean","args":[{"col":"i"}],"window":2},"mn":{"fn":"rollingMin","args":[{"col":"x"}],"window":3,"minPeriods":1},"mx":{"fn":"rollingMax","args":[{"col":"i"}],"window":4,"minPeriods":2}}}],"expected":{"fields":[{"name":"s","type":"f64"},{"name":"m","type":"f64"},{"name":"mn","type":"f64"},{"name":"mx","type":"i32"}],"columns":{"s":[null,null,null,null,null,"NaN","NaN",4.0,1.5,4.5],"m":[null,2.0,2.5,2.5,3.0,7.0,null,null,5.5,4.0],"mn":[1.5,1.5,1.5,2.0,"NaN","NaN","NaN",-1.0,-1.0,0.5],"mx":[null,3,4,4,5,9,9,9,9,6]}}},{"id":"window/over-agg","description":"aggregates broadcast over partitions","inputs":{"left":{"fields":[{"name":"g","type":"utf8"},{"name":"t","type":"i32"},{"name":"x","type":"f64"},{"name":"i","type":"i32"},{"name":"b","type":"bool"}],"columns":{"g":["a","b","a","a","b",null,"a","b","b","a"],"t":[1,1,2,3,2,1,4,3,4,5],"x":[1.5,2.0,null,4.0,"NaN",3.0,-1.0,2.0,0.5,2.0],"i":[3,1,4,1,5,9,null,6,5,3],"b":[true,false,null,true,true,false,true,null,false,true]}}},"ops":[{"op":"withColumns","exprs":{"total":{"over":{"agg":"sum","arg":{"col":"x"}},"partitionBy":[{"col":"g"}]},"n":{"over":{"agg":"len"},"partitionBy":[{"col":"g"}]},"mean":{"over":{"agg":"mean","arg":{"col":"i"}},"partitionBy":[{"col":"g"}]},"first":{"over":{"agg":"first","arg":{"col":"t"}},"partitionBy":[{"col":"g"},{"col":"b"}]}}}],"expected":{"fields":[{"name":"g","type":"utf8"},{"name":"t","type":"i32"},{"name":"x","type":"f64"},{"name":"i","type":"i32"},{"name":"b","type":"bool"},{"name":"total","type":"f64"},{"name":"n","type":"u32"},{"name":"mean","type":"f64"},{"name":"first","type":"i32"}],"columns":{"g":["a","b","a","a","b",null,"a","b","b","a"],"t":[1,1,2,3,2,1,4,3,4,5],"x":[1.5,2.0,null,4.0,"NaN",3.0,-1.0,2.0,0.5,2.0],"i":[3,1,4,1,5,9,null,6,5,3],"b":[true,false,null,true,true,false,true,null,false,true],"total":[6.5,"NaN",6.5,6.5,"NaN",3.0,6.5,"NaN","NaN",6.5],"n":[5,4,5,5,4,1,5,4,4,5],"mean":[2.75,4.25,2.75,2.75,4.25,9.0,2.75,4.25,4.25,2.75],"first":[1,1,2,1,2,1,1,3,1,1]}}},{"id":"window/over-cum","description":"cumulative and shifted values per partition, in row order","inputs":{"left":{"fields":[{"name":"g","type":"utf8"},{"name":"t","type":"i32"},{"name":"x","type":"f64"},{"name":"i","type":"i32"},{"name":"b","type":"bool"}],"columns":{"g":["a","b","a","a","b",null,"a","b","b","a"],"t":[1,1,2,3,2,1,4,3,4,5],"x":[1.5,2.0,null,4.0,"NaN",3.0,-1.0,2.0,0.5,2.0],"i":[3,1,4,1,5,9,null,6,5,3],"b":[true,false,null,true,true,false,true,null,false,true]}}},"ops":[{"op":"withColumns","exprs":{"cs":{"over":{"fn":"cumSum","args":[{"col":"i"}]},"partitionBy":[{"col":"g"}]},"prev":{"over":{"fn":"shift","args":[{"col":"x"}]},"partitionBy":[{"col":"g"}]},"d":{"over":{"fn":"diff","args":[{"col":"t"}]},"partitionBy":[{"col":"g"}]},"r":{"over":{"fn":"rank","args":[{"col":"x"}],"method":"min"},"partitionBy":[{"col":"g"}]},"roll":{"over":{"fn":"rollingSum","args":[{"col":"i"}],"window":2,"minPeriods":1},"partitionBy":[{"col":"g"}]}}}],"expected":{"fields":[{"name":"g","type":"utf8"},{"name":"t","type":"i32"},{"name":"x","type":"f64"},{"name":"i","type":"i32"},{"name":"b","type":"bool"},{"name":"cs","type":"i32"},{"name":"prev","type":"f64"},{"name":"d","type":"i32"},{"name":"r","type":"u32"},{"name":"roll","type":"i32"}],"columns":{"g":["a","b","a","a","b",null,"a","b","b","a"],"t":[1,1,2,3,2,1,4,3,4,5],"x":[1.5,2.0,null,4.0,"NaN",3.0,-1.0,2.0,0.5,2.0],"i":[3,1,4,1,5,9,null,6,5,3],"b":[true,false,null,true,true,false,true,null,false,true],"cs":[3,1,7,8,6,9,null,12,17,11],"prev":[null,null,1.5,null,2.0,null,4.0,"NaN",2.0,-1.0],"d":[null,null,1,1,1,null,1,1,1,1],"r":[2,2,null,4,4,1,1,2,1,3],"roll":[3,1,7,5,6,9,1,11,11,3]}}},{"id":"window/over-filter","description":"filters after windows see the windowed values","inputs":{"left":{"fields":[{"name":"g","type":"utf8"},{"name":"t","type":"i32"},{"name":"x","type":"f64"},{"name":"i","type":"i32"},{"name":"b","type":"bool"}],"columns":{"g":["a","b","a","a","b",null,"a","b","b","a"],"t":[1,1,2,3,2,1,4,3,4,5],"x":[1.5,2.0,null,4.0,"NaN",3.0,-1.0,2.0,0.5,2.0],"i":[3,1,4,1,5,9,null,6,5,3],"b":[true,false,null,true,true,false,true,null,false,true]}}},"ops":[{"op":"withColumns","exprs":{"cs":{"over":{"fn":"cumSum","args":[{"col":"t"}]},"partitionBy":[{"col":"g"}]}}},{"op":"filter","predicate":{"fn":"gt","args":[{"col":"t"},{"lit":2}]}}],"expected":{"fields":[{"name":"g","type":"utf8"},{"name":"t","type":"i32"},{"name":"x","type":"f64"},{"name":"i","type":"i32"},{"name":"b","type":"bool"},{"name":"cs","type":"i32"}],"columns":{"g":["a","a","b","b","a"],"t":[3,4,3,4,5],"x":[4.0,-1.0,2.0,0.5,2.0],"i":[1,null,6,5,3],"b":[true,true,null,false,true],"cs":[6,10,6,10,15]}}},{"id":"window/filter-first","description":"windows after a filter only see the remaining rows","inputs":{"left":{"fields":[{"name":"g","type":"utf8"},{"name":"t","type":"i32"},{"name":"x","type":"f64"},{"name":"i","type":"i32"},{"name":"b","type":"bool"}],"columns":{"g":["a","b","a","a","b",null,"a","b","b","a"],"t":[1,1,2,3,2,1,4,3,4,5],"x":[1.5,2.0,null,4.0,"NaN",3.0,-1.0,2.0,0.5,2.0],"i":[3,1,4,1,5,9,null,6,5,3],"b":[true,false,null,true,true,false,true,null,false,true]}}},"ops":[{"op":"filter","predicate":{"fn":"isNotNull","args":[{"col":"x"}]}},{"op":"withColumns","exprs":{"cs":{"fn":"cumSum","args":[{"col":"x"}]},"r":{"over":{"fn":"rank","args":[{"col":"x"}]},"partitionBy":[{"col":"g"}]}}},{"op":"head","n":4}],"expected":{"fields":[{"name":"g","type":"utf8"},{"name":"t","type":"i32"},{"name":"x","type":"f64"},{"name":"i","type":"i32"},{"name":"b","type":"bool"},{"name":"cs","type":"f64"},{"name":"r","type":"f64"}],"columns":{"g":["a","b","a","b"],"t":[1,1,3,2],"x":[1.5,2.0,4.0,"NaN"],"i":[3,1,1,5],"b":[true,false,true,true],"cs":[1.5,3.5,7.5,"NaN"],"r":[2.0,2.5,4.0,4.0]}}},{"id":"pivot/sum","description":"pivot with a sum per cell; missing cells are null","inputs":{"left":{"fields":[{"name":"region","type":"utf8"},{"name":"quarter","type":"utf8"},{"name":"units","type":"i32"},{"name":"price","type":"f64"}],"columns":{"region":["north","north","south","south","north","east","south","north"],"quarter":["q1","q2","q1","q3","q1","q2","q1",null],"units":[10,20,5,null,7,3,8,1],"price":[1.5,2.0,1.0,3.0,null,2.5,1.0,9.0]}}},"ops":[{"op":"pivot","on":"quarter","index":["region"],"values":"units","aggregate":"sum"}],"expected":{"fields":[{"name":"region","type":"utf8"},{"name":"q1","type":"i32"},{"name":"q2","type":"i32"},{"name":"q3","type":"i32"},{"name":"null","type":"i32"}],"columns":{"region":["north","south","east"],"q1":[17,13,0],"q2":[20,0,3],"q3":[0,0,0],"null":[1,0,0]}}},{"id":"pivot/first","description":"pivot with the first value per cell","inputs":{"left":{"fields":[{"name":"region","type":"utf8"},{"name":"quarter","type":"utf8"},{"name":"units","type":"i32"},{"name":"price","type":"f64"}],"columns":{"region":["north","north","south","south","north","east","south","north"],"quarter":["q1","q2","q1","q3","q1","q2","q1",null],"units":[10,20,5,null,7,3,8,1],"price":[1.5,2.0,1.0,3.0,null,2.5,1.0,9.0]}}},"ops":[{"op":"pivot","on":"region","index":["quarter"],"values":"price","aggregate":"first"}],"expected":{"fields":[{"name":"quarter","type":"utf8"},{"name":"north","type":"f64"},{"name":"south","type":"f64"},{"name":"east","type":"f64"}],"columns":{"quarter":["q1","q2","q3",null],"north":[1.5,2.0,null,9.0],"south":[1.0,null,3.0,null],"east":[null,2.5,null,null]}}},{"id":"pivot/len","description":"pivot counting rows per cell","inputs":{"left":{"fields":[{"name":"region","type":"utf8"},{"name":"quarter","type":"utf8"},{"name":"units","type":"i32"},{"name":"price","type":"f64"}],"columns":{"region":["north","north","south","south","north","east","south","north"],"quarter":["q1","q2","q1","q3","q1","q2","q1",null],"units":[10,20,5,null,7,3,8,1],"price":[1.5,2.0,1.0,3.0,null,2.5,1.0,9.0]}}},"ops":[{"op":"pivot","on":"quarter","index":["region"],"values":"units","aggregate":"len"}],"expected":{"fields":[{"name":"region","type":"utf8"},{"name":"q1","type":"u32"},{"name":"q2","type":"u32"},{"name":"q3","type":"u32"},{"name":"null","type":"u32"}],"columns":{"region":["north","south","east"],"q1":[2,2,0],"q2":[1,0,1],"q3":[0,1,0],"null":[1,0,0]}}},{"id":"pivot/columns","description":"pivot with declared output columns","inputs":{"left":{"fields":[{"name":"region","type":"utf8"},{"name":"quarter","type":"utf8"},{"name":"units","type":"i32"},{"name":"price","type":"f64"}],"columns":{"region":["north","north","south","south","north","east","south","north"],"quarter":["q1","q2","q1","q3","q1","q2","q1",null],"units":[10,20,5,null,7,3,8,1],"price":[1.5,2.0,1.0,3.0,null,2.5,1.0,9.0]}}},"ops":[{"op":"pivot","on":"quarter","index":["region"],"values":"price","aggregate":"max","columns":["q1","q2","q4"]}],"expected":{"fields":[{"name":"region","type":"utf8"},{"name":"q1","type":"f64"},{"name":"q2","type":"f64"},{"name":"q4","type":"f64"}],"columns":{"region":["north","south","east"],"q1":[1.5,1.0,null],"q2":[2.0,null,2.5],"q4":[null,null,null]}}},{"id":"pivot/numeric-on","description":"pivot on a numeric column names columns by value","inputs":{"left":{"fields":[{"name":"g","type":"utf8"},{"name":"t","type":"i32"},{"name":"x","type":"f64"},{"name":"i","type":"i32"},{"name":"b","type":"bool"}],"columns":{"g":["a","b","a","a","b",null,"a","b","b","a"],"t":[1,1,2,3,2,1,4,3,4,5],"x":[1.5,2.0,null,4.0,"NaN",3.0,-1.0,2.0,0.5,2.0],"i":[3,1,4,1,5,9,null,6,5,3],"b":[true,false,null,true,true,false,true,null,false,true]}}},"ops":[{"op":"filter","predicate":{"fn":"isNotNull","args":[{"col":"g"}]}},{"op":"pivot","on":"t","index":["g"],"values":"i","aggregate":"min"}],"expected":{"fields":[{"name":"g","type":"utf8"},{"name":"1","type":"i32"},{"name":"2","type":"i32"},{"name":"3","type":"i32"},{"name":"4","type":"i32"},{"name":"5","type":"i32"}],"columns":{"g":["a","b"],"1":[3,1],"2":[4,5],"3":[1,6],"4":[null,5],"5":[3,null]}}},{"id":"unpivot/basic","description":"unpivot stacks columns, column by column","inputs":{"left":{"fields":[{"name":"region","type":"utf8"},{"name":"quarter","type":"utf8"},{"name":"units","type":"i32"},{"name":"price","type":"f64"}],"columns":{"region":["north","north","south","south","north","east","south","north"],"quarter":["q1","q2","q1","q3","q1","q2","q1",null],"units":[10,20,5,null,7,3,8,1],"price":[1.5,2.0,1.0,3.0,null,2.5,1.0,9.0]}}},"ops":[{"op":"unpivot","index":["region"],"on":["units","price"]}],"expected":{"fields":[{"name":"region","type":"utf8"},{"name":"variable","type":"utf8"},{"name":"value","type":"f64"}],"columns":{"region":["north","north","south","south","north","east","south","north","north","north","south","south","north","east","south","north"],"variable":["units","units","units","units","units","units","units","units","price","price","price","price","price","price","price","price"],"value":[10.0,20.0,5.0,null,7.0,3.0,8.0,1.0,1.5,2.0,1.0,3.0,null,2.5,1.0,9.0]}}},{"id":"unpivot/names","description":"unpivot with named variable and value columns","inputs":{"left":{"fields":[{"name":"region","type":"utf8"},{"name":"quarter","type":"utf8"},{"name":"units","type":"i32"},{"name":"price","type":"f64"}],"columns":{"region":["north","north","south","south","north","east","south","north"],"quarter":["q1","q2","q1","q3","q1","q2","q1",null],"units":[10,20,5,null,7,3,8,1],"price":[1.5,2.0,1.0,3.0,null,2.5,1.0,9.0]}}},"ops":[{"op":"pick","columns":["region","quarter"]},{"op":"unpivot","index":[],"on":["region","quarter"],"variable":"field","value":"text"}],"expected":{"fields":[{"name":"field","type":"utf8"},{"name":"text","type":"utf8"}],"columns":{"field":["region","region","region","region","region","region","region","region","quarter","quarter","quarter","quarter","quarter","quarter","quarter","quarter"],"text":["north","north","south","south","north","east","south","north","q1","q2","q1","q3","q1","q2","q1",null]}}},{"id":"temporal/extract","description":"calendar fields of timestamps and dates","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"ts","type":"datetime[us]"},{"name":"ms","type":"datetime[ms]"},{"name":"d","type":"date"},{"name":"dur","type":"duration[us]"}],"columns":{"k":["a","b","c","d","e","f"],"ts":[1709214307123456,1704067199999000,1709510400000000,null,-14182940000000,1710052200000500],"ms":[1709214307123,1704067200000,null,1709251200000,0,1709942400000],"d":[19782,19722,19786,19782,null,19792],"dur":[5400000000,-1500000,null,259200000001,0,90000000000]}},"right":{"fields":[{"name":"d","type":"date"},{"name":"name","type":"utf8"}],"columns":{"d":[19782,19792,20089],"name":["leap","sunday","new year"]}}},"ops":[{"op":"select","exprs":{"y":{"fn":"year","args":[{"col":"ts"}]},"mo":{"fn":"month","args":[{"col":"ts"}]},"d":{"fn":"day","args":[{"col":"ts"}]},"h":{"fn":"hour","args":[{"col":"ts"}]},"mi":{"fn":"minute","args":[{"col":"ts"}]},"s":{"fn":"second","args":[{"col":"ts"}]},"ms":{"fn":"millisecond","args":[{"col":"ts"}]},"wd":{"fn":"weekday","args":[{"col":"ts"}]},"od":{"fn":"ordinalDay","args":[{"col":"ts"}]},"dy":{"fn":"year","args":[{"col":"d"}]},"dwd":{"fn":"weekday","args":[{"col":"d"}]},"dod":{"fn":"ordinalDay","args":[{"col":"d"}]}}}],"expected":{"fields":[{"name":"y","type":"i32"},{"name":"mo","type":"i8"},{"name":"d","type":"i8"},{"name":"h","type":"i8"},{"name":"mi","type":"i8"},{"name":"s","type":"i8"},{"name":"ms","type":"i32"},{"name":"wd","type":"i8"},{"name":"od","type":"i16"},{"name":"dy","type":"i32"},{"name":"dwd","type":"i8"},{"name":"dod","type":"i16"}],"columns":{"y":[2024,2023,2024,null,1969,2024],"mo":[2,12,3,null,7,3],"d":[29,31,4,null,20,10],"h":[13,23,0,null,20,6],"mi":[45,59,0,null,17,30],"s":[7,59,0,null,40,0],"ms":[123,999,0,null,0,0],"wd":[4,7,1,null,7,7],"od":[60,365,64,null,201,70],"dy":[2024,2023,2024,2024,null,2024],"dwd":[4,7,1,4,null,7],"dod":[60,365,64,60,null,70]}}},{"id":"temporal/arith","description":"timestamp differences are durations; durations shift timestamps","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"ts","type":"datetime[us]"},{"name":"ms","type":"datetime[ms]"},{"name":"d","type":"date"},{"name":"dur","type":"duration[us]"}],"columns":{"k":["a","b","c","d","e","f"],"ts":[1709214307123456,1704067199999000,1709510400000000,null,-14182940000000,1710052200000500],"ms":[1709214307123,1704067200000,null,1709251200000,0,1709942400000],"d":[19782,19722,19786,19782,null,19792],"dur":[5400000000,-1500000,null,259200000001,0,90000000000]}},"right":{"fields":[{"name":"d","type":"date"},{"name":"name","type":"utf8"}],"columns":{"d":[19782,19792,20089],"name":["leap","sunday","new year"]}}},"ops":[{"op":"select","exprs":{"gap":{"fn":"sub","args":[{"col":"ts"},{"col":"ms"}]},"later":{"fn":"add","args":[{"col":"ts"},{"col":"dur"}]},"earlier":{"fn":"sub","args":[{"col":"ts"},{"col":"dur"}]},"days":{"fn":"sub","args":[{"col":"d"},{"lit":18000,"type":"date"}]},"twice":{"fn":"add","args":[{"col":"dur"},{"col":"dur"}]}}}],"expected":{"fields":[{"name":"gap","type":"duration[ms]"},{"name":"later","type":"datetime[us]"},{"name":"earlier","type":"datetime[us]"},{"name":"days","type":"duration[us]"},{"name":"twice","type":"duration[us]"}],"columns":{"gap":[0,-1,null,null,-14182940000,109800000],"later":[1709219707123456,1704067198499000,null,null,-14182940000000,1710142200000500],"earlier":[1709208907123456,1704067201499000,null,null,-14182940000000,1709962200000500],"days":[153964800000000,148780800000000,154310400000000,153964800000000,null,154828800000000],"twice":[10800000000,-3000000,null,518400000002,0,180000000000]}}},{"id":"temporal/truncate","description":"truncate to a calendar unit","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"ts","type":"datetime[us]"},{"name":"ms","type":"datetime[ms]"},{"name":"d","type":"date"},{"name":"dur","type":"duration[us]"}],"columns":{"k":["a","b","c","d","e","f"],"ts":[1709214307123456,1704067199999000,1709510400000000,null,-14182940000000,1710052200000500],"ms":[1709214307123,1704067200000,null,1709251200000,0,1709942400000],"d":[19782,19722,19786,19782,null,19792],"dur":[5400000000,-1500000,null,259200000001,0,90000000000]}},"right":{"fields":[{"name":"d","type":"date"},{"name":"name","type":"utf8"}],"columns":{"d":[19782,19792,20089],"name":["leap","sunday","new year"]}}},"ops":[{"op":"select","exprs":{"h":{"fn":"truncate","args":[{"col":"ts"}],"every":"1h"},"d":{"fn":"truncate","args":[{"col":"ts"}],"every":"1d"},"w":{"fn":"truncate","args":[{"col":"ts"}],"every":"1w"},"mo":{"fn":"truncate","args":[{"col":"ts"}],"every":"1mo"},"y":{"fn":"truncate","args":[{"col":"ts"}],"every":"1y"},"dmo":{"fn":"truncate","args":[{"col":"d"}],"every":"1mo"}}}],"expected":{"fields":[{"name":"h","type":"datetime[us]"},{"name":"d","type":"datetime[us]"},{"name":"w","type":"datetime[us]"},{"name":"mo","type":"datetime[us]"},{"name":"y","type":"datetime[us]"},{"name":"dmo","type":"date"}],"columns":{"h":[1709211600000000,1704063600000000,1709510400000000,null,-14184000000000,1710050400000000],"d":[1709164800000000,1703980800000000,1709510400000000,null,-14256000000000,1710028800000000],"w":[1708905600000000,1703462400000000,1709510400000000,null,-14774400000000,1709510400000000],"mo":[1706745600000000,1701388800000000,1709251200000000,null,-15897600000000,1709251200000000],"y":[1704067200000000,1672531200000000,1704067200000000,null,-31536000000000,1704067200000000],"dmo":[19754,19692,19783,19754,null,19783]}}},{"id":"temporal/duration","description":"durations in whole units","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"ts","type":"datetime[us]"},{"name":"ms","type":"datetime[ms]"},{"name":"d","type":"date"},{"name":"dur","type":"duration[us]"}],"columns":{"k":["a","b","c","d","e","f"],"ts":[1709214307123456,1704067199999000,1709510400000000,null,-14182940000000,1710052200000500],"ms":[1709214307123,1704067200000,null,1709251200000,0,1709942400000],"d":[19782,19722,19786,19782,null,19792],"dur":[5400000000,-1500000,null,259200000001,0,90000000000]}},"right":{"fields":[{"name":"d","type":"date"},{"name":"name","type":"utf8"}],"columns":{"d":[19782,19792,20089],"name":["leap","sunday","new year"]}}},"ops":[{"op":"select","exprs":{"d":{"fn":"totalDays","args":[{"col":"dur"}]},"h":{"fn":"totalHours","args":[{"col":"dur"}]},"m":{"fn":"totalMinutes","args":[{"col":"dur"}]},"s":{"fn":"totalSeconds","args":[{"col":"dur"}]},"ms":{"fn":"totalMilliseconds","args":[{"col":"dur"}]}}}],"expected":{"fields":[{"name":"d","type":"i64"},{"name":"h","type":"i64"},{"name":"m","type":"i64"},{"name":"s","type":"i64"},{"name":"ms","type":"i64"}],"columns":{"d":["0","0",null,"3","0","1"],"h":["1","0",null,"72","0","25"],"m":["90","0",null,"4320","0","1500"],"s":["5400","-1",null,"259200","0","90000"],"ms":["5400000","-1500",null,"259200000","0","90000000"]}}},{"id":"temporal/cast","description":"casts between dates, timestamps and integers","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"ts","type":"datetime[us]"},{"name":"ms","type":"datetime[ms]"},{"name":"d","type":"date"},{"name":"dur","type":"duration[us]"}],"columns":{"k":["a","b","c","d","e","f"],"ts":[1709214307123456,1704067199999000,1709510400000000,null,-14182940000000,1710052200000500],"ms":[1709214307123,1704067200000,null,1709251200000,0,1709942400000],"d":[19782,19722,19786,19782,null,19792],"dur":[5400000000,-1500000,null,259200000001,0,90000000000]}},"right":{"fields":[{"name":"d","type":"date"},{"name":"name","type":"utf8"}],"columns":{"d":[19782,19792,20089],"name":["leap","sunday","new year"]}}},"ops":[{"op":"select","exprs":{"date":{"fn":"cast","args":[{"col":"ts"}],"to":"date"},"ts":{"fn":"cast","args":[{"col":"d"}],"to":"datetime[us]"},"us":{"fn":"cast","args":[{"col":"ts"}],"to":"i64"},"days":{"fn":"cast","args":[{"col":"d"}],"to":"i32"},"ms":{"fn":"cast","args":[{"col":"ts"}],"to":"datetime[ms]"}}}],"expected":{"fields":[{"name":"date","type":"date"},{"name":"ts","type":"datetime[us]"},{"name":"us","type":"i64"},{"name":"days","type":"i32"},{"name":"ms","type":"datetime[ms]"}],"columns":{"date":[19782,19722,19786,null,-165,19792],"ts":[1709164800000000,1703980800000000,1709510400000000,1709164800000000,null,1710028800000000],"us":["1709214307123456","1704067199999000","1709510400000000",null,"-14182940000000","1710052200000500"],"days":[19782,19722,19786,19782,null,19792],"ms":[1709214307123,1704067199999,1709510400000,null,-14182940000,1710052200000]}}},{"id":"temporal/compare","description":"comparisons with timestamp literals and between units","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"ts","type":"datetime[us]"},{"name":"ms","type":"datetime[ms]"},{"name":"d","type":"date"},{"name":"dur","type":"duration[us]"}],"columns":{"k":["a","b","c","d","e","f"],"ts":[1709214307123456,1704067199999000,1709510400000000,null,-14182940000000,1710052200000500],"ms":[1709214307123,1704067200000,null,1709251200000,0,1709942400000],"d":[19782,19722,19786,19782,null,19792],"dur":[5400000000,-1500000,null,259200000001,0,90000000000]}},"right":{"fields":[{"name":"d","type":"date"},{"name":"name","type":"utf8"}],"columns":{"d":[19782,19792,20089],"name":["leap","sunday","new year"]}}},"ops":[{"op":"filter","predicate":{"fn":"and","args":[{"fn":"gte","args":[{"col":"ts"},{"lit":1709251200000000,"type":"datetime[us]"}]},{"fn":"lt","args":[{"col":"ms"},{"col":"ts"}]}]}}],"expected":{"fields":[{"name":"k","type":"utf8"},{"name":"ts","type":"datetime[us]"},{"name":"ms","type":"datetime[ms]"},{"name":"d","type":"date"},{"name":"dur","type":"duration[us]"}],"columns":{"k":["f"],"ts":[1710052200000500],"ms":[1709942400000],"d":[19792],"dur":[90000000000]}}},{"id":"temporal/group-sort","description":"group by date, aggregate timestamps, sort","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"ts","type":"datetime[us]"},{"name":"ms","type":"datetime[ms]"},{"name":"d","type":"date"},{"name":"dur","type":"duration[us]"}],"columns":{"k":["a","b","c","d","e","f"],"ts":[1709214307123456,1704067199999000,1709510400000000,null,-14182940000000,1710052200000500],"ms":[1709214307123,1704067200000,null,1709251200000,0,1709942400000],"d":[19782,19722,19786,19782,null,19792],"dur":[5400000000,-1500000,null,259200000001,0,90000000000]}},"right":{"fields":[{"name":"d","type":"date"},{"name":"name","type":"utf8"}],"columns":{"d":[19782,19792,20089],"name":["leap","sunday","new year"]}}},"ops":[{"op":"groupBy","keys":["d"],"aggs":{"first":{"agg":"min","arg":{"col":"ts"}},"last":{"agg":"max","arg":{"col":"ts"}},"n":{"agg":"len"},"total":{"agg":"sum","arg":{"col":"dur"}}}},{"op":"sort","keys":[{"expr":{"col":"last"},"descending":true}]}],"expected":{"fields":[{"name":"d","type":"date"},{"name":"first","type":"datetime[us]"},{"name":"last","type":"datetime[us]"},{"name":"n","type":"u32"},{"name":"total","type":"duration[us]"}],"columns":{"d":[19792,19786,19782,19722,null],"first":[1710052200000500,1709510400000000,1709214307123456,1704067199999000,-14182940000000],"last":[1710052200000500,1709510400000000,1709214307123456,1704067199999000,-14182940000000],"n":[1,1,2,1,1],"total":[90000000000,0,264600000001,-1500000,0]}}},{"id":"temporal/join","description":"join on a date","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"ts","type":"datetime[us]"},{"name":"ms","type":"datetime[ms]"},{"name":"d","type":"date"},{"name":"dur","type":"duration[us]"}],"columns":{"k":["a","b","c","d","e","f"],"ts":[1709214307123456,1704067199999000,1709510400000000,null,-14182940000000,1710052200000500],"ms":[1709214307123,1704067200000,null,1709251200000,0,1709942400000],"d":[19782,19722,19786,19782,null,19792],"dur":[5400000000,-1500000,null,259200000001,0,90000000000]}},"right":{"fields":[{"name":"d","type":"date"},{"name":"name","type":"utf8"}],"columns":{"d":[19782,19792,20089],"name":["leap","sunday","new year"]}}},"ops":[{"op":"join","right":"right","on":["d"],"how":"left"}],"expected":{"fields":[{"name":"k","type":"utf8"},{"name":"ts","type":"datetime[us]"},{"name":"ms","type":"datetime[ms]"},{"name":"d","type":"date"},{"name":"dur","type":"duration[us]"},{"name":"name","type":"utf8"}],"columns":{"k":["a","b","c","d","e","f"],"ts":[1709214307123456,1704067199999000,1709510400000000,null,-14182940000000,1710052200000500],"ms":[1709214307123,1704067200000,null,1709251200000,0,1709942400000],"d":[19782,19722,19786,19782,null,19792],"dur":[5400000000,-1500000,null,259200000001,0,90000000000],"name":["leap",null,null,"leap",null,"sunday"]}}},{"id":"nested/list-ops","description":"list length, element access and aggregates","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"g","type":"i32"},{"name":"xs","type":"list"},{"name":"ws","type":"list"},{"name":"s","type":"struct"}],"columns":{"k":["a","b","c","d","e"],"g":[1,2,1,2,1],"xs":[[1,2,3],[],null,[4,null,2],[7]],"ws":[["a","b"],["c"],[],null,["z","a"]],"s":[{"a":1,"b":"x"},{"a":null,"b":"y"},null,{"a":4,"b":null},{"a":5,"b":"z"}]}},"right":{"fields":[{"name":"k","type":"utf8"},{"name":"tags","type":"list"}],"columns":{"k":["a","d","x"],"tags":[["t1"],["t2","t3"],[]]}}},"ops":[{"op":"select","exprs":{"n":{"fn":"listLength","args":[{"col":"xs"}]},"first":{"fn":"listGet","args":[{"col":"xs"}],"index":0},"last":{"fn":"listGet","args":[{"col":"xs"}],"index":-1},"oob":{"fn":"listGet","args":[{"col":"xs"}],"index":5},"sum":{"fn":"listSum","args":[{"col":"xs"}]},"mean":{"fn":"listMean","args":[{"col":"xs"}]},"min":{"fn":"listMin","args":[{"col":"xs"}]},"max":{"fn":"listMax","args":[{"col":"ws"}]},"has2":{"fn":"listContains","args":[{"col":"xs"},{"lit":2}]},"hasA":{"fn":"listContains","args":[{"col":"ws"},{"lit":"a"}]}}}],"expected":{"fields":[{"name":"n","type":"u32"},{"name":"first","type":"i32"},{"name":"last","type":"i32"},{"name":"oob","type":"i32"},{"name":"sum","type":"i32"},{"name":"mean","type":"f64"},{"name":"min","type":"i32"},{"name":"max","type":"utf8"},{"name":"has2","type":"bool"},{"name":"hasA","type":"bool"}],"columns":{"n":[3,0,null,3,1],"first":[1,null,null,4,7],"last":[3,null,null,2,7],"oob":[null,null,null,null,null],"sum":[6,0,null,6,7],"mean":[2.0,null,null,3.0,7.0],"min":[1,null,null,2,7],"max":["b","c",null,null,"z"],"has2":[true,false,null,true,false],"hasA":[true,false,false,null,true]}}},{"id":"nested/struct","description":"struct fields and struct construction","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"g","type":"i32"},{"name":"xs","type":"list"},{"name":"ws","type":"list"},{"name":"s","type":"struct"}],"columns":{"k":["a","b","c","d","e"],"g":[1,2,1,2,1],"xs":[[1,2,3],[],null,[4,null,2],[7]],"ws":[["a","b"],["c"],[],null,["z","a"]],"s":[{"a":1,"b":"x"},{"a":null,"b":"y"},null,{"a":4,"b":null},{"a":5,"b":"z"}]}},"right":{"fields":[{"name":"k","type":"utf8"},{"name":"tags","type":"list"}],"columns":{"k":["a","d","x"],"tags":[["t1"],["t2","t3"],[]]}}},"ops":[{"op":"select","exprs":{"a":{"fn":"field","args":[{"col":"s"}],"field":"a"},"b":{"fn":"field","args":[{"col":"s"}],"field":"b"},"pair":{"fn":"struct","args":[{"col":"k"},{"fn":"listLength","args":[{"col":"xs"}]}],"names":["key","size"]}}}],"expected":{"fields":[{"name":"a","type":"i32"},{"name":"b","type":"utf8"},{"name":"pair","type":"struct"}],"columns":{"a":[1,null,null,4,5],"b":["x","y",null,null,"z"],"pair":[{"key":"a","size":3},{"key":"b","size":0},{"key":"c","size":null},{"key":"d","size":3},{"key":"e","size":1}]}}},{"id":"nested/explode","description":"explode lists into rows; a null list gives a null row, an empty list no rows","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"g","type":"i32"},{"name":"xs","type":"list"},{"name":"ws","type":"list"},{"name":"s","type":"struct"}],"columns":{"k":["a","b","c","d","e"],"g":[1,2,1,2,1],"xs":[[1,2,3],[],null,[4,null,2],[7]],"ws":[["a","b"],["c"],[],null,["z","a"]],"s":[{"a":1,"b":"x"},{"a":null,"b":"y"},null,{"a":4,"b":null},{"a":5,"b":"z"}]}},"right":{"fields":[{"name":"k","type":"utf8"},{"name":"tags","type":"list"}],"columns":{"k":["a","d","x"],"tags":[["t1"],["t2","t3"],[]]}}},"ops":[{"op":"pick","columns":["k","xs"]},{"op":"explode","column":"xs"}],"expected":{"fields":[{"name":"k","type":"utf8"},{"name":"xs","type":"i32"}],"columns":{"k":["a","a","a","c","d","d","d","e"],"xs":[1,2,3,null,4,null,2,7]}}},{"id":"nested/unnest","description":"unnest a struct into columns","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"g","type":"i32"},{"name":"xs","type":"list"},{"name":"ws","type":"list"},{"name":"s","type":"struct"}],"columns":{"k":["a","b","c","d","e"],"g":[1,2,1,2,1],"xs":[[1,2,3],[],null,[4,null,2],[7]],"ws":[["a","b"],["c"],[],null,["z","a"]],"s":[{"a":1,"b":"x"},{"a":null,"b":"y"},null,{"a":4,"b":null},{"a":5,"b":"z"}]}},"right":{"fields":[{"name":"k","type":"utf8"},{"name":"tags","type":"list"}],"columns":{"k":["a","d","x"],"tags":[["t1"],["t2","t3"],[]]}}},"ops":[{"op":"pick","columns":["k","s"]},{"op":"unnest","column":"s"}],"expected":{"fields":[{"name":"k","type":"utf8"},{"name":"a","type":"i32"},{"name":"b","type":"utf8"}],"columns":{"k":["a","b","c","d","e"],"a":[1,null,null,4,5],"b":["x","y",null,null,"z"]}}},{"id":"nested/agg-list","description":"collect values into lists per group","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"g","type":"i32"},{"name":"xs","type":"list"},{"name":"ws","type":"list"},{"name":"s","type":"struct"}],"columns":{"k":["a","b","c","d","e"],"g":[1,2,1,2,1],"xs":[[1,2,3],[],null,[4,null,2],[7]],"ws":[["a","b"],["c"],[],null,["z","a"]],"s":[{"a":1,"b":"x"},{"a":null,"b":"y"},null,{"a":4,"b":null},{"a":5,"b":"z"}]}},"right":{"fields":[{"name":"k","type":"utf8"},{"name":"tags","type":"list"}],"columns":{"k":["a","d","x"],"tags":[["t1"],["t2","t3"],[]]}}},"ops":[{"op":"groupBy","keys":["g"],"aggs":{"ks":{"agg":"list","arg":{"col":"k"}},"firsts":{"agg":"first","arg":{"col":"xs"}},"n":{"agg":"len"}}}],"expected":{"fields":[{"name":"g","type":"i32"},{"name":"ks","type":"list"},{"name":"firsts","type":"list"},{"name":"n","type":"u32"}],"columns":{"g":[1,2],"ks":[["a","c","e"],["b","d"]],"firsts":[[1,2,3],[]],"n":[3,2]}}},{"id":"nested/carry","description":"lists and structs survive filters, sorts, joins and slices","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"g","type":"i32"},{"name":"xs","type":"list"},{"name":"ws","type":"list"},{"name":"s","type":"struct"}],"columns":{"k":["a","b","c","d","e"],"g":[1,2,1,2,1],"xs":[[1,2,3],[],null,[4,null,2],[7]],"ws":[["a","b"],["c"],[],null,["z","a"]],"s":[{"a":1,"b":"x"},{"a":null,"b":"y"},null,{"a":4,"b":null},{"a":5,"b":"z"}]}},"right":{"fields":[{"name":"k","type":"utf8"},{"name":"tags","type":"list"}],"columns":{"k":["a","d","x"],"tags":[["t1"],["t2","t3"],[]]}}},"ops":[{"op":"filter","predicate":{"fn":"neq","args":[{"col":"k"},{"lit":"c"}]}},{"op":"sort","keys":[{"expr":{"col":"g"},"descending":true},{"expr":{"col":"k"}}]},{"op":"join","right":"right","on":["k"],"how":"left"},{"op":"head","n":4}],"expected":{"fields":[{"name":"k","type":"utf8"},{"name":"g","type":"i32"},{"name":"xs","type":"list"},{"name":"ws","type":"list"},{"name":"s","type":"struct"},{"name":"tags","type":"list"}],"columns":{"k":["b","d","a","e"],"g":[2,2,1,1],"xs":[[],[4,null,2],[1,2,3],[7]],"ws":[["c"],null,["a","b"],["z","a"]],"s":[{"a":null,"b":"y"},{"a":4,"b":null},{"a":1,"b":"x"},{"a":5,"b":"z"}],"tags":[null,["t2","t3"],["t1"],null]}}},{"id":"decimal/arith","description":"decimal addition, subtraction and multiplication","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"price","type":"decimal[10,2]"},{"name":"fee","type":"decimal[6,3]"},{"name":"qty","type":"decimal[4,0]"},{"name":"x","type":"f64"}],"columns":{"k":["a","b","a","c","b","a"],"price":["12.34","0.10","-5.00",null,"99999999.99","10.01"],"fee":["0.125","1.000",null,"2.500","-0.001","10.010"],"qty":["3","10","2","1",null,"7"],"x":[1.005,-2.5,123.456,null,1000000.0,0.0]}}},"ops":[{"op":"select","exprs":{"sum":{"fn":"add","args":[{"col":"price"},{"col":"fee"}]},"diff":{"fn":"sub","args":[{"col":"price"},{"col":"fee"}]},"prod":{"fn":"mul","args":[{"col":"price"},{"col":"qty"}]},"plus":{"fn":"add","args":[{"col":"price"},{"lit":"0.5","type":"decimal[3,1]"}]}}}],"expected":{"fields":[{"name":"sum","type":"decimal[38,3]"},{"name":"diff","type":"decimal[38,3]"},{"name":"prod","type":"decimal[38,2]"},{"name":"plus","type":"decimal[38,2]"}],"columns":{"sum":["12.465","1.100",null,null,"99999999.989","20.020"],"diff":["12.215","-0.900",null,null,"99999999.991","0.000"],"prod":["37.02","1.00","-10.00",null,null,"70.07"],"plus":["12.84","0.60","-4.50",null,"100000000.49","10.51"]}}},{"id":"decimal/compare","description":"decimal comparisons, also across scales","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"price","type":"decimal[10,2]"},{"name":"fee","type":"decimal[6,3]"},{"name":"qty","type":"decimal[4,0]"},{"name":"x","type":"f64"}],"columns":{"k":["a","b","a","c","b","a"],"price":["12.34","0.10","-5.00",null,"99999999.99","10.01"],"fee":["0.125","1.000",null,"2.500","-0.001","10.010"],"qty":["3","10","2","1",null,"7"],"x":[1.005,-2.5,123.456,null,1000000.0,0.0]}}},"ops":[{"op":"filter","predicate":{"fn":"and","args":[{"fn":"gt","args":[{"col":"price"},{"lit":"10.00","type":"decimal[10,2]"}]},{"fn":"gte","args":[{"col":"price"},{"col":"fee"}]}]}}],"expected":{"fields":[{"name":"k","type":"utf8"},{"name":"price","type":"decimal[10,2]"},{"name":"fee","type":"decimal[6,3]"},{"name":"qty","type":"decimal[4,0]"},{"name":"x","type":"f64"}],"columns":{"k":["a","b","a"],"price":["12.34","99999999.99","10.01"],"fee":["0.125","-0.001","10.010"],"qty":["3",null,"7"],"x":[1.005,1000000.0,0.0]}}},{"id":"decimal/aggregate","description":"sums, extremes and means of decimals per group","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"price","type":"decimal[10,2]"},{"name":"fee","type":"decimal[6,3]"},{"name":"qty","type":"decimal[4,0]"},{"name":"x","type":"f64"}],"columns":{"k":["a","b","a","c","b","a"],"price":["12.34","0.10","-5.00",null,"99999999.99","10.01"],"fee":["0.125","1.000",null,"2.500","-0.001","10.010"],"qty":["3","10","2","1",null,"7"],"x":[1.005,-2.5,123.456,null,1000000.0,0.0]}}},"ops":[{"op":"groupBy","keys":["k"],"aggs":{"total":{"agg":"sum","arg":{"col":"price"}},"min":{"agg":"min","arg":{"col":"price"}},"max":{"agg":"max","arg":{"col":"fee"}},"mean":{"agg":"mean","arg":{"col":"price"}},"n":{"agg":"count","arg":{"col":"price"}}}}],"expected":{"fields":[{"name":"k","type":"utf8"},{"name":"total","type":"decimal[38,2]"},{"name":"min","type":"decimal[10,2]"},{"name":"max","type":"decimal[6,3]"},{"name":"mean","type":"f64"},{"name":"n","type":"u32"}],"columns":{"k":["a","b","c"],"total":["17.35","100000000.09","0.00"],"min":["-5.00","0.10",null],"max":["10.010","1.000","2.500"],"mean":[5.783333333333334,50000000.044999994,null],"n":[3,2,0]}}},{"id":"decimal/cast","description":"casts between decimals, floats, integers and strings","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"price","type":"decimal[10,2]"},{"name":"fee","type":"decimal[6,3]"},{"name":"qty","type":"decimal[4,0]"},{"name":"x","type":"f64"}],"columns":{"k":["a","b","a","c","b","a"],"price":["12.34","0.10","-5.00",null,"99999999.99","10.01"],"fee":["0.125","1.000",null,"2.500","-0.001","10.010"],"qty":["3","10","2","1",null,"7"],"x":[1.005,-2.5,123.456,null,1000000.0,0.0]}}},"ops":[{"op":"select","exprs":{"f":{"fn":"cast","args":[{"col":"price"}],"to":"f64"},"i":{"fn":"cast","args":[{"col":"price"}],"to":"i32"},"s":{"fn":"cast","args":[{"col":"fee"}],"to":"utf8"},"d":{"fn":"cast","args":[{"col":"x"}],"to":"decimal[10,2]"},"r":{"fn":"cast","args":[{"col":"price"}],"to":"decimal[12,1]"}}}],"expected":{"fields":[{"name":"f","type":"f64"},{"name":"i","type":"i32"},{"name":"s","type":"utf8"},{"name":"d","type":"decimal[10,2]"},{"name":"r","type":"decimal[12,1]"}],"columns":{"f":[12.34,0.1,-5.0,null,99999999.99,10.01],"i":[12,0,-5,null,99999999,10],"s":["0.125","1.000",null,"2.500","-0.001","10.010"],"d":["1.00","-2.50","123.46",null,"1000000.00","0.00"],"r":["12.3","0.1","-5.0",null,"100000000.0","10.0"]}}},{"id":"decimal/sort","description":"sort by a decimal","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"price","type":"decimal[10,2]"},{"name":"fee","type":"decimal[6,3]"},{"name":"qty","type":"decimal[4,0]"},{"name":"x","type":"f64"}],"columns":{"k":["a","b","a","c","b","a"],"price":["12.34","0.10","-5.00",null,"99999999.99","10.01"],"fee":["0.125","1.000",null,"2.500","-0.001","10.010"],"qty":["3","10","2","1",null,"7"],"x":[1.005,-2.5,123.456,null,1000000.0,0.0]}}},"ops":[{"op":"sort","keys":[{"expr":{"col":"price"},"descending":true}],"nullsLast":true}],"expected":{"fields":[{"name":"k","type":"utf8"},{"name":"price","type":"decimal[10,2]"},{"name":"fee","type":"decimal[6,3]"},{"name":"qty","type":"decimal[4,0]"},{"name":"x","type":"f64"}],"columns":{"k":["b","a","a","b","a","c"],"price":["99999999.99","12.34","10.01","0.10","-5.00",null],"fee":["-0.001","0.125","10.010","1.000",null,"2.500"],"qty":[null,"3","7","10","2","1"],"x":[1000000.0,1.005,0.0,-2.5,123.456,null]}}},{"id":"decimal/round-ties","description":"narrowing the scale rounds exact ties","inputs":{"left":{"fields":[{"name":"t","type":"decimal[6,3]"}],"columns":{"t":["0.250","0.350","-0.250","-0.350","1.050","2.500","0.249"]}}},"ops":[{"op":"select","exprs":{"one":{"fn":"cast","args":[{"col":"t"}],"to":"decimal[6,1]"},"zero":{"fn":"cast","args":[{"col":"t"}],"to":"decimal[6,0]"}}}],"expected":{"fields":[{"name":"one","type":"decimal[6,1]"},{"name":"zero","type":"decimal[6,0]"}],"columns":{"one":["0.2","0.4","-0.2","-0.4","1.0","2.5","0.2"],"zero":["0","0","0","0","1","2","0"]}}},{"id":"decimal/cast-float-ties","description":"floats cast to decimals round exact ties half to even, also below zero","inputs":{"left":{"fields":[{"name":"x","type":"f64"}],"columns":{"x":[-1.5,-2.5,1.5,2.5,-0.75]}}},"ops":[{"op":"select","exprs":{"zero":{"fn":"cast","args":[{"col":"x"}],"to":"decimal[18,0]"},"one":{"fn":"cast","args":[{"col":"x"}],"to":"decimal[18,1]"}}}],"expected":{"fields":[{"name":"zero","type":"decimal[18,0]"},{"name":"one","type":"decimal[18,1]"}],"columns":{"zero":["-2","-2","2","2","-1"],"one":["-1.5","-2.5","1.5","2.5","-0.8"]}}},{"id":"decimal/round","description":"rounding decimals keeps the scale and rounds half to even","inputs":{"left":{"fields":[{"name":"t","type":"decimal[6,3]"}],"columns":{"t":["0.250","0.350","-0.250","-0.350","1.050","2.500","0.249",null]}}},"ops":[{"op":"select","exprs":{"r0":{"fn":"round","args":[{"col":"t"}]},"r1":{"fn":"round","args":[{"col":"t"}],"decimals":1},"r2":{"fn":"round","args":[{"col":"t"}],"decimals":2},"r3":{"fn":"round","args":[{"col":"t"}],"decimals":3}}}],"expected":{"fields":[{"name":"r0","type":"decimal[7,3]"},{"name":"r1","type":"decimal[7,3]"},{"name":"r2","type":"decimal[7,3]"},{"name":"r3","type":"decimal[6,3]"}],"columns":{"r0":["0.000","0.000","0.000","0.000","1.000","2.000","0.000",null],"r1":["0.200","0.400","-0.200","-0.400","1.000","2.500","0.200",null],"r2":["0.250","0.350","-0.250","-0.350","1.050","2.500","0.250",null],"r3":["0.250","0.350","-0.250","-0.350","1.050","2.500","0.249",null]}}},{"id":"arith/u32-divide-by-zero","description":"unsigned 32-bit floor division and modulo by zero give null","inputs":{"left":{"fields":[{"name":"n","type":"u32"},{"name":"d","type":"u32"}],"columns":{"n":[7,0,5],"d":[0,0,2]}}},"ops":[{"op":"select","exprs":{"q":{"fn":"floorDiv","args":[{"col":"n"},{"col":"d"}]},"m":{"fn":"mod","args":[{"col":"n"},{"col":"d"}]},"qs":{"fn":"floorDiv","args":[{"col":"n"},{"lit":0,"type":"u32"}]},"ms":{"fn":"mod","args":[{"col":"n"},{"lit":0,"type":"u32"}]}}}],"expected":{"fields":[{"name":"q","type":"u32"},{"name":"m","type":"u32"},{"name":"qs","type":"u32"},{"name":"ms","type":"u32"}],"columns":{"q":[null,null,2],"m":[null,null,1],"qs":[null,null,null],"ms":[null,null,null]}}},{"id":"slice/clip-5+1","description":"slices reaching before the first row clip from the unclamped start","inputs":{"left":{"fields":[{"name":"x","type":"i32"}],"columns":{"x":[1,2,3]}}},"ops":[{"op":"slice","offset":-5,"length":1}],"expected":{"fields":[{"name":"x","type":"i32"}],"columns":{"x":[]}}},{"id":"slice/clip-5+3","description":"slices reaching before the first row clip from the unclamped start","inputs":{"left":{"fields":[{"name":"x","type":"i32"}],"columns":{"x":[1,2,3]}}},"ops":[{"op":"slice","offset":-5,"length":3}],"expected":{"fields":[{"name":"x","type":"i32"}],"columns":{"x":[1]}}},{"id":"slice/clip-5+4","description":"slices reaching before the first row clip from the unclamped start","inputs":{"left":{"fields":[{"name":"x","type":"i32"}],"columns":{"x":[1,2,3]}}},"ops":[{"op":"slice","offset":-5,"length":4}],"expected":{"fields":[{"name":"x","type":"i32"}],"columns":{"x":[1,2]}}},{"id":"slice/clip-2+5","description":"slices reaching before the first row clip from the unclamped start","inputs":{"left":{"fields":[{"name":"x","type":"i32"}],"columns":{"x":[1,2,3]}}},"ops":[{"op":"slice","offset":-2,"length":5}],"expected":{"fields":[{"name":"x","type":"i32"}],"columns":{"x":[2,3]}}},{"id":"slice/clip-5","description":"slices reaching before the first row clip from the unclamped start","inputs":{"left":{"fields":[{"name":"x","type":"i32"}],"columns":{"x":[1,2,3]}}},"ops":[{"op":"slice","offset":-5}],"expected":{"fields":[{"name":"x","type":"i32"}],"columns":{"x":[1,2,3]}}},{"id":"slice/clip1+0","description":"slices reaching before the first row clip from the unclamped start","inputs":{"left":{"fields":[{"name":"x","type":"i32"}],"columns":{"x":[1,2,3]}}},"ops":[{"op":"slice","offset":1,"length":0}],"expected":{"fields":[{"name":"x","type":"i32"}],"columns":{"x":[]}}},{"id":"window/filter-after-sort","description":"a window predicate sees the sorted rows","inputs":{"left":{"fields":[{"name":"g","type":"i32"},{"name":"h","type":"i32"},{"name":"x","type":"f64"}],"columns":{"g":[1,2,1,2],"h":[1,1,2,2],"x":[3.0,1.0,2.0,4.0]}},"right":{"fields":[{"name":"g","type":"i32"},{"name":"name","type":"utf8"}],"columns":{"g":[1],"name":["one"]}}},"ops":[{"op":"sort","keys":[{"expr":{"col":"x"}}]},{"op":"filter","predicate":{"fn":"gt","args":[{"fn":"cumSum","args":[{"col":"x"}]},{"lit":3.0}]}}],"expected":{"fields":[{"name":"g","type":"i32"},{"name":"h","type":"i32"},{"name":"x","type":"f64"}],"columns":{"g":[1,2],"h":[1,2],"x":[3.0,4.0]}}},{"id":"window/filter-after-sort-by-window","description":"a filter after a sort by a window leaves the sort's order alone","inputs":{"left":{"fields":[{"name":"x","type":"f64"}],"columns":{"x":[-3.0,5.0,-1.0]}}},"ops":[{"op":"sort","keys":[{"expr":{"fn":"cumSum","args":[{"col":"x"}]}}]},{"op":"filter","predicate":{"fn":"neq","args":[{"col":"x"},{"lit":5.0}]}}],"expected":{"fields":[{"name":"x","type":"f64"}],"columns":{"x":[-3.0,-1.0]}}},{"id":"window/filter-after-filter","description":"a window predicate sees only the rows an earlier filter kept","inputs":{"left":{"fields":[{"name":"g","type":"i32"},{"name":"h","type":"i32"},{"name":"x","type":"f64"}],"columns":{"g":[1,2,1,2],"h":[1,1,2,2],"x":[3.0,1.0,2.0,4.0]}},"right":{"fields":[{"name":"g","type":"i32"},{"name":"name","type":"utf8"}],"columns":{"g":[1],"name":["one"]}}},"ops":[{"op":"filter","predicate":{"fn":"gt","args":[{"col":"x"},{"lit":1.0}]}},{"op":"filter","predicate":{"fn":"gt","args":[{"fn":"shift","args":[{"col":"x"}]},{"lit":2.0}]}}],"expected":{"fields":[{"name":"g","type":"i32"},{"name":"h","type":"i32"},{"name":"x","type":"f64"}],"columns":{"g":[1],"h":[2],"x":[2.0]}}},{"id":"window/filter-after-concat","description":"a window predicate sees every input of a concat","inputs":{"left":{"fields":[{"name":"g","type":"i32"},{"name":"h","type":"i32"},{"name":"x","type":"f64"}],"columns":{"g":[1,2,1,2],"h":[1,1,2,2],"x":[3.0,1.0,2.0,4.0]}},"right":{"fields":[{"name":"g","type":"i32"},{"name":"name","type":"utf8"}],"columns":{"g":[1],"name":["one"]}}},"ops":[{"op":"concat","with":"left"},{"op":"filter","predicate":{"fn":"gt","args":[{"fn":"cumSum","args":[{"col":"x"}]},{"lit":15.0}]}}],"expected":{"fields":[{"name":"g","type":"i32"},{"name":"h","type":"i32"},{"name":"x","type":"f64"}],"columns":{"g":[1,2],"h":[2,2],"x":[2.0,4.0]}}},{"id":"window/filter-after-join","description":"a window predicate sees only the joined rows","inputs":{"left":{"fields":[{"name":"g","type":"i32"},{"name":"h","type":"i32"},{"name":"x","type":"f64"}],"columns":{"g":[1,2,1,2],"h":[1,1,2,2],"x":[3.0,1.0,2.0,4.0]}},"right":{"fields":[{"name":"g","type":"i32"},{"name":"name","type":"utf8"}],"columns":{"g":[1],"name":["one"]}}},"ops":[{"op":"join","right":"right","on":["g"],"how":"inner"},{"op":"filter","predicate":{"fn":"gt","args":[{"fn":"shift","args":[{"col":"x"}]},{"lit":2.0}]}}],"expected":{"fields":[{"name":"g","type":"i32"},{"name":"h","type":"i32"},{"name":"x","type":"f64"},{"name":"name","type":"utf8"}],"columns":{"g":[1],"h":[2],"x":[2.0],"name":["one"]}}},{"id":"window/filter-whole","description":"a window predicate sees the whole input","inputs":{"left":{"fields":[{"name":"g","type":"i32"},{"name":"h","type":"i32"},{"name":"x","type":"f64"}],"columns":{"g":[1,2,1,2],"h":[1,1,2,2],"x":[3.0,1.0,2.0,4.0]}},"right":{"fields":[{"name":"g","type":"i32"},{"name":"name","type":"utf8"}],"columns":{"g":[1],"name":["one"]}}},"ops":[{"op":"filter","predicate":{"fn":"gt","args":[{"fn":"cumSum","args":[{"col":"x"}]},{"lit":5.0}]}}],"expected":{"fields":[{"name":"g","type":"i32"},{"name":"h","type":"i32"},{"name":"x","type":"f64"}],"columns":{"g":[1,2],"h":[2,2],"x":[2.0,4.0]}}},{"id":"window/agg-of-window","description":"an aggregate of a window sees the whole input","inputs":{"left":{"fields":[{"name":"g","type":"i32"},{"name":"h","type":"i32"},{"name":"x","type":"f64"}],"columns":{"g":[1,2,1,2],"h":[1,1,2,2],"x":[3.0,1.0,2.0,4.0]}},"right":{"fields":[{"name":"g","type":"i32"},{"name":"name","type":"utf8"}],"columns":{"g":[1],"name":["one"]}}},"ops":[{"op":"agg","aggs":{"s":{"agg":"sum","arg":{"fn":"cumSum","args":[{"col":"x"}]}},"n":{"agg":"len"}}}],"expected":{"fields":[{"name":"s","type":"f64"},{"name":"n","type":"u32"}],"columns":{"s":[23.0],"n":[4]}}},{"id":"window/grouped-agg-of-window","description":"windows inside grouped aggregates run per group","inputs":{"left":{"fields":[{"name":"g","type":"i32"},{"name":"h","type":"i32"},{"name":"x","type":"f64"}],"columns":{"g":[1,2,1,2],"h":[1,1,2,2],"x":[3.0,1.0,2.0,4.0]}},"right":{"fields":[{"name":"g","type":"i32"},{"name":"name","type":"utf8"}],"columns":{"g":[1],"name":["one"]}}},"ops":[{"op":"groupBy","keys":["g"],"aggs":{"s":{"agg":"sum","arg":{"fn":"cumSum","args":[{"col":"x"}]}},"l":{"agg":"list","arg":{"fn":"cumSum","args":[{"col":"x"}]}},"o":{"agg":"sum","arg":{"over":{"agg":"sum","arg":{"col":"x"}},"partitionBy":[{"col":"h"}]}}}}],"expected":{"fields":[{"name":"g","type":"i32"},{"name":"s","type":"f64"},{"name":"l","type":"list"},{"name":"o","type":"f64"}],"columns":{"g":[1,2],"s":[8.0,6.0],"l":[[3.0,5.0],[1.0,5.0]],"o":[5.0,5.0]}}}]} +{"generator":"generate_gold.py","polars":"2.1.0","deviations":["JavaScript has no float literal distinct from an integer: an integral number literal behaves like a Polars integer literal (it takes the integer type of the column it meets), so cases never combine an integer column with an integral float literal.","Float to string casts use JavaScript number formatting (Polars prints 5.0 as '5.0', JavaScript as '5').","Joins never add '_right' suffixes: overlapping non-key columns are a type error.","Arithmetic between 64-bit integers (bigint) and other numeric types requires an explicit cast."],"cases":[{"id":"window/cum","description":"cumulative sum, min, max and count; nulls stay null","inputs":{"left":{"fields":[{"name":"g","type":"utf8"},{"name":"t","type":"i32"},{"name":"x","type":"f64"},{"name":"i","type":"i32"},{"name":"b","type":"bool"}],"columns":{"g":["a","b","a","a","b",null,"a","b","b","a"],"t":[1,1,2,3,2,1,4,3,4,5],"x":[1.5,2.0,null,4.0,"NaN",3.0,-1.0,2.0,0.5,2.0],"i":[3,1,4,1,5,9,null,6,5,3],"b":[true,false,null,true,true,false,true,null,false,true]}}},"ops":[{"op":"select","exprs":{"sx":{"fn":"cumSum","args":[{"col":"x"}]},"si":{"fn":"cumSum","args":[{"col":"i"}]},"mn":{"fn":"cumMin","args":[{"col":"i"}]},"mx":{"fn":"cumMax","args":[{"col":"x"}]},"n":{"fn":"cumCount","args":[{"col":"x"}]},"sb":{"fn":"cumSum","args":[{"col":"b"}]}}}],"expected":{"fields":[{"name":"sx","type":"f64"},{"name":"si","type":"i32"},{"name":"mn","type":"i32"},{"name":"mx","type":"f64"},{"name":"n","type":"u32"},{"name":"sb","type":"u32"}],"columns":{"sx":[1.5,3.5,null,7.5,"NaN","NaN","NaN","NaN","NaN","NaN"],"si":[3,4,8,9,14,23,null,29,34,37],"mn":[3,1,1,1,1,1,null,1,1,1],"mx":[1.5,2.0,null,4.0,4.0,4.0,4.0,4.0,4.0,4.0],"n":[1,2,2,3,4,5,6,7,8,9],"sb":[1,1,null,2,3,3,4,null,4,5]}}},{"id":"window/shift-diff","description":"shift (lag and lead) and diff","inputs":{"left":{"fields":[{"name":"g","type":"utf8"},{"name":"t","type":"i32"},{"name":"x","type":"f64"},{"name":"i","type":"i32"},{"name":"b","type":"bool"}],"columns":{"g":["a","b","a","a","b",null,"a","b","b","a"],"t":[1,1,2,3,2,1,4,3,4,5],"x":[1.5,2.0,null,4.0,"NaN",3.0,-1.0,2.0,0.5,2.0],"i":[3,1,4,1,5,9,null,6,5,3],"b":[true,false,null,true,true,false,true,null,false,true]}}},"ops":[{"op":"select","exprs":{"lag":{"fn":"shift","args":[{"col":"i"}]},"lead":{"fn":"shift","args":[{"col":"x"}],"n":-2},"d":{"fn":"diff","args":[{"col":"i"}]},"d2":{"fn":"diff","args":[{"col":"x"}],"n":2},"ls":{"fn":"shift","args":[{"col":"g"}]}}}],"expected":{"fields":[{"name":"lag","type":"i32"},{"name":"lead","type":"f64"},{"name":"d","type":"i32"},{"name":"d2","type":"f64"},{"name":"ls","type":"utf8"}],"columns":{"lag":[null,3,1,4,1,5,9,null,6,5],"lead":[null,4.0,"NaN",3.0,-1.0,2.0,0.5,2.0,null,null],"d":[null,-2,3,-3,4,4,null,null,-1,-2],"d2":[null,null,null,2.0,null,-1.0,"NaN",-1.0,1.5,0.0],"ls":[null,"a","b","a","a","b",null,"a","b","b"]}}},{"id":"window/rank","description":"rank methods, nulls and NaN","inputs":{"left":{"fields":[{"name":"g","type":"utf8"},{"name":"t","type":"i32"},{"name":"x","type":"f64"},{"name":"i","type":"i32"},{"name":"b","type":"bool"}],"columns":{"g":["a","b","a","a","b",null,"a","b","b","a"],"t":[1,1,2,3,2,1,4,3,4,5],"x":[1.5,2.0,null,4.0,"NaN",3.0,-1.0,2.0,0.5,2.0],"i":[3,1,4,1,5,9,null,6,5,3],"b":[true,false,null,true,true,false,true,null,false,true]}}},"ops":[{"op":"select","exprs":{"avg":{"fn":"rank","args":[{"col":"x"}]},"min":{"fn":"rank","args":[{"col":"i"}],"method":"min"},"max":{"fn":"rank","args":[{"col":"i"}],"method":"max"},"dense":{"fn":"rank","args":[{"col":"i"}],"method":"dense","descending":true},"ord":{"fn":"rank","args":[{"col":"x"}],"method":"ordinal"},"s":{"fn":"rank","args":[{"col":"g"}],"method":"dense"}}}],"expected":{"fields":[{"name":"avg","type":"f64"},{"name":"min","type":"u32"},{"name":"max","type":"u32"},{"name":"dense","type":"u32"},{"name":"ord","type":"u32"},{"name":"s","type":"u32"}],"columns":{"avg":[3.0,5.0,null,8.0,9.0,7.0,1.0,5.0,2.0,5.0],"min":[3,1,5,1,6,9,null,8,6,3],"max":[4,2,5,2,7,9,null,8,7,4],"dense":[5,6,4,6,3,1,null,2,3,5],"ord":[3,4,null,8,9,7,1,5,2,6],"s":[1,2,1,1,2,null,1,2,2,1]}}},{"id":"window/rolling","description":"rolling sum, mean, min and max","inputs":{"left":{"fields":[{"name":"g","type":"utf8"},{"name":"t","type":"i32"},{"name":"x","type":"f64"},{"name":"i","type":"i32"},{"name":"b","type":"bool"}],"columns":{"g":["a","b","a","a","b",null,"a","b","b","a"],"t":[1,1,2,3,2,1,4,3,4,5],"x":[1.5,2.0,null,4.0,"NaN",3.0,-1.0,2.0,0.5,2.0],"i":[3,1,4,1,5,9,null,6,5,3],"b":[true,false,null,true,true,false,true,null,false,true]}}},"ops":[{"op":"select","exprs":{"s":{"fn":"rollingSum","args":[{"col":"x"}],"window":3},"m":{"fn":"rollingMean","args":[{"col":"i"}],"window":2},"mn":{"fn":"rollingMin","args":[{"col":"x"}],"window":3,"minPeriods":1},"mx":{"fn":"rollingMax","args":[{"col":"i"}],"window":4,"minPeriods":2}}}],"expected":{"fields":[{"name":"s","type":"f64"},{"name":"m","type":"f64"},{"name":"mn","type":"f64"},{"name":"mx","type":"i32"}],"columns":{"s":[null,null,null,null,null,"NaN","NaN",4.0,1.5,4.5],"m":[null,2.0,2.5,2.5,3.0,7.0,null,null,5.5,4.0],"mn":[1.5,1.5,1.5,2.0,"NaN","NaN","NaN",-1.0,-1.0,0.5],"mx":[null,3,4,4,5,9,9,9,9,6]}}},{"id":"window/over-agg","description":"aggregates broadcast over partitions","inputs":{"left":{"fields":[{"name":"g","type":"utf8"},{"name":"t","type":"i32"},{"name":"x","type":"f64"},{"name":"i","type":"i32"},{"name":"b","type":"bool"}],"columns":{"g":["a","b","a","a","b",null,"a","b","b","a"],"t":[1,1,2,3,2,1,4,3,4,5],"x":[1.5,2.0,null,4.0,"NaN",3.0,-1.0,2.0,0.5,2.0],"i":[3,1,4,1,5,9,null,6,5,3],"b":[true,false,null,true,true,false,true,null,false,true]}}},"ops":[{"op":"withColumns","exprs":{"total":{"over":{"agg":"sum","arg":{"col":"x"}},"partitionBy":[{"col":"g"}]},"n":{"over":{"agg":"len"},"partitionBy":[{"col":"g"}]},"mean":{"over":{"agg":"mean","arg":{"col":"i"}},"partitionBy":[{"col":"g"}]},"first":{"over":{"agg":"first","arg":{"col":"t"}},"partitionBy":[{"col":"g"},{"col":"b"}]}}}],"expected":{"fields":[{"name":"g","type":"utf8"},{"name":"t","type":"i32"},{"name":"x","type":"f64"},{"name":"i","type":"i32"},{"name":"b","type":"bool"},{"name":"total","type":"f64"},{"name":"n","type":"u32"},{"name":"mean","type":"f64"},{"name":"first","type":"i32"}],"columns":{"g":["a","b","a","a","b",null,"a","b","b","a"],"t":[1,1,2,3,2,1,4,3,4,5],"x":[1.5,2.0,null,4.0,"NaN",3.0,-1.0,2.0,0.5,2.0],"i":[3,1,4,1,5,9,null,6,5,3],"b":[true,false,null,true,true,false,true,null,false,true],"total":[6.5,"NaN",6.5,6.5,"NaN",3.0,6.5,"NaN","NaN",6.5],"n":[5,4,5,5,4,1,5,4,4,5],"mean":[2.75,4.25,2.75,2.75,4.25,9.0,2.75,4.25,4.25,2.75],"first":[1,1,2,1,2,1,1,3,1,1]}}},{"id":"window/over-cum","description":"cumulative and shifted values per partition, in row order","inputs":{"left":{"fields":[{"name":"g","type":"utf8"},{"name":"t","type":"i32"},{"name":"x","type":"f64"},{"name":"i","type":"i32"},{"name":"b","type":"bool"}],"columns":{"g":["a","b","a","a","b",null,"a","b","b","a"],"t":[1,1,2,3,2,1,4,3,4,5],"x":[1.5,2.0,null,4.0,"NaN",3.0,-1.0,2.0,0.5,2.0],"i":[3,1,4,1,5,9,null,6,5,3],"b":[true,false,null,true,true,false,true,null,false,true]}}},"ops":[{"op":"withColumns","exprs":{"cs":{"over":{"fn":"cumSum","args":[{"col":"i"}]},"partitionBy":[{"col":"g"}]},"prev":{"over":{"fn":"shift","args":[{"col":"x"}]},"partitionBy":[{"col":"g"}]},"d":{"over":{"fn":"diff","args":[{"col":"t"}]},"partitionBy":[{"col":"g"}]},"r":{"over":{"fn":"rank","args":[{"col":"x"}],"method":"min"},"partitionBy":[{"col":"g"}]},"roll":{"over":{"fn":"rollingSum","args":[{"col":"i"}],"window":2,"minPeriods":1},"partitionBy":[{"col":"g"}]}}}],"expected":{"fields":[{"name":"g","type":"utf8"},{"name":"t","type":"i32"},{"name":"x","type":"f64"},{"name":"i","type":"i32"},{"name":"b","type":"bool"},{"name":"cs","type":"i32"},{"name":"prev","type":"f64"},{"name":"d","type":"i32"},{"name":"r","type":"u32"},{"name":"roll","type":"i32"}],"columns":{"g":["a","b","a","a","b",null,"a","b","b","a"],"t":[1,1,2,3,2,1,4,3,4,5],"x":[1.5,2.0,null,4.0,"NaN",3.0,-1.0,2.0,0.5,2.0],"i":[3,1,4,1,5,9,null,6,5,3],"b":[true,false,null,true,true,false,true,null,false,true],"cs":[3,1,7,8,6,9,null,12,17,11],"prev":[null,null,1.5,null,2.0,null,4.0,"NaN",2.0,-1.0],"d":[null,null,1,1,1,null,1,1,1,1],"r":[2,2,null,4,4,1,1,2,1,3],"roll":[3,1,7,5,6,9,1,11,11,3]}}},{"id":"window/over-filter","description":"filters after windows see the windowed values","inputs":{"left":{"fields":[{"name":"g","type":"utf8"},{"name":"t","type":"i32"},{"name":"x","type":"f64"},{"name":"i","type":"i32"},{"name":"b","type":"bool"}],"columns":{"g":["a","b","a","a","b",null,"a","b","b","a"],"t":[1,1,2,3,2,1,4,3,4,5],"x":[1.5,2.0,null,4.0,"NaN",3.0,-1.0,2.0,0.5,2.0],"i":[3,1,4,1,5,9,null,6,5,3],"b":[true,false,null,true,true,false,true,null,false,true]}}},"ops":[{"op":"withColumns","exprs":{"cs":{"over":{"fn":"cumSum","args":[{"col":"t"}]},"partitionBy":[{"col":"g"}]}}},{"op":"filter","predicate":{"fn":"gt","args":[{"col":"t"},{"lit":2}]}}],"expected":{"fields":[{"name":"g","type":"utf8"},{"name":"t","type":"i32"},{"name":"x","type":"f64"},{"name":"i","type":"i32"},{"name":"b","type":"bool"},{"name":"cs","type":"i32"}],"columns":{"g":["a","a","b","b","a"],"t":[3,4,3,4,5],"x":[4.0,-1.0,2.0,0.5,2.0],"i":[1,null,6,5,3],"b":[true,true,null,false,true],"cs":[6,10,6,10,15]}}},{"id":"window/filter-first","description":"windows after a filter only see the remaining rows","inputs":{"left":{"fields":[{"name":"g","type":"utf8"},{"name":"t","type":"i32"},{"name":"x","type":"f64"},{"name":"i","type":"i32"},{"name":"b","type":"bool"}],"columns":{"g":["a","b","a","a","b",null,"a","b","b","a"],"t":[1,1,2,3,2,1,4,3,4,5],"x":[1.5,2.0,null,4.0,"NaN",3.0,-1.0,2.0,0.5,2.0],"i":[3,1,4,1,5,9,null,6,5,3],"b":[true,false,null,true,true,false,true,null,false,true]}}},"ops":[{"op":"filter","predicate":{"fn":"isNotNull","args":[{"col":"x"}]}},{"op":"withColumns","exprs":{"cs":{"fn":"cumSum","args":[{"col":"x"}]},"r":{"over":{"fn":"rank","args":[{"col":"x"}]},"partitionBy":[{"col":"g"}]}}},{"op":"head","n":4}],"expected":{"fields":[{"name":"g","type":"utf8"},{"name":"t","type":"i32"},{"name":"x","type":"f64"},{"name":"i","type":"i32"},{"name":"b","type":"bool"},{"name":"cs","type":"f64"},{"name":"r","type":"f64"}],"columns":{"g":["a","b","a","b"],"t":[1,1,3,2],"x":[1.5,2.0,4.0,"NaN"],"i":[3,1,1,5],"b":[true,false,true,true],"cs":[1.5,3.5,7.5,"NaN"],"r":[2.0,2.5,4.0,4.0]}}},{"id":"pivot/sum","description":"pivot with a sum per cell; missing cells are null","inputs":{"left":{"fields":[{"name":"region","type":"utf8"},{"name":"quarter","type":"utf8"},{"name":"units","type":"i32"},{"name":"price","type":"f64"}],"columns":{"region":["north","north","south","south","north","east","south","north"],"quarter":["q1","q2","q1","q3","q1","q2","q1",null],"units":[10,20,5,null,7,3,8,1],"price":[1.5,2.0,1.0,3.0,null,2.5,1.0,9.0]}}},"ops":[{"op":"pivot","on":"quarter","index":["region"],"values":"units","aggregate":"sum"}],"expected":{"fields":[{"name":"region","type":"utf8"},{"name":"q1","type":"i32"},{"name":"q2","type":"i32"},{"name":"q3","type":"i32"},{"name":"null","type":"i32"}],"columns":{"region":["north","south","east"],"q1":[17,13,0],"q2":[20,0,3],"q3":[0,0,0],"null":[1,0,0]}}},{"id":"pivot/first","description":"pivot with the first value per cell","inputs":{"left":{"fields":[{"name":"region","type":"utf8"},{"name":"quarter","type":"utf8"},{"name":"units","type":"i32"},{"name":"price","type":"f64"}],"columns":{"region":["north","north","south","south","north","east","south","north"],"quarter":["q1","q2","q1","q3","q1","q2","q1",null],"units":[10,20,5,null,7,3,8,1],"price":[1.5,2.0,1.0,3.0,null,2.5,1.0,9.0]}}},"ops":[{"op":"pivot","on":"region","index":["quarter"],"values":"price","aggregate":"first"}],"expected":{"fields":[{"name":"quarter","type":"utf8"},{"name":"north","type":"f64"},{"name":"south","type":"f64"},{"name":"east","type":"f64"}],"columns":{"quarter":["q1","q2","q3",null],"north":[1.5,2.0,null,9.0],"south":[1.0,null,3.0,null],"east":[null,2.5,null,null]}}},{"id":"pivot/len","description":"pivot counting rows per cell","inputs":{"left":{"fields":[{"name":"region","type":"utf8"},{"name":"quarter","type":"utf8"},{"name":"units","type":"i32"},{"name":"price","type":"f64"}],"columns":{"region":["north","north","south","south","north","east","south","north"],"quarter":["q1","q2","q1","q3","q1","q2","q1",null],"units":[10,20,5,null,7,3,8,1],"price":[1.5,2.0,1.0,3.0,null,2.5,1.0,9.0]}}},"ops":[{"op":"pivot","on":"quarter","index":["region"],"values":"units","aggregate":"len"}],"expected":{"fields":[{"name":"region","type":"utf8"},{"name":"q1","type":"u32"},{"name":"q2","type":"u32"},{"name":"q3","type":"u32"},{"name":"null","type":"u32"}],"columns":{"region":["north","south","east"],"q1":[2,2,0],"q2":[1,0,1],"q3":[0,1,0],"null":[1,0,0]}}},{"id":"pivot/columns","description":"pivot with declared output columns","inputs":{"left":{"fields":[{"name":"region","type":"utf8"},{"name":"quarter","type":"utf8"},{"name":"units","type":"i32"},{"name":"price","type":"f64"}],"columns":{"region":["north","north","south","south","north","east","south","north"],"quarter":["q1","q2","q1","q3","q1","q2","q1",null],"units":[10,20,5,null,7,3,8,1],"price":[1.5,2.0,1.0,3.0,null,2.5,1.0,9.0]}}},"ops":[{"op":"pivot","on":"quarter","index":["region"],"values":"price","aggregate":"max","columns":["q1","q2","q4"]}],"expected":{"fields":[{"name":"region","type":"utf8"},{"name":"q1","type":"f64"},{"name":"q2","type":"f64"},{"name":"q4","type":"f64"}],"columns":{"region":["north","south","east"],"q1":[1.5,1.0,null],"q2":[2.0,null,2.5],"q4":[null,null,null]}}},{"id":"pivot/numeric-on","description":"pivot on a numeric column names columns by value","inputs":{"left":{"fields":[{"name":"g","type":"utf8"},{"name":"t","type":"i32"},{"name":"x","type":"f64"},{"name":"i","type":"i32"},{"name":"b","type":"bool"}],"columns":{"g":["a","b","a","a","b",null,"a","b","b","a"],"t":[1,1,2,3,2,1,4,3,4,5],"x":[1.5,2.0,null,4.0,"NaN",3.0,-1.0,2.0,0.5,2.0],"i":[3,1,4,1,5,9,null,6,5,3],"b":[true,false,null,true,true,false,true,null,false,true]}}},"ops":[{"op":"filter","predicate":{"fn":"isNotNull","args":[{"col":"g"}]}},{"op":"pivot","on":"t","index":["g"],"values":"i","aggregate":"min"}],"expected":{"fields":[{"name":"g","type":"utf8"},{"name":"1","type":"i32"},{"name":"2","type":"i32"},{"name":"3","type":"i32"},{"name":"4","type":"i32"},{"name":"5","type":"i32"}],"columns":{"g":["a","b"],"1":[3,1],"2":[4,5],"3":[1,6],"4":[null,5],"5":[3,null]}}},{"id":"unpivot/basic","description":"unpivot stacks columns, column by column","inputs":{"left":{"fields":[{"name":"region","type":"utf8"},{"name":"quarter","type":"utf8"},{"name":"units","type":"i32"},{"name":"price","type":"f64"}],"columns":{"region":["north","north","south","south","north","east","south","north"],"quarter":["q1","q2","q1","q3","q1","q2","q1",null],"units":[10,20,5,null,7,3,8,1],"price":[1.5,2.0,1.0,3.0,null,2.5,1.0,9.0]}}},"ops":[{"op":"unpivot","index":["region"],"on":["units","price"]}],"expected":{"fields":[{"name":"region","type":"utf8"},{"name":"variable","type":"utf8"},{"name":"value","type":"f64"}],"columns":{"region":["north","north","south","south","north","east","south","north","north","north","south","south","north","east","south","north"],"variable":["units","units","units","units","units","units","units","units","price","price","price","price","price","price","price","price"],"value":[10.0,20.0,5.0,null,7.0,3.0,8.0,1.0,1.5,2.0,1.0,3.0,null,2.5,1.0,9.0]}}},{"id":"unpivot/names","description":"unpivot with named variable and value columns","inputs":{"left":{"fields":[{"name":"region","type":"utf8"},{"name":"quarter","type":"utf8"},{"name":"units","type":"i32"},{"name":"price","type":"f64"}],"columns":{"region":["north","north","south","south","north","east","south","north"],"quarter":["q1","q2","q1","q3","q1","q2","q1",null],"units":[10,20,5,null,7,3,8,1],"price":[1.5,2.0,1.0,3.0,null,2.5,1.0,9.0]}}},"ops":[{"op":"pick","columns":["region","quarter"]},{"op":"unpivot","index":[],"on":["region","quarter"],"variable":"field","value":"text"}],"expected":{"fields":[{"name":"field","type":"utf8"},{"name":"text","type":"utf8"}],"columns":{"field":["region","region","region","region","region","region","region","region","quarter","quarter","quarter","quarter","quarter","quarter","quarter","quarter"],"text":["north","north","south","south","north","east","south","north","q1","q2","q1","q3","q1","q2","q1",null]}}},{"id":"temporal/extract","description":"calendar fields of timestamps and dates","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"ts","type":"datetime[us]"},{"name":"ms","type":"datetime[ms]"},{"name":"d","type":"date"},{"name":"dur","type":"duration[us]"}],"columns":{"k":["a","b","c","d","e","f"],"ts":[1709214307123456,1704067199999000,1709510400000000,null,-14182940000000,1710052200000500],"ms":[1709214307123,1704067200000,null,1709251200000,0,1709942400000],"d":[19782,19722,19786,19782,null,19792],"dur":[5400000000,-1500000,null,259200000001,0,90000000000]}},"right":{"fields":[{"name":"d","type":"date"},{"name":"name","type":"utf8"}],"columns":{"d":[19782,19792,20089],"name":["leap","sunday","new year"]}}},"ops":[{"op":"select","exprs":{"y":{"fn":"year","args":[{"col":"ts"}]},"mo":{"fn":"month","args":[{"col":"ts"}]},"d":{"fn":"day","args":[{"col":"ts"}]},"h":{"fn":"hour","args":[{"col":"ts"}]},"mi":{"fn":"minute","args":[{"col":"ts"}]},"s":{"fn":"second","args":[{"col":"ts"}]},"ms":{"fn":"millisecond","args":[{"col":"ts"}]},"wd":{"fn":"weekday","args":[{"col":"ts"}]},"od":{"fn":"ordinalDay","args":[{"col":"ts"}]},"dy":{"fn":"year","args":[{"col":"d"}]},"dwd":{"fn":"weekday","args":[{"col":"d"}]},"dod":{"fn":"ordinalDay","args":[{"col":"d"}]}}}],"expected":{"fields":[{"name":"y","type":"i32"},{"name":"mo","type":"i8"},{"name":"d","type":"i8"},{"name":"h","type":"i8"},{"name":"mi","type":"i8"},{"name":"s","type":"i8"},{"name":"ms","type":"i32"},{"name":"wd","type":"i8"},{"name":"od","type":"i16"},{"name":"dy","type":"i32"},{"name":"dwd","type":"i8"},{"name":"dod","type":"i16"}],"columns":{"y":[2024,2023,2024,null,1969,2024],"mo":[2,12,3,null,7,3],"d":[29,31,4,null,20,10],"h":[13,23,0,null,20,6],"mi":[45,59,0,null,17,30],"s":[7,59,0,null,40,0],"ms":[123,999,0,null,0,0],"wd":[4,7,1,null,7,7],"od":[60,365,64,null,201,70],"dy":[2024,2023,2024,2024,null,2024],"dwd":[4,7,1,4,null,7],"dod":[60,365,64,60,null,70]}}},{"id":"temporal/arith","description":"timestamp differences are durations; durations shift timestamps","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"ts","type":"datetime[us]"},{"name":"ms","type":"datetime[ms]"},{"name":"d","type":"date"},{"name":"dur","type":"duration[us]"}],"columns":{"k":["a","b","c","d","e","f"],"ts":[1709214307123456,1704067199999000,1709510400000000,null,-14182940000000,1710052200000500],"ms":[1709214307123,1704067200000,null,1709251200000,0,1709942400000],"d":[19782,19722,19786,19782,null,19792],"dur":[5400000000,-1500000,null,259200000001,0,90000000000]}},"right":{"fields":[{"name":"d","type":"date"},{"name":"name","type":"utf8"}],"columns":{"d":[19782,19792,20089],"name":["leap","sunday","new year"]}}},"ops":[{"op":"select","exprs":{"gap":{"fn":"sub","args":[{"col":"ts"},{"col":"ms"}]},"later":{"fn":"add","args":[{"col":"ts"},{"col":"dur"}]},"earlier":{"fn":"sub","args":[{"col":"ts"},{"col":"dur"}]},"days":{"fn":"sub","args":[{"col":"d"},{"lit":18000,"type":"date"}]},"twice":{"fn":"add","args":[{"col":"dur"},{"col":"dur"}]}}}],"expected":{"fields":[{"name":"gap","type":"duration[ms]"},{"name":"later","type":"datetime[us]"},{"name":"earlier","type":"datetime[us]"},{"name":"days","type":"duration[us]"},{"name":"twice","type":"duration[us]"}],"columns":{"gap":[0,-1,null,null,-14182940000,109800000],"later":[1709219707123456,1704067198499000,null,null,-14182940000000,1710142200000500],"earlier":[1709208907123456,1704067201499000,null,null,-14182940000000,1709962200000500],"days":[153964800000000,148780800000000,154310400000000,153964800000000,null,154828800000000],"twice":[10800000000,-3000000,null,518400000002,0,180000000000]}}},{"id":"temporal/truncate","description":"truncate to a calendar unit","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"ts","type":"datetime[us]"},{"name":"ms","type":"datetime[ms]"},{"name":"d","type":"date"},{"name":"dur","type":"duration[us]"}],"columns":{"k":["a","b","c","d","e","f"],"ts":[1709214307123456,1704067199999000,1709510400000000,null,-14182940000000,1710052200000500],"ms":[1709214307123,1704067200000,null,1709251200000,0,1709942400000],"d":[19782,19722,19786,19782,null,19792],"dur":[5400000000,-1500000,null,259200000001,0,90000000000]}},"right":{"fields":[{"name":"d","type":"date"},{"name":"name","type":"utf8"}],"columns":{"d":[19782,19792,20089],"name":["leap","sunday","new year"]}}},"ops":[{"op":"select","exprs":{"h":{"fn":"truncate","args":[{"col":"ts"}],"every":"1h"},"d":{"fn":"truncate","args":[{"col":"ts"}],"every":"1d"},"w":{"fn":"truncate","args":[{"col":"ts"}],"every":"1w"},"mo":{"fn":"truncate","args":[{"col":"ts"}],"every":"1mo"},"y":{"fn":"truncate","args":[{"col":"ts"}],"every":"1y"},"dmo":{"fn":"truncate","args":[{"col":"d"}],"every":"1mo"}}}],"expected":{"fields":[{"name":"h","type":"datetime[us]"},{"name":"d","type":"datetime[us]"},{"name":"w","type":"datetime[us]"},{"name":"mo","type":"datetime[us]"},{"name":"y","type":"datetime[us]"},{"name":"dmo","type":"date"}],"columns":{"h":[1709211600000000,1704063600000000,1709510400000000,null,-14184000000000,1710050400000000],"d":[1709164800000000,1703980800000000,1709510400000000,null,-14256000000000,1710028800000000],"w":[1708905600000000,1703462400000000,1709510400000000,null,-14774400000000,1709510400000000],"mo":[1706745600000000,1701388800000000,1709251200000000,null,-15897600000000,1709251200000000],"y":[1704067200000000,1672531200000000,1704067200000000,null,-31536000000000,1704067200000000],"dmo":[19754,19692,19783,19754,null,19783]}}},{"id":"temporal/duration","description":"durations in whole units","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"ts","type":"datetime[us]"},{"name":"ms","type":"datetime[ms]"},{"name":"d","type":"date"},{"name":"dur","type":"duration[us]"}],"columns":{"k":["a","b","c","d","e","f"],"ts":[1709214307123456,1704067199999000,1709510400000000,null,-14182940000000,1710052200000500],"ms":[1709214307123,1704067200000,null,1709251200000,0,1709942400000],"d":[19782,19722,19786,19782,null,19792],"dur":[5400000000,-1500000,null,259200000001,0,90000000000]}},"right":{"fields":[{"name":"d","type":"date"},{"name":"name","type":"utf8"}],"columns":{"d":[19782,19792,20089],"name":["leap","sunday","new year"]}}},"ops":[{"op":"select","exprs":{"d":{"fn":"totalDays","args":[{"col":"dur"}]},"h":{"fn":"totalHours","args":[{"col":"dur"}]},"m":{"fn":"totalMinutes","args":[{"col":"dur"}]},"s":{"fn":"totalSeconds","args":[{"col":"dur"}]},"ms":{"fn":"totalMilliseconds","args":[{"col":"dur"}]}}}],"expected":{"fields":[{"name":"d","type":"i64"},{"name":"h","type":"i64"},{"name":"m","type":"i64"},{"name":"s","type":"i64"},{"name":"ms","type":"i64"}],"columns":{"d":["0","0",null,"3","0","1"],"h":["1","0",null,"72","0","25"],"m":["90","0",null,"4320","0","1500"],"s":["5400","-1",null,"259200","0","90000"],"ms":["5400000","-1500",null,"259200000","0","90000000"]}}},{"id":"temporal/cast","description":"casts between dates, timestamps and integers","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"ts","type":"datetime[us]"},{"name":"ms","type":"datetime[ms]"},{"name":"d","type":"date"},{"name":"dur","type":"duration[us]"}],"columns":{"k":["a","b","c","d","e","f"],"ts":[1709214307123456,1704067199999000,1709510400000000,null,-14182940000000,1710052200000500],"ms":[1709214307123,1704067200000,null,1709251200000,0,1709942400000],"d":[19782,19722,19786,19782,null,19792],"dur":[5400000000,-1500000,null,259200000001,0,90000000000]}},"right":{"fields":[{"name":"d","type":"date"},{"name":"name","type":"utf8"}],"columns":{"d":[19782,19792,20089],"name":["leap","sunday","new year"]}}},"ops":[{"op":"select","exprs":{"date":{"fn":"cast","args":[{"col":"ts"}],"to":"date"},"ts":{"fn":"cast","args":[{"col":"d"}],"to":"datetime[us]"},"us":{"fn":"cast","args":[{"col":"ts"}],"to":"i64"},"days":{"fn":"cast","args":[{"col":"d"}],"to":"i32"},"ms":{"fn":"cast","args":[{"col":"ts"}],"to":"datetime[ms]"}}}],"expected":{"fields":[{"name":"date","type":"date"},{"name":"ts","type":"datetime[us]"},{"name":"us","type":"i64"},{"name":"days","type":"i32"},{"name":"ms","type":"datetime[ms]"}],"columns":{"date":[19782,19722,19786,null,-165,19792],"ts":[1709164800000000,1703980800000000,1709510400000000,1709164800000000,null,1710028800000000],"us":["1709214307123456","1704067199999000","1709510400000000",null,"-14182940000000","1710052200000500"],"days":[19782,19722,19786,19782,null,19792],"ms":[1709214307123,1704067199999,1709510400000,null,-14182940000,1710052200000]}}},{"id":"temporal/compare","description":"comparisons with timestamp literals and between units","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"ts","type":"datetime[us]"},{"name":"ms","type":"datetime[ms]"},{"name":"d","type":"date"},{"name":"dur","type":"duration[us]"}],"columns":{"k":["a","b","c","d","e","f"],"ts":[1709214307123456,1704067199999000,1709510400000000,null,-14182940000000,1710052200000500],"ms":[1709214307123,1704067200000,null,1709251200000,0,1709942400000],"d":[19782,19722,19786,19782,null,19792],"dur":[5400000000,-1500000,null,259200000001,0,90000000000]}},"right":{"fields":[{"name":"d","type":"date"},{"name":"name","type":"utf8"}],"columns":{"d":[19782,19792,20089],"name":["leap","sunday","new year"]}}},"ops":[{"op":"filter","predicate":{"fn":"and","args":[{"fn":"gte","args":[{"col":"ts"},{"lit":1709251200000000,"type":"datetime[us]"}]},{"fn":"lt","args":[{"col":"ms"},{"col":"ts"}]}]}}],"expected":{"fields":[{"name":"k","type":"utf8"},{"name":"ts","type":"datetime[us]"},{"name":"ms","type":"datetime[ms]"},{"name":"d","type":"date"},{"name":"dur","type":"duration[us]"}],"columns":{"k":["f"],"ts":[1710052200000500],"ms":[1709942400000],"d":[19792],"dur":[90000000000]}}},{"id":"temporal/group-sort","description":"group by date, aggregate timestamps, sort","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"ts","type":"datetime[us]"},{"name":"ms","type":"datetime[ms]"},{"name":"d","type":"date"},{"name":"dur","type":"duration[us]"}],"columns":{"k":["a","b","c","d","e","f"],"ts":[1709214307123456,1704067199999000,1709510400000000,null,-14182940000000,1710052200000500],"ms":[1709214307123,1704067200000,null,1709251200000,0,1709942400000],"d":[19782,19722,19786,19782,null,19792],"dur":[5400000000,-1500000,null,259200000001,0,90000000000]}},"right":{"fields":[{"name":"d","type":"date"},{"name":"name","type":"utf8"}],"columns":{"d":[19782,19792,20089],"name":["leap","sunday","new year"]}}},"ops":[{"op":"groupBy","keys":["d"],"aggs":{"first":{"agg":"min","arg":{"col":"ts"}},"last":{"agg":"max","arg":{"col":"ts"}},"n":{"agg":"len"},"total":{"agg":"sum","arg":{"col":"dur"}}}},{"op":"sort","keys":[{"expr":{"col":"last"},"descending":true}]}],"expected":{"fields":[{"name":"d","type":"date"},{"name":"first","type":"datetime[us]"},{"name":"last","type":"datetime[us]"},{"name":"n","type":"u32"},{"name":"total","type":"duration[us]"}],"columns":{"d":[19792,19786,19782,19722,null],"first":[1710052200000500,1709510400000000,1709214307123456,1704067199999000,-14182940000000],"last":[1710052200000500,1709510400000000,1709214307123456,1704067199999000,-14182940000000],"n":[1,1,2,1,1],"total":[90000000000,0,264600000001,-1500000,0]}}},{"id":"temporal/join","description":"join on a date","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"ts","type":"datetime[us]"},{"name":"ms","type":"datetime[ms]"},{"name":"d","type":"date"},{"name":"dur","type":"duration[us]"}],"columns":{"k":["a","b","c","d","e","f"],"ts":[1709214307123456,1704067199999000,1709510400000000,null,-14182940000000,1710052200000500],"ms":[1709214307123,1704067200000,null,1709251200000,0,1709942400000],"d":[19782,19722,19786,19782,null,19792],"dur":[5400000000,-1500000,null,259200000001,0,90000000000]}},"right":{"fields":[{"name":"d","type":"date"},{"name":"name","type":"utf8"}],"columns":{"d":[19782,19792,20089],"name":["leap","sunday","new year"]}}},"ops":[{"op":"join","right":"right","on":["d"],"how":"left"}],"expected":{"fields":[{"name":"k","type":"utf8"},{"name":"ts","type":"datetime[us]"},{"name":"ms","type":"datetime[ms]"},{"name":"d","type":"date"},{"name":"dur","type":"duration[us]"},{"name":"name","type":"utf8"}],"columns":{"k":["a","b","c","d","e","f"],"ts":[1709214307123456,1704067199999000,1709510400000000,null,-14182940000000,1710052200000500],"ms":[1709214307123,1704067200000,null,1709251200000,0,1709942400000],"d":[19782,19722,19786,19782,null,19792],"dur":[5400000000,-1500000,null,259200000001,0,90000000000],"name":["leap",null,null,"leap",null,"sunday"]}}},{"id":"nested/list-ops","description":"list length, element access and aggregates","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"g","type":"i32"},{"name":"xs","type":"list"},{"name":"ws","type":"list"},{"name":"s","type":"struct"}],"columns":{"k":["a","b","c","d","e"],"g":[1,2,1,2,1],"xs":[[1,2,3],[],null,[4,null,2],[7]],"ws":[["a","b"],["c"],[],null,["z","a"]],"s":[{"a":1,"b":"x"},{"a":null,"b":"y"},null,{"a":4,"b":null},{"a":5,"b":"z"}]}},"right":{"fields":[{"name":"k","type":"utf8"},{"name":"tags","type":"list"}],"columns":{"k":["a","d","x"],"tags":[["t1"],["t2","t3"],[]]}}},"ops":[{"op":"select","exprs":{"n":{"fn":"listLength","args":[{"col":"xs"}]},"first":{"fn":"listGet","args":[{"col":"xs"}],"index":0},"last":{"fn":"listGet","args":[{"col":"xs"}],"index":-1},"oob":{"fn":"listGet","args":[{"col":"xs"}],"index":5},"sum":{"fn":"listSum","args":[{"col":"xs"}]},"mean":{"fn":"listMean","args":[{"col":"xs"}]},"min":{"fn":"listMin","args":[{"col":"xs"}]},"max":{"fn":"listMax","args":[{"col":"ws"}]},"has2":{"fn":"listContains","args":[{"col":"xs"},{"lit":2}]},"hasA":{"fn":"listContains","args":[{"col":"ws"},{"lit":"a"}]}}}],"expected":{"fields":[{"name":"n","type":"u32"},{"name":"first","type":"i32"},{"name":"last","type":"i32"},{"name":"oob","type":"i32"},{"name":"sum","type":"i32"},{"name":"mean","type":"f64"},{"name":"min","type":"i32"},{"name":"max","type":"utf8"},{"name":"has2","type":"bool"},{"name":"hasA","type":"bool"}],"columns":{"n":[3,0,null,3,1],"first":[1,null,null,4,7],"last":[3,null,null,2,7],"oob":[null,null,null,null,null],"sum":[6,0,null,6,7],"mean":[2.0,null,null,3.0,7.0],"min":[1,null,null,2,7],"max":["b","c",null,null,"z"],"has2":[true,false,null,true,false],"hasA":[true,false,false,null,true]}}},{"id":"nested/struct","description":"struct fields and struct construction","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"g","type":"i32"},{"name":"xs","type":"list"},{"name":"ws","type":"list"},{"name":"s","type":"struct"}],"columns":{"k":["a","b","c","d","e"],"g":[1,2,1,2,1],"xs":[[1,2,3],[],null,[4,null,2],[7]],"ws":[["a","b"],["c"],[],null,["z","a"]],"s":[{"a":1,"b":"x"},{"a":null,"b":"y"},null,{"a":4,"b":null},{"a":5,"b":"z"}]}},"right":{"fields":[{"name":"k","type":"utf8"},{"name":"tags","type":"list"}],"columns":{"k":["a","d","x"],"tags":[["t1"],["t2","t3"],[]]}}},"ops":[{"op":"select","exprs":{"a":{"fn":"field","args":[{"col":"s"}],"field":"a"},"b":{"fn":"field","args":[{"col":"s"}],"field":"b"},"pair":{"fn":"struct","args":[{"col":"k"},{"fn":"listLength","args":[{"col":"xs"}]}],"names":["key","size"]}}}],"expected":{"fields":[{"name":"a","type":"i32"},{"name":"b","type":"utf8"},{"name":"pair","type":"struct"}],"columns":{"a":[1,null,null,4,5],"b":["x","y",null,null,"z"],"pair":[{"key":"a","size":3},{"key":"b","size":0},{"key":"c","size":null},{"key":"d","size":3},{"key":"e","size":1}]}}},{"id":"nested/explode","description":"explode lists into rows; a null list gives a null row, an empty list no rows","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"g","type":"i32"},{"name":"xs","type":"list"},{"name":"ws","type":"list"},{"name":"s","type":"struct"}],"columns":{"k":["a","b","c","d","e"],"g":[1,2,1,2,1],"xs":[[1,2,3],[],null,[4,null,2],[7]],"ws":[["a","b"],["c"],[],null,["z","a"]],"s":[{"a":1,"b":"x"},{"a":null,"b":"y"},null,{"a":4,"b":null},{"a":5,"b":"z"}]}},"right":{"fields":[{"name":"k","type":"utf8"},{"name":"tags","type":"list"}],"columns":{"k":["a","d","x"],"tags":[["t1"],["t2","t3"],[]]}}},"ops":[{"op":"pick","columns":["k","xs"]},{"op":"explode","column":"xs"}],"expected":{"fields":[{"name":"k","type":"utf8"},{"name":"xs","type":"i32"}],"columns":{"k":["a","a","a","c","d","d","d","e"],"xs":[1,2,3,null,4,null,2,7]}}},{"id":"nested/unnest","description":"unnest a struct into columns","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"g","type":"i32"},{"name":"xs","type":"list"},{"name":"ws","type":"list"},{"name":"s","type":"struct"}],"columns":{"k":["a","b","c","d","e"],"g":[1,2,1,2,1],"xs":[[1,2,3],[],null,[4,null,2],[7]],"ws":[["a","b"],["c"],[],null,["z","a"]],"s":[{"a":1,"b":"x"},{"a":null,"b":"y"},null,{"a":4,"b":null},{"a":5,"b":"z"}]}},"right":{"fields":[{"name":"k","type":"utf8"},{"name":"tags","type":"list"}],"columns":{"k":["a","d","x"],"tags":[["t1"],["t2","t3"],[]]}}},"ops":[{"op":"pick","columns":["k","s"]},{"op":"unnest","column":"s"}],"expected":{"fields":[{"name":"k","type":"utf8"},{"name":"a","type":"i32"},{"name":"b","type":"utf8"}],"columns":{"k":["a","b","c","d","e"],"a":[1,null,null,4,5],"b":["x","y",null,null,"z"]}}},{"id":"nested/agg-list","description":"collect values into lists per group","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"g","type":"i32"},{"name":"xs","type":"list"},{"name":"ws","type":"list"},{"name":"s","type":"struct"}],"columns":{"k":["a","b","c","d","e"],"g":[1,2,1,2,1],"xs":[[1,2,3],[],null,[4,null,2],[7]],"ws":[["a","b"],["c"],[],null,["z","a"]],"s":[{"a":1,"b":"x"},{"a":null,"b":"y"},null,{"a":4,"b":null},{"a":5,"b":"z"}]}},"right":{"fields":[{"name":"k","type":"utf8"},{"name":"tags","type":"list"}],"columns":{"k":["a","d","x"],"tags":[["t1"],["t2","t3"],[]]}}},"ops":[{"op":"groupBy","keys":["g"],"aggs":{"ks":{"agg":"list","arg":{"col":"k"}},"firsts":{"agg":"first","arg":{"col":"xs"}},"n":{"agg":"len"}}}],"expected":{"fields":[{"name":"g","type":"i32"},{"name":"ks","type":"list"},{"name":"firsts","type":"list"},{"name":"n","type":"u32"}],"columns":{"g":[1,2],"ks":[["a","c","e"],["b","d"]],"firsts":[[1,2,3],[]],"n":[3,2]}}},{"id":"nested/carry","description":"lists and structs survive filters, sorts, joins and slices","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"g","type":"i32"},{"name":"xs","type":"list"},{"name":"ws","type":"list"},{"name":"s","type":"struct"}],"columns":{"k":["a","b","c","d","e"],"g":[1,2,1,2,1],"xs":[[1,2,3],[],null,[4,null,2],[7]],"ws":[["a","b"],["c"],[],null,["z","a"]],"s":[{"a":1,"b":"x"},{"a":null,"b":"y"},null,{"a":4,"b":null},{"a":5,"b":"z"}]}},"right":{"fields":[{"name":"k","type":"utf8"},{"name":"tags","type":"list"}],"columns":{"k":["a","d","x"],"tags":[["t1"],["t2","t3"],[]]}}},"ops":[{"op":"filter","predicate":{"fn":"neq","args":[{"col":"k"},{"lit":"c"}]}},{"op":"sort","keys":[{"expr":{"col":"g"},"descending":true},{"expr":{"col":"k"}}]},{"op":"join","right":"right","on":["k"],"how":"left"},{"op":"head","n":4}],"expected":{"fields":[{"name":"k","type":"utf8"},{"name":"g","type":"i32"},{"name":"xs","type":"list"},{"name":"ws","type":"list"},{"name":"s","type":"struct"},{"name":"tags","type":"list"}],"columns":{"k":["b","d","a","e"],"g":[2,2,1,1],"xs":[[],[4,null,2],[1,2,3],[7]],"ws":[["c"],null,["a","b"],["z","a"]],"s":[{"a":null,"b":"y"},{"a":4,"b":null},{"a":1,"b":"x"},{"a":5,"b":"z"}],"tags":[null,["t2","t3"],["t1"],null]}}},{"id":"decimal/arith","description":"decimal addition, subtraction and multiplication","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"price","type":"decimal[10,2]"},{"name":"fee","type":"decimal[6,3]"},{"name":"qty","type":"decimal[4,0]"},{"name":"x","type":"f64"}],"columns":{"k":["a","b","a","c","b","a"],"price":["12.34","0.10","-5.00",null,"99999999.99","10.01"],"fee":["0.125","1.000",null,"2.500","-0.001","10.010"],"qty":["3","10","2","1",null,"7"],"x":[1.005,-2.5,123.456,null,1000000.0,0.0]}}},"ops":[{"op":"select","exprs":{"sum":{"fn":"add","args":[{"col":"price"},{"col":"fee"}]},"diff":{"fn":"sub","args":[{"col":"price"},{"col":"fee"}]},"prod":{"fn":"mul","args":[{"col":"price"},{"col":"qty"}]},"plus":{"fn":"add","args":[{"col":"price"},{"lit":"0.5","type":"decimal[3,1]"}]}}}],"expected":{"fields":[{"name":"sum","type":"decimal[38,3]"},{"name":"diff","type":"decimal[38,3]"},{"name":"prod","type":"decimal[38,2]"},{"name":"plus","type":"decimal[38,2]"}],"columns":{"sum":["12.465","1.100",null,null,"99999999.989","20.020"],"diff":["12.215","-0.900",null,null,"99999999.991","0.000"],"prod":["37.02","1.00","-10.00",null,null,"70.07"],"plus":["12.84","0.60","-4.50",null,"100000000.49","10.51"]}}},{"id":"decimal/compare","description":"decimal comparisons, also across scales","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"price","type":"decimal[10,2]"},{"name":"fee","type":"decimal[6,3]"},{"name":"qty","type":"decimal[4,0]"},{"name":"x","type":"f64"}],"columns":{"k":["a","b","a","c","b","a"],"price":["12.34","0.10","-5.00",null,"99999999.99","10.01"],"fee":["0.125","1.000",null,"2.500","-0.001","10.010"],"qty":["3","10","2","1",null,"7"],"x":[1.005,-2.5,123.456,null,1000000.0,0.0]}}},"ops":[{"op":"filter","predicate":{"fn":"and","args":[{"fn":"gt","args":[{"col":"price"},{"lit":"10.00","type":"decimal[10,2]"}]},{"fn":"gte","args":[{"col":"price"},{"col":"fee"}]}]}}],"expected":{"fields":[{"name":"k","type":"utf8"},{"name":"price","type":"decimal[10,2]"},{"name":"fee","type":"decimal[6,3]"},{"name":"qty","type":"decimal[4,0]"},{"name":"x","type":"f64"}],"columns":{"k":["a","b","a"],"price":["12.34","99999999.99","10.01"],"fee":["0.125","-0.001","10.010"],"qty":["3",null,"7"],"x":[1.005,1000000.0,0.0]}}},{"id":"decimal/aggregate","description":"sums, extremes and means of decimals per group","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"price","type":"decimal[10,2]"},{"name":"fee","type":"decimal[6,3]"},{"name":"qty","type":"decimal[4,0]"},{"name":"x","type":"f64"}],"columns":{"k":["a","b","a","c","b","a"],"price":["12.34","0.10","-5.00",null,"99999999.99","10.01"],"fee":["0.125","1.000",null,"2.500","-0.001","10.010"],"qty":["3","10","2","1",null,"7"],"x":[1.005,-2.5,123.456,null,1000000.0,0.0]}}},"ops":[{"op":"groupBy","keys":["k"],"aggs":{"total":{"agg":"sum","arg":{"col":"price"}},"min":{"agg":"min","arg":{"col":"price"}},"max":{"agg":"max","arg":{"col":"fee"}},"mean":{"agg":"mean","arg":{"col":"price"}},"n":{"agg":"count","arg":{"col":"price"}}}}],"expected":{"fields":[{"name":"k","type":"utf8"},{"name":"total","type":"decimal[38,2]"},{"name":"min","type":"decimal[10,2]"},{"name":"max","type":"decimal[6,3]"},{"name":"mean","type":"f64"},{"name":"n","type":"u32"}],"columns":{"k":["a","b","c"],"total":["17.35","100000000.09","0.00"],"min":["-5.00","0.10",null],"max":["10.010","1.000","2.500"],"mean":[5.783333333333334,50000000.044999994,null],"n":[3,2,0]}}},{"id":"decimal/cast","description":"casts between decimals, floats, integers and strings","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"price","type":"decimal[10,2]"},{"name":"fee","type":"decimal[6,3]"},{"name":"qty","type":"decimal[4,0]"},{"name":"x","type":"f64"}],"columns":{"k":["a","b","a","c","b","a"],"price":["12.34","0.10","-5.00",null,"99999999.99","10.01"],"fee":["0.125","1.000",null,"2.500","-0.001","10.010"],"qty":["3","10","2","1",null,"7"],"x":[1.005,-2.5,123.456,null,1000000.0,0.0]}}},"ops":[{"op":"select","exprs":{"f":{"fn":"cast","args":[{"col":"price"}],"to":"f64"},"i":{"fn":"cast","args":[{"col":"price"}],"to":"i32"},"s":{"fn":"cast","args":[{"col":"fee"}],"to":"utf8"},"d":{"fn":"cast","args":[{"col":"x"}],"to":"decimal[10,2]"},"r":{"fn":"cast","args":[{"col":"price"}],"to":"decimal[12,1]"}}}],"expected":{"fields":[{"name":"f","type":"f64"},{"name":"i","type":"i32"},{"name":"s","type":"utf8"},{"name":"d","type":"decimal[10,2]"},{"name":"r","type":"decimal[12,1]"}],"columns":{"f":[12.34,0.1,-5.0,null,99999999.99,10.01],"i":[12,0,-5,null,99999999,10],"s":["0.125","1.000",null,"2.500","-0.001","10.010"],"d":["1.00","-2.50","123.46",null,"1000000.00","0.00"],"r":["12.3","0.1","-5.0",null,"100000000.0","10.0"]}}},{"id":"decimal/sort","description":"sort by a decimal","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"price","type":"decimal[10,2]"},{"name":"fee","type":"decimal[6,3]"},{"name":"qty","type":"decimal[4,0]"},{"name":"x","type":"f64"}],"columns":{"k":["a","b","a","c","b","a"],"price":["12.34","0.10","-5.00",null,"99999999.99","10.01"],"fee":["0.125","1.000",null,"2.500","-0.001","10.010"],"qty":["3","10","2","1",null,"7"],"x":[1.005,-2.5,123.456,null,1000000.0,0.0]}}},"ops":[{"op":"sort","keys":[{"expr":{"col":"price"},"descending":true}],"nullsLast":true}],"expected":{"fields":[{"name":"k","type":"utf8"},{"name":"price","type":"decimal[10,2]"},{"name":"fee","type":"decimal[6,3]"},{"name":"qty","type":"decimal[4,0]"},{"name":"x","type":"f64"}],"columns":{"k":["b","a","a","b","a","c"],"price":["99999999.99","12.34","10.01","0.10","-5.00",null],"fee":["-0.001","0.125","10.010","1.000",null,"2.500"],"qty":[null,"3","7","10","2","1"],"x":[1000000.0,1.005,0.0,-2.5,123.456,null]}}},{"id":"decimal/round-ties","description":"narrowing the scale rounds exact ties","inputs":{"left":{"fields":[{"name":"t","type":"decimal[6,3]"}],"columns":{"t":["0.250","0.350","-0.250","-0.350","1.050","2.500","0.249"]}}},"ops":[{"op":"select","exprs":{"one":{"fn":"cast","args":[{"col":"t"}],"to":"decimal[6,1]"},"zero":{"fn":"cast","args":[{"col":"t"}],"to":"decimal[6,0]"}}}],"expected":{"fields":[{"name":"one","type":"decimal[6,1]"},{"name":"zero","type":"decimal[6,0]"}],"columns":{"one":["0.2","0.4","-0.2","-0.4","1.0","2.5","0.2"],"zero":["0","0","0","0","1","2","0"]}}},{"id":"decimal/cast-float-ties","description":"floats cast to decimals round exact ties half to even, also below zero","inputs":{"left":{"fields":[{"name":"x","type":"f64"}],"columns":{"x":[-1.5,-2.5,1.5,2.5,-0.75]}}},"ops":[{"op":"select","exprs":{"zero":{"fn":"cast","args":[{"col":"x"}],"to":"decimal[18,0]"},"one":{"fn":"cast","args":[{"col":"x"}],"to":"decimal[18,1]"}}}],"expected":{"fields":[{"name":"zero","type":"decimal[18,0]"},{"name":"one","type":"decimal[18,1]"}],"columns":{"zero":["-2","-2","2","2","-1"],"one":["-1.5","-2.5","1.5","2.5","-0.8"]}}},{"id":"decimal/round","description":"rounding decimals keeps the scale and rounds half to even","inputs":{"left":{"fields":[{"name":"t","type":"decimal[6,3]"}],"columns":{"t":["0.250","0.350","-0.250","-0.350","1.050","2.500","0.249",null]}}},"ops":[{"op":"select","exprs":{"r0":{"fn":"round","args":[{"col":"t"}]},"r1":{"fn":"round","args":[{"col":"t"}],"decimals":1},"r2":{"fn":"round","args":[{"col":"t"}],"decimals":2},"r3":{"fn":"round","args":[{"col":"t"}],"decimals":3}}}],"expected":{"fields":[{"name":"r0","type":"decimal[7,3]"},{"name":"r1","type":"decimal[7,3]"},{"name":"r2","type":"decimal[7,3]"},{"name":"r3","type":"decimal[6,3]"}],"columns":{"r0":["0.000","0.000","0.000","0.000","1.000","2.000","0.000",null],"r1":["0.200","0.400","-0.200","-0.400","1.000","2.500","0.200",null],"r2":["0.250","0.350","-0.250","-0.350","1.050","2.500","0.250",null],"r3":["0.250","0.350","-0.250","-0.350","1.050","2.500","0.249",null]}}},{"id":"decimal/rolling-mean","description":"rolling means of decimals are floats of their values","inputs":{"left":{"fields":[{"name":"k","type":"utf8"},{"name":"price","type":"decimal[10,2]"},{"name":"fee","type":"decimal[6,3]"},{"name":"qty","type":"decimal[4,0]"},{"name":"x","type":"f64"}],"columns":{"k":["a","b","a","c","b","a"],"price":["12.34","0.10","-5.00",null,"99999999.99","10.01"],"fee":["0.125","1.000",null,"2.500","-0.001","10.010"],"qty":["3","10","2","1",null,"7"],"x":[1.005,-2.5,123.456,null,1000000.0,0.0]}}},"ops":[{"op":"select","exprs":{"m":{"fn":"rollingMean","args":[{"col":"price"}],"window":2},"mf":{"fn":"rollingMean","args":[{"col":"fee"}],"window":3,"minPeriods":1}}}],"expected":{"fields":[{"name":"m","type":"f64"},{"name":"mf","type":"f64"}],"columns":{"m":[null,6.22,-2.45,null,null,50000005.0],"mf":[0.125,0.5625,0.5625,1.75,1.2495,4.169666666666667]}}},{"id":"arith/u32-divide-by-zero","description":"unsigned 32-bit floor division and modulo by zero give null","inputs":{"left":{"fields":[{"name":"n","type":"u32"},{"name":"d","type":"u32"}],"columns":{"n":[7,0,5],"d":[0,0,2]}}},"ops":[{"op":"select","exprs":{"q":{"fn":"floorDiv","args":[{"col":"n"},{"col":"d"}]},"m":{"fn":"mod","args":[{"col":"n"},{"col":"d"}]},"qs":{"fn":"floorDiv","args":[{"col":"n"},{"lit":0,"type":"u32"}]},"ms":{"fn":"mod","args":[{"col":"n"},{"lit":0,"type":"u32"}]}}}],"expected":{"fields":[{"name":"q","type":"u32"},{"name":"m","type":"u32"},{"name":"qs","type":"u32"},{"name":"ms","type":"u32"}],"columns":{"q":[null,null,2],"m":[null,null,1],"qs":[null,null,null],"ms":[null,null,null]}}},{"id":"slice/clip-5+1","description":"slices reaching before the first row clip from the unclamped start","inputs":{"left":{"fields":[{"name":"x","type":"i32"}],"columns":{"x":[1,2,3]}}},"ops":[{"op":"slice","offset":-5,"length":1}],"expected":{"fields":[{"name":"x","type":"i32"}],"columns":{"x":[]}}},{"id":"slice/clip-5+3","description":"slices reaching before the first row clip from the unclamped start","inputs":{"left":{"fields":[{"name":"x","type":"i32"}],"columns":{"x":[1,2,3]}}},"ops":[{"op":"slice","offset":-5,"length":3}],"expected":{"fields":[{"name":"x","type":"i32"}],"columns":{"x":[1]}}},{"id":"slice/clip-5+4","description":"slices reaching before the first row clip from the unclamped start","inputs":{"left":{"fields":[{"name":"x","type":"i32"}],"columns":{"x":[1,2,3]}}},"ops":[{"op":"slice","offset":-5,"length":4}],"expected":{"fields":[{"name":"x","type":"i32"}],"columns":{"x":[1,2]}}},{"id":"slice/clip-2+5","description":"slices reaching before the first row clip from the unclamped start","inputs":{"left":{"fields":[{"name":"x","type":"i32"}],"columns":{"x":[1,2,3]}}},"ops":[{"op":"slice","offset":-2,"length":5}],"expected":{"fields":[{"name":"x","type":"i32"}],"columns":{"x":[2,3]}}},{"id":"slice/clip-5","description":"slices reaching before the first row clip from the unclamped start","inputs":{"left":{"fields":[{"name":"x","type":"i32"}],"columns":{"x":[1,2,3]}}},"ops":[{"op":"slice","offset":-5}],"expected":{"fields":[{"name":"x","type":"i32"}],"columns":{"x":[1,2,3]}}},{"id":"slice/clip1+0","description":"slices reaching before the first row clip from the unclamped start","inputs":{"left":{"fields":[{"name":"x","type":"i32"}],"columns":{"x":[1,2,3]}}},"ops":[{"op":"slice","offset":1,"length":0}],"expected":{"fields":[{"name":"x","type":"i32"}],"columns":{"x":[]}}},{"id":"window/filter-after-sort","description":"a window predicate sees the sorted rows","inputs":{"left":{"fields":[{"name":"g","type":"i32"},{"name":"h","type":"i32"},{"name":"x","type":"f64"}],"columns":{"g":[1,2,1,2],"h":[1,1,2,2],"x":[3.0,1.0,2.0,4.0]}},"right":{"fields":[{"name":"g","type":"i32"},{"name":"name","type":"utf8"}],"columns":{"g":[1],"name":["one"]}}},"ops":[{"op":"sort","keys":[{"expr":{"col":"x"}}]},{"op":"filter","predicate":{"fn":"gt","args":[{"fn":"cumSum","args":[{"col":"x"}]},{"lit":3.0}]}}],"expected":{"fields":[{"name":"g","type":"i32"},{"name":"h","type":"i32"},{"name":"x","type":"f64"}],"columns":{"g":[1,2],"h":[1,2],"x":[3.0,4.0]}}},{"id":"window/filter-after-sort-by-window","description":"a filter after a sort by a window leaves the sort's order alone","inputs":{"left":{"fields":[{"name":"x","type":"f64"}],"columns":{"x":[-3.0,5.0,-1.0]}}},"ops":[{"op":"sort","keys":[{"expr":{"fn":"cumSum","args":[{"col":"x"}]}}]},{"op":"filter","predicate":{"fn":"neq","args":[{"col":"x"},{"lit":5.0}]}}],"expected":{"fields":[{"name":"x","type":"f64"}],"columns":{"x":[-3.0,-1.0]}}},{"id":"window/filter-after-filter","description":"a window predicate sees only the rows an earlier filter kept","inputs":{"left":{"fields":[{"name":"g","type":"i32"},{"name":"h","type":"i32"},{"name":"x","type":"f64"}],"columns":{"g":[1,2,1,2],"h":[1,1,2,2],"x":[3.0,1.0,2.0,4.0]}},"right":{"fields":[{"name":"g","type":"i32"},{"name":"name","type":"utf8"}],"columns":{"g":[1],"name":["one"]}}},"ops":[{"op":"filter","predicate":{"fn":"gt","args":[{"col":"x"},{"lit":1.0}]}},{"op":"filter","predicate":{"fn":"gt","args":[{"fn":"shift","args":[{"col":"x"}]},{"lit":2.0}]}}],"expected":{"fields":[{"name":"g","type":"i32"},{"name":"h","type":"i32"},{"name":"x","type":"f64"}],"columns":{"g":[1],"h":[2],"x":[2.0]}}},{"id":"window/filter-after-concat","description":"a window predicate sees every input of a concat","inputs":{"left":{"fields":[{"name":"g","type":"i32"},{"name":"h","type":"i32"},{"name":"x","type":"f64"}],"columns":{"g":[1,2,1,2],"h":[1,1,2,2],"x":[3.0,1.0,2.0,4.0]}},"right":{"fields":[{"name":"g","type":"i32"},{"name":"name","type":"utf8"}],"columns":{"g":[1],"name":["one"]}}},"ops":[{"op":"concat","with":"left"},{"op":"filter","predicate":{"fn":"gt","args":[{"fn":"cumSum","args":[{"col":"x"}]},{"lit":15.0}]}}],"expected":{"fields":[{"name":"g","type":"i32"},{"name":"h","type":"i32"},{"name":"x","type":"f64"}],"columns":{"g":[1,2],"h":[2,2],"x":[2.0,4.0]}}},{"id":"window/filter-after-join","description":"a window predicate sees only the joined rows","inputs":{"left":{"fields":[{"name":"g","type":"i32"},{"name":"h","type":"i32"},{"name":"x","type":"f64"}],"columns":{"g":[1,2,1,2],"h":[1,1,2,2],"x":[3.0,1.0,2.0,4.0]}},"right":{"fields":[{"name":"g","type":"i32"},{"name":"name","type":"utf8"}],"columns":{"g":[1],"name":["one"]}}},"ops":[{"op":"join","right":"right","on":["g"],"how":"inner"},{"op":"filter","predicate":{"fn":"gt","args":[{"fn":"shift","args":[{"col":"x"}]},{"lit":2.0}]}}],"expected":{"fields":[{"name":"g","type":"i32"},{"name":"h","type":"i32"},{"name":"x","type":"f64"},{"name":"name","type":"utf8"}],"columns":{"g":[1],"h":[2],"x":[2.0],"name":["one"]}}},{"id":"window/filter-whole","description":"a window predicate sees the whole input","inputs":{"left":{"fields":[{"name":"g","type":"i32"},{"name":"h","type":"i32"},{"name":"x","type":"f64"}],"columns":{"g":[1,2,1,2],"h":[1,1,2,2],"x":[3.0,1.0,2.0,4.0]}},"right":{"fields":[{"name":"g","type":"i32"},{"name":"name","type":"utf8"}],"columns":{"g":[1],"name":["one"]}}},"ops":[{"op":"filter","predicate":{"fn":"gt","args":[{"fn":"cumSum","args":[{"col":"x"}]},{"lit":5.0}]}}],"expected":{"fields":[{"name":"g","type":"i32"},{"name":"h","type":"i32"},{"name":"x","type":"f64"}],"columns":{"g":[1,2],"h":[2,2],"x":[2.0,4.0]}}},{"id":"window/agg-of-window","description":"an aggregate of a window sees the whole input","inputs":{"left":{"fields":[{"name":"g","type":"i32"},{"name":"h","type":"i32"},{"name":"x","type":"f64"}],"columns":{"g":[1,2,1,2],"h":[1,1,2,2],"x":[3.0,1.0,2.0,4.0]}},"right":{"fields":[{"name":"g","type":"i32"},{"name":"name","type":"utf8"}],"columns":{"g":[1],"name":["one"]}}},"ops":[{"op":"agg","aggs":{"s":{"agg":"sum","arg":{"fn":"cumSum","args":[{"col":"x"}]}},"n":{"agg":"len"}}}],"expected":{"fields":[{"name":"s","type":"f64"},{"name":"n","type":"u32"}],"columns":{"s":[23.0],"n":[4]}}},{"id":"window/grouped-agg-of-window","description":"windows inside grouped aggregates run per group","inputs":{"left":{"fields":[{"name":"g","type":"i32"},{"name":"h","type":"i32"},{"name":"x","type":"f64"}],"columns":{"g":[1,2,1,2],"h":[1,1,2,2],"x":[3.0,1.0,2.0,4.0]}},"right":{"fields":[{"name":"g","type":"i32"},{"name":"name","type":"utf8"}],"columns":{"g":[1],"name":["one"]}}},"ops":[{"op":"groupBy","keys":["g"],"aggs":{"s":{"agg":"sum","arg":{"fn":"cumSum","args":[{"col":"x"}]}},"l":{"agg":"list","arg":{"fn":"cumSum","args":[{"col":"x"}]}},"o":{"agg":"sum","arg":{"over":{"agg":"sum","arg":{"col":"x"}},"partitionBy":[{"col":"h"}]}}}}],"expected":{"fields":[{"name":"g","type":"i32"},{"name":"s","type":"f64"},{"name":"l","type":"list"},{"name":"o","type":"f64"}],"columns":{"g":[1,2],"s":[8.0,6.0],"l":[[3.0,5.0],[1.0,5.0]],"o":[5.0,5.0]}}}]} From c468b0d1c319203cd555d4f3a2fd4638aebbaeab Mon Sep 17 00:00:00 2001 From: Effect Bot Date: Sun, 15 Nov 2026 12:34:33 +0000 Subject: [PATCH 15/88] Keep partial decimal sums exact until the final check A streamed decimal sum failed when a batch's partial sum left 64 bits even though the total fit. Partial sums of 64-bit integers and decimals are now unbounded and only the final sum is checked. --- .../effect/src/dataframe/internal/partial.ts | 24 +++++++++++++++++++ packages/effect/test/dataframe/Query.test.ts | 11 +++++++++ 2 files changed, 35 insertions(+) diff --git a/packages/effect/src/dataframe/internal/partial.ts b/packages/effect/src/dataframe/internal/partial.ts index 0fb27a1246a..57363b26d59 100644 --- a/packages/effect/src/dataframe/internal/partial.ts +++ b/packages/effect/src/dataframe/internal/partial.ts @@ -6,16 +6,21 @@ * Sums, counts, minima, maxima, firsts and lasts combine by aggregating the * partials again; means combine from sums and counts; variances from counts, * means and M2 (Chan et al.). Medians and distinct counts don't combine. + * Sums of 64-bit integers and decimals stay exact until `finish`, so whether + * a decimal sum fits 64 bits doesn't depend on how the rows are split. * * @internal */ +import type * as S from "../../arrow/ArrowSchema.ts" import * as Agg from "./aggregate.ts" +import * as D from "./dtype.ts" import * as Hash from "./hash.ts" import * as V from "./vector.ts" /** @internal */ export type Part = | { readonly _tag: "vector"; readonly vector: V.Vector } + | Sums | { readonly _tag: "mean"; readonly sum: Float64Array; readonly count: Float64Array } | { readonly _tag: "moments" @@ -25,6 +30,13 @@ export type Part = } | { readonly _tag: "none" } +/** Exact sums of 64-bit integers or decimals, and the type of their sum. */ +interface Sums { + readonly _tag: "sums" + readonly type: S.DataType + readonly sums: ReadonlyArray +} + const reaggregated = new Set(["sum", "count", "len", "min", "max", "first", "last"]) /** Whether partial aggregates of `fn` combine. @internal */ @@ -33,6 +45,7 @@ export const combinable = (fn: Agg.AggFn): boolean => /** The partial aggregate of `fn` over `v` grouped by `g`. @internal */ export const partial = (fn: Agg.AggFn, v: V.Vector | undefined, g: Agg.Grouping): Part => { + if (fn === "sum" && v?.kind === "i64") return { _tag: "sums", type: D.sumType(v.type), sums: Agg.exactSums(v, g) } if (reaggregated.has(fn)) return { _tag: "vector", vector: Agg.aggregate(fn, v, g) } if (v === undefined) return V.fail(`${fn} needs an argument`) switch (fn) { @@ -57,6 +70,15 @@ export const partial = (fn: Agg.AggFn, v: V.Vector | undefined, g: Agg.Grouping) */ export const combine = (fn: Agg.AggFn, parts: ReadonlyArray, ids: Int32Array, count: number): Part => { const grouping: Agg.Grouping = { ids, count } + if (parts.every((p): p is Sums => p._tag === "sums")) { + const sums = new Array(count).fill(BigInt(0)) + let at = 0 + for (const p of parts) { + for (let j = 0; j < p.sums.length; j++) sums[ids[at + j]] += p.sums[j] + at += p.sums.length + } + return { _tag: "sums", type: parts[0].type, sums } + } if (reaggregated.has(fn)) { const vectors = parts.map((p) => (p as Extract).vector) const merge = fn === "count" || fn === "len" ? "sum" : fn @@ -104,6 +126,8 @@ export const finish = (fn: Agg.AggFn, part: Part, input: V.Vector | undefined): switch (part._tag) { case "vector": return part.vector + case "sums": + return Agg.fromSums(part.type, part.sums) case "mean": return Agg.meanFrom(input!.type, part.sum, part.count) case "moments": diff --git a/packages/effect/test/dataframe/Query.test.ts b/packages/effect/test/dataframe/Query.test.ts index d49ba4f9ab3..6006eae3e1a 100644 --- a/packages/effect/test/dataframe/Query.test.ts +++ b/packages/effect/test/dataframe/Query.test.ts @@ -376,6 +376,17 @@ describe("Query", () => { assert.deepStrictEqual(yield* streamed(Query.head(t, 0), Stream.die("pulled")), []) })) + it.effect("decimal sums don't depend on batch boundaries", () => + Effect.gen(function*() { + const query = Query.table("t", Schema.Struct({ s: Schema.String })).pipe( + Query.aggregate((c) => ({ total: Expr.sum(Expr.castDecimal(c.s, 19, 0)) })) + ) + const big = DataFrame.fromColumns({ s: ["6000000000000000000", "6000000000000000000"] }) + const negative = DataFrame.fromColumns({ s: ["-6000000000000000000", "-6000000000000000000"] }) + const result = yield* streamed(query, Stream.make(big, negative)) + assert.deepStrictEqual(result, [{ total: BigDecimal.fromBigInt(0n) }]) + })) + it.effect("zero-row batches aggregate like an empty table", () => Effect.gen(function*() { const query = Query.aggregate(t, (c) => ({ n: Expr.count(), total: Expr.sum(c.x), mean: Expr.mean(c.x) })) From ba6066677749264c328525c0699abf19e9cdea0d Mon Sep 17 00:00:00 2001 From: Effect Bot Date: Sun, 15 Nov 2026 12:38:21 +0000 Subject: [PATCH 16/88] Read only UTC timestamps in range as DateTimeUtc fields DateTimeUtc fields accepted timestamps in any time zone and of any value, so validated rows held DateTime.Zoned values or raw bigints. Tables and validate now fail on other time zones and on timestamps outside the range of DateTime, and drop UTC time zones so the values read as DateTime.Utc. --- packages/effect/src/dataframe/DataFrame.ts | 8 ++ packages/effect/src/dataframe/Plan.ts | 3 + .../effect/src/dataframe/internal/schema.ts | 75 +++++++++++++++---- .../effect/test/dataframe/DataFrame.test.ts | 34 ++++++++- packages/effect/test/dataframe/Query.test.ts | 9 +++ 5 files changed, 114 insertions(+), 15 deletions(-) diff --git a/packages/effect/src/dataframe/DataFrame.ts b/packages/effect/src/dataframe/DataFrame.ts index 0ec791a653f..9ff4510ad4b 100644 --- a/packages/effect/src/dataframe/DataFrame.ts +++ b/packages/effect/src/dataframe/DataFrame.ts @@ -384,6 +384,14 @@ export const empty = ( * arrays need compatible fields and items. Values of fields the schema * constrains further (literals, enums, refinements) are checked against it. * + * **Gotchas** + * + * `DateTimeUtc` fields take timestamps without a time zone, or with a UTC + * one (`UTC`, `Etc/UTC`, `+00:00`), which the result drops; other time + * zones, and values outside the range `DateTime` represents, fail. + * `DateTimeZoned` fields are not supported: read zoned timestamps from an + * untyped frame, where they are `DateTime.Zoned` values. + * * @stability experimental * @category constructors * @since 4.1.5 diff --git a/packages/effect/src/dataframe/Plan.ts b/packages/effect/src/dataframe/Plan.ts index df20b2c324c..7169d090673 100644 --- a/packages/effect/src/dataframe/Plan.ts +++ b/packages/effect/src/dataframe/Plan.ts @@ -703,6 +703,9 @@ export interface TableColumn { * bits for floats and narrower integers, any 64-bit integer for `Int64` * and `UInt64`, any decimal for `Decimal`, the same * temporal type in any unit, and lists and structs with compatible children. + * `Timestamp` columns read as `DateTime.Utc` values: their timestamps must + * have no time zone or a UTC one (dropped), and lie in the range `DateTime` + * represents. * * `schema`, set by `Query.table`, is the struct schema the rows must * satisfy: values of fields it constrains beyond their column type (literals, diff --git a/packages/effect/src/dataframe/internal/schema.ts b/packages/effect/src/dataframe/internal/schema.ts index 7d496728bb7..d29f79ebcdd 100644 --- a/packages/effect/src/dataframe/internal/schema.ts +++ b/packages/effect/src/dataframe/internal/schema.ts @@ -2,6 +2,11 @@ * Table columns for the fields of an Effect Schema struct: their Arrow types, * and the checks a frame must pass to be read as rows of the schema. * + * A `Timestamp` column reads as `DateTime.Utc` values, so its timestamps + * must have no time zone (or a UTC one, which the check drops) and lie in + * the range `DateTime` represents. `DateTimeZoned` fields have no column + * type. + * * @internal */ import * as S from "../../arrow/ArrowSchema.ts" @@ -251,7 +256,32 @@ const allNull = (v: V.Vector, rows: Uint8Array | undefined): boolean => { return true } -const check = (path: string, v: V.Vector, expected: Column, rows: Uint8Array | undefined): void => { +const utcZones: ReadonlySet = new Set(["UTC", "Etc/UTC", "+00:00"]) + +/** The stored values of a timestamp in each unit that `DateTime` represents (±8.64e15 ms). */ +const timestampRange: { readonly [U in S.TimeUnit]: readonly [bigint, bigint] } = { + second: [-BigInt("8640000000000"), BigInt("8640000000000")], + millisecond: [-BigInt("8640000000000000"), BigInt("8640000000000000")], + microsecond: [-BigInt("8640000000000000000"), BigInt("8640000000000000999")], + nanosecond: [-BigInt("9223372036854775808"), BigInt("9223372036854775807")] +} + +/** Timestamps that read as `DateTime.Utc`: without a time zone, in range. */ +const utcTimestamps = (path: string, v: V.I64, type: S.Timestamp, rows: Uint8Array | undefined): V.Vector => { + if (type.timezone !== undefined && !utcZones.has(type.timezone)) { + return V.fail(`column ${JSON.stringify(path)} has time zone ${type.timezone}, expected UTC timestamps`) + } + const [lo, hi] = timestampRange[type.unit] + for (let i = 0; i < v.length; i++) { + if ((rows === undefined || rows[i] === 1) && V.isValid(v, i) && (v.values[i] < lo || v.values[i] > hi)) { + return V.fail(`column ${JSON.stringify(path)} has timestamps outside the range of DateTime`) + } + } + return type.timezone === undefined ? v : V.withType(v, S.timestamp(type.unit)) +} + +/** Checks a column against its declaration, returning it with UTC time zones dropped. */ +const check = (path: string, v: V.Vector, expected: Column, rows: Uint8Array | undefined): V.Vector => { if (!compatible(v.type, expected, true) && !(expected.type === "Null" && allNull(v, rows))) { return V.fail(`column ${JSON.stringify(path)} is ${describe(v.type)}, expected ${describeColumn(expected)}`) } @@ -262,22 +292,35 @@ const check = (path: string, v: V.Vector, expected: Column, rows: Uint8Array | u } } } - if (expected.fields === undefined) return + if (expected.type === "Timestamp" && v.kind === "i64" && v.type._tag === "Timestamp") { + return utcTimestamps(path, v, v.type, rows) + } + if (expected.fields === undefined) return v if (v.kind === "struct" && v.type._tag === "Struct") { const present = new Uint8Array(v.length) for (let i = 0; i < v.length; i++) present[i] = (rows === undefined || rows[i] === 1) && V.isValid(v, i) ? 1 : 0 + const fields = v.type.fields + const children = v.children.slice() for (const f of expected.fields) { - const k = v.type.fields.findIndex((g) => g.name === f.name) + const k = fields.findIndex((g) => g.name === f.name) if (k === -1) return V.fail(`missing column ${JSON.stringify(`${path}.${f.name}`)}`) - check(`${path}.${f.name}`, v.children[k], f, present) + children[k] = check(`${path}.${f.name}`, v.children[k], f, present) } - } else if (v.kind === "list") { + if (children.every((c, k) => c === v.children[k])) return v + const type = S.struct(fields.map((g, k) => ({ ...g, type: children[k].type }))) + return V.struct(type, children, v.length, v.valid) + } + if (v.kind === "list" && (v.type._tag === "List" || v.type._tag === "LargeList")) { const present = new Uint8Array(v.child.length) for (let i = 0; i < v.length; i++) { if ((rows === undefined || rows[i] === 1) && V.isValid(v, i)) present.fill(1, v.offsets[i], v.offsets[i + 1]) } - check(`${path}[]`, v.child, expected.fields[0], present) + const child = check(`${path}[]`, v.child, expected.fields[0], present) + return child === v.child + ? v + : V.list({ ...v.type, child: { ...v.type.child, type: child.type } }, v.offsets, child, v.valid) } + return v } const refined = (ast: AST.AST): boolean => @@ -323,8 +366,8 @@ const valuesCheck = (ast: AST.AST): ((value: unknown) => boolean) | undefined => } /** - * Checks that a frame has the expected columns and narrows it to them. With - * a `schema`, the values of fields it constrains beyond their column type are + * Checks that a frame has the expected columns and narrows it to them, + * dropping UTC time zones from timestamps. With a `schema`, the values of fields it constrains beyond their column type are * checked against it too. * * @internal @@ -334,12 +377,16 @@ export const conform = ( columns: ReadonlyArray, schema?: { readonly ast: AST.AST } | undefined ): F.Frame => { - for (const expected of columns) { - const c = frame.columns.find((c) => c.name === expected.name) - if (c === undefined) return V.fail(`missing column ${JSON.stringify(expected.name)}`) - check(expected.name, c.values(), expected, undefined) - } - const narrowed = F.pick(frame, columns.map((c) => c.name)) + const narrowed = F.make( + columns.map((expected) => { + const c = frame.columns.find((c) => c.name === expected.name) + if (c === undefined) return V.fail(`missing column ${JSON.stringify(expected.name)}`) + const values = c.values() + const checked = check(expected.name, values, expected, undefined) + return checked === values ? c : new F.Column(c.name, checked, undefined, c.nullable) + }), + frame.length + ) if (schema !== undefined) { for (const ps of structOf(schema).propertySignatures) { const test = valuesCheck(ps.type) diff --git a/packages/effect/test/dataframe/DataFrame.test.ts b/packages/effect/test/dataframe/DataFrame.test.ts index aab6112fa43..e0d2a0e0923 100644 --- a/packages/effect/test/dataframe/DataFrame.test.ts +++ b/packages/effect/test/dataframe/DataFrame.test.ts @@ -1,5 +1,5 @@ import { assert, describe, it } from "@effect/vitest" -import { BigDecimal, Effect, Equal, Result, Schema, Stream } from "effect" +import { BigDecimal, DateTime, Effect, Equal, Result, Schema, Stream } from "effect" import { ArrowIpc, ArrowSchema, ArrowTable } from "effect/arrow" import { DataFrame, Expr, Query } from "effect/dataframe" @@ -85,6 +85,13 @@ describe("DataFrame", () => { describe("validate", () => { const untyped = DataFrame.fromArrowTable(DataFrame.toArrowTable(readings)) + const column = (type: ArrowSchema.DataType, values: ReadonlyArray) => + DataFrame.fromArrowTable( + Result.getOrThrow( + ArrowTable.fromRows(ArrowSchema.make([ArrowSchema.field("t", type)]), values.map((t) => ({ t }))) + ) + ) + const Utc = Schema.Struct({ t: Schema.DateTimeUtc }) it("returns the frame typed by the schema, in schema order", () => { const result = DataFrame.validate(untyped, Schema.Struct({ temp: Schema.Number, city: Schema.String })) @@ -116,6 +123,24 @@ describe("DataFrame", () => { DataFrame.fromColumns({ n: [1, 1.5] }), Schema.Struct({ n: Schema.Int }), `column "n" has values its schema rejects` + ], + [ + column(ArrowSchema.timestamp("millisecond", "Europe/Berlin"), [0n]), + Utc, + `column "t" has time zone Europe/Berlin, expected UTC timestamps` + ], + [ + column(ArrowSchema.timestamp("millisecond"), [9223372036854775807n]), + Utc, + `column "t" has timestamps outside the range of DateTime` + ], + [ + column(ArrowSchema.list(ArrowSchema.field("item", ArrowSchema.timestamp("microsecond"))), [[ + 8640000000000000999n, + 8640000000000001000n + ]]), + Schema.Struct({ t: Schema.Array(Schema.DateTimeUtc) }), + `column "t[]" has timestamps outside the range of DateTime` ] ] for (const [df, schema, message] of cases) { @@ -124,6 +149,13 @@ describe("DataFrame", () => { assert.strictEqual(Result.isFailure(result) && result.failure.message, message) } }) + + it("reads timestamps with a UTC time zone as DateTime.Utc", () => { + const ts = ArrowSchema.timestamp("millisecond", "UTC") + const nested = column(ArrowSchema.list(ArrowSchema.field("item", ts)), [[1000n]]) + const df = Result.getOrThrow(DataFrame.validate(nested, Schema.Struct({ t: Schema.Array(Schema.DateTimeUtc) }))) + assert.deepStrictEqual(DataFrame.toRows(df), [{ t: [DateTime.makeUnsafe(1000)] }]) + }) }) describe("Arrow interop", () => { diff --git a/packages/effect/test/dataframe/Query.test.ts b/packages/effect/test/dataframe/Query.test.ts index 6006eae3e1a..b1ae0cdb4c6 100644 --- a/packages/effect/test/dataframe/Query.test.ts +++ b/packages/effect/test/dataframe/Query.test.ts @@ -78,6 +78,15 @@ describe("Query", () => { const error = yield* Effect.flip(Query.table("t", Kind).pipe(Effect.provide(Catalog.layer({ t })))) assert.strictEqual(error.message, `column "kind" has values its schema rejects`) } + const berlin = DataFrame.fromArrowTable(Result.getOrThrow(ArrowTable.fromRows( + ArrowSchema.make([ArrowSchema.field("t", ArrowSchema.timestamp("millisecond", "Europe/Berlin"))]), + [{ t: 0n }] + ))) + const Utc = Schema.Struct({ t: Schema.DateTimeUtc }) + for (const t of [berlin, Stream.make(berlin)]) { + const error = yield* Effect.flip(Query.table("t", Utc).pipe(Effect.provide(Catalog.layer({ t })))) + assert.strictEqual(error.message, `column "t" has time zone Europe/Berlin, expected UTC timestamps`) + } })) it.effect("narrows tables to their declared columns", () => From 83e661e124fb89a5a640d7e33b3d80797442aee3 Mon Sep 17 00:00:00 2001 From: Effect Bot Date: Sun, 15 Nov 2026 12:40:56 +0000 Subject: [PATCH 17/88] Keep column types of streamed results without rows A streamed query whose input was empty typed nested declared columns as Null, so list and struct functions over them failed, and one whose rows were all filtered out lost the columns of an undeclared table. Empty tables now build their declared columns recursively, and batches without rows flow through the engine so the result keeps their types; Query.stream still emits only non-empty batches. --- .../effect/src/dataframe/internal/stream.ts | 38 +++++++++++++------ packages/effect/test/dataframe/Query.test.ts | 21 ++++++++++ 2 files changed, 47 insertions(+), 12 deletions(-) diff --git a/packages/effect/src/dataframe/internal/stream.ts b/packages/effect/src/dataframe/internal/stream.ts index 6c1301b2507..fc1f425bb6c 100644 --- a/packages/effect/src/dataframe/internal/stream.ts +++ b/packages/effect/src/dataframe/internal/stream.ts @@ -50,14 +50,22 @@ const typeOf: Record = { Duration: S.duration("microsecond") } +/** The type of a declared column without rows; decimals, whose precision is unknown, are Null. */ +const emptyType = (column: Plan.TableColumn): S.DataType => { + const fields = (column.fields ?? []).map((f) => S.field(f.name, emptyType(f), { nullable: f.nullable })) + switch (column.type) { + case "List": + return S.list(fields[0]) + case "Struct": + return S.struct(fields) + default: + return typeOf[column.type] ?? S.nullType + } +} + /** A table without rows: its declared columns, or none. */ const emptyTable = (node: Extract): F.Frame => - F.make( - (node.columns ?? []).map((c) => - new F.Column(c.name, K.nullsOf(typeOf[c.type] ?? S.nullType, 0), undefined, c.nullable) - ), - 0 - ) + F.make((node.columns ?? []).map((c) => new F.Column(c.name, K.nullsOf(emptyType(c), 0), undefined, c.nullable)), 0) /** The plan over no rows (every input empty), run in place. */ const runEmpty = (node: Q.Node): Effect.Effect => { @@ -182,18 +190,22 @@ const slice = (input: Batches, offset: number, length: number | undefined): Batc left -= end - start return F.slice(batch, start, end) }), - Stream.takeUntil(() => left === 0), - Stream.filter((batch) => batch.length > 0) + Stream.takeUntil(() => left === 0) ) }) -/** The batches of a node's result. @internal */ +/** + * The batches of a node's result. Batches without rows stay, so a result + * whose rows are all dropped still has the types of its columns. + * + * @internal + */ export const batches = (node: Q.Node, catalog: Catalog.Service): Batches => { if (perBatch(node)) { return Stream.mapEffect( batches((node as { readonly input: Q.Node }).input, catalog), (batch) => Q.attempt(() => Q.step(node, [batch])) - ).pipe(Stream.filter((batch) => batch.length > 0)) + ) } switch (node._tag) { case "Table": @@ -252,9 +264,11 @@ const prepare = (plan: Plan.Node, catalog: Catalog.Service): Effect.Effect - Stream.unwrap(Effect.map(prepare(plan, catalog), (optimized) => batches(optimized, catalog))) + Stream.unwrap(Effect.map(prepare(plan, catalog), (optimized) => batches(optimized, catalog))).pipe( + Stream.filter((batch) => batch.length > 0) + ) /** A plan's result as one frame, computed batch by batch. @internal */ export const collect = (plan: Plan.Node, catalog: Catalog.Service): Effect.Effect => diff --git a/packages/effect/test/dataframe/Query.test.ts b/packages/effect/test/dataframe/Query.test.ts index b1ae0cdb4c6..2125de649ea 100644 --- a/packages/effect/test/dataframe/Query.test.ts +++ b/packages/effect/test/dataframe/Query.test.ts @@ -403,6 +403,27 @@ describe("Query", () => { assert.deepStrictEqual(yield* streamed(query, empty), [{ n: 0, total: 0, mean: null }]) })) + it.effect("results without rows keep the types of their columns", () => + Effect.gen(function*() { + const Nested = Schema.Struct({ xs: Schema.Array(Schema.Number), s: Schema.Struct({ a: Schema.Number }) }) + const nested = Query.table("t", Nested).pipe( + Query.filter((c) => Expr.gt(Expr.listLength(c.xs), 5)), + Query.select((c) => ({ sum: Expr.listSum(c.xs), a: Expr.field(c.s, "a") })) + ) + const untyped = Query.fromPlan({ _tag: "Table", name: "t" }).pipe( + Query.filter((c) => Expr.gt(c.x as Expr.Expr, 5)) + ) + const cases: ReadonlyArray, Catalog.Source, ReadonlyArray]> = [ + [nested, Stream.empty, ["sum:Float", "a:Float"]], + [nested, Stream.make(DataFrame.fromRows(Nested, [{ xs: [1], s: { a: 1 } }])), ["sum:Float", "a:Float"]], + [untyped, Stream.make(DataFrame.fromColumns({ x: [1], y: ["a"] })), ["x:Float", "y:Utf8"]] + ] + for (const [query, source, expected] of cases) { + const df = yield* query.pipe(Effect.provide(Catalog.layer({ t: source }))) + assert.deepStrictEqual(df.schema.fields.map((f) => `${f.name}:${f.type._tag}`), expected) + } + })) + it.effect("reads a table named twice once per run", () => Effect.gen(function*() { let loads = 0 From f3ebf166428939ad3b77f6d8b1dddfb0de28b9b8 Mon Sep 17 00:00:00 2001 From: Effect Bot Date: Sun, 15 Nov 2026 12:30:21 +0000 Subject: [PATCH 18/88] Compile DataFrame plans for PostgreSQL only --- .changeset/dataframe.md | 2 +- packages/effect/src/dataframe/Sql.ts | 74 +++---- packages/effect/src/dataframe/internal/sql.ts | 184 +++++++----------- packages/sql/pglite/test/DataFrameSql.test.ts | 5 +- 4 files changed, 97 insertions(+), 168 deletions(-) diff --git a/.changeset/dataframe.md b/.changeset/dataframe.md index 500c135633f..b60b80e02cf 100644 --- a/.changeset/dataframe.md +++ b/.changeset/dataframe.md @@ -2,4 +2,4 @@ "effect": patch --- -Add `effect/dataframe`: typed, columnar data frames on top of `effect/arrow`. `DataFrame` is the immutable table value; `Query` describes filters, derived columns, sorts, groups, aggregates, joins, window functions (`Expr.over`, cumulative, shift, rank, rolling), pivots, unpivots and list explosion lazily, with column names and types checked at compile time, and runs as an `Effect` after an optimiser rewrites the plan (predicate, slice and projection pushdown, top-k). Columns can hold numbers, strings, booleans, dates, timestamps, durations, decimals, lists and structs. Queries are plain data (`Plan`, with Schema codecs to and from JSON), read named tables from a `Catalog` service (frames, effects or streams of batches, processed batch by batch), and run on an `Engine`: in memory, or on a database through `Sql.layer` and `effect/sql`. Results follow Polars semantics and are checked against a Polars-generated gold corpus. +Add `effect/dataframe`: typed, columnar data frames on top of `effect/arrow`. `DataFrame` is the immutable table value; `Query` describes filters, derived columns, sorts, groups, aggregates, joins, window functions (`Expr.over`, cumulative, shift, rank, rolling), pivots, unpivots and list explosion lazily, with column names and types checked at compile time, and runs as an `Effect` after an optimiser rewrites the plan (predicate, slice and projection pushdown, top-k). Columns can hold numbers, strings, booleans, dates, timestamps, durations, decimals, lists and structs. Queries are plain data (`Plan`, with Schema codecs to and from JSON), read named tables from a `Catalog` service (frames, effects or streams of batches, processed batch by batch), and run on an `Engine`: in memory, or on PostgreSQL through `Sql.layer` and `effect/sql`. Results follow Polars semantics and are checked against a Polars-generated gold corpus. diff --git a/packages/effect/src/dataframe/Sql.ts b/packages/effect/src/dataframe/Sql.ts index 2803dbd4c19..110df5aaea3 100644 --- a/packages/effect/src/dataframe/Sql.ts +++ b/packages/effect/src/dataframe/Sql.ts @@ -1,5 +1,5 @@ /** - * Running queries on a database: plans compiled to SQL and run through the + * Running queries on PostgreSQL: plans compiled to SQL and run through the * `effect/sql` client in the context. * * **Mental model** @@ -10,7 +10,8 @@ * - {@link layer} provides an `Engine` that compiles every query and runs it * with the `SqlClient` in the context, so `yield* query` reads from the * database. `Query.table` names become table names. - * - Two dialects: `postgres` (PostgreSQL, PGlite) and `duckdb`. + * - The SQL is PostgreSQL's (PGlite included); other databases are future + * work. * * **Example** (Compiling a query) * @@ -25,7 +26,7 @@ * Query.agg((c) => ({ total: Expr.sum(c.km) })) * ) * - * const compiled = Result.getOrThrow(Sql.compile(query.plan, { dialect: "postgres" })) + * const compiled = Result.getOrThrow(Sql.compile(query.plan)) * compiled.schema.fields.map((f) => f.name) // => ["city", "total"] * ``` * @@ -35,7 +36,7 @@ * plan (see {@link Compiled}). Values follow the database's semantics where * they differ from the in-memory engine's: integer overflow raises an error * instead of wrapping, division by zero follows the database, and string - * order follows its collation (binary in DuckDB). Pivots, unpivots, + * order follows its collation. Pivots, unpivots, * `explode` and `unnest` are not supported yet; tables need declared * columns (`Query.table` with a schema). * @@ -55,15 +56,6 @@ import * as internal from "./internal/sql.ts" import * as V from "./internal/vector.ts" import type * as Plan from "./Plan.ts" -/** - * The SQL dialects plans compile to. - * - * @stability experimental - * @category models - * @since 4.1.5 - */ -export type Dialect = "duckdb" | "postgres" - /** * A compiled plan: the query, and the schema of its result. * @@ -94,19 +86,16 @@ const toError = (cause: unknown) => { } /** - * Compiles a plan to one SQL query (dialect `postgres` by default). Fails - * for operations the SQL engine doesn't support. + * Compiles a plan to one PostgreSQL query. Fails for operations the SQL + * engine doesn't support. * * @stability experimental * @category compiling * @since 4.1.5 */ -export const compile = ( - plan: Plan.Node, - options?: { readonly dialect?: Dialect | undefined } -): Result.Result => { +export const compile = (plan: Plan.Node): Result.Result => { try { - return Result.succeed(internal.compile(plan, options?.dialect ?? "postgres")) + return Result.succeed(internal.compile(plan)) } catch (cause) { if (cause instanceof V.DataFrameDefect) return Result.fail(toError(cause)) throw cause @@ -121,31 +110,25 @@ export const compile = ( * @category engines * @since 4.1.5 */ -export const engine = (options?: { readonly dialect?: Dialect | undefined }): Effect.Effect< - Engine.Service, - never, - SqlClient -> => - Effect.gen(function*() { - const sql = yield* SqlClient - const dialect = options?.dialect ?? "postgres" - const compiled = (plan: Plan.Node) => Effect.fromResult(compile(plan, { dialect })) - const collect = (plan: Plan.Node): Effect.Effect, DataFrameError> => - Effect.gen(function*() { - const query = yield* compiled(plan) - const rows = yield* sql.unsafe(query.sql).values.pipe(Effect.mapError(toError)) - return yield* Effect.try({ - try: () => internal.decode(query.schema, rows) as unknown as DataFrame, - catch: toError - }) +export const engine: Effect.Effect = Effect.gen(function*() { + const sql = yield* SqlClient + const compiled = (plan: Plan.Node) => Effect.fromResult(compile(plan)) + const collect = (plan: Plan.Node): Effect.Effect, DataFrameError> => + Effect.gen(function*() { + const query = yield* compiled(plan) + const rows = yield* sql.unsafe(query.sql).values.pipe(Effect.mapError(toError)) + return yield* Effect.try({ + try: () => internal.decode(query.schema, rows) as unknown as DataFrame, + catch: toError }) - return { - name: `sql (${dialect})`, - collect, - stream: (plan) => Stream.fromEffect(collect(plan)), - explain: (plan) => Effect.map(compiled(plan), (c) => c.sql) - } satisfies Engine.Service - }) + }) + return { + name: "sql (postgres)", + collect, + stream: (plan) => Stream.fromEffect(collect(plan)), + explain: (plan) => Effect.map(compiled(plan), (c) => c.sql) + } satisfies Engine.Service +}) /** * A layer that runs queries on the database of the `SqlClient` in the @@ -155,5 +138,4 @@ export const engine = (options?: { readonly dialect?: Dialect | undefined }): Ef * @category layers * @since 4.1.5 */ -export const layer = (options?: { readonly dialect?: Dialect | undefined }): Layer.Layer => - Layer.effect(Engine, engine(options)) +export const layer: Layer.Layer = Layer.effect(Engine, engine) diff --git a/packages/effect/src/dataframe/internal/sql.ts b/packages/effect/src/dataframe/internal/sql.ts index 225a245d8c2..7e87b96d433 100644 --- a/packages/effect/src/dataframe/internal/sql.ts +++ b/packages/effect/src/dataframe/internal/sql.ts @@ -33,11 +33,7 @@ import * as Q from "./query.ts" import { civil } from "./temporal.ts" import * as V from "./vector.ts" -/** @internal */ -export type Dialect = "duckdb" | "postgres" - interface Compiler { - readonly dialect: Dialect /** The engine's frame for every node of the plan, on zero-row inputs. */ readonly frames: ReadonlyMap /** A quoted helper column name, disjoint from the plan's column names. */ @@ -51,21 +47,15 @@ export const quote = (name: string): string => `"${name.replaceAll(`"`, `""`)}"` const list = (names: ReadonlyArray, prefix = "") => names.map((n) => `${prefix}${quote(n)}`).join(", ") -const unsupported = (what: string, dialect: Dialect): never => - V.fail(`${what} is not supported by the SQL engine (${dialect})`) +const unsupported = (what: string): never => V.fail(`${what} is not supported by the SQL engine`) // ----------------------------------------------------------------------------- // Types and literals // ----------------------------------------------------------------------------- -const sqlType = (t: S.DataType, dialect: Dialect): string => { +const sqlType = (t: S.DataType): string => { switch (t._tag) { case "Int": - if (dialect === "duckdb") { - return `${t.signed ? "" : "U"}${ - ({ 8: "TINYINT", 16: "SMALLINT", 32: "INTEGER", 64: "BIGINT" } as const)[t.bitWidth] - }` - } return t.bitWidth <= 16 && t.signed ? "SMALLINT" : t.bitWidth <= 16 || (t.bitWidth === 32 && t.signed) ? "INTEGER" : t.bitWidth === 32 @@ -78,11 +68,11 @@ const sqlType = (t: S.DataType, dialect: Dialect): string => { case "Utf8": case "LargeUtf8": case "Utf8View": - return dialect === "duckdb" ? "VARCHAR" : "TEXT" + return "TEXT" case "Binary": case "LargeBinary": case "BinaryView": - return dialect === "duckdb" ? "BLOB" : "BYTEA" + return "BYTEA" case "Bool": return "BOOLEAN" case "Date": @@ -96,12 +86,12 @@ const sqlType = (t: S.DataType, dialect: Dialect): string => { case "Null": return "INTEGER" default: - return unsupported(`columns of type ${t._tag}`, dialect) + return unsupported(`columns of type ${t._tag}`) } } /** The type the SQL engine reads a declared table column as. */ -const tableType = (table: string, column: Plan.TableColumn, dialect: Dialect): S.DataType => { +const tableType = (table: string, column: Plan.TableColumn): S.DataType => { switch (column.type) { case "Timestamp": return S.timestamp("microsecond") @@ -113,8 +103,7 @@ const tableType = (table: string, column: Plan.TableColumn, dialect: Dialect): S return S.nullType default: return D.castTargets[column.type] ?? unsupported( - `column ${JSON.stringify(column.name)} of table ${JSON.stringify(table)}: ${column.type} table columns`, - dialect + `column ${JSON.stringify(column.name)} of table ${JSON.stringify(table)}: ${column.type} table columns` ) } } @@ -147,7 +136,7 @@ const timestampText = (value: bigint, unit: Plan.TimeUnit): string => { const NAN = "CAST('NaN' AS DOUBLE PRECISION)" -const literal = (value: unknown, type: Plan.LiteralType | undefined, dialect: Dialect): string => { +const literal = (value: unknown, type: Plan.LiteralType | undefined): string => { if (value === null || value === undefined) return "NULL" if (type !== undefined) { const big = BigInt(value as number | bigint) @@ -162,9 +151,7 @@ const literal = (value: unknown, type: Plan.LiteralType | undefined, dialect: Di : `TIMESTAMPTZ '${timestampText(big, type.unit)}+00'` case "Duration": { const micros = big * BigInt(1_000_000) / perSecond[type.unit] - return dialect === "duckdb" - ? `TO_MICROSECONDS(CAST(${micros} AS BIGINT))` - : `(${micros} * INTERVAL '1 microsecond')` + return `(${micros} * INTERVAL '1 microsecond')` } case "Decimal": { const negative = big < BigInt(0) @@ -187,7 +174,7 @@ const literal = (value: unknown, type: Plan.LiteralType | undefined, dialect: Di case "string": return string(value) default: - return unsupported(`literals like ${String(value)}`, dialect) + return unsupported(`literals like ${String(value)}`) } } @@ -215,10 +202,7 @@ const partitionBy = (scope: Scope, separator = ""): string => ? "" : `PARTITION BY ${scope.partition.join(", ")}${separator}` -const isNaNSql = ( - x: string, - d: Dialect -) => (d === "duckdb" ? `ISNAN(${x})` : `(${x} = ${NAN})`) +const isNaNSql = (x: string) => `(${x} = ${NAN})` const window = (scope: Scope, frame?: string): string => `OVER (${partitionBy(scope, " ")}ORDER BY ${scope.compiler.row}${frame === undefined ? "" : ` ${frame}`})` @@ -235,7 +219,7 @@ const power = (k: number) => k >= 0 ? `1${"0".repeat(k)}` : `0.${"0".repeat(-k - * precision, decimals half to even at `d` places into the engine's result * type, integers unchanged. */ -const round = (x: string, type: S.DataType, result: S.DataType, decimals: number, d: Dialect): string => { +const round = (x: string, type: S.DataType, result: S.DataType, decimals: number): string => { switch (type._tag) { case "Int": case "Null": @@ -246,19 +230,17 @@ const round = (x: string, type: S.DataType, result: S.DataType, decimals: number const floor = `FLOOR(${scaled})` const up = `(${scaled} - ${floor} > 0.5 OR (${scaled} - ${floor} = 0.5 AND MOD(${floor}, 2) <> 0))` const rounded = `(CASE WHEN ${up} THEN ${floor} + 1 ELSE ${floor} END)` - return `CAST(${decimals === 0 ? rounded : `(${rounded} / ${power(decimals)})`} AS ${sqlType(result, d)})` + return `CAST(${decimals === 0 ? rounded : `(${rounded} / ${power(decimals)})`} AS ${sqlType(result)})` } default: { - const rint = (y: string) => d === "duckdb" ? `ROUND_EVEN(${y}, 0)` : `ROUND(${y})` - if (decimals === 0) return rint(float(x)) - const scale = literal(10 ** decimals, undefined, d) - return `(${rint(`(${float(x)} * ${float(scale)})`)} / ${float(scale)})` + if (decimals === 0) return `ROUND(${float(x)})` + const scale = float(literal(10 ** decimals, undefined)) + return `(ROUND(${float(x)} * ${scale}) / ${scale})` } } } const aggregate = (node: Plan.AggregateCall, scope: Scope, over: string | undefined): string => { - const d = scope.compiler.dialect const row = scope.compiler.row const inner = { ...scope, partition: undefined } const x = node.arg === undefined ? "" : expression(node.arg, inner) @@ -268,11 +250,7 @@ const aggregate = (node: Plan.AggregateCall, scope: Scope, over: string | undefi case "count": return node.arg === undefined ? `COUNT(*)${o}` : `COUNT(${x})${o}` case "sum": - if (type === "Duration") { - return d === "duckdb" - ? `TO_MICROSECONDS(CAST(COALESCE(SUM(EXTRACT(EPOCH FROM ${x}) * 1000000)${o}, 0) AS BIGINT))` - : `COALESCE(SUM(${x})${o}, INTERVAL '0 seconds')` - } + if (type === "Duration") return `COALESCE(SUM(${x})${o}, INTERVAL '0 seconds')` return `COALESCE(SUM(${type === "Bool" ? `CAST(${x} AS INTEGER)` : x})${o}, 0)` case "mean": return `AVG(${x})${o}` @@ -281,7 +259,7 @@ const aggregate = (node: Plan.AggregateCall, scope: Scope, over: string | undefi const fn = node.fn.toUpperCase() // Floats: NaN is skipped unless every value is NaN. return type === "Float" - ? `COALESCE(${fn}(CASE WHEN ${isNaNSql(x, d)} THEN NULL ELSE ${x} END)${o}, ${fn}(${x})${o})` + ? `COALESCE(${fn}(CASE WHEN ${isNaNSql(x)} THEN NULL ELSE ${x} END)${o}, ${fn}(${x})${o})` : `${fn}(${x})${o}` } case "std": @@ -289,10 +267,10 @@ const aggregate = (node: Plan.AggregateCall, scope: Scope, over: string | undefi case "variance": return `VAR_SAMP(${x})${o}` case "median": - if (over !== undefined) return unsupported("median over partitions", d) - return d === "duckdb" ? `MEDIAN(${x})` : `PERCENTILE_CONT(0.5) WITHIN GROUP (ORDER BY ${x})` + if (over !== undefined) return unsupported("median over partitions") + return `PERCENTILE_CONT(0.5) WITHIN GROUP (ORDER BY ${x})` case "countDistinct": - if (over !== undefined) return unsupported("countDistinct over partitions", d) + if (over !== undefined) return unsupported("countDistinct over partitions") // Nulls count as one value. return `(COUNT(DISTINCT ${x}) + CASE WHEN COUNT(*) > COUNT(${x}) THEN 1 ELSE 0 END)` case "first": @@ -302,13 +280,11 @@ const aggregate = (node: Plan.AggregateCall, scope: Scope, over: string | undefi partitionBy(scope, " ") }ORDER BY ${row} ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING)` } - return d === "duckdb" - ? `${node.fn.toUpperCase()}(${x} ORDER BY ${row})` - : `(ARRAY_AGG(${x} ORDER BY ${row} ${node.fn === "first" ? "ASC" : "DESC"}))[1]` + return `(ARRAY_AGG(${x} ORDER BY ${row} ${node.fn === "first" ? "ASC" : "DESC"}))[1]` } case "list": - if (over !== undefined) return unsupported("list over partitions", d) - return d === "duckdb" ? `LIST(${x} ORDER BY ${row})` : `ARRAY_AGG(${x} ORDER BY ${row})` + if (over !== undefined) return unsupported("list over partitions") + return `ARRAY_AGG(${x} ORDER BY ${row})` } } @@ -322,7 +298,6 @@ const totalSeconds: Record = { } const call = (node: Plan.Call, scope: Scope): string => { - const d = scope.compiler.dialect const row = scope.compiler.row const a = node.args.map((x) => expression(x, scope)) const k = node.args.map((x) => typeOf(x, scope)._tag) @@ -342,7 +317,7 @@ const call = (node: Plan.Call, scope: Scope): string => { return `(${float(a[0])} / ${a[1]})` case "floorDivide": return ints - ? `((${a[0]} - (${intMod()})) ${d === "duckdb" ? "//" : "/"} NULLIF(${a[1]}, 0))` + ? `((${a[0]} - (${intMod()})) / NULLIF(${a[1]}, 0))` : `FLOOR(${float(a[0])} / ${a[1]})` case "mod": return ints ? `(${intMod()})` : `(${a[0]} - ${a[1]} * FLOOR(${float(a[0])} / ${a[1]}))` @@ -353,7 +328,7 @@ const call = (node: Plan.Call, scope: Scope): string => { case "sqrt": return `(CASE WHEN ${float(a[0])} < 0 THEN ${NAN} ELSE SQRT(${float(a[0])}) END)` case "round": - return round(a[0], typeOf(node.args[0], scope), typeOf(node, scope), o.decimals ?? 0, d) + return round(a[0], typeOf(node.args[0], scope), typeOf(node, scope), o.decimals ?? 0) case "eq": return `(${a[0]} = ${a[1]})` case "neq": @@ -372,7 +347,7 @@ const call = (node: Plan.Call, scope: Scope): string => { const values = (o.values ?? []).filter((v) => v !== null) return values.length === 0 ? `(${a[0]} IS NULL AND NULL)` : - `(${a[0]} IN (${values.map((v) => literal(v, undefined, d)).join(", ")}))` + `(${a[0]} IN (${values.map((v) => literal(v, undefined)).join(", ")}))` } case "and": return `(${a[0]} AND ${a[1]})` @@ -385,7 +360,7 @@ const call = (node: Plan.Call, scope: Scope): string => { case "isNotNull": return `(${a[0]} IS NOT NULL)` case "isNaN": - return isNaNSql(a[0], d) + return isNaNSql(a[0]) case "fillNull": case "coalesce": return `COALESCE(${a.join(", ")})` @@ -395,18 +370,15 @@ const call = (node: Plan.Call, scope: Scope): string => { const to = o.to ?? "" const target = to === "Decimal" ? `DECIMAL(${o.precision}, ${o.scale})` - : sqlType(D.castTargets[to] ?? V.fail(`unknown cast target ${to}`), d) + : sqlType(D.castTargets[to] ?? V.fail(`unknown cast target ${to}`)) const integer = to.startsWith("Int") || to.startsWith("UInt") let x = a[0] // Temporal values become their integers; floats and decimals truncate (as the engine does). - if (integer && k[0] === "Timestamp") { - x = d === "duckdb" - ? `EPOCH_US(${x})` - : `CAST(EXTRACT(EPOCH FROM ${x}) * 1000000 AS BIGINT)` - } else if (integer && k[0] === "Date") x = `(${x} - DATE '1970-01-01')` + if (integer && k[0] === "Timestamp") x = `CAST(EXTRACT(EPOCH FROM ${x}) * 1000000 AS BIGINT)` + else if (integer && k[0] === "Date") x = `(${x} - DATE '1970-01-01')` else if (integer && k[0] === "Duration") x = `CAST(EXTRACT(EPOCH FROM ${x}) * 1000000 AS BIGINT)` else if (integer && (k[0] === "Decimal" || k[0] === "Float")) x = `TRUNC(${x})` - return d === "duckdb" ? `TRY_CAST(${x} AS ${target})` : `CAST(${x} AS ${target})` + return `CAST(${x} AS ${target})` } case "length": return `LENGTH(${a[0]})` @@ -431,9 +403,7 @@ const call = (node: Plan.Call, scope: Scope): string => { const fn = node.fn === "cumMin" ? "MIN" : "MAX" // Floats: NaN is skipped (it stays only while no other value was seen). const value = k[0] === "Float" - ? `COALESCE(${fn}(CASE WHEN ${isNaNSql(a[0], d)} THEN NULL ELSE ${a[0]} END) ${window(scope, running)}, ${ - a[0] - })` + ? `COALESCE(${fn}(CASE WHEN ${isNaNSql(a[0])} THEN NULL ELSE ${a[0]} END) ${window(scope, running)}, ${a[0]})` : `${fn}(${a[0]}) ${window(scope, running)}` return `(CASE WHEN ${a[0]} IS NULL THEN NULL ELSE ${value} END)` } @@ -474,7 +444,7 @@ const call = (node: Plan.Call, scope: Scope): string => { const enough = `COUNT(${a[0]}) ${window(scope, frame)} >= ${o.minPeriods ?? o.window ?? 1}` // Floats: a NaN in the window makes the minimum or maximum NaN. const value = k[0] === "Float" && (fn === "MIN" || fn === "MAX") - ? `CASE WHEN MAX(CASE WHEN ${isNaNSql(a[0], d)} THEN 1 ELSE 0 END) ${ + ? `CASE WHEN MAX(CASE WHEN ${isNaNSql(a[0])} THEN 1 ELSE 0 END) ${ window(scope, frame) } = 1 THEN ${NAN} ELSE ${fn}(${a[0]}) ${window(scope, frame)} END` : `${fn}(${a[0]}) ${window(scope, frame)}` @@ -495,7 +465,7 @@ const call = (node: Plan.Call, scope: Scope): string => { case "ordinalDay": return `CAST(EXTRACT(DOY FROM ${a[0]}) AS INTEGER)` case "truncate": { - const unit = truncateUnits[o.every ?? ""] ?? unsupported(`truncate to ${o.every}`, d) + const unit = truncateUnits[o.every ?? ""] ?? unsupported(`truncate to ${o.every}`) return k[0] === "Date" ? `CAST(DATE_TRUNC('${unit}', ${a[0]}) AS DATE)` : `DATE_TRUNC('${unit}', ${a[0]})` } case "totalDays": @@ -505,36 +475,21 @@ const call = (node: Plan.Call, scope: Scope): string => { case "totalMilliseconds": return `CAST(TRUNC(EXTRACT(EPOCH FROM ${a[0]}) / ${totalSeconds[node.fn]}) AS BIGINT)` case "listLength": - return d === "duckdb" - ? `CAST(LEN(${a[0]}) AS BIGINT)` - : `COALESCE(CARDINALITY(${a[0]}), CASE WHEN ${a[0]} IS NULL THEN NULL ELSE 0 END)` + return `COALESCE(CARDINALITY(${a[0]}), CASE WHEN ${a[0]} IS NULL THEN NULL ELSE 0 END)` case "listGet": { const i = o.index ?? 0 - return d === "duckdb" - ? `${a[0]}[${i >= 0 ? i + 1 : i}]` - : `${a[0]}[${i >= 0 ? `${i + 1}` : `CARDINALITY(${a[0]}) + ${i + 1}`}]` + return `${a[0]}[${i >= 0 ? `${i + 1}` : `CARDINALITY(${a[0]}) + ${i + 1}`}]` } case "listSum": case "listMean": case "listMin": - case "listMax": { - if (d !== "duckdb") return unsupported(node.fn, d) - const fn = { listSum: "LIST_SUM", listMean: "LIST_AVG", listMin: "LIST_MIN", listMax: "LIST_MAX" }[node.fn] - return node.fn === "listSum" - ? `(CASE WHEN ${a[0]} IS NULL THEN NULL ELSE COALESCE(${fn}(${a[0]}), 0) END)` - : `${fn}(${a[0]})` - } + case "listMax": + case "struct": + return unsupported(node.fn) case "listContains": - return d === "duckdb" - ? `LIST_CONTAINS(${a[0]}, ${a[1]})` - : `(CASE WHEN ${a[0]} IS NULL THEN NULL ELSE ${a[1]} = ANY(${a[0]}) END)` + return `(CASE WHEN ${a[0]} IS NULL THEN NULL ELSE ${a[1]} = ANY(${a[0]}) END)` case "field": return `(${a[0]}).${quote(o.name ?? "")}` - case "struct": { - if (d !== "duckdb") return unsupported("struct", d) - const names = o.names ?? [] - return `STRUCT_PACK(${a.map((x, i) => `${quote(names[i])} := ${x}`).join(", ")})` - } } } @@ -543,7 +498,7 @@ const expression = (node: Plan.Expression, scope: Scope): string => { case "Column": return quote(node.name) case "Literal": - return literal(node.value, node.type, scope.compiler.dialect) + return literal(node.value, node.type) case "Call": return call(node, scope) case "AggregateCall": @@ -568,24 +523,24 @@ const unsupportedNodes: ReadonlySet = new Set(["Pivot", "Unpi * Runs the plan through the in-memory engine on zero-row inputs (tables as * their declared columns), recording every node's frame. */ -const typed = (node: Plan.Node, frames: Map, dialect: Dialect): F.Frame => { +const typed = (node: Plan.Node, frames: Map): F.Frame => { const known = frames.get(node) if (known !== undefined) return known let frame: F.Frame - if (unsupportedNodes.has(node._tag)) return unsupported(node._tag, dialect) + if (unsupportedNodes.has(node._tag)) return unsupported(node._tag) if (node._tag === "Table") { const columns = node.columns ?? V.fail( `the SQL engine needs the columns of table ${JSON.stringify(node.name)}: use Query.table with a schema` ) frame = F.fromVectors( columns.map((c) => c.name), - columns.map((c) => K.nullsOf(tableType(node.name, c, dialect), 0)), + columns.map((c) => K.nullsOf(tableType(node.name, c), 0)), 0 ) } else if (node._tag === "Values") { frame = F.slice(node.frame as unknown as F.Frame, 0, 0) } else { - const inputs = Q.children(node as Q.Node).map((n) => typed(n as Plan.Node, frames, dialect)) + const inputs = Q.children(node as Q.Node).map((n) => typed(n as Plan.Node, frames)) frame = Q.step(node as Q.Node, inputs) } frames.set(node, frame) @@ -606,7 +561,6 @@ const scopeOf = (node: Plan.Node, compiler: Compiler): Scope => ({ }) const values = (frame: F.Frame, c: Compiler): string => { - const d = c.dialect const columns = frame.columns.map((col) => { const type = col.type const vector = col.values() @@ -621,24 +575,24 @@ const values = (frame: F.Frame, c: Compiler): string => { case "Date": return literal(type.unit === "day" ? raw[i] : floorDiv(raw[i] as bigint, BigInt(86_400_000)), { _tag: "Date" - }, d) + }) case "Timestamp": - return literal(raw[i], { _tag: "Timestamp", unit: type.unit, timezone: type.timezone }, d) + return literal(raw[i], { _tag: "Timestamp", unit: type.unit, timezone: type.timezone }) case "Duration": - return literal(raw[i], { _tag: "Duration", unit: type.unit }, d) + return literal(raw[i], { _tag: "Duration", unit: type.unit }) case "Decimal": - return literal(raw[i], { _tag: "Decimal", precision: type.precision, scale: type.scale }, d) + return literal(raw[i], { _tag: "Decimal", precision: type.precision, scale: type.scale }) default: { const v = values![i] - return typeof v === "bigint" ? String(v) : literal(v, undefined, d) + return typeof v === "bigint" ? String(v) : literal(v, undefined) } } } - const sql = sqlType(type, d) + const sql = sqlType(type) return { name: quote(col.name), cell: (i: number) => `CAST(${V.isValid(vector, i) ? value(i) : "NULL"} AS ${sql})` } }) if (frame.length === 0) { - const nulls = frame.columns.map((col, i) => `CAST(NULL AS ${sqlType(col.type, d)}) AS ${columns[i].name}`) + const nulls = frame.columns.map((col, i) => `CAST(NULL AS ${sqlType(col.type)}) AS ${columns[i].name}`) return `SELECT ${[...nulls, `CAST(0 AS BIGINT) AS ${c.row}`].join(", ")} WHERE FALSE` } const tuples = Array.from( @@ -774,7 +728,7 @@ const relation = (node: Plan.Node, c: Compiler): string => { case "Unpivot": case "Explode": case "Unnest": - return unsupported(node._tag, c.dialect) + return unsupported(node._tag) } } @@ -783,20 +737,14 @@ const relation = (node: Plan.Node, c: Compiler): string => { // ----------------------------------------------------------------------------- /** A temporal value as an integer in `unit` since the epoch (or as a length). */ -const epoch = (x: string, type: S.Timestamp | S.Duration, d: Dialect): string => { - if (d === "duckdb" && type._tag === "Timestamp") { - return type.unit === "second" ? - `CAST(FLOOR(EPOCH_MS(${x}) / 1000.0) AS BIGINT)` : - `EPOCH_${({ millisecond: "MS", microsecond: "US", nanosecond: "NS" } as const)[type.unit]}(${x})` - } - return `CAST(FLOOR(EXTRACT(EPOCH FROM ${x}) * ${perSecond[type.unit]}) AS BIGINT)` -} +const epoch = (x: string, type: S.Timestamp | S.Duration): string => + `CAST(FLOOR(EXTRACT(EPOCH FROM ${x}) * ${perSecond[type.unit]}) AS BIGINT)` const scalars: ReadonlySet = new Set(["Null", "Bool", "Int", "Float", "Decimal", "Utf8"]) /** SQL that renders a result column of `type` in the form `reader` reads. */ -const output = (x: string, type: S.DataType, d: Dialect): string => { - const text = (y: string) => `CAST(${y} AS ${d === "duckdb" ? "VARCHAR" : "TEXT"})` +const output = (x: string, type: S.DataType): string => { + const text = (y: string) => `CAST(${y} AS TEXT)` switch (type._tag) { case "Null": return "NULL" @@ -805,18 +753,18 @@ const output = (x: string, type: S.DataType, d: Dialect): string => { case "Float": case "Decimal": case "Utf8": - return text(`CAST(${x} AS ${sqlType(type, d)})`) + return text(`CAST(${x} AS ${sqlType(type)})`) case "Date": return text(`(CAST(${x} AS DATE) - DATE '1970-01-01')${type.unit === "day" ? "" : " * 86400000"}`) case "Timestamp": case "Duration": - return text(epoch(x, type, d)) + return text(epoch(x, type)) case "Binary": return x case "List": - if (scalars.has(type.child.type._tag)) return `CAST(${x} AS ${d === "duckdb" ? "VARCHAR" : "TEXT"}[])` + if (scalars.has(type.child.type._tag)) return `CAST(${x} AS TEXT[])` } - return unsupported(`results of type ${D.describe(type)}`, d) + return unsupported(`results of type ${D.describe(type)}`) } const number = (s: string): number => { @@ -857,17 +805,17 @@ export interface Compiled { } /** A plan as one SQL query, its rows in the plan's row order. @internal */ -export const compile = (plan: Plan.Node, dialect: Dialect): Compiled => { +export const compile = (plan: Plan.Node): Compiled => { const frames = new Map() - const result = typed(plan, frames, dialect) + const result = typed(plan, frames) const used = new Set() for (const frame of frames.values()) for (const col of frame.columns) used.add(col.name) let prefix = "__" while ([...used].some((name) => name.startsWith(prefix))) prefix += "_" const hidden = (name: string) => quote(prefix + name) - const compiler: Compiler = { dialect, frames, hidden, row: hidden("row") } + const compiler: Compiler = { frames, hidden, row: hidden("row") } const sql = relation(plan, compiler) - const columns = result.columns.map((col) => `${output(quote(col.name), col.type, dialect)} AS ${quote(col.name)}`) + const columns = result.columns.map((col) => `${output(quote(col.name), col.type)} AS ${quote(col.name)}`) return { sql: `SELECT ${columns.length === 0 ? "1" : columns.join(", ")} FROM (${sql}) AS t ORDER BY ${compiler.row}`, schema: S.make(result.columns.map((col) => S.field(col.name, col.type, { nullable: true }))) diff --git a/packages/sql/pglite/test/DataFrameSql.test.ts b/packages/sql/pglite/test/DataFrameSql.test.ts index 8a627877aa5..eecb5d78e89 100644 --- a/packages/sql/pglite/test/DataFrameSql.test.ts +++ b/packages/sql/pglite/test/DataFrameSql.test.ts @@ -62,8 +62,7 @@ const close = (a: unknown, b: unknown) => const types = (df: DataFrame.DataFrame) => DataFrame.toArrowTable(df).schema.fields.map((f) => ({ name: f.name, type: f.type })) -const sql = Sql.layer({ dialect: "postgres" }) -const onSql = (query: Query.Query) => query.pipe(Effect.provide(sql)) +const onSql = (query: Query.Query) => query.pipe(Effect.provide(Sql.layer)) const rowsOnSql = (query: Query.Query) => Effect.map(onSql(query), DataFrame.toRows) const xs = Query.from(DataFrame.fromColumns({ x: [1, 2, 3] })) @@ -279,7 +278,7 @@ describe("effect/dataframe on PostgreSQL (PGlite)", { concurrent: false }, () => it.effect("explain shows the SQL", () => Effect.gen(function*() { const text = yield* Query.explain(table.pipe(Query.head(1))).pipe( - Effect.provide(Sql.layer({ dialect: "postgres" })) + Effect.provide(Sql.layer) ) assert.include(text, `FROM "sales"`) })) From 44e0e2c09ecbc9d22774ccdbbc3e6c98fd787737 Mon Sep 17 00:00:00 2001 From: Effect Bot Date: Sun, 15 Nov 2026 12:32:32 +0000 Subject: [PATCH 19/88] Check SQL tables against their declared columns and schema --- packages/effect/src/dataframe/Sql.ts | 49 ++++++++++++--- .../effect/src/dataframe/internal/schema.ts | 23 ++++++-- packages/effect/src/dataframe/internal/sql.ts | 59 ++++++++++++++++++- packages/sql/pglite/test/DataFrameSql.test.ts | 31 ++++++++++ 4 files changed, 146 insertions(+), 16 deletions(-) diff --git a/packages/effect/src/dataframe/Sql.ts b/packages/effect/src/dataframe/Sql.ts index 110df5aaea3..e4c8073140b 100644 --- a/packages/effect/src/dataframe/Sql.ts +++ b/packages/effect/src/dataframe/Sql.ts @@ -36,9 +36,10 @@ * plan (see {@link Compiled}). Values follow the database's semantics where * they differ from the in-memory engine's: integer overflow raises an error * instead of wrapping, division by zero follows the database, and string - * order follows its collation. Pivots, unpivots, - * `explode` and `unnest` are not supported yet; tables need declared - * columns (`Query.table` with a schema). + * order follows its collation. Pivots, unpivots, `explode` and `unnest` + * are not supported yet; tables need declared columns (`Query.table` with a + * schema), and {@link engine} checks that tables satisfy them, as the + * in-memory engine does. * * @stability experimental * @since 4.1.5 @@ -67,6 +68,11 @@ import type * as Plan from "./Plan.ts" * type, so its raw rows are meant for that decoding rather than for direct * use. * + * **Gotchas** + * + * The query doesn't check that the tables satisfy their declared columns and + * schemas: {@link engine} runs those checks as separate queries first. + * * @stability experimental * @category models * @since 4.1.5 @@ -93,9 +99,12 @@ const toError = (cause: unknown) => { * @category compiling * @since 4.1.5 */ -export const compile = (plan: Plan.Node): Result.Result => { +export const compile = (plan: Plan.Node): Result.Result => + Result.map(attempt(() => internal.compile(plan)), ({ schema, sql }) => ({ sql, schema })) + +const attempt = (f: () => A): Result.Result => { try { - return Result.succeed(internal.compile(plan)) + return Result.succeed(f()) } catch (cause) { if (cause instanceof V.DataFrameDefect) return Result.fail(toError(cause)) throw cause @@ -106,17 +115,38 @@ export const compile = (plan: Plan.Node): Result.Result = Effect.gen(function*() { const sql = yield* SqlClient - const compiled = (plan: Plan.Node) => Effect.fromResult(compile(plan)) + const compiled = (plan: Plan.Node) => Effect.fromResult(attempt(() => internal.compile(plan))) + const run = (query: string) => sql.unsafe(query).values.pipe(Effect.mapError(toError)) const collect = (plan: Plan.Node): Effect.Effect, DataFrameError> => Effect.gen(function*() { const query = yield* compiled(plan) - const rows = yield* sql.unsafe(query.sql).values.pipe(Effect.mapError(toError)) + for (const check of query.checks) { + const rows = yield* run(check.sql) + yield* Effect.fromResult(attempt(() => check.verify(rows))) + } + const rows = yield* run(query.sql) return yield* Effect.try({ try: () => internal.decode(query.schema, rows) as unknown as DataFrame, catch: toError @@ -126,13 +156,14 @@ export const engine: Effect.Effect = Effect.ge name: "sql (postgres)", collect, stream: (plan) => Stream.fromEffect(collect(plan)), - explain: (plan) => Effect.map(compiled(plan), (c) => c.sql) + explain: (plan) => Effect.map(compiled(plan), (c) => [...c.checks.map((check) => check.sql), c.sql].join(";\n")) } satisfies Engine.Service }) /** * A layer that runs queries on the database of the `SqlClient` in the - * context. + * context, with {@link engine} (tables are checked against their schemas + * first). * * @stability experimental * @category layers diff --git a/packages/effect/src/dataframe/internal/schema.ts b/packages/effect/src/dataframe/internal/schema.ts index d29f79ebcdd..a29c154a187 100644 --- a/packages/effect/src/dataframe/internal/schema.ts +++ b/packages/effect/src/dataframe/internal/schema.ts @@ -365,9 +365,24 @@ const valuesCheck = (ast: AST.AST): ((value: unknown) => boolean) | undefined => } } +/** + * The fields of a struct schema whose values it constrains beyond their + * column type, each with its test for those values. + * + * @internal + */ +export const valueChecks = ( + schema: { readonly ast: AST.AST } +): Array boolean]> => + structOf(schema).propertySignatures.flatMap((ps) => { + const test = valuesCheck(ps.type) + return test === undefined ? [] : [[String(ps.name), test] as const] + }) + /** * Checks that a frame has the expected columns and narrows it to them, - * dropping UTC time zones from timestamps. With a `schema`, the values of fields it constrains beyond their column type are + * dropping UTC time zones from timestamps. With a `schema`, the values of + * fields it constrains beyond their column type are * checked against it too. * * @internal @@ -388,10 +403,8 @@ export const conform = ( frame.length ) if (schema !== undefined) { - for (const ps of structOf(schema).propertySignatures) { - const test = valuesCheck(ps.type) - const name = String(ps.name) - if (test !== undefined && !F.toArray(narrowed, name).every(test)) { + for (const [name, test] of valueChecks(schema)) { + if (!F.toArray(narrowed, name).every(test)) { return V.fail(`column ${JSON.stringify(name)} has values its schema rejects`) } } diff --git a/packages/effect/src/dataframe/internal/sql.ts b/packages/effect/src/dataframe/internal/sql.ts index 7e87b96d433..e217c2b60ad 100644 --- a/packages/effect/src/dataframe/internal/sql.ts +++ b/packages/effect/src/dataframe/internal/sql.ts @@ -30,6 +30,7 @@ import * as E from "./expr.ts" import * as F from "./frame.ts" import * as K from "./kernels.ts" import * as Q from "./query.ts" +import * as Schema_ from "./schema.ts" import { civil } from "./temporal.ts" import * as V from "./vector.ts" @@ -798,13 +799,66 @@ const reader = (type: S.DataType): (value: unknown) => unknown => { } } +// ----------------------------------------------------------------------------- +// Table checks +// ----------------------------------------------------------------------------- + +/** + * A query over a table, and a test of its rows that fails when the table + * breaks its declared columns or schema. + * + * @internal + */ +export interface Check { + readonly sql: string + readonly verify: (rows: ReadonlyArray>) => void +} + +/** + * The checks of a table that its result's casts to the column types don't + * make: no nulls in non-nullable columns, and the values of fields its schema + * constrains (literals, enums, refinements) accepted by the schema, tested + * on each such column's distinct values. + */ +const tableChecks = (table: Plan.Table, frame: F.Frame): Array => { + const from = quote(table.name) + const checks: Array = [] + const required = (table.columns ?? []).filter((c) => !c.nullable) + if (required.length > 0) { + const read = reader(S.bool) + checks.push({ + sql: `SELECT ${ + required.map((c) => output(`EXISTS (SELECT 1 FROM ${from} WHERE ${quote(c.name)} IS NULL)`, S.bool)).join(", ") + }`, + verify: (rows) => { + const missing = required.find((_, i) => read(rows[0][i])) + if (missing !== undefined) V.fail(`column ${JSON.stringify(missing.name)} has missing values`) + } + }) + } + for (const [name, test] of table.schema === undefined ? [] : Schema_.valueChecks(table.schema)) { + const type = frame.columns.find((c) => c.name === name)!.type + checks.push({ + sql: `SELECT DISTINCT ${output(quote(name), type)} FROM ${from}`, + verify: (rows) => { + if (!F.toArray(decode(S.make([S.field(name, type, { nullable: true })]), rows), name).every(test)) { + V.fail(`column ${JSON.stringify(name)} has values its schema rejects`) + } + } + }) + } + return checks +} + /** @internal */ export interface Compiled { readonly sql: string readonly schema: S.Schema + /** The checks of the plan's tables, to run before the query. */ + readonly checks: ReadonlyArray } -/** A plan as one SQL query, its rows in the plan's row order. @internal */ +/** A plan as one SQL query, its rows in the plan's row order, and the checks of its tables. @internal */ export const compile = (plan: Plan.Node): Compiled => { const frames = new Map() const result = typed(plan, frames) @@ -818,7 +872,8 @@ export const compile = (plan: Plan.Node): Compiled => { const columns = result.columns.map((col) => `${output(quote(col.name), col.type)} AS ${quote(col.name)}`) return { sql: `SELECT ${columns.length === 0 ? "1" : columns.join(", ")} FROM (${sql}) AS t ORDER BY ${compiler.row}`, - schema: S.make(result.columns.map((col) => S.field(col.name, col.type, { nullable: true }))) + schema: S.make(result.columns.map((col) => S.field(col.name, col.type, { nullable: true }))), + checks: [...frames].flatMap(([node, frame]) => node._tag === "Table" ? tableChecks(node, frame) : []) } } diff --git a/packages/sql/pglite/test/DataFrameSql.test.ts b/packages/sql/pglite/test/DataFrameSql.test.ts index eecb5d78e89..a9e3f51e23b 100644 --- a/packages/sql/pglite/test/DataFrameSql.test.ts +++ b/packages/sql/pglite/test/DataFrameSql.test.ts @@ -36,6 +36,7 @@ const Setup = Layer.effectDiscard( const sql = yield* SqlClient yield* sql`DROP TABLE IF EXISTS sales` yield* sql`DROP TABLE IF EXISTS regions` + yield* sql`DROP TABLE IF EXISTS contract` yield* sql`CREATE TABLE sales (id DOUBLE PRECISION, region TEXT, units DOUBLE PRECISION, price DOUBLE PRECISION, at TIMESTAMP)` yield* sql`CREATE TABLE regions (region TEXT, manager TEXT)` for (const s of sales) { @@ -44,6 +45,8 @@ const Setup = Layer.effectDiscard( })` } for (const r of regions) yield* sql`INSERT INTO regions VALUES (${r.region}, ${r.manager})` + yield* sql`CREATE TABLE contract (s TEXT, n DOUBLE PRECISION)` + yield* sql`INSERT INTO contract VALUES ('bad', NULL), ('good', 1)` }) ) @@ -213,6 +216,34 @@ describe("effect/dataframe on PostgreSQL (PGlite)", { concurrent: false }, () => assert.include(error.message, "integer out of range") })) + it.effect("tables must satisfy their schema, as in memory", () => + Effect.gen(function*() { + const contract = (schema: S) => + Query.table("contract", schema) + const rejects = (query: Query.Query) => Effect.map(Effect.flip(onSql(query)), (e) => e.message) + assert.strictEqual( + yield* rejects(contract(Schema.Struct({ s: Schema.Literal("good") }))), + `column "s" has values its schema rejects` + ) + assert.strictEqual( + yield* rejects(contract(Schema.Struct({ n: Schema.Number }))), + `column "n" has missing values` + ) + // Columns the query doesn't read are checked too. + assert.strictEqual( + yield* rejects( + contract(Schema.Struct({ s: Schema.Literal("good"), n: Schema.NullOr(Schema.Number) })).pipe( + Query.pick("n") + ) + ), + `column "s" has values its schema rejects` + ) + assert.deepStrictEqual( + yield* rowsOnSql(contract(Schema.Struct({ s: Schema.Literals(["bad", "good"]) }))), + [{ s: "bad" }, { s: "good" }] + ) + })) + it.effect("filters, derives and sorts", () => both(table.pipe( Query.filter((c) => Expr.and(Expr.isNotNull(c.units), Expr.gt(c.price, 20))), From b02df05eb9495d21b62eb5afd60a3f4790d96e14 Mon Sep 17 00:00:00 2001 From: Effect Bot Date: Sun, 15 Nov 2026 12:37:04 +0000 Subject: [PATCH 20/88] Lower SQL temporal casts through their units --- packages/effect/src/dataframe/Sql.ts | 11 ++- packages/effect/src/dataframe/internal/sql.ts | 98 ++++++++++++++----- packages/sql/pglite/test/DataFrameSql.test.ts | 62 ++++++++++++ 3 files changed, 142 insertions(+), 29 deletions(-) diff --git a/packages/effect/src/dataframe/Sql.ts b/packages/effect/src/dataframe/Sql.ts index e4c8073140b..7a192b27819 100644 --- a/packages/effect/src/dataframe/Sql.ts +++ b/packages/effect/src/dataframe/Sql.ts @@ -35,11 +35,12 @@ * Results have the column types the in-memory engine gives for the same * plan (see {@link Compiled}). Values follow the database's semantics where * they differ from the in-memory engine's: integer overflow raises an error - * instead of wrapping, division by zero follows the database, and string - * order follows its collation. Pivots, unpivots, `explode` and `unnest` - * are not supported yet; tables need declared columns (`Query.table` with a - * schema), and {@link engine} checks that tables satisfy them, as the - * in-memory engine does. + * instead of wrapping, division by zero follows the database, string order + * follows its collation, and timestamps and durations keep PostgreSQL's + * microseconds (finer values round down). Pivots, unpivots, `explode` and + * `unnest` are not supported yet; tables need declared columns + * (`Query.table` with a schema), and {@link engine} checks that tables + * satisfy them, as the in-memory engine does. * * @stability experimental * @since 4.1.5 diff --git a/packages/effect/src/dataframe/internal/sql.ts b/packages/effect/src/dataframe/internal/sql.ts index e217c2b60ad..2d334b5dc67 100644 --- a/packages/effect/src/dataframe/internal/sql.ts +++ b/packages/effect/src/dataframe/internal/sql.ts @@ -150,10 +150,8 @@ const literal = (value: unknown, type: Plan.LiteralType | undefined): string => return type.timezone === undefined ? `TIMESTAMP '${timestampText(big, type.unit)}'` : `TIMESTAMPTZ '${timestampText(big, type.unit)}+00'` - case "Duration": { - const micros = big * BigInt(1_000_000) / perSecond[type.unit] - return `(${micros} * INTERVAL '1 microsecond')` - } + case "Duration": + return fromInteger(`CAST(${big} AS BIGINT)`, S.duration(type.unit)) case "Decimal": { const negative = big < BigInt(0) const digits = (negative ? -big : big).toString().padStart(type.scale + 1, "0") @@ -179,6 +177,75 @@ const literal = (value: unknown, type: Plan.LiteralType | undefined): string => } } +// ----------------------------------------------------------------------------- +// Temporal values and casts +// ----------------------------------------------------------------------------- + +type Temporal = S.Date | S.Timestamp | S.Duration + +const isTemporal = (t: S.DataType): t is Temporal => + t._tag === "Date" || t._tag === "Timestamp" || t._tag === "Duration" + +/** The integer type of a temporal type's values. */ +const integerOf = (t: Temporal): S.Int => (t._tag === "Date" && t.unit === "day" ? S.int32 : S.int64) + +/** + * A temporal value as its integer: its `unit`s (its own by default) since + * the epoch, or its length in them, rounded down. + */ +const toInteger = (x: string, type: Temporal, unit?: Plan.TimeUnit): string => { + if (type._tag !== "Date") { + return `CAST(FLOOR(EXTRACT(EPOCH FROM ${x}) * ${perSecond[unit ?? type.unit]}) AS BIGINT)` + } + const days = `(CAST(${x} AS DATE) - DATE '1970-01-01')` + return type.unit === "day" ? days : `(CAST(${days} AS BIGINT) * 86400000)` +} + +/** + * An interval of `micros` (a BIGINT) microseconds. Intervals multiply in + * double precision, so whole hours and the remaining microseconds are + * multiplied separately, each exactly. + */ +const interval = (micros: string): string => + `((${micros}) / 3600000000 * INTERVAL '1 hour' + (${micros}) % 3600000000 * INTERVAL '1 microsecond')` + +/** The temporal value of `type` whose integer (see {@link toInteger}) is `x`, of type {@link integerOf}. */ +const fromInteger = (x: string, type: Temporal): string => { + if (type._tag === "Date") { + return `(DATE '1970-01-01' + CAST(${type.unit === "day" ? x : `FLOOR(${x} / 86400000.0)`} AS INTEGER))` + } + const micros = type.unit === "nanosecond" + ? `CAST(FLOOR(${x} / 1000.0) AS BIGINT)` + : `(${x} * ${perSecond.microsecond / perSecond[type.unit]})` + if (type._tag === "Duration") return interval(micros) + const epoch = type.timezone === undefined ? "TIMESTAMP '1970-01-01 00:00:00'" : "TIMESTAMPTZ '1970-01-01 00:00:00+00'" + return `(${epoch} + ${interval(micros)})` +} + +/** + * `x` converted from `from` to `to` as the engine converts: timestamps + * and durations rescale to the target unit (rounding down), timestamps and + * dates convert at midnight, and other temporal values convert as their + * integers. + */ +const cast = (x: string, from: S.DataType, to: S.DataType): string => { + if (isTemporal(from) && isTemporal(to)) { + if (from._tag !== "Date" && from._tag === to._tag) return fromInteger(toInteger(x, from, to.unit), to) + if (from._tag === "Timestamp" && to._tag === "Date") { + return fromInteger(`CAST(FLOOR(EXTRACT(EPOCH FROM ${x}) / 86400) AS BIGINT)`, S.date("day")) + } + if (from._tag === "Date" && to._tag === "Timestamp") { + return fromInteger(`CAST(${toInteger(x, S.date("day"))} AS BIGINT) * ${BigInt(86400) * perSecond[to.unit]}`, to) + } + return fromInteger(cast(toInteger(x, from), integerOf(from), integerOf(to)), to) + } + if (isTemporal(from)) return cast(toInteger(x, from), integerOf(from), to) + if (isTemporal(to)) return fromInteger(cast(x, from, integerOf(to)), to) + // Floats and decimals truncate towards zero. + const truncated = to._tag === "Int" && (from._tag === "Decimal" || from._tag === "Float") ? `TRUNC(${x})` : x + return `CAST(${truncated} AS ${sqlType(to)})` +} + // ----------------------------------------------------------------------------- // Expressions // ----------------------------------------------------------------------------- @@ -367,20 +434,8 @@ const call = (node: Plan.Call, scope: Scope): string => { return `COALESCE(${a.join(", ")})` case "when": return `(CASE WHEN ${a[0]} THEN ${a[1]} ELSE ${a[2]} END)` - case "cast": { - const to = o.to ?? "" - const target = to === "Decimal" - ? `DECIMAL(${o.precision}, ${o.scale})` - : sqlType(D.castTargets[to] ?? V.fail(`unknown cast target ${to}`)) - const integer = to.startsWith("Int") || to.startsWith("UInt") - let x = a[0] - // Temporal values become their integers; floats and decimals truncate (as the engine does). - if (integer && k[0] === "Timestamp") x = `CAST(EXTRACT(EPOCH FROM ${x}) * 1000000 AS BIGINT)` - else if (integer && k[0] === "Date") x = `(${x} - DATE '1970-01-01')` - else if (integer && k[0] === "Duration") x = `CAST(EXTRACT(EPOCH FROM ${x}) * 1000000 AS BIGINT)` - else if (integer && (k[0] === "Decimal" || k[0] === "Float")) x = `TRUNC(${x})` - return `CAST(${x} AS ${target})` - } + case "cast": + return cast(a[0], typeOf(node.args[0], scope), typeOf(node, scope)) case "length": return `LENGTH(${a[0]})` case "toLowerCase": @@ -737,10 +792,6 @@ const relation = (node: Plan.Node, c: Compiler): string => { // Results // ----------------------------------------------------------------------------- -/** A temporal value as an integer in `unit` since the epoch (or as a length). */ -const epoch = (x: string, type: S.Timestamp | S.Duration): string => - `CAST(FLOOR(EXTRACT(EPOCH FROM ${x}) * ${perSecond[type.unit]}) AS BIGINT)` - const scalars: ReadonlySet = new Set(["Null", "Bool", "Int", "Float", "Decimal", "Utf8"]) /** SQL that renders a result column of `type` in the form `reader` reads. */ @@ -756,10 +807,9 @@ const output = (x: string, type: S.DataType): string => { case "Utf8": return text(`CAST(${x} AS ${sqlType(type)})`) case "Date": - return text(`(CAST(${x} AS DATE) - DATE '1970-01-01')${type.unit === "day" ? "" : " * 86400000"}`) case "Timestamp": case "Duration": - return text(epoch(x, type)) + return text(toInteger(x, type)) case "Binary": return x case "List": diff --git a/packages/sql/pglite/test/DataFrameSql.test.ts b/packages/sql/pglite/test/DataFrameSql.test.ts index a9e3f51e23b..715376bc9d8 100644 --- a/packages/sql/pglite/test/DataFrameSql.test.ts +++ b/packages/sql/pglite/test/DataFrameSql.test.ts @@ -83,6 +83,44 @@ const decimals = arrow<{ d: BigDecimal.BigDecimal | null }>( s === null ? null : BigDecimal.fromStringUnsafe(s) )] ) +const units = { s: "second", ms: "millisecond", us: "microsecond", ns: "nanosecond" } as const +const unitNames = ["s", "ms", "us", "ns"] as const +/** One column `v` of `type`, its integers microsecond multiples in every unit (PostgreSQL keeps microseconds). */ +const temporal = (type: ArrowSchema.DataType, values: ReadonlyArray = [1_001_000n, -1_500_000n, null]) => + arrow<{ v: unknown }>([ArrowSchema.field("v", type)], [values]) +/** Casts in both directions between temporal types of every unit and integers. */ +const temporalCasts = unitNames.flatMap((u) => [ + [ + `casts of Timestamp[${u}]`, + temporal(ArrowSchema.timestamp(units[u])).pipe(Query.select((c) => ({ + int: Expr.cast(c.v, "Int64"), + date: Expr.cast(c.v, "Date"), + duration: Expr.cast(c.v, `Duration[${u}]`), + s: Expr.cast(c.v, "Timestamp[s]"), + ms: Expr.cast(c.v, "Timestamp[ms]"), + us: Expr.cast(c.v, "Timestamp[us]"), + ns: Expr.cast(c.v, "Timestamp[ns]") + }))) + ] as const, + [ + `casts of Duration[${u}]`, + temporal(ArrowSchema.duration(units[u])).pipe(Query.select((c) => ({ + int: Expr.cast(c.v, "Int64"), + timestamp: Expr.cast(c.v, `Timestamp[${u}]`), + s: Expr.cast(c.v, "Duration[s]"), + ms: Expr.cast(c.v, "Duration[ms]"), + us: Expr.cast(c.v, "Duration[us]"), + ns: Expr.cast(c.v, "Duration[ns]") + }))) + ] as const, + [ + `casts of integers to Timestamp[${u}] and Duration[${u}]`, + temporal(ArrowSchema.int64).pipe(Query.select((c) => ({ + timestamp: Expr.cast(c.v, `Timestamp[${u}]`), + duration: Expr.cast(c.v, `Duration[${u}]`) + }))) + ] as const +]) /** Queries whose SQL result must equal the in-memory result, column types included. */ const cases: ReadonlyArray]> = [ @@ -125,6 +163,30 @@ const cases: ReadonlyArray]> = [ [BigDecimal.fromStringUnsafe("12.34"), BigDecimal.fromStringUnsafe("-0.05"), null] ]).pipe(Query.withColumns((c) => ({ s: Expr.totalSeconds(c.d), year: Expr.year(c.t) }))) ], + ...temporalCasts, + [ + "casts of integers beyond double precision to Timestamp[us] and Duration[us]", + temporal(ArrowSchema.int64, [253_402_300_799_999_999n, -62_135_596_800_000_001n, 9_007_199_254_740_993n]).pipe( + Query.select((c) => ({ timestamp: Expr.cast(c.v, "Timestamp[us]"), duration: Expr.cast(c.v, "Duration[us]") })) + ) + ], + [ + "casts of zoned timestamps", + temporal(ArrowSchema.timestamp("microsecond", "UTC")).pipe( + Query.select((c) => ({ int: Expr.cast(c.v, "Int64"), s: Expr.cast(c.v, "Timestamp[s]") })) + ) + ], + [ + "casts of dates", + temporal(ArrowSchema.date("day"), [19_000, -1, null]).pipe(Query.select((c) => ({ + int: Expr.cast(c.v, "Int32"), + long: Expr.cast(c.v, "Int64"), + text: Expr.cast(c.v, "Utf8"), + s: Expr.cast(c.v, "Timestamp[s]"), + ns: Expr.cast(c.v, "Timestamp[ns]"), + date: Expr.cast(Expr.cast(c.v, "Int32"), "Date") + }))) + ], [ "durations", Query.from(DataFrame.fromColumns({ d: [Duration.seconds(30), Duration.seconds(90)] })).pipe( From 26af0e3dca80068ba3a4e3b8a6a67a864d2fa2e4 Mon Sep 17 00:00:00 2001 From: Effect Bot Date: Sun, 15 Nov 2026 12:38:13 +0000 Subject: [PATCH 21/88] Use boolean aggregates for SQL boolean minimum and maximum --- packages/effect/src/dataframe/internal/sql.ts | 8 ++++++-- packages/sql/pglite/test/DataFrameSql.test.ts | 14 ++++++++++++++ 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/packages/effect/src/dataframe/internal/sql.ts b/packages/effect/src/dataframe/internal/sql.ts index 2d334b5dc67..4b53ceb45a5 100644 --- a/packages/effect/src/dataframe/internal/sql.ts +++ b/packages/effect/src/dataframe/internal/sql.ts @@ -308,6 +308,10 @@ const round = (x: string, type: S.DataType, result: S.DataType, decimals: number } } +/** The SQL aggregate for the minimum or maximum of a column of `type` (booleans have their own). */ +const extreme = (fn: "min" | "max", type: S.DataType["_tag"] | undefined): string => + type === "Bool" ? (fn === "min" ? "BOOL_AND" : "BOOL_OR") : fn.toUpperCase() + const aggregate = (node: Plan.AggregateCall, scope: Scope, over: string | undefined): string => { const row = scope.compiler.row const inner = { ...scope, partition: undefined } @@ -324,7 +328,7 @@ const aggregate = (node: Plan.AggregateCall, scope: Scope, over: string | undefi return `AVG(${x})${o}` case "min": case "max": { - const fn = node.fn.toUpperCase() + const fn = extreme(node.fn, type) // Floats: NaN is skipped unless every value is NaN. return type === "Float" ? `COALESCE(${fn}(CASE WHEN ${isNaNSql(x)} THEN NULL ELSE ${x} END)${o}, ${fn}(${x})${o})` @@ -456,7 +460,7 @@ const call = (node: Plan.Call, scope: Scope): string => { } END)` case "cumMin": case "cumMax": { - const fn = node.fn === "cumMin" ? "MIN" : "MAX" + const fn = extreme(node.fn === "cumMin" ? "min" : "max", k[0]) // Floats: NaN is skipped (it stays only while no other value was seen). const value = k[0] === "Float" ? `COALESCE(${fn}(CASE WHEN ${isNaNSql(a[0])} THEN NULL ELSE ${a[0]} END) ${window(scope, running)}, ${a[0]})` diff --git a/packages/sql/pglite/test/DataFrameSql.test.ts b/packages/sql/pglite/test/DataFrameSql.test.ts index 715376bc9d8..eee90383d19 100644 --- a/packages/sql/pglite/test/DataFrameSql.test.ts +++ b/packages/sql/pglite/test/DataFrameSql.test.ts @@ -70,6 +70,7 @@ const rowsOnSql = (query: Query.Query) => Effect.map(onSql(query), DataFra const xs = Query.from(DataFrame.fromColumns({ x: [1, 2, 3] })) const ints = Query.from(DataFrame.fromColumns({ x: Int32Array.of(1, 2, 3) })) +const bools = Query.from(DataFrame.fromColumns({ b: [true, null, false, true] })) const none = ints.pipe(Query.filter((c) => Expr.gt(c.x, 9))) /** A query over a frame with exactly the given Arrow columns. */ const arrow = (fields: ReadonlyArray, columns: ReadonlyArray>) => @@ -143,6 +144,19 @@ const cases: ReadonlyArray]> = [ "boolean sum aggregate", Query.from(DataFrame.fromColumns({ b: [true, false, true] })).pipe(Query.aggregate((c) => ({ s: Expr.sum(c.b) }))) ], + [ + "boolean minimum and maximum", + bools.pipe(Query.aggregate((c) => ({ min: Expr.min(c.b), max: Expr.max(c.b) }))) + ], + [ + "boolean minimum and maximum over windows", + bools.pipe(Query.select((c) => ({ + min: Expr.over(Expr.min(c.b), []), + max: Expr.over(Expr.max(c.b), []), + cumMin: Expr.cumMin(c.b), + cumMax: Expr.cumMax(c.b) + }))) + ], ["filters on a window function", xs.pipe(Query.filter((c) => Expr.gt(Expr.cumSum(c.x), 3)))], ["filters on an aggregate over the input", xs.pipe(Query.filter((c) => Expr.gt(c.x, Expr.over(Expr.mean(c.x), []))))], [ From 8ee9e43e22ca70ec84fca2ad50080599c27c7246 Mon Sep 17 00:00:00 2001 From: Effect Bot Date: Sun, 15 Nov 2026 12:38:55 +0000 Subject: [PATCH 22/88] Keep the first row for SQL unique on no columns --- packages/effect/src/dataframe/internal/sql.ts | 8 +++++--- packages/sql/pglite/test/DataFrameSql.test.ts | 1 + 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/packages/effect/src/dataframe/internal/sql.ts b/packages/effect/src/dataframe/internal/sql.ts index 4b53ceb45a5..68fa6ce3fab 100644 --- a/packages/effect/src/dataframe/internal/sql.ts +++ b/packages/effect/src/dataframe/internal/sql.ts @@ -733,9 +733,11 @@ const relation = (node: Plan.Node, c: Compiler): string => { } case "Unique": { const k = c.hidden("k") - return `SELECT ${keep} FROM (SELECT *, ROW_NUMBER() OVER (PARTITION BY ${ - list(node.columns ?? columns) - } ORDER BY ${row}) AS ${k} FROM ${from("s")}) AS t WHERE ${k} = 1` + const keys = node.columns ?? columns + const partition = keys.length === 0 ? "" : `PARTITION BY ${list(keys)} ` + return `SELECT ${keep} FROM (SELECT *, ROW_NUMBER() OVER (${partition}ORDER BY ${row}) AS ${k} FROM ${ + from("s") + }) AS t WHERE ${k} = 1` } case "DropNulls": { const keys = node.columns ?? columns diff --git a/packages/sql/pglite/test/DataFrameSql.test.ts b/packages/sql/pglite/test/DataFrameSql.test.ts index eee90383d19..a5fbdd33a71 100644 --- a/packages/sql/pglite/test/DataFrameSql.test.ts +++ b/packages/sql/pglite/test/DataFrameSql.test.ts @@ -157,6 +157,7 @@ const cases: ReadonlyArray]> = [ cumMax: Expr.cumMax(c.b) }))) ], + ["unique on no columns keeps the first row", xs.pipe(Query.unique([]))], ["filters on a window function", xs.pipe(Query.filter((c) => Expr.gt(Expr.cumSum(c.x), 3)))], ["filters on an aggregate over the input", xs.pipe(Query.filter((c) => Expr.gt(c.x, Expr.over(Expr.mean(c.x), []))))], [ From 4c15c4e6bdb52c79e278672e6abe0df3d4db93d0 Mon Sep 17 00:00:00 2001 From: Effect Bot Date: Sun, 15 Nov 2026 12:41:52 +0000 Subject: [PATCH 23/88] Compute SQL aggregate arguments per group before aggregating --- packages/effect/src/dataframe/internal/sql.ts | 58 +++++++++++++------ packages/sql/pglite/test/DataFrameSql.test.ts | 20 +++++++ 2 files changed, 59 insertions(+), 19 deletions(-) diff --git a/packages/effect/src/dataframe/internal/sql.ts b/packages/effect/src/dataframe/internal/sql.ts index 68fa6ce3fab..3acf7a9832c 100644 --- a/packages/effect/src/dataframe/internal/sql.ts +++ b/packages/effect/src/dataframe/internal/sql.ts @@ -312,11 +312,18 @@ const round = (x: string, type: S.DataType, result: S.DataType, decimals: number const extreme = (fn: "min" | "max", type: S.DataType["_tag"] | undefined): string => type === "Bool" ? (fn === "min" ? "BOOL_AND" : "BOOL_OR") : fn.toUpperCase() -const aggregate = (node: Plan.AggregateCall, scope: Scope, over: string | undefined): string => { +/** + * The aggregate `node` of `x`, the SQL of its argument (of type `type`); a + * window function when `over` (its window) is given. + */ +const aggregate = ( + node: Plan.AggregateCall, + x: string, + type: S.DataType["_tag"] | undefined, + scope: Scope, + over: string | undefined +): string => { const row = scope.compiler.row - const inner = { ...scope, partition: undefined } - const x = node.arg === undefined ? "" : expression(node.arg, inner) - const type = node.arg === undefined ? undefined : typeOf(node.arg, inner)._tag const o = over === undefined ? "" : ` ${over}` switch (node.fn) { case "count": @@ -561,15 +568,18 @@ const expression = (node: Plan.Expression, scope: Scope): string => { return literal(node.value, node.type) case "Call": return call(node, scope) - case "AggregateCall": - return scope.partition === undefined - ? V.fail(`aggregate ${node.fn} can only be used in agg or over`) - : aggregate(node, scope, `OVER (${partitionBy(scope)})`) - case "Over": - return expression(node.expression, { - ...scope, - partition: node.partitionBy.map((k) => expression(k, { ...scope, partition: undefined })) - }) + case "AggregateCall": { + if (scope.partition === undefined) return V.fail(`aggregate ${node.fn} can only be used in agg or over`) + const inner = { ...scope, partition: undefined } + const x = node.arg === undefined ? "" : expression(node.arg, inner) + const type = node.arg === undefined ? undefined : typeOf(node.arg, inner)._tag + return aggregate(node, x, type, scope, `OVER (${partitionBy(scope)})`) + } + case "Over": { + // Within an enclosing partition (another `over`, or a group), partitions refine it. + const keys = node.partitionBy.map((k) => expression(k, { ...scope, partition: undefined })) + return expression(node.expression, { ...scope, partition: [...(scope.partition ?? []), ...keys] }) + } } } @@ -747,15 +757,25 @@ const relation = (node: Plan.Node, c: Compiler): string => { } case "GroupBy": case "Aggregate": { - const s = scope() const keys = node._tag === "GroupBy" ? node.keys : [] - const aggs = node.aggregates.map((a) => - `${aggregate(a.expression as Plan.AggregateCall, s, undefined)} AS ${quote(a.name)}` + // Arguments are computed first, partitioned by the group: windows in them see only its rows. + const group: Scope = { ...scope(), partition: keys.map(quote) } + const args = node.aggregates.map((a, i) => { + const arg = (a.expression as Plan.AggregateCall).arg + return arg === undefined + ? undefined + : { name: c.hidden(`arg${i}`), sql: expression(arg, group), type: typeOf(arg, group)._tag } + }) + const computed = args.flatMap((x) => x === undefined ? [] : [`, ${x.sql} AS ${x.name}`]) + const aggs = node.aggregates.map((a, i) => + `${aggregate(a.expression as Plan.AggregateCall, args[i]?.name ?? "", args[i]?.type, group, undefined)} AS ${ + quote(a.name) + }` ) const key = keys.length === 0 ? `1 AS ${row}` : `MIN(${row}) AS ${row}` - return `SELECT ${[...keys.map(quote), ...aggs, key].join(", ")} FROM ${from()}${ - keys.length === 0 ? "" : ` GROUP BY ${list(keys)}` - }` + return `SELECT ${[...keys.map(quote), ...aggs, key].join(", ")} FROM (SELECT *${ + computed.join("") + } FROM ${from()}) AS t${keys.length === 0 ? "" : ` GROUP BY ${list(keys)}`}` } case "Join": { const l = relation(node.left, c) diff --git a/packages/sql/pglite/test/DataFrameSql.test.ts b/packages/sql/pglite/test/DataFrameSql.test.ts index a5fbdd33a71..089302014fb 100644 --- a/packages/sql/pglite/test/DataFrameSql.test.ts +++ b/packages/sql/pglite/test/DataFrameSql.test.ts @@ -71,6 +71,9 @@ const rowsOnSql = (query: Query.Query) => Effect.map(onSql(query), DataFra const xs = Query.from(DataFrame.fromColumns({ x: [1, 2, 3] })) const ints = Query.from(DataFrame.fromColumns({ x: Int32Array.of(1, 2, 3) })) const bools = Query.from(DataFrame.fromColumns({ b: [true, null, false, true] })) +const groups = Query.from( + DataFrame.fromColumns({ g: ["a", "b", "a", "b", "a"], h: [1, 1, 2, 2, 1], x: [1, 3, 2, 6, 4] }) +) const none = ints.pipe(Query.filter((c) => Expr.gt(c.x, 9))) /** A query over a frame with exactly the given Arrow columns. */ const arrow = (fields: ReadonlyArray, columns: ReadonlyArray>) => @@ -158,6 +161,23 @@ const cases: ReadonlyArray]> = [ }))) ], ["unique on no columns keeps the first row", xs.pipe(Query.unique([]))], + [ + "windows in aggregate arguments see only their group", + groups.pipe( + Query.groupBy("g"), + Query.agg((c) => ({ + running: Expr.sum(Expr.cumSum(c.x)), + previous: Expr.sum(Expr.shift(c.x)), + rank: Expr.max(Expr.rank(c.x, { method: "min" })), + centered: Expr.max(Expr.subtract(c.x, Expr.over(Expr.mean(c.x), []))), + nested: Expr.sum(Expr.over(Expr.cumSum(c.x), [c.h])) + })) + ) + ], + [ + "windows in aggregate arguments over all rows", + groups.pipe(Query.aggregate((c) => ({ running: Expr.sum(Expr.cumSum(c.x)), first: Expr.first(Expr.shift(c.x)) }))) + ], ["filters on a window function", xs.pipe(Query.filter((c) => Expr.gt(Expr.cumSum(c.x), 3)))], ["filters on an aggregate over the input", xs.pipe(Query.filter((c) => Expr.gt(c.x, Expr.over(Expr.mean(c.x), []))))], [ From 589a25612f00f4dbc5e12f220381a0831d108faa Mon Sep 17 00:00:00 2001 From: Effect Bot Date: Sun, 15 Nov 2026 13:15:31 +0000 Subject: [PATCH 24/88] Declare table column types once and convert every input to them A struct schema plus options.types now fixes each column's declared type, including decimal precision and scale and temporal units (TableColumn gains precision, scale and unit). fromRows, empty, validate, Query.table and every streamed batch convert to it through the same code: exact conversions only, UTC time zones dropped and other zones rejected. Tables without rows get typed empty columns. Value checks now look through union members, suspends, structs and arrays, and recursive schemas fail instead of overflowing. --- packages/effect/src/dataframe/DataFrame.ts | 40 +- packages/effect/src/dataframe/Plan.ts | 38 +- packages/effect/src/dataframe/Query.ts | 22 +- .../effect/src/dataframe/internal/schema.ts | 404 ++++++++++++------ .../effect/src/dataframe/internal/stream.ts | 43 +- .../effect/test/dataframe/DataFrame.test.ts | 86 +++- packages/effect/test/dataframe/Query.test.ts | 5 + packages/effect/test/dataframe/oracle/Plan.ts | 5 + packages/effect/typetest/DataFrame.tst.ts | 14 + 9 files changed, 443 insertions(+), 214 deletions(-) diff --git a/packages/effect/src/dataframe/DataFrame.ts b/packages/effect/src/dataframe/DataFrame.ts index 9ff4510ad4b..4f69943c002 100644 --- a/packages/effect/src/dataframe/DataFrame.ts +++ b/packages/effect/src/dataframe/DataFrame.ts @@ -161,14 +161,18 @@ export const isDataFrame = (u: unknown): u is DataFrame => F.isFrame(u) /** * Builds a frame from rows described by a struct schema. Column types come * from the schema's type side: `String` is Utf8, `Number` is Float64, - * `Boolean` is Bool, `BigInt` is Int64, and `NullOr`/optional fields are - * nullable. `options.types` overrides the Arrow type of individual columns - * (for example `ArrowSchema.int32`); `BigDecimal` fields need one, with - * the decimal's precision and scale. + * `Boolean` is Bool, `BigInt` is Int64, `DateTimeUtc` and `Duration` + * are microsecond timestamps and durations, and `NullOr`/optional fields + * are nullable. `options.types` overrides the Arrow type of individual + * columns (for example `ArrowSchema.int32`); `BigDecimal` fields need + * one, with the decimal's precision and scale. * * **Details** * - * Missing values read back as `null` (see {@link Normalize}). + * Missing values read back as `null` (see {@link Normalize}). An override + * must hold its field's values, as {@link validate} checks columns: a + * `DateTimeUtc` override is a timestamp without a time zone or with a UTC + * one, which the frame drops. * * Throws a {@link DataFrameError} when a value doesn't fit its column, or an * override can't hold its field's values. @@ -372,8 +376,9 @@ export const empty = ( /** * Checks that a frame has the columns of a struct schema, with compatible - * types, and returns it typed (with only the schema's columns, in its - * order). + * types, and returns it typed: with only the schema's columns, in its order, + * converted to the types {@link fromRows} gives them (`options.types` + * overrides some, and `BigDecimal` fields need one). * * **Details** * @@ -381,8 +386,11 @@ export const empty = ( * takes floats and integers below 64 bits, `BigInt` takes 64-bit integers, * `BigDecimal` takes decimals of any precision and scale, `DateTimeUtc` and * `Duration` take timestamps and durations in any unit, and structs and - * arrays need compatible fields and items. Values of fields the schema - * constrains further (literals, enums, refinements) are checked against it. + * arrays need compatible fields and items. Every value must convert to the + * declared type exactly: a decimal with more digits than the declared scale, + * or a timestamp finer than the declared unit, fails. Values of fields the + * schema constrains further (literals, enums, refinements, including inside + * structs, arrays and union members) are checked against it. * * **Gotchas** * @@ -398,15 +406,21 @@ export const empty = ( */ export const validate: { ( - schema: S + schema: S, + options?: { readonly types?: { readonly [K in keyof S["Type"]]?: ArrowSchema.DataType } | undefined } ): (self: DataFrame) => Result.Result>, DataFrameError> ( self: DataFrame, - schema: S + schema: S, + options?: { readonly types?: { readonly [K in keyof S["Type"]]?: ArrowSchema.DataType } | undefined } ): Result.Result>, DataFrameError> -} = dual(2, (self: DataFrame, schema: { readonly ast: any }) => { +} = dual((args) => isDataFrame(args[0]), ( + self: DataFrame, + schema: { readonly ast: any }, + options?: { readonly types?: { readonly [name: string]: ArrowSchema.DataType | undefined } | undefined } +) => { try { - return Result.succeed(wrap(Schema_.conform(frame(self), Schema_.columns(schema), schema))) + return Result.succeed(wrap(Schema_.conform(frame(self), Schema_.declare(schema, options?.types), schema))) } catch (cause) { return Result.fail(fromDefect(cause)) } diff --git a/packages/effect/src/dataframe/Plan.ts b/packages/effect/src/dataframe/Plan.ts index 7169d090673..34a1139ad5b 100644 --- a/packages/effect/src/dataframe/Plan.ts +++ b/packages/effect/src/dataframe/Plan.ts @@ -673,13 +673,19 @@ export const TypeName = Schema.Literals([ ]) /** - * A column a {@link Table} is expected to have. + * A column a {@link Table} is declared to have, and the type it is read as. * * **Details** * * `fields` describes the children of a nested column: the single `item` of - * a `List`, or the fields of a `Struct`. Without it, a nested column's - * children are not checked. + * a `List`, or the fields of a `Struct`. `precision` and `scale` give the + * type of a `Decimal` column, and `unit` the unit of a `Timestamp` or + * `Duration` column (microseconds when absent). + * + * **Gotchas** + * + * A `Decimal` column without `precision` and `scale` fails when its table + * is read. * * @stability experimental * @category models @@ -690,6 +696,9 @@ export interface TableColumn { readonly type: TypeName readonly nullable: boolean readonly fields?: ReadonlyArray | undefined + readonly precision?: number | undefined + readonly scale?: number | undefined + readonly unit?: TimeUnit | undefined } /** @@ -698,14 +707,16 @@ export interface TableColumn { * **Details** * * When `columns` is given, the table must have these columns and no nulls - * in non-nullable ones; the input is narrowed to these columns, in this - * order. A column's type must hold the declared values: integers below 64 - * bits for floats and narrower integers, any 64-bit integer for `Int64` - * and `UInt64`, any decimal for `Decimal`, the same - * temporal type in any unit, and lists and structs with compatible children. - * `Timestamp` columns read as `DateTime.Utc` values: their timestamps must - * have no time zone or a UTC one (dropped), and lie in the range `DateTime` - * represents. + * in non-nullable ones; the input (every batch of a streamed table) is + * narrowed to these columns, in this order, and converted to their declared + * types, so results don't depend on the types the input happens to have. A + * column converts when its type holds the declared values (integers below + * 64 bits for floats and narrower integers, any 64-bit integer for `Int64` + * and `UInt64`, any decimal for `Decimal`, the same temporal type in any + * unit, lists and structs with such children) and every value converts + * exactly. `Timestamp` columns read as `DateTime.Utc` values: their + * timestamps must have no time zone or a UTC one (dropped), and lie in the + * range `DateTime` represents. `Date` and `Time` columns keep their unit. * * `schema`, set by `Query.table`, is the struct schema the rows must * satisfy: values of fields it constrains beyond their column type (literals, @@ -1030,7 +1041,10 @@ const TableColumnSchema: Schema.Codec = Schema.Struct({ name: Schema.String, type: TypeName, nullable: Schema.Boolean, - fields: Schema.optional(Schema.Array(Schema.suspend(() => TableColumnSchema))) + fields: Schema.optional(Schema.Array(Schema.suspend(() => TableColumnSchema))), + precision: Schema.optional(Schema.Int), + scale: Schema.optional(Schema.Int), + unit: Schema.optional(TimeUnit) }) const NodeSchema = Schema.suspend((): Schema.Codec => NodeUnion) diff --git a/packages/effect/src/dataframe/Query.ts b/packages/effect/src/dataframe/Query.ts index 0564c775b15..d5998edb71c 100644 --- a/packages/effect/src/dataframe/Query.ts +++ b/packages/effect/src/dataframe/Query.ts @@ -68,6 +68,7 @@ * @stability experimental * @since 4.1.5 */ +import type * as ArrowSchema from "../arrow/ArrowSchema.ts" import type * as BigDecimal from "../BigDecimal.ts" import * as Effect from "../Effect.ts" import * as Effectable from "../Effectable.ts" @@ -235,7 +236,16 @@ export const from = (frame: DataFrame): Query => make({ _tag: "Values", * A query over the table called `name`, provided by the `Catalog` when * the query runs. The table must have the columns of the struct schema, and * its values must satisfy it, as with `DataFrame.validate`; the query sees - * only those columns, in the schema's order. + * only those columns, in the schema's order, with the types + * `DataFrame.fromRows` gives them (`options.types` overrides some, and + * `BigDecimal` fields need one). + * + * **Details** + * + * Every batch of a streamed table is converted to these types, so results + * have the same types whatever the batches hold, and when there are none. + * A table without the precision and scale of a `BigDecimal` field fails + * when it is read. * * @stability experimental * @category constructors @@ -243,9 +253,15 @@ export const from = (frame: DataFrame): Query => make({ _tag: "Values", */ export const table = ( name: string, - schema: S + schema: S, + options?: { readonly types?: { readonly [K in keyof S["Type"]]?: ArrowSchema.DataType } | undefined } ): Query> => - make({ _tag: "Table", name, columns: Schema_.columns(schema), schema: schema as unknown as Schema.Top }) + make({ + _tag: "Table", + name, + columns: Schema_.declare(schema, options?.types as any), + schema: schema as unknown as Schema.Top + }) /** * A query from its plan, with untyped rows. Decode untrusted plans with diff --git a/packages/effect/src/dataframe/internal/schema.ts b/packages/effect/src/dataframe/internal/schema.ts index a29c154a187..c164b3acb5b 100644 --- a/packages/effect/src/dataframe/internal/schema.ts +++ b/packages/effect/src/dataframe/internal/schema.ts @@ -1,20 +1,29 @@ /** - * Table columns for the fields of an Effect Schema struct: their Arrow types, - * and the checks a frame must pass to be read as rows of the schema. + * Table columns for the fields of an Effect Schema struct, and the checks a + * frame must pass to be read as rows of the schema. + * + * A field's declared column is fixed once, from the schema and the Arrow + * types given for some fields (`declare`): numbers are Float64, + * timestamps and durations microseconds, unless an override says otherwise, + * and decimals need one. Building a frame from rows, validating a frame and + * reading a table all convert to that declaration (`arrowType`, + * `conform`), so the same schema and overrides always give the same column + * types. * * A `Timestamp` column reads as `DateTime.Utc` values, so its timestamps - * must have no time zone (or a UTC one, which the check drops) and lie in - * the range `DateTime` represents. `DateTimeZoned` fields have no column - * type. + * must have no time zone (or a UTC one, which is dropped) and lie in the + * range `DateTime` represents. `DateTimeZoned` fields have no column type. * * @internal */ import * as S from "../../arrow/ArrowSchema.ts" +import * as Result from "../../Result.ts" import * as AST from "../../SchemaAST.ts" import * as Parser from "../../SchemaParser.ts" import type * as Plan from "../Plan.ts" import * as D from "./dtype.ts" import * as F from "./frame.ts" +import * as K from "./kernels.ts" import * as V from "./vector.ts" type Column = Omit @@ -39,6 +48,8 @@ const literalType = (literal: unknown): Plan.TypeName | undefined => { const sameShape = (a: Column, b: Column): boolean => a.type === b.type && JSON.stringify(a.fields ?? []) === JSON.stringify(b.fields ?? []) +const suspended = new Set() + const columnOf = (ast: AST.AST, path: string): Column => { switch (ast._tag) { case "String": @@ -77,8 +88,17 @@ const columnOf = (ast: AST.AST, path: string): Column => { } return { ...(column ?? leaf("Null")), nullable } } - case "Suspend": - return columnOf(ast.thunk(), path) + case "Suspend": { + if (suspended.has(ast)) { + return V.fail(`field ${JSON.stringify(path)}: recursive schemas have no DataFrame column type`) + } + suspended.add(ast) + try { + return columnOf(ast.thunk(), path) + } finally { + suspended.delete(ast) + } + } case "Arrays": { const element = ast.rest[0] if (ast.elements.length > 0 || element === undefined || ast.rest.length > 1) return unsupported(ast, path) @@ -118,69 +138,8 @@ const structOf = (schema: { readonly ast: AST.AST }): AST.Objects => { : V.fail("DataFrame rows must be described by a struct schema") } -/** The table columns of a struct schema's type side. @internal */ -export const columns = (schema: { readonly ast: AST.AST }): Array => - fieldsOf(structOf(schema), undefined) - -const defaultType = (column: Plan.TableColumn, path: string): S.DataType => { - switch (column.type) { - case "List": { - const item = column.fields![0] - return S.list(S.field("item", defaultType(item, `${path}[]`), { nullable: item.nullable })) - } - case "Struct": - return S.struct( - column.fields!.map((f) => S.field(f.name, defaultType(f, `${path}.${f.name}`), { nullable: f.nullable })) - ) - case "Utf8": - return S.utf8 - case "Float64": - return S.float64 - case "Bool": - return S.bool - case "Int64": - return S.int64 - case "Null": - return S.nullType - case "Timestamp": - return S.timestamp("microsecond") - case "Duration": - return S.duration("microsecond") - case "Decimal": - return V.fail( - `field ${ - JSON.stringify(path) - }: decimal columns need a precision and scale: pass options.types (for example ArrowSchema.decimal(18, 2))` - ) - default: - return V.fail(`field ${JSON.stringify(path)}: no default Arrow type for ${column.type}`) - } -} - -/** - * The Arrow schema of a struct schema's type side, with `overrides` for the - * types of some of its columns. - * - * @internal - */ -export const toArrowSchema = ( - schema: { readonly ast: AST.AST }, - overrides: { readonly [name: string]: S.DataType | undefined } | undefined -): S.Schema => - S.make( - columns(schema).map((column) => { - const override = overrides?.[column.name] - if (override !== undefined && !compatible(override, column)) { - return V.fail( - `field ${JSON.stringify(column.name)}: ${describe(override)} can't hold values of ${describeColumn(column)}` - ) - } - return S.field(column.name, override ?? defaultType(column, column.name), { nullable: column.nullable }) - }) - ) - // ----------------------------------------------------------------------------- -// Checking frames +// Declared columns // ----------------------------------------------------------------------------- /** The plan type name of an Arrow type. @internal */ @@ -206,22 +165,32 @@ export const typeName = (t: S.DataType): Plan.TypeName => { } const describe = (t: S.DataType): string => { - const name = typeName(t) if (t._tag === "List" || t._tag === "LargeList") return `List<${describe(t.child.type)}>` - return name + return t._tag === "Timestamp" || t._tag === "Duration" || t._tag === "Decimal" ? D.describe(t) : typeName(t) } const describeColumn = (c: Column): string => c.type === "List" ? `List<${describeColumn(c.fields![0])}>` : c.type +const utcZones: ReadonlySet = new Set(["UTC", "Etc/UTC", "+00:00"]) + /** - * Whether a column of Arrow type `t` reads as values of the declared - * column: integers below 64 bits for floats and narrower integers, 64-bit - * integers (bigints) for 64-bit ones, any decimal for a decimal, temporal - * types of the same kind in any unit, and lists and structs whose children - * are compatible (unless `shallow`). + * Checks that values of Arrow type `t` read as values of the column, not + * looking into lists and structs: integers below 64 bits for floats and + * narrower integers, 64-bit integers (bigints) for 64-bit ones, any decimal + * for a decimal, timestamps without a time zone or with a UTC one, and + * temporal types of the same kind in any unit. */ -const compatible = (t: S.DataType, expected: Column, shallow = false): boolean => { - const actual = typeName(t) +const checkType = (path: string, t: S.DataType, expected: Column): void => { + if (expected.type === "Timestamp" && t._tag === "Timestamp" && t.timezone !== undefined) { + if (utcZones.has(t.timezone)) return + return V.fail(`column ${JSON.stringify(path)} has time zone ${t.timezone}, expected UTC timestamps`) + } + if (!compatible(t, expected)) { + V.fail(`column ${JSON.stringify(path)} is ${describe(t)}, expected ${describeColumn(expected)}`) + } +} + +const compatible = (t: S.DataType, expected: Column): boolean => { switch (expected.type) { case "Float16": case "Float32": @@ -237,27 +206,140 @@ const compatible = (t: S.DataType, expected: Column, shallow = false): boolean = case "Int64": case "UInt64": return t._tag === "Int" && t.bitWidth === 64 - case "List": - return (t._tag === "List" || t._tag === "LargeList") && - (shallow || expected.fields === undefined || compatible(t.child.type, expected.fields[0])) + default: + return typeName(t) === expected.type + } +} + +const field = (path: string, name: string) => (path === "" ? name : `${path}.${name}`) + +const missing = (path: string): never => V.fail(`missing column ${JSON.stringify(path)}`) + +/** The column declared for a field's `column` with Arrow type `t`, or its default type. */ +const declareColumn = (path: string, column: Column, t: S.DataType | undefined): Column => { + if (t !== undefined) checkType(path, t, column) + switch (column.type) { + case "List": { + const [item] = column.fields! + const child = t !== undefined && (t._tag === "List" || t._tag === "LargeList") ? t.child.type : undefined + return { ...column, fields: [{ ...declareColumn(`${path}[]`, item, child), name: "item" }] } + } + case "Struct": + return { + ...column, + fields: column.fields!.map((f) => { + if (t === undefined || t._tag !== "Struct") { + return { ...declareColumn(field(path, f.name), f, undefined), name: f.name } + } + const g = t.fields.find((g) => g.name === f.name) + return g === undefined + ? missing(field(path, f.name)) + : { ...declareColumn(field(path, f.name), f, g.type), name: f.name } + }) + } + case "Decimal": + return t?._tag === "Decimal" ? { ...column, precision: t.precision, scale: t.scale } : column + case "Timestamp": + case "Duration": + return { ...column, unit: t?._tag === column.type ? t.unit : "microsecond" } + default: + return t === undefined || column.type === "Null" ? column : { ...column, type: typeName(t) } + } +} + +/** + * The declared columns of a struct schema's fields: their types from the + * schema, with `overrides` for the Arrow types of some of them, checked + * against the schema. Decimals have no default precision and scale. + * + * @internal + */ +export const declare = ( + schema: { readonly ast: AST.AST }, + overrides: { readonly [name: string]: S.DataType | undefined } | undefined +): Array => + fieldsOf(structOf(schema), undefined).map((c) => ({ ...declareColumn(c.name, c, overrides?.[c.name]), name: c.name })) + +const numeric: { readonly [name: string]: S.DataType } = { + Int8: S.int8, + Int16: S.int16, + Int32: S.int32, + Int64: S.int64, + UInt8: S.uint8, + UInt16: S.uint16, + UInt32: S.uint32, + UInt64: S.uint64, + Float16: S.float16, + Float32: S.float32, + Float64: S.float64 +} + +/** + * The Arrow type of a declared column. Timestamps and durations without a + * unit are microseconds, dates days and times microseconds; decimals need + * their precision and scale. + * + * @internal + */ +export const arrowType = (column: Plan.TableColumn, path: string = column.name): S.DataType => { + switch (column.type) { + case "List": { + const item = column.fields![0] + return S.list(S.field("item", arrowType(item, `${path}[]`), { nullable: item.nullable })) + } case "Struct": - return t._tag === "Struct" && - (shallow || (expected.fields ?? []).every((f) => { - const field = t.fields.find((g) => g.name === f.name) - return field !== undefined && compatible(field.type, f) - })) + return S.struct( + column.fields!.map((f) => S.field(f.name, arrowType(f, field(path, f.name)), { nullable: f.nullable })) + ) + case "Utf8": + return S.utf8 + case "Binary": + return S.binary + case "Bool": + return S.bool + case "Null": + return S.nullType + case "Date": + return S.date("day") + case "Time": + return S.time("microsecond") + case "Timestamp": + return S.timestamp(column.unit ?? "microsecond") + case "Duration": + return S.duration(column.unit ?? "microsecond") + case "Decimal": + return column.precision === undefined || column.scale === undefined + ? V.fail( + `column ${ + JSON.stringify(path) + }: decimal columns need a precision and scale: pass options.types (for example ArrowSchema.decimal(18, 2))` + ) + : S.decimal(column.precision, column.scale) default: - return actual === expected.type + return numeric[column.type] } } +/** + * The Arrow schema of a struct schema's declared columns, with `overrides` + * for the types of some of them. + * + * @internal + */ +export const toArrowSchema = ( + schema: { readonly ast: AST.AST }, + overrides: { readonly [name: string]: S.DataType | undefined } | undefined +): S.Schema => S.make(declare(schema, overrides).map((c) => S.field(c.name, arrowType(c), { nullable: c.nullable }))) + +// ----------------------------------------------------------------------------- +// Checking frames +// ----------------------------------------------------------------------------- + const allNull = (v: V.Vector, rows: Uint8Array | undefined): boolean => { for (let i = 0; i < v.length; i++) if ((rows === undefined || rows[i] === 1) && V.isValid(v, i)) return false return true } -const utcZones: ReadonlySet = new Set(["UTC", "Etc/UTC", "+00:00"]) - /** The stored values of a timestamp in each unit that `DateTime` represents (±8.64e15 ms). */ const timestampRange: { readonly [U in S.TimeUnit]: readonly [bigint, bigint] } = { second: [-BigInt("8640000000000"), BigInt("8640000000000")], @@ -266,25 +348,56 @@ const timestampRange: { readonly [U in S.TimeUnit]: readonly [bigint, bigint] } nanosecond: [-BigInt("9223372036854775808"), BigInt("9223372036854775807")] } -/** Timestamps that read as `DateTime.Utc`: without a time zone, in range. */ -const utcTimestamps = (path: string, v: V.I64, type: S.Timestamp, rows: Uint8Array | undefined): V.Vector => { - if (type.timezone !== undefined && !utcZones.has(type.timezone)) { - return V.fail(`column ${JSON.stringify(path)} has time zone ${type.timezone}, expected UTC timestamps`) - } - const [lo, hi] = timestampRange[type.unit] +const checkRange = (path: string, v: V.I64, unit: S.TimeUnit, rows: Uint8Array | undefined): void => { + const [lo, hi] = timestampRange[unit] for (let i = 0; i < v.length; i++) { if ((rows === undefined || rows[i] === 1) && V.isValid(v, i) && (v.values[i] < lo || v.values[i] > hi)) { - return V.fail(`column ${JSON.stringify(path)} has timestamps outside the range of DateTime`) + V.fail(`column ${JSON.stringify(path)} has timestamps outside the range of DateTime`) + } + } +} + +const sameValue = (a: V.Vector, b: V.Vector, i: number): boolean => { + switch (a.kind) { + case "f64": { + const x = a.values[i] + const y = (b as V.F64).values[i] + return Object.is(x, y) || (x !== x && y !== y) + } + case "i32": + case "i64": + case "bool": + return a.values[i] === (b as typeof a).values[i] + case "str": + return V.decodeString(a, i) === V.decodeString(b as V.Str, i) + default: + return true + } +} + +/** `v` converted to `target`, failing unless every present value converts exactly. */ +const convert = (path: string, v: V.Vector, target: S.DataType, rows: Uint8Array | undefined): V.Vector => { + if (D.sameType(v.type, target)) return v + const converted = K.cast(v, target) + const back = K.cast(converted, v.type) + for (let i = 0; i < v.length; i++) { + if ( + (rows === undefined || rows[i] === 1) && V.isValid(v, i) && + (!V.isValid(converted, i) || !sameValue(v, back, i)) + ) { + return V.fail(`column ${JSON.stringify(path)} has values that ${D.describe(target)} can't hold exactly`) } } - return type.timezone === undefined ? v : V.withType(v, S.timestamp(type.unit)) + return converted } -/** Checks a column against its declaration, returning it with UTC time zones dropped. */ -const check = (path: string, v: V.Vector, expected: Column, rows: Uint8Array | undefined): V.Vector => { - if (!compatible(v.type, expected, true) && !(expected.type === "Null" && allNull(v, rows))) { - return V.fail(`column ${JSON.stringify(path)} is ${describe(v.type)}, expected ${describeColumn(expected)}`) +/** A column checked against its declaration and converted to its declared type. */ +const check = (path: string, v: V.Vector, expected: Plan.TableColumn, rows: Uint8Array | undefined): V.Vector => { + if (expected.type === "Null") { + if (!allNull(v, rows)) checkType(path, v.type, expected) + return V.nulls(v.length) } + checkType(path, v.type, expected) if (!expected.nullable) { for (let i = 0; i < v.length; i++) { if ((rows === undefined || rows[i] === 1) && !V.isValid(v, i)) { @@ -292,22 +405,15 @@ const check = (path: string, v: V.Vector, expected: Column, rows: Uint8Array | u } } } - if (expected.type === "Timestamp" && v.kind === "i64" && v.type._tag === "Timestamp") { - return utcTimestamps(path, v, v.type, rows) - } - if (expected.fields === undefined) return v if (v.kind === "struct" && v.type._tag === "Struct") { const present = new Uint8Array(v.length) for (let i = 0; i < v.length; i++) present[i] = (rows === undefined || rows[i] === 1) && V.isValid(v, i) ? 1 : 0 const fields = v.type.fields - const children = v.children.slice() - for (const f of expected.fields) { + const children = expected.fields!.map((f) => { const k = fields.findIndex((g) => g.name === f.name) - if (k === -1) return V.fail(`missing column ${JSON.stringify(`${path}.${f.name}`)}`) - children[k] = check(`${path}.${f.name}`, v.children[k], f, present) - } - if (children.every((c, k) => c === v.children[k])) return v - const type = S.struct(fields.map((g, k) => ({ ...g, type: children[k].type }))) + return k === -1 ? missing(field(path, f.name)) : check(field(path, f.name), v.children[k], f, present) + }) + const type = S.struct(expected.fields!.map((f, k) => S.field(f.name, children[k].type, { nullable: f.nullable }))) return V.struct(type, children, v.length, v.valid) } if (v.kind === "list" && (v.type._tag === "List" || v.type._tag === "LargeList")) { @@ -315,38 +421,53 @@ const check = (path: string, v: V.Vector, expected: Column, rows: Uint8Array | u for (let i = 0; i < v.length; i++) { if ((rows === undefined || rows[i] === 1) && V.isValid(v, i)) present.fill(1, v.offsets[i], v.offsets[i + 1]) } - const child = check(`${path}[]`, v.child, expected.fields[0], present) - return child === v.child - ? v - : V.list({ ...v.type, child: { ...v.type.child, type: child.type } }, v.offsets, child, v.valid) + const item = expected.fields![0] + const child = check(`${path}[]`, v.child, item, present) + return V.list(S.list(S.field("item", child.type, { nullable: item.nullable })), v.offsets, child, v.valid) + } + if (expected.type === "Timestamp" && v.kind === "i64" && v.type._tag === "Timestamp") { + checkRange(path, v, v.type.unit, rows) } - return v + return expected.type === "Date" || expected.type === "Time" ? v : convert(path, v, arrowType(expected, path), rows) } -const refined = (ast: AST.AST): boolean => - ast.checks !== undefined || ast._tag === "Literal" || ast._tag === "Enum" || ast._tag === "TemplateLiteral" - /** - * A test for the values of a field whose schema constrains them beyond its - * column type (literals, enums, template literals and refinements), or + * A test for values of a schema that it constrains beyond their column type + * (literals, enums, template literals and checks), anywhere inside it, or * `undefined` when the column type says it all. Nulls pass: nullability is - * checked with the column. + * checked with the column. Union members are tried in turn. */ const valuesCheck = (ast: AST.AST): ((value: unknown) => boolean) | undefined => { - if (refined(ast)) { - const is = Parser._is(ast) - return (value) => value === null || is(value) - } + const own = ast.checks === undefined + ? undefined + : (value: unknown) => Result.isSuccess(AST.runChecks(ast.checks!, value)) + const inner = structural(ast) + if (own === undefined) return inner + return (value) => value === null || ((inner === undefined || inner(value)) && own(value)) +} + +const structural = (ast: AST.AST): ((value: unknown) => boolean) | undefined => { switch (ast._tag) { - case "Suspend": - return valuesCheck(ast.thunk()) - case "Union": { - const members = ast.types.filter((t) => t._tag !== "Null" && t._tag !== "Undefined") - if (members.length === 1) return valuesCheck(members[0]) - if (!members.some(refined)) return undefined - const is = Parser._is(ast) + case "Literal": + case "Enum": + case "TemplateLiteral": { + const is = Parser._is(AST.replaceChecks(ast, undefined)) return (value) => value === null || is(value) } + case "Suspend": { + if (suspended.has(ast)) return V.fail("recursive schemas have no DataFrame column type") + suspended.add(ast) + try { + return valuesCheck(ast.thunk()) + } finally { + suspended.delete(ast) + } + } + case "Union": { + const members = ast.types.filter((t) => t._tag !== "Null" && t._tag !== "Undefined").map(valuesCheck) + if (members.some((m) => m === undefined)) return undefined + return (value) => value === null || members.some((m) => m!(value)) + } case "Arrays": { const item = valuesCheck(ast.rest[0]) return item && ((value) => value === null || (value as ReadonlyArray).every(item)) @@ -380,10 +501,10 @@ export const valueChecks = ( }) /** - * Checks that a frame has the expected columns and narrows it to them, - * dropping UTC time zones from timestamps. With a `schema`, the values of - * fields it constrains beyond their column type are - * checked against it too. + * Checks that a frame has the declared columns and narrows it to them, in + * their order, converting each to its declared type (exactly: a value the + * declared type can't hold fails). With a `schema`, the values of fields it + * constrains beyond their column type are checked against it too. * * @internal */ @@ -394,11 +515,12 @@ export const conform = ( ): F.Frame => { const narrowed = F.make( columns.map((expected) => { - const c = frame.columns.find((c) => c.name === expected.name) - if (c === undefined) return V.fail(`missing column ${JSON.stringify(expected.name)}`) + const c = frame.columns.find((c) => c.name === expected.name) ?? missing(expected.name) const values = c.values() const checked = check(expected.name, values, expected, undefined) - return checked === values ? c : new F.Column(c.name, checked, undefined, c.nullable) + return checked === values && c.nullable === expected.nullable + ? c + : new F.Column(c.name, checked, undefined, expected.nullable) }), frame.length ) diff --git a/packages/effect/src/dataframe/internal/stream.ts b/packages/effect/src/dataframe/internal/stream.ts index fc1f425bb6c..c18ba17b1d4 100644 --- a/packages/effect/src/dataframe/internal/stream.ts +++ b/packages/effect/src/dataframe/internal/stream.ts @@ -13,7 +13,6 @@ * * @internal */ -import * as S from "../../arrow/ArrowSchema.ts" import * as Effect from "../../Effect.ts" import * as Stream from "../../Stream.ts" import type { Catalog } from "../Catalog.ts" @@ -25,47 +24,17 @@ import * as F from "./frame.ts" import * as K from "./kernels.ts" import * as Partial from "./partial.ts" import * as Q from "./query.ts" -import { conform } from "./schema.ts" +import { arrowType, conform } from "./schema.ts" import * as V from "./vector.ts" type Batches = Stream.Stream -const typeOf: Record = { - Int8: S.int8, - Int16: S.int16, - Int32: S.int32, - Int64: S.int64, - UInt8: S.uint8, - UInt16: S.uint16, - UInt32: S.uint32, - UInt64: S.uint64, - Float32: S.float32, - Float64: S.float64, - Utf8: S.utf8, - Binary: S.binary, - Bool: S.bool, - Null: S.nullType, - Date: S.date("day"), - Timestamp: S.timestamp("microsecond"), - Duration: S.duration("microsecond") -} - -/** The type of a declared column without rows; decimals, whose precision is unknown, are Null. */ -const emptyType = (column: Plan.TableColumn): S.DataType => { - const fields = (column.fields ?? []).map((f) => S.field(f.name, emptyType(f), { nullable: f.nullable })) - switch (column.type) { - case "List": - return S.list(fields[0]) - case "Struct": - return S.struct(fields) - default: - return typeOf[column.type] ?? S.nullType - } -} - /** A table without rows: its declared columns, or none. */ const emptyTable = (node: Extract): F.Frame => - F.make((node.columns ?? []).map((c) => new F.Column(c.name, K.nullsOf(emptyType(c), 0), undefined, c.nullable)), 0) + F.make( + (node.columns ?? []).map((c) => new F.Column(c.name, K.nullsOf(arrowType(c), 0), undefined, c.nullable)), + 0 + ) /** The plan over no rows (every input empty), run in place. */ const runEmpty = (node: Q.Node): Effect.Effect => { @@ -83,7 +52,7 @@ const runEmpty = (node: Q.Node): Effect.Effect => { return { ...n, input: empty((n as { readonly input: Q.Node }).input) } as Q.Node } } - return Q.execute(empty(node)) + return Effect.flatMap(Q.attempt(() => empty(node)), Q.execute) } const collectFrames = (batches: Batches, node: Q.Node): Effect.Effect => diff --git a/packages/effect/test/dataframe/DataFrame.test.ts b/packages/effect/test/dataframe/DataFrame.test.ts index e0d2a0e0923..4b88f5ba30e 100644 --- a/packages/effect/test/dataframe/DataFrame.test.ts +++ b/packages/effect/test/dataframe/DataFrame.test.ts @@ -46,23 +46,36 @@ describe("DataFrame", () => { assert.deepStrictEqual(df.schema.fields[0].type, ArrowSchema.decimal(18, 2)) assert.isTrue(BigDecimal.equals(DataFrame.toRows(df)[0].n, BigDecimal.make(123n, 2))) assert.strictEqual(DataFrame.empty(Price, { types: { n: ArrowSchema.decimal(9, 0) } }).length, 0) - assert.isTrue(Result.isSuccess(DataFrame.validate(df, Price))) + assert.isTrue(Result.isSuccess(DataFrame.validate(df, Price, { types: { n: ArrowSchema.decimal(18, 2) } }))) }) it("fromRows rejects overrides that can't hold the field's values", () => { assert.throws( () => DataFrame.fromRows(Schema.Struct({ n: Schema.Number }), [], { types: { n: ArrowSchema.int64 } }), - `field "n": Int64 can't hold values of Float64` + `column "n" is Int64, expected Float64` + ) + assert.throws( + () => + DataFrame.fromRows(Schema.Struct({ t: Schema.DateTimeUtc }), [], { + types: { t: ArrowSchema.timestamp("second", "Europe/Berlin") } + }), + `column "t" has time zone Europe/Berlin, expected UTC timestamps` ) }) it("fromRows rejects schemas without a column type", () => { - try { - DataFrame.fromRows(Schema.Struct({ xs: Schema.Record(Schema.String, Schema.Number) }), []) - assert.fail("expected an error") - } catch (e) { - assert.instanceOf(e, DataFrame.DataFrameError) - assert.include((e as DataFrame.DataFrameError).message, `"xs"`) + interface Tree { + readonly children: ReadonlyArray + } + const Tree: Schema.Codec = Schema.Struct({ children: Schema.Array(Schema.suspend(() => Tree)) }) + for (const xs of [Schema.Record(Schema.String, Schema.Number), Tree]) { + try { + DataFrame.fromRows(Schema.Struct({ xs }), []) + assert.fail("expected an error") + } catch (e) { + assert.instanceOf(e, DataFrame.DataFrameError) + assert.include((e as DataFrame.DataFrameError).message, `"xs`) + } } }) @@ -124,6 +137,32 @@ describe("DataFrame", () => { Schema.Struct({ n: Schema.Int }), `column "n" has values its schema rejects` ], + [ + DataFrame.fromRows(Schema.Struct({ s: Schema.Struct({ a: Schema.String }) }), [{ s: { a: "x" } }, { + s: { a: "z" } + }]), + Schema.Struct({ + s: Schema.Union([Schema.Struct({ a: Schema.Literal("x") }), Schema.Struct({ a: Schema.Literal("y") })]) + }), + `column "s" has values its schema rejects` + ], + [ + DataFrame.fromColumns({ s: ["x", "z"] }), + Schema.Struct({ + s: Schema.Union([Schema.suspend(() => Schema.Literal("x")), Schema.suspend(() => Schema.Literal("y"))]) + }), + `column "s" has values its schema rejects` + ], + [ + column(ArrowSchema.decimal(18, 3), [BigDecimal.fromStringUnsafe("1.234")]), + Schema.Struct({ t: Schema.BigDecimal }), + `column "t": decimal columns need a precision and scale: pass options.types (for example ArrowSchema.decimal(18, 2))` + ], + [ + column(ArrowSchema.timestamp("nanosecond"), [1n]), + Utc, + `column "t" has values that Timestamp[us] can't hold exactly` + ], [ column(ArrowSchema.timestamp("millisecond", "Europe/Berlin"), [0n]), Utc, @@ -155,6 +194,37 @@ describe("DataFrame", () => { const nested = column(ArrowSchema.list(ArrowSchema.field("item", ts)), [[1000n]]) const df = Result.getOrThrow(DataFrame.validate(nested, Schema.Struct({ t: Schema.Array(Schema.DateTimeUtc) }))) assert.deepStrictEqual(DataFrame.toRows(df), [{ t: [DateTime.makeUnsafe(1000)] }]) + const built = DataFrame.fromRows(Utc, [{ t: DateTime.makeUnsafe(0) }], { + types: { t: ArrowSchema.timestamp("second", "UTC") } + }) + assert.deepStrictEqual(built.schema.fields[0].type, ArrowSchema.timestamp("second")) + assert.deepStrictEqual(DataFrame.toRows(built), [{ t: DateTime.makeUnsafe(0) }]) + }) + + it("converts columns to the types fromRows gives the schema, exactly", () => { + const Row = Schema.Struct({ n: Schema.Number, d: Schema.BigDecimal, t: Schema.DateTimeUtc }) + const df = DataFrame.fromArrowTable(Result.getOrThrow(ArrowTable.fromRows( + ArrowSchema.make([ + ArrowSchema.field("n", ArrowSchema.int32), + ArrowSchema.field("d", ArrowSchema.decimal(18, 1)), + ArrowSchema.field("t", ArrowSchema.timestamp("millisecond")) + ]), + [{ n: 1, d: BigDecimal.fromStringUnsafe("1.2"), t: 1n }] + ))) + const types = { d: ArrowSchema.decimal(18, 2) } + const validated = Result.getOrThrow(DataFrame.validate(df, Row, { types })) + assert.deepStrictEqual(validated.schema, DataFrame.empty(Row, { types }).schema) + assert.deepStrictEqual(df.pipe(DataFrame.validate(Row, { types }), Result.getOrThrow).schema, validated.schema) + assert.deepStrictEqual(DataFrame.toRows(validated), [{ + n: 1, + d: BigDecimal.make(120n, 2), + t: DateTime.makeUnsafe(1) + }]) + const lossy = DataFrame.validate(df, Row, { types: { d: ArrowSchema.decimal(18, 0) } }) + assert.strictEqual( + Result.isFailure(lossy) && lossy.failure.message, + `column "d" has values that Decimal[18, 0] can't hold exactly` + ) }) }) diff --git a/packages/effect/test/dataframe/Query.test.ts b/packages/effect/test/dataframe/Query.test.ts index 2125de649ea..5e9f7e5fbf7 100644 --- a/packages/effect/test/dataframe/Query.test.ts +++ b/packages/effect/test/dataframe/Query.test.ts @@ -413,8 +413,13 @@ describe("Query", () => { const untyped = Query.fromPlan({ _tag: "Table", name: "t" }).pipe( Query.filter((c) => Expr.gt(c.x as Expr.Expr, 5)) ) + const Price = Schema.Struct({ d: Schema.BigDecimal }) + const prices = Query.table("t", Price, { types: { d: ArrowSchema.decimal(18, 2) } }).pipe( + Query.aggregate((c) => ({ total: Expr.sum(c.d), r: Expr.max(Expr.round(c.d, 1)) })) + ) const cases: ReadonlyArray, Catalog.Source, ReadonlyArray]> = [ [nested, Stream.empty, ["sum:Float", "a:Float"]], + [prices, Stream.empty, ["total:Decimal", "r:Decimal"]], [nested, Stream.make(DataFrame.fromRows(Nested, [{ xs: [1], s: { a: 1 } }])), ["sum:Float", "a:Float"]], [untyped, Stream.make(DataFrame.fromColumns({ x: [1], y: ["a"] })), ["x:Float", "y:Utf8"]] ] diff --git a/packages/effect/test/dataframe/oracle/Plan.ts b/packages/effect/test/dataframe/oracle/Plan.ts index bf021db866b..2e60618d410 100644 --- a/packages/effect/test/dataframe/oracle/Plan.ts +++ b/packages/effect/test/dataframe/oracle/Plan.ts @@ -78,6 +78,11 @@ const tableColumn = (name: string, t: ArrowSchema.DataType): Plan.TableColumn => return { name, type: "List", nullable: true, fields: [tableColumn("item", t.child.type)] } case "Struct": return { name, type: "Struct", nullable: true, fields: t.fields.map((f) => tableColumn(f.name, f.type)) } + case "Decimal": + return { name, type: "Decimal", nullable: true, precision: t.precision, scale: t.scale } + case "Timestamp": + case "Duration": + return { name, type: t._tag, nullable: true, unit: t.unit } default: return { name, type: t._tag as Plan.TypeName, nullable: true } } diff --git a/packages/effect/typetest/DataFrame.tst.ts b/packages/effect/typetest/DataFrame.tst.ts index 25cdce2b5a1..761587f48d8 100644 --- a/packages/effect/typetest/DataFrame.tst.ts +++ b/packages/effect/typetest/DataFrame.tst.ts @@ -1,5 +1,6 @@ import type { BigDecimal, Result, Stream } from "effect" import { DateTime, Duration, Effect, Schema } from "effect" +import { ArrowSchema } from "effect/arrow" import { DataFrame, Expr, Query } from "effect/dataframe" import { describe, expect, it } from "tstyche" @@ -41,6 +42,19 @@ describe("DataFrame", () => { expect(Query.table("t", S)).type.toBe>() }) + it("validate and Query.table take Arrow types for the schema's fields", () => { + const Price = Schema.Struct({ d: Schema.BigDecimal }) + const types = { d: ArrowSchema.decimal(18, 2) } + expect(DataFrame.validate(DataFrame.empty(Price, { types }), Price, { types })).type.toBe< + Result.Result, DataFrame.DataFrameError> + >() + expect(DataFrame.empty(Price, { types }).pipe(DataFrame.validate(Price, { types }))).type.toBe< + Result.Result, DataFrame.DataFrameError> + >() + expect(Query.table("t", Price, { types })).type.toBe>() + expect(Query.table).type.not.toBeCallableWith("t", Price, { types: { e: ArrowSchema.int32 } }) + }) + it("fromColumns infers column types", () => { expect(DataFrame.fromColumns({ a: new Float64Array(1), b: ["x", null], c: new BigInt64Array(1) })).type.toBe< DataFrame.DataFrame<{ readonly a: number; readonly b: string | null; readonly c: bigint }> From e5c71e98d0465d0bfdd4676aa80ea69cc5ee9e54 Mon Sep 17 00:00:00 2001 From: Effect Bot Date: Sun, 15 Nov 2026 13:16:44 +0000 Subject: [PATCH 25/88] Combine streamed partial aggregates in their common type Batches of an untyped streamed table may differ in type. Exact sums are rescaled to the common decimal scale, re-aggregated partials and group keys are cast to the common type before concatenating, as concatenating the frames would, instead of reading one batch's values in another's type. --- .../effect/src/dataframe/internal/frame.ts | 11 ++---- .../effect/src/dataframe/internal/kernels.ts | 18 ++++++++++ .../effect/src/dataframe/internal/partial.ts | 20 ++++++++--- packages/effect/test/dataframe/Query.test.ts | 36 +++++++++++++++++++ 4 files changed, 72 insertions(+), 13 deletions(-) diff --git a/packages/effect/src/dataframe/internal/frame.ts b/packages/effect/src/dataframe/internal/frame.ts index 5355d698e99..10ed0195a0b 100644 --- a/packages/effect/src/dataframe/internal/frame.ts +++ b/packages/effect/src/dataframe/internal/frame.ts @@ -557,14 +557,9 @@ export const dropNulls = (self: Frame, names: ReadonlyArray | undefined) /** @internal */ export const concat = (frames: ReadonlyArray): Frame => { const first = frames[0] - const columns = first.columns.map((c) => { - const parts = frames.map((f) => column(f, c.name).values()) - const type = parts.reduce((t, p) => { - const st = D.supertype(t, undefined, p.type, undefined) - return st === undefined ? V.fail(`cannot concatenate ${D.describe(t)} with ${D.describe(p.type)}`) : st.type - }, parts[0].type) - return new Column(c.name, V.concat(parts.map((p) => K.cast(p, type))), undefined) - }) + const columns = first.columns.map((c) => + new Column(c.name, K.concat(frames.map((f) => column(f, c.name).values())), undefined) + ) return make(columns, frames.reduce((s, f) => s + f.length, 0)) } diff --git a/packages/effect/src/dataframe/internal/kernels.ts b/packages/effect/src/dataframe/internal/kernels.ts index e761ae15e0f..f9f416bde1f 100644 --- a/packages/effect/src/dataframe/internal/kernels.ts +++ b/packages/effect/src/dataframe/internal/kernels.ts @@ -267,6 +267,24 @@ export const cast = (v: V.Vector, target: S.DataType): V.Vector => { : V.f64(target, out as Float64Array, V.compactValid(valid)) } +/** + * The common type of values of `types` (the larger decimal scale, the finer + * time unit, …); fails when they have none. + * + * @internal + */ +export const commonType = (types: ReadonlyArray): S.DataType => + types.reduce((t, u) => { + const common = D.supertype(t, undefined, u, undefined) + return common === undefined ? V.fail(`cannot concatenate ${D.describe(t)} with ${D.describe(u)}`) : common.type + }) + +/** Concatenates vectors, cast to their {@link commonType}. @internal */ +export const concat = (parts: ReadonlyArray): V.Vector => { + const type = commonType(parts.map((p) => p.type)) + return V.concat(parts.map((p) => cast(p, type))) +} + /** Casts a value, keeping its shape. @internal */ export const castValue = (value: Value, target: S.DataType): Value => D.sameType(value.vector.type, target) diff --git a/packages/effect/src/dataframe/internal/partial.ts b/packages/effect/src/dataframe/internal/partial.ts index 57363b26d59..1568f784fa3 100644 --- a/packages/effect/src/dataframe/internal/partial.ts +++ b/packages/effect/src/dataframe/internal/partial.ts @@ -8,13 +8,17 @@ * means and M2 (Chan et al.). Medians and distinct counts don't combine. * Sums of 64-bit integers and decimals stay exact until `finish`, so whether * a decimal sum fits 64 bits doesn't depend on how the rows are split. + * Parts may differ in type (stream batches of untyped tables): partials + * combine in their common type, as concatenated rows would. * * @internal */ import type * as S from "../../arrow/ArrowSchema.ts" import * as Agg from "./aggregate.ts" +import * as Dec from "./decimal.ts" import * as D from "./dtype.ts" import * as Hash from "./hash.ts" +import * as K from "./kernels.ts" import * as V from "./vector.ts" /** @internal */ @@ -37,6 +41,8 @@ interface Sums { readonly sums: ReadonlyArray } +const scaleOf = (t: S.DataType): number => (Dec.isDecimal(t) ? t.scale : 0) + const reaggregated = new Set(["sum", "count", "len", "min", "max", "first", "last"]) /** Whether partial aggregates of `fn` combine. @internal */ @@ -71,18 +77,22 @@ export const partial = (fn: Agg.AggFn, v: V.Vector | undefined, g: Agg.Grouping) export const combine = (fn: Agg.AggFn, parts: ReadonlyArray, ids: Int32Array, count: number): Part => { const grouping: Agg.Grouping = { ids, count } if (parts.every((p): p is Sums => p._tag === "sums")) { + const type = K.commonType(parts.map((p) => p.type)) const sums = new Array(count).fill(BigInt(0)) let at = 0 for (const p of parts) { - for (let j = 0; j < p.sums.length; j++) sums[ids[at + j]] += p.sums[j] + const factor = Dec.pow10(scaleOf(type) - scaleOf(p.type)) + for (let j = 0; j < p.sums.length; j++) sums[ids[at + j]] += p.sums[j] * factor at += p.sums.length } - return { _tag: "sums", type: parts[0].type, sums } + return { _tag: "sums", type, sums } } if (reaggregated.has(fn)) { - const vectors = parts.map((p) => (p as Extract).vector) + const vectors = parts.map((p) => + p._tag === "sums" ? Agg.fromSums(p.type, p.sums) : (p as Extract).vector + ) const merge = fn === "count" || fn === "len" ? "sum" : fn - return { _tag: "vector", vector: Agg.aggregate(merge, V.concat(vectors), grouping) } + return { _tag: "vector", vector: Agg.aggregate(merge, K.concat(vectors), grouping) } } if (fn === "mean") { const sum = new Float64Array(count) @@ -166,7 +176,7 @@ export const group = ( export const merge = (fns: ReadonlyArray, states: ReadonlyArray): State => { if (states.length === 1) return states[0] const total = states.reduce((s, x) => s + x.count, 0) - const keys = states[0].keys.map((_, k) => V.concat(states.map((s) => s.keys[k]))) + const keys = states[0].keys.map((_, k) => K.concat(states.map((s) => s.keys[k]))) const g = Hash.groups(keys, total) return { keys: keys.map((k) => V.take(k, g.firstRow)), diff --git a/packages/effect/test/dataframe/Query.test.ts b/packages/effect/test/dataframe/Query.test.ts index 5e9f7e5fbf7..f4c06082c74 100644 --- a/packages/effect/test/dataframe/Query.test.ts +++ b/packages/effect/test/dataframe/Query.test.ts @@ -396,6 +396,42 @@ describe("Query", () => { assert.deepStrictEqual(result, [{ total: BigDecimal.fromBigInt(0n) }]) })) + it.effect("batches of different types combine in their common type", () => + Effect.gen(function*() { + const Price = Schema.Struct({ d: Schema.BigDecimal }) + const price = (d: string, scale: number) => + DataFrame.fromRows(Price, [{ d: BigDecimal.fromStringUnsafe(d) }], { + types: { d: ArrowSchema.decimal(18, scale) } + }) + const source = Stream.make(price("1.2", 1), price("1.23", 2), price("1.2", 2)) + const declared = Query.table("t", Price, { types: { d: ArrowSchema.decimal(18, 2) } }) + const untyped = Query.fromPlan({ _tag: "Table", name: "t" }) as typeof declared + const d = (s: string) => BigDecimal.fromStringUnsafe(s) + for (const table of [declared, untyped]) { + const totals = table.pipe( + Query.aggregate((c) => ({ s: Expr.sum(c.d), lo: Expr.min(c.d), hi: Expr.max(c.d) })) + ) + assert.deepStrictEqual(yield* streamed(totals, source), [{ s: d("3.63"), lo: d("1.20"), hi: d("1.23") }]) + const groups = table.pipe( + Query.groupBy("d"), + Query.agg(() => ({ n: Expr.count() })), + Query.sort((c) => [c.d]) + ) + assert.deepStrictEqual(yield* streamed(groups, source), [{ d: d("1.20"), n: 2 }, { d: d("1.23"), n: 1 }]) + } + const numbers = Stream.make( + DataFrame.fromRows(Row, [{ x: 1 }], { types: { x: ArrowSchema.int32 } }), + DataFrame.fromRows(Row, [{ x: 1.5 }]) + ) + const sum = Query.fromPlan({ _tag: "Table", name: "t" }).pipe( + Query.aggregate((c) => ({ s: Expr.sum(c.x as Expr.Expr) })) + ) + assert.deepStrictEqual(yield* streamed(sum, numbers), [{ s: 2.5 }]) + const lossy = Query.table("t", Price, { types: { d: ArrowSchema.decimal(18, 1) } }) + const error = yield* Effect.flip(streamed(lossy, source)) + assert.strictEqual(error.message, `column "d" has values that Decimal[18, 1] can't hold exactly`) + })) + it.effect("zero-row batches aggregate like an empty table", () => Effect.gen(function*() { const query = Query.aggregate(t, (c) => ({ n: Expr.count(), total: Expr.sum(c.x), mean: Expr.mean(c.x) })) From ca0dfad0c6ede7f2ea171299d4ac709c0905d835 Mon Sep 17 00:00:00 2001 From: Effect Bot Date: Sun, 15 Nov 2026 13:14:09 +0000 Subject: [PATCH 26/88] Rescale durations to coarser units toward zero, as Polars does --- .../effect/src/dataframe/internal/temporal.ts | 11 ++- packages/effect/test/dataframe/Query.test.ts | 74 ++++++++++++++++++- 2 files changed, 80 insertions(+), 5 deletions(-) diff --git a/packages/effect/src/dataframe/internal/temporal.ts b/packages/effect/src/dataframe/internal/temporal.ts index d1957e77981..d62fe4b35a9 100644 --- a/packages/effect/src/dataframe/internal/temporal.ts +++ b/packages/effect/src/dataframe/internal/temporal.ts @@ -53,8 +53,11 @@ const floorDiv = (a: bigint, b: bigint): bigint => { return (a % b !== BigInt(0)) && ((a < BigInt(0)) !== (b < BigInt(0))) ? q - BigInt(1) : q } -/** Rescales integer counts of `from` units to `to` units (flooring). */ -const rescale = (values: BigInt64Array, from: S.TimeUnit, to: S.TimeUnit): BigInt64Array => { +/** + * Rescales integer counts of `from` units to `to` units, rounding down + * (toward zero when `truncate`, as Polars rescales durations). + */ +const rescale = (values: BigInt64Array, from: S.TimeUnit, to: S.TimeUnit, truncate = false): BigInt64Array => { if (from === to) return values const out = new BigInt64Array(values.length) if (fineness[to] > fineness[from]) { @@ -62,7 +65,7 @@ const rescale = (values: BigInt64Array, from: S.TimeUnit, to: S.TimeUnit): BigIn for (let i = 0; i < values.length; i++) out[i] = BigInt.asIntN(64, values[i] * f) } else { const f = BigInt(perSecond[from] / perSecond[to]) - for (let i = 0; i < values.length; i++) out[i] = floorDiv(values[i], f) + for (let i = 0; i < values.length; i++) out[i] = truncate ? values[i] / f : floorDiv(values[i], f) } return out } @@ -296,7 +299,7 @@ export const cast = (v: V.Vector, target: S.DataType): V.Vector | undefined => { return V.i64(target, rescale(asI64(v), from.unit, target.unit), v.valid) } if (from._tag === "Duration" && target._tag === "Duration") { - return V.i64(target, rescale(asI64(v), from.unit, target.unit), v.valid) + return V.i64(target, rescale(asI64(v), from.unit, target.unit, true), v.valid) } if (from._tag === "Timestamp" && target._tag === "Date") { return target.unit === "day" diff --git a/packages/effect/test/dataframe/Query.test.ts b/packages/effect/test/dataframe/Query.test.ts index f4c06082c74..b2eeedbc6ab 100644 --- a/packages/effect/test/dataframe/Query.test.ts +++ b/packages/effect/test/dataframe/Query.test.ts @@ -1,5 +1,5 @@ import { assert, describe, it } from "@effect/vitest" -import { BigDecimal, Effect, Exit, Result, Schema, Stream } from "effect" +import { BigDecimal, type DateTime, type Duration, Effect, Exit, Result, Schema, Stream } from "effect" import { ArrowSchema, ArrowTable } from "effect/arrow" import { Catalog, DataFrame, Expr, Plan, Query } from "effect/dataframe" import * as Compare from "./oracle/Compare.ts" @@ -196,6 +196,78 @@ describe("Query", () => { ) assert.deepStrictEqual(casts, [{ fromFloat: null, fromUInt64: null, toUInt64: null }]) })) + + it.effect("temporal operands of mixed units meet in the coarser unit, as in Polars", () => + Effect.gen(function*() { + // Expected values from Polars 2.1.0: timestamps round down, durations toward zero. + const df = DataFrame.fromArrowTable(Result.getOrThrow(ArrowTable.fromColumns( + ArrowSchema.make([ + ArrowSchema.field("t", ArrowSchema.timestamp("millisecond")), + ArrowSchema.field("s", ArrowSchema.timestamp("microsecond")), + ArrowSchema.field("d", ArrowSchema.duration("millisecond")), + ArrowSchema.field("e", ArrowSchema.duration("microsecond")) + ]), + [[1n, -1n, 2n], [1500n, 1500n, -1n], [1n, -1n, 2n], [1500n, 1500n, -1n]] + ))) as DataFrame.DataFrame as DataFrame.DataFrame<{ + t: DateTime.Utc + s: DateTime.Utc + d: Duration.Duration + e: Duration.Duration + }> + const int = (x: Expr.Expr) => Expr.cast(x, "Int64") + const result = yield* rows( + Query.from(df).pipe(Query.select((c) => ({ + tMinusS: int(Expr.subtract(c.t, c.s)), + tPlusE: int(Expr.add(c.t, c.e)), + tMinusE: int(Expr.subtract(c.t, c.e)), + dPlusE: int(Expr.add(c.d, c.e)), + dMinusE: int(Expr.subtract(c.d, c.e)), + eq: Expr.eq(c.t, c.s), + lt: Expr.lt(c.t, c.s), + durationsEq: Expr.eq(c.d, c.e), + s: int(Expr.cast(c.s, "Timestamp[ms]")), + e: int(Expr.cast(c.e, "Duration[ms]")) + }))) + ) + assert.deepStrictEqual(result, [ + { + tMinusS: 0n, + tPlusE: 2n, + tMinusE: 0n, + dPlusE: 2n, + dMinusE: 0n, + eq: true, + lt: false, + durationsEq: true, + s: 1n, + e: 1n + }, + { + tMinusS: -2n, + tPlusE: 0n, + tMinusE: -2n, + dPlusE: 0n, + dMinusE: -2n, + eq: false, + lt: true, + durationsEq: false, + s: 1n, + e: 1n + }, + { + tMinusS: 3n, + tPlusE: 2n, + tMinusE: 2n, + dPlusE: 2n, + dMinusE: 2n, + eq: false, + lt: false, + durationsEq: false, + s: -1n, + e: 0n + } + ]) + })) }) describe("explain", () => { From 757d2c036b2d7a775ff039aac9c23ee73542b69c Mon Sep 17 00:00:00 2001 From: Effect Bot Date: Sun, 15 Nov 2026 13:14:09 +0000 Subject: [PATCH 27/88] Bring SQL temporal operands of mixed units to the coarser unit first --- packages/effect/src/dataframe/Sql.ts | 2 +- packages/effect/src/dataframe/internal/sql.ts | 82 +++++++++++++++---- packages/sql/pglite/test/DataFrameSql.test.ts | 66 +++++++++++++++ 3 files changed, 132 insertions(+), 18 deletions(-) diff --git a/packages/effect/src/dataframe/Sql.ts b/packages/effect/src/dataframe/Sql.ts index 7a192b27819..439a7366938 100644 --- a/packages/effect/src/dataframe/Sql.ts +++ b/packages/effect/src/dataframe/Sql.ts @@ -37,7 +37,7 @@ * they differ from the in-memory engine's: integer overflow raises an error * instead of wrapping, division by zero follows the database, string order * follows its collation, and timestamps and durations keep PostgreSQL's - * microseconds (finer values round down). Pivots, unpivots, `explode` and + * microseconds (finer timestamps round down, finer durations toward zero). Pivots, unpivots, `explode` and * `unnest` are not supported yet; tables need declared columns * (`Query.table` with a schema), and {@link engine} checks that tables * satisfy them, as the in-memory engine does. diff --git a/packages/effect/src/dataframe/internal/sql.ts b/packages/effect/src/dataframe/internal/sql.ts index 3acf7a9832c..a0abaccec28 100644 --- a/packages/effect/src/dataframe/internal/sql.ts +++ b/packages/effect/src/dataframe/internal/sql.ts @@ -31,7 +31,7 @@ import * as F from "./frame.ts" import * as K from "./kernels.ts" import * as Q from "./query.ts" import * as Schema_ from "./schema.ts" -import { civil } from "./temporal.ts" +import { civil, coarser } from "./temporal.ts" import * as V from "./vector.ts" interface Compiler { @@ -191,11 +191,12 @@ const integerOf = (t: Temporal): S.Int => (t._tag === "Date" && t.unit === "day" /** * A temporal value as its integer: its `unit`s (its own by default) since - * the epoch, or its length in them, rounded down. + * the epoch, rounded down, or its length in them, rounded toward zero. */ const toInteger = (x: string, type: Temporal, unit?: Plan.TimeUnit): string => { if (type._tag !== "Date") { - return `CAST(FLOOR(EXTRACT(EPOCH FROM ${x}) * ${perSecond[unit ?? type.unit]}) AS BIGINT)` + const rounding = type._tag === "Duration" ? "TRUNC" : "FLOOR" + return `CAST(${rounding}(EXTRACT(EPOCH FROM ${x}) * ${perSecond[unit ?? type.unit]}) AS BIGINT)` } const days = `(CAST(${x} AS DATE) - DATE '1970-01-01')` return type.unit === "day" ? days : `(CAST(${days} AS BIGINT) * 86400000)` @@ -215,7 +216,7 @@ const fromInteger = (x: string, type: Temporal): string => { return `(DATE '1970-01-01' + CAST(${type.unit === "day" ? x : `FLOOR(${x} / 86400000.0)`} AS INTEGER))` } const micros = type.unit === "nanosecond" - ? `CAST(FLOOR(${x} / 1000.0) AS BIGINT)` + ? `CAST(${type._tag === "Duration" ? "TRUNC" : "FLOOR"}(${x} / 1000.0) AS BIGINT)` : `(${x} * ${perSecond.microsecond / perSecond[type.unit]})` if (type._tag === "Duration") return interval(micros) const epoch = type.timezone === undefined ? "TIMESTAMP '1970-01-01 00:00:00'" : "TIMESTAMPTZ '1970-01-01 00:00:00+00'" @@ -224,9 +225,9 @@ const fromInteger = (x: string, type: Temporal): string => { /** * `x` converted from `from` to `to` as the engine converts: timestamps - * and durations rescale to the target unit (rounding down), timestamps and - * dates convert at midnight, and other temporal values convert as their - * integers. + * and durations rescale to the target unit (rounding as {@link toInteger}), + * timestamps and dates convert at midnight, and other temporal values + * convert as their integers. */ const cast = (x: string, from: S.DataType, to: S.DataType): string => { if (isTemporal(from) && isTemporal(to)) { @@ -246,6 +247,22 @@ const cast = (x: string, from: S.DataType, to: S.DataType): string => { return `CAST(${truncated} AS ${sqlType(to)})` } +/** + * Operands as the engine combines them: timestamps and durations of mixed + * units converted to the coarsest of their units first. + */ +const sameUnit = (xs: ReadonlyArray, types: ReadonlyArray): Array => { + const units = types.flatMap((t) => (t._tag === "Timestamp" || t._tag === "Duration" ? [t.unit] : [])) + if (units.every((u) => u === units[0])) return [...xs] + const unit = units.reduce(coarser) + return xs.map((x, i) => { + const t = types[i] + if (t._tag === "Timestamp" && t.unit !== unit) return cast(x, t, S.timestamp(unit, t.timezone)) + if (t._tag === "Duration" && t.unit !== unit) return cast(x, t, S.duration(unit)) + return x + }) +} + // ----------------------------------------------------------------------------- // Expressions // ----------------------------------------------------------------------------- @@ -378,8 +395,11 @@ const totalSeconds: Record = { const call = (node: Plan.Call, scope: Scope): string => { const row = scope.compiler.row - const a = node.args.map((x) => expression(x, scope)) - const k = node.args.map((x) => typeOf(x, scope)._tag) + const types = node.args.map((x) => typeOf(x, scope)) + const raw = node.args.map((x) => expression(x, scope)) + // between compares its operand with each bound separately. + const a = node.fn === "between" ? raw : sameUnit(raw, types) + const k = types.map((t) => t._tag) const ints = k[0] === "Int" && k[1] === "Int" const o = node.options ?? {} // Floored modulo of integers, exact, null for a zero divisor. @@ -420,8 +440,11 @@ const call = (node: Plan.Call, scope: Scope): string => { return `(${a[0]} > ${a[1]})` case "gte": return `(${a[0]} >= ${a[1]})` - case "between": - return `(${a[0]} BETWEEN ${a[1]} AND ${a[2]})` + case "between": { + const [x, lo] = sameUnit([a[0], a[1]], [types[0], types[1]]) + const [y, hi] = sameUnit([a[0], a[2]], [types[0], types[2]]) + return `(${x} >= ${lo} AND ${y} <= ${hi})` + } case "isIn": { const values = (o.values ?? []).filter((v) => v !== null) return values.length === 0 ? @@ -624,6 +647,20 @@ const typed = (node: Plan.Node, frames: Map): F.Frame => { const columnsOf = (node: Plan.Node, c: Compiler): ReadonlyArray => c.frames.get(node)!.columns.map((col) => col.name) +const typeIn = (node: Plan.Node, name: string, c: Compiler): S.DataType => + c.frames.get(node)!.columns.find((col) => col.name === name)!.type + +/** + * Column `name` of `input`, as `x`, in the type the column has in `node`: + * joins and concatenations bring timestamps and durations of mixed units to + * the coarser one. + */ +const conformed = (x: string, input: Plan.Node, node: Plan.Node, name: string, c: Compiler): string => { + const from = typeIn(input, name, c) + const to = typeIn(node, name, c) + return isTemporal(from) && isTemporal(to) && !D.sameType(from, to) ? cast(x, from, to) : x +} + const scopeOf = (node: Plan.Node, compiler: Compiler): Scope => ({ compiler, frame: compiler.frames.get(node)!, @@ -780,21 +817,28 @@ const relation = (node: Plan.Node, c: Compiler): string => { case "Join": { const l = relation(node.left, c) const r = relation(node.right, c) - const on = node.on.map((k) => `l.${quote(k)} = r.${quote(k)}`).join(" AND ") + const on = node.on.map((k) => { + const [x, y] = sameUnit([`l.${quote(k)}`, `r.${quote(k)}`], [typeIn(node.left, k, c), typeIn(node.right, k, c)]) + return `${x} = ${y}` + }).join(" AND ") if (node.how === "semi" || node.how === "anti") { return `SELECT * FROM (${l}) AS l WHERE ${ node.how === "anti" ? "NOT " : "" }EXISTS (SELECT 1 FROM (${r}) AS r WHERE ${on})` } const rightOnly = columnsOf(node.right, c).filter((name) => !node.on.includes(name)) + const fromLeft = (name: string) => conformed(`l.${quote(name)}`, node.left, node, name, c) + const fromRight = (name: string) => conformed(`r.${quote(name)}`, node.right, node, name, c) const left = columnsOf(node.left, c).map((name) => - node.how === "full" && node.on.includes(name) - ? `COALESCE(l.${quote(name)}, r.${quote(name)}) AS ${quote(name)}` - : `l.${quote(name)}` + `${ + node.how === "full" && node.on.includes(name) + ? `COALESCE(${fromLeft(name)}, ${fromRight(name)})` + : fromLeft(name) + } AS ${quote(name)}` ) const join = node.how === "inner" ? "INNER JOIN" : node.how === "left" ? "LEFT JOIN" : "FULL JOIN" return `SELECT ${ - [...left, ...rightOnly.map((name) => `r.${quote(name)}`)].join(", ") + [...left, ...rightOnly.map((name) => `${fromRight(name)} AS ${quote(name)}`)].join(", ") }, ROW_NUMBER() OVER (ORDER BY l.${row}${ node.how === "full" ? " NULLS LAST" : "" }, r.${row}) AS ${row} FROM (${l}) AS l ${join} (${r}) AS r ON ${on}` @@ -802,7 +846,11 @@ const relation = (node: Plan.Node, c: Compiler): string => { case "Concat": { const source = c.hidden("source") const union = node.inputs - .map((n, i) => `SELECT ${keep}, ${i} AS ${source} FROM (${relation(n, c)}) AS t${i}`) + .map((n, i) => + `SELECT ${ + columns.map((name) => `${conformed(quote(name), n, node, name, c)} AS ${quote(name)}`).join(", ") + }, ${row}, ${i} AS ${source} FROM (${relation(n, c)}) AS t${i}` + ) .join(" UNION ALL ") return renumbered(union, `${source}, ${row}`) } diff --git a/packages/sql/pglite/test/DataFrameSql.test.ts b/packages/sql/pglite/test/DataFrameSql.test.ts index 089302014fb..434b2a45b34 100644 --- a/packages/sql/pglite/test/DataFrameSql.test.ts +++ b/packages/sql/pglite/test/DataFrameSql.test.ts @@ -125,6 +125,54 @@ const temporalCasts = unitNames.flatMap((u) => [ }))) ] as const ]) +/** Integers of a temporal column of `unit`, microsecond multiples (PostgreSQL keeps microseconds). */ +const exact = (unit: typeof unitNames[number], values: ReadonlyArray) => + values.map((v) => (v === null ? null : BigInt(v) * (unit === "ns" ? 1000n : 1n))) +/** Timestamps `t`, `s` and durations `d`, `e`: `t` and `d` of unit `u`, `s` and `e` of unit `v`. */ +const mixed = (u: typeof unitNames[number], v: typeof unitNames[number]) => + arrow<{ + t: DateTime.Utc | null + s: DateTime.Utc | null + d: Duration.Duration | null + e: Duration.Duration | null + }>([ + ArrowSchema.field("t", ArrowSchema.timestamp(units[u])), + ArrowSchema.field("s", ArrowSchema.timestamp(units[v])), + ArrowSchema.field("d", ArrowSchema.duration(units[u])), + ArrowSchema.field("e", ArrowSchema.duration(units[v])) + ], [ + exact(u, [1500, -1, 2, 1000, null]), + exact(v, [1, 1500, -1, 1000, 7]), + exact(u, [1500, -1, 2, 1000, null]), + exact(v, [1, 1500, -1, 1000, 7]) + ]) +/** Temporal operations on operands of two different units, which meet in the coarser one. */ +const mixedUnits = unitNames.flatMap((u) => + unitNames.filter((v) => v !== u).map((v) => + [ + `temporal operations mixing ${u} and ${v}`, + mixed(u, v).pipe(Query.select((c) => { + const int = (x: Expr.Expr) => Expr.cast(x, "Int64") + return { + tMinusS: int(Expr.subtract(c.t, c.s)), + tPlusE: int(Expr.add(c.t, c.e)), + tMinusE: int(Expr.subtract(c.t, c.e)), + sPlusD: int(Expr.add(c.s, c.d)), + dPlusE: int(Expr.add(c.d, c.e)), + dMinusE: int(Expr.subtract(c.d, c.e)), + eq: Expr.eq(c.t, c.s), + lt: Expr.lt(c.t, c.s), + gte: Expr.gte(c.s, c.t), + durationsEq: Expr.eq(c.d, c.e), + durationsGt: Expr.gt(c.d, c.e), + between: Expr.between(c.t, c.s, c.s), + coalesce: int(Expr.coalesce(c.t, c.s)), + when: int(Expr.when(Expr.isNull(c.d), c.e, c.d)) + } + })) + ] as const + ) +) /** Queries whose SQL result must equal the in-memory result, column types included. */ const cases: ReadonlyArray]> = [ @@ -199,6 +247,24 @@ const cases: ReadonlyArray]> = [ ]).pipe(Query.withColumns((c) => ({ s: Expr.totalSeconds(c.d), year: Expr.year(c.t) }))) ], ...temporalCasts, + ...mixedUnits, + [ + "joins on timestamps of mixed units", + mixed("s", "ms").pipe( + Query.select((c) => ({ k: c.t, x: Expr.lit(1) })), + Query.join(mixed("s", "ms").pipe(Query.select((c) => ({ k: c.s, y: Expr.lit(2) }))), { on: "k", how: "full" }), + Query.select((c) => ({ k: Expr.cast(c.k, "Int64"), x: c.x, y: c.y })) + ) + ], + [ + "concatenations of durations of mixed units", + Query.concat([ + mixed("s", "ms").pipe(Query.select((c) => ({ d: c.e }))), + mixed("s", "ms").pipe(Query.select((c) => ({ d: c.d }))) + ]).pipe( + Query.select((c) => ({ d: Expr.cast(c.d, "Int64"), ms: Expr.cast(Expr.cast(c.d, "Duration[ms]"), "Int64") })) + ) + ], [ "casts of integers beyond double precision to Timestamp[us] and Duration[us]", temporal(ArrowSchema.int64, [253_402_300_799_999_999n, -62_135_596_800_000_001n, 9_007_199_254_740_993n]).pipe( From 0980adf3b404bdefbf8f1c1ab38b9d5ad083cd8e Mon Sep 17 00:00:00 2001 From: Effect Bot Date: Sun, 15 Nov 2026 13:24:01 +0000 Subject: [PATCH 28/88] Check SQL tables in the statement that reads them, as memory checks them Row checks and the result now share one statement and snapshot; volatile views are computed once for both. Column types are checked against what memory accepts, Null columns must hold only nulls, timestamps must be in DateTime range and read as UTC whatever the session time zone, and intervals must not have months. --- packages/effect/src/dataframe/Sql.ts | 47 ++- packages/effect/src/dataframe/internal/sql.ts | 290 ++++++++++++++---- packages/sql/pglite/test/DataFrameSql.test.ts | 59 ++++ 3 files changed, 320 insertions(+), 76 deletions(-) diff --git a/packages/effect/src/dataframe/Sql.ts b/packages/effect/src/dataframe/Sql.ts index 439a7366938..3924e532311 100644 --- a/packages/effect/src/dataframe/Sql.ts +++ b/packages/effect/src/dataframe/Sql.ts @@ -72,7 +72,7 @@ import type * as Plan from "./Plan.ts" * **Gotchas** * * The query doesn't check that the tables satisfy their declared columns and - * schemas: {@link engine} runs those checks as separate queries first. + * schemas: {@link engine} runs a query with those checks instead. * * @stability experimental * @category models @@ -119,18 +119,29 @@ const attempt = (f: () => A): Result.Result => { * **Details** * * Tables must satisfy their declared columns and schemas, as with the - * in-memory engine: before the query, the engine queries every table of the - * plan for nulls in its non-nullable columns and for the distinct values of - * the columns its schema constrains beyond their type (literals, enums, - * refinements), tests those values with the schema, and fails if the table - * breaks either. Columns the query doesn't read are checked too. `explain` - * shows these queries before the query itself. + * in-memory engine, or the query fails. First the engine queries the + * PostgreSQL types of the declared columns: `smallint` or `integer` for + * integers up to 32 bits, `bigint` for 64-bit ones, `smallint`, + * `integer`, `real` or `double precision` for floats, string or enum + * types for strings, `bytea`, `boolean`, `date`, `timestamp` or + * `timestamptz` (read as UTC), `interval`, and any type for `Null` + * columns. Then one + * query checks the rows and computes the result from the same rows: only + * nulls in `Null` columns, no nulls in other non-nullable columns, + * timestamps in the range of `DateTime`, intervals without months, and the + * distinct values of the columns the schema constrains beyond their type + * (literals, enums, refinements) accepted by the schema. Columns the query + * doesn't read are checked too. `explain` shows both queries. * * **Gotchas** * - * The checks and the query are separate statements: rows written between - * them are not checked. Run the query inside a `REPEATABLE READ` - * transaction when the tables may change concurrently. + * The row checks and the result read the same rows: they are one + * statement, so they read one snapshot, and a view that calls volatile + * functions (`nextval`, `random`) is computed once for both. Functions + * that give different results within a statement without being declared + * volatile are not supported. The column types are checked in a separate + * statement first: run both inside a `REPEATABLE READ` transaction when + * the tables may be altered concurrently. * * @stability experimental * @category engines @@ -143,13 +154,14 @@ export const engine: Effect.Effect = Effect.ge const collect = (plan: Plan.Node): Effect.Effect, DataFrameError> => Effect.gen(function*() { const query = yield* compiled(plan) - for (const check of query.checks) { - const rows = yield* run(check.sql) - yield* Effect.fromResult(attempt(() => check.verify(rows))) + if (query.types !== undefined) { + const types = query.types + const rows = yield* run(types.sql) + yield* Effect.fromResult(attempt(() => internal.verify(types, rows))) } - const rows = yield* run(query.sql) + const rows = yield* run(query.query.sql) return yield* Effect.try({ - try: () => internal.decode(query.schema, rows) as unknown as DataFrame, + try: () => internal.read(query, rows) as unknown as DataFrame, catch: toError }) }) @@ -157,14 +169,15 @@ export const engine: Effect.Effect = Effect.ge name: "sql (postgres)", collect, stream: (plan) => Stream.fromEffect(collect(plan)), - explain: (plan) => Effect.map(compiled(plan), (c) => [...c.checks.map((check) => check.sql), c.sql].join(";\n")) + explain: (plan) => + Effect.map(compiled(plan), (c) => [...(c.types === undefined ? [] : [c.types.sql]), c.query.sql].join(";\n")) } satisfies Engine.Service }) /** * A layer that runs queries on the database of the `SqlClient` in the * context, with {@link engine} (tables are checked against their schemas - * first). + * as they are read). * * @stability experimental * @category layers diff --git a/packages/effect/src/dataframe/internal/sql.ts b/packages/effect/src/dataframe/internal/sql.ts index a0abaccec28..faa285acfec 100644 --- a/packages/effect/src/dataframe/internal/sql.ts +++ b/packages/effect/src/dataframe/internal/sql.ts @@ -41,6 +41,8 @@ interface Compiler { readonly hidden: (name: string) => string /** The quoted row key. */ readonly row: string + /** The quoted name of each table's source (see {@link source}). */ + readonly sources: ReadonlyMap } /** @internal */ @@ -91,24 +93,55 @@ const sqlType = (t: S.DataType): string => { } } -/** The type the SQL engine reads a declared table column as. */ -const tableType = (table: string, column: Plan.TableColumn): S.DataType => { - switch (column.type) { - case "Timestamp": - return S.timestamp("microsecond") - case "Duration": - return S.duration("microsecond") - case "Binary": - return S.binary - case "Null": - return S.nullType - default: - return D.castTargets[column.type] ?? unsupported( - `column ${JSON.stringify(column.name)} of table ${JSON.stringify(table)}: ${column.type} table columns` - ) - } +const smallIntegers = ["smallint", "integer"] + +/** + * How the SQL engine reads declared table columns of each type: as values + * of `type`, from columns of the PostgreSQL types `accepts` or of the type + * categories `categories` (as the in-memory engine accepts integers below + * 64 bits for floats and narrower integers), or of any type for `Null`. + * `undefined` for column types it doesn't read yet. + */ +const tableTypes: { + readonly [K in Plan.TypeName]: + | { + readonly type: S.DataType + readonly accepts?: ReadonlyArray + readonly categories?: ReadonlyArray + } + | undefined +} = { + Int8: { type: S.int8, accepts: smallIntegers }, + Int16: { type: S.int16, accepts: smallIntegers }, + Int32: { type: S.int32, accepts: smallIntegers }, + Int64: { type: S.int64, accepts: ["bigint"] }, + UInt8: { type: S.uint8, accepts: smallIntegers }, + UInt16: { type: S.uint16, accepts: smallIntegers }, + UInt32: { type: S.uint32, accepts: smallIntegers }, + UInt64: { type: S.uint64, accepts: ["bigint"] }, + Float16: undefined, + Float32: { type: S.float32, accepts: [...smallIntegers, "real", "double precision"] }, + Float64: { type: S.float64, accepts: [...smallIntegers, "real", "double precision"] }, + // String types and enums. + Utf8: { type: S.utf8, categories: ["S", "E"] }, + Binary: { type: S.binary, accepts: ["bytea"] }, + Bool: { type: S.bool, accepts: ["boolean"] }, + Null: { type: S.nullType }, + Date: { type: S.date("day"), accepts: ["date"] }, + Time: undefined, + Timestamp: { type: S.timestamp("microsecond"), accepts: ["timestamp without time zone", "timestamp with time zone"] }, + Duration: { type: S.duration("microsecond"), accepts: ["interval"] }, + Decimal: undefined, + List: undefined, + Struct: undefined } +/** How the SQL engine reads a declared table column (see {@link tableTypes}). */ +const tableType = (table: string, column: Plan.TableColumn) => + tableTypes[column.type] ?? unsupported( + `column ${JSON.stringify(column.name)} of table ${JSON.stringify(table)}: ${column.type} table columns` + ) + const string = (s: string): string => `'${s.replaceAll("'", "''")}'` const pad = (n: number, width = 2) => String(n).padStart(width, "0") @@ -627,7 +660,7 @@ const typed = (node: Plan.Node, frames: Map): F.Frame => { ) frame = F.fromVectors( columns.map((c) => c.name), - columns.map((c) => K.nullsOf(tableType(node.name, c), 0)), + columns.map((c) => K.nullsOf(tableType(node.name, c).type, 0)), 0 ) } else if (node._tag === "Values") { @@ -726,7 +759,9 @@ const relation = (node: Plan.Node, c: Compiler): string => { } FROM (${input}) AS t` switch (node._tag) { case "Table": - return `SELECT ${[...columns.map(quote), `ROW_NUMBER() OVER () AS ${row}`].join(", ")} FROM ${quote(node.name)}` + return `SELECT ${[...columns.map(quote), `ROW_NUMBER() OVER () AS ${row}`].join(", ")} FROM ${ + c.sources.get(node) + }` case "Values": return values(node.frame as unknown as F.Frame, c) case "Filter": { @@ -924,65 +959,142 @@ const reader = (type: S.DataType): (value: unknown) => unknown => { } // ----------------------------------------------------------------------------- -// Table checks +// Tables // ----------------------------------------------------------------------------- /** - * A query over a table, and a test of its rows that fails when the table - * breaks its declared columns or schema. - * - * @internal + * A check of a table: one SQL value, and a test of it that fails when the + * table breaks its declaration. */ -export interface Check { +interface Check { readonly sql: string - readonly verify: (rows: ReadonlyArray>) => void + readonly verify: (value: unknown) => void } /** - * The checks of a table that its result's casts to the column types don't - * make: no nulls in non-nullable columns, and the values of fields its schema - * constrains (literals, enums, refinements) accepted by the schema, tested - * on each such column's distinct values. + * The checks of a table's declared column types that don't read its rows: + * each value is the PostgreSQL type of a column the engine doesn't read as + * declared, or null. + */ +const typeChecks = (table: Plan.Table): Array => + (table.columns ?? []).flatMap((column): Array => { + if (column.type === "Null") return [] + const { accepts = [], categories = [] } = tableType(table.name, column) + const accepted = [ + ...(accepts.length === 0 ? [] : [`format_type(oid, NULL) IN (${accepts.map(string).join(", ")})`]), + ...(categories.length === 0 ? [] : [`typcategory IN (${categories.map(string).join(", ")})`]) + ] + return [{ + sql: `(SELECT CASE WHEN ${ + accepted.join(" OR ") + } THEN NULL ELSE format_type(oid, NULL) END FROM pg_type WHERE oid = pg_typeof((SELECT ${ + quote(column.name) + } FROM ${quote(table.name)} LIMIT 0)))`, + verify: (actual) => { + if (actual !== null) V.fail(`column ${JSON.stringify(column.name)} is ${actual}, expected ${column.type}`) + } + }] + }) + +/** + * The checks of a table's rows, the ones the in-memory engine makes beyond + * column types: only nulls in `Null` columns, no nulls in other columns + * unless they are nullable, timestamps in the range of `DateTime`, + * durations of fixed length (no months), and the values of the fields its + * schema constrains beyond their column type (literals, enums, refinements) + * accepted by the schema, tested on each such column's distinct values. + * Each value aggregates the rows of the table's source. */ -const tableChecks = (table: Plan.Table, frame: F.Frame): Array => { - const from = quote(table.name) - const checks: Array = [] - const required = (table.columns ?? []).filter((c) => !c.nullable) - if (required.length > 0) { - const read = reader(S.bool) - checks.push({ - sql: `SELECT ${ - required.map((c) => output(`EXISTS (SELECT 1 FROM ${from} WHERE ${quote(c.name)} IS NULL)`, S.bool)).join(", ") - }`, - verify: (rows) => { - const missing = required.find((_, i) => read(rows[0][i])) - if (missing !== undefined) V.fail(`column ${JSON.stringify(missing.name)} has missing values`) +const rowChecks = (table: Plan.Table, frame: F.Frame): Array => [ + ...(table.columns ?? []).flatMap((column) => { + const x = quote(column.name) + const path = JSON.stringify(column.name) + const any = (failed: string, message: string): Check => ({ + sql: `BOOL_OR(${failed})`, + verify: (value) => { + if (value === true) V.fail(`column ${path} ${message}`) } }) - } - for (const [name, test] of table.schema === undefined ? [] : Schema_.valueChecks(table.schema)) { + const checks: Array = [] + if (column.type === "Null") checks.push(any(`${x} IS NOT NULL`, "has values, expected Null")) + else if (!column.nullable) checks.push(any(`${x} IS NULL`, "has missing values")) + if (column.type === "Timestamp") { + checks.push(any( + `EXTRACT(EPOCH FROM ${x}) NOT BETWEEN -8640000000000 AND 8640000000000.000999`, + "has timestamps outside the range of DateTime" + )) + } + if (column.type === "Duration") { + checks.push(any( + `EXTRACT(YEAR FROM ${x}) <> 0 OR EXTRACT(MONTH FROM ${x}) <> 0`, + "has durations in months, which have no fixed length" + )) + } + return checks + }), + ...(table.schema === undefined ? [] : Schema_.valueChecks(table.schema)).map(([name, test]): Check => { const type = frame.columns.find((c) => c.name === name)!.type - checks.push({ - sql: `SELECT DISTINCT ${output(quote(name), type)} FROM ${from}`, - verify: (rows) => { + return { + sql: `ARRAY_AGG(DISTINCT ${output(quote(name), type)})`, + verify: (values) => { + const rows = ((values ?? []) as ReadonlyArray).map((v) => [v]) if (!F.toArray(decode(S.make([S.field(name, type, { nullable: true })]), rows), name).every(test)) { V.fail(`column ${JSON.stringify(name)} has values its schema rejects`) } } - }) - } - return checks + } + }) +] + +const utc = S.timestamp("microsecond") + +/** + * The source a table is read through: its declared columns, timestamps as + * UTC timestamps without a time zone (as the engine reads them, whatever the + * session's time zone). The checks and the result read it in the same + * statement, so the same snapshot: sources the checks read too are inlined + * (`NOT MATERIALIZED`, as cheap as reading the table twice), unless they + * call volatile functions, which PostgreSQL computes once for both instead. + */ +const source = (table: Plan.Table, checked: boolean): string => { + const columns = (table.columns ?? []).map((column) => + column.type === "Timestamp" + ? `${fromInteger(toInteger(quote(column.name), utc), utc)} AS ${quote(column.name)}` + : quote(column.name) + ) + return `${checked ? "NOT MATERIALIZED " : ""}(SELECT ${columns.length === 0 ? "1" : columns.join(", ")} FROM ${ + quote(table.name) + })` +} + +/** One SQL query whose first row holds the values of checks. */ +interface Checked { + readonly sql: string + readonly checks: ReadonlyArray<(value: unknown) => void> } /** @internal */ export interface Compiled { + /** The query without the checks: the result's rows in the plan's row order. */ readonly sql: string + /** + * The checks of the column types of the plan's tables, to run first (the + * query doesn't compile against tables of other column types), when the + * plan reads tables. + */ + readonly types: Checked | undefined + /** + * The query: a row of the checks of its tables' rows first, when there + * are some, then the result's rows in the plan's row order. + */ + readonly query: Checked readonly schema: S.Schema - /** The checks of the plan's tables, to run before the query. */ - readonly checks: ReadonlyArray } -/** A plan as one SQL query, its rows in the plan's row order, and the checks of its tables. @internal */ +/** + * A plan as SQL: the query, a query of its tables' column types, and the + * query with the checks of their rows. @internal + */ export const compile = (plan: Plan.Node): Compiled => { const frames = new Map() const result = typed(plan, frames) @@ -991,18 +1103,78 @@ export const compile = (plan: Plan.Node): Compiled => { let prefix = "__" while ([...used].some((name) => name.startsWith(prefix))) prefix += "_" const hidden = (name: string) => quote(prefix + name) - const compiler: Compiler = { frames, hidden, row: hidden("row") } + const tables = [...frames].flatMap(([node, frame], i) => + node._tag === "Table" ? [{ node, name: hidden(`table${i}`), checks: rowChecks(node, frame) }] : [] + ) + const compiler: Compiler = { + frames, + hidden, + row: hidden("row"), + sources: new Map(tables.map((t) => [t.node, t.name])) + } const sql = relation(plan, compiler) + const schema = S.make(result.columns.map((col) => S.field(col.name, col.type, { nullable: true }))) const columns = result.columns.map((col) => `${output(quote(col.name), col.type)} AS ${quote(col.name)}`) + const ctes = (checked: boolean) => + tables.length === 0 + ? "" + : `WITH ${tables.map((t) => `${t.name} AS ${source(t.node, checked && t.checks.length > 0)}`).join(", ")} ` + const body = `SELECT ${columns.length === 0 ? "1" : columns.join(", ")} FROM (${sql}) AS t ORDER BY ${compiler.row}` + const columnTypes = tables.flatMap((t) => typeChecks(t.node)) + const types = columnTypes.length === 0 ? undefined : { + sql: `SELECT ${columnTypes.map((check) => check.sql).join(", ")}`, + checks: columnTypes.map((check) => check.verify) + } + const checked = tables.filter((t) => t.checks.length > 0) + if (checked.length === 0) { + return { sql: ctes(false) + body, types, query: { sql: ctes(false) + body, checks: [] }, schema } + } + const names: Array = [] + const header = checked.map((t, i) => + `(SELECT ${ + t.checks.map((check) => { + names.push(hidden(`check${names.length}`)) + return `${check.sql} AS ${names[names.length - 1]}` + }).join(", ") + } FROM ${t.name}) AS ${hidden(`checks${i}`)}` + ) + const rows = `(SELECT ${[...columns, compiler.row].join(", ")} FROM (${sql}) AS t) AS r` return { - sql: `SELECT ${columns.length === 0 ? "1" : columns.join(", ")} FROM (${sql}) AS t ORDER BY ${compiler.row}`, - schema: S.make(result.columns.map((col) => S.field(col.name, col.type, { nullable: true }))), - checks: [...frames].flatMap(([node, frame]) => node._tag === "Table" ? tableChecks(node, frame) : []) + sql: ctes(false) + body, + types, + query: { + sql: `${ctes(true)}SELECT ${[...names, ...result.columns.map((col) => `r.${quote(col.name)}`)].join(", ")} FROM ${ + header.join(" CROSS JOIN ") + } FULL JOIN ${rows} ON FALSE ORDER BY r.${compiler.row} NULLS FIRST`, + checks: checked.flatMap((t) => t.checks.map((check) => check.verify)) + }, + schema } } -/** The rows of a compiled query's result, one array of values per row, as a frame. @internal */ -export const decode = (schema: S.Schema, rows: ReadonlyArray>): F.Frame => +/** + * Tests the checks in the first row of a checked query's result, failing + * when a table breaks its declaration, and returns the other rows without + * the checks. + * + * @internal + */ +export const verify = ( + checked: Checked, + rows: ReadonlyArray> +): ReadonlyArray> => { + const k = checked.checks.length + if (k === 0) return rows + checked.checks.forEach((test, i) => test(rows[0][i])) + return rows.slice(1).map((row) => row.slice(k)) +} + +/** The result rows of a compiled query, after its checks (see {@link verify}), as a frame. @internal */ +export const read = (compiled: Compiled, rows: ReadonlyArray>): F.Frame => + decode(compiled.schema, verify(compiled.query, rows)) + +/** The rows of a result, one array of values per row, as a frame of `schema`. */ +const decode = (schema: S.Schema, rows: ReadonlyArray>): F.Frame => F.make( schema.fields.map((f, i) => { const read = reader(f.type) diff --git a/packages/sql/pglite/test/DataFrameSql.test.ts b/packages/sql/pglite/test/DataFrameSql.test.ts index 434b2a45b34..1375844b87c 100644 --- a/packages/sql/pglite/test/DataFrameSql.test.ts +++ b/packages/sql/pglite/test/DataFrameSql.test.ts @@ -47,6 +47,9 @@ const Setup = Layer.effectDiscard( for (const r of regions) yield* sql`INSERT INTO regions VALUES (${r.region}, ${r.manager})` yield* sql`CREATE TABLE contract (s TEXT, n DOUBLE PRECISION)` yield* sql`INSERT INTO contract VALUES ('bad', NULL), ('good', 1)` + yield* sql`DROP TABLE IF EXISTS odd` + yield* sql`CREATE TABLE odd (nil INTEGER, nothing INTEGER, at TIMESTAMPTZ, far TIMESTAMP, months INTERVAL)` + yield* sql`INSERT INTO odd VALUES (42, NULL, '2024-01-01 00:30:00+00', '290000-01-01', '1 month')` }) ) @@ -405,6 +408,62 @@ describe("effect/dataframe on PostgreSQL (PGlite)", { concurrent: false }, () => yield* rowsOnSql(contract(Schema.Struct({ s: Schema.Literals(["bad", "good"]) }))), [{ s: "bad" }, { s: "good" }] ) + const odd = (schema: S) => Query.table("odd", schema) + for ( + const [query, message] of [ + [odd(Schema.Struct({ nil: Schema.Null })), `column "nil" has values, expected Null`], + [contract(Schema.Struct({ s: Schema.NullOr(Schema.Number) })), `column "s" is text, expected Float64`], + [ + contract(Schema.Struct({ n: Schema.NullOr(Schema.BigInt) })), + `column "n" is double precision, expected Int64` + ], + [ + odd(Schema.Struct({ far: Schema.NullOr(Schema.DateTimeUtc) })), + `column "far" has timestamps outside the range of DateTime` + ], + [ + odd(Schema.Struct({ months: Schema.Duration })), + `column "months" has durations in months, which have no fixed length` + ] + ] as const + ) { + assert.strictEqual(yield* rejects(query), message) + } + assert.deepStrictEqual(yield* rowsOnSql(odd(Schema.Struct({ nothing: Schema.Null }))), [{ nothing: null }]) + })) + + it.effect("tables are checked on the rows the query reads, even of volatile views", () => + Effect.gen(function*() { + const sql = yield* SqlClient + yield* sql`CREATE SEQUENCE seq START 1` + yield* sql`CREATE VIEW fluctuating AS SELECT CASE WHEN nextval('seq') <= 2 THEN 'good' ELSE 'bad' END AS s` + const query = Query.table("fluctuating", Schema.Struct({ s: Schema.Literal("good") })) + const read = sql.withTransaction( + Effect.andThen(sql`SET TRANSACTION ISOLATION LEVEL REPEATABLE READ`, Effect.result(rowsOnSql(query))) + ) + const results = [yield* read, yield* read, yield* read] + assert.deepStrictEqual(results.map((r) => Result.isSuccess(r) ? r.success : r.failure.message), [ + [{ s: "good" }], + [{ s: "good" }], + `column "s" has values its schema rejects` + ]) + })) + + it.effect("timestamps with time zones are UTC, whatever the session's time zone", () => + Effect.gen(function*() { + const sql = yield* SqlClient + const query = Query.table("odd", Schema.Struct({ at: Schema.DateTimeUtc })).pipe( + Query.select((c) => ({ + at: c.at, + hour: Expr.hour(c.at), + early: Expr.lt(c.at, Expr.lit(DateTime.makeUnsafe("2024-01-01T00:45:00Z"))) + })) + ) + const rows = yield* sql`SET TIME ZONE 'Europe/Berlin'`.pipe( + Effect.andThen(rowsOnSql(query)), + Effect.ensuring(Effect.orDie(sql`RESET TIME ZONE`)) + ) + assert.deepStrictEqual(rows, [{ at: DateTime.makeUnsafe("2024-01-01T00:30:00Z"), hour: 0, early: true }]) })) it.effect("filters, derives and sorts", () => From 6c55c708ad76d4c15509d0fedd9fea6da409add7 Mon Sep 17 00:00:00 2001 From: Effect Bot Date: Sun, 15 Nov 2026 13:35:31 +0000 Subject: [PATCH 29/88] Read declared SQL table columns as the types memory conforms them to The SQL engine read every declared Timestamp/Duration table column as microseconds and rejected Decimal columns, while the memory engine conforms tables to the declared unit, precision and scale. Read table columns as Schema_.arrowType gives them, accept numeric for decimals, and fail in the statement, with memory's message, on values the declared type can't hold exactly (fractions of the unit, nanoseconds beyond 64 bits, decimals that need rounding or more digits). Inexact values read as null so that reading them doesn't fail before the checks do. --- packages/effect/src/dataframe/internal/sql.ts | 122 ++++++++++++------ packages/sql/pglite/test/DataFrameSql.test.ts | 79 +++++++++++- 2 files changed, 159 insertions(+), 42 deletions(-) diff --git a/packages/effect/src/dataframe/internal/sql.ts b/packages/effect/src/dataframe/internal/sql.ts index faa285acfec..b358b24f877 100644 --- a/packages/effect/src/dataframe/internal/sql.ts +++ b/packages/effect/src/dataframe/internal/sql.ts @@ -96,51 +96,52 @@ const sqlType = (t: S.DataType): string => { const smallIntegers = ["smallint", "integer"] /** - * How the SQL engine reads declared table columns of each type: as values - * of `type`, from columns of the PostgreSQL types `accepts` or of the type + * The PostgreSQL columns the SQL engine reads as declared table columns of + * each type: columns of the PostgreSQL types `accepts` or of the type * categories `categories` (as the in-memory engine accepts integers below * 64 bits for floats and narrower integers), or of any type for `Null`. * `undefined` for column types it doesn't read yet. */ const tableTypes: { readonly [K in Plan.TypeName]: - | { - readonly type: S.DataType - readonly accepts?: ReadonlyArray - readonly categories?: ReadonlyArray - } + | { readonly accepts?: ReadonlyArray; readonly categories?: ReadonlyArray } | undefined } = { - Int8: { type: S.int8, accepts: smallIntegers }, - Int16: { type: S.int16, accepts: smallIntegers }, - Int32: { type: S.int32, accepts: smallIntegers }, - Int64: { type: S.int64, accepts: ["bigint"] }, - UInt8: { type: S.uint8, accepts: smallIntegers }, - UInt16: { type: S.uint16, accepts: smallIntegers }, - UInt32: { type: S.uint32, accepts: smallIntegers }, - UInt64: { type: S.uint64, accepts: ["bigint"] }, + Int8: { accepts: smallIntegers }, + Int16: { accepts: smallIntegers }, + Int32: { accepts: smallIntegers }, + Int64: { accepts: ["bigint"] }, + UInt8: { accepts: smallIntegers }, + UInt16: { accepts: smallIntegers }, + UInt32: { accepts: smallIntegers }, + UInt64: { accepts: ["bigint"] }, Float16: undefined, - Float32: { type: S.float32, accepts: [...smallIntegers, "real", "double precision"] }, - Float64: { type: S.float64, accepts: [...smallIntegers, "real", "double precision"] }, + Float32: { accepts: [...smallIntegers, "real", "double precision"] }, + Float64: { accepts: [...smallIntegers, "real", "double precision"] }, // String types and enums. - Utf8: { type: S.utf8, categories: ["S", "E"] }, - Binary: { type: S.binary, accepts: ["bytea"] }, - Bool: { type: S.bool, accepts: ["boolean"] }, - Null: { type: S.nullType }, - Date: { type: S.date("day"), accepts: ["date"] }, + Utf8: { categories: ["S", "E"] }, + Binary: { accepts: ["bytea"] }, + Bool: { accepts: ["boolean"] }, + Null: {}, + Date: { accepts: ["date"] }, Time: undefined, - Timestamp: { type: S.timestamp("microsecond"), accepts: ["timestamp without time zone", "timestamp with time zone"] }, - Duration: { type: S.duration("microsecond"), accepts: ["interval"] }, - Decimal: undefined, + Timestamp: { accepts: ["timestamp without time zone", "timestamp with time zone"] }, + Duration: { accepts: ["interval"] }, + Decimal: { accepts: ["numeric"] }, List: undefined, Struct: undefined } -/** How the SQL engine reads a declared table column (see {@link tableTypes}). */ -const tableType = (table: string, column: Plan.TableColumn) => - tableTypes[column.type] ?? unsupported( - `column ${JSON.stringify(column.name)} of table ${JSON.stringify(table)}: ${column.type} table columns` - ) +/** + * The type the SQL engine reads a declared table column as: the type the + * in-memory engine conforms it to. + */ +const tableType = (table: string, column: Plan.TableColumn): S.DataType => + tableTypes[column.type] === undefined + ? unsupported( + `column ${JSON.stringify(column.name)} of table ${JSON.stringify(table)}: ${column.type} table columns` + ) + : Schema_.arrowType(column) const string = (s: string): string => `'${s.replaceAll("'", "''")}'` @@ -660,7 +661,7 @@ const typed = (node: Plan.Node, frames: Map): F.Frame => { ) frame = F.fromVectors( columns.map((c) => c.name), - columns.map((c) => K.nullsOf(tableType(node.name, c).type, 0)), + columns.map((c) => K.nullsOf(tableType(node.name, c), 0)), 0 ) } else if (node._tag === "Values") { @@ -759,9 +760,13 @@ const relation = (node: Plan.Node, c: Compiler): string => { } FROM (${input}) AS t` switch (node._tag) { case "Table": - return `SELECT ${[...columns.map(quote), `ROW_NUMBER() OVER () AS ${row}`].join(", ")} FROM ${ - c.sources.get(node) - }` + return `SELECT ${ + [ + ...node.columns!.map((column) => `${declared(column)} AS ${quote(column.name)}`), + `ROW_NUMBER() OVER () AS ${row}` + ] + .join(", ") + } FROM ${c.sources.get(node)}` case "Values": return values(node.frame as unknown as F.Frame, c) case "Filter": { @@ -971,6 +976,43 @@ interface Check { readonly verify: (value: unknown) => void } +const int64 = BigInt(1) << BigInt(63) + +/** + * SQL that is true when `x`, a value of a table's source, is one its + * declared column's `type` holds exactly, as the in-memory engine converts + * table columns: timestamps and durations in whole units that fit 64 bits + * (PostgreSQL keeps microseconds), decimals without more digits than the + * precision and scale allow (no rounding). `undefined` when every value is. + */ +const exact = (x: string, type: S.DataType): string | undefined => { + switch (type._tag) { + case "Timestamp": + case "Duration": { + const v = `(EXTRACT(EPOCH FROM ${x}) * ${perSecond[type.unit]})` + return `${v} = TRUNC(${v}) AND ${v} BETWEEN ${-int64} AND ${int64 - BigInt(1)}` + } + case "Decimal": + return `ROUND(${x}, ${type.scale}) = ${x} AND ABS(${x}) < ${power(type.precision - type.scale)}` + default: + return undefined + } +} + +/** + * A declared column of a table's source as its declared type: null where it + * isn't held exactly (see {@link exact}), so that reading such values + * doesn't fail before the checks of the rows do. + */ +const declared = (column: Plan.TableColumn): string => { + const x = quote(column.name) + const type = Schema_.arrowType(column) + const held = exact(x, type) + return held === undefined + ? x + : `(CASE WHEN ${held} THEN ${type._tag === "Decimal" ? `CAST(${x} AS ${sqlType(type)})` : x} END)` +} + /** * The checks of a table's declared column types that don't read its rows: * each value is the PostgreSQL type of a column the engine doesn't read as @@ -979,7 +1021,7 @@ interface Check { const typeChecks = (table: Plan.Table): Array => (table.columns ?? []).flatMap((column): Array => { if (column.type === "Null") return [] - const { accepts = [], categories = [] } = tableType(table.name, column) + const { accepts = [], categories = [] } = tableTypes[column.type]! const accepted = [ ...(accepts.length === 0 ? [] : [`format_type(oid, NULL) IN (${accepts.map(string).join(", ")})`]), ...(categories.length === 0 ? [] : [`typcategory IN (${categories.map(string).join(", ")})`]) @@ -1000,8 +1042,9 @@ const typeChecks = (table: Plan.Table): Array => * The checks of a table's rows, the ones the in-memory engine makes beyond * column types: only nulls in `Null` columns, no nulls in other columns * unless they are nullable, timestamps in the range of `DateTime`, - * durations of fixed length (no months), and the values of the fields its - * schema constrains beyond their column type (literals, enums, refinements) + * durations of fixed length (no months), values their declared type holds + * exactly (see {@link exact}), and the values of the fields its schema + * constrains beyond their column type (literals, enums, refinements) * accepted by the schema, tested on each such column's distinct values. * Each value aggregates the rows of the table's source. */ @@ -1030,12 +1073,15 @@ const rowChecks = (table: Plan.Table, frame: F.Frame): Array => [ "has durations in months, which have no fixed length" )) } + const type = Schema_.arrowType(column) + const held = exact(x, type) + if (held !== undefined) checks.push(any(`NOT (${held})`, `has values that ${D.describe(type)} can't hold exactly`)) return checks }), ...(table.schema === undefined ? [] : Schema_.valueChecks(table.schema)).map(([name, test]): Check => { const type = frame.columns.find((c) => c.name === name)!.type return { - sql: `ARRAY_AGG(DISTINCT ${output(quote(name), type)})`, + sql: `ARRAY_AGG(DISTINCT ${output(declared(table.columns!.find((c) => c.name === name)!), type)})`, verify: (values) => { const rows = ((values ?? []) as ReadonlyArray).map((v) => [v]) if (!F.toArray(decode(S.make([S.field(name, type, { nullable: true })]), rows), name).every(test)) { diff --git a/packages/sql/pglite/test/DataFrameSql.test.ts b/packages/sql/pglite/test/DataFrameSql.test.ts index 1375844b87c..3844471206f 100644 --- a/packages/sql/pglite/test/DataFrameSql.test.ts +++ b/packages/sql/pglite/test/DataFrameSql.test.ts @@ -50,12 +50,43 @@ const Setup = Layer.effectDiscard( yield* sql`DROP TABLE IF EXISTS odd` yield* sql`CREATE TABLE odd (nil INTEGER, nothing INTEGER, at TIMESTAMPTZ, far TIMESTAMP, months INTERVAL)` yield* sql`INSERT INTO odd VALUES (42, NULL, '2024-01-01 00:30:00+00', '290000-01-01', '1 month')` + yield* sql`DROP TABLE IF EXISTS declared` + yield* sql`CREATE TABLE declared (t TIMESTAMP, d INTERVAL, n NUMERIC)` + yield* sql`INSERT INTO declared VALUES ('1970-01-01 00:00:01', '-2 seconds', 12.5), ('1969-12-31 23:59:58', '3 seconds', -0.05), (NULL, NULL, NULL)` + yield* sql`DROP TABLE IF EXISTS inexact` + yield* sql`CREATE TABLE inexact (s TIMESTAMP, ms TIMESTAMP, ns TIMESTAMP, ds INTERVAL, dms INTERVAL, dns INTERVAL, scale NUMERIC, digits NUMERIC)` + yield* sql`INSERT INTO inexact VALUES ('1970-01-01 00:00:01.5', '1970-01-01 00:00:01.0005', '2263-01-01', '-1.5 seconds', '0.0005 seconds', '110000 days', 1.005, 1000)` }) ) +/** A frame with exactly the given Arrow columns. */ +const frame = (fields: ReadonlyArray, columns: ReadonlyArray>) => + DataFrame.fromArrowTable(Result.getOrThrow(ArrowTable.fromColumns(ArrowSchema.make(fields), columns))) +const decimal = (s: string) => BigDecimal.fromStringUnsafe(s) + const memory = Catalog.layer({ sales: DataFrame.fromRows(Sale, sales), - regions: DataFrame.fromRows(Region, regions) + regions: DataFrame.fromRows(Region, regions), + // The rows of the PostgreSQL tables `declared` and `inexact`, as PostgreSQL keeps them. + declared: frame([ + ArrowSchema.field("t", ArrowSchema.timestamp("microsecond"), { nullable: true }), + ArrowSchema.field("d", ArrowSchema.duration("microsecond"), { nullable: true }), + ArrowSchema.field("n", ArrowSchema.decimal(20, 4), { nullable: true }) + ], [[1_000_000n, -2_000_000n, null], [-2_000_000n, 3_000_000n, null], [decimal("12.5"), decimal("-0.05"), null]]), + inexact: frame([ + ...["s", "ms", "ns"].map((name) => ArrowSchema.field(name, ArrowSchema.timestamp("microsecond"))), + ...["ds", "dms", "dns"].map((name) => ArrowSchema.field(name, ArrowSchema.duration("microsecond"))), + ...["scale", "digits"].map((name) => ArrowSchema.field(name, ArrowSchema.decimal(20, 4))) + ], [ + [1_500_000n], + [1_000_500n], + [9_246_182_400_000_000n], + [-1_500_000n], + [500n], + [9_504_000_000_000_000n], + [decimal("1.005")], + [decimal("1000")] + ]) }) const table = Query.table("sales", Sale) @@ -80,9 +111,7 @@ const groups = Query.from( const none = ints.pipe(Query.filter((c) => Expr.gt(c.x, 9))) /** A query over a frame with exactly the given Arrow columns. */ const arrow = (fields: ReadonlyArray, columns: ReadonlyArray>) => - Query.from( - DataFrame.fromArrowTable(Result.getOrThrow(ArrowTable.fromColumns(ArrowSchema.make(fields), columns))) - ) as Query.Query as Query.Query + Query.from(frame(fields, columns)) as Query.Query as Query.Query const ties = [0.5, 1.5, 2.5, -0.5, -1.5, -2.5, 2.675, 1.005, 0.125, 0.375, 1.25, 1.35] const decimals = arrow<{ d: BigDecimal.BigDecimal | null }>( [ArrowSchema.field("d", ArrowSchema.decimal(18, 3))], @@ -319,6 +348,27 @@ const cases: ReadonlyArray]> = [ Query.select((c) => ({ r0: Expr.round(c.x), r1: Expr.round(c.x, 1), r2: Expr.round(c.x, 2) })) ) ], + ...unitNames.map((u) => + [ + `tables declared in ${u} and with a decimal`, + Query.table( + "declared", + Schema.Struct({ + t: Schema.NullOr(Schema.DateTimeUtc), + d: Schema.NullOr(Schema.Duration), + n: Schema.NullOr(Schema.BigDecimal) + }), + { + types: { t: ArrowSchema.timestamp(units[u]), d: ArrowSchema.duration(units[u]), n: ArrowSchema.decimal(5, 2) } + } + ).pipe(Query.withColumns((c) => ({ + ti: Expr.cast(c.t, "Int64"), + di: Expr.cast(c.d, "Int64"), + text: Expr.cast(c.n, "Utf8"), + sum: Expr.add(c.n, c.n) + }))) + ] as const + ), [ "rounding integers", Query.from(DataFrame.fromColumns({ x: Int32Array.of(5, -15, 25) })).pipe( @@ -432,6 +482,27 @@ describe("effect/dataframe on PostgreSQL (PGlite)", { concurrent: false }, () => assert.deepStrictEqual(yield* rowsOnSql(odd(Schema.Struct({ nothing: Schema.Null }))), [{ nothing: null }]) })) + it.effect("tables reject values their declared types can't hold exactly, as in memory", () => + Effect.gen(function*() { + for ( + const [column, schema, type] of [ + ["s", Schema.DateTimeUtc, ArrowSchema.timestamp("second")], + ["ms", Schema.DateTimeUtc, ArrowSchema.timestamp("millisecond")], + ["ns", Schema.DateTimeUtc, ArrowSchema.timestamp("nanosecond")], + ["ds", Schema.Duration, ArrowSchema.duration("second")], + ["dms", Schema.Duration, ArrowSchema.duration("millisecond")], + ["dns", Schema.Duration, ArrowSchema.duration("nanosecond")], + ["scale", Schema.BigDecimal, ArrowSchema.decimal(5, 2)], + ["digits", Schema.BigDecimal, ArrowSchema.decimal(5, 2)] + ] as const + ) { + const query = Query.table("inexact", Schema.Struct({ [column]: schema }), { types: { [column]: type } }) + const expected = yield* Effect.flip(query.pipe(Effect.provide(memory))) + const actual = yield* Effect.flip(onSql(query)) + assert.strictEqual(actual.message, expected.message) + } + })) + it.effect("tables are checked on the rows the query reads, even of volatile views", () => Effect.gen(function*() { const sql = yield* SqlClient From 9facd6c32193cc327b3a5fbb6b9c64cedaf8069d Mon Sep 17 00:00:00 2001 From: Effect Bot Date: Sun, 15 Nov 2026 13:58:29 +0000 Subject: [PATCH 30/88] Make temporal casts that leave 64 bits null instead of wrapping Rescaling timestamps and durations to a finer unit, dates to timestamps and 64-bit integers to 32-bit temporal values wrapped on overflow. Values that don't fit become null, as for every other cast and in Polars' non-strict casts. Dates in milliseconds no longer fail to convert to second timestamps. --- .../effect/src/dataframe/internal/temporal.ts | 58 +++++++++++-------- packages/effect/test/dataframe/Query.test.ts | 26 +++++++++ 2 files changed, 60 insertions(+), 24 deletions(-) diff --git a/packages/effect/src/dataframe/internal/temporal.ts b/packages/effect/src/dataframe/internal/temporal.ts index d62fe4b35a9..3c9c52cb377 100644 --- a/packages/effect/src/dataframe/internal/temporal.ts +++ b/packages/effect/src/dataframe/internal/temporal.ts @@ -53,21 +53,28 @@ const floorDiv = (a: bigint, b: bigint): bigint => { return (a % b !== BigInt(0)) && ((a < BigInt(0)) !== (b < BigInt(0))) ? q - BigInt(1) : q } +/** `v`'s integers times `factor` as `target`; null where the product leaves 64 bits. */ +const multiply = (v: V.Vector, factor: bigint, target: S.DataType): V.Vector => { + const x = asI64(v) + const out = new BigInt64Array(x.length) + const valid = new Uint8Array(x.length) + for (let i = 0; i < x.length; i++) { + const y = x[i] * factor + out[i] = BigInt.asIntN(64, y) + valid[i] = V.isValid(v, i) && out[i] === y ? 1 : 0 + } + return V.i64(target, out, V.compactValid(valid)) +} + /** - * Rescales integer counts of `from` units to `to` units, rounding down - * (toward zero when `truncate`, as Polars rescales durations). + * Rescales integer counts of `from` units to `to` units as `target`, + * rounding down (toward zero when `truncate`, as Polars rescales + * durations); counts that leave 64 bits become null. */ -const rescale = (values: BigInt64Array, from: S.TimeUnit, to: S.TimeUnit, truncate = false): BigInt64Array => { - if (from === to) return values - const out = new BigInt64Array(values.length) - if (fineness[to] > fineness[from]) { - const f = BigInt(perSecond[to] / perSecond[from]) - for (let i = 0; i < values.length; i++) out[i] = BigInt.asIntN(64, values[i] * f) - } else { - const f = BigInt(perSecond[from] / perSecond[to]) - for (let i = 0; i < values.length; i++) out[i] = truncate ? values[i] / f : floorDiv(values[i], f) - } - return out +const rescale = (v: V.Vector, from: S.TimeUnit, to: S.TimeUnit, target: S.DataType, truncate = false): V.Vector => { + if (fineness[to] > fineness[from]) return multiply(v, BigInt(perSecond[to] / perSecond[from]), target) + const f = BigInt(perSecond[from] / perSecond[to]) + return V.i64(target, BigInt64Array.from(asI64(v), (x) => (truncate ? x / f : floorDiv(x, f))), v.valid) } const dayUnits = (unit: S.TimeUnit): bigint => BigInt(86_400 * perSecond[unit]) @@ -288,27 +295,24 @@ const asI64 = (v: V.Vector): BigInt64Array => /** * Casts between dates, timestamps, durations and their integers; `undefined` - * when neither side is temporal. + * when neither side is temporal. Values the target can't hold become null. * * @internal */ export const cast = (v: V.Vector, target: S.DataType): V.Vector | undefined => { const from = v.type if (!isTemporal(from) && !isTemporal(target)) return undefined - if (from._tag === "Timestamp" && target._tag === "Timestamp") { - return V.i64(target, rescale(asI64(v), from.unit, target.unit), v.valid) - } - if (from._tag === "Duration" && target._tag === "Duration") { - return V.i64(target, rescale(asI64(v), from.unit, target.unit, true), v.valid) - } + if (from._tag === "Timestamp" && target._tag === "Timestamp") return rescale(v, from.unit, target.unit, target) + if (from._tag === "Duration" && target._tag === "Duration") return rescale(v, from.unit, target.unit, target, true) if (from._tag === "Timestamp" && target._tag === "Date") { return target.unit === "day" ? V.i32(target, Int32Array.from(days(v)), v.valid) - : V.i64(target, rescale(asI64(v), from.unit, "millisecond"), v.valid) + : rescale(v, from.unit, "millisecond", target) } if (from._tag === "Date" && target._tag === "Timestamp") { - const per = from.unit === "day" ? dayUnits(target.unit) : BigInt(perSecond[target.unit] / 1000) - return V.i64(target, BigInt64Array.from(asI64(v), (x) => BigInt.asIntN(64, x * per)), v.valid) + return from.unit === "day" + ? multiply(v, dayUnits(target.unit), target) + : rescale(v, "millisecond", target.unit, target) } // Integers and temporal values of the same width reinterpret each other. if (target._tag === "Int" || isTemporal(target)) { @@ -316,7 +320,13 @@ export const cast = (v: V.Vector, target: S.DataType): V.Vector | undefined => { if (kind === "i64") return V.i64(target, asI64(v), v.valid) if (kind === "i32" && v.kind === "i32") return V.i32(target, v.values, v.valid) if (kind === "i32" && v.kind === "i64") { - return V.i32(target, Int32Array.from(v.values, (x) => Number(BigInt.asIntN(32, x))), v.valid) + const valid = new Uint8Array(v.length) + const out = Int32Array.from(v.values, (x, i) => { + const y = Number(BigInt.asIntN(32, x)) + valid[i] = V.isValid(v, i) && BigInt(y) === x ? 1 : 0 + return y + }) + return V.i32(target, out, V.compactValid(valid)) } } return undefined diff --git a/packages/effect/test/dataframe/Query.test.ts b/packages/effect/test/dataframe/Query.test.ts index b2eeedbc6ab..b6d5a09c033 100644 --- a/packages/effect/test/dataframe/Query.test.ts +++ b/packages/effect/test/dataframe/Query.test.ts @@ -195,6 +195,32 @@ describe("Query", () => { }))) ) assert.deepStrictEqual(casts, [{ fromFloat: null, fromUInt64: null, toUInt64: null }]) + // Expected values from Polars 2.1.0 non-strict casts. + const temporal = DataFrame.fromArrowTable(Result.getOrThrow(ArrowTable.fromColumns( + ArrowSchema.make([ + ArrowSchema.field("t", ArrowSchema.timestamp("millisecond")), + ArrowSchema.field("d", ArrowSchema.duration("millisecond")), + ArrowSchema.field("i", ArrowSchema.int64) + ]), + [[10n ** 17n], [10n ** 17n], [2n ** 40n]] + ))) as DataFrame.DataFrame as DataFrame.DataFrame<{ t: DateTime.Utc; d: Duration.Duration; i: bigint }> + const overflows = yield* rows( + Query.from(temporal).pipe(Query.select((c) => ({ + t: Expr.cast(c.t, "Timestamp[ns]"), + d: Expr.cast(c.d, "Duration[ns]"), + i: Expr.cast(c.i, "Date") + }))) + ) + assert.deepStrictEqual(overflows, [{ t: null, d: null, i: null }]) + })) + ) + assert.deepStrictEqual(DataFrame.toRows(df), [{ sum: 301, gt: false, when: 1, coalesce: 1 }]) + assert.deepStrictEqual(df.schema.fields.map((f) => f.type), [ + ArrowSchema.int16, + ArrowSchema.bool, + ArrowSchema.int16, + ArrowSchema.int16 + ]) })) it.effect("temporal operands of mixed units meet in the coarser unit, as in Polars", () => From c46f6fe116a415c487f014d3c24b7c44873563b3 Mon Sep 17 00:00:00 2001 From: Effect Bot Date: Sun, 15 Nov 2026 13:58:44 +0000 Subject: [PATCH 31/88] Fail implicit conversions to a common type that can't hold a value Concatenation, join keys, unpivot, streamed partial merges and the operand promotion of arithmetic, comparisons, when and coalesce converted values to their common type with the explicit cast, so a value the type can't hold, such as a decimal leaving 64-bit storage at the larger scale, became a silent null in a non-null column. They now go through one checked coercion that fails with a DataFrameError; Expr.cast keeps making such values null. An integer literal a column's type can't hold now widens it to the smallest signed type holding the literal, as in Polars (Int8 + 300 is Int16), instead of becoming null. --- packages/effect/src/dataframe/Expr.ts | 9 +- packages/effect/src/dataframe/Query.ts | 9 +- .../effect/src/dataframe/internal/frame.ts | 4 +- .../effect/src/dataframe/internal/kernels.ts | 96 ++++++++++++++----- packages/effect/test/dataframe/Query.test.ts | 33 +++++++ 5 files changed, 120 insertions(+), 31 deletions(-) diff --git a/packages/effect/src/dataframe/Expr.ts b/packages/effect/src/dataframe/Expr.ts index 21efa1354a2..d1df8de2c97 100644 --- a/packages/effect/src/dataframe/Expr.ts +++ b/packages/effect/src/dataframe/Expr.ts @@ -30,7 +30,11 @@ * returns a float; `floorDivide` and `mod` round towards negative infinity, * and an integer division by zero gives `null`. * - An integer literal takes the integer type of the column it meets - * (`c.int8Column + 1` stays Int8). + * (`c.int8Column + 1` stays Int8) when the type holds it, and widens it + * otherwise (`c.int8Column + 300` is Int16). + * - Operands of different types meet in their common type. A value the + * common type can't hold fails the query with a `DataFrameError`; only + * an explicit {@link cast} makes it `null`. * - Decimals are stored as 64-bit unscaled integers, so they hold up to 18 * digits. Unlike integers they never wrap: a decimal result outside that * storage (from arithmetic, sums, `abs`, `negate`, `round` or a @@ -261,7 +265,8 @@ export const fromPlan: { /** * A constant. Integers between -2^31 and 2^31 - 1 adopt the integer type of - * the column they meet; other numbers are Float64; bigints are Int64. + * the column they meet when it holds them; other numbers are Float64; + * bigints are Int64. * * @stability experimental * @category constructors diff --git a/packages/effect/src/dataframe/Query.ts b/packages/effect/src/dataframe/Query.ts index d5998edb71c..39e1b05dc96 100644 --- a/packages/effect/src/dataframe/Query.ts +++ b/packages/effect/src/dataframe/Query.ts @@ -467,6 +467,13 @@ export const dropNulls: { * The rows of every query, one after the other. The queries must have the * same columns. * + * **Gotchas** + * + * Columns of different Arrow types (decimals of different scales, integers + * of different widths) take their common type, as in Polars. A value the + * common type can't hold, such as a decimal that leaves 64-bit storage at + * the larger scale, fails with a `DataFrameError` instead of becoming null. + * * @stability experimental * @category combining * @since 4.1.5 @@ -823,7 +830,7 @@ export const pivot: { * Stacks the `on` columns into one `value` column (`"value"` by default), * one block of rows per column, with the column's name in `variable` * (`"variable"` by default) and the `index` columns repeated. The values - * take the common type of the stacked columns. + * take the common type of the stacked columns; a value it can't hold fails. * * @stability experimental * @category reshaping diff --git a/packages/effect/src/dataframe/internal/frame.ts b/packages/effect/src/dataframe/internal/frame.ts index 10ed0195a0b..4ae83a5ae12 100644 --- a/packages/effect/src/dataframe/internal/frame.ts +++ b/packages/effect/src/dataframe/internal/frame.ts @@ -379,7 +379,7 @@ const unifyKeys = (l: V.Vector, r: V.Vector): [V.Vector, V.Vector] => { if (D.sameType(l.type, r.type)) return [l, r] const st = D.supertype(l.type, undefined, r.type, undefined) if (st === undefined) return V.fail(`cannot join ${D.describe(l.type)} with ${D.describe(r.type)}`) - return [K.cast(l, st.type), K.cast(r, st.type)] + return [K.coerce(l, st.type), K.coerce(r, st.type)] } /** @@ -744,7 +744,7 @@ export const unpivot = ( return st === undefined ? V.fail(`cannot unpivot ${D.describe(t)} with ${D.describe(p.type)}`) : st.type }, parts.length === 0 ? S.nullType : parts[0].type) const variable = V.stringsFrom(S.utf8, options.on.flatMap((name) => Array.from({ length: n }, () => name))) - const value = k === 0 ? V.nulls(0) : V.concat(parts.map((p) => K.cast(p, type))) + const value = k === 0 ? V.nulls(0) : V.concat(parts.map((p) => K.coerce(p, type))) return make( [...keys, new Column(options.variable, variable, undefined, false), new Column(options.value, value, undefined)], n * k diff --git a/packages/effect/src/dataframe/internal/kernels.ts b/packages/effect/src/dataframe/internal/kernels.ts index f9f416bde1f..b286dff308f 100644 --- a/packages/effect/src/dataframe/internal/kernels.ts +++ b/packages/effect/src/dataframe/internal/kernels.ts @@ -97,8 +97,8 @@ const INT_SYNTAX = /^[+-]?\d+$/ const formatNumber = (x: number): string => String(x) /** - * Converts `v` to `target`. Values that don't fit become null (Polars' - * non-strict casts). + * Converts `v` to `target`, as `Expr.cast` does. Values that don't fit + * become null (Polars' non-strict casts). * * @internal */ @@ -279,18 +279,45 @@ export const commonType = (types: ReadonlyArray): S.DataType => return common === undefined ? V.fail(`cannot concatenate ${D.describe(t)} with ${D.describe(u)}`) : common.type }) -/** Concatenates vectors, cast to their {@link commonType}. @internal */ +/** + * Converts `v` to `target`, a type it shares with other values (their + * {@link commonType}, an operation's supertype) without being asked to: a + * value `target` can't represent fails, where {@link cast} makes it null. + * Conversions to the common type may still round, as Polars' do (integers + * to floats, timestamps to a coarser unit). + * + * @internal + */ +export const coerce = (v: V.Vector, target: S.DataType): V.Vector => { + const out = cast(v, target) + if (out.valid !== undefined || out.kind === "null") { + for (let i = 0; i < v.length; i++) { + if (V.isValid(v, i) && !V.isValid(out, i)) { + return V.fail(`${D.describe(v.type)} values don't fit their common type ${D.describe(target)}`) + } + } + } + return out +} + +/** Concatenates vectors, coerced to their {@link commonType}. @internal */ export const concat = (parts: ReadonlyArray): V.Vector => { const type = commonType(parts.map((p) => p.type)) - return V.concat(parts.map((p) => cast(p, type))) + return V.concat(parts.map((p) => coerce(p, type))) } -/** Casts a value, keeping its shape. @internal */ +/** Casts a value ({@link cast}), keeping its shape. @internal */ export const castValue = (value: Value, target: S.DataType): Value => D.sameType(value.vector.type, target) ? value : { vector: cast(value.vector, target), scalar: value.scalar, dynamic: undefined } +/** Coerces a value ({@link coerce}), keeping its shape. */ +const coerceValue = (value: Value, target: S.DataType): Value => + D.sameType(value.vector.type, target) + ? value + : { vector: coerce(value.vector, target), scalar: value.scalar, dynamic: undefined } + // ----------------------------------------------------------------------------- // Arithmetic // ----------------------------------------------------------------------------- @@ -298,8 +325,25 @@ export const castValue = (value: Value, target: S.DataType): Value => /** @internal */ export type ArithOp = "add" | "sub" | "mul" | "div" | "floorDiv" | "mod" +/** + * The type and dynamism of an operand next to one of type `other`: an + * integer literal whose value `other` can't hold commits to the smallest + * signed integer type that holds it, instead of adopting `other`, as in + * Polars (`Int8 + 300` is Int16). + */ +const operand = (v: Value, other: S.DataType): readonly [S.DataType, D.Dynamic] => { + if (v.dynamic !== "int" || other._tag !== "Int" || v.vector.kind !== "i32") return [v.vector.type, v.dynamic] + const x = v.vector.values[0] + const [lo, hi] = intRange(other) + if (x >= lo && x <= hi) return [v.vector.type, v.dynamic] + return [x >= -128 && x <= 127 ? S.int8 : x >= -32768 && x <= 32767 ? S.int16 : S.int32, undefined] +} + +/** The common type of two operands (`D.supertype`, literals fitted by {@link operand}). */ +const supertype = (a: Value, b: Value) => D.supertype(...operand(a, b.vector.type), ...operand(b, a.vector.type)) + const unify = (op: string, a: Value, b: Value) => { - const st = D.supertype(a.vector.type, a.dynamic, b.vector.type, b.dynamic) + const st = supertype(a, b) if (st === undefined || !(D.isNumeric(st.type) || st.type._tag === "Null")) { return V.fail(`${op} is not defined for ${D.describe(a.vector.type)} and ${D.describe(b.vector.type)}`) } @@ -319,7 +363,7 @@ const temporalArith = (op: ArithOp, a: Value, b: Value, length: number): Value = const at = a.vector.type const bt = b.vector.type const unsupported = () => V.fail(`${op} is not defined for ${D.describe(at)} and ${D.describe(bt)}`) - const asInt = (v: Value, type: S.DataType) => relabel(castValue(v, type), S.int64) + const asInt = (v: Value, type: S.DataType) => relabel(coerceValue(v, type), S.int64) const point = (t: S.DataType) => t._tag === "Date" || t._tag === "Timestamp" if (op === "sub" && point(at) && point(bt)) { if (at._tag === "Date" && bt._tag === "Date") { @@ -341,7 +385,7 @@ const temporalArith = (op: ArithOp, a: Value, b: Value, length: number): Value = return relabel(arith(op, asInt(a, d), asInt(b, d), length), d) } if (op === "mul" && at._tag === "Duration" && D.isInteger(bt)) { - return relabel(arith("mul", relabel(a, S.int64), castValue(b, S.int64), length), at) + return relabel(arith("mul", relabel(a, S.int64), coerceValue(b, S.int64), length), at) } if (op === "mul" && bt._tag === "Duration" && D.isInteger(at)) return temporalArith(op, b, a, length) return unsupported() @@ -352,14 +396,14 @@ const temporalArith = (op: ArithOp, a: Value, b: Value, length: number): Value = * types); other operations and mixes with floats run on floats. */ const decimalArith = (op: ArithOp, a: Value, b: Value, length: number): Value => { - const st = D.supertype(a.vector.type, a.dynamic, b.vector.type, b.dynamic) + const st = supertype(a, b) if (st === undefined) { return V.fail(`${op} is not defined for ${D.describe(a.vector.type)} and ${D.describe(b.vector.type)}`) } if (!Dec.isDecimal(st.type) || (op !== "add" && op !== "sub" && op !== "mul")) { - return arith(op, castValue(a, S.float64), castValue(b, S.float64), length) + return arith(op, coerceValue(a, S.float64), coerceValue(b, S.float64), length) } - const asDecimal = (v: Value): Value => (Dec.isDecimal(v.vector.type) ? v : castValue(v, Dec.decimal(0))) + const asDecimal = (v: Value): Value => (Dec.isDecimal(v.vector.type) ? v : coerceValue(v, Dec.decimal(0))) let left = asDecimal(a) const right = asDecimal(b) const n = left.scalar && right.scalar ? 1 : length @@ -393,8 +437,8 @@ export const arith = (op: ArithOp, a: Value, b: Value, length: number): Value => const valid = resultValid(a, b, n) if (valid === "all-null") return { vector: nullsOf(type, n), scalar: a.scalar && b.scalar, dynamic } // A scalar on the left: materialise it (rare) so kernels only see col op col / col op scalar. - let left = castValue(a, type) - const right = castValue(b, type) + let left = coerceValue(a, type) + const right = coerceValue(b, type) if (left.scalar && !right.scalar) left = column(materialize(left, n)) const lv = left.vector const rv = right.vector @@ -571,7 +615,7 @@ export const unaryNumeric = (op: "neg" | "abs" | "sqrt" | "round", a: Value, dec const v = a.vector const n = v.length if (v.kind === "null") return a - if (op === "sqrt" && v.kind !== "f64") return unaryNumeric(op, castValue(a, S.float64)) + if (op === "sqrt" && v.kind !== "f64") return unaryNumeric(op, coerceValue(a, S.float64)) if (op === "round" && !(Number.isInteger(decimals) && decimals >= 0)) { return V.fail(`round needs a non-negative whole number of decimals, got ${decimals}`) } @@ -685,23 +729,23 @@ export const compare = (op: CompareOp, a: Value, b: Value, length: number): Valu let left = a let right = b if (D.isNumeric(at) && D.isNumeric(bt) && !D.sameType(at, bt)) { - const st = D.supertype(at, a.dynamic, bt, b.dynamic)!.type + const st = supertype(a, b)!.type // Compare mixed widths as Float64 unless both are 64-bit integers. const target = V.kindOf(st) === "i64" && V.kindOf(at) === "i64" && V.kindOf(bt) === "i64" ? st : S.float64 - left = castValue(a, V.kindOf(st) === "i32" ? st : target) - right = castValue(b, V.kindOf(st) === "i32" ? st : target) + left = coerceValue(a, V.kindOf(st) === "i32" ? st : target) + right = coerceValue(b, V.kindOf(st) === "i32" ? st : target) } else if (at._tag === "Null" || bt._tag === "Null") { return { vector: nullsOf(S.bool, n), scalar: a.scalar, dynamic: undefined } } else if ((Dec.isDecimal(at) || Dec.isDecimal(bt)) && !D.sameType(at, bt)) { - const st = D.supertype(at, a.dynamic, bt, b.dynamic) + const st = supertype(a, b) if (st === undefined) return V.fail(`cannot compare ${D.describe(at)} with ${D.describe(bt)}`) - left = castValue(a, st.type) - right = castValue(b, st.type) + left = coerceValue(a, st.type) + right = coerceValue(b, st.type) } else if ((T.isTemporal(at) || T.isTemporal(bt)) && !D.sameType(at, bt)) { const st = T.supertype(at, bt) if (st === undefined) return V.fail(`cannot compare ${D.describe(at)} with ${D.describe(bt)}`) - left = castValue(a, st) - right = castValue(b, st) + left = coerceValue(a, st) + right = coerceValue(b, st) } else if (!D.sameType(at, bt) && !(D.isBinaryLike(at) && D.isBinaryLike(bt))) { return V.fail(`cannot compare ${D.describe(at)} with ${D.describe(bt)}`) } @@ -1012,7 +1056,7 @@ export const isNaN_ = (a: Value): Value => { * @internal */ export const when = (cond: Value, then: Value, otherwise: Value, length: number): Value => { - const st = D.supertype(then.vector.type, then.dynamic, otherwise.vector.type, otherwise.dynamic) + const st = supertype(then, otherwise) if (st === undefined) { return V.fail( `when branches have incompatible types ${D.describe(then.vector.type)} and ${D.describe(otherwise.vector.type)}` @@ -1025,7 +1069,7 @@ export const when = (cond: Value, then: Value, otherwise: Value, length: number) const cv = c.values const ca = c.valid for (let i = 0; i < n; i++) mask[i] = (cv[i] & (ca === undefined ? 1 : ca[i])) === 1 ? 0 : 1 - const branches = [materialize(castValue(then, st.type), n), materialize(castValue(otherwise, st.type), n)] + const branches = [materialize(coerceValue(then, st.type), n), materialize(coerceValue(otherwise, st.type), n)] return { vector: select(branches, mask), scalar: allScalar, dynamic: st.dynamic } } @@ -1088,14 +1132,14 @@ export const coalesce = (values: ReadonlyArray, length: number): Value => let type = acc.vector.type let dynamic = acc.dynamic for (const v of values.slice(1)) { - const st = D.supertype(type, dynamic, v.vector.type, v.dynamic) + const st = D.supertype(type, dynamic, ...operand(v, type)) if (st === undefined) return V.fail(`coalesce arguments have incompatible types`) type = st.type dynamic = st.dynamic } const allScalar = values.every((v) => v.scalar) const n = allScalar ? 1 : length - const vectors = values.map((v) => materialize(castValue(v, type), n)) + const vectors = values.map((v) => materialize(coerceValue(v, type), n)) const choice = new Int32Array(n) for (let i = 0; i < n; i++) { let k = 0 diff --git a/packages/effect/test/dataframe/Query.test.ts b/packages/effect/test/dataframe/Query.test.ts index b6d5a09c033..590c91c15f9 100644 --- a/packages/effect/test/dataframe/Query.test.ts +++ b/packages/effect/test/dataframe/Query.test.ts @@ -213,6 +213,39 @@ describe("Query", () => { ) assert.deepStrictEqual(overflows, [{ t: null, d: null, i: null }]) })) + + it.effect("implicit conversions to a common type fail on values it can't hold", () => + Effect.gen(function*() { + const Price = Schema.Struct({ d: Schema.BigDecimal }) + const price = (d: string, precision: number, scale: number) => + DataFrame.fromRows(Price, [{ d: BigDecimal.fromStringUnsafe(d) }], { + types: { d: ArrowSchema.decimal(precision, scale) } + }) + const big = Query.from(price("6000000000000000000", 19, 0)) + const small = Query.from(price("1.20", 18, 2)) + const groups = Query.fromPlan({ _tag: "Table", name: "t" }).pipe( + Query.groupBy("d"), + Query.agg(() => ({ n: Expr.count() })), + Effect.provide(Catalog.layer({ t: Stream.make(price("6000000000000000000", 19, 0), price("1.20", 18, 2)) })) + ) + const failures: ReadonlyArray> = [ + Query.concat([big, small]), + groups, + Query.join(big, small, { on: "d", how: "left" }), + Query.filter(big, (c) => Expr.eq(c.d, BigDecimal.fromStringUnsafe("1.20"))) + ] + for (const query of failures) { + const error = yield* Effect.flip(query) + assert.strictEqual(error.message, "Decimal[19, 0] values don't fit their common type Decimal[38, 2]") + } + // An integer literal a column's type can't hold widens it, as in Polars 2.1.0. + const df = yield* Query.from(DataFrame.fromColumns({ x: Int8Array.of(1) })).pipe( + Query.select((c) => ({ + sum: Expr.add(c.x, 300), + gt: Expr.gt(c.x, 1000), + when: Expr.when(Expr.gt(c.x, 0), c.x, 1000), + coalesce: Expr.coalesce(c.x, 1000) + })) ) assert.deepStrictEqual(DataFrame.toRows(df), [{ sum: 301, gt: false, when: 1, coalesce: 1 }]) assert.deepStrictEqual(df.schema.fields.map((f) => f.type), [ From 76f7ebdf37ed395d930936c8885d42678423979b Mon Sep 17 00:00:00 2001 From: Effect Bot Date: Sun, 15 Nov 2026 14:09:29 +0000 Subject: [PATCH 32/88] Give streamed aggregates the value and type of the concatenated batches Partial sums wrapped in an early batch's type before a later batch widened it, and means, variances and other partials finished in the first batch's type. Partial sums now stay exact (Float64 for float inputs) and each state keeps, per aggregate input, its smallest and largest value converted to the inputs' common type: a value that type can't hold fails as concatenating fails, and finish applies the common type's result type and overflow once. Zero-row batches add their types without rows, so they no longer drop out of the common type, and key columns are nullable when any batch's is. --- .../src/dataframe/internal/aggregate.ts | 86 +++++---- .../effect/src/dataframe/internal/partial.ts | 170 ++++++++++++------ .../effect/src/dataframe/internal/stream.ts | 72 ++++---- packages/effect/test/dataframe/Query.test.ts | 96 +++++++++- 4 files changed, 286 insertions(+), 138 deletions(-) diff --git a/packages/effect/src/dataframe/internal/aggregate.ts b/packages/effect/src/dataframe/internal/aggregate.ts index 0bdcbd27f1a..ce55952e26e 100644 --- a/packages/effect/src/dataframe/internal/aggregate.ts +++ b/packages/effect/src/dataframe/internal/aggregate.ts @@ -115,62 +115,70 @@ export const aggregate = (fn: AggFn, v: V.Vector | undefined, g: Grouping): V.Ve } } -const sum = (v: V.Vector, g: Grouping): V.Vector => { +const sum = (v: V.Vector, g: Grouping): V.Vector => sumColumn(v.type, sums(v, g)) + +/** + * Per-group sums of the valid values: Float64 for float columns, exact + * integers (unscaled for decimals) for every other column `sum` takes. + * + * @internal + */ +export type Sums = Float64Array | ReadonlyArray + +/** Per-group {@link Sums} of a column. @internal */ +export const sums = (v: V.Vector, g: Grouping): Sums => { const ids = g.ids const n = ids.length - const groups = g.count - const type = D.sumType(v.type) switch (v.kind) { case "null": - return V.i32(S.int32, new Int32Array(groups)) - case "f64": { - const x = v.values - const acc = new Float64Array(groups) - const valid = v.valid - if (valid === undefined) { for (let i = 0; i < n; i++) acc[ids[i]] += x[i] } - else for (let i = 0; i < n; i++) if (valid[i] === 1) acc[ids[i]] += x[i] - V.wrapInPlace(type, acc) - V.roundFloat32InPlace(type, acc) - return V.f64(type, acc) + return new Array(g.count).fill(I0) + case "i64": { + const x = V.isUnsigned64(v.type) ? new BigUint64Array(v.values.buffer, v.values.byteOffset, v.length) : v.values + const acc = new Array(g.count).fill(I0) + for (let i = 0; i < n; i++) if (V.isValid(v, i)) acc[ids[i]] += x[i] + return acc } - case "i32": { + case "f64": + case "i32": + case "bool": { const x = v.values - // Accumulate in Float64 (exact up to 2^53), then wrap to Int32. - const acc = new Float64Array(groups) + const acc = new Float64Array(g.count) const valid = v.valid if (valid === undefined) { for (let i = 0; i < n; i++) acc[ids[i]] += x[i] } else for (let i = 0; i < n; i++) if (valid[i] === 1) acc[ids[i]] += x[i] - const out = new Int32Array(groups) - for (let k = 0; k < groups; k++) out[k] = Number(BigInt.asIntN(32, BigInt(acc[k]))) - return V.i32(type, out) - } - case "bool": { - const x = v.values - const acc = new Float64Array(groups) - for (let i = 0; i < n; i++) if (V.isValid(v, i)) acc[ids[i]] += x[i] - return V.f64(S.uint32, acc) + // Integers accumulate exactly in Float64 up to 2^53. + return v.type._tag === "Float" ? acc : Array.from(acc, BigInt) } - case "i64": - return fromSums(type, exactSums(v, g)) default: return V.fail(`sum is not defined for ${D.describe(v.type)}`) } } -/** Per-group sums of 64-bit integers or decimals, exact (unbounded). @internal */ -export const exactSums = (v: V.I64, g: Grouping): Array => { - const x = v.values - const ids = g.ids - const acc = new Array(g.count).fill(I0) - const valid = v.valid - for (let i = 0; i < ids.length; i++) if (valid === undefined || valid[i] === 1) acc[ids[i]] += x[i] - return acc +/** + * The `sum` column of an input of type `inputType` from its {@link Sums}: + * integer sums wrap to the sum type, decimal sums fail outside 64 bits. + * + * @internal + */ +export const sumColumn = (inputType: S.DataType, sums: Sums): V.Vector => { + const type = D.sumType(inputType) + if (sums instanceof Float64Array || type._tag === "Float") { + const out = Float64Array.from(sums as ArrayLike, Number) + V.roundFloat32InPlace(type, out) + return V.f64(type, out) + } + switch (V.kindOf(type)) { + case "null": + return V.i32(S.int32, new Int32Array(sums.length)) + case "i32": + return V.i32(type, Int32Array.from(sums, (x) => Number(BigInt.asIntN(32, x)))) + case "f64": + return V.f64(type, Float64Array.from(sums, (x) => Number(BigInt.asUintN(32, x)))) + default: + return V.i64(type, BigInt64Array.from(sums, Dec.fit(type))) + } } -/** Exact sums as a column of `type`, stored as `Dec.fit` says. @internal */ -export const fromSums = (type: S.DataType, sums: ReadonlyArray): V.Vector => - V.i64(type, BigInt64Array.from(sums, Dec.fit(type))) - /** Every value as a float, with validity. */ const asFloat = (v: V.Vector): { readonly x: Float64Array; readonly valid: Uint8Array | undefined } => { switch (v.kind) { diff --git a/packages/effect/src/dataframe/internal/partial.ts b/packages/effect/src/dataframe/internal/partial.ts index 1568f784fa3..a180bdc74ef 100644 --- a/packages/effect/src/dataframe/internal/partial.ts +++ b/packages/effect/src/dataframe/internal/partial.ts @@ -3,13 +3,17 @@ * batch) into partial aggregates that combine with other parts' into the * aggregates of all rows. * - * Sums, counts, minima, maxima, firsts and lasts combine by aggregating the - * partials again; means combine from sums and counts; variances from counts, - * means and M2 (Chan et al.). Medians and distinct counts don't combine. - * Sums of 64-bit integers and decimals stay exact until `finish`, so whether - * a decimal sum fits 64 bits doesn't depend on how the rows are split. - * Parts may differ in type (stream batches of untyped tables): partials - * combine in their common type, as concatenated rows would. + * Counts, minima, maxima, firsts and lasts combine by aggregating the + * partials again; sums combine from per-part sums (exact for integers and + * decimals); means from sums and counts; variances from counts, means and + * M2 (Chan et al.). Medians and distinct counts don't combine. + * + * Parts may differ in type (stream batches of untyped tables). The result + * is the aggregate of the concatenated rows: each state keeps the smallest + * and largest input value converted to the inputs' common type, so a value + * that type can't hold fails as concatenating would, and `finish` gives + * the result the type and overflow of that common type. Float results may + * differ in rounding, as float sums of the same values in another order do. * * @internal */ @@ -19,12 +23,13 @@ import * as Dec from "./decimal.ts" import * as D from "./dtype.ts" import * as Hash from "./hash.ts" import * as K from "./kernels.ts" +import * as T from "./temporal.ts" import * as V from "./vector.ts" /** @internal */ export type Part = | { readonly _tag: "vector"; readonly vector: V.Vector } - | Sums + | { readonly _tag: "sums"; readonly sums: Agg.Sums } | { readonly _tag: "mean"; readonly sum: Float64Array; readonly count: Float64Array } | { readonly _tag: "moments" @@ -34,27 +39,21 @@ export type Part = } | { readonly _tag: "none" } -/** Exact sums of 64-bit integers or decimals, and the type of their sum. */ -interface Sums { - readonly _tag: "sums" - readonly type: S.DataType - readonly sums: ReadonlyArray -} - const scaleOf = (t: S.DataType): number => (Dec.isDecimal(t) ? t.scale : 0) -const reaggregated = new Set(["sum", "count", "len", "min", "max", "first", "last"]) +const reaggregated = new Set(["count", "len", "min", "max", "first", "last"]) /** Whether partial aggregates of `fn` combine. @internal */ export const combinable = (fn: Agg.AggFn): boolean => - reaggregated.has(fn) || fn === "mean" || fn === "std" || fn === "variance" + reaggregated.has(fn) || fn === "sum" || fn === "mean" || fn === "std" || fn === "variance" /** The partial aggregate of `fn` over `v` grouped by `g`. @internal */ export const partial = (fn: Agg.AggFn, v: V.Vector | undefined, g: Agg.Grouping): Part => { - if (fn === "sum" && v?.kind === "i64") return { _tag: "sums", type: D.sumType(v.type), sums: Agg.exactSums(v, g) } if (reaggregated.has(fn)) return { _tag: "vector", vector: Agg.aggregate(fn, v, g) } if (v === undefined) return V.fail(`${fn} needs an argument`) switch (fn) { + case "sum": + return { _tag: "sums", sums: Agg.sums(v, g) } case "mean": { const { count, sum } = Agg.sumCount(v, g) return { _tag: "mean", sum, count } @@ -67,32 +66,60 @@ export const partial = (fn: Agg.AggFn, v: V.Vector | undefined, g: Agg.Grouping) } } +/** Sums of parts whose inputs have `types`, in their common type `type`. */ +const combineSums = ( + parts: ReadonlyArray, + types: ReadonlyArray, + type: S.DataType, + ids: Int32Array, + count: number +): Agg.Sums => { + const float = type._tag === "Float" + const out = float ? new Float64Array(count) : new Array(count).fill(BigInt(0)) + let at = 0 + parts.forEach((p, c) => { + if (T.isTemporal(type) && !D.sameType(types[c], type)) { + V.fail(`sums of ${D.describe(types[c])} and ${D.describe(type)} values don't combine`) + } + if (float) { + const scale = 10 ** scaleOf(types[c]) + for (let j = 0; j < p.length; j++) (out as Float64Array)[ids[at + j]] += Number(p[j]) / scale + } else { + const factor = Dec.pow10(scaleOf(type) - scaleOf(types[c])) + for (let j = 0; j < p.length; j++) (out as Array)[ids[at + j]] += (p[j] as bigint) * factor + } + at += p.length + }) + return out +} + /** * Combines the partials of several parts: `parts[c]` holds part `c`'s - * partials for its groups, `ids` maps the concatenated groups of all parts - * to `count` combined groups. + * partials for its groups over inputs of `types[c]`, whose common type is + * `type`; `ids` maps the concatenated groups of all parts to `count` + * combined groups. * * @internal */ -export const combine = (fn: Agg.AggFn, parts: ReadonlyArray, ids: Int32Array, count: number): Part => { +export const combine = ( + fn: Agg.AggFn, + parts: ReadonlyArray, + types: ReadonlyArray, + type: S.DataType | undefined, + ids: Int32Array, + count: number +): Part => { const grouping: Agg.Grouping = { ids, count } - if (parts.every((p): p is Sums => p._tag === "sums")) { - const type = K.commonType(parts.map((p) => p.type)) - const sums = new Array(count).fill(BigInt(0)) - let at = 0 - for (const p of parts) { - const factor = Dec.pow10(scaleOf(type) - scaleOf(p.type)) - for (let j = 0; j < p.sums.length; j++) sums[ids[at + j]] += p.sums[j] * factor - at += p.sums.length + if (reaggregated.has(fn)) { + const vectors = parts.map((p) => (p as Extract).vector) + if (fn === "count" || fn === "len") { + return { _tag: "vector", vector: Agg.aggregate("sum", K.concat(vectors), grouping) } } - return { _tag: "sums", type, sums } + return { _tag: "vector", vector: Agg.aggregate(fn, V.concat(vectors.map((v) => K.coerce(v, type!))), grouping) } } - if (reaggregated.has(fn)) { - const vectors = parts.map((p) => - p._tag === "sums" ? Agg.fromSums(p.type, p.sums) : (p as Extract).vector - ) - const merge = fn === "count" || fn === "len" ? "sum" : fn - return { _tag: "vector", vector: Agg.aggregate(merge, K.concat(vectors), grouping) } + if (fn === "sum") { + const sums = parts.map((p) => (p as Extract).sums) + return { _tag: "sums", sums: combineSums(sums, types as ReadonlyArray, type!, ids, count) } } if (fn === "mean") { const sum = new Float64Array(count) @@ -131,17 +158,17 @@ export const combine = (fn: Agg.AggFn, parts: ReadonlyArray, ids: Int32Arr return V.fail(`${fn} cannot be computed from partial aggregates`) } -/** The aggregate column of a combined partial. @internal */ -export const finish = (fn: Agg.AggFn, part: Part, input: V.Vector | undefined): V.Vector => { +/** The aggregate column of a combined partial over inputs of `type`. @internal */ +export const finish = (fn: Agg.AggFn, part: Part, type: S.DataType | undefined): V.Vector => { switch (part._tag) { case "vector": return part.vector case "sums": - return Agg.fromSums(part.type, part.sums) + return Agg.sumColumn(type!, part.sums) case "mean": - return Agg.meanFrom(input!.type, part.sum, part.count) + return Agg.meanFrom(type!, part.sum, part.count) case "moments": - return Agg.varianceFrom(fn as "std" | "variance", input!.type, part.count, part.m2) + return Agg.varianceFrom(fn as "std" | "variance", type!, part.count, part.m2) case "none": return V.fail(`${fn} cannot be computed from partial aggregates`) } @@ -151,36 +178,79 @@ export const finish = (fn: Agg.AggFn, part: Part, input: V.Vector | undefined): // Grouped states // ----------------------------------------------------------------------------- -/** The groups of some rows: their keys (one row per group) and partial aggregates. @internal */ +/** + * The groups of some rows: their keys (one row per group), partial + * aggregates and, per aggregate input, its domain: the smallest and + * largest value in the inputs' common type. + * + * @internal + */ export interface State { readonly keys: ReadonlyArray + readonly domains: ReadonlyArray readonly parts: ReadonlyArray readonly count: number + readonly rows: number +} + +/** + * The smallest and largest valid value of `v`, the values that decide + * whether a numeric or temporal column converts to a type; no values for + * other columns. + */ +const domain = (v: V.Vector): V.Vector => { + if (v.kind !== "i32" && v.kind !== "i64" && v.kind !== "f64") return V.take(v, new Int32Array(0)) + const all = { ids: new Int32Array(v.length), count: 1 } + return V.concat([Agg.aggregate("min", v, all), Agg.aggregate("max", v, all)]) } -/** Groups some rows. @internal */ +/** Groups some rows; without keys, they form one group even when there are none. @internal */ export const group = ( keys: ReadonlyArray, inputs: ReadonlyArray<{ readonly fn: Agg.AggFn; readonly vector: V.Vector | undefined }>, n: number ): State => { - const g = Hash.groups(keys, n) + const g = keys.length === 0 ? { ids: new Int32Array(n), count: 1, firstRow: new Int32Array(1) } : Hash.groups(keys, n) return { keys: keys.map((k) => V.take(k, g.firstRow)), + domains: inputs.map(({ vector }) => vector === undefined ? undefined : domain(vector)), parts: inputs.map(({ fn, vector }) => partial(fn, vector, g)), - count: g.count + count: g.count, + rows: n } } -/** Combines states, keeping groups in order of first appearance. @internal */ +/** + * Combines states, keeping groups in order of first appearance. States + * without rows only add their types. + * + * @internal + */ export const merge = (fns: ReadonlyArray, states: ReadonlyArray): State => { if (states.length === 1) return states[0] - const total = states.reduce((s, x) => s + x.count, 0) const keys = states[0].keys.map((_, k) => K.concat(states.map((s) => s.keys[k]))) - const g = Hash.groups(keys, total) + const domains = states[0].domains.map((d, a) => + d === undefined ? undefined : domain(K.concat(states.map((s) => s.domains[a]!))) + ) + const filled = states.filter((s) => s.rows > 0) + if (filled.length === 0) { + return group(keys, fns.map((fn, a) => ({ fn, vector: domains[a] && V.take(domains[a], new Int32Array(0)) })), 0) + } + const g = Hash.groups(keys, filled.reduce((n, s) => n + s.count, 0)) return { keys: keys.map((k) => V.take(k, g.firstRow)), - parts: fns.map((fn, a) => combine(fn, states.map((s) => s.parts[a]), g.ids, g.count)), - count: g.count + domains, + parts: fns.map((fn, a) => + combine( + fn, + filled.map((s) => s.parts[a]), + filled.map((s) => s.domains[a]?.type), + domains[a]?.type, + g.ids, + g.count + ) + ), + count: g.count, + rows: filled.reduce((n, s) => n + s.rows, 0) } } diff --git a/packages/effect/src/dataframe/internal/stream.ts b/packages/effect/src/dataframe/internal/stream.ts index c18ba17b1d4..84bd17d8557 100644 --- a/packages/effect/src/dataframe/internal/stream.ts +++ b/packages/effect/src/dataframe/internal/stream.ts @@ -103,48 +103,38 @@ const aggregate = ( return collected(node, catalog) } const fns = specs.map((s) => s.fn) - // The first non-empty batch's input vectors and key columns, for result types. - let template: { readonly inputs: ReadonlyArray; readonly frame: F.Frame } | undefined - // The first batch, to aggregate over no rows when every batch is empty. - let first: F.Frame | undefined - return Stream.unwrap( - Stream.runFoldEffect( - batches(node.input, catalog), - (): Partial.State | undefined => undefined, - (state, batch) => - Q.attempt(() => { - first ??= batch - if (batch.length === 0) return state - const ctx = F.context(batch) - const inputs = specs.map((s) => (s.arg === undefined ? undefined : E.evaluateColumn(s.arg, ctx))) - template ??= { inputs, frame: batch } - const part = Partial.group( - keys.map((k) => F.column(batch, k).values()), - fns.map((fn, a) => ({ fn, vector: inputs[a] })), - batch.length - ) - return state === undefined ? part : Partial.merge(fns, [state, part]) - }) - ).pipe( - Effect.flatMap((state) => { - if (state === undefined || template === undefined) { - const empty = first - return empty === undefined ? runEmpty(node) : Q.attempt(() => Q.step(node, [empty])) - } - const t = template - return Q.attempt(() => { - const keyColumns = keys.map((k, i) => - new F.Column(k, state.keys[i], undefined, F.column(t.frame, k).nullable) - ) - const columns = specs.map((s, a) => - new F.Column(s.name, Partial.finish(s.fn, state.parts[a], t.inputs[a]), undefined) - ) - return F.make([...keyColumns, ...columns], node._tag === "Aggregate" ? 1 : state.count) - }) - }), - Effect.map(Stream.succeed) + return Stream.suspend(() => { + // Whether any batch's key column may hold nulls. + const nullable = keys.map(() => false) + return Stream.unwrap( + Stream.runFoldEffect( + batches(node.input, catalog), + (): Partial.State | undefined => undefined, + (state, batch) => + Q.attempt(() => { + const ctx = F.context(batch) + keys.forEach((k, i) => nullable[i] ||= F.column(batch, k).nullable) + const part = Partial.group( + keys.map((k) => F.column(batch, k).values()), + specs.map((s) => ({ fn: s.fn, vector: s.arg === undefined ? undefined : E.evaluateColumn(s.arg, ctx) })), + batch.length + ) + return state === undefined ? part : Partial.merge(fns, [state, part]) + }) + ).pipe( + Effect.flatMap((state) => + state === undefined ? runEmpty(node) : Q.attempt(() => { + const keyColumns = keys.map((k, i) => new F.Column(k, state.keys[i], undefined, nullable[i])) + const columns = specs.map((s, a) => + new F.Column(s.name, Partial.finish(s.fn, state.parts[a], state.domains[a]?.type), undefined) + ) + return F.make([...keyColumns, ...columns], state.count) + }) + ), + Effect.map(Stream.succeed) + ) ) - ) + }) } const slice = (input: Batches, offset: number, length: number | undefined): Batches => diff --git a/packages/effect/test/dataframe/Query.test.ts b/packages/effect/test/dataframe/Query.test.ts index 590c91c15f9..30b4754b01f 100644 --- a/packages/effect/test/dataframe/Query.test.ts +++ b/packages/effect/test/dataframe/Query.test.ts @@ -550,19 +550,99 @@ describe("Query", () => { ) assert.deepStrictEqual(yield* streamed(groups, source), [{ d: d("1.20"), n: 2 }, { d: d("1.23"), n: 1 }]) } - const numbers = Stream.make( - DataFrame.fromRows(Row, [{ x: 1 }], { types: { x: ArrowSchema.int32 } }), - DataFrame.fromRows(Row, [{ x: 1.5 }]) - ) - const sum = Query.fromPlan({ _tag: "Table", name: "t" }).pipe( - Query.aggregate((c) => ({ s: Expr.sum(c.x as Expr.Expr) })) - ) - assert.deepStrictEqual(yield* streamed(sum, numbers), [{ s: 2.5 }]) const lossy = Query.table("t", Price, { types: { d: ArrowSchema.decimal(18, 1) } }) const error = yield* Effect.flip(streamed(lossy, source)) assert.strictEqual(error.message, `column "d" has values that Decimal[18, 1] can't hold exactly`) })) + it.effect("aggregates of batches of different types are those of the batches concatenated, in either order", () => + Effect.gen(function*() { + const x = (type: ArrowSchema.DataType, values: ReadonlyArray) => + DataFrame.fromRows(Row, values.map((x) => ({ x })), { types: { x: type } }) + const untyped = Query.fromPlan({ _tag: "Table", name: "t" }) as typeof t + const query = (table: typeof t) => + Query.aggregate(table, (c) => ({ + sum: Expr.sum(c.x), + mean: Expr.mean(c.x), + min: Expr.min(c.x), + max: Expr.max(c.x), + count: Expr.count(c.x), + first: Expr.first(c.x), + last: Expr.last(c.x), + std: Expr.std(c.x), + variance: Expr.variance(c.x) + })) + // Expected values from Polars 2.1.0 over the concatenated batches, in + // order and reversed (first, last). Every result is Float64 but count. + // std and variance are left out where floats round them differently + // depending on how the rows are split. + const { float32, float64, int32 } = ArrowSchema + const cases: ReadonlyArray< + readonly [ + ReadonlyArray>, + Record, + readonly [readonly [number, number], readonly [number, number]] + ] + > = [ + [ + [x(int32, [2147483647]), x(int32, [1]), x(float64, [0.5])], + { sum: 2147483648.5, mean: 715827882.8333334, min: 0.5, max: 2147483647, count: 3 }, + [[2147483647, 0.5], [0.5, 2147483647]] + ], + [ + [x(float32, [1]), x(float64, [1.0000001])], + { sum: 2.0000001000000003, mean: 1.0000000500000001, min: 1, max: 1.0000001, count: 2 }, + [[1, 1.0000001], [1.0000001, 1]] + ], + [ + [x(float32, [1, 3]), x(float64, [2])], + { sum: 6, mean: 2, min: 1, max: 3, count: 3, std: 1, variance: 1 }, + [[1, 2], [2, 3]] + ], + [ + [x(float64, []), x(int32, [1, 2])], + { sum: 3, mean: 1.5, min: 1, max: 2, count: 2, std: Math.SQRT1_2, variance: 0.5 }, + [[1, 2], [1, 2]] + ] + ] + for (const [batches, expected, ends] of cases) { + for (const [order, [first, last]] of [[batches, ends[0]], [[...batches].reverse(), ends[1]]] as const) { + const results = [ + yield* query(untyped).pipe(Effect.provide(Catalog.layer({ t: Stream.fromIterable(order) }))), + yield* query(Query.concat(order.map(Query.from) as [typeof t, ...Array])) + ] + for (const df of results) { + const [row] = DataFrame.toRows(df) + const keys = Object.keys(expected) + assert.deepStrictEqual( + Object.fromEntries([...keys, "first", "last"].map((k) => [k, row[k as keyof typeof row]])), + { ...expected, first, last } + ) + assert.deepStrictEqual( + df.schema.fields.map((f) => f.type), + df.schema.fields.map((f) => f.name === "count" ? ArrowSchema.uint32 : float64) + ) + } + } + } + // A value the common type can't hold fails, as concatenating fails, though no minimum holds it. + const wide = [ + DataFrame.fromColumns({ x: BigUint64Array.of(2n ** 63n, 0n) }), + DataFrame.fromColumns({ x: BigInt64Array.of(-1n) }) + ] as const + const min = (table: Query.Query<{ readonly x: bigint }>) => + Query.aggregate(table, (c) => ({ min: Expr.min(c.x) })) + const failures = [ + min(Query.fromPlan({ _tag: "Table", name: "t" }) as Query.Query<{ readonly x: bigint }>).pipe( + Effect.provide(Catalog.layer({ t: Stream.make(...wide) })) + ), + min(Query.concat([Query.from(wide[0]), Query.from(wide[1])])) + ] + for (const query of failures) { + assert.strictEqual((yield* Effect.flip(query)).message, "UInt64 values don't fit their common type Int64") + } + })) + it.effect("zero-row batches aggregate like an empty table", () => Effect.gen(function*() { const query = Query.aggregate(t, (c) => ({ n: Expr.count(), total: Expr.sum(c.x), mean: Expr.mean(c.x) })) From d20786793f4807a3c4e908b3f992214a8ab4ee4e Mon Sep 17 00:00:00 2001 From: Effect Bot Date: Sun, 15 Nov 2026 14:12:39 +0000 Subject: [PATCH 33/88] Check a struct schema's checks on whole rows DataFrame.validate and table reads checked the values of fields a schema constrains, but ignored the struct's own checks (Schema.Struct(...).check), so rows the schema rejects were accepted. conform now runs them on the rows when the struct has any. A SQL engine checks tables column by column, so a query over a table whose schema has them fails when it is compiled. --- packages/effect/src/dataframe/DataFrame.ts | 3 ++- packages/effect/src/dataframe/Query.ts | 3 +++ .../effect/src/dataframe/internal/schema.ts | 25 ++++++++++++++++++- packages/effect/src/dataframe/internal/sql.ts | 2 +- .../effect/test/dataframe/DataFrame.test.ts | 7 ++++++ packages/effect/test/dataframe/Query.test.ts | 21 +++++++++++++--- packages/sql/pglite/test/DataFrameSql.test.ts | 4 +++ 7 files changed, 58 insertions(+), 7 deletions(-) diff --git a/packages/effect/src/dataframe/DataFrame.ts b/packages/effect/src/dataframe/DataFrame.ts index 4f69943c002..44c45fe56a7 100644 --- a/packages/effect/src/dataframe/DataFrame.ts +++ b/packages/effect/src/dataframe/DataFrame.ts @@ -390,7 +390,8 @@ export const empty = ( * declared type exactly: a decimal with more digits than the declared scale, * or a timestamp finer than the declared unit, fails. Values of fields the * schema constrains further (literals, enums, refinements, including inside - * structs, arrays and union members) are checked against it. + * structs, arrays and union members) are checked against it, and so are the + * rows when the struct itself has checks (`Schema.Struct(...).check(...)`). * * **Gotchas** * diff --git a/packages/effect/src/dataframe/Query.ts b/packages/effect/src/dataframe/Query.ts index 39e1b05dc96..2d204b952d8 100644 --- a/packages/effect/src/dataframe/Query.ts +++ b/packages/effect/src/dataframe/Query.ts @@ -247,6 +247,9 @@ export const from = (frame: DataFrame): Query => make({ _tag: "Values", * A table without the precision and scale of a `BigDecimal` field fails * when it is read. * + * A SQL engine can't check a struct's own checks on whole rows + * (`Schema.Struct(...).check(...)`): a query over such a table fails. + * * @stability experimental * @category constructors * @since 4.1.5 diff --git a/packages/effect/src/dataframe/internal/schema.ts b/packages/effect/src/dataframe/internal/schema.ts index c164b3acb5b..43fedeed970 100644 --- a/packages/effect/src/dataframe/internal/schema.ts +++ b/packages/effect/src/dataframe/internal/schema.ts @@ -500,11 +500,32 @@ export const valueChecks = ( return test === undefined ? [] : [[String(ps.name), test] as const] }) +/** A test of whole rows against a struct schema's own checks; `undefined` without any. */ +const rowsCheck = (schema: { readonly ast: AST.AST }): ((row: unknown) => boolean) | undefined => { + const checks = structOf(schema).checks + return checks && ((row) => Result.isSuccess(AST.runChecks(checks, row))) +} + +/** + * The {@link valueChecks} of a schema whose tables are checked column by + * column, as a SQL engine checks them: a schema with checks on whole rows + * (`Schema.Struct(...).check(...)`) fails. + * + * @internal + */ +export const columnChecks = ( + schema: { readonly ast: AST.AST } +): Array boolean]> => + rowsCheck(schema) === undefined + ? valueChecks(schema) + : V.fail("schemas with checks on whole rows aren't supported on SQL tables; check their fields instead") + /** * Checks that a frame has the declared columns and narrows it to them, in * their order, converting each to its declared type (exactly: a value the * declared type can't hold fails). With a `schema`, the values of fields it - * constrains beyond their column type are checked against it too. + * constrains beyond their column type, and its checks on whole rows, are + * checked against it too. * * @internal */ @@ -530,6 +551,8 @@ export const conform = ( return V.fail(`column ${JSON.stringify(name)} has values its schema rejects`) } } + const rows = rowsCheck(schema) + if (rows !== undefined && !F.toRows(narrowed).every(rows)) return V.fail("rows have values their schema rejects") } return narrowed } diff --git a/packages/effect/src/dataframe/internal/sql.ts b/packages/effect/src/dataframe/internal/sql.ts index b358b24f877..a23f5f4d7e2 100644 --- a/packages/effect/src/dataframe/internal/sql.ts +++ b/packages/effect/src/dataframe/internal/sql.ts @@ -1078,7 +1078,7 @@ const rowChecks = (table: Plan.Table, frame: F.Frame): Array => [ if (held !== undefined) checks.push(any(`NOT (${held})`, `has values that ${D.describe(type)} can't hold exactly`)) return checks }), - ...(table.schema === undefined ? [] : Schema_.valueChecks(table.schema)).map(([name, test]): Check => { + ...(table.schema === undefined ? [] : Schema_.columnChecks(table.schema)).map(([name, test]): Check => { const type = frame.columns.find((c) => c.name === name)!.type return { sql: `ARRAY_AGG(DISTINCT ${output(declared(table.columns!.find((c) => c.name === name)!), type)})`, diff --git a/packages/effect/test/dataframe/DataFrame.test.ts b/packages/effect/test/dataframe/DataFrame.test.ts index 4b88f5ba30e..d59f4221203 100644 --- a/packages/effect/test/dataframe/DataFrame.test.ts +++ b/packages/effect/test/dataframe/DataFrame.test.ts @@ -137,6 +137,13 @@ describe("DataFrame", () => { Schema.Struct({ n: Schema.Int }), `column "n" has values its schema rejects` ], + [ + DataFrame.fromColumns({ lo: [1, 2], hi: [2, 1] }), + Schema.Struct({ lo: Schema.Number, hi: Schema.Number }).check( + Schema.makeFilter((row) => row.lo < row.hi || "lo must be below hi") + ), + "rows have values their schema rejects" + ], [ DataFrame.fromRows(Schema.Struct({ s: Schema.Struct({ a: Schema.String }) }), [{ s: { a: "x" } }, { s: { a: "z" } diff --git a/packages/effect/test/dataframe/Query.test.ts b/packages/effect/test/dataframe/Query.test.ts index 30b4754b01f..84d0f076eed 100644 --- a/packages/effect/test/dataframe/Query.test.ts +++ b/packages/effect/test/dataframe/Query.test.ts @@ -73,11 +73,24 @@ describe("Query", () => { const streamed = yield* Effect.flip(Query.table("t", City).pipe(Effect.provide(Catalog.layer({ t: bad })))) assert.strictEqual(streamed.message, `column "city" is Float64, expected Utf8`) const Kind = Schema.Struct({ kind: Schema.Literals(["a", "b"]) }) - const kinds = DataFrame.fromColumns({ kind: ["a", "c"] }) - for (const t of [kinds, Stream.make(kinds)]) { - const error = yield* Effect.flip(Query.table("t", Kind).pipe(Effect.provide(Catalog.layer({ t })))) - assert.strictEqual(error.message, `column "kind" has values its schema rejects`) + const Range = Schema.Struct({ lo: Schema.Number, hi: Schema.Number }).check( + Schema.makeFilter((row) => row.lo < row.hi || "lo must be below hi") + ) + for ( + const [schema, frame, message] of [ + [Kind, DataFrame.fromColumns({ kind: ["a", "c"] }), `column "kind" has values its schema rejects`], + [Range, DataFrame.fromColumns({ lo: [1, 2], hi: [2, 1] }), "rows have values their schema rejects"] + ] as const + ) { + for (const t of [frame, Stream.make(frame)]) { + const error = yield* Effect.flip(Query.table("t", schema).pipe(Effect.provide(Catalog.layer({ t })))) + assert.strictEqual(error.message, message) + } } + const ranges = Query.table("t", Range).pipe( + Effect.provide(Catalog.layer({ t: DataFrame.fromColumns({ lo: [1], hi: [2] }) })) + ) + assert.deepStrictEqual(DataFrame.toRows(yield* ranges), [{ lo: 1, hi: 2 }]) const berlin = DataFrame.fromArrowTable(Result.getOrThrow(ArrowTable.fromRows( ArrowSchema.make([ArrowSchema.field("t", ArrowSchema.timestamp("millisecond", "Europe/Berlin"))]), [{ t: 0n }] diff --git a/packages/sql/pglite/test/DataFrameSql.test.ts b/packages/sql/pglite/test/DataFrameSql.test.ts index 3844471206f..a152a7a50d8 100644 --- a/packages/sql/pglite/test/DataFrameSql.test.ts +++ b/packages/sql/pglite/test/DataFrameSql.test.ts @@ -474,6 +474,10 @@ describe("effect/dataframe on PostgreSQL (PGlite)", { concurrent: false }, () => [ odd(Schema.Struct({ months: Schema.Duration })), `column "months" has durations in months, which have no fixed length` + ], + [ + contract(Schema.Struct({ s: Schema.String }).check(Schema.makeFilter((row) => row.s !== ""))), + "schemas with checks on whole rows aren't supported on SQL tables; check their fields instead" ] ] as const ) { From 190650d897ed292802bee89062f7d01ba1859d22 Mon Sep 17 00:00:00 2001 From: Effect Bot Date: Sun, 15 Nov 2026 13:59:49 +0000 Subject: [PATCH 34/88] Read each SQL table once, through a materialized source The row checks and the result read a table through a NOT MATERIALIZED source unless it called volatile functions, so PostgreSQL could scan it twice and a view that samples rows (TABLESAMPLE) or calls a function that differs per scan could pass the checks on rows the result doesn't return. Read every table name once, through one MATERIALIZED source that PostgreSQL computes once per statement, for the checks and the result of every node that reads it; timestamps are read as UTC by the nodes, since the source is shared. --- packages/effect/src/dataframe/Sql.ts | 35 +++++---- packages/effect/src/dataframe/internal/sql.ts | 74 ++++++++----------- packages/sql/pglite/test/DataFrameSql.test.ts | 14 +++- 3 files changed, 64 insertions(+), 59 deletions(-) diff --git a/packages/effect/src/dataframe/Sql.ts b/packages/effect/src/dataframe/Sql.ts index 3924e532311..b1afa36a2a2 100644 --- a/packages/effect/src/dataframe/Sql.ts +++ b/packages/effect/src/dataframe/Sql.ts @@ -72,7 +72,8 @@ import type * as Plan from "./Plan.ts" * **Gotchas** * * The query doesn't check that the tables satisfy their declared columns and - * schemas: {@link engine} runs a query with those checks instead. + * schemas: {@link engine} runs a query that checks the rows it reads + * instead. * * @stability experimental * @category models @@ -124,22 +125,23 @@ const attempt = (f: () => A): Result.Result => { * integers up to 32 bits, `bigint` for 64-bit ones, `smallint`, * `integer`, `real` or `double precision` for floats, string or enum * types for strings, `bytea`, `boolean`, `date`, `timestamp` or - * `timestamptz` (read as UTC), `interval`, and any type for `Null` - * columns. Then one - * query checks the rows and computes the result from the same rows: only - * nulls in `Null` columns, no nulls in other non-nullable columns, - * timestamps in the range of `DateTime`, intervals without months, and the - * distinct values of the columns the schema constrains beyond their type - * (literals, enums, refinements) accepted by the schema. Columns the query - * doesn't read are checked too. `explain` shows both queries. + * `timestamptz` (read as UTC), `interval`, `numeric` for decimals, and + * any type for `Null` columns. Then one query reads each table once, as a + * `MATERIALIZED` common table expression (computed once per statement), + * checks those rows and computes the result from exactly them, whatever + * the relation: views that sample rows or call functions included, and a + * table read twice gives the same rows twice. The checks: only nulls in + * `Null` columns, no nulls in other non-nullable columns, timestamps in + * the range of `DateTime`, intervals without months, and the distinct + * values of the columns the schema constrains beyond their type (literals, + * enums, refinements) accepted by the schema. Columns the query doesn't + * read are checked too. `explain` shows both queries. * * **Gotchas** * - * The row checks and the result read the same rows: they are one - * statement, so they read one snapshot, and a view that calls volatile - * functions (`nextval`, `random`) is computed once for both. Functions - * that give different results within a statement without being declared - * volatile are not supported. The column types are checked in a separate + * Every query reads all rows of its tables, which the checks need, and + * copies their declared columns once: filters can't use the tables' + * indexes. The column types are checked in a separate * statement first: run both inside a `REPEATABLE READ` transaction when * the tables may be altered concurrently. * @@ -176,8 +178,9 @@ export const engine: Effect.Effect = Effect.ge /** * A layer that runs queries on the database of the `SqlClient` in the - * context, with {@link engine} (tables are checked against their schemas - * as they are read). + * context, with {@link engine}: each query reads its tables once, checks + * those rows against their declared columns and schemas, and computes its + * result from exactly those rows. * * @stability experimental * @category layers diff --git a/packages/effect/src/dataframe/internal/sql.ts b/packages/effect/src/dataframe/internal/sql.ts index a23f5f4d7e2..f3da17a6e41 100644 --- a/packages/effect/src/dataframe/internal/sql.ts +++ b/packages/effect/src/dataframe/internal/sql.ts @@ -41,8 +41,8 @@ interface Compiler { readonly hidden: (name: string) => string /** The quoted row key. */ readonly row: string - /** The quoted name of each table's source (see {@link source}). */ - readonly sources: ReadonlyMap + /** The quoted name of the source of each table name (see {@link source}). */ + readonly sources: ReadonlyMap } /** @internal */ @@ -766,7 +766,7 @@ const relation = (node: Plan.Node, c: Compiler): string => { `ROW_NUMBER() OVER () AS ${row}` ] .join(", ") - } FROM ${c.sources.get(node)}` + } FROM ${c.sources.get(node.name)}` case "Values": return values(node.frame as unknown as F.Frame, c) case "Filter": { @@ -999,18 +999,24 @@ const exact = (x: string, type: S.DataType): string | undefined => { } } +const utc = S.timestamp("microsecond") + /** - * A declared column of a table's source as its declared type: null where it - * isn't held exactly (see {@link exact}), so that reading such values - * doesn't fail before the checks of the rows do. + * A declared column of a table's source as its declared type (timestamps + * as UTC timestamps without a time zone, whatever the session's time + * zone): null where it isn't held exactly (see {@link exact}), so that + * reading such values doesn't fail before the checks of the rows do. */ const declared = (column: Plan.TableColumn): string => { const x = quote(column.name) const type = Schema_.arrowType(column) + const value = type._tag === "Timestamp" + ? fromInteger(toInteger(x, utc), utc) + : type._tag === "Decimal" + ? `CAST(${x} AS ${sqlType(type)})` + : x const held = exact(x, type) - return held === undefined - ? x - : `(CASE WHEN ${held} THEN ${type._tag === "Decimal" ? `CAST(${x} AS ${sqlType(type)})` : x} END)` + return held === undefined ? value : `(CASE WHEN ${held} THEN ${value} END)` } /** @@ -1092,25 +1098,15 @@ const rowChecks = (table: Plan.Table, frame: F.Frame): Array => [ }) ] -const utc = S.timestamp("microsecond") - /** - * The source a table is read through: its declared columns, timestamps as - * UTC timestamps without a time zone (as the engine reads them, whatever the - * session's time zone). The checks and the result read it in the same - * statement, so the same snapshot: sources the checks read too are inlined - * (`NOT MATERIALIZED`, as cheap as reading the table twice), unless they - * call volatile functions, which PostgreSQL computes once for both instead. + * The source of the tables named `name` among `tables`: the columns they + * declare, read once. It is `MATERIALIZED`, which PostgreSQL computes once + * per statement, so the checks and the result read the same rows, whatever + * the relation: views that sample rows or call functions included. */ -const source = (table: Plan.Table, checked: boolean): string => { - const columns = (table.columns ?? []).map((column) => - column.type === "Timestamp" - ? `${fromInteger(toInteger(quote(column.name), utc), utc)} AS ${quote(column.name)}` - : quote(column.name) - ) - return `${checked ? "NOT MATERIALIZED " : ""}(SELECT ${columns.length === 0 ? "1" : columns.join(", ")} FROM ${ - quote(table.name) - })` +const source = (name: string, tables: ReadonlyArray): string => { + const columns = [...new Set(tables.flatMap((t) => t.name === name ? t.columns!.map((c) => c.name) : []))] + return `MATERIALIZED (SELECT ${columns.length === 0 ? "1" : list(columns)} FROM ${quote(name)})` } /** One SQL query whose first row holds the values of checks. */ @@ -1149,22 +1145,16 @@ export const compile = (plan: Plan.Node): Compiled => { let prefix = "__" while ([...used].some((name) => name.startsWith(prefix))) prefix += "_" const hidden = (name: string) => quote(prefix + name) - const tables = [...frames].flatMap(([node, frame], i) => - node._tag === "Table" ? [{ node, name: hidden(`table${i}`), checks: rowChecks(node, frame) }] : [] + const tables = [...frames].flatMap(([node, frame]) => + node._tag === "Table" ? [{ node, checks: rowChecks(node, frame) }] : [] ) - const compiler: Compiler = { - frames, - hidden, - row: hidden("row"), - sources: new Map(tables.map((t) => [t.node, t.name])) - } + const sources = new Map([...new Set(tables.map((t) => t.node.name))].map((name, i) => [name, hidden(`table${i}`)])) + const compiler: Compiler = { frames, hidden, row: hidden("row"), sources } const sql = relation(plan, compiler) const schema = S.make(result.columns.map((col) => S.field(col.name, col.type, { nullable: true }))) const columns = result.columns.map((col) => `${output(quote(col.name), col.type)} AS ${quote(col.name)}`) - const ctes = (checked: boolean) => - tables.length === 0 - ? "" - : `WITH ${tables.map((t) => `${t.name} AS ${source(t.node, checked && t.checks.length > 0)}`).join(", ")} ` + const ctes = [...sources].map(([name, cte]) => `${cte} AS ${source(name, tables.map((t) => t.node))}`) + const prelude = ctes.length === 0 ? "" : `WITH ${ctes.join(", ")} ` const body = `SELECT ${columns.length === 0 ? "1" : columns.join(", ")} FROM (${sql}) AS t ORDER BY ${compiler.row}` const columnTypes = tables.flatMap((t) => typeChecks(t.node)) const types = columnTypes.length === 0 ? undefined : { @@ -1173,7 +1163,7 @@ export const compile = (plan: Plan.Node): Compiled => { } const checked = tables.filter((t) => t.checks.length > 0) if (checked.length === 0) { - return { sql: ctes(false) + body, types, query: { sql: ctes(false) + body, checks: [] }, schema } + return { sql: prelude + body, types, query: { sql: prelude + body, checks: [] }, schema } } const names: Array = [] const header = checked.map((t, i) => @@ -1182,14 +1172,14 @@ export const compile = (plan: Plan.Node): Compiled => { names.push(hidden(`check${names.length}`)) return `${check.sql} AS ${names[names.length - 1]}` }).join(", ") - } FROM ${t.name}) AS ${hidden(`checks${i}`)}` + } FROM ${sources.get(t.node.name)}) AS ${hidden(`checks${i}`)}` ) const rows = `(SELECT ${[...columns, compiler.row].join(", ")} FROM (${sql}) AS t) AS r` return { - sql: ctes(false) + body, + sql: prelude + body, types, query: { - sql: `${ctes(true)}SELECT ${[...names, ...result.columns.map((col) => `r.${quote(col.name)}`)].join(", ")} FROM ${ + sql: `${prelude}SELECT ${[...names, ...result.columns.map((col) => `r.${quote(col.name)}`)].join(", ")} FROM ${ header.join(" CROSS JOIN ") } FULL JOIN ${rows} ON FALSE ORDER BY r.${compiler.row} NULLS FIRST`, checks: checked.flatMap((t) => t.checks.map((check) => check.verify)) diff --git a/packages/sql/pglite/test/DataFrameSql.test.ts b/packages/sql/pglite/test/DataFrameSql.test.ts index a152a7a50d8..49a91fa8fa0 100644 --- a/packages/sql/pglite/test/DataFrameSql.test.ts +++ b/packages/sql/pglite/test/DataFrameSql.test.ts @@ -507,9 +507,21 @@ describe("effect/dataframe on PostgreSQL (PGlite)", { concurrent: false }, () => } })) - it.effect("tables are checked on the rows the query reads, even of volatile views", () => + it.effect("tables are checked on the rows the query reads, whatever the relation", () => Effect.gen(function*() { const sql = yield* SqlClient + // A view whose rows differ in each scan, without volatile functions. + yield* sql.unsafe( + `CREATE FUNCTION tick() RETURNS INTEGER STABLE LANGUAGE plpgsql AS $$ + DECLARE n INTEGER := COALESCE(NULLIF(current_setting('df.tick', TRUE), ''), '0')::INTEGER + 1; + BEGIN PERFORM set_config('df.tick', n::TEXT, FALSE); RETURN n; END $$` + ) + yield* sql`CREATE VIEW ticking AS SELECT CASE WHEN tick() <= 1 THEN 'good' ELSE 'bad' END AS s` + const ticking = () => Query.table("ticking", Schema.Struct({ s: Schema.Literal("good") })) + const tick = (query: Query.Query) => + Effect.andThen(sql`SELECT set_config('df.tick', '0', FALSE)`, rowsOnSql(query)) + assert.deepStrictEqual(yield* tick(ticking()), [{ s: "good" }]) + assert.deepStrictEqual(yield* tick(Query.concat([ticking(), ticking()])), [{ s: "good" }, { s: "good" }]) yield* sql`CREATE SEQUENCE seq START 1` yield* sql`CREATE VIEW fluctuating AS SELECT CASE WHEN nextval('seq') <= 2 THEN 'good' ELSE 'bad' END AS s` const query = Query.table("fluctuating", Schema.Struct({ s: Schema.Literal("good") })) From ab0d642854e74fdcb1793ce8b81818f7a9056551 Mon Sep 17 00:00:00 2001 From: Effect Bot Date: Sun, 15 Nov 2026 14:00:40 +0000 Subject: [PATCH 35/88] Check SQL table values against their whole declared type The SQL engine checked only temporal units and decimal digits, so an INTEGER 256 declared Int8, a DOUBLE 0.1 declared Float32 or a NUMERIC 1 declared decimal(38, 20) passed, silently rounded or failed to decode, while memory rejects them. Give every column type in tableTypes the conditions its values must meet for the declared type to hold them exactly (integer ranges, Float32 round trips, decimal precision, scale and 64-bit storage, finite dates, timestamp range and whole units, intervals without months), and derive both the row checks and the null-guarded reads from them, with memory's messages. The checks read the source, so columns the query doesn't read are checked too. --- packages/effect/src/dataframe/Sql.ts | 13 +- packages/effect/src/dataframe/internal/sql.ts | 252 +++++++++++------- packages/sql/pglite/test/DataFrameSql.test.ts | 70 ++++- 3 files changed, 216 insertions(+), 119 deletions(-) diff --git a/packages/effect/src/dataframe/Sql.ts b/packages/effect/src/dataframe/Sql.ts index b1afa36a2a2..0507d514788 100644 --- a/packages/effect/src/dataframe/Sql.ts +++ b/packages/effect/src/dataframe/Sql.ts @@ -131,11 +131,14 @@ const attempt = (f: () => A): Result.Result => { * checks those rows and computes the result from exactly them, whatever * the relation: views that sample rows or call functions included, and a * table read twice gives the same rows twice. The checks: only nulls in - * `Null` columns, no nulls in other non-nullable columns, timestamps in - * the range of `DateTime`, intervals without months, and the distinct - * values of the columns the schema constrains beyond their type (literals, - * enums, refinements) accepted by the schema. Columns the query doesn't - * read are checked too. `explain` shows both queries. + * `Null` columns, no nulls in other non-nullable columns, values their + * declared types hold exactly (integers in range, numbers that `Float32` + * holds, decimals within their precision, scale and 64-bit storage, finite + * dates, timestamps and durations in whole units), timestamps in the range + * of `DateTime`, intervals without months, and the distinct values of the + * columns the schema constrains beyond their type (literals, enums, + * refinements) accepted by the schema. Columns the query doesn't read are + * checked too. `explain` shows both queries. * * **Gotchas** * diff --git a/packages/effect/src/dataframe/internal/sql.ts b/packages/effect/src/dataframe/internal/sql.ts index f3da17a6e41..8a818e08445 100644 --- a/packages/effect/src/dataframe/internal/sql.ts +++ b/packages/effect/src/dataframe/internal/sql.ts @@ -93,56 +93,6 @@ const sqlType = (t: S.DataType): string => { } } -const smallIntegers = ["smallint", "integer"] - -/** - * The PostgreSQL columns the SQL engine reads as declared table columns of - * each type: columns of the PostgreSQL types `accepts` or of the type - * categories `categories` (as the in-memory engine accepts integers below - * 64 bits for floats and narrower integers), or of any type for `Null`. - * `undefined` for column types it doesn't read yet. - */ -const tableTypes: { - readonly [K in Plan.TypeName]: - | { readonly accepts?: ReadonlyArray; readonly categories?: ReadonlyArray } - | undefined -} = { - Int8: { accepts: smallIntegers }, - Int16: { accepts: smallIntegers }, - Int32: { accepts: smallIntegers }, - Int64: { accepts: ["bigint"] }, - UInt8: { accepts: smallIntegers }, - UInt16: { accepts: smallIntegers }, - UInt32: { accepts: smallIntegers }, - UInt64: { accepts: ["bigint"] }, - Float16: undefined, - Float32: { accepts: [...smallIntegers, "real", "double precision"] }, - Float64: { accepts: [...smallIntegers, "real", "double precision"] }, - // String types and enums. - Utf8: { categories: ["S", "E"] }, - Binary: { accepts: ["bytea"] }, - Bool: { accepts: ["boolean"] }, - Null: {}, - Date: { accepts: ["date"] }, - Time: undefined, - Timestamp: { accepts: ["timestamp without time zone", "timestamp with time zone"] }, - Duration: { accepts: ["interval"] }, - Decimal: { accepts: ["numeric"] }, - List: undefined, - Struct: undefined -} - -/** - * The type the SQL engine reads a declared table column as: the type the - * in-memory engine conforms it to. - */ -const tableType = (table: string, column: Plan.TableColumn): S.DataType => - tableTypes[column.type] === undefined - ? unsupported( - `column ${JSON.stringify(column.name)} of table ${JSON.stringify(table)}: ${column.type} table columns` - ) - : Schema_.arrowType(column) - const string = (s: string): string => `'${s.replaceAll("'", "''")}'` const pad = (n: number, width = 2) => String(n).padStart(width, "0") @@ -967,36 +917,147 @@ const reader = (type: S.DataType): (value: unknown) => unknown => { // Tables // ----------------------------------------------------------------------------- +const smallIntegers = ["smallint", "integer"] + +/** A condition, as SQL on a value `x`, and the failure of a column with values that don't satisfy it. */ +interface Condition { + readonly holds: string + readonly failure: string +} + +const exactly = (type: S.DataType): string => `has values that ${D.describe(type)} can't hold exactly` + +/** Integers in the range of an integer type. */ +const inRange = (x: string, type: S.DataType): Array => { + const { bitWidth, signed } = type as S.Int + const lo = signed ? -(BigInt(1) << BigInt(bitWidth - 1)) : BigInt(0) + const hi = (BigInt(1) << BigInt(signed ? bitWidth - 1 : bitWidth)) - BigInt(1) + return [{ holds: `${x} BETWEEN ${lo} AND ${hi}`, failure: exactly(type) }] +} + /** - * A check of a table: one SQL value, and a test of it that fails when the - * table breaks its declaration. + * Numbers that are 32-bit floats: zero, infinities, NaN, or numbers between + * the least and the greatest 32-bit float in magnitude that convert to + * `REAL` and back unchanged (outside that range the conversion fails). */ -interface Check { - readonly sql: string - readonly verify: (value: unknown) => void +const float32 = (x: string, type: S.DataType): Array => { + const d = float(x) + return [{ + holds: `(CASE WHEN ABS(${d}) BETWEEN ${float(`'${2 ** -149}'`)} AND ${ + float(`'${(2 - 2 ** -23) * 2 ** 127}'`) + } THEN CAST(CAST(${d} AS REAL) AS DOUBLE PRECISION) = ${d} ELSE ${d} = 0 OR ABS(${d}) = ${ + float("'Infinity'") + } OR ${d} = ${NAN} END)`, + failure: exactly(type) + }] } const int64 = BigInt(1) << BigInt(63) +/** Timestamps and durations in whole units of the type that fit 64 bits (PostgreSQL keeps microseconds). */ +const wholeUnits = (x: string, type: S.DataType): Condition => { + const v = `(EXTRACT(EPOCH FROM ${x}) * ${perSecond[(type as S.Timestamp | S.Duration).unit]})` + return { holds: `${v} = TRUNC(${v}) AND ${v} BETWEEN ${-int64} AND ${int64 - BigInt(1)}`, failure: exactly(type) } +} + /** - * SQL that is true when `x`, a value of a table's source, is one its - * declared column's `type` holds exactly, as the in-memory engine converts - * table columns: timestamps and durations in whole units that fit 64 bits - * (PostgreSQL keeps microseconds), decimals without more digits than the - * precision and scale allow (no rounding). `undefined` when every value is. + * How the SQL engine reads declared table columns of each type, as the + * in-memory engine conforms table columns to their declaration: + * + * - `accepts` and `categories`: the PostgreSQL types, and type categories, + * of the columns it reads as such columns (integers below 64 bits for + * floats and narrower integers, as the in-memory engine; any type for + * `Null`, which holds only nulls). + * - `conditions`: the conditions every value satisfies, beyond its type, + * for its declared type to hold it exactly. + * + * `undefined` for column types it doesn't read yet. */ -const exact = (x: string, type: S.DataType): string | undefined => { - switch (type._tag) { - case "Timestamp": - case "Duration": { - const v = `(EXTRACT(EPOCH FROM ${x}) * ${perSecond[type.unit]})` - return `${v} = TRUNC(${v}) AND ${v} BETWEEN ${-int64} AND ${int64 - BigInt(1)}` +const tableTypes: { + readonly [K in Plan.TypeName]: + | { + readonly accepts?: ReadonlyArray + readonly categories?: ReadonlyArray + readonly conditions?: (x: string, type: S.DataType) => ReadonlyArray } - case "Decimal": - return `ROUND(${x}, ${type.scale}) = ${x} AND ABS(${x}) < ${power(type.precision - type.scale)}` - default: - return undefined - } + | undefined +} = { + Int8: { accepts: smallIntegers, conditions: inRange }, + Int16: { accepts: smallIntegers, conditions: inRange }, + Int32: { accepts: smallIntegers }, + Int64: { accepts: ["bigint"] }, + UInt8: { accepts: smallIntegers, conditions: inRange }, + UInt16: { accepts: smallIntegers, conditions: inRange }, + UInt32: { accepts: smallIntegers, conditions: inRange }, + UInt64: { accepts: ["bigint"], conditions: inRange }, + Float16: undefined, + Float32: { accepts: [...smallIntegers, "real", "double precision"], conditions: float32 }, + Float64: { accepts: [...smallIntegers, "real", "double precision"] }, + // String types and enums. + Utf8: { categories: ["S", "E"] }, + Binary: { accepts: ["bytea"] }, + Bool: { accepts: ["boolean"] }, + Null: {}, + Date: { accepts: ["date"], conditions: (x, type) => [{ holds: `isfinite(${x})`, failure: exactly(type) }] }, + Time: undefined, + Timestamp: { + accepts: ["timestamp without time zone", "timestamp with time zone"], + conditions: (x, type) => [ + { + holds: `EXTRACT(EPOCH FROM ${x}) BETWEEN -8640000000000 AND 8640000000000.000999`, + failure: "has timestamps outside the range of DateTime" + }, + wholeUnits(x, type) + ] + }, + Duration: { + accepts: ["interval"], + conditions: (x, type) => [ + { + holds: `EXTRACT(YEAR FROM ${x}) = 0 AND EXTRACT(MONTH FROM ${x}) = 0`, + failure: "has durations in months, which have no fixed length" + }, + wholeUnits(x, type) + ] + }, + Decimal: { + accepts: ["numeric"], + conditions: (x, type) => { + const { precision, scale } = type as S.Decimal + return [{ + holds: `ROUND(${x}, ${scale}) = ${x} AND ABS(${x}) < ${power(precision - scale)} AND ${x} * ${ + power(scale) + } BETWEEN ${-int64} AND ${int64 - BigInt(1)}`, + failure: exactly(type) + }] + } + }, + List: undefined, + Struct: undefined +} + +/** + * The type the SQL engine reads a declared table column as: the type the + * in-memory engine conforms it to. + */ +const tableType = (table: string, column: Plan.TableColumn): S.DataType => + tableTypes[column.type] === undefined + ? unsupported( + `column ${JSON.stringify(column.name)} of table ${JSON.stringify(table)}: ${column.type} table columns` + ) + : Schema_.arrowType(column) + +/** The conditions of a declared column's values, `x` (see {@link tableTypes}). */ +const conditions = (x: string, column: Plan.TableColumn): ReadonlyArray => + tableTypes[column.type]!.conditions?.(x, Schema_.arrowType(column)) ?? [] + +/** + * A check of a table: one SQL value, and a test of it that fails when the + * table breaks its declaration. + */ +interface Check { + readonly sql: string + readonly verify: (value: unknown) => void } const utc = S.timestamp("microsecond") @@ -1004,7 +1065,7 @@ const utc = S.timestamp("microsecond") /** * A declared column of a table's source as its declared type (timestamps * as UTC timestamps without a time zone, whatever the session's time - * zone): null where it isn't held exactly (see {@link exact}), so that + * zone): null where it breaks a condition (see {@link conditions}), so that * reading such values doesn't fail before the checks of the rows do. */ const declared = (column: Plan.TableColumn): string => { @@ -1015,8 +1076,8 @@ const declared = (column: Plan.TableColumn): string => { : type._tag === "Decimal" ? `CAST(${x} AS ${sqlType(type)})` : x - const held = exact(x, type) - return held === undefined ? value : `(CASE WHEN ${held} THEN ${value} END)` + const holds = conditions(x, column) + return holds.length === 0 ? value : `(CASE WHEN ${holds.map((c) => c.holds).join(" AND ")} THEN ${value} END)` } /** @@ -1047,42 +1108,29 @@ const typeChecks = (table: Plan.Table): Array => /** * The checks of a table's rows, the ones the in-memory engine makes beyond * column types: only nulls in `Null` columns, no nulls in other columns - * unless they are nullable, timestamps in the range of `DateTime`, - * durations of fixed length (no months), values their declared type holds - * exactly (see {@link exact}), and the values of the fields its schema - * constrains beyond their column type (literals, enums, refinements) - * accepted by the schema, tested on each such column's distinct values. - * Each value aggregates the rows of the table's source. + * unless they are nullable, the conditions of their values (see + * {@link conditions}), and the values of the fields its schema constrains + * beyond their column type (literals, enums, refinements) accepted by the + * schema, tested on each such column's distinct values. Each value + * aggregates the rows of the table's source. */ const rowChecks = (table: Plan.Table, frame: F.Frame): Array => [ ...(table.columns ?? []).flatMap((column) => { const x = quote(column.name) - const path = JSON.stringify(column.name) - const any = (failed: string, message: string): Check => ({ + const any = (failed: string, failure: string): Check => ({ sql: `BOOL_OR(${failed})`, verify: (value) => { - if (value === true) V.fail(`column ${path} ${message}`) + if (value === true) V.fail(`column ${JSON.stringify(column.name)} ${failure}`) } }) - const checks: Array = [] - if (column.type === "Null") checks.push(any(`${x} IS NOT NULL`, "has values, expected Null")) - else if (!column.nullable) checks.push(any(`${x} IS NULL`, "has missing values")) - if (column.type === "Timestamp") { - checks.push(any( - `EXTRACT(EPOCH FROM ${x}) NOT BETWEEN -8640000000000 AND 8640000000000.000999`, - "has timestamps outside the range of DateTime" - )) - } - if (column.type === "Duration") { - checks.push(any( - `EXTRACT(YEAR FROM ${x}) <> 0 OR EXTRACT(MONTH FROM ${x}) <> 0`, - "has durations in months, which have no fixed length" - )) - } - const type = Schema_.arrowType(column) - const held = exact(x, type) - if (held !== undefined) checks.push(any(`NOT (${held})`, `has values that ${D.describe(type)} can't hold exactly`)) - return checks + return [ + ...(column.type === "Null" + ? [any(`${x} IS NOT NULL`, "has values, expected Null")] + : column.nullable + ? [] + : [any(`${x} IS NULL`, "has missing values")]), + ...conditions(x, column).map((c) => any(`NOT (${c.holds})`, c.failure)) + ] }), ...(table.schema === undefined ? [] : Schema_.columnChecks(table.schema)).map(([name, test]): Check => { const type = frame.columns.find((c) => c.name === name)!.type diff --git a/packages/sql/pglite/test/DataFrameSql.test.ts b/packages/sql/pglite/test/DataFrameSql.test.ts index 49a91fa8fa0..4068bb2478d 100644 --- a/packages/sql/pglite/test/DataFrameSql.test.ts +++ b/packages/sql/pglite/test/DataFrameSql.test.ts @@ -48,14 +48,19 @@ const Setup = Layer.effectDiscard( yield* sql`CREATE TABLE contract (s TEXT, n DOUBLE PRECISION)` yield* sql`INSERT INTO contract VALUES ('bad', NULL), ('good', 1)` yield* sql`DROP TABLE IF EXISTS odd` - yield* sql`CREATE TABLE odd (nil INTEGER, nothing INTEGER, at TIMESTAMPTZ, far TIMESTAMP, months INTERVAL)` - yield* sql`INSERT INTO odd VALUES (42, NULL, '2024-01-01 00:30:00+00', '290000-01-01', '1 month')` + yield* sql`CREATE TABLE odd (nil INTEGER, nothing INTEGER, at TIMESTAMPTZ, far TIMESTAMP, months INTERVAL, forever TIMESTAMP, endless DATE)` + yield* sql`INSERT INTO odd VALUES (42, NULL, '2024-01-01 00:30:00+00', '290000-01-01', '1 month', 'infinity', 'infinity')` yield* sql`DROP TABLE IF EXISTS declared` yield* sql`CREATE TABLE declared (t TIMESTAMP, d INTERVAL, n NUMERIC)` yield* sql`INSERT INTO declared VALUES ('1970-01-01 00:00:01', '-2 seconds', 12.5), ('1969-12-31 23:59:58', '3 seconds', -0.05), (NULL, NULL, NULL)` yield* sql`DROP TABLE IF EXISTS inexact` - yield* sql`CREATE TABLE inexact (s TIMESTAMP, ms TIMESTAMP, ns TIMESTAMP, ds INTERVAL, dms INTERVAL, dns INTERVAL, scale NUMERIC, digits NUMERIC)` - yield* sql`INSERT INTO inexact VALUES ('1970-01-01 00:00:01.5', '1970-01-01 00:00:01.0005', '2263-01-01', '-1.5 seconds', '0.0005 seconds', '110000 days', 1.005, 1000)` + yield* sql`CREATE TABLE inexact (s TIMESTAMP, ms TIMESTAMP, ns TIMESTAMP, ds INTERVAL, dms INTERVAL, dns INTERVAL, scale NUMERIC, digits NUMERIC, wide NUMERIC, i8 INTEGER, u32 INTEGER, u64 BIGINT, f32 DOUBLE PRECISION, f32i INTEGER)` + yield* sql`INSERT INTO inexact VALUES ('1970-01-01 00:00:01.5', '1970-01-01 00:00:01.0005', '2263-01-01', '-1.5 seconds', '0.0005 seconds', '110000 days', 1.005, 1000, 1, 256, -1, -1, 0.1, 16777217)` + yield* sql`DROP TABLE IF EXISTS exact` + yield* sql`CREATE TABLE exact (i8 INTEGER, f32 DOUBLE PRECISION)` + yield* sql`INSERT INTO exact VALUES (-128, 0.5), (127, 'NaN'), (0, '-Infinity'), (1, ${ + 2 ** -149 + }), (2, '-0'), (3, ${(2 - 2 ** -23) * 2 ** 127})` }) ) @@ -76,7 +81,11 @@ const memory = Catalog.layer({ inexact: frame([ ...["s", "ms", "ns"].map((name) => ArrowSchema.field(name, ArrowSchema.timestamp("microsecond"))), ...["ds", "dms", "dns"].map((name) => ArrowSchema.field(name, ArrowSchema.duration("microsecond"))), - ...["scale", "digits"].map((name) => ArrowSchema.field(name, ArrowSchema.decimal(20, 4))) + ...["scale", "digits", "wide"].map((name) => ArrowSchema.field(name, ArrowSchema.decimal(20, 4))), + ...["i8", "u32"].map((name) => ArrowSchema.field(name, ArrowSchema.int32)), + ArrowSchema.field("u64", ArrowSchema.int64), + ArrowSchema.field("f32", ArrowSchema.float64), + ArrowSchema.field("f32i", ArrowSchema.int32) ], [ [1_500_000n], [1_000_500n], @@ -85,8 +94,18 @@ const memory = Catalog.layer({ [500n], [9_504_000_000_000_000n], [decimal("1.005")], - [decimal("1000")] - ]) + [decimal("1000")], + [decimal("1")], + [256], + [-1], + [-1n], + [0.1], + [16777217] + ]), + exact: frame([ + ArrowSchema.field("i8", ArrowSchema.int32), + ArrowSchema.field("f32", ArrowSchema.float64) + ], [[-128, 127, 0, 1, 2, 3], [0.5, NaN, -Infinity, 2 ** -149, -0, (2 - 2 ** -23) * 2 ** 127]]) }) const table = Query.table("sales", Sale) @@ -369,6 +388,12 @@ const cases: ReadonlyArray]> = [ }))) ] as const ), + [ + "tables declared with narrower types, holding them exactly", + Query.table("exact", Schema.Struct({ i8: Schema.Number, f32: Schema.Number }), { + types: { i8: ArrowSchema.int8, f32: ArrowSchema.float32 } + }) + ], [ "rounding integers", Query.from(DataFrame.fromColumns({ x: Int32Array.of(5, -15, 25) })).pipe( @@ -478,6 +503,18 @@ describe("effect/dataframe on PostgreSQL (PGlite)", { concurrent: false }, () => [ contract(Schema.Struct({ s: Schema.String }).check(Schema.makeFilter((row) => row.s !== ""))), "schemas with checks on whole rows aren't supported on SQL tables; check their fields instead" + ], + [ + odd(Schema.Struct({ forever: Schema.DateTimeUtc })), + `column "forever" has timestamps outside the range of DateTime` + ], + [ + Query.fromPlan({ + _tag: "Table", + name: "odd", + columns: [{ name: "endless", type: "Date", nullable: true }] + }), + `column "endless" has values that Date can't hold exactly` ] ] as const ) { @@ -497,13 +534,22 @@ describe("effect/dataframe on PostgreSQL (PGlite)", { concurrent: false }, () => ["dms", Schema.Duration, ArrowSchema.duration("millisecond")], ["dns", Schema.Duration, ArrowSchema.duration("nanosecond")], ["scale", Schema.BigDecimal, ArrowSchema.decimal(5, 2)], - ["digits", Schema.BigDecimal, ArrowSchema.decimal(5, 2)] + ["digits", Schema.BigDecimal, ArrowSchema.decimal(5, 2)], + ["wide", Schema.BigDecimal, ArrowSchema.decimal(38, 20)], + ["i8", Schema.Number, ArrowSchema.int8], + ["u32", Schema.Number, ArrowSchema.uint32], + ["u64", Schema.BigInt, ArrowSchema.uint64], + ["f32", Schema.Number, ArrowSchema.float32], + ["f32i", Schema.Number, ArrowSchema.float32] ] as const ) { - const query = Query.table("inexact", Schema.Struct({ [column]: schema }), { types: { [column]: type } }) - const expected = yield* Effect.flip(query.pipe(Effect.provide(memory))) - const actual = yield* Effect.flip(onSql(query)) - assert.strictEqual(actual.message, expected.message) + const table = Query.table("inexact", Schema.Struct({ [column]: schema }), { types: { [column]: type } }) + // A count reads none of the table's columns. + for (const query of [table, table.pipe(Query.aggregate(() => ({ n: Expr.count() })))]) { + const expected = yield* Effect.flip(query.pipe(Effect.provide(memory))) + const actual = yield* Effect.flip(onSql(query)) + assert.strictEqual(actual.message, expected.message) + } } })) From 216666cd167ead4146239d0c31f59424a3d9767d Mon Sep 17 00:00:00 2001 From: Effect Bot Date: Sun, 15 Nov 2026 14:01:18 +0000 Subject: [PATCH 36/88] Read SQL table columns as their declared types Accepted source columns kept their PostgreSQL types inside the query, so expressions saw a different type than the declared one: lower() failed on enums, enum comparisons parsed literals as labels and enums sorted by declaration, char(n) compared without padding, smallint sums overflowed where Int32 doesn't, and real arithmetic rounded to Float32 where Float64 doesn't. Read every declared column as the PostgreSQL type of its declared type (strings, enums and char(n) as text, as PostgreSQL converts them) where the result reads the source, so every expression sees it. --- packages/effect/src/dataframe/Sql.ts | 5 ++- packages/effect/src/dataframe/internal/sql.ts | 17 +++++----- packages/sql/pglite/test/DataFrameSql.test.ts | 34 ++++++++++++++++++- 3 files changed, 46 insertions(+), 10 deletions(-) diff --git a/packages/effect/src/dataframe/Sql.ts b/packages/effect/src/dataframe/Sql.ts index 0507d514788..4dc8abd03a0 100644 --- a/packages/effect/src/dataframe/Sql.ts +++ b/packages/effect/src/dataframe/Sql.ts @@ -138,7 +138,10 @@ const attempt = (f: () => A): Result.Result => { * of `DateTime`, intervals without months, and the distinct values of the * columns the schema constrains beyond their type (literals, enums, * refinements) accepted by the schema. Columns the query doesn't read are - * checked too. `explain` shows both queries. + * checked too. The query reads every column as its declared type: strings, + * enums and `char(n)` as text (`char(n)` without trailing spaces, as + * PostgreSQL converts it), numbers as the PostgreSQL type of their + * declared type. `explain` shows both queries. * * **Gotchas** * diff --git a/packages/effect/src/dataframe/internal/sql.ts b/packages/effect/src/dataframe/internal/sql.ts index 8a818e08445..1c25e57a4b4 100644 --- a/packages/effect/src/dataframe/internal/sql.ts +++ b/packages/effect/src/dataframe/internal/sql.ts @@ -1063,19 +1063,20 @@ interface Check { const utc = S.timestamp("microsecond") /** - * A declared column of a table's source as its declared type (timestamps - * as UTC timestamps without a time zone, whatever the session's time - * zone): null where it breaks a condition (see {@link conditions}), so that - * reading such values doesn't fail before the checks of the rows do. + * A declared column of a table's source as the engine reads it: in the + * PostgreSQL type of its declared type (strings and enums as text, + * timestamps as UTC timestamps without a time zone, whatever the session's + * time zone), and null where it breaks a condition (see {@link conditions}), + * so that reading such values doesn't fail before the checks of the rows do. */ const declared = (column: Plan.TableColumn): string => { const x = quote(column.name) const type = Schema_.arrowType(column) - const value = type._tag === "Timestamp" + const value = type._tag === "Null" + ? `CAST(NULL AS ${sqlType(type)})` + : type._tag === "Timestamp" ? fromInteger(toInteger(x, utc), utc) - : type._tag === "Decimal" - ? `CAST(${x} AS ${sqlType(type)})` - : x + : `CAST(${x} AS ${sqlType(type)})` const holds = conditions(x, column) return holds.length === 0 ? value : `(CASE WHEN ${holds.map((c) => c.holds).join(" AND ")} THEN ${value} END)` } diff --git a/packages/sql/pglite/test/DataFrameSql.test.ts b/packages/sql/pglite/test/DataFrameSql.test.ts index 4068bb2478d..9dd5dae93c3 100644 --- a/packages/sql/pglite/test/DataFrameSql.test.ts +++ b/packages/sql/pglite/test/DataFrameSql.test.ts @@ -61,6 +61,11 @@ const Setup = Layer.effectDiscard( yield* sql`INSERT INTO exact VALUES (-128, 0.5), (127, 'NaN'), (0, '-Infinity'), (1, ${ 2 ** -149 }), (2, '-0'), (3, ${(2 - 2 ** -23) * 2 ** 127})` + yield* sql`DROP TABLE IF EXISTS texts` + yield* sql`DROP TYPE IF EXISTS mood` + yield* sql`CREATE TYPE mood AS ENUM ('zzz', 'aaa')` + yield* sql`CREATE TABLE texts (m mood, c CHAR(3), v VARCHAR(5), sm SMALLINT, r REAL)` + yield* sql`INSERT INTO texts VALUES ('zzz', 'ab', 'xy', 20000, 16777216), ('aaa', 'abc', 'x', -20000, 0.5)` }) ) @@ -105,7 +110,13 @@ const memory = Catalog.layer({ exact: frame([ ArrowSchema.field("i8", ArrowSchema.int32), ArrowSchema.field("f32", ArrowSchema.float64) - ], [[-128, 127, 0, 1, 2, 3], [0.5, NaN, -Infinity, 2 ** -149, -0, (2 - 2 ** -23) * 2 ** 127]]) + ], [[-128, 127, 0, 1, 2, 3], [0.5, NaN, -Infinity, 2 ** -149, -0, (2 - 2 ** -23) * 2 ** 127]]), + // PostgreSQL converts char(n) values to text without their trailing spaces. + texts: frame([ + ...["m", "c", "v"].map((name) => ArrowSchema.field(name, ArrowSchema.utf8)), + ArrowSchema.field("sm", ArrowSchema.int16), + ArrowSchema.field("r", ArrowSchema.float32) + ], [["zzz", "aaa"], ["ab", "abc"], ["xy", "x"], [20000, -20000], [16777216, 0.5]]) }) const table = Query.table("sales", Sale) @@ -394,6 +405,27 @@ const cases: ReadonlyArray]> = [ types: { i8: ArrowSchema.int8, f32: ArrowSchema.float32 } }) ], + [ + "enum, char, varchar, smallint and real columns, as their declared types", + Query.table( + "texts", + Schema.Struct({ m: Schema.String, c: Schema.String, v: Schema.String, sm: Schema.Number, r: Schema.Number }), + { types: { sm: ArrowSchema.int32 } } + ).pipe( + Query.select((c) => ({ + m: c.m, + lower: Expr.toLowerCase(c.m), + before: Expr.lt(c.m, "b"), + c: c.c, + padded: Expr.eq(c.c, "ab "), + length: Expr.length(c.c), + v: Expr.concat(c.v, "|"), + sm: Expr.add(c.sm, c.sm), + r: Expr.add(c.r, 1) + })), + Query.sort((c) => [c.m]) + ) + ], [ "rounding integers", Query.from(DataFrame.fromColumns({ x: Int32Array.of(5, -15, 25) })).pipe( From 0fd10c3811c97d1ae2c859b05d6794a237ff9f01 Mon Sep 17 00:00:00 2001 From: Effect Bot Date: Sun, 15 Nov 2026 14:18:30 +0000 Subject: [PATCH 37/88] Document that SQL tables with whole-row checks are rejected --- packages/effect/src/dataframe/Sql.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/effect/src/dataframe/Sql.ts b/packages/effect/src/dataframe/Sql.ts index 4dc8abd03a0..3958f50696d 100644 --- a/packages/effect/src/dataframe/Sql.ts +++ b/packages/effect/src/dataframe/Sql.ts @@ -141,7 +141,8 @@ const attempt = (f: () => A): Result.Result => { * checked too. The query reads every column as its declared type: strings, * enums and `char(n)` as text (`char(n)` without trailing spaces, as * PostgreSQL converts it), numbers as the PostgreSQL type of their - * declared type. `explain` shows both queries. + * declared type. `explain` shows both queries. Tables whose schemas have + * checks on whole rows are rejected when the query compiles. * * **Gotchas** * From b536556a8240e91d10686a7d87a76f210679d920 Mon Sep 17 00:00:00 2001 From: Effect Bot Date: Sun, 15 Nov 2026 14:43:17 +0000 Subject: [PATCH 38/88] Keep predicates above concats and off the right side's join keys --- packages/effect/src/dataframe/internal/query.ts | 17 ++++++++--------- packages/effect/test/dataframe/Query.test.ts | 11 +++++++++++ 2 files changed, 19 insertions(+), 9 deletions(-) diff --git a/packages/effect/src/dataframe/internal/query.ts b/packages/effect/src/dataframe/internal/query.ts index ba7ca9a10b0..ab5a6767af5 100644 --- a/packages/effect/src/dataframe/internal/query.ts +++ b/packages/effect/src/dataframe/internal/query.ts @@ -6,8 +6,11 @@ * * - row-wise predicates move below the row-wise operations they commute * with (sorts, projections, derived columns they don't read, group keys, - * either side of a join, every input of a concat), and filters that meet - * merge; + * either side of a join), and filters that meet merge. A predicate only + * moves where it sees the values it sees in place: it stays above a concat + * (whose inputs may differ in type from their concatenation), and one on + * join keys moves to the left side only (the side whose keys the join + * keeps); * - slices move below row-wise operations that keep every row in place, and * a slice of a sort becomes a top-k selection. * @@ -319,23 +322,19 @@ const push = (node: Node, predicates: ReadonlyArray): Node => { const { down, here } = split((refs) => subset(refs, node.keys)) return filtered({ ...node, input: push(node.input, down) }, here) } - case "Concat": - return { ...node, inputs: node.inputs.map((input) => push(input, predicates)) } case "Join": { const leftNames = names(node.left) const rightNames = names(node.right) if (node.how === "full" || leftNames === undefined || rightNames === undefined) return filtered(node, predicates) const left = new Set(leftNames) - const right = new Set(rightNames) + // The join keeps the left side's keys, which the right side's may differ from in type. + const right = new Set(rightNames.filter((n) => !node.on.includes(n))) const toLeft: Array = [] const toRight: Array = [] const here: Array = [] for (const p of predicates) { const refs = E.columns(p) - if (node.how === "inner" && subset(refs, node.on)) { - toLeft.push(p) - toRight.push(p) - } else if (subset(refs, left)) toLeft.push(p) + if (subset(refs, left)) toLeft.push(p) else if (node.how === "inner" && subset(refs, right)) toRight.push(p) else here.push(p) } diff --git a/packages/effect/test/dataframe/Query.test.ts b/packages/effect/test/dataframe/Query.test.ts index 84d0f076eed..73a84fc6724 100644 --- a/packages/effect/test/dataframe/Query.test.ts +++ b/packages/effect/test/dataframe/Query.test.ts @@ -393,6 +393,17 @@ describe("Query", () => { assert.deepStrictEqual(yield* rows(query), [{ town: "Vienna", n: 2, mean: 20.25 }]) })) + it.effect("keeps predicates on join keys off the right side, whose keys the join drops", () => + Effect.gen(function*() { + // The result has the left side's Int8 key, where 100 + 100 wraps to -56; + // in the right side's Int64 it is 200. + const query = Query.from(DataFrame.fromColumns({ x: Int8Array.of(100), a: [1] })).pipe( + Query.join(Query.from(DataFrame.fromColumns({ x: BigInt64Array.of(100n), b: [2] })), { on: "x" }), + Query.filter((c) => Expr.lt(Expr.add(c.x, c.x), 0)) + ) + assert.deepStrictEqual(yield* rows(query), [{ x: 100, a: 1, b: 2 }]) + })) + it.effect("keeps predicates above left joins when they read the right side", () => Effect.gen(function*() { const query = Query.from(readings).pipe( From d06a16299d95af985014aff3ea3e8bad1048276d Mon Sep 17 00:00:00 2001 From: Effect Bot Date: Sun, 15 Nov 2026 14:45:01 +0000 Subject: [PATCH 39/88] Stream batches in the column types of their node's whole result --- packages/effect/src/dataframe/Catalog.ts | 5 +- packages/effect/src/dataframe/Plan.ts | 3 + packages/effect/src/dataframe/Query.ts | 3 + .../effect/src/dataframe/internal/frame.ts | 17 ++ .../effect/src/dataframe/internal/partial.ts | 111 +++-------- .../effect/src/dataframe/internal/query.ts | 4 +- .../effect/src/dataframe/internal/stream.ts | 71 ++++++- packages/effect/test/dataframe/Query.test.ts | 178 ++++++++++++------ 8 files changed, 243 insertions(+), 149 deletions(-) diff --git a/packages/effect/src/dataframe/Catalog.ts b/packages/effect/src/dataframe/Catalog.ts index c0a9ffbd2d0..51be5cde800 100644 --- a/packages/effect/src/dataframe/Catalog.ts +++ b/packages/effect/src/dataframe/Catalog.ts @@ -11,7 +11,10 @@ * run of a query reads each table it names once. * - A query over a streamed table runs batch by batch: row-wise operations * per batch, grouping and aggregation folded into partial aggregates, so a - * stream larger than memory can be filtered and summarised. + * stream larger than memory can be filtered and summarised. The batches, + * as the query reads them (converted to its declared columns, if any), + * must have the column names and types of the first, as the batches of an + * Arrow stream share its schema. * - The default catalog is empty; provide one with {@link layer} or * `Effect.provideService(Catalog, Catalog.make(...))`. * diff --git a/packages/effect/src/dataframe/Plan.ts b/packages/effect/src/dataframe/Plan.ts index 34a1139ad5b..0bef15b39c6 100644 --- a/packages/effect/src/dataframe/Plan.ts +++ b/packages/effect/src/dataframe/Plan.ts @@ -718,6 +718,9 @@ export interface TableColumn { * timestamps must have no time zone or a UTC one (dropped), and lie in the * range `DateTime` represents. `Date` and `Time` columns keep their unit. * + * Every batch of a streamed table, after this conversion (as given, without + * `columns`), must have the column names and types of its first batch. + * * `schema`, set by `Query.table`, is the struct schema the rows must * satisfy: values of fields it constrains beyond their column type (literals, * enums, refinements) are checked against it. diff --git a/packages/effect/src/dataframe/Query.ts b/packages/effect/src/dataframe/Query.ts index 2d204b952d8..30bc31169b7 100644 --- a/packages/effect/src/dataframe/Query.ts +++ b/packages/effect/src/dataframe/Query.ts @@ -477,6 +477,9 @@ export const dropNulls: { * common type can't hold, such as a decimal that leaves 64-bit storage at * the larger scale, fails with a `DataFrameError` instead of becoming null. * + * Over streamed tables, the concatenation reads the first batch of every + * query, which gives its types, before it returns its first row. + * * @stability experimental * @category combining * @since 4.1.5 diff --git a/packages/effect/src/dataframe/internal/frame.ts b/packages/effect/src/dataframe/internal/frame.ts index 4ae83a5ae12..1545af1766f 100644 --- a/packages/effect/src/dataframe/internal/frame.ts +++ b/packages/effect/src/dataframe/internal/frame.ts @@ -554,6 +554,23 @@ export const dropNulls = (self: Frame, names: ReadonlyArray | undefined) return take(self, Int32Array.from(keep)) } +/** + * `self` with the columns of `like`, each converted to its type in `like` + * (`K.coerce`: a value that type can't hold fails). + * + * @internal + */ +export const coerce = (self: Frame, like: Frame): Frame => + make( + like.columns.map((c) => { + const own = column(self, c.name) + return D.sameType(own.type, c.type) + ? own + : new Column(c.name, K.coerce(own.values(), c.type), undefined, own.nullable) + }), + self.length + ) + /** @internal */ export const concat = (frames: ReadonlyArray): Frame => { const first = frames[0] diff --git a/packages/effect/src/dataframe/internal/partial.ts b/packages/effect/src/dataframe/internal/partial.ts index a180bdc74ef..716db45176d 100644 --- a/packages/effect/src/dataframe/internal/partial.ts +++ b/packages/effect/src/dataframe/internal/partial.ts @@ -8,22 +8,17 @@ * decimals); means from sums and counts; variances from counts, means and * M2 (Chan et al.). Medians and distinct counts don't combine. * - * Parts may differ in type (stream batches of untyped tables). The result - * is the aggregate of the concatenated rows: each state keeps the smallest - * and largest input value converted to the inputs' common type, so a value - * that type can't hold fails as concatenating would, and `finish` gives - * the result the type and overflow of that common type. Float results may + * Parts have the types of the whole input (every batch of a streamed node + * has the types of its whole result), so their partials combine as they + * are and finish in that input type. Float results may * differ in rounding, as float sums of the same values in another order do. * * @internal */ import type * as S from "../../arrow/ArrowSchema.ts" import * as Agg from "./aggregate.ts" -import * as Dec from "./decimal.ts" -import * as D from "./dtype.ts" import * as Hash from "./hash.ts" import * as K from "./kernels.ts" -import * as T from "./temporal.ts" import * as V from "./vector.ts" /** @internal */ @@ -39,8 +34,6 @@ export type Part = } | { readonly _tag: "none" } -const scaleOf = (t: S.DataType): number => (Dec.isDecimal(t) ? t.scale : 0) - const reaggregated = new Set(["count", "len", "min", "max", "first", "last"]) /** Whether partial aggregates of `fn` combine. @internal */ @@ -66,60 +59,37 @@ export const partial = (fn: Agg.AggFn, v: V.Vector | undefined, g: Agg.Grouping) } } -/** Sums of parts whose inputs have `types`, in their common type `type`. */ -const combineSums = ( - parts: ReadonlyArray, - types: ReadonlyArray, - type: S.DataType, - ids: Int32Array, - count: number -): Agg.Sums => { - const float = type._tag === "Float" - const out = float ? new Float64Array(count) : new Array(count).fill(BigInt(0)) - let at = 0 - parts.forEach((p, c) => { - if (T.isTemporal(type) && !D.sameType(types[c], type)) { - V.fail(`sums of ${D.describe(types[c])} and ${D.describe(type)} values don't combine`) - } - if (float) { - const scale = 10 ** scaleOf(types[c]) - for (let j = 0; j < p.length; j++) (out as Float64Array)[ids[at + j]] += Number(p[j]) / scale - } else { - const factor = Dec.pow10(scaleOf(type) - scaleOf(types[c])) - for (let j = 0; j < p.length; j++) (out as Array)[ids[at + j]] += (p[j] as bigint) * factor - } - at += p.length - }) - return out -} - /** * Combines the partials of several parts: `parts[c]` holds part `c`'s - * partials for its groups over inputs of `types[c]`, whose common type is - * `type`; `ids` maps the concatenated groups of all parts to `count` - * combined groups. + * partials for its groups over inputs of `type`; `ids` maps the + * concatenated groups of all parts to `count` combined groups. * * @internal */ export const combine = ( fn: Agg.AggFn, parts: ReadonlyArray, - types: ReadonlyArray, type: S.DataType | undefined, ids: Int32Array, count: number ): Part => { const grouping: Agg.Grouping = { ids, count } if (reaggregated.has(fn)) { - const vectors = parts.map((p) => (p as Extract).vector) - if (fn === "count" || fn === "len") { - return { _tag: "vector", vector: Agg.aggregate("sum", K.concat(vectors), grouping) } - } - return { _tag: "vector", vector: Agg.aggregate(fn, V.concat(vectors.map((v) => K.coerce(v, type!))), grouping) } + const vector = K.concat(parts.map((p) => (p as Extract).vector)) + return { _tag: "vector", vector: Agg.aggregate(fn === "count" || fn === "len" ? "sum" : fn, vector, grouping) } } if (fn === "sum") { - const sums = parts.map((p) => (p as Extract).sums) - return { _tag: "sums", sums: combineSums(sums, types as ReadonlyArray, type!, ids, count) } + const float = type!._tag === "Float" + const sums = float ? new Float64Array(count) : new Array(count).fill(BigInt(0)) + let at = 0 + for (const p of parts as ReadonlyArray>) { + for (let j = 0; j < p.sums.length; j++) { + if (float) (sums as Float64Array)[ids[at + j]] += p.sums[j] as number + else (sums as Array)[ids[at + j]] += p.sums[j] as bigint + } + at += p.sums.length + } + return { _tag: "sums", sums } } if (fn === "mean") { const sum = new Float64Array(count) @@ -180,30 +150,18 @@ export const finish = (fn: Agg.AggFn, part: Part, type: S.DataType | undefined): /** * The groups of some rows: their keys (one row per group), partial - * aggregates and, per aggregate input, its domain: the smallest and - * largest value in the inputs' common type. + * aggregates and the type of each aggregate's input. * * @internal */ export interface State { readonly keys: ReadonlyArray - readonly domains: ReadonlyArray + readonly types: ReadonlyArray readonly parts: ReadonlyArray readonly count: number readonly rows: number } -/** - * The smallest and largest valid value of `v`, the values that decide - * whether a numeric or temporal column converts to a type; no values for - * other columns. - */ -const domain = (v: V.Vector): V.Vector => { - if (v.kind !== "i32" && v.kind !== "i64" && v.kind !== "f64") return V.take(v, new Int32Array(0)) - const all = { ids: new Int32Array(v.length), count: 1 } - return V.concat([Agg.aggregate("min", v, all), Agg.aggregate("max", v, all)]) -} - /** Groups some rows; without keys, they form one group even when there are none. @internal */ export const group = ( keys: ReadonlyArray, @@ -213,7 +171,7 @@ export const group = ( const g = keys.length === 0 ? { ids: new Int32Array(n), count: 1, firstRow: new Int32Array(1) } : Hash.groups(keys, n) return { keys: keys.map((k) => V.take(k, g.firstRow)), - domains: inputs.map(({ vector }) => vector === undefined ? undefined : domain(vector)), + types: inputs.map(({ vector }) => vector?.type), parts: inputs.map(({ fn, vector }) => partial(fn, vector, g)), count: g.count, rows: n @@ -221,35 +179,20 @@ export const group = ( } /** - * Combines states, keeping groups in order of first appearance. States - * without rows only add their types. + * Combines states of rows of the same types, keeping groups in order of + * first appearance. States without rows add nothing. * * @internal */ export const merge = (fns: ReadonlyArray, states: ReadonlyArray): State => { - if (states.length === 1) return states[0] - const keys = states[0].keys.map((_, k) => K.concat(states.map((s) => s.keys[k]))) - const domains = states[0].domains.map((d, a) => - d === undefined ? undefined : domain(K.concat(states.map((s) => s.domains[a]!))) - ) const filled = states.filter((s) => s.rows > 0) - if (filled.length === 0) { - return group(keys, fns.map((fn, a) => ({ fn, vector: domains[a] && V.take(domains[a], new Int32Array(0)) })), 0) - } + if (filled.length <= 1) return filled[0] ?? states[0] + const keys = filled[0].keys.map((_, k) => K.concat(filled.map((s) => s.keys[k]))) const g = Hash.groups(keys, filled.reduce((n, s) => n + s.count, 0)) return { keys: keys.map((k) => V.take(k, g.firstRow)), - domains, - parts: fns.map((fn, a) => - combine( - fn, - filled.map((s) => s.parts[a]), - filled.map((s) => s.domains[a]?.type), - domains[a]?.type, - g.ids, - g.count - ) - ), + types: filled[0].types, + parts: fns.map((fn, a) => combine(fn, filled.map((s) => s.parts[a]), filled[0].types[a], g.ids, g.count)), count: g.count, rows: filled.reduce((n, s) => n + s.rows, 0) } diff --git a/packages/effect/src/dataframe/internal/query.ts b/packages/effect/src/dataframe/internal/query.ts index ab5a6767af5..0335c99f99e 100644 --- a/packages/effect/src/dataframe/internal/query.ts +++ b/packages/effect/src/dataframe/internal/query.ts @@ -105,8 +105,8 @@ export const mapTables = (plan: Plan.Node, f: (table: Plan.Table) => Node): Node return replace(plan) } -/** A table's frame, checked against and narrowed to its declared columns. @internal */ -export const tableValues = (table: Plan.Table, frame: F.Frame): Node => +/** A table's frame, checked against and narrowed to its declared columns. */ +const tableValues = (table: Plan.Table, frame: F.Frame): Node => values(table.columns === undefined ? frame : conform(frame, table.columns, table.schema)) /** Replaces every `Table` with the catalog's frame, reading each table once. @internal */ diff --git a/packages/effect/src/dataframe/internal/stream.ts b/packages/effect/src/dataframe/internal/stream.ts index 84bd17d8557..3bdea4e72bd 100644 --- a/packages/effect/src/dataframe/internal/stream.ts +++ b/packages/effect/src/dataframe/internal/stream.ts @@ -11,14 +11,24 @@ * aggregates of windows collect instead). Every other operation collects its * inputs first. * + * Every batch of a node has the column types of the node's whole result, so + * each operation sees the values it would see over all rows at once: the + * batches of a table, as read (converted to its declared columns, if any), + * must have the columns of its first batch, as the batches of an Arrow + * stream share its schema, and a concatenation converts its inputs' batches + * to their common types, as concatenating their results would. + * * @internal */ import * as Effect from "../../Effect.ts" +import * as Option from "../../Option.ts" +import * as Sink from "../../Sink.ts" import * as Stream from "../../Stream.ts" import type { Catalog } from "../Catalog.ts" import type { DataFrameError } from "../DataFrame.ts" import type * as Plan from "../Plan.ts" import type * as Agg from "./aggregate.ts" +import * as D from "./dtype.ts" import * as E from "./expr.ts" import * as F from "./frame.ts" import * as K from "./kernels.ts" @@ -126,7 +136,7 @@ const aggregate = ( state === undefined ? runEmpty(node) : Q.attempt(() => { const keyColumns = keys.map((k, i) => new F.Column(k, state.keys[i], undefined, nullable[i])) const columns = specs.map((s, a) => - new F.Column(s.name, Partial.finish(s.fn, state.parts[a], state.domains[a]?.type), undefined) + new F.Column(s.name, Partial.finish(s.fn, state.parts[a], state.types[a]), undefined) ) return F.make([...keyColumns, ...columns], state.count) }) @@ -137,6 +147,47 @@ const aggregate = ( }) } +/** + * The inputs' batches one after the other, converted to the inputs' common + * types. An input's first batch gives its types (its result without rows when + * it has none), so every input starts before the first batch is emitted. + */ +const concat = (inputs: ReadonlyArray, catalog: Catalog.Service): Batches => + Stream.unwrap(Effect.gen(function*() { + const peeled = yield* Effect.forEach( + inputs, + (input) => + Effect.flatMap(Stream.peel(batches(input, catalog), Sink.head()), ([first, rest]) => + Option.isSome(first) + ? Effect.succeed({ first: first.value, rest: Stream.concat(Stream.succeed(first.value), rest) }) + : Effect.map(runEmpty(input), (empty) => ({ first: empty, rest: Stream.empty }))) + ) + const types = yield* Q.attempt(() => F.concat(peeled.map(({ first }) => F.slice(first, 0, 0)))) + return peeled.map(({ rest }) => Stream.mapEffect(rest, (batch) => Q.attempt(() => F.coerce(batch, types)))) + .reduce((a, b) => Stream.concat(a, b)) + })) + +const describeColumns = (frame: F.Frame): string => + `[${frame.columns.map((c) => `${c.name}: ${D.describe(c.type)}`).join(", ")}]` + +/** + * Reads the batches of a table in turn: each converted to the declared + * columns, if any, and checked to have the columns of the first. + */ +const reader = (table: Plan.Table) => { + let first: F.Frame | undefined + return (frame: F.Frame): F.Frame => { + const batch = table.columns === undefined ? frame : conform(frame, table.columns, table.schema) + first ??= batch + const same = batch.columns.length === first.columns.length && + batch.columns.every((c, i) => c.name === first!.columns[i].name && D.sameType(c.type, first!.columns[i].type)) + return same ? batch : V.fail( + `a batch of table ${JSON.stringify(table.name)} has columns ${describeColumns(batch)}, ` + + `its first batch ${describeColumns(first)}: the batches of a streamed table share their columns` + ) + } +} + const slice = (input: Batches, offset: number, length: number | undefined): Batches => length === 0 ? Stream.empty : Stream.suspend(() => { let skip = offset @@ -168,12 +219,13 @@ export const batches = (node: Q.Node, catalog: Catalog.Service): Batches => { } switch (node._tag) { case "Table": - return Stream.suspend(() => catalog.stream(node.name)).pipe( - Stream.mapEffect((frame) => { - const batch = frame as unknown as F.Frame - return Q.attempt(() => (node.columns === undefined ? batch : conform(batch, node.columns, node.schema))) - }) - ) + return Stream.suspend(() => { + const read = reader(node) + return Stream.mapEffect( + catalog.stream(node.name), + (frame) => Q.attempt(() => read(frame as unknown as F.Frame)) + ) + }) case "Values": return Stream.succeed(node.frame as unknown as F.Frame) case "Slice": @@ -190,7 +242,7 @@ export const batches = (node: Q.Node, catalog: Catalog.Service): Batches => { case "Concat": return node.inputs.length === 0 ? collected(node, catalog) - : node.inputs.map((input) => batches(input, catalog)).reduce((a, b) => Stream.concat(a, b)) + : concat(node.inputs, catalog) case "GroupBy": case "Aggregate": return aggregate(node, catalog) @@ -216,7 +268,8 @@ const prepare = (plan: Plan.Node, catalog: Catalog.Service): Effect.Effect | undefined if (frames === undefined) return table if (frames.length === 0) return Q.values(emptyTable(table)) - return Q.tableValues(table, frames.length === 1 ? frames[0] : F.concat(frames)) + const batches = frames.map(reader(table)) + return Q.values(batches.length === 1 ? batches[0] : F.concat(batches)) })) ) }) diff --git a/packages/effect/test/dataframe/Query.test.ts b/packages/effect/test/dataframe/Query.test.ts index 73a84fc6724..ca35c03a707 100644 --- a/packages/effect/test/dataframe/Query.test.ts +++ b/packages/effect/test/dataframe/Query.test.ts @@ -236,14 +236,8 @@ describe("Query", () => { }) const big = Query.from(price("6000000000000000000", 19, 0)) const small = Query.from(price("1.20", 18, 2)) - const groups = Query.fromPlan({ _tag: "Table", name: "t" }).pipe( - Query.groupBy("d"), - Query.agg(() => ({ n: Expr.count() })), - Effect.provide(Catalog.layer({ t: Stream.make(price("6000000000000000000", 19, 0), price("1.20", 18, 2)) })) - ) const failures: ReadonlyArray> = [ Query.concat([big, small]), - groups, Query.join(big, small, { on: "d", how: "left" }), Query.filter(big, (c) => Expr.eq(c.d, BigDecimal.fromStringUnsafe("1.20"))) ] @@ -551,7 +545,7 @@ describe("Query", () => { assert.deepStrictEqual(result, [{ total: BigDecimal.fromBigInt(0n) }]) })) - it.effect("batches of different types combine in their common type", () => + it.effect("batches convert to the declared column types; undeclared, they must share their columns", () => Effect.gen(function*() { const Price = Schema.Struct({ d: Schema.BigDecimal }) const price = (d: string, scale: number) => @@ -560,31 +554,63 @@ describe("Query", () => { }) const source = Stream.make(price("1.2", 1), price("1.23", 2), price("1.2", 2)) const declared = Query.table("t", Price, { types: { d: ArrowSchema.decimal(18, 2) } }) - const untyped = Query.fromPlan({ _tag: "Table", name: "t" }) as typeof declared const d = (s: string) => BigDecimal.fromStringUnsafe(s) - for (const table of [declared, untyped]) { - const totals = table.pipe( - Query.aggregate((c) => ({ s: Expr.sum(c.d), lo: Expr.min(c.d), hi: Expr.max(c.d) })) - ) - assert.deepStrictEqual(yield* streamed(totals, source), [{ s: d("3.63"), lo: d("1.20"), hi: d("1.23") }]) - const groups = table.pipe( - Query.groupBy("d"), - Query.agg(() => ({ n: Expr.count() })), - Query.sort((c) => [c.d]) + const totals = declared.pipe( + Query.aggregate((c) => ({ s: Expr.sum(c.d), lo: Expr.min(c.d), hi: Expr.max(c.d) })) + ) + assert.deepStrictEqual(yield* streamed(totals, source), [{ s: d("3.63"), lo: d("1.20"), hi: d("1.23") }]) + const groups = declared.pipe( + Query.groupBy("d"), + Query.agg(() => ({ n: Expr.count() })), + Query.sort((c) => [c.d]) + ) + assert.deepStrictEqual(yield* streamed(groups, source), [{ d: d("1.20"), n: 2 }, { d: d("1.23"), n: 1 }]) + // Undeclared, a batch of other types would make the column types of + // earlier results depend on later batches. + const untyped = Query.fromPlan({ _tag: "Table", name: "t" }) + for ( + const query of [ + untyped, + Query.concat([untyped, untyped]), + Query.aggregate(untyped, () => ({ n: Expr.count() })) + ] + ) { + assert.strictEqual( + (yield* Effect.flip(streamed(query, source))).message, + `a batch of table "t" has columns [d: Decimal[18, 2]], its first batch [d: Decimal[18, 1]]: ` + + `the batches of a streamed table share their columns` ) - assert.deepStrictEqual(yield* streamed(groups, source), [{ d: d("1.20"), n: 2 }, { d: d("1.23"), n: 1 }]) } const lossy = Query.table("t", Price, { types: { d: ArrowSchema.decimal(18, 1) } }) const error = yield* Effect.flip(streamed(lossy, source)) assert.strictEqual(error.message, `column "d" has values that Decimal[18, 1] can't hold exactly`) })) - it.effect("aggregates of batches of different types are those of the batches concatenated, in either order", () => + it.effect("results over inputs of different types are those over their concatenation, streamed or not", () => Effect.gen(function*() { + type Frame = DataFrame.DataFrame<{ readonly x: number }> const x = (type: ArrowSchema.DataType, values: ReadonlyArray) => DataFrame.fromRows(Row, values.map((x) => ({ x })), { types: { x: type } }) - const untyped = Query.fromPlan({ _tag: "Table", name: "t" }) as typeof t - const query = (table: typeof t) => + const wide = (values: BigUint64Array | BigInt64Array) => + DataFrame.fromColumns({ x: values }) as unknown as Frame + const Price = Schema.Struct({ x: Schema.BigDecimal }) + const price = (d: string, precision: number, scale: number) => + DataFrame.fromRows(Price, [{ x: BigDecimal.fromStringUnsafe(d) }], { + types: { x: ArrowSchema.decimal(precision, scale) } + }) as unknown as Frame + // A query over the frames concatenated: in memory, and streamed with + // each frame a streamed table of its own. + const results = (frames: ReadonlyArray, query: (table: typeof t) => Query.Query) => { + const names = frames.map((_, i) => `t${i}`) + const concat = (tables: ReadonlyArray) => Query.concat(tables as [typeof t, ...Array]) + return [ + query(concat(frames.map(Query.from))), + query(concat(names.map((name) => Query.fromPlan({ _tag: "Table", name }) as typeof t))).pipe( + Effect.provide(Catalog.layer(Object.fromEntries(names.map((name, i) => [name, Stream.make(frames[i])])))) + ) + ] + } + const aggregates = (table: typeof t) => Query.aggregate(table, (c) => ({ sum: Expr.sum(c.x), mean: Expr.mean(c.x), @@ -596,50 +622,74 @@ describe("Query", () => { std: Expr.std(c.x), variance: Expr.variance(c.x) })) - // Expected values from Polars 2.1.0 over the concatenated batches, in - // order and reversed (first, last). Every result is Float64 but count. - // std and variance are left out where floats round them differently - // depending on how the rows are split. + const groups = (table: typeof t) => table.pipe(Query.groupBy("x"), Query.agg(() => ({ n: Expr.count() }))) + // Expected values from Polars 2.1.0 over the frames concatenated + // (`how="vertical_relaxed"`), in order and reversed: first and last, + // and groups in order of appearance. Every result is Float64 but + // count. std and variance are left out where floats round them + // differently depending on how the rows are split. const { float32, float64, int32 } = ArrowSchema const cases: ReadonlyArray< readonly [ - ReadonlyArray>, + ReadonlyArray, Record, - readonly [readonly [number, number], readonly [number, number]] + readonly [readonly [number, number], readonly [number, number]], + readonly [ReadonlyArray, ReadonlyArray] ] > = [ [ [x(int32, [2147483647]), x(int32, [1]), x(float64, [0.5])], { sum: 2147483648.5, mean: 715827882.8333334, min: 0.5, max: 2147483647, count: 3 }, - [[2147483647, 0.5], [0.5, 2147483647]] + [[2147483647, 0.5], [0.5, 2147483647]], + [[[2147483647, 1], [1, 1], [0.5, 1]], [[0.5, 1], [1, 1], [2147483647, 1]]] ], [ [x(float32, [1]), x(float64, [1.0000001])], { sum: 2.0000001000000003, mean: 1.0000000500000001, min: 1, max: 1.0000001, count: 2 }, - [[1, 1.0000001], [1.0000001, 1]] + [[1, 1.0000001], [1.0000001, 1]], + [[[1, 1], [1.0000001, 1]], [[1.0000001, 1], [1, 1]]] ], [ [x(float32, [1, 3]), x(float64, [2])], { sum: 6, mean: 2, min: 1, max: 3, count: 3, std: 1, variance: 1 }, - [[1, 2], [2, 3]] + [[1, 2], [2, 3]], + [[[1, 1], [3, 1], [2, 1]], [[2, 1], [1, 1], [3, 1]]] ], [ [x(float64, []), x(int32, [1, 2])], { sum: 3, mean: 1.5, min: 1, max: 2, count: 2, std: Math.SQRT1_2, variance: 0.5 }, - [[1, 2], [1, 2]] + [[1, 2], [1, 2]], + [[[1, 1], [2, 1]], [[1, 1], [2, 1]]] + ], + [ + [wide(BigUint64Array.of(2n ** 63n)), wide(BigInt64Array.of(-1n)), x(float64, [0.5])], + { sum: 2 ** 63, mean: 3074457345618258400, min: -1, max: 2 ** 63, count: 3 }, + [[2 ** 63, 0.5], [0.5, 2 ** 63]], + [[[2 ** 63, 1], [-1, 1], [0.5, 1]], [[0.5, 1], [-1, 1], [2 ** 63, 1]]] + ], + [ + [price("6000000000000000000", 19, 0), price("1.20", 18, 2), x(float64, [0.5])], + { sum: 6e18, mean: 2e18, min: 0.5, max: 6e18, count: 3 }, + [[6e18, 0.5], [0.5, 6e18]], + [[[6e18, 1], [1.2, 1], [0.5, 1]], [[0.5, 1], [1.2, 1], [6e18, 1]]] + ], + [ + [x(int32, [1, 2]), x(float64, [2, 0.5])], + { sum: 5.5, mean: 1.375, min: 0.5, max: 2, count: 4 }, + [[1, 0.5], [2, 2]], + [[[1, 1], [2, 2], [0.5, 1]], [[2, 2], [0.5, 1], [1, 1]]] ] ] - for (const [batches, expected, ends] of cases) { - for (const [order, [first, last]] of [[batches, ends[0]], [[...batches].reverse(), ends[1]]] as const) { - const results = [ - yield* query(untyped).pipe(Effect.provide(Catalog.layer({ t: Stream.fromIterable(order) }))), - yield* query(Query.concat(order.map(Query.from) as [typeof t, ...Array])) - ] - for (const df of results) { + for (const [frames, expected, ends, grouped] of cases) { + const orders = [[frames, ends[0], grouped[0]], [[...frames].reverse(), ends[1], grouped[1]]] as const + for (const [order, [first, last], keys] of orders) { + for (const query of results(order, aggregates)) { + const df = yield* query const [row] = DataFrame.toRows(df) - const keys = Object.keys(expected) assert.deepStrictEqual( - Object.fromEntries([...keys, "first", "last"].map((k) => [k, row[k as keyof typeof row]])), + Object.fromEntries( + [...Object.keys(expected), "first", "last"].map((k) => [k, row[k as keyof typeof row]]) + ), { ...expected, first, last } ) assert.deepStrictEqual( @@ -647,23 +697,45 @@ describe("Query", () => { df.schema.fields.map((f) => f.name === "count" ? ArrowSchema.uint32 : float64) ) } + for (const query of results(order, groups)) { + assert.deepStrictEqual(DataFrame.toRows(yield* query), keys.map(([x, n]) => ({ x, n }))) + } } } + // Row-wise operations see the common type too, as in Polars 2.1.0: Int8 + // 100 doubles to 200 in Int64, where doubling in Int8 wraps to -56. + const small = [DataFrame.fromColumns({ x: Int8Array.of(100) }), wide(BigInt64Array.of(1n))] + const rowWise: ReadonlyArray Query.Query, ReadonlyArray]> = [ + [(table) => Query.select(table, (c) => ({ y: Expr.multiply(c.x, 2) })), [{ y: 200n }, { y: 2n }]], + [(table) => Query.filter(table, (c) => Expr.gt(Expr.add(c.x, c.x), 150)), [{ x: 100n }]] + ] + for (const [query, expected] of rowWise) { + for (const result of results(small, query)) assert.deepStrictEqual(DataFrame.toRows(yield* result), expected) + } // A value the common type can't hold fails, as concatenating fails, though no minimum holds it. - const wide = [ - DataFrame.fromColumns({ x: BigUint64Array.of(2n ** 63n, 0n) }), - DataFrame.fromColumns({ x: BigInt64Array.of(-1n) }) - ] as const - const min = (table: Query.Query<{ readonly x: bigint }>) => - Query.aggregate(table, (c) => ({ min: Expr.min(c.x) })) - const failures = [ - min(Query.fromPlan({ _tag: "Table", name: "t" }) as Query.Query<{ readonly x: bigint }>).pipe( - Effect.provide(Catalog.layer({ t: Stream.make(...wide) })) - ), - min(Query.concat([Query.from(wide[0]), Query.from(wide[1])])) + const failures: ReadonlyArray, string]> = [ + [ + [wide(BigUint64Array.of(2n ** 63n, 0n)), wide(BigInt64Array.of(-1n))], + "UInt64 values don't fit their common type Int64" + ], + [ + [wide(BigInt64Array.of(-1n)), wide(BigUint64Array.of(2n ** 63n))], + "UInt64 values don't fit their common type Int64" + ], + [ + [price("6000000000000000000", 19, 0), price("1.20", 18, 2)], + "Decimal[19, 0] values don't fit their common type Decimal[38, 2]" + ] ] - for (const query of failures) { - assert.strictEqual((yield* Effect.flip(query)).message, "UInt64 values don't fit their common type Int64") + const min = (table: typeof t) => Query.aggregate(table, (c) => ({ min: Expr.min(c.x) })) + for (const [frames, message] of failures) { + const queries: ReadonlyArray> = [ + ...results(frames, min), + ...results(frames, groups) + ] + for (const query of queries) { + assert.strictEqual((yield* Effect.flip(query)).message, message) + } } })) From fc9ec8c98898a981f5517507ad87a64ebb3f9bdc Mon Sep 17 00:00:00 2001 From: Effect Bot Date: Sun, 15 Nov 2026 14:49:01 +0000 Subject: [PATCH 40/88] Fit integer literals to the other operands wherever they appear --- packages/effect/src/dataframe/Expr.ts | 5 +- .../effect/src/dataframe/internal/kernels.ts | 60 ++++++++++++------- packages/effect/test/dataframe/Query.test.ts | 59 +++++++++++++----- 3 files changed, 85 insertions(+), 39 deletions(-) diff --git a/packages/effect/src/dataframe/Expr.ts b/packages/effect/src/dataframe/Expr.ts index d1df8de2c97..ab17e674310 100644 --- a/packages/effect/src/dataframe/Expr.ts +++ b/packages/effect/src/dataframe/Expr.ts @@ -29,9 +29,10 @@ * - Integer arithmetic wraps on overflow; `divide` is true division and * returns a float; `floorDivide` and `mod` round towards negative infinity, * and an integer division by zero gives `null`. - * - An integer literal takes the integer type of the column it meets + * - An integer literal takes the integer type of the columns it meets * (`c.int8Column + 1` stays Int8) when the type holds it, and widens it - * otherwise (`c.int8Column + 300` is Int16). + * otherwise (`c.int8Column + 300` is Int16), wherever it appears among + * the operands (`coalesce(300, 70000, c.int8Column)` is Int32). * - Operands of different types meet in their common type. A value the * common type can't hold fails the query with a `DataFrameError`; only * an explicit {@link cast} makes it `null`. diff --git a/packages/effect/src/dataframe/internal/kernels.ts b/packages/effect/src/dataframe/internal/kernels.ts index b286dff308f..e8dbf5c56bd 100644 --- a/packages/effect/src/dataframe/internal/kernels.ts +++ b/packages/effect/src/dataframe/internal/kernels.ts @@ -326,21 +326,43 @@ const coerceValue = (value: Value, target: S.DataType): Value => export type ArithOp = "add" | "sub" | "mul" | "div" | "floorDiv" | "mod" /** - * The type and dynamism of an operand next to one of type `other`: an - * integer literal whose value `other` can't hold commits to the smallest - * signed integer type that holds it, instead of adopting `other`, as in - * Polars (`Int8 + 300` is Int16). + * The type and dynamism of an operand next to one of type `other`: integer + * literals (every value of an operand built from them) that `other` can't + * hold commit to the smallest signed integer type that holds them, instead + * of adopting `other`, as in Polars (`Int8 + 300` is Int16). */ const operand = (v: Value, other: S.DataType): readonly [S.DataType, D.Dynamic] => { - if (v.dynamic !== "int" || other._tag !== "Int" || v.vector.kind !== "i32") return [v.vector.type, v.dynamic] - const x = v.vector.values[0] - const [lo, hi] = intRange(other) - if (x >= lo && x <= hi) return [v.vector.type, v.dynamic] - return [x >= -128 && x <= 127 ? S.int8 : x >= -32768 && x <= 32767 ? S.int16 : S.int32, undefined] + const vector = v.vector + if (v.dynamic !== "int" || other._tag !== "Int" || vector.kind !== "i32") return [vector.type, v.dynamic] + let lo = 0 + let hi = 0 + for (let i = 0; i < vector.length; i++) { + if (!V.isValid(vector, i)) continue + lo = Math.min(lo, vector.values[i]) + hi = Math.max(hi, vector.values[i]) + } + const [min, max] = intRange(other) + if (lo >= min && hi <= max) return [vector.type, v.dynamic] + return [lo >= -128 && hi <= 127 ? S.int8 : lo >= -32768 && hi <= 32767 ? S.int16 : S.int32, undefined] } -/** The common type of two operands (`D.supertype`, literals fitted by {@link operand}). */ -const supertype = (a: Value, b: Value) => D.supertype(...operand(a, b.vector.type), ...operand(b, a.vector.type)) +/** + * The common type of operands (`D.supertype`): integer literals are fitted + * by {@link operand} to the common type of the other operands, wherever they + * appear among them, so the result doesn't depend on their order. + */ +const supertype = (...values: ReadonlyArray) => { + const fold = (types: ReadonlyArray) => + types.reduce>( + (acc, [type, dynamic]) => acc && D.supertype(acc.type, acc.dynamic, type, dynamic), + { type: S.nullType, dynamic: undefined } + ) + const fixed = fold(values.flatMap((v) => v.dynamic === undefined ? [[v.vector.type, undefined] as const] : [])) + return fixed && fold([ + [fixed.type, undefined], + ...values.flatMap((v) => v.dynamic === undefined ? [] : [operand(v, fixed.type)]) + ]) +} const unify = (op: string, a: Value, b: Value) => { const st = supertype(a, b) @@ -1128,26 +1150,18 @@ export const select = (branches: ReadonlyArray, choice: Int32Array): V /** First non-null value per slot. @internal */ export const coalesce = (values: ReadonlyArray, length: number): Value => { - let acc = values[0] - let type = acc.vector.type - let dynamic = acc.dynamic - for (const v of values.slice(1)) { - const st = D.supertype(type, dynamic, ...operand(v, type)) - if (st === undefined) return V.fail(`coalesce arguments have incompatible types`) - type = st.type - dynamic = st.dynamic - } + const st = supertype(...values) + if (st === undefined) return V.fail(`coalesce arguments have incompatible types`) const allScalar = values.every((v) => v.scalar) const n = allScalar ? 1 : length - const vectors = values.map((v) => materialize(coerceValue(v, type), n)) + const vectors = values.map((v) => materialize(coerceValue(v, st.type), n)) const choice = new Int32Array(n) for (let i = 0; i < n; i++) { let k = 0 while (k < vectors.length - 1 && !V.isValid(vectors[k], i)) k++ choice[i] = k } - acc = { vector: select(vectors, choice), scalar: allScalar, dynamic } - return acc + return { vector: select(vectors, choice), scalar: allScalar, dynamic: st.dynamic } } // ----------------------------------------------------------------------------- diff --git a/packages/effect/test/dataframe/Query.test.ts b/packages/effect/test/dataframe/Query.test.ts index ca35c03a707..fcfe8868106 100644 --- a/packages/effect/test/dataframe/Query.test.ts +++ b/packages/effect/test/dataframe/Query.test.ts @@ -245,22 +245,53 @@ describe("Query", () => { const error = yield* Effect.flip(query) assert.strictEqual(error.message, "Decimal[19, 0] values don't fit their common type Decimal[38, 2]") } - // An integer literal a column's type can't hold widens it, as in Polars 2.1.0. - const df = yield* Query.from(DataFrame.fromColumns({ x: Int8Array.of(1) })).pipe( - Query.select((c) => ({ - sum: Expr.add(c.x, 300), - gt: Expr.gt(c.x, 1000), - when: Expr.when(Expr.gt(c.x, 0), c.x, 1000), - coalesce: Expr.coalesce(c.x, 1000) + // Integer literals a column's type can't hold widen it, wherever they + // appear among the operands. Expected values from Polars 2.1.0 unless noted. + const { bool, float64, int8, int16, int32 } = ArrowSchema + const literals: ReadonlyArray< + readonly [ + string, + (c: Query.Columns<{ readonly x: number | null; readonly u: number | null }>) => Expr.Expr, + ArrowSchema.DataType, + ReadonlyArray + ] + > = [ + ["x + 300", (c) => Expr.add(c.x, 300), int16, [null, 305]], + ["300 + x", (c) => Expr.add(Expr.lit(300), c.x), int16, [null, 305]], + ["x > 1000", (c) => Expr.gt(c.x, 1000), bool, [null, false]], + ["when(x > 0, x, 300)", (c) => Expr.when(Expr.gt(c.x, 0), c.x, 300), int16, [300, 5]], + ["when(x > 0, 300, x)", (c) => Expr.when(Expr.gt(c.x, 0), 300, c.x), int16, [null, 300]], + ["coalesce(x, 300)", (c) => Expr.coalesce(c.x, 300), int16, [300, 5]], + ["coalesce(300, x)", (c) => Expr.coalesce(300, c.x), int16, [300, 300]], + ["fillNull(x, 300)", (c) => Expr.fillNull(c.x, 300), int16, [300, 5]], + ["coalesce(null, 300, x)", (c) => Expr.coalesce(null, 300, c.x), int16, [300, 300]], + ["coalesce(x, null, 300)", (c) => Expr.coalesce(c.x, null, 300), int16, [300, 5]], + ["coalesce(x, 300, 70000)", (c) => Expr.coalesce(c.x, 300, 70000), int32, [300, 5]], + // Polars 2.1.0 gives Int16, which can't hold 70000. + ["coalesce(300, 70000, x)", (c) => Expr.coalesce(300, 70000, c.x), int32, [300, 300]], + ["coalesce(1, x)", (c) => Expr.coalesce(1, c.x), int8, [1, 1]], + ["coalesce(300, 1.5, x)", (c) => Expr.coalesce(300, 1.5, c.x), float64, [300, 300]], + ["coalesce(u, -1)", (c) => Expr.coalesce(c.u, -1), int16, [-1, 5]], + ["coalesce(-1, u)", (c) => Expr.coalesce(-1, c.u), int16, [-1, -1]], + // Every value of an operand built from literals is fitted, not only + // the first row's; Polars 2.1.0 gives Int8, making 5 + 300 null. + ["x + when(x is null, 1, 300)", (c) => Expr.add(c.x, Expr.when(Expr.isNull(c.x), 1, 300)), int16, [null, 305]] + ] + // x is Int8 and u UInt8, both [null, 5]. + const ints = Query.from(DataFrame.fromColumns({ x: Int8Array.of(0, 5), u: Uint8Array.of(0, 5) })).pipe( + Query.withColumns((c) => ({ + x: Expr.when(Expr.gt(c.x, 0), c.x, null), + u: Expr.when(Expr.gt(c.u, 0), c.u, null) })) ) - assert.deepStrictEqual(DataFrame.toRows(df), [{ sum: 301, gt: false, when: 1, coalesce: 1 }]) - assert.deepStrictEqual(df.schema.fields.map((f) => f.type), [ - ArrowSchema.int16, - ArrowSchema.bool, - ArrowSchema.int16, - ArrowSchema.int16 - ]) + for (const [name, expr, type, values] of literals) { + const df = yield* Query.select(ints, (c) => ({ r: expr(c) })) + assert.deepStrictEqual( + [df.schema.fields[0].type, DataFrame.toRows(df).map((row) => row.r)], + [type, values], + name + ) + } })) it.effect("temporal operands of mixed units meet in the coarser unit, as in Polars", () => From be557ebd22a156e806702cd24fdcb83d71bdd99b Mon Sep 17 00:00:00 2001 From: Effect Bot Date: Sun, 15 Nov 2026 14:32:01 +0000 Subject: [PATCH 41/88] Reserve table names from the names the SQL compiler invents --- packages/effect/src/dataframe/internal/sql.ts | 34 ++++++++++++++----- packages/sql/pglite/test/DataFrameSql.test.ts | 25 ++++++++++++++ 2 files changed, 51 insertions(+), 8 deletions(-) diff --git a/packages/effect/src/dataframe/internal/sql.ts b/packages/effect/src/dataframe/internal/sql.ts index 1c25e57a4b4..c6f0a118d58 100644 --- a/packages/effect/src/dataframe/internal/sql.ts +++ b/packages/effect/src/dataframe/internal/sql.ts @@ -15,8 +15,10 @@ * renders every column as text (temporal values as integers in their unit), * and `decode` reads the rows back as columns of those types. * - * Helper columns (the row key and the like) share one prefix, chosen per - * plan so that no column of the plan starts with it. + * The names the compiler invents (helper columns like the row key, the + * sources of tables) share one prefix, chosen per plan so that no name the + * plan supplies (its tables' and its columns') starts with it (see + * {@link inventor}). * * @internal */ @@ -37,7 +39,7 @@ import * as V from "./vector.ts" interface Compiler { /** The engine's frame for every node of the plan, on zero-row inputs. */ readonly frames: ReadonlyMap - /** A quoted helper column name, disjoint from the plan's column names. */ + /** A quoted invented name, disjoint from the plan's table and column names (see {@link inventor}). */ readonly hidden: (name: string) => string /** The quoted row key. */ readonly row: string @@ -1158,6 +1160,26 @@ const source = (name: string, tables: ReadonlyArray): string => { return `MATERIALIZED (SELECT ${columns.length === 0 ? "1" : list(columns)} FROM ${quote(name)})` } +/** + * Quotes the names the compiler invents for the plan of `frames`, all with + * one prefix that no table or column name of the plan starts with, so that + * no invented name equals a name the plan supplies: a helper column can't + * hide a column, and a source can't shadow a table it reads (a common table + * expression shadows the table of its name in every `FROM` in its scope, + * the other sources' included; qualifying the tables' names with their + * schema is no way out, as the schema isn't known). Names PostgreSQL + * truncates keep their start, so this holds for them too. + */ +const inventor = (frames: ReadonlyMap): (name: string) => string => { + const names = [...frames].flatMap(([node, frame]) => [ + ...(node._tag === "Table" ? [node.name] : []), + ...frame.columns.map((col) => col.name) + ]) + let prefix = "__" + while (names.some((name) => name.startsWith(prefix))) prefix += "_" + return (name) => quote(prefix + name) +} + /** One SQL query whose first row holds the values of checks. */ interface Checked { readonly sql: string @@ -1189,11 +1211,7 @@ export interface Compiled { export const compile = (plan: Plan.Node): Compiled => { const frames = new Map() const result = typed(plan, frames) - const used = new Set() - for (const frame of frames.values()) for (const col of frame.columns) used.add(col.name) - let prefix = "__" - while ([...used].some((name) => name.startsWith(prefix))) prefix += "_" - const hidden = (name: string) => quote(prefix + name) + const hidden = inventor(frames) const tables = [...frames].flatMap(([node, frame]) => node._tag === "Table" ? [{ node, checks: rowChecks(node, frame) }] : [] ) diff --git a/packages/sql/pglite/test/DataFrameSql.test.ts b/packages/sql/pglite/test/DataFrameSql.test.ts index 9dd5dae93c3..4b425fb4471 100644 --- a/packages/sql/pglite/test/DataFrameSql.test.ts +++ b/packages/sql/pglite/test/DataFrameSql.test.ts @@ -37,6 +37,9 @@ const Setup = Layer.effectDiscard( yield* sql`DROP TABLE IF EXISTS sales` yield* sql`DROP TABLE IF EXISTS regions` yield* sql`DROP TABLE IF EXISTS contract` + yield* sql`DROP TABLE IF EXISTS second` + yield* sql`DROP TABLE IF EXISTS "__table0"` + yield* sql`DROP TABLE IF EXISTS "__table1"` yield* sql`CREATE TABLE sales (id DOUBLE PRECISION, region TEXT, units DOUBLE PRECISION, price DOUBLE PRECISION, at TIMESTAMP)` yield* sql`CREATE TABLE regions (region TEXT, manager TEXT)` for (const s of sales) { @@ -47,6 +50,13 @@ const Setup = Layer.effectDiscard( for (const r of regions) yield* sql`INSERT INTO regions VALUES (${r.region}, ${r.manager})` yield* sql`CREATE TABLE contract (s TEXT, n DOUBLE PRECISION)` yield* sql`INSERT INTO contract VALUES ('bad', NULL), ('good', 1)` + // Tables named like the sources of other tables. + yield* sql`CREATE TABLE second (s TEXT)` + yield* sql`INSERT INTO second VALUES ('good')` + yield* sql`CREATE TABLE "__table0" (s TEXT)` + yield* sql`INSERT INTO "__table0" VALUES ('bad')` + yield* sql`CREATE TABLE "__table1" (s TEXT)` + yield* sql`INSERT INTO "__table1" VALUES ('bad')` yield* sql`DROP TABLE IF EXISTS odd` yield* sql`CREATE TABLE odd (nil INTEGER, nothing INTEGER, at TIMESTAMPTZ, far TIMESTAMP, months INTERVAL, forever TIMESTAMP, endless DATE)` yield* sql`INSERT INTO odd VALUES (42, NULL, '2024-01-01 00:30:00+00', '290000-01-01', '1 month', 'infinity', 'infinity')` @@ -614,6 +624,21 @@ describe("effect/dataframe on PostgreSQL (PGlite)", { concurrent: false }, () => ]) })) + it.effect("tables are read from their names, whatever the compiler names", () => + Effect.gen(function*() { + const s = (name: string) => Query.table(name, Schema.Struct({ s: Schema.String })) + assert.deepStrictEqual( + yield* rowsOnSql(Query.concat([s("contract"), s("__table0"), s("second"), s("__table1")])), + [{ s: "bad" }, { s: "good" }, { s: "bad" }, { s: "good" }, { s: "bad" }] + ) + const error = yield* Effect.flip(onSql(Query.concat([ + s("contract").pipe(Query.filter((c) => Expr.eq(c.s, "good"))), + s("second"), + Query.table("__table1", Schema.Struct({ s: Schema.Literal("good") })) + ]))) + assert.strictEqual(error.message, `column "s" has values its schema rejects`) + })) + it.effect("timestamps with time zones are UTC, whatever the session's time zone", () => Effect.gen(function*() { const sql = yield* SqlClient From 3f0fea02098029ce6d3fc1361531e5e5cae169db Mon Sep 17 00:00:00 2001 From: Effect Bot Date: Sun, 15 Nov 2026 14:32:23 +0000 Subject: [PATCH 42/88] Qualify the columns the SQL column type checks read by their table --- packages/effect/src/dataframe/internal/sql.ts | 7 ++++--- packages/sql/pglite/test/DataFrameSql.test.ts | 5 +++++ 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/packages/effect/src/dataframe/internal/sql.ts b/packages/effect/src/dataframe/internal/sql.ts index c6f0a118d58..7f6e9567cc7 100644 --- a/packages/effect/src/dataframe/internal/sql.ts +++ b/packages/effect/src/dataframe/internal/sql.ts @@ -1086,7 +1086,8 @@ const declared = (column: Plan.TableColumn): string => { /** * The checks of a table's declared column types that don't read its rows: * each value is the PostgreSQL type of a column the engine doesn't read as - * declared, or null. + * declared, or null. Each column is qualified by its table: unqualified, a + * column the table lacks would be the `pg_type` column of its name. */ const typeChecks = (table: Plan.Table): Array => (table.columns ?? []).flatMap((column): Array => { @@ -1099,9 +1100,9 @@ const typeChecks = (table: Plan.Table): Array => return [{ sql: `(SELECT CASE WHEN ${ accepted.join(" OR ") - } THEN NULL ELSE format_type(oid, NULL) END FROM pg_type WHERE oid = pg_typeof((SELECT ${ + } THEN NULL ELSE format_type(oid, NULL) END FROM pg_type WHERE oid = pg_typeof((SELECT t.${ quote(column.name) - } FROM ${quote(table.name)} LIMIT 0)))`, + } FROM ${quote(table.name)} AS t LIMIT 0)))`, verify: (actual) => { if (actual !== null) V.fail(`column ${JSON.stringify(column.name)} is ${actual}, expected ${column.type}`) } diff --git a/packages/sql/pglite/test/DataFrameSql.test.ts b/packages/sql/pglite/test/DataFrameSql.test.ts index 4b425fb4471..71ad1799c77 100644 --- a/packages/sql/pglite/test/DataFrameSql.test.ts +++ b/packages/sql/pglite/test/DataFrameSql.test.ts @@ -563,6 +563,11 @@ describe("effect/dataframe on PostgreSQL (PGlite)", { concurrent: false }, () => assert.strictEqual(yield* rejects(query), message) } assert.deepStrictEqual(yield* rowsOnSql(odd(Schema.Struct({ nothing: Schema.Null }))), [{ nothing: null }]) + // A declared column the table doesn't have, named like a column of PostgreSQL's catalog. + for (const schema of [Schema.Struct({ oid: Schema.BigInt }), Schema.Struct({ typname: Schema.String })]) { + const missing = yield* Effect.flip(onSql(Query.table("contract", schema))) + assert.include(missing.message, `.${Object.keys(schema.fields)[0]} does not exist`) + } })) it.effect("tables reject values their declared types can't hold exactly, as in memory", () => From 93f09f6660f0023cb1584e47dd8f3eaf486216e7 Mon Sep 17 00:00:00 2001 From: Effect Bot Date: Sun, 15 Nov 2026 14:33:36 +0000 Subject: [PATCH 43/88] Document that SQL table names are single quoted identifiers --- packages/effect/src/dataframe/Sql.ts | 4 +++- packages/sql/pglite/test/DataFrameSql.test.ts | 8 ++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/packages/effect/src/dataframe/Sql.ts b/packages/effect/src/dataframe/Sql.ts index 3958f50696d..4030eeda697 100644 --- a/packages/effect/src/dataframe/Sql.ts +++ b/packages/effect/src/dataframe/Sql.ts @@ -9,7 +9,9 @@ * joins in left row order, as the in-memory engine. * - {@link layer} provides an `Engine` that compiles every query and runs it * with the `SqlClient` in the context, so `yield* query` reads from the - * database. `Query.table` names become table names. + * database. `Query.table` names become table names, each one quoted + * identifier: `"a.b"` reads the table called `a.b`, not table `b` of + * schema `a`. * - The SQL is PostgreSQL's (PGlite included); other databases are future * work. * diff --git a/packages/sql/pglite/test/DataFrameSql.test.ts b/packages/sql/pglite/test/DataFrameSql.test.ts index 71ad1799c77..b36af22b411 100644 --- a/packages/sql/pglite/test/DataFrameSql.test.ts +++ b/packages/sql/pglite/test/DataFrameSql.test.ts @@ -57,6 +57,13 @@ const Setup = Layer.effectDiscard( yield* sql`INSERT INTO "__table0" VALUES ('bad')` yield* sql`CREATE TABLE "__table1" (s TEXT)` yield* sql`INSERT INTO "__table1" VALUES ('bad')` + yield* sql`DROP TABLE IF EXISTS "a.b"` + yield* sql`CREATE TABLE "a.b" (s TEXT)` + yield* sql`INSERT INTO "a.b" VALUES ('good')` + yield* sql`CREATE SCHEMA IF NOT EXISTS a` + yield* sql`DROP TABLE IF EXISTS a.b` + yield* sql`CREATE TABLE a.b (s TEXT)` + yield* sql`INSERT INTO a.b VALUES ('bad')` yield* sql`DROP TABLE IF EXISTS odd` yield* sql`CREATE TABLE odd (nil INTEGER, nothing INTEGER, at TIMESTAMPTZ, far TIMESTAMP, months INTERVAL, forever TIMESTAMP, endless DATE)` yield* sql`INSERT INTO odd VALUES (42, NULL, '2024-01-01 00:30:00+00', '290000-01-01', '1 month', 'infinity', 'infinity')` @@ -642,6 +649,7 @@ describe("effect/dataframe on PostgreSQL (PGlite)", { concurrent: false }, () => Query.table("__table1", Schema.Struct({ s: Schema.Literal("good") })) ]))) assert.strictEqual(error.message, `column "s" has values its schema rejects`) + assert.deepStrictEqual(yield* rowsOnSql(s("a.b")), [{ s: "good" }]) })) it.effect("timestamps with time zones are UTC, whatever the session's time zone", () => From f79b5821db66bc38bc9331ab3c29a69cbda759f4 Mon Sep 17 00:00:00 2001 From: Effect Bot Date: Sun, 15 Nov 2026 15:11:18 +0000 Subject: [PATCH 44/88] Fit integer literals of when and coalesce by their plan-time range The type of an operand built from integer literals was chosen from the values of the current batch, so results depended on batch boundaries and slice pushdown. A when or coalesce of literals now carries the range of its literals; a scalar fits by its value; other operands computed from literals count as Int32. --- packages/effect/src/dataframe/Expr.ts | 5 ++ .../effect/src/dataframe/internal/kernels.ts | 53 +++++++++++++------ packages/effect/test/dataframe/Query.test.ts | 44 ++++++++++----- packages/sql/pglite/test/DataFrameSql.test.ts | 6 +++ 4 files changed, 81 insertions(+), 27 deletions(-) diff --git a/packages/effect/src/dataframe/Expr.ts b/packages/effect/src/dataframe/Expr.ts index ab17e674310..a75df8fae6d 100644 --- a/packages/effect/src/dataframe/Expr.ts +++ b/packages/effect/src/dataframe/Expr.ts @@ -33,6 +33,11 @@ * (`c.int8Column + 1` stays Int8) when the type holds it, and widens it * otherwise (`c.int8Column + 300` is Int16), wherever it appears among * the operands (`coalesce(300, 70000, c.int8Column)` is Int32). + * A `when`, `coalesce` or `fillNull` of literals fits all its literals, + * whichever rows select them (`c.int8Column + when(cond, 300, 1)` is + * Int16), so result types never depend on the rows. Other values computed + * from integer literals (`negate`, arithmetic) widen narrower columns to + * Int32. * - Operands of different types meet in their common type. A value the * common type can't hold fails the query with a `DataFrameError`; only * an explicit {@link cast} makes it `null`. diff --git a/packages/effect/src/dataframe/internal/kernels.ts b/packages/effect/src/dataframe/internal/kernels.ts index e8dbf5c56bd..71af53679df 100644 --- a/packages/effect/src/dataframe/internal/kernels.ts +++ b/packages/effect/src/dataframe/internal/kernels.ts @@ -27,6 +27,12 @@ export interface Value { readonly vector: V.Vector readonly scalar: boolean readonly dynamic: D.Dynamic + /** + * For a dynamic integer column (a `when` or `coalesce` of literals), the + * bounds of its literals, known from the plan; Int32's when unknown. See + * {@link literalRange}. + */ + readonly range?: readonly [number, number] | undefined } /** @internal */ @@ -325,24 +331,36 @@ const coerceValue = (value: Value, target: S.DataType): Value => /** @internal */ export type ArithOp = "add" | "sub" | "mul" | "div" | "floorDiv" | "mod" +/** + * The bounds of a dynamic integer's values, from the plan alone, so that its + * type never depends on the rows (a batch, a slice of them): a scalar's + * value (built from literals only), or the {@link Value.range} of a column. + */ +const literalRange = (v: Value): readonly [number, number] => { + if (!v.scalar) return v.range ?? intRange(S.int32) + const x = (v.vector as V.I32).values[0] + return V.isValid(v.vector, 0) ? [x, x] : [0, 0] +} + +/** The range of `when` or `coalesce` of `values` whose common type is `dynamic`. */ +const rangeOf = (values: ReadonlyArray, dynamic: D.Dynamic): readonly [number, number] | undefined => { + if (dynamic !== "int") return undefined + const ranges = values.flatMap((v) => v.dynamic === "int" ? [literalRange(v)] : []) + return [Math.min(...ranges.map((r) => r[0])), Math.max(...ranges.map((r) => r[1]))] +} + /** * The type and dynamism of an operand next to one of type `other`: integer - * literals (every value of an operand built from them) that `other` can't - * hold commit to the smallest signed integer type that holds them, instead - * of adopting `other`, as in Polars (`Int8 + 300` is Int16). + * literals (every literal of an operand built from them, whichever rows + * select it) that `other` can't hold commit to the smallest signed integer + * type that holds them, instead of adopting `other`, as in Polars (`Int8 + + * 300` is Int16). */ const operand = (v: Value, other: S.DataType): readonly [S.DataType, D.Dynamic] => { - const vector = v.vector - if (v.dynamic !== "int" || other._tag !== "Int" || vector.kind !== "i32") return [vector.type, v.dynamic] - let lo = 0 - let hi = 0 - for (let i = 0; i < vector.length; i++) { - if (!V.isValid(vector, i)) continue - lo = Math.min(lo, vector.values[i]) - hi = Math.max(hi, vector.values[i]) - } + if (v.dynamic !== "int" || other._tag !== "Int" || v.vector.kind !== "i32") return [v.vector.type, v.dynamic] + const [lo, hi] = literalRange(v) const [min, max] = intRange(other) - if (lo >= min && hi <= max) return [vector.type, v.dynamic] + if (lo >= min && hi <= max) return [v.vector.type, v.dynamic] return [lo >= -128 && hi <= 127 ? S.int8 : lo >= -32768 && hi <= 32767 ? S.int16 : S.int32, undefined] } @@ -1092,7 +1110,12 @@ export const when = (cond: Value, then: Value, otherwise: Value, length: number) const ca = c.valid for (let i = 0; i < n; i++) mask[i] = (cv[i] & (ca === undefined ? 1 : ca[i])) === 1 ? 0 : 1 const branches = [materialize(coerceValue(then, st.type), n), materialize(coerceValue(otherwise, st.type), n)] - return { vector: select(branches, mask), scalar: allScalar, dynamic: st.dynamic } + return { + vector: select(branches, mask), + scalar: allScalar, + dynamic: st.dynamic, + range: rangeOf([then, otherwise], st.dynamic) + } } /** Picks slot `i` from `branches[choice[i]]`; all branches share a type. @internal */ @@ -1161,7 +1184,7 @@ export const coalesce = (values: ReadonlyArray, length: number): Value => while (k < vectors.length - 1 && !V.isValid(vectors[k], i)) k++ choice[i] = k } - return { vector: select(vectors, choice), scalar: allScalar, dynamic: st.dynamic } + return { vector: select(vectors, choice), scalar: allScalar, dynamic: st.dynamic, range: rangeOf(values, st.dynamic) } } // ----------------------------------------------------------------------------- diff --git a/packages/effect/test/dataframe/Query.test.ts b/packages/effect/test/dataframe/Query.test.ts index fcfe8868106..5d3c9299aca 100644 --- a/packages/effect/test/dataframe/Query.test.ts +++ b/packages/effect/test/dataframe/Query.test.ts @@ -248,6 +248,7 @@ describe("Query", () => { // Integer literals a column's type can't hold widen it, wherever they // appear among the operands. Expected values from Polars 2.1.0 unless noted. const { bool, float64, int8, int16, int32 } = ArrowSchema + const w = (x: Expr.Expr) => Expr.when(Expr.isNull(x), 1, 300) const literals: ReadonlyArray< readonly [ string, @@ -273,24 +274,43 @@ describe("Query", () => { ["coalesce(300, 1.5, x)", (c) => Expr.coalesce(300, 1.5, c.x), float64, [300, 300]], ["coalesce(u, -1)", (c) => Expr.coalesce(c.u, -1), int16, [-1, 5]], ["coalesce(-1, u)", (c) => Expr.coalesce(-1, c.u), int16, [-1, -1]], - // Every value of an operand built from literals is fitted, not only - // the first row's; Polars 2.1.0 gives Int8, making 5 + 300 null. - ["x + when(x is null, 1, 300)", (c) => Expr.add(c.x, Expr.when(Expr.isNull(c.x), 1, 300)), int16, [null, 305]] + // Every literal of an operand built from literals is fitted, whichever + // rows select it (w is when(x is null, 1, 300)); Polars 2.1.0 gives + // Int8, making 5 + 300 null and (100 + 1) * 2 wrap to -54. + ["x + w", (c) => Expr.add(c.x, w(c.x)), int16, [null, 305]], + [ + "(coalesce(x, 100) + w) * 2", + (c) => Expr.multiply(Expr.add(Expr.coalesce(c.x, 100), w(c.x)), 2), + int16, + [202, 610] + ], + // Other operands computed from literals count as Int32; Polars 2.1.0 + // gives Int8 with nulls. + ["x + -w", (c) => Expr.add(c.x, Expr.negate(w(c.x))), int32, [null, -295]] ] - // x is Int8 and u UInt8, both [null, 5]. - const ints = Query.from(DataFrame.fromColumns({ x: Int8Array.of(0, 5), u: Uint8Array.of(0, 5) })).pipe( - Query.withColumns((c) => ({ + // x is Int8 and u UInt8, both [null, 5]: whole, streamed a row at a + // time and sliced to the first row, results have the same types. + const ints = (xs: ReadonlyArray) => + DataFrame.fromColumns({ x: Int8Array.from(xs), u: Uint8Array.from(xs) }) + const nullable = (input: Query.Query<{ readonly x: number; readonly u: number }>) => + Query.withColumns(input, (c) => ({ x: Expr.when(Expr.gt(c.x, 0), c.x, null), u: Expr.when(Expr.gt(c.u, 0), c.u, null) })) + const whole = nullable(Query.from(ints([0, 5]))) + const streamed = nullable( + Query.table("t", Schema.Struct({ x: Schema.Number, u: Schema.Number }), { + types: { x: ArrowSchema.int8, u: ArrowSchema.uint8 } + }) ) + const batches = Catalog.layer({ t: Stream.make(ints([0]), ints([5])) }) + const result = (query: Query.Query<{ readonly r: unknown }>) => + Effect.map(query, (df) => [df.schema.fields[0].type, DataFrame.toArray(df, "r")]) for (const [name, expr, type, values] of literals) { - const df = yield* Query.select(ints, (c) => ({ r: expr(c) })) - assert.deepStrictEqual( - [df.schema.fields[0].type, DataFrame.toRows(df).map((row) => row.r)], - [type, values], - name - ) + const select = (input: typeof whole) => Query.select(input, (c) => ({ r: expr(c) })) + assert.deepStrictEqual(yield* result(select(whole)), [type, values], name) + assert.deepStrictEqual(yield* result(select(streamed)).pipe(Effect.provide(batches)), [type, values], name) + assert.deepStrictEqual(yield* result(Query.head(select(whole), 1)), [type, values.slice(0, 1)], name) } })) diff --git a/packages/sql/pglite/test/DataFrameSql.test.ts b/packages/sql/pglite/test/DataFrameSql.test.ts index b36af22b411..62475354d94 100644 --- a/packages/sql/pglite/test/DataFrameSql.test.ts +++ b/packages/sql/pglite/test/DataFrameSql.test.ts @@ -448,6 +448,12 @@ const cases: ReadonlyArray]> = [ Query.from(DataFrame.fromColumns({ x: Int32Array.of(5, -15, 25) })).pipe( Query.select((c) => ({ r0: Expr.round(c.x), r1: Expr.round(c.x, 1) })) ) + ], + [ + "integer literals fitted whichever rows select them", + Query.from(DataFrame.fromColumns({ x: Int8Array.of(100, 0) })).pipe( + Query.select((c) => ({ r: Expr.multiply(Expr.add(c.x, Expr.when(Expr.eq(c.x, 0), 300, 1)), 2) })) + ) ] ] From 28fa44562d2d7aabfd395e0c28ab92b1441f03c3 Mon Sep 17 00:00:00 2001 From: Effect Bot Date: Sun, 15 Nov 2026 15:11:18 +0000 Subject: [PATCH 45/88] Fix the type of an integer literal cast to its own type A cast to Int32 kept an integer literal flexible, so it still adopted the type of the column it met (Int8 + cast(5, Int32) was Int8). --- packages/effect/src/dataframe/Expr.ts | 2 +- packages/effect/src/dataframe/internal/kernels.ts | 11 ++++++----- packages/effect/test/dataframe/Query.test.ts | 1 + 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/packages/effect/src/dataframe/Expr.ts b/packages/effect/src/dataframe/Expr.ts index a75df8fae6d..cac85f9fa6c 100644 --- a/packages/effect/src/dataframe/Expr.ts +++ b/packages/effect/src/dataframe/Expr.ts @@ -37,7 +37,7 @@ * whichever rows select them (`c.int8Column + when(cond, 300, 1)` is * Int16), so result types never depend on the rows. Other values computed * from integer literals (`negate`, arithmetic) widen narrower columns to - * Int32. + * Int32, and a {@link cast} fixes a literal's type. * - Operands of different types meet in their common type. A value the * common type can't hold fails the query with a `DataFrameError`; only * an explicit {@link cast} makes it `null`. diff --git a/packages/effect/src/dataframe/internal/kernels.ts b/packages/effect/src/dataframe/internal/kernels.ts index 71af53679df..c98e89b2abc 100644 --- a/packages/effect/src/dataframe/internal/kernels.ts +++ b/packages/effect/src/dataframe/internal/kernels.ts @@ -312,11 +312,12 @@ export const concat = (parts: ReadonlyArray): V.Vector => { return V.concat(parts.map((p) => coerce(p, type))) } -/** Casts a value ({@link cast}), keeping its shape. @internal */ -export const castValue = (value: Value, target: S.DataType): Value => - D.sameType(value.vector.type, target) - ? value - : { vector: cast(value.vector, target), scalar: value.scalar, dynamic: undefined } +/** Casts a value ({@link cast}), keeping its shape; a cast literal has a fixed type. @internal */ +export const castValue = (value: Value, target: S.DataType): Value => ({ + vector: D.sameType(value.vector.type, target) ? value.vector : cast(value.vector, target), + scalar: value.scalar, + dynamic: undefined +}) /** Coerces a value ({@link coerce}), keeping its shape. */ const coerceValue = (value: Value, target: S.DataType): Value => diff --git a/packages/effect/test/dataframe/Query.test.ts b/packages/effect/test/dataframe/Query.test.ts index 5d3c9299aca..a8bf3bbd505 100644 --- a/packages/effect/test/dataframe/Query.test.ts +++ b/packages/effect/test/dataframe/Query.test.ts @@ -274,6 +274,7 @@ describe("Query", () => { ["coalesce(300, 1.5, x)", (c) => Expr.coalesce(300, 1.5, c.x), float64, [300, 300]], ["coalesce(u, -1)", (c) => Expr.coalesce(c.u, -1), int16, [-1, 5]], ["coalesce(-1, u)", (c) => Expr.coalesce(-1, c.u), int16, [-1, -1]], + ["x + cast(5, Int32)", (c) => Expr.add(c.x, Expr.cast(Expr.lit(5), "Int32")), int32, [null, 10]], // Every literal of an operand built from literals is fitted, whichever // rows select it (w is when(x is null, 1, 300)); Polars 2.1.0 gives // Int8, making 5 + 300 null and (100 + 1) * 2 wrap to -54. From 2a10bd3cfa8366b7c31f8102f060d1ce08daf432 Mon Sep 17 00:00:00 2001 From: Effect Bot Date: Sun, 15 Nov 2026 15:15:49 +0000 Subject: [PATCH 46/88] Import Arrow tables without batches with the nullability of their fields A frame without rows is now built in one place (F.empty), with the declared nullability and no validity bitmap, which made non-nullable fields nullable. --- .../effect/src/dataframe/internal/frame.ts | 24 ++++++++++++++++--- .../effect/test/dataframe/DataFrame.test.ts | 11 +++++++++ 2 files changed, 32 insertions(+), 3 deletions(-) diff --git a/packages/effect/src/dataframe/internal/frame.ts b/packages/effect/src/dataframe/internal/frame.ts index 1545af1766f..acb6a66586a 100644 --- a/packages/effect/src/dataframe/internal/frame.ts +++ b/packages/effect/src/dataframe/internal/frame.ts @@ -115,6 +115,23 @@ export const make = (columns: ReadonlyArray, length: number): Frame => { return self } +/** + * A frame without rows with the given columns. Its vectors have no validity + * bitmap, which would make a column nullable where it meets others. + * + * @internal + */ +export const empty = ( + fields: ReadonlyArray<{ readonly name: string; readonly type: S.DataType; readonly nullable: boolean }> +): Frame => + make( + fields.map((f) => { + const vector = V.withValid(K.nullsOf(f.type, 0), undefined) + return new Column(f.name, vector, undefined, f.nullable || vector.kind === "null") + }), + 0 + ) + /** @internal */ export const fromVectors = (names: ReadonlyArray, vectors: ReadonlyArray, length?: number): Frame => { const n = length ?? vectors[0]?.length ?? 0 @@ -142,10 +159,11 @@ export const context = (self: Frame): E.Context => ({ /** @internal */ export const fromArrowTable = (table: ArrowTable.Table): Frame => { + if (table.batches.length === 0) { + return empty(table.schema.fields.map((f) => ({ name: f.name, type: V.storageType(f.type), nullable: f.nullable }))) + } const length = table.batches.reduce((s, b) => s + b.length, 0) - const vectors = table.batches.length === 0 - ? table.schema.fields.map((f) => K.nullsOf(V.storageType(f.type), 0)) - : table.schema.fields.map((_, c) => V.concat(table.batches.map((b) => V.fromArrowData(b.columns[c])))) + const vectors = table.schema.fields.map((_, c) => V.concat(table.batches.map((b) => V.fromArrowData(b.columns[c])))) return make( table.schema.fields.map((f, c) => new Column( diff --git a/packages/effect/test/dataframe/DataFrame.test.ts b/packages/effect/test/dataframe/DataFrame.test.ts index d59f4221203..e80b7e7fef0 100644 --- a/packages/effect/test/dataframe/DataFrame.test.ts +++ b/packages/effect/test/dataframe/DataFrame.test.ts @@ -266,6 +266,17 @@ describe("DataFrame", () => { assert.deepStrictEqual(DataFrame.toRows(DataFrame.fromArrowTable(table)), DataFrame.toRows(df)) }) + it("imports a table without batches with the nullability of its fields", () => { + const schema = ArrowSchema.make([ + ArrowSchema.field("x", ArrowSchema.int8, { nullable: false }), + ArrowSchema.field("s", ArrowSchema.utf8) + ]) + assert.deepStrictEqual(types(DataFrame.fromArrowTable(ArrowTable.make(schema, []))), [ + "x:Int:false", + "s:Utf8:true" + ]) + }) + it("imports sliced list columns with only their own elements", () => { const Lists = Schema.Struct({ xs: Schema.Array(Schema.Number) }) const df = DataFrame.fromRows(Lists, [{ xs: [1] }, { xs: [2] }, { xs: [3] }]) From 766686fb4639e203ac466261e938fc6ad2079893 Mon Sep 17 00:00:00 2001 From: Effect Bot Date: Sun, 15 Nov 2026 15:15:49 +0000 Subject: [PATCH 47/88] Keep the columns of streamed results without rows, or fail where they are unknown Every streamed node now has at least one batch unless its columns are unknown: a declared table without batches, and a slice without rows (which reads nothing), give their result from the declared columns. A node without batches depends on an undeclared table of which no batch was read; collecting, concatenating, joining or aggregating it fails asking for the columns to be declared, instead of inventing a result without columns. Tables named more than once are read into memory as batches of the same table, so they follow the same rule. --- packages/effect/src/dataframe/Plan.ts | 7 ++ .../effect/src/dataframe/internal/stream.ts | 115 ++++++++++++------ packages/effect/test/dataframe/Query.test.ts | 62 ++++++++++ 3 files changed, 145 insertions(+), 39 deletions(-) diff --git a/packages/effect/src/dataframe/Plan.ts b/packages/effect/src/dataframe/Plan.ts index 0bef15b39c6..6b1526062b2 100644 --- a/packages/effect/src/dataframe/Plan.ts +++ b/packages/effect/src/dataframe/Plan.ts @@ -721,6 +721,13 @@ export interface TableColumn { * Every batch of a streamed table, after this conversion (as given, without * `columns`), must have the column names and types of its first batch. * + * Without `columns`, a streamed table's columns come from its batches. When + * no batch is read (the table has none, or a slice without rows skips it), + * they are unknown: a result that needs them (collecting one, concatenating, + * joining, aggregating, sorting) fails with a `DataFrameError` asking for + * them to be declared, and `Query.stream` emits no batch for one that has no + * rows. + * * `schema`, set by `Query.table`, is the struct schema the rows must * satisfy: values of fields it constrains beyond their column type (literals, * enums, refinements) are checked against it. diff --git a/packages/effect/src/dataframe/internal/stream.ts b/packages/effect/src/dataframe/internal/stream.ts index 3bdea4e72bd..07cd1593368 100644 --- a/packages/effect/src/dataframe/internal/stream.ts +++ b/packages/effect/src/dataframe/internal/stream.ts @@ -18,6 +18,14 @@ * stream share its schema, and a concatenation converts its inputs' batches * to their common types, as concatenating their results would. * + * A node has at least one batch, possibly without rows, unless its columns + * are unknown: they depend on a table without declared columns of which no + * batch was read (it has none, or a slice without rows doesn't read it). + * Where no batch is read, a declared table's result is its declared columns + * without rows. Unknown columns are never made up: an operation that needs + * them (collecting a result, concatenating, joining, aggregating, any + * operation that collects its input) fails, asking for them to be declared. + * * @internal */ import * as Effect from "../../Effect.ts" @@ -31,7 +39,6 @@ import type * as Agg from "./aggregate.ts" import * as D from "./dtype.ts" import * as E from "./expr.ts" import * as F from "./frame.ts" -import * as K from "./kernels.ts" import * as Partial from "./partial.ts" import * as Q from "./query.ts" import { arrowType, conform } from "./schema.ts" @@ -39,21 +46,38 @@ import * as V from "./vector.ts" type Batches = Stream.Stream -/** A table without rows: its declared columns, or none. */ -const emptyTable = (node: Extract): F.Frame => - F.make( - (node.columns ?? []).map((c) => new F.Column(c.name, K.nullsOf(arrowType(c), 0), undefined, c.nullable)), - 0 - ) +/** Where a run reads the batches of its tables. */ +type Tables = Pick + +const reads = (node: Q.Node): boolean => node._tag === "Table" || Q.children(node).some(reads) + +/** + * What a node's columns depend on besides declared ones, so that they are + * unknown without reading its tables: tables without declared columns, and + * pivots without declared columns over tables (they take theirs from the + * values). + */ +const undeclared = (node: Q.Node): ReadonlyArray => [ + ...(node._tag === "Table" && node.columns === undefined ? [`table ${JSON.stringify(node.name)}`] : []), + ...(node._tag === "Pivot" && node.columns === undefined && reads(node.input) + ? [`the pivot on ${JSON.stringify(node.on)}`] + : []), + ...Q.children(node).flatMap(undeclared) +] -/** The plan over no rows (every input empty), run in place. */ -const runEmpty = (node: Q.Node): Effect.Effect => { +/** + * The result of a node that reads no batch of its tables, as one batch: the + * node run over its tables' declared columns without rows. No batch when its + * columns are unknown. + */ +const unread = (node: Q.Node): Batches => { + if (undeclared(node).length > 0) return Stream.empty const empty = (n: Q.Node): Q.Node => { switch (n._tag) { case "Table": - return Q.values(emptyTable(n)) + return Q.values(F.empty(n.columns!.map((c) => ({ name: c.name, type: arrowType(c), nullable: c.nullable })))) case "Values": - return Q.values(F.slice(n.frame as unknown as F.Frame, 0, 0)) + return n case "Join": return { ...n, left: empty(n.left), right: empty(n.right) } case "Concat": @@ -62,13 +86,22 @@ const runEmpty = (node: Q.Node): Effect.Effect => { return { ...n, input: empty((n as { readonly input: Q.Node }).input) } as Q.Node } } - return Effect.flatMap(Q.attempt(() => empty(node)), Q.execute) + return Stream.fromEffect(Effect.flatMap(Q.attempt(() => empty(node)), Q.execute)) } +/** Fails for a node without batches, whose columns are unknown. */ +const unknownColumns = (node: Q.Node): Effect.Effect => + Q.attempt(() => + V.fail( + `the columns of a result without rows are unknown, as no batch they depend on was read: ` + + `declare the columns of ${undeclared(node).join(" and ")}` + ) + ) + const collectFrames = (batches: Batches, node: Q.Node): Effect.Effect => Effect.flatMap(Stream.runCollect(batches), (frames) => frames.length === 0 - ? runEmpty(node) + ? unknownColumns(node) : Q.attempt(() => frames.length === 1 ? frames[0] : F.concat(frames))) /** Each output row depends only on its input row, so the node runs per batch. */ @@ -90,7 +123,7 @@ const perBatch = (node: Q.Node): boolean => { } /** Runs a node by collecting its inputs (an operation that needs all rows). */ -const collected = (node: Q.Node, catalog: Catalog.Service): Batches => +const collected = (node: Q.Node, catalog: Tables): Batches => Stream.unwrap( Effect.forEach(Q.children(node), (c) => collectFrames(batches(c, catalog), c)).pipe( Effect.flatMap((inputs) => Q.attempt(() => Q.step(node, inputs))), @@ -100,7 +133,7 @@ const collected = (node: Q.Node, catalog: Catalog.Service): Batches => const aggregate = ( node: Extract, - catalog: Catalog.Service + catalog: Tables ): Batches => { const keys = node._tag === "GroupBy" ? node.keys : [] const specs = node.aggregates.map(({ expression, name }) => { @@ -133,7 +166,7 @@ const aggregate = ( }) ).pipe( Effect.flatMap((state) => - state === undefined ? runEmpty(node) : Q.attempt(() => { + state === undefined ? unknownColumns(node.input) : Q.attempt(() => { const keyColumns = keys.map((k, i) => new F.Column(k, state.keys[i], undefined, nullable[i])) const columns = specs.map((s, a) => new F.Column(s.name, Partial.finish(s.fn, state.parts[a], state.types[a]), undefined) @@ -149,10 +182,10 @@ const aggregate = ( /** * The inputs' batches one after the other, converted to the inputs' common - * types. An input's first batch gives its types (its result without rows when - * it has none), so every input starts before the first batch is emitted. + * types. An input's first batch gives its types, so every input starts before + * the first batch is emitted. */ -const concat = (inputs: ReadonlyArray, catalog: Catalog.Service): Batches => +const concat = (inputs: ReadonlyArray, catalog: Tables): Batches => Stream.unwrap(Effect.gen(function*() { const peeled = yield* Effect.forEach( inputs, @@ -160,7 +193,7 @@ const concat = (inputs: ReadonlyArray, catalog: Catalog.Service): Batche Effect.flatMap(Stream.peel(batches(input, catalog), Sink.head()), ([first, rest]) => Option.isSome(first) ? Effect.succeed({ first: first.value, rest: Stream.concat(Stream.succeed(first.value), rest) }) - : Effect.map(runEmpty(input), (empty) => ({ first: empty, rest: Stream.empty }))) + : unknownColumns(input)) ) const types = yield* Q.attempt(() => F.concat(peeled.map(({ first }) => F.slice(first, 0, 0)))) return peeled.map(({ rest }) => Stream.mapEffect(rest, (batch) => Q.attempt(() => F.coerce(batch, types)))) @@ -189,7 +222,7 @@ const reader = (table: Plan.Table) => { } const slice = (input: Batches, offset: number, length: number | undefined): Batches => - length === 0 ? Stream.empty : Stream.suspend(() => { + Stream.suspend(() => { let skip = offset let left = length ?? Infinity return input.pipe( @@ -205,12 +238,13 @@ const slice = (input: Batches, offset: number, length: number | undefined): Batc }) /** - * The batches of a node's result. Batches without rows stay, so a result - * whose rows are all dropped still has the types of its columns. + * The batches of a node's result: at least one unless its columns are + * unknown. Batches without rows stay, so a result whose rows are all dropped + * still has the types of its columns. * * @internal */ -export const batches = (node: Q.Node, catalog: Catalog.Service): Batches => { +export const batches = (node: Q.Node, catalog: Tables): Batches => { if (perBatch(node)) { return Stream.mapEffect( batches((node as { readonly input: Q.Node }).input, catalog), @@ -225,10 +259,11 @@ export const batches = (node: Q.Node, catalog: Catalog.Service): Batches => { catalog.stream(node.name), (frame) => Q.attempt(() => read(frame as unknown as F.Frame)) ) - }) + }).pipe(Stream.orElseIfEmpty(() => unread(node))) case "Values": return Stream.succeed(node.frame as unknown as F.Frame) case "Slice": + if (node.length === 0) return unread(node) return node.offset >= 0 ? slice(batches(node.input, catalog), node.offset, node.length) : collected(node, catalog) case "Join": { if (node.how === "full") return collected(node, catalog) @@ -252,10 +287,13 @@ export const batches = (node: Q.Node, catalog: Catalog.Service): Batches => { } /** - * The optimised plan, with the tables it names more than once read once, - * as frames: each run reads each table once. + * The optimised plan, and its tables with those it names more than once + * read into memory: each run reads each table once. */ -const prepare = (plan: Plan.Node, catalog: Catalog.Service): Effect.Effect => { +const prepare = ( + plan: Plan.Node, + catalog: Catalog.Service +): Effect.Effect => { const repeated = [...Q.tables(plan)].flatMap(([name, n]) => n > 1 ? [name] : []) return Effect.forEach( repeated, @@ -263,25 +301,24 @@ const prepare = (plan: Plan.Node, catalog: Catalog.Service): Effect.Effect { const read = new Map(entries) - return Q.attempt(() => - Q.optimize(Q.mapTables(plan, (table) => { - const frames = read.get(table.name) as ReadonlyArray | undefined - if (frames === undefined) return table - if (frames.length === 0) return Q.values(emptyTable(table)) - const batches = frames.map(reader(table)) - return Q.values(batches.length === 1 ? batches[0] : F.concat(batches)) - })) - ) + const stream = (name: string) => { + const frames = read.get(name) + return frames === undefined ? catalog.stream(name) : Stream.fromIterable(frames) + } + return Q.attempt(() => [Q.optimize(plan as Q.Node), { stream }] as const) }) ) } /** The non-empty batches of a plan's result, after optimisation. @internal */ export const stream = (plan: Plan.Node, catalog: Catalog.Service): Batches => - Stream.unwrap(Effect.map(prepare(plan, catalog), (optimized) => batches(optimized, catalog))).pipe( + Stream.unwrap(Effect.map(prepare(plan, catalog), ([optimized, read]) => batches(optimized, read))).pipe( Stream.filter((batch) => batch.length > 0) ) /** A plan's result as one frame, computed batch by batch. @internal */ export const collect = (plan: Plan.Node, catalog: Catalog.Service): Effect.Effect => - Effect.flatMap(prepare(plan, catalog), (optimized) => collectFrames(batches(optimized, catalog), optimized)) + Effect.flatMap( + prepare(plan, catalog), + ([optimized, read]) => collectFrames(batches(optimized, read), optimized) + ) diff --git a/packages/effect/test/dataframe/Query.test.ts b/packages/effect/test/dataframe/Query.test.ts index a8bf3bbd505..64eec1743dd 100644 --- a/packages/effect/test/dataframe/Query.test.ts +++ b/packages/effect/test/dataframe/Query.test.ts @@ -824,6 +824,68 @@ describe("Query", () => { } })) + it.effect("results without rows are those in memory, or fail when their columns are undeclared and unread", () => + Effect.gen(function*() { + const x = (...values: Array) => DataFrame.fromColumns({ x: Int8Array.from(values) }) + const xy = (x: number, y: number) => DataFrame.fromColumns({ x: Int8Array.of(x), y: Int8Array.of(y) }) + const undeclared = (name: string) => Query.fromPlan({ _tag: "Table", name }) as typeof t + const [u, r, e] = [undeclared("u"), undeclared("r"), undeclared("e")] + const d = Query.table("d", Row, { types: { x: ArrowSchema.int8 } }) + const p = Query.table("p", Schema.Struct({ i: Schema.Number, k: Schema.String, v: Schema.Number })) + // Streamed, no batch of u is read and the declared d has none. + const memory = Catalog.layer({ + u: x(1), + r: xy(1, 2), + d: x(), + p: DataFrame.fromColumns({ i: [1], k: ["a"], v: [1] }) + }) + const streamed = Catalog.layer({ + u: Stream.die("read"), + r: Stream.make(xy(1, 2)), + d: Stream.empty, + p: Stream.die("read"), + e: Stream.empty + }) + const declare = (what: string) => + `the columns of a result without rows are unknown, as no batch they depend on was read: declare the columns of ${what}` + const sums = (query: typeof t) => Query.aggregate(query, (c) => ({ n: Expr.count(), s: Expr.sum(c.x) })) + const cases: ReadonlyArray, Array, string?]> = [ + [Query.concat([Query.head(u, 0), Query.from(x(9))]), [{ x: 9 }], declare(`table "u"`)], + [Query.head(u, 0), [], declare(`table "u"`)], + [Query.unique(Query.head(u, 0)), [], declare(`table "u"`)], + [sums(Query.head(u, 0)), [{ n: 0, s: 0 }], declare(`table "u"`)], + [Query.join(d, Query.head(r, 0), { on: "x" }), [], declare(`table "r"`)], + [ + Query.head(Query.pivot(p, { on: "k", index: ["i"], values: "v", aggregate: "sum" }), 0), + [], + declare(`the pivot on "k"`) + ], + [Query.concat([Query.join(d, r, { on: "x" }), Query.from(xy(9, 3))]), [{ x: 9, y: 3 }]], + [Query.concat([Query.head(d, 0), Query.from(x(9))]), [{ x: 9 }]], + [sums(d), [{ n: 0, s: 0 }]] + ] + for (const [query, expected, error] of cases) { + const df = yield* Effect.provide(query, memory) + assert.deepStrictEqual(DataFrame.toRows(df), expected) + if (error === undefined) { + const result = yield* Effect.provide(query, streamed) + assert.deepStrictEqual([DataFrame.toRows(result), result.schema], [expected, df.schema]) + } else { + assert.strictEqual((yield* Effect.flip(Effect.provide(query, streamed))).message, error) + } + } + // A table without declared columns or batches has no in-memory counterpart. + const unknown: ReadonlyArray> = [ + e, + Query.concat([e, Query.from(x(9))]), + Query.concat([e, e]), + Query.aggregate(e, () => ({ n: Expr.count() })) + ] + for (const query of unknown) { + assert.strictEqual((yield* Effect.flip(Effect.provide(query, streamed))).message, declare(`table "e"`)) + } + })) + it.effect("reads a table named twice once per run", () => Effect.gen(function*() { let loads = 0 From 0da84fd5c486fcc1b818f4db7ce29b270c7e239b Mon Sep 17 00:00:00 2001 From: Effect Bot Date: Sun, 15 Nov 2026 15:15:49 +0000 Subject: [PATCH 48/88] Read no batch for a top-k selection without rows A head without rows of a sort is optimised into a top-k of no rows, which collected its input; it now reads nothing, as a slice without rows does. --- packages/effect/src/dataframe/internal/stream.ts | 2 ++ packages/effect/test/dataframe/Query.test.ts | 4 +++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/packages/effect/src/dataframe/internal/stream.ts b/packages/effect/src/dataframe/internal/stream.ts index 07cd1593368..761732a2a08 100644 --- a/packages/effect/src/dataframe/internal/stream.ts +++ b/packages/effect/src/dataframe/internal/stream.ts @@ -265,6 +265,8 @@ export const batches = (node: Q.Node, catalog: Tables): Batches => { case "Slice": if (node.length === 0) return unread(node) return node.offset >= 0 ? slice(batches(node.input, catalog), node.offset, node.length) : collected(node, catalog) + case "TopK": + return node.limit === 0 ? unread(node) : collected(node, catalog) case "Join": { if (node.how === "full") return collected(node, catalog) const right = collectFrames(batches(node.right, catalog), node.right) diff --git a/packages/effect/test/dataframe/Query.test.ts b/packages/effect/test/dataframe/Query.test.ts index 64eec1743dd..b04a124f68d 100644 --- a/packages/effect/test/dataframe/Query.test.ts +++ b/packages/effect/test/dataframe/Query.test.ts @@ -583,7 +583,9 @@ describe("Query", () => { Effect.gen(function*() { const one = Stream.concat(Stream.succeed(DataFrame.fromColumns({ x: [1] })), Stream.die("pulled again")) assert.deepStrictEqual(yield* streamed(Query.head(t, 1), one), [{ x: 1 }]) - assert.deepStrictEqual(yield* streamed(Query.head(t, 0), Stream.die("pulled")), []) + for (const none of [Query.head(t, 0), Query.head(Query.sort(t, (c) => [c.x]), 0)]) { + assert.deepStrictEqual(yield* streamed(none, Stream.die("pulled")), []) + } })) it.effect("decimal sums don't depend on batch boundaries", () => From 5d3535ea74332c24adc3d9c47d798fa401625b25 Mon Sep 17 00:00:00 2001 From: Effect Bot Date: Sun, 15 Nov 2026 15:06:58 +0000 Subject: [PATCH 49/88] Keep the names the SQL compiler invents distinct after truncation The prefix of invented names grew with the plan's names, so a 63-byte table name made every helper name truncate to one identifier, and a 57-byte one dropped the suffixes of the tables' sources. Number the prefix instead: __, then __1_, __2_, ... A name starts with at most one of those, so invented names stay short and PostgreSQL keeps them whole. --- packages/effect/src/dataframe/internal/sql.ts | 17 ++++++++++------ packages/sql/pglite/test/DataFrameSql.test.ts | 20 ++++++++++++++++++- 2 files changed, 30 insertions(+), 7 deletions(-) diff --git a/packages/effect/src/dataframe/internal/sql.ts b/packages/effect/src/dataframe/internal/sql.ts index 7f6e9567cc7..1b1b08996c7 100644 --- a/packages/effect/src/dataframe/internal/sql.ts +++ b/packages/effect/src/dataframe/internal/sql.ts @@ -16,9 +16,9 @@ * and `decode` reads the rows back as columns of those types. * * The names the compiler invents (helper columns like the row key, the - * sources of tables) share one prefix, chosen per plan so that no name the - * plan supplies (its tables' and its columns') starts with it (see - * {@link inventor}). + * sources of tables) are short and share one prefix, chosen per plan so + * that no name the plan supplies (its tables' and its columns') starts with + * it (see {@link inventor}). * * @internal */ @@ -1168,8 +1168,13 @@ const source = (name: string, tables: ReadonlyArray): string => { * hide a column, and a source can't shadow a table it reads (a common table * expression shadows the table of its name in every `FROM` in its scope, * the other sources' included; qualifying the tables' names with their - * schema is no way out, as the schema isn't known). Names PostgreSQL - * truncates keep their start, so this holds for them too. + * schema is no way out, as the schema isn't known). + * + * The prefix is `__`, or else the first `__1_`, `__2_`, … that no name + * starts with: a name starts with at most one of those, so the prefix stays + * short, and so do the invented names. PostgreSQL keeps them whole (it + * keeps 63 bytes of a name), and a name it truncates keeps its start, so + * they stay distinct from each other and from the plan's names. */ const inventor = (frames: ReadonlyMap): (name: string) => string => { const names = [...frames].flatMap(([node, frame]) => [ @@ -1177,7 +1182,7 @@ const inventor = (frames: ReadonlyMap): (name: string) => st ...frame.columns.map((col) => col.name) ]) let prefix = "__" - while (names.some((name) => name.startsWith(prefix))) prefix += "_" + for (let k = 1; names.some((name) => name.startsWith(prefix)); k++) prefix = `__${k}_` return (name) => quote(prefix + name) } diff --git a/packages/sql/pglite/test/DataFrameSql.test.ts b/packages/sql/pglite/test/DataFrameSql.test.ts index 62475354d94..dac1648a3d5 100644 --- a/packages/sql/pglite/test/DataFrameSql.test.ts +++ b/packages/sql/pglite/test/DataFrameSql.test.ts @@ -17,6 +17,9 @@ const Sale = Schema.Struct({ at: Schema.DateTimeUtc }) const Region = Schema.Struct({ region: Schema.String, manager: Schema.String }) +/** Table names starting like the compiler's names: 63 bytes, the most PostgreSQL keeps, and 57. */ +const long = "_".repeat(62) + "a" +const underscores = "_".repeat(57) const sales = Array.from({ length: 40 }, (_, i) => ({ id: i, @@ -57,6 +60,11 @@ const Setup = Layer.effectDiscard( yield* sql`INSERT INTO "__table0" VALUES ('bad')` yield* sql`CREATE TABLE "__table1" (s TEXT)` yield* sql`INSERT INTO "__table1" VALUES ('bad')` + for (const name of [long, underscores]) { + yield* sql.unsafe(`DROP TABLE IF EXISTS "${name}"`) + yield* sql.unsafe(`CREATE TABLE "${name}" (s TEXT)`) + yield* sql.unsafe(`INSERT INTO "${name}" VALUES ('${name.length}')`) + } yield* sql`DROP TABLE IF EXISTS "a.b"` yield* sql`CREATE TABLE "a.b" (s TEXT)` yield* sql`INSERT INTO "a.b" VALUES ('good')` @@ -377,7 +385,14 @@ const cases: ReadonlyArray]> = [ "columns named like helpers", Query.from(DataFrame.fromColumns({ __row: [1, 2], __n: [3, 4], __source: ["a", "b"] })) ], - ["tail of a column named __n", Query.from(DataFrame.fromColumns({ __n: [1, 2] })).pipe(Query.tail(1))], + [ + "tail of columns named __n and long", + Query.from(DataFrame.fromColumns({ __n: [1, 2], [long]: [3, 4] })).pipe(Query.tail(1)) + ], + [ + "columns of 63 bytes, multi-byte characters included", + Query.from(DataFrame.fromColumns({ ["é".repeat(31) + "a"]: [1], ["é".repeat(31) + "b"]: [2] })) + ], [ "concat of a column named __source", Query.concat([ @@ -656,6 +671,9 @@ describe("effect/dataframe on PostgreSQL (PGlite)", { concurrent: false }, () => ]))) assert.strictEqual(error.message, `column "s" has values its schema rejects`) assert.deepStrictEqual(yield* rowsOnSql(s("a.b")), [{ s: "good" }]) + assert.deepStrictEqual(yield* rowsOnSql(s(long)), [{ s: "63" }]) + const two = Query.concat([s(underscores), s("second")]) + assert.deepStrictEqual(yield* rowsOnSql(two), [{ s: "57" }, { s: "good" }]) })) it.effect("timestamps with time zones are UTC, whatever the session's time zone", () => From c43ddd8c06e3ae9cdc1d5f6bd5e0d86cadb2422b Mon Sep 17 00:00:00 2001 From: Effect Bot Date: Sun, 15 Nov 2026 15:09:21 +0000 Subject: [PATCH 50/88] Reject columns PostgreSQL can't tell apart in SQL plans PostgreSQL keeps the whole characters within a name's first 63 bytes, so two columns of one relation that share them are one column to it, and the query failed with an ambiguous reference. Fail at compile time instead, naming both columns. --- packages/effect/src/dataframe/Sql.ts | 4 +- packages/effect/src/dataframe/internal/sql.ts | 48 +++++++++++++++++-- packages/sql/pglite/test/DataFrameSql.test.ts | 13 +++++ 3 files changed, 60 insertions(+), 5 deletions(-) diff --git a/packages/effect/src/dataframe/Sql.ts b/packages/effect/src/dataframe/Sql.ts index 4030eeda697..7d78282815a 100644 --- a/packages/effect/src/dataframe/Sql.ts +++ b/packages/effect/src/dataframe/Sql.ts @@ -97,7 +97,9 @@ const toError = (cause: unknown) => { /** * Compiles a plan to one PostgreSQL query. Fails for operations the SQL - * engine doesn't support. + * engine doesn't support, and for columns of one relation whose names + * share their first 63 bytes (in UTF-8): PostgreSQL keeps only those of a + * name, so it can't tell such columns apart. * * @stability experimental * @category compiling diff --git a/packages/effect/src/dataframe/internal/sql.ts b/packages/effect/src/dataframe/internal/sql.ts index 1b1b08996c7..4e64f4517c3 100644 --- a/packages/effect/src/dataframe/internal/sql.ts +++ b/packages/effect/src/dataframe/internal/sql.ts @@ -18,7 +18,9 @@ * The names the compiler invents (helper columns like the row key, the * sources of tables) are short and share one prefix, chosen per plan so * that no name the plan supplies (its tables' and its columns') starts with - * it (see {@link inventor}). + * it (see {@link inventor}). PostgreSQL keeps only the first 63 bytes of a + * name, so the columns of each relation must differ within them (see + * {@link distinguishable}). * * @internal */ @@ -1172,9 +1174,9 @@ const source = (name: string, tables: ReadonlyArray): string => { * * The prefix is `__`, or else the first `__1_`, `__2_`, … that no name * starts with: a name starts with at most one of those, so the prefix stays - * short, and so do the invented names. PostgreSQL keeps them whole (it - * keeps 63 bytes of a name), and a name it truncates keeps its start, so - * they stay distinct from each other and from the plan's names. + * short, and so do the invented names. PostgreSQL keeps them whole (see + * {@link identifier}), and a name it truncates keeps its start, so they stay + * distinct from each other and from the plan's names. */ const inventor = (frames: ReadonlyMap): (name: string) => string => { const names = [...frames].flatMap(([node, frame]) => [ @@ -1186,6 +1188,43 @@ const inventor = (frames: ReadonlyMap): (name: string) => st return (name) => quote(prefix + name) } +/** + * A name as PostgreSQL keeps it: the whole characters within its first 63 + * bytes in UTF-8 (`NAMEDATALEN - 1`). + */ +const identifier = (name: string): string => { + let bytes = 0 + let end = 0 + for (const char of name) { + const code = char.codePointAt(0)! + bytes += code < 0x80 ? 1 : code < 0x800 ? 2 : code < 0x10000 ? 3 : 4 + if (bytes > 63) break + end += char.length + } + return name.slice(0, end) +} + +/** + * Fails when two columns of a relation of the plan of `frames` are one + * identifier to PostgreSQL (see {@link identifier}): the in-memory engine + * tells them apart, the database can't. + */ +const distinguishable = (frames: ReadonlyMap): void => { + for (const frame of frames.values()) { + const seen = new Map() + for (const { name } of frame.columns) { + const id = identifier(name) + const other = seen.get(id) + if (other !== undefined) { + V.fail( + `columns ${JSON.stringify(other)} and ${JSON.stringify(name)} are one name to PostgreSQL (its first 63 bytes)` + ) + } + seen.set(id, name) + } + } +} + /** One SQL query whose first row holds the values of checks. */ interface Checked { readonly sql: string @@ -1217,6 +1256,7 @@ export interface Compiled { export const compile = (plan: Plan.Node): Compiled => { const frames = new Map() const result = typed(plan, frames) + distinguishable(frames) const hidden = inventor(frames) const tables = [...frames].flatMap(([node, frame]) => node._tag === "Table" ? [{ node, checks: rowChecks(node, frame) }] : [] diff --git a/packages/sql/pglite/test/DataFrameSql.test.ts b/packages/sql/pglite/test/DataFrameSql.test.ts index dac1648a3d5..a16987f3c59 100644 --- a/packages/sql/pglite/test/DataFrameSql.test.ts +++ b/packages/sql/pglite/test/DataFrameSql.test.ts @@ -676,6 +676,19 @@ describe("effect/dataframe on PostgreSQL (PGlite)", { concurrent: false }, () => assert.deepStrictEqual(yield* rowsOnSql(two), [{ s: "57" }, { s: "good" }]) })) + it.effect("columns PostgreSQL can't tell apart are rejected", () => + Effect.gen(function*() { + // PostgreSQL keeps the whole characters within an identifier's first 63 bytes. + const pairs = [["x".repeat(63) + "a", "x".repeat(63) + "b"], ["x".repeat(62) + "é", "x".repeat(62) + "ж"]] + for (const [a, b] of pairs) { + const added = Query.from(DataFrame.fromColumns({ [a]: [1] })).pipe( + Query.withColumns(() => ({ [b]: Expr.lit(2) })) + ) + const error = yield* Effect.flip(onSql(added)) + assert.include(error.message, `columns ${JSON.stringify(a)} and ${JSON.stringify(b)} are one name`) + } + })) + it.effect("timestamps with time zones are UTC, whatever the session's time zone", () => Effect.gen(function*() { const sql = yield* SqlClient From 059c5655af85c8f9e7a7e6798756337a7b15b797 Mon Sep 17 00:00:00 2001 From: Effect Bot Date: Sun, 15 Nov 2026 15:32:25 +0000 Subject: [PATCH 51/88] Read a table named more than once only when a run needs its batches Streamed runs collected every repeated table while preparing, before a zero-row slice or top-k could skip it, so head(concat([t, t]), 0) pulled, died on or hung over its source. Each repeated table is now an Effect.cached collection that starts on first demand and is shared by every use in the run. The error for unknown columns names each undeclared table once. --- .../effect/src/dataframe/internal/stream.ts | 35 +++++++++---------- packages/effect/test/dataframe/Query.test.ts | 23 +++++++++--- 2 files changed, 35 insertions(+), 23 deletions(-) diff --git a/packages/effect/src/dataframe/internal/stream.ts b/packages/effect/src/dataframe/internal/stream.ts index 761732a2a08..2985d6c8aad 100644 --- a/packages/effect/src/dataframe/internal/stream.ts +++ b/packages/effect/src/dataframe/internal/stream.ts @@ -94,7 +94,7 @@ const unknownColumns = (node: Q.Node): Effect.Effect => Q.attempt(() => V.fail( `the columns of a result without rows are unknown, as no batch they depend on was read: ` + - `declare the columns of ${undeclared(node).join(" and ")}` + `declare the columns of ${[...new Set(undeclared(node))].join(" and ")}` ) ) @@ -289,28 +289,27 @@ export const batches = (node: Q.Node, catalog: Tables): Batches => { } /** - * The optimised plan, and its tables with those it names more than once - * read into memory: each run reads each table once. + * The optimised plan, and its tables with those it names more than once read + * into memory on first use: each run reads each table at most once, and only + * if a node needs its batches. */ const prepare = ( plan: Plan.Node, catalog: Catalog.Service -): Effect.Effect => { - const repeated = [...Q.tables(plan)].flatMap(([name, n]) => n > 1 ? [name] : []) - return Effect.forEach( - repeated, - (name) => Effect.map(Stream.runCollect(catalog.stream(name)), (frames) => [name, frames] as const) - ).pipe( - Effect.flatMap((entries) => { - const read = new Map(entries) - const stream = (name: string) => { - const frames = read.get(name) - return frames === undefined ? catalog.stream(name) : Stream.fromIterable(frames) +): Effect.Effect => + Effect.gen(function*() { + const optimized = yield* Q.attempt(() => Q.optimize(plan as Q.Node)) + const read = new Map>() + for (const [name, n] of Q.tables(optimized as Plan.Node)) { + if (n > 1) { + read.set( + name, + Stream.unwrap(Effect.map(yield* Effect.cached(Stream.runCollect(catalog.stream(name))), Stream.fromIterable)) + ) } - return Q.attempt(() => [Q.optimize(plan as Q.Node), { stream }] as const) - }) - ) -} + } + return [optimized, { stream: (name: string) => read.get(name) ?? catalog.stream(name) }] as const + }) /** The non-empty batches of a plan's result, after optimisation. @internal */ export const stream = (plan: Plan.Node, catalog: Catalog.Service): Batches => diff --git a/packages/effect/test/dataframe/Query.test.ts b/packages/effect/test/dataframe/Query.test.ts index b04a124f68d..5be9bb1f3ad 100644 --- a/packages/effect/test/dataframe/Query.test.ts +++ b/packages/effect/test/dataframe/Query.test.ts @@ -854,6 +854,7 @@ describe("Query", () => { const cases: ReadonlyArray, Array, string?]> = [ [Query.concat([Query.head(u, 0), Query.from(x(9))]), [{ x: 9 }], declare(`table "u"`)], [Query.head(u, 0), [], declare(`table "u"`)], + [Query.head(Query.concat([u, u]), 0), [], declare(`table "u"`)], [Query.unique(Query.head(u, 0)), [], declare(`table "u"`)], [sums(Query.head(u, 0)), [{ n: 0, s: 0 }], declare(`table "u"`)], [Query.join(d, Query.head(r, 0), { on: "x" }), [], declare(`table "r"`)], @@ -888,19 +889,31 @@ describe("Query", () => { } })) - it.effect("reads a table named twice once per run", () => + it.effect("reads a table named twice once per run, only if its batches are needed", () => Effect.gen(function*() { let loads = 0 const load = Effect.sync(() => DataFrame.fromColumns({ x: [++loads] })) - assert.deepStrictEqual(yield* streamed(Query.concat([t, t]), Stream.fromEffect(load)), [{ x: 1 }, { x: 1 }]) - assert.strictEqual(loads, 1) + const cases: ReadonlyArray, Array, number]> = [ + [Query.concat([t, t]), [{ x: 1 }, { x: 1 }], 1], + [Query.join(t, t, { on: "x" }), [{ x: 1 }], 1], + [Query.concat([t, Query.head(t, 0)]), [{ x: 1 }], 1], + [Query.head(Query.concat([t, t]), 0), [], 0], + [Query.concat([Query.head(t, 0), Query.head(t, 0)]), [], 0], + [Query.head(Query.sort(Query.concat([t, t]), (c) => [c.x]), 0), [], 0] + ] + for (const [query, expected, reads] of cases) { + loads = 0 + assert.deepStrictEqual(yield* streamed(query, Stream.fromEffect(load)), expected) + assert.strictEqual(loads, reads) + } + loads = 0 const frames = yield* Query.concat([t, t]).pipe( Query.stream, Stream.runCollect, Effect.provide(Catalog.layer({ t: load })) ) - assert.deepStrictEqual(frames.flatMap((frame) => DataFrame.toArray(frame, "x")), [2, 2]) - assert.strictEqual(loads, 2) + assert.deepStrictEqual(frames.flatMap((frame) => DataFrame.toArray(frame, "x")), [1, 1]) + assert.strictEqual(loads, 1) })) it.effect("explain shows the plan without reading the stream", () => From 158f530359c64b8df13f5ffd393a4418364e60b2 Mon Sep 17 00:00:00 2001 From: Effect Bot Date: Sun, 15 Nov 2026 15:32:19 +0000 Subject: [PATCH 52/88] Share one SQL source among names PostgreSQL keeps as one Declarations of one table whose column names differ only past their first 63 bytes put that one column twice in the table's MATERIALIZED source, so PostgreSQL rejected the query as ambiguous although each declaration ran alone. The source now names its columns, and is keyed by its table's name, as PostgreSQL keeps them: every such declaration reads the one column or relation, once, each with its own checks. --- packages/effect/src/dataframe/Sql.ts | 4 ++- packages/effect/src/dataframe/internal/sql.ts | 31 ++++++++++++------- packages/sql/pglite/test/DataFrameSql.test.ts | 25 ++++++++++++--- 3 files changed, 43 insertions(+), 17 deletions(-) diff --git a/packages/effect/src/dataframe/Sql.ts b/packages/effect/src/dataframe/Sql.ts index 7d78282815a..42289c72eaf 100644 --- a/packages/effect/src/dataframe/Sql.ts +++ b/packages/effect/src/dataframe/Sql.ts @@ -99,7 +99,9 @@ const toError = (cause: unknown) => { * Compiles a plan to one PostgreSQL query. Fails for operations the SQL * engine doesn't support, and for columns of one relation whose names * share their first 63 bytes (in UTF-8): PostgreSQL keeps only those of a - * name, so it can't tell such columns apart. + * name, so it can't tell such columns apart. Declarations of tables, or of + * their columns, whose names share those bytes read one table or column, + * each with its own checks. * * @stability experimental * @category compiling diff --git a/packages/effect/src/dataframe/internal/sql.ts b/packages/effect/src/dataframe/internal/sql.ts index 4e64f4517c3..96e9a06b35b 100644 --- a/packages/effect/src/dataframe/internal/sql.ts +++ b/packages/effect/src/dataframe/internal/sql.ts @@ -19,8 +19,10 @@ * sources of tables) are short and share one prefix, chosen per plan so * that no name the plan supplies (its tables' and its columns') starts with * it (see {@link inventor}). PostgreSQL keeps only the first 63 bytes of a - * name, so the columns of each relation must differ within them (see - * {@link distinguishable}). + * name (see {@link identifier}), so the columns of each relation must + * differ within them (see {@link distinguishable}), and names alike within + * them are one table, or one column of it, to every declaration (see + * {@link source}). * * @internal */ @@ -45,7 +47,7 @@ interface Compiler { readonly hidden: (name: string) => string /** The quoted row key. */ readonly row: string - /** The quoted name of the source of each table name (see {@link source}). */ + /** The quoted name of the source of each table name, as PostgreSQL keeps it (see {@link source}). */ readonly sources: ReadonlyMap } @@ -720,7 +722,7 @@ const relation = (node: Plan.Node, c: Compiler): string => { `ROW_NUMBER() OVER () AS ${row}` ] .join(", ") - } FROM ${c.sources.get(node.name)}` + } FROM ${c.sources.get(identifier(node.name))}` case "Values": return values(node.frame as unknown as F.Frame, c) case "Filter": { @@ -1153,13 +1155,18 @@ const rowChecks = (table: Plan.Table, frame: F.Frame): Array => [ ] /** - * The source of the tables named `name` among `tables`: the columns they - * declare, read once. It is `MATERIALIZED`, which PostgreSQL computes once - * per statement, so the checks and the result read the same rows, whatever - * the relation: views that sample rows or call functions included. + * The source of the tables named `name` among `tables`, as PostgreSQL + * keeps their names (see {@link identifier}): the columns they declare, + * read once. It is `MATERIALIZED`, which PostgreSQL computes once per + * statement, so the checks and the result read the same rows, whatever the + * relation: views that sample rows or call functions included. Columns are + * named as PostgreSQL keeps them too, so declarations naming one column + * differently past its first 63 bytes all read it, each with its checks. */ const source = (name: string, tables: ReadonlyArray): string => { - const columns = [...new Set(tables.flatMap((t) => t.name === name ? t.columns!.map((c) => c.name) : []))] + const columns = [ + ...new Set(tables.flatMap((t) => identifier(t.name) === name ? t.columns!.map((c) => identifier(c.name)) : [])) + ] return `MATERIALIZED (SELECT ${columns.length === 0 ? "1" : list(columns)} FROM ${quote(name)})` } @@ -1261,7 +1268,9 @@ export const compile = (plan: Plan.Node): Compiled => { const tables = [...frames].flatMap(([node, frame]) => node._tag === "Table" ? [{ node, checks: rowChecks(node, frame) }] : [] ) - const sources = new Map([...new Set(tables.map((t) => t.node.name))].map((name, i) => [name, hidden(`table${i}`)])) + const sources = new Map( + [...new Set(tables.map((t) => identifier(t.node.name)))].map((name, i) => [name, hidden(`table${i}`)]) + ) const compiler: Compiler = { frames, hidden, row: hidden("row"), sources } const sql = relation(plan, compiler) const schema = S.make(result.columns.map((col) => S.field(col.name, col.type, { nullable: true }))) @@ -1285,7 +1294,7 @@ export const compile = (plan: Plan.Node): Compiled => { names.push(hidden(`check${names.length}`)) return `${check.sql} AS ${names[names.length - 1]}` }).join(", ") - } FROM ${sources.get(t.node.name)}) AS ${hidden(`checks${i}`)}` + } FROM ${sources.get(identifier(t.node.name))}) AS ${hidden(`checks${i}`)}` ) const rows = `(SELECT ${[...columns, compiler.row].join(", ")} FROM (${sql}) AS t) AS r` return { diff --git a/packages/sql/pglite/test/DataFrameSql.test.ts b/packages/sql/pglite/test/DataFrameSql.test.ts index a16987f3c59..f61bf86418a 100644 --- a/packages/sql/pglite/test/DataFrameSql.test.ts +++ b/packages/sql/pglite/test/DataFrameSql.test.ts @@ -65,6 +65,9 @@ const Setup = Layer.effectDiscard( yield* sql.unsafe(`CREATE TABLE "${name}" (s TEXT)`) yield* sql.unsafe(`INSERT INTO "${name}" VALUES ('${name.length}')`) } + yield* sql.unsafe(`DROP TABLE IF EXISTS shared`) + yield* sql.unsafe(`CREATE TABLE shared ("${"z".repeat(63)}" DOUBLE PRECISION)`) + yield* sql`INSERT INTO shared VALUES (1)` yield* sql`DROP TABLE IF EXISTS "a.b"` yield* sql`CREATE TABLE "a.b" (s TEXT)` yield* sql`INSERT INTO "a.b" VALUES ('good')` @@ -637,12 +640,17 @@ describe("effect/dataframe on PostgreSQL (PGlite)", { concurrent: false }, () => DECLARE n INTEGER := COALESCE(NULLIF(current_setting('df.tick', TRUE), ''), '0')::INTEGER + 1; BEGIN PERFORM set_config('df.tick', n::TEXT, FALSE); RETURN n; END $$` ) - yield* sql`CREATE VIEW ticking AS SELECT CASE WHEN tick() <= 1 THEN 'good' ELSE 'bad' END AS s` - const ticking = () => Query.table("ticking", Schema.Struct({ s: Schema.Literal("good") })) + // Named with 63 bytes: names longer than that are one with it to PostgreSQL. + const view = "ticking".padEnd(63, "_") + yield* sql.unsafe(`CREATE VIEW "${view}" AS SELECT CASE WHEN tick() <= 1 THEN 'good' ELSE 'bad' END AS s`) + const ticking = (name = view) => Query.table(name, Schema.Struct({ s: Schema.Literal("good") })) const tick = (query: Query.Query) => Effect.andThen(sql`SELECT set_config('df.tick', '0', FALSE)`, rowsOnSql(query)) assert.deepStrictEqual(yield* tick(ticking()), [{ s: "good" }]) - assert.deepStrictEqual(yield* tick(Query.concat([ticking(), ticking()])), [{ s: "good" }, { s: "good" }]) + assert.deepStrictEqual( + yield* tick(Query.concat([ticking(), ticking(), ticking(view + "s")])), + [{ s: "good" }, { s: "good" }, { s: "good" }] + ) yield* sql`CREATE SEQUENCE seq START 1` yield* sql`CREATE VIEW fluctuating AS SELECT CASE WHEN nextval('seq') <= 2 THEN 'good' ELSE 'bad' END AS s` const query = Query.table("fluctuating", Schema.Struct({ s: Schema.Literal("good") })) @@ -676,9 +684,16 @@ describe("effect/dataframe on PostgreSQL (PGlite)", { concurrent: false }, () => assert.deepStrictEqual(yield* rowsOnSql(two), [{ s: "57" }, { s: "good" }]) })) - it.effect("columns PostgreSQL can't tell apart are rejected", () => + it.effect("columns are one to PostgreSQL when their first 63 bytes are", () => Effect.gen(function*() { - // PostgreSQL keeps the whole characters within an identifier's first 63 bytes. + // Declarations of one table read its one column `"z".repeat(63)`, each with its checks. + const shared = (name: string, schema: Schema.Top & { readonly Type: number } = Schema.Number) => + Query.table("shared", Schema.Struct({ [name]: schema })).pipe(Query.select((c) => ({ x: c[name] }))) + const [za, zb] = ["z".repeat(63) + "a", "z".repeat(63) + "b"] + assert.deepStrictEqual(yield* rowsOnSql(Query.concat([shared(za), shared(zb)])), [{ x: 1 }, { x: 1 }]) + const error = yield* Effect.flip(onSql(Query.concat([shared(za), shared(zb, Schema.Literal(2))]))) + assert.strictEqual(error.message, `column ${JSON.stringify(zb)} has values its schema rejects`) + // Columns of one relation are rejected: PostgreSQL keeps the whole characters within an identifier's first 63 bytes. const pairs = [["x".repeat(63) + "a", "x".repeat(63) + "b"], ["x".repeat(62) + "é", "x".repeat(62) + "ж"]] for (const [a, b] of pairs) { const added = Query.from(DataFrame.fromColumns({ [a]: [1] })).pipe( From e2db400695746977abac33672a34bc6f3629c21f Mon Sep 17 00:00:00 2001 From: Effect Bot Date: Sun, 15 Nov 2026 19:14:28 +0000 Subject: [PATCH 53/88] Simplify the dataframe public modules --- packages/effect/src/dataframe/Catalog.ts | 35 ++--- packages/effect/src/dataframe/DataFrame.ts | 108 +++++++------- packages/effect/src/dataframe/Engine.ts | 14 +- packages/effect/src/dataframe/Expr.ts | 155 ++++++++++----------- packages/effect/src/dataframe/Plan.ts | 110 +++++---------- packages/effect/src/dataframe/Query.ts | 146 ++++++++++--------- packages/effect/src/dataframe/Sql.ts | 11 +- 7 files changed, 243 insertions(+), 336 deletions(-) diff --git a/packages/effect/src/dataframe/Catalog.ts b/packages/effect/src/dataframe/Catalog.ts index 51be5cde800..afc346e6021 100644 --- a/packages/effect/src/dataframe/Catalog.ts +++ b/packages/effect/src/dataframe/Catalog.ts @@ -104,27 +104,11 @@ export type Source = const unknownTable = (name: string) => Effect.fail(new DataFrameError({ message: `unknown table ${JSON.stringify(name)}` })) -/** - * A catalog without tables. - * - * @stability experimental - * @category constructors - * @since 4.1.5 - */ -export const empty: Catalog.Service = { - table: unknownTable, - stream: (name) => Stream.fromEffect(unknownTable(name)), - streaming: () => false -} - -const concatenated = (frames: ReadonlyArray>): Effect.Effect, DataFrameError> => { - if (frames.length === 1) return Effect.succeed(frames[0]) - try { - return Effect.succeed(F.concat(frames as unknown as ReadonlyArray) as unknown as DataFrame) - } catch (cause) { - return Effect.fail(new DataFrameError({ message: cause instanceof Error ? cause.message : String(cause), cause })) - } -} +const concatenated = (frames: ReadonlyArray>): Effect.Effect, DataFrameError> => + frames.length === 1 ? Effect.succeed(frames[0]) : Effect.try({ + try: () => F.concat(frames as unknown as ReadonlyArray) as unknown as DataFrame, + catch: (cause) => new DataFrameError({ message: cause instanceof Error ? cause.message : String(cause), cause }) + }) /** * A catalog of the given tables. @@ -162,6 +146,15 @@ export const make = (tables: { readonly [name: string]: Source }): Catalog.Servi } } +/** + * A catalog without tables. + * + * @stability experimental + * @category constructors + * @since 4.1.5 + */ +export const empty: Catalog.Service = make({}) + /** * A layer providing a catalog of the given tables. * diff --git a/packages/effect/src/dataframe/DataFrame.ts b/packages/effect/src/dataframe/DataFrame.ts index 44c45fe56a7..844ceb066cf 100644 --- a/packages/effect/src/dataframe/DataFrame.ts +++ b/packages/effect/src/dataframe/DataFrame.ts @@ -38,6 +38,7 @@ * @stability experimental * @since 4.1.5 */ +import * as ArrowData from "../arrow/ArrowData.ts" import * as ArrowIpc from "../arrow/ArrowIpc.ts" import * as ArrowSchema from "../arrow/ArrowSchema.ts" import * as ArrowTable from "../arrow/ArrowTable.ts" @@ -111,6 +112,12 @@ export type Normalize = { type Simplify = { readonly [K in keyof A]: A[K] } extends infer B ? B : never +type StructSchema = { readonly Type: object; readonly ast: any } + +type TypeOptions = { + readonly types?: { readonly [K in keyof S["Type"]]?: ArrowSchema.DataType } | undefined +} + type NormalizeValue = A extends undefined ? null : A extends BigDecimal.BigDecimal | DateTime.DateTime | Duration.Duration | Uint8Array ? A : A extends Array ? Array> @@ -119,9 +126,9 @@ type NormalizeValue = A extends undefined ? null : A /** - * Raised for invalid input that the types can't rule out: a row that doesn't - * match its schema, an Arrow table whose columns don't match a schema, or a - * column type the DataFrame doesn't support. + * Raised for invalid input that the types can't rule out (a row that doesn't + * match its schema, an Arrow table whose columns don't match a schema, a + * column type frames can't hold), and by queries that fail when they run. * * @stability experimental * @category errors @@ -145,6 +152,7 @@ const unsafe = (f: () => A): A => { const frame = (self: DataFrame): F.Frame => self as unknown as F.Frame const wrap = (f: F.Frame): DataFrame => f as unknown as DataFrame + /** * Returns `true` for data frames. * @@ -181,18 +189,16 @@ export const isDataFrame = (u: unknown): u is DataFrame => F.isFrame(u) * @category constructors * @since 4.1.5 */ -export const fromRows = ( +export const fromRows = ( schema: S, rows: Iterable, - options?: { readonly types?: { readonly [K in keyof S["Type"]]?: ArrowSchema.DataType } | undefined } + options?: TypeOptions ): DataFrame> => unsafe(() => { const arrow = Schema_.toArrowSchema(schema, options?.types as any) - const table = Result.getOrElse( + const table = Result.getOrThrowWith( ArrowTable.fromRows(arrow, Array.isArray(rows) ? rows : Array.from(rows) as any), - (error) => { - throw fromDefect(error) - } + fromDefect ) return wrap(F.fromArrowTable(table)) }) @@ -270,40 +276,23 @@ export const fromColumns = ( ): DataFrame }>> => unsafe(() => { const names = Object.keys(columns) - const fields: Array = [] - const data = names.map((name) => { + const types = names.map((name) => { const c = columns[name] - if (ArrayBuffer.isView(c)) { - const type = typedArrayTypes.find(([ctor]) => c instanceof ctor)![1] - fields.push(ArrowSchema.field(name, type, { nullable: false })) - const bytes = new Uint8Array(c.buffer, c.byteOffset, c.byteLength) - return { - type, - length: (c as Float64Array).length, - offset: 0, - nullCount: 0, - validity: undefined, - buffers: [bytes], - children: [], - dictionary: undefined - } - } - const type = inferType(c as ReadonlyArray) - fields.push(ArrowSchema.field(name, type)) - return undefined + return ArrayBuffer.isView(c) + ? typedArrayTypes.find(([ctor]) => c instanceof ctor)![1] + : inferType(c as ReadonlyArray) }) - const lengths = new Set(names.map((n) => (columns[n] as { length: number }).length)) + const lengths = new Set(names.map((n) => columns[n].length)) if (lengths.size > 1) V.fail(`columns have different lengths: ${[...lengths].join(", ")}`) - const length = names.length === 0 ? 0 : (columns[names[0]] as { length: number }).length + const length = names.length === 0 ? 0 : columns[names[0]].length const vectors = names.map((name, i) => { - const d = data[i] - if (d !== undefined) return V.fromArrowData(d) - const schema = ArrowSchema.make([fields[i]]) - const values = columns[name] as ReadonlyArray - const rows = values.map((v) => ({ [name]: v })) - const table = Result.getOrElse(ArrowTable.fromRows(schema, rows), (error) => { - throw fromDefect(error) - }) + const c = columns[name] + if (ArrayBuffer.isView(c)) { + const bytes = new Uint8Array(c.buffer, c.byteOffset, c.byteLength) + return V.fromArrowData(ArrowData.make({ type: types[i], length: c.length, buffers: [bytes] })) + } + const schema = ArrowSchema.make([ArrowSchema.field(name, types[i])]) + const table = Result.getOrThrowWith(ArrowTable.fromColumns(schema, [c]), fromDefect) return F.fromArrowTable(table).columns[0].vector }) return wrap(F.fromVectors(names, vectors, length)) @@ -315,8 +304,8 @@ export const fromColumns = ( * * **Details** * - * Throws a {@link DataFrameError} for column types the DataFrame doesn't - * support yet (decimals, nested types, intervals). + * Throws a {@link DataFrameError} for column types frames can't hold, such + * as Float16, intervals, maps, unions, or decimals beyond 18 digits. * * @stability experimental * @category constructors @@ -352,13 +341,12 @@ export const decodeArrowStream = ( Stream.mapError((error) => ArrowIpc.isArrowIpcError(error) ? new DataFrameError({ message: error.message, cause: error }) : error as E ), - Stream.mapEffect((batch) => { - try { - return Effect.succeed(fromRecordBatch(batch)) - } catch (cause) { - return Effect.fail(cause instanceof DataFrameError ? cause : fromDefect(cause)) - } - }) + Stream.mapEffect((batch) => + Effect.try({ + try: () => fromRecordBatch(batch), + catch: (cause) => cause instanceof DataFrameError ? cause : fromDefect(cause) + }) + ) ) /** @@ -369,9 +357,9 @@ export const decodeArrowStream = ( * @category constructors * @since 4.1.5 */ -export const empty = ( +export const empty = ( schema: S, - options?: { readonly types?: { readonly [K in keyof S["Type"]]?: ArrowSchema.DataType } | undefined } + options?: TypeOptions ): DataFrame> => fromRows(schema, [], options) /** @@ -406,26 +394,24 @@ export const empty = ( * @since 4.1.5 */ export const validate: { - ( + ( schema: S, - options?: { readonly types?: { readonly [K in keyof S["Type"]]?: ArrowSchema.DataType } | undefined } + options?: TypeOptions ): (self: DataFrame) => Result.Result>, DataFrameError> - ( + ( self: DataFrame, schema: S, - options?: { readonly types?: { readonly [K in keyof S["Type"]]?: ArrowSchema.DataType } | undefined } + options?: TypeOptions ): Result.Result>, DataFrameError> } = dual((args) => isDataFrame(args[0]), ( self: DataFrame, schema: { readonly ast: any }, options?: { readonly types?: { readonly [name: string]: ArrowSchema.DataType | undefined } | undefined } -) => { - try { - return Result.succeed(wrap(Schema_.conform(frame(self), Schema_.declare(schema, options?.types), schema))) - } catch (cause) { - return Result.fail(fromDefect(cause)) - } -}) +) => + Result.try({ + try: () => wrap(Schema_.conform(frame(self), Schema_.declare(schema, options?.types), schema)), + catch: fromDefect + })) // ----------------------------------------------------------------------------- // Conversions diff --git a/packages/effect/src/dataframe/Engine.ts b/packages/effect/src/dataframe/Engine.ts index 7d346b18c4a..ec5378e6a2f 100644 --- a/packages/effect/src/dataframe/Engine.ts +++ b/packages/effect/src/dataframe/Engine.ts @@ -16,7 +16,7 @@ * @since 4.1.5 */ import * as Context from "../Context.ts" -import * as Effect from "../Effect.ts" +import type * as Effect from "../Effect.ts" import * as Layer from "../Layer.ts" import * as Stream from "../Stream.ts" import { Catalog } from "./Catalog.ts" @@ -51,9 +51,6 @@ export declare namespace Engine { } } -const frame = (effect: Effect.Effect): Effect.Effect, DataFrameError> => - effect as any - /** * The in-memory engine: tables from the `Catalog`, the plan rewritten * (predicate, slice and projection pushdown, top-k), streamed tables batch by @@ -65,13 +62,8 @@ const frame = (effect: Effect.Effect): Effect.Effect Catalog.use((catalog) => frame(Q.collect(plan, catalog))), - stream: (plan) => - Stream.unwrap( - Catalog.use((catalog) => - Effect.succeed(Streaming.stream(plan, catalog) as unknown as Stream.Stream, DataFrameError>) - ) - ), + collect: (plan) => Catalog.use((catalog) => Q.collect(plan, catalog) as any), + stream: (plan) => Stream.unwrap(Catalog.useSync((catalog) => Streaming.stream(plan, catalog) as any)), explain: (plan) => Catalog.use((catalog) => Q.explain(plan, catalog)) } diff --git a/packages/effect/src/dataframe/Expr.ts b/packages/effect/src/dataframe/Expr.ts index cac85f9fa6c..d7854f0bdc9 100644 --- a/packages/effect/src/dataframe/Expr.ts +++ b/packages/effect/src/dataframe/Expr.ts @@ -288,6 +288,8 @@ export const lit = < type ArithResult = Expr | Nullish | Nullish> +const unary = (fn: Plan.FunctionName) => (self: Expr) => internal.call(fn, [self]) + const binary = (fn: Plan.FunctionName) => dual(2, (self: unknown, that: unknown) => internal.call(fn, [self, that])) as any @@ -376,6 +378,13 @@ export const divide: { ): Expr | Nullish> } = binary("divide") +type FloorDivision = { + ( + that: Input + ): | null>(self: Expr) => Expr | null> + | null>(self: Expr, that: Input): Expr | null> +} + /** * Division rounded towards negative infinity. Integer division by zero * gives `null`. @@ -384,12 +393,7 @@ export const divide: { * @category arithmetic * @since 4.1.5 */ -export const floorDivide: { - ( - that: Input - ): | null>(self: Expr) => Expr | null> - | null>(self: Expr, that: Input): Expr | null> -} = binary("floorDivide") +export const floorDivide: FloorDivision = binary("floorDivide") /** * The remainder of {@link floorDivide}: it has the sign of the divisor @@ -399,12 +403,7 @@ export const floorDivide: { * @category arithmetic * @since 4.1.5 */ -export const mod: { - ( - that: Input - ): | null>(self: Expr) => Expr | null> - | null>(self: Expr, that: Input): Expr | null> -} = binary("mod") +export const mod: FloorDivision = binary("mod") /** * `-self`. @@ -558,6 +557,16 @@ export const isIn: { // Logic // ----------------------------------------------------------------------------- +type Logical = { + ( + that: Input + ): (self: Expr) => Expr | Nullish> + ( + self: Expr, + that: Input + ): Expr | Nullish> +} + /** * Three-valued AND: `false` if either side is `false`, otherwise `null` if * either side is `null`. @@ -566,15 +575,7 @@ export const isIn: { * @category logic * @since 4.1.5 */ -export const and: { - ( - that: Input - ): (self: Expr) => Expr | Nullish> - ( - self: Expr, - that: Input - ): Expr | Nullish> -} = binary("and") +export const and: Logical = binary("and") /** * Three-valued OR: `true` if either side is `true`, otherwise `null` if @@ -584,15 +585,7 @@ export const and: { * @category logic * @since 4.1.5 */ -export const or: { - ( - that: Input - ): (self: Expr) => Expr | Nullish> - ( - self: Expr, - that: Input - ): Expr | Nullish> -} = binary("or") +export const or: Logical = binary("or") /** * Negation; `null` stays `null`. @@ -700,7 +693,7 @@ export const cast: { * 64-bit decimal storage) become null; extra digits round half to even. * * @stability experimental - * @category casts + * @category conversions * @since 4.1.5 */ export const castDecimal: { @@ -744,7 +737,12 @@ export const toLowerCase = (self: Expr): Expr => */ export const toUpperCase = (self: Expr): Expr => internal.call("toUpperCase", [self]) -const pattern = (fn: "includes" | "startsWith" | "endsWith") => +type Pattern = { + (text: string): (self: Expr) => Expr> + (self: Expr, text: string): Expr> +} + +const pattern = (fn: "includes" | "startsWith" | "endsWith"): Pattern => dual(2, (self: unknown, text: string) => internal.call(fn, [self], { pattern: text })) as any /** @@ -754,10 +752,7 @@ const pattern = (fn: "includes" | "startsWith" | "endsWith") => * @category strings * @since 4.1.5 */ -export const includes: { - (text: string): (self: Expr) => Expr> - (self: Expr, text: string): Expr> -} = pattern("includes") +export const includes: Pattern = pattern("includes") /** * Whether the string starts with `text`. @@ -766,10 +761,7 @@ export const includes: { * @category strings * @since 4.1.5 */ -export const startsWith: { - (text: string): (self: Expr) => Expr> - (self: Expr, text: string): Expr> -} = pattern("startsWith") +export const startsWith: Pattern = pattern("startsWith") /** * Whether the string ends with `text`. @@ -778,10 +770,7 @@ export const startsWith: { * @category strings * @since 4.1.5 */ -export const endsWith: { - (text: string): (self: Expr) => Expr> - (self: Expr, text: string): Expr> -} = pattern("endsWith") +export const endsWith: Pattern = pattern("endsWith") /** * Concatenates the arguments as strings; `null` if any argument is `null`. @@ -978,8 +967,6 @@ export const over: { }) ) as any -type CumSum = Expr | Nullish> - /** * The running sum: on each row, the sum of the values so far (in the * partition, with {@link over}). Nulls stay null and are skipped; NaN @@ -989,7 +976,9 @@ type CumSum = Expr | Nullish> * @category windows * @since 4.1.5 */ -export const cumSum = (self: Expr): CumSum => internal.call("cumSum", [self]) +export const cumSum = ( + self: Expr +): Expr | Nullish> => internal.call("cumSum", [self]) /** * The running minimum. Nulls stay null; NaN is skipped. @@ -1018,6 +1007,9 @@ export const cumMax = (self: Expr): Expr => internal.call("cumMax", [se */ export const cumCount = (self: Expr): Expr => internal.call("cumCount", [self]) +const lag = (fn: "shift" | "diff") => + dual((args) => isExpr(args[0]), (self: unknown, n = 1) => internal.call(fn, [self], { n })) as any + /** * The value `n` rows earlier (1 by default); a negative `n` looks ahead. * Null where there is no such row. @@ -1029,10 +1021,7 @@ export const cumCount = (self: Expr): Expr => internal.call("cu export const shift: { (n?: number): (self: Expr) => Expr (self: Expr, n?: number): Expr -} = dual( - (args) => isExpr(args[0]), - (self: unknown, n = 1) => internal.call("shift", [self], { n }) -) as any +} = lag("shift") /** * The difference to the value `n` rows earlier (1 by default). @@ -1044,10 +1033,12 @@ export const shift: { export const diff: { (n?: number): (self: Expr) => Expr (self: Expr, n?: number): Expr -} = dual( - (args) => isExpr(args[0]), - (self: unknown, n = 1) => internal.call("diff", [self], { n }) -) as any +} = lag("diff") + +type RankOptions = { + readonly method?: Plan.RankMethod | undefined + readonly descending?: boolean | undefined +} /** * The rank of each value, 1 for the smallest (largest with `descending`). @@ -1060,38 +1051,34 @@ export const diff: { * @since 4.1.5 */ export const rank: { - (options?: { - readonly method?: Plan.RankMethod | undefined - readonly descending?: boolean | undefined - }): (self: Expr) => Expr> - (self: Expr, options?: { - readonly method?: Plan.RankMethod | undefined - readonly descending?: boolean | undefined - }): Expr> + (options?: RankOptions): (self: Expr) => Expr> + (self: Expr, options?: RankOptions): Expr> } = dual( (args) => isExpr(args[0]), - (self: unknown, options?: { readonly method?: Plan.RankMethod; readonly descending?: boolean }) => + (self: unknown, options?: RankOptions) => internal.call("rank", [self], { method: options?.method ?? "average", descending: options?.descending ?? false }) ) as any +type RollingOptions = { readonly minPeriods?: number | undefined } + type Rolling = { - (window: number, options?: { readonly minPeriods?: number | undefined }): ( + (window: number, options?: RollingOptions): ( self: Expr ) => Expr | null> ( self: Expr, window: number, - options?: { readonly minPeriods?: number | undefined } + options?: RollingOptions ): Expr | null> } const rolling = (fn: "rollingSum" | "rollingMean" | "rollingMin" | "rollingMax"): Rolling => dual( (args) => isExpr(args[0]), - (self: unknown, window: number, options?: { readonly minPeriods?: number | undefined }) => + (self: unknown, window: number, options?: RollingOptions) => internal.call(fn, [self], { window, minPeriods: options?.minPeriods ?? window }) ) as any @@ -1113,13 +1100,13 @@ export const rollingSum: Rolling = rolling("rollingSum") * @since 4.1.5 */ export const rollingMean: { - (window: number, options?: { readonly minPeriods?: number | undefined }): ( + (window: number, options?: RollingOptions): ( self: Expr ) => Expr ( self: Expr, window: number, - options?: { readonly minPeriods?: number | undefined } + options?: RollingOptions ): Expr } = rolling("rollingMean") as any @@ -1146,7 +1133,6 @@ export const rollingMax: Rolling = rolling("rollingMax") // ----------------------------------------------------------------------------- type Field = (self: Expr) => Expr> -const datePart = (fn: Plan.FunctionName): Field => (self) => internal.call(fn, [self]) /** * The calendar year of a date or timestamp (UTC). @@ -1171,7 +1157,7 @@ const datePart = (fn: Plan.FunctionName): Field => (self) => internal.call(fn, [ * @category dates and times * @since 4.1.5 */ -export const year: Field = datePart("year") +export const year: Field = unary("year") /** * The month (1–12) of a date or timestamp. @@ -1180,7 +1166,7 @@ export const year: Field = datePart("year") * @category dates and times * @since 4.1.5 */ -export const month: Field = datePart("month") +export const month: Field = unary("month") /** * The day of the month (1–31) of a date or timestamp. @@ -1189,7 +1175,7 @@ export const month: Field = datePart("month") * @category dates and times * @since 4.1.5 */ -export const day: Field = datePart("day") +export const day: Field = unary("day") /** * The hour (0–23) of a timestamp. @@ -1198,7 +1184,7 @@ export const day: Field = datePart("day") * @category dates and times * @since 4.1.5 */ -export const hour: Field = datePart("hour") +export const hour: Field = unary("hour") /** * The minute (0–59) of a timestamp. @@ -1207,7 +1193,7 @@ export const hour: Field = datePart("hour") * @category dates and times * @since 4.1.5 */ -export const minute: Field = datePart("minute") +export const minute: Field = unary("minute") /** * The second (0–59) of a timestamp. @@ -1216,7 +1202,7 @@ export const minute: Field = datePart("minute") * @category dates and times * @since 4.1.5 */ -export const second: Field = datePart("second") +export const second: Field = unary("second") /** * The millisecond within the second (0–999) of a timestamp. @@ -1225,7 +1211,7 @@ export const second: Field = datePart("second") * @category dates and times * @since 4.1.5 */ -export const millisecond: Field = datePart("millisecond") +export const millisecond: Field = unary("millisecond") /** * The ISO weekday of a date or timestamp: 1 for Monday to 7 for Sunday. @@ -1234,7 +1220,7 @@ export const millisecond: Field = datePart("millisecond") * @category dates and times * @since 4.1.5 */ -export const weekday: Field = datePart("weekday") +export const weekday: Field = unary("weekday") /** * The day of the year (1–366) of a date or timestamp. @@ -1243,7 +1229,7 @@ export const weekday: Field = datePart("weekday") * @category dates and times * @since 4.1.5 */ -export const ordinalDay: Field = datePart("ordinalDay") +export const ordinalDay: Field = unary("ordinalDay") /** * Rounds dates and timestamps down to a multiple of `every`: `"30s"`, @@ -1260,7 +1246,6 @@ export const truncate: { } = dual(2, (self: unknown, every: string) => internal.call("truncate", [self], { every })) as any type Total = (self: Expr) => Expr> -const total = (fn: Plan.FunctionName): Total => (self) => internal.call(fn, [self]) /** * A duration in whole days, rounded toward zero. @@ -1269,7 +1254,7 @@ const total = (fn: Plan.FunctionName): Total => (self) => internal.call(fn, [sel * @category dates and times * @since 4.1.5 */ -export const totalDays: Total = total("totalDays") +export const totalDays: Total = unary("totalDays") /** * A duration in whole hours, rounded toward zero. @@ -1278,7 +1263,7 @@ export const totalDays: Total = total("totalDays") * @category dates and times * @since 4.1.5 */ -export const totalHours: Total = total("totalHours") +export const totalHours: Total = unary("totalHours") /** * A duration in whole minutes, rounded toward zero. @@ -1287,7 +1272,7 @@ export const totalHours: Total = total("totalHours") * @category dates and times * @since 4.1.5 */ -export const totalMinutes: Total = total("totalMinutes") +export const totalMinutes: Total = unary("totalMinutes") /** * A duration in whole seconds, rounded toward zero. @@ -1296,7 +1281,7 @@ export const totalMinutes: Total = total("totalMinutes") * @category dates and times * @since 4.1.5 */ -export const totalSeconds: Total = total("totalSeconds") +export const totalSeconds: Total = unary("totalSeconds") /** * A duration in whole milliseconds, rounded toward zero. @@ -1305,7 +1290,7 @@ export const totalSeconds: Total = total("totalSeconds") * @category dates and times * @since 4.1.5 */ -export const totalMilliseconds: Total = total("totalMilliseconds") +export const totalMilliseconds: Total = unary("totalMilliseconds") // ----------------------------------------------------------------------------- // Lists and structs diff --git a/packages/effect/src/dataframe/Plan.ts b/packages/effect/src/dataframe/Plan.ts index 6b1526062b2..cd64163b61e 100644 --- a/packages/effect/src/dataframe/Plan.ts +++ b/packages/effect/src/dataframe/Plan.ts @@ -59,7 +59,7 @@ import * as Result from "../Result.ts" import * as Schema from "../Schema.ts" import * as SchemaIssue from "../SchemaIssue.ts" import * as SchemaTransformation from "../SchemaTransformation.ts" -import type { DataFrame } from "./DataFrame.ts" +import { type DataFrame, isDataFrame } from "./DataFrame.ts" import * as F from "./internal/frame.ts" // ----------------------------------------------------------------------------- @@ -110,11 +110,7 @@ const valueFromJson: SchemaTransformation.Transformation { if (typeof value === "bigint") return Effect.succeed({ bigint: value.toString() }) if (typeof value === "number" && (!Number.isFinite(value) || Object.is(value, -0))) { - return Effect.succeed({ - float: Number.isNaN(value) ? "NaN" as const : value === Infinity ? "Infinity" as const : value === -Infinity - ? "-Infinity" as const - : "-0" as const - }) + return Effect.succeed({ float: (Object.is(value, -0) ? "-0" : String(value)) as typeof FloatName.Type }) } return Effect.succeed(value as string | number | boolean | null) } @@ -132,8 +128,6 @@ export const Value: Schema.declare = Schema.declare(isValue, { toCodecJson: () => Schema.link()(ValueJson, valueFromJson) }) -const isDataFrame = (u: unknown): u is DataFrame => F.isFrame(u) - const frameFromBytes: SchemaTransformation.Transformation, Uint8Array> = SchemaTransformation .transformEffect({ decode: (bytes, options) => { @@ -370,10 +364,10 @@ export type TimeUnit = typeof TimeUnit.Type export const TimeUnit = Schema.Literals(["second", "millisecond", "microsecond", "nanosecond"]) const LiteralTypeSchema: Schema.Codec = Schema.Union([ - Schema.Struct({ _tag: Schema.tag("Date") }), - Schema.Struct({ _tag: Schema.tag("Timestamp"), unit: TimeUnit, timezone: Schema.optional(Schema.String) }), - Schema.Struct({ _tag: Schema.tag("Duration"), unit: TimeUnit }), - Schema.Struct({ _tag: Schema.tag("Decimal"), precision: Schema.Int, scale: Schema.Int }) + Schema.TaggedStruct("Date", {}), + Schema.TaggedStruct("Timestamp", { unit: TimeUnit, timezone: Schema.optional(Schema.String) }), + Schema.TaggedStruct("Duration", { unit: TimeUnit }), + Schema.TaggedStruct("Decimal", { precision: Schema.Int, scale: Schema.Int }) ]) as any /** @@ -507,13 +501,10 @@ export interface Over { const ExpressionSchema = Schema.suspend((): Schema.Codec => ExpressionUnion) -const ColumnSchema = Schema.Struct({ _tag: Schema.tag("Column"), name: Schema.String }) -const LiteralSchema = Schema.Struct({ - _tag: Schema.tag("Literal"), - value: Value, - type: Schema.optional(LiteralTypeSchema) -}) +const ColumnSchema = Schema.TaggedStruct("Column", { name: Schema.String }) +const LiteralSchema = Schema.TaggedStruct("Literal", { value: Value, type: Schema.optional(LiteralTypeSchema) }) const PositiveInt = Schema.Int.check(Schema.isGreaterThanOrEqualTo(1)) +const NonNegativeInt = Schema.Int.check(Schema.isGreaterThanOrEqualTo(0)) const CallOptionsSchema = Schema.Struct({ decimals: Schema.optional(Schema.Int), to: Schema.optional(CastTarget), @@ -529,21 +520,18 @@ const CallOptionsSchema = Schema.Struct({ name: Schema.optional(Schema.String), names: Schema.optional(Schema.Array(Schema.String)), precision: Schema.optional(PositiveInt), - scale: Schema.optional(Schema.Int.check(Schema.isGreaterThanOrEqualTo(0))) + scale: Schema.optional(NonNegativeInt) }) -const CallSchema = Schema.Struct({ - _tag: Schema.tag("Call"), +const CallSchema = Schema.TaggedStruct("Call", { fn: FunctionName, args: Schema.Array(ExpressionSchema), options: Schema.optional(CallOptionsSchema) }) -const AggregateCallSchema = Schema.Struct({ - _tag: Schema.tag("AggregateCall"), +const AggregateCallSchema = Schema.TaggedStruct("AggregateCall", { fn: AggregateName, arg: Schema.optional(ExpressionSchema) }) -const OverSchema = Schema.Struct({ - _tag: Schema.tag("Over"), +const OverSchema = Schema.TaggedStruct("Over", { expression: ExpressionSchema, partitionBy: Schema.Array(ExpressionSchema) }) @@ -1059,56 +1047,25 @@ const TableColumnSchema: Schema.Codec = Schema.Struct({ const NodeSchema = Schema.suspend((): Schema.Codec => NodeUnion) const Columns = Schema.Array(Schema.String) -const NonNegativeInt = Schema.Int.check(Schema.isGreaterThanOrEqualTo(0)) const NodeUnion = Schema.Union([ - Schema.Struct({ _tag: Schema.tag("Values"), frame: Frame }), - Schema.Struct({ - _tag: Schema.tag("Table"), - name: Schema.String, - columns: Schema.optional(Schema.Array(TableColumnSchema)) - }), - Schema.Struct({ _tag: Schema.tag("Filter"), input: NodeSchema, predicate: ExpressionSchema }), - Schema.Struct({ _tag: Schema.tag("Select"), input: NodeSchema, columns: Schema.Array(NamedSchema) }), - Schema.Struct({ _tag: Schema.tag("WithColumns"), input: NodeSchema, columns: Schema.Array(NamedSchema) }), - Schema.Struct({ - _tag: Schema.tag("Sort"), - input: NodeSchema, - keys: Schema.Array(SortKeySchema), - nullsLast: Schema.Boolean - }), - Schema.Struct({ - _tag: Schema.tag("Slice"), - input: NodeSchema, - offset: Schema.Int, - length: Schema.optional(NonNegativeInt) - }), - Schema.Struct({ _tag: Schema.tag("Unique"), input: NodeSchema, columns: Schema.optional(Columns) }), - Schema.Struct({ _tag: Schema.tag("DropNulls"), input: NodeSchema, columns: Schema.optional(Columns) }), - Schema.Struct({ _tag: Schema.tag("Pick"), input: NodeSchema, columns: Columns }), - Schema.Struct({ _tag: Schema.tag("Omit"), input: NodeSchema, columns: Columns }), - Schema.Struct({ - _tag: Schema.tag("Rename"), - input: NodeSchema, - mapping: Schema.Record(Schema.String, Schema.String) - }), - Schema.Struct({ - _tag: Schema.tag("GroupBy"), - input: NodeSchema, - keys: Columns, - aggregates: Schema.Array(NamedSchema) - }), - Schema.Struct({ _tag: Schema.tag("Aggregate"), input: NodeSchema, aggregates: Schema.Array(NamedSchema) }), - Schema.Struct({ - _tag: Schema.tag("Join"), - left: NodeSchema, - right: NodeSchema, - on: Columns, - how: JoinHow - }), - Schema.Struct({ _tag: Schema.tag("Concat"), inputs: Schema.Array(NodeSchema) }), - Schema.Struct({ - _tag: Schema.tag("Pivot"), + Schema.TaggedStruct("Values", { frame: Frame }), + Schema.TaggedStruct("Table", { name: Schema.String, columns: Schema.optional(Schema.Array(TableColumnSchema)) }), + Schema.TaggedStruct("Filter", { input: NodeSchema, predicate: ExpressionSchema }), + Schema.TaggedStruct("Select", { input: NodeSchema, columns: Schema.Array(NamedSchema) }), + Schema.TaggedStruct("WithColumns", { input: NodeSchema, columns: Schema.Array(NamedSchema) }), + Schema.TaggedStruct("Sort", { input: NodeSchema, keys: Schema.Array(SortKeySchema), nullsLast: Schema.Boolean }), + Schema.TaggedStruct("Slice", { input: NodeSchema, offset: Schema.Int, length: Schema.optional(NonNegativeInt) }), + Schema.TaggedStruct("Unique", { input: NodeSchema, columns: Schema.optional(Columns) }), + Schema.TaggedStruct("DropNulls", { input: NodeSchema, columns: Schema.optional(Columns) }), + Schema.TaggedStruct("Pick", { input: NodeSchema, columns: Columns }), + Schema.TaggedStruct("Omit", { input: NodeSchema, columns: Columns }), + Schema.TaggedStruct("Rename", { input: NodeSchema, mapping: Schema.Record(Schema.String, Schema.String) }), + Schema.TaggedStruct("GroupBy", { input: NodeSchema, keys: Columns, aggregates: Schema.Array(NamedSchema) }), + Schema.TaggedStruct("Aggregate", { input: NodeSchema, aggregates: Schema.Array(NamedSchema) }), + Schema.TaggedStruct("Join", { left: NodeSchema, right: NodeSchema, on: Columns, how: JoinHow }), + Schema.TaggedStruct("Concat", { inputs: Schema.Array(NodeSchema) }), + Schema.TaggedStruct("Pivot", { input: NodeSchema, on: Schema.String, index: Columns, @@ -1116,10 +1073,9 @@ const NodeUnion = Schema.Union([ aggregate: Schema.Union([AggregateName, Schema.Literal("len")]), columns: Schema.optional(Columns) }), - Schema.Struct({ _tag: Schema.tag("Explode"), input: NodeSchema, column: Schema.String }), - Schema.Struct({ _tag: Schema.tag("Unnest"), input: NodeSchema, column: Schema.String }), - Schema.Struct({ - _tag: Schema.tag("Unpivot"), + Schema.TaggedStruct("Explode", { input: NodeSchema, column: Schema.String }), + Schema.TaggedStruct("Unnest", { input: NodeSchema, column: Schema.String }), + Schema.TaggedStruct("Unpivot", { input: NodeSchema, index: Columns, on: Columns, diff --git a/packages/effect/src/dataframe/Query.ts b/packages/effect/src/dataframe/Query.ts index 30bc31169b7..20c9754dd80 100644 --- a/packages/effect/src/dataframe/Query.ts +++ b/packages/effect/src/dataframe/Query.ts @@ -70,7 +70,7 @@ */ import type * as ArrowSchema from "../arrow/ArrowSchema.ts" import type * as BigDecimal from "../BigDecimal.ts" -import * as Effect from "../Effect.ts" +import type * as Effect from "../Effect.ts" import * as Effectable from "../Effectable.ts" import { dual } from "../Function.ts" import { NodeInspectSymbol } from "../Inspectable.ts" @@ -216,9 +216,6 @@ const columns: any = new Proxy({}, { const named = (exprs: { readonly [name: string]: unknown }): Array => Object.keys(exprs).map((name) => ({ name, expression: E.toNode(exprs[name]) })) -const isGroupBy = (u: unknown): u is GroupBy => - typeof u === "object" && u !== null && (u as { _tag?: unknown })._tag === "GroupBy" && "query" in u - // ----------------------------------------------------------------------------- // Constructors // ----------------------------------------------------------------------------- @@ -311,7 +308,7 @@ export const fromPlan = (plan: Plan.Node): Query<{ readonly [name: string]: unkn * @since 4.1.5 */ export const stream = (self: Query): Stream.Stream, DataFrameError> => - Stream.unwrap(Engine.use((engine) => Effect.succeed(engine.stream(self.plan) as any))) + Stream.unwrap(Engine.useSync((engine) => engine.stream(self.plan) as any)) /** * The plan that runs for a query, after tables are resolved and the plan is @@ -370,14 +367,10 @@ export const sort: { make({ _tag: "Sort", input: self.plan, - keys: keys(columns).map((k) => - E.isExpr(k) - ? { expression: k.node, descending: false } - : { - expression: E.toNode((k as Expr.SortOrder).expr), - descending: (k as Expr.SortOrder).descending - } - ), + keys: keys(columns).map((k) => { + const { expr, descending } = E.isExpr(k) ? { expr: k, descending: false } : k as Expr.SortOrder + return { expression: E.toNode(expr), descending } + }), nullsLast: options?.nullsLast ?? false })) @@ -445,6 +438,8 @@ export const unique: { make({ _tag: "Unique", input: self.plan, columns }) ) +type NonNull = Simplify & { [P in K]: NonNullable }> + /** * Removes the rows that are `null` in any of the given columns (all * columns by default). @@ -456,11 +451,11 @@ export const unique: { export const dropNulls: { ( columns?: ReadonlyArray - ): (self: Query) => Query & { [P in K]: NonNullable }>> + ): (self: Query) => Query> ( self: Query, columns?: ReadonlyArray - ): Query & { [P in K]: NonNullable }>> + ): Query> } = dual( (args) => isQuery(args[0]), (self: Query, columns?: ReadonlyArray) => make({ _tag: "DropNulls", input: self.plan, columns }) @@ -531,6 +526,10 @@ export const omit: { } = (...names: ReadonlyArray) => (self: Query) => make({ _tag: "Omit", input: self.plan, columns: names }) as any +type Renamed = Simplify< + Omit & { [K in keyof M & keyof R as M[K] extends string ? M[K] : never]: R[K] } +> + /** * Renames columns: `{ old: "new" }`. * @@ -541,13 +540,8 @@ export const omit: { export const rename: { ( mapping: M - ): (self: Query) => Query< - Simplify & { [K in keyof M & keyof R as M[K] extends string ? M[K] : never]: R[K] }> - > - ( - self: Query, - mapping: M - ): Query & { [K in keyof M & keyof R as M[K] extends string ? M[K] : never]: R[K] }>> + ): (self: Query) => Query> + (self: Query, mapping: M): Query> } = dual( 2, (self: Query, mapping: { readonly [name: string]: string | undefined }) => { @@ -557,6 +551,8 @@ export const rename: { } ) +type Computed = Simplify<{ [K in keyof O]: Expr.Type }> + /** * Replaces the columns with the given expressions; the record's keys name * the new columns. @@ -568,11 +564,11 @@ export const rename: { export const select: { }>( f: (columns: Columns) => O - ): (self: Query) => Query }>> + ): (self: Query) => Query> }>( self: Query, f: (columns: Columns) => O - ): Query }>> + ): Query> } = dual( 2, (self: Query, f: (columns: any) => { readonly [name: string]: unknown }) => @@ -589,11 +585,11 @@ export const select: { export const withColumns: { }>( f: (columns: Columns) => O - ): (self: Query) => Query & { [K in keyof O]: Expr.Type }>> + ): (self: Query) => Query & Computed>> }>( self: Query, f: (columns: Columns) => O - ): Query & { [K in keyof O]: Expr.Type }>> + ): Query & Computed>> } = dual( 2, (self: Query, f: (columns: any) => { readonly [name: string]: unknown }) => @@ -631,13 +627,13 @@ export const groupBy: { export const agg: { }>( f: (columns: Columns) => O - ): (self: GroupBy) => Query & { [P in keyof O]: Expr.Type }>> + ): (self: GroupBy) => Query & Computed>> }>( self: GroupBy, f: (columns: Columns) => O - ): Query & { [P in keyof O]: Expr.Type }>> + ): Query & Computed>> } = dual( - (args) => isGroupBy(args[0]), + 2, (self: GroupBy, f: (columns: any) => { readonly [name: string]: unknown }) => make({ _tag: "GroupBy", @@ -657,11 +653,11 @@ export const agg: { export const aggregate: { }>( f: (columns: Columns) => O - ): (self: Query) => Query }>> + ): (self: Query) => Query> }>( self: Query, f: (columns: Columns) => O - ): Query }>> + ): Query> } = dual( 2, (self: Query, f: (columns: any) => { readonly [name: string]: unknown }) => @@ -832,6 +828,25 @@ export const pivot: { columns: options.columns })) as any +interface UnpivotOptions { + readonly on: On + readonly index?: Index | undefined + readonly variable?: Variable | undefined + readonly value?: Value | undefined +} + +type Unpivoted< + R, + On extends ReadonlyArray, + Index extends ReadonlyArray, + Variable extends string, + Value extends string +> = Simplify< + & Pick + & { readonly [K in Variable]: On[number] } + & { readonly [K in Value]: R[On[number]] } +> + /** * Stacks the `on` columns into one `value` column (`"value"` by default), * one block of rows per column, with the column's name in `variable` @@ -849,52 +864,31 @@ export const unpivot: { const Index extends ReadonlyArray = readonly [], const Variable extends string = "variable", const Value extends string = "value" - >(options: { - readonly on: On - readonly index?: Index | undefined - readonly variable?: Variable | undefined - readonly value?: Value | undefined - }): ( - self: Query - ) => Query< - Simplify< - & Pick - & { readonly [K in Variable]: On[number] } - & { readonly [K in Value]: R[On[number]] } - > - > + >( + options: UnpivotOptions + ): (self: Query) => Query> < R, const On extends ReadonlyArray, const Index extends ReadonlyArray = readonly [], const Variable extends string = "variable", const Value extends string = "value" - >(self: Query, options: { - readonly on: On - readonly index?: Index | undefined - readonly variable?: Variable | undefined - readonly value?: Value | undefined - }): Query< - Simplify< - & Pick - & { readonly [K in Variable]: On[number] } - & { readonly [K in Value]: R[On[number]] } - > - > -} = dual(2, (self: Query, options: { - readonly on: ReadonlyArray - readonly index?: ReadonlyArray | undefined - readonly variable?: string | undefined - readonly value?: string | undefined -}) => - make({ - _tag: "Unpivot", - input: self.plan, - index: options.index ?? [], - on: options.on, - variable: options.variable ?? "variable", - value: options.value ?? "value" - })) as any + >( + self: Query, + options: UnpivotOptions + ): Query> +} = dual( + 2, + (self: Query, options: UnpivotOptions, ReadonlyArray, string, string>) => + make({ + _tag: "Unpivot", + input: self.plan, + index: options.index ?? [], + on: options.on, + variable: options.variable ?? "variable", + value: options.value ?? "value" + }) +) as any /** * One row per element of the list column `column`, the other columns @@ -908,11 +902,11 @@ export const unpivot: { export const explode: { >( column: K - ): (self: Query) => Query | null : R[P] }>> + ): (self: Query) => Query> >( self: Query, column: K - ): Query | null : R[P] }>> + ): Query> } = dual(2, (self: Query, column: string) => make({ _tag: "Explode", input: self.plan, column })) as any /** @@ -926,14 +920,15 @@ export const explode: { export const unnest: { >( column: K - ): (self: Query) => Query & Fields>> - >(self: Query, column: K): Query & Fields>> + ): (self: Query) => Query> + >(self: Query, column: K): Query> } = dual(2, (self: Query, column: string) => make({ _tag: "Unnest", input: self.plan, column })) as any type ListKeys = { [K in keyof R & string]: NonNullable extends ReadonlyArray ? K : never }[keyof R & string] type ElementOf = NonNullable extends ReadonlyArray ? E : never +type Exploded = Simplify<{ readonly [P in keyof R]: P extends K ? ElementOf | null : R[P] }> type StructKeys = { [K in keyof R & string]: NonNullable extends ReadonlyArray ? never : NonNullable extends object ? K @@ -941,3 +936,4 @@ type StructKeys = { }[keyof R & string] type Fields = null extends A ? { readonly [K in keyof NonNullable]: NonNullable[K] | null } : { readonly [K in keyof NonNullable]: NonNullable[K] } +type Unnested = Simplify & Fields> diff --git a/packages/effect/src/dataframe/Sql.ts b/packages/effect/src/dataframe/Sql.ts index 42289c72eaf..e39b787db15 100644 --- a/packages/effect/src/dataframe/Sql.ts +++ b/packages/effect/src/dataframe/Sql.ts @@ -39,10 +39,10 @@ * they differ from the in-memory engine's: integer overflow raises an error * instead of wrapping, division by zero follows the database, string order * follows its collation, and timestamps and durations keep PostgreSQL's - * microseconds (finer timestamps round down, finer durations toward zero). Pivots, unpivots, `explode` and - * `unnest` are not supported yet; tables need declared columns - * (`Query.table` with a schema), and {@link engine} checks that tables - * satisfy them, as the in-memory engine does. + * microseconds (finer timestamps round down, finer durations toward zero). + * Pivots, unpivots, `explode` and `unnest` are not supported yet; tables + * need declared columns (`Query.table` with a schema), and {@link engine} + * checks that tables satisfy them, as the in-memory engine does. * * @stability experimental * @since 4.1.5 @@ -53,8 +53,7 @@ import * as Layer from "../Layer.ts" import * as Result from "../Result.ts" import { SqlClient } from "../sql/SqlClient.ts" import * as Stream from "../Stream.ts" -import { DataFrameError } from "./DataFrame.ts" -import type { DataFrame } from "./DataFrame.ts" +import { type DataFrame, DataFrameError } from "./DataFrame.ts" import { Engine } from "./Engine.ts" import * as internal from "./internal/sql.ts" import * as V from "./internal/vector.ts" From 71c00539a11ce719937cb647fcb58f02665d70ac Mon Sep 17 00:00:00 2001 From: Effect Bot Date: Sun, 15 Nov 2026 19:14:28 +0000 Subject: [PATCH 54/88] Shorten the dataframe changeset --- .changeset/dataframe.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.changeset/dataframe.md b/.changeset/dataframe.md index b60b80e02cf..fcdfc935899 100644 --- a/.changeset/dataframe.md +++ b/.changeset/dataframe.md @@ -2,4 +2,4 @@ "effect": patch --- -Add `effect/dataframe`: typed, columnar data frames on top of `effect/arrow`. `DataFrame` is the immutable table value; `Query` describes filters, derived columns, sorts, groups, aggregates, joins, window functions (`Expr.over`, cumulative, shift, rank, rolling), pivots, unpivots and list explosion lazily, with column names and types checked at compile time, and runs as an `Effect` after an optimiser rewrites the plan (predicate, slice and projection pushdown, top-k). Columns can hold numbers, strings, booleans, dates, timestamps, durations, decimals, lists and structs. Queries are plain data (`Plan`, with Schema codecs to and from JSON), read named tables from a `Catalog` service (frames, effects or streams of batches, processed batch by batch), and run on an `Engine`: in memory, or on PostgreSQL through `Sql.layer` and `effect/sql`. Results follow Polars semantics and are checked against a Polars-generated gold corpus. +Add `effect/dataframe`: typed, columnar data frames on `effect/arrow` with lazy, type-checked queries that run in memory or on PostgreSQL. From 260c88a4c602f2730d3518b070529b759e283ec0 Mon Sep 17 00:00:00 2001 From: Effect Bot Date: Sun, 15 Nov 2026 19:16:37 +0000 Subject: [PATCH 55/88] Simplify dataframe value kernels, vectors and type helpers --- .../effect/src/dataframe/internal/decimal.ts | 53 ++-- .../effect/src/dataframe/internal/dtype.ts | 11 +- .../effect/src/dataframe/internal/kernels.ts | 237 ++++++------------ .../effect/src/dataframe/internal/nested.ts | 14 +- .../effect/src/dataframe/internal/temporal.ts | 36 +-- .../effect/src/dataframe/internal/vector.ts | 154 +++++------- 6 files changed, 202 insertions(+), 303 deletions(-) diff --git a/packages/effect/src/dataframe/internal/decimal.ts b/packages/effect/src/dataframe/internal/decimal.ts index 59180a663a9..70529950b9e 100644 --- a/packages/effect/src/dataframe/internal/decimal.ts +++ b/packages/effect/src/dataframe/internal/decimal.ts @@ -13,17 +13,18 @@ import * as BigDecimal from "../../BigDecimal.ts" import * as V from "./vector.ts" const powers: Array = [] -/** `10^k` as a bigint. @internal */ -export const pow10 = (k: number): bigint => (powers[k] ??= BigInt(10) ** BigInt(k)) +/** `10^k` as a bigint. */ +const pow10 = (k: number): bigint => (powers[k] ??= BigInt(10) ** BigInt(k)) +const I0 = BigInt(0) const MAX = BigInt("9223372036854775807") const MIN = -BigInt("9223372036854775808") /** @internal */ export const isDecimal = (t: S.DataType): t is S.Decimal => t._tag === "Decimal" -/** Precision of results, as in Polars. @internal */ -export const PRECISION = 38 +/** Precision of results, as in Polars. */ +const PRECISION = 38 /** @internal */ export const decimal = (scale: number): S.Decimal => S.decimal(PRECISION, scale) @@ -60,9 +61,9 @@ export const fit = (type: S.DataType): (x: bigint) => bigint => isDecimal(type) const divRound = (x: bigint, k: number): bigint => k === 0 ? x : BigDecimal.round(BigDecimal.make(x, k), { scale: 0, mode: "half-even" }).value -/** `unscaled / 10^scale` as a decimal string, all `scale` digits shown. @internal */ -export const format = (unscaled: bigint, scale: number): string => { - const negative = unscaled < BigInt(0) +/** `unscaled / 10^scale` as a decimal string, all `scale` digits shown. */ +const format = (unscaled: bigint, scale: number): string => { + const negative = unscaled < I0 const digits = (negative ? -unscaled : unscaled).toString().padStart(scale + 1, "0") const text = scale === 0 ? digits : `${digits.slice(0, digits.length - scale)}.${digits.slice(digits.length - scale)}` return negative ? `-${text}` : text @@ -70,8 +71,8 @@ export const format = (unscaled: bigint, scale: number): string => { const DECIMAL_SYNTAX = /^\s*([+-]?)(\d*)(?:\.(\d*))?\s*$/ -/** Parses a decimal string to unscaled units of `scale`; `undefined` when invalid. @internal */ -export const parse = (s: string, scale: number): bigint | undefined => { +/** Parses a decimal string to unscaled units of `scale`; `undefined` when invalid. */ +const parse = (s: string, scale: number): bigint | undefined => { const m = DECIMAL_SYNTAX.exec(s) if (m === null || (m[2] === "" && (m[3] ?? "") === "")) return undefined const fraction = m[3] ?? "" @@ -80,10 +81,6 @@ export const parse = (s: string, scale: number): bigint | undefined => { return m[1] === "-" ? -x : x } -const digitsOf = (x: bigint): number => (x < BigInt(0) ? -x : x).toString().length - -const fits = (x: bigint, t: S.Decimal): boolean => inRange(x) && digitsOf(x) <= t.precision - /** * Casts to and from decimals; `undefined` when neither side is a decimal. * Values that don't fit the target become null. @@ -97,8 +94,10 @@ export const cast = (v: V.Vector, target: S.DataType): V.Vector | undefined => { const valid = new Uint8Array(n) if (isDecimal(target)) { const out = new BigInt64Array(n) + // At most `precision` digits: |x| < 10^precision. + const limit = target.precision >= 1 ? pow10(target.precision) : I0 const put = (i: number, x: bigint | undefined) => { - if (x !== undefined && fits(x, target)) { + if (x !== undefined && inRange(x) && x < limit && -x < limit) { out[i] = x valid[i] = 1 } @@ -119,6 +118,7 @@ export const cast = (v: V.Vector, target: S.DataType): V.Vector | undefined => { break } case "i32": + case "bool": put(i, BigInt(v.values[i]) * pow10(target.scale)) break case "f64": { @@ -133,9 +133,6 @@ export const cast = (v: V.Vector, target: S.DataType): V.Vector | undefined => { case "str": put(i, parse(V.decodeString(v, i), target.scale)) break - case "bool": - put(i, BigInt(v.values[i]) * pow10(target.scale)) - break } } return V.i64(target, out, valid) @@ -144,25 +141,21 @@ export const cast = (v: V.Vector, target: S.DataType): V.Vector | undefined => { if (v.kind !== "i64" || !isDecimal(from)) return V.fail("expected decimal storage") const scale = pow10(from.scale) switch (target._tag) { - case "Float": { - const out = new Float64Array(n) - for (let i = 0; i < n; i++) out[i] = Number(v.values[i]) / 10 ** from.scale - return V.f64(target, out, v.valid) - } + case "Float": + return V.f64(target, toFloat(v.values, from.scale), v.valid) case "Int": { if (target.bitWidth === 64) { const out = new BigInt64Array(n) for (let i = 0; i < n; i++) { const x = v.values[i] / scale - if (V.isValid(v, i) && (target.signed || x >= BigInt(0))) { + if (V.isValid(v, i) && (target.signed || x >= I0)) { out[i] = x valid[i] = 1 } } return V.i64(target, out, valid) } - const lo = target.signed ? -(2 ** (target.bitWidth - 1)) : 0 - const hi = target.signed ? 2 ** (target.bitWidth - 1) - 1 : 2 ** target.bitWidth - 1 + const [lo, hi] = V.intRange(target) const out = new Float64Array(n) for (let i = 0; i < n; i++) { if (!V.isValid(v, i)) continue @@ -180,7 +173,7 @@ export const cast = (v: V.Vector, target: S.DataType): V.Vector | undefined => { Array.from({ length: n }, (_, i) => (V.isValid(v, i) ? format(v.values[i], from.scale) : null)) ) case "Bool": - return V.bool(Uint8Array.from(v.values, (x) => (x === BigInt(0) ? 0 : 1)), v.valid) + return V.bool(Uint8Array.from(v.values, (x) => (x === I0 ? 0 : 1)), v.valid) default: return V.fail(`cannot cast a decimal to ${target._tag}`) } @@ -230,5 +223,9 @@ export const round = (v: V.I64, type: S.Decimal, places: number): V.Vector => { } /** Unscaled values as floats. @internal */ -export const toFloat = (values: BigInt64Array, scale: number): Float64Array => - Float64Array.from(values, (x) => Number(x) / 10 ** scale) +export const toFloat = (values: BigInt64Array, scale: number): Float64Array => { + const out = new Float64Array(values.length) + const unit = 10 ** scale + for (let i = 0; i < values.length; i++) out[i] = Number(values[i]) / unit + return out +} diff --git a/packages/effect/src/dataframe/internal/dtype.ts b/packages/effect/src/dataframe/internal/dtype.ts index b423ebda5fd..b1924ff8769 100644 --- a/packages/effect/src/dataframe/internal/dtype.ts +++ b/packages/effect/src/dataframe/internal/dtype.ts @@ -110,20 +110,15 @@ export const supertype = ( if (ad !== undefined || bd !== undefined) { const [lit, col] = ad !== undefined ? [ad, b] : [bd, a] if (lit === "int") return { type: col, dynamic: undefined } - // float literal return { type: col._tag === "Float" ? col : S.float64, dynamic: undefined } } if (a._tag === "Int" && b._tag === "Int") return { type: intSupertype(a, b), dynamic: undefined } const f32 = (t: S.DataType) => t._tag === "Float" ? t.bitWidth === 32 : (t as S.Int).bitWidth <= 16 return { type: f32(a) && f32(b) ? S.float32 : S.float64, dynamic: undefined } } - if (T.isTemporal(a) || T.isTemporal(b)) { - if (sameType(a, b)) return { type: a, dynamic: undefined } - const t = T.supertype(a, b) - return t === undefined ? undefined : { type: t, dynamic: undefined } - } if (sameType(a, b)) return { type: a, dynamic: undefined } - return undefined + const t = T.supertype(a, b) + return t === undefined ? undefined : { type: t, dynamic: undefined } } /** Type of a true division. @internal */ @@ -139,7 +134,7 @@ export const sumType = (t: S.DataType): S.DataType => { } /** Type of `mean`, `median`, `std` and `var` over a column of type `t`. @internal */ -export const meanType = (t: S.DataType): S.DataType => t._tag === "Float" && t.bitWidth === 32 ? S.float32 : S.float64 +export const meanType = divisionType /** @internal */ export const describe = (t: S.DataType): string => { diff --git a/packages/effect/src/dataframe/internal/kernels.ts b/packages/effect/src/dataframe/internal/kernels.ts index c98e89b2abc..a5a74bbb171 100644 --- a/packages/effect/src/dataframe/internal/kernels.ts +++ b/packages/effect/src/dataframe/internal/kernels.ts @@ -1,7 +1,8 @@ /** - * Element-wise kernels. Each kernel has one loop per physical layout and - * operand shape (column/column, column/scalar), written out so that V8 sees - * monomorphic typed-array access in every loop. + * Element-wise kernels. Loops are specialised per physical layout so that + * typed-array access stays monomorphic. A scalar right operand is read with a + * `step` of 0 (`y[i * step]`), except in Int32 arithmetic and Int32 and + * Float64 comparisons, which keep a separate, faster scalar loop. * * @internal */ @@ -12,7 +13,6 @@ import * as T from "./temporal.ts" import * as V from "./vector.ts" const I0 = BigInt(0) -const I1 = BigInt(1) const I2 = BigInt(2) const I63 = BigInt(63) const I64 = BigInt(64) @@ -41,13 +41,15 @@ export const column = (vector: V.Vector): Value => ({ vector, scalar: false, dyn /** @internal */ export const scalar = (vector: V.Vector, dynamic: D.Dynamic = undefined): Value => ({ vector, scalar: true, dynamic }) +const result = (vector: V.Vector, scalar: boolean, dynamic?: D.Dynamic): Value => ({ vector, scalar, dynamic }) + /** Expands a scalar to `length` slots. @internal */ export const materialize = (value: Value, length: number): V.Vector => value.scalar ? V.broadcast(value.vector, length) : value.vector const scalarNull = (value: Value): boolean => value.scalar && !V.isValid(value.vector, 0) -/** Validity of a binary result. @internal */ +/** Validity of a binary result. */ const resultValid = (a: Value, b: Value, length: number): Uint8Array | undefined | "all-null" => { if (scalarNull(a) || scalarNull(b)) return "all-null" const av = a.scalar ? undefined : a.vector.valid @@ -88,20 +90,9 @@ export const nullsOf = (type: S.DataType, length: number): V.Vector => { // Casts // ----------------------------------------------------------------------------- -const intRange = (t: S.DataType): [number, number] => { - if (t._tag !== "Int") return [-Infinity, Infinity] - if (t.signed) { - const max = 2 ** (t.bitWidth - 1) - return [-max, max - 1] - } - return [0, 2 ** t.bitWidth - 1] -} - const FLOAT_SYNTAX = /^[+-]?(?:\d+\.?\d*(?:[eE][+-]?\d+)?|\.\d+(?:[eE][+-]?\d+)?|inf|infinity|nan)$/i const INT_SYNTAX = /^[+-]?\d+$/ -const formatNumber = (x: number): string => String(x) - /** * Converts `v` to `target`, as `Expr.cast` does. Values that don't fit * become null (Polars' non-strict casts). @@ -138,8 +129,6 @@ export const cast = (v: V.Vector, target: S.DataType): V.Vector => { else { switch (v.kind) { case "f64": - out[i] = formatNumber(v.values[i]) - break case "i32": out[i] = String(v.values[i]) break @@ -185,18 +174,19 @@ export const cast = (v: V.Vector, target: S.DataType): V.Vector => { return V.bool(out, V.compactValid(valid)) } // numeric targets: read every source slot as a number (or bigint for i64 targets) - const [lo, hi] = intRange(target) + const [lo, hi] = V.intRange(target) const isIntTarget = D.isInteger(target) || (target._tag !== "Float" && targetKind !== "f64") if (targetKind === "i64") { const out = new BigInt64Array(n) const unsigned = V.isUnsigned64(target) + const min = unsigned ? I0 : -(I2 ** I63) + const max = unsigned ? I2 ** I64 : I2 ** I63 for (let i = 0; i < n; i++) { if (valid !== undefined && valid[i] === 0) continue switch (v.kind) { case "i64": { const x = v.values[i] - if (unsigned && !V.isUnsigned64(v.type) && x < I0) invalidate(i) - else if (!unsigned && V.isUnsigned64(v.type) && x < I0) invalidate(i) + if (unsigned !== V.isUnsigned64(v.type) && x < I0) invalidate(i) else out[i] = x break } @@ -216,7 +206,7 @@ export const cast = (v: V.Vector, target: S.DataType): V.Vector => { if (!INT_SYNTAX.test(s)) invalidate(i) else { const x = BigInt(s) - if (x < (unsigned ? I0 : -(I2 ** I63)) || x >= (unsigned ? I2 ** I64 : I2 ** I63)) invalidate(i) + if (x < min || x >= max) invalidate(i) else out[i] = BigInt.asIntN(64, x) } break @@ -313,17 +303,13 @@ export const concat = (parts: ReadonlyArray): V.Vector => { } /** Casts a value ({@link cast}), keeping its shape; a cast literal has a fixed type. @internal */ -export const castValue = (value: Value, target: S.DataType): Value => ({ - vector: D.sameType(value.vector.type, target) ? value.vector : cast(value.vector, target), - scalar: value.scalar, - dynamic: undefined -}) +export const castValue = (value: Value, target: S.DataType): Value => result(cast(value.vector, target), value.scalar) /** Coerces a value ({@link coerce}), keeping its shape. */ const coerceValue = (value: Value, target: S.DataType): Value => D.sameType(value.vector.type, target) ? value - : { vector: coerce(value.vector, target), scalar: value.scalar, dynamic: undefined } + : result(coerce(value.vector, target), value.scalar) // ----------------------------------------------------------------------------- // Arithmetic @@ -338,7 +324,7 @@ export type ArithOp = "add" | "sub" | "mul" | "div" | "floorDiv" | "mod" * value (built from literals only), or the {@link Value.range} of a column. */ const literalRange = (v: Value): readonly [number, number] => { - if (!v.scalar) return v.range ?? intRange(S.int32) + if (!v.scalar) return v.range ?? V.intRange(S.int32) const x = (v.vector as V.I32).values[0] return V.isValid(v.vector, 0) ? [x, x] : [0, 0] } @@ -360,7 +346,7 @@ const rangeOf = (values: ReadonlyArray, dynamic: D.Dynamic): readonly [nu const operand = (v: Value, other: S.DataType): readonly [S.DataType, D.Dynamic] => { if (v.dynamic !== "int" || other._tag !== "Int" || v.vector.kind !== "i32") return [v.vector.type, v.dynamic] const [lo, hi] = literalRange(v) - const [min, max] = intRange(other) + const [min, max] = V.intRange(other) if (lo >= min && hi <= max) return [v.vector.type, v.dynamic] return [lo >= -128 && hi <= 127 ? S.int8 : lo >= -32768 && hi <= 32767 ? S.int16 : S.int32, undefined] } @@ -383,12 +369,12 @@ const supertype = (...values: ReadonlyArray) => { ]) } +const notDefined = (op: string, a: Value, b: Value): never => + V.fail(`${op} is not defined for ${D.describe(a.vector.type)} and ${D.describe(b.vector.type)}`) + const unify = (op: string, a: Value, b: Value) => { const st = supertype(a, b) - if (st === undefined || !(D.isNumeric(st.type) || st.type._tag === "Null")) { - return V.fail(`${op} is not defined for ${D.describe(a.vector.type)} and ${D.describe(b.vector.type)}`) - } - return st + return st === undefined || !(D.isNumeric(st.type) || st.type._tag === "Null") ? notDefined(op, a, b) : st } /** Relabels a value's vector (same storage). */ @@ -403,7 +389,6 @@ const relabel = (value: Value, type: S.DataType): Value => ({ ...value, vector: const temporalArith = (op: ArithOp, a: Value, b: Value, length: number): Value => { const at = a.vector.type const bt = b.vector.type - const unsupported = () => V.fail(`${op} is not defined for ${D.describe(at)} and ${D.describe(bt)}`) const asInt = (v: Value, type: S.DataType) => relabel(coerceValue(v, type), S.int64) const point = (t: S.DataType) => t._tag === "Date" || t._tag === "Timestamp" if (op === "sub" && point(at) && point(bt)) { @@ -412,7 +397,7 @@ const temporalArith = (op: ArithOp, a: Value, b: Value, length: number): Value = return relabel(arith("sub", asInt(a, ts), asInt(b, ts), length), S.duration("microsecond")) } const st = T.supertype(at, bt) as S.Timestamp | undefined - if (st === undefined) return unsupported() + if (st === undefined) return notDefined(op, a, b) return relabel(arith("sub", asInt(a, st), asInt(b, st), length), S.duration(st.unit)) } if ((op === "add" || op === "sub") && at._tag === "Timestamp" && bt._tag === "Duration") { @@ -429,7 +414,7 @@ const temporalArith = (op: ArithOp, a: Value, b: Value, length: number): Value = return relabel(arith("mul", relabel(a, S.int64), coerceValue(b, S.int64), length), at) } if (op === "mul" && bt._tag === "Duration" && D.isInteger(at)) return temporalArith(op, b, a, length) - return unsupported() + return notDefined(op, a, b) } /** @@ -438,9 +423,7 @@ const temporalArith = (op: ArithOp, a: Value, b: Value, length: number): Value = */ const decimalArith = (op: ArithOp, a: Value, b: Value, length: number): Value => { const st = supertype(a, b) - if (st === undefined) { - return V.fail(`${op} is not defined for ${D.describe(a.vector.type)} and ${D.describe(b.vector.type)}`) - } + if (st === undefined) return notDefined(op, a, b) if (!Dec.isDecimal(st.type) || (op !== "add" && op !== "sub" && op !== "mul")) { return arith(op, coerceValue(a, S.float64), coerceValue(b, S.float64), length) } @@ -452,7 +435,7 @@ const decimalArith = (op: ArithOp, a: Value, b: Value, length: number): Value => const valid = resultValid(left, right, n) const ls = (left.vector.type as S.Decimal).scale const rs = (right.vector.type as S.Decimal).scale - const result = Dec.arith( + const { type, values } = Dec.arith( op, (left.vector as V.I64).values, ls, @@ -460,8 +443,7 @@ const decimalArith = (op: ArithOp, a: Value, b: Value, length: number): Value => rs, right.scalar && !left.scalar ) - if (valid === "all-null") return { vector: nullsOf(result.type, n), scalar: a.scalar && b.scalar, dynamic: undefined } - return { vector: V.i64(result.type, result.values, valid), scalar: a.scalar && b.scalar, dynamic: undefined } + return result(valid === "all-null" ? nullsOf(type, n) : V.i64(type, values, valid), a.scalar && b.scalar) } /** @internal */ @@ -476,14 +458,14 @@ export const arith = (op: ArithOp, a: Value, b: Value, length: number): Value => if (type._tag === "Null") type = S.int32 const n = a.scalar && b.scalar ? 1 : length const valid = resultValid(a, b, n) - if (valid === "all-null") return { vector: nullsOf(type, n), scalar: a.scalar && b.scalar, dynamic } + if (valid === "all-null") return result(nullsOf(type, n), a.scalar && b.scalar, dynamic) // A scalar on the left: materialise it (rare) so kernels only see col op col / col op scalar. let left = coerceValue(a, type) const right = coerceValue(b, type) if (left.scalar && !right.scalar) left = column(materialize(left, n)) const lv = left.vector const rv = right.vector - const rs = right.scalar + const step = right.scalar ? 0 : 1 let out: V.Vector let nullOut: Uint8Array | undefined switch (lv.kind) { @@ -492,9 +474,9 @@ export const arith = (op: ArithOp, a: Value, b: Value, length: number): Value => const o = new Float64Array(n) const y = (rv as V.F64).values const unsigned32 = type._tag === "Int" - arithF64(op, x, y, rs ? 0 : 1, o, n, unsigned32) + arithF64(op, x, y, step, o, n, unsigned32) if (unsigned32 && (op === "floorDiv" || op === "mod")) { - nullOut = Uint8Array.from({ length: n }, (_, i) => y[rs ? 0 : i] === 0 ? 0 : 1) + nullOut = Uint8Array.from({ length: n }, (_, i) => y[i * step] === 0 ? 0 : 1) } V.roundFloat32InPlace(type, o) out = V.f64(type, o, valid) @@ -505,8 +487,8 @@ export const arith = (op: ArithOp, a: Value, b: Value, length: number): Value => const o = new Int32Array(n) if (op === "floorDiv" || op === "mod") { nullOut = new Uint8Array(n).fill(1) - intDivMod(op, x, (rv as V.I32).values, rs, o, n, nullOut) - } else if (rs) arithI32Scalar(op, x, (rv as V.I32).values[0], o, n) + intDivMod(op, x, (rv as V.I32).values, step, o, n, nullOut) + } else if (step === 0) arithI32Scalar(op, x, (rv as V.I32).values[0], o, n) else arithI32(op, x, (rv as V.I32).values, o, n) V.wrapInPlace(type, o) out = V.i32(type, o, valid) @@ -519,7 +501,7 @@ export const arith = (op: ArithOp, a: Value, b: Value, length: number): Value => const unsigned = V.isUnsigned64(type) for (let i = 0; i < n; i++) { let p = x[i] - let q = rs ? y[0] : y[i] + let q = y[i * step] if (unsigned) { p = BigInt.asUintN(64, p) q = BigInt.asUintN(64, q) @@ -542,8 +524,7 @@ export const arith = (op: ArithOp, a: Value, b: Value, length: number): Value => r = I0 break } - let d = p / q - if ((p % q !== I0) && ((p < I0) !== (q < I0))) d -= I1 + const d = T.floorDiv(p, q) r = op === "floorDiv" ? d : p - d * q } } @@ -556,7 +537,7 @@ export const arith = (op: ArithOp, a: Value, b: Value, length: number): Value => return V.fail(`${op} is not defined for ${D.describe(type)}`) } if (nullOut !== undefined) out = V.withValid(out, V.compactValid(V.andValid(out.valid, nullOut, n))) - return { vector: out, scalar: a.scalar && b.scalar, dynamic } + return result(out, a.scalar && b.scalar, dynamic) } /** Float modulo as Polars computes it: `a - b * floor(a / b)`. */ @@ -634,14 +615,14 @@ const intDivMod = ( op: "floorDiv" | "mod", x: Int32Array, y: Int32Array, - ys: boolean, + step: 0 | 1, o: Int32Array, n: number, valid: Uint8Array ) => { for (let i = 0; i < n; i++) { const a = x[i] - const b = ys ? y[0] : y[i] + const b = y[i * step] if (b === 0) { valid[i] = 0 continue @@ -661,7 +642,11 @@ export const unaryNumeric = (op: "neg" | "abs" | "sqrt" | "round", a: Value, dec return V.fail(`round needs a non-negative whole number of decimals, got ${decimals}`) } if (op === "round" && v.kind === "i64" && Dec.isDecimal(v.type)) { - return { vector: Dec.round(v, v.type, decimals), scalar: a.scalar, dynamic: a.dynamic } + return result(Dec.round(v, v.type, decimals), a.scalar, a.dynamic) + } + // Rounding leaves integers (and dates) as they are. + if (op === "round" && (v.kind === "i32" || v.kind === "i64" || (v.kind === "f64" && !D.isFloat(v.type)))) { + return result(v, a.scalar, a.dynamic) } switch (v.kind) { case "f64": { @@ -670,29 +655,26 @@ export const unaryNumeric = (op: "neg" | "abs" | "sqrt" | "round", a: Value, dec if (op === "neg") { for (let i = 0; i < n; i++) o[i] = -x[i] } else if (op === "abs") { for (let i = 0; i < n; i++) o[i] = Math.abs(x[i]) } else if (op === "sqrt") { for (let i = 0; i < n; i++) o[i] = Math.sqrt(x[i]) } - else if (D.isFloat(v.type)) roundHalfEven(x, o, decimals) - else o.set(x) - if (v.type._tag === "Int") V.wrapInPlace(v.type, o) + else roundHalfEven(x, o, decimals) + V.wrapInPlace(v.type, o) V.roundFloat32InPlace(v.type, o) - return { vector: V.f64(v.type, o, v.valid), scalar: a.scalar, dynamic: a.dynamic } + return result(V.f64(v.type, o, v.valid), a.scalar, a.dynamic) } case "i32": { const x = v.values const o = new Int32Array(n) if (op === "neg") { for (let i = 0; i < n; i++) o[i] = -x[i] | 0 } - else if (op === "abs") { for (let i = 0; i < n; i++) o[i] = Math.abs(x[i]) | 0 } - else o.set(x) + else for (let i = 0; i < n; i++) o[i] = Math.abs(x[i]) | 0 V.wrapInPlace(v.type, o) - return { vector: V.i32(v.type, o, v.valid), scalar: a.scalar, dynamic: a.dynamic } + return result(V.i32(v.type, o, v.valid), a.scalar, a.dynamic) } case "i64": { const x = v.values const o = new BigInt64Array(n) const fit = Dec.fit(v.type) if (op === "neg") { for (let i = 0; i < n; i++) o[i] = fit(-x[i]) } - else if (op === "abs") { for (let i = 0; i < n; i++) o[i] = fit(x[i] < I0 ? -x[i] : x[i]) } - else o.set(x) - return { vector: V.i64(v.type, o, v.valid), scalar: a.scalar, dynamic: a.dynamic } + else for (let i = 0; i < n; i++) o[i] = fit(x[i] < I0 ? -x[i] : x[i]) + return result(V.i64(v.type, o, v.valid), a.scalar, a.dynamic) } default: return V.fail(`${op} is not defined for ${D.describe(v.type)}`) @@ -771,31 +753,28 @@ export const compare = (op: CompareOp, a: Value, b: Value, length: number): Valu let right = b if (D.isNumeric(at) && D.isNumeric(bt) && !D.sameType(at, bt)) { const st = supertype(a, b)!.type - // Compare mixed widths as Float64 unless both are 64-bit integers. - const target = V.kindOf(st) === "i64" && V.kindOf(at) === "i64" && V.kindOf(bt) === "i64" ? st : S.float64 - left = coerceValue(a, V.kindOf(st) === "i32" ? st : target) - right = coerceValue(b, V.kindOf(st) === "i32" ? st : target) + // Mixed widths compare as Float64 unless the common type has Int32 storage + // or both sides are 64-bit integers. + const k = V.kindOf(st) + const target = k === "i32" || (k === "i64" && V.kindOf(at) === "i64" && V.kindOf(bt) === "i64") ? st : S.float64 + left = coerceValue(a, target) + right = coerceValue(b, target) } else if (at._tag === "Null" || bt._tag === "Null") { - return { vector: nullsOf(S.bool, n), scalar: a.scalar, dynamic: undefined } - } else if ((Dec.isDecimal(at) || Dec.isDecimal(bt)) && !D.sameType(at, bt)) { + return result(nullsOf(S.bool, n), a.scalar) + } else if (!D.sameType(at, bt) && !(D.isBinaryLike(at) && D.isBinaryLike(bt))) { + // Decimals and temporal values compare in their common type. const st = supertype(a, b) if (st === undefined) return V.fail(`cannot compare ${D.describe(at)} with ${D.describe(bt)}`) left = coerceValue(a, st.type) right = coerceValue(b, st.type) - } else if ((T.isTemporal(at) || T.isTemporal(bt)) && !D.sameType(at, bt)) { - const st = T.supertype(at, bt) - if (st === undefined) return V.fail(`cannot compare ${D.describe(at)} with ${D.describe(bt)}`) - left = coerceValue(a, st) - right = coerceValue(b, st) - } else if (!D.sameType(at, bt) && !(D.isBinaryLike(at) && D.isBinaryLike(bt))) { - return V.fail(`cannot compare ${D.describe(at)} with ${D.describe(bt)}`) } const valid = resultValid(left, right, n) - if (valid === "all-null") return { vector: nullsOf(S.bool, n), scalar: a.scalar, dynamic: undefined } + if (valid === "all-null") return result(nullsOf(S.bool, n), a.scalar) const o = new Uint8Array(n) const lv = left.vector const rv = right.vector const rs = right.scalar + const step = rs ? 0 : 1 switch (lv.kind) { case "f64": if (rs) compareF64Scalar(op, lv.values, (rv as V.F64).values[0], o, n) @@ -808,7 +787,7 @@ export const compare = (op: CompareOp, a: Value, b: Value, length: number): Valu case "bool": { const x = lv.values const y = (rv as V.Bool).values - for (let i = 0; i < n; i++) o[i] = fromOrdering(op, x[i] - y[rs ? 0 : i]) + for (let i = 0; i < n; i++) o[i] = fromOrdering(op, x[i] - y[i * step]) break } case "i64": { @@ -817,7 +796,7 @@ export const compare = (op: CompareOp, a: Value, b: Value, length: number): Valu const yv = (rv as V.I64).values const y = new Int32Array(yv.buffer, yv.byteOffset, yv.length * 2) for (let i = 0; i < n; i++) { - const j = rs ? 0 : i + const j = i * step const xh = x[2 * i + 1] const yh = y[2 * j + 1] let c: number @@ -854,7 +833,7 @@ export const compare = (op: CompareOp, a: Value, b: Value, length: number): Valu } } else { for (let i = 0; i < n; i++) { - const j = rs ? 0 : i + const j = i * step o[i] = fromOrdering(op, compareBytes(ad, ao[i], ao[i + 1], bd, bo[j], bo[j + 1])) } } @@ -863,7 +842,7 @@ export const compare = (op: CompareOp, a: Value, b: Value, length: number): Valu default: return V.fail(`cannot compare ${D.describe(at)}`) } - return { vector: V.bool(o, valid), scalar: a.scalar, dynamic: undefined } + return result(V.bool(o, valid), a.scalar) } // Float64 comparisons use a total order: NaN equals NaN and sorts above every @@ -1024,7 +1003,7 @@ export const logical = (op: "and" | "or", a: Value, b: Value, length: number): V if (x.valid === undefined && y.valid === undefined) { if (op === "and") { for (let i = 0; i < n; i++) o[i] = xv[i] & yv[i] } else for (let i = 0; i < n; i++) o[i] = xv[i] | yv[i] - return { vector: V.bool(o), scalar: a.scalar && b.scalar, dynamic: undefined } + return result(V.bool(o), a.scalar && b.scalar) } const valid = new Uint8Array(n) const xa = x.valid @@ -1045,7 +1024,7 @@ export const logical = (op: "and" | "or", a: Value, b: Value, length: number): V valid[i] = definitelyTrue || (p === 1 && q === 1) ? 1 : 0 } } - return { vector: V.bool(o, V.compactValid(valid)), scalar: a.scalar && b.scalar, dynamic: undefined } + return result(V.bool(o, V.compactValid(valid)), a.scalar && b.scalar) } /** @internal */ @@ -1056,7 +1035,7 @@ export const not = (a: Value): Value => { const o = new Uint8Array(v.length) const x = v.values for (let i = 0; i < o.length; i++) o[i] = x[i] ^ 1 - return { vector: V.bool(o, v.valid), scalar: a.scalar, dynamic: undefined } + return result(V.bool(o, v.valid), a.scalar) } /** @internal */ @@ -1067,7 +1046,7 @@ export const isNull = (a: Value, negate: boolean): Value => { else if (v.valid === undefined) o.fill(negate ? 1 : 0) else if (negate) o.set(v.valid) else for (let i = 0; i < o.length; i++) o[i] = v.valid[i] ^ 1 - return { vector: V.bool(o), scalar: a.scalar, dynamic: undefined } + return result(V.bool(o), a.scalar) } /** @internal */ @@ -1080,11 +1059,7 @@ export const isNaN_ = (a: Value): Value => { } else if (v.kind !== "f64" && v.kind !== "i32" && v.kind !== "i64" && v.kind !== "null") { return V.fail(`isNaN is not defined for ${D.describe(v.type)}`) } - return { - vector: V.bool(o, v.kind === "null" ? new Uint8Array(v.length) : v.valid), - scalar: a.scalar, - dynamic: undefined - } + return result(V.bool(o, v.kind === "null" ? new Uint8Array(v.length) : v.valid), a.scalar) } // ----------------------------------------------------------------------------- @@ -1119,57 +1094,13 @@ export const when = (cond: Value, then: Value, otherwise: Value, length: number) } } -/** Picks slot `i` from `branches[choice[i]]`; all branches share a type. @internal */ -export const select = (branches: ReadonlyArray, choice: Int32Array): V.Vector => { +/** Picks slot `i` from `branches[choice[i]]`; all branches share a type. */ +const select = (branches: ReadonlyArray, choice: Int32Array): V.Vector => { const n = choice.length - const first = branches.find((b) => b.kind !== "null") ?? branches[0] - if (first.kind === "null") return V.nulls(n) - if (first.kind === "list" || first.kind === "struct") { - // Nested values: gather slot i from the chosen branch. - const all = V.concat(branches.map((b) => (b.kind === "null" ? nullsOf(first.type, n) : b))) - const indices = new Int32Array(n) - for (let i = 0; i < n; i++) indices[i] = choice[i] * n + i - return V.take(all, indices) - } - const valid = new Uint8Array(n) - for (let i = 0; i < n; i++) { - const b = branches[choice[i]] - valid[i] = V.isValid(b, i) ? 1 : 0 - } - switch (first.kind) { - case "f64": - case "i32": - case "bool": - case "i64": { - const out = first.kind === "f64" - ? new Float64Array(n) - : first.kind === "i32" - ? new Int32Array(n) - : first.kind === "bool" - ? new Uint8Array(n) - : new BigInt64Array(n) - for (let i = 0; i < n; i++) { - const b = branches[choice[i]] - if (b.kind !== "null") (out as any)[i] = (b as V.F64).values[i] - } - const vector = first.kind === "f64" - ? V.f64(first.type, out as Float64Array) - : first.kind === "i32" - ? V.i32(first.type, out as Int32Array) - : first.kind === "bool" - ? V.bool(out as Uint8Array) - : V.i64(first.type, out as BigInt64Array) - return V.withValid(vector, V.compactValid(valid)) - } - case "str": { - const indices = new Int32Array(n) - // Concatenate branches, then gather from the concatenation. - const strs = branches.map((b) => b.kind === "str" ? b : nullsOf(first.type, n)) - const all = V.concat(strs) - for (let i = 0; i < n; i++) indices[i] = choice[i] * n + i - return V.withValid(V.take(all, indices), V.compactValid(valid)) - } - } + const indices = new Int32Array(n) + for (let i = 0; i < n; i++) indices[i] = choice[i] * n + i + const out = V.take(V.concat(branches), indices) + return V.withValid(out, V.compactValid(out.valid)) } /** First non-null value per slot. @internal */ @@ -1234,9 +1165,9 @@ export const isIn = (a: Value, candidates: ReadonlyArray): Value => { break } case "null": - return { vector: nullsOf(S.bool, n), scalar: a.scalar, dynamic: undefined } + return result(nullsOf(S.bool, n), a.scalar) } - return { vector: V.bool(o, v.valid), scalar: a.scalar, dynamic: undefined } + return result(V.bool(o, v.valid), a.scalar) } // ----------------------------------------------------------------------------- @@ -1251,9 +1182,7 @@ const asStr = (value: Value, name: string): V.Str => { /** Number of code points. @internal */ export const strLength = (a: Value): Value => { - if (a.vector.kind === "null") { - return { vector: nullsOf(S.uint32, a.vector.length), scalar: a.scalar, dynamic: undefined } - } + if (a.vector.kind === "null") return result(nullsOf(S.uint32, a.vector.length), a.scalar) const v = asStr(a, "length") const n = v.length const o = new Float64Array(n) @@ -1264,7 +1193,7 @@ export const strLength = (a: Value): Value => { for (let k = off[i]; k < off[i + 1]; k++) c += (d[k] & 0xc0) !== 0x80 ? 1 : 0 o[i] = c } - return { vector: V.f64(S.uint32, o, v.valid), scalar: a.scalar, dynamic: undefined } + return result(V.f64(S.uint32, o, v.valid), a.scalar) } /** @internal */ @@ -1273,14 +1202,12 @@ export const strMap = (a: Value, f: (s: string) => string): Value => { const v = asStr(a, "string function") const out: Array = new Array(v.length) for (let i = 0; i < v.length; i++) out[i] = V.isValid(v, i) ? f(V.decodeString(v, i)) : null - return { vector: V.stringsFrom(v.type, out), scalar: a.scalar, dynamic: undefined } + return result(V.stringsFrom(v.type, out), a.scalar) } /** @internal */ export const strMatch = (a: Value, mode: "includes" | "startsWith" | "endsWith", pattern: string): Value => { - if (a.vector.kind === "null") { - return { vector: nullsOf(S.bool, a.vector.length), scalar: a.scalar, dynamic: undefined } - } + if (a.vector.kind === "null") return result(nullsOf(S.bool, a.vector.length), a.scalar) const v = asStr(a, mode) const p = V.encodeString(pattern) const m = p.length @@ -1310,7 +1237,7 @@ export const strMatch = (a: Value, mode: "includes" | "startsWith" | "endsWith", } } } - return { vector: V.bool(o, v.valid), scalar: a.scalar, dynamic: undefined } + return result(V.bool(o, v.valid), a.scalar) } /** Concatenation; a null argument makes the slot null. @internal */ @@ -1343,5 +1270,5 @@ export const strConcat = (values: ReadonlyArray, length: number): Value = at += e - s } } - return { vector: V.str(S.utf8, offsets, data, valid), scalar: allScalar, dynamic: undefined } + return result(V.str(S.utf8, offsets, data, valid), allScalar) } diff --git a/packages/effect/src/dataframe/internal/nested.ts b/packages/effect/src/dataframe/internal/nested.ts index 498d4459cc9..d9be6608401 100644 --- a/packages/effect/src/dataframe/internal/nested.ts +++ b/packages/effect/src/dataframe/internal/nested.ts @@ -53,15 +53,13 @@ export const contains = (x: V.Vector, item: K.Value): V.Vector => { const v = asList(x, "listContains") const n = v.length const items = V.take(K.materialize(item, n), elementRows(v)) - const eq = K.materialize(K.compare("eq", K.column(v.child), K.column(items), v.child.length), v.child.length) + const eq = K.compare("eq", K.column(v.child), K.column(items), v.child.length).vector as V.Bool const out = new Uint8Array(n) - if (eq.kind === "bool") { - for (let i = 0; i < n; i++) { - for (let e = v.offsets[i]; e < v.offsets[i + 1]; e++) { - if (V.isValid(eq, e) && eq.values[e] === 1) { - out[i] = 1 - break - } + for (let i = 0; i < n; i++) { + for (let e = v.offsets[i]; e < v.offsets[i + 1]; e++) { + if (V.isValid(eq, e) && eq.values[e] === 1) { + out[i] = 1 + break } } } diff --git a/packages/effect/src/dataframe/internal/temporal.ts b/packages/effect/src/dataframe/internal/temporal.ts index 3c9c52cb377..40b9f14ea51 100644 --- a/packages/effect/src/dataframe/internal/temporal.ts +++ b/packages/effect/src/dataframe/internal/temporal.ts @@ -22,10 +22,9 @@ const perSecond: Record = { microsecond: 1_000_000, nanosecond: 1_000_000_000 } -const fineness: Record = { second: 0, millisecond: 1, microsecond: 2, nanosecond: 3 } /** The coarser of two units (the common unit of mixed operands, as in Polars). @internal */ -export const coarser = (a: S.TimeUnit, b: S.TimeUnit): S.TimeUnit => (fineness[a] <= fineness[b] ? a : b) +export const coarser = (a: S.TimeUnit, b: S.TimeUnit): S.TimeUnit => (perSecond[a] <= perSecond[b] ? a : b) const zoned = (t: S.Timestamp): boolean => t.timezone !== undefined && t.timezone !== "UTC" && t.timezone !== "+00:00" @@ -48,9 +47,13 @@ export const supertype = (a: S.DataType, b: S.DataType): S.DataType | undefined // Integer helpers // ----------------------------------------------------------------------------- -const floorDiv = (a: bigint, b: bigint): bigint => { +const I0 = BigInt(0) +const I1 = BigInt(1) + +/** Integer division rounding toward negative infinity. @internal */ +export const floorDiv = (a: bigint, b: bigint): bigint => { const q = a / b - return (a % b !== BigInt(0)) && ((a < BigInt(0)) !== (b < BigInt(0))) ? q - BigInt(1) : q + return a % b !== I0 && (a < I0) !== (b < I0) ? q - I1 : q } /** `v`'s integers times `factor` as `target`; null where the product leaves 64 bits. */ @@ -72,20 +75,23 @@ const multiply = (v: V.Vector, factor: bigint, target: S.DataType): V.Vector => * durations); counts that leave 64 bits become null. */ const rescale = (v: V.Vector, from: S.TimeUnit, to: S.TimeUnit, target: S.DataType, truncate = false): V.Vector => { - if (fineness[to] > fineness[from]) return multiply(v, BigInt(perSecond[to] / perSecond[from]), target) + if (perSecond[to] > perSecond[from]) return multiply(v, BigInt(perSecond[to] / perSecond[from]), target) const f = BigInt(perSecond[from] / perSecond[to]) return V.i64(target, BigInt64Array.from(asI64(v), (x) => (truncate ? x / f : floorDiv(x, f))), v.valid) } const dayUnits = (unit: S.TimeUnit): bigint => BigInt(86_400 * perSecond[unit]) +/** Units per day of a 64-bit date (milliseconds) or timestamp. */ +const unitsPerDay = (t: S.DataType): bigint => dayUnits(t._tag === "Timestamp" ? t.unit : "millisecond") + /** Days since the epoch of every row (dates and timestamps). */ const days = (v: V.Vector): Float64Array => { const out = new Float64Array(v.length) if (v.kind === "i32") { for (let i = 0; i < v.length; i++) out[i] = v.values[i] } else if (v.kind === "i64") { - const per = v.type._tag === "Timestamp" ? dayUnits(v.type.unit) : BigInt(86_400_000) // Date64: milliseconds + const per = unitsPerDay(v.type) for (let i = 0; i < v.length; i++) out[i] = Number(floorDiv(v.values[i], per)) } return out @@ -108,8 +114,8 @@ export const civil = (z: number): readonly [number, number, number] => { return [yoe + era * 400 + (m <= 2 ? 1 : 0), m, d] } -/** The day count of a date. @internal */ -export const daysFromCivil = (year: number, month: number, day: number): number => { +/** The day count of a date. */ +const daysFromCivil = (year: number, month: number, day: number): number => { const y = month <= 2 ? year - 1 : year const era = Math.floor(y / 400) const yoe = y - era * 400 @@ -158,7 +164,6 @@ export const field = (fn: Field, v: V.Vector): V.Vector => { checkTimestamp(v, fn) const n = v.length const out = new Int32Array(n) - const d = days(v) if (fn === "hour" || fn === "minute" || fn === "second" || fn === "millisecond") { if (v.type._tag !== "Timestamp" || v.kind !== "i64") return V.i32(fieldType[fn], out, v.valid) const unit = v.type.unit @@ -177,6 +182,7 @@ export const field = (fn: Field, v: V.Vector): V.Vector => { } return V.i32(fieldType[fn], out, v.valid) } + const d = days(v) for (let i = 0; i < n; i++) { const z = d[i] switch (fn) { @@ -209,7 +215,7 @@ const parseEvery = (every: string): { readonly n: number; readonly unit: string return { n, unit: match[2] } } -const fixed: Record = { s: 1, m: 60, h: 3600, d: 86_400, w: 7 * 86_400 } +const seconds: Record = { ns: 1e-9, us: 1e-6, ms: 1e-3, s: 1, m: 60, h: 3600, d: 86_400, w: 604_800 } /** * Every date or timestamp rounded down to a multiple of `every`: seconds, @@ -243,11 +249,9 @@ export const truncate = (v: V.Vector, every: string): V.Vector => { return V.i32(v.type, out, v.valid) } if (v.kind !== "i64" || v.type._tag !== "Timestamp") return V.fail(`truncate is not defined for ${v.type._tag}`) - const per = perSecond[v.type.unit] - const sub: Record = { ns: 1e-9, us: 1e-6, ms: 1e-3 } - const size = BigInt(Math.round((fixed[unit] ?? sub[unit]) * k * per)) - if (size <= BigInt(0)) return v - const offset = unit === "w" ? BigInt(3) * dayUnits(v.type.unit) : BigInt(0) + const size = BigInt(Math.round(seconds[unit] * k * perSecond[v.type.unit])) + if (size <= I0) return v + const offset = unit === "w" ? BigInt(3) * dayUnits(v.type.unit) : I0 const out = new BigInt64Array(n) for (let i = 0; i < n; i++) out[i] = floorDiv(v.values[i] + offset, size) * size - offset return V.i64(v.type, out, v.valid) @@ -256,7 +260,7 @@ export const truncate = (v: V.Vector, every: string): V.Vector => { /** Day counts as the input's type (dates stay dates). */ const fromDays = (v: V.Vector, d: Float64Array): V.Vector => { if (v.kind === "i32") return V.i32(v.type, Int32Array.from(d), v.valid) - const per = v.type._tag === "Timestamp" ? dayUnits(v.type.unit) : BigInt(86_400_000) + const per = unitsPerDay(v.type) return V.i64(v.type, BigInt64Array.from(d, (x) => BigInt(x) * per), v.valid) } diff --git a/packages/effect/src/dataframe/internal/vector.ts b/packages/effect/src/dataframe/internal/vector.ts index ccfc769f788..e3b9ec8fe09 100644 --- a/packages/effect/src/dataframe/internal/vector.ts +++ b/packages/effect/src/dataframe/internal/vector.ts @@ -23,7 +23,7 @@ */ import type * as ArrowData from "../../arrow/ArrowData.ts" import * as S from "../../arrow/ArrowSchema.ts" -import { typed } from "../../arrow/internal/buffers.ts" +import { typed, type TypedArrayConstructor } from "../../arrow/internal/buffers.ts" /** @internal */ export type Vector = F64 | I32 | I64 | Bool | Str | Null | List | Struct @@ -78,7 +78,6 @@ export interface Null { readonly valid: Uint8Array | undefined } -/** @internal */ /** @internal */ export interface List { readonly kind: "list" @@ -195,8 +194,8 @@ export const withType = (v: V, type: S.DataType): V => ({ ...v export const isValid = (v: Vector, i: number): boolean => v.kind === "null" ? false : v.valid === undefined || v.valid[i] === 1 -/** Number of null slots. @internal */ -export const nullCount = (v: Vector): number => { +/** Number of null slots. */ +const nullCount = (v: Vector): number => { if (v.kind === "null") return v.length const valid = v.valid if (valid === undefined) return 0 @@ -304,8 +303,6 @@ export const wrapInPlace = (type: S.DataType, values: Int32Array | Float64Array) else for (let i = 0; i < n; i++) values[i] = values[i] & 0xffff } else if (type.bitWidth === 32 && !type.signed) { for (let i = 0; i < n; i++) values[i] = values[i] >>> 0 - } else if (type.bitWidth === 32) { - for (let i = 0; i < n; i++) values[i] = values[i] | 0 } } @@ -316,9 +313,19 @@ export const roundFloat32InPlace = (type: S.DataType, values: Float64Array): voi } } -/** Wraps 64-bit values to an unsigned type's bit pattern view. @internal */ +/** Whether `type` is UInt64 (stored as its Int64 bit pattern). @internal */ export const isUnsigned64 = (type: S.DataType): boolean => type._tag === "Int" && type.bitWidth === 64 && !type.signed +/** The bounds of an integer type's values; unbounded for other types. @internal */ +export const intRange = (t: S.DataType): [number, number] => { + if (t._tag !== "Int") return [-Infinity, Infinity] + if (t.signed) { + const max = 2 ** (t.bitWidth - 1) + return [-max, max - 1] + } + return [0, 2 ** t.bitWidth - 1] +} + // ----------------------------------------------------------------------------- // Arrow interop // ----------------------------------------------------------------------------- @@ -350,8 +357,8 @@ const unpackBits = (bits: Uint8Array, offset: number, length: number): Uint8Arra return out } -/** Packs a byte-per-slot mask into an Arrow bitmap. @internal */ -export const packBits = (bytes: Uint8Array): Uint8Array => { +/** Packs a byte-per-slot mask into an Arrow bitmap. */ +const packBits = (bytes: Uint8Array): Uint8Array => { const n = bytes.length const out = new Uint8Array((n + 7) >> 3) const full = n >> 3 @@ -364,6 +371,9 @@ export const packBits = (bytes: Uint8Array): Uint8Array => { return out } +const I0 = BigInt(0) +const IM1 = BigInt(-1) + const encoder = new TextEncoder() const decoder = new TextDecoder() @@ -488,58 +498,36 @@ export const fromArrowData = (data: ArrowData.ArrowData): Vector => { ? undefined : unpackBits(data.validity, data.offset, n) const storage = storageType(type) + /** Slots of the first buffer as `Ctor` (`extra` more for offsets). */ + const view = (Ctor: TypedArrayConstructor, extra = 0) => + typed(Ctor, data.buffers[0]).subarray(data.offset, data.offset + n + extra) switch (type._tag) { case "Null": return nulls(n) case "Bool": return bool(unpackBits(data.buffers[0], data.offset, n), valid) - case "Int": { - const o = data.offset - const b = data.buffers[0] + case "Int": switch (type.bitWidth) { case 8: - return i32( - type, - type.signed - ? Int32Array.from(typed(Int8Array, b).subarray(o, o + n)) - : Int32Array.from(b.subarray(o, o + n)), - valid - ) + return i32(type, Int32Array.from(type.signed ? view(Int8Array) : view(Uint8Array)), valid) case 16: - return i32( - type, - Int32Array.from((type.signed ? typed(Int16Array, b) : typed(Uint16Array, b)).subarray(o, o + n)), - valid - ) + return i32(type, Int32Array.from(type.signed ? view(Int16Array) : view(Uint16Array)), valid) case 32: return type.signed - ? i32(type, typed(Int32Array, b).subarray(o, o + n), valid) - : f64(type, Float64Array.from(typed(Uint32Array, b).subarray(o, o + n)), valid) + ? i32(type, view(Int32Array), valid) + : f64(type, Float64Array.from(view(Uint32Array)), valid) default: - return i64(type, typed(BigInt64Array, b).subarray(o, o + n), valid) + return i64(type, view(BigInt64Array), valid) } - } case "Float": - if (type.bitWidth === 64) { - return f64(type, typed(Float64Array, data.buffers[0]).subarray(data.offset, data.offset + n), valid) - } - if (type.bitWidth === 32) { - return f64( - type, - Float64Array.from(typed(Float32Array, data.buffers[0]).subarray(data.offset, data.offset + n)), - valid - ) - } + if (type.bitWidth === 64) return f64(type, view(Float64Array), valid) + if (type.bitWidth === 32) return f64(type, Float64Array.from(view(Float32Array)), valid) return fail("Float16 columns are not supported") case "Date": case "Time": case "Timestamp": - case "Duration": { - const k = kindOf(type) - return k === "i32" - ? i32(type, typed(Int32Array, data.buffers[0]).subarray(data.offset, data.offset + n), valid) - : i64(type, typed(BigInt64Array, data.buffers[0]).subarray(data.offset, data.offset + n), valid) - } + case "Duration": + return kindOf(type) === "i32" ? i32(type, view(Int32Array), valid) : i64(type, view(BigInt64Array), valid) case "Utf8": case "Binary": case "LargeUtf8": @@ -551,20 +539,16 @@ export const fromArrowData = (data: ArrowData.ArrowData): Vector => { case "BinaryView": return fromViews(storage, data, valid) case "Decimal": { - const o = data.offset - const b = data.buffers[0] - if (type.bitWidth === 32) { - return i64(type, BigInt64Array.from(typed(Int32Array, b).subarray(o, o + n), BigInt), valid) - } - if (type.bitWidth === 64) return i64(type, typed(BigInt64Array, b).subarray(o, o + n), valid) + if (type.bitWidth === 32) return i64(type, BigInt64Array.from(view(Int32Array), BigInt), valid) + if (type.bitWidth === 64) return i64(type, view(BigInt64Array), valid) // 128 or 256 bits: keep the low word when the rest only extends its sign. const words = type.bitWidth / 64 - const all = typed(BigInt64Array, b) + const all = typed(BigInt64Array, data.buffers[0]) const out = new BigInt64Array(n) for (let i = 0; i < n; i++) { - const base = (o + i) * words + const base = (data.offset + i) * words const lo = all[base] - const sign = lo < BigInt(0) ? BigInt(-1) : BigInt(0) + const sign = lo < I0 ? IM1 : I0 for (let w = 1; w < words; w++) { if (all[base + w] !== sign && (valid === undefined || valid[i] === 1)) { return fail(`decimal values beyond 18 digits are not supported (column of ${type.bitWidth}-bit decimals)`) @@ -576,9 +560,7 @@ export const fromArrowData = (data: ArrowData.ArrowData): Vector => { } case "List": case "LargeList": { - const raw = type._tag === "LargeList" - ? Int32Array.from(typed(BigInt64Array, data.buffers[0]).subarray(data.offset, data.offset + n + 1), Number) - : typed(Int32Array, data.buffers[0]).subarray(data.offset, data.offset + n + 1) + const raw = type._tag === "LargeList" ? Int32Array.from(view(BigInt64Array, 1), Number) : view(Int32Array, 1) const child = fromArrowData(data.children[0]) const first = raw[0] if (first === 0 && raw[n] === child.length) return list(storage, raw, child, valid) @@ -631,7 +613,7 @@ export const toArrowData = (v: Vector): ArrowData.ArrowData => { const out = new BigInt64Array(n * words) for (let i = 0; i < n; i++) { out[i * words] = v.values[i] - const sign = v.values[i] < BigInt(0) ? BigInt(-1) : BigInt(0) + const sign = v.values[i] < I0 ? IM1 : I0 for (let w = 1; w < words; w++) out[i * words + w] = sign } return { ...base, buffers: [bytes(out)] } @@ -662,8 +644,8 @@ export const toArrowData = (v: Vector): ArrowData.ArrowData => { // ----------------------------------------------------------------------------- /** - * Gathers `indices` from `v`. A negative index produces a null slot. `valid` - * (optional) marks additional null output slots. + * Gathers `indices` from `v`. A negative index produces a null slot. + * `extraValid` (optional) marks additional null output slots. * * @internal */ @@ -815,7 +797,6 @@ export const slice = (v: Vector, start: number, end: number): Vector => { /** Concatenates vectors of the same kind and type. @internal */ export const concat = (parts: ReadonlyArray): Vector => { if (parts.length === 1) return parts[0] - const first = parts[0] const n = parts.reduce((s, p) => s + p.length, 0) let valid: Uint8Array | undefined if (parts.some((p) => p.kind === "null" || p.valid !== undefined)) { @@ -829,32 +810,9 @@ export const concat = (parts: ReadonlyArray): Vector => { at += p.length } } - const kind = parts.find((p) => p.kind !== "null")?.kind ?? "null" - if (kind === "null") return nulls(n) - const type = parts.find((p) => p.kind !== "null")!.type - if (kind === "list") { - const offsets = new Int32Array(n + 1) - const children: Array = [] - let row = 0 - let at = 0 - for (const p of parts) { - if (p.kind === "list") { - for (let i = 1; i <= p.length; i++) offsets[row + i] = at + p.offsets[i] - at += p.offsets[p.length] - children.push(p.child) - } else for (let i = 1; i <= p.length; i++) offsets[row + i] = at - row += p.length - } - return list(type, offsets, concat(children.length === 0 ? [nulls(0)] : children), valid) - } - if (kind === "struct") { - const fields = (parts.find((p) => p.kind === "struct") as Struct).children.length - const children = Array.from( - { length: fields }, - (_, f) => concat(parts.map((p) => (p.kind === "struct" ? p.children[f] : nulls(p.length)))) - ) - return struct(type, children, n, valid) - } + const head = parts.find((p) => p.kind !== "null") + if (head === undefined) return nulls(n) + const type = head.type const fill = (out: T): T => { let at = 0 for (const p of parts) { @@ -863,7 +821,7 @@ export const concat = (parts: ReadonlyArray): Vector => { } return out } - switch (kind) { + switch (head.kind) { case "f64": return f64(type, fill(new Float64Array(n)), valid) case "i32": @@ -872,6 +830,27 @@ export const concat = (parts: ReadonlyArray): Vector => { return i64(type, fill(new BigInt64Array(n)), valid) case "bool": return bool(fill(new Uint8Array(n)), valid) + case "list": { + const offsets = new Int32Array(n + 1) + const children: Array = [] + let row = 0 + let at = 0 + for (const p of parts) { + if (p.kind === "list") { + for (let i = 1; i <= p.length; i++) offsets[row + i] = at + p.offsets[i] + at += p.offsets[p.length] + children.push(p.child) + } else for (let i = 1; i <= p.length; i++) offsets[row + i] = at + row += p.length + } + return list(type, offsets, concat(children.length === 0 ? [nulls(0)] : children), valid) + } + case "struct": { + const children = head.children.map((_, f) => + concat(parts.map((p) => (p.kind === "struct" ? p.children[f] : nulls(p.length)))) + ) + return struct(type, children, n, valid) + } case "str": { const total = parts.reduce((s, p) => s + (p.kind === "str" ? p.offsets[p.length] - p.offsets[0] : 0), 0) const data = new Uint8Array(total) @@ -892,7 +871,6 @@ export const concat = (parts: ReadonlyArray): Vector => { return str(type, offsets, data, valid) } } - return first } /** A vector of `length` copies of one value. @internal */ From 91e37a0dee8c59eac6e902fc4cd1fd30c5dbe9e1 Mon Sep 17 00:00:00 2001 From: Effect Bot Date: Sun, 15 Nov 2026 19:15:36 +0000 Subject: [PATCH 56/88] Share row grouping, ordering and key helpers across the dataframe kernels --- .../src/dataframe/internal/aggregate.ts | 54 ++++---- .../effect/src/dataframe/internal/frame.ts | 77 ++++-------- .../effect/src/dataframe/internal/hash.ts | 119 +++++++++++------- .../effect/src/dataframe/internal/partial.ts | 74 +++++------ .../effect/src/dataframe/internal/sort.ts | 79 +++++++----- .../effect/src/dataframe/internal/stream.ts | 42 +++---- .../effect/src/dataframe/internal/window.ts | 81 ++---------- 7 files changed, 223 insertions(+), 303 deletions(-) diff --git a/packages/effect/src/dataframe/internal/aggregate.ts b/packages/effect/src/dataframe/internal/aggregate.ts index ce55952e26e..6eef9883f96 100644 --- a/packages/effect/src/dataframe/internal/aggregate.ts +++ b/packages/effect/src/dataframe/internal/aggregate.ts @@ -13,7 +13,7 @@ import * as S from "../../arrow/ArrowSchema.ts" import * as Dec from "./decimal.ts" import * as D from "./dtype.ts" import * as Hash from "./hash.ts" -import { compareBytes } from "./kernels.ts" +import * as Sort from "./sort.ts" import * as V from "./vector.ts" const I0 = BigInt(0) @@ -34,6 +34,15 @@ export type AggFn = | "median" | "list" +/** + * The kernel of an aggregate call with argument `arg`: `count` without an + * argument counts rows; every other function needs one. + * + * @internal + */ +export const kernel = (fn: AggFn, arg: unknown): AggFn => + arg !== undefined ? fn : fn === "count" ? "len" : V.fail(`${fn} needs an argument`) + /** @internal */ export interface Grouping { readonly ids: Int32Array @@ -75,13 +84,7 @@ export const aggregate = (fn: AggFn, v: V.Vector | undefined, g: Grouping): V.Ve if (v === undefined) return V.fail(`${fn} needs an argument`) switch (fn) { case "list": { - // Each group's values, in row order. - const starts = new Int32Array(groups + 1) - for (let i = 0; i < n; i++) starts[ids[i] + 1]++ - for (let k = 0; k < groups; k++) starts[k + 1] += starts[k] - const at = starts.slice(0, groups) - const rows = new Int32Array(n) - for (let i = 0; i < n; i++) rows[at[ids[i]]++] = i + const { rows, starts } = Hash.rowsByGroup(ids, groups) return V.list(S.list(S.field("item", v.type)), starts, V.take(v, rows)) } case "count": @@ -212,10 +215,18 @@ const asFloat = (v: V.Vector): { readonly x: Float64Array; readonly valid: Uint8 export const sumCount = ( v: V.Vector, g: Grouping +): { readonly sum: Float64Array; readonly count: Float64Array } => { + const { valid, x } = asFloat(v) + return sumCountOf(x, valid, g) +} + +const sumCountOf = ( + x: Float64Array, + valid: Uint8Array | undefined, + g: Grouping ): { readonly sum: Float64Array; readonly count: Float64Array } => { const ids = g.ids const n = ids.length - const { valid, x } = asFloat(v) const sum = new Float64Array(g.count) const count = new Float64Array(g.count) if (valid === undefined) { @@ -262,16 +273,15 @@ export const moments = ( const n = ids.length const { valid, x } = asFloat(v) const groups = g.count - const { count, sum } = sumCount(v, g) - const m = sum - for (let k = 0; k < groups; k++) m[k] = sum[k] / count[k] + const { count, sum } = sumCountOf(x, valid, g) + for (let k = 0; k < groups; k++) sum[k] /= count[k] const m2 = new Float64Array(groups) for (let i = 0; i < n; i++) { if (valid !== undefined && valid[i] === 0) continue - const d = x[i] - m[ids[i]] + const d = x[i] - sum[ids[i]] m2[ids[i]] += d * d } - return { count, mean: m, m2 } + return { count, mean: sum, m2 } } /** The std or variance column from per-group counts and M2. @internal */ @@ -360,7 +370,7 @@ const minMax = (fn: "min" | "max", v: V.Vector, g: Grouping): V.Vector => { case "str": { // Track the best row per group, then gather it. const best = new Int32Array(groups).fill(-1) - const cmp = rowComparator(v) + const cmp = Sort.rowOrder(v) for (let i = 0; i < n; i++) { if (!V.isValid(v, i)) continue const k = ids[i] @@ -372,20 +382,6 @@ const minMax = (fn: "min" | "max", v: V.Vector, g: Grouping): V.Vector => { } } -const rowComparator = (v: V.I64 | V.Str): (a: number, b: number) => number => { - if (v.kind === "str") { - const d = v.data - const o = v.offsets - return (a, b) => compareBytes(d, o[a], o[a + 1], d, o[b], o[b + 1]) - } - const x = v.values - if (V.isUnsigned64(v.type)) { - const u = new BigUint64Array(x.buffer, x.byteOffset, x.length) - return (a, b) => (u[a] < u[b] ? -1 : u[a] > u[b] ? 1 : 0) - } - return (a, b) => (x[a] < x[b] ? -1 : x[a] > x[b] ? 1 : 0) -} - const median = (v: V.Vector, g: Grouping): V.Vector => { const ids = g.ids const n = ids.length diff --git a/packages/effect/src/dataframe/internal/frame.ts b/packages/effect/src/dataframe/internal/frame.ts index acb6a66586a..5bad6c484d0 100644 --- a/packages/effect/src/dataframe/internal/frame.ts +++ b/packages/effect/src/dataframe/internal/frame.ts @@ -10,7 +10,7 @@ * * @internal */ -import * as ArrowData from "../../arrow/ArrowData.ts" +import * as ArrowColumn from "../../arrow/ArrowColumn.ts" import * as S from "../../arrow/ArrowSchema.ts" import * as ArrowTable from "../../arrow/ArrowTable.ts" import * as Equal from "../../Equal.ts" @@ -139,13 +139,8 @@ export const fromVectors = (names: ReadonlyArray, vectors: ReadonlyArray } /** @internal */ -export const columnIndex = (self: Frame, name: string): number => { - const i = self.columns.findIndex((c) => c.name === name) - return i < 0 ? V.fail(`unknown column ${JSON.stringify(name)}`) : i -} - -/** @internal */ -export const column = (self: Frame, name: string): Column => self.columns[columnIndex(self, name)] +export const column = (self: Frame, name: string): Column => + self.columns.find((c) => c.name === name) ?? V.fail(`unknown column ${JSON.stringify(name)}`) /** @internal */ export const context = (self: Frame): E.Context => ({ @@ -179,13 +174,12 @@ export const fromArrowTable = (table: ArrowTable.Table): Frame => { /** @internal */ export const toArrowTable = (self: Frame): ArrowTable.Table => { - const schema = S.make(self.columns.map((c) => S.field(c.name, c.type, { nullable: c.nullable }))) + const schema = self.schema const data = self.columns.map((c) => V.toArrowData(c.values())) return ArrowTable.make(schema, [ArrowTable.recordBatch(schema, self.length, data)]) } -/** @internal */ -export const rows = (self: Frame): Iterable => ArrowTable.rows(toArrowTable(self)) +const rows = (self: Frame): Iterable => ArrowTable.rows(toArrowTable(self)) /** @internal */ export const toRows = (self: Frame): Array => ArrowTable.toRows(toArrowTable(self)) @@ -193,21 +187,15 @@ export const toRows = (self: Frame): Array => ArrowTable.toRows(toArrowTabl /** @internal */ export const toArray = (self: Frame, name: string): Array => { const vector = column(self, name).values() - const field = S.field(name, vector.type) - const t = ArrowTable.make(S.make([field]), [ - ArrowTable.recordBatch(S.make([field]), vector.length, [V.toArrowData(vector)]) - ]) - return ArrowTable.toRows(t).map((r) => r[name]) + return ArrowColumn.toArray(ArrowColumn.make(S.field(name, vector.type), [V.toArrowData(vector)])) } -void ArrowData - // ----------------------------------------------------------------------------- // Row selection // ----------------------------------------------------------------------------- -/** Re-selects every column through `indices` (positions in the current frame; -1 = null). @internal */ -export const take = (self: Frame, indices: Int32Array): Frame => { +/** Re-selects every column through `indices` (positions in the current frame; -1 = null). */ +const take = (self: Frame, indices: Int32Array): Frame => { let missing = false for (let i = 0; i < indices.length; i++) { if (indices[i] < 0) { @@ -281,11 +269,9 @@ export const slice = (self: Frame, start: number, end: number): Frame => { ) } -/** @internal */ -export const head = (self: Frame, n: number): Frame => slice(self, 0, n) +const head = (self: Frame, n: number): Frame => slice(self, 0, n) -/** @internal */ -export const tail = (self: Frame, n: number): Frame => slice(self, Math.max(0, self.length - n), self.length) +const tail = (self: Frame, n: number): Frame => slice(self, Math.max(0, self.length - n), self.length) /** @internal */ export const sort = ( @@ -334,11 +320,9 @@ const evaluateAll = (self: Frame, exprs: { readonly [name: string]: E.Node }): A } /** @internal */ -export const select = (self: Frame, exprs: { readonly [name: string]: E.Node }): Frame => { - const columns = evaluateAll(self, exprs) +export const select = (self: Frame, exprs: { readonly [name: string]: E.Node }): Frame => // A frame of only scalars (e.g. `select(() => ({ one: Expr.lit(1) }))`) keeps the frame's length. - return make(columns, self.length) -} + make(evaluateAll(self, exprs), self.length) /** @internal */ export const withColumns = (self: Frame, exprs: { readonly [name: string]: E.Node }): Frame => { @@ -366,9 +350,9 @@ const aggregateColumns = ( return Object.keys(aggs).map((name) => { const node = aggs[name] if (node._tag !== "AggregateCall") return V.fail(`${name} is not an aggregate expression`) - if (node.arg === undefined && node.fn !== "count") return V.fail(`${node.fn} needs an argument`) + const fn = Agg.kernel(node.fn, node.arg) const input = node.arg === undefined ? undefined : E.evaluateColumn(node.arg, ctx) - return new Column(name, Agg.aggregate(node.arg === undefined ? "len" : node.fn, input, grouping), undefined) + return new Column(name, Agg.aggregate(fn, input, grouping), undefined) }) } @@ -408,20 +392,13 @@ const joinIds = ( pairs: ReadonlyArray<[V.Vector, V.Vector]>, nl: number, nr: number -): { leftIds: Int32Array; rightIds: Int32Array; count: number; nulls: undefined } => { +): { leftIds: Int32Array; rightIds: Int32Array; count: number } => { if (pairs.length === 1 && pairs[0][0].kind === "i32" && pairs[0][1].kind === "i32") { // One Int32 key: index the right keys directly by value when their range is small. const l = pairs[0][0] as V.I32 const r = pairs[0][1] as V.I32 - let min = 2147483647 - let max = -2147483648 - for (let j = 0; j < nr; j++) { - if (r.valid !== undefined && r.valid[j] === 0) continue - const v = r.values[j] - if (v < min) min = v - if (v > max) max = v - } - if (max < min || max - min < Math.max(1 << 16, 2 * (nl + nr))) { + const { max, min } = Hash.int32Range(r.values, r.valid, nr) + if (max < min || max - min < Hash.directLimit(nl + nr)) { const range = max < min ? 0 : max - min + 1 const table = new Int32Array(range).fill(-1) const rightIds = new Int32Array(nr) @@ -443,7 +420,7 @@ const joinIds = ( const k = lv[i] - min leftIds[i] = (la !== undefined && la[i] === 0) || k < 0 || k >= range ? -1 : table[k] } - return { leftIds, rightIds, count, nulls: undefined } + return { leftIds, rightIds, count } } } // General case: ids over both sides at once. @@ -456,7 +433,7 @@ const joinIds = ( for (let i = 0; i < nl; i++) if (nulls[i] === 1) leftIds[i] = -1 for (let j = 0; j < nr; j++) if (nulls[nl + j] === 1) rightIds[j] = -1 } - return { leftIds, rightIds, count: g.count, nulls: undefined } + return { leftIds, rightIds, count: g.count } } /** @internal */ @@ -464,16 +441,8 @@ export const join = (self: Frame, that: Frame, on: ReadonlyArray, how: J const nl = self.length const nr = that.length const pairs = on.map((k) => unifyKeys(column(self, k).values(), column(that, k).values())) - const { leftIds, nulls, rightIds, count } = joinIds(pairs, nl, nr) - // CSR buckets of right rows per id - const start = new Int32Array(count + 1) - for (let j = 0; j < nr; j++) if (rightIds[j] >= 0) start[rightIds[j] + 1]++ - for (let k = 0; k < count; k++) start[k + 1] += start[k] - const rows = new Int32Array(start[count]) - const fill = start.slice(0, count) - for (let j = 0; j < nr; j++) if (rightIds[j] >= 0) rows[fill[rightIds[j]]++] = j - const ids = leftIds - void nulls + const { count, leftIds: ids, rightIds } = joinIds(pairs, nl, nr) + const { rows, starts: start } = Hash.rowsByGroup(rightIds, count) if (how === "semi" || how === "anti") { const want = how === "semi" ? 1 : 0 @@ -746,9 +715,7 @@ export const unnest = (self: Frame, name: string): Frame => { fields.forEach((f, k) => { if (existing.has(f.name)) V.fail(`unnest would duplicate the column ${JSON.stringify(f.name)}`) const child = v.children[k] - const valid = v.valid === undefined - ? (child.kind === "null" ? undefined : child.valid) - : V.andValid(child.kind === "null" ? undefined : child.valid, v.valid, v.length) + const valid = V.andValid(child.kind === "null" ? undefined : child.valid, v.valid, v.length) columns.push(new Column(f.name, V.withValid(child, valid), undefined)) }) } diff --git a/packages/effect/src/dataframe/internal/hash.ts b/packages/effect/src/dataframe/internal/hash.ts index 67c8e27ba4f..302ce2b4e1a 100644 --- a/packages/effect/src/dataframe/internal/hash.ts +++ b/packages/effect/src/dataframe/internal/hash.ts @@ -29,8 +29,8 @@ const mix = (h: number): number => { return (h ^ (h >>> 16)) >>> 0 } -/** Largest table the direct (array-indexed) strategy may allocate for `n` rows. */ -const directLimit = (n: number) => Math.max(1 << 16, n * 2) +/** Largest table the direct (array-indexed) strategy may allocate for `n` rows. @internal */ +export const directLimit = (n: number) => Math.max(1 << 16, n * 2) /** * Ids for keys in `[0, range)`, by direct indexing: no hashing, one array @@ -60,8 +60,12 @@ const directIds = ( return { ids, count, firstRow: firstRow.subarray(0, count) } } -/** Ids for one Int32 column. @internal */ -const int32Ids = (x: Int32Array, valid: Uint8Array | undefined, n: number): Groups => { +/** The smallest and largest valid value; `max < min` when there is none. @internal */ +export const int32Range = ( + x: Int32Array, + valid: Uint8Array | undefined, + n: number +): { readonly min: number; readonly max: number } => { let min = 2147483647 let max = -2147483648 if (valid === undefined) { @@ -78,6 +82,12 @@ const int32Ids = (x: Int32Array, valid: Uint8Array | undefined, n: number): Grou if (v > max) max = v } } + return { min, max } +} + +/** Ids for one Int32 column. */ +const int32Ids = (x: Int32Array, valid: Uint8Array | undefined, n: number): Groups => { + const { max, min } = int32Range(x, valid, n) if (max >= min && max - min < directLimit(n)) return directIds(x, min, max - min + 1, valid, n) return hashInt32Ids(x, valid, n) } @@ -139,7 +149,7 @@ const hashInt32Ids = (x: Int32Array, valid: Uint8Array | undefined, n: number): return { ids, count, firstRow: firstRow.subarray(0, count) } } -/** Ids for 64-bit keys given as (lo, hi) pairs. @internal */ +/** Ids for 64-bit keys given as (lo, hi) pairs. */ const pairIds = (pairs: Int32Array, valid: Uint8Array | undefined, n: number): Groups => { const ids = new Int32Array(n) let firstRow: Int32Array = new Int32Array(64) @@ -202,13 +212,33 @@ const pairIds = (pairs: Int32Array, valid: Uint8Array | undefined, n: number): G return { ids, count, firstRow: firstRow.subarray(0, count) } } -const canonicalBits = (x: Float64Array, n: number): Int32Array => { +/** `x` with `-0` as `0` and every NaN as one NaN, so equal keys have equal bits. @internal */ +export const canonicalBits = (x: Float64Array, n: number): Float64Array => { const out = new Float64Array(n) for (let i = 0; i < n; i++) { const v = x[i] out[i] = v === 0 ? 0 : v !== v ? NaN : v } - return new Int32Array(out.buffer) + return out +} + +/** Moves a table of group ids and their stored hashes into one of `cap` slots. */ +const rehash = ( + slots: Int32Array, + hashes: Int32Array, + cap: number +): [Int32Array, Int32Array] => { + const mask = cap - 1 + const s = new Int32Array(cap).fill(-1) + const h = new Int32Array(cap) + for (let t = 0; t < slots.length; t++) { + if (slots[t] === -1) continue + let u = hashes[t] & mask + while (s[u] !== -1) u = (u + 1) & mask + s[u] = slots[t] + h[u] = hashes[t] + } + return [s, h] } const stringIds = (v: V.Str, n: number): Groups => { @@ -248,19 +278,9 @@ const stringIds = (v: V.Str, n: number): Groups => { hashes[s] = h ids[i] = count++ if (count * 2 > cap) { - const os = slots - const oh = hashes cap <<= 1 mask = cap - 1 - slots = new Int32Array(cap).fill(-1) - hashes = new Int32Array(cap) - for (let t = 0; t < os.length; t++) { - if (os[t] === -1) continue - let u = oh[t] & mask - while (slots[u] !== -1) u = (u + 1) & mask - slots[u] = os[t] - hashes[u] = oh[t] - } + ;[slots, hashes] = rehash(slots, hashes, cap) } break } @@ -318,7 +338,7 @@ export const ids = (v: V.Vector): Groups => { case "i32": return int32Ids(v.values, v.valid, n) case "f64": - return pairIds(canonicalBits(v.values, n), v.valid, n) + return pairIds(new Int32Array(canonicalBits(v.values, n).buffer), v.valid, n) case "i64": return pairIds(new Int32Array(v.values.buffer, v.values.byteOffset, n * 2), v.valid, n) case "bool": @@ -326,31 +346,30 @@ export const ids = (v: V.Vector): Groups => { case "str": return stringIds(v, n) case "null": - return { ids: new Int32Array(n), count: n > 0 ? 1 : 0, firstRow: new Int32Array(n > 0 ? 1 : 0) } + return single(n) case "list": case "struct": return V.fail(`cannot group, join or deduplicate by ${v.type._tag} columns yet`) } } -/** Combines two id columns into ids of the pair. @internal */ +/** Every row in one group (none without rows). */ +const single = (n: number): Groups => ({ + ids: new Int32Array(n), + count: n > 0 ? 1 : 0, + firstRow: new Int32Array(n > 0 ? 1 : 0) +}) + +/** Ids of the pairs of two id columns whose product of counts is at most `directLimit(n)`. */ const combine = (a: Groups, b: Groups, n: number): Groups => { if (a.count === 1) return b if (b.count === 1) return a const bc = b.count const ai = a.ids const bi = b.ids - if (a.count * bc <= directLimit(n)) { - const keys = new Int32Array(n) - for (let i = 0; i < n; i++) keys[i] = ai[i] * bc + bi[i] - return directIds(keys, 0, a.count * bc, undefined, n) - } - const pairs = new Int32Array(n * 2) - for (let i = 0; i < n; i++) { - pairs[2 * i] = ai[i] - pairs[2 * i + 1] = bi[i] - } - return pairIds(pairs, undefined, n) + const keys = new Int32Array(n) + for (let i = 0; i < n; i++) keys[i] = ai[i] * bc + bi[i] + return directIds(keys, 0, a.count * bc, undefined, n) } /** @@ -380,19 +399,9 @@ const tupleIds = (cols: ReadonlyArray, n: number): Groups => { hashes[s] = h ids[i] = count++ if (count * 2 > cap) { - const os = slots - const oh = hashes cap <<= 1 mask = cap - 1 - slots = new Int32Array(cap).fill(-1) - hashes = new Int32Array(cap) - for (let t = 0; t < os.length; t++) { - if (os[t] === -1) continue - let u = oh[t] & mask - while (slots[u] !== -1) u = (u + 1) & mask - slots[u] = os[t] - hashes[u] = oh[t] - } + ;[slots, hashes] = rehash(slots, hashes, cap) } break } @@ -418,9 +427,7 @@ const tupleIds = (cols: ReadonlyArray, n: number): Groups => { /** Dense ids for the rows of several key columns together. @internal */ export const groups = (keys: ReadonlyArray, n: number): Groups => { - if (keys.length === 0) { - return { ids: new Int32Array(n), count: n > 0 ? 1 : 0, firstRow: new Int32Array(n > 0 ? 1 : 0) } - } + if (keys.length === 0) return single(n) const parts = keys.map(ids) // Combine by direct indexing while the key space stays small, then hash // the remaining columns as one tuple. @@ -431,6 +438,26 @@ export const groups = (keys: ReadonlyArray, n: number): Groups => { return tupleIds([acc.ids, ...parts.slice(k).map((p) => p.ids)], n) } +/** + * The rows of every group, in row order: `rows[starts[g]..starts[g + 1])`. + * Rows with a negative id are left out. + * + * @internal + */ +export const rowsByGroup = ( + ids: Int32Array, + count: number +): { readonly starts: Int32Array; readonly rows: Int32Array } => { + const n = ids.length + const starts = new Int32Array(count + 1) + for (let i = 0; i < n; i++) if (ids[i] >= 0) starts[ids[i] + 1]++ + for (let g = 0; g < count; g++) starts[g + 1] += starts[g] + const at = starts.slice(0, count) + const rows = new Int32Array(starts[count]) + for (let i = 0; i < n; i++) if (ids[i] >= 0) rows[at[ids[i]]++] = i + return { starts, rows } +} + /** Rows where any key column is null. @internal */ export const anyNull = (keys: ReadonlyArray, n: number): Uint8Array | undefined => { let out: Uint8Array | undefined diff --git a/packages/effect/src/dataframe/internal/partial.ts b/packages/effect/src/dataframe/internal/partial.ts index 716db45176d..2b7fca9f6b1 100644 --- a/packages/effect/src/dataframe/internal/partial.ts +++ b/packages/effect/src/dataframe/internal/partial.ts @@ -9,9 +9,9 @@ * M2 (Chan et al.). Medians and distinct counts don't combine. * * Parts have the types of the whole input (every batch of a streamed node - * has the types of its whole result), so their partials combine as they - * are and finish in that input type. Float results may - * differ in rounding, as float sums of the same values in another order do. + * has the types of its whole result), so their partials combine as they are + * and finish in that input type. Float results may differ in rounding, as + * float sums of the same values in another order do. * * @internal */ @@ -21,8 +21,7 @@ import * as Hash from "./hash.ts" import * as K from "./kernels.ts" import * as V from "./vector.ts" -/** @internal */ -export type Part = +type Part = | { readonly _tag: "vector"; readonly vector: V.Vector } | { readonly _tag: "sums"; readonly sums: Agg.Sums } | { readonly _tag: "mean"; readonly sum: Float64Array; readonly count: Float64Array } @@ -32,7 +31,6 @@ export type Part = readonly mean: Float64Array readonly m2: Float64Array } - | { readonly _tag: "none" } const reaggregated = new Set(["count", "len", "min", "max", "first", "last"]) @@ -40,8 +38,8 @@ const reaggregated = new Set(["count", "len", "min", "max", "first", export const combinable = (fn: Agg.AggFn): boolean => reaggregated.has(fn) || fn === "sum" || fn === "mean" || fn === "std" || fn === "variance" -/** The partial aggregate of `fn` over `v` grouped by `g`. @internal */ -export const partial = (fn: Agg.AggFn, v: V.Vector | undefined, g: Agg.Grouping): Part => { +/** The partial aggregate of `fn` over `v` grouped by `g`. */ +const partial = (fn: Agg.AggFn, v: V.Vector | undefined, g: Agg.Grouping): Part => { if (reaggregated.has(fn)) return { _tag: "vector", vector: Agg.aggregate(fn, v, g) } if (v === undefined) return V.fail(`${fn} needs an argument`) switch (fn) { @@ -55,31 +53,23 @@ export const partial = (fn: Agg.AggFn, v: V.Vector | undefined, g: Agg.Grouping) case "variance": return { _tag: "moments", ...Agg.moments(v, g) } default: - return { _tag: "none" } + return V.fail(`${fn} cannot be computed from partial aggregates`) } } /** * Combines the partials of several parts: `parts[c]` holds part `c`'s - * partials for its groups over inputs of `type`; `ids` maps the - * concatenated groups of all parts to `count` combined groups. - * - * @internal + * partials for its groups; `ids` maps the concatenated groups of all parts + * to `count` combined groups. */ -export const combine = ( - fn: Agg.AggFn, - parts: ReadonlyArray, - type: S.DataType | undefined, - ids: Int32Array, - count: number -): Part => { +const combine = (fn: Agg.AggFn, parts: ReadonlyArray, ids: Int32Array, count: number): Part => { const grouping: Agg.Grouping = { ids, count } if (reaggregated.has(fn)) { const vector = K.concat(parts.map((p) => (p as Extract).vector)) return { _tag: "vector", vector: Agg.aggregate(fn === "count" || fn === "len" ? "sum" : fn, vector, grouping) } } if (fn === "sum") { - const float = type!._tag === "Float" + const float = (parts[0] as Extract).sums instanceof Float64Array const sums = float ? new Float64Array(count) : new Array(count).fill(BigInt(0)) let at = 0 for (const p of parts as ReadonlyArray>) { @@ -104,28 +94,26 @@ export const combine = ( } return { _tag: "mean", sum, count: total } } - if (fn === "std" || fn === "variance") { - const n = new Float64Array(count) - const mean = new Float64Array(count) - const m2 = new Float64Array(count) - let at = 0 - for (const p of parts as ReadonlyArray>) { - for (let j = 0; j < p.count.length; j++) { - const nb = p.count[j] - if (nb === 0) continue - const g = ids[at + j] - const na = n[g] - const nab = na + nb - const delta = p.mean[j] - mean[g] - mean[g] += delta * nb / nab - m2[g] += p.m2[j] + delta * delta * na * nb / nab - n[g] = nab - } - at += p.count.length + // std and variance + const n = new Float64Array(count) + const mean = new Float64Array(count) + const m2 = new Float64Array(count) + let at = 0 + for (const p of parts as ReadonlyArray>) { + for (let j = 0; j < p.count.length; j++) { + const nb = p.count[j] + if (nb === 0) continue + const g = ids[at + j] + const na = n[g] + const nab = na + nb + const delta = p.mean[j] - mean[g] + mean[g] += delta * nb / nab + m2[g] += p.m2[j] + delta * delta * na * nb / nab + n[g] = nab } - return { _tag: "moments", count: n, mean, m2 } + at += p.count.length } - return V.fail(`${fn} cannot be computed from partial aggregates`) + return { _tag: "moments", count: n, mean, m2 } } /** The aggregate column of a combined partial over inputs of `type`. @internal */ @@ -139,8 +127,6 @@ export const finish = (fn: Agg.AggFn, part: Part, type: S.DataType | undefined): return Agg.meanFrom(type!, part.sum, part.count) case "moments": return Agg.varianceFrom(fn as "std" | "variance", type!, part.count, part.m2) - case "none": - return V.fail(`${fn} cannot be computed from partial aggregates`) } } @@ -192,7 +178,7 @@ export const merge = (fns: ReadonlyArray, states: ReadonlyArray V.take(k, g.firstRow)), types: filled[0].types, - parts: fns.map((fn, a) => combine(fn, filled.map((s) => s.parts[a]), filled[0].types[a], g.ids, g.count)), + parts: fns.map((fn, a) => combine(fn, filled.map((s) => s.parts[a]), g.ids, g.count)), count: g.count, rows: filled.reduce((n, s) => n + s.rows, 0) } diff --git a/packages/effect/src/dataframe/internal/sort.ts b/packages/effect/src/dataframe/internal/sort.ts index db4877b7162..816f54a905f 100644 --- a/packages/effect/src/dataframe/internal/sort.ts +++ b/packages/effect/src/dataframe/internal/sort.ts @@ -56,13 +56,7 @@ export const words = (key: SortKey): { readonly hi: Uint32Array | undefined; rea return { hi: undefined, lo } } case "f64": { - const tmp = new Float64Array(n) - const x = v.values - for (let i = 0; i < n; i++) { - const a = x[i] - tmp[i] = a === 0 ? 0 : a !== a ? NaN : a - } - const bits = new Uint32Array(tmp.buffer) + const bits = new Uint32Array(Hash.canonicalBits(v.values, n).buffer) const hi = new Uint32Array(n) const lo = new Uint32Array(n) for (let i = 0; i < n; i++) { @@ -90,11 +84,9 @@ export const words = (key: SortKey): { readonly hi: Uint32Array | undefined; rea } case "str": { const g = Hash.ids(v) - const order = new Int32Array(g.count) - for (let i = 0; i < g.count; i++) order[i] = i const off = v.offsets const d = v.data - const sorted = Array.from(order).sort((a, b) => { + const sorted = Int32Array.from({ length: g.count }, (_, i) => i).sort((a, b) => { const ra = g.firstRow[a] const rb = g.firstRow[b] return compareBytes(d, off[ra], off[ra + 1], d, off[rb], off[rb + 1]) @@ -134,7 +126,6 @@ const radix = (perm: Int32Array, hi: Uint32Array | undefined, lo: Uint32Array): const src = useHi ? h! : l counts.fill(0) for (let i = 0; i < n; i++) counts[((src[i] >>> shift) & 0xffff) + 1]++ - // constant digit: nothing to do let constant = false for (let b = 1; b <= 65536; b++) { if (counts[b] === n) { @@ -161,7 +152,7 @@ const radix = (perm: Int32Array, hi: Uint32Array | undefined, lo: Uint32Array): const sortByKey = (key: SortKey, nullsLast: boolean, perm: Int32Array): Int32Array => { const n = perm.length const w = words(key) - // keys in current perm order + // The key's words in the current perm order. const lo = new Uint32Array(n) const hi = w.hi === undefined ? undefined : new Uint32Array(n) for (let i = 0; i < n; i++) { @@ -173,8 +164,8 @@ const sortByKey = (key: SortKey, nullsLast: boolean, perm: Int32Array): Int32Arr return partitionNulls(key.vector, sorted, nullsLast) } -/** Moves null keys to the front (or back), keeping order on both sides. @internal */ -export const partitionNulls = (v: V.Vector, perm: Int32Array, nullsLast: boolean): Int32Array => { +/** Moves null keys to the front (or back), keeping order on both sides. */ +const partitionNulls = (v: V.Vector, perm: Int32Array, nullsLast: boolean): Int32Array => { if (v.kind !== "null" && v.valid === undefined) return perm const n = perm.length const out = new Int32Array(n) @@ -213,7 +204,7 @@ const comparator = (keys: ReadonlyArray, nullsLast: boolean): (a: numbe } const smallSort = (keys: ReadonlyArray, nullsLast: boolean, perm: Int32Array): Int32Array => - Int32Array.from(Array.from(perm).sort(comparator(keys, nullsLast))) + perm.sort(comparator(keys, nullsLast)) /** * The first `k` rows of the order that sorts `keys`, without sorting the @@ -227,13 +218,12 @@ export const topK = (keys: ReadonlyArray, nullsLast: boolean, n: number const cmp = comparator(keys, nullsLast) // A max-heap: the root is the worst of the best `k` rows so far. const heap = new Int32Array(k) - const down = (size: number) => { - let i = 0 + const down = (i: number) => { while (true) { const l = 2 * i + 1 - if (l >= size) return + if (l >= k) return const r = l + 1 - const c = r < size && cmp(heap[r], heap[l]) > 0 ? r : l + const c = r < k && cmp(heap[r], heap[l]) > 0 ? r : l if (cmp(heap[c], heap[i]) <= 0) return const t = heap[c] heap[c] = heap[i] @@ -241,23 +231,46 @@ export const topK = (keys: ReadonlyArray, nullsLast: boolean, n: number i = c } } - for (let i = 0; i < k; i++) { - heap[i] = i - let j = i - while (j > 0) { - const p = (j - 1) >> 1 - if (cmp(heap[j], heap[p]) <= 0) break - const t = heap[p] - heap[p] = heap[j] - heap[j] = t - j = p - } - } + for (let i = 0; i < k; i++) heap[i] = i + for (let i = (k >> 1) - 1; i >= 0; i--) down(i) for (let i = k; i < n; i++) { if (cmp(i, heap[0]) < 0) { heap[0] = i - down(k) + down(0) + } + } + return heap.sort(cmp) +} + +// ----------------------------------------------------------------------------- +// Row comparison +// ----------------------------------------------------------------------------- + +/** `order(i, j) < 0` when row i sorts before row j (valid, non-NaN rows). @internal */ +export const rowOrder = (v: V.Vector): (i: number, j: number) => number => { + switch (v.kind) { + case "list": + case "struct": + return V.fail(`cannot order ${v.type._tag} values`) + case "f64": { + const x = v.values + return (i, j) => (x[i] < x[j] ? -1 : x[i] > x[j] ? 1 : 0) + } + case "i32": + case "bool": { + const x = v.values + return (i, j) => x[i] - x[j] } + case "i64": { + const x = V.isUnsigned64(v.type) ? new BigUint64Array(v.values.buffer, v.values.byteOffset, v.length) : v.values + return (i, j) => (x[i] < x[j] ? -1 : x[i] > x[j] ? 1 : 0) + } + case "str": { + const o = v.offsets + const d = v.data + return (i, j) => compareBytes(d, o[i], o[i + 1], d, o[j], o[j + 1]) + } + case "null": + return () => 0 } - return Int32Array.from(Array.from(heap).sort(cmp)) } diff --git a/packages/effect/src/dataframe/internal/stream.ts b/packages/effect/src/dataframe/internal/stream.ts index 2985d6c8aad..8e948018baf 100644 --- a/packages/effect/src/dataframe/internal/stream.ts +++ b/packages/effect/src/dataframe/internal/stream.ts @@ -11,20 +11,17 @@ * aggregates of windows collect instead). Every other operation collects its * inputs first. * - * Every batch of a node has the column types of the node's whole result, so - * each operation sees the values it would see over all rows at once: the - * batches of a table, as read (converted to its declared columns, if any), - * must have the columns of its first batch, as the batches of an Arrow - * stream share its schema, and a concatenation converts its inputs' batches - * to their common types, as concatenating their results would. + * Every batch of a node has the column types of the node's whole result: a + * table's batches (converted to its declared columns, if any) must share the + * columns of its first, and a concatenation converts its inputs to their + * common types. * * A node has at least one batch, possibly without rows, unless its columns * are unknown: they depend on a table without declared columns of which no - * batch was read (it has none, or a slice without rows doesn't read it). - * Where no batch is read, a declared table's result is its declared columns - * without rows. Unknown columns are never made up: an operation that needs - * them (collecting a result, concatenating, joining, aggregating, any - * operation that collects its input) fails, asking for them to be declared. + * batch was read. Where no batch is read, a declared table's result is its + * declared columns without rows. Operations that need unknown columns + * (collecting, concatenating, joining, aggregating) fail, asking for them to + * be declared. * * @internal */ @@ -35,7 +32,7 @@ import * as Stream from "../../Stream.ts" import type { Catalog } from "../Catalog.ts" import type { DataFrameError } from "../DataFrame.ts" import type * as Plan from "../Plan.ts" -import type * as Agg from "./aggregate.ts" +import * as Agg from "./aggregate.ts" import * as D from "./dtype.ts" import * as E from "./expr.ts" import * as F from "./frame.ts" @@ -124,10 +121,10 @@ const perBatch = (node: Q.Node): boolean => { /** Runs a node by collecting its inputs (an operation that needs all rows). */ const collected = (node: Q.Node, catalog: Tables): Batches => - Stream.unwrap( - Effect.forEach(Q.children(node), (c) => collectFrames(batches(c, catalog), c)).pipe( - Effect.flatMap((inputs) => Q.attempt(() => Q.step(node, inputs))), - Effect.map(Stream.succeed) + Stream.fromEffect( + Effect.flatMap( + Effect.forEach(Q.children(node), (c) => collectFrames(batches(c, catalog), c)), + (inputs) => Q.attempt(() => Q.step(node, inputs)) ) ) @@ -138,9 +135,7 @@ const aggregate = ( const keys = node._tag === "GroupBy" ? node.keys : [] const specs = node.aggregates.map(({ expression, name }) => { if (expression._tag !== "AggregateCall") return V.fail(`${name} is not an aggregate expression`) - if (expression.arg === undefined && expression.fn !== "count") return V.fail(`${expression.fn} needs an argument`) - const fn: Agg.AggFn = expression.arg === undefined ? "len" : expression.fn - return { name, fn, arg: expression.arg } + return { name, fn: Agg.kernel(expression.fn, expression.arg), arg: expression.arg } }) if (!Q.isRowWise(node) || !specs.every((s) => Partial.combinable(s.fn))) { return collected(node, catalog) @@ -149,7 +144,7 @@ const aggregate = ( return Stream.suspend(() => { // Whether any batch's key column may hold nulls. const nullable = keys.map(() => false) - return Stream.unwrap( + return Stream.fromEffect( Stream.runFoldEffect( batches(node.input, catalog), (): Partial.State | undefined => undefined, @@ -173,8 +168,7 @@ const aggregate = ( ) return F.make([...keyColumns, ...columns], state.count) }) - ), - Effect.map(Stream.succeed) + ) ) ) }) @@ -241,10 +235,8 @@ const slice = (input: Batches, offset: number, length: number | undefined): Batc * The batches of a node's result: at least one unless its columns are * unknown. Batches without rows stay, so a result whose rows are all dropped * still has the types of its columns. - * - * @internal */ -export const batches = (node: Q.Node, catalog: Tables): Batches => { +const batches = (node: Q.Node, catalog: Tables): Batches => { if (perBatch(node)) { return Stream.mapEffect( batches((node as { readonly input: Q.Node }).input, catalog), diff --git a/packages/effect/src/dataframe/internal/window.ts b/packages/effect/src/dataframe/internal/window.ts index 0278346ed0b..4da4b1209d9 100644 --- a/packages/effect/src/dataframe/internal/window.ts +++ b/packages/effect/src/dataframe/internal/window.ts @@ -7,33 +7,19 @@ */ import * as S from "../../arrow/ArrowSchema.ts" import type * as Plan from "../Plan.ts" +import type * as Agg from "./aggregate.ts" import * as Dec from "./decimal.ts" import * as D from "./dtype.ts" -import { compareBytes } from "./kernels.ts" +import * as Hash from "./hash.ts" import * as Sort from "./sort.ts" import * as V from "./vector.ts" /** Rows' partition ids (dense, in order of first appearance). @internal */ -export interface Partition { - readonly ids: Int32Array - readonly count: number -} +export type Partition = Agg.Grouping /** One partition holding every row. @internal */ export const whole = (n: number): Partition => ({ ids: new Int32Array(n), count: n > 0 ? 1 : 0 }) -/** The rows of every partition, in row order: `rows[starts[g]..starts[g + 1])`. */ -const rowsByPartition = (p: Partition): { readonly starts: Int32Array; readonly rows: Int32Array } => { - const n = p.ids.length - const starts = new Int32Array(p.count + 1) - for (let i = 0; i < n; i++) starts[p.ids[i] + 1]++ - for (let g = 0; g < p.count; g++) starts[g + 1] += starts[g] - const at = starts.slice(0, p.count) - const rows = new Int32Array(n) - for (let i = 0; i < n; i++) rows[at[p.ids[i]]++] = i - return { starts, rows } -} - const validOf = (v: V.Vector): Uint8Array | undefined => v.kind === "null" ? new Uint8Array(v.length) : v.valid === undefined ? undefined : v.valid.slice() @@ -50,7 +36,8 @@ export const cumSum = (v: V.Vector, p: Partition): V.Vector => { switch (v.kind) { case "null": return V.nulls(n) - case "f64": { + case "f64": + case "bool": { const x = v.values const acc = new Float64Array(p.count) const out = new Float64Array(n) @@ -68,13 +55,6 @@ export const cumSum = (v: V.Vector, p: Partition): V.Vector => { } return V.i32(type, out, valid) } - case "bool": { - const x = v.values - const acc = new Float64Array(p.count) - const out = new Float64Array(n) - for (let i = 0; i < n; i++) if (valid === undefined || valid[i] === 1) out[i] = acc[ids[i]] += x[i] - return V.f64(S.uint32, out, valid) - } case "i64": { const x = v.values const acc = new BigInt64Array(p.count) @@ -102,51 +82,12 @@ export const cumCount = (v: V.Vector, p: Partition): V.Vector => { return V.f64(S.uint32, out) } -/** `compare(i, j) < 0` when row i sorts before row j (valid, non-NaN rows). */ -const rowOrder = (v: V.Vector): (i: number, j: number) => number => { - switch (v.kind) { - case "list": - case "struct": - return V.fail(`cannot order ${v.type._tag} values`) - case "f64": { - const x = v.values - return (i, j) => (x[i] < x[j] ? -1 : x[i] > x[j] ? 1 : 0) - } - case "i32": - case "bool": { - const x = v.values - return (i, j) => x[i] - x[j] - } - case "i64": { - const x = v.values - const unsigned = V.isUnsigned64(v.type) - return unsigned - ? ( - i, - j - ) => (BigInt.asUintN(64, x[i]) < BigInt.asUintN(64, x[j]) - ? -1 - : BigInt.asUintN(64, x[i]) > BigInt.asUintN(64, x[j]) - ? 1 - : 0) - : (i, j) => (x[i] < x[j] ? -1 : x[i] > x[j] ? 1 : 0) - } - case "str": { - const o = v.offsets - const d = v.data - return (i, j) => compareBytes(d, o[i], o[i + 1], d, o[j], o[j + 1]) - } - case "null": - return () => 0 - } -} - const isNaNAt = (v: V.Vector, i: number): boolean => v.kind === "f64" && v.values[i] !== v.values[i] /** Running minimum or maximum per partition; NaNs are skipped, nulls stay null. @internal */ export const cumMinMax = (fn: "cumMin" | "cumMax", v: V.Vector, p: Partition): V.Vector => { const n = v.length - const order = rowOrder(v) + const order = Sort.rowOrder(v) const sign = fn === "cumMin" ? -1 : 1 const best = new Int32Array(p.count).fill(-1) const rows = new Int32Array(n) @@ -168,7 +109,7 @@ export const cumMinMax = (fn: "cumMin" | "cumMax", v: V.Vector, p: Partition): V /** The value `n` rows earlier in the partition (later for negative `n`). @internal */ export const shift = (v: V.Vector, p: Partition, n: number): V.Vector => { - const { rows, starts } = rowsByPartition(p) + const { rows, starts } = Hash.rowsByGroup(p.ids, p.count) const idx = new Int32Array(v.length).fill(-1) for (let g = 0; g < p.count; g++) { const s = starts[g] @@ -200,7 +141,7 @@ export const rank = (v: V.Vector, p: Partition, method: Plan.RankMethod, descend const valid = validOf(v) let i = 0 while (i < n) { - // one partition: perm[i..end) + // One partition: perm[i..end). const g = p.ids[perm[i]] let end = i while (end < n && p.ids[perm[end]] === g) end++ @@ -252,7 +193,7 @@ export const rolling = ( minPeriods: number ): V.Vector => { const n = v.length - const { rows, starts } = rowsByPartition(p) + const { rows, starts } = Hash.rowsByGroup(p.ids, p.count) /** Calls `f` with each row and the window `rows[first..last]` ending at it. */ const windows = (f: (row: number, first: number, last: number) => void) => { for (let g = 0; g < p.count; g++) { @@ -261,7 +202,7 @@ export const rolling = ( } const ok = new Uint8Array(n) if (fn === "rollingMin" || fn === "rollingMax") { - const order = rowOrder(v) + const order = Sort.rowOrder(v) const sign = fn === "rollingMin" ? -1 : 1 const idx = new Int32Array(n).fill(-1) windows((row, first, last) => { @@ -324,8 +265,6 @@ export const rolling = ( switch (v.kind) { case "i32": return V.i32(type, Int32Array.from(out, (a) => a | 0), ok) - case "bool": - return V.f64(S.uint32, out, ok) default: V.wrapInPlace(type, out) return V.f64(type, out, ok) From 6d17c5648588fb1cdbe7a9877bd937e4d92dfc3d Mon Sep 17 00:00:00 2001 From: Effect Bot Date: Sun, 15 Nov 2026 19:11:11 +0000 Subject: [PATCH 57/88] Simplify dataframe plan rewriting, expression evaluation and schema declarations --- .../effect/src/dataframe/internal/expr.ts | 38 ++-- .../effect/src/dataframe/internal/query.ts | 203 ++++++++---------- .../effect/src/dataframe/internal/schema.ts | 124 +++++------ .../effect/src/dataframe/internal/stream.ts | 18 +- 4 files changed, 158 insertions(+), 225 deletions(-) diff --git a/packages/effect/src/dataframe/internal/expr.ts b/packages/effect/src/dataframe/internal/expr.ts index 65b552ca7f0..38f5ede55f5 100644 --- a/packages/effect/src/dataframe/internal/expr.ts +++ b/packages/effect/src/dataframe/internal/expr.ts @@ -196,9 +196,9 @@ export const evaluate = (node: Node, ctx: Context): K.Value => { case "AggregateCall": { // Inside `over` or an aggregate: the partition's aggregate on each of its rows. if (ctx.partition === undefined) return V.fail(`aggregate ${show(node)} can only be used in agg or over`) - if (node.arg === undefined && node.fn !== "count") return V.fail(`${node.fn} needs an argument`) - const arg = node.arg === undefined ? undefined : evaluateColumn(node.arg, ctx) - const values = Agg.aggregate(node.arg === undefined ? "len" : node.fn, arg, ctx.partition) + // `count()` without an argument counts rows; `Agg.aggregate` fails for other aggregates without one. + const fn = node.fn === "count" && node.arg === undefined ? "len" : node.fn + const values = Agg.aggregate(fn, node.arg && evaluateColumn(node.arg, ctx), ctx.partition) return K.column(V.take(values, ctx.partition.ids)) } case "Call": @@ -328,6 +328,8 @@ const arithmetic = { mod: "mod" } as const +const listAggregates = { listSum: "sum", listMean: "mean", listMin: "min", listMax: "max" } as const + const evaluateCall = (node: Plan.Call, ctx: Context): K.Value => { const n = ctx.length const bounds = arity[node.fn] @@ -344,6 +346,8 @@ const evaluateCall = (node: Plan.Call, ctx: Context): K.Value => { ) } const arg = (i: number) => evaluate(node.args[i], ctx) + const col = (i: number) => K.materialize(arg(i), n) + const args = () => node.args.map((a) => evaluate(a, ctx)) const options = node.options ?? {} const option = (name: keyof Plan.CallOptions): A => { const value = options[name] @@ -384,7 +388,7 @@ const evaluateCall = (node: Plan.Call, ctx: Context): K.Value => { return K.isNaN_(arg(0)) case "fillNull": case "coalesce": - return K.coalesce(node.args.map((a) => evaluate(a, ctx)), n) + return K.coalesce(args(), n) case "when": return K.when(arg(0), arg(1), arg(2), n) case "cast": { @@ -413,7 +417,7 @@ const evaluateCall = (node: Plan.Call, ctx: Context): K.Value => { case "endsWith": return K.strMatch(arg(0), node.fn, option("pattern")) case "concat": - return K.strConcat(node.args.map((a) => evaluate(a, ctx)), n) + return K.strConcat(args(), n) case "year": case "month": case "day": @@ -423,35 +427,33 @@ const evaluateCall = (node: Plan.Call, ctx: Context): K.Value => { case "millisecond": case "weekday": case "ordinalDay": - return K.column(T.field(node.fn, K.materialize(arg(0), n))) + return K.column(T.field(node.fn, col(0))) case "truncate": - return K.column(T.truncate(K.materialize(arg(0), n), option("every"))) + return K.column(T.truncate(col(0), option("every"))) case "listLength": - return K.column(Nested.length(K.materialize(arg(0), n))) + return K.column(Nested.length(col(0))) case "listGet": - return K.column(Nested.get(K.materialize(arg(0), n), option("index"))) + return K.column(Nested.get(col(0), option("index"))) case "listSum": case "listMean": case "listMin": - case "listMax": { - const fn = node.fn === "listSum" ? "sum" : node.fn === "listMean" ? "mean" : node.fn === "listMin" ? "min" : "max" - return K.column(Nested.aggregate(fn, K.materialize(arg(0), n))) - } + case "listMax": + return K.column(Nested.aggregate(listAggregates[node.fn], col(0))) case "listContains": - return K.column(Nested.contains(K.materialize(arg(0), n), arg(1))) + return K.column(Nested.contains(col(0), arg(1))) case "field": - return K.column(Nested.field(K.materialize(arg(0), n), option("name"))) + return K.column(Nested.field(col(0), option("name"))) case "struct": { const names = option>("names") if (names.length !== node.args.length) return V.fail("struct needs one name per argument") - return K.column(Nested.struct(names, node.args.map((a) => K.materialize(evaluate(a, ctx), n)), n)) + return K.column(Nested.struct(names, node.args.map((_, i) => col(i)), n)) } case "totalDays": case "totalHours": case "totalMinutes": case "totalSeconds": case "totalMilliseconds": - return K.column(T.total(node.fn, K.materialize(arg(0), n))) + return K.column(T.total(node.fn, col(0))) case "cumSum": case "cumMin": case "cumMax": @@ -463,7 +465,7 @@ const evaluateCall = (node: Plan.Call, ctx: Context): K.Value => { case "rollingMean": case "rollingMin": case "rollingMax": { - const x = K.materialize(evaluate(node.args[0], ctx), n) + const x = col(0) const p = ctx.partition ?? W.whole(n) switch (node.fn) { case "cumSum": diff --git a/packages/effect/src/dataframe/internal/query.ts b/packages/effect/src/dataframe/internal/query.ts index 0335c99f99e..48242009b10 100644 --- a/packages/effect/src/dataframe/internal/query.ts +++ b/packages/effect/src/dataframe/internal/query.ts @@ -12,15 +12,14 @@ * join keys moves to the left side only (the side whose keys the join * keeps); * - slices move below row-wise operations that keep every row in place, and - * a slice of a sort becomes a top-k selection. + * a slice of a sort becomes a top-k selection; + * - columns nothing reads are dropped at the inputs, and derived columns and + * aggregates nothing reads are not computed. * * An operation is row-wise when its expressions are (see `isRowWise`): a * window in a predicate, a sort key or a derived column sees every row of its * input, so moving a filter or slice below it changes its result. * - * - columns nothing reads are dropped at the inputs, and derived columns and - * aggregates nothing reads are not computed. - * * @internal */ import * as Effect from "../../Effect.ts" @@ -52,21 +51,46 @@ export type Node = type Rebuild = N extends { readonly input: Plan.Node } ? Omit & { readonly input: Node } : never -/** @internal */ -export const toError = (cause: unknown): DataFrameError => - new DataFrameError({ message: cause instanceof Error ? cause.message : String(cause), cause }) - /** Runs a kernel computation, turning its failures into `DataFrameError`s. @internal */ export const attempt = (f: () => A): Effect.Effect => { try { return Effect.succeed(f()) } catch (cause) { - if (cause instanceof V.DataFrameDefect) return Effect.fail(toError(cause)) + if (cause instanceof V.DataFrameDefect) return Effect.fail(new DataFrameError({ message: cause.message, cause })) if (cause instanceof DataFrameError) return Effect.fail(cause) throw cause } } +/** @internal */ +export const children = (node: Node): ReadonlyArray => { + switch (node._tag) { + case "Values": + case "Table": + return [] + case "Join": + return [node.left, node.right] + case "Concat": + return node.inputs + default: + return [node.input] + } +} + +const mapChildren = (node: Node, f: (child: Node) => Node): Node => { + switch (node._tag) { + case "Values": + case "Table": + return node + case "Join": + return { ...node, left: f(node.left), right: f(node.right) } + case "Concat": + return { ...node, inputs: node.inputs.map(f) } + default: + return { ...node, input: f(node.input) } as Node + } +} + // ----------------------------------------------------------------------------- // Resolving tables // ----------------------------------------------------------------------------- @@ -87,21 +111,8 @@ export const tables = (plan: Plan.Node): Map => { } /** A plan with each `Table` replaced by `f` of it. @internal */ -export const mapTables = (plan: Plan.Node, f: (table: Plan.Table) => Node): Node => { - const replace = (node: Plan.Node): Node => { - switch (node._tag) { - case "Table": - return f(node) - case "Values": - return node - case "Join": - return { ...node, left: replace(node.left), right: replace(node.right) } - case "Concat": - return { ...node, inputs: node.inputs.map(replace) } - default: - return { ...node, input: replace(node.input) } as Node - } - } +export const mapTables = (plan: Node, f: (table: Plan.Table) => Node): Node => { + const replace = (node: Node): Node => node._tag === "Table" ? f(node) : mapChildren(node, replace) return replace(plan) } @@ -109,8 +120,8 @@ export const mapTables = (plan: Plan.Node, f: (table: Plan.Table) => Node): Node const tableValues = (table: Plan.Table, frame: F.Frame): Node => values(table.columns === undefined ? frame : conform(frame, table.columns, table.schema)) -/** Replaces every `Table` with the catalog's frame, reading each table once. @internal */ -export const resolve = (plan: Plan.Node, catalog: Catalog.Service): Effect.Effect => { +/** Replaces every `Table` with the catalog's frame, reading each table once. */ +const resolve = (plan: Plan.Node, catalog: Catalog.Service): Effect.Effect => { const names = [...tables(plan).keys()] if (names.length === 0) return Effect.succeed(plan as Node) return Effect.forEach( @@ -120,7 +131,7 @@ export const resolve = (plan: Plan.Node, catalog: Catalog.Service): Effect.Effec .pipe( Effect.flatMap((entries) => { const frames = new Map(entries) - return attempt(() => mapTables(plan, (table) => tableValues(table, frames.get(table.name)!))) + return attempt(() => mapTables(plan as Node, (table) => tableValues(table, frames.get(table.name)!))) }) ) } @@ -134,10 +145,8 @@ const namesCache = new WeakMap | /** * The output column names of a resolved plan, or `undefined` when they * depend on the data (a pivot without declared columns). - * - * @internal */ -export const names = (node: Node): ReadonlyArray | undefined => { +const names = (node: Node): ReadonlyArray | undefined => { const cached = namesCache.get(node) if (cached !== undefined) return cached.names const result = computeNames(node) @@ -157,6 +166,7 @@ const computeNames = (node: Node): ReadonlyArray | undefined => { case "Slice": case "Unique": case "DropNulls": + case "Explode": return names(node.input) case "Select": return node.columns.map((c) => c.name) @@ -190,8 +200,6 @@ const computeNames = (node: Node): ReadonlyArray | undefined => { return node.inputs.length === 0 ? [] : names(node.inputs[0]) case "Pivot": return node.columns === undefined ? undefined : [...node.index, ...node.columns] - case "Explode": - return names(node.input) case "Unnest": return undefined case "Unpivot": @@ -224,10 +232,9 @@ const expressions = (node: Node): ReadonlyArray => { } /** - * Whether an operation's expressions are row-wise (`E.isRowWise`; for - * aggregations, their arguments), so that each input row contributes the - * same values whichever other rows are present. Only row-wise operations - * let filters and slices move below them, and run batch by batch. + * Whether an operation's expressions are row-wise (for aggregations, their + * arguments). Only such operations let filters and slices move below them, + * and run batch by batch. * * @internal */ @@ -244,25 +251,13 @@ const conjuncts = (e: Plan.Expression): Array => const and = (es: ReadonlyArray): Plan.Expression => es.reduce((a, b) => ({ _tag: "Call", fn: "and", args: [a, b] })) -const subset = (a: Set, b: ReadonlySet | ReadonlyArray): boolean => { - const s = b instanceof Set ? b : new Set(b as ReadonlyArray) - for (const x of a) if (!s.has(x)) return false +const subset = (a: Set, b: ReadonlySet): boolean => { + for (const x of a) if (!b.has(x)) return false return true } -const mapChildren = (node: Node, f: (child: Node) => Node): Node => { - switch (node._tag) { - case "Values": - case "Table": - return node - case "Join": - return { ...node, left: f(node.left), right: f(node.right) } - case "Concat": - return { ...node, inputs: node.inputs.map(f) } - default: - return { ...node, input: f(node.input) } as Node - } -} +const inverse = (mapping: { readonly [from: string]: string }): Map => + new Map(Object.entries(mapping).map(([from, to]) => [to, from])) const pushFilters = (node: Node): Node => { const optimized = mapChildren(node, pushFilters) @@ -275,17 +270,18 @@ const filtered = (input: Node, predicates: ReadonlyArray): Node /** * Places each predicate as deep as it can go below `node`. Predicates that * are not row-wise (windows) depend on exactly the rows they see, so a set - * containing one stays where it is, as do predicates above an operation that - * is not row-wise. + * containing one stays where it is. */ const push = (node: Node, predicates: ReadonlyArray): Node => { if (predicates.length === 0) return node if (!predicates.every(E.isRowWise) || !isRowWise(node)) return filtered(node, predicates) - const split = (canPush: (refs: Set, p: Plan.Expression) => boolean) => { + /** Pushes the predicates reading only columns that pass `ok` into the input, rewritten. */ + const below = (ok: (name: string) => boolean, rewrite = (p: Plan.Expression) => p): Node => { const down: Array = [] const here: Array = [] - for (const p of predicates) (canPush(E.columns(p), p) ? down : here).push(p) - return { down, here } + for (const p of predicates) ([...E.columns(p)].every(ok) ? down : here).push(p) + const input = push((node as { readonly input: Node }).input, down.map(rewrite)) + return filtered({ ...node, input } as Node, here) } switch (node._tag) { case "Filter": @@ -294,34 +290,25 @@ const push = (node: Node, predicates: ReadonlyArray): Node => { case "DropNulls": case "Pick": case "Omit": - return { ...node, input: push(node.input, predicates) } + return below(() => true) case "Rename": { - const inverse = new Map(Object.entries(node.mapping).map(([from, to]) => [to, from])) - const renamed = predicates.map((p) => - E.mapColumns(p, (name) => ({ _tag: "Column", name: inverse.get(name) ?? name })) - ) - return { ...node, input: push(node.input, renamed) } + const from = inverse(node.mapping) + return below(() => true, (p) => E.mapColumns(p, (name) => ({ _tag: "Column", name: from.get(name) ?? name }))) } case "WithColumns": { const added = new Set(node.columns.map((c) => c.name)) - const { down, here } = split((refs) => [...refs].every((r) => !added.has(r))) - return filtered({ ...node, input: push(node.input, down) }, here) + return below((r) => !added.has(r)) } case "Select": { const byName = new Map(node.columns.map((c) => [c.name, c.expression])) - const { down, here } = split((refs) => [...refs].every((r) => byName.get(r)?._tag === "Column")) - const inlined = down.map((p) => E.mapColumns(p, (name) => byName.get(name)!)) - return filtered({ ...node, input: push(node.input, inlined) }, here) + return below((r) => byName.get(r)?._tag === "Column", (p) => E.mapColumns(p, (name) => byName.get(name)!)) } case "Unique": { const columns = node.columns - const { down, here } = split((refs) => columns === undefined || subset(refs, columns)) - return filtered({ ...node, input: push(node.input, down) }, here) - } - case "GroupBy": { - const { down, here } = split((refs) => subset(refs, node.keys)) - return filtered({ ...node, input: push(node.input, down) }, here) + return below((r) => columns === undefined || columns.includes(r)) } + case "GroupBy": + return below((r) => node.keys.includes(r)) case "Join": { const leftNames = names(node.left) const rightNames = names(node.right) @@ -380,10 +367,10 @@ const slice = (input: Node, offset: number, length: number | undefined): Node => type Required = ReadonlySet | undefined -const union = (required: Required, ...more: ReadonlyArray>): Required => { +const union = (required: Required, more: Iterable): Required => { if (required === undefined) return undefined const out = new Set(required) - for (const m of more) for (const x of m) out.add(x) + for (const x of more) out.add(x) return out } @@ -411,12 +398,10 @@ const prune = (node: Node, required: Required): Node => { case "Table": return node case "Filter": - return { ...node, input: prune(node.input, union(required, E.columns(node.predicate))) } case "Sort": case "TopK": - return { ...node, input: prune(node.input, union(required, exprColumns(node.keys.map((k) => k.expression)))) } case "Slice": - return { ...node, input: prune(node.input, required) } + return { ...node, input: prune(node.input, union(required, exprColumns(expressions(node)))) } case "Unique": case "DropNulls": return { @@ -443,13 +428,13 @@ const prune = (node: Node, required: Required): Node => { const all = names(node.input) if (all === undefined) return { ...node, input: prune(node.input, undefined) } const drop = new Set(node.columns) - const below = all.filter((n) => !drop.has(n) && (required === undefined || required.has(n))) - return { ...node, input: prune(node.input, new Set(below.length === 0 ? all.slice(0, 1) : below)) } + const below = keep(all, (n) => !drop.has(n) && (required === undefined || required.has(n))) + return { ...node, input: prune(node.input, new Set(below)) } } case "Rename": { if (required === undefined) return { ...node, input: prune(node.input, undefined) } - const inverse = new Map(Object.entries(node.mapping).map(([from, to]) => [to, from])) - return { ...node, input: prune(node.input, new Set([...required].map((n) => inverse.get(n) ?? n))) } + const from = inverse(node.mapping) + return { ...node, input: prune(node.input, new Set([...required].map((n) => from.get(n) ?? n))) } } case "GroupBy": { const aggregates = required === undefined @@ -510,9 +495,8 @@ const line = (node: Node): string => { case "Filter": return `Filter ${E.show(node.predicate)}` case "Select": - return `Select ${named(node.columns)}` case "WithColumns": - return `WithColumns ${named(node.columns)}` + return `${node._tag} ${named(node.columns)}` case "Sort": return `Sort ${keys(node.keys)}${node.nullsLast ? " nulls last" : ""}` case "TopK": @@ -520,13 +504,11 @@ const line = (node: Node): string => { case "Slice": return `Slice offset ${node.offset}${node.length === undefined ? "" : ` length ${node.length}`}` case "Unique": - return `Unique${node.columns === undefined ? "" : ` ${list(node.columns)}`}` case "DropNulls": - return `DropNulls${node.columns === undefined ? "" : ` ${list(node.columns)}`}` + return `${node._tag}${node.columns === undefined ? "" : ` ${list(node.columns)}`}` case "Pick": - return `Pick ${list(node.columns)}` case "Omit": - return `Omit ${list(node.columns)}` + return `${node._tag} ${list(node.columns)}` case "Rename": return `Rename ${list(Object.entries(node.mapping).map(([a, b]) => `${a} -> ${b}`))}` case "GroupBy": @@ -544,24 +526,8 @@ const line = (node: Node): string => { case "Unpivot": return `Unpivot ${list(node.on)} index ${list(node.index)} as ${node.variable}, ${node.value}` case "Explode": - return `Explode ${node.column}` case "Unnest": - return `Unnest ${node.column}` - } -} - -/** @internal */ -export const children = (node: Node): ReadonlyArray => { - switch (node._tag) { - case "Values": - case "Table": - return [] - case "Join": - return [node.left, node.right] - case "Concat": - return node.inputs - default: - return [node.input] + return `${node._tag} ${node.column}` } } @@ -662,18 +628,18 @@ export const execute = (plan: Node): Effect.Effect => { * * @internal */ -export const collect = (plan: Plan.Node, catalog: Catalog.Service): Effect.Effect => - isStreaming(plan, catalog) - ? Streaming.collect(plan, catalog).pipe( - Effect.withSpan("dataframe.Query", { - attributes: { "dataframe.operations": count(plan), "dataframe.streaming": true } - }) - ) +export const collect = (plan: Plan.Node, catalog: Catalog.Service): Effect.Effect => { + const streaming = isStreaming(plan, catalog) + const result = streaming + ? Streaming.collect(plan, catalog) : resolve(plan, catalog).pipe( Effect.flatMap((resolved) => attempt(() => optimize(resolved))), - Effect.flatMap(execute), - Effect.withSpan("dataframe.Query", { attributes: { "dataframe.operations": count(plan) } }) + Effect.flatMap(execute) ) + return result.pipe(Effect.withSpan("dataframe.Query", { + attributes: { "dataframe.operations": count(plan), ...(streaming ? { "dataframe.streaming": true } : {}) } + })) +} /** * The optimised plan that runs, as text. Plans over streamed tables show @@ -682,9 +648,10 @@ export const collect = (plan: Plan.Node, catalog: Catalog.Service): Effect.Effec * @internal */ export const explain = (plan: Plan.Node, catalog: Catalog.Service): Effect.Effect => - isStreaming(plan, catalog) - ? attempt(() => show(optimize(plan as Node))) - : resolve(plan, catalog).pipe(Effect.flatMap((resolved) => attempt(() => show(optimize(resolved))))) + Effect.flatMap( + isStreaming(plan, catalog) ? Effect.succeed(plan as Node) : resolve(plan, catalog), + (resolved) => attempt(() => show(optimize(resolved))) + ) const isStreaming = (plan: Plan.Node, catalog: Catalog.Service): boolean => [...tables(plan).keys()].some(catalog.streaming) diff --git a/packages/effect/src/dataframe/internal/schema.ts b/packages/effect/src/dataframe/internal/schema.ts index 43fedeed970..47bec3533a0 100644 --- a/packages/effect/src/dataframe/internal/schema.ts +++ b/packages/effect/src/dataframe/internal/schema.ts @@ -28,6 +28,8 @@ import * as V from "./vector.ts" type Column = Omit +const field = (path: string, name: string) => (path === "" ? name : `${path}.${name}`) + const leaf = (type: Plan.TypeName, nullable = false): Column => ({ type, nullable }) const literalType = (literal: unknown): Plan.TypeName | undefined => { @@ -50,6 +52,17 @@ const sameShape = (a: Column, b: Column): boolean => const suspended = new Set() +/** `f` of a suspended schema, failing (after `prefix`) when it recurses. */ +const unsuspend = (ast: AST.Suspend, prefix: string, f: (ast: AST.AST) => A): A => { + if (suspended.has(ast)) return V.fail(`${prefix}recursive schemas have no DataFrame column type`) + suspended.add(ast) + try { + return f(ast.thunk()) + } finally { + suspended.delete(ast) + } +} + const columnOf = (ast: AST.AST, path: string): Column => { switch (ast._tag) { case "String": @@ -69,12 +82,8 @@ const columnOf = (ast: AST.AST, path: string): Column => { return type === undefined ? unsupported(ast, path) : leaf(type) } case "Enum": { - const kinds = new Set(ast.enums.map(([, v]) => typeof v)) - return kinds.size === 1 && kinds.has("string") - ? leaf("Utf8") - : kinds.size === 1 - ? leaf("Float64") - : unsupported(ast, path) + const types = new Set(ast.enums.map(([, v]) => literalType(v))) + return types.size === 1 ? leaf([...types][0]!) : unsupported(ast, path) } case "Union": { let column: Column | undefined @@ -88,17 +97,8 @@ const columnOf = (ast: AST.AST, path: string): Column => { } return { ...(column ?? leaf("Null")), nullable } } - case "Suspend": { - if (suspended.has(ast)) { - return V.fail(`field ${JSON.stringify(path)}: recursive schemas have no DataFrame column type`) - } - suspended.add(ast) - try { - return columnOf(ast.thunk(), path) - } finally { - suspended.delete(ast) - } - } + case "Suspend": + return unsuspend(ast, `field ${JSON.stringify(path)}: `, (ast) => columnOf(ast, path)) case "Arrays": { const element = ast.rest[0] if (ast.elements.length > 0 || element === undefined || ast.rest.length > 1) return unsupported(ast, path) @@ -124,10 +124,10 @@ const columnOf = (ast: AST.AST, path: string): Column => { const unsupported = (ast: AST.AST, path: string): never => V.fail(`field ${JSON.stringify(path)}: schemas of kind ${ast._tag} have no DataFrame column type yet`) -const fieldsOf = (ast: AST.Objects, prefix: string | undefined): Array => +const fieldsOf = (ast: AST.Objects, path: string): Array => ast.propertySignatures.map((ps) => { const name = String(ps.name) - const column = columnOf(ps.type, prefix === undefined ? name : `${prefix}.${name}`) + const column = columnOf(ps.type, field(path, name)) return { name, ...column, nullable: column.nullable || AST.isOptional(ps.type) } }) @@ -142,8 +142,8 @@ const structOf = (schema: { readonly ast: AST.AST }): AST.Objects => { // Declared columns // ----------------------------------------------------------------------------- -/** The plan type name of an Arrow type. @internal */ -export const typeName = (t: S.DataType): Plan.TypeName => { +/** The plan type name of an Arrow type. */ +const typeName = (t: S.DataType): Plan.TypeName => { switch (t._tag) { case "Int": case "Float": @@ -174,11 +174,8 @@ const describeColumn = (c: Column): string => c.type === "List" ? `List<${descri const utcZones: ReadonlySet = new Set(["UTC", "Etc/UTC", "+00:00"]) /** - * Checks that values of Arrow type `t` read as values of the column, not - * looking into lists and structs: integers below 64 bits for floats and - * narrower integers, 64-bit integers (bigints) for 64-bit ones, any decimal - * for a decimal, timestamps without a time zone or with a UTC one, and - * temporal types of the same kind in any unit. + * Fails unless values of Arrow type `t` read as values of the column (not + * looking into lists and structs). Timestamps need no time zone or a UTC one. */ const checkType = (path: string, t: S.DataType, expected: Column): void => { if (expected.type === "Timestamp" && t._tag === "Timestamp" && t.timezone !== undefined) { @@ -190,6 +187,11 @@ const checkType = (path: string, t: S.DataType, expected: Column): void => { } } +/** + * Integers below 64 bits read as floats and narrower integers, 64-bit + * integers (bigints) as 64-bit ones, any decimal as a decimal, and temporal + * types as the same kind in any unit. + */ const compatible = (t: S.DataType, expected: Column): boolean => { switch (expected.type) { case "Float16": @@ -211,30 +213,23 @@ const compatible = (t: S.DataType, expected: Column): boolean => { } } -const field = (path: string, name: string) => (path === "" ? name : `${path}.${name}`) - const missing = (path: string): never => V.fail(`missing column ${JSON.stringify(path)}`) /** The column declared for a field's `column` with Arrow type `t`, or its default type. */ -const declareColumn = (path: string, column: Column, t: S.DataType | undefined): Column => { +const declareColumn = (path: string, column: Plan.TableColumn, t: S.DataType | undefined): Plan.TableColumn => { if (t !== undefined) checkType(path, t, column) switch (column.type) { case "List": { - const [item] = column.fields! const child = t !== undefined && (t._tag === "List" || t._tag === "LargeList") ? t.child.type : undefined - return { ...column, fields: [{ ...declareColumn(`${path}[]`, item, child), name: "item" }] } + return { ...column, fields: [declareColumn(`${path}[]`, column.fields![0], child)] } } case "Struct": return { ...column, fields: column.fields!.map((f) => { - if (t === undefined || t._tag !== "Struct") { - return { ...declareColumn(field(path, f.name), f, undefined), name: f.name } - } - const g = t.fields.find((g) => g.name === f.name) - return g === undefined - ? missing(field(path, f.name)) - : { ...declareColumn(field(path, f.name), f, g.type), name: f.name } + const p = field(path, f.name) + const g = t?._tag === "Struct" ? (t.fields.find((g) => g.name === f.name) ?? missing(p)).type : undefined + return declareColumn(p, f, g) }) } case "Decimal": @@ -257,22 +252,7 @@ const declareColumn = (path: string, column: Column, t: S.DataType | undefined): export const declare = ( schema: { readonly ast: AST.AST }, overrides: { readonly [name: string]: S.DataType | undefined } | undefined -): Array => - fieldsOf(structOf(schema), undefined).map((c) => ({ ...declareColumn(c.name, c, overrides?.[c.name]), name: c.name })) - -const numeric: { readonly [name: string]: S.DataType } = { - Int8: S.int8, - Int16: S.int16, - Int32: S.int32, - Int64: S.int64, - UInt8: S.uint8, - UInt16: S.uint16, - UInt32: S.uint32, - UInt64: S.uint64, - Float16: S.float16, - Float32: S.float32, - Float64: S.float64 -} +): Array => fieldsOf(structOf(schema), "").map((c) => declareColumn(c.name, c, overrides?.[c.name])) /** * The Arrow type of a declared column. Timestamps and durations without a @@ -315,8 +295,11 @@ export const arrowType = (column: Plan.TableColumn, path: string = column.name): }: decimal columns need a precision and scale: pass options.types (for example ArrowSchema.decimal(18, 2))` ) : S.decimal(column.precision, column.scale) + case "Float16": + return S.float16 default: - return numeric[column.type] + // The other numeric types, named as cast targets. + return D.castTargets[column.type] } } @@ -335,8 +318,10 @@ export const toArrowSchema = ( // Checking frames // ----------------------------------------------------------------------------- +const inRows = (rows: Uint8Array | undefined, i: number): boolean => rows === undefined || rows[i] === 1 + const allNull = (v: V.Vector, rows: Uint8Array | undefined): boolean => { - for (let i = 0; i < v.length; i++) if ((rows === undefined || rows[i] === 1) && V.isValid(v, i)) return false + for (let i = 0; i < v.length; i++) if (inRows(rows, i) && V.isValid(v, i)) return false return true } @@ -351,7 +336,7 @@ const timestampRange: { readonly [U in S.TimeUnit]: readonly [bigint, bigint] } const checkRange = (path: string, v: V.I64, unit: S.TimeUnit, rows: Uint8Array | undefined): void => { const [lo, hi] = timestampRange[unit] for (let i = 0; i < v.length; i++) { - if ((rows === undefined || rows[i] === 1) && V.isValid(v, i) && (v.values[i] < lo || v.values[i] > hi)) { + if (inRows(rows, i) && V.isValid(v, i) && (v.values[i] < lo || v.values[i] > hi)) { V.fail(`column ${JSON.stringify(path)} has timestamps outside the range of DateTime`) } } @@ -382,7 +367,7 @@ const convert = (path: string, v: V.Vector, target: S.DataType, rows: Uint8Array const back = K.cast(converted, v.type) for (let i = 0; i < v.length; i++) { if ( - (rows === undefined || rows[i] === 1) && V.isValid(v, i) && + inRows(rows, i) && V.isValid(v, i) && (!V.isValid(converted, i) || !sameValue(v, back, i)) ) { return V.fail(`column ${JSON.stringify(path)} has values that ${D.describe(target)} can't hold exactly`) @@ -400,14 +385,14 @@ const check = (path: string, v: V.Vector, expected: Plan.TableColumn, rows: Uint checkType(path, v.type, expected) if (!expected.nullable) { for (let i = 0; i < v.length; i++) { - if ((rows === undefined || rows[i] === 1) && !V.isValid(v, i)) { + if (inRows(rows, i) && !V.isValid(v, i)) { return V.fail(`column ${JSON.stringify(path)} has missing values`) } } } if (v.kind === "struct" && v.type._tag === "Struct") { const present = new Uint8Array(v.length) - for (let i = 0; i < v.length; i++) present[i] = (rows === undefined || rows[i] === 1) && V.isValid(v, i) ? 1 : 0 + for (let i = 0; i < v.length; i++) present[i] = inRows(rows, i) && V.isValid(v, i) ? 1 : 0 const fields = v.type.fields const children = expected.fields!.map((f) => { const k = fields.findIndex((g) => g.name === f.name) @@ -416,10 +401,10 @@ const check = (path: string, v: V.Vector, expected: Plan.TableColumn, rows: Uint const type = S.struct(expected.fields!.map((f, k) => S.field(f.name, children[k].type, { nullable: f.nullable }))) return V.struct(type, children, v.length, v.valid) } - if (v.kind === "list" && (v.type._tag === "List" || v.type._tag === "LargeList")) { + if (v.kind === "list") { const present = new Uint8Array(v.child.length) for (let i = 0; i < v.length; i++) { - if ((rows === undefined || rows[i] === 1) && V.isValid(v, i)) present.fill(1, v.offsets[i], v.offsets[i + 1]) + if (inRows(rows, i) && V.isValid(v, i)) present.fill(1, v.offsets[i], v.offsets[i + 1]) } const item = expected.fields![0] const child = check(`${path}[]`, v.child, item, present) @@ -454,15 +439,8 @@ const structural = (ast: AST.AST): ((value: unknown) => boolean) | undefined => const is = Parser._is(AST.replaceChecks(ast, undefined)) return (value) => value === null || is(value) } - case "Suspend": { - if (suspended.has(ast)) return V.fail("recursive schemas have no DataFrame column type") - suspended.add(ast) - try { - return valuesCheck(ast.thunk()) - } finally { - suspended.delete(ast) - } - } + case "Suspend": + return unsuspend(ast, "", valuesCheck) case "Union": { const members = ast.types.filter((t) => t._tag !== "Null" && t._tag !== "Undefined").map(valuesCheck) if (members.some((m) => m === undefined)) return undefined @@ -489,10 +467,8 @@ const structural = (ast: AST.AST): ((value: unknown) => boolean) | undefined => /** * The fields of a struct schema whose values it constrains beyond their * column type, each with its test for those values. - * - * @internal */ -export const valueChecks = ( +const valueChecks = ( schema: { readonly ast: AST.AST } ): Array boolean]> => structOf(schema).propertySignatures.flatMap((ps) => { diff --git a/packages/effect/src/dataframe/internal/stream.ts b/packages/effect/src/dataframe/internal/stream.ts index 8e948018baf..0809cccf7f6 100644 --- a/packages/effect/src/dataframe/internal/stream.ts +++ b/packages/effect/src/dataframe/internal/stream.ts @@ -69,21 +69,9 @@ const undeclared = (node: Q.Node): ReadonlyArray => [ */ const unread = (node: Q.Node): Batches => { if (undeclared(node).length > 0) return Stream.empty - const empty = (n: Q.Node): Q.Node => { - switch (n._tag) { - case "Table": - return Q.values(F.empty(n.columns!.map((c) => ({ name: c.name, type: arrowType(c), nullable: c.nullable })))) - case "Values": - return n - case "Join": - return { ...n, left: empty(n.left), right: empty(n.right) } - case "Concat": - return { ...n, inputs: n.inputs.map(empty) } - default: - return { ...n, input: empty((n as { readonly input: Q.Node }).input) } as Q.Node - } - } - return Stream.fromEffect(Effect.flatMap(Q.attempt(() => empty(node)), Q.execute)) + const empty = (table: Plan.Table) => + Q.values(F.empty(table.columns!.map((c) => ({ name: c.name, type: arrowType(c), nullable: c.nullable })))) + return Stream.fromEffect(Effect.flatMap(Q.attempt(() => Q.mapTables(node, empty)), Q.execute)) } /** Fails for a node without batches, whose columns are unknown. */ From fff9a121cb62256b2ec8087f2e61c92056aab0d6 Mon Sep 17 00:00:00 2001 From: Effect Bot Date: Sun, 15 Nov 2026 19:11:26 +0000 Subject: [PATCH 58/88] Simplify the SQL compiler and shorten its comments --- packages/effect/src/dataframe/internal/sql.ts | 258 +++++++----------- 1 file changed, 106 insertions(+), 152 deletions(-) diff --git a/packages/effect/src/dataframe/internal/sql.ts b/packages/effect/src/dataframe/internal/sql.ts index 96e9a06b35b..4c6b9a20bd2 100644 --- a/packages/effect/src/dataframe/internal/sql.ts +++ b/packages/effect/src/dataframe/internal/sql.ts @@ -15,14 +15,9 @@ * renders every column as text (temporal values as integers in their unit), * and `decode` reads the rows back as columns of those types. * - * The names the compiler invents (helper columns like the row key, the - * sources of tables) are short and share one prefix, chosen per plan so - * that no name the plan supplies (its tables' and its columns') starts with - * it (see {@link inventor}). PostgreSQL keeps only the first 63 bytes of a - * name (see {@link identifier}), so the columns of each relation must - * differ within them (see {@link distinguishable}), and names alike within - * them are one table, or one column of it, to every declaration (see - * {@link source}). + * Invented names (the row key, table sources) share a per-plan prefix that + * no name of the plan starts with (see {@link inventor}). PostgreSQL keeps + * the first 63 bytes of a name (see {@link identifier}). * * @internal */ @@ -51,10 +46,9 @@ interface Compiler { readonly sources: ReadonlyMap } -/** @internal */ -export const quote = (name: string): string => `"${name.replaceAll(`"`, `""`)}"` +const quote = (name: string): string => `"${name.replaceAll(`"`, `""`)}"` -const list = (names: ReadonlyArray, prefix = "") => names.map((n) => `${prefix}${quote(n)}`).join(", ") +const list = (names: ReadonlyArray) => names.map(quote).join(", ") const unsupported = (what: string): never => V.fail(`${what} is not supported by the SQL engine`) @@ -65,13 +59,9 @@ const unsupported = (what: string): never => V.fail(`${what} is not supported by const sqlType = (t: S.DataType): string => { switch (t._tag) { case "Int": - return t.bitWidth <= 16 && t.signed ? "SMALLINT" : t.bitWidth <= 16 || (t.bitWidth === 32 && t.signed) - ? "INTEGER" - : t.bitWidth === 32 - ? "BIGINT" - : t.signed - ? "BIGINT" - : "NUMERIC(20, 0)" + if (t.bitWidth <= 16) return t.signed ? "SMALLINT" : "INTEGER" + if (t.bitWidth === 32) return t.signed ? "INTEGER" : "BIGINT" + return t.signed ? "BIGINT" : "NUMERIC(20, 0)" case "Float": return t.bitWidth === 32 ? "REAL" : "DOUBLE PRECISION" case "Utf8": @@ -228,7 +218,6 @@ const cast = (x: string, from: S.DataType, to: S.DataType): string => { if (from._tag === "Date" && to._tag === "Timestamp") { return fromInteger(`CAST(${toInteger(x, S.date("day"))} AS BIGINT) * ${BigInt(86400) * perSecond[to.unit]}`, to) } - return fromInteger(cast(toInteger(x, from), integerOf(from), integerOf(to)), to) } if (isTemporal(from)) return cast(toInteger(x, from), integerOf(from), to) if (isTemporal(to)) return fromInteger(cast(x, from, integerOf(to)), to) @@ -362,9 +351,9 @@ const aggregate = ( case "first": case "last": { if (over !== undefined) { - return `${node.fn === "first" ? "FIRST_VALUE" : "LAST_VALUE"}(${x}) OVER (${ - partitionBy(scope, " ") - }ORDER BY ${row} ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING)` + return `${node.fn === "first" ? "FIRST_VALUE" : "LAST_VALUE"}(${x}) ${ + window(scope, "ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING") + }` } return `(ARRAY_AGG(${x} ORDER BY ${row} ${node.fn === "first" ? "ASC" : "DESC"}))[1]` } @@ -383,6 +372,20 @@ const totalSeconds: Record = { totalMilliseconds: 0.001 } +const infix = { + add: "+", + multiply: "*", + eq: "=", + neq: "<>", + lt: "<", + lte: "<=", + gt: ">", + gte: ">=", + and: "AND", + or: "OR" +} as const +const unary = { abs: "ABS", length: "LENGTH", toLowerCase: "LOWER", toUpperCase: "UPPER" } as const + const call = (node: Plan.Call, scope: Scope): string => { const row = scope.compiler.row const types = node.args.map((x) => typeOf(x, scope)) @@ -396,12 +399,19 @@ const call = (node: Plan.Call, scope: Scope): string => { const intMod = () => `((${a[0]} % NULLIF(${a[1]}, 0)) + ${a[1]}) % NULLIF(${a[1]}, 0)` switch (node.fn) { case "add": - return `(${a[0]} + ${a[1]})` + case "multiply": + case "eq": + case "neq": + case "lt": + case "lte": + case "gt": + case "gte": + case "and": + case "or": + return `(${a[0]} ${infix[node.fn]} ${a[1]})` case "subtract": if (k[0] === "Date" && k[1] === "Date") return `((${a[0]} - ${a[1]}) * INTERVAL '1 day')` return `(${a[0]} - ${a[1]})` - case "multiply": - return `(${a[0]} * ${a[1]})` case "divide": return `(${float(a[0])} / ${a[1]})` case "floorDivide": @@ -413,23 +423,14 @@ const call = (node: Plan.Call, scope: Scope): string => { case "negate": return `(-(${a[0]}))` case "abs": - return `ABS(${a[0]})` + case "length": + case "toLowerCase": + case "toUpperCase": + return `${unary[node.fn]}(${a[0]})` case "sqrt": return `(CASE WHEN ${float(a[0])} < 0 THEN ${NAN} ELSE SQRT(${float(a[0])}) END)` case "round": - return round(a[0], typeOf(node.args[0], scope), typeOf(node, scope), o.decimals ?? 0) - case "eq": - return `(${a[0]} = ${a[1]})` - case "neq": - return `(${a[0]} <> ${a[1]})` - case "lt": - return `(${a[0]} < ${a[1]})` - case "lte": - return `(${a[0]} <= ${a[1]})` - case "gt": - return `(${a[0]} > ${a[1]})` - case "gte": - return `(${a[0]} >= ${a[1]})` + return round(a[0], types[0], typeOf(node, scope), o.decimals ?? 0) case "between": { const [x, lo] = sameUnit([a[0], a[1]], [types[0], types[1]]) const [y, hi] = sameUnit([a[0], a[2]], [types[0], types[2]]) @@ -441,10 +442,6 @@ const call = (node: Plan.Call, scope: Scope): string => { `(${a[0]} IS NULL AND NULL)` : `(${a[0]} IN (${values.map((v) => literal(v, undefined)).join(", ")}))` } - case "and": - return `(${a[0]} AND ${a[1]})` - case "or": - return `(${a[0]} OR ${a[1]})` case "not": return `(NOT ${a[0]})` case "isNull": @@ -459,13 +456,7 @@ const call = (node: Plan.Call, scope: Scope): string => { case "when": return `(CASE WHEN ${a[0]} THEN ${a[1]} ELSE ${a[2]} END)` case "cast": - return cast(a[0], typeOf(node.args[0], scope), typeOf(node, scope)) - case "length": - return `LENGTH(${a[0]})` - case "toLowerCase": - return `LOWER(${a[0]})` - case "toUpperCase": - return `UPPER(${a[0]})` + return cast(a[0], types[0], typeOf(node, scope)) case "includes": return `(STRPOS(${a[0]}, ${string(o.pattern ?? "")}) > 0)` case "startsWith": @@ -497,20 +488,20 @@ const call = (node: Plan.Call, scope: Scope): string => { return `(${a[0]} - LAG(${a[0]}, ${o.n ?? 1}) ${window(scope)})` case "rank": { const order = `${a[0]} ${o.descending ? "DESC" : "ASC"} NULLS LAST` - const part = partitionBy(scope, " ") + const over = `OVER (${partitionBy(scope, " ")}ORDER BY ${order})` const ties = `COUNT(*) OVER (PARTITION BY ${[...(scope.partition ?? []), a[0]].join(", ")})` const ranked = (() => { switch (o.method ?? "average") { case "min": - return `RANK() OVER (${part}ORDER BY ${order})` + return `RANK() ${over}` case "max": - return `(RANK() OVER (${part}ORDER BY ${order}) + ${ties} - 1)` + return `(RANK() ${over} + ${ties} - 1)` case "dense": - return `DENSE_RANK() OVER (${part}ORDER BY ${order})` + return `DENSE_RANK() ${over}` case "ordinal": - return `ROW_NUMBER() OVER (${part}ORDER BY ${order}, ${row})` + return `ROW_NUMBER() OVER (${partitionBy(scope, " ")}ORDER BY ${order}, ${row})` case "average": - return `(RANK() OVER (${part}ORDER BY ${order}) + (${ties} - 1) / 2.0)` + return `(RANK() ${over} + (${ties} - 1) / 2.0)` } })() return `(CASE WHEN ${a[0]} IS NULL THEN NULL ELSE ${ranked} END)` @@ -519,15 +510,14 @@ const call = (node: Plan.Call, scope: Scope): string => { case "rollingMean": case "rollingMin": case "rollingMax": { - const frame = `ROWS BETWEEN ${(o.window ?? 1) - 1} PRECEDING AND CURRENT ROW` + const w = window(scope, `ROWS BETWEEN ${(o.window ?? 1) - 1} PRECEDING AND CURRENT ROW`) const fn = { rollingSum: "SUM", rollingMean: "AVG", rollingMin: "MIN", rollingMax: "MAX" }[node.fn] - const enough = `COUNT(${a[0]}) ${window(scope, frame)} >= ${o.minPeriods ?? o.window ?? 1}` + const enough = `COUNT(${a[0]}) ${w} >= ${o.minPeriods ?? o.window ?? 1}` // Floats: a NaN in the window makes the minimum or maximum NaN. + const nan = `MAX(CASE WHEN ${isNaNSql(a[0])} THEN 1 ELSE 0 END) ${w} = 1` const value = k[0] === "Float" && (fn === "MIN" || fn === "MAX") - ? `CASE WHEN MAX(CASE WHEN ${isNaNSql(a[0])} THEN 1 ELSE 0 END) ${ - window(scope, frame) - } = 1 THEN ${NAN} ELSE ${fn}(${a[0]}) ${window(scope, frame)} END` - : `${fn}(${a[0]}) ${window(scope, frame)}` + ? `CASE WHEN ${nan} THEN ${NAN} ELSE ${fn}(${a[0]}) ${w} END` + : `${fn}(${a[0]}) ${w}` return `(CASE WHEN ${enough} THEN ${value} END)` } case "year": @@ -651,47 +641,29 @@ const conformed = (x: string, input: Plan.Node, node: Plan.Node, name: string, c return isTemporal(from) && isTemporal(to) && !D.sameType(from, to) ? cast(x, from, to) : x } -const scopeOf = (node: Plan.Node, compiler: Compiler): Scope => ({ - compiler, - frame: compiler.frames.get(node)!, - partition: undefined -}) - const values = (frame: F.Frame, c: Compiler): string => { + if (frame.length === 0) { + const nulls = frame.columns.map((col) => `CAST(NULL AS ${sqlType(col.type)}) AS ${quote(col.name)}`) + return `SELECT ${[...nulls, `CAST(0 AS BIGINT) AS ${c.row}`].join(", ")} WHERE FALSE` + } const columns = frame.columns.map((col) => { const type = col.type const vector = col.values() const raw = (vector as V.I32 | V.I64).values - const values = type._tag === "Date" || type._tag === "Timestamp" || type._tag === "Duration" || - type._tag === "Decimal" - ? undefined - : F.toArray(frame, col.name) + const integers = isTemporal(type) || type._tag === "Decimal" + const values = integers ? undefined : F.toArray(frame, col.name) // Temporal and decimal values are written from their integers, at full precision. const value = (i: number): string => { - switch (type._tag) { - case "Date": - return literal(type.unit === "day" ? raw[i] : floorDiv(raw[i] as bigint, BigInt(86_400_000)), { - _tag: "Date" - }) - case "Timestamp": - return literal(raw[i], { _tag: "Timestamp", unit: type.unit, timezone: type.timezone }) - case "Duration": - return literal(raw[i], { _tag: "Duration", unit: type.unit }) - case "Decimal": - return literal(raw[i], { _tag: "Decimal", precision: type.precision, scale: type.scale }) - default: { - const v = values![i] - return typeof v === "bigint" ? String(v) : literal(v, undefined) - } + if (type._tag === "Date" && type.unit !== "day") { + return literal(floorDiv(raw[i] as bigint, BigInt(86_400_000)), type) } + if (integers) return literal(raw[i], type as Plan.LiteralType) + const v = values![i] + return typeof v === "bigint" ? String(v) : literal(v, undefined) } const sql = sqlType(type) return { name: quote(col.name), cell: (i: number) => `CAST(${V.isValid(vector, i) ? value(i) : "NULL"} AS ${sql})` } }) - if (frame.length === 0) { - const nulls = frame.columns.map((col, i) => `CAST(NULL AS ${sqlType(col.type)}) AS ${columns[i].name}`) - return `SELECT ${[...nulls, `CAST(0 AS BIGINT) AS ${c.row}`].join(", ")} WHERE FALSE` - } const tuples = Array.from( { length: frame.length }, (_, r) => `(${[...columns.map((col) => col.cell(r)), String(r + 1)].join(", ")})` @@ -707,13 +679,14 @@ const relation = (node: Plan.Node, c: Compiler): string => { const row = c.row const columns = columnsOf(node, c) const keep = [...columns.map(quote), row].join(", ") - const from = (alias = "t") => `(${relation((node as { readonly input: Plan.Node }).input, c)}) AS ${alias}` - const scope = () => scopeOf((node as { readonly input: Plan.Node }).input, c) - /** The rows of `input` renumbered in `order`. */ - const renumbered = (input: string, order: string) => + const input = (node as { readonly input: Plan.Node }).input + const from = (alias = "t") => `(${relation(input, c)}) AS ${alias}` + const scope = (): Scope => ({ compiler: c, frame: c.frames.get(input)!, partition: undefined }) + // The rows of `query` renumbered in `order`. + const renumbered = (query: string, order: string) => `SELECT ${ [...columns.map(quote), `ROW_NUMBER() OVER (ORDER BY ${order}) AS ${row}`].join(", ") - } FROM (${input}) AS t` + } FROM (${query}) AS t` switch (node._tag) { case "Table": return `SELECT ${ @@ -888,20 +861,13 @@ const output = (x: string, type: S.DataType): string => { return unsupported(`results of type ${D.describe(type)}`) } -const number = (s: string): number => { - const n = Number(s) - if (!Number.isNaN(n)) return n - const lower = s.toLowerCase() - return lower.endsWith("inf") ? (lower.startsWith("-") ? -Infinity : Infinity) : NaN -} - /** Reads one non-null value of a result column of `type`. */ const reader = (type: S.DataType): (value: unknown) => unknown => { switch (type._tag) { case "Int": - return type.bitWidth === 64 ? (v) => BigInt(String(v)) : (v) => Number(v) + return type.bitWidth === 64 ? (v) => BigInt(String(v)) : Number case "Float": - return (v) => number(String(v)) + return Number case "Bool": return (v) => v === true || v === "true" || v === "t" case "Decimal": @@ -1114,15 +1080,12 @@ const typeChecks = (table: Plan.Table): Array => }) /** - * The checks of a table's rows, the ones the in-memory engine makes beyond - * column types: only nulls in `Null` columns, no nulls in other columns - * unless they are nullable, the conditions of their values (see - * {@link conditions}), and the values of the fields its schema constrains - * beyond their column type (literals, enums, refinements) accepted by the - * schema, tested on each such column's distinct values. Each value - * aggregates the rows of the table's source. + * The checks of a table's rows beyond column types: nulls, the + * {@link conditions} of values, and fields its schema constrains (literals, + * enums, refinements), tested on distinct values. Each check aggregates the + * rows of the table's source. */ -const rowChecks = (table: Plan.Table, frame: F.Frame): Array => [ +const rowChecks = (table: Plan.Table): Array => [ ...(table.columns ?? []).flatMap((column) => { const x = quote(column.name) const any = (failed: string, failure: string): Check => ({ @@ -1141,9 +1104,10 @@ const rowChecks = (table: Plan.Table, frame: F.Frame): Array => [ ] }), ...(table.schema === undefined ? [] : Schema_.columnChecks(table.schema)).map(([name, test]): Check => { - const type = frame.columns.find((c) => c.name === name)!.type + const column = table.columns!.find((c) => c.name === name)! + const type = Schema_.arrowType(column) return { - sql: `ARRAY_AGG(DISTINCT ${output(declared(table.columns!.find((c) => c.name === name)!), type)})`, + sql: `ARRAY_AGG(DISTINCT ${output(declared(column), type)})`, verify: (values) => { const rows = ((values ?? []) as ReadonlyArray).map((v) => [v]) if (!F.toArray(decode(S.make([S.field(name, type, { nullable: true })]), rows), name).every(test)) { @@ -1155,13 +1119,11 @@ const rowChecks = (table: Plan.Table, frame: F.Frame): Array => [ ] /** - * The source of the tables named `name` among `tables`, as PostgreSQL - * keeps their names (see {@link identifier}): the columns they declare, - * read once. It is `MATERIALIZED`, which PostgreSQL computes once per - * statement, so the checks and the result read the same rows, whatever the - * relation: views that sample rows or call functions included. Columns are - * named as PostgreSQL keeps them too, so declarations naming one column - * differently past its first 63 bytes all read it, each with its checks. + * The `MATERIALIZED` source of the tables PostgreSQL sees as table `name` + * (see {@link identifier}): their declared columns, read once per + * statement, so checks and result see the same rows. Columns are named as + * PostgreSQL keeps them, so declarations of one column that differ past its + * first 63 bytes all read it. */ const source = (name: string, tables: ReadonlyArray): string => { const columns = [ @@ -1171,19 +1133,12 @@ const source = (name: string, tables: ReadonlyArray): string => { } /** - * Quotes the names the compiler invents for the plan of `frames`, all with - * one prefix that no table or column name of the plan starts with, so that - * no invented name equals a name the plan supplies: a helper column can't - * hide a column, and a source can't shadow a table it reads (a common table - * expression shadows the table of its name in every `FROM` in its scope, - * the other sources' included; qualifying the tables' names with their - * schema is no way out, as the schema isn't known). - * - * The prefix is `__`, or else the first `__1_`, `__2_`, … that no name - * starts with: a name starts with at most one of those, so the prefix stays - * short, and so do the invented names. PostgreSQL keeps them whole (see - * {@link identifier}), and a name it truncates keeps its start, so they stay - * distinct from each other and from the plan's names. + * Quotes invented names with a prefix that no table or column name of the + * plan starts with (`__`, else the first free `__1_`, `__2_`, …), so a + * helper column can't hide a column and a source can't shadow a table it + * reads. Schemas are unknown, so tables can't be qualified instead. The + * prefix is short and truncation keeps a name's start, so invented names + * stay distinct after PostgreSQL truncates (see {@link identifier}). */ const inventor = (frames: ReadonlyMap): (name: string) => string => { const names = [...frames].flatMap(([node, frame]) => [ @@ -1265,9 +1220,7 @@ export const compile = (plan: Plan.Node): Compiled => { const result = typed(plan, frames) distinguishable(frames) const hidden = inventor(frames) - const tables = [...frames].flatMap(([node, frame]) => - node._tag === "Table" ? [{ node, checks: rowChecks(node, frame) }] : [] - ) + const tables = [...frames.keys()].flatMap((node) => node._tag === "Table" ? [{ node, checks: rowChecks(node) }] : []) const sources = new Map( [...new Set(tables.map((t) => identifier(t.node.name)))].map((name, i) => [name, hidden(`table${i}`)]) ) @@ -1275,33 +1228,34 @@ export const compile = (plan: Plan.Node): Compiled => { const sql = relation(plan, compiler) const schema = S.make(result.columns.map((col) => S.field(col.name, col.type, { nullable: true }))) const columns = result.columns.map((col) => `${output(quote(col.name), col.type)} AS ${quote(col.name)}`) - const ctes = [...sources].map(([name, cte]) => `${cte} AS ${source(name, tables.map((t) => t.node))}`) + const nodes = tables.map((t) => t.node) + const ctes = [...sources].map(([name, cte]) => `${cte} AS ${source(name, nodes)}`) const prelude = ctes.length === 0 ? "" : `WITH ${ctes.join(", ")} ` const body = `SELECT ${columns.length === 0 ? "1" : columns.join(", ")} FROM (${sql}) AS t ORDER BY ${compiler.row}` - const columnTypes = tables.flatMap((t) => typeChecks(t.node)) + const columnTypes = nodes.flatMap(typeChecks) const types = columnTypes.length === 0 ? undefined : { sql: `SELECT ${columnTypes.map((check) => check.sql).join(", ")}`, checks: columnTypes.map((check) => check.verify) } const checked = tables.filter((t) => t.checks.length > 0) - if (checked.length === 0) { - return { sql: prelude + body, types, query: { sql: prelude + body, checks: [] }, schema } - } - const names: Array = [] + const plain = prelude + body + if (checked.length === 0) return { sql: plain, types, query: { sql: plain, checks: [] }, schema } + // Check columns are read by position, so they need no names. const header = checked.map((t, i) => - `(SELECT ${ - t.checks.map((check) => { - names.push(hidden(`check${names.length}`)) - return `${check.sql} AS ${names[names.length - 1]}` - }).join(", ") - } FROM ${sources.get(identifier(t.node.name))}) AS ${hidden(`checks${i}`)}` + `(SELECT ${t.checks.map((check) => check.sql).join(", ")} FROM ${sources.get(identifier(t.node.name))}) AS ${ + hidden(`checks${i}`) + }` ) const rows = `(SELECT ${[...columns, compiler.row].join(", ")} FROM (${sql}) AS t) AS r` + const selected = [ + ...checked.map((_, i) => `${hidden(`checks${i}`)}.*`), + ...result.columns.map((col) => `r.${quote(col.name)}`) + ] return { - sql: prelude + body, + sql: plain, types, query: { - sql: `${prelude}SELECT ${[...names, ...result.columns.map((col) => `r.${quote(col.name)}`)].join(", ")} FROM ${ + sql: `${prelude}SELECT ${selected.join(", ")} FROM ${ header.join(" CROSS JOIN ") } FULL JOIN ${rows} ON FALSE ORDER BY r.${compiler.row} NULLS FIRST`, checks: checked.flatMap((t) => t.checks.map((check) => check.verify)) From 88fa4baa51419361cbcfd1f4ab08798300f7e3dd Mon Sep 17 00:00:00 2001 From: Effect Bot Date: Sun, 15 Nov 2026 19:11:26 +0000 Subject: [PATCH 59/88] Trim setup and share error helpers in the PGlite DataFrame tests --- packages/sql/pglite/test/DataFrameSql.test.ts | 68 ++++++++----------- 1 file changed, 29 insertions(+), 39 deletions(-) diff --git a/packages/sql/pglite/test/DataFrameSql.test.ts b/packages/sql/pglite/test/DataFrameSql.test.ts index f61bf86418a..fc2b479aed8 100644 --- a/packages/sql/pglite/test/DataFrameSql.test.ts +++ b/packages/sql/pglite/test/DataFrameSql.test.ts @@ -37,12 +37,6 @@ const regions = [ const Setup = Layer.effectDiscard( Effect.gen(function*() { const sql = yield* SqlClient - yield* sql`DROP TABLE IF EXISTS sales` - yield* sql`DROP TABLE IF EXISTS regions` - yield* sql`DROP TABLE IF EXISTS contract` - yield* sql`DROP TABLE IF EXISTS second` - yield* sql`DROP TABLE IF EXISTS "__table0"` - yield* sql`DROP TABLE IF EXISTS "__table1"` yield* sql`CREATE TABLE sales (id DOUBLE PRECISION, region TEXT, units DOUBLE PRECISION, price DOUBLE PRECISION, at TIMESTAMP)` yield* sql`CREATE TABLE regions (region TEXT, manager TEXT)` for (const s of sales) { @@ -61,36 +55,26 @@ const Setup = Layer.effectDiscard( yield* sql`CREATE TABLE "__table1" (s TEXT)` yield* sql`INSERT INTO "__table1" VALUES ('bad')` for (const name of [long, underscores]) { - yield* sql.unsafe(`DROP TABLE IF EXISTS "${name}"`) yield* sql.unsafe(`CREATE TABLE "${name}" (s TEXT)`) yield* sql.unsafe(`INSERT INTO "${name}" VALUES ('${name.length}')`) } - yield* sql.unsafe(`DROP TABLE IF EXISTS shared`) yield* sql.unsafe(`CREATE TABLE shared ("${"z".repeat(63)}" DOUBLE PRECISION)`) yield* sql`INSERT INTO shared VALUES (1)` - yield* sql`DROP TABLE IF EXISTS "a.b"` yield* sql`CREATE TABLE "a.b" (s TEXT)` yield* sql`INSERT INTO "a.b" VALUES ('good')` - yield* sql`CREATE SCHEMA IF NOT EXISTS a` - yield* sql`DROP TABLE IF EXISTS a.b` + yield* sql`CREATE SCHEMA a` yield* sql`CREATE TABLE a.b (s TEXT)` yield* sql`INSERT INTO a.b VALUES ('bad')` - yield* sql`DROP TABLE IF EXISTS odd` yield* sql`CREATE TABLE odd (nil INTEGER, nothing INTEGER, at TIMESTAMPTZ, far TIMESTAMP, months INTERVAL, forever TIMESTAMP, endless DATE)` yield* sql`INSERT INTO odd VALUES (42, NULL, '2024-01-01 00:30:00+00', '290000-01-01', '1 month', 'infinity', 'infinity')` - yield* sql`DROP TABLE IF EXISTS declared` yield* sql`CREATE TABLE declared (t TIMESTAMP, d INTERVAL, n NUMERIC)` yield* sql`INSERT INTO declared VALUES ('1970-01-01 00:00:01', '-2 seconds', 12.5), ('1969-12-31 23:59:58', '3 seconds', -0.05), (NULL, NULL, NULL)` - yield* sql`DROP TABLE IF EXISTS inexact` yield* sql`CREATE TABLE inexact (s TIMESTAMP, ms TIMESTAMP, ns TIMESTAMP, ds INTERVAL, dms INTERVAL, dns INTERVAL, scale NUMERIC, digits NUMERIC, wide NUMERIC, i8 INTEGER, u32 INTEGER, u64 BIGINT, f32 DOUBLE PRECISION, f32i INTEGER)` yield* sql`INSERT INTO inexact VALUES ('1970-01-01 00:00:01.5', '1970-01-01 00:00:01.0005', '2263-01-01', '-1.5 seconds', '0.0005 seconds', '110000 days', 1.005, 1000, 1, 256, -1, -1, 0.1, 16777217)` - yield* sql`DROP TABLE IF EXISTS exact` yield* sql`CREATE TABLE exact (i8 INTEGER, f32 DOUBLE PRECISION)` yield* sql`INSERT INTO exact VALUES (-128, 0.5), (127, 'NaN'), (0, '-Infinity'), (1, ${ 2 ** -149 }), (2, '-0'), (3, ${(2 - 2 ** -23) * 2 ** 127})` - yield* sql`DROP TABLE IF EXISTS texts` - yield* sql`DROP TYPE IF EXISTS mood` yield* sql`CREATE TYPE mood AS ENUM ('zzz', 'aaa')` yield* sql`CREATE TABLE texts (m mood, c CHAR(3), v VARCHAR(5), sm SMALLINT, r REAL)` yield* sql`INSERT INTO texts VALUES ('zzz', 'ab', 'xy', 20000, 16777216), ('aaa', 'abc', 'x', -20000, 0.5)` @@ -159,6 +143,11 @@ const types = (df: DataFrame.DataFrame) => const onSql = (query: Query.Query) => query.pipe(Effect.provide(Sql.layer)) const rowsOnSql = (query: Query.Query) => Effect.map(onSql(query), DataFrame.toRows) +/** The message of the error `query` fails with on SQL. */ +const rejects = (query: Query.Query) => Effect.map(Effect.flip(onSql(query)), (e) => e.message) +/** Queries of the table `name` declared with a schema. */ +const tableOf = (name: string) => (schema: S) => + Query.table(name, schema) const xs = Query.from(DataFrame.fromColumns({ x: [1, 2, 3] })) const ints = Query.from(DataFrame.fromColumns({ x: Int32Array.of(1, 2, 3) })) @@ -174,7 +163,7 @@ const ties = [0.5, 1.5, 2.5, -0.5, -1.5, -2.5, 2.675, 1.005, 0.125, 0.375, 1.25, const decimals = arrow<{ d: BigDecimal.BigDecimal | null }>( [ArrowSchema.field("d", ArrowSchema.decimal(18, 3))], [["0.5", "1.5", "2.5", "-0.5", "-1.5", "-2.5", "2.675", "1.005", "0.125", null].map((s) => - s === null ? null : BigDecimal.fromStringUnsafe(s) + s === null ? null : decimal(s) )] ) const units = { s: "second", ms: "millisecond", us: "microsecond", ns: "nanosecond" } as const @@ -333,7 +322,7 @@ const cases: ReadonlyArray]> = [ [1500n, -90_000_001n, null], [1_700_000_000_123_456n, -1n, null], [19_000, -1, null], - [BigDecimal.fromStringUnsafe("12.34"), BigDecimal.fromStringUnsafe("-0.05"), null] + [decimal("12.34"), decimal("-0.05"), null] ]).pipe(Query.withColumns((c) => ({ s: Expr.totalSeconds(c.d), year: Expr.year(c.t) }))) ], ...temporalCasts, @@ -526,15 +515,16 @@ describe("effect/dataframe on PostgreSQL (PGlite)", { concurrent: false }, () => it.effect("errors carry the database's message", () => Effect.gen(function*() { - const error = yield* Effect.flip(onSql(ints.pipe(Query.select((c) => ({ y: Expr.add(c.x, 2147483647) }))))) - assert.include(error.message, "integer out of range") + assert.include( + yield* rejects(ints.pipe(Query.select((c) => ({ y: Expr.add(c.x, 2147483647) })))), + "integer out of range" + ) })) it.effect("tables must satisfy their schema, as in memory", () => Effect.gen(function*() { - const contract = (schema: S) => - Query.table("contract", schema) - const rejects = (query: Query.Query) => Effect.map(Effect.flip(onSql(query)), (e) => e.message) + const contract = tableOf("contract") + const odd = tableOf("odd") assert.strictEqual( yield* rejects(contract(Schema.Struct({ s: Schema.Literal("good") }))), `column "s" has values its schema rejects` @@ -556,7 +546,6 @@ describe("effect/dataframe on PostgreSQL (PGlite)", { concurrent: false }, () => yield* rowsOnSql(contract(Schema.Struct({ s: Schema.Literals(["bad", "good"]) }))), [{ s: "bad" }, { s: "good" }] ) - const odd = (schema: S) => Query.table("odd", schema) for ( const [query, message] of [ [odd(Schema.Struct({ nil: Schema.Null })), `column "nil" has values, expected Null`], @@ -596,8 +585,7 @@ describe("effect/dataframe on PostgreSQL (PGlite)", { concurrent: false }, () => assert.deepStrictEqual(yield* rowsOnSql(odd(Schema.Struct({ nothing: Schema.Null }))), [{ nothing: null }]) // A declared column the table doesn't have, named like a column of PostgreSQL's catalog. for (const schema of [Schema.Struct({ oid: Schema.BigInt }), Schema.Struct({ typname: Schema.String })]) { - const missing = yield* Effect.flip(onSql(Query.table("contract", schema))) - assert.include(missing.message, `.${Object.keys(schema.fields)[0]} does not exist`) + assert.include(yield* rejects(contract(schema)), `.${Object.keys(schema.fields)[0]} does not exist`) } })) @@ -625,8 +613,7 @@ describe("effect/dataframe on PostgreSQL (PGlite)", { concurrent: false }, () => // A count reads none of the table's columns. for (const query of [table, table.pipe(Query.aggregate(() => ({ n: Expr.count() })))]) { const expected = yield* Effect.flip(query.pipe(Effect.provide(memory))) - const actual = yield* Effect.flip(onSql(query)) - assert.strictEqual(actual.message, expected.message) + assert.strictEqual(yield* rejects(query), expected.message) } } })) @@ -672,12 +659,14 @@ describe("effect/dataframe on PostgreSQL (PGlite)", { concurrent: false }, () => yield* rowsOnSql(Query.concat([s("contract"), s("__table0"), s("second"), s("__table1")])), [{ s: "bad" }, { s: "good" }, { s: "bad" }, { s: "good" }, { s: "bad" }] ) - const error = yield* Effect.flip(onSql(Query.concat([ - s("contract").pipe(Query.filter((c) => Expr.eq(c.s, "good"))), - s("second"), - Query.table("__table1", Schema.Struct({ s: Schema.Literal("good") })) - ]))) - assert.strictEqual(error.message, `column "s" has values its schema rejects`) + assert.strictEqual( + yield* rejects(Query.concat([ + s("contract").pipe(Query.filter((c) => Expr.eq(c.s, "good"))), + s("second"), + Query.table("__table1", Schema.Struct({ s: Schema.Literal("good") })) + ])), + `column "s" has values its schema rejects` + ) assert.deepStrictEqual(yield* rowsOnSql(s("a.b")), [{ s: "good" }]) assert.deepStrictEqual(yield* rowsOnSql(s(long)), [{ s: "63" }]) const two = Query.concat([s(underscores), s("second")]) @@ -691,16 +680,17 @@ describe("effect/dataframe on PostgreSQL (PGlite)", { concurrent: false }, () => Query.table("shared", Schema.Struct({ [name]: schema })).pipe(Query.select((c) => ({ x: c[name] }))) const [za, zb] = ["z".repeat(63) + "a", "z".repeat(63) + "b"] assert.deepStrictEqual(yield* rowsOnSql(Query.concat([shared(za), shared(zb)])), [{ x: 1 }, { x: 1 }]) - const error = yield* Effect.flip(onSql(Query.concat([shared(za), shared(zb, Schema.Literal(2))]))) - assert.strictEqual(error.message, `column ${JSON.stringify(zb)} has values its schema rejects`) + assert.strictEqual( + yield* rejects(Query.concat([shared(za), shared(zb, Schema.Literal(2))])), + `column ${JSON.stringify(zb)} has values its schema rejects` + ) // Columns of one relation are rejected: PostgreSQL keeps the whole characters within an identifier's first 63 bytes. const pairs = [["x".repeat(63) + "a", "x".repeat(63) + "b"], ["x".repeat(62) + "é", "x".repeat(62) + "ж"]] for (const [a, b] of pairs) { const added = Query.from(DataFrame.fromColumns({ [a]: [1] })).pipe( Query.withColumns(() => ({ [b]: Expr.lit(2) })) ) - const error = yield* Effect.flip(onSql(added)) - assert.include(error.message, `columns ${JSON.stringify(a)} and ${JSON.stringify(b)} are one name`) + assert.include(yield* rejects(added), `columns ${JSON.stringify(a)} and ${JSON.stringify(b)} are one name`) } })) From ace6f35cc470f3c2c5e1387380e55352ee37015c Mon Sep 17 00:00:00 2001 From: Effect Bot Date: Sun, 15 Nov 2026 19:26:05 +0000 Subject: [PATCH 60/88] Simplify the dataframe tests, oracle, gold generator and benchmark --- packages/effect/benchmark/dataframe/h2o.py | 27 +- packages/effect/benchmark/dataframe/h2o.ts | 27 +- .../effect/test/dataframe/DataFrame.test.ts | 86 +++-- packages/effect/test/dataframe/Query.test.ts | 298 +++++++++--------- .../effect/test/dataframe/oracle/Compare.ts | 26 +- packages/effect/test/dataframe/oracle/Gold.ts | 27 +- packages/effect/test/dataframe/oracle/Plan.ts | 150 +++------ .../test/fixtures/dataframe/generate_gold.py | 55 ++-- 8 files changed, 292 insertions(+), 404 deletions(-) diff --git a/packages/effect/benchmark/dataframe/h2o.py b/packages/effect/benchmark/dataframe/h2o.py index 7a697343ca0..77da7859445 100644 --- a/packages/effect/benchmark/dataframe/h2o.py +++ b/packages/effect/benchmark/dataframe/h2o.py @@ -26,12 +26,13 @@ HERE = Path(__file__).parent DEFAULT_DATA = HERE / ".data" +# Table keys of the queries, and their file names (as -.arrow). +TABLES = {"g": "groupby", "x": "join-x", "small": "join-small", "medium": "join-medium", "big": "join-big"} def generate(rows, data): import numpy as np import pyarrow as pa - import pyarrow.ipc as ipc rng = np.random.default_rng(20261114) n = rows @@ -138,8 +139,6 @@ def pandas_queries(pd): def checksum_polars(df): import polars as pl - if not isinstance(df, pl.DataFrame): - return {"rows": 1, "sum": float(df)} total = 0.0 for name, dtype in df.schema.items(): if dtype.is_numeric(): @@ -148,15 +147,8 @@ def checksum_polars(df): return {"rows": df.height, "sum": total} -def load_polars(pl, data, rows): - read = lambda name: pl.read_ipc(data / name) - return { - "g": read(f"groupby-{rows}.arrow"), - "x": read(f"join-x-{rows}.arrow"), - "small": read(f"join-small-{rows}.arrow"), - "medium": read(f"join-medium-{rows}.arrow"), - "big": read(f"join-big-{rows}.arrow"), - } +def load(read, data, rows): + return {key: read(data / f"{name}-{rows}.arrow") for key, name in TABLES.items()} def time_best(f, reps): @@ -172,7 +164,7 @@ def time_best(f, reps): def run_polars(data, rows, reps): import polars as pl - tables = load_polars(pl, data, rows) + tables = load(pl.read_ipc, data, rows) out = {"version": pl.__version__, "threads": pl.thread_pool_size(), "queries": {}} for name, q in polars_queries(pl).items(): ms, result = time_best(lambda: q(tables), reps) @@ -184,14 +176,7 @@ def run_pandas(data, rows, reps): import pandas as pd import pyarrow.ipc as ipc - read = lambda name: ipc.open_file(str(data / name)).read_all().to_pandas() - tables = { - "g": read(f"groupby-{rows}.arrow"), - "x": read(f"join-x-{rows}.arrow"), - "small": read(f"join-small-{rows}.arrow"), - "medium": read(f"join-medium-{rows}.arrow"), - "big": read(f"join-big-{rows}.arrow"), - } + tables = load(lambda path: ipc.open_file(str(path)).read_all().to_pandas(), data, rows) out = {"version": pd.__version__, "queries": {}} for name, q in pandas_queries(pd).items(): ms, _ = time_best(lambda: q(tables), reps) diff --git a/packages/effect/benchmark/dataframe/h2o.ts b/packages/effect/benchmark/dataframe/h2o.ts index 5a400ce23eb..bdd0ace4cff 100644 --- a/packages/effect/benchmark/dataframe/h2o.ts +++ b/packages/effect/benchmark/dataframe/h2o.ts @@ -1,19 +1,17 @@ /** - * effect/dataframe performance against Polars and pandas: the performance - * half of the dataframe oracle. + * effect/dataframe against Polars and pandas on H2O db-benchmark style queries. * * python benchmark/dataframe/h2o.py generate --rows 1000000 * python benchmark/dataframe/h2o.py run --rows 1000000 - * node benchmark/dataframe/h2o.ts [--rows=1000000] [--reps=3] [--only=] [--out=] + * node [--expose-gc] benchmark/dataframe/h2o.ts [--rows=1000000] [--reps=3] [--only=] + * [--out=] [--data=] [--gc] * - * Every query reads the same Arrow IPC files as the Python baseline, and runs - * as a lazy query (so the time includes optimising the plan). Before - * its time counts, each result's checksum (row count and the sum of its - * numeric columns) has to match the Polars result, so a fast wrong answer - * can't win. Times are the best of `--reps` runs, in milliseconds. - `node --expose-gc h2o.ts --gc` - * collects garbage before every timed run, so earlier queries' garbage - * doesn't count against later ones. + * Every query reads the same Arrow IPC files as the Python baseline and runs + * as a lazy query, so the time includes optimising the plan. A result's + * checksum (row count and the sum of its numeric columns) must match Polars + * before its time counts. Times are the best of `--reps` runs, in + * milliseconds. `--gc` (with `node --expose-gc`) collects garbage before + * every timed run. */ import { Effect } from "effect" import { ArrowIpc } from "effect/arrow" @@ -28,7 +26,6 @@ const reps = Number(arg("reps") ?? 3) const only = arg("only") const out = arg("out") const data = arg("data") ?? Path.join(import.meta.dirname, ".data") -/** With `node --expose-gc ... --gc`, collect garbage before every timed run. */ const gc = process.argv.includes("--gc") ? (globalThis as { gc?: () => void }).gc : undefined const load = (name: string) => { @@ -115,9 +112,6 @@ const checksum = (df: DataFrame.DataFrame) => { return { rows: df.length, sum } } -/** Forces every column of a result without converting values to JS. */ -const force = (df: DataFrame.DataFrame) => DataFrame.toArrowTable(df) - const baselinePath = Path.join(data, `baseline-${rows}.json`) const baseline = Fs.existsSync(baselinePath) ? JSON.parse(Fs.readFileSync(baselinePath, "utf8")) : undefined if (baseline === undefined) console.warn(`no baseline at ${baselinePath}: run h2o.py run --rows ${rows}`) @@ -149,7 +143,8 @@ for (const [name, query] of Object.entries(queries)) { gc?.() const start = performance.now() result = Effect.runSync(query()) - force(result) + // Forces every column of the result without converting values to JS. + DataFrame.toArrowTable(result) const ms = performance.now() - start if (r === 0) first = ms best = Math.min(best, ms) diff --git a/packages/effect/test/dataframe/DataFrame.test.ts b/packages/effect/test/dataframe/DataFrame.test.ts index e80b7e7fef0..e6ad911fb63 100644 --- a/packages/effect/test/dataframe/DataFrame.test.ts +++ b/packages/effect/test/dataframe/DataFrame.test.ts @@ -69,13 +69,7 @@ describe("DataFrame", () => { } const Tree: Schema.Codec = Schema.Struct({ children: Schema.Array(Schema.suspend(() => Tree)) }) for (const xs of [Schema.Record(Schema.String, Schema.Number), Tree]) { - try { - DataFrame.fromRows(Schema.Struct({ xs }), []) - assert.fail("expected an error") - } catch (e) { - assert.instanceOf(e, DataFrame.DataFrameError) - assert.include((e as DataFrame.DataFrameError).message, `"xs`) - } + assert.throws(() => DataFrame.fromRows(Schema.Struct({ xs }), []), DataFrame.DataFrameError, `"xs`) } }) @@ -98,12 +92,12 @@ describe("DataFrame", () => { describe("validate", () => { const untyped = DataFrame.fromArrowTable(DataFrame.toArrowTable(readings)) - const column = (type: ArrowSchema.DataType, values: ReadonlyArray) => - DataFrame.fromArrowTable( - Result.getOrThrow( - ArrowTable.fromRows(ArrowSchema.make([ArrowSchema.field("t", type)]), values.map((t) => ({ t }))) - ) - ) + const arrowFrame = (columns: Record]>) => + DataFrame.fromArrowTable(Result.getOrThrow(ArrowTable.fromColumns( + ArrowSchema.make(Object.entries(columns).map(([name, [type]]) => ArrowSchema.field(name, type))), + Object.values(columns).map(([, values]) => values) + ))) + const column = (type: ArrowSchema.DataType, values: ReadonlyArray) => arrowFrame({ t: [type, values] }) const Utc = Schema.Struct({ t: Schema.DateTimeUtc }) it("returns the frame typed by the schema, in schema order", () => { @@ -191,7 +185,6 @@ describe("DataFrame", () => { ] for (const [df, schema, message] of cases) { const result = DataFrame.validate(df, schema) - assert.isTrue(Result.isFailure(result)) assert.strictEqual(Result.isFailure(result) && result.failure.message, message) } }) @@ -210,14 +203,11 @@ describe("DataFrame", () => { it("converts columns to the types fromRows gives the schema, exactly", () => { const Row = Schema.Struct({ n: Schema.Number, d: Schema.BigDecimal, t: Schema.DateTimeUtc }) - const df = DataFrame.fromArrowTable(Result.getOrThrow(ArrowTable.fromRows( - ArrowSchema.make([ - ArrowSchema.field("n", ArrowSchema.int32), - ArrowSchema.field("d", ArrowSchema.decimal(18, 1)), - ArrowSchema.field("t", ArrowSchema.timestamp("millisecond")) - ]), - [{ n: 1, d: BigDecimal.fromStringUnsafe("1.2"), t: 1n }] - ))) + const df = arrowFrame({ + n: [ArrowSchema.int32, [1]], + d: [ArrowSchema.decimal(18, 1), [BigDecimal.fromStringUnsafe("1.2")]], + t: [ArrowSchema.timestamp("millisecond"), [1n]] + }) const types = { d: ArrowSchema.decimal(18, 2) } const validated = Result.getOrThrow(DataFrame.validate(df, Row, { types })) assert.deepStrictEqual(validated.schema, DataFrame.empty(Row, { types }).schema) @@ -255,16 +245,16 @@ describe("DataFrame", () => { for (const frame of frames) assert.isTrue(Equal.equals(frame, readings)) })) - it("exports late-selected columns", () => { - const df = Query.from(readings).pipe( - Query.filter((c) => Expr.isNotNull(c.rain)), - Query.sort((c) => [c.temp]), - Effect.runSync - ) - const table = DataFrame.toArrowTable(df) - assert.strictEqual(table.batches[0].length, 2) - assert.deepStrictEqual(DataFrame.toRows(DataFrame.fromArrowTable(table)), DataFrame.toRows(df)) - }) + it.effect("exports late-selected columns", () => + Effect.gen(function*() { + const df = yield* Query.from(readings).pipe( + Query.filter((c) => Expr.isNotNull(c.rain)), + Query.sort((c) => [c.temp]) + ) + const table = DataFrame.toArrowTable(df) + assert.strictEqual(table.batches[0].length, 2) + assert.deepStrictEqual(DataFrame.toRows(DataFrame.fromArrowTable(table)), DataFrame.toRows(df)) + })) it("imports a table without batches with the nullability of its fields", () => { const schema = ArrowSchema.make([ @@ -277,22 +267,22 @@ describe("DataFrame", () => { ]) }) - it("imports sliced list columns with only their own elements", () => { - const Lists = Schema.Struct({ xs: Schema.Array(Schema.Number) }) - const df = DataFrame.fromRows(Lists, [{ xs: [1] }, { xs: [2] }, { xs: [3] }]) - const table = DataFrame.toArrowTable(df) - const sliced = DataFrame.fromArrowTable(ArrowTable.slice(table, 0, 1)) - const sums = Query.from(sliced).pipe( - Query.select((c) => ({ s: Expr.listSum(c.xs as Expr.Expr>) })), - Effect.runSync - ) - assert.deepStrictEqual(DataFrame.toRows(sums), [{ s: 1 }]) - const two = ArrowTable.make(table.schema, [ - ...ArrowTable.slice(table, 0, 1).batches, - ...ArrowTable.slice(table, 2, 3).batches - ]) - assert.deepStrictEqual(DataFrame.toRows(DataFrame.fromArrowTable(two)), [{ xs: [1] }, { xs: [3] }]) - }) + it.effect("imports sliced list columns with only their own elements", () => + Effect.gen(function*() { + const Lists = Schema.Struct({ xs: Schema.Array(Schema.Number) }) + const df = DataFrame.fromRows(Lists, [{ xs: [1] }, { xs: [2] }, { xs: [3] }]) + const table = DataFrame.toArrowTable(df) + const sliced = DataFrame.fromArrowTable(ArrowTable.slice(table, 0, 1)) + const sums = yield* Query.from(sliced).pipe( + Query.select((c) => ({ s: Expr.listSum(c.xs as Expr.Expr>) })) + ) + assert.deepStrictEqual(DataFrame.toRows(sums), [{ s: 1 }]) + const two = ArrowTable.make(table.schema, [ + ...ArrowTable.slice(table, 0, 1).batches, + ...ArrowTable.slice(table, 2, 3).batches + ]) + assert.deepStrictEqual(DataFrame.toRows(DataFrame.fromArrowTable(two)), [{ xs: [1] }, { xs: [3] }]) + })) }) describe("values", () => { diff --git a/packages/effect/test/dataframe/Query.test.ts b/packages/effect/test/dataframe/Query.test.ts index 5be9bb1f3ad..f8241828a39 100644 --- a/packages/effect/test/dataframe/Query.test.ts +++ b/packages/effect/test/dataframe/Query.test.ts @@ -1,5 +1,5 @@ import { assert, describe, it } from "@effect/vitest" -import { BigDecimal, type DateTime, type Duration, Effect, Exit, Result, Schema, Stream } from "effect" +import { BigDecimal, type DateTime, type Duration, Effect, Result, Schema, Stream } from "effect" import { ArrowSchema, ArrowTable } from "effect/arrow" import { Catalog, DataFrame, Expr, Plan, Query } from "effect/dataframe" import * as Compare from "./oracle/Compare.ts" @@ -28,6 +28,19 @@ const tables = Catalog.layer({ readings, cities }) const rows = (query: Query.Query) => Effect.map(query, DataFrame.toRows) +/** A frame of Arrow-typed columns. */ +const arrowFrame = (columns: Record]>) => + DataFrame.fromArrowTable(Result.getOrThrow(ArrowTable.fromColumns( + ArrowSchema.make(Object.entries(columns).map(([name, [type]]) => ArrowSchema.field(name, type))), + Object.values(columns).map(([, values]) => values) + ))) as DataFrame.DataFrame + +const Price = Schema.Struct({ d: Schema.BigDecimal }) +const price = (d: string, precision: number, scale: number) => + DataFrame.fromRows(Price, [{ d: BigDecimal.fromStringUnsafe(d) }], { + types: { d: ArrowSchema.decimal(precision, scale) } + }) + describe("Query", () => { describe("running", () => { it.effect("is an Effect producing a frame", () => @@ -76,10 +89,13 @@ describe("Query", () => { const Range = Schema.Struct({ lo: Schema.Number, hi: Schema.Number }).check( Schema.makeFilter((row) => row.lo < row.hi || "lo must be below hi") ) + const Utc = Schema.Struct({ t: Schema.DateTimeUtc }) + const berlin = arrowFrame({ t: [ArrowSchema.timestamp("millisecond", "Europe/Berlin"), [0n]] }) for ( const [schema, frame, message] of [ [Kind, DataFrame.fromColumns({ kind: ["a", "c"] }), `column "kind" has values its schema rejects`], - [Range, DataFrame.fromColumns({ lo: [1, 2], hi: [2, 1] }), "rows have values their schema rejects"] + [Range, DataFrame.fromColumns({ lo: [1, 2], hi: [2, 1] }), "rows have values their schema rejects"], + [Utc, berlin, `column "t" has time zone Europe/Berlin, expected UTC timestamps`] ] as const ) { for (const t of [frame, Stream.make(frame)]) { @@ -91,15 +107,6 @@ describe("Query", () => { Effect.provide(Catalog.layer({ t: DataFrame.fromColumns({ lo: [1], hi: [2] }) })) ) assert.deepStrictEqual(DataFrame.toRows(yield* ranges), [{ lo: 1, hi: 2 }]) - const berlin = DataFrame.fromArrowTable(Result.getOrThrow(ArrowTable.fromRows( - ArrowSchema.make([ArrowSchema.field("t", ArrowSchema.timestamp("millisecond", "Europe/Berlin"))]), - [{ t: 0n }] - ))) - const Utc = Schema.Struct({ t: Schema.DateTimeUtc }) - for (const t of [berlin, Stream.make(berlin)]) { - const error = yield* Effect.flip(Query.table("t", Utc).pipe(Effect.provide(Catalog.layer({ t })))) - assert.strictEqual(error.message, `column "t" has time zone Europe/Berlin, expected UTC timestamps`) - } })) it.effect("narrows tables to their declared columns", () => @@ -111,10 +118,7 @@ describe("Query", () => { it.effect("fails with DataFrameError on invalid expressions", () => Effect.gen(function*() { - const query = Query.from(readings).pipe(Query.filter((c) => c.city as any)) - const exit = yield* Effect.exit(query) - assert.isTrue(Exit.isFailure(exit)) - const error = yield* Effect.flip(query) + const error = yield* Effect.flip(Query.from(readings).pipe(Query.filter((c) => c.city as any))) assert.instanceOf(error, DataFrame.DataFrameError) assert.strictEqual(error.message, "filter predicate must be boolean, got Utf8") })) @@ -209,14 +213,11 @@ describe("Query", () => { ) assert.deepStrictEqual(casts, [{ fromFloat: null, fromUInt64: null, toUInt64: null }]) // Expected values from Polars 2.1.0 non-strict casts. - const temporal = DataFrame.fromArrowTable(Result.getOrThrow(ArrowTable.fromColumns( - ArrowSchema.make([ - ArrowSchema.field("t", ArrowSchema.timestamp("millisecond")), - ArrowSchema.field("d", ArrowSchema.duration("millisecond")), - ArrowSchema.field("i", ArrowSchema.int64) - ]), - [[10n ** 17n], [10n ** 17n], [2n ** 40n]] - ))) as DataFrame.DataFrame as DataFrame.DataFrame<{ t: DateTime.Utc; d: Duration.Duration; i: bigint }> + const temporal = arrowFrame<{ t: DateTime.Utc; d: Duration.Duration; i: bigint }>({ + t: [ArrowSchema.timestamp("millisecond"), [10n ** 17n]], + d: [ArrowSchema.duration("millisecond"), [10n ** 17n]], + i: [ArrowSchema.int64, [2n ** 40n]] + }) const overflows = yield* rows( Query.from(temporal).pipe(Query.select((c) => ({ t: Expr.cast(c.t, "Timestamp[ns]"), @@ -229,11 +230,6 @@ describe("Query", () => { it.effect("implicit conversions to a common type fail on values it can't hold", () => Effect.gen(function*() { - const Price = Schema.Struct({ d: Schema.BigDecimal }) - const price = (d: string, precision: number, scale: number) => - DataFrame.fromRows(Price, [{ d: BigDecimal.fromStringUnsafe(d) }], { - types: { d: ArrowSchema.decimal(precision, scale) } - }) const big = Query.from(price("6000000000000000000", 19, 0)) const small = Query.from(price("1.20", 18, 2)) const failures: ReadonlyArray> = [ @@ -268,16 +264,15 @@ describe("Query", () => { ["coalesce(null, 300, x)", (c) => Expr.coalesce(null, 300, c.x), int16, [300, 300]], ["coalesce(x, null, 300)", (c) => Expr.coalesce(c.x, null, 300), int16, [300, 5]], ["coalesce(x, 300, 70000)", (c) => Expr.coalesce(c.x, 300, 70000), int32, [300, 5]], - // Polars 2.1.0 gives Int16, which can't hold 70000. + // Polars gives Int16, which can't hold 70000. ["coalesce(300, 70000, x)", (c) => Expr.coalesce(300, 70000, c.x), int32, [300, 300]], ["coalesce(1, x)", (c) => Expr.coalesce(1, c.x), int8, [1, 1]], ["coalesce(300, 1.5, x)", (c) => Expr.coalesce(300, 1.5, c.x), float64, [300, 300]], ["coalesce(u, -1)", (c) => Expr.coalesce(c.u, -1), int16, [-1, 5]], ["coalesce(-1, u)", (c) => Expr.coalesce(-1, c.u), int16, [-1, -1]], ["x + cast(5, Int32)", (c) => Expr.add(c.x, Expr.cast(Expr.lit(5), "Int32")), int32, [null, 10]], - // Every literal of an operand built from literals is fitted, whichever - // rows select it (w is when(x is null, 1, 300)); Polars 2.1.0 gives - // Int8, making 5 + 300 null and (100 + 1) * 2 wrap to -54. + // Literal-only operands are fitted as a whole (w is when(x is null, 1, 300)); + // Polars gives Int8, making 5 + 300 null and (100 + 1) * 2 wrap to -54. ["x + w", (c) => Expr.add(c.x, w(c.x)), int16, [null, 305]], [ "(coalesce(x, 100) + w) * 2", @@ -285,8 +280,8 @@ describe("Query", () => { int16, [202, 610] ], - // Other operands computed from literals count as Int32; Polars 2.1.0 - // gives Int8 with nulls. + // Other operands computed from literals count as Int32; Polars gives + // Int8 with nulls. ["x + -w", (c) => Expr.add(c.x, Expr.negate(w(c.x))), int32, [null, -295]] ] // x is Int8 and u UInt8, both [null, 5]: whole, streamed a row at a @@ -318,20 +313,12 @@ describe("Query", () => { it.effect("temporal operands of mixed units meet in the coarser unit, as in Polars", () => Effect.gen(function*() { // Expected values from Polars 2.1.0: timestamps round down, durations toward zero. - const df = DataFrame.fromArrowTable(Result.getOrThrow(ArrowTable.fromColumns( - ArrowSchema.make([ - ArrowSchema.field("t", ArrowSchema.timestamp("millisecond")), - ArrowSchema.field("s", ArrowSchema.timestamp("microsecond")), - ArrowSchema.field("d", ArrowSchema.duration("millisecond")), - ArrowSchema.field("e", ArrowSchema.duration("microsecond")) - ]), - [[1n, -1n, 2n], [1500n, 1500n, -1n], [1n, -1n, 2n], [1500n, 1500n, -1n]] - ))) as DataFrame.DataFrame as DataFrame.DataFrame<{ - t: DateTime.Utc - s: DateTime.Utc - d: Duration.Duration - e: Duration.Duration - }> + const df = arrowFrame<{ t: DateTime.Utc; s: DateTime.Utc; d: Duration.Duration; e: Duration.Duration }>({ + t: [ArrowSchema.timestamp("millisecond"), [1n, -1n, 2n]], + s: [ArrowSchema.timestamp("microsecond"), [1500n, 1500n, -1n]], + d: [ArrowSchema.duration("millisecond"), [1n, -1n, 2n]], + e: [ArrowSchema.duration("microsecond"), [1500n, 1500n, -1n]] + }) const int = (x: Expr.Expr) => Expr.cast(x, "Int64") const result = yield* rows( Query.from(df).pipe(Query.select((c) => ({ @@ -560,13 +547,15 @@ describe("Query", () => { // Every case of the Polars gold corpus (test/fixtures/dataframe/gold), with // its inputs inline, as catalog frames and as streams of small batches. describe("Polars oracle", () => { - const cases = [...Gold.spec().cases, ...Gold.extended().cases] + const cases = Gold.cases() for (const source of ["values", "frames", "batches"] as const) { describe(source, () => { for (const c of cases) { - it(`${c.id}: ${c.description}`, () => { - assert.strictEqual(Compare.diff(c.id, Oracle.run(c, source), c.expected), undefined) - }) + it.effect(`${c.id}: ${c.description}`, () => + Effect.gen(function*() { + const df = yield* Oracle.run(c, source) + assert.strictEqual(yield* Compare.diff(c.id, df, c.expected), undefined) + })) } }) } @@ -601,12 +590,7 @@ describe("Query", () => { it.effect("batches convert to the declared column types; undeclared, they must share their columns", () => Effect.gen(function*() { - const Price = Schema.Struct({ d: Schema.BigDecimal }) - const price = (d: string, scale: number) => - DataFrame.fromRows(Price, [{ d: BigDecimal.fromStringUnsafe(d) }], { - types: { d: ArrowSchema.decimal(18, scale) } - }) - const source = Stream.make(price("1.2", 1), price("1.23", 2), price("1.2", 2)) + const source = Stream.make(price("1.2", 18, 1), price("1.23", 18, 2), price("1.2", 18, 2)) const declared = Query.table("t", Price, { types: { d: ArrowSchema.decimal(18, 2) } }) const d = (s: string) => BigDecimal.fromStringUnsafe(s) const totals = declared.pipe( @@ -647,9 +631,8 @@ describe("Query", () => { DataFrame.fromRows(Row, values.map((x) => ({ x })), { types: { x: type } }) const wide = (values: BigUint64Array | BigInt64Array) => DataFrame.fromColumns({ x: values }) as unknown as Frame - const Price = Schema.Struct({ x: Schema.BigDecimal }) - const price = (d: string, precision: number, scale: number) => - DataFrame.fromRows(Price, [{ x: BigDecimal.fromStringUnsafe(d) }], { + const decimal = (d: string, precision: number, scale: number) => + DataFrame.fromRows(Schema.Struct({ x: Schema.BigDecimal }), [{ x: BigDecimal.fromStringUnsafe(d) }], { types: { x: ArrowSchema.decimal(precision, scale) } }) as unknown as Frame // A query over the frames concatenated: in memory, and streamed with @@ -722,7 +705,7 @@ describe("Query", () => { [[[2 ** 63, 1], [-1, 1], [0.5, 1]], [[0.5, 1], [-1, 1], [2 ** 63, 1]]] ], [ - [price("6000000000000000000", 19, 0), price("1.20", 18, 2), x(float64, [0.5])], + [decimal("6000000000000000000", 19, 0), decimal("1.20", 18, 2), x(float64, [0.5])], { sum: 6e18, mean: 2e18, min: 0.5, max: 6e18, count: 3 }, [[6e18, 0.5], [0.5, 6e18]], [[[6e18, 1], [1.2, 1], [0.5, 1]], [[0.5, 1], [1.2, 1], [6e18, 1]]] @@ -756,7 +739,7 @@ describe("Query", () => { } } } - // Row-wise operations see the common type too, as in Polars 2.1.0: Int8 + // Row-wise operations see the common type too, as in Polars: Int8 // 100 doubles to 200 in Int64, where doubling in Int8 wraps to -56. const small = [DataFrame.fromColumns({ x: Int8Array.of(100) }), wide(BigInt64Array.of(1n))] const rowWise: ReadonlyArray Query.Query, ReadonlyArray]> = [ @@ -777,7 +760,7 @@ describe("Query", () => { "UInt64 values don't fit their common type Int64" ], [ - [price("6000000000000000000", 19, 0), price("1.20", 18, 2)], + [decimal("6000000000000000000", 19, 0), decimal("1.20", 18, 2)], "Decimal[19, 0] values don't fit their common type Decimal[38, 2]" ] ] @@ -810,7 +793,6 @@ describe("Query", () => { const untyped = Query.fromPlan({ _tag: "Table", name: "t" }).pipe( Query.filter((c) => Expr.gt(c.x as Expr.Expr, 5)) ) - const Price = Schema.Struct({ d: Schema.BigDecimal }) const prices = Query.table("t", Price, { types: { d: ArrowSchema.decimal(18, 2) } }).pipe( Query.aggregate((c) => ({ total: Expr.sum(c.d), r: Expr.max(Expr.round(c.d, 1)) })) ) @@ -928,7 +910,7 @@ describe("Query", () => { Effect.gen(function*() { let pulled = 0 const source = Stream.fromIterable(batches).pipe(Stream.tap(() => Effect.sync(() => pulled++))) - const df = yield* Query.table("t", Row).pipe( + const df = yield* t.pipe( Query.filter((c) => Expr.gt(c.x, 0)), Query.head(2), Effect.provide(Catalog.layer({ t: source })) @@ -939,7 +921,7 @@ describe("Query", () => { it.effect("Query.stream emits a frame per non-empty batch", () => Effect.gen(function*() { - const sizes = yield* Query.table("t", Row).pipe( + const sizes = yield* t.pipe( Query.filter((c) => Expr.lt(c.x, 5)), Query.stream, Stream.map((frame) => frame.length), @@ -962,13 +944,6 @@ describe("Query", () => { } type Key = { readonly name: keyof Row; readonly descending: boolean } - const schema = ArrowSchema.make([ - ArrowSchema.field("f", ArrowSchema.float64), - ArrowSchema.field("i", ArrowSchema.int32), - ArrowSchema.field("b", ArrowSchema.int64), - ArrowSchema.field("s", ArrowSchema.utf8), - ArrowSchema.field("p", ArrowSchema.bool) - ]) const strings = ["a", "b", "ab", "B", "é", "😀", "\uffff", "", "zz", "a\u0000"] const generate = (seed: number, n: number, cardinality: number): Array => { @@ -1028,8 +1003,7 @@ describe("Query", () => { const keyOf = (values: ReadonlyArray) => JSON.stringify( values, - (_, v) => - typeof v === "bigint" ? `${v}n` : typeof v === "number" && !Number.isFinite(v) ? String(v) : v === 0 ? 0 : v + (_, v) => typeof v === "bigint" ? `${v}n` : typeof v === "number" && !Number.isFinite(v) ? String(v) : v ) const groups = new Map>() for (const r of rows) { @@ -1041,95 +1015,105 @@ describe("Query", () => { return [...groups.values()] } - const rows = generate(0x2545f491, 6000, 3000) - const df = Query.from( - DataFrame.fromArrowTable(Result.getOrThrow(ArrowTable.fromRows(schema, rows as any))) as DataFrame.DataFrame - ) - const rowsOf = (query: Query.Query): Array => DataFrame.toRows(Effect.runSync(query)) + const data = generate(0x2545f491, 6000, 3000) + const df = Query.from(arrowFrame({ + f: [ArrowSchema.float64, data.map((r) => r.f)], + i: [ArrowSchema.int32, data.map((r) => r.i)], + b: [ArrowSchema.int64, data.map((r) => r.b)], + s: [ArrowSchema.utf8, data.map((r) => r.s)], + p: [ArrowSchema.bool, data.map((r) => r.p)] + })) const sortKey = (c: { readonly [K in keyof Row]: Expr.Expr }, key: Key) => key.descending ? Expr.desc(c[key.name] as Expr.Expr) : Expr.asc(c[key.name] as Expr.Expr) - it("sorts by every column type, in both directions, nulls first or last", () => { - for (const name of ["f", "i", "b", "s", "p"] as const) { - for (const descending of [false, true]) { - for (const nullsLast of [false, true]) { - const sorted = Query.sort(df, (c) => [sortKey(c, { name, descending })], { nullsLast }) - assert.deepStrictEqual(rowsOf(sorted), referenceSort(rows, [{ name, descending }], nullsLast)) + it.effect("sorts by every column type, in both directions, nulls first or last", () => + Effect.gen(function*() { + for (const name of ["f", "i", "b", "s", "p"] as const) { + for (const descending of [false, true]) { + for (const nullsLast of [false, true]) { + const sorted = Query.sort(df, (c) => [sortKey(c, { name, descending })], { nullsLast }) + assert.deepStrictEqual(yield* rows(sorted), referenceSort(data, [{ name, descending }], nullsLast)) + } } } - } - }) + })) - it("sorts stably by several keys, also when sliced (top-k)", () => { - const keys: ReadonlyArray = [ - { name: "p", descending: true }, - { name: "s", descending: false }, - { name: "f", descending: true } - ] - const sorted = Query.sort(df, (c) => keys.map((key) => sortKey(c, key))) - assert.deepStrictEqual(rowsOf(sorted), referenceSort(rows, keys, false)) - for ( - const [name, descending, nullsLast, offset, length] of [ - ["f", true, false, 0, 10], - ["i", false, true, 0, 1], - ["s", false, false, 5, 20], - ["b", true, true, 0, 700] - ] as const - ) { - const top = df.pipe( - Query.sort((c) => [sortKey(c, { name, descending }), Expr.asc(c.i)], { nullsLast }), - Query.slice(offset, length) - ) - const expected = referenceSort(rows, [{ name, descending }, { name: "i", descending: false }], nullsLast) - assert.deepStrictEqual(rowsOf(top), expected.slice(offset, offset + length)) - } - }) + it.effect("sorts stably by several keys, also when sliced (top-k)", () => + Effect.gen(function*() { + const keys: ReadonlyArray = [ + { name: "p", descending: true }, + { name: "s", descending: false }, + { name: "f", descending: true } + ] + const sorted = Query.sort(df, (c) => keys.map((key) => sortKey(c, key))) + assert.deepStrictEqual(yield* rows(sorted), referenceSort(data, keys, false)) + for ( + const [name, descending, nullsLast, offset, length] of [ + ["f", true, false, 0, 10], + ["i", false, true, 0, 1], + ["s", false, false, 5, 20], + ["b", true, true, 0, 700] + ] as const + ) { + const top = df.pipe( + Query.sort((c) => [sortKey(c, { name, descending }), Expr.asc(c.i)], { nullsLast }), + Query.slice(offset, length) + ) + const expected = referenceSort(data, [{ name, descending }, { name: "i", descending: false }], nullsLast) + assert.deepStrictEqual(yield* rows(top), expected.slice(offset, offset + length)) + } + })) - it("groups by keys of every type", () => { - for (const keys of [["f"], ["b"], ["p", "i"], ["s", "b", "p"]] as const) { - const result = df.pipe( - Query.groupBy(...keys), - Query.agg((c) => ({ n: Expr.count(), ni: Expr.count(c.i), max: Expr.max(c.s), first: Expr.first(c.b) })) + it.effect("groups by keys of every type", () => + Effect.gen(function*() { + for (const keys of [["f"], ["b"], ["p", "i"], ["s", "b", "p"]] as const) { + const result = df.pipe( + Query.groupBy(...keys), + Query.agg((c) => ({ n: Expr.count(), ni: Expr.count(c.i), max: Expr.max(c.s), first: Expr.first(c.b) })) + ) + const expected = referenceGroups(data, keys).map((g) => { + const out: Record = {} + for (const k of keys) out[k] = g[0][k] + const present = g.map((r) => r.s).filter((s): s is string => s !== null) + out.n = g.length + out.ni = g.filter((r) => r.i !== null).length + out.max = present.length === 0 ? null : present.reduce((a, b) => (compareCodePoints(a, b) >= 0 ? a : b)) + out.first = g[0].b + return out + }) + assert.deepStrictEqual(yield* rows(result), expected) + } + assert.deepStrictEqual( + yield* rows(Query.unique(df, ["i", "p"])), + referenceGroups(data, ["i", "p"]).map((g) => g[0]) ) - const expected = referenceGroups(rows, keys).map((g) => { - const out: Record = {} - for (const k of keys) out[k] = g[0][k] - const present = g.map((r) => r.s).filter((s): s is string => s !== null) - out.n = g.length - out.ni = g.filter((r) => r.i !== null).length - out.max = present.length === 0 ? null : present.reduce((a, b) => (compareCodePoints(a, b) >= 0 ? a : b)) - out.first = g[0].b - return out - }) - assert.deepStrictEqual(rowsOf(result), expected) - } - assert.deepStrictEqual(rowsOf(Query.unique(df, ["i", "p"])), referenceGroups(rows, ["i", "p"]).map((g) => g[0])) - }) + })) - it("joins on a high-cardinality key", () => { - const right = generate(0x1b873593, 4000, 3000).map((r) => ({ i: r.i ?? 0, rs: r.s })) - const rightFrame = DataFrame.fromColumns({ i: Int32Array.from(right, (r) => r.i), rs: right.map((r) => r.rs) }) - const byKey = new Map>() - for (const r of right) byKey.set(r.i, [...(byKey.get(r.i) ?? []), r.rs]) - for (const how of ["inner", "left", "semi", "anti"] as const) { - const result = Query.join(df.pipe(Query.pick("i", "s")), Query.from(rightFrame), { on: "i", how }) - const expected = rows.flatMap((r) => { - const matches = r.i === null ? [] : byKey.get(r.i) ?? [] - switch (how) { - case "inner": - return matches.map((rs) => ({ i: r.i, s: r.s, rs })) - case "left": - return matches.length === 0 - ? [{ i: r.i, s: r.s, rs: null }] - : matches.map((rs) => ({ i: r.i, s: r.s, rs })) - case "semi": - return matches.length > 0 ? [{ i: r.i, s: r.s }] : [] - case "anti": - return matches.length === 0 ? [{ i: r.i, s: r.s }] : [] - } - }) - assert.deepStrictEqual(rowsOf(result), expected) - } - }) + it.effect("joins on a high-cardinality key", () => + Effect.gen(function*() { + const right = generate(0x1b873593, 4000, 3000).map((r) => ({ i: r.i ?? 0, rs: r.s })) + const rightFrame = DataFrame.fromColumns({ i: Int32Array.from(right, (r) => r.i), rs: right.map((r) => r.rs) }) + const byKey = new Map>() + for (const r of right) byKey.set(r.i, [...(byKey.get(r.i) ?? []), r.rs]) + for (const how of ["inner", "left", "semi", "anti"] as const) { + const result = Query.join(df.pipe(Query.pick("i", "s")), Query.from(rightFrame), { on: "i", how }) + const expected = data.flatMap((r) => { + const matches = r.i === null ? [] : byKey.get(r.i) ?? [] + switch (how) { + case "inner": + return matches.map((rs) => ({ i: r.i, s: r.s, rs })) + case "left": + return matches.length === 0 + ? [{ i: r.i, s: r.s, rs: null }] + : matches.map((rs) => ({ i: r.i, s: r.s, rs })) + case "semi": + return matches.length > 0 ? [{ i: r.i, s: r.s }] : [] + case "anti": + return matches.length === 0 ? [{ i: r.i, s: r.s }] : [] + } + }) + assert.deepStrictEqual(yield* rows(result), expected) + } + })) }) }) diff --git a/packages/effect/test/dataframe/oracle/Compare.ts b/packages/effect/test/dataframe/oracle/Compare.ts index d236ab261c0..fb31463e00f 100644 --- a/packages/effect/test/dataframe/oracle/Compare.ts +++ b/packages/effect/test/dataframe/oracle/Compare.ts @@ -14,13 +14,9 @@ import * as Plan from "./Plan.ts" * Deliberate differences from Polars, by case id: the column types * effect/dataframe produces instead. Values still have to match. */ -export const deviations: { - readonly [id: string]: { readonly reason: string; readonly types: { readonly [column: string]: string } } -} = { - "group/small-ints": { - reason: "sums of 8/16-bit integers are Int32 (Polars: Int64, which would be a bigint in TypeScript)", - types: { i8: "i32", u8: "i32" } - } +const deviations: { readonly [id: string]: { readonly [column: string]: string } } = { + // Sums of 8/16-bit integers are Int32 (Polars: Int64, which would be a bigint in TypeScript). + "group/small-ints": { i8: "i32", u8: "i32" } } const close = (a: number, b: number): boolean => { @@ -58,8 +54,8 @@ const show = (u: unknown): string => : JSON.stringify(u) ?? String(u) /** `undefined` when equal, otherwise a description of the first difference. */ -export const diff = (id: string, actual: DataFrame.DataFrame, expected: Gold.Table): string | undefined => { - const names = actual.schema.fields.map((f) => f.name) +export const diff = Effect.fnUntraced(function*(id: string, actual: DataFrame.DataFrame, expected: Gold.Table) { + const names = DataFrame.columnNames(actual) const expectedNames = expected.fields.map((f) => f.name) if (JSON.stringify(names) !== JSON.stringify(expectedNames)) { return `columns: got ${JSON.stringify(names)}, expected ${JSON.stringify(expectedNames)}` @@ -67,14 +63,14 @@ export const diff = (id: string, actual: DataFrame.DataFrame, expected: Gol const deviation = deviations[id] for (const [i, f] of expected.fields.entries()) { const got = Plan.typeName(actual.schema.fields[i].type) - const want = deviation?.types[f.name] ?? f.type + const want = deviation?.[f.name] ?? f.type if (got !== want) return `column ${f.name}: type ${got}, expected ${want}` } - const length = expected.fields.length === 0 ? 0 : expected.columns[expected.fields[0].name].length + const length = Gold.rowCount(expected) if (actual.length !== length) return `length ${actual.length}, expected ${length}` // Temporal columns compare through their integers (days or units since the epoch). - const rows = DataFrame.toRows(Effect.runSync( - Query.from(actual).pipe(Query.select((c: any) => { + const rows = DataFrame.toRows( + yield* Query.from(actual).pipe(Query.select((c: any) => { const out: Record> = {} for (const f of expected.fields) { const t = Plan.temporal[f.type] @@ -82,7 +78,7 @@ export const diff = (id: string, actual: DataFrame.DataFrame, expected: Gol } return out })) - )) + ) for (const f of expected.fields) { const column = expected.columns[f.name] for (let i = 0; i < length; i++) { @@ -92,4 +88,4 @@ export const diff = (id: string, actual: DataFrame.DataFrame, expected: Gol } } return undefined -} +}) diff --git a/packages/effect/test/dataframe/oracle/Gold.ts b/packages/effect/test/dataframe/oracle/Gold.ts index 86945da2300..0f1bee3c673 100644 --- a/packages/effect/test/dataframe/oracle/Gold.ts +++ b/packages/effect/test/dataframe/oracle/Gold.ts @@ -18,7 +18,6 @@ export type TypeName = | "f32" | "f64" | "utf8" - | "null" | "date" | "datetime[us]" | "datetime[ms]" @@ -33,11 +32,19 @@ export interface Table { readonly columns: { readonly [name: string]: ReadonlyArray } } +export const rowCount = (table: Table): number => Object.values(table.columns)[0]?.length ?? 0 + +export interface Call { + readonly fn: string + readonly args: ReadonlyArray + readonly [option: string]: unknown +} + export type Expr = | { readonly over: Expr; readonly partitionBy: ReadonlyArray } | { readonly col: string } | { readonly lit: unknown; readonly type?: TypeName } - | { readonly fn: string; readonly args: ReadonlyArray; readonly [option: string]: unknown } + | Call | { readonly agg: string; readonly arg?: Expr } export type Op = @@ -87,20 +94,14 @@ export interface Case { readonly expected: Table } -export interface Corpus { - readonly polars: string - readonly deviations: ReadonlyArray - readonly cases: ReadonlyArray -} - -const load = (name: string): Corpus => - JSON.parse(Fs.readFileSync(new URL(`../../fixtures/dataframe/gold/${name}.json`, import.meta.url), "utf8")) +const load = (name: string): ReadonlyArray => + JSON.parse(Fs.readFileSync(new URL(`../../fixtures/dataframe/gold/${name}.json`, import.meta.url), "utf8")).cases -export const spec = (): Corpus => load("spec") -export const extended = (): Corpus => load("extended") +/** The cases of gold/spec.json and gold/extended.json. */ +export const cases = (): ReadonlyArray => [...load("spec"), ...load("extended")] /** Splits `a:x,b:list` on top-level commas. */ -export const splitTop = (s: string): Array => { +const splitTop = (s: string): Array => { const parts: Array = [] let depth = 0 let current = "" diff --git a/packages/effect/test/dataframe/oracle/Plan.ts b/packages/effect/test/dataframe/oracle/Plan.ts index 2e60618d410..25f6f6c4e57 100644 --- a/packages/effect/test/dataframe/oracle/Plan.ts +++ b/packages/effect/test/dataframe/oracle/Plan.ts @@ -20,7 +20,6 @@ const arrowTypes = { f32: ArrowSchema.float32, f64: ArrowSchema.float64, utf8: ArrowSchema.utf8, - null: ArrowSchema.nullType, date: ArrowSchema.date("day"), "datetime[us]": ArrowSchema.timestamp("microsecond"), "datetime[ms]": ArrowSchema.timestamp("millisecond"), @@ -40,12 +39,12 @@ export const temporal: Partial< } /** Precision and scale of a `decimal[p,s]` name. */ -export const decimalOf = (name: string): readonly [number, number] => { +const decimalOf = (name: string): readonly [number, number] => { const [p, s] = name.slice(8, -1).split(",").map(Number) return [p, s] } -export const arrowType = (name: Gold.TypeName): ArrowSchema.DataType => { +const arrowType = (name: Gold.TypeName): ArrowSchema.DataType => { if (name.startsWith("decimal[")) return ArrowSchema.decimal(...decimalOf(name)) if (name.startsWith("list<")) { return ArrowSchema.list(ArrowSchema.field("item", arrowType(name.slice(5, -1) as Gold.TypeName))) @@ -89,16 +88,11 @@ const tableColumn = (name: string, t: ArrowSchema.DataType): Plan.TableColumn => } /** A gold input table as a frame. */ -export const frameOf = (table: Gold.Table): DataFrame.DataFrame => { - const schema = ArrowSchema.make(table.fields.map((f) => ArrowSchema.field(f.name, arrowType(f.type)))) - const length = table.fields.length === 0 ? 0 : table.columns[table.fields[0].name].length - const rows = Array.from({ length }, (_, i) => { - const row: Record = {} - for (const f of table.fields) row[f.name] = Gold.decodeValue(table.columns[f.name][i], f.type) - return row - }) - return DataFrame.fromArrowTable(Result.getOrThrow(ArrowTable.fromRows(schema, rows))) -} +const frameOf = (table: Gold.Table): DataFrame.DataFrame => + DataFrame.fromArrowTable(Result.getOrThrow(ArrowTable.fromColumns( + ArrowSchema.make(table.fields.map((f) => ArrowSchema.field(f.name, arrowType(f.type)))), + table.fields.map((f) => table.columns[f.name].map((v) => Gold.decodeValue(v, f.type))) + ))) const castNames: Partial> = { i8: "Int8", @@ -120,15 +114,6 @@ const castNames: Partial> = { "duration[ms]": "Duration[ms]" } -const floatSpecials = new Set(["NaN", "Infinity", "-Infinity", "-0"]) - -const literal = (e: { readonly lit: unknown; readonly type?: Gold.TypeName }): Plan.Value => - (e.type !== undefined - ? Gold.decodeValue(e.lit, e.type) - : typeof e.lit === "string" && floatSpecials.has(e.lit) - ? Gold.decodeValue(e.lit, "f64") - : e.lit) as Plan.Value - /** Gold function names that differ from the plan's. */ const functionNames: Record = { sub: "subtract", @@ -145,86 +130,51 @@ const functionNames: Record = { strConcat: "concat" } -const options = (e: { readonly [option: string]: unknown }): Plan.CallOptions | undefined => { - switch (e.fn) { - case "round": - return { decimals: (e.decimals as number | undefined) ?? 0 } - case "cast": { - const to = e.to as string - if (to.startsWith("decimal[")) { - const [precision, scale] = decimalOf(to) - return { to: "Decimal", precision, scale } - } - return { to: castNames[to as Gold.TypeName] } - } - case "truncate": - return { every: e.every as string } - case "listGet": - return { index: e.index as number } - case "field": - return { name: e.field as string } - case "struct": - return { names: e.names as ReadonlyArray } - case "isIn": - return { - values: (e.values as ReadonlyArray).map((v) => - Gold.decodeValue(v, (e.type as Gold.TypeName) ?? "utf8") as Plan.Value - ) - } - case "strContains": - return { pattern: e.pattern as string } - case "strStartsWith": - return { pattern: e.prefix as string } - case "strEndsWith": - return { pattern: e.suffix as string } - case "shift": - case "diff": - return { n: (e.n as number | undefined) ?? 1 } - case "rank": - return { - method: (e.method as Plan.RankMethod | undefined) ?? "average", - descending: (e.descending as boolean | undefined) ?? false - } - case "rollingSum": - case "rollingMean": - case "rollingMin": - case "rollingMax": - return { window: e.window as number, minPeriods: (e.minPeriods as number | undefined) ?? (e.window as number) } - default: - return undefined +/** Gold option names that differ from the plan's. */ +const optionNames: Record = { prefix: "pattern", suffix: "pattern", field: "name" } + +const options = ({ args: _, fn, ...rest }: Gold.Call): Plan.CallOptions | undefined => { + if (fn === "cast") { + const to = rest.to as Gold.TypeName + if (!to.startsWith("decimal[")) return { to: castNames[to] } + const [precision, scale] = decimalOf(to) + return { to: "Decimal", precision, scale } } + const entries = Object.entries(rest) + return entries.length === 0 ? undefined : Object.fromEntries(entries.map(([k, v]) => [optionNames[k] ?? k, v])) } const aggregateNames: Record = { len: "count", nUnique: "countDistinct", var: "variance" } /** A gold expression as plan data. */ -export const expression = (e: Gold.Expr): Plan.Expression => { +const expression = (e: Gold.Expr): Plan.Expression => { if ("over" in e) { const o = e as { readonly over: Gold.Expr; readonly partitionBy: ReadonlyArray } return { _tag: "Over", expression: expression(o.over), partitionBy: o.partitionBy.map(expression) } } if ("col" in e) return { _tag: "Column", name: e.col as string } if ("lit" in e) { - const l = e as { readonly lit: unknown; readonly type?: Gold.TypeName } - const t = l.type === undefined ? undefined : temporal[l.type] - if (l.type !== undefined && l.type.startsWith("decimal[")) { - const [precision, scale] = decimalOf(l.type) - const value = BigDecimal.scale(BigDecimal.fromStringUnsafe(l.lit as string), scale).value + const { lit, type } = e as { readonly lit: unknown; readonly type?: Gold.TypeName } + if (type === undefined) return { _tag: "Literal", value: lit as Plan.Value } + if (type.startsWith("decimal[")) { + const [precision, scale] = decimalOf(type) + const value = BigDecimal.scale(BigDecimal.fromStringUnsafe(lit as string), scale).value return { _tag: "Literal", value, type: { _tag: "Decimal", precision, scale } } } + const t = temporal[type] return t === undefined - ? { _tag: "Literal", value: literal(l) } - : { _tag: "Literal", value: l.lit as number, type: t.literal } + ? { _tag: "Literal", value: Gold.decodeValue(lit, type) as Plan.Value } + : { _tag: "Literal", value: lit as number, type: t.literal } } if ("agg" in e) { const a = e as { readonly agg: string; readonly arg?: Gold.Expr } return { _tag: "AggregateCall", fn: aggregateNames[a.agg] ?? a.agg as Plan.AggregateName, - arg: a.arg === undefined || a.agg === "len" ? undefined : expression(a.arg) + arg: a.arg === undefined ? undefined : expression(a.arg) } } - const call = e as { readonly fn: string; readonly args: ReadonlyArray; readonly [option: string]: unknown } + const call = e as Gold.Call return { _tag: "Call", fn: functionNames[call.fn] ?? call.fn as Plan.FunctionName, @@ -237,17 +187,19 @@ const named = (record: { readonly [name: string]: Gold.Expr }): Array ({ name, expression: expression(e) })) /** - * A case's operations as one plan over its inputs. Inputs are `Table` nodes - * read from the catalog, or with `inline`, `Values` nodes holding the frames. + * How a case's inputs reach the query: inline `Values` nodes, frames in the + * catalog, or streams of two-row batches in the catalog, declared with their + * columns. */ -export const plan = (gold: Gold.Case, inline: boolean, declare = false): Plan.Node => { - const columns = (name: string): Array => - gold.inputs[name].fields.map((f) => tableColumn(f.name, arrowType(f.type))) +export type Source = "values" | "frames" | "batches" + +/** A case's operations as one plan over its inputs. */ +const plan = (gold: Gold.Case, source: Source): Plan.Node => { const input = (name: string): Plan.Node => - inline + source === "values" ? { _tag: "Values", frame: frameOf(gold.inputs[name]) } - : declare - ? { _tag: "Table", name, columns: columns(name) } + : source === "batches" + ? { _tag: "Table", name, columns: gold.inputs[name].fields.map((f) => tableColumn(f.name, arrowType(f.type))) } : { _tag: "Table", name } let node = input("left") for (const op of gold.ops) { @@ -314,8 +266,7 @@ export const plan = (gold: Gold.Case, inline: boolean, declare = false): Plan.No on: op.on, index: op.index, values: op.values, - aggregate: - (op.aggregate === "len" ? "len" : aggregateNames[op.aggregate] ?? op.aggregate) as Plan.PivotAggregate, + aggregate: op.aggregate as Plan.PivotAggregate, columns: op.columns } break @@ -340,33 +291,28 @@ export const plan = (gold: Gold.Case, inline: boolean, declare = false): Plan.No return node } -/** - * How a case's inputs reach the query: inline `Values` nodes, frames in the - * catalog, or streams of two-row batches in the catalog. - */ -export type Source = "values" | "frames" | "batches" - -const batchesOf = (table: Gold.Table): Stream.Stream> => { - const length = table.fields.length === 0 ? 0 : table.columns[table.fields[0].name].length - return Stream.fromIterable(Array.from({ length: Math.ceil(length / 2) }, (_, b) => +const batchesOf = (table: Gold.Table): Stream.Stream> => + Stream.fromIterable(Array.from({ length: Math.ceil(Gold.rowCount(table) / 2) }, (_, b) => frameOf({ fields: table.fields, columns: Object.fromEntries( Object.entries(table.columns).map(([name, values]) => [name, values.slice(b * 2, b * 2 + 2)]) ) }))) -} /** * Runs a case: its plan goes through its JSON form (so the codecs are part * of what is checked) and then runs as a query. */ -export const run = (gold: Gold.Case, source: Source): DataFrame.DataFrame => { - const node = plan(gold, source === "values", source === "batches") +export const run = ( + gold: Gold.Case, + source: Source +): Effect.Effect, DataFrame.DataFrameError> => { + const node = plan(gold, source) const json = JSON.parse(JSON.stringify(Schema.encodeSync(Plan.NodeFromJson)(node))) const decoded = Schema.decodeUnknownSync(Plan.NodeFromJson)(json) const tables = Object.fromEntries( Object.entries(gold.inputs).map(([name, t]) => [name, source === "batches" ? batchesOf(t) : frameOf(t)]) ) - return Query.fromPlan(decoded).pipe(Effect.provide(Catalog.layer(tables)), Effect.runSync) + return Query.fromPlan(decoded).pipe(Effect.provide(Catalog.layer(tables))) } diff --git a/packages/effect/test/fixtures/dataframe/generate_gold.py b/packages/effect/test/fixtures/dataframe/generate_gold.py index 56dc3190d24..3c9ca7e0013 100644 --- a/packages/effect/test/fixtures/dataframe/generate_gold.py +++ b/packages/effect/test/fixtures/dataframe/generate_gold.py @@ -8,14 +8,14 @@ uv run --with polars==2.1.0 python generate_gold.py # writes gold/spec.json and gold/extended.json -Needs polars (tested with 2.1). Polars is the semantic reference: where -effect/dataframe deliberately differs, the case is left out here and the -difference is listed in DEVIATIONS below. +Polars is the semantic reference. Cases avoid the differences listed in +DEVIATIONS; differences in result types alone are listed in oracle/Compare.ts. """ import json import math -import sys +from datetime import date, datetime +from decimal import Decimal from pathlib import Path import polars as pl @@ -48,7 +48,6 @@ "f32": pl.Float32, "f64": pl.Float64, "utf8": pl.String, - "null": pl.Null, "date": pl.Date, "datetime[us]": pl.Datetime("us"), "datetime[ms]": pl.Datetime("ms"), @@ -79,6 +78,10 @@ def split_top(s): return parts +def struct_fields(t): + return [f.split(":", 1) for f in split_top(t[7:-1])] + + def dtype_of(name): if name.startswith("decimal["): p, sc = name[8:-1].split(",") @@ -86,7 +89,7 @@ def dtype_of(name): if name.startswith("list<"): return pl.List(dtype_of(name[5:-1])) if name.startswith("struct<"): - return pl.Struct({f.split(":", 1)[0]: dtype_of(f.split(":", 1)[1]) for f in split_top(name[7:-1])}) + return pl.Struct({n: dtype_of(ft) for n, ft in struct_fields(name)}) return DTYPES[name] @@ -111,8 +114,7 @@ def encode_value(value, t): if t.startswith("list<"): return [encode_value(v, t[5:-1]) for v in value] if t.startswith("struct<"): - fields = [f.split(":", 1) for f in split_top(t[7:-1])] - return {n: encode_value(value.get(n), ft) for n, ft in fields} + return {n: encode_value(value.get(n), ft) for n, ft in struct_fields(t)} if t in ("f32", "f64"): if math.isnan(value): return "NaN" @@ -130,13 +132,11 @@ def decode_value(value, t): if value is None: return None if t.startswith("decimal["): - from decimal import Decimal as PyDecimal - return PyDecimal(value) + return Decimal(value) if t.startswith("list<"): return [decode_value(v, t[5:-1]) for v in value] if t.startswith("struct<"): - fields = [f.split(":", 1) for f in split_top(t[7:-1])] - return {n: decode_value(value.get(n), ft) for n, ft in fields} + return {n: decode_value(value.get(n), ft) for n, ft in struct_fields(t)} if t in ("f32", "f64"): if isinstance(value, str): return {"NaN": float("nan"), "Infinity": float("inf"), "-Infinity": float("-inf"), "-0": -0.0}[value] @@ -179,10 +179,7 @@ def values(name, t): def lit(value, t=None): - out = {"lit": encode_value(value, t or ("f64" if isinstance(value, float) else "")) if value is not None else None} - if t is not None: - out["type"] = t - return out + return {"lit": value} if t is None else {"lit": encode_value(value, t), "type": t} def col(name): @@ -199,16 +196,12 @@ def agg(name, arg=None): def p_lit(e): t = e.get("type") - v = e["lit"] - if t is not None and t.startswith("decimal["): - return pl.lit(decode_value(v, t), dtype=dtype_of(t)) - if t is not None and t in TEMPORAL: - return pl.lit(decode_value(v, t), dtype=TEMPORAL[t]).cast(DTYPES[t]) - if t is not None: - return pl.lit(decode_value(v, t), dtype=DTYPES[t]) - if isinstance(v, str) and v in ("NaN", "Infinity", "-Infinity", "-0"): - return pl.lit(decode_value(v, "f64")) - return pl.lit(v) + if t is None: + return pl.lit(e["lit"]) + value = decode_value(e["lit"], t) + if t in TEMPORAL: + return pl.lit(value, dtype=TEMPORAL[t]).cast(DTYPES[t]) + return pl.lit(value, dtype=dtype_of(t)) def p_expr(e): @@ -261,7 +254,7 @@ def p_expr(e): if f == "cast": return a[0].cast(dtype_of(e["to"]), strict=False) if f == "isIn": - return a[0].is_in([decode_value(v, e.get("type", "")) for v in e["values"]]) + return a[0].is_in(e["values"]) if f == "between": return a[0].is_between(a[1], a[2]) if f == "round": @@ -406,11 +399,12 @@ def run(inputs, ops): elif kind == "dropNulls": df = df.drop_nulls(subset=op.get("columns")) elif kind == "pivot": + source = df df = df.pivot(on=op["on"], index=op["index"], values=op["values"], aggregate_function=op["aggregate"], sort_columns=False) if "columns" in op: # Declared columns that never occur are cells without rows: # the aggregate of an empty set (0 for sums and counts). - empty = run({"left": inputs["left"]}, [o for o in ops[: ops.index(op)]]).clear().select( + empty = source.clear().select( p_agg(agg(op["aggregate"], col(op["values"])) if op["aggregate"] != "len" else agg("len")).alias("v") ) for c in op["columns"]: @@ -703,8 +697,6 @@ def spec_cases(): }, ) -from datetime import datetime, date, timedelta - EPOCH = datetime(1970, 1, 1) @@ -825,7 +817,6 @@ def extended_cases(): [{"op": "pick", "columns": ["region", "quarter"]}, {"op": "unpivot", "index": [], "on": ["region", "quarter"], "variable": "field", "value": "text"}]) T = {"left": times, "right": holidays} - us = lambda dt: int((dt - EPOCH).total_seconds() * 1_000_000) add("temporal/extract", "calendar fields of timestamps and dates", T, [{"op": "select", "exprs": {"y": fn("year", col("ts")), "mo": fn("month", col("ts")), "d": fn("day", col("ts")), "h": fn("hour", col("ts")), "mi": fn("minute", col("ts")), "s": fn("second", col("ts")), @@ -847,7 +838,7 @@ def extended_cases(): "us": fn("cast", col("ts"), to="i64"), "days": fn("cast", col("d"), to="i32"), "ms": fn("cast", col("ts"), to="datetime[ms]")}}]) add("temporal/compare", "comparisons with timestamp literals and between units", T, - [{"op": "filter", "predicate": fn("and", fn("gte", col("ts"), lit(us(datetime(2024, 3, 1)), "datetime[us]")), fn("lt", col("ms"), col("ts")))}]) + [{"op": "filter", "predicate": fn("and", fn("gte", col("ts"), lit(_us(datetime(2024, 3, 1)), "datetime[us]")), fn("lt", col("ms"), col("ts")))}]) add("temporal/group-sort", "group by date, aggregate timestamps, sort", T, [{"op": "groupBy", "keys": ["d"], "aggs": {"first": agg("min", col("ts")), "last": agg("max", col("ts")), "n": agg("len"), "total": agg("sum", col("dur"))}}, From e3474e992a21853ed8a362a230e32bfec948487b Mon Sep 17 00:00:00 2001 From: Effect Bot Date: Sun, 15 Nov 2026 19:27:28 +0000 Subject: [PATCH 61/88] Clarify the comment on fitting integer literals to their operand --- packages/effect/src/dataframe/internal/kernels.ts | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/packages/effect/src/dataframe/internal/kernels.ts b/packages/effect/src/dataframe/internal/kernels.ts index a5a74bbb171..572dd1f6390 100644 --- a/packages/effect/src/dataframe/internal/kernels.ts +++ b/packages/effect/src/dataframe/internal/kernels.ts @@ -337,11 +337,10 @@ const rangeOf = (values: ReadonlyArray, dynamic: D.Dynamic): readonly [nu } /** - * The type and dynamism of an operand next to one of type `other`: integer - * literals (every literal of an operand built from them, whichever rows - * select it) that `other` can't hold commit to the smallest signed integer - * type that holds them, instead of adopting `other`, as in Polars (`Int8 + - * 300` is Int16). + * The type and dynamism of an operand next to one of type `other`. Integer + * literals that `other` can't hold take the smallest signed integer type + * that holds them instead of adopting `other`, as in Polars (`Int8 + 300` + * is Int16). An operand built only from literals is fitted as a whole. */ const operand = (v: Value, other: S.DataType): readonly [S.DataType, D.Dynamic] => { if (v.dynamic !== "int" || other._tag !== "Int" || v.vector.kind !== "i32") return [v.vector.type, v.dynamic] From a415abf8910268ddb6774f46713e70f0b6e66ce1 Mon Sep 17 00:00:00 2001 From: Effect Bot Date: Sun, 15 Nov 2026 19:44:17 +0000 Subject: [PATCH 62/88] Map DataFrame API contracts to runtime and type coverage --- .../effect/test/dataframe/DataFrame.test.ts | 20 ++ packages/effect/test/dataframe/Query.test.ts | 56 ++++++ packages/effect/test/dataframe/README.md | 186 ++++++++++++++++++ packages/sql/pglite/test/DataFrameSql.test.ts | 17 ++ 4 files changed, 279 insertions(+) create mode 100644 packages/effect/test/dataframe/README.md diff --git a/packages/effect/test/dataframe/DataFrame.test.ts b/packages/effect/test/dataframe/DataFrame.test.ts index e6ad911fb63..9a8ef34cb07 100644 --- a/packages/effect/test/dataframe/DataFrame.test.ts +++ b/packages/effect/test/dataframe/DataFrame.test.ts @@ -21,6 +21,7 @@ const types = (df: DataFrame.DataFrame) => describe("DataFrame", () => { describe("constructors", () => { + // DF-C001 (API contract): DataFrame.fromRows; pinned source in README.md. it("fromRows maps schema fields to Arrow types", () => { assert.deepStrictEqual(types(readings), [ "city:Utf8:false", @@ -31,6 +32,7 @@ describe("DataFrame", () => { assert.deepStrictEqual(DataFrame.toArray(readings, "station") as Array, ["s1", null, null]) }) + // DF-C002 (API contract): DataFrame.fromRows; pinned source in README.md. it("fromRows takes per-column type overrides", () => { const df = DataFrame.fromRows(Schema.Struct({ n: Schema.Number }), [{ n: 1 }, { n: 2 }], { types: { n: ArrowSchema.int32 } @@ -38,6 +40,7 @@ describe("DataFrame", () => { assert.deepStrictEqual(df.schema.fields[0].type, ArrowSchema.int32) }) + // DF-C003 (API contract): DataFrame.fromRows; pinned source in README.md. it("fromRows builds decimal columns from options.types", () => { const Price = Schema.Struct({ n: Schema.BigDecimal }) const df = DataFrame.fromRows(Price, [{ n: BigDecimal.make(123n, 2) }], { @@ -49,6 +52,7 @@ describe("DataFrame", () => { assert.isTrue(Result.isSuccess(DataFrame.validate(df, Price, { types: { n: ArrowSchema.decimal(18, 2) } }))) }) + // DF-C004 (API contract): DataFrame.fromRows; pinned source in README.md. it("fromRows rejects overrides that can't hold the field's values", () => { assert.throws( () => DataFrame.fromRows(Schema.Struct({ n: Schema.Number }), [], { types: { n: ArrowSchema.int64 } }), @@ -63,6 +67,7 @@ describe("DataFrame", () => { ) }) + // DF-C005 (API contract): DataFrame.fromRows; pinned source in README.md. it("fromRows rejects schemas without a column type", () => { interface Tree { readonly children: ReadonlyArray @@ -73,16 +78,19 @@ describe("DataFrame", () => { } }) + // DF-C006 (API contract): DataFrame.fromColumns; pinned source in README.md. it("fromColumns keeps typed array element types", () => { const df = DataFrame.fromColumns({ i: Int32Array.of(1, 2), b: BigInt64Array.of(1n, 2n), s: ["a", null] }) assert.deepStrictEqual(types(df), ["i:Int:false", "b:Int:false", "s:Utf8:true"]) assert.deepStrictEqual(DataFrame.toRows(df), [{ i: 1, b: 1n, s: "a" }, { i: 2, b: 2n, s: null }]) }) + // DF-C007 (API contract): DataFrame.fromColumns; pinned source in README.md. it("fromColumns rejects columns of different lengths", () => { assert.throws(() => DataFrame.fromColumns({ a: [1, 2], b: [1] }), DataFrame.DataFrameError) }) + // DF-C008 (API contract): DataFrame.empty; pinned source in README.md. it("empty has the schema's columns and no rows", () => { const df = DataFrame.empty(Reading) assert.strictEqual(df.length, 0) @@ -100,11 +108,13 @@ describe("DataFrame", () => { const column = (type: ArrowSchema.DataType, values: ReadonlyArray) => arrowFrame({ t: [type, values] }) const Utc = Schema.Struct({ t: Schema.DateTimeUtc }) + // DF-C009 (API contract): DataFrame.validate; pinned source in README.md. it("returns the frame typed by the schema, in schema order", () => { const result = DataFrame.validate(untyped, Schema.Struct({ temp: Schema.Number, city: Schema.String })) assert.deepStrictEqual(DataFrame.columnNames(Result.getOrThrow(result)), ["temp", "city"]) }) + // DF-C010 (API contract): DataFrame.validate; pinned source in README.md. it("fails on missing columns, wrong types, unexpected nulls and values the schema rejects", () => { const nested = DataFrame.fromRows( Schema.Struct({ p: Schema.Struct({ x: Schema.String }), xs: Schema.Array(Schema.NullOr(Schema.Number)) }), @@ -189,6 +199,7 @@ describe("DataFrame", () => { } }) + // DF-C011 (API contract): DataFrame.validate; pinned source in README.md. it("reads timestamps with a UTC time zone as DateTime.Utc", () => { const ts = ArrowSchema.timestamp("millisecond", "UTC") const nested = column(ArrowSchema.list(ArrowSchema.field("item", ts)), [[1000n]]) @@ -201,6 +212,7 @@ describe("DataFrame", () => { assert.deepStrictEqual(DataFrame.toRows(built), [{ t: DateTime.makeUnsafe(0) }]) }) + // DF-C012 (API contract): DataFrame.validate; pinned source in README.md. it("converts columns to the types fromRows gives the schema, exactly", () => { const Row = Schema.Struct({ n: Schema.Number, d: Schema.BigDecimal, t: Schema.DateTimeUtc }) const df = arrowFrame({ @@ -226,12 +238,14 @@ describe("DataFrame", () => { }) describe("Arrow interop", () => { + // DF-C013 (API contract): DataFrame.toArrowTable; pinned source in README.md. it("round-trips through the IPC stream format", () => { const bytes = Result.getOrThrow(ArrowIpc.encodeStream(DataFrame.toArrowTable(readings))) const back = DataFrame.fromArrowTable(Result.getOrThrow(ArrowIpc.decodeStream(bytes))) assert.isTrue(Equal.equals(back, readings)) }) + // DF-C014 (API contract): DataFrame.decodeArrowStream; pinned source in README.md. it.effect("decodes IPC stream bytes into a frame per batch", () => Effect.gen(function*() { const table = DataFrame.toArrowTable(readings) @@ -245,6 +259,7 @@ describe("DataFrame", () => { for (const frame of frames) assert.isTrue(Equal.equals(frame, readings)) })) + // DF-C015 (API contract): DataFrame.toArrowTable; pinned source in README.md. it.effect("exports late-selected columns", () => Effect.gen(function*() { const df = yield* Query.from(readings).pipe( @@ -256,6 +271,7 @@ describe("DataFrame", () => { assert.deepStrictEqual(DataFrame.toRows(DataFrame.fromArrowTable(table)), DataFrame.toRows(df)) })) + // DF-C016 (API contract): DataFrame.fromArrowTable; pinned source in README.md. it("imports a table without batches with the nullability of its fields", () => { const schema = ArrowSchema.make([ ArrowSchema.field("x", ArrowSchema.int8, { nullable: false }), @@ -267,6 +283,7 @@ describe("DataFrame", () => { ]) }) + // DF-C017 (API contract): DataFrame.fromArrowTable; pinned source in README.md. it.effect("imports sliced list columns with only their own elements", () => Effect.gen(function*() { const Lists = Schema.Struct({ xs: Schema.Array(Schema.Number) }) @@ -286,6 +303,7 @@ describe("DataFrame", () => { }) describe("values", () => { + // DF-C018 (API contract): DataFrame.equals; pinned source in README.md. it("compare structurally, NaN included", () => { const a = DataFrame.fromColumns({ x: [1, NaN, null] }) const b = DataFrame.fromColumns({ x: [1, NaN, null] }) @@ -294,6 +312,7 @@ describe("DataFrame", () => { assert.isFalse(Equal.equals(a, c)) }) + // DF-C019 (API contract): DataFrame.DataFrame; pinned source in README.md. it("iterate as rows", () => { assert.deepStrictEqual(Array.from(DataFrame.fromColumns({ city: DataFrame.toArray(readings, "city") })), [ { city: "Vienna" }, @@ -302,6 +321,7 @@ describe("DataFrame", () => { ]) }) + // DF-C020 (local policy): DataFrame.format; pinned source in README.md. it("format as a text table", () => { assert.strictEqual( DataFrame.format( diff --git a/packages/effect/test/dataframe/Query.test.ts b/packages/effect/test/dataframe/Query.test.ts index f8241828a39..6e992741e2a 100644 --- a/packages/effect/test/dataframe/Query.test.ts +++ b/packages/effect/test/dataframe/Query.test.ts @@ -43,12 +43,29 @@ const price = (d: string, precision: number, scale: number) => describe("Query", () => { describe("running", () => { + // DF-Q001 (API contract): Query.Query; pinned source in README.md. it.effect("is an Effect producing a frame", () => Effect.gen(function*() { const df = yield* Query.from(readings).pipe(Query.filter((c) => Expr.gt(c.temp, 20))) assert.deepStrictEqual(DataFrame.toArray(df, "city"), ["Vienna", "Berlin"]) })) + // DF-Q002 (API contract): Query.select; pinned source in README.md. + it.effect("expression callbacks run once at construction, not on execution", () => + Effect.gen(function*() { + let calls = 0 + const query = Query.from(readings).pipe(Query.select((c) => { + calls++ + return { city: c.city } + })) + assert.strictEqual(calls, 1) + const expected = [{ city: "Vienna" }, { city: "Berlin" }, { city: "Vienna" }] + assert.deepStrictEqual(yield* rows(query), expected) + assert.deepStrictEqual(yield* rows(query), expected) + assert.strictEqual(calls, 1) + })) + + // DF-Q003 (API contract): Query.Query; pinned source in README.md. it.effect("does no work until it runs", () => Effect.gen(function*() { const query = Query.table("missing", City).pipe(Query.filter((c) => Expr.eq(c.city, "Vienna"))) @@ -56,6 +73,7 @@ describe("Query", () => { assert.strictEqual(error.message, `unknown table "missing"`) })) + // DF-Q004 (API contract): Catalog.Catalog; pinned source in README.md. it.effect("reads each table once per run, from effects too", () => Effect.gen(function*() { let loads = 0 @@ -70,6 +88,7 @@ describe("Query", () => { assert.strictEqual(loads, 2) })) + // DF-Q005 (API contract): Catalog.Catalog; pinned source in README.md. it.effect("tables can be queries (views)", () => Effect.gen(function*() { const warm = Query.table("readings", Reading).pipe(Query.filter((c) => Expr.gt(c.temp, 20))) @@ -78,6 +97,7 @@ describe("Query", () => { assert.deepStrictEqual(result, [{ city: "Vienna" }, { city: "Berlin" }]) })) + // DF-Q006 (API contract): Query.table; pinned source in README.md. it.effect("checks tables against their declared columns", () => Effect.gen(function*() { const error = yield* Effect.flip(Query.table("readings", City).pipe(Effect.provide(tables))) @@ -109,6 +129,7 @@ describe("Query", () => { assert.deepStrictEqual(DataFrame.toRows(yield* ranges), [{ lo: 1, hi: 2 }]) })) + // DF-Q007 (API contract): Query.table; pinned source in README.md. it.effect("narrows tables to their declared columns", () => Effect.gen(function*() { const Temps = Schema.Struct({ temp: Schema.Number, city: Schema.String }) @@ -116,6 +137,7 @@ describe("Query", () => { assert.deepStrictEqual(DataFrame.columnNames(df), ["temp", "city"]) })) + // DF-Q008 (API contract): DataFrame.DataFrameError; pinned source in README.md. it.effect("fails with DataFrameError on invalid expressions", () => Effect.gen(function*() { const error = yield* Effect.flip(Query.from(readings).pipe(Query.filter((c) => c.city as any))) @@ -123,6 +145,7 @@ describe("Query", () => { assert.strictEqual(error.message, "filter predicate must be boolean, got Utf8") })) + // DF-Q009 (API contract): Expr.Expr; pinned source in README.md. it.effect("decimal results fail instead of overflowing 64 bits", () => Effect.gen(function*() { const nines = (rows: number) => @@ -148,6 +171,7 @@ describe("Query", () => { } })) + // DF-Q010 (API contract): Expr.round; pinned source in README.md. it.effect("round rejects negative or fractional places", () => Effect.gen(function*() { for (const places of [-1, 0.5]) { @@ -160,6 +184,7 @@ describe("Query", () => { }) describe("operations", () => { + // DF-Q011 (API contract): Query.slice; pinned source in README.md. it.effect("slice takes an offset and a length; negative offsets count from the end", () => Effect.gen(function*() { const n = Query.from(DataFrame.fromColumns({ n: Int32Array.of(0, 1, 2, 3, 4) })) @@ -173,12 +198,14 @@ describe("Query", () => { assert.deepStrictEqual(yield* values(Query.slice(n, 9)), []) })) + // DF-Q012 (API contract): Query.append; pinned source in README.md. it.effect("append stacks rows; dropNulls removes incomplete rows", () => Effect.gen(function*() { const df = yield* Query.from(readings).pipe(Query.append(Query.from(readings)), Query.dropNulls(["rain"])) assert.deepStrictEqual(DataFrame.toArray(df, "rain"), [0, 2, 0, 2]) })) + // DF-Q013 (API contract): Query.join; pinned source in README.md. it.effect("joins on several keys", () => Effect.gen(function*() { const right = DataFrame.fromColumns({ city: ["Vienna", "Vienna"], temp: [19, 21.5], note: ["cool", "warm"] }) @@ -191,6 +218,7 @@ describe("Query", () => { ]) })) + // DF-Q014 (API contract): Expr.cast; pinned source in README.md. it.effect("cast reports values that don't fit as null", () => Effect.gen(function*() { const df = DataFrame.fromColumns({ x: [1.9, -1.9, 3e10, NaN] }) @@ -228,6 +256,7 @@ describe("Query", () => { assert.deepStrictEqual(overflows, [{ t: null, d: null, i: null }]) })) + // DF-Q015 (API contract): Expr.Expr; pinned source in README.md. it.effect("implicit conversions to a common type fail on values it can't hold", () => Effect.gen(function*() { const big = Query.from(price("6000000000000000000", 19, 0)) @@ -310,6 +339,7 @@ describe("Query", () => { } })) + // DF-Q016 (API contract): Expr.Expr; pinned source in README.md. it.effect("temporal operands of mixed units meet in the coarser unit, as in Polars", () => Effect.gen(function*() { // Expected values from Polars 2.1.0: timestamps round down, durations toward zero. @@ -376,6 +406,7 @@ describe("Query", () => { }) describe("explain", () => { + // DF-Q017 (local policy): Query.explain; pinned source in README.md. it.effect("shows the rewritten plan", () => Effect.gen(function*() { const query = Query.table("readings", Reading).pipe( @@ -404,6 +435,7 @@ describe("Query", () => { assert.deepStrictEqual(yield* rows(query).pipe(Effect.provide(tables)), [{ city: "Vienna", fahrenheit: 70.7 }]) })) + // DF-Q018 (local policy): Query.explain; pinned source in README.md. it.effect("pushes key predicates below groups and through renames", () => Effect.gen(function*() { const query = Query.from(readings).pipe( @@ -426,6 +458,7 @@ describe("Query", () => { assert.deepStrictEqual(yield* rows(query), [{ town: "Vienna", n: 2, mean: 20.25 }]) })) + // DF-Q019 (local policy): Query.explain; pinned source in README.md. it.effect("keeps predicates on join keys off the right side, whose keys the join drops", () => Effect.gen(function*() { // The result has the left side's Int8 key, where 100 + 100 wraps to -56; @@ -437,6 +470,7 @@ describe("Query", () => { assert.deepStrictEqual(yield* rows(query), [{ x: 100, a: 1, b: 2 }]) })) + // DF-Q020 (local policy): Query.explain; pinned source in README.md. it.effect("keeps predicates above left joins when they read the right side", () => Effect.gen(function*() { const query = Query.from(readings).pipe( @@ -456,6 +490,7 @@ describe("Query", () => { }) describe("plans", () => { + // DF-Q021 (API contract): Plan.Node; pinned source in README.md. it("are plain data", () => { const query = Query.from(readings).pipe(Query.filter((c) => Expr.gt(c.temp, 20)), Query.pick("city")) assert.deepStrictEqual(query.plan, { @@ -474,6 +509,7 @@ describe("Query", () => { }) }) + // DF-Q022 (API contract): Plan.NodeFromJson; pinned source in README.md. it("encode to JSON without losing literal types", () => { const values: ReadonlyArray = ["NaN", NaN, -0, Infinity, 2n ** 70n, "12", 12, true, null] const expression: Plan.Expression = { @@ -507,6 +543,7 @@ describe("Query", () => { assert.deepStrictEqual(back, expression) }) + // DF-Q023 (API contract): Plan.NodeFromJson; pinned source in README.md. it.effect("round-trip through JSON with their inputs", () => Effect.gen(function*() { const query = Query.from(readings).pipe(Query.groupBy("city"), Query.agg((c) => ({ max: Expr.max(c.temp) }))) @@ -515,11 +552,13 @@ describe("Query", () => { assert.deepStrictEqual(yield* rows(decoded), yield* rows(query)) })) + // DF-Q024 (API contract): Plan.NodeFromJson; pinned source in README.md. it("reject malformed JSON when decoding", () => { const json = { _tag: "Filter", input: { _tag: "Table", name: "t" }, predicate: { _tag: "Call", fn: "explode" } } assert.throws(() => Schema.decodeUnknownSync(Plan.NodeFromJson)(json)) }) + // DF-Q025 (API contract): Plan.Expression; pinned source in README.md. it.effect("fail at run time on the wrong number of arguments", () => Effect.gen(function*() { const plan: Plan.Node = { @@ -531,6 +570,7 @@ describe("Query", () => { assert.strictEqual(error.message, "gt takes 2 argument(s), got 1") })) + // DF-Q026 (API contract): Expr.toPlan; pinned source in README.md. it("expressions convert to and from plans", () => { const e = Expr.add(Expr.lit(1n), 2n) assert.deepStrictEqual(Expr.toPlan(e), { @@ -551,6 +591,7 @@ describe("Query", () => { for (const source of ["values", "frames", "batches"] as const) { describe(source, () => { for (const c of cases) { + // DF-I-POLARS//: interoperability evidence; pinned oracle provenance in README.md. it.effect(`${c.id}: ${c.description}`, () => Effect.gen(function*() { const df = yield* Oracle.run(c, source) @@ -568,6 +609,7 @@ describe("Query", () => { const streamed = (query: Query.Query, source: Catalog.Source) => rows(query).pipe(Effect.provide(Catalog.layer({ t: source }))) + // DF-Q027 (API contract): Catalog.Catalog; pinned source in README.md. it.effect("a slice that has its rows pulls no further batch", () => Effect.gen(function*() { const one = Stream.concat(Stream.succeed(DataFrame.fromColumns({ x: [1] })), Stream.die("pulled again")) @@ -577,6 +619,7 @@ describe("Query", () => { } })) + // DF-Q028 (API contract): Expr.sum; pinned source in README.md. it.effect("decimal sums don't depend on batch boundaries", () => Effect.gen(function*() { const query = Query.table("t", Schema.Struct({ s: Schema.String })).pipe( @@ -588,6 +631,7 @@ describe("Query", () => { assert.deepStrictEqual(result, [{ total: BigDecimal.fromBigInt(0n) }]) })) + // DF-Q029 (API contract): Catalog.Catalog; pinned source in README.md. it.effect("batches convert to the declared column types; undeclared, they must share their columns", () => Effect.gen(function*() { const source = Stream.make(price("1.2", 18, 1), price("1.23", 18, 2), price("1.2", 18, 2)) @@ -624,6 +668,7 @@ describe("Query", () => { assert.strictEqual(error.message, `column "d" has values that Decimal[18, 1] can't hold exactly`) })) + // DF-Q030 (API contract): Catalog.Catalog; pinned source in README.md. it.effect("results over inputs of different types are those over their concatenation, streamed or not", () => Effect.gen(function*() { type Frame = DataFrame.DataFrame<{ readonly x: number }> @@ -776,6 +821,7 @@ describe("Query", () => { } })) + // DF-Q031 (API contract): Query.aggregate; pinned source in README.md. it.effect("zero-row batches aggregate like an empty table", () => Effect.gen(function*() { const query = Query.aggregate(t, (c) => ({ n: Expr.count(), total: Expr.sum(c.x), mean: Expr.mean(c.x) })) @@ -783,6 +829,7 @@ describe("Query", () => { assert.deepStrictEqual(yield* streamed(query, empty), [{ n: 0, total: 0, mean: null }]) })) + // DF-Q032 (API contract): Query.stream; pinned source in README.md. it.effect("results without rows keep the types of their columns", () => Effect.gen(function*() { const Nested = Schema.Struct({ xs: Schema.Array(Schema.Number), s: Schema.Struct({ a: Schema.Number }) }) @@ -808,6 +855,7 @@ describe("Query", () => { } })) + // DF-Q033 (API contract): Query.stream; pinned source in README.md. it.effect("results without rows are those in memory, or fail when their columns are undeclared and unread", () => Effect.gen(function*() { const x = (...values: Array) => DataFrame.fromColumns({ x: Int8Array.from(values) }) @@ -871,6 +919,7 @@ describe("Query", () => { } })) + // DF-Q034 (API contract): Catalog.Catalog; pinned source in README.md. it.effect("reads a table named twice once per run, only if its batches are needed", () => Effect.gen(function*() { let loads = 0 @@ -898,6 +947,7 @@ describe("Query", () => { assert.strictEqual(loads, 1) })) + // DF-Q035 (API contract): Query.explain; pinned source in README.md. it.effect("explain shows the plan without reading the stream", () => Effect.gen(function*() { const plan = yield* Query.explain(Query.head(t, 1)).pipe( @@ -906,6 +956,7 @@ describe("Query", () => { assert.strictEqual(plan, [`Slice offset 0 length 1`, ` Table t [x]`].join("\n")) })) + // DF-Q036 (API contract): Query.head; pinned source in README.md. it.effect("a head stops reading the stream", () => Effect.gen(function*() { let pulled = 0 @@ -919,6 +970,7 @@ describe("Query", () => { assert.isBelow(pulled, batches.length) })) + // DF-Q037 (API contract): Query.stream; pinned source in README.md. it.effect("Query.stream emits a frame per non-empty batch", () => Effect.gen(function*() { const sizes = yield* t.pipe( @@ -1026,6 +1078,7 @@ describe("Query", () => { const sortKey = (c: { readonly [K in keyof Row]: Expr.Expr }, key: Key) => key.descending ? Expr.desc(c[key.name] as Expr.Expr) : Expr.asc(c[key.name] as Expr.Expr) + // DF-Q038 (API contract): Query.sort; pinned source in README.md. it.effect("sorts by every column type, in both directions, nulls first or last", () => Effect.gen(function*() { for (const name of ["f", "i", "b", "s", "p"] as const) { @@ -1038,6 +1091,7 @@ describe("Query", () => { } })) + // DF-Q039 (API contract): Query.sort; pinned source in README.md. it.effect("sorts stably by several keys, also when sliced (top-k)", () => Effect.gen(function*() { const keys: ReadonlyArray = [ @@ -1064,6 +1118,7 @@ describe("Query", () => { } })) + // DF-Q040 (API contract): Query.groupBy; pinned source in README.md. it.effect("groups by keys of every type", () => Effect.gen(function*() { for (const keys of [["f"], ["b"], ["p", "i"], ["s", "b", "p"]] as const) { @@ -1089,6 +1144,7 @@ describe("Query", () => { ) })) + // DF-Q041 (API contract): Query.join; pinned source in README.md. it.effect("joins on a high-cardinality key", () => Effect.gen(function*() { const right = generate(0x1b873593, 4000, 3000).map((r) => ({ i: r.i ?? 0, rs: r.s })) diff --git a/packages/effect/test/dataframe/README.md b/packages/effect/test/dataframe/README.md new file mode 100644 index 00000000000..fcacf6adff8 --- /dev/null +++ b/packages/effect/test/dataframe/README.md @@ -0,0 +1,186 @@ +# DataFrame contract assertions + +Reference for maintainers reviewing the DataFrame layer. This is an assertion-to-test inventory, not a claim of complete standards or API coverage. Existing regressions remain in their conventional suites. + +## Authority and classification + +The approved API baseline is commit `d7121ccbd69f9f437ac60b09a704f216c0ca44ee`. Links below pin that immutable revision of `packages/effect/src/dataframe`; API documentation, not an external dataframe standard, defines the contract. The source symbol locates the surrounding documentation; it is not an assertion that every implementation detail is normative. + +- **C**: approved public API contract; **P**: local regression/policy detail (not an external requirement). +- **I**: interoperability/differential evidence; an independent engine is an oracle, not the authority for this API. +- No row is an external normative requirement. Gaps and exclusions are explicit below. +- IDs are permanent: retain them when tests move or titles change; do not renumber. Test comments resolve to this inventory and its pinned source. + +## Runtime assertions + +The assertion column is the exact leaf test title. Combine it with the file and surrounding describe blocks to select the test; each listed title is unique within its file. A row covers the concrete inputs/assertions in that test, not all possible values of the API. + +### `DataFrame.test.ts` + +| ID | Kind | Assertion / exact test title | API source | +| ------- | ---- | ------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| DF-C001 | C | fromRows maps schema fields to Arrow types | [DataFrame.fromRows](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/DataFrame.ts#L192) | +| DF-C002 | C | fromRows takes per-column type overrides | [DataFrame.fromRows](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/DataFrame.ts#L192) | +| DF-C003 | C | fromRows builds decimal columns from options.types | [DataFrame.fromRows](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/DataFrame.ts#L192) | +| DF-C004 | C | fromRows rejects overrides that can't hold the field's values | [DataFrame.fromRows](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/DataFrame.ts#L192) | +| DF-C005 | C | fromRows rejects schemas without a column type | [DataFrame.fromRows](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/DataFrame.ts#L192) | +| DF-C006 | C | fromColumns keeps typed array element types | [DataFrame.fromColumns](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/DataFrame.ts#L274) | +| DF-C007 | C | fromColumns rejects columns of different lengths | [DataFrame.fromColumns](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/DataFrame.ts#L274) | +| DF-C008 | C | empty has the schema's columns and no rows | [DataFrame.empty](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/DataFrame.ts#L360) | +| DF-C009 | C | returns the frame typed by the schema, in schema order | [DataFrame.validate](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/DataFrame.ts#L396) | +| DF-C010 | C | fails on missing columns, wrong types, unexpected nulls and values the schema rejects | [DataFrame.validate](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/DataFrame.ts#L396) | +| DF-C011 | C | reads timestamps with a UTC time zone as DateTime.Utc | [DataFrame.validate](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/DataFrame.ts#L396) | +| DF-C012 | C | converts columns to the types fromRows gives the schema, exactly | [DataFrame.validate](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/DataFrame.ts#L396) | +| DF-C013 | C | round-trips through the IPC stream format | [DataFrame.toArrowTable](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/DataFrame.ts#L448) | +| DF-C014 | C | decodes IPC stream bytes into a frame per batch | [DataFrame.decodeArrowStream](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/DataFrame.ts#L336) | +| DF-C015 | C | exports late-selected columns | [DataFrame.toArrowTable](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/DataFrame.ts#L448) | +| DF-C016 | C | imports a table without batches with the nullability of its fields | [DataFrame.fromArrowTable](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/DataFrame.ts#L314) | +| DF-C017 | C | imports sliced list columns with only their own elements | [DataFrame.fromArrowTable](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/DataFrame.ts#L314) | +| DF-C018 | C | compare structurally, NaN included | [DataFrame.equals](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/DataFrame.ts#L478) | +| DF-C019 | C | iterate as rows | [DataFrame.DataFrame](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/DataFrame.ts#L81) | +| DF-C020 | P | format as a text table | [DataFrame.format](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/DataFrame.ts#L467) | + +### `Query.test.ts` + +| ID | Kind | Assertion / exact test title | API source | +| ------- | ---- | ---------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| DF-Q001 | C | is an Effect producing a frame | [Query.Query](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Query.ts#L108) | +| DF-Q002 | C | expression callbacks run once at construction, not on execution | [Query.select](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Query.ts#L7-L18) | +| DF-Q003 | C | does no work until it runs | [Query.Query](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Query.ts#L108) | +| DF-Q004 | C | reads each table once per run, from effects too | [Catalog.Catalog](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Catalog.ts#L6-L19) | +| DF-Q005 | C | tables can be queries (views) | [Catalog.Catalog](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Catalog.ts#L6-L19) | +| DF-Q006 | C | checks tables against their declared columns | [Query.table](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Query.ts#L254) | +| DF-Q007 | C | narrows tables to their declared columns | [Query.table](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Query.ts#L254) | +| DF-Q008 | C | fails with DataFrameError on invalid expressions | [DataFrame.DataFrameError](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/DataFrame.ts#L137) | +| DF-Q009 | C | decimal results fail instead of overflowing 64 bits | [Expr.Expr](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Expr.ts#L24-L48) | +| DF-Q010 | C | round rejects negative or fractional places | [Expr.round](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Expr.ts#L455) | +| DF-Q011 | C | slice takes an offset and a length; negative offsets count from the end | [Query.slice](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Query.ts#L391) | +| DF-Q012 | C | append stacks rows; dropNulls removes incomplete rows | [Query.append](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Query.ts#L492) | +| DF-Q013 | C | joins on several keys | [Query.join](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Query.ts#L680) | +| DF-Q014 | C | cast reports values that don't fit as null | [Expr.cast](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Expr.ts#L685) | +| DF-Q015 | C | implicit conversions to a common type fail on values it can't hold | [Expr.Expr](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Expr.ts#L24-L48) | +| DF-Q016 | C | temporal operands of mixed units meet in the coarser unit, as in Polars | [Expr.Expr](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Expr.ts#L24-L48) | +| DF-Q017 | P | shows the rewritten plan | [Query.explain](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Query.ts#L321) | +| DF-Q018 | P | pushes key predicates below groups and through renames | [Query.explain](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Query.ts#L321) | +| DF-Q019 | P | keeps predicates on join keys off the right side, whose keys the join drops | [Query.explain](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Query.ts#L321) | +| DF-Q020 | P | keeps predicates above left joins when they read the right side | [Query.explain](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Query.ts#L321) | +| DF-Q021 | C | are plain data | [Plan.Node](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Plan.ts#L588) | +| DF-Q022 | C | encode to JSON without losing literal types | [Plan.NodeFromJson](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Plan.ts#L1103) | +| DF-Q023 | C | round-trip through JSON with their inputs | [Plan.NodeFromJson](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Plan.ts#L1103) | +| DF-Q024 | C | reject malformed JSON when decoding | [Plan.NodeFromJson](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Plan.ts#L1103) | +| DF-Q025 | C | fail at run time on the wrong number of arguments | [Plan.Expression](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Plan.ts#L380) | +| DF-Q026 | C | expressions convert to and from plans | [Expr.toPlan](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Expr.ts#L251) | +| DF-Q027 | C | a slice that has its rows pulls no further batch | [Catalog.Catalog](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Catalog.ts#L6-L19) | +| DF-Q028 | C | decimal sums don't depend on batch boundaries | [Expr.sum](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Expr.ts#L827) | +| DF-Q029 | C | batches convert to the declared column types; undeclared, they must share their columns | [Catalog.Catalog](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Catalog.ts#L6-L19) | +| DF-Q030 | C | results over inputs of different types are those over their concatenation, streamed or not | [Catalog.Catalog](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Catalog.ts#L6-L19) | +| DF-Q031 | C | zero-row batches aggregate like an empty table | [Query.aggregate](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Query.ts#L653) | +| DF-Q032 | C | results without rows keep the types of their columns | [Query.stream](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Query.ts#L310) | +| DF-Q033 | C | results without rows are those in memory, or fail when their columns are undeclared and unread | [Query.stream](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Query.ts#L310) | +| DF-Q034 | C | reads a table named twice once per run, only if its batches are needed | [Catalog.Catalog](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Catalog.ts#L6-L19) | +| DF-Q035 | C | explain shows the plan without reading the stream | [Query.explain](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Query.ts#L321) | +| DF-Q036 | C | a head stops reading the stream | [Query.head](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Query.ts#L407) | +| DF-Q037 | C | Query.stream emits a frame per non-empty batch | [Query.stream](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Query.ts#L310) | +| DF-Q038 | C | sorts by every column type, in both directions, nulls first or last | [Query.sort](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Query.ts#L352) | +| DF-Q039 | C | sorts stably by several keys, also when sliced (top-k) | [Query.sort](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Query.ts#L352) | +| DF-Q040 | C | groups by keys of every type | [Query.groupBy](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Query.ts#L610) | +| DF-Q041 | C | joins on a high-cardinality key | [Query.join](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Query.ts#L680) | + +## Oracle provenance and differential assertions + +**DF-I-POLARS//** (I) resolves exactly to `Query.test.ts > Query > Polars oracle > > : `. Sources are `values`, `frames`, and `batches`; case IDs and descriptions are the existing entries in the pinned [spec.json](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/test/fixtures/dataframe/gold/spec.json) and [extended.json](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/test/fixtures/dataframe/gold/extended.json). This expansion rule identifies each assertion without duplicating the corpus or renaming tests. The three modes exercise inline frames, catalog frames and streamed batches against the same expected columns, types, values and row order. + +The API authority is [Query semantics](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Query.ts#L46-L57) and [Expr semantics](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Expr.ts#L22-L48). Separately, expected results were generated with **Python Polars 2.1.0**, recorded in the corpus metadata and [generator invocation](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/test/fixtures/dataframe/generate_gold.py#L1-L32). No oracle was regenerated in this change. + +Comparison policy is pinned in [Compare.ts](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/test/dataframe/oracle/Compare.ts#L1-L45): float relative tolerance `1e-9` plus `1e-300`, NaN equals NaN, and signed zeros compare equal. `group/small-ints` allows Int32 sums for i8/u8 instead of Polars Int64; values must still match. Temporal values compare via epoch integers. These are local comparison policies, not Arrow requirements. + +## PostgreSQL / PGlite assertions + +The actual SQL suite is **`packages/sql/pglite/test/DataFrameSql.test.ts`**, not `packages/sql/sql-pg/test`. Its public contract is [Sql.ts](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Sql.ts#L1-L45). Each title below is under `effect/dataframe on PostgreSQL (PGlite)`. This is PostgreSQL integration evidence, not SQL-standard conformance. The dependency and resolved version are reproducible from the pinned [package manifest](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/sql/pglite/package.json) and [pnpm lockfile](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/pnpm-lock.yaml). + +**DF-I-SQL/** (I) identifies the exact test named by each `cases` entry, executed by `it.effect(name, () => both(query))`. This includes generated temporal-cast and mixed-unit names: their literal templates and unit arrays in the [pinned suite](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/sql/pglite/test/DataFrameSql.test.ts#L169-L485) define the finite expansion. `both` compares in-memory results with PGlite, including types, length and ordered values (numeric tolerance `1e-9 * max(1, abs(actual))`). Shared compiler/API bugs can escape differential agreement; it is not an independent proof of the whole API. + +| ID | Kind | Exact test title | API authority | +| ------- | ----- | -------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------- | +| DF-S001 | C / I | negative slices clamp both ends | [Sql contract](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Sql.ts#L1-L45) | +| DF-S002 | C / I | rounds decimals half to even, keeping the scale | [Sql contract](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Sql.ts#L1-L45) | +| DF-S003 | C / I | errors carry the database's message | [Sql contract](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Sql.ts#L1-L45) | +| DF-S004 | C / I | tables must satisfy their schema, as in memory | [Sql contract](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Sql.ts#L1-L45) | +| DF-S005 | C / I | tables reject values their declared types can't hold exactly, as in memory | [Sql contract](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Sql.ts#L1-L45) | +| DF-S006 | C / I | tables are checked on the rows the query reads, whatever the relation | [Sql contract](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Sql.ts#L1-L45) | +| DF-S007 | C / I | tables are read from their names, whatever the compiler names | [Sql contract](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Sql.ts#L1-L45) | +| DF-S008 | C / I | columns are one to PostgreSQL when their first 63 bytes are | [Sql contract](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Sql.ts#L1-L45) | +| DF-S009 | C / I | timestamps with time zones are UTC, whatever the session's time zone | [Sql contract](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Sql.ts#L1-L45) | +| DF-S010 | C / I | filters, derives and sorts | [Sql contract](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Sql.ts#L1-L45) | +| DF-S011 | C / I | groups in order of first appearance | [Sql contract](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Sql.ts#L1-L45) | +| DF-S012 | C / I | joins keep the left row order | [Sql contract](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Sql.ts#L1-L45) | +| DF-S013 | C / I | semi and anti joins, unique and slices | [Sql contract](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Sql.ts#L1-L45) | +| DF-S014 | C / I | window functions over partitions | [Sql contract](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Sql.ts#L1-L45) | +| DF-S015 | C / I | dates and times | [Sql contract](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Sql.ts#L1-L45) | +| DF-S016 | C / I | explain shows the SQL | [Sql contract](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Sql.ts#L1-L45) | + +## Compiler assertions + +`packages/effect/typetest/DataFrame.tst.ts` contains the following exact `describe` / `it` names. These are public API inference and rejection checks, not runtime or wire-format evidence. Each entry covers only its concrete expressions; it does not establish every overload or editor quick-info rendering. + +### `DataFrame` + +- `fromRows takes the row type from the schema` +- `fromRows, empty, validate and Query.table read missing values back as null` +- `validate and Query.table take Arrow types for the schema's fields` +- `fromColumns infers column types` + +### `Query` + +- `from and table take the row type from the frame or schema` +- `is an Effect producing a frame of its rows` +- `filter keeps the row type` +- `select and withColumns compute the row type, nulls included` +- `pick, omit and rename` +- `groupBy and agg` +- `joins add the other side's columns; left joins make them nullable` +- `rejects mistakes` + +### `Expr` + +- `propagates nullability` +- `data-last forms` + +### `Windows and reshaping` + +- `over keeps the value type; window functions type their results` +- `pivot with declared columns is typed; without them rows are untyped` +- `unpivot names the variable and value columns` +- `explode and unnest` + +### `Types` + +- `dates, times and durations` +- `lists and structs` +- `decimals` + +### `Streaming` + +- `Query.stream emits frames of the row type` + +Rejection assertions include unknown Arrow override fields (`DataFrame`), unknown columns/group keys, incompatible comparisons/arithmetic, non-aggregate aggregation, aggregate predicates and duplicate non-key join columns (`Query` / `rejects mistakes`), exploding a non-list column (`Windows and reshaping` / `explode and unnest`), and extracting a year from a number (`Types` / `dates, times and durations`). Other entries assert exact inferred types, including nullability, Effect errors and stream elements. + +## Exclusions and remaining gaps + +- This inventory does not assert Arrow IPC or FlatBuffers wire conformance; DataFrame Arrow tests cover the adapter API and same-library round trips only. Wire formats belong to their own layer inventories. +- Polars corpus exclusions remain as recorded in `generate_gold.py:DEVIATIONS`: integral float literals, float-to-string formatting, automatic join suffixes, and mixed bigint/number arithmetic. No claim of universal Polars equivalence. +- SQL intentionally excludes pivot/unpivot/explode/unnest and databases other than PostgreSQL. Overflow, division by zero, collation and sub-microsecond temporal behavior follow the documented SQL deviations, not unconditional parity. PGlite is not a live PostgreSQL server/version matrix. +- The callback timing contract now has a direct test (DF-Q002); it supplements, rather than replaces, lazy loading and per-run catalog tests. +- Dedicated coverage for public guards, every dual overload, cancellation/fairness, aliasing/zero-copy ownership, and every malformed plan/IPC input is not established by this inventory. Those are gaps, not passing assertions. +- Compiler inference/rejection coverage is inventoried above and is not counted as runtime coverage. Exact diagnostic strings and explain/format layouts are local regression details even where a test also verifies a public failure or result contract. + +## Validation + +Run from the repository root: + +```sh +nix develop -c pnpm test --run packages/effect/test/dataframe/DataFrame.test.ts packages/effect/test/dataframe/Query.test.ts packages/sql/pglite/test/DataFrameSql.test.ts +nix develop -c pnpm test-types DataFrame +``` + +Keep existing regressions and oracle cases when updating the inventory. Do not turn an excluded case into a coverage claim without a focused assertion and an authoritative contract source. diff --git a/packages/sql/pglite/test/DataFrameSql.test.ts b/packages/sql/pglite/test/DataFrameSql.test.ts index fc2b479aed8..108f51e5e50 100644 --- a/packages/sql/pglite/test/DataFrameSql.test.ts +++ b/packages/sql/pglite/test/DataFrameSql.test.ts @@ -482,14 +482,17 @@ describe("effect/dataframe on PostgreSQL (PGlite)", { concurrent: false }, () => } }) + // DF-I-SQL/: differential evidence; exact expansion and pins in effect/test/dataframe/README.md. for (const [name, query] of cases) it.effect(name, () => both(query)) + // DF-S001: Sql API contract; pins and oracle policy in effect/test/dataframe/README.md. it.effect("negative slices clamp both ends", () => Effect.gen(function*() { assert.deepStrictEqual(yield* rowsOnSql(xs.pipe(Query.slice(-5, 1))), []) assert.deepStrictEqual(yield* rowsOnSql(xs.pipe(Query.slice(-5, 3))), [{ x: 1 }]) })) + // DF-S002: Sql API contract; pins and oracle policy in effect/test/dataframe/README.md. it.effect("rounds decimals half to even, keeping the scale", () => Effect.gen(function*() { const df = yield* onSql( @@ -513,6 +516,7 @@ describe("effect/dataframe on PostgreSQL (PGlite)", { concurrent: false }, () => ) })) + // DF-S003: Sql API contract; pins and oracle policy in effect/test/dataframe/README.md. it.effect("errors carry the database's message", () => Effect.gen(function*() { assert.include( @@ -521,6 +525,7 @@ describe("effect/dataframe on PostgreSQL (PGlite)", { concurrent: false }, () => ) })) + // DF-S004: Sql API contract; pins and oracle policy in effect/test/dataframe/README.md. it.effect("tables must satisfy their schema, as in memory", () => Effect.gen(function*() { const contract = tableOf("contract") @@ -589,6 +594,7 @@ describe("effect/dataframe on PostgreSQL (PGlite)", { concurrent: false }, () => } })) + // DF-S005: Sql API contract; pins and oracle policy in effect/test/dataframe/README.md. it.effect("tables reject values their declared types can't hold exactly, as in memory", () => Effect.gen(function*() { for ( @@ -618,6 +624,7 @@ describe("effect/dataframe on PostgreSQL (PGlite)", { concurrent: false }, () => } })) + // DF-S006: Sql API contract; pins and oracle policy in effect/test/dataframe/README.md. it.effect("tables are checked on the rows the query reads, whatever the relation", () => Effect.gen(function*() { const sql = yield* SqlClient @@ -652,6 +659,7 @@ describe("effect/dataframe on PostgreSQL (PGlite)", { concurrent: false }, () => ]) })) + // DF-S007: Sql API contract; pins and oracle policy in effect/test/dataframe/README.md. it.effect("tables are read from their names, whatever the compiler names", () => Effect.gen(function*() { const s = (name: string) => Query.table(name, Schema.Struct({ s: Schema.String })) @@ -673,6 +681,7 @@ describe("effect/dataframe on PostgreSQL (PGlite)", { concurrent: false }, () => assert.deepStrictEqual(yield* rowsOnSql(two), [{ s: "57" }, { s: "good" }]) })) + // DF-S008: Sql API contract; pins and oracle policy in effect/test/dataframe/README.md. it.effect("columns are one to PostgreSQL when their first 63 bytes are", () => Effect.gen(function*() { // Declarations of one table read its one column `"z".repeat(63)`, each with its checks. @@ -694,6 +703,7 @@ describe("effect/dataframe on PostgreSQL (PGlite)", { concurrent: false }, () => } })) + // DF-S009: Sql API contract; pins and oracle policy in effect/test/dataframe/README.md. it.effect("timestamps with time zones are UTC, whatever the session's time zone", () => Effect.gen(function*() { const sql = yield* SqlClient @@ -711,6 +721,7 @@ describe("effect/dataframe on PostgreSQL (PGlite)", { concurrent: false }, () => assert.deepStrictEqual(rows, [{ at: DateTime.makeUnsafe("2024-01-01T00:30:00Z"), hour: 0, early: true }]) })) + // DF-S010: Sql API contract; pins and oracle policy in effect/test/dataframe/README.md. it.effect("filters, derives and sorts", () => both(table.pipe( Query.filter((c) => Expr.and(Expr.isNotNull(c.units), Expr.gt(c.price, 20))), @@ -719,6 +730,7 @@ describe("effect/dataframe on PostgreSQL (PGlite)", { concurrent: false }, () => Query.head(10) ))) + // DF-S011: Sql API contract; pins and oracle policy in effect/test/dataframe/README.md. it.effect("groups in order of first appearance", () => both(table.pipe( Query.groupBy("region"), @@ -735,6 +747,7 @@ describe("effect/dataframe on PostgreSQL (PGlite)", { concurrent: false }, () => })) ))) + // DF-S012: Sql API contract; pins and oracle policy in effect/test/dataframe/README.md. it.effect("joins keep the left row order", () => both(table.pipe( Query.pick("id", "region"), @@ -742,6 +755,7 @@ describe("effect/dataframe on PostgreSQL (PGlite)", { concurrent: false }, () => Query.filter((c) => Expr.lt(c.id, 15)) ))) + // DF-S013: Sql API contract; pins and oracle policy in effect/test/dataframe/README.md. it.effect("semi and anti joins, unique and slices", () => Effect.gen(function*() { yield* both(table.pipe(Query.join(Query.table("regions", Region), { on: "region", how: "anti" }))) @@ -750,6 +764,7 @@ describe("effect/dataframe on PostgreSQL (PGlite)", { concurrent: false }, () => yield* both(table.pipe(Query.slice(7, 4))) })) + // DF-S014: Sql API contract; pins and oracle policy in effect/test/dataframe/README.md. it.effect("window functions over partitions", () => both(table.pipe( Query.select((c) => ({ @@ -762,6 +777,7 @@ describe("effect/dataframe on PostgreSQL (PGlite)", { concurrent: false }, () => })) ))) + // DF-S015: Sql API contract; pins and oracle policy in effect/test/dataframe/README.md. it.effect("dates and times", () => both(table.pipe( Query.select((c) => ({ @@ -773,6 +789,7 @@ describe("effect/dataframe on PostgreSQL (PGlite)", { concurrent: false }, () => })) ))) + // DF-S016: Sql API contract; pins and oracle policy in effect/test/dataframe/README.md. it.effect("explain shows the SQL", () => Effect.gen(function*() { const text = yield* Query.explain(table.pipe(Query.head(1))).pipe( From 23727596c217a0a913c2e66f6d26b8f42fedd689 Mon Sep 17 00:00:00 2001 From: Effect Bot Date: Sun, 15 Nov 2026 20:18:46 +0000 Subject: [PATCH 63/88] Cover DataFrame cooperative interruption and buffer reuse contracts --- .../effect/test/dataframe/DataFrame.test.ts | 20 ++++++ packages/effect/test/dataframe/Query.test.ts | 71 ++++++++++++++++++- packages/effect/test/dataframe/README.md | 8 ++- 3 files changed, 97 insertions(+), 2 deletions(-) diff --git a/packages/effect/test/dataframe/DataFrame.test.ts b/packages/effect/test/dataframe/DataFrame.test.ts index 9a8ef34cb07..5a1ed205857 100644 --- a/packages/effect/test/dataframe/DataFrame.test.ts +++ b/packages/effect/test/dataframe/DataFrame.test.ts @@ -98,6 +98,26 @@ describe("DataFrame", () => { }) }) + // DF-C021 (API contract): compatible Arrow layouts; pinned source in README.md. + it("reuses compatible numeric buffers across Arrow conversion", () => { + const schema = ArrowSchema.make([ + ArrowSchema.field("i", ArrowSchema.int32), + ArrowSchema.field("n", ArrowSchema.float64) + ]) + const input = Result.getOrThrow(ArrowTable.fromColumns(schema, [[1, -2, 3], [1.5, -2.5, 0]])) + const frame = DataFrame.fromArrowTable(input) + const output = DataFrame.toArrowTable(frame) + assert.deepStrictEqual(DataFrame.toRows(frame), [{ i: 1, n: 1.5 }, { i: -2, n: -2.5 }, { i: 3, n: 0 }]) + assert.deepStrictEqual(ArrowTable.toRows(output), DataFrame.toRows(frame)) + for (let i = 0; i < 2; i++) { + const before = input.batches[0].columns[i].buffers[0] + const after = output.batches[0].columns[i].buffers[0] + assert.strictEqual(after.buffer, before.buffer) + assert.strictEqual(after.byteOffset, before.byteOffset) + assert.strictEqual(after.byteLength, before.byteLength) + } + }) + describe("validate", () => { const untyped = DataFrame.fromArrowTable(DataFrame.toArrowTable(readings)) const arrowFrame = (columns: Record]>) => diff --git a/packages/effect/test/dataframe/Query.test.ts b/packages/effect/test/dataframe/Query.test.ts index 6e992741e2a..a5df84d36b5 100644 --- a/packages/effect/test/dataframe/Query.test.ts +++ b/packages/effect/test/dataframe/Query.test.ts @@ -1,7 +1,19 @@ import { assert, describe, it } from "@effect/vitest" -import { BigDecimal, type DateTime, type Duration, Effect, Result, Schema, Stream } from "effect" +import { + BigDecimal, + type DateTime, + type Duration, + Effect, + Exit, + Fiber, + Result, + Scheduler, + Schema, + Stream +} from "effect" import { ArrowSchema, ArrowTable } from "effect/arrow" import { Catalog, DataFrame, Expr, Plan, Query } from "effect/dataframe" +import { vi } from "vitest" import * as Compare from "./oracle/Compare.ts" import * as Gold from "./oracle/Gold.ts" import * as Oracle from "./oracle/Plan.ts" @@ -50,6 +62,63 @@ describe("Query", () => { assert.deepStrictEqual(DataFrame.toArray(df, "city"), ["Vienna", "Berlin"]) })) + // DF-Q042 (API contract): Query yields between operations; pinned source in README.md. + it.effect("yields a long query so another fiber can interrupt it", () => + Effect.gen(function*() { + // Control elapsed time, not machine speed; disable automatic runtime yields + // so only the query's cooperative yielding can let the parent resume. + let now = 0 + const clock = vi.spyOn(performance, "now").mockImplementation(() => (now += 1_000)) + const tasks: Array<() => void> = [] + const scheduler: Scheduler.Scheduler = { + executionMode: "async", + shouldYield: () => false, + makeDispatcher: () => ({ + scheduleTask: (task) => { + tasks.push(task) + }, + flush() {} + }) + } + let loaded = false + let completed = false + try { + const query = Query.table("readings", Reading).pipe( + Query.sort((c) => [Expr.asc(c.temp)]), + Query.select((c) => ({ temp: Expr.add(c.temp, 1) })) + ) + const child = yield* query.pipe( + Effect.provideService( + Catalog.Catalog, + Catalog.make({ + readings: Effect.sync(() => { + loaded = true + return readings + }) + }) + ), + Effect.tap(() => + Effect.sync(() => { + completed = true + }) + ), + Effect.provideService(Scheduler.Scheduler, scheduler), + Effect.forkChild({ startImmediately: true }) + ) + try { + assert.isTrue(loaded) + assert.isUndefined(child.pollUnsafe()) + } finally { + child.interruptUnsafe() + while (tasks.length > 0) tasks.shift()!() + } + assert.isTrue(Exit.hasInterrupts(yield* Fiber.await(child))) + assert.isFalse(completed) + } finally { + clock.mockRestore() + } + })) + // DF-Q002 (API contract): Query.select; pinned source in README.md. it.effect("expression callbacks run once at construction, not on execution", () => Effect.gen(function*() { diff --git a/packages/effect/test/dataframe/README.md b/packages/effect/test/dataframe/README.md index fcacf6adff8..4752576d566 100644 --- a/packages/effect/test/dataframe/README.md +++ b/packages/effect/test/dataframe/README.md @@ -39,6 +39,9 @@ The assertion column is the exact leaf test title. Combine it with the file and | DF-C018 | C | compare structurally, NaN included | [DataFrame.equals](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/DataFrame.ts#L478) | | DF-C019 | C | iterate as rows | [DataFrame.DataFrame](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/DataFrame.ts#L81) | | DF-C020 | P | format as a text table | [DataFrame.format](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/DataFrame.ts#L467) | +| DF-C021 | C | reuses compatible numeric buffers across Arrow conversion | [DataFrame layouts](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/DataFrame.ts#L6-L9) | + +DF-C021 checks a single, unsliced, non-null batch of Int32 and Float64 columns: import/export preserves the backing buffers, byte offsets and byte lengths, as well as values. It does not require typed-array wrapper identity or mutation-based aliasing. The compatible fixed-width layouts are described in Apache Arrow’s [Columnar format](https://github.com/apache/arrow/blob/ee4d09ebef61c663c1efbfa4c18e518a03b798be/docs/source/format/Columnar.rst#fixed-size-primitive-layout) and [Schema.fbs](https://github.com/apache/arrow/blob/ee4d09ebef61c663c1efbfa4c18e518a03b798be/format/Schema.fbs); the no-copy adapter promise is the Effect API contract, not a requirement that Arrow imposes on every adapter. ### `Query.test.ts` @@ -85,6 +88,7 @@ The assertion column is the exact leaf test title. Combine it with the file and | DF-Q039 | C | sorts stably by several keys, also when sliced (top-k) | [Query.sort](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Query.ts#L352) | | DF-Q040 | C | groups by keys of every type | [Query.groupBy](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Query.ts#L610) | | DF-Q041 | C | joins on a high-cardinality key | [Query.join](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Query.ts#L680) | +| DF-Q042 | C | yields a long query so another fiber can interrupt it | [Query yielding](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Query.ts#L65-L66) | ## Oracle provenance and differential assertions @@ -171,7 +175,9 @@ Rejection assertions include unknown Arrow override fields (`DataFrame`), unknow - Polars corpus exclusions remain as recorded in `generate_gold.py:DEVIATIONS`: integral float literals, float-to-string formatting, automatic join suffixes, and mixed bigint/number arithmetic. No claim of universal Polars equivalence. - SQL intentionally excludes pivot/unpivot/explode/unnest and databases other than PostgreSQL. Overflow, division by zero, collation and sub-microsecond temporal behavior follow the documented SQL deviations, not unconditional parity. PGlite is not a live PostgreSQL server/version matrix. - The callback timing contract now has a direct test (DF-Q002); it supplements, rather than replaces, lazy loading and per-run catalog tests. -- Dedicated coverage for public guards, every dual overload, cancellation/fairness, aliasing/zero-copy ownership, and every malformed plan/IPC input is not established by this inventory. Those are gaps, not passing assertions. +- DF-Q042 controls the host monotonic clock and disables automatic scheduler yields, then verifies a started public query lets its parent resume and interrupt it before completion. It does not assert a yield interval, timeout behavior, fairness among many fibers, or interruption within an operation. +- DF-C021 covers buffer reuse only for the compatible numeric layouts described above. Sliced/multi-batch inputs, validity buffers, other layouts and mutation/ownership guarantees remain outside that assertion. +- Dedicated coverage for public guards, every dual overload, and every malformed plan/IPC input is not established by this inventory. Those are gaps, not passing assertions. - Compiler inference/rejection coverage is inventoried above and is not counted as runtime coverage. Exact diagnostic strings and explain/format layouts are local regression details even where a test also verifies a public failure or result contract. ## Validation From 8bed2f0c30db5d288ca0b1d7b56f5db9aef6fb65 Mon Sep 17 00:00:00 2001 From: Effect Bot Date: Mon, 16 Nov 2026 08:19:14 +0000 Subject: [PATCH 64/88] Keep Effect Schema out of table execution and declare tables by Arrow schemas --- packages/effect/src/dataframe/DataFrame.ts | 121 +++-- packages/effect/src/dataframe/Plan.ts | 42 +- packages/effect/src/dataframe/Query.ts | 88 +++- .../effect/src/dataframe/internal/query.ts | 13 +- .../effect/src/dataframe/internal/schema.ts | 428 ++++------------- packages/effect/src/dataframe/internal/sql.ts | 29 +- .../effect/src/dataframe/internal/stream.ts | 4 +- .../effect/src/dataframe/internal/table.ts | 444 ++++++++++++++++++ .../effect/test/dataframe/DataFrame.test.ts | 219 ++++++++- packages/effect/test/dataframe/Query.test.ts | 93 ++++ packages/effect/typetest/DataFrame.tst.ts | 16 + 11 files changed, 1074 insertions(+), 423 deletions(-) create mode 100644 packages/effect/src/dataframe/internal/table.ts diff --git a/packages/effect/src/dataframe/DataFrame.ts b/packages/effect/src/dataframe/DataFrame.ts index 844ceb066cf..7553fb5fbab 100644 --- a/packages/effect/src/dataframe/DataFrame.ts +++ b/packages/effect/src/dataframe/DataFrame.ts @@ -56,6 +56,7 @@ import * as Stream from "../Stream.ts" import type * as Types from "../Types.ts" import * as F from "./internal/frame.ts" import * as Schema_ from "./internal/schema.ts" +import * as T from "./internal/table.ts" import * as V from "./internal/vector.ts" // ----------------------------------------------------------------------------- @@ -168,22 +169,41 @@ export const isDataFrame = (u: unknown): u is DataFrame => F.isFrame(u) /** * Builds a frame from rows described by a struct schema. Column types come - * from the schema's type side: `String` is Utf8, `Number` is Float64, + * from the schema's fields: `String` is Utf8, `Number` is Float64, * `Boolean` is Bool, `BigInt` is Int64, `DateTimeUtc` and `Duration` - * are microsecond timestamps and durations, and `NullOr`/optional fields - * are nullable. `options.types` overrides the Arrow type of individual - * columns (for example `ArrowSchema.int32`); `BigDecimal` fields need - * one, with the decimal's precision and scale. + * are microsecond timestamps and durations, arrays are lists, structs are + * structs, and `NullOr`/optional fields are nullable. `options.types` + * overrides the Arrow type of individual columns (for example + * `ArrowSchema.int32`); `BigDecimal` fields need one, with the decimal's + * precision and scale. * * **Details** * + * The rows are decoded values (the schema's `Type`). The schema describes + * them and is not run as a decoder. Its refinements, literals and enums are + * tested on the value of every field that has them (inside structs, arrays + * and union members too), and a struct's own checks + * (`Schema.Struct(...).check(...)`) on every row, so the frame holds only + * rows that {@link validate} accepts. Fields without them are stored without + * further work. + * * Missing values read back as `null` (see {@link Normalize}). An override * must hold its field's values, as {@link validate} checks columns: a * `DateTimeUtc` override is a timestamp without a time zone or with a UTC * one, which the frame drops. * - * Throws a {@link DataFrameError} when a value doesn't fit its column, or an - * override can't hold its field's values. + * Throws a {@link DataFrameError} when a value doesn't fit its column or + * fails a check of the schema, an override can't hold its field's values, + * or the schema has no column types. + * + * **Gotchas** + * + * - A schema that transforms its rows is rejected, as frames hold decoded + * values and run no transformation: `Schema.NumberFromString`, + * `Schema.decodeTo`, decoding defaults, renamed keys. Pass + * `Schema.toType(schema)` to describe the decoded rows of such a schema. + * - Tuples, records, classes, `DateTimeZoned`, unions of different column + * types and recursive schemas have no column type. * * @stability experimental * @category constructors @@ -195,11 +215,11 @@ export const fromRows = ( options?: TypeOptions ): DataFrame> => unsafe(() => { - const arrow = Schema_.toArrowSchema(schema, options?.types as any) - const table = Result.getOrThrowWith( - ArrowTable.fromRows(arrow, Array.isArray(rows) ? rows : Array.from(rows) as any), - fromDefect - ) + const { checks, columns } = Schema_.declare(schema, options?.types as any) + const input: ReadonlyArray = Array.isArray(rows) ? rows : Array.from(rows) + const table = Result.getOrThrowWith(ArrowTable.fromRows(T.toArrowSchema(columns), input), fromDefect) + // The rows are values already: test them as given instead of reading them back from the frame. + if (checks !== undefined) T.runChecks(checks, (name) => input.map((row) => row[name]), () => input) return wrap(F.fromArrowTable(table)) }) @@ -353,6 +373,11 @@ export const decodeArrowStream = ( * An empty frame with the columns of a struct schema, typed as in * {@link fromRows}, which also describes `options.types`. * + * **Gotchas** + * + * Throws a {@link DataFrameError} for the schemas {@link fromRows} rejects: + * one that transforms its rows, or has fields without a column type. + * * @stability experimental * @category constructors * @since 4.1.5 @@ -363,31 +388,51 @@ export const empty = ( ): DataFrame> => fromRows(schema, [], options) /** - * Checks that a frame has the columns of a struct schema, with compatible - * types, and returns it typed: with only the schema's columns, in its order, - * converted to the types {@link fromRows} gives them (`options.types` - * overrides some, and `BigDecimal` fields need one). + * Checks that a frame has the columns a struct schema or an Arrow schema + * declares, with compatible types, and returns it with only those columns, + * in the declared order, converted to the declared types. * * **Details** * - * A column is compatible when its values read back as the field's: `Number` - * takes floats and integers below 64 bits, `BigInt` takes 64-bit integers, - * `BigDecimal` takes decimals of any precision and scale, `DateTimeUtc` and - * `Duration` take timestamps and durations in any unit, and structs and - * arrays need compatible fields and items. Every value must convert to the - * declared type exactly: a decimal with more digits than the declared scale, - * or a timestamp finer than the declared unit, fails. Values of fields the - * schema constrains further (literals, enums, refinements, including inside - * structs, arrays and union members) are checked against it, and so are the - * rows when the struct itself has checks (`Schema.Struct(...).check(...)`). + * A struct schema declares the columns {@link fromRows} gives its fields + * (`options.types` overrides some, and `BigDecimal` fields need one), and + * the result is typed by it. A column is compatible when its values read + * back as the field's: `Number` takes floats and integers below 64 bits, + * `BigInt` takes 64-bit integers, `BigDecimal` takes decimals of any + * precision and scale, `DateTimeUtc` and `Duration` take timestamps and + * durations in any unit, and structs and arrays need compatible fields and + * items. Every value must convert to the declared type exactly: a decimal + * with more digits than the declared scale, or a timestamp finer than the + * declared unit, fails. + * + * The types are checked and converted on the typed columns, without reading + * values. A field the schema constrains further (literals, enums, + * refinements, including inside structs, arrays and union members) has its + * values read and tested one by one, and a struct with checks of its own + * (`Schema.Struct(...).check(...)`) has every row read and tested. The + * schema is not run as a decoder: one that transforms its rows fails, as in + * {@link fromRows}. + * + * An Arrow schema declares its fields as they are: type, nullability, + * nested fields, decimal precision and scale, and time unit. The result has + * untyped rows, and no values are tested beyond their column type. The + * fields can be `Null` (nullable), `Bool`, integers of any width, `Float32`, + * `Float64`, `Utf8`, `Binary`, `Date` in days, `Time` in microseconds, + * `Timestamp` and `Duration` in any unit, 128-bit `Decimal`, and `List` + * and `Struct` of these. Other Arrow types fail: frames can't hold some + * (`Float16`, intervals, maps, unions) and hold others as one of the types + * above, which is the type to declare (`Utf8` for `LargeUtf8` and + * `Utf8View`, `List` for `LargeList`, the value type for a dictionary). A + * list's item field is named `item` in the result. Dates in milliseconds + * and times in other units convert to the declared days and microseconds. * * **Gotchas** * - * `DateTimeUtc` fields take timestamps without a time zone, or with a UTC - * one (`UTC`, `Etc/UTC`, `+00:00`), which the result drops; other time - * zones, and values outside the range `DateTime` represents, fail. - * `DateTimeZoned` fields are not supported: read zoned timestamps from an - * untyped frame, where they are `DateTime.Zoned` values. + * `DateTimeUtc` fields and Arrow timestamps take timestamps without a time + * zone, or with a UTC one (`UTC`, `Etc/UTC`, `+00:00`), which the result + * drops; other time zones, and values outside the range `DateTime` + * represents, fail. `DateTimeZoned` fields are not supported: read zoned + * timestamps from an untyped frame, where they are `DateTime.Zoned` values. * * @stability experimental * @category constructors @@ -398,18 +443,28 @@ export const validate: { schema: S, options?: TypeOptions ): (self: DataFrame) => Result.Result>, DataFrameError> + ( + schema: ArrowSchema.Schema + ): (self: DataFrame) => Result.Result, DataFrameError> ( self: DataFrame, schema: S, options?: TypeOptions ): Result.Result>, DataFrameError> + ( + self: DataFrame, + schema: ArrowSchema.Schema + ): Result.Result, DataFrameError> } = dual((args) => isDataFrame(args[0]), ( self: DataFrame, - schema: { readonly ast: any }, + schema: { readonly ast: any } | ArrowSchema.Schema, options?: { readonly types?: { readonly [name: string]: ArrowSchema.DataType | undefined } | undefined } ) => Result.try({ - try: () => wrap(Schema_.conform(frame(self), Schema_.declare(schema, options?.types), schema)), + try: () => { + const { checks, columns } = Schema_.declare(schema, options?.types) + return wrap(T.conform(frame(self), columns, checks)) + }, catch: fromDefect })) diff --git a/packages/effect/src/dataframe/Plan.ts b/packages/effect/src/dataframe/Plan.ts index cd64163b61e..7b8e27b3389 100644 --- a/packages/effect/src/dataframe/Plan.ts +++ b/packages/effect/src/dataframe/Plan.ts @@ -689,6 +689,36 @@ export interface TableColumn { readonly unit?: TimeUnit | undefined } +/** + * The tests of a {@link Table}'s values that its column types don't cover, + * compiled once from a struct schema's literals, enums and refinements when + * the table is declared. + * + * **Details** + * + * `columns` pairs the name of a declared column with a test of one value + * of it. Every value of that column is read (as `DataFrame.toArray` gives + * it) and tested; a test must accept `null`, as nullability is part of the + * column's declaration. `rows` tests one row object, as `DataFrame.toRows` + * gives it, and is `undefined` when there is nothing to test across + * columns. A table fails with a `DataFrameError` when a test returns + * `false`. + * + * **Gotchas** + * + * The tests are functions, so they are not part of the JSON form of a plan. + * A SQL engine runs `columns` tests on the distinct values of each column + * and fails for a table with a `rows` test. + * + * @stability experimental + * @category models + * @since 4.1.5 + */ +export interface TableChecks { + readonly columns: ReadonlyArray boolean]> + readonly rows: ((row: unknown) => boolean) | undefined +} + /** * The rows of a table the `Catalog` provides by name. * @@ -716,13 +746,15 @@ export interface TableColumn { * them to be declared, and `Query.stream` emits no batch for one that has no * rows. * - * `schema`, set by `Query.table`, is the struct schema the rows must - * satisfy: values of fields it constrains beyond their column type (literals, - * enums, refinements) are checked against it. + * `checks`, set by `Query.table` for a struct schema that constrains + * values beyond their column type, holds the tests the rows must also pass + * (see {@link TableChecks}). They run after the conversion, on every batch + * of a streamed table. A column without a test is checked by its type alone + * and its values are never read. * * **Gotchas** * - * `schema` is not part of the JSON form: a plan decoded from JSON checks + * `checks` is not part of the JSON form: a plan decoded from JSON checks * column types only. * * @stability experimental @@ -733,7 +765,7 @@ export interface Table { readonly _tag: "Table" readonly name: string readonly columns?: ReadonlyArray | undefined - readonly schema?: Schema.Top | undefined + readonly checks?: TableChecks | undefined } /** diff --git a/packages/effect/src/dataframe/Query.ts b/packages/effect/src/dataframe/Query.ts index 20c9754dd80..3501688f077 100644 --- a/packages/effect/src/dataframe/Query.ts +++ b/packages/effect/src/dataframe/Query.ts @@ -74,7 +74,6 @@ import type * as Effect from "../Effect.ts" import * as Effectable from "../Effectable.ts" import { dual } from "../Function.ts" import { NodeInspectSymbol } from "../Inspectable.ts" -import type * as Schema from "../Schema.ts" import * as Stream from "../Stream.ts" import type * as Types from "../Types.ts" import type { DataFrame, DataFrameError, Normalize } from "./DataFrame.ts" @@ -231,36 +230,83 @@ export const from = (frame: DataFrame): Query => make({ _tag: "Values", /** * A query over the table called `name`, provided by the `Catalog` when - * the query runs. The table must have the columns of the struct schema, and - * its values must satisfy it, as with `DataFrame.validate`; the query sees - * only those columns, in the schema's order, with the types - * `DataFrame.fromRows` gives them (`options.types` overrides some, and - * `BigDecimal` fields need one). + * the query runs, with the columns a struct schema or an Arrow schema + * declares. The table must have those columns, as `DataFrame.validate` + * checks a frame; the query sees only them, in the declared order and with + * the declared types. * * **Details** * - * Every batch of a streamed table is converted to these types, so results - * have the same types whatever the batches hold, and when there are none. - * A table without the precision and scale of a `BigDecimal` field fails - * when it is read. + * A struct schema declares the columns `DataFrame.fromRows` gives its + * fields (`options.types` overrides some, and `BigDecimal` fields need + * one) and types the rows. The schema is read once, here, and compiled to + * the tests the table's rows must pass (`Plan.TableChecks`). A field with + * refinements, literals or enums has its values read and tested one by one, + * and a struct with checks of its own (`Schema.Struct(...).check(...)`) has + * every row read and tested. Other fields are checked by column type only, + * on the typed columns. + * + * An Arrow schema declares its fields as they are (type, nullability, + * nested fields, decimal precision and scale, time unit), with untyped rows + * and no tests of values. `DataFrame.validate` lists the Arrow types a + * declaration takes. + * + * Every batch of a streamed table is converted to the declared types, so + * results have the same types whatever the batches hold, and when there are + * none. A table without the precision and scale of a `BigDecimal` field + * fails when it is read. * - * A SQL engine can't check a struct's own checks on whole rows - * (`Schema.Struct(...).check(...)`): a query over such a table fails. + * **Gotchas** + * + * - Throws a `DataFrameError` for a schema that declares no columns: a + * struct schema that transforms its rows or has fields without a column + * type (see `DataFrame.fromRows`), or an Arrow schema with types a + * declaration can't take. + * - A SQL engine can't check a struct's own checks on whole rows: a query + * over such a table fails. + * - The tests are functions and not part of the plan's JSON form: a plan + * decoded from JSON checks column types only. + * + * **Example** (Declaring a table by an Arrow schema) + * + * ```ts import.meta.vitest + * import { Effect } from "effect" + * import { ArrowSchema } from "effect/arrow" + * import { Catalog, DataFrame, Query } from "effect/dataframe" + * + * const readings = Query.table( + * "readings", + * ArrowSchema.make([ + * ArrowSchema.field("city", ArrowSchema.utf8, { nullable: false }), + * ArrowSchema.field("temp", ArrowSchema.float32) + * ]) + * ) + * + * const frame = DataFrame.fromColumns({ temp: Int32Array.of(21, 19), city: ["Vienna", "Berlin"] }) + * const result = Effect.runSync(readings.pipe(Effect.provide(Catalog.layer({ readings: frame })))) + * result.schema.fields.map((f) => f.type) // => [ArrowSchema.utf8, ArrowSchema.float32] + * DataFrame.toRows(result) // => [{ city: "Vienna", temp: 21 }, { city: "Berlin", temp: 19 }] + * ``` * * @stability experimental * @category constructors * @since 4.1.5 */ -export const table = ( +export const table: { + ( + name: string, + schema: S, + options?: { readonly types?: { readonly [K in keyof S["Type"]]?: ArrowSchema.DataType } | undefined } + ): Query> + (name: string, schema: ArrowSchema.Schema): Query<{ readonly [name: string]: unknown }> +} = ( name: string, - schema: S, - options?: { readonly types?: { readonly [K in keyof S["Type"]]?: ArrowSchema.DataType } | undefined } -): Query> => - make({ - _tag: "Table", - name, - columns: Schema_.declare(schema, options?.types as any), - schema: schema as unknown as Schema.Top + schema: { readonly ast: any } | ArrowSchema.Schema, + options?: { readonly types?: { readonly [name: string]: ArrowSchema.DataType | undefined } | undefined } +): Query => + Q.unsafe(() => { + const { checks, columns } = Schema_.declare(schema, options?.types) + return make(checks === undefined ? { _tag: "Table", name, columns } : { _tag: "Table", name, columns, checks }) }) /** diff --git a/packages/effect/src/dataframe/internal/query.ts b/packages/effect/src/dataframe/internal/query.ts index 48242009b10..a8afc4b8426 100644 --- a/packages/effect/src/dataframe/internal/query.ts +++ b/packages/effect/src/dataframe/internal/query.ts @@ -28,8 +28,8 @@ import { DataFrameError } from "../DataFrame.ts" import type * as Plan from "../Plan.ts" import * as E from "./expr.ts" import * as F from "./frame.ts" -import { conform } from "./schema.ts" import * as Streaming from "./stream.ts" +import { conform } from "./table.ts" import * as V from "./vector.ts" /** A sort that only keeps its first `limit` rows. @internal */ @@ -62,6 +62,15 @@ export const attempt = (f: () => A): Effect.Effect => { } } +/** Runs a kernel computation, throwing its failures as `DataFrameError`s. @internal */ +export const unsafe = (f: () => A): A => { + try { + return f() + } catch (cause) { + throw cause instanceof V.DataFrameDefect ? new DataFrameError({ message: cause.message, cause }) : cause + } +} + /** @internal */ export const children = (node: Node): ReadonlyArray => { switch (node._tag) { @@ -118,7 +127,7 @@ export const mapTables = (plan: Node, f: (table: Plan.Table) => Node): Node => { /** A table's frame, checked against and narrowed to its declared columns. */ const tableValues = (table: Plan.Table, frame: F.Frame): Node => - values(table.columns === undefined ? frame : conform(frame, table.columns, table.schema)) + values(table.columns === undefined ? frame : conform(frame, table.columns, table.checks)) /** Replaces every `Table` with the catalog's frame, reading each table once. */ const resolve = (plan: Plan.Node, catalog: Catalog.Service): Effect.Effect => { diff --git a/packages/effect/src/dataframe/internal/schema.ts b/packages/effect/src/dataframe/internal/schema.ts index 47bec3533a0..999bdd2383a 100644 --- a/packages/effect/src/dataframe/internal/schema.ts +++ b/packages/effect/src/dataframe/internal/schema.ts @@ -1,35 +1,29 @@ /** - * Table columns for the fields of an Effect Schema struct, and the checks a - * frame must pass to be read as rows of the schema. + * Effect Schema at the boundary of frames: the declared columns of a struct + * schema's fields, and its checks compiled to plain tests. * - * A field's declared column is fixed once, from the schema and the Arrow - * types given for some fields (`declare`): numbers are Float64, - * timestamps and durations microseconds, unless an override says otherwise, - * and decimals need one. Building a frame from rows, validating a frame and - * reading a table all convert to that declaration (`arrowType`, - * `conform`), so the same schema and overrides always give the same column - * types. + * A schema is read once, when a frame is built from rows or validated, or a + * table is declared (`declare`). Everything after that works on the declared + * columns and the compiled tests (see `./table.ts`), without the schema. * - * A `Timestamp` column reads as `DateTime.Utc` values, so its timestamps - * must have no time zone (or a UTC one, which is dropped) and lie in the - * range `DateTime` represents. `DateTimeZoned` fields have no column type. + * Frames hold decoded values, so a schema describes the columns of its type + * side and must not transform: a node with an encoding fails. Numbers are + * Float64, timestamps and durations microseconds, unless an override says + * otherwise, and decimals need one. `DateTimeZoned` fields have no column + * type. * * @internal */ -import * as S from "../../arrow/ArrowSchema.ts" +import type * as S from "../../arrow/ArrowSchema.ts" import * as Result from "../../Result.ts" import * as AST from "../../SchemaAST.ts" import * as Parser from "../../SchemaParser.ts" import type * as Plan from "../Plan.ts" -import * as D from "./dtype.ts" -import * as F from "./frame.ts" -import * as K from "./kernels.ts" +import * as T from "./table.ts" import * as V from "./vector.ts" type Column = Omit -const field = (path: string, name: string) => (path === "" ? name : `${path}.${name}`) - const leaf = (type: Plan.TypeName, nullable = false): Column => ({ type, nullable }) const literalType = (literal: unknown): Plan.TypeName | undefined => { @@ -127,299 +121,51 @@ const unsupported = (ast: AST.AST, path: string): never => const fieldsOf = (ast: AST.Objects, path: string): Array => ast.propertySignatures.map((ps) => { const name = String(ps.name) - const column = columnOf(ps.type, field(path, name)) + const column = columnOf(ps.type, T.field(path, name)) return { name, ...column, nullable: column.nullable || AST.isOptional(ps.type) } }) -const structOf = (schema: { readonly ast: AST.AST }): AST.Objects => { - const ast = AST.toType(schema.ast) - return ast._tag === "Objects" && ast.indexSignatures.length === 0 - ? ast +/** The struct a schema describes its rows by. */ +const structOf = (ast: AST.AST): AST.Objects => { + const type = AST.toType(ast) + return type._tag === "Objects" && type.indexSignatures.length === 0 + ? type : V.fail("DataFrame rows must be described by a struct schema") } -// ----------------------------------------------------------------------------- -// Declared columns -// ----------------------------------------------------------------------------- - -/** The plan type name of an Arrow type. */ -const typeName = (t: S.DataType): Plan.TypeName => { - switch (t._tag) { - case "Int": - case "Float": - return D.describe(t) as Plan.TypeName - case "Utf8": - case "LargeUtf8": - case "Utf8View": - return "Utf8" - case "Binary": - case "LargeBinary": - case "BinaryView": - case "FixedSizeBinary": - return "Binary" - case "LargeList": - return "List" - default: - return t._tag as Plan.TypeName - } -} - -const describe = (t: S.DataType): string => { - if (t._tag === "List" || t._tag === "LargeList") return `List<${describe(t.child.type)}>` - return t._tag === "Timestamp" || t._tag === "Duration" || t._tag === "Decimal" ? D.describe(t) : typeName(t) -} - -const describeColumn = (c: Column): string => c.type === "List" ? `List<${describeColumn(c.fields![0])}>` : c.type - -const utcZones: ReadonlySet = new Set(["UTC", "Etc/UTC", "+00:00"]) - -/** - * Fails unless values of Arrow type `t` read as values of the column (not - * looking into lists and structs). Timestamps need no time zone or a UTC one. - */ -const checkType = (path: string, t: S.DataType, expected: Column): void => { - if (expected.type === "Timestamp" && t._tag === "Timestamp" && t.timezone !== undefined) { - if (utcZones.has(t.timezone)) return - return V.fail(`column ${JSON.stringify(path)} has time zone ${t.timezone}, expected UTC timestamps`) - } - if (!compatible(t, expected)) { - V.fail(`column ${JSON.stringify(path)} is ${describe(t)}, expected ${describeColumn(expected)}`) - } -} - -/** - * Integers below 64 bits read as floats and narrower integers, 64-bit - * integers (bigints) as 64-bit ones, any decimal as a decimal, and temporal - * types as the same kind in any unit. - */ -const compatible = (t: S.DataType, expected: Column): boolean => { - switch (expected.type) { - case "Float16": - case "Float32": - case "Float64": - return t._tag === "Float" || (t._tag === "Int" && t.bitWidth < 64) - case "Int8": - case "Int16": - case "Int32": - case "UInt8": - case "UInt16": - case "UInt32": - return t._tag === "Int" && t.bitWidth < 64 - case "Int64": - case "UInt64": - return t._tag === "Int" && t.bitWidth === 64 - default: - return typeName(t) === expected.type - } -} - -const missing = (path: string): never => V.fail(`missing column ${JSON.stringify(path)}`) - -/** The column declared for a field's `column` with Arrow type `t`, or its default type. */ -const declareColumn = (path: string, column: Plan.TableColumn, t: S.DataType | undefined): Plan.TableColumn => { - if (t !== undefined) checkType(path, t, column) - switch (column.type) { - case "List": { - const child = t !== undefined && (t._tag === "List" || t._tag === "LargeList") ? t.child.type : undefined - return { ...column, fields: [declareColumn(`${path}[]`, column.fields![0], child)] } - } - case "Struct": - return { - ...column, - fields: column.fields!.map((f) => { - const p = field(path, f.name) - const g = t?._tag === "Struct" ? (t.fields.find((g) => g.name === f.name) ?? missing(p)).type : undefined - return declareColumn(p, f, g) - }) - } - case "Decimal": - return t?._tag === "Decimal" ? { ...column, precision: t.precision, scale: t.scale } : column - case "Timestamp": - case "Duration": - return { ...column, unit: t?._tag === column.type ? t.unit : "microsecond" } - default: - return t === undefined || column.type === "Null" ? column : { ...column, type: typeName(t) } - } -} - -/** - * The declared columns of a struct schema's fields: their types from the - * schema, with `overrides` for the Arrow types of some of them, checked - * against the schema. Decimals have no default precision and scale. - * - * @internal - */ -export const declare = ( - schema: { readonly ast: AST.AST }, - overrides: { readonly [name: string]: S.DataType | undefined } | undefined -): Array => fieldsOf(structOf(schema), "").map((c) => declareColumn(c.name, c, overrides?.[c.name])) +const transformed = (path: string): never => + V.fail( + `${path === "" ? "the schema" : `field ${JSON.stringify(path)}`} has a transformation: ` + + "frames hold decoded values and transformations are not run; Schema.toType(schema) describes the decoded rows" + ) /** - * The Arrow type of a declared column. Timestamps and durations without a - * unit are microseconds, dates days and times microseconds; decimals need - * their precision and scale. - * - * @internal + * Fails when a schema on the way to a column has an encoding: its decoded + * values differ from the encoded ones, and frames run no transformation. */ -export const arrowType = (column: Plan.TableColumn, path: string = column.name): S.DataType => { - switch (column.type) { - case "List": { - const item = column.fields![0] - return S.list(S.field("item", arrowType(item, `${path}[]`), { nullable: item.nullable })) - } - case "Struct": - return S.struct( - column.fields!.map((f) => S.field(f.name, arrowType(f, field(path, f.name)), { nullable: f.nullable })) - ) - case "Utf8": - return S.utf8 - case "Binary": - return S.binary - case "Bool": - return S.bool - case "Null": - return S.nullType - case "Date": - return S.date("day") - case "Time": - return S.time("microsecond") - case "Timestamp": - return S.timestamp(column.unit ?? "microsecond") - case "Duration": - return S.duration(column.unit ?? "microsecond") - case "Decimal": - return column.precision === undefined || column.scale === undefined - ? V.fail( - `column ${ - JSON.stringify(path) - }: decimal columns need a precision and scale: pass options.types (for example ArrowSchema.decimal(18, 2))` - ) - : S.decimal(column.precision, column.scale) - case "Float16": - return S.float16 - default: - // The other numeric types, named as cast targets. - return D.castTargets[column.type] +const rejectEncodings = (ast: AST.AST, path: string): void => { + if (ast.encoding !== undefined) return transformed(path) + switch (ast._tag) { + case "Objects": + return ast.propertySignatures.forEach((ps) => rejectEncodings(ps.type, T.field(path, String(ps.name)))) + case "Arrays": + return [...ast.elements, ...ast.rest].forEach((item) => rejectEncodings(item, `${path}[]`)) + case "Union": + return ast.types.forEach((member) => rejectEncodings(member, path)) + case "Suspend": + return unsuspend(ast, `field ${JSON.stringify(path)}: `, (ast) => rejectEncodings(ast, path)) } } -/** - * The Arrow schema of a struct schema's declared columns, with `overrides` - * for the types of some of them. - * - * @internal - */ -export const toArrowSchema = ( - schema: { readonly ast: AST.AST }, - overrides: { readonly [name: string]: S.DataType | undefined } | undefined -): S.Schema => S.make(declare(schema, overrides).map((c) => S.field(c.name, arrowType(c), { nullable: c.nullable }))) - // ----------------------------------------------------------------------------- -// Checking frames +// Checks // ----------------------------------------------------------------------------- -const inRows = (rows: Uint8Array | undefined, i: number): boolean => rows === undefined || rows[i] === 1 - -const allNull = (v: V.Vector, rows: Uint8Array | undefined): boolean => { - for (let i = 0; i < v.length; i++) if (inRows(rows, i) && V.isValid(v, i)) return false - return true -} - -/** The stored values of a timestamp in each unit that `DateTime` represents (±8.64e15 ms). */ -const timestampRange: { readonly [U in S.TimeUnit]: readonly [bigint, bigint] } = { - second: [-BigInt("8640000000000"), BigInt("8640000000000")], - millisecond: [-BigInt("8640000000000000"), BigInt("8640000000000000")], - microsecond: [-BigInt("8640000000000000000"), BigInt("8640000000000000999")], - nanosecond: [-BigInt("9223372036854775808"), BigInt("9223372036854775807")] -} - -const checkRange = (path: string, v: V.I64, unit: S.TimeUnit, rows: Uint8Array | undefined): void => { - const [lo, hi] = timestampRange[unit] - for (let i = 0; i < v.length; i++) { - if (inRows(rows, i) && V.isValid(v, i) && (v.values[i] < lo || v.values[i] > hi)) { - V.fail(`column ${JSON.stringify(path)} has timestamps outside the range of DateTime`) - } - } -} - -const sameValue = (a: V.Vector, b: V.Vector, i: number): boolean => { - switch (a.kind) { - case "f64": { - const x = a.values[i] - const y = (b as V.F64).values[i] - return Object.is(x, y) || (x !== x && y !== y) - } - case "i32": - case "i64": - case "bool": - return a.values[i] === (b as typeof a).values[i] - case "str": - return V.decodeString(a, i) === V.decodeString(b as V.Str, i) - default: - return true - } -} - -/** `v` converted to `target`, failing unless every present value converts exactly. */ -const convert = (path: string, v: V.Vector, target: S.DataType, rows: Uint8Array | undefined): V.Vector => { - if (D.sameType(v.type, target)) return v - const converted = K.cast(v, target) - const back = K.cast(converted, v.type) - for (let i = 0; i < v.length; i++) { - if ( - inRows(rows, i) && V.isValid(v, i) && - (!V.isValid(converted, i) || !sameValue(v, back, i)) - ) { - return V.fail(`column ${JSON.stringify(path)} has values that ${D.describe(target)} can't hold exactly`) - } - } - return converted -} - -/** A column checked against its declaration and converted to its declared type. */ -const check = (path: string, v: V.Vector, expected: Plan.TableColumn, rows: Uint8Array | undefined): V.Vector => { - if (expected.type === "Null") { - if (!allNull(v, rows)) checkType(path, v.type, expected) - return V.nulls(v.length) - } - checkType(path, v.type, expected) - if (!expected.nullable) { - for (let i = 0; i < v.length; i++) { - if (inRows(rows, i) && !V.isValid(v, i)) { - return V.fail(`column ${JSON.stringify(path)} has missing values`) - } - } - } - if (v.kind === "struct" && v.type._tag === "Struct") { - const present = new Uint8Array(v.length) - for (let i = 0; i < v.length; i++) present[i] = inRows(rows, i) && V.isValid(v, i) ? 1 : 0 - const fields = v.type.fields - const children = expected.fields!.map((f) => { - const k = fields.findIndex((g) => g.name === f.name) - return k === -1 ? missing(field(path, f.name)) : check(field(path, f.name), v.children[k], f, present) - }) - const type = S.struct(expected.fields!.map((f, k) => S.field(f.name, children[k].type, { nullable: f.nullable }))) - return V.struct(type, children, v.length, v.valid) - } - if (v.kind === "list") { - const present = new Uint8Array(v.child.length) - for (let i = 0; i < v.length; i++) { - if (inRows(rows, i) && V.isValid(v, i)) present.fill(1, v.offsets[i], v.offsets[i + 1]) - } - const item = expected.fields![0] - const child = check(`${path}[]`, v.child, item, present) - return V.list(S.list(S.field("item", child.type, { nullable: item.nullable })), v.offsets, child, v.valid) - } - if (expected.type === "Timestamp" && v.kind === "i64" && v.type._tag === "Timestamp") { - checkRange(path, v, v.type.unit, rows) - } - return expected.type === "Date" || expected.type === "Time" ? v : convert(path, v, arrowType(expected, path), rows) -} - /** * A test for values of a schema that it constrains beyond their column type * (literals, enums, template literals and checks), anywhere inside it, or - * `undefined` when the column type says it all. Nulls pass: nullability is + * `undefined` when the column type says it all. Missing values (`null`, and + * `undefined` in rows that are not in a frame yet) pass: nullability is * checked with the column. Union members are tried in turn. */ const valuesCheck = (ast: AST.AST): ((value: unknown) => boolean) | undefined => { @@ -428,7 +174,7 @@ const valuesCheck = (ast: AST.AST): ((value: unknown) => boolean) | undefined => : (value: unknown) => Result.isSuccess(AST.runChecks(ast.checks!, value)) const inner = structural(ast) if (own === undefined) return inner - return (value) => value === null || ((inner === undefined || inner(value)) && own(value)) + return (value) => value == null || ((inner === undefined || inner(value)) && own(value)) } const structural = (ast: AST.AST): ((value: unknown) => boolean) | undefined => { @@ -437,18 +183,18 @@ const structural = (ast: AST.AST): ((value: unknown) => boolean) | undefined => case "Enum": case "TemplateLiteral": { const is = Parser._is(AST.replaceChecks(ast, undefined)) - return (value) => value === null || is(value) + return (value) => value == null || is(value) } case "Suspend": return unsuspend(ast, "", valuesCheck) case "Union": { const members = ast.types.filter((t) => t._tag !== "Null" && t._tag !== "Undefined").map(valuesCheck) if (members.some((m) => m === undefined)) return undefined - return (value) => value === null || members.some((m) => m!(value)) + return (value) => value == null || members.some((m) => m!(value)) } case "Arrays": { const item = valuesCheck(ast.rest[0]) - return item && ((value) => value === null || (value as ReadonlyArray).every(item)) + return item && ((value) => value == null || (value as ReadonlyArray).every(item)) } case "Objects": { const fields = ast.propertySignatures.flatMap((ps) => { @@ -457,7 +203,7 @@ const structural = (ast: AST.AST): ((value: unknown) => boolean) | undefined => }) return fields.length === 0 ? undefined - : (value) => value === null || fields.every(([name, f]) => f((value as any)[name])) + : (value) => value == null || fields.every(([name, f]) => f((value as any)[name])) } default: return undefined @@ -465,70 +211,54 @@ const structural = (ast: AST.AST): ((value: unknown) => boolean) | undefined => } /** - * The fields of a struct schema whose values it constrains beyond their - * column type, each with its test for those values. + * The tests of a struct schema: one for each field whose values it + * constrains beyond their column type, and one of whole rows when the struct + * has checks of its own. `undefined` when the column types say it all. */ -const valueChecks = ( - schema: { readonly ast: AST.AST } -): Array boolean]> => - structOf(schema).propertySignatures.flatMap((ps) => { +const checksOf = (struct: AST.Objects): Plan.TableChecks | undefined => { + const columns = struct.propertySignatures.flatMap((ps) => { const test = valuesCheck(ps.type) return test === undefined ? [] : [[String(ps.name), test] as const] }) - -/** A test of whole rows against a struct schema's own checks; `undefined` without any. */ -const rowsCheck = (schema: { readonly ast: AST.AST }): ((row: unknown) => boolean) | undefined => { - const checks = structOf(schema).checks - return checks && ((row) => Result.isSuccess(AST.runChecks(checks, row))) + const checks = struct.checks + const rows = checks && ((row: unknown) => Result.isSuccess(AST.runChecks(checks, row))) + return columns.length === 0 && rows === undefined ? undefined : { columns, rows } } +// ----------------------------------------------------------------------------- +// Declarations +// ----------------------------------------------------------------------------- + /** - * The {@link valueChecks} of a schema whose tables are checked column by - * column, as a SQL engine checks them: a schema with checks on whole rows - * (`Schema.Struct(...).check(...)`) fails. + * A table as declared by a schema: its columns, and the tests of the values + * and rows the columns' types don't rule out. * * @internal */ -export const columnChecks = ( - schema: { readonly ast: AST.AST } -): Array boolean]> => - rowsCheck(schema) === undefined - ? valueChecks(schema) - : V.fail("schemas with checks on whole rows aren't supported on SQL tables; check their fields instead") +export interface Declared { + readonly columns: Array + readonly checks: Plan.TableChecks | undefined +} /** - * Checks that a frame has the declared columns and narrows it to them, in - * their order, converting each to its declared type (exactly: a value the - * declared type can't hold fails). With a `schema`, the values of fields it - * constrains beyond their column type, and its checks on whole rows, are - * checked against it too. + * The table a struct schema declares: the columns of its fields, with + * `overrides` for the Arrow types of some of them, checked against the + * schema, and its checks compiled. Decimals have no default precision and + * scale. A schema that transforms its rows fails. + * + * An Arrow schema declares its fields as columns, without checks: no Effect + * Schema is involved. * * @internal */ -export const conform = ( - frame: F.Frame, - columns: ReadonlyArray, - schema?: { readonly ast: AST.AST } | undefined -): F.Frame => { - const narrowed = F.make( - columns.map((expected) => { - const c = frame.columns.find((c) => c.name === expected.name) ?? missing(expected.name) - const values = c.values() - const checked = check(expected.name, values, expected, undefined) - return checked === values && c.nullable === expected.nullable - ? c - : new F.Column(c.name, checked, undefined, expected.nullable) - }), - frame.length - ) - if (schema !== undefined) { - for (const [name, test] of valueChecks(schema)) { - if (!F.toArray(narrowed, name).every(test)) { - return V.fail(`column ${JSON.stringify(name)} has values its schema rejects`) - } - } - const rows = rowsCheck(schema) - if (rows !== undefined && !F.toRows(narrowed).every(rows)) return V.fail("rows have values their schema rejects") - } - return narrowed +export const declare = ( + schema: { readonly ast: AST.AST } | S.Schema, + overrides?: { readonly [name: string]: S.DataType | undefined } | undefined +): Declared => { + if (!("ast" in schema)) return { columns: T.fromArrowSchema(schema), checks: undefined } + const struct = structOf(schema.ast) + const columns = fieldsOf(struct, "").map((c) => T.declareColumn(c.name, c, overrides?.[c.name])) + // After the columns: a decoded side without column types fails for that first. + rejectEncodings(schema.ast, "") + return { columns, checks: checksOf(struct) } } diff --git a/packages/effect/src/dataframe/internal/sql.ts b/packages/effect/src/dataframe/internal/sql.ts index 4c6b9a20bd2..0d4feeb3888 100644 --- a/packages/effect/src/dataframe/internal/sql.ts +++ b/packages/effect/src/dataframe/internal/sql.ts @@ -31,7 +31,7 @@ import * as E from "./expr.ts" import * as F from "./frame.ts" import * as K from "./kernels.ts" import * as Q from "./query.ts" -import * as Schema_ from "./schema.ts" +import { arrowType } from "./table.ts" import { civil, coarser } from "./temporal.ts" import * as V from "./vector.ts" @@ -1017,11 +1017,11 @@ const tableType = (table: string, column: Plan.TableColumn): S.DataType => ? unsupported( `column ${JSON.stringify(column.name)} of table ${JSON.stringify(table)}: ${column.type} table columns` ) - : Schema_.arrowType(column) + : arrowType(column) /** The conditions of a declared column's values, `x` (see {@link tableTypes}). */ const conditions = (x: string, column: Plan.TableColumn): ReadonlyArray => - tableTypes[column.type]!.conditions?.(x, Schema_.arrowType(column)) ?? [] + tableTypes[column.type]!.conditions?.(x, arrowType(column)) ?? [] /** * A check of a table: one SQL value, and a test of it that fails when the @@ -1043,7 +1043,7 @@ const utc = S.timestamp("microsecond") */ const declared = (column: Plan.TableColumn): string => { const x = quote(column.name) - const type = Schema_.arrowType(column) + const type = arrowType(column) const value = type._tag === "Null" ? `CAST(NULL AS ${sqlType(type)})` : type._tag === "Timestamp" @@ -1079,11 +1079,22 @@ const typeChecks = (table: Plan.Table): Array => }] }) +/** + * The tests of a table's checks, each of the values of one column, as a SQL + * engine runs them: a table with a test of whole rows fails. + */ +const columnChecks = (table: Plan.Table): Plan.TableChecks["columns"] => + table.checks === undefined + ? [] + : table.checks.rows === undefined + ? table.checks.columns + : V.fail("schemas with checks on whole rows aren't supported on SQL tables; check their fields instead") + /** * The checks of a table's rows beyond column types: nulls, the - * {@link conditions} of values, and fields its schema constrains (literals, - * enums, refinements), tested on distinct values. Each check aggregates the - * rows of the table's source. + * {@link conditions} of values, and the tests of the table's own checks + * (fields its schema constrains: literals, enums, refinements), run on + * distinct values. Each check aggregates the rows of the table's source. */ const rowChecks = (table: Plan.Table): Array => [ ...(table.columns ?? []).flatMap((column) => { @@ -1103,9 +1114,9 @@ const rowChecks = (table: Plan.Table): Array => [ ...conditions(x, column).map((c) => any(`NOT (${c.holds})`, c.failure)) ] }), - ...(table.schema === undefined ? [] : Schema_.columnChecks(table.schema)).map(([name, test]): Check => { + ...columnChecks(table).map(([name, test]): Check => { const column = table.columns!.find((c) => c.name === name)! - const type = Schema_.arrowType(column) + const type = arrowType(column) return { sql: `ARRAY_AGG(DISTINCT ${output(declared(column), type)})`, verify: (values) => { diff --git a/packages/effect/src/dataframe/internal/stream.ts b/packages/effect/src/dataframe/internal/stream.ts index 0809cccf7f6..8170f048b35 100644 --- a/packages/effect/src/dataframe/internal/stream.ts +++ b/packages/effect/src/dataframe/internal/stream.ts @@ -38,7 +38,7 @@ import * as E from "./expr.ts" import * as F from "./frame.ts" import * as Partial from "./partial.ts" import * as Q from "./query.ts" -import { arrowType, conform } from "./schema.ts" +import { arrowType, conform } from "./table.ts" import * as V from "./vector.ts" type Batches = Stream.Stream @@ -192,7 +192,7 @@ const describeColumns = (frame: F.Frame): string => const reader = (table: Plan.Table) => { let first: F.Frame | undefined return (frame: F.Frame): F.Frame => { - const batch = table.columns === undefined ? frame : conform(frame, table.columns, table.schema) + const batch = table.columns === undefined ? frame : conform(frame, table.columns, table.checks) first ??= batch const same = batch.columns.length === first.columns.length && batch.columns.every((c, i) => c.name === first!.columns[i].name && D.sameType(c.type, first!.columns[i].type)) diff --git a/packages/effect/src/dataframe/internal/table.ts b/packages/effect/src/dataframe/internal/table.ts new file mode 100644 index 00000000000..39f1a37f299 --- /dev/null +++ b/packages/effect/src/dataframe/internal/table.ts @@ -0,0 +1,444 @@ +/** + * Declared table columns: what a frame must hold to be read as a table, and + * the conversion of a frame to them. + * + * A declared column names its type once (`declareColumn`, `fromArrowSchema`). + * Validating a frame and reading a table convert to that declaration + * (`arrowType`, `conform`), so the same declaration always gives the same + * column types. The conversion works on the typed buffers of the columns and + * builds no value objects. + * + * A `Timestamp` column reads as `DateTime.Utc` values, so its timestamps + * must have no time zone (or a UTC one, which is dropped) and lie in the + * range `DateTime` represents. + * + * A declaration can carry tests of values and rows (`Plan.TableChecks`). + * They are compiled by whoever declares the table and run here as given, on + * the values of the columns they name and on the rows. + * + * @internal + */ +import * as S from "../../arrow/ArrowSchema.ts" +import type * as Plan from "../Plan.ts" +import * as D from "./dtype.ts" +import * as F from "./frame.ts" +import * as K from "./kernels.ts" +import * as V from "./vector.ts" + +type Column = Omit + +/** The path of field `name` under `path`. @internal */ +export const field = (path: string, name: string) => (path === "" ? name : `${path}.${name}`) + +// ----------------------------------------------------------------------------- +// Declared columns +// ----------------------------------------------------------------------------- + +/** The plan type name of an Arrow type. */ +const typeName = (t: S.DataType): Plan.TypeName => { + switch (t._tag) { + case "Int": + case "Float": + return D.describe(t) as Plan.TypeName + case "Utf8": + case "LargeUtf8": + case "Utf8View": + return "Utf8" + case "Binary": + case "LargeBinary": + case "BinaryView": + case "FixedSizeBinary": + return "Binary" + case "LargeList": + return "List" + default: + return t._tag as Plan.TypeName + } +} + +const describe = (t: S.DataType): string => { + if (t._tag === "List" || t._tag === "LargeList") return `List<${describe(t.child.type)}>` + return t._tag === "Timestamp" || t._tag === "Duration" || t._tag === "Decimal" ? D.describe(t) : typeName(t) +} + +const describeColumn = (c: Column): string => c.type === "List" ? `List<${describeColumn(c.fields![0])}>` : c.type + +const utcZones: ReadonlySet = new Set(["UTC", "Etc/UTC", "+00:00"]) + +/** + * Fails unless values of Arrow type `t` read as values of the column (not + * looking into lists and structs). Timestamps need no time zone or a UTC one. + */ +const checkType = (path: string, t: S.DataType, expected: Column): void => { + if (expected.type === "Timestamp" && t._tag === "Timestamp" && t.timezone !== undefined) { + if (utcZones.has(t.timezone)) return + return V.fail(`column ${JSON.stringify(path)} has time zone ${t.timezone}, expected UTC timestamps`) + } + if (!compatible(t, expected)) { + V.fail(`column ${JSON.stringify(path)} is ${describe(t)}, expected ${describeColumn(expected)}`) + } +} + +/** + * Integers below 64 bits read as floats and narrower integers, 64-bit + * integers (bigints) as 64-bit ones, any decimal as a decimal, and temporal + * types as the same kind in any unit. + */ +const compatible = (t: S.DataType, expected: Column): boolean => { + switch (expected.type) { + case "Float16": + case "Float32": + case "Float64": + return t._tag === "Float" || (t._tag === "Int" && t.bitWidth < 64) + case "Int8": + case "Int16": + case "Int32": + case "UInt8": + case "UInt16": + case "UInt32": + return t._tag === "Int" && t.bitWidth < 64 + case "Int64": + case "UInt64": + return t._tag === "Int" && t.bitWidth === 64 + default: + return typeName(t) === expected.type + } +} + +const missing = (path: string): never => V.fail(`missing column ${JSON.stringify(path)}`) + +/** The column declared for a field's `column` with Arrow type `t`, or its default type. @internal */ +export const declareColumn = (path: string, column: Plan.TableColumn, t: S.DataType | undefined): Plan.TableColumn => { + if (t !== undefined) checkType(path, t, column) + switch (column.type) { + case "List": { + const child = t !== undefined && (t._tag === "List" || t._tag === "LargeList") ? t.child.type : undefined + return { ...column, fields: [declareColumn(`${path}[]`, column.fields![0], child)] } + } + case "Struct": + return { + ...column, + fields: column.fields!.map((f) => { + const p = field(path, f.name) + const g = t?._tag === "Struct" ? (t.fields.find((g) => g.name === f.name) ?? missing(p)).type : undefined + return declareColumn(p, f, g) + }) + } + case "Decimal": + return t?._tag === "Decimal" ? { ...column, precision: t.precision, scale: t.scale } : column + case "Timestamp": + case "Duration": + return { ...column, unit: t?._tag === column.type ? t.unit : "microsecond" } + default: + return t === undefined || column.type === "Null" ? column : { ...column, type: typeName(t) } + } +} + +/** Arrow types frames hold as another type, which is the one to declare. */ +const heldAs: { readonly [tag: string]: string } = { + LargeUtf8: "Utf8", + Utf8View: "Utf8", + LargeBinary: "Binary", + BinaryView: "Binary", + LargeList: "List", + Dictionary: "their value type" +} + +const undeclarable = (path: string, what: string, why?: string): never => + V.fail(`column ${JSON.stringify(path)}: ${what} can't be declared${why === undefined ? "" : `: ${why}`}`) + +/** + * The declared column of an Arrow field. Only types that {@link arrowType} + * gives back are declared: a type frames can't hold, or hold as another + * type, fails. A UTC time zone is dropped, as {@link checkType} reads it. + */ +const fieldColumn = (path: string, f: S.Field): Plan.TableColumn => { + const t = f.type + const column = { name: f.name, nullable: f.nullable } + switch (t._tag) { + case "Null": + return f.nullable + ? { ...column, type: "Null" } + : V.fail(`column ${JSON.stringify(path)}: Null columns are nullable`) + case "Bool": + case "Utf8": + case "Binary": + case "Int": + return { ...column, type: typeName(t) } + case "Float": + return t.bitWidth === 16 ? undeclarable(path, "Float16 columns") : { ...column, type: typeName(t) } + case "Date": + return t.unit === "day" + ? { ...column, type: "Date" } + : undeclarable(path, `dates in ${t.unit}s`, "declared dates are in days") + case "Time": + return t.unit === "microsecond" + ? { ...column, type: "Time" } + : undeclarable(path, `times in ${t.unit}s`, "declared times are in microseconds") + case "Timestamp": + checkType(path, t, { type: "Timestamp", nullable: f.nullable }) + return { ...column, type: "Timestamp", unit: t.unit } + case "Duration": + return { ...column, type: "Duration", unit: t.unit } + case "Decimal": + return t.bitWidth === 128 + ? { ...column, type: "Decimal", precision: t.precision, scale: t.scale } + : undeclarable(path, `${t.bitWidth}-bit decimals`, "declared decimals are 128-bit") + case "List": + return { ...column, type: "List", fields: [fieldColumn(`${path}[]`, { ...t.child, name: "item" })] } + case "Struct": + return { ...column, type: "Struct", fields: t.fields.map((g) => fieldColumn(field(path, g.name), g)) } + default: + return undeclarable( + path, + `${t._tag} columns`, + heldAs[t._tag] === undefined ? undefined : `frames hold them as ${heldAs[t._tag]}` + ) + } +} + +/** + * The declared columns of an Arrow schema's fields: their types, + * nullability, nested fields, decimal precision and scale, and time units. + * + * @internal + */ +export const fromArrowSchema = (schema: S.Schema): Array => + schema.fields.map((f) => fieldColumn(f.name, f)) + +/** + * The Arrow type of a declared column. Timestamps and durations without a + * unit are microseconds, dates days and times microseconds; decimals need + * their precision and scale. + * + * @internal + */ +export const arrowType = (column: Plan.TableColumn, path: string = column.name): S.DataType => { + switch (column.type) { + case "List": { + const item = column.fields![0] + return S.list(S.field("item", arrowType(item, `${path}[]`), { nullable: item.nullable })) + } + case "Struct": + return S.struct( + column.fields!.map((f) => S.field(f.name, arrowType(f, field(path, f.name)), { nullable: f.nullable })) + ) + case "Utf8": + return S.utf8 + case "Binary": + return S.binary + case "Bool": + return S.bool + case "Null": + return S.nullType + case "Date": + return S.date("day") + case "Time": + return S.time("microsecond") + case "Timestamp": + return S.timestamp(column.unit ?? "microsecond") + case "Duration": + return S.duration(column.unit ?? "microsecond") + case "Decimal": + return column.precision === undefined || column.scale === undefined + ? V.fail( + `column ${ + JSON.stringify(path) + }: decimal columns need a precision and scale: pass options.types (for example ArrowSchema.decimal(18, 2))` + ) + : S.decimal(column.precision, column.scale) + case "Float16": + return S.float16 + default: + // The other numeric types, named as cast targets. + return D.castTargets[column.type] + } +} + +/** + * The Arrow schema of declared columns. + * + * @internal + */ +export const toArrowSchema = (columns: ReadonlyArray): S.Schema => + S.make(columns.map((c) => S.field(c.name, arrowType(c), { nullable: c.nullable }))) + +// ----------------------------------------------------------------------------- +// Checking frames +// ----------------------------------------------------------------------------- + +const inRows = (rows: Uint8Array | undefined, i: number): boolean => rows === undefined || rows[i] === 1 + +const allNull = (v: V.Vector, rows: Uint8Array | undefined): boolean => { + for (let i = 0; i < v.length; i++) if (inRows(rows, i) && V.isValid(v, i)) return false + return true +} + +/** The stored values of a timestamp in each unit that `DateTime` represents (±8.64e15 ms). */ +const timestampRange: { readonly [U in S.TimeUnit]: readonly [bigint, bigint] } = { + second: [-BigInt("8640000000000"), BigInt("8640000000000")], + millisecond: [-BigInt("8640000000000000"), BigInt("8640000000000000")], + microsecond: [-BigInt("8640000000000000000"), BigInt("8640000000000000999")], + nanosecond: [-BigInt("9223372036854775808"), BigInt("9223372036854775807")] +} + +const checkRange = (path: string, v: V.I64, unit: S.TimeUnit, rows: Uint8Array | undefined): void => { + const [lo, hi] = timestampRange[unit] + for (let i = 0; i < v.length; i++) { + if (inRows(rows, i) && V.isValid(v, i) && (v.values[i] < lo || v.values[i] > hi)) { + V.fail(`column ${JSON.stringify(path)} has timestamps outside the range of DateTime`) + } + } +} + +const sameValue = (a: V.Vector, b: V.Vector, i: number): boolean => { + switch (a.kind) { + case "f64": { + const x = a.values[i] + const y = (b as V.F64).values[i] + return Object.is(x, y) || (x !== x && y !== y) + } + case "i32": + case "i64": + case "bool": + return a.values[i] === (b as typeof a).values[i] + case "str": + return V.decodeString(a, i) === V.decodeString(b as V.Str, i) + default: + return true + } +} + +/** `v` converted to `target`, failing unless every present value converts exactly. */ +const convert = (path: string, v: V.Vector, target: S.DataType, rows: Uint8Array | undefined): V.Vector => { + if (D.sameType(v.type, target)) return v + const converted = K.cast(v, target) + const back = K.cast(converted, v.type) + for (let i = 0; i < v.length; i++) { + if ( + inRows(rows, i) && V.isValid(v, i) && + (!V.isValid(converted, i) || !sameValue(v, back, i)) + ) { + return inexact(path, target) + } + } + return converted +} + +const inexact = (path: string, target: S.DataType): never => + V.fail(`column ${JSON.stringify(path)} has values that ${D.describe(target)} can't hold exactly`) + +/** Microseconds in each unit of a time, as a fraction. */ +const microseconds: { readonly [U in S.TimeUnit]: readonly [bigint, bigint] } = { + second: [BigInt(1_000_000), BigInt(1)], + millisecond: [BigInt(1_000), BigInt(1)], + microsecond: [BigInt(1), BigInt(1)], + nanosecond: [BigInt(1), BigInt(1_000)] +} + +/** + * A date in days or a time in microseconds (`target`) from one in another + * unit, failing unless every present value converts exactly. `K.cast` + * reads the stored numbers of these types as they are, in any unit. + */ +const rescale = (path: string, v: V.I32 | V.I64, target: S.Date | S.Time, rows: Uint8Array | undefined): V.Vector => { + const [mul, div] = v.type._tag === "Time" ? microseconds[v.type.unit] : [BigInt(1), BigInt(86_400_000)] + const out = target._tag === "Date" ? new Int32Array(v.length) : new BigInt64Array(v.length) + for (let i = 0; i < v.length; i++) { + if (!inRows(rows, i) || !V.isValid(v, i)) continue + const x = BigInt(v.values[i]) * mul + const y = x / div + if (y * div !== x || BigInt.asIntN(out.BYTES_PER_ELEMENT * 8, y) !== y) return inexact(path, target) + out[i] = (target._tag === "Date" ? Number(y) : y) as never + } + return out instanceof Int32Array ? V.i32(target, out, v.valid) : V.i64(target, out, v.valid) +} + +/** A column checked against its declaration and converted to its declared type. */ +const check = (path: string, v: V.Vector, expected: Plan.TableColumn, rows: Uint8Array | undefined): V.Vector => { + if (expected.type === "Null") { + if (!allNull(v, rows)) checkType(path, v.type, expected) + return V.nulls(v.length) + } + checkType(path, v.type, expected) + if (!expected.nullable) { + for (let i = 0; i < v.length; i++) { + if (inRows(rows, i) && !V.isValid(v, i)) { + return V.fail(`column ${JSON.stringify(path)} has missing values`) + } + } + } + if (v.kind === "struct" && v.type._tag === "Struct") { + const present = new Uint8Array(v.length) + for (let i = 0; i < v.length; i++) present[i] = inRows(rows, i) && V.isValid(v, i) ? 1 : 0 + const fields = v.type.fields + const children = expected.fields!.map((f) => { + const k = fields.findIndex((g) => g.name === f.name) + return k === -1 ? missing(field(path, f.name)) : check(field(path, f.name), v.children[k], f, present) + }) + const type = S.struct(expected.fields!.map((f, k) => S.field(f.name, children[k].type, { nullable: f.nullable }))) + return V.struct(type, children, v.length, v.valid) + } + if (v.kind === "list") { + const present = new Uint8Array(v.child.length) + for (let i = 0; i < v.length; i++) { + if (inRows(rows, i) && V.isValid(v, i)) present.fill(1, v.offsets[i], v.offsets[i + 1]) + } + const item = expected.fields![0] + const child = check(`${path}[]`, v.child, item, present) + return V.list(S.list(S.field("item", child.type, { nullable: item.nullable })), v.offsets, child, v.valid) + } + if (expected.type === "Timestamp" && v.kind === "i64" && v.type._tag === "Timestamp") { + checkRange(path, v, v.type.unit, rows) + } + const target = arrowType(expected, path) + return (target._tag === "Date" || target._tag === "Time") && !D.sameType(v.type, target) + ? rescale(path, v as V.I32 | V.I64, target, rows) + : convert(path, v, target, rows) +} + +/** + * Fails unless the values of the columns that `checks` names pass their + * test, and the rows the row test. + * + * @internal + */ +export const runChecks = ( + checks: Plan.TableChecks, + values: (name: string) => ReadonlyArray, + rows: () => ReadonlyArray +): void => { + for (const [name, test] of checks.columns) { + if (!values(name).every(test)) V.fail(`column ${JSON.stringify(name)} has values its schema rejects`) + } + if (checks.rows !== undefined && !rows().every(checks.rows)) V.fail("rows have values their schema rejects") +} + +/** + * Checks that a frame has the declared columns and narrows it to them, in + * their order, converting each to its declared type (exactly: a value the + * declared type can't hold fails). With `checks`, the values of the columns + * they name, and the rows when they test rows, are then read and tested; + * other columns are never read as values. + * + * @internal + */ +export const conform = ( + frame: F.Frame, + columns: ReadonlyArray, + checks?: Plan.TableChecks | undefined +): F.Frame => { + const narrowed = F.make( + columns.map((expected) => { + const c = frame.columns.find((c) => c.name === expected.name) ?? missing(expected.name) + const values = c.values() + const checked = check(expected.name, values, expected, undefined) + return checked === values && c.nullable === expected.nullable + ? c + : new F.Column(c.name, checked, undefined, expected.nullable) + }), + frame.length + ) + if (checks !== undefined) runChecks(checks, (name) => F.toArray(narrowed, name), () => F.toRows(narrowed)) + return narrowed +} diff --git a/packages/effect/test/dataframe/DataFrame.test.ts b/packages/effect/test/dataframe/DataFrame.test.ts index 5a1ed205857..7b84da13233 100644 --- a/packages/effect/test/dataframe/DataFrame.test.ts +++ b/packages/effect/test/dataframe/DataFrame.test.ts @@ -1,7 +1,9 @@ import { assert, describe, it } from "@effect/vitest" -import { BigDecimal, DateTime, Effect, Equal, Result, Schema, Stream } from "effect" +import { BigDecimal, DateTime, Duration, Effect, Equal, Result, Schema, Stream } from "effect" import { ArrowIpc, ArrowSchema, ArrowTable } from "effect/arrow" -import { DataFrame, Expr, Query } from "effect/dataframe" +import { Catalog, DataFrame, Expr, Query } from "effect/dataframe" +import * as InternalDateTime from "effect/internal/dateTime" +import { vi } from "vitest" const Reading = Schema.Struct({ city: Schema.String, @@ -96,6 +98,62 @@ describe("DataFrame", () => { assert.strictEqual(df.length, 0) assert.deepStrictEqual(DataFrame.columnNames(df), ["city", "temp", "rain", "station"]) }) + + // DF-C022 (API contract): DataFrame.fromRows; pinned source in README.md. + it("fromRows checks rows against the schema's refinements, literals and row checks", () => { + const Positive = Schema.Number.check(Schema.isGreaterThan(0)) + const Order = Schema.Struct({ + n: Positive, + kind: Schema.Literals(["x", "y"]), + note: Schema.optional(Schema.NonEmptyString), + lines: Schema.Array( + Schema.Struct({ qty: Schema.NullOr(Positive), unit: Schema.optional(Schema.Literal("kg")) }) + ) + }).check(Schema.makeFilter((row) => row.lines.length <= row.n || "more lines than n")) + const valid = { n: 2, kind: "x", lines: [{ qty: 1 }, { qty: null, unit: undefined }] } as const + const df = DataFrame.fromRows(Order, [valid, { ...valid, note: undefined }, { ...valid, note: "late" }]) + assert.deepStrictEqual(DataFrame.toArray(df, "note"), [null, null, "late"]) + assert.isTrue(Result.isSuccess(DataFrame.validate(df, Order))) + const cases: ReadonlyArray = [ + [{ n: -1 }, `column "n" has values its schema rejects`], + [{ kind: "z" }, `column "kind" has values its schema rejects`], + [{ note: "" }, `column "note" has values its schema rejects`], + [{ lines: [{ qty: 1 }, { qty: 0 }] }, `column "lines" has values its schema rejects`], + [{ lines: [{ qty: 1, unit: "lb" }] }, `column "lines" has values its schema rejects`], + [{ n: 1 }, "rows have values their schema rejects"] + ] + for (const [fields, message] of cases) { + const rows = [valid, { ...valid, ...fields }] as unknown as ReadonlyArray + assert.throws(() => DataFrame.fromRows(Order, rows), DataFrame.DataFrameError, message) + } + }) + + // DF-C023 (API contract): DataFrame.fromRows; pinned source in README.md. + it("fromRows, empty and validate reject schemas that transform their rows", () => { + const cases: ReadonlyArray = [ + [Schema.Struct({ a: Schema.NumberFromString }), `field "a"`], + [Schema.Struct({ t: Schema.optional(Schema.DateTimeUtcFromString) }), `field "t"`], + [ + Schema.Struct({ p: Schema.Struct({ xs: Schema.Array(Schema.NullOr(Schema.NumberFromString)) }) }), + `field "p.xs[]"` + ], + [Schema.Struct({ a: Schema.Number.pipe(Schema.withDecodingDefault(Effect.succeed(1))) }), `field "a"`], + [Schema.Struct({ a: Schema.suspend(() => Schema.NumberFromString) }), `field "a"`], + [Schema.Struct({ a: Schema.Number }).pipe(Schema.encodeKeys({ a: "b" })), "the schema"] + ] + for (const [schema, subject] of cases) { + const message = + `${subject} has a transformation: frames hold decoded values and transformations are not run; ` + + "Schema.toType(schema) describes the decoded rows" + assert.throws(() => DataFrame.fromRows(schema, []), DataFrame.DataFrameError, message) + assert.throws(() => DataFrame.empty(schema), DataFrame.DataFrameError, message) + const result = DataFrame.validate(DataFrame.fromColumns({ a: ["1"] }), schema) + assert.strictEqual(Result.isFailure(result) && result.failure.message, message) + } + const Decoded = Schema.toType(Schema.Struct({ a: Schema.NumberFromString, t: Schema.DateTimeUtcFromString })) + const row = { a: 1, t: DateTime.makeUnsafe(0) } + assert.deepStrictEqual(DataFrame.toRows(DataFrame.fromRows(Decoded, [row])), [row]) + }) }) // DF-C021 (API contract): compatible Arrow layouts; pinned source in README.md. @@ -255,6 +313,163 @@ describe("DataFrame", () => { `column "d" has values that Decimal[18, 0] can't hold exactly` ) }) + + // DF-C024 (local policy): DataFrame.validate; pinned source in README.md. + it("builds values only for the fields the schema constrains", () => { + const Row = Schema.Struct({ t: Schema.DateTimeUtc, d: Schema.Duration, m: Schema.BigDecimal }) + const types = { m: ArrowSchema.decimal(18, 2) } + const df = arrowFrame({ + t: [ArrowSchema.timestamp("millisecond"), [1n, 2n, 3n]], + d: [ArrowSchema.duration("second"), [1n, 2n, 3n]], + m: [ArrowSchema.decimal(9, 2), ["1.5", "2.5", "3.5"].map(BigDecimal.fromStringUnsafe)] + }) + const built = [vi.spyOn(InternalDateTime, "makeUtc"), vi.spyOn(Duration, "nanos"), vi.spyOn(BigDecimal, "make")] + const counts = () => built.map((spy) => spy.mock.calls.length) + try { + const validated = Result.getOrThrow(DataFrame.validate(df, Row, { types })) + const table = Query.table("t", Row, { types }).pipe(Effect.provide(Catalog.layer({ t: Stream.make(df, df) }))) + assert.strictEqual(Effect.runSync(table).length, 6) + assert.deepStrictEqual(counts(), [0, 0, 0]) + const After = Schema.Struct({ + ...Row.fields, + t: Schema.DateTimeUtc.check(Schema.makeFilter((t) => DateTime.toEpochMillis(t) > 0)) + }) + assert.isTrue(Result.isSuccess(DataFrame.validate(df, After, { types }))) + assert.deepStrictEqual(counts(), [3, 0, 0]) + // Reading rows builds the values, which shows that the spies see them. + const rows = DataFrame.toRows(validated) + assert.deepStrictEqual(counts().map((n, i) => n > [3, 0, 0][i]), [true, true, true]) + assert.deepStrictEqual(rows[0], { + t: DateTime.makeUnsafe(1), + d: Duration.nanos(1_000_000_000n), + m: BigDecimal.make(150n, 2) + }) + } finally { + for (const spy of built) spy.mockRestore() + } + }) + + // DF-C025 (API contract): DataFrame.validate; pinned source in README.md. + it("takes an Arrow schema, converting the frame to its columns", () => { + const item = ArrowSchema.field("element", ArrowSchema.decimal(18, 2), { nullable: false }) + const schema = ArrowSchema.make([ + ArrowSchema.field("t", ArrowSchema.timestamp("second", "UTC")), + ArrowSchema.field("n", ArrowSchema.int32, { nullable: false }), + ArrowSchema.field("xs", ArrowSchema.list(item)), + ArrowSchema.field("p", ArrowSchema.struct([ArrowSchema.field("s", ArrowSchema.utf8, { nullable: false })])) + ]) + const frame = (n: number | null, unit: ArrowSchema.TimeUnit = "nanosecond", t = 1_000_000_000n) => + arrowFrame({ + extra: [ArrowSchema.bool, [true]], + p: [ArrowSchema.struct([ArrowSchema.field("s", ArrowSchema.largeUtf8)]), [{ s: "a" }]], + xs: [ArrowSchema.list(ArrowSchema.field("item", ArrowSchema.decimal(9, 1))), [[BigDecimal.make(15n, 1)]]], + n: [ArrowSchema.int8, [n]], + t: [ArrowSchema.timestamp(unit), [t]] + }) + const validated = Result.getOrThrow(DataFrame.validate(frame(1), schema)) + assert.deepStrictEqual( + validated.schema, + ArrowSchema.make([ + ArrowSchema.field("t", ArrowSchema.timestamp("second")), + schema.fields[1], + ArrowSchema.field("xs", ArrowSchema.list({ ...item, name: "item" })), + schema.fields[3] + ]) + ) + assert.deepStrictEqual(frame(1).pipe(DataFrame.validate(schema), Result.getOrThrow), validated) + assert.deepStrictEqual(DataFrame.toRows(validated), [{ + t: DateTime.makeUnsafe(1000), + n: 1, + xs: [BigDecimal.make(150n, 2)], + p: { s: "a" } + }]) + for ( + const [df, message] of [ + [frame(null), `column "n" has missing values`], + // A finer unit converts only when no digit is lost (DF-C010 has the same for struct schemas). + [frame(1, "millisecond", 1_500n), `column "t" has values that Timestamp[s] can't hold exactly`] + ] as const + ) { + const result = DataFrame.validate(df, schema) + assert.strictEqual(Result.isFailure(result) && result.failure.message, message) + } + }) + + // DF-C027 (API contract): DataFrame.validate; pinned source in README.md. + it("converts dates and times to the declared unit", () => { + const schema = ArrowSchema.make([ + ArrowSchema.field("d", ArrowSchema.date("day")), + ArrowSchema.field("at", ArrowSchema.time("microsecond")) + ]) + const frame = (d: bigint | null, at: number | null) => + arrowFrame({ + d: [ArrowSchema.date("millisecond"), [d, null]], + at: [ArrowSchema.time("millisecond"), [at, null]] + }) + const source = frame(172_800_000n, 1_500) + const validated = Result.getOrThrow(DataFrame.validate(source, schema)) + assert.deepStrictEqual(validated.schema, schema) + assert.deepStrictEqual(DataFrame.toArray(validated, "d"), DataFrame.toArray(source, "d")) + // A time reads as its count of the column's unit. + assert.deepStrictEqual(DataFrame.toArray(validated, "at"), [1_500_000n, null]) + const result = DataFrame.validate(frame(172_800_001n, 1), schema) + assert.strictEqual( + Result.isFailure(result) && result.failure.message, + `column "d" has values that Date can't hold exactly` + ) + }) + + // DF-C026 (API contract): DataFrame.validate; pinned source in README.md. + it("rejects Arrow schemas with types a table can't declare", () => { + const { field } = ArrowSchema + const cases: ReadonlyArray = [ + [ + ArrowSchema.timestamp("second", "Europe/Berlin"), + `column "c" has time zone Europe/Berlin, expected UTC timestamps` + ], + [ArrowSchema.interval("yearMonth"), `column "c": Interval columns can't be declared`], + [ArrowSchema.float16, `column "c": Float16 columns can't be declared`], + [ + ArrowSchema.map(field("k", ArrowSchema.utf8), field("v", ArrowSchema.int32)), + `column "c": Map columns can't be declared` + ], + [ + ArrowSchema.dictionary(ArrowSchema.utf8), + `column "c": Dictionary columns can't be declared: frames hold them as their value type` + ], + [ArrowSchema.largeUtf8, `column "c": LargeUtf8 columns can't be declared: frames hold them as Utf8`], + [ + ArrowSchema.largeList(field("item", ArrowSchema.int32)), + `column "c": LargeList columns can't be declared: frames hold them as List` + ], + [ + ArrowSchema.decimal(18, 2, 64), + `column "c": 64-bit decimals can't be declared: declared decimals are 128-bit` + ], + [ + ArrowSchema.date("millisecond"), + `column "c": dates in milliseconds can't be declared: declared dates are in days` + ], + [ + ArrowSchema.time("second"), + `column "c": times in seconds can't be declared: declared times are in microseconds` + ], + [ArrowSchema.list(field("item", ArrowSchema.float16)), `column "c[]": Float16 columns can't be declared`], + [ + ArrowSchema.struct([field("x", ArrowSchema.utf8View)]), + `column "c.x": Utf8View columns can't be declared: frames hold them as Utf8` + ] + ] + for (const [type, message] of cases) { + const result = DataFrame.validate(readings, ArrowSchema.make([field("c", type)])) + assert.strictEqual(Result.isFailure(result) && result.failure.message, message) + } + const notNull = DataFrame.validate( + readings, + ArrowSchema.make([field("c", ArrowSchema.nullType, { nullable: false })]) + ) + assert.strictEqual(Result.isFailure(notNull) && notNull.failure.message, `column "c": Null columns are nullable`) + }) }) describe("Arrow interop", () => { diff --git a/packages/effect/test/dataframe/Query.test.ts b/packages/effect/test/dataframe/Query.test.ts index a5df84d36b5..6eba4cf04ae 100644 --- a/packages/effect/test/dataframe/Query.test.ts +++ b/packages/effect/test/dataframe/Query.test.ts @@ -206,6 +206,99 @@ describe("Query", () => { assert.deepStrictEqual(DataFrame.columnNames(df), ["temp", "city"]) })) + // DF-Q043 (API contract): Query.table; pinned source in README.md. + it.effect("compiles a schema's checks when the table is declared, not per batch", () => + Effect.gen(function*() { + const Kind = Schema.Struct({ kind: Schema.Literals(["a", "b"]) }) + let reads = 0 + const counted = { + Type: Kind.Type, + get ast() { + reads++ + return Kind.ast + } + } + const query = Query.table("t", counted) + const declared = reads + const good = DataFrame.fromColumns({ kind: ["a", "b"] }) + const bad = DataFrame.fromColumns({ kind: ["c"] }) + const read = (query: Query.Query, batches: ReadonlyArray>) => + query.pipe(Effect.provide(Catalog.layer({ t: Stream.fromIterable(batches) }))) + assert.strictEqual((yield* read(query, [good, good, good, good])).length, 8) + const error = yield* Effect.flip(read(query, [good, good, bad])) + assert.strictEqual(error.message, `column "kind" has values its schema rejects`) + assert.strictEqual(reads, declared) + // The checks are functions: they stay out of the JSON form of the plan. + const json = JSON.stringify(Schema.encodeSync(Plan.NodeFromJson)(query.plan)) + const decoded = Query.fromPlan(Schema.decodeUnknownSync(Plan.NodeFromJson)(JSON.parse(json))) + assert.deepStrictEqual(decoded.plan, { + _tag: "Table", + name: "t", + columns: [{ name: "kind", type: "Utf8", nullable: false }] + }) + assert.strictEqual((yield* read(decoded, [good, bad])).length, 3) + })) + + // DF-Q044 (API contract): Query.table; pinned source in README.md. + it.effect("tables declared by an Arrow schema convert their input to its columns", () => + Effect.gen(function*() { + const schema = ArrowSchema.make([ + ArrowSchema.field("d", ArrowSchema.decimal(18, 2), { nullable: false }), + ArrowSchema.field("n", ArrowSchema.int32) + ]) + const t = Query.table("t", schema) + assert.deepStrictEqual(t.plan, { + _tag: "Table", + name: "t", + columns: [ + { name: "d", type: "Decimal", nullable: false, precision: 18, scale: 2 }, + { name: "n", type: "Int32", nullable: true } + ] + }) + const batch = (d: string | null, scale: number, n: number | null) => + arrowFrame({ + n: [ArrowSchema.int8, [n]], + extra: [ArrowSchema.utf8, ["x"]], + d: [ArrowSchema.decimal(9, scale), [d === null ? null : BigDecimal.fromStringUnsafe(d)]] + }) + const read = (source: Catalog.Source) => t.pipe(Effect.provide(Catalog.layer({ t: source }))) + const expected = [{ d: BigDecimal.make(120n, 2), n: 1 }] + for (const source of [batch("1.2", 1, 1), Stream.make(batch("1.2", 1, 1)), Stream.make(batch("1.20", 2, 1))]) { + const df = yield* read(source) + assert.deepStrictEqual(df.schema, schema) + assert.deepStrictEqual(DataFrame.toRows(df), expected) + } + const two = yield* read(Stream.make(batch("1.2", 1, 1), batch("1.20", 2, null))) + assert.deepStrictEqual(DataFrame.toRows(two), [...expected, { d: BigDecimal.make(120n, 2), n: null }]) + assert.deepStrictEqual((yield* read(Stream.empty)).schema, schema) + for ( + const [bad, message] of [ + [batch(null, 1, 1), `column "d" has missing values`], + [batch("1.234", 3, 1), `column "d" has values that Decimal[18, 2] can't hold exactly`] + ] as const + ) { + for (const source of [bad, Stream.make(batch("1.2", 1, 1), bad)]) { + assert.strictEqual((yield* Effect.flip(read(source))).message, message) + } + } + })) + + // DF-Q045 (API contract): Query.table; pinned source in README.md. + it("fails when a table is declared by a schema that has no columns for it", () => { + assert.throws( + () => Query.table("t", Schema.Struct({ a: Schema.NumberFromString })), + DataFrame.DataFrameError, + `field "a" has a transformation: frames hold decoded values and transformations are not run; ` + + "Schema.toType(schema) describes the decoded rows" + ) + const zoned = ArrowSchema.make([ArrowSchema.field("t", ArrowSchema.timestamp("second", "Europe/Berlin"))]) + assert.throws( + () => Query.table("t", zoned), + DataFrame.DataFrameError, + `column "t" has time zone Europe/Berlin, expected UTC timestamps` + ) + }) + // DF-Q008 (API contract): DataFrame.DataFrameError; pinned source in README.md. it.effect("fails with DataFrameError on invalid expressions", () => Effect.gen(function*() { diff --git a/packages/effect/typetest/DataFrame.tst.ts b/packages/effect/typetest/DataFrame.tst.ts index 761587f48d8..21b6748c78b 100644 --- a/packages/effect/typetest/DataFrame.tst.ts +++ b/packages/effect/typetest/DataFrame.tst.ts @@ -55,6 +55,22 @@ describe("DataFrame", () => { expect(Query.table).type.not.toBeCallableWith("t", Price, { types: { e: ArrowSchema.int32 } }) }) + it("validate and Query.table take an Arrow schema and give untyped rows", () => { + const schema = ArrowSchema.make([ArrowSchema.field("n", ArrowSchema.int32)]) + type Untyped = { readonly [name: string]: unknown } + expect(DataFrame.validate(DataFrame.empty(Station), schema)).type.toBe< + Result.Result, DataFrame.DataFrameError> + >() + expect(DataFrame.empty(Station).pipe(DataFrame.validate(schema))).type.toBe< + Result.Result, DataFrame.DataFrameError> + >() + expect(Query.table("t", schema)).type.toBe>() + expect(Query.table).type.not.toBeCallableWith("t", schema, { types: { n: ArrowSchema.int64 } }) + expect(DataFrame.validate).type.not.toBeCallableWith(DataFrame.empty(Station), schema, { + types: { n: ArrowSchema.int64 } + }) + }) + it("fromColumns infers column types", () => { expect(DataFrame.fromColumns({ a: new Float64Array(1), b: ["x", null], c: new BigInt64Array(1) })).type.toBe< DataFrame.DataFrame<{ readonly a: number; readonly b: string | null; readonly c: bigint }> From 29e15e49e13494f9331b358031da5ea37f141001 Mon Sep 17 00:00:00 2001 From: Effect Bot Date: Mon, 16 Nov 2026 08:19:14 +0000 Subject: [PATCH 65/88] List the schema check and Arrow declaration tests in the dataframe inventory --- packages/effect/test/dataframe/README.md | 39 ++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/packages/effect/test/dataframe/README.md b/packages/effect/test/dataframe/README.md index 4752576d566..5d545051117 100644 --- a/packages/effect/test/dataframe/README.md +++ b/packages/effect/test/dataframe/README.md @@ -40,6 +40,12 @@ The assertion column is the exact leaf test title. Combine it with the file and | DF-C019 | C | iterate as rows | [DataFrame.DataFrame](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/DataFrame.ts#L81) | | DF-C020 | P | format as a text table | [DataFrame.format](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/DataFrame.ts#L467) | | DF-C021 | C | reuses compatible numeric buffers across Arrow conversion | [DataFrame layouts](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/DataFrame.ts#L6-L9) | +| DF-C022 | C | fromRows checks rows against the schema's refinements, literals and row checks | [DataFrame.fromRows](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/DataFrame.ts#L192) | +| DF-C023 | C | fromRows, empty and validate reject schemas that transform their rows | [DataFrame.fromRows](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/DataFrame.ts#L192) | +| DF-C024 | P | builds values only for the fields the schema constrains | [DataFrame.validate](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/DataFrame.ts#L396) | +| DF-C025 | C | takes an Arrow schema, converting the frame to its columns | [DataFrame.validate](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/DataFrame.ts#L396) | +| DF-C026 | C | rejects Arrow schemas with types a table can't declare | [DataFrame.validate](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/DataFrame.ts#L396) | +| DF-C027 | C | converts dates and times to the declared unit | [DataFrame.validate](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/DataFrame.ts#L396) | DF-C021 checks a single, unsliced, non-null batch of Int32 and Float64 columns: import/export preserves the backing buffers, byte offsets and byte lengths, as well as values. It does not require typed-array wrapper identity or mutation-based aliasing. The compatible fixed-width layouts are described in Apache Arrow’s [Columnar format](https://github.com/apache/arrow/blob/ee4d09ebef61c663c1efbfa4c18e518a03b798be/docs/source/format/Columnar.rst#fixed-size-primitive-layout) and [Schema.fbs](https://github.com/apache/arrow/blob/ee4d09ebef61c663c1efbfa4c18e518a03b798be/format/Schema.fbs); the no-copy adapter promise is the Effect API contract, not a requirement that Arrow imposes on every adapter. @@ -89,6 +95,37 @@ DF-C021 checks a single, unsliced, non-null batch of Int32 and Float64 columns: | DF-Q040 | C | groups by keys of every type | [Query.groupBy](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Query.ts#L610) | | DF-Q041 | C | joins on a high-cardinality key | [Query.join](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Query.ts#L680) | | DF-Q042 | C | yields a long query so another fiber can interrupt it | [Query yielding](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Query.ts#L65-L66) | +| DF-Q043 | C | compiles a schema's checks when the table is declared, not per batch | [Query.table](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Query.ts#L254) | +| DF-Q044 | C | tables declared by an Arrow schema convert their input to its columns | [Query.table](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Query.ts#L254) | +| DF-Q045 | C | fails when a table is declared by a schema that has no columns for it | [Query.table](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Query.ts#L254) | + +DF-C022 to DF-C027 and DF-Q043 to DF-Q045 assert behaviour added after the baseline: checks compiled from the schema, rejected transformations, and tables declared by an Arrow schema. Their links locate the symbol at the baseline; the documentation of that symbol in the checked-out source is their contract. DF-C024 observes which value constructors run (`DateTime`, `Duration`, `BigDecimal`) and covers only a frame whose columns already have the declared types. + +## Supported Schema subset + +`DataFrame.fromRows`, `DataFrame.empty`, `DataFrame.validate` and `Query.table` read a struct schema once and take from it column types and checks. They never decode or encode with it. + +| Schema | Column type | +| ----------------------------------------- | ------------------------------------------------------ | +| `String`, template literals | Utf8 | +| `Number` | Float64, unless `options.types` names another type | +| `Boolean` | Bool | +| `BigInt` | Int64 | +| `Null`, `Undefined` | Null | +| literals and enums of one primitive type | the type of their values | +| `DateTimeUtc` | Timestamp in microseconds, without a time zone | +| `Duration` | Duration in microseconds | +| `BigDecimal` | Decimal; `options.types` must give precision and scale | +| `Array` of a supported schema | List | +| `Struct` of supported schemas | Struct | +| `NullOr`, `UndefinedOr`, optional fields | the member's type, nullable | +| unions whose members have one column type | that type | + +Rejected with a `DataFrameError` that names the field: transformations (any schema with an encoding, such as `NumberFromString`, `decodeTo`, decoding defaults and renamed keys; `Schema.toType` gives the decoded side), `DateTimeZoned`, tuples, records, classes, unions of different column types, and recursive schemas. + +Refinements, literals, enums and template literals are value checks: every value of such a field is read and tested, inside structs, arrays and union members too. A struct's own checks are row checks: every row is read and tested. Fields without checks are checked by column type only, on the typed columns. `DataFrame.fromRows` tests the rows it is given; `DataFrame.validate` and `Query.table` test what they read. The checks are compiled when the schema is read (`Plan.TableChecks`) and are not part of a serialized plan: a plan decoded from JSON checks column types only. A SQL engine tests value checks on distinct values and fails for a table with row checks. + +A table declared by an Arrow schema (DF-C025, DF-C026, DF-Q044) involves no Effect Schema and has no checks. ## Oracle provenance and differential assertions @@ -132,6 +169,7 @@ The actual SQL suite is **`packages/sql/pglite/test/DataFrameSql.test.ts`**, not - `fromRows takes the row type from the schema` - `fromRows, empty, validate and Query.table read missing values back as null` - `validate and Query.table take Arrow types for the schema's fields` +- `validate and Query.table take an Arrow schema and give untyped rows` - `fromColumns infers column types` ### `Query` @@ -177,6 +215,7 @@ Rejection assertions include unknown Arrow override fields (`DataFrame`), unknow - The callback timing contract now has a direct test (DF-Q002); it supplements, rather than replaces, lazy loading and per-run catalog tests. - DF-Q042 controls the host monotonic clock and disables automatic scheduler yields, then verifies a started public query lets its parent resume and interrupt it before completion. It does not assert a yield interval, timeout behavior, fairness among many fibers, or interruption within an operation. - DF-C021 covers buffer reuse only for the compatible numeric layouts described above. Sliced/multi-batch inputs, validity buffers, other layouts and mutation/ownership guarantees remain outside that assertion. +- DF-C022 and DF-Q043 cover the check kinds named in their tests (a refinement, literals, an optional field, an array of structs, a row check). The subset table above is documentation: its rows are not each asserted here. - Dedicated coverage for public guards, every dual overload, and every malformed plan/IPC input is not established by this inventory. Those are gaps, not passing assertions. - Compiler inference/rejection coverage is inventoried above and is not counted as runtime coverage. Exact diagnostic strings and explain/format layouts are local regression details even where a test also verifies a public failure or result contract. From 0e15174824597eb9ab230b2733fc04689e2bfffe Mon Sep 17 00:00:00 2001 From: Effect Bot Date: Mon, 16 Nov 2026 08:22:25 +0000 Subject: [PATCH 66/88] Round decimals half to even with integer arithmetic --- .../effect/src/dataframe/internal/decimal.ts | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/packages/effect/src/dataframe/internal/decimal.ts b/packages/effect/src/dataframe/internal/decimal.ts index 70529950b9e..cd7e25b422c 100644 --- a/packages/effect/src/dataframe/internal/decimal.ts +++ b/packages/effect/src/dataframe/internal/decimal.ts @@ -9,7 +9,6 @@ * @internal */ import * as S from "../../arrow/ArrowSchema.ts" -import * as BigDecimal from "../../BigDecimal.ts" import * as V from "./vector.ts" const powers: Array = [] @@ -17,6 +16,8 @@ const powers: Array = [] const pow10 = (k: number): bigint => (powers[k] ??= BigInt(10) ** BigInt(k)) const I0 = BigInt(0) +const I1 = BigInt(1) +const I2 = BigInt(2) const MAX = BigInt("9223372036854775807") const MIN = -BigInt("9223372036854775808") @@ -57,9 +58,17 @@ const wrap = (x: bigint): bigint => BigInt.asIntN(64, x) */ export const fit = (type: S.DataType): (x: bigint) => bigint => isDecimal(type) ? checked : wrap -/** Rounds `x / 10^k` half to even (`BigDecimal.round`). */ -const divRound = (x: bigint, k: number): bigint => - k === 0 ? x : BigDecimal.round(BigDecimal.make(x, k), { scale: 0, mode: "half-even" }).value +/** Rounds `x / 10^k` half to even. */ +const divRound = (x: bigint, k: number): bigint => { + if (k === 0) return x + const divisor = pow10(k) + // Truncated towards zero; the remainder has the sign of `x`. + const quotient = x / divisor + const remainder = x % divisor + const twice = (remainder < I0 ? -remainder : remainder) * I2 + if (twice < divisor || (twice === divisor && quotient % I2 === I0)) return quotient + return x < I0 ? quotient - I1 : quotient + I1 +} /** `unscaled / 10^scale` as a decimal string, all `scale` digits shown. */ const format = (unscaled: bigint, scale: number): string => { From f93416451c5f047cc2a0797b3b5181854e8a1a29 Mon Sep 17 00:00:00 2001 From: Effect Bot Date: Mon, 16 Nov 2026 08:22:25 +0000 Subject: [PATCH 67/88] Read decimal SQL results into columns without a BigDecimal per value --- .../effect/src/dataframe/internal/decimal.ts | 52 ++++++++++++++++--- packages/effect/src/dataframe/internal/sql.ts | 35 ++++++++++++- 2 files changed, 79 insertions(+), 8 deletions(-) diff --git a/packages/effect/src/dataframe/internal/decimal.ts b/packages/effect/src/dataframe/internal/decimal.ts index cd7e25b422c..e673970aa81 100644 --- a/packages/effect/src/dataframe/internal/decimal.ts +++ b/packages/effect/src/dataframe/internal/decimal.ts @@ -79,15 +79,55 @@ const format = (unscaled: bigint, scale: number): string => { } const DECIMAL_SYNTAX = /^\s*([+-]?)(\d*)(?:\.(\d*))?\s*$/ +/** {@link DECIMAL_SYNTAX} without the spaces around it. */ +const PLAIN_SYNTAX = /^([+-]?)(\d*)(?:\.(\d*))?$/ -/** Parses a decimal string to unscaled units of `scale`; `undefined` when invalid. */ -const parse = (s: string, scale: number): bigint | undefined => { - const m = DECIMAL_SYNTAX.exec(s) +/** + * Parses a decimal string to unscaled units of `scale`; `undefined` when + * invalid. `drop` removes the `k` digits a fraction has beyond the scale. + */ +const parse = ( + s: string, + scale: number, + syntax: RegExp = DECIMAL_SYNTAX, + drop: (x: bigint, k: number) => bigint | undefined = divRound +): bigint | undefined => { + const m = syntax.exec(s) if (m === null || (m[2] === "" && (m[3] ?? "") === "")) return undefined const fraction = m[3] ?? "" - const all = BigInt(`${m[2] || "0"}${fraction}`) - const x = fraction.length >= scale ? divRound(all, fraction.length - scale) : all * pow10(scale - fraction.length) - return m[1] === "-" ? -x : x + const all = BigInt(`${m[1]}${m[2] || "0"}${fraction}`) + return fraction.length >= scale ? drop(all, fraction.length - scale) : all * pow10(scale - fraction.length) +} + +/** For {@link parse}: a fraction longer than the scale is invalid. */ +const dropNone = (x: bigint, k: number): bigint | undefined => (k === 0 ? x : undefined) + +/** + * A decimal vector of `type` from decimal strings; `null` and `undefined` + * are nulls. Reads plain decimal notation (`-12.34`) only: `undefined` when + * a string has another form, more fraction digits than the scale, or a value + * outside the type's bit width or the 64 bits of a vector. + * + * @internal + */ +export const fromStrings = (type: S.Decimal, values: ReadonlyArray): V.I64 | undefined => { + const n = values.length + const out = new BigInt64Array(n) + const valid = new Uint8Array(n) + const limit = I1 << BigInt(Math.min(type.bitWidth, 64) - 1) + let nulls = false + for (let i = 0; i < n; i++) { + const v = values[i] + if (v === null || v === undefined) { + nulls = true + continue + } + const x = parse(String(v), type.scale, PLAIN_SYNTAX, dropNone) + if (x === undefined || x >= limit || x < -limit) return undefined + out[i] = x + valid[i] = 1 + } + return V.i64(type, out, nulls ? valid : undefined) } /** diff --git a/packages/effect/src/dataframe/internal/sql.ts b/packages/effect/src/dataframe/internal/sql.ts index 0d4feeb3888..17e0b4ae85d 100644 --- a/packages/effect/src/dataframe/internal/sql.ts +++ b/packages/effect/src/dataframe/internal/sql.ts @@ -26,6 +26,7 @@ import * as S from "../../arrow/ArrowSchema.ts" import * as BigDecimal from "../../BigDecimal.ts" import * as Result from "../../Result.ts" import type * as Plan from "../Plan.ts" +import * as Dec from "./decimal.ts" import * as D from "./dtype.ts" import * as E from "./expr.ts" import * as F from "./frame.ts" @@ -1296,14 +1297,44 @@ export const verify = ( export const read = (compiled: Compiled, rows: ReadonlyArray>): F.Frame => decode(compiled.schema, verify(compiled.query, rows)) +/** + * The vector of a result column of decimals, or of lists of decimals, parsed + * from the strings of its values without a `BigDecimal` per value. + * `undefined` for other types and for values `Dec.fromStrings` doesn't read. + * `decode` then builds the column from what `reader` reads, which also + * reports invalid values. + */ +const decimals = (type: S.DataType, values: ReadonlyArray): V.Vector | undefined => { + if (type._tag === "Decimal") return Dec.fromStrings(type, values) + if (type._tag !== "List" || type.child.type._tag !== "Decimal") return undefined + const offsets = new Int32Array(values.length + 1) + const valid = new Uint8Array(values.length) + const items: Array = [] + let nulls = false + values.forEach((v, i) => { + if (v === null || v === undefined) nulls = true + else { + valid[i] = 1 + for (const item of v as ReadonlyArray) items.push(item) + } + offsets[i + 1] = items.length + }) + const child = Dec.fromStrings(type.child.type, items) + // A null item in a field declared without nulls is left for `decode` to report. + if (child === undefined || (child.valid !== undefined && !type.child.nullable)) return undefined + return V.list(V.storageType(type), offsets, child, nulls ? valid : undefined) +} + /** The rows of a result, one array of values per row, as a frame of `schema`. */ const decode = (schema: S.Schema, rows: ReadonlyArray>): F.Frame => F.make( schema.fields.map((f, i) => { + const cells = rows.map((row) => row[i]) + const vector = decimals(f.type, cells) + if (vector !== undefined) return new F.Column(f.name, vector, undefined, vector.valid !== undefined) const read = reader(f.type) let nullable = false - const values = rows.map((row) => { - const v = row[i] + const values = cells.map((v) => { if (v !== null && v !== undefined) return read(v) nullable = true return null From 194d0fe30f4149e024668d816d5c72aa4ec7abf0 Mon Sep 17 00:00:00 2001 From: Effect Bot Date: Mon, 16 Nov 2026 08:22:25 +0000 Subject: [PATCH 68/88] Cover the columnar path of temporal and decimal columns --- packages/effect/test/dataframe/Query.test.ts | 301 +++++++++++++++++- packages/effect/test/dataframe/README.md | 17 + packages/sql/pglite/test/DataFrameSql.test.ts | 69 ++++ 3 files changed, 374 insertions(+), 13 deletions(-) diff --git a/packages/effect/test/dataframe/Query.test.ts b/packages/effect/test/dataframe/Query.test.ts index 6eba4cf04ae..82f69caddcb 100644 --- a/packages/effect/test/dataframe/Query.test.ts +++ b/packages/effect/test/dataframe/Query.test.ts @@ -1,18 +1,8 @@ import { assert, describe, it } from "@effect/vitest" -import { - BigDecimal, - type DateTime, - type Duration, - Effect, - Exit, - Fiber, - Result, - Scheduler, - Schema, - Stream -} from "effect" -import { ArrowSchema, ArrowTable } from "effect/arrow" +import { BigDecimal, DateTime, Duration, Effect, Equal, Exit, Fiber, Result, Scheduler, Schema, Stream } from "effect" +import { ArrowIpc, ArrowSchema, ArrowTable } from "effect/arrow" import { Catalog, DataFrame, Expr, Plan, Query } from "effect/dataframe" +import * as InternalDateTime from "effect/internal/dateTime" import { vi } from "vitest" import * as Compare from "./oracle/Compare.ts" import * as Gold from "./oracle/Gold.ts" @@ -1334,4 +1324,289 @@ describe("Query", () => { } })) }) + + // Columns hold temporal and decimal values as integers. Only rows and + // arrays of values turn them into DateTime, Duration and BigDecimal. + describe("columnar path", () => { + type Event = { + readonly us: DateTime.Utc | null + readonly zoned: DateTime.Utc | null + readonly ns: DateTime.Utc | null + readonly nsUtc: DateTime.Utc | null + readonly day: DateTime.Utc | null + readonly took: Duration.Duration | null + readonly amount: BigDecimal.BigDecimal | null + readonly key: string | null + } + const ns = [1_700_000_000_123_456_789n, 1_700_000_100_000_000_001n, null, -1n, 1_600_000_000_999_999_999n, 5n] + const micros = ns.map((x) => (x === null ? null : x / 1000n)) + const input = Result.getOrThrow(ArrowTable.fromColumns( + ArrowSchema.make([ + ArrowSchema.field("us", ArrowSchema.timestamp("microsecond")), + ArrowSchema.field("zoned", ArrowSchema.timestamp("microsecond", "Europe/Berlin")), + ArrowSchema.field("ns", ArrowSchema.timestamp("nanosecond")), + ArrowSchema.field("nsUtc", ArrowSchema.timestamp("nanosecond", "UTC")), + ArrowSchema.field("day", ArrowSchema.date("day")), + ArrowSchema.field("took", ArrowSchema.duration("microsecond")), + ArrowSchema.field("amount", ArrowSchema.decimal(18, 2)), + ArrowSchema.field("key", ArrowSchema.utf8) + ]), + [ + micros, + micros, + ns, + ns, + [19_000, 19_001, 19_000, null, -1, 0], + [1_500_000n, -250n, 90_000_000_000n, null, 7n, 0n], + ["12.34", "-0.05", null, "100.25", "0.35", "7.45"].map((s) => + s === null ? null : BigDecimal.fromStringUnsafe(s) + ), + ["a", "b", "a", "b", "a", null] + ] + )) + const bytes = Result.getOrThrow(ArrowIpc.encodeStream(input)) + /** Non-null values per column of `input`. */ + const present = 5 + + /** + * The DateTime, Duration and BigDecimal values built while `f` runs, by + * the functions `effect/arrow` and `effect/dataframe` build them with. + * `f` is synchronous, so no other test of this file runs meanwhile. + */ + const built = (f: () => A) => { + const count = (spies: ReadonlyArray<{ readonly mock: { readonly calls: ReadonlyArray } }>) => + spies.reduce((n, spy) => n + spy.mock.calls.length, 0) + const dateTimes = [vi.spyOn(InternalDateTime, "makeUtc"), vi.spyOn(InternalDateTime, "makeZonedProto")] + const durations = [vi.spyOn(Duration, "nanos")] + const decimals = [ + vi.spyOn(BigDecimal, "make"), + vi.spyOn(BigDecimal, "round"), + vi.spyOn(BigDecimal, "fromStringUnsafe") + ] + try { + const value = f() + return { value, dateTimes: count(dateTimes), durations: count(durations), decimals: count(decimals) } + } finally { + for (const spy of [...dateTimes, ...durations, ...decimals]) spy.mockRestore() + } + } + const none = { dateTimes: 0, durations: 0, decimals: 0 } + + // DF-N001 (local policy): columnar path; pinned source in README.md. + it("decoding, querying and encoding build no DateTime, Duration or BigDecimal", () => { + const since = DateTime.makeUnsafe(1_650_000_000_000) + const selected = (events: Query.Query) => + events.pipe( + Query.filter((c) => Expr.gt(c.us, since)), + Query.sort((c) => [Expr.desc(c.us), c.ns]), + Query.select((c) => ({ + us: c.us, + zoned: c.zoned, + ns: c.ns, + year: Expr.year(c.us), + month: Expr.month(c.ns), + weekday: Expr.weekday(c.day), + millisecond: Expr.millisecond(c.us), + hourly: Expr.truncate(c.ns, "1h"), + later: Expr.add(c.us, c.took), + elapsed: Expr.subtract(c.ns, c.us), + seconds: Expr.totalSeconds(c.took), + tenths: Expr.castDecimal(c.amount, 18, 1), + rounded: Expr.round(c.amount, 1) + })) + ) + const grouped = (events: Query.Query) => + events.pipe( + Query.groupBy("key"), + Query.agg((c) => ({ + total: Expr.sum(c.amount), + mean: Expr.mean(c.amount), + least: Expr.min(c.amount), + most: Expr.max(c.amount), + tookTotal: Expr.sum(Expr.totalMilliseconds(c.took)), + tookMean: Expr.mean(Expr.totalSeconds(c.took)), + tookLeast: Expr.min(c.took), + tookMost: Expr.max(c.took), + first: Expr.min(c.us), + last: Expr.max(c.ns) + })) + ) + const table = Query.fromPlan({ _tag: "Table", name: "events" }) as Query.Query as Query.Query + const run = built(() => { + const frame = DataFrame.fromArrowTable(Result.getOrThrow(ArrowIpc.decodeStream(bytes))) as DataFrame.DataFrame< + unknown + > as DataFrame.DataFrame + const batches = Effect.runSync(Stream.runCollect( + DataFrame.decodeArrowStream(Stream.make(bytes.subarray(0, 200), bytes.subarray(200))) + )) + const streamed = (query: Query.Query) => + Effect.runSync( + Stream.runCollect(Query.stream(query)).pipe( + Effect.provide(Catalog.layer({ events: Stream.fromIterable([...batches, ...batches]) })) + ) + ) + const results = { + selected: Effect.runSync(selected(Query.from(frame))), + grouped: Effect.runSync(grouped(Query.from(frame))), + streamed: [...streamed(selected(table)), ...streamed(grouped(table))] + } + const frames: ReadonlyArray> = [ + results.selected, + results.grouped, + ...results.streamed + ] + return { + ...results, + encoded: frames.map((df) => + [df, Result.getOrThrow(ArrowIpc.encodeStream(DataFrame.toArrowTable(df)))] as const + ) + } + }) + assert.deepStrictEqual({ dateTimes: run.dateTimes, durations: run.durations, decimals: run.decimals }, none) + // The results, read as rows afterwards: nothing was built, though the work was done. + const { encoded, grouped: groups, selected: events, streamed } = run.value + assert.deepStrictEqual( + DataFrame.toRows(events).map((r) => [ + r.us && DateTime.toEpochMillis(r.us), + r.year, + r.tenths && BigDecimal.format(r.tenths), + r.rounded && BigDecimal.format(r.rounded), + r.seconds + ]), + [ + [1_700_000_100_000, 2023, "0", "0", 0n], + [1_700_000_000_123, 2023, "12.3", "12.3", 1n] + ] + ) + assert.deepStrictEqual( + DataFrame.toRows(groups).map((r) => [ + r.key, + r.total && BigDecimal.format(r.total), + r.tookMost && Duration.toMillis(r.tookMost) + ]), + [["a", "12.69", 90_000_000], ["b", "100.2", -0.25], [null, "7.45", 0]] + ) + // Over the two batches: the sorted rows of both, then one row per group. + assert.deepStrictEqual(streamed.map((df) => df.length), [4, 3]) + for (const [df, bytes] of encoded) { + assert.isTrue(Equal.equals(DataFrame.fromArrowTable(Result.getOrThrow(ArrowIpc.decodeStream(bytes))), df)) + } + }) + + // DF-N002 (local policy): DataFrame.toRows and DataFrame.toArray; pinned source in README.md. + it("rows build one DateTime, Duration or BigDecimal per value; columns keep nanoseconds", () => { + const frame = DataFrame.fromArrowTable(input) + const rows = built(() => DataFrame.toRows(frame)) + // us, zoned, ns, nsUtc and day; took; amount. + assert.deepStrictEqual( + { dateTimes: rows.dateTimes, durations: rows.durations, decimals: rows.decimals }, + { dateTimes: 5 * present, durations: present, decimals: present } + ) + for (const name of ["us", "zoned", "ns", "nsUtc", "day", "took", "amount"]) { + const column = built(() => DataFrame.toArray(frame, name)) + assert.strictEqual(column.dateTimes + column.durations + column.decimals, present, name) + } + // Rows read nanoseconds as DateTime, which keeps milliseconds. + assert.deepStrictEqual( + rows.value.map((r) => r.ns === null ? null : DateTime.toEpochMillis(r.ns as DateTime.Utc)), + [1_700_000_000_123, 1_700_000_100_000, null, -1, 1_600_000_000_999, 0] + ) + const output = built(() => Result.getOrThrow(ArrowIpc.encodeStream(DataFrame.toArrowTable(frame)))) + assert.deepStrictEqual( + { dateTimes: output.dateTimes, durations: output.durations, decimals: output.decimals }, + none + ) + assert.deepStrictEqual(output.value, bytes) + const stored = Effect.runSync( + Query.from(DataFrame.fromArrowTable(Result.getOrThrow(ArrowIpc.decodeStream(output.value)))).pipe( + Query.select((c) => ({ ns: Expr.cast(c.ns as Expr.Expr, "Int64") })) + ) + ) + assert.deepStrictEqual(DataFrame.toArray(stored, "ns"), ns) + }) + + // DF-N003 (API contract): Expr.castDecimal and Expr.round; pinned source in README.md. + it.effect("dropping decimal digits rounds half to even, exactly", () => + Effect.gen(function*() { + /** + * Texts with `k` fraction digits brought to scale 0: parsed from the + * text, cast from a decimal of scale `k`, and rounded at that scale, + * the last two when 64 bits hold the text at scale `k`. + */ + const whole = (texts: ReadonlyArray, k: number) => + Query.from(DataFrame.fromColumns({ s: texts })).pipe( + Query.select((c) => { + const exact = Expr.castDecimal(c.s, 38, k) + return { + parsed: Expr.cast(Expr.castDecimal(c.s, 38, 0), "Int64"), + ...(k <= 18 && { + cast: Expr.cast(Expr.castDecimal(exact, 38, 0), "Int64"), + rounded: Expr.cast(Expr.round(exact, 0), "Int64") + }) + } + }), + Effect.map((df) => DataFrame.columnNames(df).map((name) => DataFrame.toArray(df, name))) + ) + const zeros = "0".repeat(38) + const pinned: ReadonlyArray = [ + [0, "7", 7n], + [0, "-7", -7n], + [1, "0.5", 0n], + [1, "1.5", 2n], + [1, "2.5", 2n], + [1, "-0.5", 0n], + [1, "-1.5", -2n], + [1, "-2.5", -2n], + [2, "0.51", 1n], + [2, "0.49", 0n], + [2, "-0.51", -1n], + [2, "-0.49", 0n], + [2, "-2.50", -2n], + [2, "-3.50", -4n], + [18, "8.500000000000000000", 8n], + [18, "-7.500000000000000000", -8n], + [18, "8.500000000000000001", 9n], + [18, "-8.499999999999999999", -8n], + [39, `0.5${zeros}`, 0n], + [39, `1.5${zeros}`, 2n], + [39, `-2.5${zeros}`, -2n], + [40, `0.5${zeros}1`, 1n], + [40, `-2.5${zeros}1`, -3n], + [39, `0.4${"9".repeat(38)}`, 0n] + ] + for (const k of new Set(pinned.map(([k]) => k))) { + const cases = pinned.filter((c) => c[0] === k) + for (const column of yield* whole(cases.map(([, text]) => text), k)) { + assert.deepStrictEqual(column, cases.map(([, , expected]) => expected), `${k} digits`) + } + } + // Against BigDecimal.round, over random values around and away from ties. + let state = 0x9e3779b9 + const next = (bound: number) => { + state ^= state << 13 + state ^= state >>> 17 + state ^= state << 5 + return Math.floor(((state >>> 0) / 4294967296) * bound) + } + const digits = (n: number) => Array.from({ length: n }, () => next(10)).join("") + for (const k of [0, 1, 2, 9, 17, 18, 25, 40]) { + const fractions = k === 0 ? [""] : [ + "0".repeat(k), + `5${"0".repeat(k - 1)}`, + `4${"9".repeat(k - 1)}`, + `5${"0".repeat(k - 1)}`.replace(/0$/, "1"), + digits(k) + ] + // Up to 18 digits in all (below 9 at 18 fraction digits), so 64 bits hold the texts at scale `k`, rounded up too. + const integer = () => k === 18 ? String(next(9)) : digits(1 + next(k < 18 ? 18 - k : 17)) + const texts = Array.from({ length: 60 }, integer).flatMap((integer) => + fractions.map((fraction) => `${next(2) === 0 ? "-" : ""}${integer}${k === 0 ? "" : "."}${fraction}`) + ) + const expected = texts.map((text) => + BigDecimal.round(BigDecimal.make(BigInt(text.replace(".", "")), k), { scale: 0, mode: "half-even" }).value + ) + for (const column of yield* whole(texts, k)) assert.deepStrictEqual(column, expected, `${k} digits`) + } + })) + }) }) diff --git a/packages/effect/test/dataframe/README.md b/packages/effect/test/dataframe/README.md index 5d545051117..4d527d769af 100644 --- a/packages/effect/test/dataframe/README.md +++ b/packages/effect/test/dataframe/README.md @@ -127,6 +127,23 @@ Refinements, literals, enums and template literals are value checks: every value A table declared by an Arrow schema (DF-C025, DF-C026, DF-Q044) involves no Effect Schema and has no checks. +### Columnar path + +Columns hold timestamps, dates, durations and decimals as integers under their Arrow type. `DateTime`, `Duration` and `BigDecimal` values are built only for rows and arrays of values (`DataFrame.toRows`, `DataFrame.toArray`). The tests are in `Query.test.ts`, under `Query > columnar path`, and in `packages/sql/pglite/test/DataFrameSql.test.ts`. + +| ID | Kind | File | Assertion / exact test title | API source | +| ------- | ---- | ---------------------- | ----------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| DF-N001 | P | `Query.test.ts` | decoding, querying and encoding build no DateTime, Duration or BigDecimal | [DataFrame layouts](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/DataFrame.ts#L6-L9) | +| DF-N002 | P | `Query.test.ts` | rows build one DateTime, Duration or BigDecimal per value; columns keep nanoseconds | [DataFrame.toRows](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/DataFrame.ts#L427) | +| DF-N003 | C | `Query.test.ts` | dropping decimal digits rounds half to even, exactly | [Expr.castDecimal](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Expr.ts#L699), [Expr.round](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Expr.ts#L455) | +| DF-N004 | P | `DataFrameSql.test.ts` | decimal results are read from their text, without BigDecimal values | [Sql contract](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Sql.ts#L1-L45) | + +DF-N001, DF-N002 and DF-N004 count calls of the functions `effect/arrow` and `effect/dataframe` build these values with: `makeUtc` and `makeZonedProto` of `effect/internal/dateTime`, `Duration.nanos`, and `BigDecimal.make`, `BigDecimal.round` and `BigDecimal.fromStringUnsafe`. DF-N002 shows the counters see the values of rows. A value built another way would not be counted, and the tests measure no memory. + +DF-N001 covers one pipeline over timestamps in microseconds and nanoseconds (with and without a time zone), dates, durations, decimals and strings: IPC stream bytes decoded as a table and as a stream of batches, a filter against a timestamp literal, a sort, date and time fields, `truncate`, timestamp and duration arithmetic, a decimal cast to a lower scale, `round` on a decimal, and sums, means and extremes per group, run over a frame and over streamed batches, then exported and encoded. It makes no claim about other operations. Tables declared with a schema (`Query.table`) and `DataFrame.validate` are not part of it. + +DF-N003 compares with `BigDecimal.round` in mode `half-even` over generated values, besides the listed ties. DF-N004 covers the plain notation PostgreSQL writes decimals in (`-12.34`), alone and in lists. A column with a value in another form, with more fraction digits than its scale or beyond 64 bits is still read through `BigDecimal`, which reports the invalid ones. DF-N004 also pins the message of a decimal result beyond 64 bits, a local detail. + ## Oracle provenance and differential assertions **DF-I-POLARS//** (I) resolves exactly to `Query.test.ts > Query > Polars oracle > > : `. Sources are `values`, `frames`, and `batches`; case IDs and descriptions are the existing entries in the pinned [spec.json](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/test/fixtures/dataframe/gold/spec.json) and [extended.json](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/test/fixtures/dataframe/gold/extended.json). This expansion rule identifies each assertion without duplicating the corpus or renaming tests. The three modes exercise inline frames, catalog frames and streamed batches against the same expected columns, types, values and row order. diff --git a/packages/sql/pglite/test/DataFrameSql.test.ts b/packages/sql/pglite/test/DataFrameSql.test.ts index 108f51e5e50..1faa17bce4a 100644 --- a/packages/sql/pglite/test/DataFrameSql.test.ts +++ b/packages/sql/pglite/test/DataFrameSql.test.ts @@ -8,6 +8,7 @@ import { BigDecimal, DateTime, Duration, Effect, Layer, Result, Schema } from "e import { ArrowSchema, ArrowTable } from "effect/arrow" import { Catalog, DataFrame, Expr, Query, Sql } from "effect/dataframe" import { SqlClient } from "effect/sql/SqlClient" +import { vi } from "vitest" const Sale = Schema.Struct({ id: Schema.Number, @@ -797,5 +798,73 @@ describe("effect/dataframe on PostgreSQL (PGlite)", { concurrent: false }, () => ) assert.include(text, `FROM "sales"`) })) + + // DF-N004: local policy for SQL results; pins in effect/test/dataframe/README.md. + it.effect("decimal results are read from their text, without BigDecimal values", () => + Effect.gen(function*() { + const amounts = arrow<{ g: string; d: BigDecimal.BigDecimal | null }>( + [ArrowSchema.field("g", ArrowSchema.utf8), ArrowSchema.field("d", ArrowSchema.decimal(18, 2))], + [ + ["a", "a", "b", "b", "c"], + [decimal("-12.34"), decimal("0"), decimal("9999999999999999.99"), null, decimal("-9999999999999999.99")] + ] + ) + const query = amounts.pipe( + Query.groupBy("g"), + Query.agg((c) => ({ total: Expr.sum(c.d), all: Expr.list(c.d) })) + ) + const { calls, grouped, plain } = yield* Effect.acquireUseRelease( + Effect.sync(() => [ + vi.spyOn(BigDecimal, "make"), + vi.spyOn(BigDecimal, "round"), + vi.spyOn(BigDecimal, "fromStringUnsafe") + ]), + (spies) => + Effect.gen(function*() { + const count = () => spies.reduce((n, spy) => n + spy.mock.calls.length, 0) + const plain = yield* onSql(amounts) + const grouped = yield* onSql(query) + const read = count() + DataFrame.toRows(plain) + DataFrame.toRows(grouped) + return { plain, grouped, calls: [read, count() - read] } + }), + (spies) => Effect.sync(() => spies.forEach((spy) => spy.mockRestore())) + ) + // None to read the results; one per decimal of their rows: 4 in `d`, 3 in `total`, 4 in `all`. + assert.deepStrictEqual(calls, [0, 11]) + const unscaled = (d: unknown) => BigDecimal.isBigDecimal(d) ? [d.value, d.scale] : d + assert.deepStrictEqual(types(plain).map((f) => f.type), [ArrowSchema.utf8, ArrowSchema.decimal(18, 2)]) + assert.deepStrictEqual(DataFrame.toArray(plain, "d").map(unscaled), [ + [-1234n, 2], + [0n, 2], + [999999999999999999n, 2], + null, + [-999999999999999999n, 2] + ]) + assert.deepStrictEqual(types(grouped).map((f) => f.type), [ + ArrowSchema.utf8, + ArrowSchema.decimal(38, 2), + ArrowSchema.list(ArrowSchema.field("item", ArrowSchema.decimal(18, 2))) + ]) + assert.deepStrictEqual( + DataFrame.toRows(grouped).map((r) => [r.g, unscaled(r.total), r.all.map(unscaled)]), + [ + ["a", [-1234n, 2], [[-1234n, 2], [0n, 2]]], + ["b", [999999999999999999n, 2], [[999999999999999999n, 2], null]], + ["c", [-999999999999999999n, 2], [[-999999999999999999n, 2]]] + ] + ) + // Results beyond the 64 bits of a decimal column fail, in lists too. + const beyond = "decimal values beyond 18 digits are not supported (column of 128-bit decimals)" + assert.strictEqual( + yield* rejects(amounts.pipe(Query.select((c) => ({ p: Expr.multiply(c.d, c.d) })))), + beyond + ) + assert.strictEqual( + yield* rejects(amounts.pipe(Query.aggregate((c) => ({ p: Expr.list(Expr.multiply(c.d, c.d)) })))), + beyond + ) + })) }) }) From 72e493750378f58913f4e9ef704f128c65842c51 Mon Sep 17 00:00:00 2001 From: Effect Bot Date: Mon, 16 Nov 2026 08:22:25 +0000 Subject: [PATCH 69/88] Add a benchmark of columnar work against row output --- .../effect/benchmark/dataframe/columnar.ts | 146 ++++++++++++++++++ 1 file changed, 146 insertions(+) create mode 100644 packages/effect/benchmark/dataframe/columnar.ts diff --git a/packages/effect/benchmark/dataframe/columnar.ts b/packages/effect/benchmark/dataframe/columnar.ts new file mode 100644 index 00000000000..ce023371bd5 --- /dev/null +++ b/packages/effect/benchmark/dataframe/columnar.ts @@ -0,0 +1,146 @@ +/** + * effect/dataframe on timestamp, duration and decimal columns, which frames + * hold as integers. Only rows turn them into `DateTime`, `Duration` and + * `BigDecimal` values, so the cases compare work that stays in columns with + * the same input read as rows. + * + * node [--expose-gc] benchmark/dataframe/columnar.ts [--rows=1000000] [--reps=5] [--only=] + * [--out=] [--gc] + * + * The data is generated in memory: a timestamp and a duration in + * microseconds, a decimal(18, 2), a float64 and a string key of 100 values, + * without nulls, encoded once as an Arrow IPC stream. Times are the best of + * `--reps` runs, in milliseconds. + * + * `--gc` (with `node --expose-gc`) collects garbage before every timed run, + * and again after it while the case's result is still referenced. The memory + * columns are the growth between the two, in MiB, in the last run: `heap` is + * `heapUsed`, the JS objects the result keeps alive, and `buffers` is + * `arrayBuffers`, the memory of its typed arrays. Neither counts what a run + * allocated and released. + */ +import { BigDecimal, DateTime, Effect } from "effect" +import { ArrowIpc, ArrowSchema, ArrowTable } from "effect/arrow" +import { DataFrame, Expr, Query } from "effect/dataframe" +import * as Result from "effect/Result" +import * as Fs from "node:fs" + +const arg = (name: string) => process.argv.find((a) => a.startsWith(`--${name}=`))?.slice(name.length + 3) +const rows = Number(arg("rows") ?? 1_000_000) +const reps = Number(arg("reps") ?? 5) +const only = arg("only") +const out = arg("out") +const gc = process.argv.includes("--gc") ? (globalThis as { gc?: () => void }).gc : undefined + +let state = 0x9e3779b9 +const next = (bound: number) => { + state ^= state << 13 + state ^= state >>> 17 + state ^= state << 5 + return Math.floor(((state >>> 0) / 4294967296) * bound) +} +/** 2024-01-01T00:00:00Z, in milliseconds. */ +const start = 1_704_067_200_000 +const bytes = Result.getOrThrow(ArrowIpc.encodeStream(Result.getOrThrow(ArrowTable.fromColumns( + ArrowSchema.make([ + ArrowSchema.field("at", ArrowSchema.timestamp("microsecond")), + ArrowSchema.field("took", ArrowSchema.duration("microsecond")), + ArrowSchema.field("amount", ArrowSchema.decimal(18, 2)), + ArrowSchema.field("v", ArrowSchema.float64), + ArrowSchema.field("key", ArrowSchema.utf8) + ]), + [ + // One row a second, up to a second late. + Array.from({ length: rows }, (_, i) => BigInt(start + i * 1000) * 1000n + BigInt(next(1_000_000))), + Array.from({ length: rows }, () => BigInt(next(3_600_000_000))), + Array.from({ length: rows }, () => BigDecimal.make(BigInt(next(2_000_000_000) - 1_000_000_000), 2)), + Array.from({ length: rows }, () => next(1_000_000) / 1000), + Array.from({ length: rows }, () => `key${next(100)}`) + ] +)))) + +const decode = () => + DataFrame.fromArrowTable(Result.getOrThrow(ArrowIpc.decodeStream(bytes))) as DataFrame.DataFrame +const encode = (df: DataFrame.DataFrame) => Result.getOrThrow(ArrowIpc.encodeStream(DataFrame.toArrowTable(df))) +/** The later half of the rows. */ +const since = DateTime.makeUnsafe(start + (rows / 2) * 1000) +const input = decode() + +const cases: Record { readonly result: unknown; readonly rows: number }> = { + "a decode, filter, group, encode": () => { + const result = Effect.runSync( + Query.from(decode()).pipe( + Query.filter((c) => Expr.gt(c.at, since)), + Query.groupBy("key"), + Query.agg((c) => ({ + amount: Expr.sum(c.amount), + took: Expr.mean(c.took), + last: Expr.max(c.at), + v: Expr.sum(c.v) + })) + ) + ) + return { result: encode(result), rows: result.length } + }, + "b decode, rows": () => { + const result = DataFrame.toRows(decode()) + return { result, rows: result.length } + }, + "c cast decimal(18, 2) to (18, 1)": () => { + const result = Effect.runSync( + Query.from(input).pipe(Query.select((c) => ({ amount: Expr.castDecimal(c.amount, 18, 1) }))) + ) + // Forces the column without converting values to JS. + return { result: DataFrame.toArrowTable(result), rows: result.length } + } +} + +const results: Record = {} +/** References the result of the last run, so that collecting garbage keeps it. */ +const held: { result?: unknown } = {} +/** Runs a case in a call of its own, so that `held` has the only reference to its result afterwards. */ +const time = (run: () => { readonly result: unknown; readonly rows: number }) => { + const start = performance.now() + const { result, rows } = run() + const ms = performance.now() - start + held.result = result + return { ms, rows } +} +const fmt = (n: number | undefined) => (n === undefined ? "—" : n.toFixed(1)).padStart(9) +const mib = (n: number) => n / 1024 / 1024 + +console.log( + `effect/dataframe columnar path, ${rows} rows (${mib(bytes.length).toFixed(1)} MiB of IPC), best of ${reps}` +) +console.log( + `${"case".padEnd(34)} ${"best ms".padStart(9)} ${"first ms".padStart(9)} ${"heap MiB".padStart(9)} ${ + "buffers".padStart(9) + } rows` +) +for (const [name, run] of Object.entries(cases)) { + if (only !== undefined && !name.includes(only)) continue + let best = Infinity + let first = 0 + let count = 0 + let heap: number | undefined + let buffers: number | undefined + for (let r = 0; r < reps; r++) { + held.result = undefined + gc?.() + const before = process.memoryUsage() + const { ms, rows } = time(run) + if (r === 0) first = ms + best = Math.min(best, ms) + count = rows + if (gc !== undefined) { + gc() + const after = process.memoryUsage() + heap = mib(after.heapUsed - before.heapUsed) + buffers = mib(after.arrayBuffers - before.arrayBuffers) + } + } + console.log(`${name.padEnd(34)} ${fmt(best)} ${fmt(first)} ${fmt(heap)} ${fmt(buffers)} ${count}`) + results[name] = { ms: best, first, heapMiB: heap, arrayBuffersMiB: buffers, rows: count } +} + +if (out !== undefined) Fs.writeFileSync(out, JSON.stringify({ rows, reps, results }, null, 2) + "\n") From eb6629905b294b767fbe4e69cee9ede1a46178c5 Mon Sep 17 00:00:00 2001 From: Effect Bot Date: Mon, 16 Nov 2026 08:25:48 +0000 Subject: [PATCH 70/88] Add DataFrame.column to read a column's buffers without building rows --- packages/effect/src/dataframe/DataFrame.ts | 61 +++++++++++++++++++ packages/effect/src/dataframe/Plan.ts | 4 +- .../effect/src/dataframe/internal/frame.ts | 12 ++-- .../effect/test/dataframe/DataFrame.test.ts | 54 +++++++++++++++- packages/effect/test/dataframe/README.md | 4 +- packages/effect/typetest/DataFrame.tst.ts | 13 +++- 6 files changed, 139 insertions(+), 9 deletions(-) diff --git a/packages/effect/src/dataframe/DataFrame.ts b/packages/effect/src/dataframe/DataFrame.ts index 7553fb5fbab..c51141f4e76 100644 --- a/packages/effect/src/dataframe/DataFrame.ts +++ b/packages/effect/src/dataframe/DataFrame.ts @@ -38,6 +38,7 @@ * @stability experimental * @since 4.1.5 */ +import type * as ArrowColumn from "../arrow/ArrowColumn.ts" import * as ArrowData from "../arrow/ArrowData.ts" import * as ArrowIpc from "../arrow/ArrowIpc.ts" import * as ArrowSchema from "../arrow/ArrowSchema.ts" @@ -493,6 +494,66 @@ export const toArray: { (self: DataFrame, name: K): Array } = dual(2, (self: DataFrame, name: string) => F.toArray(frame(self), name)) as any +/** + * One column as an Arrow column, without building rows or values. + * + * **When to use** + * + * Use to read or compute over a column's stored numbers, or to read single + * values, without paying for a row object or a `DateTime`, `Duration` or + * `BigDecimal` per value. + * + * **Details** + * + * The column has one chunk, laid out as {@link toArrowTable} lays it out: + * its type, validity bitmap, buffers and child arrays are those of the + * Arrow table, and share the frame's memory where the layouts agree. + * `ArrowData.values` and `ArrowData.offsets` give its numbers and slot + * boundaries as typed arrays, with timestamps, dates and durations as the + * integers they are stored as, in the unit of the column's type. + * `ArrowColumn.get`, `ArrowColumn.toArray` and iteration build values + * as {@link toArray} does, typed by the frame's rows, one at a time. + * + * **Gotchas** + * + * - The typed array to expect follows from the column's Arrow type + * (`self.schema`), not from the row type: a `number` column can be + * `Int32` or `Float64`. + * - Treat the buffers as read-only: they can be the frame's own. + * - Frames hold dictionary-encoded and chunked input as one plain array per + * column, so the column has no dictionary and a single chunk. + * + * **Example** (Reading timestamps as stored) + * + * ```ts import.meta.vitest + * import { DateTime, Option, Schema } from "effect" + * import { ArrowColumn, ArrowData } from "effect/arrow" + * import { DataFrame } from "effect/dataframe" + * + * const Event = Schema.Struct({ at: Schema.NullOr(Schema.DateTimeUtc) }) + * const events = DataFrame.fromRows(Event, [{ at: DateTime.makeUnsafe(1500) }, { at: null }]) + * + * const at = DataFrame.column(events, "at") + * const micros = Option.getOrThrow(ArrowData.values(at.chunks[0])) + * micros[0] // => 1500000n + * ArrowData.isValid(at.chunks[0], 1) // => false + * ArrowColumn.getUnsafe(at, 0) // => DateTime.makeUnsafe(1500) + * ``` + * + * @see {@link toArray} for the values of a column as an array + * @see {@link toArrowTable} for every column as one record batch + * @stability experimental + * @category conversions + * @since 4.1.5 + */ +export const column: { + (name: K): (self: DataFrame) => ArrowColumn.ArrowColumn> + (self: DataFrame, name: K): ArrowColumn.ArrowColumn> +} = dual( + 2, + (self: DataFrame, name: string) => unsafe(() => F.toArrowColumn(frame(self), name)) +) as any + /** * The frame as an Arrow table with one record batch. * diff --git a/packages/effect/src/dataframe/Plan.ts b/packages/effect/src/dataframe/Plan.ts index 7b8e27b3389..5be593af572 100644 --- a/packages/effect/src/dataframe/Plan.ts +++ b/packages/effect/src/dataframe/Plan.ts @@ -734,7 +734,9 @@ export interface TableChecks { * unit, lists and structs with such children) and every value converts * exactly. `Timestamp` columns read as `DateTime.Utc` values: their * timestamps must have no time zone or a UTC one (dropped), and lie in the - * range `DateTime` represents. `Date` and `Time` columns keep their unit. + * range `DateTime` represents. `Date` columns are in days and `Time` + * columns in microseconds: dates in milliseconds and times in other units + * convert to them. * * Every batch of a streamed table, after this conversion (as given, without * `columns`), must have the column names and types of its first batch. diff --git a/packages/effect/src/dataframe/internal/frame.ts b/packages/effect/src/dataframe/internal/frame.ts index 5bad6c484d0..8c8bd72ed45 100644 --- a/packages/effect/src/dataframe/internal/frame.ts +++ b/packages/effect/src/dataframe/internal/frame.ts @@ -184,12 +184,16 @@ const rows = (self: Frame): Iterable => ArrowTable.rows(toArrowTable(self)) /** @internal */ export const toRows = (self: Frame): Array => ArrowTable.toRows(toArrowTable(self)) -/** @internal */ -export const toArray = (self: Frame, name: string): Array => { - const vector = column(self, name).values() - return ArrowColumn.toArray(ArrowColumn.make(S.field(name, vector.type), [V.toArrowData(vector)])) +/** One column as an Arrow column of one chunk, over the frame's buffers where the layouts agree. @internal */ +export const toArrowColumn = (self: Frame, name: string): ArrowColumn.ArrowColumn => { + const c = column(self, name) + const vector = c.values() + return ArrowColumn.make(S.field(name, vector.type, { nullable: c.nullable }), [V.toArrowData(vector)]) } +/** @internal */ +export const toArray = (self: Frame, name: string): Array => ArrowColumn.toArray(toArrowColumn(self, name)) + // ----------------------------------------------------------------------------- // Row selection // ----------------------------------------------------------------------------- diff --git a/packages/effect/test/dataframe/DataFrame.test.ts b/packages/effect/test/dataframe/DataFrame.test.ts index 7b84da13233..d7276ba21d8 100644 --- a/packages/effect/test/dataframe/DataFrame.test.ts +++ b/packages/effect/test/dataframe/DataFrame.test.ts @@ -1,6 +1,6 @@ import { assert, describe, it } from "@effect/vitest" -import { BigDecimal, DateTime, Duration, Effect, Equal, Result, Schema, Stream } from "effect" -import { ArrowIpc, ArrowSchema, ArrowTable } from "effect/arrow" +import { BigDecimal, DateTime, Duration, Effect, Equal, Option, Result, Schema, Stream } from "effect" +import { ArrowColumn, ArrowData, ArrowIpc, ArrowSchema, ArrowTable } from "effect/arrow" import { Catalog, DataFrame, Expr, Query } from "effect/dataframe" import * as InternalDateTime from "effect/internal/dateTime" import { vi } from "vitest" @@ -176,6 +176,56 @@ describe("DataFrame", () => { } }) + // DF-C028 (API contract): DataFrame.column; pinned source in README.md. + it("column gives a column's buffers and values without building rows", () => { + const schema = ArrowSchema.make([ + ArrowSchema.field("at", ArrowSchema.timestamp("nanosecond", "Europe/Vienna")), + ArrowSchema.field("n", ArrowSchema.int32, { nullable: false }), + ArrowSchema.field("s", ArrowSchema.utf8) + ]) + const input = Result.getOrThrow( + ArrowTable.fromColumns(schema, [[1700000000123456789n, null, -1n], [1, -2, 3], ["a", null, "bc"]]) + ) + const frame = DataFrame.fromArrowTable(input) + const built = [vi.spyOn(InternalDateTime, "makeUtc"), vi.spyOn(InternalDateTime, "makeZonedProto")] + try { + const at = DataFrame.column(frame, "at") + assert.strictEqual(at.length, 3) + assert.deepStrictEqual(at.field.type, ArrowSchema.timestamp("nanosecond", "Europe/Vienna")) + assert.strictEqual(at.chunks.length, 1) + const nanos = Option.getOrThrow(ArrowData.values(at.chunks[0])) + assert.isTrue(nanos instanceof BigInt64Array) + assert.deepStrictEqual([nanos[0], nanos[2]], [1700000000123456789n, -1n]) + assert.deepStrictEqual([0, 1, 2].map((i) => ArrowData.isValid(at.chunks[0], i)), [true, false, true]) + + const n = frame.pipe(DataFrame.column("n")) + assert.strictEqual(n.field.nullable, false) + const ints = Option.getOrThrow(ArrowData.values(n.chunks[0])) + assert.deepStrictEqual(ints, Int32Array.of(1, -2, 3)) + assert.strictEqual(ints.buffer, input.batches[0].columns[1].buffers[0].buffer) + + const s = DataFrame.column(frame, "s").chunks[0] + assert.deepStrictEqual(Array.from(Option.getOrThrow(ArrowData.offsets(s)), Number), [0, 1, 1, 3]) + assert.deepStrictEqual(new TextDecoder().decode(s.buffers[1]), "abc") + assert.strictEqual(built.reduce((sum, spy) => sum + spy.mock.calls.length, 0), 0) + + // A value is built when it is asked for, at the millisecond DateTime holds. + const first = ArrowColumn.getUnsafe(at, 0) as DateTime.Zoned + assert.strictEqual(DateTime.toEpochMillis(first), 1700000000123) + assert.strictEqual(DateTime.zoneToString(first.zone), "Europe/Vienna") + assert.strictEqual(ArrowColumn.getUnsafe(at, 1), null) + assert.strictEqual(built.reduce((sum, spy) => sum + spy.mock.calls.length, 0), 1) + } finally { + built.forEach((spy) => spy.mockRestore()) + } + assert.deepStrictEqual(ArrowColumn.toArray(DataFrame.column(readings, "rain")), [0, null, 2]) + assert.throws( + () => DataFrame.column(frame as DataFrame.DataFrame, "missing"), + DataFrame.DataFrameError, + `unknown column "missing"` + ) + }) + describe("validate", () => { const untyped = DataFrame.fromArrowTable(DataFrame.toArrowTable(readings)) const arrowFrame = (columns: Record]>) => diff --git a/packages/effect/test/dataframe/README.md b/packages/effect/test/dataframe/README.md index 4d527d769af..b8f14351861 100644 --- a/packages/effect/test/dataframe/README.md +++ b/packages/effect/test/dataframe/README.md @@ -46,6 +46,7 @@ The assertion column is the exact leaf test title. Combine it with the file and | DF-C025 | C | takes an Arrow schema, converting the frame to its columns | [DataFrame.validate](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/DataFrame.ts#L396) | | DF-C026 | C | rejects Arrow schemas with types a table can't declare | [DataFrame.validate](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/DataFrame.ts#L396) | | DF-C027 | C | converts dates and times to the declared unit | [DataFrame.validate](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/DataFrame.ts#L396) | +| DF-C028 | C | column gives a column's buffers and values without building rows | [DataFrame layouts](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/DataFrame.ts#L6-L9) | DF-C021 checks a single, unsliced, non-null batch of Int32 and Float64 columns: import/export preserves the backing buffers, byte offsets and byte lengths, as well as values. It does not require typed-array wrapper identity or mutation-based aliasing. The compatible fixed-width layouts are described in Apache Arrow’s [Columnar format](https://github.com/apache/arrow/blob/ee4d09ebef61c663c1efbfa4c18e518a03b798be/docs/source/format/Columnar.rst#fixed-size-primitive-layout) and [Schema.fbs](https://github.com/apache/arrow/blob/ee4d09ebef61c663c1efbfa4c18e518a03b798be/format/Schema.fbs); the no-copy adapter promise is the Effect API contract, not a requirement that Arrow imposes on every adapter. @@ -99,7 +100,7 @@ DF-C021 checks a single, unsliced, non-null batch of Int32 and Float64 columns: | DF-Q044 | C | tables declared by an Arrow schema convert their input to its columns | [Query.table](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Query.ts#L254) | | DF-Q045 | C | fails when a table is declared by a schema that has no columns for it | [Query.table](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Query.ts#L254) | -DF-C022 to DF-C027 and DF-Q043 to DF-Q045 assert behaviour added after the baseline: checks compiled from the schema, rejected transformations, and tables declared by an Arrow schema. Their links locate the symbol at the baseline; the documentation of that symbol in the checked-out source is their contract. DF-C024 observes which value constructors run (`DateTime`, `Duration`, `BigDecimal`) and covers only a frame whose columns already have the declared types. +DF-C022 to DF-C028 and DF-Q043 to DF-Q045 assert behaviour added after the baseline: checks compiled from the schema, rejected transformations, tables declared by an Arrow schema, and `DataFrame.column`, which has no symbol at the baseline and links to the layouts it exposes. Their links locate the symbol at the baseline; the documentation of that symbol in the checked-out source is their contract. DF-C024 observes which value constructors run (`DateTime`, `Duration`, `BigDecimal`) and covers only a frame whose columns already have the declared types. ## Supported Schema subset @@ -187,6 +188,7 @@ The actual SQL suite is **`packages/sql/pglite/test/DataFrameSql.test.ts`**, not - `fromRows, empty, validate and Query.table read missing values back as null` - `validate and Query.table take Arrow types for the schema's fields` - `validate and Query.table take an Arrow schema and give untyped rows` +- `column types its values by the frame's rows` - `fromColumns infers column types` ### `Query` diff --git a/packages/effect/typetest/DataFrame.tst.ts b/packages/effect/typetest/DataFrame.tst.ts index 21b6748c78b..f40a42b7289 100644 --- a/packages/effect/typetest/DataFrame.tst.ts +++ b/packages/effect/typetest/DataFrame.tst.ts @@ -1,6 +1,6 @@ import type { BigDecimal, Result, Stream } from "effect" import { DateTime, Duration, Effect, Schema } from "effect" -import { ArrowSchema } from "effect/arrow" +import { ArrowColumn, ArrowSchema } from "effect/arrow" import { DataFrame, Expr, Query } from "effect/dataframe" import { describe, expect, it } from "tstyche" @@ -71,6 +71,17 @@ describe("DataFrame", () => { }) }) + it("column types its values by the frame's rows", () => { + const frame = DataFrame.empty(Reading) + expect(DataFrame.column(frame, "temp")).type.toBe>() + expect(frame.pipe(DataFrame.column("rain"))).type.toBe>() + expect(ArrowColumn.toArray(DataFrame.column(frame, "rain"))).type.toBe>() + expect(ArrowColumn.getUnsafe(DataFrame.column(frame, "id"), 0)).type.toBe() + expect(DataFrame.column).type.not.toBeCallableWith(frame, "missing") + const untyped = DataFrame.fromArrowTable(DataFrame.toArrowTable(frame)) + expect(DataFrame.column(untyped, "anything")).type.toBe>() + }) + it("fromColumns infers column types", () => { expect(DataFrame.fromColumns({ a: new Float64Array(1), b: ["x", null], c: new BigInt64Array(1) })).type.toBe< DataFrame.DataFrame<{ readonly a: number; readonly b: string | null; readonly c: bigint }> From b8d296127e1a919a4ddd36d3477fb037793d885f Mon Sep 17 00:00:00 2001 From: Effect Bot Date: Mon, 16 Nov 2026 08:27:10 +0000 Subject: [PATCH 71/88] Run a table declared by an Arrow schema on PostgreSQL --- .changeset/dataframe.md | 2 +- packages/sql/pglite/test/DataFrameSql.test.ts | 22 +++++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/.changeset/dataframe.md b/.changeset/dataframe.md index fcdfc935899..da9ea19c56c 100644 --- a/.changeset/dataframe.md +++ b/.changeset/dataframe.md @@ -2,4 +2,4 @@ "effect": patch --- -Add `effect/dataframe`: typed, columnar data frames on `effect/arrow` with lazy, type-checked queries that run in memory or on PostgreSQL. +Add `effect/dataframe`: typed, columnar data frames on `effect/arrow` with lazy, type-checked queries that run in memory or on PostgreSQL. Tables are declared by an Effect Schema, whose checks run and whose transformations are rejected, or by an Arrow schema. diff --git a/packages/sql/pglite/test/DataFrameSql.test.ts b/packages/sql/pglite/test/DataFrameSql.test.ts index 1faa17bce4a..29fe4e95821 100644 --- a/packages/sql/pglite/test/DataFrameSql.test.ts +++ b/packages/sql/pglite/test/DataFrameSql.test.ts @@ -76,6 +76,8 @@ const Setup = Layer.effectDiscard( yield* sql`INSERT INTO exact VALUES (-128, 0.5), (127, 'NaN'), (0, '-Infinity'), (1, ${ 2 ** -149 }), (2, '-0'), (3, ${(2 - 2 ** -23) * 2 ** 127})` + yield* sql`CREATE TABLE arrowed (d DATE, at TIMESTAMP, n INTEGER, s TEXT, extra TEXT)` + yield* sql`INSERT INTO arrowed VALUES ('2024-01-01', '1970-01-01 00:00:01.5', 7, 'a', 'x'), (NULL, NULL, -7, NULL, 'y')` yield* sql`CREATE TYPE mood AS ENUM ('zzz', 'aaa')` yield* sql`CREATE TABLE texts (m mood, c CHAR(3), v VARCHAR(5), sm SMALLINT, r REAL)` yield* sql`INSERT INTO texts VALUES ('zzz', 'ab', 'xy', 20000, 16777216), ('aaa', 'abc', 'x', -20000, 0.5)` @@ -124,6 +126,14 @@ const memory = Catalog.layer({ ArrowSchema.field("i8", ArrowSchema.int32), ArrowSchema.field("f32", ArrowSchema.float64) ], [[-128, 127, 0, 1, 2, 3], [0.5, NaN, -Infinity, 2 ** -149, -0, (2 - 2 ** -23) * 2 ** 127]]), + // Dates in milliseconds and timestamps in seconds convert to the declared days and milliseconds. + arrowed: frame([ + ArrowSchema.field("extra", ArrowSchema.utf8), + ArrowSchema.field("d", ArrowSchema.date("millisecond")), + ArrowSchema.field("at", ArrowSchema.timestamp("microsecond")), + ArrowSchema.field("n", ArrowSchema.int32), + ArrowSchema.field("s", ArrowSchema.utf8) + ], [["x", "y"], [BigInt(19723 * 86_400_000), null], [1_500_000n, null], [7, -7], ["a", null]]), // PostgreSQL converts char(n) values to text without their trailing spaces. texts: frame([ ...["m", "c", "v"].map((name) => ArrowSchema.field(name, ArrowSchema.utf8)), @@ -430,6 +440,18 @@ const cases: ReadonlyArray]> = [ types: { i8: ArrowSchema.int8, f32: ArrowSchema.float32 } }) ], + [ + "tables declared by an Arrow schema", + Query.table( + "arrowed", + ArrowSchema.make([ + ArrowSchema.field("d", ArrowSchema.date("day")), + ArrowSchema.field("at", ArrowSchema.timestamp("millisecond")), + ArrowSchema.field("n", ArrowSchema.int32, { nullable: false }), + ArrowSchema.field("s", ArrowSchema.utf8) + ]) + ) + ], [ "enum, char, varchar, smallint and real columns, as their declared types", Query.table( From 86e0fef050d3234a26cec4b6285dbeae349ea91c Mon Sep 17 00:00:00 2001 From: Effect Bot Date: Mon, 16 Nov 2026 08:57:30 +0000 Subject: [PATCH 72/88] Check the frame fromRows built, and reject temporal and decimal values its columns can't hold exactly --- packages/effect/src/dataframe/DataFrame.ts | 31 +++++--- .../effect/src/dataframe/internal/table.ts | 78 +++++++++++++++++++ .../effect/test/dataframe/DataFrame.test.ts | 53 +++++++++++++ packages/effect/test/dataframe/README.md | 5 +- 4 files changed, 154 insertions(+), 13 deletions(-) diff --git a/packages/effect/src/dataframe/DataFrame.ts b/packages/effect/src/dataframe/DataFrame.ts index c51141f4e76..8c4a891fbcf 100644 --- a/packages/effect/src/dataframe/DataFrame.ts +++ b/packages/effect/src/dataframe/DataFrame.ts @@ -181,12 +181,21 @@ export const isDataFrame = (u: unknown): u is DataFrame => F.isFrame(u) * **Details** * * The rows are decoded values (the schema's `Type`). The schema describes - * them and is not run as a decoder. Its refinements, literals and enums are - * tested on the value of every field that has them (inside structs, arrays - * and union members too), and a struct's own checks - * (`Schema.Struct(...).check(...)`) on every row, so the frame holds only - * rows that {@link validate} accepts. Fields without them are stored without - * further work. + * them and is not run as a decoder. The frame is built and then checked as + * {@link validate} checks one, so it validates against the same schema and + * options. The schema's refinements, literals and enums are tested on the + * values the frame stores, in every field that has them (inside structs, + * arrays and union members too), and a struct's own checks + * (`Schema.Struct(...).check(...)`) on every stored row. Fields without + * them are not read back as values. + * + * A value is stored exactly or the row is rejected. A `DateTime` or + * `Duration` finer than its column's unit fails, and so do a `DateTime` + * beyond the range of valid dates, a `BigDecimal` with more digits than + * its column's scale or precision, and a number an integer column can't + * hold. A + * `Float32` override is the exception: it rounds each number to the + * nearest 32-bit float, and the schema's checks test the rounded number. * * Missing values read back as `null` (see {@link Normalize}). An override * must hold its field's values, as {@link validate} checks columns: a @@ -219,9 +228,9 @@ export const fromRows = ( const { checks, columns } = Schema_.declare(schema, options?.types as any) const input: ReadonlyArray = Array.isArray(rows) ? rows : Array.from(rows) const table = Result.getOrThrowWith(ArrowTable.fromRows(T.toArrowSchema(columns), input), fromDefect) - // The rows are values already: test them as given instead of reading them back from the frame. - if (checks !== undefined) T.runChecks(checks, (name) => input.map((row) => row[name]), () => input) - return wrap(F.fromArrowTable(table)) + T.checkExact(columns, input) + // The frame is checked as validate checks one, so it holds what its declaration accepts. + return wrap(T.conform(F.fromArrowTable(table), columns, checks)) }) type ColumnInput = @@ -403,8 +412,8 @@ export const empty = ( * precision and scale, `DateTimeUtc` and `Duration` take timestamps and * durations in any unit, and structs and arrays need compatible fields and * items. Every value must convert to the declared type exactly: a decimal - * with more digits than the declared scale, or a timestamp finer than the - * declared unit, fails. + * with more digits than the declared scale or precision, or a timestamp + * finer than the declared unit, fails. * * The types are checked and converted on the typed columns, without reading * values. A field the schema constrains further (literals, enums, diff --git a/packages/effect/src/dataframe/internal/table.ts b/packages/effect/src/dataframe/internal/table.ts index 39f1a37f299..7446faa2d14 100644 --- a/packages/effect/src/dataframe/internal/table.ts +++ b/packages/effect/src/dataframe/internal/table.ts @@ -19,6 +19,8 @@ * @internal */ import * as S from "../../arrow/ArrowSchema.ts" +import * as DateTime from "../../DateTime.ts" +import * as Duration from "../../Duration.ts" import type * as Plan from "../Plan.ts" import * as D from "./dtype.ts" import * as F from "./frame.ts" @@ -291,6 +293,17 @@ const checkRange = (path: string, v: V.I64, unit: S.TimeUnit, rows: Uint8Array | } } +/** + * Fails for a decimal with more digits than its column's precision. A + * column holds one all the same, and a conversion to its own type finds none. + */ +const checkPrecision = (path: string, v: V.I64, type: S.Decimal, rows: Uint8Array | undefined): void => { + const limit = BigInt(10) ** BigInt(type.precision) + for (let i = 0; i < v.length; i++) { + if (inRows(rows, i) && V.isValid(v, i) && (v.values[i] >= limit || v.values[i] <= -limit)) inexact(path, type) + } +} + const sameValue = (a: V.Vector, b: V.Vector, i: number): boolean => { switch (a.kind) { case "f64": { @@ -354,6 +367,68 @@ const rescale = (path: string, v: V.I32 | V.I64, target: S.Date | S.Time, rows: return out instanceof Int32Array ? V.i32(target, out, v.valid) : V.i64(target, out, v.valid) } +/** + * A test of the values rows give a column, failing for a `DateTime` or + * `Duration` that is not a whole number of the column's unit: building the + * column rounds it down. `undefined` when the unit holds them all (a + * `DateTime` is in milliseconds, a `Duration` in nanoseconds). + */ +const exact = (path: string, column: Plan.TableColumn): ((value: unknown) => void) | undefined => { + switch (column.type) { + case "Timestamp": + return column.unit !== "second" ? undefined : (value) => { + if (DateTime.isDateTime(value) && DateTime.toEpochMillis(value) % 1000 !== 0) { + inexact(path, arrowType(column, path)) + } + } + case "Duration": { + const [mul, div] = microseconds[column.unit ?? "microsecond"] + const nanos = mul * BigInt(1000) / div + return nanos === BigInt(1) ? undefined : (value) => { + if (Duration.isDuration(value) && Duration.toNanosUnsafe(value) % nanos !== BigInt(0)) { + inexact(path, arrowType(column, path)) + } + } + } + case "List": { + const item = exact(`${path}[]`, column.fields![0]) + return item && ((value) => { + if (value == null) return + // Any other iterable may be spent by now, with nothing left to test. + if (!Array.isArray(value)) return V.fail(`column ${JSON.stringify(path)} needs its lists as arrays`) + value.forEach(item) + }) + } + case "Struct": { + const fields = column.fields!.flatMap((f, k) => { + const test = exact(field(path, f.name), f) + return test === undefined ? [] : [[f.name, k, test] as const] + }) + return fields.length === 0 ? undefined : (value) => { + if (value == null) return + // An array gives a struct's values by position. + for (const [name, k, test] of fields) test(Array.isArray(value) ? value[k] : (value as any)[name]) + } + } + default: + return undefined + } +} + +/** + * Fails unless building the columns from `rows` keeps their `DateTime` + * and `Duration` values exactly: one finer than its column's unit fails, + * as it does when a frame is converted ({@link conform}). + * + * @internal + */ +export const checkExact = (columns: ReadonlyArray, rows: ReadonlyArray): void => { + for (const column of columns) { + const test = exact(column.name, column) + if (test !== undefined) { for (const row of rows) test(row[column.name]) } + } +} + /** A column checked against its declaration and converted to its declared type. */ const check = (path: string, v: V.Vector, expected: Plan.TableColumn, rows: Uint8Array | undefined): V.Vector => { if (expected.type === "Null") { @@ -392,6 +467,9 @@ const check = (path: string, v: V.Vector, expected: Plan.TableColumn, rows: Uint checkRange(path, v, v.type.unit, rows) } const target = arrowType(expected, path) + if (target._tag === "Decimal" && v.kind === "i64" && D.sameType(v.type, target)) { + checkPrecision(path, v, target, rows) + } return (target._tag === "Date" || target._tag === "Time") && !D.sameType(v.type, target) ? rescale(path, v as V.I32 | V.I64, target, rows) : convert(path, v, target, rows) diff --git a/packages/effect/test/dataframe/DataFrame.test.ts b/packages/effect/test/dataframe/DataFrame.test.ts index d7276ba21d8..16424fa1148 100644 --- a/packages/effect/test/dataframe/DataFrame.test.ts +++ b/packages/effect/test/dataframe/DataFrame.test.ts @@ -128,6 +128,57 @@ describe("DataFrame", () => { } }) + // DF-C029 (API contract): DataFrame.fromRows; pinned source in README.md. + it("fromRows returns a frame that validates against the same declaration", () => { + const Row = Schema.Struct({ + n: Schema.Number.check(Schema.isGreaterThan(0)), + at: Schema.NullOr(Schema.DateTimeUtc), + waits: Schema.Array(Schema.Struct({ d: Schema.optional(Schema.Duration) })), + m: Schema.BigDecimal + }) + const types = { n: ArrowSchema.float32, at: ArrowSchema.timestamp("second"), m: ArrowSchema.decimal(4, 2) } + const row = { + n: 0.1, + at: DateTime.makeUnsafe(2000), + waits: [{ d: Duration.nanos(1_000n) }, {}], + m: BigDecimal.make(15n, 1) + } + const df = DataFrame.fromRows(Row, [row, { ...row, at: null }], { types }) + // A Float32 column rounds its numbers, and the frame holds the rounded ones. + assert.strictEqual(DataFrame.toArray(df, "n")[0], Math.fround(0.1)) + for (const built of [df, DataFrame.empty(Row, { types })]) { + const validated = Result.getOrThrow(DataFrame.validate(built, Row, { types })) + assert.deepStrictEqual(validated.schema, built.schema) + assert.deepStrictEqual(DataFrame.toRows(validated), DataFrame.toRows(built)) + } + + const beyond = DateTime.fromEpochSeconds(8_640_000_000_001) + const cases: ReadonlyArray = [ + // The check sees what the column stores: Math.fround(1e-46) is 0. + [{ n: 1e-46 }, `column "n" has values its schema rejects`], + [{ at: DateTime.makeUnsafe(1500) }, `column "at" has values that Timestamp[s] can't hold exactly`], + [{ at: beyond }, `column "at" has timestamps outside the range of DateTime`], + [ + { waits: [{ d: Duration.nanos(1_001n) }] }, + `column "waits[].d" has values that Duration[us] can't hold exactly` + ], + [{ m: BigDecimal.make(1_234n, 3) }, "1.234 has more than 2 decimal places"], + [{ m: BigDecimal.make(1_234n, 1) }, `column "m" has values that Decimal[4, 2] can't hold exactly`] + ] + for (const [fields, message] of cases) { + assert.throws( + () => DataFrame.fromRows(Row, [row, { ...row, ...fields }], { types }), + DataFrame.DataFrameError, + message + ) + } + // A frame that holds such a decimal under the declared type fails too. + const Amount = Schema.Struct({ m: Schema.BigDecimal }) + const held = ArrowTable.fromRows(DataFrame.empty(Amount, { types }).schema, [{ m: BigDecimal.make(1_234n, 1) }]) + const validated = DataFrame.validate(DataFrame.fromArrowTable(Result.getOrThrow(held)), Amount, { types }) + assert.include(Result.isFailure(validated) && validated.failure.message, cases[5][1]) + }) + // DF-C023 (API contract): DataFrame.fromRows; pinned source in README.md. it("fromRows, empty and validate reject schemas that transform their rows", () => { const cases: ReadonlyArray = [ @@ -373,9 +424,11 @@ describe("DataFrame", () => { d: [ArrowSchema.duration("second"), [1n, 2n, 3n]], m: [ArrowSchema.decimal(9, 2), ["1.5", "2.5", "3.5"].map(BigDecimal.fromStringUnsafe)] }) + const row = { t: DateTime.makeUnsafe(1), d: Duration.seconds(1), m: BigDecimal.fromStringUnsafe("1.5") } const built = [vi.spyOn(InternalDateTime, "makeUtc"), vi.spyOn(Duration, "nanos"), vi.spyOn(BigDecimal, "make")] const counts = () => built.map((spy) => spy.mock.calls.length) try { + assert.strictEqual(DataFrame.fromRows(Row, [row, row], { types }).length, 2) const validated = Result.getOrThrow(DataFrame.validate(df, Row, { types })) const table = Query.table("t", Row, { types }).pipe(Effect.provide(Catalog.layer({ t: Stream.make(df, df) }))) assert.strictEqual(Effect.runSync(table).length, 6) diff --git a/packages/effect/test/dataframe/README.md b/packages/effect/test/dataframe/README.md index b8f14351861..72591fbdda6 100644 --- a/packages/effect/test/dataframe/README.md +++ b/packages/effect/test/dataframe/README.md @@ -47,6 +47,7 @@ The assertion column is the exact leaf test title. Combine it with the file and | DF-C026 | C | rejects Arrow schemas with types a table can't declare | [DataFrame.validate](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/DataFrame.ts#L396) | | DF-C027 | C | converts dates and times to the declared unit | [DataFrame.validate](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/DataFrame.ts#L396) | | DF-C028 | C | column gives a column's buffers and values without building rows | [DataFrame layouts](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/DataFrame.ts#L6-L9) | +| DF-C029 | C | fromRows returns a frame that validates against the same declaration | [DataFrame.fromRows](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/DataFrame.ts#L192) | DF-C021 checks a single, unsliced, non-null batch of Int32 and Float64 columns: import/export preserves the backing buffers, byte offsets and byte lengths, as well as values. It does not require typed-array wrapper identity or mutation-based aliasing. The compatible fixed-width layouts are described in Apache Arrow’s [Columnar format](https://github.com/apache/arrow/blob/ee4d09ebef61c663c1efbfa4c18e518a03b798be/docs/source/format/Columnar.rst#fixed-size-primitive-layout) and [Schema.fbs](https://github.com/apache/arrow/blob/ee4d09ebef61c663c1efbfa4c18e518a03b798be/format/Schema.fbs); the no-copy adapter promise is the Effect API contract, not a requirement that Arrow imposes on every adapter. @@ -100,7 +101,7 @@ DF-C021 checks a single, unsliced, non-null batch of Int32 and Float64 columns: | DF-Q044 | C | tables declared by an Arrow schema convert their input to its columns | [Query.table](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Query.ts#L254) | | DF-Q045 | C | fails when a table is declared by a schema that has no columns for it | [Query.table](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Query.ts#L254) | -DF-C022 to DF-C028 and DF-Q043 to DF-Q045 assert behaviour added after the baseline: checks compiled from the schema, rejected transformations, tables declared by an Arrow schema, and `DataFrame.column`, which has no symbol at the baseline and links to the layouts it exposes. Their links locate the symbol at the baseline; the documentation of that symbol in the checked-out source is their contract. DF-C024 observes which value constructors run (`DateTime`, `Duration`, `BigDecimal`) and covers only a frame whose columns already have the declared types. +DF-C022 to DF-C029 and DF-Q043 to DF-Q045 assert behaviour added after the baseline: checks compiled from the schema, rejected transformations, tables declared by an Arrow schema, `DataFrame.column`, which has no symbol at the baseline and links to the layouts it exposes, and the checks `DataFrame.fromRows` runs on the frame it built. Their links locate the symbol at the baseline; the documentation of that symbol in the checked-out source is their contract. DF-C024 observes which value constructors run (`DateTime`, `Duration`, `BigDecimal`) and covers only `DataFrame.fromRows` on rows that hold such values already, and a frame whose columns already have the declared types. ## Supported Schema subset @@ -124,7 +125,7 @@ DF-C022 to DF-C028 and DF-Q043 to DF-Q045 assert behaviour added after the basel Rejected with a `DataFrameError` that names the field: transformations (any schema with an encoding, such as `NumberFromString`, `decodeTo`, decoding defaults and renamed keys; `Schema.toType` gives the decoded side), `DateTimeZoned`, tuples, records, classes, unions of different column types, and recursive schemas. -Refinements, literals, enums and template literals are value checks: every value of such a field is read and tested, inside structs, arrays and union members too. A struct's own checks are row checks: every row is read and tested. Fields without checks are checked by column type only, on the typed columns. `DataFrame.fromRows` tests the rows it is given; `DataFrame.validate` and `Query.table` test what they read. The checks are compiled when the schema is read (`Plan.TableChecks`) and are not part of a serialized plan: a plan decoded from JSON checks column types only. A SQL engine tests value checks on distinct values and fails for a table with row checks. +Refinements, literals, enums and template literals are value checks: every value of such a field is read and tested, inside structs, arrays and union members too. A struct's own checks are row checks: every row is read and tested. Fields without checks are checked by column type only, on the typed columns. `DataFrame.fromRows` builds its frame and then checks it as `DataFrame.validate` does, so all of them test what the frame stores: a `Float32` column's check sees the rounded number. `DataFrame.fromRows` rejects a `DateTime` or `Duration` finer than its column's unit instead of rounding it, and a decimal column holds no more digits than its precision, whichever of them checks it (DF-C029). The checks are compiled when the schema is read (`Plan.TableChecks`) and are not part of a serialized plan: a plan decoded from JSON checks column types only. A SQL engine tests value checks on distinct values and fails for a table with row checks. A table declared by an Arrow schema (DF-C025, DF-C026, DF-Q044) involves no Effect Schema and has no checks. From 0ad0fb972d5e4340634f7c99a1b5562124405aa6 Mon Sep 17 00:00:00 2001 From: Effect Bot Date: Mon, 16 Nov 2026 08:59:26 +0000 Subject: [PATCH 73/88] Keep the checks of a union's members when its null member is suspended --- .../effect/src/dataframe/internal/schema.ts | 6 ++++- .../effect/test/dataframe/DataFrame.test.ts | 25 +++++++++++++++++++ packages/effect/test/dataframe/README.md | 4 ++- packages/sql/pglite/test/DataFrameSql.test.ts | 15 +++++++++++ 4 files changed, 48 insertions(+), 2 deletions(-) diff --git a/packages/effect/src/dataframe/internal/schema.ts b/packages/effect/src/dataframe/internal/schema.ts index 999bdd2383a..7cd32af531e 100644 --- a/packages/effect/src/dataframe/internal/schema.ts +++ b/packages/effect/src/dataframe/internal/schema.ts @@ -177,6 +177,10 @@ const valuesCheck = (ast: AST.AST): ((value: unknown) => boolean) | undefined => return (value) => value == null || ((inner === undefined || inner(value)) && own(value)) } +/** Whether a schema holds only missing values, behind any `suspend`. */ +const isMissing = (ast: AST.AST): boolean => + ast._tag === "Suspend" ? unsuspend(ast, "", isMissing) : ast._tag === "Null" || ast._tag === "Undefined" + const structural = (ast: AST.AST): ((value: unknown) => boolean) | undefined => { switch (ast._tag) { case "Literal": @@ -188,7 +192,7 @@ const structural = (ast: AST.AST): ((value: unknown) => boolean) | undefined => case "Suspend": return unsuspend(ast, "", valuesCheck) case "Union": { - const members = ast.types.filter((t) => t._tag !== "Null" && t._tag !== "Undefined").map(valuesCheck) + const members = ast.types.filter((t) => !isMissing(t)).map(valuesCheck) if (members.some((m) => m === undefined)) return undefined return (value) => value == null || members.some((m) => m!(value)) } diff --git a/packages/effect/test/dataframe/DataFrame.test.ts b/packages/effect/test/dataframe/DataFrame.test.ts index 16424fa1148..245a03fdd77 100644 --- a/packages/effect/test/dataframe/DataFrame.test.ts +++ b/packages/effect/test/dataframe/DataFrame.test.ts @@ -21,6 +21,23 @@ const readings = DataFrame.fromRows(Reading, [ const types = (df: DataFrame.DataFrame) => df.schema.fields.map((f) => `${f.name}:${f.type._tag}:${f.nullable}`) +/** + * What `fromRows`, `validate` and `Query.table` (in memory) say to rows + * held in the schema's columns: `"ok"`, or the message they fail with. + */ +const outcomes = ( + schema: Schema.Struct, + rows: ReadonlyArray> +): ReadonlyArray => { + const stored = DataFrame.fromArrowTable(Result.getOrThrow(ArrowTable.fromRows(DataFrame.empty(schema).schema, rows))) + const table = Query.table("t", schema).pipe(Effect.provide(Catalog.layer({ t: stored }))) + return [ + Result.try({ try: () => DataFrame.fromRows(schema, rows), catch: (e) => e as DataFrame.DataFrameError }), + DataFrame.validate(stored, schema), + Effect.runSync(Effect.result(table)) + ].map((result) => Result.isFailure(result) ? result.failure.message : "ok") +} + describe("DataFrame", () => { describe("constructors", () => { // DF-C001 (API contract): DataFrame.fromRows; pinned source in README.md. @@ -179,6 +196,14 @@ describe("DataFrame", () => { assert.include(Result.isFailure(validated) && validated.failure.message, cases[5][1]) }) + // DF-C030 (API contract): DataFrame.fromRows; pinned source in README.md. + it("a null member of a union keeps the checks of the other members", () => { + const Row = Schema.Struct({ s: Schema.Union([Schema.suspend(() => Schema.Null), Schema.Literal("ok")]) }) + assert.deepStrictEqual(outcomes(Row, [{ s: "ok" }, { s: null }]), ["ok", "ok", "ok"]) + const message = `column "s" has values its schema rejects` + assert.deepStrictEqual(outcomes(Row, [{ s: "ok" }, { s: "bad" }]), [message, message, message]) + }) + // DF-C023 (API contract): DataFrame.fromRows; pinned source in README.md. it("fromRows, empty and validate reject schemas that transform their rows", () => { const cases: ReadonlyArray = [ diff --git a/packages/effect/test/dataframe/README.md b/packages/effect/test/dataframe/README.md index 72591fbdda6..924ba7286a3 100644 --- a/packages/effect/test/dataframe/README.md +++ b/packages/effect/test/dataframe/README.md @@ -48,6 +48,7 @@ The assertion column is the exact leaf test title. Combine it with the file and | DF-C027 | C | converts dates and times to the declared unit | [DataFrame.validate](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/DataFrame.ts#L396) | | DF-C028 | C | column gives a column's buffers and values without building rows | [DataFrame layouts](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/DataFrame.ts#L6-L9) | | DF-C029 | C | fromRows returns a frame that validates against the same declaration | [DataFrame.fromRows](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/DataFrame.ts#L192) | +| DF-C030 | C | a null member of a union keeps the checks of the other members | [DataFrame.fromRows](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/DataFrame.ts#L192) | DF-C021 checks a single, unsliced, non-null batch of Int32 and Float64 columns: import/export preserves the backing buffers, byte offsets and byte lengths, as well as values. It does not require typed-array wrapper identity or mutation-based aliasing. The compatible fixed-width layouts are described in Apache Arrow’s [Columnar format](https://github.com/apache/arrow/blob/ee4d09ebef61c663c1efbfa4c18e518a03b798be/docs/source/format/Columnar.rst#fixed-size-primitive-layout) and [Schema.fbs](https://github.com/apache/arrow/blob/ee4d09ebef61c663c1efbfa4c18e518a03b798be/format/Schema.fbs); the no-copy adapter promise is the Effect API contract, not a requirement that Arrow imposes on every adapter. @@ -101,7 +102,7 @@ DF-C021 checks a single, unsliced, non-null batch of Int32 and Float64 columns: | DF-Q044 | C | tables declared by an Arrow schema convert their input to its columns | [Query.table](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Query.ts#L254) | | DF-Q045 | C | fails when a table is declared by a schema that has no columns for it | [Query.table](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Query.ts#L254) | -DF-C022 to DF-C029 and DF-Q043 to DF-Q045 assert behaviour added after the baseline: checks compiled from the schema, rejected transformations, tables declared by an Arrow schema, `DataFrame.column`, which has no symbol at the baseline and links to the layouts it exposes, and the checks `DataFrame.fromRows` runs on the frame it built. Their links locate the symbol at the baseline; the documentation of that symbol in the checked-out source is their contract. DF-C024 observes which value constructors run (`DateTime`, `Duration`, `BigDecimal`) and covers only `DataFrame.fromRows` on rows that hold such values already, and a frame whose columns already have the declared types. +DF-C022 to DF-C030 and DF-Q043 to DF-Q045 assert behaviour added after the baseline: checks compiled from the schema, rejected transformations, tables declared by an Arrow schema, `DataFrame.column`, which has no symbol at the baseline and links to the layouts it exposes, and the checks `DataFrame.fromRows` runs on the frame it built. Their links locate the symbol at the baseline; the documentation of that symbol in the checked-out source is their contract. DF-C024 observes which value constructors run (`DateTime`, `Duration`, `BigDecimal`) and covers only `DataFrame.fromRows` on rows that hold such values already, and a frame whose columns already have the declared types. ## Supported Schema subset @@ -178,6 +179,7 @@ The actual SQL suite is **`packages/sql/pglite/test/DataFrameSql.test.ts`**, not | DF-S014 | C / I | window functions over partitions | [Sql contract](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Sql.ts#L1-L45) | | DF-S015 | C / I | dates and times | [Sql contract](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Sql.ts#L1-L45) | | DF-S016 | C / I | explain shows the SQL | [Sql contract](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Sql.ts#L1-L45) | +| DF-S017 | C / I | unions and nullable fields are checked as in memory | [Sql contract](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Sql.ts#L1-L45) | ## Compiler assertions diff --git a/packages/sql/pglite/test/DataFrameSql.test.ts b/packages/sql/pglite/test/DataFrameSql.test.ts index 29fe4e95821..856791551e5 100644 --- a/packages/sql/pglite/test/DataFrameSql.test.ts +++ b/packages/sql/pglite/test/DataFrameSql.test.ts @@ -888,5 +888,20 @@ describe("effect/dataframe on PostgreSQL (PGlite)", { concurrent: false }, () => beyond ) })) + + // DF-S017: Sql API contract; pins and oracle policy in effect/test/dataframe/README.md. + it.effect("unions and nullable fields are checked as in memory", () => + Effect.gen(function*() { + const contract = tableOf("contract") + const orNull = (member: S) => Schema.Union([Schema.suspend(() => Schema.Null), member]) + assert.deepStrictEqual( + yield* rowsOnSql(contract(Schema.Struct({ s: orNull(Schema.Literals(["bad", "good"])) }))), + [{ s: "bad" }, { s: "good" }] + ) + assert.strictEqual( + yield* rejects(contract(Schema.Struct({ s: orNull(Schema.Literal("good")) }))), + `column "s" has values its schema rejects` + ) + })) }) }) From ac3ae0909989377544334947e16073742c1863e9 Mon Sep 17 00:00:00 2001 From: Effect Bot Date: Mon, 16 Nov 2026 09:02:10 +0000 Subject: [PATCH 74/88] Hand checks the stored values as the schema writes them, and reject schemas a frame can't do that for --- packages/effect/src/dataframe/DataFrame.ts | 21 ++- .../effect/src/dataframe/internal/schema.ts | 142 +++++++++++++++++- .../effect/test/dataframe/DataFrame.test.ts | 46 ++++++ packages/effect/test/dataframe/README.md | 12 +- 4 files changed, 212 insertions(+), 9 deletions(-) diff --git a/packages/effect/src/dataframe/DataFrame.ts b/packages/effect/src/dataframe/DataFrame.ts index 8c4a891fbcf..1a40eae096a 100644 --- a/packages/effect/src/dataframe/DataFrame.ts +++ b/packages/effect/src/dataframe/DataFrame.ts @@ -189,6 +189,12 @@ export const isDataFrame = (u: unknown): u is DataFrame => F.isFrame(u) * (`Schema.Struct(...).check(...)`) on every stored row. Fields without * them are not read back as values. * + * A check is handed what the schema describes. The frame holds `null` for + * every missing value, and a check sees it as the schema writes it: an + * absent key for an optional field, `undefined` for `Schema.UndefinedOr` + * and `null` for `Schema.NullOr`. The rows `{}` and `{ a: undefined }` + * are one row to a frame, so a check sees an absent key for both. + * * A value is stored exactly or the row is rejected. A `DateTime` or * `Duration` finer than its column's unit fails, and so do a `DateTime` * beyond the range of valid dates, a `BigDecimal` with more digits than @@ -214,6 +220,12 @@ export const isDataFrame = (u: unknown): u is DataFrame => F.isFrame(u) * `Schema.toType(schema)` to describe the decoded rows of such a schema. * - Tuples, records, classes, `DateTimeZoned`, unions of different column * types and recursive schemas have no column type. + * - A schema is rejected when a check would see a missing value that the + * schema writes in several ways, as the frame holds `null` for each of + * them: a struct with checks and a + * `Schema.optional(Schema.NullOr(...))` field, or a check on + * `Schema.UndefinedOr(Schema.NullOr(...))`. Such a field is fine where + * no check is handed its value. * * @stability experimental * @category constructors @@ -386,7 +398,8 @@ export const decodeArrowStream = ( * **Gotchas** * * Throws a {@link DataFrameError} for the schemas {@link fromRows} rejects: - * one that transforms its rows, or has fields without a column type. + * one that transforms its rows, has fields without a column type, or has a + * check that would see a missing value the schema writes in several ways. * * @stability experimental * @category constructors @@ -420,8 +433,10 @@ export const empty = ( * refinements, including inside structs, arrays and union members) has its * values read and tested one by one, and a struct with checks of its own * (`Schema.Struct(...).check(...)`) has every row read and tested. The - * schema is not run as a decoder: one that transforms its rows fails, as in - * {@link fromRows}. + * checks see missing values as the schema writes them (an absent key, + * `undefined` or `null`), and a schema that writes one in several ways + * under a check fails, as in {@link fromRows}. The schema is not run as a + * decoder: one that transforms its rows fails, as in {@link fromRows}. * * An Arrow schema declares its fields as they are: type, nullability, * nested fields, decimal precision and scale, and time unit. The result has diff --git a/packages/effect/src/dataframe/internal/schema.ts b/packages/effect/src/dataframe/internal/schema.ts index 7cd32af531e..d3598ed7913 100644 --- a/packages/effect/src/dataframe/internal/schema.ts +++ b/packages/effect/src/dataframe/internal/schema.ts @@ -157,6 +157,124 @@ const rejectEncodings = (ast: AST.AST, path: string): void => { } } +// ----------------------------------------------------------------------------- +// Missing values +// ----------------------------------------------------------------------------- + +type Missing = "null" | "undefined" | "absent" + +/** + * How a schema writes the values a frame stores. A frame holds `null` for + * every missing value, and a schema writes one as `null`, as `undefined` + * or as an absent key: `missing` is the one it writes where values can be + * missing, for a list's items and a struct's fields too. + */ +interface Shape { + readonly missing?: Missing | undefined + readonly item?: Shape | undefined + readonly fields?: ReadonlyArray | undefined +} + +const written: { readonly [M in Missing]: string } = { null: "null", undefined: "undefined", absent: "an absent key" } + +/** + * The shape of the values of a schema, the type of an `optional` key or + * not. `seen` says that a check around them is handed these values (one of + * a struct, an array or a union they are in). + * + * A check must be handed what the schema describes, so a missing value it + * sees needs one way of being written. An optional field's is an absent + * key, whether or not its type has `undefined` too. It fails when the + * schema writes it as `null` and another way. + */ +const shapeOf = (ast: AST.AST, path: string, seen: boolean, optional = false): Shape => { + const ways = new Set(optional ? ["absent"] : []) + const members: Array = [] + // Whether a check inside this field is handed a missing value. + let own = false + const visit = (ast: AST.AST, checked: boolean): void => { + checked ||= ast.checks !== undefined + switch (ast._tag) { + case "Null": + case "Undefined": + ways.add(ast._tag === "Null" ? "null" : "undefined") + own ||= checked + return + case "Suspend": + return unsuspend(ast, "", (inner) => visit(inner, checked)) + case "Union": + // Matching exactly one member tells the members apart, as a check does. + return ast.types.forEach((member) => visit(member, checked || ast.options?.mode === "oneOf")) + case "Arrays": + members.push({ item: shapeOf(ast.rest[0], `${path}[]`, seen || checked) }) + return + case "Objects": + members.push({ + fields: ast.propertySignatures.map((ps) => { + const name = String(ps.name) + return [name, shapeOf(ps.type, T.field(path, name), seen || checked, AST.isOptional(ps.type))] as const + }) + }) + } + } + visit(ast, false) + if (ways.size > 1 && ways.has("null") && (own || seen)) { + V.fail( + `field ${JSON.stringify(path)}: a check sees a missing value that the schema writes in several ways (${ + [...ways].map((way) => written[way]).join(", ") + }), and a frame holds null for each of them` + ) + } + if (new Set(members.map((member) => JSON.stringify(member))).size > 1) { + V.fail( + `field ${JSON.stringify(path)}: the members of a union write missing values in different ways, ` + + "and a frame holds null for each of them" + ) + } + const missing = ways.has("null") + ? "null" + : ways.has("absent") + ? "absent" + : ways.has("undefined") + ? "undefined" + : undefined + return { missing, ...members[0] } +} + +/** + * The values a frame stores for `shape`, as the schema writes them, or + * `undefined` when it stores them that way. + */ +const restorer = (shape: Shape): ((value: unknown) => unknown) | undefined => { + const present = shape.item !== undefined + ? items(shape.item) + : shape.fields !== undefined + ? struct(shape.fields) + : undefined + const missing = shape.missing === "undefined" ? undefined : null + if (present === undefined && missing === null) return undefined + return (value) => value === null ? missing : present === undefined ? value : present(value) +} + +const items = (shape: Shape): ((value: unknown) => unknown) | undefined => { + const item = restorer(shape) + return item && ((value) => (value as ReadonlyArray).map(item)) +} + +const struct = (shapes: NonNullable): ((value: unknown) => unknown) | undefined => { + const fields = shapes.map(([name, shape]) => [name, shape.missing === "absent", restorer(shape)] as const) + if (fields.every(([, absent, restore]) => !absent && restore === undefined)) return undefined + return (value) => { + const out: Record = {} + for (const [name, absent, restore] of fields) { + const v = (value as Record)[name] + if (v === null && absent) continue + out[name] = restore === undefined ? v : restore(v) + } + return out + } +} + // ----------------------------------------------------------------------------- // Checks // ----------------------------------------------------------------------------- @@ -218,14 +336,28 @@ const structural = (ast: AST.AST): ((value: unknown) => boolean) | undefined => * The tests of a struct schema: one for each field whose values it * constrains beyond their column type, and one of whole rows when the struct * has checks of its own. `undefined` when the column types say it all. + * + * The tests take values and rows as a frame stores them, and hand them to + * the schema's checks as the schema writes them (see {@link shapeOf}). */ -const checksOf = (struct: AST.Objects): Plan.TableChecks | undefined => { - const columns = struct.propertySignatures.flatMap((ps) => { +const checksOf = (ast: AST.Objects): Plan.TableChecks | undefined => { + const checks = ast.checks + const shapes: Array = [] + const columns = ast.propertySignatures.flatMap((ps) => { + const name = String(ps.name) const test = valuesCheck(ps.type) - return test === undefined ? [] : [[String(ps.name), test] as const] + if (test === undefined && checks === undefined) return [] + const shape = shapeOf(ps.type, name, checks !== undefined, AST.isOptional(ps.type)) + shapes.push([name, shape]) + const restore = restorer(shape) + return test === undefined + ? [] + : [[name, restore === undefined ? test : (value: unknown) => test(restore(value))] as const] }) - const checks = struct.checks - const rows = checks && ((row: unknown) => Result.isSuccess(AST.runChecks(checks, row))) + // With checks of rows, every field has its shape. + const restore = checks && struct(shapes) + const rows = checks && + ((row: unknown) => Result.isSuccess(AST.runChecks(checks, restore === undefined ? row : restore(row)))) return columns.length === 0 && rows === undefined ? undefined : { columns, rows } } diff --git a/packages/effect/test/dataframe/DataFrame.test.ts b/packages/effect/test/dataframe/DataFrame.test.ts index 245a03fdd77..b06560a6a11 100644 --- a/packages/effect/test/dataframe/DataFrame.test.ts +++ b/packages/effect/test/dataframe/DataFrame.test.ts @@ -204,6 +204,52 @@ describe("DataFrame", () => { assert.deepStrictEqual(outcomes(Row, [{ s: "ok" }, { s: "bad" }]), [message, message, message]) }) + // DF-C031 (API contract): DataFrame.fromRows; pinned source in README.md. + it("checks see stored values as the schema writes them, missing ones included", () => { + const seen: Array = [] + const record = Schema.makeFilter((value: unknown) => { + seen.push(value) + return true + }) + const Item = Schema.Struct({ unit: Schema.optional(Schema.String), qty: Schema.UndefinedOr(Schema.Number) }) + const Row = Schema.Struct({ + note: Schema.optional(Schema.String), + rain: Schema.NullOr(Schema.Number), + items: Schema.Array(Item).check(record) + }).check(record) + const rows = [{ rain: null, items: [{ qty: undefined }] }, { + note: undefined, + rain: 1, + items: [{ unit: "kg", qty: 2 }] + }] + assert.deepStrictEqual(outcomes(Row, rows), ["ok", "ok", "ok"]) + // What each of the three hands to the checks: the items of both rows, then both rows. + const items = [[{ qty: undefined }], [{ unit: "kg", qty: 2 }]] + const checked = [...items, { rain: null, items: items[0] }, { rain: 1, items: items[1] }] + assert.deepStrictEqual(seen, [...checked, ...checked, ...checked]) + + // A frame can't tell null from undefined or an absent key, so a check that could is rejected. + const Either = Schema.optional(Schema.NullOr(Schema.String)) + const rejected: ReadonlyArray, string]> = [ + [Schema.Struct({ a: Either }).check(record), `field "a"`], + [ + Schema.Struct({ p: Schema.Struct({ xs: Schema.Array(Schema.Struct({ a: Either })) }).check(record) }), + `field "p.xs[].a"` + ], + [Schema.Struct({ a: Schema.Union([Schema.Null, Schema.Undefined, Schema.String]).check(record) }), `field "a"`] + ] + for (const [schema, field] of rejected) { + const message = `${field}: a check sees a missing value that the schema writes in several ways` + assert.throws(() => DataFrame.fromRows(schema, []), DataFrame.DataFrameError, message) + assert.throws(() => Query.table("t", schema), DataFrame.DataFrameError, message) + const validated = DataFrame.validate(readings, schema) + assert.include(Result.isFailure(validated) && validated.failure.message, message) + } + // No check sees it here: the literal is tested on its own column. + const Unseen = Schema.Struct({ kind: Schema.Literal("x"), a: Either }) + assert.deepStrictEqual(outcomes(Unseen, [{ kind: "x" }, { kind: "x", a: null }]), ["ok", "ok", "ok"]) + }) + // DF-C023 (API contract): DataFrame.fromRows; pinned source in README.md. it("fromRows, empty and validate reject schemas that transform their rows", () => { const cases: ReadonlyArray = [ diff --git a/packages/effect/test/dataframe/README.md b/packages/effect/test/dataframe/README.md index 924ba7286a3..08f53fc2ae1 100644 --- a/packages/effect/test/dataframe/README.md +++ b/packages/effect/test/dataframe/README.md @@ -49,6 +49,7 @@ The assertion column is the exact leaf test title. Combine it with the file and | DF-C028 | C | column gives a column's buffers and values without building rows | [DataFrame layouts](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/DataFrame.ts#L6-L9) | | DF-C029 | C | fromRows returns a frame that validates against the same declaration | [DataFrame.fromRows](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/DataFrame.ts#L192) | | DF-C030 | C | a null member of a union keeps the checks of the other members | [DataFrame.fromRows](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/DataFrame.ts#L192) | +| DF-C031 | C | checks see stored values as the schema writes them, missing ones included | [DataFrame.fromRows](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/DataFrame.ts#L192) | DF-C021 checks a single, unsliced, non-null batch of Int32 and Float64 columns: import/export preserves the backing buffers, byte offsets and byte lengths, as well as values. It does not require typed-array wrapper identity or mutation-based aliasing. The compatible fixed-width layouts are described in Apache Arrow’s [Columnar format](https://github.com/apache/arrow/blob/ee4d09ebef61c663c1efbfa4c18e518a03b798be/docs/source/format/Columnar.rst#fixed-size-primitive-layout) and [Schema.fbs](https://github.com/apache/arrow/blob/ee4d09ebef61c663c1efbfa4c18e518a03b798be/format/Schema.fbs); the no-copy adapter promise is the Effect API contract, not a requirement that Arrow imposes on every adapter. @@ -102,7 +103,7 @@ DF-C021 checks a single, unsliced, non-null batch of Int32 and Float64 columns: | DF-Q044 | C | tables declared by an Arrow schema convert their input to its columns | [Query.table](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Query.ts#L254) | | DF-Q045 | C | fails when a table is declared by a schema that has no columns for it | [Query.table](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Query.ts#L254) | -DF-C022 to DF-C030 and DF-Q043 to DF-Q045 assert behaviour added after the baseline: checks compiled from the schema, rejected transformations, tables declared by an Arrow schema, `DataFrame.column`, which has no symbol at the baseline and links to the layouts it exposes, and the checks `DataFrame.fromRows` runs on the frame it built. Their links locate the symbol at the baseline; the documentation of that symbol in the checked-out source is their contract. DF-C024 observes which value constructors run (`DateTime`, `Duration`, `BigDecimal`) and covers only `DataFrame.fromRows` on rows that hold such values already, and a frame whose columns already have the declared types. +DF-C022 to DF-C031 and DF-Q043 to DF-Q045 assert behaviour added after the baseline: checks compiled from the schema, rejected transformations, tables declared by an Arrow schema, `DataFrame.column`, which has no symbol at the baseline and links to the layouts it exposes, and the checks `DataFrame.fromRows` runs on the frame it built. Their links locate the symbol at the baseline; the documentation of that symbol in the checked-out source is their contract. DF-C024 observes which value constructors run (`DateTime`, `Duration`, `BigDecimal`) and covers only `DataFrame.fromRows` on rows that hold such values already, and a frame whose columns already have the declared types. ## Supported Schema subset @@ -128,6 +129,15 @@ Rejected with a `DataFrameError` that names the field: transformations (any sche Refinements, literals, enums and template literals are value checks: every value of such a field is read and tested, inside structs, arrays and union members too. A struct's own checks are row checks: every row is read and tested. Fields without checks are checked by column type only, on the typed columns. `DataFrame.fromRows` builds its frame and then checks it as `DataFrame.validate` does, so all of them test what the frame stores: a `Float32` column's check sees the rounded number. `DataFrame.fromRows` rejects a `DateTime` or `Duration` finer than its column's unit instead of rounding it, and a decimal column holds no more digits than its precision, whichever of them checks it (DF-C029). The checks are compiled when the schema is read (`Plan.TableChecks`) and are not part of a serialized plan: a plan decoded from JSON checks column types only. A SQL engine tests value checks on distinct values and fails for a table with row checks. +Checks are handed stored values as the schema writes them (DF-C031), in `DataFrame.fromRows`, `DataFrame.empty`, `DataFrame.validate` and `Query.table` alike. A frame holds `null` for every missing value, and a check sees an absent key for an optional field, `undefined` for `UndefinedOr` and `null` for `NullOr`. The rows `{}` and `{ a: undefined }` are one stored row, and a check sees an absent key for both. + +A declaration is rejected with a `DataFrameError` that names the field when a check would see a missing value the schema writes in several ways, as the frame holds `null` for each of them: + +- `null` next to `undefined` or an optional key (`optional(NullOr(String))`, `UndefinedOr(NullOr(String))`), under any check that is handed the value: a check of the field itself, or of a struct, array or union around it, row checks included. +- a union whose struct or array members write the same missing value in different ways, when its values are checked. + +Such fields are accepted where no check is handed their value. + A table declared by an Arrow schema (DF-C025, DF-C026, DF-Q044) involves no Effect Schema and has no checks. ### Columnar path From e1559942d64c24b9754f4130cf7c7d014804d684 Mon Sep 17 00:00:00 2001 From: Effect Bot Date: Mon, 16 Nov 2026 09:07:37 +0000 Subject: [PATCH 75/88] Test a checked field with the schema's own guard, so null and oneOf unions are checked as Schema.is checks them --- packages/effect/src/dataframe/DataFrame.ts | 5 ++ packages/effect/src/dataframe/Plan.ts | 10 +-- .../effect/src/dataframe/internal/schema.ts | 77 +++++++------------ .../effect/test/dataframe/DataFrame.test.ts | 58 ++++++++++++++ packages/effect/test/dataframe/README.md | 5 +- packages/sql/pglite/test/DataFrameSql.test.ts | 16 +++- 6 files changed, 110 insertions(+), 61 deletions(-) diff --git a/packages/effect/src/dataframe/DataFrame.ts b/packages/effect/src/dataframe/DataFrame.ts index 1a40eae096a..5337d4f744b 100644 --- a/packages/effect/src/dataframe/DataFrame.ts +++ b/packages/effect/src/dataframe/DataFrame.ts @@ -189,6 +189,11 @@ export const isDataFrame = (u: unknown): u is DataFrame => F.isFrame(u) * (`Schema.Struct(...).check(...)`) on every stored row. Fields without * them are not read back as values. * + * A field that has them accepts the stored values that `Schema.is` accepts + * for it: a check on a nullable field runs on its missing values too, a + * check on a union member runs on that member's values, and a union in + * `oneOf` mode takes a value of exactly one member. + * * A check is handed what the schema describes. The frame holds `null` for * every missing value, and a check sees it as the schema writes it: an * absent key for an optional field, `undefined` for `Schema.UndefinedOr` diff --git a/packages/effect/src/dataframe/Plan.ts b/packages/effect/src/dataframe/Plan.ts index 5be593af572..95cf0e7b34c 100644 --- a/packages/effect/src/dataframe/Plan.ts +++ b/packages/effect/src/dataframe/Plan.ts @@ -698,11 +698,11 @@ export interface TableColumn { * * `columns` pairs the name of a declared column with a test of one value * of it. Every value of that column is read (as `DataFrame.toArray` gives - * it) and tested; a test must accept `null`, as nullability is part of the - * column's declaration. `rows` tests one row object, as `DataFrame.toRows` - * gives it, and is `undefined` when there is nothing to test across - * columns. A table fails with a `DataFrameError` when a test returns - * `false`. + * it) and tested, `null` included: a nullable column's test decides + * whether its missing values pass, as a schema's check on a nullable field + * does. `rows` tests one row object, as `DataFrame.toRows` gives it, and + * is `undefined` when there is nothing to test across columns. A table + * fails with a `DataFrameError` when a test returns `false`. * * **Gotchas** * diff --git a/packages/effect/src/dataframe/internal/schema.ts b/packages/effect/src/dataframe/internal/schema.ts index d3598ed7913..3b6a7f4109b 100644 --- a/packages/effect/src/dataframe/internal/schema.ts +++ b/packages/effect/src/dataframe/internal/schema.ts @@ -15,6 +15,7 @@ * @internal */ import type * as S from "../../arrow/ArrowSchema.ts" +import { identity } from "../../Function.ts" import * as Result from "../../Result.ts" import * as AST from "../../SchemaAST.ts" import * as Parser from "../../SchemaParser.ts" @@ -280,55 +281,27 @@ const struct = (shapes: NonNullable): ((value: unknown) => unkn // ----------------------------------------------------------------------------- /** - * A test for values of a schema that it constrains beyond their column type - * (literals, enums, template literals and checks), anywhere inside it, or - * `undefined` when the column type says it all. Missing values (`null`, and - * `undefined` in rows that are not in a frame yet) pass: nullability is - * checked with the column. Union members are tried in turn. + * Whether a schema constrains its values beyond their column type and + * nullability, anywhere inside it: by a check, a literal, an enum, a + * template literal, or a union that takes the values of exactly one member. */ -const valuesCheck = (ast: AST.AST): ((value: unknown) => boolean) | undefined => { - const own = ast.checks === undefined - ? undefined - : (value: unknown) => Result.isSuccess(AST.runChecks(ast.checks!, value)) - const inner = structural(ast) - if (own === undefined) return inner - return (value) => value == null || ((inner === undefined || inner(value)) && own(value)) -} - -/** Whether a schema holds only missing values, behind any `suspend`. */ -const isMissing = (ast: AST.AST): boolean => - ast._tag === "Suspend" ? unsuspend(ast, "", isMissing) : ast._tag === "Null" || ast._tag === "Undefined" - -const structural = (ast: AST.AST): ((value: unknown) => boolean) | undefined => { +const constrains = (ast: AST.AST): boolean => { + if (ast.checks !== undefined) return true switch (ast._tag) { case "Literal": case "Enum": - case "TemplateLiteral": { - const is = Parser._is(AST.replaceChecks(ast, undefined)) - return (value) => value == null || is(value) - } + case "TemplateLiteral": + return true case "Suspend": - return unsuspend(ast, "", valuesCheck) - case "Union": { - const members = ast.types.filter((t) => !isMissing(t)).map(valuesCheck) - if (members.some((m) => m === undefined)) return undefined - return (value) => value == null || members.some((m) => m!(value)) - } - case "Arrays": { - const item = valuesCheck(ast.rest[0]) - return item && ((value) => value == null || (value as ReadonlyArray).every(item)) - } - case "Objects": { - const fields = ast.propertySignatures.flatMap((ps) => { - const f = valuesCheck(ps.type) - return f === undefined ? [] : [[ps.name, f] as const] - }) - return fields.length === 0 - ? undefined - : (value) => value == null || fields.every(([name, f]) => f((value as any)[name])) - } + return unsuspend(ast, "", constrains) + case "Union": + return ast.options?.mode === "oneOf" || ast.types.some(constrains) + case "Arrays": + return constrains(ast.rest[0]) + case "Objects": + return ast.propertySignatures.some((ps) => constrains(ps.type)) default: - return undefined + return false } } @@ -338,21 +311,25 @@ const structural = (ast: AST.AST): ((value: unknown) => boolean) | undefined => * has checks of its own. `undefined` when the column types say it all. * * The tests take values and rows as a frame stores them, and hand them to - * the schema's checks as the schema writes them (see {@link shapeOf}). + * the schema as it writes them (see {@link shapeOf}). A field's test is the + * schema's own: a value passes when the field's type has it, so checks run + * on every value that reaches them (`null` too) and unions match as they + * do in Schema. An absent key has no value to test. */ const checksOf = (ast: AST.Objects): Plan.TableChecks | undefined => { const checks = ast.checks const shapes: Array = [] const columns = ast.propertySignatures.flatMap((ps) => { const name = String(ps.name) - const test = valuesCheck(ps.type) - if (test === undefined && checks === undefined) return [] + const constrained = constrains(ps.type) + if (!constrained && checks === undefined) return [] const shape = shapeOf(ps.type, name, checks !== undefined, AST.isOptional(ps.type)) shapes.push([name, shape]) - const restore = restorer(shape) - return test === undefined - ? [] - : [[name, restore === undefined ? test : (value: unknown) => test(restore(value))] as const] + if (!constrained) return [] + const is = Parser._is(ps.type) + const restore = restorer(shape) ?? identity + const absent = shape.missing === "absent" + return [[name, (value: unknown) => (absent && value === null) || is(restore(value))] as const] }) // With checks of rows, every field has its shape. const restore = checks && struct(shapes) diff --git a/packages/effect/test/dataframe/DataFrame.test.ts b/packages/effect/test/dataframe/DataFrame.test.ts index b06560a6a11..e1490378f2d 100644 --- a/packages/effect/test/dataframe/DataFrame.test.ts +++ b/packages/effect/test/dataframe/DataFrame.test.ts @@ -250,6 +250,64 @@ describe("DataFrame", () => { assert.deepStrictEqual(outcomes(Unseen, [{ kind: "x" }, { kind: "x", a: null }]), ["ok", "ok", "ok"]) }) + // DF-C032 (API contract): DataFrame.fromRows; pinned source in README.md. + it("accepts the stored values that Schema.is accepts", () => { + const rejects = `column "v" has values its schema rejects` + const NotNull = Schema.Struct({ v: Schema.NullOr(Schema.String).check(Schema.makeFilter((v) => v !== null)) }) + assert.deepStrictEqual(outcomes(NotNull, [{ v: "a" }, { v: null }]), [rejects, rejects, rejects]) + const OneOf = Schema.Struct({ v: Schema.Union([Schema.String, Schema.Literal("ok")], { mode: "oneOf" }) }) + assert.deepStrictEqual(outcomes(OneOf, [{ v: "no" }, { v: "ok" }]), [rejects, rejects, rejects]) + + const Tagged = Schema.Union([ + Schema.Struct({ kind: Schema.Literal("a"), n: Schema.NullOr(Schema.Int) }), + Schema.Struct({ kind: Schema.Literals(["a", "b"]), n: Schema.NullOr(Schema.Number) }) + ], { mode: "oneOf" }) + const cases: ReadonlyArray, ReadonlyArray>]> = [ + [NotNull, [{ v: "a" }, { v: null }]], + [OneOf, [{ v: "no" }, { v: "ok" }]], + [Schema.Struct({ v: Schema.Array(Tagged) }), [ + { v: [{ kind: "b", n: 1.5 }, { kind: "a", n: 1.5 }] }, + { v: [{ kind: "a", n: 1 }] }, + { v: [{ kind: "a", n: null }] }, + { v: [{ kind: "c", n: 1 }] } + ]], + [Schema.Struct({ v: Schema.optional(Schema.String.check(Schema.isMinLength(2))) }), [ + {}, + { v: undefined }, + { v: "ab" }, + { v: "a" } + ]], + [Schema.Struct({ v: Schema.UndefinedOr(Schema.Number).check(Schema.makeFilter((v) => v !== 0)) }), [ + { v: undefined }, + { v: 1 }, + { v: 0 } + ]], + [Schema.Struct({ v: Schema.Union([Schema.Null.check(Schema.makeFilter(() => false)), Schema.String]) }), [ + { v: "a" }, + { v: null } + ]], + [ + Schema.Struct({ v: Schema.optionalKey(Schema.Number), w: Schema.NullOr(Schema.Number) }).check( + Schema.makeFilter((row) => ("v" in row) === (row.w === null)) + ), + [{ v: 1, w: null }, { w: 1 }, { w: null }, { v: 1, w: 1 }] + ] + ] + const verdicts = new Set() + for (const [schema, rows] of cases) { + for (const row of rows) { + const accepted = Schema.is(schema)(row) + verdicts.add(accepted) + assert.deepStrictEqual( + outcomes(schema, [row]).map((outcome) => outcome === "ok"), + [accepted, accepted, accepted], + JSON.stringify(row) + ) + } + } + assert.strictEqual(verdicts.size, 2) + }) + // DF-C023 (API contract): DataFrame.fromRows; pinned source in README.md. it("fromRows, empty and validate reject schemas that transform their rows", () => { const cases: ReadonlyArray = [ diff --git a/packages/effect/test/dataframe/README.md b/packages/effect/test/dataframe/README.md index 08f53fc2ae1..d9b42a16c64 100644 --- a/packages/effect/test/dataframe/README.md +++ b/packages/effect/test/dataframe/README.md @@ -50,6 +50,7 @@ The assertion column is the exact leaf test title. Combine it with the file and | DF-C029 | C | fromRows returns a frame that validates against the same declaration | [DataFrame.fromRows](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/DataFrame.ts#L192) | | DF-C030 | C | a null member of a union keeps the checks of the other members | [DataFrame.fromRows](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/DataFrame.ts#L192) | | DF-C031 | C | checks see stored values as the schema writes them, missing ones included | [DataFrame.fromRows](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/DataFrame.ts#L192) | +| DF-C032 | C | accepts the stored values that Schema.is accepts | [DataFrame.fromRows](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/DataFrame.ts#L192) | DF-C021 checks a single, unsliced, non-null batch of Int32 and Float64 columns: import/export preserves the backing buffers, byte offsets and byte lengths, as well as values. It does not require typed-array wrapper identity or mutation-based aliasing. The compatible fixed-width layouts are described in Apache Arrow’s [Columnar format](https://github.com/apache/arrow/blob/ee4d09ebef61c663c1efbfa4c18e518a03b798be/docs/source/format/Columnar.rst#fixed-size-primitive-layout) and [Schema.fbs](https://github.com/apache/arrow/blob/ee4d09ebef61c663c1efbfa4c18e518a03b798be/format/Schema.fbs); the no-copy adapter promise is the Effect API contract, not a requirement that Arrow imposes on every adapter. @@ -103,7 +104,7 @@ DF-C021 checks a single, unsliced, non-null batch of Int32 and Float64 columns: | DF-Q044 | C | tables declared by an Arrow schema convert their input to its columns | [Query.table](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Query.ts#L254) | | DF-Q045 | C | fails when a table is declared by a schema that has no columns for it | [Query.table](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Query.ts#L254) | -DF-C022 to DF-C031 and DF-Q043 to DF-Q045 assert behaviour added after the baseline: checks compiled from the schema, rejected transformations, tables declared by an Arrow schema, `DataFrame.column`, which has no symbol at the baseline and links to the layouts it exposes, and the checks `DataFrame.fromRows` runs on the frame it built. Their links locate the symbol at the baseline; the documentation of that symbol in the checked-out source is their contract. DF-C024 observes which value constructors run (`DateTime`, `Duration`, `BigDecimal`) and covers only `DataFrame.fromRows` on rows that hold such values already, and a frame whose columns already have the declared types. +DF-C022 to DF-C032 and DF-Q043 to DF-Q045 assert behaviour added after the baseline: checks compiled from the schema, rejected transformations, tables declared by an Arrow schema, `DataFrame.column`, which has no symbol at the baseline and links to the layouts it exposes, and the checks `DataFrame.fromRows` runs on the frame it built. Their links locate the symbol at the baseline; the documentation of that symbol in the checked-out source is their contract. DF-C024 observes which value constructors run (`DateTime`, `Duration`, `BigDecimal`) and covers only `DataFrame.fromRows` on rows that hold such values already, and a frame whose columns already have the declared types. ## Supported Schema subset @@ -127,7 +128,7 @@ DF-C022 to DF-C031 and DF-Q043 to DF-Q045 assert behaviour added after the basel Rejected with a `DataFrameError` that names the field: transformations (any schema with an encoding, such as `NumberFromString`, `decodeTo`, decoding defaults and renamed keys; `Schema.toType` gives the decoded side), `DateTimeZoned`, tuples, records, classes, unions of different column types, and recursive schemas. -Refinements, literals, enums and template literals are value checks: every value of such a field is read and tested, inside structs, arrays and union members too. A struct's own checks are row checks: every row is read and tested. Fields without checks are checked by column type only, on the typed columns. `DataFrame.fromRows` builds its frame and then checks it as `DataFrame.validate` does, so all of them test what the frame stores: a `Float32` column's check sees the rounded number. `DataFrame.fromRows` rejects a `DateTime` or `Duration` finer than its column's unit instead of rounding it, and a decimal column holds no more digits than its precision, whichever of them checks it (DF-C029). The checks are compiled when the schema is read (`Plan.TableChecks`) and are not part of a serialized plan: a plan decoded from JSON checks column types only. A SQL engine tests value checks on distinct values and fails for a table with row checks. +Refinements, literals, enums and template literals are value checks: every value of such a field is read and tested, inside structs, arrays and union members too. A struct's own checks are row checks: every row is read and tested. A field with a value check accepts the stored values that `Schema.is` accepts for it (DF-C032): a check on a nullable field runs on its nulls too, a check on a union member runs on that member's values, and a union in `oneOf` mode takes a value of exactly one member. Fields without checks are checked by column type only, on the typed columns. `DataFrame.fromRows` builds its frame and then checks it as `DataFrame.validate` does, so all of them test what the frame stores: a `Float32` column's check sees the rounded number. `DataFrame.fromRows` rejects a `DateTime` or `Duration` finer than its column's unit instead of rounding it, and a decimal column holds no more digits than its precision, whichever of them checks it (DF-C029). The checks are compiled when the schema is read (`Plan.TableChecks`) and are not part of a serialized plan: a plan decoded from JSON checks column types only. A SQL engine tests value checks on distinct values and fails for a table with row checks. Checks are handed stored values as the schema writes them (DF-C031), in `DataFrame.fromRows`, `DataFrame.empty`, `DataFrame.validate` and `Query.table` alike. A frame holds `null` for every missing value, and a check sees an absent key for an optional field, `undefined` for `UndefinedOr` and `null` for `NullOr`. The rows `{}` and `{ a: undefined }` are one stored row, and a check sees an absent key for both. diff --git a/packages/sql/pglite/test/DataFrameSql.test.ts b/packages/sql/pglite/test/DataFrameSql.test.ts index 856791551e5..26a8300611a 100644 --- a/packages/sql/pglite/test/DataFrameSql.test.ts +++ b/packages/sql/pglite/test/DataFrameSql.test.ts @@ -898,10 +898,18 @@ describe("effect/dataframe on PostgreSQL (PGlite)", { concurrent: false }, () => yield* rowsOnSql(contract(Schema.Struct({ s: orNull(Schema.Literals(["bad", "good"])) }))), [{ s: "bad" }, { s: "good" }] ) - assert.strictEqual( - yield* rejects(contract(Schema.Struct({ s: orNull(Schema.Literal("good")) }))), - `column "s" has values its schema rejects` - ) + const notNull = Schema.makeFilter((n: number | null) => n !== null) + for ( + const [schema, column] of [ + [Schema.Struct({ s: orNull(Schema.Literal("good")) }), "s"], + // A check of a nullable field is run on its nulls. + [Schema.Struct({ n: Schema.NullOr(Schema.Number).check(notNull) }), "n"], + // "good" is a value of both members. + [Schema.Struct({ s: Schema.Union([Schema.String, Schema.Literal("good")], { mode: "oneOf" }) }), "s"] + ] as const + ) { + assert.strictEqual(yield* rejects(contract(schema)), `column "${column}" has values its schema rejects`) + } })) }) }) From 9c238dfeee53830687d34a43f5ce86162d953724 Mon Sep 17 00:00:00 2001 From: Effect Bot Date: Mon, 16 Nov 2026 09:02:20 +0000 Subject: [PATCH 76/88] Refuse to encode a plan whose tables carry checks instead of dropping them --- packages/effect/src/dataframe/Plan.ts | 129 ++++++++++++++++++- packages/effect/src/dataframe/Query.ts | 6 +- packages/effect/test/dataframe/Query.test.ts | 29 ++++- packages/effect/test/dataframe/README.md | 4 +- 4 files changed, 153 insertions(+), 15 deletions(-) diff --git a/packages/effect/src/dataframe/Plan.ts b/packages/effect/src/dataframe/Plan.ts index 95cf0e7b34c..b2a42d0ef78 100644 --- a/packages/effect/src/dataframe/Plan.ts +++ b/packages/effect/src/dataframe/Plan.ts @@ -57,10 +57,13 @@ import * as ArrowIpc from "../arrow/ArrowIpc.ts" import * as Effect from "../Effect.ts" import * as Result from "../Result.ts" import * as Schema from "../Schema.ts" +import * as SchemaAST from "../SchemaAST.ts" import * as SchemaIssue from "../SchemaIssue.ts" +import * as SchemaParser from "../SchemaParser.ts" import * as SchemaTransformation from "../SchemaTransformation.ts" import { type DataFrame, isDataFrame } from "./DataFrame.ts" import * as F from "./internal/frame.ts" +import * as Q from "./internal/query.ts" // ----------------------------------------------------------------------------- // Values @@ -706,9 +709,10 @@ export interface TableColumn { * * **Gotchas** * - * The tests are functions, so they are not part of the JSON form of a plan. - * A SQL engine runs `columns` tests on the distinct values of each column - * and fails for a table with a `rows` test. + * The tests are functions and have no JSON form: {@link NodeFromJson} fails + * for a table that has them (see {@link withoutChecks}). A SQL engine runs + * `columns` tests on the distinct values of each column and fails for a + * table with a `rows` test. * * @stability experimental * @category models @@ -756,7 +760,9 @@ export interface TableChecks { * * **Gotchas** * - * `checks` is not part of the JSON form: a plan decoded from JSON checks + * `checks` has no JSON form. {@link NodeFromJson} fails to encode a table + * that has them, naming the table, and fails to decode JSON with a `checks` + * key. {@link withoutChecks} removes them: the plan then encodes, and checks * column types only. * * @stability experimental @@ -1079,12 +1085,63 @@ const TableColumnSchema: Schema.Codec = Schema.Struct({ unit: Schema.optional(TimeUnit) }) +const isTableChecks = (u: unknown): u is TableChecks => + typeof u === "object" && u !== null && "columns" in u && Array.isArray(u.columns) && + u.columns.every((c) => + Array.isArray(c) && c.length === 2 && typeof c[0] === "string" && typeof c[1] === "function" + ) && + (!("rows" in u) || u.rows === undefined || typeof u.rows === "function") + +const tableFields = { name: Schema.String, columns: Schema.optional(Schema.Array(TableColumnSchema)) } + +const TableStruct = Schema.TaggedStruct("Table", { + ...tableFields, + checks: Schema.optional(Schema.declare(isTableChecks, { expected: "TableChecks" })) +}) + +/** A table in JSON; `checks` is only there to be refused. */ +const TableJson = Schema.TaggedStruct("Table", { ...tableFields, checks: Schema.optional(Schema.Unknown) }) + +/** Fails for a table with checks: dropping them would drop the validation they do. */ +const layoutOnly = ( + table: A +): Effect.Effect => + table.checks === undefined + ? Effect.succeed(table) + : Effect.fail( + new SchemaIssue.Forbidden({ + message: `table ${JSON.stringify(table.name)} has checks, which are functions without a JSON form: ` + + "encode Plan.withoutChecks(plan), and declare the table again with Query.table to check the decoded plan's rows" + }, table) + ) + +const tableFromJson: SchemaTransformation.Transformation = SchemaTransformation + .transformEffect({ + decode: (json) => layoutOnly(json as Table), + encode: (table) => layoutOnly(table as typeof TableJson.Type) + }) + +/** + * A table as it is, checks included, so that validating a plan keeps them; + * the JSON form refuses a table with checks instead of dropping them. + */ +const TableSchema = Schema.declareConstructor()( + [], + () => (u, _, options) => Effect.as(SchemaParser.decodeUnknownEffect(TableStruct)(u, options), u as Table), + { + expected: "Table", + // A union picks this member by its tag, as it does a struct. + "~sentinels": SchemaAST.collectSentinels(TableStruct.ast), + toCodecJson: () => Schema.link
()(TableJson, tableFromJson) + } +) + const NodeSchema = Schema.suspend((): Schema.Codec => NodeUnion) const Columns = Schema.Array(Schema.String) const NodeUnion = Schema.Union([ Schema.TaggedStruct("Values", { frame: Frame }), - Schema.TaggedStruct("Table", { name: Schema.String, columns: Schema.optional(Schema.Array(TableColumnSchema)) }), + TableSchema, Schema.TaggedStruct("Filter", { input: NodeSchema, predicate: ExpressionSchema }), Schema.TaggedStruct("Select", { input: NodeSchema, columns: Schema.Array(NamedSchema) }), Schema.TaggedStruct("WithColumns", { input: NodeSchema, columns: Schema.Array(NamedSchema) }), @@ -1121,6 +1178,10 @@ const NodeUnion = Schema.Union([ /** * Schema of {@link Node}. * + * **Details** + * + * A valid plan decodes and encodes to itself: tables keep their `checks`. + * * @stability experimental * @category schemas * @since 4.1.5 @@ -1130,12 +1191,70 @@ export const Node: Schema.Codec = NodeSchema /** * A {@link Node} from its JSON form. * + * **Gotchas** + * + * A table's `checks` are functions and have no JSON form. Encoding a plan + * fails for a table that has them instead of leaving them out, and so does + * decoding JSON with a `checks` key (the plan as `JSON.stringify` writes + * it). Encode {@link withoutChecks} of the plan to send it without them. A + * decoded plan never has checks: to test its tables' rows, build the query + * from `Query.table` with the schema where the plan is decoded. + * * @stability experimental * @category schemas * @since 4.1.5 */ export const NodeFromJson: Schema.Codec = Schema.toCodecJson(Node) as any +/** + * The plan with the {@link TableChecks} of its tables removed, so that + * {@link NodeFromJson} encodes it. + * + * **When to use** + * + * Use to send or store a plan whose tables were declared by a struct schema + * with refinements, literals or enums, when the reader of the plan doesn't + * need those values tested. + * + * **Details** + * + * Tables keep their names and declared columns: the result still fails for a + * table without those columns, with nulls in non-nullable ones or with values + * a column's type can't hold. + * + * **Gotchas** + * + * The result accepts values the schema rejects, here and after decoding. + * Checks can't be attached to a decoded plan: to test the rows again, build + * the query from `Query.table` with the schema where the plan is decoded. + * + * **Example** (Encoding a plan over a table with refinements) + * + * ```ts import.meta.vitest + * import { Result, Schema } from "effect" + * import { Plan, Query } from "effect/dataframe" + * + * const Reading = Schema.Struct({ temp: Schema.Number.check(Schema.isGreaterThan(-273.15)) }) + * const query = Query.table("readings", Reading) + * + * const encode = Schema.encodeResult(Plan.NodeFromJson) + * Result.isFailure(encode(query.plan)) // => true + * + * const columns = [{ name: "temp", type: "Float64", nullable: false }] + * Result.getOrThrow(encode(Plan.withoutChecks(query.plan))) // => { _tag: "Table", name: "readings", columns } + * ``` + * + * @stability experimental + * @category combinators + * @since 4.1.5 + */ +export const withoutChecks = (plan: Node): Node => + Q.mapTables(plan as Q.Node, (table) => { + if (table.checks === undefined) return table + const { checks: _, ...layout } = table + return layout + }) as Node + /** * An {@link Expression} from its JSON form. * diff --git a/packages/effect/src/dataframe/Query.ts b/packages/effect/src/dataframe/Query.ts index 3501688f077..92ca73a7892 100644 --- a/packages/effect/src/dataframe/Query.ts +++ b/packages/effect/src/dataframe/Query.ts @@ -264,8 +264,10 @@ export const from = (frame: DataFrame): Query => make({ _tag: "Values", * declaration can't take. * - A SQL engine can't check a struct's own checks on whole rows: a query * over such a table fails. - * - The tests are functions and not part of the plan's JSON form: a plan - * decoded from JSON checks column types only. + * - The tests are functions and have no JSON form: `Plan.NodeFromJson` + * fails to encode a plan over a table that has them. Encode + * `Plan.withoutChecks` of the plan to send it without them, and declare + * the table again where the plan is decoded to test its rows there. * * **Example** (Declaring a table by an Arrow schema) * diff --git a/packages/effect/test/dataframe/Query.test.ts b/packages/effect/test/dataframe/Query.test.ts index 82f69caddcb..e20eda9dcb4 100644 --- a/packages/effect/test/dataframe/Query.test.ts +++ b/packages/effect/test/dataframe/Query.test.ts @@ -197,7 +197,7 @@ describe("Query", () => { })) // DF-Q043 (API contract): Query.table; pinned source in README.md. - it.effect("compiles a schema's checks when the table is declared, not per batch", () => + it.effect("compiles a schema's checks when the table is declared, and does not encode them to JSON", () => Effect.gen(function*() { const Kind = Schema.Struct({ kind: Schema.Literals(["a", "b"]) }) let reads = 0 @@ -218,13 +218,30 @@ describe("Query", () => { const error = yield* Effect.flip(read(query, [good, good, bad])) assert.strictEqual(error.message, `column "kind" has values its schema rejects`) assert.strictEqual(reads, declared) - // The checks are functions: they stay out of the JSON form of the plan. - const json = JSON.stringify(Schema.encodeSync(Plan.NodeFromJson)(query.plan)) + // The checks are functions without a JSON form: a plan that carries them is not encoded. + const first = query.pipe(Query.head(3)) + const rejected = /table "t" has checks[^]*Plan\.withoutChecks/ + assert.throws(() => Schema.encodeSync(Plan.NodeFromJson)(first.plan), rejected) + assert.throws(() => Schema.encodeSync(Schema.toCodecJson(Schema.Struct({ plan: Plan.Node })))(first), rejected) + // Nor is one decoded from JSON written without the codec, where the functions are lost. + assert.throws( + () => Schema.decodeUnknownSync(Plan.NodeFromJson)(JSON.parse(JSON.stringify(first.plan))), + /table "t" has checks/ + ) + // Validating a plan keeps its checks. + const validated = Query.fromPlan(Schema.decodeUnknownSync(Plan.Node)(first.plan)) + assert.strictEqual( + (yield* Effect.flip(read(validated, [good, bad]))).message, + `column "kind" has values its schema rejects` + ) + // Removing them is explicit: the plan then encodes, and checks column types only. + const json = JSON.stringify(Schema.encodeSync(Plan.NodeFromJson)(Plan.withoutChecks(first.plan))) const decoded = Query.fromPlan(Schema.decodeUnknownSync(Plan.NodeFromJson)(JSON.parse(json))) assert.deepStrictEqual(decoded.plan, { - _tag: "Table", - name: "t", - columns: [{ name: "kind", type: "Utf8", nullable: false }] + _tag: "Slice", + input: { _tag: "Table", name: "t", columns: [{ name: "kind", type: "Utf8", nullable: false }] }, + offset: 0, + length: 3 }) assert.strictEqual((yield* read(decoded, [good, bad])).length, 3) })) diff --git a/packages/effect/test/dataframe/README.md b/packages/effect/test/dataframe/README.md index d9b42a16c64..90b61ae5e60 100644 --- a/packages/effect/test/dataframe/README.md +++ b/packages/effect/test/dataframe/README.md @@ -100,7 +100,7 @@ DF-C021 checks a single, unsliced, non-null batch of Int32 and Float64 columns: | DF-Q040 | C | groups by keys of every type | [Query.groupBy](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Query.ts#L610) | | DF-Q041 | C | joins on a high-cardinality key | [Query.join](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Query.ts#L680) | | DF-Q042 | C | yields a long query so another fiber can interrupt it | [Query yielding](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Query.ts#L65-L66) | -| DF-Q043 | C | compiles a schema's checks when the table is declared, not per batch | [Query.table](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Query.ts#L254) | +| DF-Q043 | C | compiles a schema's checks when the table is declared, and does not encode them to JSON | [Query.table](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Query.ts#L254) | | DF-Q044 | C | tables declared by an Arrow schema convert their input to its columns | [Query.table](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Query.ts#L254) | | DF-Q045 | C | fails when a table is declared by a schema that has no columns for it | [Query.table](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Query.ts#L254) | @@ -128,7 +128,7 @@ DF-C022 to DF-C032 and DF-Q043 to DF-Q045 assert behaviour added after the basel Rejected with a `DataFrameError` that names the field: transformations (any schema with an encoding, such as `NumberFromString`, `decodeTo`, decoding defaults and renamed keys; `Schema.toType` gives the decoded side), `DateTimeZoned`, tuples, records, classes, unions of different column types, and recursive schemas. -Refinements, literals, enums and template literals are value checks: every value of such a field is read and tested, inside structs, arrays and union members too. A struct's own checks are row checks: every row is read and tested. A field with a value check accepts the stored values that `Schema.is` accepts for it (DF-C032): a check on a nullable field runs on its nulls too, a check on a union member runs on that member's values, and a union in `oneOf` mode takes a value of exactly one member. Fields without checks are checked by column type only, on the typed columns. `DataFrame.fromRows` builds its frame and then checks it as `DataFrame.validate` does, so all of them test what the frame stores: a `Float32` column's check sees the rounded number. `DataFrame.fromRows` rejects a `DateTime` or `Duration` finer than its column's unit instead of rounding it, and a decimal column holds no more digits than its precision, whichever of them checks it (DF-C029). The checks are compiled when the schema is read (`Plan.TableChecks`) and are not part of a serialized plan: a plan decoded from JSON checks column types only. A SQL engine tests value checks on distinct values and fails for a table with row checks. +Refinements, literals, enums and template literals are value checks: every value of such a field is read and tested, inside structs, arrays and union members too. A struct's own checks are row checks: every row is read and tested. A field with a value check accepts the stored values that `Schema.is` accepts for it (DF-C032): a check on a nullable field runs on its nulls too, a check on a union member runs on that member's values, and a union in `oneOf` mode takes a value of exactly one member. Fields without checks are checked by column type only, on the typed columns. `DataFrame.fromRows` builds its frame and then checks it as `DataFrame.validate` does, so all of them test what the frame stores: a `Float32` column's check sees the rounded number. `DataFrame.fromRows` rejects a `DateTime` or `Duration` finer than its column's unit instead of rounding it, and a decimal column holds no more digits than its precision, whichever of them checks it (DF-C029). The checks are compiled when the schema is read (`Plan.TableChecks`). They are functions and have no JSON form: `Plan.NodeFromJson` fails to encode a plan over a table that has them, naming the table, and fails to decode JSON that carries a `checks` key. `Plan.withoutChecks` removes them explicitly; the plan then encodes and checks column types only. A decoded plan gets its checks back by declaring the table again with `Query.table`. `Plan.Node` validates a plan as it is and keeps its checks. A SQL engine tests value checks on distinct values and fails for a table with row checks. Checks are handed stored values as the schema writes them (DF-C031), in `DataFrame.fromRows`, `DataFrame.empty`, `DataFrame.validate` and `Query.table` alike. A frame holds `null` for every missing value, and a check sees an absent key for an optional field, `undefined` for `UndefinedOr` and `null` for `NullOr`. The rows `{}` and `{ a: undefined }` are one stored row, and a check sees an absent key for both. From 035eb7bcff5f376b430aeefd4d4d90c513fbe9ff Mon Sep 17 00:00:00 2001 From: Effect Bot Date: Mon, 16 Nov 2026 09:12:47 +0000 Subject: [PATCH 77/88] Type duration sums as durations and reject the means and sums the engines disagree on --- packages/effect/src/dataframe/Expr.ts | 50 +++++-- packages/effect/src/dataframe/Query.ts | 6 + .../src/dataframe/internal/aggregate.ts | 9 +- .../effect/src/dataframe/internal/dtype.ts | 9 ++ .../effect/src/dataframe/internal/window.ts | 6 +- packages/effect/test/dataframe/Query.test.ts | 128 ++++++++++++++++++ packages/effect/test/dataframe/README.md | 11 +- packages/effect/typetest/DataFrame.tst.ts | 28 ++++ packages/sql/pglite/test/DataFrameSql.test.ts | 64 ++++++++- 9 files changed, 291 insertions(+), 20 deletions(-) diff --git a/packages/effect/src/dataframe/Expr.ts b/packages/effect/src/dataframe/Expr.ts index d7854f0bdc9..de1ba44a95b 100644 --- a/packages/effect/src/dataframe/Expr.ts +++ b/packages/effect/src/dataframe/Expr.ts @@ -190,6 +190,8 @@ type Numeric = number | bigint | BigDecimal.BigDecimal type NumericOf = NonNullable extends bigint ? bigint : NonNullable extends BigDecimal.BigDecimal ? BigDecimal.BigDecimal : number +/** The type of a sum of `A`s: durations add up to a duration, the rest to their numeric family. */ +type SumOf = NonNullable extends Numeric | boolean ? NumericOf : Duration.Duration type InputType = I extends Expr ? A : I /** @@ -820,17 +822,34 @@ const agg = (fn: Plan.AggregateName, self?: unknown) => * The sum of the values; 0 for a group without values. Sums of 8- and 16-bit * integers are Int32. * + * **Details** + * + * Durations sum to a duration in the column's unit, the zero duration for a + * group without values. Like 64-bit integer sums, they wrap on overflow. + * + * **Gotchas** + * + * Dates, times and timestamps have no sum: a query that sums them fails with + * a `DataFrameError`. + * * @stability experimental * @category aggregates * @since 4.1.5 */ -export const sum = ( +export const sum = ( self: Expr -): Aggregate> => agg("sum", self) +): Aggregate> => agg("sum", self) /** * The mean of the values; `null` for a group without values. * + * **Gotchas** + * + * Durations, dates, times and timestamps are not numbers: a query that takes + * their {@link mean}, {@link median}, {@link std} or {@link variance} fails + * with a `DataFrameError`. Aggregate a count of units instead, such as + * {@link totalMilliseconds} of a duration. + * * @stability experimental * @category aggregates * @since 4.1.5 @@ -970,15 +989,15 @@ export const over: { /** * The running sum: on each row, the sum of the values so far (in the * partition, with {@link over}). Nulls stay null and are skipped; NaN - * propagates. + * propagates. Durations sum to durations, as with {@link sum}. * * @stability experimental * @category windows * @since 4.1.5 */ -export const cumSum = ( +export const cumSum = ( self: Expr -): Expr | Nullish> => internal.call("cumSum", [self]) +): Expr | Nullish> => internal.call("cumSum", [self]) /** * The running minimum. Nulls stay null; NaN is skipped. @@ -1024,15 +1043,16 @@ export const shift: { } = lag("shift") /** - * The difference to the value `n` rows earlier (1 by default). + * The difference to the value `n` rows earlier (1 by default), of numbers + * or of durations. * * @stability experimental * @category windows * @since 4.1.5 */ export const diff: { - (n?: number): (self: Expr) => Expr - (self: Expr, n?: number): Expr + (n?: number): (self: Expr) => Expr + (self: Expr, n?: number): Expr } = lag("diff") type RankOptions = { @@ -1065,10 +1085,10 @@ export const rank: { type RollingOptions = { readonly minPeriods?: number | undefined } type Rolling = { - (window: number, options?: RollingOptions): ( + (window: number, options?: RollingOptions): ( self: Expr ) => Expr | null> - ( + ( self: Expr, window: number, options?: RollingOptions @@ -1085,6 +1105,7 @@ const rolling = (fn: "rollingSum" | "rollingMean" | "rollingMin" | "rollingMax") /** * The sum of the last `window` values (the current one included). Null * while fewer than `minPeriods` (default: `window`) of them are non-null. + * Durations sum to durations, as with {@link sum}. * * @stability experimental * @category windows @@ -1093,7 +1114,8 @@ const rolling = (fn: "rollingSum" | "rollingMean" | "rollingMin" | "rollingMax") export const rollingSum: Rolling = rolling("rollingSum") /** - * The mean of the last `window` values; see {@link rollingSum}. + * The mean of the last `window` values; see {@link rollingSum}. Durations + * have no mean, as with {@link mean}. * * @stability experimental * @category windows @@ -1340,15 +1362,15 @@ export const listGet: { /** * The sum of each list's elements: 0 for an empty list, null for a null - * list. + * list. Lists of durations sum to durations, as with {@link sum}. * * @stability experimental * @category lists and structs * @since 4.1.5 */ -export const listSum = | null>( +export const listSum = | null>( self: Expr -): Expr> | Nullish> => internal.call("listSum", [self]) +): Expr> | Nullish> => internal.call("listSum", [self]) /** * The mean of each list's elements; null for an empty or null list. diff --git a/packages/effect/src/dataframe/Query.ts b/packages/effect/src/dataframe/Query.ts index 92ca73a7892..dea117af87e 100644 --- a/packages/effect/src/dataframe/Query.ts +++ b/packages/effect/src/dataframe/Query.ts @@ -70,6 +70,7 @@ */ import type * as ArrowSchema from "../arrow/ArrowSchema.ts" import type * as BigDecimal from "../BigDecimal.ts" +import type * as Duration from "../Duration.ts" import type * as Effect from "../Effect.ts" import * as Effectable from "../Effectable.ts" import { dual } from "../Function.ts" @@ -758,6 +759,10 @@ export const join: { /** * The value type of a pivot cell for an aggregate of values of type `A`. * + * **Details** + * + * Sums have the type `Expr.sum` gives: durations sum to a duration. + * * @stability experimental * @category type-level * @since 4.1.5 @@ -765,6 +770,7 @@ export const join: { export type PivotCell = F extends "len" | "count" | "countDistinct" ? number : F extends "sum" ? (NonNullable extends bigint ? bigint : NonNullable extends BigDecimal.BigDecimal ? BigDecimal.BigDecimal + : NonNullable extends Duration.Duration ? Duration.Duration : number) : F extends "mean" | "median" | "std" | "variance" ? number | null : NonNullable | null diff --git a/packages/effect/src/dataframe/internal/aggregate.ts b/packages/effect/src/dataframe/internal/aggregate.ts index 6eef9883f96..4f08c1c7a1d 100644 --- a/packages/effect/src/dataframe/internal/aggregate.ts +++ b/packages/effect/src/dataframe/internal/aggregate.ts @@ -132,6 +132,7 @@ export type Sums = Float64Array | ReadonlyArray export const sums = (v: V.Vector, g: Grouping): Sums => { const ids = g.ids const n = ids.length + if (D.isPointInTime(v.type)) return V.fail(`sum is not defined for ${D.describe(v.type)}`) switch (v.kind) { case "null": return new Array(g.count).fill(I0) @@ -182,8 +183,14 @@ export const sumColumn = (inputType: S.DataType, sums: Sums): V.Vector => { } } -/** Every value as a float, with validity. */ +/** + * Every value as a float, with validity. Durations and points in time are + * not numbers: their integers count units, which a mean would lose. + */ const asFloat = (v: V.Vector): { readonly x: Float64Array; readonly valid: Uint8Array | undefined } => { + if (v.type._tag === "Duration" || D.isPointInTime(v.type)) { + return V.fail(`expected a numeric column, got ${D.describe(v.type)}`) + } switch (v.kind) { case "f64": return { x: v.values, valid: v.valid } diff --git a/packages/effect/src/dataframe/internal/dtype.ts b/packages/effect/src/dataframe/internal/dtype.ts index b1924ff8769..b6f1f51cf58 100644 --- a/packages/effect/src/dataframe/internal/dtype.ts +++ b/packages/effect/src/dataframe/internal/dtype.ts @@ -133,6 +133,15 @@ export const sumType = (t: S.DataType): S.DataType => { return t } +/** + * Whether values of type `t` are points in time, which have neither a sum + * nor a mean: dates, times of day and timestamps. Durations add up. + * + * @internal + */ +export const isPointInTime = (t: S.DataType): boolean => + t._tag === "Date" || t._tag === "Time" || t._tag === "Timestamp" + /** Type of `mean`, `median`, `std` and `var` over a column of type `t`. @internal */ export const meanType = divisionType diff --git a/packages/effect/src/dataframe/internal/window.ts b/packages/effect/src/dataframe/internal/window.ts index 4da4b1209d9..6081c89adb7 100644 --- a/packages/effect/src/dataframe/internal/window.ts +++ b/packages/effect/src/dataframe/internal/window.ts @@ -33,6 +33,7 @@ export const cumSum = (v: V.Vector, p: Partition): V.Vector => { const ids = p.ids const type = D.sumType(v.type) const valid = validOf(v) + if (D.isPointInTime(v.type)) return V.fail(`cumSum is not defined for ${D.describe(v.type)}`) switch (v.kind) { case "null": return V.nulls(n) @@ -219,7 +220,10 @@ export const rolling = ( }) return V.take(v, idx) } - if (v.kind === "str" || v.kind === "null" || v.kind === "list" || v.kind === "struct") { + if ( + v.kind === "str" || v.kind === "null" || v.kind === "list" || v.kind === "struct" || D.isPointInTime(v.type) || + (fn === "rollingMean" && v.type._tag === "Duration") + ) { return V.fail(`${fn} is not defined for ${D.describe(v.type)}`) } const type = fn === "rollingMean" ? D.meanType(v.type) : D.sumType(v.type) diff --git a/packages/effect/test/dataframe/Query.test.ts b/packages/effect/test/dataframe/Query.test.ts index e20eda9dcb4..28c1dec7600 100644 --- a/packages/effect/test/dataframe/Query.test.ts +++ b/packages/effect/test/dataframe/Query.test.ts @@ -572,6 +572,134 @@ describe("Query", () => { } ]) })) + + // DF-Q046 (API contract): Expr.sum; pinned source in README.md. + it.effect("sums durations to durations in the column's unit", () => + Effect.gen(function*() { + type Row = { readonly g: string; readonly q: string; readonly d: Duration.Duration | null } + const unit = ArrowSchema.duration("nanosecond") + const frame = (g: ReadonlyArray, q: ReadonlyArray, d: ReadonlyArray) => + arrowFrame({ g: [ArrowSchema.utf8, g], q: [ArrowSchema.utf8, q], d: [unit, d] }) + const whole = Query.from(frame(["a", "a", "a", "b", "c"], ["x", "y", "x", "x", "y"], [7n, 9n, null, 4n, null])) + const streamed = Query.table("t", whole.pipe(Effect.runSync).schema) as unknown as Query.Query + const batches = Catalog.layer({ + t: Stream.make( + frame(["a", "a"], ["x", "y"], [7n, 9n]), + frame(["a", "b", "c"], ["x", "x", "y"], [null, 4n, null]) + ) + }) + // Every column but the keys: its type and its values in nanoseconds. + const nanos = (query: Query.Query) => + Effect.map(query, (df) => + Object.fromEntries( + df.schema.fields.filter((f) => f.name !== "g").map((f) => [f.name, [ + f.type, + (DataFrame.toArray(df, f.name) as Array).map((d) => + d === null ? null : Duration.toNanosUnsafe(d) + ) + ]]) + )) + // A group without values sums to the zero duration. + const grouped = (input: Query.Query) => + input.pipe(Query.groupBy("g"), Query.agg((c) => ({ total: Expr.sum(c.d) }))) + assert.deepStrictEqual(yield* nanos(grouped(whole)), { total: [unit, [16n, 4n, 0n]] }) + assert.deepStrictEqual(yield* nanos(grouped(streamed)).pipe(Effect.provide(batches)), { + total: [unit, [16n, 4n, 0n]] + }) + assert.deepStrictEqual( + yield* nanos(whole.pipe(Query.select((c) => ({ + over: Expr.over(Expr.sum(c.d), [c.g]), + running: Expr.cumSum(c.d), + last2: Expr.rollingSum(c.d, 2), + least2: Expr.rollingMin(c.d, 2), + step: Expr.diff(c.d) + })))), + { + over: [unit, [16n, 16n, 16n, 4n, 0n]], + running: [unit, [7n, 16n, null, 20n, null]], + last2: [unit, [null, 16n, null, null, null]], + least2: [unit, [null, 7n, null, null, null]], + step: [unit, [null, 2n, null, null, null]] + } + ) + assert.deepStrictEqual( + yield* nanos(whole.pipe( + Query.groupBy("g"), + Query.agg((c) => ({ all: Expr.list(c.d) })), + Query.select((c) => ({ g: c.g, total: Expr.listSum(c.all) })) + )), + { total: [unit, [16n, 4n, 0n]] } + ) + assert.deepStrictEqual( + yield* nanos( + whole.pipe(Query.pivot({ on: "q", index: ["g"], values: "d", aggregate: "sum", columns: ["x", "y"] })) + ), + { x: [unit, [7n, 4n, 0n]], y: [unit, [9n, 0n, 0n]] } + ) + })) + + // DF-Q047 (API contract): Expr.mean; pinned source in README.md. + it.effect("rejects aggregates that durations, timestamps, dates and times don't have", () => + Effect.gen(function*() { + const frame = (type: ArrowSchema.DataType, values: ReadonlyArray) => + arrowFrame<{ readonly g: string; readonly v: number | null }>({ + g: [ArrowSchema.utf8, values.map(() => "a")], + v: [type, values] + }) + // The types reject these columns too: `v` is read as a number here. + const message = (query: Query.Query) => Effect.map(Effect.flip(query), (e) => e.message) + const aggregated = + (input: Query.Query<{ readonly g: string; readonly v: number | null }>) => + (f: (v: Expr.Expr) => Expr.Aggregate) => + message(input.pipe(Query.groupBy("g"), Query.agg((c) => ({ r: f(c.v) })))) + const selected = + (input: Query.Query<{ readonly g: string; readonly v: number | null }>) => + (f: (v: Expr.Expr) => Expr.Expr) => + message(input.pipe(Query.select((c) => ({ r: f(c.v) })))) + + const durations = frame(ArrowSchema.duration("microsecond"), [7n, 9n, null]) + const notNumeric = "expected a numeric column, got Duration[us]" + for (const f of [Expr.mean, Expr.median, Expr.std, Expr.variance]) { + assert.strictEqual(yield* aggregated(Query.from(durations))(f), notNumeric) + } + // A streamed table's means combine from partial aggregates. + const streamed = Query.table("t", durations.schema) as unknown as Query.Query<{ g: string; v: number | null }> + const batches = Catalog.layer({ t: Stream.make(durations, durations) }) + for (const f of [Expr.mean, Expr.std]) { + assert.strictEqual(yield* aggregated(streamed)(f).pipe(Effect.provide(batches)), notNumeric) + } + assert.strictEqual( + yield* selected(Query.from(durations))((v) => Expr.rollingMean(v, 2)), + "rollingMean is not defined for Duration[us]" + ) + assert.strictEqual( + yield* message( + Query.from(durations).pipe( + Query.groupBy("g"), + Query.agg((c) => ({ all: Expr.list(c.v) })), + Query.select((c) => ({ r: Expr.listMean(c.all) })) + ) + ), + notNumeric + ) + + // Points in time have neither a sum nor a mean. + const points: ReadonlyArray, string]> = [ + [ArrowSchema.timestamp("microsecond"), [7n, 9n, null], "Timestamp[us]"], + [ArrowSchema.date("day"), [7, 9, null], "Date"], + [ArrowSchema.time("microsecond"), [7n, 9n, null], "Time"] + ] + for (const [type, values, name] of points) { + const input = Query.from(frame(type, values)) + assert.strictEqual(yield* aggregated(input)(Expr.sum), `sum is not defined for ${name}`) + assert.strictEqual(yield* aggregated(input)(Expr.mean), `expected a numeric column, got ${name}`) + assert.strictEqual(yield* selected(input)(Expr.cumSum), `cumSum is not defined for ${name}`) + assert.strictEqual( + yield* selected(input)((v) => Expr.rollingSum(v, 2)), + `rollingSum is not defined for ${name}` + ) + } + })) }) describe("explain", () => { diff --git a/packages/effect/test/dataframe/README.md b/packages/effect/test/dataframe/README.md index 90b61ae5e60..ce81a731a93 100644 --- a/packages/effect/test/dataframe/README.md +++ b/packages/effect/test/dataframe/README.md @@ -103,8 +103,10 @@ DF-C021 checks a single, unsliced, non-null batch of Int32 and Float64 columns: | DF-Q043 | C | compiles a schema's checks when the table is declared, and does not encode them to JSON | [Query.table](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Query.ts#L254) | | DF-Q044 | C | tables declared by an Arrow schema convert their input to its columns | [Query.table](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Query.ts#L254) | | DF-Q045 | C | fails when a table is declared by a schema that has no columns for it | [Query.table](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Query.ts#L254) | +| DF-Q046 | C | sums durations to durations in the column's unit | [Expr.sum](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Expr.ts#L827) | +| DF-Q047 | C | rejects aggregates that durations, timestamps, dates and times don't have | [Expr.mean](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Expr.ts#L838) | -DF-C022 to DF-C032 and DF-Q043 to DF-Q045 assert behaviour added after the baseline: checks compiled from the schema, rejected transformations, tables declared by an Arrow schema, `DataFrame.column`, which has no symbol at the baseline and links to the layouts it exposes, and the checks `DataFrame.fromRows` runs on the frame it built. Their links locate the symbol at the baseline; the documentation of that symbol in the checked-out source is their contract. DF-C024 observes which value constructors run (`DateTime`, `Duration`, `BigDecimal`) and covers only `DataFrame.fromRows` on rows that hold such values already, and a frame whose columns already have the declared types. +DF-C022 to DF-C032 and DF-Q043 to DF-Q047 assert behaviour added after the baseline: checks compiled from the schema and refused by the JSON form of a plan, rejected transformations, tables declared by an Arrow schema, sums of durations, aggregates rejected for durations and points in time, `DataFrame.column`, which has no symbol at the baseline and links to the layouts it exposes, and the checks `DataFrame.fromRows` runs on the frame it built. Their links locate the symbol at the baseline; the documentation of that symbol in the checked-out source is their contract. DF-C024 observes which value constructors run (`DateTime`, `Duration`, `BigDecimal`) and covers only `DataFrame.fromRows` on rows that hold such values already, and a frame whose columns already have the declared types. ## Supported Schema subset @@ -190,7 +192,9 @@ The actual SQL suite is **`packages/sql/pglite/test/DataFrameSql.test.ts`**, not | DF-S014 | C / I | window functions over partitions | [Sql contract](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Sql.ts#L1-L45) | | DF-S015 | C / I | dates and times | [Sql contract](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Sql.ts#L1-L45) | | DF-S016 | C / I | explain shows the SQL | [Sql contract](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Sql.ts#L1-L45) | -| DF-S017 | C / I | unions and nullable fields are checked as in memory | [Sql contract](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Sql.ts#L1-L45) | +| DF-S017 | C / I | sums durations to durations, as in memory | [Sql contract](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Sql.ts#L1-L45) | +| DF-S018 | C / I | rejects aggregates that durations and timestamps don't have, as in memory | [Sql contract](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Sql.ts#L1-L45) | +| DF-S019 | C / I | unions and nullable fields are checked as in memory | [Sql contract](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Sql.ts#L1-L45) | ## Compiler assertions @@ -220,6 +224,7 @@ The actual SQL suite is **`packages/sql/pglite/test/DataFrameSql.test.ts`**, not - `propagates nullability` - `data-last forms` +- `durations sum to durations and have no mean` ### `Windows and reshaping` @@ -238,7 +243,7 @@ The actual SQL suite is **`packages/sql/pglite/test/DataFrameSql.test.ts`**, not - `Query.stream emits frames of the row type` -Rejection assertions include unknown Arrow override fields (`DataFrame`), unknown columns/group keys, incompatible comparisons/arithmetic, non-aggregate aggregation, aggregate predicates and duplicate non-key join columns (`Query` / `rejects mistakes`), exploding a non-list column (`Windows and reshaping` / `explode and unnest`), and extracting a year from a number (`Types` / `dates, times and durations`). Other entries assert exact inferred types, including nullability, Effect errors and stream elements. +Rejection assertions include unknown Arrow override fields (`DataFrame`), unknown columns/group keys, incompatible comparisons/arithmetic, non-aggregate aggregation, aggregate predicates and duplicate non-key join columns (`Query` / `rejects mistakes`), exploding a non-list column (`Windows and reshaping` / `explode and unnest`), extracting a year from a number (`Types` / `dates, times and durations`), and the mean, median, deviation and rolling mean of a duration and the sum of a timestamp (`Expr` / `durations sum to durations and have no mean`). Other entries assert exact inferred types, including nullability, Effect errors and stream elements. ## Exclusions and remaining gaps diff --git a/packages/effect/typetest/DataFrame.tst.ts b/packages/effect/typetest/DataFrame.tst.ts index f40a42b7289..951da8c4e4c 100644 --- a/packages/effect/typetest/DataFrame.tst.ts +++ b/packages/effect/typetest/DataFrame.tst.ts @@ -221,6 +221,31 @@ describe("Expr", () => { expect(s.pipe(Expr.startsWith("a"))).type.toBe>() expect(n.pipe(Expr.round(2))).type.toBe>() }) + + it("durations sum to durations and have no mean", () => { + expect(Expr.sum(took)).type.toBe>() + expect(Expr.cumSum(took)).type.toBe>() + expect(Expr.rollingSum(took, 2)).type.toBe>() + expect(took.pipe(Expr.rollingMin(2))).type.toBe>() + expect(Expr.diff(took)).type.toBe>() + expect(Expr.listSum(tookEach)).type.toBe>() + expect(events.pipe(Query.pivot({ on: "id", index: [], values: "took", aggregate: "sum", columns: ["a", "b"] }))) + .type.toBe>() + // @ts-expect-error: is not assignable to parameter of type 'Expr' + Expr.mean(took) + // @ts-expect-error: is not assignable to parameter of type 'Expr' + Expr.median(took) + // @ts-expect-error: is not assignable to parameter of type 'Expr' + Expr.std(took) + // @ts-expect-error: is not assignable to parameter of type 'Expr' + Expr.variance(took) + // @ts-expect-error: is not assignable to parameter of type 'Expr' + Expr.rollingMean(took, 2) + // @ts-expect-error: is not assignable to parameter of type + Expr.listMean(tookEach) + // @ts-expect-error: is not assignable to parameter of type + Expr.sum(at) + }) }) const Event = Schema.Struct({ @@ -233,6 +258,9 @@ const Event = Schema.Struct({ }) declare const events: Query.Query declare const price: Expr.Expr +declare const at: Expr.Expr +declare const took: Expr.Expr +declare const tookEach: Expr.Expr> describe("Windows and reshaping", () => { it("over keeps the value type; window functions type their results", () => { diff --git a/packages/sql/pglite/test/DataFrameSql.test.ts b/packages/sql/pglite/test/DataFrameSql.test.ts index 26a8300611a..a0cc682efd1 100644 --- a/packages/sql/pglite/test/DataFrameSql.test.ts +++ b/packages/sql/pglite/test/DataFrameSql.test.ts @@ -177,6 +177,11 @@ const decimals = arrow<{ d: BigDecimal.BigDecimal | null }>( s === null ? null : decimal(s) )] ) +/** Durations in microseconds, with a group that has none. */ +const spans = arrow<{ g: string; d: Duration.Duration | null }>( + [ArrowSchema.field("g", ArrowSchema.utf8), ArrowSchema.field("d", ArrowSchema.duration("microsecond"))], + [["a", "a", "a", "b", "c"], [7n, 9n, null, 4n, null]] +) const units = { s: "second", ms: "millisecond", us: "microsecond", ns: "nanosecond" } as const const unitNames = ["s", "ms", "us", "ns"] as const /** One column `v` of `type`, its integers microsecond multiples in every unit (PostgreSQL keeps microseconds). */ @@ -821,6 +826,63 @@ describe("effect/dataframe on PostgreSQL (PGlite)", { concurrent: false }, () => assert.include(text, `FROM "sales"`) })) + // DF-S017: Sql API contract; pins and oracle policy in effect/test/dataframe/README.md. + it.effect("sums durations to durations, as in memory", () => + Effect.gen(function*() { + const totals = spans.pipe(Query.groupBy("g"), Query.agg((c) => ({ total: Expr.sum(c.d) }))) + yield* both(totals) + yield* both(spans.pipe(Query.select((c) => ({ + over: Expr.over(Expr.sum(c.d), [c.g]), + running: Expr.cumSum(c.d), + last2: Expr.rollingSum(c.d, 2), + least2: Expr.rollingMin(c.d, 2), + step: Expr.diff(c.d) + })))) + // A group without values sums to the zero duration. + const result = yield* onSql(totals) + assert.deepStrictEqual(types(result)[1].type, ArrowSchema.duration("microsecond")) + assert.deepStrictEqual(DataFrame.toArray(result, "total").map(Duration.toNanosUnsafe), [16_000n, 4_000n, 0n]) + })) + + // DF-S018: Sql API contract; pins and oracle policy in effect/test/dataframe/README.md. + it.effect("rejects aggregates that durations and timestamps don't have, as in memory", () => + Effect.gen(function*() { + // The types reject these columns too: they are read as numbers here. + type Numbers = Query.Query<{ readonly g: string; readonly d: number | null }> + const inMemory = (query: Query.Query) => Effect.map(Effect.flip(query), (e) => e.message) + const rejected = Effect.fnUntraced(function*(query: Query.Query, message: string) { + assert.strictEqual(yield* rejects(query), message) + assert.strictEqual(yield* inMemory(query), message) + }) + const durations = spans as unknown as Numbers + for (const f of [Expr.mean, Expr.median, Expr.std, Expr.variance]) { + yield* rejected( + durations.pipe(Query.groupBy("g"), Query.agg((c) => ({ r: f(c.d) }))), + "expected a numeric column, got Duration[us]" + ) + } + yield* rejected( + durations.pipe(Query.select((c) => ({ r: Expr.over(Expr.mean(c.d), [c.g]) }))), + "expected a numeric column, got Duration[us]" + ) + yield* rejected( + durations.pipe(Query.select((c) => ({ r: Expr.rollingMean(c.d, 2) }))), + "rollingMean is not defined for Duration[us]" + ) + const timestamps = arrow<{ g: string; d: number | null }>( + [ArrowSchema.field("g", ArrowSchema.utf8), ArrowSchema.field("d", ArrowSchema.timestamp("microsecond"))], + [["a", "a"], [7n, null]] + ) + yield* rejected( + timestamps.pipe(Query.groupBy("g"), Query.agg((c) => ({ r: Expr.sum(c.d) }))), + "sum is not defined for Timestamp[us]" + ) + yield* rejected( + timestamps.pipe(Query.select((c) => ({ r: Expr.cumSum(c.d) }))), + "cumSum is not defined for Timestamp[us]" + ) + })) + // DF-N004: local policy for SQL results; pins in effect/test/dataframe/README.md. it.effect("decimal results are read from their text, without BigDecimal values", () => Effect.gen(function*() { @@ -889,7 +951,7 @@ describe("effect/dataframe on PostgreSQL (PGlite)", { concurrent: false }, () => ) })) - // DF-S017: Sql API contract; pins and oracle policy in effect/test/dataframe/README.md. + // DF-S019: Sql API contract; pins and oracle policy in effect/test/dataframe/README.md. it.effect("unions and nullable fields are checked as in memory", () => Effect.gen(function*() { const contract = tableOf("contract") From 88544471e37a7fb1299ead4503a97f687b5b29b0 Mon Sep 17 00:00:00 2001 From: Effect Bot Date: Mon, 16 Nov 2026 09:14:56 +0000 Subject: [PATCH 78/88] Use the shared temporal check for the columns a mean rejects --- packages/effect/src/dataframe/internal/aggregate.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/effect/src/dataframe/internal/aggregate.ts b/packages/effect/src/dataframe/internal/aggregate.ts index 4f08c1c7a1d..bf295c5f3da 100644 --- a/packages/effect/src/dataframe/internal/aggregate.ts +++ b/packages/effect/src/dataframe/internal/aggregate.ts @@ -14,6 +14,7 @@ import * as Dec from "./decimal.ts" import * as D from "./dtype.ts" import * as Hash from "./hash.ts" import * as Sort from "./sort.ts" +import * as T from "./temporal.ts" import * as V from "./vector.ts" const I0 = BigInt(0) @@ -188,7 +189,7 @@ export const sumColumn = (inputType: S.DataType, sums: Sums): V.Vector => { * not numbers: their integers count units, which a mean would lose. */ const asFloat = (v: V.Vector): { readonly x: Float64Array; readonly valid: Uint8Array | undefined } => { - if (v.type._tag === "Duration" || D.isPointInTime(v.type)) { + if (T.isTemporal(v.type)) { return V.fail(`expected a numeric column, got ${D.describe(v.type)}`) } switch (v.kind) { From 8968ad5116bcf265dfab57aa0703f315fc343643 Mon Sep 17 00:00:00 2001 From: Effect Bot Date: Mon, 16 Nov 2026 09:21:41 +0000 Subject: [PATCH 79/88] Align the DataFrame decimal control and Query.table docs with the stricter builders and checks --- packages/effect/src/dataframe/Query.ts | 10 +++++---- .../effect/test/dataframe/DataFrame.test.ts | 21 ++++++++++++++----- 2 files changed, 22 insertions(+), 9 deletions(-) diff --git a/packages/effect/src/dataframe/Query.ts b/packages/effect/src/dataframe/Query.ts index dea117af87e..040437a2eb6 100644 --- a/packages/effect/src/dataframe/Query.ts +++ b/packages/effect/src/dataframe/Query.ts @@ -245,7 +245,8 @@ export const from = (frame: DataFrame): Query => make({ _tag: "Values", * refinements, literals or enums has its values read and tested one by one, * and a struct with checks of its own (`Schema.Struct(...).check(...)`) has * every row read and tested. Other fields are checked by column type only, - * on the typed columns. + * on the typed columns. The tests see missing values as the schema writes + * them (an absent key, `undefined` or `null`), as in `DataFrame.fromRows`. * * An Arrow schema declares its fields as they are (type, nullability, * nested fields, decimal precision and scale, time unit), with untyped rows @@ -260,9 +261,10 @@ export const from = (frame: DataFrame): Query => make({ _tag: "Values", * **Gotchas** * * - Throws a `DataFrameError` for a schema that declares no columns: a - * struct schema that transforms its rows or has fields without a column - * type (see `DataFrame.fromRows`), or an Arrow schema with types a - * declaration can't take. + * struct schema that transforms its rows, has fields without a column + * type or has a check that would see a missing value the schema writes in + * several ways (see `DataFrame.fromRows`), or an Arrow schema with types + * a declaration can't take. * - A SQL engine can't check a struct's own checks on whole rows: a query * over such a table fails. * - The tests are functions and have no JSON form: `Plan.NodeFromJson` diff --git a/packages/effect/test/dataframe/DataFrame.test.ts b/packages/effect/test/dataframe/DataFrame.test.ts index e1490378f2d..3e81aeca164 100644 --- a/packages/effect/test/dataframe/DataFrame.test.ts +++ b/packages/effect/test/dataframe/DataFrame.test.ts @@ -180,7 +180,7 @@ describe("DataFrame", () => { `column "waits[].d" has values that Duration[us] can't hold exactly` ], [{ m: BigDecimal.make(1_234n, 3) }, "1.234 has more than 2 decimal places"], - [{ m: BigDecimal.make(1_234n, 1) }, `column "m" has values that Decimal[4, 2] can't hold exactly`] + [{ m: BigDecimal.make(1_234n, 1) }, "123.4 has more than 4 digits at scale 2"] ] for (const [fields, message] of cases) { assert.throws( @@ -189,11 +189,22 @@ describe("DataFrame", () => { message ) } - // A frame that holds such a decimal under the declared type fails too. + // A frame that holds such a decimal under the declared type fails too. Arrow builders + // refuse to store one, so the integers of a wider decimal are put under the declared type. const Amount = Schema.Struct({ m: Schema.BigDecimal }) - const held = ArrowTable.fromRows(DataFrame.empty(Amount, { types }).schema, [{ m: BigDecimal.make(1_234n, 1) }]) - const validated = DataFrame.validate(DataFrame.fromArrowTable(Result.getOrThrow(held)), Amount, { types }) - assert.include(Result.isFailure(validated) && validated.failure.message, cases[5][1]) + const schema = DataFrame.empty(Amount, { types }).schema + const wide = Result.getOrThrow(ArrowTable.fromRows( + ArrowSchema.make([ArrowSchema.field("m", ArrowSchema.decimal(5, 2), { nullable: false })]), + [{ m: BigDecimal.make(1_234n, 1) }] + )) + const held = ArrowTable.make(schema, [ + ArrowTable.recordBatch(schema, 1, [{ ...wide.batches[0].columns[0], type: types.m }]) + ]) + const validated = DataFrame.validate(DataFrame.fromArrowTable(held), Amount, { types }) + assert.include( + Result.isFailure(validated) && validated.failure.message, + `column "m" has values that Decimal[4, 2] can't hold exactly` + ) }) // DF-C030 (API contract): DataFrame.fromRows; pinned source in README.md. From ab350ad29140c8db2e08cf7d0ca96ea899dd394c Mon Sep 17 00:00:00 2001 From: Effect Bot Date: Mon, 16 Nov 2026 11:23:15 +0000 Subject: [PATCH 80/88] Keep __proto__ an own property of the rows that checks see A row is rebuilt for a check when a field is optional, and assigning __proto__ set the prototype instead of the field, so the check saw an object without it. --- .../effect/src/dataframe/internal/schema.ts | 5 +++-- .../effect/test/dataframe/DataFrame.test.ts | 22 +++++++++++++++++++ 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/packages/effect/src/dataframe/internal/schema.ts b/packages/effect/src/dataframe/internal/schema.ts index 3b6a7f4109b..da9fa5e8078 100644 --- a/packages/effect/src/dataframe/internal/schema.ts +++ b/packages/effect/src/dataframe/internal/schema.ts @@ -15,6 +15,7 @@ * @internal */ import type * as S from "../../arrow/ArrowSchema.ts" +import * as Values from "../../arrow/internal/values.ts" import { identity } from "../../Function.ts" import * as Result from "../../Result.ts" import * as AST from "../../SchemaAST.ts" @@ -268,9 +269,9 @@ const struct = (shapes: NonNullable): ((value: unknown) => unkn return (value) => { const out: Record = {} for (const [name, absent, restore] of fields) { - const v = (value as Record)[name] + const v = Values.getProperty(value as object, name) if (v === null && absent) continue - out[name] = restore === undefined ? v : restore(v) + Values.setProperty(out, name, restore === undefined ? v : restore(v)) } return out } diff --git a/packages/effect/test/dataframe/DataFrame.test.ts b/packages/effect/test/dataframe/DataFrame.test.ts index 3e81aeca164..3dd288bac96 100644 --- a/packages/effect/test/dataframe/DataFrame.test.ts +++ b/packages/effect/test/dataframe/DataFrame.test.ts @@ -302,6 +302,28 @@ describe("DataFrame", () => { Schema.makeFilter((row) => ("v" in row) === (row.w === null)) ), [{ v: 1, w: null }, { w: 1 }, { w: null }, { v: 1, w: 1 }] + ], + // An optional field makes the check see a rebuilt row, which must keep + // `__proto__` as an own property + [ + Schema.Struct({ ["__proto__"]: Schema.String, opt: Schema.optional(Schema.Number) }).check( + Schema.makeFilter((row) => Object.keys(row).length === 0) + ), + [{ ["__proto__"]: "good" }] + ], + [ + Schema.Struct({ ["__proto__"]: Schema.String, opt: Schema.optional(Schema.Number) }).check( + Schema.makeFilter((row) => Object.hasOwn(row, "__proto__") && row["__proto__"] === "good") + ), + [{ ["__proto__"]: "good" }, { ["__proto__"]: "bad", opt: 1 }] + ], + [ + Schema.Struct({ + s: Schema.Struct({ ["__proto__"]: Schema.String, opt: Schema.optional(Schema.Number) }).check( + Schema.makeFilter((s) => Object.hasOwn(s, "__proto__") && s["__proto__"] === "good") + ) + }), + [{ s: { ["__proto__"]: "good" } }, { s: { ["__proto__"]: "bad" } }] ] ] const verdicts = new Set() From 6524aa2f40e746565e86bf5017452db4c5854e54 Mon Sep 17 00:00:00 2001 From: Effect Bot Date: Mon, 16 Nov 2026 11:56:23 +0000 Subject: [PATCH 81/88] Simplify the table checks, the Table decoder and two unit lookups in the data frame internals --- packages/effect/src/dataframe/Plan.ts | 5 ++++- .../effect/src/dataframe/internal/table.ts | 18 ++++++------------ .../effect/src/dataframe/internal/window.ts | 2 +- 3 files changed, 11 insertions(+), 14 deletions(-) diff --git a/packages/effect/src/dataframe/Plan.ts b/packages/effect/src/dataframe/Plan.ts index b2a42d0ef78..b846b093f71 100644 --- a/packages/effect/src/dataframe/Plan.ts +++ b/packages/effect/src/dataframe/Plan.ts @@ -1127,7 +1127,10 @@ const tableFromJson: SchemaTransformation.Transformation()( [], - () => (u, _, options) => Effect.as(SchemaParser.decodeUnknownEffect(TableStruct)(u, options), u as Table), + () => { + const decode = SchemaParser.decodeUnknownEffect(TableStruct) + return (u, _, options) => Effect.as(decode(u, options), u as Table) + }, { expected: "Table", // A union picks this member by its tag, as it does a struct. diff --git a/packages/effect/src/dataframe/internal/table.ts b/packages/effect/src/dataframe/internal/table.ts index 7446faa2d14..7dfa8c37c6a 100644 --- a/packages/effect/src/dataframe/internal/table.ts +++ b/packages/effect/src/dataframe/internal/table.ts @@ -19,6 +19,7 @@ * @internal */ import * as S from "../../arrow/ArrowSchema.ts" +import * as Values from "../../arrow/internal/values.ts" import * as DateTime from "../../DateTime.ts" import * as Duration from "../../Duration.ts" import type * as Plan from "../Plan.ts" @@ -382,8 +383,7 @@ const exact = (path: string, column: Plan.TableColumn): ((value: unknown) => voi } } case "Duration": { - const [mul, div] = microseconds[column.unit ?? "microsecond"] - const nanos = mul * BigInt(1000) / div + const nanos = Values.nanosPerUnit[column.unit ?? "microsecond"] return nanos === BigInt(1) ? undefined : (value) => { if (Duration.isDuration(value) && Duration.toNanosUnsafe(value) % nanos !== BigInt(0)) { inexact(path, arrowType(column, path)) @@ -478,18 +478,12 @@ const check = (path: string, v: V.Vector, expected: Plan.TableColumn, rows: Uint /** * Fails unless the values of the columns that `checks` names pass their * test, and the rows the row test. - * - * @internal */ -export const runChecks = ( - checks: Plan.TableChecks, - values: (name: string) => ReadonlyArray, - rows: () => ReadonlyArray -): void => { +const runChecks = (checks: Plan.TableChecks, frame: F.Frame): void => { for (const [name, test] of checks.columns) { - if (!values(name).every(test)) V.fail(`column ${JSON.stringify(name)} has values its schema rejects`) + if (!F.toArray(frame, name).every(test)) V.fail(`column ${JSON.stringify(name)} has values its schema rejects`) } - if (checks.rows !== undefined && !rows().every(checks.rows)) V.fail("rows have values their schema rejects") + if (checks.rows !== undefined && !F.toRows(frame).every(checks.rows)) V.fail("rows have values their schema rejects") } /** @@ -517,6 +511,6 @@ export const conform = ( }), frame.length ) - if (checks !== undefined) runChecks(checks, (name) => F.toArray(narrowed, name), () => F.toRows(narrowed)) + if (checks !== undefined) runChecks(checks, narrowed) return narrowed } diff --git a/packages/effect/src/dataframe/internal/window.ts b/packages/effect/src/dataframe/internal/window.ts index 6081c89adb7..cacd3c03dd0 100644 --- a/packages/effect/src/dataframe/internal/window.ts +++ b/packages/effect/src/dataframe/internal/window.ts @@ -31,9 +31,9 @@ const validOf = (v: V.Vector): Uint8Array | undefined => export const cumSum = (v: V.Vector, p: Partition): V.Vector => { const n = v.length const ids = p.ids + if (D.isPointInTime(v.type)) return V.fail(`cumSum is not defined for ${D.describe(v.type)}`) const type = D.sumType(v.type) const valid = validOf(v) - if (D.isPointInTime(v.type)) return V.fail(`cumSum is not defined for ${D.describe(v.type)}`) switch (v.kind) { case "null": return V.nulls(n) From 283ec06fc54f805804877f28d43e5286330962f5 Mon Sep 17 00:00:00 2001 From: Effect Bot Date: Mon, 16 Nov 2026 12:00:36 +0000 Subject: [PATCH 82/88] Shorten and clarify data frame comments, docs and the changeset --- .changeset/dataframe.md | 2 +- packages/effect/src/dataframe/DataFrame.ts | 6 ++---- packages/effect/src/dataframe/Plan.ts | 20 +++++++++---------- .../src/dataframe/internal/aggregate.ts | 4 ++-- .../effect/src/dataframe/internal/schema.ts | 18 ++++++++--------- .../effect/src/dataframe/internal/table.ts | 3 ++- packages/effect/test/dataframe/README.md | 6 +++++- 7 files changed, 30 insertions(+), 29 deletions(-) diff --git a/.changeset/dataframe.md b/.changeset/dataframe.md index da9ea19c56c..bc332451c31 100644 --- a/.changeset/dataframe.md +++ b/.changeset/dataframe.md @@ -2,4 +2,4 @@ "effect": patch --- -Add `effect/dataframe`: typed, columnar data frames on `effect/arrow` with lazy, type-checked queries that run in memory or on PostgreSQL. Tables are declared by an Effect Schema, whose checks run and whose transformations are rejected, or by an Arrow schema. +Add the experimental `effect/dataframe` modules for typed, columnar data frames on `effect/arrow`, with lazy, type-checked queries that run in memory or on PostgreSQL. Tables are declared by an Effect Schema or an Arrow schema. diff --git a/packages/effect/src/dataframe/DataFrame.ts b/packages/effect/src/dataframe/DataFrame.ts index 5337d4f744b..083ddaca33d 100644 --- a/packages/effect/src/dataframe/DataFrame.ts +++ b/packages/effect/src/dataframe/DataFrame.ts @@ -204,9 +204,8 @@ export const isDataFrame = (u: unknown): u is DataFrame => F.isFrame(u) * `Duration` finer than its column's unit fails, and so do a `DateTime` * beyond the range of valid dates, a `BigDecimal` with more digits than * its column's scale or precision, and a number an integer column can't - * hold. A - * `Float32` override is the exception: it rounds each number to the - * nearest 32-bit float, and the schema's checks test the rounded number. + * hold. A `Float32` override is the exception: it rounds each number to + * the nearest 32-bit float, and the schema's checks test the rounded number. * * Missing values read back as `null` (see {@link Normalize}). An override * must hold its field's values, as {@link validate} checks columns: a @@ -246,7 +245,6 @@ export const fromRows = ( const input: ReadonlyArray = Array.isArray(rows) ? rows : Array.from(rows) const table = Result.getOrThrowWith(ArrowTable.fromRows(T.toArrowSchema(columns), input), fromDefect) T.checkExact(columns, input) - // The frame is checked as validate checks one, so it holds what its declaration accepts. return wrap(T.conform(F.fromArrowTable(table), columns, checks)) }) diff --git a/packages/effect/src/dataframe/Plan.ts b/packages/effect/src/dataframe/Plan.ts index b846b093f71..7ccb2aa6e09 100644 --- a/packages/effect/src/dataframe/Plan.ts +++ b/packages/effect/src/dataframe/Plan.ts @@ -760,10 +760,8 @@ export interface TableChecks { * * **Gotchas** * - * `checks` has no JSON form. {@link NodeFromJson} fails to encode a table - * that has them, naming the table, and fails to decode JSON with a `checks` - * key. {@link withoutChecks} removes them: the plan then encodes, and checks - * column types only. + * `checks` has no JSON form. See {@link NodeFromJson} and + * {@link withoutChecks}. * * @stability experimental * @category models @@ -1197,11 +1195,12 @@ export const Node: Schema.Codec = NodeSchema * **Gotchas** * * A table's `checks` are functions and have no JSON form. Encoding a plan - * fails for a table that has them instead of leaving them out, and so does - * decoding JSON with a `checks` key (the plan as `JSON.stringify` writes - * it). Encode {@link withoutChecks} of the plan to send it without them. A - * decoded plan never has checks: to test its tables' rows, build the query - * from `Query.table` with the schema where the plan is decoded. + * fails for a table that has them, naming the table, instead of leaving + * them out, and so does decoding JSON with a `checks` key (the plan as + * `JSON.stringify` writes it). Encode {@link withoutChecks} of the plan to + * send it without them. A decoded plan never has checks: to test its + * tables' rows, build the query from `Query.table` with the schema where + * the plan is decoded. * * @stability experimental * @category schemas @@ -1228,8 +1227,7 @@ export const NodeFromJson: Schema.Codec = Schema.toCodecJson( * **Gotchas** * * The result accepts values the schema rejects, here and after decoding. - * Checks can't be attached to a decoded plan: to test the rows again, build - * the query from `Query.table` with the schema where the plan is decoded. + * {@link NodeFromJson} says how to test the rows of a decoded plan. * * **Example** (Encoding a plan over a table with refinements) * diff --git a/packages/effect/src/dataframe/internal/aggregate.ts b/packages/effect/src/dataframe/internal/aggregate.ts index bf295c5f3da..e7d74692d70 100644 --- a/packages/effect/src/dataframe/internal/aggregate.ts +++ b/packages/effect/src/dataframe/internal/aggregate.ts @@ -185,8 +185,8 @@ export const sumColumn = (inputType: S.DataType, sums: Sums): V.Vector => { } /** - * Every value as a float, with validity. Durations and points in time are - * not numbers: their integers count units, which a mean would lose. + * Every value as a float, with validity. Fails for durations and points in + * time: their integers count units, which a mean would lose. */ const asFloat = (v: V.Vector): { readonly x: Float64Array; readonly valid: Uint8Array | undefined } => { if (T.isTemporal(v.type)) { diff --git a/packages/effect/src/dataframe/internal/schema.ts b/packages/effect/src/dataframe/internal/schema.ts index da9fa5e8078..c88321a1747 100644 --- a/packages/effect/src/dataframe/internal/schema.ts +++ b/packages/effect/src/dataframe/internal/schema.ts @@ -180,14 +180,14 @@ interface Shape { const written: { readonly [M in Missing]: string } = { null: "null", undefined: "undefined", absent: "an absent key" } /** - * The shape of the values of a schema, the type of an `optional` key or - * not. `seen` says that a check around them is handed these values (one of - * a struct, an array or a union they are in). + * The shape of the values of `ast`. `optional` says that `ast` is the type + * of an optional key. `seen` says that a check of a struct, array or union + * around the values is handed them. * * A check must be handed what the schema describes, so a missing value it - * sees needs one way of being written. An optional field's is an absent - * key, whether or not its type has `undefined` too. It fails when the - * schema writes it as `null` and another way. + * sees needs one way of being written. For an optional field it is an + * absent key, whether or not its type has `undefined` too. Fails when a + * check sees a missing value the schema writes as `null` and another way. */ const shapeOf = (ast: AST.AST, path: string, seen: boolean, optional = false): Shape => { const ways = new Set(optional ? ["absent"] : []) @@ -244,8 +244,8 @@ const shapeOf = (ast: AST.AST, path: string, seen: boolean, optional = false): S } /** - * The values a frame stores for `shape`, as the schema writes them, or - * `undefined` when it stores them that way. + * The function that turns a value a frame stores for `shape` into the form + * the schema writes. `undefined` when the frame stores that form already. */ const restorer = (shape: Shape): ((value: unknown) => unknown) | undefined => { const present = shape.item !== undefined @@ -332,7 +332,7 @@ const checksOf = (ast: AST.Objects): Plan.TableChecks | undefined => { const absent = shape.missing === "absent" return [[name, (value: unknown) => (absent && value === null) || is(restore(value))] as const] }) - // With checks of rows, every field has its shape. + // When the struct has checks, `shapes` holds every field. const restore = checks && struct(shapes) const rows = checks && ((row: unknown) => Result.isSuccess(AST.runChecks(checks, restore === undefined ? row : restore(row)))) diff --git a/packages/effect/src/dataframe/internal/table.ts b/packages/effect/src/dataframe/internal/table.ts index 7dfa8c37c6a..d743382b7fd 100644 --- a/packages/effect/src/dataframe/internal/table.ts +++ b/packages/effect/src/dataframe/internal/table.ts @@ -296,7 +296,8 @@ const checkRange = (path: string, v: V.I64, unit: S.TimeUnit, rows: Uint8Array | /** * Fails for a decimal with more digits than its column's precision. A - * column holds one all the same, and a conversion to its own type finds none. + * column can hold one (IPC decoding does not check stored values), and + * `convert` returns a column that has the target type as it is. */ const checkPrecision = (path: string, v: V.I64, type: S.Decimal, rows: Uint8Array | undefined): void => { const limit = BigInt(10) ** BigInt(type.precision) diff --git a/packages/effect/test/dataframe/README.md b/packages/effect/test/dataframe/README.md index ce81a731a93..c7259942b1e 100644 --- a/packages/effect/test/dataframe/README.md +++ b/packages/effect/test/dataframe/README.md @@ -130,7 +130,11 @@ DF-C022 to DF-C032 and DF-Q043 to DF-Q047 assert behaviour added after the basel Rejected with a `DataFrameError` that names the field: transformations (any schema with an encoding, such as `NumberFromString`, `decodeTo`, decoding defaults and renamed keys; `Schema.toType` gives the decoded side), `DateTimeZoned`, tuples, records, classes, unions of different column types, and recursive schemas. -Refinements, literals, enums and template literals are value checks: every value of such a field is read and tested, inside structs, arrays and union members too. A struct's own checks are row checks: every row is read and tested. A field with a value check accepts the stored values that `Schema.is` accepts for it (DF-C032): a check on a nullable field runs on its nulls too, a check on a union member runs on that member's values, and a union in `oneOf` mode takes a value of exactly one member. Fields without checks are checked by column type only, on the typed columns. `DataFrame.fromRows` builds its frame and then checks it as `DataFrame.validate` does, so all of them test what the frame stores: a `Float32` column's check sees the rounded number. `DataFrame.fromRows` rejects a `DateTime` or `Duration` finer than its column's unit instead of rounding it, and a decimal column holds no more digits than its precision, whichever of them checks it (DF-C029). The checks are compiled when the schema is read (`Plan.TableChecks`). They are functions and have no JSON form: `Plan.NodeFromJson` fails to encode a plan over a table that has them, naming the table, and fails to decode JSON that carries a `checks` key. `Plan.withoutChecks` removes them explicitly; the plan then encodes and checks column types only. A decoded plan gets its checks back by declaring the table again with `Query.table`. `Plan.Node` validates a plan as it is and keeps its checks. A SQL engine tests value checks on distinct values and fails for a table with row checks. +Refinements, literals, enums and template literals are value checks: every value of such a field is read and tested, inside structs, arrays and union members too. A struct's own checks are row checks: every row is read and tested. A field with a value check accepts the stored values that `Schema.is` accepts for it (DF-C032): a check on a nullable field runs on its nulls too, a check on a union member runs on that member's values, and a union in `oneOf` mode takes a value of exactly one member. Fields without checks are checked by column type only, on the typed columns. + +`DataFrame.fromRows` builds its frame and then checks it as `DataFrame.validate` does, so all of them test what the frame stores: a `Float32` column's check sees the rounded number. `DataFrame.fromRows` rejects a `DateTime` or `Duration` finer than its column's unit instead of rounding it, and a decimal column holds no more digits than its precision, whichever of them checks it (DF-C029). + +The checks are compiled when the schema is read (`Plan.TableChecks`). They are functions and have no JSON form: `Plan.NodeFromJson` fails to encode a plan over a table that has them, naming the table, and fails to decode JSON that carries a `checks` key. `Plan.withoutChecks` removes them explicitly; the plan then encodes and checks column types only. A decoded plan gets its checks back by declaring the table again with `Query.table`. `Plan.Node` validates a plan as it is and keeps its checks. A SQL engine tests value checks on distinct values and fails for a table with row checks. Checks are handed stored values as the schema writes them (DF-C031), in `DataFrame.fromRows`, `DataFrame.empty`, `DataFrame.validate` and `Query.table` alike. A frame holds `null` for every missing value, and a check sees an absent key for an optional field, `undefined` for `UndefinedOr` and `null` for `NullOr`. The rows `{}` and `{ a: undefined }` are one stored row, and a check sees an absent key for both. From ced3afd8d6266b1d409bca3a8a55b01d764f0361 Mon Sep 17 00:00:00 2001 From: Effect Bot Date: Mon, 16 Nov 2026 12:33:51 +0000 Subject: [PATCH 83/88] Reject a declared decimal precision or scale that a 128-bit decimal can't have before values are converted --- packages/effect/src/dataframe/DataFrame.ts | 5 ++- packages/effect/src/dataframe/Plan.ts | 5 ++- .../effect/src/dataframe/internal/table.ts | 30 ++++++++++--- .../effect/test/dataframe/DataFrame.test.ts | 4 ++ packages/effect/test/dataframe/Query.test.ts | 43 +++++++++++++++++++ packages/effect/test/dataframe/README.md | 3 +- 6 files changed, 78 insertions(+), 12 deletions(-) diff --git a/packages/effect/src/dataframe/DataFrame.ts b/packages/effect/src/dataframe/DataFrame.ts index 083ddaca33d..1192dc87f14 100644 --- a/packages/effect/src/dataframe/DataFrame.ts +++ b/packages/effect/src/dataframe/DataFrame.ts @@ -446,8 +446,9 @@ export const empty = ( * untyped rows, and no values are tested beyond their column type. The * fields can be `Null` (nullable), `Bool`, integers of any width, `Float32`, * `Float64`, `Utf8`, `Binary`, `Date` in days, `Time` in microseconds, - * `Timestamp` and `Duration` in any unit, 128-bit `Decimal`, and `List` - * and `Struct` of these. Other Arrow types fail: frames can't hold some + * `Timestamp` and `Duration` in any unit, 128-bit `Decimal` (a precision + * from 1 to 38 and a 32-bit scale, or it fails), and `List` and `Struct` of + * these. Other Arrow types fail: frames can't hold some * (`Float16`, intervals, maps, unions) and hold others as one of the types * above, which is the type to declare (`Utf8` for `LargeUtf8` and * `Utf8View`, `List` for `LargeList`, the value type for a dictionary). A diff --git a/packages/effect/src/dataframe/Plan.ts b/packages/effect/src/dataframe/Plan.ts index 7ccb2aa6e09..4c4572cf91c 100644 --- a/packages/effect/src/dataframe/Plan.ts +++ b/packages/effect/src/dataframe/Plan.ts @@ -675,8 +675,9 @@ export const TypeName = Schema.Literals([ * * **Gotchas** * - * A `Decimal` column without `precision` and `scale` fails when its table - * is read. + * A `Decimal` column fails when its table is read unless it has a + * `precision` that is an integer from 1 to 38 and a `scale` that is a + * 32-bit integer. * * @stability experimental * @category models diff --git a/packages/effect/src/dataframe/internal/table.ts b/packages/effect/src/dataframe/internal/table.ts index d743382b7fd..49f422453b8 100644 --- a/packages/effect/src/dataframe/internal/table.ts +++ b/packages/effect/src/dataframe/internal/table.ts @@ -19,6 +19,7 @@ * @internal */ import * as S from "../../arrow/ArrowSchema.ts" +import * as Layout from "../../arrow/internal/layout.ts" import * as Values from "../../arrow/internal/values.ts" import * as DateTime from "../../DateTime.ts" import * as Duration from "../../Duration.ts" @@ -110,6 +111,17 @@ const compatible = (t: S.DataType, expected: Column): boolean => { const missing = (path: string): never => V.fail(`missing column ${JSON.stringify(path)}`) +/** + * The 128-bit decimal type of a declared precision and scale, failing unless + * Arrow allows them (`Layout.checkDecimal`). Declarations and plans give + * them unchecked, and converting values computes with them. + */ +const decimalType = (path: string, precision: number, scale: number): S.Decimal => { + const type = S.decimal(precision, scale) + Layout.checkDecimal(type, (message) => V.fail(`column ${JSON.stringify(path)}: ${message}`)) + return type +} + /** The column declared for a field's `column` with Arrow type `t`, or its default type. @internal */ export const declareColumn = (path: string, column: Plan.TableColumn, t: S.DataType | undefined): Plan.TableColumn => { if (t !== undefined) checkType(path, t, column) @@ -127,8 +139,11 @@ export const declareColumn = (path: string, column: Plan.TableColumn, t: S.DataT return declareColumn(p, f, g) }) } - case "Decimal": - return t?._tag === "Decimal" ? { ...column, precision: t.precision, scale: t.scale } : column + case "Decimal": { + if (t?._tag !== "Decimal") return column + const { precision, scale } = decimalType(path, t.precision, t.scale) + return { ...column, precision, scale } + } case "Timestamp": case "Duration": return { ...column, unit: t?._tag === column.type ? t.unit : "microsecond" } @@ -183,10 +198,11 @@ const fieldColumn = (path: string, f: S.Field): Plan.TableColumn => { return { ...column, type: "Timestamp", unit: t.unit } case "Duration": return { ...column, type: "Duration", unit: t.unit } - case "Decimal": - return t.bitWidth === 128 - ? { ...column, type: "Decimal", precision: t.precision, scale: t.scale } - : undeclarable(path, `${t.bitWidth}-bit decimals`, "declared decimals are 128-bit") + case "Decimal": { + if (t.bitWidth !== 128) return undeclarable(path, `${t.bitWidth}-bit decimals`, "declared decimals are 128-bit") + const { precision, scale } = decimalType(path, t.precision, t.scale) + return { ...column, type: "Decimal", precision, scale } + } case "List": return { ...column, type: "List", fields: [fieldColumn(`${path}[]`, { ...t.child, name: "item" })] } case "Struct": @@ -249,7 +265,7 @@ export const arrowType = (column: Plan.TableColumn, path: string = column.name): JSON.stringify(path) }: decimal columns need a precision and scale: pass options.types (for example ArrowSchema.decimal(18, 2))` ) - : S.decimal(column.precision, column.scale) + : decimalType(path, column.precision, column.scale) case "Float16": return S.float16 default: diff --git a/packages/effect/test/dataframe/DataFrame.test.ts b/packages/effect/test/dataframe/DataFrame.test.ts index 3dd288bac96..1be4ec15271 100644 --- a/packages/effect/test/dataframe/DataFrame.test.ts +++ b/packages/effect/test/dataframe/DataFrame.test.ts @@ -711,6 +711,10 @@ describe("DataFrame", () => { ArrowSchema.decimal(18, 2, 64), `column "c": 64-bit decimals can't be declared: declared decimals are 128-bit` ], + [ + ArrowSchema.struct([field("x", ArrowSchema.decimal(39, 0))]), + `column "c.x": invalid decimal128 precision 39, expected an integer in 1..38` + ], [ ArrowSchema.date("millisecond"), `column "c": dates in milliseconds can't be declared: declared dates are in days` diff --git a/packages/effect/test/dataframe/Query.test.ts b/packages/effect/test/dataframe/Query.test.ts index 28c1dec7600..e59a247061a 100644 --- a/packages/effect/test/dataframe/Query.test.ts +++ b/packages/effect/test/dataframe/Query.test.ts @@ -306,6 +306,49 @@ describe("Query", () => { ) }) + // DF-Q048 (API contract): Query.table; pinned source in README.md. + it.effect("rejects a decimal precision or scale that a 128-bit decimal can't have", () => + Effect.gen(function*() { + const precision = (p: number) => `column "d": invalid decimal128 precision ${p}, expected an integer in 1..38` + const scale = (s: number) => `column "d": invalid decimal scale ${s}, expected a 32-bit integer` + const source = Catalog.layer({ t: price("7", 2, 0) }) + const native = (type: ArrowSchema.DataType) => + Query.table("t", ArrowSchema.make([ArrowSchema.field("d", type, { nullable: false })])) + const override = (type: ArrowSchema.DataType) => Query.table("t", Price, { types: { d: type } }) + const plan = (precision: number, scale: number): Plan.Node => ({ + _tag: "Table", + name: "t", + columns: [{ name: "d", type: "Decimal", nullable: false, precision, scale }] + }) + // The plan schema takes any integer: a decoded plan carries those, a hand-made one the others. + const cases: ReadonlyArray = [ + [2.5, 0, precision(2.5), false], + [0, 0, precision(0), true], + [39, 0, precision(39), true], + [2, 0.5, scale(0.5), false], + [2, 2 ** 31, scale(2 ** 31), true] + ] + for (const [p, s, message, decodes] of cases) { + const type = ArrowSchema.decimal(p, s) + assert.throws(() => native(type), DataFrame.DataFrameError, message) + assert.throws(() => override(type), DataFrame.DataFrameError, message) + const decoded = Schema.decodeUnknownResult(Plan.Node)(plan(p, s)) + assert.strictEqual(Result.isSuccess(decoded), decodes) + const query = Query.fromPlan(Result.getOrElse(decoded, () => plan(p, s))) + const error = yield* Effect.flip(query.pipe(Effect.provide(source))) + assert.instanceOf(error, DataFrame.DataFrameError) + assert.strictEqual(error.message, message) + const streamed = Query.stream(query).pipe(Stream.runCollect, Effect.provide(source)) + assert.strictEqual((yield* Effect.flip(streamed)).message, message) + } + const valid = ArrowSchema.decimal(38, 0) + const decoded = Query.fromPlan(Schema.decodeUnknownSync(Plan.Node)(plan(38, 0))) + for (const query of [native(valid), override(valid), decoded]) { + const df = yield* query.pipe(Effect.provide(source)) + assert.deepStrictEqual(DataFrame.toRows(df), [{ d: BigDecimal.make(7n, 0) }]) + } + })) + // DF-Q008 (API contract): DataFrame.DataFrameError; pinned source in README.md. it.effect("fails with DataFrameError on invalid expressions", () => Effect.gen(function*() { diff --git a/packages/effect/test/dataframe/README.md b/packages/effect/test/dataframe/README.md index c7259942b1e..229c817bc3a 100644 --- a/packages/effect/test/dataframe/README.md +++ b/packages/effect/test/dataframe/README.md @@ -105,8 +105,9 @@ DF-C021 checks a single, unsliced, non-null batch of Int32 and Float64 columns: | DF-Q045 | C | fails when a table is declared by a schema that has no columns for it | [Query.table](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Query.ts#L254) | | DF-Q046 | C | sums durations to durations in the column's unit | [Expr.sum](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Expr.ts#L827) | | DF-Q047 | C | rejects aggregates that durations, timestamps, dates and times don't have | [Expr.mean](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Expr.ts#L838) | +| DF-Q048 | C | rejects a decimal precision or scale that a 128-bit decimal can't have | [Query.table](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Query.ts#L254) | -DF-C022 to DF-C032 and DF-Q043 to DF-Q047 assert behaviour added after the baseline: checks compiled from the schema and refused by the JSON form of a plan, rejected transformations, tables declared by an Arrow schema, sums of durations, aggregates rejected for durations and points in time, `DataFrame.column`, which has no symbol at the baseline and links to the layouts it exposes, and the checks `DataFrame.fromRows` runs on the frame it built. Their links locate the symbol at the baseline; the documentation of that symbol in the checked-out source is their contract. DF-C024 observes which value constructors run (`DateTime`, `Duration`, `BigDecimal`) and covers only `DataFrame.fromRows` on rows that hold such values already, and a frame whose columns already have the declared types. +DF-C022 to DF-C032 and DF-Q043 to DF-Q048 assert behaviour added after the baseline: checks compiled from the schema and refused by the JSON form of a plan, rejected transformations, tables declared by an Arrow schema, sums of durations, aggregates rejected for durations and points in time, the precision and scale of declared decimals, `DataFrame.column`, which has no symbol at the baseline and links to the layouts it exposes, and the checks `DataFrame.fromRows` runs on the frame it built. Their links locate the symbol at the baseline; the documentation of that symbol in the checked-out source is their contract. DF-Q048 covers a table declared by an Arrow schema, by a struct schema with `options.types`, and by a plan (decoded where the plan schema takes the numbers), run and streamed, with a precision of 38 as the passing control. The limits are those of a 128-bit Arrow decimal; the exact messages are local detail. DF-C024 observes which value constructors run (`DateTime`, `Duration`, `BigDecimal`) and covers only `DataFrame.fromRows` on rows that hold such values already, and a frame whose columns already have the declared types. ## Supported Schema subset From dbf60e63c014b13783006512c7250fd2f219579f Mon Sep 17 00:00:00 2001 From: Effect Bot Date: Mon, 16 Nov 2026 13:02:05 +0000 Subject: [PATCH 84/88] Decide decimal scales far apart from the 64-bit storage range and reject decimal widths, cast targets and SQL scales that can't be held --- packages/effect/src/dataframe/DataFrame.ts | 4 +- packages/effect/src/dataframe/Expr.ts | 5 +- packages/effect/src/dataframe/Sql.ts | 1 + .../effect/src/dataframe/internal/decimal.ts | 110 +++++++++--- .../effect/src/dataframe/internal/expr.ts | 5 +- packages/effect/src/dataframe/internal/sql.ts | 18 +- .../effect/src/dataframe/internal/table.ts | 14 +- .../effect/src/dataframe/internal/window.ts | 3 +- .../effect/test/dataframe/DataFrame.test.ts | 15 ++ packages/effect/test/dataframe/Query.test.ts | 159 ++++++++++++++++++ packages/effect/test/dataframe/README.md | 9 +- packages/sql/pglite/test/DataFrameSql.test.ts | 36 ++++ 12 files changed, 336 insertions(+), 43 deletions(-) diff --git a/packages/effect/src/dataframe/DataFrame.ts b/packages/effect/src/dataframe/DataFrame.ts index 1192dc87f14..1be386fa3f5 100644 --- a/packages/effect/src/dataframe/DataFrame.ts +++ b/packages/effect/src/dataframe/DataFrame.ts @@ -175,8 +175,8 @@ export const isDataFrame = (u: unknown): u is DataFrame => F.isFrame(u) * are microsecond timestamps and durations, arrays are lists, structs are * structs, and `NullOr`/optional fields are nullable. `options.types` * overrides the Arrow type of individual columns (for example - * `ArrowSchema.int32`); `BigDecimal` fields need one, with the decimal's - * precision and scale. + * `ArrowSchema.int32`); `BigDecimal` fields need one, a 128-bit decimal + * with the precision and scale (decimals of other widths fail). * * **Details** * diff --git a/packages/effect/src/dataframe/Expr.ts b/packages/effect/src/dataframe/Expr.ts index de1ba44a95b..5ba4210399f 100644 --- a/packages/effect/src/dataframe/Expr.ts +++ b/packages/effect/src/dataframe/Expr.ts @@ -692,7 +692,10 @@ export const cast: { /** * Converts to a decimal with `precision` digits, `scale` of them after the * point. Values that don't fit (more than `precision` digits, or outside - * 64-bit decimal storage) become null; extra digits round half to even. + * 64-bit decimal storage) become null; extra digits round half to even. A + * negative scale rounds before the point: at scale `-2`, `1250` is `1200`. + * The query fails for a precision outside 1 to 38 or a scale that isn't a + * 32-bit integer. * * @stability experimental * @category conversions diff --git a/packages/effect/src/dataframe/Sql.ts b/packages/effect/src/dataframe/Sql.ts index e39b787db15..2947b006bf0 100644 --- a/packages/effect/src/dataframe/Sql.ts +++ b/packages/effect/src/dataframe/Sql.ts @@ -40,6 +40,7 @@ * instead of wrapping, division by zero follows the database, string order * follows its collation, and timestamps and durations keep PostgreSQL's * microseconds (finer timestamps round down, finer durations toward zero). + * Decimals need a scale PostgreSQL's `numeric` takes, from -1000 to 1000. * Pivots, unpivots, `explode` and `unnest` are not supported yet; tables * need declared columns (`Query.table` with a schema), and {@link engine} * checks that tables satisfy them, as the in-memory engine does. diff --git a/packages/effect/src/dataframe/internal/decimal.ts b/packages/effect/src/dataframe/internal/decimal.ts index e673970aa81..9515a5ea992 100644 --- a/packages/effect/src/dataframe/internal/decimal.ts +++ b/packages/effect/src/dataframe/internal/decimal.ts @@ -4,22 +4,40 @@ * (precision 38, scales added for products, the larger scale for sums). * Every computed decimal is stored through `checked` (or `fit`), so a * value outside 64 bits fails instead of wrapping; casts make such values - * null instead. Rescaling down rounds half to even. + * null instead. Rescaling down rounds half to even. Scales are any 32-bit + * integers, negative ones included; see {@link FAR} for scales far apart. * * @internal */ import * as S from "../../arrow/ArrowSchema.ts" import * as V from "./vector.ts" +/** + * From this many digits on, the 64 bits of storage decide a rescale, so no + * larger power of ten is built for one: `10^20` is more than twice any + * 64-bit value (a quotient rounds to zero, see {@link divRound}), and a + * nonzero value times `10^20` is outside 64 bits and stays outside with a + * 64-bit value added (see {@link factor}). + */ +const FAR = 20 + const powers: Array = [] -/** `10^k` as a bigint. */ -const pow10 = (k: number): bigint => (powers[k] ??= BigInt(10) ** BigInt(k)) +/** `10^k` as a bigint, for `k >= 0`. Only the powers up to {@link FAR} are kept. */ +const pow10 = (k: number): bigint => k <= FAR ? (powers[k] ??= BigInt(10) ** BigInt(k)) : BigInt(10) ** BigInt(k) + +/** + * What to multiply by to add `k >= 0` digits: `10^k`, or `10^FAR` from + * {@link FAR} digits on. The product is then zero for zero and outside 64 + * bits otherwise, as it is with `10^k`, and every product is range-checked. + */ +const factor = (k: number): bigint => pow10(Math.min(k, FAR)) const I0 = BigInt(0) const I1 = BigInt(1) const I2 = BigInt(2) const MAX = BigInt("9223372036854775807") const MIN = -BigInt("9223372036854775808") +const UMAX = BigInt("18446744073709551615") /** @internal */ export const isDecimal = (t: S.DataType): t is S.Decimal => t._tag === "Decimal" @@ -61,6 +79,8 @@ export const fit = (type: S.DataType): (x: bigint) => bigint => isDecimal(type) /** Rounds `x / 10^k` half to even. */ const divRound = (x: bigint, k: number): bigint => { if (k === 0) return x + // `10^k` is more than twice `x`, which has fewer than `k` digits. + if (k >= FAR && (inRange(x) || k > x.toString().length)) return I0 const divisor = pow10(k) // Truncated towards zero; the remainder has the sign of `x`. const quotient = x / divisor @@ -70,14 +90,47 @@ const divRound = (x: bigint, k: number): bigint => { return x < I0 ? quotient - I1 : quotient + I1 } -/** `unscaled / 10^scale` as a decimal string, all `scale` digits shown. */ -const format = (unscaled: bigint, scale: number): string => { +/** + * `unscaled / 10^scale` as a decimal string, all `scale` digits shown. A + * negative scale writes its zeros out (`7` at scale `-2` is `700`). + * + * @internal + */ +export const format = (unscaled: bigint, scale: number): string => { const negative = unscaled < I0 - const digits = (negative ? -unscaled : unscaled).toString().padStart(scale + 1, "0") - const text = scale === 0 ? digits : `${digits.slice(0, digits.length - scale)}.${digits.slice(digits.length - scale)}` + const digits = (negative ? -unscaled : unscaled).toString() + if (scale <= 0) return unscaled === I0 ? digits : `${negative ? "-" : ""}${digits}${"0".repeat(-scale)}` + const padded = digits.padStart(scale + 1, "0") + const text = `${padded.slice(0, padded.length - scale)}.${padded.slice(padded.length - scale)}` return negative ? `-${text}` : text } +/** + * {@link format} for every value. The text has as many digits as the scale, + * so it fails where the scale is more than a string can hold. + */ +const formatAll = (v: V.I64, type: S.Decimal): Array => { + try { + return Array.from({ length: v.length }, (_, i) => (V.isValid(v, i) ? format(v.values[i], type.scale) : null)) + } catch (error) { + if (!(error instanceof RangeError)) throw error + return V.fail(`Decimal[${type.precision}, ${type.scale}] values are too long to write as text`) + } +} + +/** + * The integer part of `unscaled / 10^scale`, truncated towards zero. A + * negative scale can take it outside 64 bits (see {@link factor}). + */ +const integerPart = (unscaled: bigint, scale: number): bigint => + scale < 0 ? unscaled * factor(-scale) : scale >= FAR ? I0 : unscaled / pow10(scale) + +/** + * The integer `x` in unscaled units of `scale`: rounded half to even for a + * negative scale, and possibly outside 64 bits otherwise (see {@link factor}). + */ +const toScale = (x: bigint, scale: number): bigint => scale < 0 ? divRound(x, -scale) : x * factor(scale) + const DECIMAL_SYNTAX = /^\s*([+-]?)(\d*)(?:\.(\d*))?\s*$/ /** {@link DECIMAL_SYNTAX} without the spaces around it. */ const PLAIN_SYNTAX = /^([+-]?)(\d*)(?:\.(\d*))?$/ @@ -96,7 +149,7 @@ const parse = ( if (m === null || (m[2] === "" && (m[3] ?? "") === "")) return undefined const fraction = m[3] ?? "" const all = BigInt(`${m[1]}${m[2] || "0"}${fraction}`) - return fraction.length >= scale ? drop(all, fraction.length - scale) : all * pow10(scale - fraction.length) + return fraction.length >= scale ? drop(all, fraction.length - scale) : all * factor(scale - fraction.length) } /** For {@link parse}: a fraction longer than the scale is invalid. */ @@ -144,7 +197,8 @@ export const cast = (v: V.Vector, target: S.DataType): V.Vector | undefined => { if (isDecimal(target)) { const out = new BigInt64Array(n) // At most `precision` digits: |x| < 10^precision. - const limit = target.precision >= 1 ? pow10(target.precision) : I0 + // From `FAR` digits on, the 64 bits of storage are the tighter limit. + const limit = target.precision >= 1 ? factor(target.precision) : I0 const put = (i: number, x: bigint | undefined) => { if (x !== undefined && inRange(x) && x < limit && -x < limit) { out[i] = x @@ -161,17 +215,18 @@ export const cast = (v: V.Vector, target: S.DataType): V.Vector | undefined => { isDecimal(from) ? from.scale >= target.scale ? divRound(x, from.scale - target.scale) - : x * pow10(target.scale - from.scale) - : x * pow10(target.scale) + : x * factor(target.scale - from.scale) + : toScale(x, target.scale) ) break } case "i32": case "bool": - put(i, BigInt(v.values[i]) * pow10(target.scale)) + put(i, toScale(BigInt(v.values[i]), target.scale)) break case "f64": { - const scaled = v.values[i] * 10 ** target.scale + // Zero is zero at any scale, also where `10^scale` is infinite. + const scaled = v.values[i] === 0 ? 0 : v.values[i] * 10 ** target.scale if (Number.isFinite(scaled)) { const floor = Math.floor(scaled) const rest = scaled - floor @@ -188,7 +243,6 @@ export const cast = (v: V.Vector, target: S.DataType): V.Vector | undefined => { } // from a decimal if (v.kind !== "i64" || !isDecimal(from)) return V.fail("expected decimal storage") - const scale = pow10(from.scale) switch (target._tag) { case "Float": return V.f64(target, toFloat(v.values, from.scale), v.valid) @@ -196,8 +250,8 @@ export const cast = (v: V.Vector, target: S.DataType): V.Vector | undefined => { if (target.bitWidth === 64) { const out = new BigInt64Array(n) for (let i = 0; i < n; i++) { - const x = v.values[i] / scale - if (V.isValid(v, i) && (target.signed || x >= I0)) { + const x = integerPart(v.values[i], from.scale) + if (V.isValid(v, i) && (target.signed ? inRange(x) : x >= I0 && x <= UMAX)) { out[i] = x valid[i] = 1 } @@ -208,19 +262,16 @@ export const cast = (v: V.Vector, target: S.DataType): V.Vector | undefined => { const out = new Float64Array(n) for (let i = 0; i < n; i++) { if (!V.isValid(v, i)) continue - const x = Number(v.values[i] / scale) + const x = integerPart(v.values[i], from.scale) if (x >= lo && x <= hi) { - out[i] = x + out[i] = Number(x) valid[i] = 1 } } return V.kindOf(target) === "f64" ? V.f64(target, out, valid) : V.i32(target, Int32Array.from(out), valid) } case "Utf8": - return V.stringsFrom( - S.utf8, - Array.from({ length: n }, (_, i) => (V.isValid(v, i) ? format(v.values[i], from.scale) : null)) - ) + return V.stringsFrom(S.utf8, formatAll(v, from)) case "Bool": return V.bool(Uint8Array.from(v.values, (x) => (x === I0 ? 0 : 1)), v.valid) default: @@ -240,12 +291,14 @@ export const arith = ( const n = a.length const out = new BigInt64Array(n) if (op === "mul") { + const scale = aScale + bScale + if ((scale | 0) !== scale) V.fail(`the scale ${scale} of a decimal product is not a 32-bit integer`) for (let i = 0; i < n; i++) out[i] = checked(a[i] * (scalar ? b[0] : b[i])) - return { values: out, type: decimal(aScale + bScale) } + return { values: out, type: decimal(scale) } } const scale = Math.max(aScale, bScale) - const fa = pow10(scale - aScale) - const fb = pow10(scale - bScale) + const fa = factor(scale - aScale) + const fb = factor(scale - bScale) for (let i = 0; i < n; i++) { const x = a[i] * fa const y = (scalar ? b[0] : b[i]) * fb @@ -263,10 +316,10 @@ export const arith = ( export const round = (v: V.I64, type: S.Decimal, places: number): V.Vector => { if (places >= type.scale) return v const k = type.scale - places - const factor = pow10(k) + const up = factor(k) return V.i64( S.decimal(Math.min(type.precision + 1, PRECISION), type.scale), - BigInt64Array.from(v.values, (x) => checked(divRound(x, k) * factor)), + BigInt64Array.from(v.values, (x) => checked(divRound(x, k) * up)), v.valid ) } @@ -275,6 +328,7 @@ export const round = (v: V.I64, type: S.Decimal, places: number): V.Vector => { export const toFloat = (values: BigInt64Array, scale: number): Float64Array => { const out = new Float64Array(values.length) const unit = 10 ** scale - for (let i = 0; i < values.length; i++) out[i] = Number(values[i]) / unit + // Zero is zero at any scale, also where `10^scale` is zero or infinite. + for (let i = 0; i < values.length; i++) out[i] = values[i] === I0 ? 0 : Number(values[i]) / unit return out } diff --git a/packages/effect/src/dataframe/internal/expr.ts b/packages/effect/src/dataframe/internal/expr.ts index 38f5ede55f5..b2b51aba8a5 100644 --- a/packages/effect/src/dataframe/internal/expr.ts +++ b/packages/effect/src/dataframe/internal/expr.ts @@ -4,6 +4,7 @@ * @internal */ import * as S from "../../arrow/ArrowSchema.ts" +import * as Layout from "../../arrow/internal/layout.ts" import * as BigDecimal from "../../BigDecimal.ts" import * as DateTime from "../../DateTime.ts" import * as Duration from "../../Duration.ts" @@ -394,7 +395,9 @@ const evaluateCall = (node: Plan.Call, ctx: Context): K.Value => { case "cast": { const to = option("to") if (to === "Decimal") { - return K.castValue(arg(0), S.decimal(option("precision"), option("scale"))) + const type = S.decimal(option("precision"), option("scale")) + Layout.checkDecimal(type, (message) => V.fail(`cast: ${message}`)) + return K.castValue(arg(0), type) } const target = D.castTargets[to] if (target === undefined) return V.fail(`unknown cast target ${to}`) diff --git a/packages/effect/src/dataframe/internal/sql.ts b/packages/effect/src/dataframe/internal/sql.ts index 17e0b4ae85d..d2dab62ca43 100644 --- a/packages/effect/src/dataframe/internal/sql.ts +++ b/packages/effect/src/dataframe/internal/sql.ts @@ -53,6 +53,13 @@ const list = (names: ReadonlyArray) => names.map(quote).join(", ") const unsupported = (what: string): never => V.fail(`${what} is not supported by the SQL engine`) +/** + * A decimal scale PostgreSQL's `numeric` can declare (-1000 to 1000). The + * SQL for a decimal writes out as many digits as its scale. + */ +const numericScale = (scale: number): number => + scale >= -1000 && scale <= 1000 ? scale : unsupported(`a decimal scale of ${scale}`) + // ----------------------------------------------------------------------------- // Types and literals // ----------------------------------------------------------------------------- @@ -82,7 +89,7 @@ const sqlType = (t: S.DataType): string => { case "Duration": return "INTERVAL" case "Decimal": - return `DECIMAL(${t.precision}, ${t.scale})` + return `DECIMAL(${t.precision}, ${numericScale(t.scale)})` case "Null": return "INTEGER" default: @@ -134,10 +141,8 @@ const literal = (value: unknown, type: Plan.LiteralType | undefined): string => case "Duration": return fromInteger(`CAST(${big} AS BIGINT)`, S.duration(type.unit)) case "Decimal": { - const negative = big < BigInt(0) - const digits = (negative ? -big : big).toString().padStart(type.scale + 1, "0") - const text = type.scale === 0 ? digits : `${digits.slice(0, -type.scale)}.${digits.slice(-type.scale)}` - return `CAST('${negative ? "-" : ""}${text}' AS DECIMAL(${type.precision}, ${type.scale}))` + const scale = numericScale(type.scale) + return `CAST('${Dec.format(big, scale)}' AS DECIMAL(${type.precision}, ${scale}))` } } } @@ -996,7 +1001,8 @@ const tableTypes: { Decimal: { accepts: ["numeric"], conditions: (x, type) => { - const { precision, scale } = type as S.Decimal + const { precision } = type as S.Decimal + const scale = numericScale((type as S.Decimal).scale) return [{ holds: `ROUND(${x}, ${scale}) = ${x} AND ABS(${x}) < ${power(precision - scale)} AND ${x} * ${ power(scale) diff --git a/packages/effect/src/dataframe/internal/table.ts b/packages/effect/src/dataframe/internal/table.ts index 49f422453b8..c25760d668d 100644 --- a/packages/effect/src/dataframe/internal/table.ts +++ b/packages/effect/src/dataframe/internal/table.ts @@ -122,6 +122,15 @@ const decimalType = (path: string, precision: number, scale: number): S.Decimal return type } +/** + * The decimal type an Arrow decimal declares. Declared decimals are 128-bit, + * so another width fails instead of being declared as one. + */ +const declaredDecimal = (path: string, t: S.Decimal): S.Decimal => + t.bitWidth === 128 + ? decimalType(path, t.precision, t.scale) + : undeclarable(path, `${t.bitWidth}-bit decimals`, "declared decimals are 128-bit") + /** The column declared for a field's `column` with Arrow type `t`, or its default type. @internal */ export const declareColumn = (path: string, column: Plan.TableColumn, t: S.DataType | undefined): Plan.TableColumn => { if (t !== undefined) checkType(path, t, column) @@ -141,7 +150,7 @@ export const declareColumn = (path: string, column: Plan.TableColumn, t: S.DataT } case "Decimal": { if (t?._tag !== "Decimal") return column - const { precision, scale } = decimalType(path, t.precision, t.scale) + const { precision, scale } = declaredDecimal(path, t) return { ...column, precision, scale } } case "Timestamp": @@ -199,8 +208,7 @@ const fieldColumn = (path: string, f: S.Field): Plan.TableColumn => { case "Duration": return { ...column, type: "Duration", unit: t.unit } case "Decimal": { - if (t.bitWidth !== 128) return undeclarable(path, `${t.bitWidth}-bit decimals`, "declared decimals are 128-bit") - const { precision, scale } = decimalType(path, t.precision, t.scale) + const { precision, scale } = declaredDecimal(path, t) return { ...column, type: "Decimal", precision, scale } } case "List": diff --git a/packages/effect/src/dataframe/internal/window.ts b/packages/effect/src/dataframe/internal/window.ts index cacd3c03dd0..736bbb22e44 100644 --- a/packages/effect/src/dataframe/internal/window.ts +++ b/packages/effect/src/dataframe/internal/window.ts @@ -244,7 +244,8 @@ export const rolling = ( } if (count < minPeriods) return ok[row] = 1 - if (fn === "rollingMean") means[row] = Number(sum) / unit / count + // Zero is zero at any scale, also where `10^scale` is zero. + if (fn === "rollingMean") means[row] = sum === BigInt(0) ? 0 : Number(sum) / unit / count else sums[row] = fit(sum) }) return fn === "rollingMean" ? V.f64(type, means, ok) : V.i64(type, sums, ok) diff --git a/packages/effect/test/dataframe/DataFrame.test.ts b/packages/effect/test/dataframe/DataFrame.test.ts index 1be4ec15271..4f1acf2f2b9 100644 --- a/packages/effect/test/dataframe/DataFrame.test.ts +++ b/packages/effect/test/dataframe/DataFrame.test.ts @@ -86,6 +86,21 @@ describe("DataFrame", () => { ) }) + // DF-C033 (API contract): DataFrame.fromRows; pinned source in README.md. + it("rejects decimal overrides that aren't 128-bit", () => { + const Price = Schema.Struct({ n: Schema.BigDecimal }) + const df = DataFrame.fromRows(Price, [{ n: BigDecimal.make(7n, 0) }], { types: { n: ArrowSchema.decimal(9, 0) } }) + for (const width of [32, 64, 256] as const) { + const types = { n: ArrowSchema.decimal(9, 0, width) } + const message = `column "n": ${width}-bit decimals can't be declared: declared decimals are 128-bit` + assert.throws(() => DataFrame.fromRows(Price, [], { types }), DataFrame.DataFrameError, message) + assert.throws(() => DataFrame.empty(Price, { types }), DataFrame.DataFrameError, message) + assert.throws(() => Query.table("t", Price, { types }), DataFrame.DataFrameError, message) + const result = DataFrame.validate(df, Price, { types }) + assert.strictEqual(Result.isFailure(result) && result.failure.message, message) + } + }) + // DF-C005 (API contract): DataFrame.fromRows; pinned source in README.md. it("fromRows rejects schemas without a column type", () => { interface Tree { diff --git a/packages/effect/test/dataframe/Query.test.ts b/packages/effect/test/dataframe/Query.test.ts index e59a247061a..d58635e76bf 100644 --- a/packages/effect/test/dataframe/Query.test.ts +++ b/packages/effect/test/dataframe/Query.test.ts @@ -349,6 +349,165 @@ describe("Query", () => { } })) + // DF-Q049 (API contract): Query.table, Expr.castDecimal; pinned source in README.md. + it.effect("decides decimal scales far apart from the 64 bits of storage", () => + Effect.gen(function*() { + const source = Catalog.layer({ t: price("7", 2, 0), zero: price("0", 2, 0) }) + const declared = (name: string, scale: number) => + Query.table( + name, + ArrowSchema.make([ArrowSchema.field("d", ArrowSchema.decimal(38, scale), { nullable: false })]) + ) + const cast = (name: string, scale: number) => + declared(name, 0).pipe(Query.select((c) => ({ d: Expr.castDecimal(c.d, 38, scale) })), rows) + for (const scale of [2147483647, -2147483648, 10_000_000, 20, -20]) { + // The exact conversion of a declaration fails, the explicit cast gives null or rounds. + const inexact = `column "d" has values that Decimal[38, ${scale}] can't hold exactly` + const error = yield* Effect.flip(declared("t", scale).pipe(Effect.provide(source))) + assert.instanceOf(error, DataFrame.DataFrameError) + assert.strictEqual(error.message, inexact) + const streamed = Query.stream(declared("t", scale)).pipe(Stream.runCollect, Effect.provide(source)) + assert.strictEqual((yield* Effect.flip(streamed)).message, inexact) + assert.deepStrictEqual(yield* cast("t", scale).pipe(Effect.provide(source)), [ + { d: scale > 0 ? null : BigDecimal.make(0n, scale) } + ]) + // Zero is zero at every scale. + const zero = [{ d: BigDecimal.make(0n, scale) }] + assert.deepStrictEqual(yield* rows(declared("zero", scale)).pipe(Effect.provide(source)), zero) + assert.deepStrictEqual(yield* cast("zero", scale).pipe(Effect.provide(source)), zero) + } + // Up to 19 digits apart, values are rescaled as before: exactly, or half to even. + assert.deepStrictEqual(yield* rows(declared("t", 18)).pipe(Effect.provide(source)), [ + { d: BigDecimal.make(7_000_000_000_000_000_000n, 18) } + ]) + const halves = Query.from(DataFrame.fromColumns({ s: ["0.5", "1.5", "-2.5", "0.5000000000000000001"] })).pipe( + Query.select((c) => ({ d: Expr.castDecimal(c.s, 38, 19) })), + Query.select((c) => ({ d: Expr.castDecimal(c.d, 38, 0) })) + ) + assert.deepStrictEqual(yield* rows(halves), [ + { d: BigDecimal.make(0n, 0) }, + { d: null }, + { d: null }, + { d: BigDecimal.make(1n, 0) } + ]) + // Values held at the far scales: 7 and 0 units of 10^-2147483647 and of 10^2147483648. + const held = (scale: number) => + Query.from(arrowFrame<{ d: BigDecimal.BigDecimal }>({ + d: [ArrowSchema.decimal(38, scale), [BigDecimal.make(7n, scale), BigDecimal.make(0n, scale)]] + })) + const read = (scale: number) => + held(scale).pipe( + Query.select((c) => ({ + round: Expr.round(c.d, 0), + unit: Expr.castDecimal(c.d, 38, 0), + int: Expr.cast(c.d, "Int64"), + float: Expr.cast(c.d, "Float64"), + mean: Expr.rollingMean(c.d, 1) + })), + rows + ) + const whole = (n: bigint) => Expr.lit(BigDecimal.make(n, 0)) + const none = { unit: BigDecimal.make(0n, 0), int: 0n, float: 0, mean: 0 } + assert.deepStrictEqual(yield* read(2147483647), [ + { round: BigDecimal.make(0n, 2147483647), ...none }, + { round: BigDecimal.make(0n, 2147483647), ...none } + ]) + assert.deepStrictEqual(yield* read(-2147483648), [ + { round: BigDecimal.make(7n, -2147483648), unit: null, int: null, float: Infinity, mean: Infinity }, + { round: BigDecimal.make(0n, -2147483648), ...none } + ]) + for (const scale of [2147483647, -2147483648]) { + const sum = yield* Effect.flip(held(scale).pipe(Query.select((c) => ({ d: Expr.add(c.d, whole(1n)) })))) + assert.strictEqual(sum.message, "decimal value overflows 64 bits") + const text = yield* Effect.flip(held(scale).pipe(Query.select((c) => ({ d: Expr.cast(c.d, "Utf8") })))) + assert.strictEqual(text.message, `Decimal[38, ${scale}] values are too long to write as text`) + assert.deepStrictEqual( + yield* held(scale).pipe(Query.aggregate((c) => ({ mean: Expr.mean(Expr.multiply(c.d, whole(0n))) })), rows), + [{ mean: 0 }] + ) + } + })) + + // DF-Q050 (API contract): Expr.cast, Expr.castDecimal; pinned source in README.md. + it.effect("casts to and from decimals with a negative scale", () => + Effect.gen(function*() { + const tens = (unscaled: bigint) => BigDecimal.make(unscaled, -1) + const values = Query.from(DataFrame.fromColumns({ + i: Int32Array.of(0, 7, -15, 25), + b: BigInt64Array.of(0n, 7n, -15n, 25n), + f: [0, 7.5, -15, 26], + s: ["0", "7", "-15", "25.01"] + })) + // Half to even at the tens: 7 is 10, -15 is -20, 25 is 20. + const to = values.pipe(Query.select((c) => ({ + i: Expr.castDecimal(c.i, 38, -1), + b: Expr.castDecimal(c.b, 38, -1), + f: Expr.castDecimal(c.f, 38, -1), + s: Expr.castDecimal(c.s, 38, -1) + }))) + assert.deepStrictEqual(yield* rows(to), [ + { i: tens(0n), b: tens(0n), f: tens(0n), s: tens(0n) }, + { i: tens(1n), b: tens(1n), f: tens(1n), s: tens(1n) }, + { i: tens(-2n), b: tens(-2n), f: tens(-2n), s: tens(-2n) }, + { i: tens(2n), b: tens(2n), f: tens(3n), s: tens(3n) } + ]) + const from = (scale: number) => + values.pipe( + Query.select((c) => ({ d: Expr.castDecimal(c.b, 38, 0) })), + Query.select((c) => ({ d: Expr.castDecimal(c.d, 38, scale) })), + Query.select((c) => ({ + int: Expr.cast(c.d, "Int64"), + small: Expr.cast(c.d, "Int8"), + float: Expr.cast(c.d, "Float64"), + text: Expr.cast(c.d, "Utf8"), + above: Expr.gt(c.d, Expr.lit(BigDecimal.make(5n, 0))) + })), + rows + ) + assert.deepStrictEqual(yield* from(-1), [ + { int: 0n, small: 0, float: 0, text: "0", above: false }, + { int: 10n, small: 10, float: 10, text: "10", above: true }, + { int: -20n, small: -20, float: -20, text: "-20", above: false }, + { int: 20n, small: 20, float: 20, text: "20", above: true } + ]) + })) + + // DF-Q051 (API contract): Expr.castDecimal, Expr.multiply; pinned source in README.md. + it.effect("rejects decimal types a cast or a product can't have", () => + Effect.gen(function*() { + const sevens = Query.from(DataFrame.fromColumns({ b: BigInt64Array.of(7n) })) + const cast = (precision: number, scale: number) => + sevens.pipe(Query.select((c) => ({ d: Expr.castDecimal(c.b, precision, scale) }))) + const precision = (p: number) => `cast: invalid decimal128 precision ${p}, expected an integer in 1..38` + const scale = (s: number) => `cast: invalid decimal scale ${s}, expected a 32-bit integer` + const cases: ReadonlyArray = [ + [2.5, 0, precision(2.5)], + [0, 0, precision(0)], + [39, 0, precision(39)], + [1e9, 0, precision(1e9)], + [2, 0.5, scale(0.5)], + [2, 2 ** 31, scale(2 ** 31)] + ] + for (const [p, s, message] of cases) { + const error = yield* Effect.flip(cast(p, s)) + assert.instanceOf(error, DataFrame.DataFrameError) + assert.strictEqual(error.message, message) + } + assert.deepStrictEqual(yield* rows(cast(38, 2)), [{ d: BigDecimal.make(700n, 2) }]) + // A product adds the scales of its factors. + const square = (s: number) => + cast(38, 0).pipe( + Query.select((c) => ({ d: Expr.castDecimal(c.d, 38, s) })), + Query.select((c) => ({ d: Expr.multiply(c.d, c.d) })) + ) + assert.strictEqual( + (yield* Effect.flip(square(-2147483648))).message, + "the scale -4294967296 of a decimal product is not a 32-bit integer" + ) + assert.deepStrictEqual(yield* rows(square(-1073741824)), [{ d: BigDecimal.make(0n, -2147483648) }]) + assert.deepStrictEqual(yield* rows(square(1)), [{ d: BigDecimal.make(4900n, 2) }]) + })) + // DF-Q008 (API contract): DataFrame.DataFrameError; pinned source in README.md. it.effect("fails with DataFrameError on invalid expressions", () => Effect.gen(function*() { diff --git a/packages/effect/test/dataframe/README.md b/packages/effect/test/dataframe/README.md index 229c817bc3a..247f5e0fcc1 100644 --- a/packages/effect/test/dataframe/README.md +++ b/packages/effect/test/dataframe/README.md @@ -51,6 +51,7 @@ The assertion column is the exact leaf test title. Combine it with the file and | DF-C030 | C | a null member of a union keeps the checks of the other members | [DataFrame.fromRows](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/DataFrame.ts#L192) | | DF-C031 | C | checks see stored values as the schema writes them, missing ones included | [DataFrame.fromRows](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/DataFrame.ts#L192) | | DF-C032 | C | accepts the stored values that Schema.is accepts | [DataFrame.fromRows](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/DataFrame.ts#L192) | +| DF-C033 | C | rejects decimal overrides that aren't 128-bit | [DataFrame.fromRows](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/DataFrame.ts#L192) | DF-C021 checks a single, unsliced, non-null batch of Int32 and Float64 columns: import/export preserves the backing buffers, byte offsets and byte lengths, as well as values. It does not require typed-array wrapper identity or mutation-based aliasing. The compatible fixed-width layouts are described in Apache Arrow’s [Columnar format](https://github.com/apache/arrow/blob/ee4d09ebef61c663c1efbfa4c18e518a03b798be/docs/source/format/Columnar.rst#fixed-size-primitive-layout) and [Schema.fbs](https://github.com/apache/arrow/blob/ee4d09ebef61c663c1efbfa4c18e518a03b798be/format/Schema.fbs); the no-copy adapter promise is the Effect API contract, not a requirement that Arrow imposes on every adapter. @@ -106,8 +107,11 @@ DF-C021 checks a single, unsliced, non-null batch of Int32 and Float64 columns: | DF-Q046 | C | sums durations to durations in the column's unit | [Expr.sum](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Expr.ts#L827) | | DF-Q047 | C | rejects aggregates that durations, timestamps, dates and times don't have | [Expr.mean](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Expr.ts#L838) | | DF-Q048 | C | rejects a decimal precision or scale that a 128-bit decimal can't have | [Query.table](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Query.ts#L254) | +| DF-Q049 | C | decides decimal scales far apart from the 64 bits of storage | [Expr.castDecimal](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Expr.ts#L699) | +| DF-Q050 | C | casts to and from decimals with a negative scale | [Expr.castDecimal](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Expr.ts#L699) | +| DF-Q051 | C | rejects decimal types a cast or a product can't have | [Expr.castDecimal](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Expr.ts#L699) | -DF-C022 to DF-C032 and DF-Q043 to DF-Q048 assert behaviour added after the baseline: checks compiled from the schema and refused by the JSON form of a plan, rejected transformations, tables declared by an Arrow schema, sums of durations, aggregates rejected for durations and points in time, the precision and scale of declared decimals, `DataFrame.column`, which has no symbol at the baseline and links to the layouts it exposes, and the checks `DataFrame.fromRows` runs on the frame it built. Their links locate the symbol at the baseline; the documentation of that symbol in the checked-out source is their contract. DF-Q048 covers a table declared by an Arrow schema, by a struct schema with `options.types`, and by a plan (decoded where the plan schema takes the numbers), run and streamed, with a precision of 38 as the passing control. The limits are those of a 128-bit Arrow decimal; the exact messages are local detail. DF-C024 observes which value constructors run (`DateTime`, `Duration`, `BigDecimal`) and covers only `DataFrame.fromRows` on rows that hold such values already, and a frame whose columns already have the declared types. +DF-C022 to DF-C033 and DF-Q043 to DF-Q051 assert behaviour added after the baseline: checks compiled from the schema and refused by the JSON form of a plan, rejected transformations, tables declared by an Arrow schema, sums of durations, aggregates rejected for durations and points in time, the precision and scale of declared decimals, `DataFrame.column`, which has no symbol at the baseline and links to the layouts it exposes, and the checks `DataFrame.fromRows` runs on the frame it built. Their links locate the symbol at the baseline; the documentation of that symbol in the checked-out source is their contract. DF-Q048 covers a table declared by an Arrow schema, by a struct schema with `options.types`, and by a plan (decoded where the plan schema takes the numbers), run and streamed, with a precision of 38 as the passing control. The limits are those of a 128-bit Arrow decimal; the exact messages are local detail. DF-C033 covers the same contract for the width: an override of 32, 64 or 256 bits fails in `DataFrame.fromRows`, `DataFrame.empty`, `DataFrame.validate` and `Query.table`, as a field of that width does in an Arrow schema, instead of being declared as a 128-bit decimal. DF-Q049 to DF-Q051 cover scales over the whole 32-bit range, which stays the only limit on a scale. DF-Q049 takes scales of 2147483647, -2147483648, 10000000, 20 and -20 against a value at scale 0: the declaration fails because the conversion must be exact, the explicit cast gives null where the value leaves 64 bits and zero where it rounds away, and zero stays zero in both; a scale of 18 and ties 19 digits apart are the controls for exact rescaling and half-to-even rounding. It also reads values held at the two extreme scales (rounding, casts to a decimal, an integer and a float, a rolling mean, a sum that fails, the text that fails, and a mean of zeros). DF-Q050 covers a scale of -1 in casts from integers, floats and strings and to integers, floats, text and a comparison. DF-Q051 covers a cast target that a 128-bit decimal can't have and a product whose scale leaves 32 bits, with passing controls. The text of a decimal has as many digits as its scale, so it fails where a string can't hold it; that failure and the exact messages are local detail. DF-C024 observes which value constructors run (`DateTime`, `Duration`, `BigDecimal`) and covers only `DataFrame.fromRows` on rows that hold such values already, and a frame whose columns already have the declared types. ## Supported Schema subset @@ -200,6 +204,9 @@ The actual SQL suite is **`packages/sql/pglite/test/DataFrameSql.test.ts`**, not | DF-S017 | C / I | sums durations to durations, as in memory | [Sql contract](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Sql.ts#L1-L45) | | DF-S018 | C / I | rejects aggregates that durations and timestamps don't have, as in memory | [Sql contract](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Sql.ts#L1-L45) | | DF-S019 | C / I | unions and nullable fields are checked as in memory | [Sql contract](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Sql.ts#L1-L45) | +| DF-S020 | C / I | decimals take the scales numeric does | [Sql contract](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Sql.ts#L1-L45) | + +DF-S020 asserts behaviour added after the baseline. A frame of decimals with a negative scale gives the same rows on PostgreSQL as in memory, through a literal, a cast, a comparison and its text. A declared column or a literal with a scale outside -1000 to 1000, the range PostgreSQL documents for `numeric`, fails to compile with a `DataFrameError` and fails the query the same way; the int32 extremes are among the cases. ## Compiler assertions diff --git a/packages/sql/pglite/test/DataFrameSql.test.ts b/packages/sql/pglite/test/DataFrameSql.test.ts index a0cc682efd1..e701e81a587 100644 --- a/packages/sql/pglite/test/DataFrameSql.test.ts +++ b/packages/sql/pglite/test/DataFrameSql.test.ts @@ -951,6 +951,42 @@ describe("effect/dataframe on PostgreSQL (PGlite)", { concurrent: false }, () => ) })) + // DF-S020: Sql API contract; pins and oracle policy in effect/test/dataframe/README.md. + it.effect("decimals take the scales numeric does", () => + Effect.gen(function*() { + const tens = (unscaled: bigint) => BigDecimal.make(unscaled, -1) + const negative = arrow<{ d: BigDecimal.BigDecimal | null }>( + [ArrowSchema.field("d", ArrowSchema.decimal(18, -1))], + [[tens(7n), tens(-15n), tens(0n), null]] + ) + yield* both(negative.pipe(Query.select((c) => ({ + d: c.d, + lit: Expr.lit(tens(-7n)), + cast: Expr.castDecimal(c.d, 38, 2), + text: Expr.cast(c.d, "Utf8"), + above: Expr.gt(c.d, Expr.lit(tens(1n))) + })))) + assert.deepStrictEqual( + yield* rowsOnSql(negative.pipe(Query.select((c) => ({ d: c.d, lit: Expr.lit(tens(-7n)) })))), + [ + { d: tens(7n), lit: tens(-7n) }, + { d: tens(-15n), lit: tens(-7n) }, + { d: tens(0n), lit: tens(-7n) }, + { d: null, lit: tens(-7n) } + ] + ) + for (const scale of [1001, -1001, 2147483647, -2147483648]) { + const message = `a decimal scale of ${scale} is not supported by the SQL engine` + const declared = Query.table("t", ArrowSchema.make([ArrowSchema.field("d", ArrowSchema.decimal(38, scale))])) + const literal = ints.pipe(Query.select(() => ({ d: Expr.lit(BigDecimal.make(7n, scale)) }))) + for (const query of [declared, literal]) { + const compiled = Sql.compile(query.plan) + assert.strictEqual(Result.isFailure(compiled) && compiled.failure.message, message) + assert.strictEqual(yield* rejects(query), message) + } + } + })) + // DF-S019: Sql API contract; pins and oracle policy in effect/test/dataframe/README.md. it.effect("unions and nullable fields are checked as in memory", () => Effect.gen(function*() { From 7dde5f32a33b7271f70ba0744f2435e9cfca10d0 Mon Sep 17 00:00:00 2001 From: Effect Bot Date: Mon, 16 Nov 2026 13:25:22 +0000 Subject: [PATCH 85/88] Keep integers exact next to negative-scale decimals, give decimal literals a valid checked type and convert floats at scales past 308 --- packages/effect/src/dataframe/Expr.ts | 5 +- .../effect/src/dataframe/internal/decimal.ts | 61 ++++++- .../effect/src/dataframe/internal/expr.ts | 8 +- .../effect/src/dataframe/internal/window.ts | 5 +- packages/effect/test/dataframe/Query.test.ts | 150 ++++++++++++++++++ packages/effect/test/dataframe/README.md | 8 +- packages/sql/pglite/test/DataFrameSql.test.ts | 37 +++++ 7 files changed, 258 insertions(+), 16 deletions(-) diff --git a/packages/effect/src/dataframe/Expr.ts b/packages/effect/src/dataframe/Expr.ts index 5ba4210399f..faef2a98c17 100644 --- a/packages/effect/src/dataframe/Expr.ts +++ b/packages/effect/src/dataframe/Expr.ts @@ -41,6 +41,8 @@ * - Operands of different types meet in their common type. A value the * common type can't hold fails the query with a `DataFrameError`; only * an explicit {@link cast} makes it `null`. + * An integer and a decimal meet in a decimal with a scale of at least + * zero, so the integer keeps its value next to a negative scale. * - Decimals are stored as 64-bit unscaled integers, so they hold up to 18 * digits. Unlike integers they never wrap: a decimal result outside that * storage (from arithmetic, sums, `abs`, `negate`, `round` or a @@ -274,7 +276,8 @@ export const fromPlan: { /** * A constant. Integers between -2^31 and 2^31 - 1 adopt the integer type of * the column they meet when it holds them; other numbers are Float64; - * bigints are Int64. + * bigints are Int64. A `BigDecimal` is a decimal of its scale, with a precision of + * at least that scale and at most 38. * * @stability experimental * @category constructors diff --git a/packages/effect/src/dataframe/internal/decimal.ts b/packages/effect/src/dataframe/internal/decimal.ts index 9515a5ea992..b0ad526e945 100644 --- a/packages/effect/src/dataframe/internal/decimal.ts +++ b/packages/effect/src/dataframe/internal/decimal.ts @@ -48,7 +48,14 @@ const PRECISION = 38 /** @internal */ export const decimal = (scale: number): S.Decimal => S.decimal(PRECISION, scale) -/** The common type of a decimal and another numeric type. @internal */ +/** + * The common type of a decimal and another numeric type. It holds every + * value of both exactly or not at all: the larger scale of two decimals, + * and with an integer a scale of at least zero, as a negative scale would + * round the integer. + * + * @internal + */ export const supertype = (a: S.DataType, b: S.DataType): S.DataType | undefined => { if (isDecimal(a) && isDecimal(b)) { return a.scale === b.scale && a.precision === b.precision @@ -56,7 +63,7 @@ export const supertype = (a: S.DataType, b: S.DataType): S.DataType | undefined : decimal(Math.max(a.scale, b.scale)) } const [d, other] = isDecimal(a) ? [a, b] : [b as S.Decimal, a] - if (other._tag === "Int") return decimal(d.scale) + if (other._tag === "Int") return decimal(Math.max(d.scale, 0)) if (other._tag === "Float") return S.float64 return undefined } @@ -66,6 +73,49 @@ const inRange = (x: bigint): boolean => x <= MAX && x >= MIN /** `x`, or a failure when it overflows 64-bit decimal storage. @internal */ export const checked = (x: bigint): bigint => (inRange(x) ? x : V.fail("decimal value overflows 64 bits")) +/** + * `x` as the unscaled value of a literal of `type`, or a failure when it + * overflows 64-bit storage or has more digits than the type's precision. + * + * @internal + */ +export const literal = (x: bigint, type: S.Decimal): bigint => { + const limit = factor(type.precision) + return checked(x) < limit && -x < limit + ? x + : V.fail(`decimal literal ${x} has more digits than its precision ${type.precision}`) +} + +/** + * The largest power of ten a float holds is `10^308`. A larger one is + * applied in two steps, so that a finite result isn't lost to an infinite + * factor. + */ +const FLOAT_DIGITS = 308 + +/** `x * 10^scale` as a float. */ +const scaleFloat = (x: number, scale: number): number => + // Zero is zero at any scale, also where `10^scale` is infinite. + x === 0 + ? 0 + : scale > FLOAT_DIGITS + ? x * 10 ** FLOAT_DIGITS * 10 ** (scale - FLOAT_DIGITS) + : x * 10 ** scale + +/** + * `unscaled / 10^scale` as a float. Past {@link FLOAT_DIGITS} digits it + * reads the value's decimal notation, which rounds once. + * + * @internal + */ +export const toNumber = (unscaled: bigint, scale: number): number => + // Zero is zero at any scale, also where `10^scale` is zero. + unscaled === I0 + ? 0 + : scale > FLOAT_DIGITS + ? Number(`${unscaled}e-${scale}`) + : Number(unscaled) / 10 ** scale + const wrap = (x: bigint): bigint => BigInt.asIntN(64, x) /** @@ -225,8 +275,7 @@ export const cast = (v: V.Vector, target: S.DataType): V.Vector | undefined => { put(i, toScale(BigInt(v.values[i]), target.scale)) break case "f64": { - // Zero is zero at any scale, also where `10^scale` is infinite. - const scaled = v.values[i] === 0 ? 0 : v.values[i] * 10 ** target.scale + const scaled = scaleFloat(v.values[i], target.scale) if (Number.isFinite(scaled)) { const floor = Math.floor(scaled) const rest = scaled - floor @@ -327,8 +376,6 @@ export const round = (v: V.I64, type: S.Decimal, places: number): V.Vector => { /** Unscaled values as floats. @internal */ export const toFloat = (values: BigInt64Array, scale: number): Float64Array => { const out = new Float64Array(values.length) - const unit = 10 ** scale - // Zero is zero at any scale, also where `10^scale` is zero or infinite. - for (let i = 0; i < values.length; i++) out[i] = values[i] === I0 ? 0 : Number(values[i]) / unit + for (let i = 0; i < values.length; i++) out[i] = toNumber(values[i], scale) return out } diff --git a/packages/effect/src/dataframe/internal/expr.ts b/packages/effect/src/dataframe/internal/expr.ts index b2b51aba8a5..e1863a3a013 100644 --- a/packages/effect/src/dataframe/internal/expr.ts +++ b/packages/effect/src/dataframe/internal/expr.ts @@ -102,7 +102,8 @@ export const literalNode = (u: unknown): Plan.Literal => { return { _tag: "Literal", value: u.value, - type: { _tag: "Decimal", precision: Math.max(digits, u.scale, 1), scale: u.scale } + // At least the scale, as SQL has it, up to the 38 digits of a 128-bit decimal. + type: { _tag: "Decimal", precision: Math.min(Math.max(digits, u.scale, 1), 38), scale: u.scale } } } return { _tag: "Literal", value: u as Plan.Value } @@ -162,8 +163,9 @@ const literalOfType = (value: unknown, type: Plan.LiteralType): K.Value => { case "Duration": return K.scalar(V.i64(S.duration(type.unit), BigInt64Array.of(big))) case "Decimal": { - const unscaled = BigInt64Array.of(Dec.checked(big)) - return K.scalar(V.i64(S.decimal(type.precision, type.scale), unscaled)) + const decimal = S.decimal(type.precision, type.scale) + Layout.checkDecimal(decimal, (message) => V.fail(`decimal literal: ${message}`)) + return K.scalar(V.i64(decimal, BigInt64Array.of(Dec.literal(big, decimal)))) } } } diff --git a/packages/effect/src/dataframe/internal/window.ts b/packages/effect/src/dataframe/internal/window.ts index 736bbb22e44..8bb0ecb9238 100644 --- a/packages/effect/src/dataframe/internal/window.ts +++ b/packages/effect/src/dataframe/internal/window.ts @@ -230,7 +230,7 @@ export const rolling = ( if (v.kind === "i64") { // Exact sums: 64-bit integers wrap and decimals fail outside 64 bits. const fit = Dec.fit(type) - const unit = Dec.isDecimal(v.type) ? 10 ** v.type.scale : 1 + const scale = Dec.isDecimal(v.type) ? v.type.scale : 0 const sums = new BigInt64Array(n) const means = new Float64Array(n) windows((row, first, last) => { @@ -244,8 +244,7 @@ export const rolling = ( } if (count < minPeriods) return ok[row] = 1 - // Zero is zero at any scale, also where `10^scale` is zero. - if (fn === "rollingMean") means[row] = sum === BigInt(0) ? 0 : Number(sum) / unit / count + if (fn === "rollingMean") means[row] = Dec.toNumber(sum, scale) / count else sums[row] = fit(sum) }) return fn === "rollingMean" ? V.f64(type, means, ok) : V.i64(type, sums, ok) diff --git a/packages/effect/test/dataframe/Query.test.ts b/packages/effect/test/dataframe/Query.test.ts index d58635e76bf..faee0164b70 100644 --- a/packages/effect/test/dataframe/Query.test.ts +++ b/packages/effect/test/dataframe/Query.test.ts @@ -508,6 +508,156 @@ describe("Query", () => { assert.deepStrictEqual(yield* rows(square(1)), [{ d: BigDecimal.make(4900n, 2) }]) })) + // DF-Q052 (API contract): Expr.Expr; pinned source in README.md. + it.effect("integers keep their value next to a decimal with a negative scale", () => + Effect.gen(function*() { + const tens = (unscaled: bigint) => BigDecimal.make(unscaled, -1) + const whole = (n: bigint) => BigDecimal.make(n, 0) + // 70 in every row, next to 69, 70 and 71. The columns are untyped, as in a frame + // read from Arrow: typed columns don't compare a decimal with an integer. + const mixed = Query.from(arrowFrame({ + d: [ArrowSchema.decimal(18, -1), [tens(7n), tens(7n), tens(7n)]], + i: [ArrowSchema.int32, [69, 70, 71]], + b: [ArrowSchema.int64, [69n, 70n, 71n]], + f: [ArrowSchema.float64, [69, 70, 71]] + })) + const compared = mixed.pipe(Query.select((c) => ({ + i: Expr.eq(c.d, c.i), + b: Expr.gt(c.d, c.b), + literal: Expr.eq(c.d, 69), + f: Expr.eq(c.d, c.f) + }))) + assert.deepStrictEqual(yield* rows(compared), [ + { i: false, b: true, literal: false, f: false }, + { i: true, b: false, literal: false, f: true }, + { i: false, b: false, literal: false, f: false } + ]) + const chosen = mixed.pipe(Query.select((c) => ({ + when: Expr.when(Expr.lit(false), c.d, c.i), + first: Expr.coalesce(c.i, c.d) + }))) + assert.deepStrictEqual(yield* rows(chosen), [ + { when: whole(69n), first: whole(69n) }, + { when: whole(70n), first: whole(70n) }, + { when: whole(71n), first: whole(71n) } + ]) + const stacked = Query.concat([ + mixed.pipe(Query.select((c) => ({ x: c.d }))), + mixed.pipe(Query.select((c) => ({ x: c.i }))) + ]) + assert.deepStrictEqual( + yield* rows(stacked), + [70n, 70n, 70n, 69n, 70n, 71n].map((n) => ({ x: whole(n) })) + ) + // Only the explicit cast rounds. + assert.deepStrictEqual( + yield* rows(mixed.pipe(Query.select((c) => ({ d: Expr.castDecimal(c.i, 38, -1) })))), + [{ d: tens(7n) }, { d: tens(7n) }, { d: tens(7n) }] + ) + // A decimal that scale 0 can't hold fails, as with any common type. + const far = Query.from(arrowFrame({ + d: [ArrowSchema.decimal(38, -400), [BigDecimal.make(7n, -400)]], + i: [ArrowSchema.int32, [1]] + })) + const error = yield* Effect.flip(far.pipe(Query.select((c) => ({ x: Expr.gt(c.d, c.i) })))) + assert.instanceOf(error, DataFrame.DataFrameError) + assert.strictEqual(error.message, "Decimal[38, -400] values don't fit their common type Decimal[38, 0]") + })) + + // DF-Q053 (API contract): Expr.lit; pinned source in README.md. + it.effect("decimal literals have a type their value fits", () => + Effect.gen(function*() { + const one = Query.from(DataFrame.fromColumns({ x: [1] })) + const select = (d: Expr.Expr) => one.pipe(Query.select(() => ({ d }))) + const typed = (value: bigint, precision: number, scale: number) => + select(Expr.fromPlan({ _tag: "Literal", value, type: { _tag: "Decimal", precision, scale } })) + /** The frame's only type and value, after a round trip through Arrow IPC. */ + const read = (df: DataFrame.DataFrame) => { + const bytes = Result.getOrThrow(ArrowIpc.encodeStream(DataFrame.toArrowTable(df))) + const table = Result.getOrThrow(ArrowIpc.decodeStream(bytes)) + return [table.schema.fields[0].type, DataFrame.toRows(DataFrame.fromArrowTable(table))] as const + } + // The precision is at least the scale, up to the 38 digits of a 128-bit decimal. + const inferred: ReadonlyArray = [ + [BigDecimal.make(5n, 2), ArrowSchema.decimal(2, 2)], + [BigDecimal.make(12345n, 2), ArrowSchema.decimal(5, 2)], + [BigDecimal.make(7n, -5), ArrowSchema.decimal(1, -5)], + [BigDecimal.make(1n, 400), ArrowSchema.decimal(38, 400)], + [BigDecimal.make(0n, 400), ArrowSchema.decimal(38, 400)], + [BigDecimal.make(-1n, 2147483647), ArrowSchema.decimal(38, 2147483647)] + ] + for (const [value, type] of inferred) { + assert.deepStrictEqual(read(yield* select(Expr.lit(value))), [type, [{ d: value }]]) + } + assert.deepStrictEqual(read(yield* typed(1n, 1, 400)), [ + ArrowSchema.decimal(1, 400), + [{ d: BigDecimal.make(1n, 400) }] + ]) + assert.deepStrictEqual(read(yield* typed(-9n, 1, 0)), [ArrowSchema.decimal(1, 0), [{ + d: BigDecimal.make(-9n, 0) + }]]) + const precision = (p: number) => + `decimal literal: invalid decimal128 precision ${p}, expected an integer in 1..38` + const failures: ReadonlyArray, string]> = [ + [typed(1n, 39, 0), precision(39)], + [typed(1n, 0, 0), precision(0)], + [typed(1n, 2, 2 ** 31), "decimal literal: invalid decimal scale 2147483648, expected a 32-bit integer"], + [ + select(Expr.lit(BigDecimal.make(1n, 2 ** 31))), + "decimal literal: invalid decimal scale 2147483648, expected a 32-bit integer" + ], + [typed(10n, 1, 0), "decimal literal 10 has more digits than its precision 1"], + [typed(-10n, 1, 0), "decimal literal -10 has more digits than its precision 1"], + [typed(2n ** 63n, 38, 0), "decimal value overflows 64 bits"] + ] + for (const [query, message] of failures) { + const error = yield* Effect.flip(query) + assert.instanceOf(error, DataFrame.DataFrameError) + assert.strictEqual(error.message, message) + } + })) + + // DF-Q054 (API contract): Expr.cast, Expr.castDecimal, Expr.mean; pinned source in README.md. + it.effect("converts between floats and decimals at scales past a float's largest power of ten", () => + Effect.gen(function*() { + const tiny = Query.from(DataFrame.fromColumns({ f: [1e-310, -1e-310, 0, 1] })).pipe( + Query.select((c) => ({ d: Expr.castDecimal(c.f, 38, 310) })) + ) + assert.deepStrictEqual(yield* rows(tiny), [ + { d: BigDecimal.make(1n, 310) }, + { d: BigDecimal.make(-1n, 310) }, + { d: BigDecimal.make(0n, 310) }, + { d: null } + ]) + // 10^18 units of 10^-310 are 10^-292. + const held = (scale: number, unscaled: ReadonlyArray) => + Query.from(arrowFrame<{ d: BigDecimal.BigDecimal }>({ + d: [ArrowSchema.decimal(38, scale), unscaled.map((x) => BigDecimal.make(x, scale))] + })) + const read = (query: Query.Query<{ d: BigDecimal.BigDecimal }>) => + Effect.all([ + query.pipe( + Query.select((c) => ({ float: Expr.cast(c.d, "Float64"), rolling: Expr.rollingMean(c.d, 2) })), + rows + ), + query.pipe(Query.aggregate((c) => ({ mean: Expr.mean(c.d) })), rows) + ]) + const big = 10n ** 18n + assert.deepStrictEqual(yield* read(held(310, [big, big, -big])), [ + [ + { float: 1e-292, rolling: null }, + { float: 1e-292, rolling: 1e-292 }, + { float: -1e-292, rolling: 0 } + ], + [{ mean: 1e-292 / 3 }] + ]) + // Below the smallest float the value is zero. + assert.deepStrictEqual(yield* read(held(400, [big, big])), [ + [{ float: 0, rolling: null }, { float: 0, rolling: 0 }], + [{ mean: 0 }] + ]) + })) + // DF-Q008 (API contract): DataFrame.DataFrameError; pinned source in README.md. it.effect("fails with DataFrameError on invalid expressions", () => Effect.gen(function*() { diff --git a/packages/effect/test/dataframe/README.md b/packages/effect/test/dataframe/README.md index 247f5e0fcc1..89687888be3 100644 --- a/packages/effect/test/dataframe/README.md +++ b/packages/effect/test/dataframe/README.md @@ -110,8 +110,11 @@ DF-C021 checks a single, unsliced, non-null batch of Int32 and Float64 columns: | DF-Q049 | C | decides decimal scales far apart from the 64 bits of storage | [Expr.castDecimal](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Expr.ts#L699) | | DF-Q050 | C | casts to and from decimals with a negative scale | [Expr.castDecimal](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Expr.ts#L699) | | DF-Q051 | C | rejects decimal types a cast or a product can't have | [Expr.castDecimal](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Expr.ts#L699) | +| DF-Q052 | C | integers keep their value next to a decimal with a negative scale | [Expr.Expr](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Expr.ts#L24-L48) | +| DF-Q053 | C | decimal literals have a type their value fits | [Expr.lit](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Expr.ts#L281) | +| DF-Q054 | C | converts between floats and decimals at scales past a float's largest power of ten | [Expr.cast](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Expr.ts#L685) | -DF-C022 to DF-C033 and DF-Q043 to DF-Q051 assert behaviour added after the baseline: checks compiled from the schema and refused by the JSON form of a plan, rejected transformations, tables declared by an Arrow schema, sums of durations, aggregates rejected for durations and points in time, the precision and scale of declared decimals, `DataFrame.column`, which has no symbol at the baseline and links to the layouts it exposes, and the checks `DataFrame.fromRows` runs on the frame it built. Their links locate the symbol at the baseline; the documentation of that symbol in the checked-out source is their contract. DF-Q048 covers a table declared by an Arrow schema, by a struct schema with `options.types`, and by a plan (decoded where the plan schema takes the numbers), run and streamed, with a precision of 38 as the passing control. The limits are those of a 128-bit Arrow decimal; the exact messages are local detail. DF-C033 covers the same contract for the width: an override of 32, 64 or 256 bits fails in `DataFrame.fromRows`, `DataFrame.empty`, `DataFrame.validate` and `Query.table`, as a field of that width does in an Arrow schema, instead of being declared as a 128-bit decimal. DF-Q049 to DF-Q051 cover scales over the whole 32-bit range, which stays the only limit on a scale. DF-Q049 takes scales of 2147483647, -2147483648, 10000000, 20 and -20 against a value at scale 0: the declaration fails because the conversion must be exact, the explicit cast gives null where the value leaves 64 bits and zero where it rounds away, and zero stays zero in both; a scale of 18 and ties 19 digits apart are the controls for exact rescaling and half-to-even rounding. It also reads values held at the two extreme scales (rounding, casts to a decimal, an integer and a float, a rolling mean, a sum that fails, the text that fails, and a mean of zeros). DF-Q050 covers a scale of -1 in casts from integers, floats and strings and to integers, floats, text and a comparison. DF-Q051 covers a cast target that a 128-bit decimal can't have and a product whose scale leaves 32 bits, with passing controls. The text of a decimal has as many digits as its scale, so it fails where a string can't hold it; that failure and the exact messages are local detail. DF-C024 observes which value constructors run (`DateTime`, `Duration`, `BigDecimal`) and covers only `DataFrame.fromRows` on rows that hold such values already, and a frame whose columns already have the declared types. +DF-C022 to DF-C033 and DF-Q043 to DF-Q054 assert behaviour added after the baseline: checks compiled from the schema and refused by the JSON form of a plan, rejected transformations, tables declared by an Arrow schema, sums of durations, aggregates rejected for durations and points in time, the precision and scale of declared decimals, `DataFrame.column`, which has no symbol at the baseline and links to the layouts it exposes, and the checks `DataFrame.fromRows` runs on the frame it built. Their links locate the symbol at the baseline; the documentation of that symbol in the checked-out source is their contract. DF-Q048 covers a table declared by an Arrow schema, by a struct schema with `options.types`, and by a plan (decoded where the plan schema takes the numbers), run and streamed, with a precision of 38 as the passing control. The limits are those of a 128-bit Arrow decimal; the exact messages are local detail. DF-C033 covers the same contract for the width: an override of 32, 64 or 256 bits fails in `DataFrame.fromRows`, `DataFrame.empty`, `DataFrame.validate` and `Query.table`, as a field of that width does in an Arrow schema, instead of being declared as a 128-bit decimal. DF-Q049 to DF-Q051 cover scales over the whole 32-bit range, which stays the only limit on a scale. DF-Q049 takes scales of 2147483647, -2147483648, 10000000, 20 and -20 against a value at scale 0: the declaration fails because the conversion must be exact, the explicit cast gives null where the value leaves 64 bits and zero where it rounds away, and zero stays zero in both; a scale of 18 and ties 19 digits apart are the controls for exact rescaling and half-to-even rounding. It also reads values held at the two extreme scales (rounding, casts to a decimal, an integer and a float, a rolling mean, a sum that fails, the text that fails, and a mean of zeros). DF-Q050 covers a scale of -1 in casts from integers, floats and strings and to integers, floats, text and a comparison. DF-Q051 covers a cast target that a 128-bit decimal can't have and a product whose scale leaves 32 bits, with passing controls. The text of a decimal has as many digits as its scale, so it fails where a string can't hold it; that failure and the exact messages are local detail. DF-Q052 to DF-Q054 cover the rest of the decimal contract at those scales. DF-Q052 holds 70 at scale -1 next to the integers 69, 70 and 71: comparisons, `when`, `coalesce` and `Query.concat` meet at scale 0, so no integer is rounded; only the explicit cast rounds, and a decimal that scale 0 can't hold fails as with any common type. DF-Q053 reads the type of a literal back through Arrow IPC: its precision is at least its scale, as SQL has it, and at most 38, so a scale of 400 or 2147483647 is a valid 128-bit decimal; a literal of a plan with a precision outside 1 to 38, a scale outside 32 bits or more digits than its precision fails when it is evaluated. DF-Q054 takes scales of 310 and 400, where `10^scale` is infinite as a float: a float of `1e-310` casts to one unit of scale 310, and a cast, a mean and a rolling mean of `10^18` units give `1e-292`; below the smallest float the value is zero. DF-C024 observes which value constructors run (`DateTime`, `Duration`, `BigDecimal`) and covers only `DataFrame.fromRows` on rows that hold such values already, and a frame whose columns already have the declared types. ## Supported Schema subset @@ -205,8 +208,9 @@ The actual SQL suite is **`packages/sql/pglite/test/DataFrameSql.test.ts`**, not | DF-S018 | C / I | rejects aggregates that durations and timestamps don't have, as in memory | [Sql contract](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Sql.ts#L1-L45) | | DF-S019 | C / I | unions and nullable fields are checked as in memory | [Sql contract](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Sql.ts#L1-L45) | | DF-S020 | C / I | decimals take the scales numeric does | [Sql contract](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Sql.ts#L1-L45) | +| DF-S021 | C / I | decimals meet integers, literals and floats as in memory | [Sql contract](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Sql.ts#L1-L45) | -DF-S020 asserts behaviour added after the baseline. A frame of decimals with a negative scale gives the same rows on PostgreSQL as in memory, through a literal, a cast, a comparison and its text. A declared column or a literal with a scale outside -1000 to 1000, the range PostgreSQL documents for `numeric`, fails to compile with a `DataFrameError` and fails the query the same way; the int32 extremes are among the cases. +DF-S020 asserts behaviour added after the baseline. A frame of decimals with a negative scale gives the same rows on PostgreSQL as in memory, through a literal, a cast, a comparison and its text. A declared column or a literal with a scale outside -1000 to 1000, the range PostgreSQL documents for `numeric`, fails to compile with a `DataFrameError` and fails the query the same way; the int32 extremes are among the cases. DF-S021 asserts the same for the repairs of DF-Q052 to DF-Q054: a decimal of scale -1 next to integers compares and selects the same values and types on PostgreSQL as in memory, a literal of scale 400 comes back with a precision of 38, and a decimal of scale 310 casts to the same float. PostgreSQL's own rounding in explicit casts is not asserted. ## Compiler assertions diff --git a/packages/sql/pglite/test/DataFrameSql.test.ts b/packages/sql/pglite/test/DataFrameSql.test.ts index e701e81a587..fff32cb5e7c 100644 --- a/packages/sql/pglite/test/DataFrameSql.test.ts +++ b/packages/sql/pglite/test/DataFrameSql.test.ts @@ -987,6 +987,43 @@ describe("effect/dataframe on PostgreSQL (PGlite)", { concurrent: false }, () => } })) + // DF-S021: Sql API contract; pins and oracle policy in effect/test/dataframe/README.md. + it.effect("decimals meet integers, literals and floats as in memory", () => + Effect.gen(function*() { + // 70 in every row, next to 69, 70 and 71. The columns are untyped, as in a frame + // read from Arrow: typed columns don't compare a decimal with an integer. + const mixed = arrow( + [ArrowSchema.field("d", ArrowSchema.decimal(18, -1)), ArrowSchema.field("i", ArrowSchema.int32)], + [[7n, 7n, 7n].map((x) => BigDecimal.make(x, -1)), [69, 70, 71]] + ) + const chosen = mixed.pipe(Query.select((c) => ({ + equal: Expr.eq(c.d, c.i), + above: Expr.gt(c.d, c.i), + literal: Expr.eq(c.d, 69), + when: Expr.when(Expr.lit(false), c.d, c.i), + first: Expr.coalesce(c.i, c.d) + }))) + yield* both(chosen) + assert.deepStrictEqual(yield* rowsOnSql(chosen), [ + { equal: false, above: true, literal: false, when: BigDecimal.make(69n, 0), first: BigDecimal.make(69n, 0) }, + { equal: true, above: false, literal: false, when: BigDecimal.make(70n, 0), first: BigDecimal.make(70n, 0) }, + { equal: false, above: false, literal: false, when: BigDecimal.make(71n, 0), first: BigDecimal.make(71n, 0) } + ]) + // A literal's precision is at most 38, whatever its scale. + const small = BigDecimal.make(1n, 400) + const literal = ints.pipe(Query.head(1), Query.select(() => ({ d: Expr.lit(small) }))) + yield* both(literal) + assert.deepStrictEqual(yield* rowsOnSql(literal), [{ d: small }]) + // 10^18 units of 10^-310 are 10^-292, past the largest power of ten a float holds. + const held = arrow<{ d: BigDecimal.BigDecimal }>( + [ArrowSchema.field("d", ArrowSchema.decimal(38, 310))], + [[BigDecimal.make(10n ** 18n, 310), BigDecimal.make(-(10n ** 18n), 310)]] + ) + const floats = held.pipe(Query.select((c) => ({ f: Expr.cast(c.d, "Float64") }))) + yield* both(floats) + assert.deepStrictEqual(yield* rowsOnSql(floats), [{ f: 1e-292 }, { f: -1e-292 }]) + })) + // DF-S019: Sql API contract; pins and oracle policy in effect/test/dataframe/README.md. it.effect("unions and nullable fields are checked as in memory", () => Effect.gen(function*() { From 643c22ba2c7d6250f684570c0d52f816eab410c1 Mon Sep 17 00:00:00 2001 From: Effect Bot Date: Mon, 16 Nov 2026 13:35:05 +0000 Subject: [PATCH 86/88] Compare isIn candidates with decimal values instead of their unscaled units and compare typed candidates with eq --- packages/effect/src/dataframe/Expr.ts | 6 ++- .../effect/src/dataframe/internal/decimal.ts | 42 +++++++++++++++ .../effect/src/dataframe/internal/expr.ts | 24 +++++++++ .../effect/src/dataframe/internal/kernels.ts | 5 +- packages/effect/test/dataframe/Query.test.ts | 46 ++++++++++++++++ packages/effect/test/dataframe/README.md | 6 ++- packages/sql/pglite/test/DataFrameSql.test.ts | 53 +++++++++++++++++++ 7 files changed, 177 insertions(+), 5 deletions(-) diff --git a/packages/effect/src/dataframe/Expr.ts b/packages/effect/src/dataframe/Expr.ts index faef2a98c17..f3a6b5741b0 100644 --- a/packages/effect/src/dataframe/Expr.ts +++ b/packages/effect/src/dataframe/Expr.ts @@ -547,7 +547,9 @@ export const between: { } = dual(3, (self: unknown, lower: unknown, upper: unknown) => internal.call("between", [self, lower, upper])) as any /** - * Whether the value is one of `values`. + * Whether the value is one of `values`; `null` for `null`. A decimal is one + * of the numbers equal to it, whatever its scale. `BigDecimal`, `DateTime` + * and `Duration` candidates are compared as {@link eq} compares them. * * @stability experimental * @category comparisons @@ -556,7 +558,7 @@ export const between: { export const isIn: { (values: ReadonlyArray): | null>(self: Expr) => Expr> (self: Expr, values: ReadonlyArray>): Expr> -} = dual(2, (self: unknown, values: ReadonlyArray) => internal.call("isIn", [self], { values })) as any +} = dual(2, internal.isIn) as any // ----------------------------------------------------------------------------- // Logic diff --git a/packages/effect/src/dataframe/internal/decimal.ts b/packages/effect/src/dataframe/internal/decimal.ts index b0ad526e945..cd153cd20b4 100644 --- a/packages/effect/src/dataframe/internal/decimal.ts +++ b/packages/effect/src/dataframe/internal/decimal.ts @@ -373,6 +373,48 @@ export const round = (v: V.I64, type: S.Decimal, places: number): V.Vector => { ) } +/** + * The integer `x` in unscaled units of `scale`, or `undefined` when the + * scale or the 64 bits of storage can't hold it exactly. + */ +const exactly = (x: bigint, scale: number): bigint | undefined => { + if (scale >= 0) { + const unscaled = x * factor(scale) + return inRange(unscaled) ? unscaled : undefined + } + if (x === I0) return I0 + // A nonzero multiple of `10^-scale` has more digits than that. + if (-scale >= (x < I0 ? -x : x).toString().length) return undefined + const divisor = pow10(-scale) + return x % divisor === I0 && inRange(x / divisor) ? x / divisor : undefined +} + +/** + * Whether each decimal is one of `candidates`. A candidate is a value and + * not a number of unscaled units: an integer is the decimal equal to it, + * and another number is the decimals whose float it is, as in a comparison + * with a float. Candidates that aren't numbers match nothing. + * + * @internal + */ +export const isIn = (values: BigInt64Array, scale: number, candidates: ReadonlyArray): Uint8Array => { + const exact = new Set() + const floats = new Set() + for (const c of candidates) { + if (typeof c === "bigint" || (typeof c === "number" && Number.isInteger(c))) { + const unscaled = exactly(BigInt(c), scale) + if (unscaled !== undefined) exact.add(unscaled) + } else if (typeof c === "number" && Number.isFinite(c)) { + floats.add(c) + } + } + const out = new Uint8Array(values.length) + for (let i = 0; i < values.length; i++) { + out[i] = +(exact.has(values[i]) || (floats.size > 0 && floats.has(toNumber(values[i], scale)))) + } + return out +} + /** Unscaled values as floats. @internal */ export const toFloat = (values: BigInt64Array, scale: number): Float64Array => { const out = new Float64Array(values.length) diff --git a/packages/effect/src/dataframe/internal/expr.ts b/packages/effect/src/dataframe/internal/expr.ts index e1863a3a013..45b15735dfa 100644 --- a/packages/effect/src/dataframe/internal/expr.ts +++ b/packages/effect/src/dataframe/internal/expr.ts @@ -116,6 +116,30 @@ export const call = ( options?: { readonly [key: string]: unknown } ): E => makeExpr({ _tag: "Call", fn, args: args.map(toNode), options }) +/** + * `isIn`: plain candidates are the `values` of the call. A `BigDecimal`, + * `DateTime` or `Duration` has a literal type, which a plan value can't + * carry, so it is compared with `eq`. + * + * @internal + */ +export const isIn = (self: unknown, values: ReadonlyArray): E => { + const literals = values.map(literalNode) + const typed = literals.filter((literal) => literal.type !== undefined) + if (typed.length === 0) return call("isIn", [self], { values }) + const node = toNode(self) + const test = (fn: Plan.FunctionName, args: ReadonlyArray, options?: Plan.CallOptions): Node => ({ + _tag: "Call", + fn, + args, + options + }) + const plain = literals.filter((literal) => literal.type === undefined).map((literal) => literal.value) + const tests = typed.map((literal) => test("eq", [node, literal])) + if (plain.length > 0) tests.unshift(test("isIn", [node], { values: plain })) + return makeExpr(tests.reduce((a, b) => test("or", [a, b]))) +} + /** Readable form of an expression, used by `toString`. @internal */ export const show = (node: Node): string => { switch (node._tag) { diff --git a/packages/effect/src/dataframe/internal/kernels.ts b/packages/effect/src/dataframe/internal/kernels.ts index 572dd1f6390..083be82be82 100644 --- a/packages/effect/src/dataframe/internal/kernels.ts +++ b/packages/effect/src/dataframe/internal/kernels.ts @@ -1151,8 +1151,11 @@ export const isIn = (a: Value, candidates: ReadonlyArray): Value => { break } case "i64": { + // A decimal's candidates are values, not the unscaled units it stores. + if (Dec.isDecimal(v.type)) return result(V.bool(Dec.isIn(v.values, v.type.scale, candidates), v.valid), a.scalar) + // Only an integer can be one of these values. const set = new Set( - candidates.filter((c) => typeof c === "bigint" || typeof c === "number").map((c) => BigInt(c as any)) + candidates.filter((c): c is bigint | number => typeof c === "bigint" || Number.isInteger(c)).map(BigInt) ) for (let i = 0; i < n; i++) o[i] = +set.has(v.values[i]) break diff --git a/packages/effect/test/dataframe/Query.test.ts b/packages/effect/test/dataframe/Query.test.ts index faee0164b70..14f0464c4ac 100644 --- a/packages/effect/test/dataframe/Query.test.ts +++ b/packages/effect/test/dataframe/Query.test.ts @@ -658,6 +658,52 @@ describe("Query", () => { ]) })) + // DF-Q055 (API contract): Expr.isIn; pinned source in README.md. + it.effect("isIn compares decimals, points in time and durations by value", () => + Effect.gen(function*() { + const dec = (unscaled: bigint, scale: number) => BigDecimal.make(unscaled, scale) + // The columns are untyped, as in a frame read from Arrow: typed columns take candidates of their own type. + const frame = arrowFrame({ + // 7.00, 7.50 and -7.00 + d: [ArrowSchema.decimal(18, 2), [dec(700n, 2), dec(750n, 2), dec(-700n, 2), null]], + // 70, 0 and 700 + n: [ArrowSchema.decimal(18, -1), [dec(7n, -1), dec(0n, -1), dec(70n, -1), null]], + b: [ArrowSchema.int64, [7n, 700n, 70n, null]], + at: [ArrowSchema.timestamp("millisecond", "UTC"), [0n, 1000n, 5n, null]], + took: [ArrowSchema.duration("millisecond"), [0n, 1000n, 5n, null]] + }) + const cases: ReadonlyArray, ReadonlyArray]> = [ + ["d", [dec(700n, 2)], [true, false, false, null]], + // Equal at another scale. + ["d", [dec(7n, 0), dec(7500n, 3)], [true, true, false, null]], + ["d", [dec(7001n, 3)], [false, false, false, null]], + ["d", [7], [true, false, false, null]], + ["d", [7n], [true, false, false, null]], + ["d", [7.5], [false, true, false, null]], + // Not the unscaled units 7.00 is stored as. + ["d", [700, 700n], [false, false, false, null]], + ["d", [7, dec(-700n, 2)], [true, false, true, null]], + ["d", [-7, null], [false, false, true, null]], + ["d", [], [false, false, false, null]], + ["d", ["7.00", true, NaN, Infinity, 2n ** 70n], [false, false, false, null]], + ["n", [70], [true, false, false, null]], + ["n", [7, 75], [false, false, false, null]], + ["n", [0, dec(700n, 0)], [false, true, true, null]], + // Controls: integers hold no fraction, and take numbers and bigints. + ["b", [7.5, NaN], [false, false, false, null]], + ["b", [7, 70n], [true, false, true, null]], + ["at", [DateTime.makeUnsafe(1000)], [false, true, false, null]], + ["took", [Duration.seconds(1), Duration.millis(5)], [false, true, true, null]] + ] + for (const [column, candidates, expected] of cases) { + const query = Query.from(frame).pipe(Query.select((c) => ({ x: Expr.isIn(c[column], candidates) }))) + assert.deepStrictEqual(yield* rows(query), expected.map((x) => ({ x })), `${column} in ${candidates}`) + // The plan holds no value its JSON form can't. + const plan = Schema.decodeSync(Plan.NodeFromJson)(Schema.encodeSync(Plan.NodeFromJson)(query.plan)) + assert.deepStrictEqual(yield* rows(Query.fromPlan(plan)), expected.map((x) => ({ x }))) + } + })) + // DF-Q008 (API contract): DataFrame.DataFrameError; pinned source in README.md. it.effect("fails with DataFrameError on invalid expressions", () => Effect.gen(function*() { diff --git a/packages/effect/test/dataframe/README.md b/packages/effect/test/dataframe/README.md index 89687888be3..0fbae1018aa 100644 --- a/packages/effect/test/dataframe/README.md +++ b/packages/effect/test/dataframe/README.md @@ -113,8 +113,9 @@ DF-C021 checks a single, unsliced, non-null batch of Int32 and Float64 columns: | DF-Q052 | C | integers keep their value next to a decimal with a negative scale | [Expr.Expr](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Expr.ts#L24-L48) | | DF-Q053 | C | decimal literals have a type their value fits | [Expr.lit](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Expr.ts#L281) | | DF-Q054 | C | converts between floats and decimals at scales past a float's largest power of ten | [Expr.cast](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Expr.ts#L685) | +| DF-Q055 | C | isIn compares decimals, points in time and durations by value | [Expr.isIn](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Expr.ts#L551) | -DF-C022 to DF-C033 and DF-Q043 to DF-Q054 assert behaviour added after the baseline: checks compiled from the schema and refused by the JSON form of a plan, rejected transformations, tables declared by an Arrow schema, sums of durations, aggregates rejected for durations and points in time, the precision and scale of declared decimals, `DataFrame.column`, which has no symbol at the baseline and links to the layouts it exposes, and the checks `DataFrame.fromRows` runs on the frame it built. Their links locate the symbol at the baseline; the documentation of that symbol in the checked-out source is their contract. DF-Q048 covers a table declared by an Arrow schema, by a struct schema with `options.types`, and by a plan (decoded where the plan schema takes the numbers), run and streamed, with a precision of 38 as the passing control. The limits are those of a 128-bit Arrow decimal; the exact messages are local detail. DF-C033 covers the same contract for the width: an override of 32, 64 or 256 bits fails in `DataFrame.fromRows`, `DataFrame.empty`, `DataFrame.validate` and `Query.table`, as a field of that width does in an Arrow schema, instead of being declared as a 128-bit decimal. DF-Q049 to DF-Q051 cover scales over the whole 32-bit range, which stays the only limit on a scale. DF-Q049 takes scales of 2147483647, -2147483648, 10000000, 20 and -20 against a value at scale 0: the declaration fails because the conversion must be exact, the explicit cast gives null where the value leaves 64 bits and zero where it rounds away, and zero stays zero in both; a scale of 18 and ties 19 digits apart are the controls for exact rescaling and half-to-even rounding. It also reads values held at the two extreme scales (rounding, casts to a decimal, an integer and a float, a rolling mean, a sum that fails, the text that fails, and a mean of zeros). DF-Q050 covers a scale of -1 in casts from integers, floats and strings and to integers, floats, text and a comparison. DF-Q051 covers a cast target that a 128-bit decimal can't have and a product whose scale leaves 32 bits, with passing controls. The text of a decimal has as many digits as its scale, so it fails where a string can't hold it; that failure and the exact messages are local detail. DF-Q052 to DF-Q054 cover the rest of the decimal contract at those scales. DF-Q052 holds 70 at scale -1 next to the integers 69, 70 and 71: comparisons, `when`, `coalesce` and `Query.concat` meet at scale 0, so no integer is rounded; only the explicit cast rounds, and a decimal that scale 0 can't hold fails as with any common type. DF-Q053 reads the type of a literal back through Arrow IPC: its precision is at least its scale, as SQL has it, and at most 38, so a scale of 400 or 2147483647 is a valid 128-bit decimal; a literal of a plan with a precision outside 1 to 38, a scale outside 32 bits or more digits than its precision fails when it is evaluated. DF-Q054 takes scales of 310 and 400, where `10^scale` is infinite as a float: a float of `1e-310` casts to one unit of scale 310, and a cast, a mean and a rolling mean of `10^18` units give `1e-292`; below the smallest float the value is zero. DF-C024 observes which value constructors run (`DateTime`, `Duration`, `BigDecimal`) and covers only `DataFrame.fromRows` on rows that hold such values already, and a frame whose columns already have the declared types. +DF-C022 to DF-C033 and DF-Q043 to DF-Q055 assert behaviour added after the baseline: checks compiled from the schema and refused by the JSON form of a plan, rejected transformations, tables declared by an Arrow schema, sums of durations, aggregates rejected for durations and points in time, the precision and scale of declared decimals, `DataFrame.column`, which has no symbol at the baseline and links to the layouts it exposes, and the checks `DataFrame.fromRows` runs on the frame it built. Their links locate the symbol at the baseline; the documentation of that symbol in the checked-out source is their contract. DF-Q048 covers a table declared by an Arrow schema, by a struct schema with `options.types`, and by a plan (decoded where the plan schema takes the numbers), run and streamed, with a precision of 38 as the passing control. The limits are those of a 128-bit Arrow decimal; the exact messages are local detail. DF-C033 covers the same contract for the width: an override of 32, 64 or 256 bits fails in `DataFrame.fromRows`, `DataFrame.empty`, `DataFrame.validate` and `Query.table`, as a field of that width does in an Arrow schema, instead of being declared as a 128-bit decimal. DF-Q049 to DF-Q051 cover scales over the whole 32-bit range, which stays the only limit on a scale. DF-Q049 takes scales of 2147483647, -2147483648, 10000000, 20 and -20 against a value at scale 0: the declaration fails because the conversion must be exact, the explicit cast gives null where the value leaves 64 bits and zero where it rounds away, and zero stays zero in both; a scale of 18 and ties 19 digits apart are the controls for exact rescaling and half-to-even rounding. It also reads values held at the two extreme scales (rounding, casts to a decimal, an integer and a float, a rolling mean, a sum that fails, the text that fails, and a mean of zeros). DF-Q050 covers a scale of -1 in casts from integers, floats and strings and to integers, floats, text and a comparison. DF-Q051 covers a cast target that a 128-bit decimal can't have and a product whose scale leaves 32 bits, with passing controls. The text of a decimal has as many digits as its scale, so it fails where a string can't hold it; that failure and the exact messages are local detail. DF-Q052 to DF-Q054 cover the rest of the decimal contract at those scales. DF-Q052 holds 70 at scale -1 next to the integers 69, 70 and 71: comparisons, `when`, `coalesce` and `Query.concat` meet at scale 0, so no integer is rounded; only the explicit cast rounds, and a decimal that scale 0 can't hold fails as with any common type. DF-Q053 reads the type of a literal back through Arrow IPC: its precision is at least its scale, as SQL has it, and at most 38, so a scale of 400 or 2147483647 is a valid 128-bit decimal; a literal of a plan with a precision outside 1 to 38, a scale outside 32 bits or more digits than its precision fails when it is evaluated. DF-Q054 takes scales of 310 and 400, where `10^scale` is infinite as a float: a float of `1e-310` casts to one unit of scale 310, and a cast, a mean and a rolling mean of `10^18` units give `1e-292`; below the smallest float the value is zero. DF-Q055 covers `Expr.isIn` on decimals: a candidate is a value and not a number of the unscaled units a decimal stores, so 7.00 at scale 2 is one of `7`, `7n` and a `BigDecimal` equal to it at any scale, and not one of `700`; a number with a fraction matches as a comparison with a float does. `BigDecimal`, `DateTime` and `Duration` candidates have a literal type that a plan value can't carry, so they are compared with `eq`, and every plan of the test survives its JSON form. Null rows stay null, null candidates and candidates of another kind match nothing, and a 64-bit integer column with a candidate that has a fraction is the control for integers. DF-C024 observes which value constructors run (`DateTime`, `Duration`, `BigDecimal`) and covers only `DataFrame.fromRows` on rows that hold such values already, and a frame whose columns already have the declared types. ## Supported Schema subset @@ -209,8 +210,9 @@ The actual SQL suite is **`packages/sql/pglite/test/DataFrameSql.test.ts`**, not | DF-S019 | C / I | unions and nullable fields are checked as in memory | [Sql contract](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Sql.ts#L1-L45) | | DF-S020 | C / I | decimals take the scales numeric does | [Sql contract](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Sql.ts#L1-L45) | | DF-S021 | C / I | decimals meet integers, literals and floats as in memory | [Sql contract](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Sql.ts#L1-L45) | +| DF-S022 | C / I | isIn compares decimals by value as in memory | [Sql contract](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Sql.ts#L1-L45) | -DF-S020 asserts behaviour added after the baseline. A frame of decimals with a negative scale gives the same rows on PostgreSQL as in memory, through a literal, a cast, a comparison and its text. A declared column or a literal with a scale outside -1000 to 1000, the range PostgreSQL documents for `numeric`, fails to compile with a `DataFrameError` and fails the query the same way; the int32 extremes are among the cases. DF-S021 asserts the same for the repairs of DF-Q052 to DF-Q054: a decimal of scale -1 next to integers compares and selects the same values and types on PostgreSQL as in memory, a literal of scale 400 comes back with a precision of 38, and a decimal of scale 310 casts to the same float. PostgreSQL's own rounding in explicit casts is not asserted. +DF-S020 asserts behaviour added after the baseline. A frame of decimals with a negative scale gives the same rows on PostgreSQL as in memory, through a literal, a cast, a comparison and its text. A declared column or a literal with a scale outside -1000 to 1000, the range PostgreSQL documents for `numeric`, fails to compile with a `DataFrameError` and fails the query the same way; the int32 extremes are among the cases. DF-S021 asserts the same for the repairs of DF-Q052 to DF-Q054: a decimal of scale -1 next to integers compares and selects the same values and types on PostgreSQL as in memory, a literal of scale 400 comes back with a precision of 38, and a decimal of scale 310 casts to the same float. PostgreSQL's own rounding in explicit casts is not asserted. DF-S022 asserts the same for DF-Q055: `Expr.isIn` over decimals of scale 2 and -1 with `BigDecimal`, integer, float and mixed candidates, and over a timestamp and a duration with `DateTime` and `Duration` candidates, gives the same rows on PostgreSQL as in memory. Candidates of another kind than the column (a string against a decimal) are not asserted: PostgreSQL converts or rejects them. ## Compiler assertions diff --git a/packages/sql/pglite/test/DataFrameSql.test.ts b/packages/sql/pglite/test/DataFrameSql.test.ts index fff32cb5e7c..70d0c3ea837 100644 --- a/packages/sql/pglite/test/DataFrameSql.test.ts +++ b/packages/sql/pglite/test/DataFrameSql.test.ts @@ -1024,6 +1024,59 @@ describe("effect/dataframe on PostgreSQL (PGlite)", { concurrent: false }, () => assert.deepStrictEqual(yield* rowsOnSql(floats), [{ f: 1e-292 }, { f: -1e-292 }]) })) + // DF-S022: Sql API contract; pins and oracle policy in effect/test/dataframe/README.md. + it.effect("isIn compares decimals by value as in memory", () => + Effect.gen(function*() { + const dec = (unscaled: bigint, scale: number) => BigDecimal.make(unscaled, scale) + // 7.00, 7.50 and -7.00 next to 70, 0 and 700. The columns are untyped, as in a frame + // read from Arrow: typed columns take candidates of their own type. + const held = arrow( + [ + ArrowSchema.field("d", ArrowSchema.decimal(18, 2)), + ArrowSchema.field("n", ArrowSchema.decimal(18, -1)), + ArrowSchema.field("at", ArrowSchema.timestamp("microsecond", "UTC")), + ArrowSchema.field("took", ArrowSchema.duration("microsecond")) + ], + [ + [dec(700n, 2), dec(750n, 2), dec(-700n, 2), null], + [dec(7n, -1), dec(0n, -1), dec(70n, -1), null], + [0n, 1_000_000n, 5n, null], + [0n, 1_000_000n, 5n, null] + ] + ) + const chosen = held.pipe(Query.select((c) => ({ + same: Expr.isIn(c.d, [dec(700n, 2)]), + rescaled: Expr.isIn(c.d, [dec(7n, 0), dec(7500n, 3)]), + integer: Expr.isIn(c.d, [7, -7n]), + float: Expr.isIn(c.d, [7.5]), + units: Expr.isIn(c.d, [700, 750n]), + mixed: Expr.isIn(c.d, [7, dec(-700n, 2), null]), + none: Expr.not(Expr.isIn(c.d, [7, dec(750n, 2)])), + negative: Expr.isIn(c.n, [70, dec(700n, 0)]), + at: Expr.isIn(c.at, [DateTime.makeUnsafe(1000)]), + took: Expr.isIn(c.took, [Duration.seconds(1), Duration.micros(5n)]) + }))) + yield* both(chosen) + const row = (...values: ReadonlyArray) => ({ + same: values[0], + rescaled: values[1], + integer: values[2], + float: values[3], + units: values[4], + mixed: values[5], + none: values[6], + negative: values[7], + at: values[8], + took: values[9] + }) + assert.deepStrictEqual(yield* rowsOnSql(chosen), [ + row(true, true, true, false, false, true, false, true, false, false), + row(false, true, false, true, false, false, false, false, true, true), + row(false, false, true, false, false, true, true, true, false, true), + row(null, null, null, null, null, null, null, null, null, null) + ]) + })) + // DF-S019: Sql API contract; pins and oracle policy in effect/test/dataframe/README.md. it.effect("unions and nullable fields are checked as in memory", () => Effect.gen(function*() { From 1dcc764db1718d0b06761e6dd502b79afdde417b Mon Sep 17 00:00:00 2001 From: Effect Bot Date: Mon, 16 Nov 2026 13:56:57 +0000 Subject: [PATCH 87/88] Join typed isIn candidates in halves, keep fractional candidates in a list of their own on SQL and compare UInt64 candidates unsigned --- packages/effect/src/dataframe/Expr.ts | 16 +++++- packages/effect/src/dataframe/Sql.ts | 4 ++ .../effect/src/dataframe/internal/expr.ts | 12 +++-- .../effect/src/dataframe/internal/kernels.ts | 4 +- packages/effect/src/dataframe/internal/sql.ts | 23 ++++++-- packages/effect/test/dataframe/Query.test.ts | 34 +++++++++++- packages/effect/test/dataframe/README.md | 6 ++- packages/sql/pglite/test/DataFrameSql.test.ts | 52 +++++++++++++++++++ 8 files changed, 139 insertions(+), 12 deletions(-) diff --git a/packages/effect/src/dataframe/Expr.ts b/packages/effect/src/dataframe/Expr.ts index f3a6b5741b0..7ec2bb525d9 100644 --- a/packages/effect/src/dataframe/Expr.ts +++ b/packages/effect/src/dataframe/Expr.ts @@ -276,8 +276,8 @@ export const fromPlan: { /** * A constant. Integers between -2^31 and 2^31 - 1 adopt the integer type of * the column they meet when it holds them; other numbers are Float64; - * bigints are Int64. A `BigDecimal` is a decimal of its scale, with a precision of - * at least that scale and at most 38. + * bigints are Int64. A `BigDecimal` is a decimal of its scale. Its precision + * is the larger of its number of digits and its scale, capped at 38. * * @stability experimental * @category constructors @@ -551,6 +551,18 @@ export const between: { * of the numbers equal to it, whatever its scale. `BigDecimal`, `DateTime` * and `Duration` candidates are compared as {@link eq} compares them. * + * **Details** + * + * Candidates of the value's kind give the same result on every engine: + * strings for strings, booleans for booleans, numbers for floats and for + * integers of up to 32 bits, numbers and bigints for 64-bit integers and + * decimals, and `BigDecimal`, `DateTime` and `Duration` for decimals, + * timestamps and durations. A number with a fraction is compared with a + * decimal as a float and is never a 64-bit integer. Other candidates (a + * string or a bigint for an Int32, a number for a timestamp) are not + * portable: what they match depends on the engine, and a database can + * reject them. + * * @stability experimental * @category comparisons * @since 4.1.5 diff --git a/packages/effect/src/dataframe/Sql.ts b/packages/effect/src/dataframe/Sql.ts index 2947b006bf0..3c3e0dbf109 100644 --- a/packages/effect/src/dataframe/Sql.ts +++ b/packages/effect/src/dataframe/Sql.ts @@ -41,6 +41,10 @@ * follows its collation, and timestamps and durations keep PostgreSQL's * microseconds (finer timestamps round down, finer durations toward zero). * Decimals need a scale PostgreSQL's `numeric` takes, from -1000 to 1000. + * `Expr.isIn` gives the in-memory result for candidates of the value's + * kind. Candidates of another kind are written as they are and are not + * portable: PostgreSQL converts some (the string `"1"` is one of the + * integers equal to 1) and rejects others with its own error. * Pivots, unpivots, `explode` and `unnest` are not supported yet; tables * need declared columns (`Query.table` with a schema), and {@link engine} * checks that tables satisfy them, as the in-memory engine does. diff --git a/packages/effect/src/dataframe/internal/expr.ts b/packages/effect/src/dataframe/internal/expr.ts index 45b15735dfa..041adf5219a 100644 --- a/packages/effect/src/dataframe/internal/expr.ts +++ b/packages/effect/src/dataframe/internal/expr.ts @@ -102,7 +102,7 @@ export const literalNode = (u: unknown): Plan.Literal => { return { _tag: "Literal", value: u.value, - // At least the scale, as SQL has it, up to the 38 digits of a 128-bit decimal. + // The larger of the digits and the scale, as SQL has it, capped at the 38 digits of a 128-bit decimal. type: { _tag: "Decimal", precision: Math.min(Math.max(digits, u.scale, 1), 38), scale: u.scale } } } @@ -119,7 +119,8 @@ export const call = ( /** * `isIn`: plain candidates are the `values` of the call. A `BigDecimal`, * `DateTime` or `Duration` has a literal type, which a plan value can't - * carry, so it is compared with `eq`. + * carry, so it is compared with `eq`. The tests are joined in halves, so + * the expression is as deep as the logarithm of their number. * * @internal */ @@ -137,7 +138,12 @@ export const isIn = (self: unknown, values: ReadonlyArray): E => { const plain = literals.filter((literal) => literal.type === undefined).map((literal) => literal.value) const tests = typed.map((literal) => test("eq", [node, literal])) if (plain.length > 0) tests.unshift(test("isIn", [node], { values: plain })) - return makeExpr(tests.reduce((a, b) => test("or", [a, b]))) + const anyOf = (tests: ReadonlyArray): Node => { + if (tests.length === 1) return tests[0] + const half = tests.length >> 1 + return test("or", [anyOf(tests.slice(0, half)), anyOf(tests.slice(half))]) + } + return makeExpr(anyOf(tests)) } /** Readable form of an expression, used by `toString`. @internal */ diff --git a/packages/effect/src/dataframe/internal/kernels.ts b/packages/effect/src/dataframe/internal/kernels.ts index 083be82be82..663ddce5edc 100644 --- a/packages/effect/src/dataframe/internal/kernels.ts +++ b/packages/effect/src/dataframe/internal/kernels.ts @@ -1157,7 +1157,9 @@ export const isIn = (a: Value, candidates: ReadonlyArray): Value => { const set = new Set( candidates.filter((c): c is bigint | number => typeof c === "bigint" || Number.isInteger(c)).map(BigInt) ) - for (let i = 0; i < n; i++) o[i] = +set.has(v.values[i]) + // A UInt64 is stored as the signed integer with the same bits. + const unsigned = V.isUnsigned64(v.type) + for (let i = 0; i < n; i++) o[i] = +set.has(unsigned ? BigInt.asUintN(64, v.values[i]) : v.values[i]) break } case "bool": { diff --git a/packages/effect/src/dataframe/internal/sql.ts b/packages/effect/src/dataframe/internal/sql.ts index d2dab62ca43..1714b9356e6 100644 --- a/packages/effect/src/dataframe/internal/sql.ts +++ b/packages/effect/src/dataframe/internal/sql.ts @@ -444,9 +444,26 @@ const call = (node: Plan.Call, scope: Scope): string => { } case "isIn": { const values = (o.values ?? []).filter((v) => v !== null) - return values.length === 0 ? - `(${a[0]} IS NULL AND NULL)` : - `(${a[0]} IN (${values.map((v) => literal(v, undefined)).join(", ")}))` + const isIn = (x: string, list: ReadonlyArray) => `(${x} IN (${list.join(", ")}))` + // One float in a list makes PostgreSQL compare the whole list as floats, which + // rounds decimals and 64-bit integers. They take their floats in a list of its own. + const t = types[0] + if (t._tag !== "Decimal" && !(t._tag === "Int" && t.bitWidth === 64)) { + return values.length === 0 ? + `(${a[0]} IS NULL AND NULL)` : + isIn(a[0], values.map((v) => literal(v, undefined))) + } + const fraction = (v: Plan.Value) => typeof v === "number" && !Number.isInteger(v) + // An integer as its digits: exact, whatever its size. + const exact = values.filter((v) => !fraction(v)).map((v) => + typeof v === "string" || typeof v === "boolean" ? literal(v, undefined) : String(BigInt(v as number | bigint)) + ) + const floats = values.filter(fraction).map((v) => literal(v, undefined)) + const tests = [ + ...(exact.length > 0 ? [isIn(a[0], exact)] : []), + ...(floats.length > 0 ? [isIn(`CAST(${a[0]} AS DOUBLE PRECISION)`, floats)] : []) + ] + return tests.length === 0 ? `(${a[0]} IS NULL AND NULL)` : `(${tests.join(" OR ")})` } case "not": return `(NOT ${a[0]})` diff --git a/packages/effect/test/dataframe/Query.test.ts b/packages/effect/test/dataframe/Query.test.ts index 14f0464c4ac..1077ca5cfd7 100644 --- a/packages/effect/test/dataframe/Query.test.ts +++ b/packages/effect/test/dataframe/Query.test.ts @@ -577,7 +577,7 @@ describe("Query", () => { const table = Result.getOrThrow(ArrowIpc.decodeStream(bytes)) return [table.schema.fields[0].type, DataFrame.toRows(DataFrame.fromArrowTable(table))] as const } - // The precision is at least the scale, up to the 38 digits of a 128-bit decimal. + // The precision is the larger of the digits and the scale, capped at the 38 digits of a 128-bit decimal. const inferred: ReadonlyArray = [ [BigDecimal.make(5n, 2), ArrowSchema.decimal(2, 2)], [BigDecimal.make(12345n, 2), ArrowSchema.decimal(5, 2)], @@ -704,6 +704,38 @@ describe("Query", () => { } })) + // DF-Q056 (API contract): Expr.isIn; pinned source in README.md. + it.effect("isIn takes long lists of typed candidates", () => + Effect.gen(function*() { + const count = 5000 + const last = BigInt(count - 1) + const frame = arrowFrame<{ + d: BigDecimal.BigDecimal | null + at: DateTime.Utc | null + took: Duration.Duration | null + }>({ + d: [ArrowSchema.decimal(18, 0), [ + BigDecimal.make(0n, 0), + BigDecimal.make(last, 0), + BigDecimal.make(last + 1n, 0), + null + ]], + at: [ArrowSchema.timestamp("millisecond", "UTC"), [0n, last, last + 1n, null]], + took: [ArrowSchema.duration("millisecond"), [0n, last, last + 1n, null]] + }) + const candidates = (make: (i: number) => A) => Array.from({ length: count }, (_, i) => make(i)) + const query = Query.from(frame).pipe(Query.select((c) => ({ + d: Expr.isIn(c.d, candidates((i) => BigDecimal.make(BigInt(i), 0))), + at: Expr.isIn(c.at, candidates((i) => DateTime.makeUnsafe(i))), + took: Expr.isIn(c.took, candidates((i) => Duration.millis(i))) + }))) + const expected = [true, true, false, null].map((x) => ({ d: x, at: x, took: x })) + assert.deepStrictEqual(yield* rows(query), expected) + // The plan is as deep as the logarithm of the list, in its JSON form too. + const plan = Schema.decodeSync(Plan.NodeFromJson)(Schema.encodeSync(Plan.NodeFromJson)(query.plan)) + assert.deepStrictEqual(yield* rows(Query.fromPlan(plan)), expected) + })) + // DF-Q008 (API contract): DataFrame.DataFrameError; pinned source in README.md. it.effect("fails with DataFrameError on invalid expressions", () => Effect.gen(function*() { diff --git a/packages/effect/test/dataframe/README.md b/packages/effect/test/dataframe/README.md index 0fbae1018aa..d7a48b903dc 100644 --- a/packages/effect/test/dataframe/README.md +++ b/packages/effect/test/dataframe/README.md @@ -114,8 +114,9 @@ DF-C021 checks a single, unsliced, non-null batch of Int32 and Float64 columns: | DF-Q053 | C | decimal literals have a type their value fits | [Expr.lit](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Expr.ts#L281) | | DF-Q054 | C | converts between floats and decimals at scales past a float's largest power of ten | [Expr.cast](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Expr.ts#L685) | | DF-Q055 | C | isIn compares decimals, points in time and durations by value | [Expr.isIn](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Expr.ts#L551) | +| DF-Q056 | C | isIn takes long lists of typed candidates | [Expr.isIn](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Expr.ts#L551) | -DF-C022 to DF-C033 and DF-Q043 to DF-Q055 assert behaviour added after the baseline: checks compiled from the schema and refused by the JSON form of a plan, rejected transformations, tables declared by an Arrow schema, sums of durations, aggregates rejected for durations and points in time, the precision and scale of declared decimals, `DataFrame.column`, which has no symbol at the baseline and links to the layouts it exposes, and the checks `DataFrame.fromRows` runs on the frame it built. Their links locate the symbol at the baseline; the documentation of that symbol in the checked-out source is their contract. DF-Q048 covers a table declared by an Arrow schema, by a struct schema with `options.types`, and by a plan (decoded where the plan schema takes the numbers), run and streamed, with a precision of 38 as the passing control. The limits are those of a 128-bit Arrow decimal; the exact messages are local detail. DF-C033 covers the same contract for the width: an override of 32, 64 or 256 bits fails in `DataFrame.fromRows`, `DataFrame.empty`, `DataFrame.validate` and `Query.table`, as a field of that width does in an Arrow schema, instead of being declared as a 128-bit decimal. DF-Q049 to DF-Q051 cover scales over the whole 32-bit range, which stays the only limit on a scale. DF-Q049 takes scales of 2147483647, -2147483648, 10000000, 20 and -20 against a value at scale 0: the declaration fails because the conversion must be exact, the explicit cast gives null where the value leaves 64 bits and zero where it rounds away, and zero stays zero in both; a scale of 18 and ties 19 digits apart are the controls for exact rescaling and half-to-even rounding. It also reads values held at the two extreme scales (rounding, casts to a decimal, an integer and a float, a rolling mean, a sum that fails, the text that fails, and a mean of zeros). DF-Q050 covers a scale of -1 in casts from integers, floats and strings and to integers, floats, text and a comparison. DF-Q051 covers a cast target that a 128-bit decimal can't have and a product whose scale leaves 32 bits, with passing controls. The text of a decimal has as many digits as its scale, so it fails where a string can't hold it; that failure and the exact messages are local detail. DF-Q052 to DF-Q054 cover the rest of the decimal contract at those scales. DF-Q052 holds 70 at scale -1 next to the integers 69, 70 and 71: comparisons, `when`, `coalesce` and `Query.concat` meet at scale 0, so no integer is rounded; only the explicit cast rounds, and a decimal that scale 0 can't hold fails as with any common type. DF-Q053 reads the type of a literal back through Arrow IPC: its precision is at least its scale, as SQL has it, and at most 38, so a scale of 400 or 2147483647 is a valid 128-bit decimal; a literal of a plan with a precision outside 1 to 38, a scale outside 32 bits or more digits than its precision fails when it is evaluated. DF-Q054 takes scales of 310 and 400, where `10^scale` is infinite as a float: a float of `1e-310` casts to one unit of scale 310, and a cast, a mean and a rolling mean of `10^18` units give `1e-292`; below the smallest float the value is zero. DF-Q055 covers `Expr.isIn` on decimals: a candidate is a value and not a number of the unscaled units a decimal stores, so 7.00 at scale 2 is one of `7`, `7n` and a `BigDecimal` equal to it at any scale, and not one of `700`; a number with a fraction matches as a comparison with a float does. `BigDecimal`, `DateTime` and `Duration` candidates have a literal type that a plan value can't carry, so they are compared with `eq`, and every plan of the test survives its JSON form. Null rows stay null, null candidates and candidates of another kind match nothing, and a 64-bit integer column with a candidate that has a fraction is the control for integers. DF-C024 observes which value constructors run (`DateTime`, `Duration`, `BigDecimal`) and covers only `DataFrame.fromRows` on rows that hold such values already, and a frame whose columns already have the declared types. +DF-C022 to DF-C033 and DF-Q043 to DF-Q056 assert behaviour added after the baseline: checks compiled from the schema and refused by the JSON form of a plan, rejected transformations, tables declared by an Arrow schema, sums of durations, aggregates rejected for durations and points in time, the precision and scale of declared decimals, `DataFrame.column`, which has no symbol at the baseline and links to the layouts it exposes, and the checks `DataFrame.fromRows` runs on the frame it built. Their links locate the symbol at the baseline; the documentation of that symbol in the checked-out source is their contract. DF-Q048 covers a table declared by an Arrow schema, by a struct schema with `options.types`, and by a plan (decoded where the plan schema takes the numbers), run and streamed, with a precision of 38 as the passing control. The limits are those of a 128-bit Arrow decimal; the exact messages are local detail. DF-C033 covers the same contract for the width: an override of 32, 64 or 256 bits fails in `DataFrame.fromRows`, `DataFrame.empty`, `DataFrame.validate` and `Query.table`, as a field of that width does in an Arrow schema, instead of being declared as a 128-bit decimal. DF-Q049 to DF-Q051 cover scales over the whole 32-bit range, which stays the only limit on a scale. DF-Q049 takes scales of 2147483647, -2147483648, 10000000, 20 and -20 against a value at scale 0: the declaration fails because the conversion must be exact, the explicit cast gives null where the value leaves 64 bits and zero where it rounds away, and zero stays zero in both; a scale of 18 and ties 19 digits apart are the controls for exact rescaling and half-to-even rounding. It also reads values held at the two extreme scales (rounding, casts to a decimal, an integer and a float, a rolling mean, a sum that fails, the text that fails, and a mean of zeros). DF-Q050 covers a scale of -1 in casts from integers, floats and strings and to integers, floats, text and a comparison. DF-Q051 covers a cast target that a 128-bit decimal can't have and a product whose scale leaves 32 bits, with passing controls. The text of a decimal has as many digits as its scale, so it fails where a string can't hold it; that failure and the exact messages are local detail. DF-Q052 to DF-Q054 cover the rest of the decimal contract at those scales. DF-Q052 holds 70 at scale -1 next to the integers 69, 70 and 71: comparisons, `when`, `coalesce` and `Query.concat` meet at scale 0, so no integer is rounded; only the explicit cast rounds, and a decimal that scale 0 can't hold fails as with any common type. DF-Q053 reads the type of a literal back through Arrow IPC: its precision is the larger of its number of digits and its scale, as SQL has it, capped at 38, so a scale of 400 or 2147483647 is a valid 128-bit decimal; a literal of a plan with a precision outside 1 to 38, a scale outside 32 bits or more digits than its precision fails when it is evaluated. DF-Q054 takes scales of 310 and 400, where `10^scale` is infinite as a float: a float of `1e-310` casts to one unit of scale 310, and a cast, a mean and a rolling mean of `10^18` units give `1e-292`; below the smallest float the value is zero. DF-Q055 covers `Expr.isIn` on decimals: a candidate is a value and not a number of the unscaled units a decimal stores, so 7.00 at scale 2 is one of `7`, `7n` and a `BigDecimal` equal to it at any scale, and not one of `700`; a number with a fraction matches as a comparison with a float does. `BigDecimal`, `DateTime` and `Duration` candidates have a literal type that a plan value can't carry, so they are compared with `eq`, and every plan of the test survives its JSON form. Null rows stay null, null candidates and candidates of another kind match nothing, and a 64-bit integer column with a candidate that has a fraction is the control for integers. DF-Q056 gives `Expr.isIn` 5000 `BigDecimal`, `DateTime` and `Duration` candidates each: their `eq` tests are joined in halves, so the plan is as deep as the logarithm of the list and runs, also after its JSON form, where a chain of that length overflowed the stack. DF-C024 observes which value constructors run (`DateTime`, `Duration`, `BigDecimal`) and covers only `DataFrame.fromRows` on rows that hold such values already, and a frame whose columns already have the declared types. ## Supported Schema subset @@ -211,8 +212,9 @@ The actual SQL suite is **`packages/sql/pglite/test/DataFrameSql.test.ts`**, not | DF-S020 | C / I | decimals take the scales numeric does | [Sql contract](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Sql.ts#L1-L45) | | DF-S021 | C / I | decimals meet integers, literals and floats as in memory | [Sql contract](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Sql.ts#L1-L45) | | DF-S022 | C / I | isIn compares decimals by value as in memory | [Sql contract](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Sql.ts#L1-L45) | +| DF-S023 | C / I | isIn keeps exact candidates apart from fractions and takes long lists | [Sql contract](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Sql.ts#L1-L45) | -DF-S020 asserts behaviour added after the baseline. A frame of decimals with a negative scale gives the same rows on PostgreSQL as in memory, through a literal, a cast, a comparison and its text. A declared column or a literal with a scale outside -1000 to 1000, the range PostgreSQL documents for `numeric`, fails to compile with a `DataFrameError` and fails the query the same way; the int32 extremes are among the cases. DF-S021 asserts the same for the repairs of DF-Q052 to DF-Q054: a decimal of scale -1 next to integers compares and selects the same values and types on PostgreSQL as in memory, a literal of scale 400 comes back with a precision of 38, and a decimal of scale 310 casts to the same float. PostgreSQL's own rounding in explicit casts is not asserted. DF-S022 asserts the same for DF-Q055: `Expr.isIn` over decimals of scale 2 and -1 with `BigDecimal`, integer, float and mixed candidates, and over a timestamp and a duration with `DateTime` and `Duration` candidates, gives the same rows on PostgreSQL as in memory. Candidates of another kind than the column (a string against a decimal) are not asserted: PostgreSQL converts or rejects them. +DF-S020 asserts behaviour added after the baseline. A frame of decimals with a negative scale gives the same rows on PostgreSQL as in memory, through a literal, a cast, a comparison and its text. A declared column or a literal with a scale outside -1000 to 1000, the range PostgreSQL documents for `numeric`, fails to compile with a `DataFrameError` and fails the query the same way; the int32 extremes are among the cases. DF-S021 asserts the same for the repairs of DF-Q052 to DF-Q054: a decimal of scale -1 next to integers compares and selects the same values and types on PostgreSQL as in memory, a literal of scale 400 comes back with a precision of 38, and a decimal of scale 310 casts to the same float. PostgreSQL's own rounding in explicit casts is not asserted. DF-S022 asserts the same for DF-Q055: `Expr.isIn` over decimals of scale 2 and -1 with `BigDecimal`, integer, float and mixed candidates, and over a timestamp and a duration with `DateTime` and `Duration` candidates, gives the same rows on PostgreSQL as in memory. Candidates of another kind than the column (a string against a decimal, a bigint against an Int32) are not asserted: the documentation of `Expr.isIn` and of `Sql` calls them not portable, and PostgreSQL converts or rejects them. DF-S023 holds 9007199254740993, the first integer a float can't hold, in a decimal, an Int64 and a UInt64 column. PostgreSQL compares a list with one float in it as floats, so the compiler writes the candidates with a fraction in a list of their own, against the column cast to a float, and the integers as their digits: a fraction next to 9007199254740992 doesn't make that candidate match, a bigint outside 64 bits matches nothing instead of failing, and a UInt64 above the signed range is one of its value and not of the signed integer with the same bits. The last column runs 3000 `BigDecimal` candidates on both engines. ## Compiler assertions diff --git a/packages/sql/pglite/test/DataFrameSql.test.ts b/packages/sql/pglite/test/DataFrameSql.test.ts index 70d0c3ea837..4d4ac65e91b 100644 --- a/packages/sql/pglite/test/DataFrameSql.test.ts +++ b/packages/sql/pglite/test/DataFrameSql.test.ts @@ -1077,6 +1077,58 @@ describe("effect/dataframe on PostgreSQL (PGlite)", { concurrent: false }, () => ]) })) + // DF-S023: Sql API contract; pins and oracle policy in effect/test/dataframe/README.md. + it.effect("isIn keeps exact candidates apart from fractions and takes long lists", () => + Effect.gen(function*() { + const dec = (unscaled: bigint, scale: number) => BigDecimal.make(unscaled, scale) + // The first integer a float can't hold, next to the float below it. + const odd = 9007199254740993n + const below = 9007199254740992 + const held = arrow( + [ + ArrowSchema.field("d", ArrowSchema.decimal(18, 0)), + ArrowSchema.field("e", ArrowSchema.decimal(18, 2)), + ArrowSchema.field("b", ArrowSchema.int64), + ArrowSchema.field("u", ArrowSchema.uint64) + ], + [ + [dec(odd, 0), dec(0n, 0), null], + [dec(odd * 100n + 50n, 2), dec(50n, 2), null], + [odd, 0n, null], + [2n ** 64n - 1n, odd, null] + ] + ) + const chosen = held.pipe(Query.select((c) => ({ + // A fraction in the list doesn't round the other candidates or the column. + rounded: Expr.isIn(c.d, [0.5, below]), + exact: Expr.isIn(c.d, [0.5, odd]), + zero: Expr.isIn(c.d, [-0, NaN, 2n ** 70n]), + fraction: Expr.isIn(c.e, [0.5, below]), + integer: Expr.isIn(c.b, [0.5, below]), + wide: Expr.isIn(c.b, [2n ** 70n, -0, odd]), + unsigned: Expr.isIn(c.u, [0.5, 2n ** 64n - 1n, below]), + signed: Expr.isIn(c.u, [-1n]), + long: Expr.isIn(c.d, Array.from({ length: 3000 }, (_, i) => dec(BigInt(i), 0))) + }))) + yield* both(chosen) + const row = (...values: ReadonlyArray) => ({ + rounded: values[0], + exact: values[1], + zero: values[2], + fraction: values[3], + integer: values[4], + wide: values[5], + unsigned: values[6], + signed: values[7], + long: values[8] + }) + assert.deepStrictEqual(yield* rowsOnSql(chosen), [ + row(false, true, false, false, false, true, true, false, false), + row(false, false, true, true, false, true, false, false, true), + row(null, null, null, null, null, null, null, null, null) + ]) + })) + // DF-S019: Sql API contract; pins and oracle policy in effect/test/dataframe/README.md. it.effect("unions and nullable fields are checked as in memory", () => Effect.gen(function*() { From eddf41910280060aecaf7c3c2d85ecd4b6e89873 Mon Sep 17 00:00:00 2001 From: Effect Bot Date: Mon, 16 Nov 2026 14:10:12 +0000 Subject: [PATCH 88/88] Say that isIn candidates of the value's kind follow each engine's comparison rules --- packages/effect/src/dataframe/Expr.ts | 30 +++++++++++++++++------- packages/effect/src/dataframe/Sql.ts | 13 ++++++---- packages/effect/test/dataframe/README.md | 2 +- 3 files changed, 31 insertions(+), 14 deletions(-) diff --git a/packages/effect/src/dataframe/Expr.ts b/packages/effect/src/dataframe/Expr.ts index 7ec2bb525d9..eb8e11100a9 100644 --- a/packages/effect/src/dataframe/Expr.ts +++ b/packages/effect/src/dataframe/Expr.ts @@ -553,15 +553,27 @@ export const between: { * * **Details** * - * Candidates of the value's kind give the same result on every engine: - * strings for strings, booleans for booleans, numbers for floats and for - * integers of up to 32 bits, numbers and bigints for 64-bit integers and - * decimals, and `BigDecimal`, `DateTime` and `Duration` for decimals, - * timestamps and durations. A number with a fraction is compared with a - * decimal as a float and is never a 64-bit integer. Other candidates (a - * string or a bigint for an Int32, a number for a timestamp) are not - * portable: what they match depends on the engine, and a database can - * reject them. + * The supported candidates are those of the value's kind: strings for + * strings, booleans for booleans, numbers for floats and for integers of + * up to 32 bits, numbers and bigints for 64-bit integers and decimals, and + * `BigDecimal`, `DateTime` and `Duration` for decimals, timestamps and + * durations. A number with a fraction is compared with a decimal as a + * float and is never a 64-bit integer. + * + * Each engine compares them by its own rules, as it does for {@link eq}, + * so two comparisons can differ between engines: + * + * - A decimal with a number that is not an integer (`NaN` and the + * infinities included). Engines round a decimal of more digits than a + * float holds differently, and a database can fail on a decimal outside + * the range of a float. + * - A timestamp that has a time zone with a `DateTime`, which is a + * timestamp without one. A database reads it in its session's time + * zone, so the results agree in a UTC session only. + * + * Other candidates (a string or a bigint for an Int32, a number for a + * timestamp) are not portable: what they match depends on the engine, and + * a database can reject them. * * @stability experimental * @category comparisons diff --git a/packages/effect/src/dataframe/Sql.ts b/packages/effect/src/dataframe/Sql.ts index 3c3e0dbf109..82639421174 100644 --- a/packages/effect/src/dataframe/Sql.ts +++ b/packages/effect/src/dataframe/Sql.ts @@ -41,10 +41,15 @@ * follows its collation, and timestamps and durations keep PostgreSQL's * microseconds (finer timestamps round down, finer durations toward zero). * Decimals need a scale PostgreSQL's `numeric` takes, from -1000 to 1000. - * `Expr.isIn` gives the in-memory result for candidates of the value's - * kind. Candidates of another kind are written as they are and are not - * portable: PostgreSQL converts some (the string `"1"` is one of the - * integers equal to 1) and rejects others with its own error. + * Comparisons and `Expr.isIn` follow the database too. A decimal next to + * a number that is not an integer is cast to `DOUBLE PRECISION`, with + * PostgreSQL's rounding and its error for a decimal outside that type's + * range. A timestamp without a time zone (a `DateTime` literal is one) + * next to one that has a time zone is read in the session's time zone, so + * it gives the in-memory result in a UTC session only. `Expr.isIn` + * candidates of another kind than the value are written as they are and + * are not portable: PostgreSQL converts some (the string `"1"` is one of + * the integers equal to 1) and rejects others with its own error. * Pivots, unpivots, `explode` and `unnest` are not supported yet; tables * need declared columns (`Query.table` with a schema), and {@link engine} * checks that tables satisfy them, as the in-memory engine does. diff --git a/packages/effect/test/dataframe/README.md b/packages/effect/test/dataframe/README.md index d7a48b903dc..205e8d54ab4 100644 --- a/packages/effect/test/dataframe/README.md +++ b/packages/effect/test/dataframe/README.md @@ -214,7 +214,7 @@ The actual SQL suite is **`packages/sql/pglite/test/DataFrameSql.test.ts`**, not | DF-S022 | C / I | isIn compares decimals by value as in memory | [Sql contract](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Sql.ts#L1-L45) | | DF-S023 | C / I | isIn keeps exact candidates apart from fractions and takes long lists | [Sql contract](https://github.com/Effect-TS/effect/blob/d7121ccbd69f9f437ac60b09a704f216c0ca44ee/packages/effect/src/dataframe/Sql.ts#L1-L45) | -DF-S020 asserts behaviour added after the baseline. A frame of decimals with a negative scale gives the same rows on PostgreSQL as in memory, through a literal, a cast, a comparison and its text. A declared column or a literal with a scale outside -1000 to 1000, the range PostgreSQL documents for `numeric`, fails to compile with a `DataFrameError` and fails the query the same way; the int32 extremes are among the cases. DF-S021 asserts the same for the repairs of DF-Q052 to DF-Q054: a decimal of scale -1 next to integers compares and selects the same values and types on PostgreSQL as in memory, a literal of scale 400 comes back with a precision of 38, and a decimal of scale 310 casts to the same float. PostgreSQL's own rounding in explicit casts is not asserted. DF-S022 asserts the same for DF-Q055: `Expr.isIn` over decimals of scale 2 and -1 with `BigDecimal`, integer, float and mixed candidates, and over a timestamp and a duration with `DateTime` and `Duration` candidates, gives the same rows on PostgreSQL as in memory. Candidates of another kind than the column (a string against a decimal, a bigint against an Int32) are not asserted: the documentation of `Expr.isIn` and of `Sql` calls them not portable, and PostgreSQL converts or rejects them. DF-S023 holds 9007199254740993, the first integer a float can't hold, in a decimal, an Int64 and a UInt64 column. PostgreSQL compares a list with one float in it as floats, so the compiler writes the candidates with a fraction in a list of their own, against the column cast to a float, and the integers as their digits: a fraction next to 9007199254740992 doesn't make that candidate match, a bigint outside 64 bits matches nothing instead of failing, and a UInt64 above the signed range is one of its value and not of the signed integer with the same bits. The last column runs 3000 `BigDecimal` candidates on both engines. +DF-S020 asserts behaviour added after the baseline. A frame of decimals with a negative scale gives the same rows on PostgreSQL as in memory, through a literal, a cast, a comparison and its text. A declared column or a literal with a scale outside -1000 to 1000, the range PostgreSQL documents for `numeric`, fails to compile with a `DataFrameError` and fails the query the same way; the int32 extremes are among the cases. DF-S021 asserts the same for the repairs of DF-Q052 to DF-Q054: a decimal of scale -1 next to integers compares and selects the same values and types on PostgreSQL as in memory, a literal of scale 400 comes back with a precision of 38, and a decimal of scale 310 casts to the same float. PostgreSQL's own rounding in explicit casts is not asserted. DF-S022 asserts the same for DF-Q055: `Expr.isIn` over decimals of scale 2 and -1 with `BigDecimal`, integer, float and mixed candidates, and over a timestamp and a duration with `DateTime` and `Duration` candidates, gives the same rows on PostgreSQL as in memory. Candidates of another kind than the column (a string against a decimal, a bigint against an Int32) are not asserted: the documentation of `Expr.isIn` and of `Sql` calls them not portable, and PostgreSQL converts or rejects them. Two comparisons of supported candidates are not asserted either, because they follow the database, as the same documentation says: a number that is not an integer next to a decimal of more digits than a float holds or outside a float's range, and a `DateTime` next to a zoned timestamp outside a UTC session. DF-S023 holds 9007199254740993, the first integer a float can't hold, in a decimal, an Int64 and a UInt64 column. PostgreSQL compares a list with one float in it as floats, so the compiler writes the candidates with a fraction in a list of their own, against the column cast to a float, and the integers as their digits: a fraction next to 9007199254740992 doesn't make that candidate match, a bigint outside 64 bits matches nothing instead of failing, and a UInt64 above the signed range is one of its value and not of the signed integer with the same bits. The last column runs 3000 `BigDecimal` candidates on both engines. ## Compiler assertions