Skip to content

Commit 2348228

Browse files
fubhyeffect-bot
authored andcommitted
Add effect/dataframe
Typed columnar data frames and lazy queries on top of effect/arrow, with in-memory, streaming and SQL engines. Split out of #10641.
1 parent d175dc8 commit 2348228

46 files changed

Lines changed: 15675 additions & 0 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.changeset/dataframe.md‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
---
2+
"effect": patch
3+
---
4+
5+
Add `effect/dataframe`: typed, columnar data frames on top of `effect/arrow`. `DataFrame` is the immutable table value; `Query` describes filters, derived columns, sorts, groups, aggregates, joins, window functions (`Expr.over`, cumulative, shift, rank, rolling), pivots, unpivots and list explosion lazily, with column names and types checked at compile time, and runs as an `Effect` after an optimiser rewrites the plan (predicate, slice and projection pushdown, top-k). Columns can hold numbers, strings, booleans, dates, timestamps, durations, decimals, lists and structs. Queries are plain data (`Plan`, with Schema codecs to and from JSON), read named tables from a `Catalog` service (frames, effects or streams of batches, processed batch by batch), and run on an `Engine`: in memory, or on a database through `Sql.layer` and `effect/sql`. Results follow Polars semantics and are checked against a Polars-generated gold corpus.

‎.gitignore‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,3 +39,6 @@ scratchpad/**/*
3939

4040
# Agent config
4141
opencode.json
42+
43+
# Python bytecode (dev scripts)
44+
__pycache__/
Lines changed: 237 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,237 @@
1+
"""
2+
Performance baseline for effect/dataframe: H2O.ai db-benchmark style group-by
3+
and join queries, run with Polars and pandas on the same data the TypeScript
4+
benchmark reads (benchmark/dataframe/h2o.ts).
5+
6+
python h2o.py generate --rows 1000000 [--data DIR] # writes Arrow IPC files
7+
python h2o.py run [--data DIR] [--reps 3] # writes DIR/baseline-<rows>.json
8+
9+
`run` times every query with Polars (all threads and one thread, each in its
10+
own process) and pandas, and records a checksum of every result so the
11+
TypeScript side can check it computed the same thing before comparing times.
12+
13+
Group-by data follows db-benchmark's groupby-datagen (K = 100 groups for
14+
id1/id2/id4/id5, N/K for id3/id6). Integer columns are Int32 on every side.
15+
The join queries are db-benchmark-style (small/medium/big right tables) on
16+
synthetic keys; see JOIN_QUERIES.
17+
"""
18+
19+
import argparse
20+
import json
21+
import os
22+
import subprocess
23+
import sys
24+
import time
25+
from pathlib import Path
26+
27+
HERE = Path(__file__).parent
28+
DEFAULT_DATA = HERE / ".data"
29+
30+
31+
def generate(rows, data):
32+
import numpy as np
33+
import pyarrow as pa
34+
import pyarrow.ipc as ipc
35+
36+
rng = np.random.default_rng(20261114)
37+
n = rows
38+
k = 100
39+
data.mkdir(parents=True, exist_ok=True)
40+
41+
def ids(fmt, count, size):
42+
pool = np.array([fmt % (i + 1) for i in range(count)], dtype=object)
43+
return pa.array(pool[rng.integers(0, count, size)], type=pa.string())
44+
45+
v3_pool = np.round(rng.uniform(0, 100, 100), 6)
46+
groupby = pa.table({
47+
"id1": ids("id%03d", k, n),
48+
"id2": ids("id%03d", k, n),
49+
"id3": ids("id%010d", n // k, n),
50+
"id4": pa.array(rng.integers(1, k + 1, n), type=pa.int32()),
51+
"id5": pa.array(rng.integers(1, k + 1, n), type=pa.int32()),
52+
"id6": pa.array(rng.integers(1, n // k + 1, n), type=pa.int32()),
53+
"v1": pa.array(rng.integers(1, 6, n), type=pa.int32()),
54+
"v2": pa.array(rng.integers(1, 16, n), type=pa.int32()),
55+
"v3": pa.array(v3_pool[rng.integers(0, 100, n)], type=pa.float64()),
56+
})
57+
write(groupby, data / f"groupby-{rows}.arrow")
58+
59+
x_id2 = rng.integers(1, 10_001, n)
60+
x = pa.table({
61+
"id1": pa.array(rng.integers(1, 11, n), type=pa.int32()),
62+
"id2": pa.array(x_id2, type=pa.int32()),
63+
"id3": pa.array(rng.integers(1, n + 1, n), type=pa.int32()),
64+
"id5": pa.array(np.array([f"id{i}" for i in range(10_001)], dtype=object)[x_id2], type=pa.string()),
65+
"v1": pa.array(np.round(rng.uniform(0, 100, n), 6), type=pa.float64()),
66+
})
67+
small = pa.table({"id1": pa.array(np.arange(1, 11), type=pa.int32()), "v2": pa.array(rng.uniform(0, 100, 10))})
68+
medium_ids = np.arange(1, 10_001)
69+
medium = pa.table({
70+
"id2": pa.array(medium_ids, type=pa.int32()),
71+
"id5": pa.array([f"id{i}" for i in medium_ids], type=pa.string()),
72+
"v2": pa.array(rng.uniform(0, 100, 10_000)),
73+
})
74+
big = pa.table({"id3": pa.array(rng.permutation(n) + 1, type=pa.int32()), "v2": pa.array(rng.uniform(0, 100, n))})
75+
for name, t in [("x", x), ("small", small), ("medium", medium), ("big", big)]:
76+
write(t, data / f"join-{name}-{rows}.arrow")
77+
print(f"wrote {rows} rows to {data}")
78+
79+
80+
def write(table, path):
81+
import pyarrow.ipc as ipc
82+
83+
with ipc.new_file(str(path), table.schema) as w:
84+
w.write_table(table)
85+
86+
87+
# ---------------------------------------------------------------------------
88+
# Queries (Polars)
89+
# ---------------------------------------------------------------------------
90+
91+
92+
def polars_queries(pl):
93+
c = pl.col
94+
gb = lambda df, keys, *aggs: df.group_by(keys, maintain_order=True).agg(*aggs)
95+
return {
96+
"q1 sum v1 by id1": lambda t: gb(t["g"], ["id1"], c("v1").sum()),
97+
"q2 sum v1 by id1:id2": lambda t: gb(t["g"], ["id1", "id2"], c("v1").sum()),
98+
"q3 sum v1, mean v3 by id3": lambda t: gb(t["g"], ["id3"], c("v1").sum(), c("v3").mean()),
99+
"q4 mean v1:v3 by id4": lambda t: gb(t["g"], ["id4"], c("v1").mean(), c("v2").mean(), c("v3").mean()),
100+
"q5 sum v1:v3 by id6": lambda t: gb(t["g"], ["id6"], c("v1").sum(), c("v2").sum(), c("v3").sum()),
101+
"q6 median, sd v3 by id4:id5": lambda t: gb(t["g"], ["id4", "id5"], c("v3").median().alias("median"), c("v3").std().alias("sd")),
102+
"q7 max v1 - min v2 by id3": lambda t: gb(t["g"], ["id3"], (c("v1").max() - c("v2").min()).alias("range")),
103+
"q10 sum v3, count by id1:id6": lambda t: gb(t["g"], ["id1", "id2", "id3", "id4", "id5", "id6"], c("v3").sum(), pl.len()),
104+
"filter v3 > 50, sum v3": lambda t: t["g"].filter(c("v3") > 50).select(c("v3").sum()),
105+
"sort by v3 desc, id6": lambda t: t["g"].sort(["v3", "id6"], descending=[True, False], maintain_order=True),
106+
"top 10 by v3 desc, id6": lambda t: t["g"].lazy().sort(["v3", "id6"], descending=[True, False], maintain_order=True).head(10).collect(),
107+
"j1 inner small on id1": lambda t: t["x"].join(t["small"], on="id1", how="inner", maintain_order="left_right"),
108+
"j2 inner medium on id2": lambda t: t["x"].join(t["medium"].drop("id5"), on="id2", how="inner", maintain_order="left_right"),
109+
"j3 left medium on id2": lambda t: t["x"].join(t["medium"].drop("id5"), on="id2", how="left", maintain_order="left_right"),
110+
"j4 inner medium on id5 (string)": lambda t: t["x"].drop("id2").join(t["medium"].drop("id2"), on="id5", how="inner", maintain_order="left_right"),
111+
"j5 inner big on id3": lambda t: t["x"].join(t["big"], on="id3", how="inner", maintain_order="left_right"),
112+
}
113+
114+
115+
def pandas_queries(pd):
116+
gb = lambda df, keys, spec: df.groupby(keys, sort=False, observed=True).agg(**spec).reset_index()
117+
return {
118+
"q1 sum v1 by id1": lambda t: gb(t["g"], ["id1"], {"v1": ("v1", "sum")}),
119+
"q2 sum v1 by id1:id2": lambda t: gb(t["g"], ["id1", "id2"], {"v1": ("v1", "sum")}),
120+
"q3 sum v1, mean v3 by id3": lambda t: gb(t["g"], ["id3"], {"v1": ("v1", "sum"), "v3": ("v3", "mean")}),
121+
"q4 mean v1:v3 by id4": lambda t: gb(t["g"], ["id4"], {"v1": ("v1", "mean"), "v2": ("v2", "mean"), "v3": ("v3", "mean")}),
122+
"q5 sum v1:v3 by id6": lambda t: gb(t["g"], ["id6"], {"v1": ("v1", "sum"), "v2": ("v2", "sum"), "v3": ("v3", "sum")}),
123+
"q6 median, sd v3 by id4:id5": lambda t: gb(t["g"], ["id4", "id5"], {"median": ("v3", "median"), "sd": ("v3", "std")}),
124+
"q7 max v1 - min v2 by id3": lambda t: (lambda r: r.assign(range=r["v1"] - r["v2"])[["id3", "range"]])(
125+
gb(t["g"], ["id3"], {"v1": ("v1", "max"), "v2": ("v2", "min")})),
126+
"q10 sum v3, count by id1:id6": lambda t: gb(t["g"], ["id1", "id2", "id3", "id4", "id5", "id6"], {"v3": ("v3", "sum"), "len": ("v1", "size")}),
127+
"filter v3 > 50, sum v3": lambda t: t["g"]["v3"][t["g"]["v3"] > 50].sum(),
128+
"sort by v3 desc, id6": lambda t: t["g"].sort_values(["v3", "id6"], ascending=[False, True], kind="stable"),
129+
"top 10 by v3 desc, id6": lambda t: t["g"].sort_values(["v3", "id6"], ascending=[False, True], kind="stable").head(10),
130+
"j1 inner small on id1": lambda t: t["x"].merge(t["small"], on="id1", how="inner"),
131+
"j2 inner medium on id2": lambda t: t["x"].merge(t["medium"].drop(columns="id5"), on="id2", how="inner"),
132+
"j3 left medium on id2": lambda t: t["x"].merge(t["medium"].drop(columns="id5"), on="id2", how="left"),
133+
"j4 inner medium on id5 (string)": lambda t: t["x"].drop(columns="id2").merge(t["medium"].drop(columns="id2"), on="id5", how="inner"),
134+
"j5 inner big on id3": lambda t: t["x"].merge(t["big"], on="id3", how="inner"),
135+
}
136+
137+
138+
def checksum_polars(df):
139+
import polars as pl
140+
141+
if not isinstance(df, pl.DataFrame):
142+
return {"rows": 1, "sum": float(df)}
143+
total = 0.0
144+
for name, dtype in df.schema.items():
145+
if dtype.is_numeric():
146+
s = df.get_column(name).cast(pl.Float64).fill_nan(None).sum()
147+
total += s or 0.0
148+
return {"rows": df.height, "sum": total}
149+
150+
151+
def load_polars(pl, data, rows):
152+
read = lambda name: pl.read_ipc(data / name)
153+
return {
154+
"g": read(f"groupby-{rows}.arrow"),
155+
"x": read(f"join-x-{rows}.arrow"),
156+
"small": read(f"join-small-{rows}.arrow"),
157+
"medium": read(f"join-medium-{rows}.arrow"),
158+
"big": read(f"join-big-{rows}.arrow"),
159+
}
160+
161+
162+
def time_best(f, reps):
163+
best = float("inf")
164+
result = None
165+
for _ in range(reps):
166+
start = time.perf_counter()
167+
result = f()
168+
best = min(best, time.perf_counter() - start)
169+
return best * 1000, result
170+
171+
172+
def run_polars(data, rows, reps):
173+
import polars as pl
174+
175+
tables = load_polars(pl, data, rows)
176+
out = {"version": pl.__version__, "threads": pl.thread_pool_size(), "queries": {}}
177+
for name, q in polars_queries(pl).items():
178+
ms, result = time_best(lambda: q(tables), reps)
179+
out["queries"][name] = {"ms": ms, "checksum": checksum_polars(result)}
180+
return out
181+
182+
183+
def run_pandas(data, rows, reps):
184+
import pandas as pd
185+
import pyarrow.ipc as ipc
186+
187+
read = lambda name: ipc.open_file(str(data / name)).read_all().to_pandas()
188+
tables = {
189+
"g": read(f"groupby-{rows}.arrow"),
190+
"x": read(f"join-x-{rows}.arrow"),
191+
"small": read(f"join-small-{rows}.arrow"),
192+
"medium": read(f"join-medium-{rows}.arrow"),
193+
"big": read(f"join-big-{rows}.arrow"),
194+
}
195+
out = {"version": pd.__version__, "queries": {}}
196+
for name, q in pandas_queries(pd).items():
197+
ms, _ = time_best(lambda: q(tables), reps)
198+
out["queries"][name] = {"ms": ms}
199+
return out
200+
201+
202+
def main():
203+
parser = argparse.ArgumentParser()
204+
parser.add_argument("command", choices=["generate", "run", "polars", "pandas"])
205+
parser.add_argument("--rows", type=int, default=1_000_000)
206+
parser.add_argument("--data", type=Path, default=DEFAULT_DATA)
207+
parser.add_argument("--reps", type=int, default=3)
208+
args = parser.parse_args()
209+
if args.command == "generate":
210+
generate(args.rows, args.data)
211+
elif args.command in ("polars", "pandas"):
212+
result = run_polars(args.data, args.rows, args.reps) if args.command == "polars" else run_pandas(args.data, args.rows, args.reps)
213+
print(json.dumps(result))
214+
else:
215+
def child(command, env=None):
216+
out = subprocess.run(
217+
[sys.executable, __file__, command, "--rows", str(args.rows), "--data", str(args.data), "--reps", str(args.reps)],
218+
check=True, capture_output=True, text=True, env={**os.environ, **(env or {})},
219+
)
220+
return json.loads(out.stdout)
221+
222+
baseline = {
223+
"rows": args.rows,
224+
"machine": {"cpus": os.cpu_count(), "platform": sys.platform},
225+
"polars": child("polars"),
226+
"polars1": child("polars", {"POLARS_MAX_THREADS": "1"}),
227+
"pandas": child("pandas"),
228+
}
229+
path = args.data / f"baseline-{args.rows}.json"
230+
path.write_text(json.dumps(baseline, indent=2) + "\n")
231+
print(f"wrote {path}")
232+
for name, q in baseline["polars"]["queries"].items():
233+
print(f"{name:36} polars {q['ms']:9.1f} ms polars(1t) {baseline['polars1']['queries'][name]['ms']:9.1f} ms pandas {baseline['pandas']['queries'][name]['ms']:9.1f} ms")
234+
235+
236+
if __name__ == "__main__":
237+
main()

0 commit comments

Comments
 (0)