Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
90 commits
Select commit Hold shift + click to select a range
b145ed1
Add effect/dataframe
fubhy Nov 15, 2026
19b4a9c
Focus effect/dataframe tests
effect-bot Nov 15, 2026
971ace2
Check table columns against their schema structurally
effect-bot Nov 15, 2026
ebd2e24
Run windows over the rows they see in filters and grouped aggregates
effect-bot Nov 15, 2026
ae74459
Fix streamed reads, slices, empty batches and explain
effect-bot Nov 15, 2026
6271716
Return null for UInt32 floor division and modulo by zero
effect-bot Nov 15, 2026
3263c37
Round decimals half to even, also when casting negative floats
effect-bot Nov 15, 2026
fb715f2
Fail decimal sums and running sums on 64-bit overflow
effect-bot Nov 15, 2026
8aa5020
Trim list children when importing sliced Arrow data
effect-bot Nov 15, 2026
2a6cb37
Clip slices from their unclamped start, as Polars does
effect-bot Nov 15, 2026
465db76
Type SQL engine results from the in-memory engine
effect-bot Nov 15, 2026
4985a7e
Name the plan-level row rules after what they guarantee
effect-bot Nov 15, 2026
5038d8e
Move filters and slices only below row-wise operations
effect-bot Nov 15, 2026
8687484
Store every computed decimal through one 64-bit check
effect-bot Nov 15, 2026
c468b0d
Keep partial decimal sums exact until the final check
effect-bot Nov 15, 2026
ba60666
Read only UTC timestamps in range as DateTimeUtc fields
effect-bot Nov 15, 2026
83e661e
Keep column types of streamed results without rows
effect-bot Nov 15, 2026
f3ebf16
Compile DataFrame plans for PostgreSQL only
effect-bot Nov 15, 2026
44e0e2c
Check SQL tables against their declared columns and schema
effect-bot Nov 15, 2026
b02df05
Lower SQL temporal casts through their units
effect-bot Nov 15, 2026
26af0e3
Use boolean aggregates for SQL boolean minimum and maximum
effect-bot Nov 15, 2026
8ee9e43
Keep the first row for SQL unique on no columns
effect-bot Nov 15, 2026
4c15c4e
Compute SQL aggregate arguments per group before aggregating
effect-bot Nov 15, 2026
589a256
Declare table column types once and convert every input to them
effect-bot Nov 15, 2026
e5c71e9
Combine streamed partial aggregates in their common type
effect-bot Nov 15, 2026
ca0dfad
Rescale durations to coarser units toward zero, as Polars does
effect-bot Nov 15, 2026
757d2c0
Bring SQL temporal operands of mixed units to the coarser unit first
effect-bot Nov 15, 2026
0980adf
Check SQL tables in the statement that reads them, as memory checks them
effect-bot Nov 15, 2026
6c55c70
Read declared SQL table columns as the types memory conforms them to
effect-bot Nov 15, 2026
9facd6c
Make temporal casts that leave 64 bits null instead of wrapping
effect-bot Nov 15, 2026
c46f6fe
Fail implicit conversions to a common type that can't hold a value
effect-bot Nov 15, 2026
76f7ebd
Give streamed aggregates the value and type of the concatenated batches
effect-bot Nov 15, 2026
d207867
Check a struct schema's checks on whole rows
effect-bot Nov 15, 2026
190650d
Read each SQL table once, through a materialized source
effect-bot Nov 15, 2026
ab0d642
Check SQL table values against their whole declared type
effect-bot Nov 15, 2026
216666c
Read SQL table columns as their declared types
effect-bot Nov 15, 2026
0fd10c3
Document that SQL tables with whole-row checks are rejected
effect-bot Nov 15, 2026
b536556
Keep predicates above concats and off the right side's join keys
effect-bot Nov 15, 2026
d06a162
Stream batches in the column types of their node's whole result
effect-bot Nov 15, 2026
fc9ec8c
Fit integer literals to the other operands wherever they appear
effect-bot Nov 15, 2026
be557eb
Reserve table names from the names the SQL compiler invents
effect-bot Nov 15, 2026
3f0fea0
Qualify the columns the SQL column type checks read by their table
effect-bot Nov 15, 2026
93f09f6
Document that SQL table names are single quoted identifiers
effect-bot Nov 15, 2026
f79b582
Fit integer literals of when and coalesce by their plan-time range
effect-bot Nov 15, 2026
28fa445
Fix the type of an integer literal cast to its own type
effect-bot Nov 15, 2026
2a10bd3
Import Arrow tables without batches with the nullability of their fields
effect-bot Nov 15, 2026
766686f
Keep the columns of streamed results without rows, or fail where they…
effect-bot Nov 15, 2026
0da84fd
Read no batch for a top-k selection without rows
effect-bot Nov 15, 2026
5d3535e
Keep the names the SQL compiler invents distinct after truncation
effect-bot Nov 15, 2026
c43ddd8
Reject columns PostgreSQL can't tell apart in SQL plans
effect-bot Nov 15, 2026
059c565
Read a table named more than once only when a run needs its batches
effect-bot Nov 15, 2026
158f530
Share one SQL source among names PostgreSQL keeps as one
effect-bot Nov 15, 2026
e2db400
Simplify the dataframe public modules
effect-bot Nov 15, 2026
71c0053
Shorten the dataframe changeset
effect-bot Nov 15, 2026
260c88a
Simplify dataframe value kernels, vectors and type helpers
effect-bot Nov 15, 2026
91e37a0
Share row grouping, ordering and key helpers across the dataframe ker…
effect-bot Nov 15, 2026
6d17c56
Simplify dataframe plan rewriting, expression evaluation and schema d…
effect-bot Nov 15, 2026
fff9a12
Simplify the SQL compiler and shorten its comments
effect-bot Nov 15, 2026
88fa4ba
Trim setup and share error helpers in the PGlite DataFrame tests
effect-bot Nov 15, 2026
ace6f35
Simplify the dataframe tests, oracle, gold generator and benchmark
effect-bot Nov 15, 2026
e3474e9
Clarify the comment on fitting integer literals to their operand
effect-bot Nov 15, 2026
a415abf
Map DataFrame API contracts to runtime and type coverage
effect-bot Nov 15, 2026
2372759
Cover DataFrame cooperative interruption and buffer reuse contracts
effect-bot Nov 15, 2026
8bed2f0
Keep Effect Schema out of table execution and declare tables by Arrow…
effect-bot Nov 16, 2026
29e15e4
List the schema check and Arrow declaration tests in the dataframe in…
effect-bot Nov 16, 2026
0e15174
Round decimals half to even with integer arithmetic
effect-bot Nov 16, 2026
f934164
Read decimal SQL results into columns without a BigDecimal per value
effect-bot Nov 16, 2026
194d0fe
Cover the columnar path of temporal and decimal columns
effect-bot Nov 16, 2026
72e4937
Add a benchmark of columnar work against row output
effect-bot Nov 16, 2026
eb66299
Add DataFrame.column to read a column's buffers without building rows
effect-bot Nov 16, 2026
b8d2961
Run a table declared by an Arrow schema on PostgreSQL
effect-bot Nov 16, 2026
86e0fef
Check the frame fromRows built, and reject temporal and decimal value…
effect-bot Nov 16, 2026
0ad0fb9
Keep the checks of a union's members when its null member is suspended
effect-bot Nov 16, 2026
ac3ae09
Hand checks the stored values as the schema writes them, and reject s…
effect-bot Nov 16, 2026
e155994
Test a checked field with the schema's own guard, so null and oneOf u…
effect-bot Nov 16, 2026
9c238df
Refuse to encode a plan whose tables carry checks instead of dropping…
effect-bot Nov 16, 2026
035eb7b
Type duration sums as durations and reject the means and sums the eng…
effect-bot Nov 16, 2026
8854447
Use the shared temporal check for the columns a mean rejects
effect-bot Nov 16, 2026
8968ad5
Align the DataFrame decimal control and Query.table docs with the str…
effect-bot Nov 16, 2026
ab350ad
Keep __proto__ an own property of the rows that checks see
effect-bot Nov 16, 2026
13730c7
Merge feat/arrow-support into feat/dataframe-support
effect-bot Nov 16, 2026
6524aa2
Simplify the table checks, the Table decoder and two unit lookups in …
effect-bot Nov 16, 2026
283ec06
Shorten and clarify data frame comments, docs and the changeset
effect-bot Nov 16, 2026
9509abd
Merge feat/arrow-support into feat/dataframe-support
effect-bot Nov 16, 2026
ced3afd
Reject a declared decimal precision or scale that a 128-bit decimal c…
effect-bot Nov 16, 2026
dbf60e6
Decide decimal scales far apart from the 64-bit storage range and rej…
effect-bot Nov 16, 2026
7dde5f3
Keep integers exact next to negative-scale decimals, give decimal lit…
effect-bot Nov 16, 2026
643c22b
Compare isIn candidates with decimal values instead of their unscaled…
effect-bot Nov 16, 2026
1dcc764
Join typed isIn candidates in halves, keep fractional candidates in a…
effect-bot Nov 16, 2026
eddf419
Say that isIn candidates of the value's kind follow each engine's com…
effect-bot Nov 16, 2026
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions .changeset/dataframe.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
---
"effect": patch
---

Add the experimental `effect/dataframe` modules for typed, columnar data frames on `effect/arrow`, with lazy, type-checked queries that run in memory or on PostgreSQL. Tables are declared by an Effect Schema or an Arrow schema.
1 change: 1 addition & 0 deletions dprint.json
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
"**/*.tsbuildinfo",
"**/CHANGELOG.md",
"packages/effect/src/StandardSchema.ts",
"packages/effect/test/fixtures/dataframe/gold/*.json",
"packages/tools/graphql-generator/test/generated/**/*.graphql.ts",
"!scratchpad/**/*",
"**/.*",
Expand Down
146 changes: 146 additions & 0 deletions packages/effect/benchmark/dataframe/columnar.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,146 @@
/**
* effect/dataframe on timestamp, duration and decimal columns, which frames
* hold as integers. Only rows turn them into `DateTime`, `Duration` and
* `BigDecimal` values, so the cases compare work that stays in columns with
* the same input read as rows.
*
* node [--expose-gc] benchmark/dataframe/columnar.ts [--rows=1000000] [--reps=5] [--only=<substring>]
* [--out=<file.json>] [--gc]
*
* The data is generated in memory: a timestamp and a duration in
* microseconds, a decimal(18, 2), a float64 and a string key of 100 values,
* without nulls, encoded once as an Arrow IPC stream. Times are the best of
* `--reps` runs, in milliseconds.
*
* `--gc` (with `node --expose-gc`) collects garbage before every timed run,
* and again after it while the case's result is still referenced. The memory
* columns are the growth between the two, in MiB, in the last run: `heap` is
* `heapUsed`, the JS objects the result keeps alive, and `buffers` is
* `arrayBuffers`, the memory of its typed arrays. Neither counts what a run
* allocated and released.
*/
import { BigDecimal, DateTime, Effect } from "effect"
import { ArrowIpc, ArrowSchema, ArrowTable } from "effect/arrow"
import { DataFrame, Expr, Query } from "effect/dataframe"
import * as Result from "effect/Result"
import * as Fs from "node:fs"

const arg = (name: string) => process.argv.find((a) => a.startsWith(`--${name}=`))?.slice(name.length + 3)
const rows = Number(arg("rows") ?? 1_000_000)
const reps = Number(arg("reps") ?? 5)
const only = arg("only")
const out = arg("out")
const gc = process.argv.includes("--gc") ? (globalThis as { gc?: () => void }).gc : undefined

let state = 0x9e3779b9
const next = (bound: number) => {
state ^= state << 13
state ^= state >>> 17
state ^= state << 5
return Math.floor(((state >>> 0) / 4294967296) * bound)
}
/** 2024-01-01T00:00:00Z, in milliseconds. */
const start = 1_704_067_200_000
const bytes = Result.getOrThrow(ArrowIpc.encodeStream(Result.getOrThrow(ArrowTable.fromColumns(
ArrowSchema.make([
ArrowSchema.field("at", ArrowSchema.timestamp("microsecond")),
ArrowSchema.field("took", ArrowSchema.duration("microsecond")),
ArrowSchema.field("amount", ArrowSchema.decimal(18, 2)),
ArrowSchema.field("v", ArrowSchema.float64),
ArrowSchema.field("key", ArrowSchema.utf8)
]),
[
// One row a second, up to a second late.
Array.from({ length: rows }, (_, i) => BigInt(start + i * 1000) * 1000n + BigInt(next(1_000_000))),
Array.from({ length: rows }, () => BigInt(next(3_600_000_000))),
Array.from({ length: rows }, () => BigDecimal.make(BigInt(next(2_000_000_000) - 1_000_000_000), 2)),
Array.from({ length: rows }, () => next(1_000_000) / 1000),
Array.from({ length: rows }, () => `key${next(100)}`)
]
))))

const decode = () =>
DataFrame.fromArrowTable(Result.getOrThrow(ArrowIpc.decodeStream(bytes))) as DataFrame.DataFrame<any>
const encode = (df: DataFrame.DataFrame<any>) => Result.getOrThrow(ArrowIpc.encodeStream(DataFrame.toArrowTable(df)))
/** The later half of the rows. */
const since = DateTime.makeUnsafe(start + (rows / 2) * 1000)
const input = decode()

const cases: Record<string, () => { readonly result: unknown; readonly rows: number }> = {
"a decode, filter, group, encode": () => {
const result = Effect.runSync(
Query.from(decode()).pipe(
Query.filter((c) => Expr.gt(c.at, since)),
Query.groupBy("key"),
Query.agg((c) => ({
amount: Expr.sum(c.amount),
took: Expr.mean(c.took),
last: Expr.max(c.at),
v: Expr.sum(c.v)
}))
)
)
return { result: encode(result), rows: result.length }
},
"b decode, rows": () => {
const result = DataFrame.toRows(decode())
return { result, rows: result.length }
},
"c cast decimal(18, 2) to (18, 1)": () => {
const result = Effect.runSync(
Query.from(input).pipe(Query.select((c) => ({ amount: Expr.castDecimal(c.amount, 18, 1) })))
)
// Forces the column without converting values to JS.
return { result: DataFrame.toArrowTable(result), rows: result.length }
}
}

const results: Record<string, unknown> = {}
/** References the result of the last run, so that collecting garbage keeps it. */
const held: { result?: unknown } = {}
/** Runs a case in a call of its own, so that `held` has the only reference to its result afterwards. */
const time = (run: () => { readonly result: unknown; readonly rows: number }) => {
const start = performance.now()
const { result, rows } = run()
const ms = performance.now() - start
held.result = result
return { ms, rows }
}
const fmt = (n: number | undefined) => (n === undefined ? "—" : n.toFixed(1)).padStart(9)
const mib = (n: number) => n / 1024 / 1024

console.log(
`effect/dataframe columnar path, ${rows} rows (${mib(bytes.length).toFixed(1)} MiB of IPC), best of ${reps}`
)
console.log(
`${"case".padEnd(34)} ${"best ms".padStart(9)} ${"first ms".padStart(9)} ${"heap MiB".padStart(9)} ${
"buffers".padStart(9)
} rows`
)
for (const [name, run] of Object.entries(cases)) {
if (only !== undefined && !name.includes(only)) continue
let best = Infinity
let first = 0
let count = 0
let heap: number | undefined
let buffers: number | undefined
for (let r = 0; r < reps; r++) {
held.result = undefined
gc?.()
const before = process.memoryUsage()
const { ms, rows } = time(run)
if (r === 0) first = ms
best = Math.min(best, ms)
count = rows
if (gc !== undefined) {
gc()
const after = process.memoryUsage()
heap = mib(after.heapUsed - before.heapUsed)
buffers = mib(after.arrayBuffers - before.arrayBuffers)
}
}
console.log(`${name.padEnd(34)} ${fmt(best)} ${fmt(first)} ${fmt(heap)} ${fmt(buffers)} ${count}`)
results[name] = { ms: best, first, heapMiB: heap, arrayBuffersMiB: buffers, rows: count }
}

if (out !== undefined) Fs.writeFileSync(out, JSON.stringify({ rows, reps, results }, null, 2) + "\n")
222 changes: 222 additions & 0 deletions packages/effect/benchmark/dataframe/h2o.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,222 @@
"""
Performance baseline for effect/dataframe: H2O.ai db-benchmark style group-by
and join queries, run with Polars and pandas on the same data the TypeScript
benchmark reads (benchmark/dataframe/h2o.ts).

python h2o.py generate --rows 1000000 [--data DIR] # writes Arrow IPC files
python h2o.py run [--data DIR] [--reps 3] # writes DIR/baseline-<rows>.json

`run` times every query with Polars (all threads and one thread, each in its
own process) and pandas, and records a checksum of every result so the
TypeScript side can check it computed the same thing before comparing times.

Group-by data follows db-benchmark's groupby-datagen (K = 100 groups for
id1/id2/id4/id5, N/K for id3/id6). Integer columns are Int32 on every side.
The join queries are db-benchmark-style (small/medium/big right tables) on
synthetic keys; see JOIN_QUERIES.
"""

import argparse
import json
import os
import subprocess
import sys
import time
from pathlib import Path

HERE = Path(__file__).parent
DEFAULT_DATA = HERE / ".data"
# Table keys of the queries, and their file names (as <name>-<rows>.arrow).
TABLES = {"g": "groupby", "x": "join-x", "small": "join-small", "medium": "join-medium", "big": "join-big"}


def generate(rows, data):
import numpy as np
import pyarrow as pa

rng = np.random.default_rng(20261114)
n = rows
k = 100
data.mkdir(parents=True, exist_ok=True)

def ids(fmt, count, size):
pool = np.array([fmt % (i + 1) for i in range(count)], dtype=object)
return pa.array(pool[rng.integers(0, count, size)], type=pa.string())

v3_pool = np.round(rng.uniform(0, 100, 100), 6)
groupby = pa.table({
"id1": ids("id%03d", k, n),
"id2": ids("id%03d", k, n),
"id3": ids("id%010d", n // k, n),
"id4": pa.array(rng.integers(1, k + 1, n), type=pa.int32()),
"id5": pa.array(rng.integers(1, k + 1, n), type=pa.int32()),
"id6": pa.array(rng.integers(1, n // k + 1, n), type=pa.int32()),
"v1": pa.array(rng.integers(1, 6, n), type=pa.int32()),
"v2": pa.array(rng.integers(1, 16, n), type=pa.int32()),
"v3": pa.array(v3_pool[rng.integers(0, 100, n)], type=pa.float64()),
})
write(groupby, data / f"groupby-{rows}.arrow")

x_id2 = rng.integers(1, 10_001, n)
x = pa.table({
"id1": pa.array(rng.integers(1, 11, n), type=pa.int32()),
"id2": pa.array(x_id2, type=pa.int32()),
"id3": pa.array(rng.integers(1, n + 1, n), type=pa.int32()),
"id5": pa.array(np.array([f"id{i}" for i in range(10_001)], dtype=object)[x_id2], type=pa.string()),
"v1": pa.array(np.round(rng.uniform(0, 100, n), 6), type=pa.float64()),
})
small = pa.table({"id1": pa.array(np.arange(1, 11), type=pa.int32()), "v2": pa.array(rng.uniform(0, 100, 10))})
medium_ids = np.arange(1, 10_001)
medium = pa.table({
"id2": pa.array(medium_ids, type=pa.int32()),
"id5": pa.array([f"id{i}" for i in medium_ids], type=pa.string()),
"v2": pa.array(rng.uniform(0, 100, 10_000)),
})
big = pa.table({"id3": pa.array(rng.permutation(n) + 1, type=pa.int32()), "v2": pa.array(rng.uniform(0, 100, n))})
for name, t in [("x", x), ("small", small), ("medium", medium), ("big", big)]:
write(t, data / f"join-{name}-{rows}.arrow")
print(f"wrote {rows} rows to {data}")


def write(table, path):
import pyarrow.ipc as ipc

with ipc.new_file(str(path), table.schema) as w:
w.write_table(table)


# ---------------------------------------------------------------------------
# Queries (Polars)
# ---------------------------------------------------------------------------


def polars_queries(pl):
c = pl.col
gb = lambda df, keys, *aggs: df.group_by(keys, maintain_order=True).agg(*aggs)
return {
"q1 sum v1 by id1": lambda t: gb(t["g"], ["id1"], c("v1").sum()),
"q2 sum v1 by id1:id2": lambda t: gb(t["g"], ["id1", "id2"], c("v1").sum()),
"q3 sum v1, mean v3 by id3": lambda t: gb(t["g"], ["id3"], c("v1").sum(), c("v3").mean()),
"q4 mean v1:v3 by id4": lambda t: gb(t["g"], ["id4"], c("v1").mean(), c("v2").mean(), c("v3").mean()),
"q5 sum v1:v3 by id6": lambda t: gb(t["g"], ["id6"], c("v1").sum(), c("v2").sum(), c("v3").sum()),
"q6 median, sd v3 by id4:id5": lambda t: gb(t["g"], ["id4", "id5"], c("v3").median().alias("median"), c("v3").std().alias("sd")),
"q7 max v1 - min v2 by id3": lambda t: gb(t["g"], ["id3"], (c("v1").max() - c("v2").min()).alias("range")),
"q10 sum v3, count by id1:id6": lambda t: gb(t["g"], ["id1", "id2", "id3", "id4", "id5", "id6"], c("v3").sum(), pl.len()),
"filter v3 > 50, sum v3": lambda t: t["g"].filter(c("v3") > 50).select(c("v3").sum()),
"sort by v3 desc, id6": lambda t: t["g"].sort(["v3", "id6"], descending=[True, False], maintain_order=True),
"top 10 by v3 desc, id6": lambda t: t["g"].lazy().sort(["v3", "id6"], descending=[True, False], maintain_order=True).head(10).collect(),
"j1 inner small on id1": lambda t: t["x"].join(t["small"], on="id1", how="inner", maintain_order="left_right"),
"j2 inner medium on id2": lambda t: t["x"].join(t["medium"].drop("id5"), on="id2", how="inner", maintain_order="left_right"),
"j3 left medium on id2": lambda t: t["x"].join(t["medium"].drop("id5"), on="id2", how="left", maintain_order="left_right"),
"j4 inner medium on id5 (string)": lambda t: t["x"].drop("id2").join(t["medium"].drop("id2"), on="id5", how="inner", maintain_order="left_right"),
"j5 inner big on id3": lambda t: t["x"].join(t["big"], on="id3", how="inner", maintain_order="left_right"),
}


def pandas_queries(pd):
gb = lambda df, keys, spec: df.groupby(keys, sort=False, observed=True).agg(**spec).reset_index()
return {
"q1 sum v1 by id1": lambda t: gb(t["g"], ["id1"], {"v1": ("v1", "sum")}),
"q2 sum v1 by id1:id2": lambda t: gb(t["g"], ["id1", "id2"], {"v1": ("v1", "sum")}),
"q3 sum v1, mean v3 by id3": lambda t: gb(t["g"], ["id3"], {"v1": ("v1", "sum"), "v3": ("v3", "mean")}),
"q4 mean v1:v3 by id4": lambda t: gb(t["g"], ["id4"], {"v1": ("v1", "mean"), "v2": ("v2", "mean"), "v3": ("v3", "mean")}),
"q5 sum v1:v3 by id6": lambda t: gb(t["g"], ["id6"], {"v1": ("v1", "sum"), "v2": ("v2", "sum"), "v3": ("v3", "sum")}),
"q6 median, sd v3 by id4:id5": lambda t: gb(t["g"], ["id4", "id5"], {"median": ("v3", "median"), "sd": ("v3", "std")}),
"q7 max v1 - min v2 by id3": lambda t: (lambda r: r.assign(range=r["v1"] - r["v2"])[["id3", "range"]])(
gb(t["g"], ["id3"], {"v1": ("v1", "max"), "v2": ("v2", "min")})),
"q10 sum v3, count by id1:id6": lambda t: gb(t["g"], ["id1", "id2", "id3", "id4", "id5", "id6"], {"v3": ("v3", "sum"), "len": ("v1", "size")}),
"filter v3 > 50, sum v3": lambda t: t["g"]["v3"][t["g"]["v3"] > 50].sum(),
"sort by v3 desc, id6": lambda t: t["g"].sort_values(["v3", "id6"], ascending=[False, True], kind="stable"),
"top 10 by v3 desc, id6": lambda t: t["g"].sort_values(["v3", "id6"], ascending=[False, True], kind="stable").head(10),
"j1 inner small on id1": lambda t: t["x"].merge(t["small"], on="id1", how="inner"),
"j2 inner medium on id2": lambda t: t["x"].merge(t["medium"].drop(columns="id5"), on="id2", how="inner"),
"j3 left medium on id2": lambda t: t["x"].merge(t["medium"].drop(columns="id5"), on="id2", how="left"),
"j4 inner medium on id5 (string)": lambda t: t["x"].drop(columns="id2").merge(t["medium"].drop(columns="id2"), on="id5", how="inner"),
"j5 inner big on id3": lambda t: t["x"].merge(t["big"], on="id3", how="inner"),
}


def checksum_polars(df):
import polars as pl

total = 0.0
for name, dtype in df.schema.items():
if dtype.is_numeric():
s = df.get_column(name).cast(pl.Float64).fill_nan(None).sum()
total += s or 0.0
return {"rows": df.height, "sum": total}


def load(read, data, rows):
return {key: read(data / f"{name}-{rows}.arrow") for key, name in TABLES.items()}


def time_best(f, reps):
best = float("inf")
result = None
for _ in range(reps):
start = time.perf_counter()
result = f()
best = min(best, time.perf_counter() - start)
return best * 1000, result


def run_polars(data, rows, reps):
import polars as pl

tables = load(pl.read_ipc, data, rows)
out = {"version": pl.__version__, "threads": pl.thread_pool_size(), "queries": {}}
for name, q in polars_queries(pl).items():
ms, result = time_best(lambda: q(tables), reps)
out["queries"][name] = {"ms": ms, "checksum": checksum_polars(result)}
return out


def run_pandas(data, rows, reps):
import pandas as pd
import pyarrow.ipc as ipc

tables = load(lambda path: ipc.open_file(str(path)).read_all().to_pandas(), data, rows)
out = {"version": pd.__version__, "queries": {}}
for name, q in pandas_queries(pd).items():
ms, _ = time_best(lambda: q(tables), reps)
out["queries"][name] = {"ms": ms}
return out


def main():
parser = argparse.ArgumentParser()
parser.add_argument("command", choices=["generate", "run", "polars", "pandas"])
parser.add_argument("--rows", type=int, default=1_000_000)
parser.add_argument("--data", type=Path, default=DEFAULT_DATA)
parser.add_argument("--reps", type=int, default=3)
args = parser.parse_args()
if args.command == "generate":
generate(args.rows, args.data)
elif args.command in ("polars", "pandas"):
result = run_polars(args.data, args.rows, args.reps) if args.command == "polars" else run_pandas(args.data, args.rows, args.reps)
print(json.dumps(result))
else:
def child(command, env=None):
out = subprocess.run(
[sys.executable, __file__, command, "--rows", str(args.rows), "--data", str(args.data), "--reps", str(args.reps)],
check=True, capture_output=True, text=True, env={**os.environ, **(env or {})},
)
return json.loads(out.stdout)

baseline = {
"rows": args.rows,
"machine": {"cpus": os.cpu_count(), "platform": sys.platform},
"polars": child("polars"),
"polars1": child("polars", {"POLARS_MAX_THREADS": "1"}),
"pandas": child("pandas"),
}
path = args.data / f"baseline-{args.rows}.json"
path.write_text(json.dumps(baseline, indent=2) + "\n")
print(f"wrote {path}")
for name, q in baseline["polars"]["queries"].items():
print(f"{name:36} polars {q['ms']:9.1f} ms polars(1t) {baseline['polars1']['queries'][name]['ms']:9.1f} ms pandas {baseline['pandas']['queries'][name]['ms']:9.1f} ms")


if __name__ == "__main__":
main()
Loading