|
| 1 | +""" |
| 2 | +Performance baseline for effect/dataframe: H2O.ai db-benchmark style group-by |
| 3 | +and join queries, run with Polars and pandas on the same data the TypeScript |
| 4 | +benchmark reads (benchmark/dataframe/h2o.ts). |
| 5 | +
|
| 6 | + python h2o.py generate --rows 1000000 [--data DIR] # writes Arrow IPC files |
| 7 | + python h2o.py run [--data DIR] [--reps 3] # writes DIR/baseline-<rows>.json |
| 8 | +
|
| 9 | +`run` times every query with Polars (all threads and one thread, each in its |
| 10 | +own process) and pandas, and records a checksum of every result so the |
| 11 | +TypeScript side can check it computed the same thing before comparing times. |
| 12 | +
|
| 13 | +Group-by data follows db-benchmark's groupby-datagen (K = 100 groups for |
| 14 | +id1/id2/id4/id5, N/K for id3/id6). Integer columns are Int32 on every side. |
| 15 | +The join queries are db-benchmark-style (small/medium/big right tables) on |
| 16 | +synthetic keys; see JOIN_QUERIES. |
| 17 | +""" |
| 18 | + |
| 19 | +import argparse |
| 20 | +import json |
| 21 | +import os |
| 22 | +import subprocess |
| 23 | +import sys |
| 24 | +import time |
| 25 | +from pathlib import Path |
| 26 | + |
| 27 | +HERE = Path(__file__).parent |
| 28 | +DEFAULT_DATA = HERE / ".data" |
| 29 | + |
| 30 | + |
| 31 | +def generate(rows, data): |
| 32 | + import numpy as np |
| 33 | + import pyarrow as pa |
| 34 | + import pyarrow.ipc as ipc |
| 35 | + |
| 36 | + rng = np.random.default_rng(20261114) |
| 37 | + n = rows |
| 38 | + k = 100 |
| 39 | + data.mkdir(parents=True, exist_ok=True) |
| 40 | + |
| 41 | + def ids(fmt, count, size): |
| 42 | + pool = np.array([fmt % (i + 1) for i in range(count)], dtype=object) |
| 43 | + return pa.array(pool[rng.integers(0, count, size)], type=pa.string()) |
| 44 | + |
| 45 | + v3_pool = np.round(rng.uniform(0, 100, 100), 6) |
| 46 | + groupby = pa.table({ |
| 47 | + "id1": ids("id%03d", k, n), |
| 48 | + "id2": ids("id%03d", k, n), |
| 49 | + "id3": ids("id%010d", n // k, n), |
| 50 | + "id4": pa.array(rng.integers(1, k + 1, n), type=pa.int32()), |
| 51 | + "id5": pa.array(rng.integers(1, k + 1, n), type=pa.int32()), |
| 52 | + "id6": pa.array(rng.integers(1, n // k + 1, n), type=pa.int32()), |
| 53 | + "v1": pa.array(rng.integers(1, 6, n), type=pa.int32()), |
| 54 | + "v2": pa.array(rng.integers(1, 16, n), type=pa.int32()), |
| 55 | + "v3": pa.array(v3_pool[rng.integers(0, 100, n)], type=pa.float64()), |
| 56 | + }) |
| 57 | + write(groupby, data / f"groupby-{rows}.arrow") |
| 58 | + |
| 59 | + x_id2 = rng.integers(1, 10_001, n) |
| 60 | + x = pa.table({ |
| 61 | + "id1": pa.array(rng.integers(1, 11, n), type=pa.int32()), |
| 62 | + "id2": pa.array(x_id2, type=pa.int32()), |
| 63 | + "id3": pa.array(rng.integers(1, n + 1, n), type=pa.int32()), |
| 64 | + "id5": pa.array(np.array([f"id{i}" for i in range(10_001)], dtype=object)[x_id2], type=pa.string()), |
| 65 | + "v1": pa.array(np.round(rng.uniform(0, 100, n), 6), type=pa.float64()), |
| 66 | + }) |
| 67 | + small = pa.table({"id1": pa.array(np.arange(1, 11), type=pa.int32()), "v2": pa.array(rng.uniform(0, 100, 10))}) |
| 68 | + medium_ids = np.arange(1, 10_001) |
| 69 | + medium = pa.table({ |
| 70 | + "id2": pa.array(medium_ids, type=pa.int32()), |
| 71 | + "id5": pa.array([f"id{i}" for i in medium_ids], type=pa.string()), |
| 72 | + "v2": pa.array(rng.uniform(0, 100, 10_000)), |
| 73 | + }) |
| 74 | + big = pa.table({"id3": pa.array(rng.permutation(n) + 1, type=pa.int32()), "v2": pa.array(rng.uniform(0, 100, n))}) |
| 75 | + for name, t in [("x", x), ("small", small), ("medium", medium), ("big", big)]: |
| 76 | + write(t, data / f"join-{name}-{rows}.arrow") |
| 77 | + print(f"wrote {rows} rows to {data}") |
| 78 | + |
| 79 | + |
| 80 | +def write(table, path): |
| 81 | + import pyarrow.ipc as ipc |
| 82 | + |
| 83 | + with ipc.new_file(str(path), table.schema) as w: |
| 84 | + w.write_table(table) |
| 85 | + |
| 86 | + |
| 87 | +# --------------------------------------------------------------------------- |
| 88 | +# Queries (Polars) |
| 89 | +# --------------------------------------------------------------------------- |
| 90 | + |
| 91 | + |
| 92 | +def polars_queries(pl): |
| 93 | + c = pl.col |
| 94 | + gb = lambda df, keys, *aggs: df.group_by(keys, maintain_order=True).agg(*aggs) |
| 95 | + return { |
| 96 | + "q1 sum v1 by id1": lambda t: gb(t["g"], ["id1"], c("v1").sum()), |
| 97 | + "q2 sum v1 by id1:id2": lambda t: gb(t["g"], ["id1", "id2"], c("v1").sum()), |
| 98 | + "q3 sum v1, mean v3 by id3": lambda t: gb(t["g"], ["id3"], c("v1").sum(), c("v3").mean()), |
| 99 | + "q4 mean v1:v3 by id4": lambda t: gb(t["g"], ["id4"], c("v1").mean(), c("v2").mean(), c("v3").mean()), |
| 100 | + "q5 sum v1:v3 by id6": lambda t: gb(t["g"], ["id6"], c("v1").sum(), c("v2").sum(), c("v3").sum()), |
| 101 | + "q6 median, sd v3 by id4:id5": lambda t: gb(t["g"], ["id4", "id5"], c("v3").median().alias("median"), c("v3").std().alias("sd")), |
| 102 | + "q7 max v1 - min v2 by id3": lambda t: gb(t["g"], ["id3"], (c("v1").max() - c("v2").min()).alias("range")), |
| 103 | + "q10 sum v3, count by id1:id6": lambda t: gb(t["g"], ["id1", "id2", "id3", "id4", "id5", "id6"], c("v3").sum(), pl.len()), |
| 104 | + "filter v3 > 50, sum v3": lambda t: t["g"].filter(c("v3") > 50).select(c("v3").sum()), |
| 105 | + "sort by v3 desc, id6": lambda t: t["g"].sort(["v3", "id6"], descending=[True, False], maintain_order=True), |
| 106 | + "top 10 by v3 desc, id6": lambda t: t["g"].lazy().sort(["v3", "id6"], descending=[True, False], maintain_order=True).head(10).collect(), |
| 107 | + "j1 inner small on id1": lambda t: t["x"].join(t["small"], on="id1", how="inner", maintain_order="left_right"), |
| 108 | + "j2 inner medium on id2": lambda t: t["x"].join(t["medium"].drop("id5"), on="id2", how="inner", maintain_order="left_right"), |
| 109 | + "j3 left medium on id2": lambda t: t["x"].join(t["medium"].drop("id5"), on="id2", how="left", maintain_order="left_right"), |
| 110 | + "j4 inner medium on id5 (string)": lambda t: t["x"].drop("id2").join(t["medium"].drop("id2"), on="id5", how="inner", maintain_order="left_right"), |
| 111 | + "j5 inner big on id3": lambda t: t["x"].join(t["big"], on="id3", how="inner", maintain_order="left_right"), |
| 112 | + } |
| 113 | + |
| 114 | + |
| 115 | +def pandas_queries(pd): |
| 116 | + gb = lambda df, keys, spec: df.groupby(keys, sort=False, observed=True).agg(**spec).reset_index() |
| 117 | + return { |
| 118 | + "q1 sum v1 by id1": lambda t: gb(t["g"], ["id1"], {"v1": ("v1", "sum")}), |
| 119 | + "q2 sum v1 by id1:id2": lambda t: gb(t["g"], ["id1", "id2"], {"v1": ("v1", "sum")}), |
| 120 | + "q3 sum v1, mean v3 by id3": lambda t: gb(t["g"], ["id3"], {"v1": ("v1", "sum"), "v3": ("v3", "mean")}), |
| 121 | + "q4 mean v1:v3 by id4": lambda t: gb(t["g"], ["id4"], {"v1": ("v1", "mean"), "v2": ("v2", "mean"), "v3": ("v3", "mean")}), |
| 122 | + "q5 sum v1:v3 by id6": lambda t: gb(t["g"], ["id6"], {"v1": ("v1", "sum"), "v2": ("v2", "sum"), "v3": ("v3", "sum")}), |
| 123 | + "q6 median, sd v3 by id4:id5": lambda t: gb(t["g"], ["id4", "id5"], {"median": ("v3", "median"), "sd": ("v3", "std")}), |
| 124 | + "q7 max v1 - min v2 by id3": lambda t: (lambda r: r.assign(range=r["v1"] - r["v2"])[["id3", "range"]])( |
| 125 | + gb(t["g"], ["id3"], {"v1": ("v1", "max"), "v2": ("v2", "min")})), |
| 126 | + "q10 sum v3, count by id1:id6": lambda t: gb(t["g"], ["id1", "id2", "id3", "id4", "id5", "id6"], {"v3": ("v3", "sum"), "len": ("v1", "size")}), |
| 127 | + "filter v3 > 50, sum v3": lambda t: t["g"]["v3"][t["g"]["v3"] > 50].sum(), |
| 128 | + "sort by v3 desc, id6": lambda t: t["g"].sort_values(["v3", "id6"], ascending=[False, True], kind="stable"), |
| 129 | + "top 10 by v3 desc, id6": lambda t: t["g"].sort_values(["v3", "id6"], ascending=[False, True], kind="stable").head(10), |
| 130 | + "j1 inner small on id1": lambda t: t["x"].merge(t["small"], on="id1", how="inner"), |
| 131 | + "j2 inner medium on id2": lambda t: t["x"].merge(t["medium"].drop(columns="id5"), on="id2", how="inner"), |
| 132 | + "j3 left medium on id2": lambda t: t["x"].merge(t["medium"].drop(columns="id5"), on="id2", how="left"), |
| 133 | + "j4 inner medium on id5 (string)": lambda t: t["x"].drop(columns="id2").merge(t["medium"].drop(columns="id2"), on="id5", how="inner"), |
| 134 | + "j5 inner big on id3": lambda t: t["x"].merge(t["big"], on="id3", how="inner"), |
| 135 | + } |
| 136 | + |
| 137 | + |
| 138 | +def checksum_polars(df): |
| 139 | + import polars as pl |
| 140 | + |
| 141 | + if not isinstance(df, pl.DataFrame): |
| 142 | + return {"rows": 1, "sum": float(df)} |
| 143 | + total = 0.0 |
| 144 | + for name, dtype in df.schema.items(): |
| 145 | + if dtype.is_numeric(): |
| 146 | + s = df.get_column(name).cast(pl.Float64).fill_nan(None).sum() |
| 147 | + total += s or 0.0 |
| 148 | + return {"rows": df.height, "sum": total} |
| 149 | + |
| 150 | + |
| 151 | +def load_polars(pl, data, rows): |
| 152 | + read = lambda name: pl.read_ipc(data / name) |
| 153 | + return { |
| 154 | + "g": read(f"groupby-{rows}.arrow"), |
| 155 | + "x": read(f"join-x-{rows}.arrow"), |
| 156 | + "small": read(f"join-small-{rows}.arrow"), |
| 157 | + "medium": read(f"join-medium-{rows}.arrow"), |
| 158 | + "big": read(f"join-big-{rows}.arrow"), |
| 159 | + } |
| 160 | + |
| 161 | + |
| 162 | +def time_best(f, reps): |
| 163 | + best = float("inf") |
| 164 | + result = None |
| 165 | + for _ in range(reps): |
| 166 | + start = time.perf_counter() |
| 167 | + result = f() |
| 168 | + best = min(best, time.perf_counter() - start) |
| 169 | + return best * 1000, result |
| 170 | + |
| 171 | + |
| 172 | +def run_polars(data, rows, reps): |
| 173 | + import polars as pl |
| 174 | + |
| 175 | + tables = load_polars(pl, data, rows) |
| 176 | + out = {"version": pl.__version__, "threads": pl.thread_pool_size(), "queries": {}} |
| 177 | + for name, q in polars_queries(pl).items(): |
| 178 | + ms, result = time_best(lambda: q(tables), reps) |
| 179 | + out["queries"][name] = {"ms": ms, "checksum": checksum_polars(result)} |
| 180 | + return out |
| 181 | + |
| 182 | + |
| 183 | +def run_pandas(data, rows, reps): |
| 184 | + import pandas as pd |
| 185 | + import pyarrow.ipc as ipc |
| 186 | + |
| 187 | + read = lambda name: ipc.open_file(str(data / name)).read_all().to_pandas() |
| 188 | + tables = { |
| 189 | + "g": read(f"groupby-{rows}.arrow"), |
| 190 | + "x": read(f"join-x-{rows}.arrow"), |
| 191 | + "small": read(f"join-small-{rows}.arrow"), |
| 192 | + "medium": read(f"join-medium-{rows}.arrow"), |
| 193 | + "big": read(f"join-big-{rows}.arrow"), |
| 194 | + } |
| 195 | + out = {"version": pd.__version__, "queries": {}} |
| 196 | + for name, q in pandas_queries(pd).items(): |
| 197 | + ms, _ = time_best(lambda: q(tables), reps) |
| 198 | + out["queries"][name] = {"ms": ms} |
| 199 | + return out |
| 200 | + |
| 201 | + |
| 202 | +def main(): |
| 203 | + parser = argparse.ArgumentParser() |
| 204 | + parser.add_argument("command", choices=["generate", "run", "polars", "pandas"]) |
| 205 | + parser.add_argument("--rows", type=int, default=1_000_000) |
| 206 | + parser.add_argument("--data", type=Path, default=DEFAULT_DATA) |
| 207 | + parser.add_argument("--reps", type=int, default=3) |
| 208 | + args = parser.parse_args() |
| 209 | + if args.command == "generate": |
| 210 | + generate(args.rows, args.data) |
| 211 | + elif args.command in ("polars", "pandas"): |
| 212 | + result = run_polars(args.data, args.rows, args.reps) if args.command == "polars" else run_pandas(args.data, args.rows, args.reps) |
| 213 | + print(json.dumps(result)) |
| 214 | + else: |
| 215 | + def child(command, env=None): |
| 216 | + out = subprocess.run( |
| 217 | + [sys.executable, __file__, command, "--rows", str(args.rows), "--data", str(args.data), "--reps", str(args.reps)], |
| 218 | + check=True, capture_output=True, text=True, env={**os.environ, **(env or {})}, |
| 219 | + ) |
| 220 | + return json.loads(out.stdout) |
| 221 | + |
| 222 | + baseline = { |
| 223 | + "rows": args.rows, |
| 224 | + "machine": {"cpus": os.cpu_count(), "platform": sys.platform}, |
| 225 | + "polars": child("polars"), |
| 226 | + "polars1": child("polars", {"POLARS_MAX_THREADS": "1"}), |
| 227 | + "pandas": child("pandas"), |
| 228 | + } |
| 229 | + path = args.data / f"baseline-{args.rows}.json" |
| 230 | + path.write_text(json.dumps(baseline, indent=2) + "\n") |
| 231 | + print(f"wrote {path}") |
| 232 | + for name, q in baseline["polars"]["queries"].items(): |
| 233 | + print(f"{name:36} polars {q['ms']:9.1f} ms polars(1t) {baseline['polars1']['queries'][name]['ms']:9.1f} ms pandas {baseline['pandas']['queries'][name]['ms']:9.1f} ms") |
| 234 | + |
| 235 | + |
| 236 | +if __name__ == "__main__": |
| 237 | + main() |
0 commit comments