Skip to content

Commit 08c00e3

Browse files
mariadb-AlekseiBukhalovmariadb-AlekseiBukhalov
committed
mariadb-duckdb: switch to Parquet, remove PRIMARY KEY, add 2026-07-13 results
c6a.4xlarge, CI build 70621, no PRIMARY KEY: - Load time: 100.4 s, data size: 20.5 GB - Concurrent QPS: 1.530, error ratio: 0.000 - Sequential hot total: ~25.5 s
1 parent 2be17ef commit 08c00e3

7 files changed

Lines changed: 109 additions & 29 deletions

File tree

‎mariadb-duckdb/benchmark.sh‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,7 @@
11
#!/bin/bash
2-
export BENCH_DOWNLOAD_SCRIPT="download-hits-tsv"
2+
# Thin shim — actual flow is in ../lib/benchmark-common.sh.
3+
#export BENCH_DOWNLOAD_SCRIPT="download-hits-tsv"
4+
export BENCH_DOWNLOAD_SCRIPT="download-hits-parquet-single"
5+
export BENCH_DURABLE=yes
6+
export BENCH_RESTARTABLE=yes
37
exec ../lib/benchmark-common.sh

‎mariadb-duckdb/create.sql‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
set global duckdb_require_primary_key = OFF;
12
CREATE TABLE hits
23
(
34
WatchID BIGINT NOT NULL,
@@ -104,6 +105,5 @@ CREATE TABLE hits
104105
HasGCLID SMALLINT NOT NULL,
105106
RefererHash BIGINT NOT NULL,
106107
URLHash BIGINT NOT NULL,
107-
CLID INTEGER NOT NULL,
108-
PRIMARY KEY (CounterID, EventDate, UserID, EventTime, WatchID)
108+
CLID INTEGER NOT NULL
109109
) ENGINE=DuckDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;

‎mariadb-duckdb/install‎

Lines changed: 15 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
#!/bin/bash
22
set -eu
33

4-
CI_SOURCES_URL="https://ci.mariadb.org/68929/amd64-ubuntu-2404-deb-autobake/mariadb.sources"
4+
CI_SOURCES_URL="https://ci.mariadb.org/70621/amd64-ubuntu-2404-deb-autobake/mariadb.sources"
55

66
# Idempotent: if DuckDB plugin is already active, nothing to do.
77
if sudo mariadb -e "SELECT PLUGIN_STATUS FROM information_schema.PLUGINS WHERE PLUGIN_NAME='DuckDB';" 2>/dev/null | grep -q 'ACTIVE'; then
@@ -20,22 +20,23 @@ if dpkg -l | grep -qE '^ii\s+(mariadb-server|mysql-server)'; then
2020
fi
2121

2222
# Add MariaDB CI build as apt source (Trusted: yes — unsigned CI packages, testing only).
23-
sudo wget -q -O /etc/apt/sources.list.d/mariadb-duckdb-ci.sources "$CI_SOURCES_URL"
23+
#sudo wget -q -O /etc/apt/sources.list.d/mariadb-duckdb-ci.sources "$CI_SOURCES_URL"
2424

25-
sudo apt-get update -y
26-
sudo DEBIAN_FRONTEND=noninteractive apt-get install -y \
27-
mariadb-server mariadb-client
25+
#sudo apt-get update -y
26+
#sudo DEBIAN_FRONTEND=noninteractive apt-get install -y \
27+
# mariadb-server mariadb-client
2828

2929
# Configure the DuckDB storage engine plugin (from mariadb.org article defaults).
30-
sudo tee /etc/mysql/mariadb.conf.d/duckdb.cnf >/dev/null <<'EOF'
31-
[mysqld]
32-
plugin-maturity=alpha
33-
plugin-load-add=ha_duckdb.so
34-
duckdb-memory-limit=24G
35-
EOF
36-
37-
sudo systemctl enable mariadb
38-
sudo systemctl restart mariadb
30+
#sudo tee /etc/mysql/mariadb.conf.d/duckdb.cnf >/dev/null <<'EOF'
31+
#[mysqld]
32+
#plugin-maturity=alpha
33+
#plugin-load-add=ha_duckdb.so
34+
#duckdb-memory-limit=24G
35+
#duckdb-allow-run-in-duckdb=ON
36+
#EOF
37+
38+
#sudo systemctl enable mariadb
39+
#sudo systemctl restart mariadb
3940

4041
# Smoke-test: DuckDB plugin must be ACTIVE.
4142
sudo mariadb -e "SELECT PLUGIN_STATUS FROM information_schema.PLUGINS WHERE PLUGIN_NAME='DuckDB';" \

‎mariadb-duckdb/load‎

Lines changed: 17 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -13,22 +13,29 @@ sudo mariadb -e "DROP DATABASE IF EXISTS test"
1313
sudo mariadb -e "CREATE DATABASE test"
1414
sudo mariadb test < create.sql
1515

16-
# Load via DuckDB's native vectorized CSV reader (12x faster than LOAD DATA LOCAL
16+
# Load via DuckDB's native vectorized parquet reader (faster than LOAD DATA LOCAL
1717
# INFILE which routes every row through MariaDB's single-threaded write_row path).
18-
# hits.tsv is tab-delimited with datetime strings in TIMESTAMP columns, matching
19-
# the table schema — no type conversion needed.
2018
#
2119
# DuckDB glob expander requires read permission on all parent directories.
2220
# /home/ubuntu is mode 751 (o+x only) so DuckDB can traverse but not list it,
2321
# causing "No files found" errors. Hardlink to /var/lib/mysql/ (mysql-owned,
24-
# mode 755) resolves this instantly without copying 70 GB.
25-
HITS_LINK=/var/lib/mysql/hits_load.tsv
26-
sudo ln "$(realpath hits.tsv)" "$HITS_LINK"
27-
sudo chmod 644 "$HITS_LINK"
22+
# mode 755) resolves this instantly without copying 14 GB.
23+
HITS_LINK=/var/lib/mysql/hits.parquet
24+
if [ ! -f "$HITS_LINK" ]; then
25+
sudo ln "$(realpath hits.parquet)" "$HITS_LINK"
26+
sudo chmod 644 "$HITS_LINK"
27+
fi
2828
# NULLSTR '\N' matches MySQL/MariaDB convention: only \N is NULL, empty fields → ''.
2929
# Without this, DuckDB treats every empty TSV field as NULL → NOT NULL violations.
30-
duck "COPY test.hits FROM '$HITS_LINK' (FORMAT CSV, DELIMITER '\t', NULLSTR '\N')"
30+
# duck "COPY test.hits FROM '$HITS_LINK' (FORMAT CSV, DELIMITER '\t', NULLSTR '\N')"
31+
duck "INSERT INTO test.hits
32+
SELECT * REPLACE (
33+
make_date(EventDate) AS EventDate,
34+
epoch_ms(EventTime * 1000) AS EventTime,
35+
epoch_ms(ClientEventTime * 1000) AS ClientEventTime,
36+
epoch_ms(LocalEventTime * 1000) AS LocalEventTime)
37+
FROM read_parquet('$HITS_LINK', binary_as_string=True);"
3138

32-
sudo rm -f "$HITS_LINK"
33-
rm -f hits.tsv
39+
#sudo rm -f "$HITS_LINK"
40+
#rm -f hits.parquet
3441
sync

‎mariadb-duckdb/results/20260713/c6a.4xlarge.json‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
{
2+
"system": "MariaDB (DuckDB)",
3+
"date": "2026-07-13",
4+
"machine": "c6a.4xlarge",
5+
"cluster_size": 1,
6+
"proprietary": "no",
7+
"hardware": "cpu",
8+
"tuned": "no",
9+
"tags": ["C++", "column-oriented", "MySQL compatible", "DuckDB derivative"],
10+
"load_time": 100.378,
11+
"data_size": 20458778624,
12+
"concurrent_qps": 1.530,
13+
"concurrent_error_ratio": 0.000,
14+
"result": [
15+
[0.039, 0.002, 0.001],
16+
[0.145, 0.005, 0.005],
17+
[0.245, 0.021, 0.021],
18+
[0.316, 0.032, 0.032],
19+
[0.503, 0.301, 0.299],
20+
[1.004, 0.258, 0.261],
21+
[0.114, 0.003, 0.003],
22+
[0.163, 0.007, 0.007],
23+
[0.900, 0.378, 0.375],
24+
[1.613, 0.528, 0.525],
25+
[0.541, 0.072, 0.071],
26+
[0.782, 0.082, 0.080],
27+
[0.986, 0.333, 0.332],
28+
[2.308, 0.655, 0.677],
29+
[1.391, 0.382, 0.386],
30+
[0.548, 0.341, 0.340],
31+
[2.219, 0.779, 0.790],
32+
[1.977, 0.524, 0.532],
33+
[4.615, 2.761, 2.781],
34+
[0.221, 0.005, 0.005],
35+
[10.688, 0.484, 0.485],
36+
[12.324, 0.500, 0.498],
37+
[14.490, 1.657, 2.319],
38+
[0.596, 0.068, 0.066],
39+
[0.216, 0.021, 0.024],
40+
[0.922, 0.103, 0.102],
41+
[0.232, 0.021, 0.020],
42+
[10.865, 0.404, 0.405],
43+
[8.675, 6.651, 6.601],
44+
[0.201, 0.027, 0.026],
45+
[2.796, 0.287, 0.285],
46+
[5.822, 0.373, 0.380],
47+
[5.005, 1.663, 1.750],
48+
[11.170, 1.654, 1.649],
49+
[11.192, 1.852, 1.827],
50+
[0.610, 0.428, 0.431],
51+
[0.134, 0.029, 0.026],
52+
[0.122, 0.010, 0.009],
53+
[0.121, 0.015, 0.015],
54+
[0.229, 0.054, 0.064],
55+
[0.136, 0.006, 0.006],
56+
[0.148, 0.006, 0.006],
57+
[0.238, 0.150, 0.154]
58+
]
59+
}

‎mariadb-duckdb/start‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,4 +4,12 @@ set -eu
44
if sudo mariadb -e "SELECT 1" >/dev/null 2>&1; then
55
exit 0
66
fi
7-
sudo systemctl start mariadb
7+
/bin/sh /usr/bin/mariadbd-safe --datadir=/var/lib/mysql &>/dev/null &
8+
#sudo systemctl start mariadb
9+
# Wait up to 30s for the server to accept connections.
10+
for _ in $(seq 1 30); do
11+
sudo mariadb -e "SELECT 1" >/dev/null 2>&1 && exit 0
12+
sleep 1
13+
done
14+
echo "mariadbd-safe did not accept connections within 30s" >&2
15+
exit 1

‎mariadb-duckdb/stop‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,3 @@
11
#!/bin/bash
2-
sudo systemctl stop mariadb || true
2+
#sudo systemctl stop mariadb || true
3+
/usr/bin/mariadb-admin shutdown

0 commit comments

Comments
 (0)