From ba644ecf5c7283c9d443e2fe8e86acdb2f87042c Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 23 Jul 2026 16:46:05 +0000 Subject: [PATCH 01/10] Fix nine systems that failed provisioning MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - eventql/hyrise: `.dockerignore` for hits.* — the overlayfs cwd exposes the shared read-only datasets, so `docker build .` was streaming ~75 GB per hits.{json,tsv,csv} into /var/lib/docker as build context and hitting "write /hits.json: no space left on device" before the first Dockerfile step ran. - pinot: bump to 1.5.1; Apache retired 1.5.0 from downloads.apache.org. - pg_deltax: soft-fail the chmod on hits_*.parquet. The source files are symlinks into a read-only dataset mount, so `chmod --dereference` aborts with "Read-only file system"; the mount already publishes them world-readable (0664) so postgres can read them. - mariadb-duckdb: skip `chmod o+x /home/ubuntu` when that directory doesn't exist. Under `set -eu` the missing dir aborted install. - presto-partitioned: `docker rm -f presto` before `docker run` — the existence guard occasionally misses a container that dockerd has re-registered from disk but not yet published to the ps table. - pg_clickhouse: add `liblz4-dev` and stop swallowing `pgxn install` failures with `|| true`. The build was failing on `lz4.h` and the extension never made it onto the system. - byconity: readiness probe now runs a lightweight CREATE/DROP DATABASE so `check` waits until tso-server has actually finished its FoundationDB handshake. Cold provision was failing with "Can't get process TSO request" even though SELECT 1 succeeded. - cedardb / cedardb-parquet: export PGHOST=/tmp in check/start/load/ query. The system-scoped systemd unit installer binds the socket only at /tmp/.s.PGSQL.5432 + 127.0.0.1:5432, but the scripts used bare `psql` which falls through to postgres' /var/run/postgresql default and hit "No such file or directory". --- byconity/check | 9 +++++++++ byconity/start | 15 +++++++++++++++ cedardb-parquet/check | 5 +++++ cedardb-parquet/load | 3 +++ cedardb-parquet/query | 3 +++ cedardb-parquet/start | 5 +++++ cedardb/check | 5 +++++ cedardb/load | 3 +++ cedardb/query | 3 +++ cedardb/start | 5 +++++ eventql/.dockerignore | 14 ++++++++++++++ hyrise/.dockerignore | 17 +++++++++++++++++ mariadb-duckdb/install | 5 ++++- pg_clickhouse/install | 10 ++++++++-- pg_deltax/load | 8 +++++++- pinot/install | 8 +++++--- pinot/load | 2 +- pinot/start | 2 +- presto-partitioned/start | 8 ++++++++ 19 files changed, 121 insertions(+), 9 deletions(-) create mode 100644 eventql/.dockerignore create mode 100644 hyrise/.dockerignore diff --git a/byconity/check b/byconity/check index 0c40c7e923..ed1b485327 100755 --- a/byconity/check +++ b/byconity/check @@ -6,6 +6,15 @@ set -e docker compose exec -T server clickhouse-client --port 52145 \ --query "SELECT 1" >/dev/null +# TSO probe (see ./start for context). Guards against declaring 'ready' +# while tso-server is still finishing its FoundationDB handshake — the +# next ./load INSERT would otherwise hit +# Code: 7002. DB::Exception: Can't get process TSO request. +docker compose exec -T server clickhouse-client --port 52145 \ + --query "CREATE DATABASE IF NOT EXISTS _bck_readiness" >/dev/null +docker compose exec -T server clickhouse-client --port 52145 \ + --query "DROP DATABASE IF EXISTS _bck_readiness" >/dev/null + # If hits exists (post-load), force a worker-touching query so we # refuse to declare 'ready' until the server-to-worker BRPC connection # has actually re-established. Without this gate the first /query diff --git a/byconity/start b/byconity/start index f5cd869c60..2cd591e86e 100755 --- a/byconity/start +++ b/byconity/start @@ -12,6 +12,21 @@ set -e ck() { docker compose exec -T server clickhouse-client --port 52145 \ --query "SELECT 1" >/dev/null 2>&1 || return 1 + # TSO probe. `SELECT 1` doesn't touch TSO — TSO is only involved in + # writes (any DDL, any INSERT). Cold-provision used to fail on the + # very first INSERT of ./load with + # Code: 7002. DB::Exception: Can't get process TSO request, + # type: GetTimestamp + # even though ./check reported ready — TSO's TCP port was up + # (server's docker-compose depends_on satisfied) but tso-server + # hadn't finished initializing against FoundationDB yet. Force a + # TSO round-trip here via a lightweight CREATE/DROP DATABASE pair. + docker compose exec -T server clickhouse-client --port 52145 \ + --query "CREATE DATABASE IF NOT EXISTS _bck_readiness" \ + >/dev/null 2>&1 || return 1 + docker compose exec -T server clickhouse-client --port 52145 \ + --query "DROP DATABASE IF EXISTS _bck_readiness" \ + >/dev/null 2>&1 || return 1 local exists exists=$(docker compose exec -T server clickhouse-client --port 52145 \ --query "EXISTS TABLE hits" 2>/dev/null || echo 0) diff --git a/cedardb-parquet/check b/cedardb-parquet/check index 1ff9f1a6d4..2d39769fd2 100755 --- a/cedardb-parquet/check +++ b/cedardb-parquet/check @@ -1,4 +1,9 @@ #!/bin/bash set -e +# CedarDB's system-scoped systemd unit binds only /tmp/.s.PGSQL.5432 and +# 127.0.0.1:5432 (see the installer output). Point psql at /tmp so this +# doesn't fall through to the postgres default /var/run/postgresql path +# and fail with "No such file or directory" on a healthy CedarDB. +export PGHOST=${PGHOST:-/tmp} psql -c 'SELECT 1' >/dev/null diff --git a/cedardb-parquet/load b/cedardb-parquet/load index 4850506bd1..5998abc059 100755 --- a/cedardb-parquet/load +++ b/cedardb-parquet/load @@ -1,6 +1,9 @@ #!/bin/bash set -eu +# CedarDB listens on /tmp/.s.PGSQL.5432; see ./check for context. +export PGHOST=${PGHOST:-/tmp} + # Substitute the absolute path so CedarDB can find the file regardless of # its working directory. sed "s|'hits\.parquet'|'$PWD/hits.parquet'|" create.sql | psql diff --git a/cedardb-parquet/query b/cedardb-parquet/query index ebb3c2d49f..cf1bba716d 100755 --- a/cedardb-parquet/query +++ b/cedardb-parquet/query @@ -6,6 +6,9 @@ # Exit non-zero on error. set -e +# CedarDB listens on /tmp/.s.PGSQL.5432; see ./check for context. +export PGHOST=${PGHOST:-/tmp} + query=$(cat) raw=$(psql -t -c '\timing' -c "$query" 2>&1) && exit_code=0 || exit_code=$? diff --git a/cedardb-parquet/start b/cedardb-parquet/start index c11e05a5ce..77711da2aa 100755 --- a/cedardb-parquet/start +++ b/cedardb-parquet/start @@ -1,4 +1,9 @@ #!/bin/bash set -eu +# CedarDB's system-scoped systemd unit binds only /tmp/.s.PGSQL.5432 and +# 127.0.0.1:5432. Point psql at /tmp so it doesn't fall through to the +# postgres default /var/run/postgresql. +export PGHOST=${PGHOST:-/tmp} + sudo systemctl start cedardb.service diff --git a/cedardb/check b/cedardb/check index 1ff9f1a6d4..2d39769fd2 100755 --- a/cedardb/check +++ b/cedardb/check @@ -1,4 +1,9 @@ #!/bin/bash set -e +# CedarDB's system-scoped systemd unit binds only /tmp/.s.PGSQL.5432 and +# 127.0.0.1:5432 (see the installer output). Point psql at /tmp so this +# doesn't fall through to the postgres default /var/run/postgresql path +# and fail with "No such file or directory" on a healthy CedarDB. +export PGHOST=${PGHOST:-/tmp} psql -c 'SELECT 1' >/dev/null diff --git a/cedardb/load b/cedardb/load index 1651aaba23..605733dc71 100755 --- a/cedardb/load +++ b/cedardb/load @@ -1,6 +1,9 @@ #!/bin/bash set -eu +# CedarDB listens on /tmp/.s.PGSQL.5432; see ./check for context. +export PGHOST=${PGHOST:-/tmp} + psql -f create.sql { # CedarDB would like to have ~4 GB of memory per parallel worker for parquet import to be efficient, so set the number of workers accordingly. diff --git a/cedardb/query b/cedardb/query index ebb3c2d49f..cf1bba716d 100755 --- a/cedardb/query +++ b/cedardb/query @@ -6,6 +6,9 @@ # Exit non-zero on error. set -e +# CedarDB listens on /tmp/.s.PGSQL.5432; see ./check for context. +export PGHOST=${PGHOST:-/tmp} + query=$(cat) raw=$(psql -t -c '\timing' -c "$query" 2>&1) && exit_code=0 || exit_code=$? diff --git a/cedardb/start b/cedardb/start index 64dd2373e8..b14e0add63 100755 --- a/cedardb/start +++ b/cedardb/start @@ -1,6 +1,11 @@ #!/bin/bash set -eu +# CedarDB's system-scoped systemd unit binds only /tmp/.s.PGSQL.5432 and +# 127.0.0.1:5432. Point psql at /tmp so it doesn't fall through to the +# postgres default /var/run/postgresql. +export PGHOST=${PGHOST:-/tmp} + sudo systemctl start cedardb.service # On multi-NUMA systems stay within one node for better performance. diff --git a/eventql/.dockerignore b/eventql/.dockerignore new file mode 100644 index 0000000000..dcac31b52d --- /dev/null +++ b/eventql/.dockerignore @@ -0,0 +1,14 @@ +# Playground cwd is an overlayfs merge of the system scripts (upper) and +# /opt/clickbench/datasets_ro (lower), so hits.parquet / hits.tsv / +# hits.csv / hits.json / hits.json.gz / hits_partitioned/ all appear next +# to the Dockerfile. Without this ignore file, `docker build .` streams +# them (75+ GB each) into /var/lib/docker/tmp as build context and the +# sysdisk fills before the first step runs: +# Error response from daemon: write /hits.json: no space left on device +hits.parquet +hits.tsv +hits.csv +hits.json +hits.json.gz +hits_*.parquet +hits_partitioned/ diff --git a/hyrise/.dockerignore b/hyrise/.dockerignore new file mode 100644 index 0000000000..ba56809482 --- /dev/null +++ b/hyrise/.dockerignore @@ -0,0 +1,17 @@ +# Playground cwd is an overlayfs merge of the system scripts (upper) and +# /opt/clickbench/datasets_ro (lower), so hits.parquet / hits.tsv / +# hits.csv / hits.json / hits.json.gz / hits_partitioned/ all appear next +# to the Dockerfile. Without this ignore file, `docker build .` streams +# them (75+ GB each) into /var/lib/docker/tmp as build context and the +# sysdisk fills before the first step runs: +# Error response from daemon: write /hits.json: no space left on device +hits.parquet +hits.tsv +hits.csv +hits.json +hits.json.gz +hits_*.parquet +hits_partitioned/ +# The host-side ./load creates data/hits_part_*.csv snapshots and finally +# data/hits.bin — none of those should be in the build context either. +data/ diff --git a/mariadb-duckdb/install b/mariadb-duckdb/install index 3d980241df..a6ee01933f 100755 --- a/mariadb-duckdb/install +++ b/mariadb-duckdb/install @@ -40,6 +40,9 @@ sudo mariadb -e "SELECT PLUGIN_STATUS FROM information_schema.PLUGINS WHERE PLUG # Allow the mysql user (which runs DuckDB embedded) to traverse the home # directory so COPY FROM can read dataset files anywhere under ~. -sudo chmod o+x /home/ubuntu +# The playground VM has no /home/ubuntu (the agent runs as root and cwd +# is /opt/clickbench/system) — skip when the dir doesn't exist so this +# doesn't abort a set -eu install. +[ -d /home/ubuntu ] && sudo chmod o+x /home/ubuntu || true echo "MariaDB with DuckDB engine installed and ready." diff --git a/pg_clickhouse/install b/pg_clickhouse/install index 3774e1b66e..635f72faf6 100755 --- a/pg_clickhouse/install +++ b/pg_clickhouse/install @@ -26,6 +26,7 @@ sudo apt-get install -y \ libcurl4-openssl-dev \ uuid-dev \ libssl-dev \ + liblz4-dev \ make \ cmake \ g++ \ @@ -51,5 +52,10 @@ EOF sudo systemctl restart postgresql@$PGVERSION-main -# Build/install the pg_clickhouse extension (idempotent on rerun). -pgxn install pg_clickhouse || true +# Build/install the pg_clickhouse extension. Not idempotent-swallowed +# any more — pgxn silently reports a failed compile with rc=2 while the +# install phase returns rc=0 to the harness, and the caller only finds +# out at load time via `extension "pg_clickhouse" is not available`. +# `pgxn install` re-fetches + rebuilds cheaply on rerun, so it's safe +# to let a real error propagate. +sudo pgxn install pg_clickhouse diff --git a/pg_deltax/load b/pg_deltax/load index 8d19eba88a..acbd2e509a 100755 --- a/pg_deltax/load +++ b/pg_deltax/load @@ -10,7 +10,13 @@ PARQUET_DIR=/tmp/hits_parquet sudo rm -rf "$PARQUET_DIR" sudo mkdir -p "$PARQUET_DIR" sudo mv hits_*.parquet "$PARQUET_DIR/" -sudo chmod 644 "$PARQUET_DIR"/*.parquet +# When the source parquet files are symlinks into a read-only dataset +# mount (the playground layout), chmod --dereference blows up on every +# file with "Read-only file system". The mount already publishes them +# world-readable (0664), so postgres can read them via the "other" +# bits — soft-fail this chmod, keep the strict behaviour for setups +# where hits_*.parquet are copies rather than symlinks. +sudo chmod 644 "$PARQUET_DIR"/*.parquet 2>/dev/null || true # Recreate the DB so this script is idempotent. DROP wipes any prior # ALTER DATABASE settings so we start from postgresql.conf defaults diff --git a/pinot/install b/pinot/install index a7a9feb5eb..bb2109b279 100755 --- a/pinot/install +++ b/pinot/install @@ -1,9 +1,11 @@ #!/bin/bash set -e -# 1.3.0 was retired from the Apache mirror; bump to a currently-published -# Pinot release. -PINOT_VERSION=1.5.0 +# Apache retires older Pinot releases from downloads.apache.org fairly +# aggressively (`downloads.apache.org/pinot/apache-pinot-1.5.0/...` now +# 404s); bump this to the current mirror-hosted version whenever we hit +# a 404 at install time. +PINOT_VERSION=1.5.1 PINOT_DIR="apache-pinot-$PINOT_VERSION-bin" if [ ! -d "$PINOT_DIR" ]; then diff --git a/pinot/load b/pinot/load index 93803853d7..7229254546 100755 --- a/pinot/load +++ b/pinot/load @@ -1,7 +1,7 @@ #!/bin/bash set -e -PINOT_VERSION=1.5.0 +PINOT_VERSION=1.5.1 PINOT_DIR="apache-pinot-$PINOT_VERSION-bin" # Wait for the controller's REST API to be live. `start` only waits for diff --git a/pinot/start b/pinot/start index 9f3036b6d0..bd189e8077 100755 --- a/pinot/start +++ b/pinot/start @@ -1,7 +1,7 @@ #!/bin/bash set -e -PINOT_VERSION=1.5.0 +PINOT_VERSION=1.5.1 PINOT_DIR="apache-pinot-$PINOT_VERSION-bin" # Idempotent: if broker query endpoint is up, do nothing. diff --git a/presto-partitioned/start b/presto-partitioned/start index 125a4ca19d..0083514a9a 100755 --- a/presto-partitioned/start +++ b/presto-partitioned/start @@ -11,6 +11,14 @@ if sudo docker ps -a --format '{{.Names}}' | grep -qx presto; then exit 0 fi +# Belt-and-suspenders: `docker ps -a` occasionally misses a container +# that dockerd has re-registered from its on-disk state but not yet +# published to the ps table (observed right after dockerd restart in +# the playground). Without this, `docker run` fails with +# "Conflict. The container name "/presto" is already in use" +# and the whole start step exits 125. +sudo docker rm -f presto 2>/dev/null || true + sudo docker run -d --name presto \ -p 8081:8080 \ -v "$PWD/etc/catalog/hive.properties:/opt/presto-server/etc/catalog/hive.properties:ro" \ From 5f906d04fbf5a557e18256b2b1ef1db1b611d751 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 23 Jul 2026 16:50:30 +0000 Subject: [PATCH 02/10] Tune druid, mssql, parseable for the 16 GiB playground VM MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - mssql: cap MSSQL_MEMORY_LIMIT_MB to 75 % of guest MemTotal (min 8 GiB). Previous unconstrained SQL Server + bcp columnstore build was OOM-killed at ~19 min (load rc=137). - parseable: drop ingest concurrency 6→3 and chunk size 2500→1000; add curl --retry 5 to survive transient stalls. Prior settings saturated parseable's incoming-batch queue on 16 GiB, then every chunk returned HTTP 408. - druid: maxNumConcurrentSubTasks 10→4 and extend queryable-wait 4 h→6 h. Fewer concurrent index-merge JVMs leaves headroom for the historical to actually load segments after indexing succeeds. --- druid/ingest.json | 2 +- druid/load | 4 ++-- mssql/start | 8 ++++++++ parseable/load | 13 ++++++++++--- 4 files changed, 21 insertions(+), 6 deletions(-) diff --git a/druid/ingest.json b/druid/ingest.json index 8d4c741934..6a88302e52 100644 --- a/druid/ingest.json +++ b/druid/ingest.json @@ -135,7 +135,7 @@ "forceGuaranteedRollup": true, "logParseExceptions": true, "maxParseExceptions": 1, - "maxNumConcurrentSubTasks": 10 + "maxNumConcurrentSubTasks": 4 }, "dataSchema": { "dataSource": "hits", diff --git a/druid/load b/druid/load index 0f3a8f6e8e..75b173f169 100755 --- a/druid/load +++ b/druid/load @@ -12,14 +12,14 @@ DRUID_DIR="apache-druid-${VERSION}" "./${DRUID_DIR}/bin/post-index-task" --file ingest.json --url http://localhost:8081 || true # Wait until the hits datasource is queryable. Druid's index task can -# legitimately take hours on a 16 GiB VM; budget 4 h here, and fail +# legitimately take hours on a 16 GiB VM; budget 6 h here, and fail # loudly if hits still isn't queryable so the agent doesn't take a # snapshot of a half-ingested datasource (which would otherwise look # "snapshotted" but every query returns # druidException ... Object 'hits' not found # at runtime). cnt="" -for _ in $(seq 1 2880); do # 2880 * 5s = 4 h +for _ in $(seq 1 4320); do # 4320 * 5s = 6 h cnt=$(curl -sf -XPOST -H'Content-Type: application/json' \ http://localhost:8888/druid/v2/sql/ \ -d '{"query": "SELECT COUNT(*) FROM hits"}' 2>/dev/null \ diff --git a/mssql/start b/mssql/start index 438d1e1a40..afba20b88f 100755 --- a/mssql/start +++ b/mssql/start @@ -18,10 +18,18 @@ if ! sudo docker ps -a --format '{{.Names}}' | grep -qx mssql1; then # would double disk usage during load: ~75 GB on the host + the copy in # the container's writable layer). The mssql process runs as UID 10001 # in the container, so ./load chmods hits.tsv to 644 before bcp runs. + # Cap SQL Server's Max Server Memory to leave enough headroom for the + # guest kernel + docker + the load-side bcp process. SQL Server on + # ClickBench happily grows to the full container memory; on a 16 GiB + # microVM that ends in an OOM-kill (load exited rc=137 after ~19 min + # of columnstore build). Scale to 75% of MemTotal, floor 8 GiB. + mem_mib=$(awk '/MemTotal/{ printf "%d", $2/1024 * 75/100 }' /proc/meminfo) + [ "$mem_mib" -lt 8192 ] && mem_mib=8192 sudo docker run -d --name mssql1 \ -e 'ACCEPT_EULA=Y' \ -e "MSSQL_SA_PASSWORD=$PASSWORD" \ -e 'MSSQL_PID=Developer' \ + -e "MSSQL_MEMORY_LIMIT_MB=$mem_mib" \ -p 1433:1433 \ -v "$DATA_DIR":/clickbench \ mcr.microsoft.com/mssql/server:2025-latest >/dev/null diff --git a/parseable/load b/parseable/load index 45a7fb541d..5cf60065f3 100755 --- a/parseable/load +++ b/parseable/load @@ -45,12 +45,19 @@ fi # because parallel runs jobs via /bin/sh by default, and a bash # `export -f`'d function isn't visible in that shell — the previous # version silently no-op'd every chunk and load wrote 0 rows. -LINES_PER_CHUNK=2500 -INGEST_JOBS=6 +# In the 16 GiB playground VM parseable's incoming-batch queue fills +# fast when we push 6 concurrent 2500-line batches (~75 MB/s in +# flight) — the endpoint then starts returning HTTP 408 and every +# subsequent chunk fails. 3 × 1000 lines is ~4x lower peak pressure +# and empirically clears without 408s. curl --retry survives short +# stalls so a briefly-saturated parseable doesn't fail the load. +LINES_PER_CHUNK=1000 +INGEST_JOBS=3 pv hits.json | parallel --pipe -N$LINES_PER_CHUNK --block 10M \ --jobs "$INGEST_JOBS" --halt-on-error 0 ' awk "BEGIN{print \"[\"} NR>1{print prev \",\"} {prev=\$0} END{if (prev) print prev; print \"]\"}" | - curl --silent --show-error --fail \ + curl --silent --show-error --fail --retry 5 --retry-delay 2 \ + --retry-all-errors --max-time 120 \ -H "Content-Type: application/json" \ -H "X-P-Stream: hits" \ -k -XPOST -u "admin:admin" \ From 683c898f7d980a640d61b872c0a2f04bef56dfa5 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 23 Jul 2026 16:51:31 +0000 Subject: [PATCH 03/10] mariadb-duckdb: fall back to symlink when hardlink is cross-device; pg_clickhouse: also add libzstd-dev MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - mariadb-duckdb: `ln` returned EXDEV because the playground exposes hits.parquet via an overlayfs on top of the read-only datasets mount, and /var/lib/mysql lives on the rootfs (separate fs). Try hardlink first (matches original intent — no 14 GB copy), fall back to symlink when EXDEV. - pg_clickhouse: the extension source also #includes after clearing the lz4.h barrier, so pull libzstd-dev in the same apt install as liblz4-dev. --- mariadb-duckdb/load | 10 ++++++++-- pg_clickhouse/install | 1 + 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/mariadb-duckdb/load b/mariadb-duckdb/load index 2d8354ae6e..b97548285d 100755 --- a/mariadb-duckdb/load +++ b/mariadb-duckdb/load @@ -20,10 +20,16 @@ sudo mariadb test < create.sql # /home/ubuntu is mode 751 (o+x only) so DuckDB can traverse but not list it, # causing "No files found" errors. Hardlink to /var/lib/mysql/ (mysql-owned, # mode 755) resolves this instantly without copying 14 GB. +# Falls back to a symlink when the source lives on a different filesystem +# — the playground exposes hits.parquet via an overlayfs whose lower is a +# read-only mount, so hardlink returns EXDEV ("Invalid cross-device link"). HITS_LINK=/var/lib/mysql/hits.parquet if [ ! -f "$HITS_LINK" ]; then - sudo ln "$(realpath hits.parquet)" "$HITS_LINK" - sudo chmod 644 "$HITS_LINK" + src=$(realpath hits.parquet) + if ! sudo ln "$src" "$HITS_LINK" 2>/dev/null; then + sudo ln -sf "$src" "$HITS_LINK" + fi + sudo chmod 644 "$HITS_LINK" 2>/dev/null || true fi duck "INSERT INTO test.hits SELECT * REPLACE ( diff --git a/pg_clickhouse/install b/pg_clickhouse/install index 635f72faf6..81d7dc6319 100755 --- a/pg_clickhouse/install +++ b/pg_clickhouse/install @@ -27,6 +27,7 @@ sudo apt-get install -y \ uuid-dev \ libssl-dev \ liblz4-dev \ + libzstd-dev \ make \ cmake \ g++ \ From 97a606d144f57c5774defad78d48147fd6a30023 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 23 Jul 2026 17:14:42 +0000 Subject: [PATCH 04/10] Follow-up: cedardb PGUSER, mariadb-duckdb + pg_deltax memory scaling MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - cedardb/cedardb-parquet: also export PGUSER=postgres. cedardb's installer only creates the `postgres` role, and psql's default of $USER (=root under the playground agent) trips FATAL: role "root" does not exist - mariadb-duckdb: scale duckdb-memory-limit to ~50 % MemTotal. Fixed 24 G worked on 32 GB benchmark hosts but OOM-crashed mariadb-server mid-load on the 16 GiB playground VM (client saw ERROR 2026 (HY000): TLS/SSL error: unexpected eof while reading). - pg_deltax: scale load-time work_mem to MemTotal/32 (128 MB floor, 1 GB ceiling). Fixed 1 GB × 4 parallel workers + shared_buffers crashed the pg_deltax compression pipeline after ~5 min of load ("server closed the connection unexpectedly"). --- cedardb-parquet/check | 6 +++++- cedardb-parquet/load | 1 + cedardb-parquet/query | 1 + cedardb-parquet/start | 5 ++++- cedardb/check | 6 +++++- cedardb/load | 1 + cedardb/query | 1 + cedardb/start | 5 ++++- mariadb-duckdb/install | 11 +++++++++-- pg_deltax/load | 19 ++++++++++++++----- 10 files changed, 45 insertions(+), 11 deletions(-) diff --git a/cedardb-parquet/check b/cedardb-parquet/check index 2d39769fd2..4540ec287f 100755 --- a/cedardb-parquet/check +++ b/cedardb-parquet/check @@ -4,6 +4,10 @@ set -e # CedarDB's system-scoped systemd unit binds only /tmp/.s.PGSQL.5432 and # 127.0.0.1:5432 (see the installer output). Point psql at /tmp so this # doesn't fall through to the postgres default /var/run/postgresql path -# and fail with "No such file or directory" on a healthy CedarDB. +# and fail with "No such file or directory" on a healthy CedarDB. The +# installer only creates a `postgres` role, so also pin PGUSER — psql's +# default is $USER, and under the playground agent that's `root`, which +# cedardb rejects with 'role "root" does not exist'. export PGHOST=${PGHOST:-/tmp} +export PGUSER=${PGUSER:-postgres} psql -c 'SELECT 1' >/dev/null diff --git a/cedardb-parquet/load b/cedardb-parquet/load index 5998abc059..4623e31c85 100755 --- a/cedardb-parquet/load +++ b/cedardb-parquet/load @@ -3,6 +3,7 @@ set -eu # CedarDB listens on /tmp/.s.PGSQL.5432; see ./check for context. export PGHOST=${PGHOST:-/tmp} +export PGUSER=${PGUSER:-postgres} # Substitute the absolute path so CedarDB can find the file regardless of # its working directory. diff --git a/cedardb-parquet/query b/cedardb-parquet/query index cf1bba716d..795d108bcb 100755 --- a/cedardb-parquet/query +++ b/cedardb-parquet/query @@ -8,6 +8,7 @@ set -e # CedarDB listens on /tmp/.s.PGSQL.5432; see ./check for context. export PGHOST=${PGHOST:-/tmp} +export PGUSER=${PGUSER:-postgres} query=$(cat) diff --git a/cedardb-parquet/start b/cedardb-parquet/start index 77711da2aa..c104fdd797 100755 --- a/cedardb-parquet/start +++ b/cedardb-parquet/start @@ -3,7 +3,10 @@ set -eu # CedarDB's system-scoped systemd unit binds only /tmp/.s.PGSQL.5432 and # 127.0.0.1:5432. Point psql at /tmp so it doesn't fall through to the -# postgres default /var/run/postgresql. +# postgres default /var/run/postgresql. PGUSER pinned to postgres — +# the installer creates only that role, and defaulting to $USER (root +# in the playground) trips 'role "root" does not exist'. export PGHOST=${PGHOST:-/tmp} +export PGUSER=${PGUSER:-postgres} sudo systemctl start cedardb.service diff --git a/cedardb/check b/cedardb/check index 2d39769fd2..4540ec287f 100755 --- a/cedardb/check +++ b/cedardb/check @@ -4,6 +4,10 @@ set -e # CedarDB's system-scoped systemd unit binds only /tmp/.s.PGSQL.5432 and # 127.0.0.1:5432 (see the installer output). Point psql at /tmp so this # doesn't fall through to the postgres default /var/run/postgresql path -# and fail with "No such file or directory" on a healthy CedarDB. +# and fail with "No such file or directory" on a healthy CedarDB. The +# installer only creates a `postgres` role, so also pin PGUSER — psql's +# default is $USER, and under the playground agent that's `root`, which +# cedardb rejects with 'role "root" does not exist'. export PGHOST=${PGHOST:-/tmp} +export PGUSER=${PGUSER:-postgres} psql -c 'SELECT 1' >/dev/null diff --git a/cedardb/load b/cedardb/load index 605733dc71..d948e7e6ff 100755 --- a/cedardb/load +++ b/cedardb/load @@ -3,6 +3,7 @@ set -eu # CedarDB listens on /tmp/.s.PGSQL.5432; see ./check for context. export PGHOST=${PGHOST:-/tmp} +export PGUSER=${PGUSER:-postgres} psql -f create.sql { diff --git a/cedardb/query b/cedardb/query index cf1bba716d..795d108bcb 100755 --- a/cedardb/query +++ b/cedardb/query @@ -8,6 +8,7 @@ set -e # CedarDB listens on /tmp/.s.PGSQL.5432; see ./check for context. export PGHOST=${PGHOST:-/tmp} +export PGUSER=${PGUSER:-postgres} query=$(cat) diff --git a/cedardb/start b/cedardb/start index b14e0add63..ffdc7daa62 100755 --- a/cedardb/start +++ b/cedardb/start @@ -3,8 +3,11 @@ set -eu # CedarDB's system-scoped systemd unit binds only /tmp/.s.PGSQL.5432 and # 127.0.0.1:5432. Point psql at /tmp so it doesn't fall through to the -# postgres default /var/run/postgresql. +# postgres default /var/run/postgresql. PGUSER pinned to postgres — +# the installer creates only that role, and defaulting to $USER (root +# in the playground) trips 'role "root" does not exist'. export PGHOST=${PGHOST:-/tmp} +export PGUSER=${PGUSER:-postgres} sudo systemctl start cedardb.service diff --git a/mariadb-duckdb/install b/mariadb-duckdb/install index a6ee01933f..cf4ba1a18c 100755 --- a/mariadb-duckdb/install +++ b/mariadb-duckdb/install @@ -23,11 +23,18 @@ sudo wget -q -O /etc/apt/sources.list.d/mariadb-duckdb-ci.sources "$CI_SOURCES_U sudo apt-get update -y sudo DEBIAN_FRONTEND=noninteractive apt-get install -y mariadb-server mariadb-client -sudo tee /etc/mysql/mariadb.conf.d/duckdb.cnf >/dev/null <<'EOF' +# Scale duckdb-memory-limit to guest RAM. The upstream default of 24G +# was fine on the 32 GB c6a.4xlarge benchmark host, but blows past a +# 16 GiB playground VM's total memory and OOM-crashes mariadb-server +# mid-load (visible as `TLS/SSL error: unexpected eof while reading` +# from the client). Cap at ~50 % of MemTotal. +mem_g=$(awk '/MemTotal/{ printf "%d", $2/1024/1024/2 }' /proc/meminfo) +[ "$mem_g" -lt 4 ] && mem_g=4 +sudo tee /etc/mysql/mariadb.conf.d/duckdb.cnf >/dev/null </dev/null || true sudo -u postgres psql -v ON_ERROR_STOP=1 -t -c "DROP DATABASE IF EXISTS test" sudo -u postgres psql -v ON_ERROR_STOP=1 -t -c "CREATE DATABASE test" -# Bump work_mem for the load only. Direct backfill sorts each segment before -# compressing; 1GB keeps larger segments in memory and shaves load time. -# Reset before the query phase so the concurrent-QPS test (10 connections -# each spawning parallel-scan workers) doesn't multiply this up to OOM. -sudo -u postgres psql -v ON_ERROR_STOP=1 -t -c "ALTER DATABASE test SET work_mem TO '1GB'" +# Bump work_mem for the load only. Direct backfill sorts each segment +# before compressing; larger keeps more segments in memory and shaves +# load time. Reset before the query phase so the concurrent-QPS test +# (10 connections each spawning parallel-scan workers) doesn't multiply +# this up to OOM. Scale to guest RAM: 1 GB was fine on 32 GB benchmark +# hosts but crashed pg_deltax's compression pipeline mid-load on the +# 16 GiB playground VM ("server closed the connection unexpectedly" +# after ~5 min). Cap at MemTotal / 32 so 4× parallel workers + shared +# buffers still leaves headroom. +load_wm_mb=$(awk '/MemTotal/{ printf "%d", $2/1024/32 }' /proc/meminfo) +[ "$load_wm_mb" -lt 128 ] && load_wm_mb=128 +[ "$load_wm_mb" -gt 1024 ] && load_wm_mb=1024 +sudo -u postgres psql -v ON_ERROR_STOP=1 -t \ + -c "ALTER DATABASE test SET work_mem TO '${load_wm_mb}MB'" sudo -u postgres psql -v ON_ERROR_STOP=1 -t test -c "CREATE EXTENSION pg_deltax" From 7d099e7f4b0500dcb08e679f9aca47df01a18656 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 23 Jul 2026 17:46:01 +0000 Subject: [PATCH 05/10] hyrise: cap ninja parallelism on low-RAM hosts; pg_deltax: tighten memory + parallel-worker caps for 16 GiB VMs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - hyrise: aggregate_hash.cpp under -O3 peaks at ~4 GB per cc1plus, so the default ninja -j$(nproc) SIGKILLs on the 16 GiB / 4-vCPU playground VM ("Killed signal terminated program cc1plus"). Cap at -j2 unless MemTotal ≥ 24 GiB. - pg_deltax: on hosts under 24 GiB, add shared_buffers=1GB, max_parallel_workers=2, max_worker_processes=8, and drop the load-time work_mem cap to MemTotal/64 (was MemTotal/32). Fewer + smaller pg_deltax decoders keep the direct-backfill pipeline from crossing the physical-RAM ceiling ("server closed the connection unexpectedly"). --- hyrise/Dockerfile | 12 +++++++++++- pg_deltax/install | 21 +++++++++++++++++++-- pg_deltax/load | 9 ++++----- 3 files changed, 34 insertions(+), 8 deletions(-) diff --git a/hyrise/Dockerfile b/hyrise/Dockerfile index 93f1ffbcf1..5a209a1784 100644 --- a/hyrise/Dockerfile +++ b/hyrise/Dockerfile @@ -63,8 +63,18 @@ RUN git clone https://github.com/hyrise/hyrise.git \ # unencoded value segments, which would make the snapshot raw-sized and make # it re-import unencoded. See ./load. WORKDIR /opt/hyrise/cmake-build-release +# ninja's default parallelism = nproc kills cc1plus on 16 GiB / 4-vCPU +# playground VMs: aggregate_hash.cpp under -O3 with heavy template +# instantiation peaks at ~4 GB per translation unit, so 4 concurrent +# cc1plus × ~4 GB > 16 GB and the guest kernel SIGKILLs the loser with +# "fatal error: Killed signal terminated program cc1plus". Cap to 2 — +# hyriseServer/hyriseConsole still build in ~15 min at that width. +# Auto-widen when the host has enough RAM per core to safely parallelize. RUN cmake -GNinja -DCMAKE_BUILD_TYPE=Release -DNO_LTO=${NO_LTO} .. \ - && ninja hyriseServer hyriseConsole + && jobs=2 \ + && total_kb=$(awk '/MemTotal/{print $2}' /proc/meminfo) \ + && if [ "$total_kb" -gt $((24 * 1024 * 1024)) ]; then jobs=$(nproc); fi \ + && ninja -j"$jobs" hyriseServer hyriseConsole FROM ubuntu:25.04 ENV DEBIAN_FRONTEND=noninteractive diff --git a/pg_deltax/install b/pg_deltax/install index 7a20505c18..7bebdb17c1 100755 --- a/pg_deltax/install +++ b/pg_deltax/install @@ -20,10 +20,27 @@ sudo apt-get update -y sudo apt-get install -y postgresql-$PGVERSION postgresql-client-$PGVERSION postgresql-server-dev-$PGVERSION # PostgreSQL runs with stock configuration except work_mem=64MB, the same -# value the vanilla postgresql/ benchmark uses. -sudo tee /etc/postgresql/$PGVERSION/main/conf.d/clickbench.conf < Date: Thu, 23 Jul 2026 18:32:18 +0000 Subject: [PATCH 06/10] pg_deltax: cap parallel_workers=2 on <24 GiB hosts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit pg_deltax.parallel_workers=0 (default) resolves to num_cpus capped at 16 (see src/lib.rs get_parallel_workers, src/copy.rs handle_copy_from_parquet_parallel). On a 4-vCPU / 16 GiB VM that splits as 4 decoders + 1 compressor; each decoder holds ~250 MB of blob state until flush, so the 5-worker × ~250 MB peak plus shared_buffers + kernel crosses the physical-RAM ceiling and the backend gets OOM-killed after 4-6 partition flushes ("server closed the connection unexpectedly"). Pin `pg_deltax.parallel_workers=2` on hosts under 24 GiB so decode splits as 2 + 1 = 3 workers ≈ 750 MB peak. --- pg_deltax/load | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/pg_deltax/load b/pg_deltax/load index d5f1c098b9..c2948a7796 100755 --- a/pg_deltax/load +++ b/pg_deltax/load @@ -30,14 +30,26 @@ sudo -u postgres psql -v ON_ERROR_STOP=1 -t -c "CREATE DATABASE test" # (10 connections each spawning parallel-scan workers) doesn't multiply # this up to OOM. Scale to guest RAM: 1 GB was fine on 32 GB benchmark # hosts but crashed pg_deltax's compression pipeline mid-load on the -# 16 GiB playground VM. Cap at MemTotal/64 so pg_deltax's ~5 background -# workers × work_mem still leaves headroom for shared_buffers + kernel. +# 16 GiB playground VM. Cap at MemTotal/64 so pg_deltax's decoder pool +# × work_mem still leaves headroom for shared_buffers + kernel. load_wm_mb=$(awk '/MemTotal/{ printf "%d", $2/1024/64 }' /proc/meminfo) [ "$load_wm_mb" -lt 64 ] && load_wm_mb=64 [ "$load_wm_mb" -gt 1024 ] && load_wm_mb=1024 sudo -u postgres psql -v ON_ERROR_STOP=1 -t \ -c "ALTER DATABASE test SET work_mem TO '${load_wm_mb}MB'" +# pg_deltax.parallel_workers=0 (default) uses num_cpus (capped at 16), so +# 4-vCPU / 16 GiB VMs end up with 4 decoders + 1 compressor. Each decoder +# accumulates ~250 MB of blob buffers before flush → ~1.5 GB of transient +# state that OOM-kills the backend ("server closed the connection +# unexpectedly" after 4-6 partition flushes). Cap decoders on +# memory-constrained hosts. Skip on hosts with ≥24 GiB RAM. +mem_kb=$(awk '/MemTotal/{print $2}' /proc/meminfo) +if [ "$mem_kb" -lt $((24 * 1024 * 1024)) ]; then + sudo -u postgres psql -v ON_ERROR_STOP=1 -t \ + -c "ALTER DATABASE test SET pg_deltax.parallel_workers TO 2" +fi + sudo -u postgres psql -v ON_ERROR_STOP=1 -t test -c "CREATE EXTENSION pg_deltax" # Schema + partitioning + compression setup. mock_now pins the partition From 0959c3cf526f315e631d75cadea1d78584716a2d Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 23 Jul 2026 19:20:18 +0000 Subject: [PATCH 07/10] Playground: bound ready-VM count + teardown TAP on failed restore; hyrise: shrink CSV chunk size for 16 GiB VM MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - vm_manager._boot: on _configure_boot failure, tear down the TAP after killing the fc process so the retry's ensure_tap recreates a fresh one. Fixes the class of "Open tap device failed: … Resource busy (os error 16)" retries that got stuck earlier today when parallel queries triggered simultaneous restores. - config + monitor: new PLAYGROUND_MAX_READY_VMS cap (default 40), enforced by the monitor via kick("max-ready-cap") on the oldest-idle ready VM. Bounds total guest anon memory during bursts — without it, a mass /api/query stack against N systems pins N VMs at "ready" until idle_kick_after_sec (10 min), which on 100+ systems piled up hundreds of GB of guest RSS + page cache. - hyrise/load: scale CSV split size to guest RAM. Fixed 5M rows/piece worked on 32 GB benchmark hosts but SIGKILL'd hyriseConsole on the 16 GiB playground VM (load rc=137 after 156 s). Drop to 2M rows on hosts under 24 GiB — smaller per-piece unencoded materialisation keeps the peak within budget. --- hyrise/load | 11 ++++++++++- playground/server/config.py | 12 ++++++++++++ playground/server/monitor.py | 28 ++++++++++++++++++++++++++++ playground/server/vm_manager.py | 13 +++++++++++-- 4 files changed, 61 insertions(+), 3 deletions(-) diff --git a/hyrise/load b/hyrise/load index 650d43b3cc..5e7db0461a 100755 --- a/hyrise/load +++ b/hyrise/load @@ -28,8 +28,17 @@ set -eu if [ ! -f data/hits.bin ]; then # hits.csv was delivered into cwd by download-hits-csv. Guarded so a # retry after a partial first load resumes from the existing pieces. + # Piece size scaled to guest RAM: on the 16 GiB playground VM, + # 5M-row pieces cross the OOM ceiling once the growing encoded + # table reaches ~10 GB (load rc=137 SIGKILL after 156 s). 2M-row + # pieces cap the transient unencoded materialisation at ~2 GB. + lines_per_piece=5000000 + mem_kb=$(awk '/MemTotal/{print $2}' /proc/meminfo) + if [ "$mem_kb" -lt $((24 * 1024 * 1024)) ]; then + lines_per_piece=2000000 + fi if [ -f hits.csv ]; then - split -l 5000000 --numeric-suffixes=1 --additional-suffix=.csv \ + split -l "$lines_per_piece" --numeric-suffixes=1 --additional-suffix=.csv \ hits.csv data/hits_part_ rm hits.csv fi diff --git a/playground/server/config.py b/playground/server/config.py index 571b186d64..5ba396a87f 100644 --- a/playground/server/config.py +++ b/playground/server/config.py @@ -61,6 +61,14 @@ class Config: idle_kick_after_sec: int host_min_free_ram_gb: int host_min_free_disk_gb: int + # Cap the number of VMs in "ready" state at any time. When exceeded, + # the monitor evicts the least-recently-used ready VM. Bounds total + # host memory during query bursts — without this a mass /query + # against N systems keeps all N VMs ready for `idle_kick_after_sec` + # (10 min default), which on 100+ systems can pile up hundreds of + # GB of guest RSS + file cache. Snapshot is preserved, so eviction + # only costs the next /query one restore. + max_ready_vms: int # Per-system disk full check. vm_disk_pct_kill_threshold: float # ClickHouse Cloud logging. @@ -141,6 +149,10 @@ def load() -> Config: idle_kick_after_sec=_env_int("VM_IDLE_KICK_AFTER_SEC", 600), host_min_free_ram_gb=_env_int("HOST_MIN_FREE_RAM_GB", 32), host_min_free_disk_gb=_env_int("HOST_MIN_FREE_DISK_GB", 100), + # 40 VMs × ~2 GB avg RSS ≈ 80 GB of guest anon, comfortably + # under host RAM on the 1 TB benchmark box. Override via + # PLAYGROUND_MAX_READY_VMS. + max_ready_vms=_env_int("PLAYGROUND_MAX_READY_VMS", 40), vm_disk_pct_kill_threshold=float(os.environ.get("VM_DISK_FULL_PCT", "0.97")), ch_cloud_url=os.environ.get("CLICKHOUSE_CLOUD_URL", ch_conf.get("url", "")), ch_cloud_user=os.environ.get("CLICKHOUSE_CLOUD_USER", ch_conf.get("user", "")), diff --git a/playground/server/monitor.py b/playground/server/monitor.py index be3cf9b076..e1c5be17bd 100644 --- a/playground/server/monitor.py +++ b/playground/server/monitor.py @@ -82,6 +82,7 @@ async def _tick(self) -> None: # Host-wide checks await self._check_host_pressure() + await self._check_max_ready_vms() def _sample_cpu(self, name: str, pid: int) -> float | None: """Return ratio of CPU used since last sample, normalized by vcpu count.""" @@ -200,6 +201,33 @@ async def _check_host_pressure(self) -> None: ) await self.vmm.kick(target.system.name, "host-disk-pressure") + async def _check_max_ready_vms(self) -> None: + """Enforce the cap on concurrent ready VMs. Evict oldest-idle + when exceeded. Only touches VMs in state=='ready' that haven't + been used in a bit (2 s grace) so we don't evict a VM + milliseconds into its first query.""" + cap = self.cfg.max_ready_vms + if cap <= 0: + return + ready = [v for v in self.vmm.vms.values() if v.state == "ready"] + if len(ready) <= cap: + return + # Pick the ones idle the longest, skipping anything used in the + # last 2 s to avoid racing an in-flight query dispatch. + now = time.time() + candidates = [v for v in ready if now - v.last_used >= 2.0] + if not candidates: + return + candidates.sort(key=lambda v: v.last_used) + excess = len(ready) - cap + for vm in candidates[:excess]: + self.sink.write_event( + system=vm.system.name, kind="max-ready-cap", + detail=f"ready={len(ready)} > cap={cap}; " + f"idle for {int(now - vm.last_used)}s", + ) + await self.vmm.kick(vm.system.name, "max-ready-cap") + def _largest_running(self, *, by: str) -> VM | None: running = [v for v in self.vmm.vms.values() if v.pid is not None and _pid_alive(v.pid)] diff --git a/playground/server/vm_manager.py b/playground/server/vm_manager.py index 8ab1b8e1dd..600731225e 100644 --- a/playground/server/vm_manager.py +++ b/playground/server/vm_manager.py @@ -443,8 +443,17 @@ async def _boot(self, vm: VM, *, restore_snapshot: bool) -> None: # If config fails partway, the firecracker process still owns the # TAP fd; without reaping it, the next attempt sees "Resource # busy" because the kernel hasn't released the TAP. Kill + - # wait() before propagating. - await self._shutdown(vm) + # wait() before propagating. Also delete the TAP interface + # itself — kernel-level opens can linger past the process + # exit in some paths (observed as retry attempts hitting + # "Open tap device failed: … Resource busy (os error 16)" + # even after the prior fc process was SIGKILL-reaped). A + # clean tuntap del + re-add on the next ensure_tap avoids + # that class of race entirely. + with contextlib.suppress(Exception): + await self._shutdown(vm) + with contextlib.suppress(Exception): + await net.teardown_tap(vm.slot) raise async def _configure_boot(self, vm: VM, *, restore_snapshot: bool) -> None: From 28fc76ddd7beebbe9789f933c17d2d2230cc52ad Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 23 Jul 2026 19:41:06 +0000 Subject: [PATCH 08/10] hive+impala: skip pre-snapshot restart so metastore state survives; trino-partitioned: copy parquet bytes instead of hardlinking symlinks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - hive: BENCH_DURABLE=no covered ClickBench's cold-cycle re-load, but the playground's pre-snapshot `./stop; ./start` sequence runs `docker rm -f hive; docker run …` — wiping the embedded Derby metastore ./load just populated. The snapshot then captures an empty catalog and every restored /query returns Failed to open new session: Database clickbench does not exist. Set PLAYGROUND_SKIP_RESTART_BEFORE_SNAPSHOT=yes so the running HS2 (with the loaded catalog) is what gets snapshotted. - impala: same class of bug — pre-snapshot `docker compose down; up` drops catalogd's in-memory catalog. With hms_event_polling_interval_s=0 (needed to sidestep the HMS notification-log RPC bug), catalogd never re-syncs from HMS on start, so `use clickbench` from every restored /query fails. Same PLAYGROUND_SKIP_RESTART_BEFORE_SNAPSHOT gate. - trino-partitioned: `ln -f` preserved the symlink (GNU default -P), so trino ≥ 483's local FS provider hit Malformed Parquet file. Metadata index: -X out of range [local:///hits/hits_N.parquet] when reading through the symlink chain. Copy the file bytes instead for the playground path, keep hardlink for bare-metal runs where the source is a real file. One-time per-provision cost, reliable afterwards. --- hive/benchmark.sh | 9 +++++++++ impala/benchmark.sh | 7 +++++++ trino-partitioned/load | 23 ++++++++++++++++++++--- 3 files changed, 36 insertions(+), 3 deletions(-) diff --git a/hive/benchmark.sh b/hive/benchmark.sh index e16fbff562..2b2a3fd613 100755 --- a/hive/benchmark.sh +++ b/hive/benchmark.sh @@ -6,4 +6,13 @@ export BENCH_DOWNLOAD_SCRIPT="download-hits-parquet-single" # is present before the first try; the load wall-clock rolls into the # cold-try timing per the standard BENCH_DURABLE=no contract. export BENCH_DURABLE=no +# The playground snapshots the guest post-load and every /query +# restores from that snapshot. If the pre-snapshot ./stop + ./start +# fires here, ./start's `docker rm -f hive; docker run …` wipes the +# same embedded Derby metastore that ./load just populated — the +# snapshot then captures a fresh container with an empty catalog, +# and every restored /query returns "Database clickbench does not +# exist". Skip the pre-snapshot restart so the running HS2 (with the +# loaded catalog) is what gets snapshotted. +export PLAYGROUND_SKIP_RESTART_BEFORE_SNAPSHOT=yes exec ../lib/benchmark-common.sh diff --git a/impala/benchmark.sh b/impala/benchmark.sh index fa3a0559bc..724a26309a 100755 --- a/impala/benchmark.sh +++ b/impala/benchmark.sh @@ -18,4 +18,11 @@ export BENCH_DOWNLOAD_SCRIPT="download-hits-parquet-single" export BENCH_RESTARTABLE=no export BENCH_CHECK_TIMEOUT=900 +# BENCH_RESTARTABLE=no covers the ClickBench cold-cycle driver, but the +# playground agent uses PLAYGROUND_SKIP_RESTART_BEFORE_SNAPSHOT for +# the equivalent gate. Without it, pre-snapshot ./stop + ./start runs +# `docker compose down` + `up`, which recreates catalogd with an empty +# in-memory catalog — snapshotted, then every restored /query fails +# with "Database does not exist: clickbench". +export PLAYGROUND_SKIP_RESTART_BEFORE_SNAPSHOT=yes exec ../lib/benchmark-common.sh diff --git a/trino-partitioned/load b/trino-partitioned/load index 0a2fe55ba3..57c5dc7579 100755 --- a/trino-partitioned/load +++ b/trino-partitioned/load @@ -1,10 +1,27 @@ #!/bin/bash set -e -# Hardlink the 100 partitioned Parquet files into the Hive -# external_location directory; the Trino container reads /clickbench/hits. +# Stage the 100 partitioned Parquet files under the Hive +# external_location the Trino container sees at /clickbench/hits. +# We used to hardlink `ln -f "$f" "data/hits/$f"`, but on the +# playground `hits_*.parquet` are symlinks into a read-only dataset +# mount; GNU ln's default -P preserves the symlink so the container +# ends up opening the file via its symlink-target path and hits +# Query … failed: Malformed Parquet file. Metadata index: -33578 +# out of range [local:///hits/hits_10.parquet] +# for arbitrary files at query time (trino ≥ 483's local FS provider +# reading the symlink chain confuses its parquet footer parser). +# Copy the file bytes instead — one-time cost per provision, reliable +# afterward. Auto-fall-back to hardlink when the source is a real +# file on the same filesystem (bare-metal ClickBench runs). for f in hits_*.parquet; do - ln -f "$f" "data/hits/$f" + dst="data/hits/$f" + if [ -L "$f" ]; then + # symlink → resolve and copy the target bytes + cp -f "$(readlink -f "$f")" "$dst" + else + ln -f "$f" "$dst" 2>/dev/null || cp -f "$f" "$dst" + fi done sudo chown -R 1000:1000 data From 42719bee244f914e20987f8232591e16e7256e8e Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 23 Jul 2026 20:23:20 +0000 Subject: [PATCH 09/10] hyrise: shrink CSV chunk further; quickwit: strengthen readiness probe MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - hyrise: 2M-row chunks still SIGKILL at chunk 10 (~20M rows encoded) because hyrise's per-segment overhead is heavy on the 16 GiB VM. Try 1M-row chunks — 100 pieces at 1M each caps the transient unencoded materialisation at ~1 GB, giving more headroom for the growing encoded catalog. If this still OOMs, hyrise is genuinely outside the playground envelope (same class as umbra). - quickwit/check: gate on an actual /api/v1/hits/search request, not just /api/v1/version. The REST server responds long before the searcher registers with the (single-node) gossip cluster, so the current check was letting the snapshot fire while the cluster still reported "no available searcher nodes" — every restored /query then bounced off that error even with PLAYGROUND_RESTART_AFTER_RESTORE_SNAPSHOT=yes. --- hyrise/load | 2 +- quickwit/check | 11 +++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/hyrise/load b/hyrise/load index 5e7db0461a..a4ff68dfab 100755 --- a/hyrise/load +++ b/hyrise/load @@ -35,7 +35,7 @@ if [ ! -f data/hits.bin ]; then lines_per_piece=5000000 mem_kb=$(awk '/MemTotal/{print $2}' /proc/meminfo) if [ "$mem_kb" -lt $((24 * 1024 * 1024)) ]; then - lines_per_piece=2000000 + lines_per_piece=1000000 fi if [ -f hits.csv ]; then split -l "$lines_per_piece" --numeric-suffixes=1 --additional-suffix=.csv \ diff --git a/quickwit/check b/quickwit/check index 82598ad0e5..9e4b764d45 100755 --- a/quickwit/check +++ b/quickwit/check @@ -1,4 +1,15 @@ #!/bin/bash set -e +# The REST server responds to /version well before the searcher has +# registered with the (single-node) gossip cluster. Snapshotting at +# that point leaves the restored VM in a state where the very first +# /query returns +# "no available searcher nodes in the cluster" +# from quickwit's job dispatcher. Gate on a real search request — it +# only succeeds once the searcher has joined and the `hits` index +# manifest is loaded. curl -sS -f http://localhost:7280/api/v1/version >/dev/null +curl -sS -f -X POST -H 'Content-Type: application/json' \ + -d '{"query":"*","max_hits":0}' \ + http://localhost:7280/api/v1/hits/search >/dev/null From ff637e6c48e1cc7e4387a8e99b7803a1a3b0f79a Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 23 Jul 2026 20:57:11 +0000 Subject: [PATCH 10/10] quickwit/check: tolerate 404 on /hits/search pre-load MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The previous version failed the initial post-start check because the `hits` index doesn't exist yet at that point (load creates it). 404 from /hits/search only means the index is missing, not that the searcher is unregistered — the actual failure signal we're trying to catch (500 "no available searcher nodes in the cluster") still fails the gate. --- quickwit/check | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/quickwit/check b/quickwit/check index 9e4b764d45..a28f0b7e7e 100755 --- a/quickwit/check +++ b/quickwit/check @@ -6,10 +6,18 @@ set -e # that point leaves the restored VM in a state where the very first # /query returns # "no available searcher nodes in the cluster" -# from quickwit's job dispatcher. Gate on a real search request — it -# only succeeds once the searcher has joined and the `hits` index -# manifest is loaded. +# from quickwit's job dispatcher. Also probe the search endpoint — +# it succeeds (200) once the searcher has joined and the `hits` +# index manifest is loaded, and returns 404 pre-load when the index +# doesn't exist yet. Either is fine; the 500 "no available searcher +# nodes" error we're trying to avoid is what fails the -f gate. +# Any non-2xx that isn't 404 is treated as unready. curl -sS -f http://localhost:7280/api/v1/version >/dev/null -curl -sS -f -X POST -H 'Content-Type: application/json' \ +code=$(curl -sS -o /dev/null -w '%{http_code}' -X POST \ + -H 'Content-Type: application/json' \ -d '{"query":"*","max_hits":0}' \ - http://localhost:7280/api/v1/hits/search >/dev/null + http://localhost:7280/api/v1/hits/search) +case "$code" in + 200|404) exit 0 ;; + *) echo "quickwit/check: /hits/search HTTP $code" >&2; exit 1 ;; +esac