Fix failed provisions

byconity/check

@@ -6,6 +6,15 @@ set -e
 docker compose exec -T server clickhouse-client --port 52145 \
     --query "SELECT 1" >/dev/null
 
+# TSO probe (see ./start for context). Guards against declaring 'ready'
+# while tso-server is still finishing its FoundationDB handshake — the
+# next ./load INSERT would otherwise hit
+#   Code: 7002. DB::Exception: Can't get process TSO request.
+docker compose exec -T server clickhouse-client --port 52145 \
+    --query "CREATE DATABASE IF NOT EXISTS _bck_readiness" >/dev/null
+docker compose exec -T server clickhouse-client --port 52145 \
+    --query "DROP DATABASE IF EXISTS _bck_readiness" >/dev/null
+
 # If hits exists (post-load), force a worker-touching query so we
 # refuse to declare 'ready' until the server-to-worker BRPC connection
 # has actually re-established. Without this gate the first /query

byconity/start

@@ -12,6 +12,21 @@ set -e
 ck() {
     docker compose exec -T server clickhouse-client --port 52145 \
         --query "SELECT 1" >/dev/null 2>&1 || return 1
+    # TSO probe. `SELECT 1` doesn't touch TSO — TSO is only involved in
+    # writes (any DDL, any INSERT). Cold-provision used to fail on the
+    # very first INSERT of ./load with
+    #   Code: 7002. DB::Exception: Can't get process TSO request,
+    #                type: GetTimestamp
+    # even though ./check reported ready — TSO's TCP port was up
+    # (server's docker-compose depends_on satisfied) but tso-server
+    # hadn't finished initializing against FoundationDB yet. Force a
+    # TSO round-trip here via a lightweight CREATE/DROP DATABASE pair.
+    docker compose exec -T server clickhouse-client --port 52145 \
+        --query "CREATE DATABASE IF NOT EXISTS _bck_readiness" \
+        >/dev/null 2>&1 || return 1
+    docker compose exec -T server clickhouse-client --port 52145 \
+        --query "DROP DATABASE IF EXISTS _bck_readiness" \
+        >/dev/null 2>&1 || return 1
     local exists
     exists=$(docker compose exec -T server clickhouse-client --port 52145 \
         --query "EXISTS TABLE hits" 2>/dev/null || echo 0)

cedardb-parquet/check

@@ -1,4 +1,13 @@
 #!/bin/bash
 set -e
 
+# CedarDB's system-scoped systemd unit binds only /tmp/.s.PGSQL.5432 and
+# 127.0.0.1:5432 (see the installer output). Point psql at /tmp so this
+# doesn't fall through to the postgres default /var/run/postgresql path
+# and fail with "No such file or directory" on a healthy CedarDB. The
+# installer only creates a `postgres` role, so also pin PGUSER — psql's
+# default is $USER, and under the playground agent that's `root`, which
+# cedardb rejects with 'role "root" does not exist'.
+export PGHOST=${PGHOST:-/tmp}
+export PGUSER=${PGUSER:-postgres}
 psql -c 'SELECT 1' >/dev/null

cedardb-parquet/load

@@ -1,6 +1,10 @@
 #!/bin/bash
 set -eu
 
+# CedarDB listens on /tmp/.s.PGSQL.5432; see ./check for context.
+export PGHOST=${PGHOST:-/tmp}
+export PGUSER=${PGUSER:-postgres}
+
 # Substitute the absolute path so CedarDB can find the file regardless of
 # its working directory.
 sed "s|'hits\.parquet'|'$PWD/hits.parquet'|" create.sql | psql

cedardb-parquet/query

@@ -6,6 +6,10 @@
 # Exit non-zero on error.
 set -e
 
+# CedarDB listens on /tmp/.s.PGSQL.5432; see ./check for context.
+export PGHOST=${PGHOST:-/tmp}
+export PGUSER=${PGUSER:-postgres}
+
 query=$(cat)
 
 raw=$(psql -t -c '\timing' -c "$query" 2>&1) && exit_code=0 || exit_code=$?

cedardb-parquet/start

@@ -1,4 +1,12 @@
 #!/bin/bash
 set -eu
 
+# CedarDB's system-scoped systemd unit binds only /tmp/.s.PGSQL.5432 and
+# 127.0.0.1:5432. Point psql at /tmp so it doesn't fall through to the
+# postgres default /var/run/postgresql. PGUSER pinned to postgres —
+# the installer creates only that role, and defaulting to $USER (root
+# in the playground) trips 'role "root" does not exist'.
+export PGHOST=${PGHOST:-/tmp}
+export PGUSER=${PGUSER:-postgres}
+
 sudo systemctl start cedardb.service

cedardb/check

@@ -1,4 +1,13 @@
 #!/bin/bash
 set -e
 
+# CedarDB's system-scoped systemd unit binds only /tmp/.s.PGSQL.5432 and
+# 127.0.0.1:5432 (see the installer output). Point psql at /tmp so this
+# doesn't fall through to the postgres default /var/run/postgresql path
+# and fail with "No such file or directory" on a healthy CedarDB. The
+# installer only creates a `postgres` role, so also pin PGUSER — psql's
+# default is $USER, and under the playground agent that's `root`, which
+# cedardb rejects with 'role "root" does not exist'.
+export PGHOST=${PGHOST:-/tmp}
+export PGUSER=${PGUSER:-postgres}
 psql -c 'SELECT 1' >/dev/null

cedardb/load

@@ -1,6 +1,10 @@
 #!/bin/bash
 set -eu
 
+# CedarDB listens on /tmp/.s.PGSQL.5432; see ./check for context.
+export PGHOST=${PGHOST:-/tmp}
+export PGUSER=${PGUSER:-postgres}
+
 psql -f create.sql
 {
     # CedarDB would like to have ~4 GB of memory per parallel worker for parquet import to be efficient, so set the number of workers accordingly.

cedardb/query

@@ -6,6 +6,10 @@
 # Exit non-zero on error.
 set -e
 
+# CedarDB listens on /tmp/.s.PGSQL.5432; see ./check for context.
+export PGHOST=${PGHOST:-/tmp}
+export PGUSER=${PGUSER:-postgres}
+
 query=$(cat)
 
 raw=$(psql -t -c '\timing' -c "$query" 2>&1) && exit_code=0 || exit_code=$?

cedardb/start

@@ -1,6 +1,14 @@
 #!/bin/bash
 set -eu
 
+# CedarDB's system-scoped systemd unit binds only /tmp/.s.PGSQL.5432 and
+# 127.0.0.1:5432. Point psql at /tmp so it doesn't fall through to the
+# postgres default /var/run/postgresql. PGUSER pinned to postgres —
+# the installer creates only that role, and defaulting to $USER (root
+# in the playground) trips 'role "root" does not exist'.
+export PGHOST=${PGHOST:-/tmp}
+export PGUSER=${PGUSER:-postgres}
+
 sudo systemctl start cedardb.service
 
 # On multi-NUMA systems stay within one node for better performance.

druid/ingest.json

@@ -135,7 +135,7 @@
       "forceGuaranteedRollup": true,
       "logParseExceptions": true,
       "maxParseExceptions": 1,
-      "maxNumConcurrentSubTasks": 10
+      "maxNumConcurrentSubTasks": 4
     },
     "dataSchema": {
       "dataSource": "hits",

druid/load

@@ -12,14 +12,14 @@ DRUID_DIR="apache-druid-${VERSION}"
 "./${DRUID_DIR}/bin/post-index-task" --file ingest.json --url http://localhost:8081 || true
 
 # Wait until the hits datasource is queryable. Druid's index task can
-# legitimately take hours on a 16 GiB VM; budget 4 h here, and fail
+# legitimately take hours on a 16 GiB VM; budget 6 h here, and fail
 # loudly if hits still isn't queryable so the agent doesn't take a
 # snapshot of a half-ingested datasource (which would otherwise look
 # "snapshotted" but every query returns
 #   druidException ... Object 'hits' not found
 # at runtime).
 cnt=""
-for _ in $(seq 1 2880); do  # 2880 * 5s = 4 h
+for _ in $(seq 1 4320); do  # 4320 * 5s = 6 h
     cnt=$(curl -sf -XPOST -H'Content-Type: application/json' \
         http://localhost:8888/druid/v2/sql/ \
         -d '{"query": "SELECT COUNT(*) FROM hits"}' 2>/dev/null \

eventql/.dockerignore

@@ -0,0 +1,14 @@
+# Playground cwd is an overlayfs merge of the system scripts (upper) and
+# /opt/clickbench/datasets_ro (lower), so hits.parquet / hits.tsv /
+# hits.csv / hits.json / hits.json.gz / hits_partitioned/ all appear next
+# to the Dockerfile. Without this ignore file, `docker build .` streams
+# them (75+ GB each) into /var/lib/docker/tmp as build context and the
+# sysdisk fills before the first step runs:
+#   Error response from daemon: write /hits.json: no space left on device
+hits.parquet
+hits.tsv
+hits.csv
+hits.json
+hits.json.gz
+hits_*.parquet
+hits_partitioned/

hive/benchmark.sh

@@ -6,4 +6,13 @@ export BENCH_DOWNLOAD_SCRIPT="download-hits-parquet-single"
 # is present before the first try; the load wall-clock rolls into the
 # cold-try timing per the standard BENCH_DURABLE=no contract.
 export BENCH_DURABLE=no
+# The playground snapshots the guest post-load and every /query
+# restores from that snapshot. If the pre-snapshot ./stop + ./start
+# fires here, ./start's `docker rm -f hive; docker run …` wipes the
+# same embedded Derby metastore that ./load just populated — the
+# snapshot then captures a fresh container with an empty catalog,
+# and every restored /query returns "Database clickbench does not
+# exist". Skip the pre-snapshot restart so the running HS2 (with the
+# loaded catalog) is what gets snapshotted.
+export PLAYGROUND_SKIP_RESTART_BEFORE_SNAPSHOT=yes
 exec ../lib/benchmark-common.sh

hyrise/.dockerignore

@@ -0,0 +1,17 @@
+# Playground cwd is an overlayfs merge of the system scripts (upper) and
+# /opt/clickbench/datasets_ro (lower), so hits.parquet / hits.tsv /
+# hits.csv / hits.json / hits.json.gz / hits_partitioned/ all appear next
+# to the Dockerfile. Without this ignore file, `docker build .` streams
+# them (75+ GB each) into /var/lib/docker/tmp as build context and the
+# sysdisk fills before the first step runs:
+#   Error response from daemon: write /hits.json: no space left on device
+hits.parquet
+hits.tsv
+hits.csv
+hits.json
+hits.json.gz
+hits_*.parquet
+hits_partitioned/
+# The host-side ./load creates data/hits_part_*.csv snapshots and finally
+# data/hits.bin — none of those should be in the build context either.
+data/

hyrise/Dockerfile

@@ -63,8 +63,18 @@ RUN git clone https://github.com/hyrise/hyrise.git \
 # unencoded value segments, which would make the snapshot raw-sized and make
 # it re-import unencoded. See ./load.
 WORKDIR /opt/hyrise/cmake-build-release
+# ninja's default parallelism = nproc kills cc1plus on 16 GiB / 4-vCPU
+# playground VMs: aggregate_hash.cpp under -O3 with heavy template
+# instantiation peaks at ~4 GB per translation unit, so 4 concurrent
+# cc1plus × ~4 GB > 16 GB and the guest kernel SIGKILLs the loser with
+# "fatal error: Killed signal terminated program cc1plus". Cap to 2 —
+# hyriseServer/hyriseConsole still build in ~15 min at that width.
+# Auto-widen when the host has enough RAM per core to safely parallelize.
 RUN cmake -GNinja -DCMAKE_BUILD_TYPE=Release -DNO_LTO=${NO_LTO} .. \
-    && ninja hyriseServer hyriseConsole
+    && jobs=2 \
+    && total_kb=$(awk '/MemTotal/{print $2}' /proc/meminfo) \
+    && if [ "$total_kb" -gt $((24 * 1024 * 1024)) ]; then jobs=$(nproc); fi \
+    && ninja -j"$jobs" hyriseServer hyriseConsole
 
 FROM ubuntu:25.04
 ENV DEBIAN_FRONTEND=noninteractive

hyrise/load

@@ -28,8 +28,17 @@ set -eu
 if [ ! -f data/hits.bin ]; then
     # hits.csv was delivered into cwd by download-hits-csv. Guarded so a
     # retry after a partial first load resumes from the existing pieces.
+    # Piece size scaled to guest RAM: on the 16 GiB playground VM,
+    # 5M-row pieces cross the OOM ceiling once the growing encoded
+    # table reaches ~10 GB (load rc=137 SIGKILL after 156 s). 2M-row
+    # pieces cap the transient unencoded materialisation at ~2 GB.
+    lines_per_piece=5000000
+    mem_kb=$(awk '/MemTotal/{print $2}' /proc/meminfo)
+    if [ "$mem_kb" -lt $((24 * 1024 * 1024)) ]; then
+        lines_per_piece=1000000
+    fi
     if [ -f hits.csv ]; then
-        split -l 5000000 --numeric-suffixes=1 --additional-suffix=.csv \
+        split -l "$lines_per_piece" --numeric-suffixes=1 --additional-suffix=.csv \
             hits.csv data/hits_part_
         rm hits.csv
     fi

impala/benchmark.sh

@@ -18,4 +18,11 @@
 export BENCH_DOWNLOAD_SCRIPT="download-hits-parquet-single"
 export BENCH_RESTARTABLE=no
 export BENCH_CHECK_TIMEOUT=900
+# BENCH_RESTARTABLE=no covers the ClickBench cold-cycle driver, but the
+# playground agent uses PLAYGROUND_SKIP_RESTART_BEFORE_SNAPSHOT for
+# the equivalent gate. Without it, pre-snapshot ./stop + ./start runs
+# `docker compose down` + `up`, which recreates catalogd with an empty
+# in-memory catalog — snapshotted, then every restored /query fails
+# with "Database does not exist: clickbench".
+export PLAYGROUND_SKIP_RESTART_BEFORE_SNAPSHOT=yes
 exec ../lib/benchmark-common.sh

mariadb-duckdb/install

@@ -23,11 +23,18 @@ sudo wget -q -O /etc/apt/sources.list.d/mariadb-duckdb-ci.sources "$CI_SOURCES_U
 sudo apt-get update -y
 sudo DEBIAN_FRONTEND=noninteractive apt-get install -y mariadb-server mariadb-client
 
-sudo tee /etc/mysql/mariadb.conf.d/duckdb.cnf >/dev/null <<'EOF'
+# Scale duckdb-memory-limit to guest RAM. The upstream default of 24G
+# was fine on the 32 GB c6a.4xlarge benchmark host, but blows past a
+# 16 GiB playground VM's total memory and OOM-crashes mariadb-server
+# mid-load (visible as `TLS/SSL error: unexpected eof while reading`
+# from the client). Cap at ~50 % of MemTotal.
+mem_g=$(awk '/MemTotal/{ printf "%d", $2/1024/1024/2 }' /proc/meminfo)
+[ "$mem_g" -lt 4 ] && mem_g=4
+sudo tee /etc/mysql/mariadb.conf.d/duckdb.cnf >/dev/null <<EOF
 [mysqld]
 plugin-maturity=alpha
 plugin-load-add=ha_duckdb.so
-duckdb-memory-limit=24G
+duckdb-memory-limit=${mem_g}G
 duckdb-allow-run-in-duckdb=ON
 EOF
 
@@ -40,6 +47,9 @@ sudo mariadb -e "SELECT PLUGIN_STATUS FROM information_schema.PLUGINS WHERE PLUG
 
 # Allow the mysql user (which runs DuckDB embedded) to traverse the home
 # directory so COPY FROM can read dataset files anywhere under ~.
-sudo chmod o+x /home/ubuntu
+# The playground VM has no /home/ubuntu (the agent runs as root and cwd
+# is /opt/clickbench/system) — skip when the dir doesn't exist so this
+# doesn't abort a set -eu install.
+[ -d /home/ubuntu ] && sudo chmod o+x /home/ubuntu || true
 
 echo "MariaDB with DuckDB engine installed and ready."

mariadb-duckdb/load

@@ -20,10 +20,16 @@ sudo mariadb test < create.sql
 # /home/ubuntu is mode 751 (o+x only) so DuckDB can traverse but not list it,
 # causing "No files found" errors. Hardlink to /var/lib/mysql/ (mysql-owned,
 # mode 755) resolves this instantly without copying 14 GB.
+# Falls back to a symlink when the source lives on a different filesystem
+# — the playground exposes hits.parquet via an overlayfs whose lower is a
+# read-only mount, so hardlink returns EXDEV ("Invalid cross-device link").
 HITS_LINK=/var/lib/mysql/hits.parquet
 if [ ! -f "$HITS_LINK" ]; then
-    sudo ln "$(realpath hits.parquet)" "$HITS_LINK"
-    sudo chmod 644 "$HITS_LINK"
+    src=$(realpath hits.parquet)
+    if ! sudo ln "$src" "$HITS_LINK" 2>/dev/null; then
+        sudo ln -sf "$src" "$HITS_LINK"
+    fi
+    sudo chmod 644 "$HITS_LINK" 2>/dev/null || true
 fi
 duck "INSERT INTO test.hits
 SELECT * REPLACE (

mssql/start

@@ -18,10 +18,18 @@ if ! sudo docker ps -a --format '{{.Names}}' | grep -qx mssql1; then
     # would double disk usage during load: ~75 GB on the host + the copy in
     # the container's writable layer). The mssql process runs as UID 10001
     # in the container, so ./load chmods hits.tsv to 644 before bcp runs.
+    # Cap SQL Server's Max Server Memory to leave enough headroom for the
+    # guest kernel + docker + the load-side bcp process. SQL Server on
+    # ClickBench happily grows to the full container memory; on a 16 GiB
+    # microVM that ends in an OOM-kill (load exited rc=137 after ~19 min
+    # of columnstore build). Scale to 75% of MemTotal, floor 8 GiB.
+    mem_mib=$(awk '/MemTotal/{ printf "%d", $2/1024 * 75/100 }' /proc/meminfo)
+    [ "$mem_mib" -lt 8192 ] && mem_mib=8192
     sudo docker run -d --name mssql1 \
         -e 'ACCEPT_EULA=Y' \
         -e "MSSQL_SA_PASSWORD=$PASSWORD" \
         -e 'MSSQL_PID=Developer' \
+        -e "MSSQL_MEMORY_LIMIT_MB=$mem_mib" \
         -p 1433:1433 \
         -v "$DATA_DIR":/clickbench \
         mcr.microsoft.com/mssql/server:2025-latest >/dev/null

parseable/load

@@ -45,12 +45,19 @@ fi
 # because parallel runs jobs via /bin/sh by default, and a bash
 # `export -f`'d function isn't visible in that shell — the previous
 # version silently no-op'd every chunk and load wrote 0 rows.
-LINES_PER_CHUNK=2500
-INGEST_JOBS=6
+# In the 16 GiB playground VM parseable's incoming-batch queue fills
+# fast when we push 6 concurrent 2500-line batches (~75 MB/s in
+# flight) — the endpoint then starts returning HTTP 408 and every
+# subsequent chunk fails. 3 × 1000 lines is ~4x lower peak pressure
+# and empirically clears without 408s. curl --retry survives short
+# stalls so a briefly-saturated parseable doesn't fail the load.
+LINES_PER_CHUNK=1000
+INGEST_JOBS=3
 pv hits.json | parallel --pipe -N$LINES_PER_CHUNK --block 10M \
     --jobs "$INGEST_JOBS" --halt-on-error 0 '
         awk "BEGIN{print \"[\"} NR>1{print prev \",\"} {prev=\$0} END{if (prev) print prev; print \"]\"}" |
-        curl --silent --show-error --fail \
+        curl --silent --show-error --fail --retry 5 --retry-delay 2 \
+            --retry-all-errors --max-time 120 \
             -H "Content-Type: application/json" \
             -H "X-P-Stream: hits" \
             -k -XPOST -u "admin:admin" \

pg_clickhouse/install

@@ -26,6 +26,8 @@ sudo apt-get install -y \
     libcurl4-openssl-dev \
     uuid-dev \
     libssl-dev \
+    liblz4-dev \
+    libzstd-dev \
     make \
     cmake \
     g++ \
@@ -51,5 +53,10 @@ EOF
 
 sudo systemctl restart postgresql@$PGVERSION-main
 
-# Build/install the pg_clickhouse extension (idempotent on rerun).
-pgxn install pg_clickhouse || true
+# Build/install the pg_clickhouse extension. Not idempotent-swallowed
+# any more — pgxn silently reports a failed compile with rc=2 while the
+# install phase returns rc=0 to the harness, and the caller only finds
+# out at load time via `extension "pg_clickhouse" is not available`.
+# `pgxn install` re-fetches + rebuilds cheaply on rerun, so it's safe
+# to let a real error propagate.
+sudo pgxn install pg_clickhouse