diff --git a/byconity/check b/byconity/check index 0c40c7e923..ed1b485327 100755 --- a/byconity/check +++ b/byconity/check @@ -6,6 +6,15 @@ set -e docker compose exec -T server clickhouse-client --port 52145 \ --query "SELECT 1" >/dev/null +# TSO probe (see ./start for context). Guards against declaring 'ready' +# while tso-server is still finishing its FoundationDB handshake — the +# next ./load INSERT would otherwise hit +# Code: 7002. DB::Exception: Can't get process TSO request. +docker compose exec -T server clickhouse-client --port 52145 \ + --query "CREATE DATABASE IF NOT EXISTS _bck_readiness" >/dev/null +docker compose exec -T server clickhouse-client --port 52145 \ + --query "DROP DATABASE IF EXISTS _bck_readiness" >/dev/null + # If hits exists (post-load), force a worker-touching query so we # refuse to declare 'ready' until the server-to-worker BRPC connection # has actually re-established. Without this gate the first /query diff --git a/byconity/start b/byconity/start index f5cd869c60..2cd591e86e 100755 --- a/byconity/start +++ b/byconity/start @@ -12,6 +12,21 @@ set -e ck() { docker compose exec -T server clickhouse-client --port 52145 \ --query "SELECT 1" >/dev/null 2>&1 || return 1 + # TSO probe. `SELECT 1` doesn't touch TSO — TSO is only involved in + # writes (any DDL, any INSERT). Cold-provision used to fail on the + # very first INSERT of ./load with + # Code: 7002. DB::Exception: Can't get process TSO request, + # type: GetTimestamp + # even though ./check reported ready — TSO's TCP port was up + # (server's docker-compose depends_on satisfied) but tso-server + # hadn't finished initializing against FoundationDB yet. Force a + # TSO round-trip here via a lightweight CREATE/DROP DATABASE pair. + docker compose exec -T server clickhouse-client --port 52145 \ + --query "CREATE DATABASE IF NOT EXISTS _bck_readiness" \ + >/dev/null 2>&1 || return 1 + docker compose exec -T server clickhouse-client --port 52145 \ + --query "DROP DATABASE IF EXISTS _bck_readiness" \ + >/dev/null 2>&1 || return 1 local exists exists=$(docker compose exec -T server clickhouse-client --port 52145 \ --query "EXISTS TABLE hits" 2>/dev/null || echo 0) diff --git a/cedardb-parquet/check b/cedardb-parquet/check index 1ff9f1a6d4..4540ec287f 100755 --- a/cedardb-parquet/check +++ b/cedardb-parquet/check @@ -1,4 +1,13 @@ #!/bin/bash set -e +# CedarDB's system-scoped systemd unit binds only /tmp/.s.PGSQL.5432 and +# 127.0.0.1:5432 (see the installer output). Point psql at /tmp so this +# doesn't fall through to the postgres default /var/run/postgresql path +# and fail with "No such file or directory" on a healthy CedarDB. The +# installer only creates a `postgres` role, so also pin PGUSER — psql's +# default is $USER, and under the playground agent that's `root`, which +# cedardb rejects with 'role "root" does not exist'. +export PGHOST=${PGHOST:-/tmp} +export PGUSER=${PGUSER:-postgres} psql -c 'SELECT 1' >/dev/null diff --git a/cedardb-parquet/load b/cedardb-parquet/load index 4850506bd1..4623e31c85 100755 --- a/cedardb-parquet/load +++ b/cedardb-parquet/load @@ -1,6 +1,10 @@ #!/bin/bash set -eu +# CedarDB listens on /tmp/.s.PGSQL.5432; see ./check for context. +export PGHOST=${PGHOST:-/tmp} +export PGUSER=${PGUSER:-postgres} + # Substitute the absolute path so CedarDB can find the file regardless of # its working directory. sed "s|'hits\.parquet'|'$PWD/hits.parquet'|" create.sql | psql diff --git a/cedardb-parquet/query b/cedardb-parquet/query index ebb3c2d49f..795d108bcb 100755 --- a/cedardb-parquet/query +++ b/cedardb-parquet/query @@ -6,6 +6,10 @@ # Exit non-zero on error. set -e +# CedarDB listens on /tmp/.s.PGSQL.5432; see ./check for context. +export PGHOST=${PGHOST:-/tmp} +export PGUSER=${PGUSER:-postgres} + query=$(cat) raw=$(psql -t -c '\timing' -c "$query" 2>&1) && exit_code=0 || exit_code=$? diff --git a/cedardb-parquet/start b/cedardb-parquet/start index c11e05a5ce..c104fdd797 100755 --- a/cedardb-parquet/start +++ b/cedardb-parquet/start @@ -1,4 +1,12 @@ #!/bin/bash set -eu +# CedarDB's system-scoped systemd unit binds only /tmp/.s.PGSQL.5432 and +# 127.0.0.1:5432. Point psql at /tmp so it doesn't fall through to the +# postgres default /var/run/postgresql. PGUSER pinned to postgres — +# the installer creates only that role, and defaulting to $USER (root +# in the playground) trips 'role "root" does not exist'. +export PGHOST=${PGHOST:-/tmp} +export PGUSER=${PGUSER:-postgres} + sudo systemctl start cedardb.service diff --git a/cedardb/check b/cedardb/check index 1ff9f1a6d4..4540ec287f 100755 --- a/cedardb/check +++ b/cedardb/check @@ -1,4 +1,13 @@ #!/bin/bash set -e +# CedarDB's system-scoped systemd unit binds only /tmp/.s.PGSQL.5432 and +# 127.0.0.1:5432 (see the installer output). Point psql at /tmp so this +# doesn't fall through to the postgres default /var/run/postgresql path +# and fail with "No such file or directory" on a healthy CedarDB. The +# installer only creates a `postgres` role, so also pin PGUSER — psql's +# default is $USER, and under the playground agent that's `root`, which +# cedardb rejects with 'role "root" does not exist'. +export PGHOST=${PGHOST:-/tmp} +export PGUSER=${PGUSER:-postgres} psql -c 'SELECT 1' >/dev/null diff --git a/cedardb/load b/cedardb/load index 1651aaba23..d948e7e6ff 100755 --- a/cedardb/load +++ b/cedardb/load @@ -1,6 +1,10 @@ #!/bin/bash set -eu +# CedarDB listens on /tmp/.s.PGSQL.5432; see ./check for context. +export PGHOST=${PGHOST:-/tmp} +export PGUSER=${PGUSER:-postgres} + psql -f create.sql { # CedarDB would like to have ~4 GB of memory per parallel worker for parquet import to be efficient, so set the number of workers accordingly. diff --git a/cedardb/query b/cedardb/query index ebb3c2d49f..795d108bcb 100755 --- a/cedardb/query +++ b/cedardb/query @@ -6,6 +6,10 @@ # Exit non-zero on error. set -e +# CedarDB listens on /tmp/.s.PGSQL.5432; see ./check for context. +export PGHOST=${PGHOST:-/tmp} +export PGUSER=${PGUSER:-postgres} + query=$(cat) raw=$(psql -t -c '\timing' -c "$query" 2>&1) && exit_code=0 || exit_code=$? diff --git a/cedardb/start b/cedardb/start index 64dd2373e8..ffdc7daa62 100755 --- a/cedardb/start +++ b/cedardb/start @@ -1,6 +1,14 @@ #!/bin/bash set -eu +# CedarDB's system-scoped systemd unit binds only /tmp/.s.PGSQL.5432 and +# 127.0.0.1:5432. Point psql at /tmp so it doesn't fall through to the +# postgres default /var/run/postgresql. PGUSER pinned to postgres — +# the installer creates only that role, and defaulting to $USER (root +# in the playground) trips 'role "root" does not exist'. +export PGHOST=${PGHOST:-/tmp} +export PGUSER=${PGUSER:-postgres} + sudo systemctl start cedardb.service # On multi-NUMA systems stay within one node for better performance. diff --git a/druid/ingest.json b/druid/ingest.json index 8d4c741934..6a88302e52 100644 --- a/druid/ingest.json +++ b/druid/ingest.json @@ -135,7 +135,7 @@ "forceGuaranteedRollup": true, "logParseExceptions": true, "maxParseExceptions": 1, - "maxNumConcurrentSubTasks": 10 + "maxNumConcurrentSubTasks": 4 }, "dataSchema": { "dataSource": "hits", diff --git a/druid/load b/druid/load index 0f3a8f6e8e..75b173f169 100755 --- a/druid/load +++ b/druid/load @@ -12,14 +12,14 @@ DRUID_DIR="apache-druid-${VERSION}" "./${DRUID_DIR}/bin/post-index-task" --file ingest.json --url http://localhost:8081 || true # Wait until the hits datasource is queryable. Druid's index task can -# legitimately take hours on a 16 GiB VM; budget 4 h here, and fail +# legitimately take hours on a 16 GiB VM; budget 6 h here, and fail # loudly if hits still isn't queryable so the agent doesn't take a # snapshot of a half-ingested datasource (which would otherwise look # "snapshotted" but every query returns # druidException ... Object 'hits' not found # at runtime). cnt="" -for _ in $(seq 1 2880); do # 2880 * 5s = 4 h +for _ in $(seq 1 4320); do # 4320 * 5s = 6 h cnt=$(curl -sf -XPOST -H'Content-Type: application/json' \ http://localhost:8888/druid/v2/sql/ \ -d '{"query": "SELECT COUNT(*) FROM hits"}' 2>/dev/null \ diff --git a/eventql/.dockerignore b/eventql/.dockerignore new file mode 100644 index 0000000000..dcac31b52d --- /dev/null +++ b/eventql/.dockerignore @@ -0,0 +1,14 @@ +# Playground cwd is an overlayfs merge of the system scripts (upper) and +# /opt/clickbench/datasets_ro (lower), so hits.parquet / hits.tsv / +# hits.csv / hits.json / hits.json.gz / hits_partitioned/ all appear next +# to the Dockerfile. Without this ignore file, `docker build .` streams +# them (75+ GB each) into /var/lib/docker/tmp as build context and the +# sysdisk fills before the first step runs: +# Error response from daemon: write /hits.json: no space left on device +hits.parquet +hits.tsv +hits.csv +hits.json +hits.json.gz +hits_*.parquet +hits_partitioned/ diff --git a/hive/benchmark.sh b/hive/benchmark.sh index e16fbff562..2b2a3fd613 100755 --- a/hive/benchmark.sh +++ b/hive/benchmark.sh @@ -6,4 +6,13 @@ export BENCH_DOWNLOAD_SCRIPT="download-hits-parquet-single" # is present before the first try; the load wall-clock rolls into the # cold-try timing per the standard BENCH_DURABLE=no contract. export BENCH_DURABLE=no +# The playground snapshots the guest post-load and every /query +# restores from that snapshot. If the pre-snapshot ./stop + ./start +# fires here, ./start's `docker rm -f hive; docker run …` wipes the +# same embedded Derby metastore that ./load just populated — the +# snapshot then captures a fresh container with an empty catalog, +# and every restored /query returns "Database clickbench does not +# exist". Skip the pre-snapshot restart so the running HS2 (with the +# loaded catalog) is what gets snapshotted. +export PLAYGROUND_SKIP_RESTART_BEFORE_SNAPSHOT=yes exec ../lib/benchmark-common.sh diff --git a/hyrise/.dockerignore b/hyrise/.dockerignore new file mode 100644 index 0000000000..ba56809482 --- /dev/null +++ b/hyrise/.dockerignore @@ -0,0 +1,17 @@ +# Playground cwd is an overlayfs merge of the system scripts (upper) and +# /opt/clickbench/datasets_ro (lower), so hits.parquet / hits.tsv / +# hits.csv / hits.json / hits.json.gz / hits_partitioned/ all appear next +# to the Dockerfile. Without this ignore file, `docker build .` streams +# them (75+ GB each) into /var/lib/docker/tmp as build context and the +# sysdisk fills before the first step runs: +# Error response from daemon: write /hits.json: no space left on device +hits.parquet +hits.tsv +hits.csv +hits.json +hits.json.gz +hits_*.parquet +hits_partitioned/ +# The host-side ./load creates data/hits_part_*.csv snapshots and finally +# data/hits.bin — none of those should be in the build context either. +data/ diff --git a/hyrise/Dockerfile b/hyrise/Dockerfile index 93f1ffbcf1..5a209a1784 100644 --- a/hyrise/Dockerfile +++ b/hyrise/Dockerfile @@ -63,8 +63,18 @@ RUN git clone https://github.com/hyrise/hyrise.git \ # unencoded value segments, which would make the snapshot raw-sized and make # it re-import unencoded. See ./load. WORKDIR /opt/hyrise/cmake-build-release +# ninja's default parallelism = nproc kills cc1plus on 16 GiB / 4-vCPU +# playground VMs: aggregate_hash.cpp under -O3 with heavy template +# instantiation peaks at ~4 GB per translation unit, so 4 concurrent +# cc1plus × ~4 GB > 16 GB and the guest kernel SIGKILLs the loser with +# "fatal error: Killed signal terminated program cc1plus". Cap to 2 — +# hyriseServer/hyriseConsole still build in ~15 min at that width. +# Auto-widen when the host has enough RAM per core to safely parallelize. RUN cmake -GNinja -DCMAKE_BUILD_TYPE=Release -DNO_LTO=${NO_LTO} .. \ - && ninja hyriseServer hyriseConsole + && jobs=2 \ + && total_kb=$(awk '/MemTotal/{print $2}' /proc/meminfo) \ + && if [ "$total_kb" -gt $((24 * 1024 * 1024)) ]; then jobs=$(nproc); fi \ + && ninja -j"$jobs" hyriseServer hyriseConsole FROM ubuntu:25.04 ENV DEBIAN_FRONTEND=noninteractive diff --git a/hyrise/load b/hyrise/load index 650d43b3cc..a4ff68dfab 100755 --- a/hyrise/load +++ b/hyrise/load @@ -28,8 +28,17 @@ set -eu if [ ! -f data/hits.bin ]; then # hits.csv was delivered into cwd by download-hits-csv. Guarded so a # retry after a partial first load resumes from the existing pieces. + # Piece size scaled to guest RAM: on the 16 GiB playground VM, + # 5M-row pieces cross the OOM ceiling once the growing encoded + # table reaches ~10 GB (load rc=137 SIGKILL after 156 s). 2M-row + # pieces cap the transient unencoded materialisation at ~2 GB. + lines_per_piece=5000000 + mem_kb=$(awk '/MemTotal/{print $2}' /proc/meminfo) + if [ "$mem_kb" -lt $((24 * 1024 * 1024)) ]; then + lines_per_piece=1000000 + fi if [ -f hits.csv ]; then - split -l 5000000 --numeric-suffixes=1 --additional-suffix=.csv \ + split -l "$lines_per_piece" --numeric-suffixes=1 --additional-suffix=.csv \ hits.csv data/hits_part_ rm hits.csv fi diff --git a/impala/benchmark.sh b/impala/benchmark.sh index fa3a0559bc..724a26309a 100755 --- a/impala/benchmark.sh +++ b/impala/benchmark.sh @@ -18,4 +18,11 @@ export BENCH_DOWNLOAD_SCRIPT="download-hits-parquet-single" export BENCH_RESTARTABLE=no export BENCH_CHECK_TIMEOUT=900 +# BENCH_RESTARTABLE=no covers the ClickBench cold-cycle driver, but the +# playground agent uses PLAYGROUND_SKIP_RESTART_BEFORE_SNAPSHOT for +# the equivalent gate. Without it, pre-snapshot ./stop + ./start runs +# `docker compose down` + `up`, which recreates catalogd with an empty +# in-memory catalog — snapshotted, then every restored /query fails +# with "Database does not exist: clickbench". +export PLAYGROUND_SKIP_RESTART_BEFORE_SNAPSHOT=yes exec ../lib/benchmark-common.sh diff --git a/mariadb-duckdb/install b/mariadb-duckdb/install index 3d980241df..cf4ba1a18c 100755 --- a/mariadb-duckdb/install +++ b/mariadb-duckdb/install @@ -23,11 +23,18 @@ sudo wget -q -O /etc/apt/sources.list.d/mariadb-duckdb-ci.sources "$CI_SOURCES_U sudo apt-get update -y sudo DEBIAN_FRONTEND=noninteractive apt-get install -y mariadb-server mariadb-client -sudo tee /etc/mysql/mariadb.conf.d/duckdb.cnf >/dev/null <<'EOF' +# Scale duckdb-memory-limit to guest RAM. The upstream default of 24G +# was fine on the 32 GB c6a.4xlarge benchmark host, but blows past a +# 16 GiB playground VM's total memory and OOM-crashes mariadb-server +# mid-load (visible as `TLS/SSL error: unexpected eof while reading` +# from the client). Cap at ~50 % of MemTotal. +mem_g=$(awk '/MemTotal/{ printf "%d", $2/1024/1024/2 }' /proc/meminfo) +[ "$mem_g" -lt 4 ] && mem_g=4 +sudo tee /etc/mysql/mariadb.conf.d/duckdb.cnf >/dev/null </dev/null; then + sudo ln -sf "$src" "$HITS_LINK" + fi + sudo chmod 644 "$HITS_LINK" 2>/dev/null || true fi duck "INSERT INTO test.hits SELECT * REPLACE ( diff --git a/mssql/start b/mssql/start index 438d1e1a40..afba20b88f 100755 --- a/mssql/start +++ b/mssql/start @@ -18,10 +18,18 @@ if ! sudo docker ps -a --format '{{.Names}}' | grep -qx mssql1; then # would double disk usage during load: ~75 GB on the host + the copy in # the container's writable layer). The mssql process runs as UID 10001 # in the container, so ./load chmods hits.tsv to 644 before bcp runs. + # Cap SQL Server's Max Server Memory to leave enough headroom for the + # guest kernel + docker + the load-side bcp process. SQL Server on + # ClickBench happily grows to the full container memory; on a 16 GiB + # microVM that ends in an OOM-kill (load exited rc=137 after ~19 min + # of columnstore build). Scale to 75% of MemTotal, floor 8 GiB. + mem_mib=$(awk '/MemTotal/{ printf "%d", $2/1024 * 75/100 }' /proc/meminfo) + [ "$mem_mib" -lt 8192 ] && mem_mib=8192 sudo docker run -d --name mssql1 \ -e 'ACCEPT_EULA=Y' \ -e "MSSQL_SA_PASSWORD=$PASSWORD" \ -e 'MSSQL_PID=Developer' \ + -e "MSSQL_MEMORY_LIMIT_MB=$mem_mib" \ -p 1433:1433 \ -v "$DATA_DIR":/clickbench \ mcr.microsoft.com/mssql/server:2025-latest >/dev/null diff --git a/parseable/load b/parseable/load index 45a7fb541d..5cf60065f3 100755 --- a/parseable/load +++ b/parseable/load @@ -45,12 +45,19 @@ fi # because parallel runs jobs via /bin/sh by default, and a bash # `export -f`'d function isn't visible in that shell — the previous # version silently no-op'd every chunk and load wrote 0 rows. -LINES_PER_CHUNK=2500 -INGEST_JOBS=6 +# In the 16 GiB playground VM parseable's incoming-batch queue fills +# fast when we push 6 concurrent 2500-line batches (~75 MB/s in +# flight) — the endpoint then starts returning HTTP 408 and every +# subsequent chunk fails. 3 × 1000 lines is ~4x lower peak pressure +# and empirically clears without 408s. curl --retry survives short +# stalls so a briefly-saturated parseable doesn't fail the load. +LINES_PER_CHUNK=1000 +INGEST_JOBS=3 pv hits.json | parallel --pipe -N$LINES_PER_CHUNK --block 10M \ --jobs "$INGEST_JOBS" --halt-on-error 0 ' awk "BEGIN{print \"[\"} NR>1{print prev \",\"} {prev=\$0} END{if (prev) print prev; print \"]\"}" | - curl --silent --show-error --fail \ + curl --silent --show-error --fail --retry 5 --retry-delay 2 \ + --retry-all-errors --max-time 120 \ -H "Content-Type: application/json" \ -H "X-P-Stream: hits" \ -k -XPOST -u "admin:admin" \ diff --git a/pg_clickhouse/install b/pg_clickhouse/install index 3774e1b66e..81d7dc6319 100755 --- a/pg_clickhouse/install +++ b/pg_clickhouse/install @@ -26,6 +26,8 @@ sudo apt-get install -y \ libcurl4-openssl-dev \ uuid-dev \ libssl-dev \ + liblz4-dev \ + libzstd-dev \ make \ cmake \ g++ \ @@ -51,5 +53,10 @@ EOF sudo systemctl restart postgresql@$PGVERSION-main -# Build/install the pg_clickhouse extension (idempotent on rerun). -pgxn install pg_clickhouse || true +# Build/install the pg_clickhouse extension. Not idempotent-swallowed +# any more — pgxn silently reports a failed compile with rc=2 while the +# install phase returns rc=0 to the harness, and the caller only finds +# out at load time via `extension "pg_clickhouse" is not available`. +# `pgxn install` re-fetches + rebuilds cheaply on rerun, so it's safe +# to let a real error propagate. +sudo pgxn install pg_clickhouse diff --git a/pg_deltax/install b/pg_deltax/install index 7a20505c18..7bebdb17c1 100755 --- a/pg_deltax/install +++ b/pg_deltax/install @@ -20,10 +20,27 @@ sudo apt-get update -y sudo apt-get install -y postgresql-$PGVERSION postgresql-client-$PGVERSION postgresql-server-dev-$PGVERSION # PostgreSQL runs with stock configuration except work_mem=64MB, the same -# value the vanilla postgresql/ benchmark uses. -sudo tee /etc/postgresql/$PGVERSION/main/conf.d/clickbench.conf </dev/null || true # Recreate the DB so this script is idempotent. DROP wipes any prior # ALTER DATABASE settings so we start from postgresql.conf defaults @@ -18,11 +24,31 @@ sudo chmod 644 "$PARQUET_DIR"/*.parquet sudo -u postgres psql -v ON_ERROR_STOP=1 -t -c "DROP DATABASE IF EXISTS test" sudo -u postgres psql -v ON_ERROR_STOP=1 -t -c "CREATE DATABASE test" -# Bump work_mem for the load only. Direct backfill sorts each segment before -# compressing; 1GB keeps larger segments in memory and shaves load time. -# Reset before the query phase so the concurrent-QPS test (10 connections -# each spawning parallel-scan workers) doesn't multiply this up to OOM. -sudo -u postgres psql -v ON_ERROR_STOP=1 -t -c "ALTER DATABASE test SET work_mem TO '1GB'" +# Bump work_mem for the load only. Direct backfill sorts each segment +# before compressing; larger keeps more segments in memory and shaves +# load time. Reset before the query phase so the concurrent-QPS test +# (10 connections each spawning parallel-scan workers) doesn't multiply +# this up to OOM. Scale to guest RAM: 1 GB was fine on 32 GB benchmark +# hosts but crashed pg_deltax's compression pipeline mid-load on the +# 16 GiB playground VM. Cap at MemTotal/64 so pg_deltax's decoder pool +# × work_mem still leaves headroom for shared_buffers + kernel. +load_wm_mb=$(awk '/MemTotal/{ printf "%d", $2/1024/64 }' /proc/meminfo) +[ "$load_wm_mb" -lt 64 ] && load_wm_mb=64 +[ "$load_wm_mb" -gt 1024 ] && load_wm_mb=1024 +sudo -u postgres psql -v ON_ERROR_STOP=1 -t \ + -c "ALTER DATABASE test SET work_mem TO '${load_wm_mb}MB'" + +# pg_deltax.parallel_workers=0 (default) uses num_cpus (capped at 16), so +# 4-vCPU / 16 GiB VMs end up with 4 decoders + 1 compressor. Each decoder +# accumulates ~250 MB of blob buffers before flush → ~1.5 GB of transient +# state that OOM-kills the backend ("server closed the connection +# unexpectedly" after 4-6 partition flushes). Cap decoders on +# memory-constrained hosts. Skip on hosts with ≥24 GiB RAM. +mem_kb=$(awk '/MemTotal/{print $2}' /proc/meminfo) +if [ "$mem_kb" -lt $((24 * 1024 * 1024)) ]; then + sudo -u postgres psql -v ON_ERROR_STOP=1 -t \ + -c "ALTER DATABASE test SET pg_deltax.parallel_workers TO 2" +fi sudo -u postgres psql -v ON_ERROR_STOP=1 -t test -c "CREATE EXTENSION pg_deltax" diff --git a/pinot/install b/pinot/install index a7a9feb5eb..bb2109b279 100755 --- a/pinot/install +++ b/pinot/install @@ -1,9 +1,11 @@ #!/bin/bash set -e -# 1.3.0 was retired from the Apache mirror; bump to a currently-published -# Pinot release. -PINOT_VERSION=1.5.0 +# Apache retires older Pinot releases from downloads.apache.org fairly +# aggressively (`downloads.apache.org/pinot/apache-pinot-1.5.0/...` now +# 404s); bump this to the current mirror-hosted version whenever we hit +# a 404 at install time. +PINOT_VERSION=1.5.1 PINOT_DIR="apache-pinot-$PINOT_VERSION-bin" if [ ! -d "$PINOT_DIR" ]; then diff --git a/pinot/load b/pinot/load index 93803853d7..7229254546 100755 --- a/pinot/load +++ b/pinot/load @@ -1,7 +1,7 @@ #!/bin/bash set -e -PINOT_VERSION=1.5.0 +PINOT_VERSION=1.5.1 PINOT_DIR="apache-pinot-$PINOT_VERSION-bin" # Wait for the controller's REST API to be live. `start` only waits for diff --git a/pinot/start b/pinot/start index 9f3036b6d0..bd189e8077 100755 --- a/pinot/start +++ b/pinot/start @@ -1,7 +1,7 @@ #!/bin/bash set -e -PINOT_VERSION=1.5.0 +PINOT_VERSION=1.5.1 PINOT_DIR="apache-pinot-$PINOT_VERSION-bin" # Idempotent: if broker query endpoint is up, do nothing. diff --git a/playground/server/config.py b/playground/server/config.py index 571b186d64..5ba396a87f 100644 --- a/playground/server/config.py +++ b/playground/server/config.py @@ -61,6 +61,14 @@ class Config: idle_kick_after_sec: int host_min_free_ram_gb: int host_min_free_disk_gb: int + # Cap the number of VMs in "ready" state at any time. When exceeded, + # the monitor evicts the least-recently-used ready VM. Bounds total + # host memory during query bursts — without this a mass /query + # against N systems keeps all N VMs ready for `idle_kick_after_sec` + # (10 min default), which on 100+ systems can pile up hundreds of + # GB of guest RSS + file cache. Snapshot is preserved, so eviction + # only costs the next /query one restore. + max_ready_vms: int # Per-system disk full check. vm_disk_pct_kill_threshold: float # ClickHouse Cloud logging. @@ -141,6 +149,10 @@ def load() -> Config: idle_kick_after_sec=_env_int("VM_IDLE_KICK_AFTER_SEC", 600), host_min_free_ram_gb=_env_int("HOST_MIN_FREE_RAM_GB", 32), host_min_free_disk_gb=_env_int("HOST_MIN_FREE_DISK_GB", 100), + # 40 VMs × ~2 GB avg RSS ≈ 80 GB of guest anon, comfortably + # under host RAM on the 1 TB benchmark box. Override via + # PLAYGROUND_MAX_READY_VMS. + max_ready_vms=_env_int("PLAYGROUND_MAX_READY_VMS", 40), vm_disk_pct_kill_threshold=float(os.environ.get("VM_DISK_FULL_PCT", "0.97")), ch_cloud_url=os.environ.get("CLICKHOUSE_CLOUD_URL", ch_conf.get("url", "")), ch_cloud_user=os.environ.get("CLICKHOUSE_CLOUD_USER", ch_conf.get("user", "")), diff --git a/playground/server/monitor.py b/playground/server/monitor.py index be3cf9b076..e1c5be17bd 100644 --- a/playground/server/monitor.py +++ b/playground/server/monitor.py @@ -82,6 +82,7 @@ async def _tick(self) -> None: # Host-wide checks await self._check_host_pressure() + await self._check_max_ready_vms() def _sample_cpu(self, name: str, pid: int) -> float | None: """Return ratio of CPU used since last sample, normalized by vcpu count.""" @@ -200,6 +201,33 @@ async def _check_host_pressure(self) -> None: ) await self.vmm.kick(target.system.name, "host-disk-pressure") + async def _check_max_ready_vms(self) -> None: + """Enforce the cap on concurrent ready VMs. Evict oldest-idle + when exceeded. Only touches VMs in state=='ready' that haven't + been used in a bit (2 s grace) so we don't evict a VM + milliseconds into its first query.""" + cap = self.cfg.max_ready_vms + if cap <= 0: + return + ready = [v for v in self.vmm.vms.values() if v.state == "ready"] + if len(ready) <= cap: + return + # Pick the ones idle the longest, skipping anything used in the + # last 2 s to avoid racing an in-flight query dispatch. + now = time.time() + candidates = [v for v in ready if now - v.last_used >= 2.0] + if not candidates: + return + candidates.sort(key=lambda v: v.last_used) + excess = len(ready) - cap + for vm in candidates[:excess]: + self.sink.write_event( + system=vm.system.name, kind="max-ready-cap", + detail=f"ready={len(ready)} > cap={cap}; " + f"idle for {int(now - vm.last_used)}s", + ) + await self.vmm.kick(vm.system.name, "max-ready-cap") + def _largest_running(self, *, by: str) -> VM | None: running = [v for v in self.vmm.vms.values() if v.pid is not None and _pid_alive(v.pid)] diff --git a/playground/server/vm_manager.py b/playground/server/vm_manager.py index 8ab1b8e1dd..600731225e 100644 --- a/playground/server/vm_manager.py +++ b/playground/server/vm_manager.py @@ -443,8 +443,17 @@ async def _boot(self, vm: VM, *, restore_snapshot: bool) -> None: # If config fails partway, the firecracker process still owns the # TAP fd; without reaping it, the next attempt sees "Resource # busy" because the kernel hasn't released the TAP. Kill + - # wait() before propagating. - await self._shutdown(vm) + # wait() before propagating. Also delete the TAP interface + # itself — kernel-level opens can linger past the process + # exit in some paths (observed as retry attempts hitting + # "Open tap device failed: … Resource busy (os error 16)" + # even after the prior fc process was SIGKILL-reaped). A + # clean tuntap del + re-add on the next ensure_tap avoids + # that class of race entirely. + with contextlib.suppress(Exception): + await self._shutdown(vm) + with contextlib.suppress(Exception): + await net.teardown_tap(vm.slot) raise async def _configure_boot(self, vm: VM, *, restore_snapshot: bool) -> None: diff --git a/presto-partitioned/start b/presto-partitioned/start index 125a4ca19d..0083514a9a 100755 --- a/presto-partitioned/start +++ b/presto-partitioned/start @@ -11,6 +11,14 @@ if sudo docker ps -a --format '{{.Names}}' | grep -qx presto; then exit 0 fi +# Belt-and-suspenders: `docker ps -a` occasionally misses a container +# that dockerd has re-registered from its on-disk state but not yet +# published to the ps table (observed right after dockerd restart in +# the playground). Without this, `docker run` fails with +# "Conflict. The container name "/presto" is already in use" +# and the whole start step exits 125. +sudo docker rm -f presto 2>/dev/null || true + sudo docker run -d --name presto \ -p 8081:8080 \ -v "$PWD/etc/catalog/hive.properties:/opt/presto-server/etc/catalog/hive.properties:ro" \ diff --git a/quickwit/check b/quickwit/check index 82598ad0e5..a28f0b7e7e 100755 --- a/quickwit/check +++ b/quickwit/check @@ -1,4 +1,23 @@ #!/bin/bash set -e +# The REST server responds to /version well before the searcher has +# registered with the (single-node) gossip cluster. Snapshotting at +# that point leaves the restored VM in a state where the very first +# /query returns +# "no available searcher nodes in the cluster" +# from quickwit's job dispatcher. Also probe the search endpoint — +# it succeeds (200) once the searcher has joined and the `hits` +# index manifest is loaded, and returns 404 pre-load when the index +# doesn't exist yet. Either is fine; the 500 "no available searcher +# nodes" error we're trying to avoid is what fails the -f gate. +# Any non-2xx that isn't 404 is treated as unready. curl -sS -f http://localhost:7280/api/v1/version >/dev/null +code=$(curl -sS -o /dev/null -w '%{http_code}' -X POST \ + -H 'Content-Type: application/json' \ + -d '{"query":"*","max_hits":0}' \ + http://localhost:7280/api/v1/hits/search) +case "$code" in + 200|404) exit 0 ;; + *) echo "quickwit/check: /hits/search HTTP $code" >&2; exit 1 ;; +esac diff --git a/trino-partitioned/load b/trino-partitioned/load index 0a2fe55ba3..57c5dc7579 100755 --- a/trino-partitioned/load +++ b/trino-partitioned/load @@ -1,10 +1,27 @@ #!/bin/bash set -e -# Hardlink the 100 partitioned Parquet files into the Hive -# external_location directory; the Trino container reads /clickbench/hits. +# Stage the 100 partitioned Parquet files under the Hive +# external_location the Trino container sees at /clickbench/hits. +# We used to hardlink `ln -f "$f" "data/hits/$f"`, but on the +# playground `hits_*.parquet` are symlinks into a read-only dataset +# mount; GNU ln's default -P preserves the symlink so the container +# ends up opening the file via its symlink-target path and hits +# Query … failed: Malformed Parquet file. Metadata index: -33578 +# out of range [local:///hits/hits_10.parquet] +# for arbitrary files at query time (trino ≥ 483's local FS provider +# reading the symlink chain confuses its parquet footer parser). +# Copy the file bytes instead — one-time cost per provision, reliable +# afterward. Auto-fall-back to hardlink when the source is a real +# file on the same filesystem (bare-metal ClickBench runs). for f in hits_*.parquet; do - ln -f "$f" "data/hits/$f" + dst="data/hits/$f" + if [ -L "$f" ]; then + # symlink → resolve and copy the target bytes + cp -f "$(readlink -f "$f")" "$dst" + else + ln -f "$f" "$dst" 2>/dev/null || cp -f "$f" "$dst" + fi done sudo chown -R 1000:1000 data