From 63f31e24854388ab44246720f07b71dc0e516bb3 Mon Sep 17 00:00:00 2001 From: RealDiligent Date: Mon, 20 Jul 2026 22:52:17 +0800 Subject: [PATCH 1/3] fix(validator): guard legacy ensure_schema alters for fresh DBs (#120) Postgres has no IF EXISTS for ALTER COLUMN. Gate DROP NOT NULL and legacy back-fills on information_schema so first-run create_all + ensure_schema works. Co-authored-by: Cursor --- docs/JOURNAL.md | 11 ++++ validator/src/eval_backend/db.py | 39 ++++++++---- validator/tests/test_ensure_schema.py | 87 +++++++++++++++++++++++++++ 3 files changed, 124 insertions(+), 13 deletions(-) create mode 100644 validator/tests/test_ensure_schema.py diff --git a/docs/JOURNAL.md b/docs/JOURNAL.md index 117bb79..0198789 100644 --- a/docs/JOURNAL.md +++ b/docs/JOURNAL.md @@ -18,6 +18,17 @@ protocol. **Newest entries at the top.** Tag each entry with one or more of: --- +## 2026-07-20 — Validator crashed on first startup: ensure_schema altered missing columns #mistake #repro #decision +**Context:** issue #120 — deploying the validator against an empty Postgres. +**Expected:** first-run `create_all` then `ensure_schema` brings up a fresh schema cleanly. +**Actual:** startup aborted with `UndefinedColumn: column "artifact_name" does not exist`. +**Root cause:** legacy columns became `@property` accessors, so `create_all` never creates them; +Postgres has no `IF EXISTS` for `ALTER COLUMN`, and unguarded `DROP NOT NULL` / back-fill +`UPDATE`s crashed on every fresh DB. +**Fix / decision:** gate legacy relaxations/back-fills on `information_schema.columns`; leave +additive `ADD COLUMN IF NOT EXISTS` untouched. Added `validator/tests/test_ensure_schema.py`. +**Follow-up:** none. + ## 2026-07-12 — Validator Postgres tests no longer silently skip in CI #decision #repro **Context:** issue #118 flagged that validator DB-backed tests could ``pytest.skip`` whenever Postgres was unreachable, including on CI where no database service was provisioned. diff --git a/validator/src/eval_backend/db.py b/validator/src/eval_backend/db.py index 457d1d9..f2891db 100644 --- a/validator/src/eval_backend/db.py +++ b/validator/src/eval_backend/db.py @@ -66,19 +66,32 @@ def ensure_schema(engine) -> None: conn.exec_driver_sql( "ALTER TABLE competition_runtime_config ADD COLUMN IF NOT EXISTS king_score DOUBLE PRECISION" ) - conn.exec_driver_sql("ALTER TABLE submissions ALTER COLUMN artifact_name DROP NOT NULL") - conn.exec_driver_sql("ALTER TABLE submissions ALTER COLUMN artifact_path DROP NOT NULL") - conn.exec_driver_sql("ALTER TABLE submissions ALTER COLUMN artifact_sha256 DROP NOT NULL") - conn.exec_driver_sql("ALTER TABLE submissions ALTER COLUMN checkpoint_path DROP NOT NULL") - conn.exec_driver_sql("ALTER TABLE submissions ALTER COLUMN benchmark DROP NOT NULL") - conn.exec_driver_sql( - "UPDATE submissions SET miner_id = COALESCE(miner_id, team_name) " - "WHERE miner_id IS NULL AND team_name IS NOT NULL" - ) - conn.exec_driver_sql( - "UPDATE submissions SET benchmark_names_json = COALESCE(benchmark_names_json, json_build_array(benchmark)) " - "WHERE benchmark_names_json IS NULL AND benchmark IS NOT NULL" - ) + + # Legacy columns below only exist on DBs created before those fields became + # ORM @property accessors. Postgres has no IF EXISTS for ALTER COLUMN, so + # gate each statement on information_schema — otherwise fresh-DB startup + # crashes with UndefinedColumn (issue #120). + legacy_columns = { + row[0] + for row in conn.exec_driver_sql( + "SELECT column_name FROM information_schema.columns " + "WHERE table_name = 'submissions' AND table_schema = current_schema()" + ) + } + for column in ("artifact_name", "artifact_path", "artifact_sha256", "checkpoint_path", "benchmark"): + if column in legacy_columns: + conn.exec_driver_sql(f"ALTER TABLE submissions ALTER COLUMN {column} DROP NOT NULL") + + if "team_name" in legacy_columns: + conn.exec_driver_sql( + "UPDATE submissions SET miner_id = COALESCE(miner_id, team_name) " + "WHERE miner_id IS NULL AND team_name IS NOT NULL" + ) + if "benchmark" in legacy_columns: + conn.exec_driver_sql( + "UPDATE submissions SET benchmark_names_json = COALESCE(benchmark_names_json, json_build_array(benchmark)) " + "WHERE benchmark_names_json IS NULL AND benchmark IS NOT NULL" + ) conn.exec_driver_sql( "UPDATE competition_runtime_config " "SET default_eval_execution_mode = COALESCE(NULLIF(default_eval_execution_mode, ''), 'remote_gpu')" diff --git a/validator/tests/test_ensure_schema.py b/validator/tests/test_ensure_schema.py new file mode 100644 index 0000000..fdbfa2d --- /dev/null +++ b/validator/tests/test_ensure_schema.py @@ -0,0 +1,87 @@ +from __future__ import annotations + +from sqlalchemy import text + +from eval_backend.db import Base, ensure_schema +import eval_backend.models # noqa: F401 — register ORM metadata + + +def test_ensure_schema_succeeds_on_fresh_create_all(validator_engine) -> None: + """Fresh DBs have no legacy submission columns; ensure_schema must not crash.""" + Base.metadata.create_all(validator_engine) + ensure_schema(validator_engine) + + with validator_engine.connect() as conn: + columns = { + row[0] + for row in conn.execute( + text( + "SELECT column_name FROM information_schema.columns " + "WHERE table_name = 'submissions' AND table_schema = current_schema()" + ) + ) + } + assert "miner_id" in columns + assert "benchmark_names_json" in columns + # These are @property accessors on the current model — not real columns. + assert "artifact_name" not in columns + assert "team_name" not in columns + + +def test_ensure_schema_relaxes_and_backfills_legacy_columns(validator_engine) -> None: + """When legacy columns exist, ensure_schema should relax NOT NULL and back-fill.""" + Base.metadata.create_all(validator_engine) + + with validator_engine.begin() as conn: + for column, coltype in ( + ("artifact_name", "VARCHAR(255)"), + ("artifact_path", "VARCHAR(1024)"), + ("artifact_sha256", "VARCHAR(64)"), + ("checkpoint_path", "VARCHAR(1024)"), + ("benchmark", "VARCHAR(64)"), + ("team_name", "VARCHAR(255)"), + ): + conn.execute( + text( + f"ALTER TABLE submissions ADD COLUMN IF NOT EXISTS {column} {coltype}" + ) + ) + # Force NOT NULL so DROP NOT NULL is meaningful, then allow nulls for insert. + conn.execute(text(f"UPDATE submissions SET {column} = '' WHERE {column} IS NULL")) + conn.execute(text(f"ALTER TABLE submissions ALTER COLUMN {column} SET NOT NULL")) + + conn.execute(text("DELETE FROM submissions WHERE id = 'legacy-1'")) + conn.execute( + text( + """ + INSERT INTO submissions ( + id, source, team_name, artifact_name, artifact_path, + artifact_sha256, checkpoint_path, benchmark, status, + benchmark_names_json, created_at, updated_at + ) VALUES ( + 'legacy-1', 'upload', 'old-team', 'a.npy', '/tmp/a.npy', + 'deadbeef', '/tmp/ckpt.npy', 'math500', 'queued', + '[]'::json, NOW(), NOW() + ) + """ + ) + ) + + ensure_schema(validator_engine) + + with validator_engine.connect() as conn: + row = conn.execute( + text( + "SELECT miner_id, benchmark_names_json::text " + "FROM submissions WHERE id = 'legacy-1'" + ) + ).one() + # Nullable after DROP NOT NULL — insert NULL succeeds. + conn.execute( + text( + "UPDATE submissions SET artifact_name = NULL, benchmark = NULL " + "WHERE id = 'legacy-1'" + ) + ) + assert row[0] == "old-team" + assert "math500" in (row[1] or "") From af0b06e4a8028c3ad90bfe273732b7ffab0a42e2 Mon Sep 17 00:00:00 2001 From: RealDiligent Date: Mon, 20 Jul 2026 22:52:32 +0800 Subject: [PATCH 2/3] test(validator): assert legacy ensure_schema back-fill and DROP NOT NULL Co-authored-by: Cursor --- validator/tests/test_ensure_schema.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/validator/tests/test_ensure_schema.py b/validator/tests/test_ensure_schema.py index fdbfa2d..f7be64e 100644 --- a/validator/tests/test_ensure_schema.py +++ b/validator/tests/test_ensure_schema.py @@ -66,22 +66,23 @@ def test_ensure_schema_relaxes_and_backfills_legacy_columns(validator_engine) -> """ ) ) + # Make the back-fill path fire: NULL benchmark_names_json + non-null benchmark. + conn.execute(text("ALTER TABLE submissions ALTER COLUMN benchmark_names_json DROP NOT NULL")) + conn.execute(text("UPDATE submissions SET benchmark_names_json = NULL WHERE id = 'legacy-1'")) + conn.execute(text("UPDATE submissions SET miner_id = NULL WHERE id = 'legacy-1'")) ensure_schema(validator_engine) with validator_engine.connect() as conn: row = conn.execute( text( - "SELECT miner_id, benchmark_names_json::text " + "SELECT miner_id, benchmark_names_json::text, " + "(SELECT is_nullable FROM information_schema.columns " + " WHERE table_name = 'submissions' AND column_name = 'artifact_name' " + " AND table_schema = current_schema()) " "FROM submissions WHERE id = 'legacy-1'" ) ).one() - # Nullable after DROP NOT NULL — insert NULL succeeds. - conn.execute( - text( - "UPDATE submissions SET artifact_name = NULL, benchmark = NULL " - "WHERE id = 'legacy-1'" - ) - ) assert row[0] == "old-team" assert "math500" in (row[1] or "") + assert row[2] == "YES" \ No newline at end of file From f843b50e7e2cf0bfc7934340c7c53dcffcb3946f Mon Sep 17 00:00:00 2001 From: RealDiligent Date: Mon, 20 Jul 2026 22:55:10 +0800 Subject: [PATCH 3/3] test(validator): drop legacy columns after ensure_schema back-fill test Leaving NOT NULL team_name/artifact_* columns polluted the shared Postgres test DB and broke later suite inserts. Co-authored-by: Cursor --- validator/tests/test_ensure_schema.py | 114 +++++++++++++++----------- 1 file changed, 68 insertions(+), 46 deletions(-) diff --git a/validator/tests/test_ensure_schema.py b/validator/tests/test_ensure_schema.py index f7be64e..423436e 100644 --- a/validator/tests/test_ensure_schema.py +++ b/validator/tests/test_ensure_schema.py @@ -5,6 +5,15 @@ from eval_backend.db import Base, ensure_schema import eval_backend.models # noqa: F401 — register ORM metadata +_LEGACY_COLUMNS = ( + "artifact_name", + "artifact_path", + "artifact_sha256", + "checkpoint_path", + "benchmark", + "team_name", +) + def test_ensure_schema_succeeds_on_fresh_create_all(validator_engine) -> None: """Fresh DBs have no legacy submission columns; ensure_schema must not crash.""" @@ -32,57 +41,70 @@ def test_ensure_schema_relaxes_and_backfills_legacy_columns(validator_engine) -> """When legacy columns exist, ensure_schema should relax NOT NULL and back-fill.""" Base.metadata.create_all(validator_engine) - with validator_engine.begin() as conn: - for column, coltype in ( - ("artifact_name", "VARCHAR(255)"), - ("artifact_path", "VARCHAR(1024)"), - ("artifact_sha256", "VARCHAR(64)"), - ("checkpoint_path", "VARCHAR(1024)"), - ("benchmark", "VARCHAR(64)"), - ("team_name", "VARCHAR(255)"), - ): + try: + with validator_engine.begin() as conn: + for column, coltype in ( + ("artifact_name", "VARCHAR(255)"), + ("artifact_path", "VARCHAR(1024)"), + ("artifact_sha256", "VARCHAR(64)"), + ("checkpoint_path", "VARCHAR(1024)"), + ("benchmark", "VARCHAR(64)"), + ("team_name", "VARCHAR(255)"), + ): + conn.execute( + text( + f"ALTER TABLE submissions ADD COLUMN IF NOT EXISTS {column} {coltype}" + ) + ) + conn.execute(text(f"UPDATE submissions SET {column} = '' WHERE {column} IS NULL")) + conn.execute(text(f"ALTER TABLE submissions ALTER COLUMN {column} SET NOT NULL")) + + conn.execute(text("DELETE FROM submissions WHERE id = 'legacy-1'")) conn.execute( text( - f"ALTER TABLE submissions ADD COLUMN IF NOT EXISTS {column} {coltype}" + """ + INSERT INTO submissions ( + id, source, team_name, artifact_name, artifact_path, + artifact_sha256, checkpoint_path, benchmark, status, + benchmark_names_json, created_at, updated_at + ) VALUES ( + 'legacy-1', 'upload', 'old-team', 'a.npy', '/tmp/a.npy', + 'deadbeef', '/tmp/ckpt.npy', 'math500', 'queued', + '[]'::json, NOW(), NOW() + ) + """ ) ) - # Force NOT NULL so DROP NOT NULL is meaningful, then allow nulls for insert. - conn.execute(text(f"UPDATE submissions SET {column} = '' WHERE {column} IS NULL")) - conn.execute(text(f"ALTER TABLE submissions ALTER COLUMN {column} SET NOT NULL")) - - conn.execute(text("DELETE FROM submissions WHERE id = 'legacy-1'")) - conn.execute( - text( - """ - INSERT INTO submissions ( - id, source, team_name, artifact_name, artifact_path, - artifact_sha256, checkpoint_path, benchmark, status, - benchmark_names_json, created_at, updated_at - ) VALUES ( - 'legacy-1', 'upload', 'old-team', 'a.npy', '/tmp/a.npy', - 'deadbeef', '/tmp/ckpt.npy', 'math500', 'queued', - '[]'::json, NOW(), NOW() - ) - """ + # Make the back-fill path fire: NULL benchmark_names_json + non-null benchmark. + conn.execute( + text("ALTER TABLE submissions ALTER COLUMN benchmark_names_json DROP NOT NULL") ) - ) - # Make the back-fill path fire: NULL benchmark_names_json + non-null benchmark. - conn.execute(text("ALTER TABLE submissions ALTER COLUMN benchmark_names_json DROP NOT NULL")) - conn.execute(text("UPDATE submissions SET benchmark_names_json = NULL WHERE id = 'legacy-1'")) - conn.execute(text("UPDATE submissions SET miner_id = NULL WHERE id = 'legacy-1'")) + conn.execute( + text("UPDATE submissions SET benchmark_names_json = NULL WHERE id = 'legacy-1'") + ) + conn.execute(text("UPDATE submissions SET miner_id = NULL WHERE id = 'legacy-1'")) - ensure_schema(validator_engine) + ensure_schema(validator_engine) - with validator_engine.connect() as conn: - row = conn.execute( - text( - "SELECT miner_id, benchmark_names_json::text, " - "(SELECT is_nullable FROM information_schema.columns " - " WHERE table_name = 'submissions' AND column_name = 'artifact_name' " - " AND table_schema = current_schema()) " - "FROM submissions WHERE id = 'legacy-1'" + with validator_engine.connect() as conn: + row = conn.execute( + text( + "SELECT miner_id, benchmark_names_json::text, " + "(SELECT is_nullable FROM information_schema.columns " + " WHERE table_name = 'submissions' AND column_name = 'artifact_name' " + " AND table_schema = current_schema()) " + "FROM submissions WHERE id = 'legacy-1'" + ) + ).one() + assert row[0] == "old-team" + assert "math500" in (row[1] or "") + assert row[2] == "YES" + finally: + # Restore a clean modern schema for later tests sharing this engine. + with validator_engine.begin() as conn: + for column in _LEGACY_COLUMNS: + conn.execute(text(f"ALTER TABLE submissions DROP COLUMN IF EXISTS {column}")) + conn.execute( + text("ALTER TABLE submissions ALTER COLUMN benchmark_names_json SET NOT NULL") ) - ).one() - assert row[0] == "old-team" - assert "math500" in (row[1] or "") - assert row[2] == "YES" \ No newline at end of file + conn.execute(text("DELETE FROM submissions WHERE id = 'legacy-1'"))