diff --git a/docs/JOURNAL.md b/docs/JOURNAL.md index 13a8715..aa2b259 100644 --- a/docs/JOURNAL.md +++ b/docs/JOURNAL.md @@ -18,6 +18,16 @@ protocol. **Newest entries at the top.** Tag each entry with one or more of: --- +## 2026-07-30 — ensure_schema still crashed on fresh DBs after lock-timeout helpers #mistake #repro #decision +**Context:** issue #120; main added `_drop_not_null_if_needed` / `_column_nullable` for lock avoidance. +**Expected:** fresh `create_all` + `ensure_schema` succeeds. +**Actual:** `_drop_not_null_if_needed` treated a missing column as "not nullable" and still issued +`ALTER COLUMN … DROP NOT NULL`; legacy `UPDATE … team_name` / `benchmark` also ran unconditionally. +**Root cause:** existence was not checked before ALTER/UPDATE of legacy columns. +**Fix / decision:** require `_column_exists` before DROP NOT NULL; gate legacy back-fills the same way. +Added `validator/tests/test_ensure_schema.py`. +**Follow-up:** none. + ## 2026-07-12 — code grader: add resource limits on top of the HOME/secrets fix #security #decision **Context:** issue #71 — the code grader (`run_pass_at_1`) runs untrusted miner/LLM candidate code. The core secret-leak fix (isolated throwaway HOME/cwd, scrubbed env, `python -I`) already landed on main. **Finding:** main's sandbox closes the HOME/secrets exfiltration vector but has **no resource limits** — an untrusted candidate can still exhaust host memory or fork-bomb the eval box within its wall-clock timeout (verified: a 4 GiB `bytearray` allocation runs to completion and "passes" on main). diff --git a/validator/src/eval_backend/db.py b/validator/src/eval_backend/db.py index 68369be..c3698fc 100644 --- a/validator/src/eval_backend/db.py +++ b/validator/src/eval_backend/db.py @@ -62,7 +62,11 @@ def _add_column_if_missing(conn, table: str, column: str, ddl_type: str) -> None def _drop_not_null_if_needed(conn, table: str, column: str) -> None: - if not _column_nullable(conn, table, column): + # Missing columns must be skipped: on a fresh DB the legacy submission + # fields are ORM @property accessors, not real columns (issue #120). + # `_column_nullable` returns False when the column is absent, so without + # an existence check we would still issue ALTER COLUMN and crash. + if _column_exists(conn, table, column) and not _column_nullable(conn, table, column): conn.exec_driver_sql(f"ALTER TABLE {table} ALTER COLUMN {column} DROP NOT NULL") @@ -101,14 +105,17 @@ def ensure_schema(engine) -> None: _drop_not_null_if_needed(conn, "submissions", "artifact_sha256") _drop_not_null_if_needed(conn, "submissions", "checkpoint_path") _drop_not_null_if_needed(conn, "submissions", "benchmark") - conn.exec_driver_sql( - "UPDATE submissions SET miner_id = COALESCE(miner_id, team_name) " - "WHERE miner_id IS NULL AND team_name IS NOT NULL" - ) - conn.exec_driver_sql( - "UPDATE submissions SET benchmark_names_json = COALESCE(benchmark_names_json, json_build_array(benchmark)) " - "WHERE benchmark_names_json IS NULL AND benchmark IS NOT NULL" - ) + # Legacy back-fills reference columns that only exist on pre-property DBs. + if _column_exists(conn, "submissions", "team_name"): + conn.exec_driver_sql( + "UPDATE submissions SET miner_id = COALESCE(miner_id, team_name) " + "WHERE miner_id IS NULL AND team_name IS NOT NULL" + ) + if _column_exists(conn, "submissions", "benchmark"): + conn.exec_driver_sql( + "UPDATE submissions SET benchmark_names_json = COALESCE(benchmark_names_json, json_build_array(benchmark)) " + "WHERE benchmark_names_json IS NULL AND benchmark IS NOT NULL" + ) conn.exec_driver_sql( "UPDATE competition_runtime_config " "SET default_eval_execution_mode = COALESCE(NULLIF(default_eval_execution_mode, ''), 'remote_gpu')" diff --git a/validator/tests/test_ensure_schema.py b/validator/tests/test_ensure_schema.py new file mode 100644 index 0000000..05f51ee --- /dev/null +++ b/validator/tests/test_ensure_schema.py @@ -0,0 +1,105 @@ +from __future__ import annotations + +from sqlalchemy import text + +from eval_backend.db import Base, ensure_schema +import eval_backend.models # noqa: F401 — register ORM metadata + +_LEGACY_COLUMNS = ( + "artifact_name", + "artifact_path", + "artifact_sha256", + "checkpoint_path", + "benchmark", + "team_name", +) + + +def test_ensure_schema_succeeds_on_fresh_create_all(validator_engine) -> None: + """Fresh DBs have no legacy submission columns; ensure_schema must not crash.""" + Base.metadata.create_all(validator_engine) + ensure_schema(validator_engine) + + with validator_engine.connect() as conn: + columns = { + row[0] + for row in conn.execute( + text( + "SELECT column_name FROM information_schema.columns " + "WHERE table_name = 'submissions' AND table_schema = current_schema()" + ) + ) + } + assert "miner_id" in columns + assert "benchmark_names_json" in columns + assert "artifact_name" not in columns + assert "team_name" not in columns + + +def test_ensure_schema_relaxes_and_backfills_legacy_columns(validator_engine) -> None: + """When legacy columns exist, ensure_schema should relax NOT NULL and back-fill.""" + Base.metadata.create_all(validator_engine) + + try: + with validator_engine.begin() as conn: + for column, coltype in ( + ("artifact_name", "VARCHAR(255)"), + ("artifact_path", "VARCHAR(1024)"), + ("artifact_sha256", "VARCHAR(64)"), + ("checkpoint_path", "VARCHAR(1024)"), + ("benchmark", "VARCHAR(64)"), + ("team_name", "VARCHAR(255)"), + ): + conn.execute( + text(f"ALTER TABLE submissions ADD COLUMN IF NOT EXISTS {column} {coltype}") + ) + conn.execute(text(f"UPDATE submissions SET {column} = '' WHERE {column} IS NULL")) + conn.execute(text(f"ALTER TABLE submissions ALTER COLUMN {column} SET NOT NULL")) + + conn.execute(text("DELETE FROM submissions WHERE id = 'legacy-1'")) + conn.execute( + text( + """ + INSERT INTO submissions ( + id, source, team_name, artifact_name, artifact_path, + artifact_sha256, checkpoint_path, benchmark, status, + benchmark_names_json, created_at, updated_at + ) VALUES ( + 'legacy-1', 'upload', 'old-team', 'a.npy', '/tmp/a.npy', + 'deadbeef', '/tmp/ckpt.npy', 'math500', 'queued', + '[]'::json, NOW(), NOW() + ) + """ + ) + ) + conn.execute( + text("ALTER TABLE submissions ALTER COLUMN benchmark_names_json DROP NOT NULL") + ) + conn.execute( + text("UPDATE submissions SET benchmark_names_json = NULL WHERE id = 'legacy-1'") + ) + conn.execute(text("UPDATE submissions SET miner_id = NULL WHERE id = 'legacy-1'")) + + ensure_schema(validator_engine) + + with validator_engine.connect() as conn: + row = conn.execute( + text( + "SELECT miner_id, benchmark_names_json::text, " + "(SELECT is_nullable FROM information_schema.columns " + " WHERE table_name = 'submissions' AND column_name = 'artifact_name' " + " AND table_schema = current_schema()) " + "FROM submissions WHERE id = 'legacy-1'" + ) + ).one() + assert row[0] == "old-team" + assert "math500" in (row[1] or "") + assert row[2] == "YES" + finally: + with validator_engine.begin() as conn: + for column in _LEGACY_COLUMNS: + conn.execute(text(f"ALTER TABLE submissions DROP COLUMN IF EXISTS {column}")) + conn.execute( + text("ALTER TABLE submissions ALTER COLUMN benchmark_names_json SET NOT NULL") + ) + conn.execute(text("DELETE FROM submissions WHERE id = 'legacy-1'"))