refactor: wave 3 — split db/queries.py (1363 LOC) into a package

skillforge/db/queries/__init__.py

@@ -0,0 +1,126 @@
+"""Database CRUD package — split by entity for maintainability.
+
+Every function previously on ``skillforge.db.queries`` is re-exported here
+so ``from skillforge.db.queries import save_run`` etc. keeps working.
+New code should prefer importing from the submodule directly:
+
+    from skillforge.db.queries.runs import save_run, get_run
+
+Submodule ownership:
+  _helpers       — connection context manager + row helpers
+  challenges     — Challenge rows (per-run evaluation tasks)
+  genomes        — SkillGenome, CompetitionResult, Generation (per-generation data)
+  runs           — EvolutionRun + lineage + leaked skills + zombies
+  seeds          — candidate seeds (user-facing registry of starter skills)
+  taxonomy       — TaxonomyNode + SkillFamily + Variant + VariantEvolution
+  transcripts    — dispatch transcripts (audit trail of every LLM call)
+"""
+
+from __future__ import annotations
+
+from skillforge.db.queries._helpers import _connect, _int_or_none, _row_get
+from skillforge.db.queries.challenges import (
+    _get_challenges_for_run,
+    save_challenge,
+)
+from skillforge.db.queries.genomes import (
+    _get_generations_for_run,
+    _get_genome_by_id,
+    _get_genomes_for_run_gen,
+    _get_results_for_run_gen,
+    _row_to_genome,
+    _row_to_result,
+    save_generation,
+    save_genome,
+    save_result,
+)
+from skillforge.db.queries.runs import (
+    delete_leaked_skill,
+    get_lineage,
+    get_run,
+    list_leaked_skills,
+    list_runs,
+    log_leaked_skill,
+    mark_zombie_runs,
+    save_run,
+)
+from skillforge.db.queries.seeds import (
+    list_candidate_seeds,
+    save_candidate_seed,
+    update_candidate_seed_status,
+)
+from skillforge.db.queries.taxonomy import (
+    _row_to_family,
+    _row_to_taxonomy_node,
+    _row_to_variant,
+    _row_to_variant_evolution,
+    get_active_variants,
+    get_family,
+    get_family_by_slug,
+    get_taxonomy_node,
+    get_taxonomy_node_by_slug,
+    get_taxonomy_tree,
+    get_variant_evolution,
+    get_variant_evolutions_for_run,
+    get_variants_for_family,
+    list_families,
+    save_skill_family,
+    save_taxonomy_node,
+    save_variant,
+    save_variant_evolution,
+)
+from skillforge.db.queries.transcripts import save_transcript
+
+__all__ = [
+    # _helpers
+    "_connect",
+    "_int_or_none",
+    "_row_get",
+    # challenges
+    "save_challenge",
+    "_get_challenges_for_run",
+    # genomes
+    "save_genome",
+    "save_result",
+    "save_generation",
+    "_get_genome_by_id",
+    "_get_genomes_for_run_gen",
+    "_get_generations_for_run",
+    "_get_results_for_run_gen",
+    "_row_to_genome",
+    "_row_to_result",
+    # runs
+    "save_run",
+    "get_run",
+    "list_runs",
+    "get_lineage",
+    "log_leaked_skill",
+    "list_leaked_skills",
+    "delete_leaked_skill",
+    "mark_zombie_runs",
+    # seeds
+    "save_candidate_seed",
+    "list_candidate_seeds",
+    "update_candidate_seed_status",
+    # taxonomy
+    "save_taxonomy_node",
+    "get_taxonomy_node",
+    "get_taxonomy_node_by_slug",
+    "get_taxonomy_tree",
+    "save_skill_family",
+    "get_family",
+    "get_family_by_slug",
+    "list_families",
+    "save_variant",
+    "get_variants_for_family",
+    "get_active_variants",
+    "save_variant_evolution",
+    "get_variant_evolution",
+    "get_variant_evolutions_for_run",
+    "_row_to_family",
+    "_row_to_taxonomy_node",
+    "_row_to_variant",
+    "_row_to_variant_evolution",
+    # transcripts
+    "save_transcript",
+]

skillforge/db/queries/_helpers.py

@@ -0,0 +1,53 @@
+"""Shared helpers for the db.queries submodules.
+
+These are internal to the package but also imported by a handful of
+external call sites that manipulate connections or rows directly
+(``api/bench.py``, ``api/llms.py``, ``api/taxonomy.py``, etc.). The
+package ``__init__`` re-exports them so those imports keep working.
+"""
+
+from __future__ import annotations
+
+from collections.abc import AsyncIterator
+from contextlib import asynccontextmanager
+from pathlib import Path
+
+import aiosqlite
+
+from skillforge.db.database import get_connection
+
+
+@asynccontextmanager
+async def _connect(db_path: Path | None = None) -> AsyncIterator[aiosqlite.Connection]:
+    """Async context manager: open a connection, yield it, then close.
+
+    This avoids the ``async with await get_connection(...)`` double-entry
+    anti-pattern.  ``get_connection`` is kept for callers that need an
+    already-open connection handed back (e.g., the API layer).
+    """
+    conn = await get_connection(db_path)
+    try:
+        yield conn
+    finally:
+        await conn.close()
+
+
+def _int_or_none(v: bool | int | None) -> int | None:
+    """Convert a bool/None to 0/1/None for SQLite INTEGER columns."""
+    if v is None:
+        return None
+    return int(v)
+
+
+def _row_get(row: aiosqlite.Row, column: str, default=None):
+    """Defensive column lookup on an aiosqlite.Row.
+
+    ``aiosqlite.Row`` does not implement ``dict.get()`` and indexing a
+    missing column raises ``IndexError``. Used for v2.0 columns that may
+    be absent on legacy databases that haven't migrated yet (init_db
+    handles the migration but tests sometimes pre-build a partial schema).
+    """
+    try:
+        return row[column]
+    except (IndexError, KeyError):
+        return default

skillforge/db/queries/challenges.py

@@ -0,0 +1,63 @@
+"""CRUD for Challenge rows (evaluation challenges produced by the challenge designer)."""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+import aiosqlite
+
+from skillforge.db.queries._helpers import _connect
+from skillforge.models import Challenge
+
+
+async def save_challenge(
+    challenge: Challenge,
+    run_id: str,
+    db_path: Path | None = None,
+) -> None:
+    """Upsert a Challenge row linked to ``run_id``."""
+    d = challenge.to_dict()
+    async with _connect(db_path) as conn:
+        await conn.execute(
+            """
+            INSERT OR REPLACE INTO challenges
+                (id, run_id, prompt, difficulty, evaluation_criteria,
+                 verification_method, setup_files, gold_standard_hints)
+            VALUES (?, ?, ?, ?, ?, ?, ?, ?)
+            """,
+            (
+                d["id"],
+                run_id,
+                d["prompt"],
+                d["difficulty"],
+                json.dumps(d["evaluation_criteria"]),
+                d["verification_method"],
+                json.dumps(d["setup_files"]),
+                d["gold_standard_hints"],
+            ),
+        )
+        await conn.commit()
+
+
+async def _get_challenges_for_run(
+    run_id: str,
+    conn: aiosqlite.Connection,
+) -> list[Challenge]:
+    async with conn.execute(
+        "SELECT * FROM challenges WHERE run_id = ?", (run_id,)
+    ) as cur:
+        rows = await cur.fetchall()
+    challenges = []
+    for row in rows:
+        d = {
+            "id": row["id"],
+            "prompt": row["prompt"],
+            "difficulty": row["difficulty"],
+            "evaluation_criteria": json.loads(row["evaluation_criteria"]),
+            "verification_method": row["verification_method"],
+            "setup_files": json.loads(row["setup_files"]),
+            "gold_standard_hints": row["gold_standard_hints"],
+        }
+        challenges.append(Challenge.from_dict(d))
+    return challenges