refactor: wave 3b — split api/routes.py (704 LOC) into a package

skillforge/api/routes/__init__.py

@@ -0,0 +1,22 @@
+"""API routes package — split by lifecycle concern.
+
+- ``evolve`` — POST endpoints that start / fork evolution runs
+- ``runs``   — POST cancel + all GET endpoints for reading run state
+
+``router`` here is the single ``APIRouter(prefix="/api")`` that
+``skillforge.main`` mounts. Submodules expose their own un-prefixed
+routers and we include them below — same public URLs as before the
+split.
+"""
+
+from __future__ import annotations
+
+from fastapi import APIRouter
+
+from skillforge.api.routes import evolve, runs
+
+router = APIRouter(prefix="/api")
+router.include_router(evolve.router)
+router.include_router(runs.router)
+
+__all__ = ["router"]

skillforge/api/routes/_helpers.py

@@ -0,0 +1,126 @@
+"""Shared helpers for the routes submodules.
+
+Kept out of ``__init__.py`` so the barrel stays focused on wiring the
+router hierarchy, and out of the route modules so both ``evolve.py``
+and ``runs.py`` can import the same classifier implementation.
+"""
+
+from __future__ import annotations
+
+import logging
+
+from skillforge.models import EvolutionRun
+
+logger = logging.getLogger("skillforge.api")
+
+
+async def classify_run_via_taxonomist(
+    run: EvolutionRun, requested_mode: str | None
+) -> None:
+    """Best-effort: classify the run, persist family + new nodes, stamp the run.
+
+    Sets ``run.family_id`` and ``run.evolution_mode`` in place. If
+    ``requested_mode`` is "atomic" or "molecular" the explicit value wins
+    over whatever the Taxonomist returns. If the Taxonomist call fails for
+    any reason — missing API key, network error, JSON parse failure — we
+    log it, leave ``family_id`` as None, default ``evolution_mode`` to
+    "molecular", and let the run proceed.
+    """
+    from skillforge.config import ANTHROPIC_API_KEY
+    from skillforge.db import get_taxonomy_tree, list_families
+    from skillforge.engine.events import emit
+
+    # No API key → skip classification entirely
+    if not ANTHROPIC_API_KEY:
+        run.evolution_mode = requested_mode or "molecular"
+        return
+
+    # Skip the LLM call when the caller explicitly forced a mode AND specified
+    # no specialization that needs classification (the autoclassify is the
+    # whole point of running the agent — if mode is forced, just stamp it).
+    if requested_mode in {"atomic", "molecular"} and not run.specialization:
+        run.evolution_mode = requested_mode
+        return
+
+    try:
+        from skillforge.agents.taxonomist import classify_and_decompose
+
+        taxonomy_tree = await get_taxonomy_tree()
+        existing_families = await list_families()
+        result = await classify_and_decompose(
+            run.specialization,
+            taxonomy_tree,
+            existing_families,
+        )
+    except Exception:  # noqa: BLE001 — taxonomist best-effort; fall back to molecular
+        logger.exception(
+            "run=%s taxonomist classification failed — defaulting to molecular",
+            run.id[:8],
+        )
+        run.evolution_mode = requested_mode or "molecular"
+        return
+
+    run.family_id = result.family.id
+    # Caller's explicit mode wins over the Taxonomist's recommendation
+    run.evolution_mode = requested_mode or result.evolution_mode
+
+    await emit(
+        run.id,
+        "taxonomy_classified",
+        family_id=result.family.id,
+        family_slug=result.family.slug,
+        domain_slug=result.domain.slug,
+        focus_slug=result.focus.slug,
+        language_slug=result.language.slug,
+        evolution_mode=run.evolution_mode,
+        created_new_nodes=result.created_new_nodes,
+    )
+
+    if result.evolution_mode == "atomic":
+        await emit(
+            run.id,
+            "decomposition_complete",
+            dimension_count=len(result.variant_dimensions),
+            dimensions=[d.to_dict() for d in result.variant_dimensions],
+            reuse_recommendations=[
+                r.to_dict() for r in result.reuse_recommendations
+            ],
+        )
+
+    # Persist a VariantEvolution row per dimension ONLY if the run will
+    # actually execute in atomic mode (the final stamped mode, which may
+    # have been overridden by the caller). The variant_evolutions FK
+    # requires the parent run to exist, so we save_run first.
+    if (
+        run.evolution_mode == "atomic"
+        and result.evolution_mode == "atomic"
+        and result.variant_dimensions
+    ):
+        from datetime import UTC as _UTC
+        from datetime import datetime as _dt
+        from uuid import uuid4 as _uuid4
+
+        # Insert the parent run row first so the FK on
+        # variant_evolutions.parent_run_id is satisfied. save_run is
+        # idempotent (INSERT OR REPLACE) so the second save_run later in
+        # the route handler is a no-op refresh.
+        from skillforge.db import save_run as _save_run
+        from skillforge.db import save_variant_evolution
+        from skillforge.models import VariantEvolution
+
+        await _save_run(run)
+
+        for dim in result.variant_dimensions:
+            await save_variant_evolution(
+                VariantEvolution(
+                    id=f"vevo_{_uuid4().hex[:12]}",
+                    family_id=result.family.id,
+                    dimension=dim.name,
+                    tier=dim.tier,
+                    parent_run_id=run.id,
+                    population_size=2,
+                    num_generations=1,
+                    status="pending",
+                    created_at=_dt.now(_UTC),
+                )
+            )

skillforge/api/routes/evolve.py

@@ -0,0 +1,214 @@
+"""POST endpoints that start evolution runs.
+
+- POST /api/evolve — start a fresh run from a specialization
+- POST /api/evolve/from-parent — fork from a registry skill, upload, or
+  inline generated skill
+"""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+import uuid
+from datetime import UTC, datetime
+
+from fastapi import APIRouter, HTTPException
+from pydantic import BaseModel, Field
+
+from skillforge.api.routes._helpers import classify_run_via_taxonomist
+from skillforge.api.schemas import EvolveRequest, EvolveResponse, Mode
+from skillforge.api.uploads import clear_upload, get_upload
+from skillforge.config import invite_code_valid
+from skillforge.db.database import init_db
+from skillforge.db.queries import get_run, save_run
+from skillforge.engine.evolution import run_evolution
+from skillforge.engine.run_registry import registry
+from skillforge.models import EvolutionRun, SkillGenome
+
+logger = logging.getLogger("skillforge.api.evolve")
+router = APIRouter()
+
+
+@router.post("/evolve", response_model=EvolveResponse)
+async def start_evolution(req: EvolveRequest) -> EvolveResponse:
+    """Start a new evolution run and return its ID + WebSocket URL.
+
+    Validates the request, creates an EvolutionRun, persists it, and spawns
+    a background task that runs the evolution loop. Returns immediately —
+    the client subscribes to the WebSocket to watch progress.
+    """
+    # Invite gating — returns True if gating is disabled OR the code is valid
+    if not invite_code_valid(req.invite_code):
+        raise HTTPException(
+            status_code=403,
+            detail="This platform is invite-only. Enter a valid invite code or request one.",
+        )
+
+    # Mode-specific validation
+    if req.mode == Mode.domain and not req.specialization:
+        raise HTTPException(status_code=400, detail="domain mode requires 'specialization'")
+    if req.mode == Mode.meta:
+        raise HTTPException(status_code=501, detail="meta mode is v1.1, not yet supported")
+
+    # Ensure DB is initialized (idempotent)
+    await init_db()
+
+    run = EvolutionRun(
+        id=str(uuid.uuid4()),
+        mode=req.mode.value,
+        specialization=req.specialization or "",
+        population_size=req.population_size,
+        num_generations=req.num_generations,
+        max_budget_usd=req.max_budget_usd,
+        status="pending",
+        created_at=datetime.now(UTC),
+    )
+
+    # v2.0 — Taxonomist classification before evolution starts. Best-effort:
+    # the run still proceeds in molecular mode if the LLM call fails or no
+    # API key is available, so we never block submission on classification.
+    await classify_run_via_taxonomist(run, req.evolution_mode)
+
+    await save_run(run)
+
+    # Spawn background task — store reference so it isn't GC'd
+    task = asyncio.create_task(run_evolution(run))
+    registry.set_task(run.id, task)
+    logger.info("run=%s started: spec=%s pop=%d gens=%d",
+                run.id[:8], run.specialization[:60], run.population_size, run.num_generations)
+
+    # Cleanup callback removes the task from the registry when it finishes
+    def _cleanup(t: asyncio.Task) -> None:
+        registry.clear_task(run.id)
+        exc = t.exception() if not t.cancelled() else None
+        if exc:
+            logger.error("run=%s task failed: %s", run.id[:8], exc)
+        else:
+            logger.info("run=%s task completed", run.id[:8])
+
+    task.add_done_callback(_cleanup)
+
+    return EvolveResponse(run_id=run.id, ws_url=f"/ws/evolve/{run.id}")
+
+
+# ---------------------------------------------------------------------------
+# Fork-and-evolve: start a run from an existing Skill (registry seed or upload)
+# ---------------------------------------------------------------------------
+
+
+class EvolveFromParentRequest(BaseModel):
+    parent_source: str = Field(..., description='"registry", "upload", or "generated"')
+    parent_id: str = Field("", description="skill_id (registry) or upload_id (upload)")
+    specialization: str | None = None
+    population_size: int = 5
+    num_generations: int = 3
+    max_budget_usd: float = 10.0
+    invite_code: str | None = None
+    # For parent_source="generated" — inline skill content
+    skill_md_content: str | None = None
+    supporting_files: dict[str, str] | None = None
+
+
+@router.post("/evolve/from-parent", response_model=EvolveResponse)
+async def start_evolution_from_parent(req: EvolveFromParentRequest) -> EvolveResponse:
+    """Start a new evolution run using an existing Skill as the gen-0 parent.
+
+    Supports two parent sources:
+      - ``registry``: ``parent_id`` is a skill_id inside the seed-library run
+        (or any other run's skill). Resolved via get_run(seed-library).
+      - ``upload``: ``parent_id`` is an upload_id from POST /api/uploads/skill.
+        Resolved via the in-memory upload cache.
+
+    The parent is stashed in the ``RunRegistry`` (see ``engine/run_registry.py``)
+    keyed by the new run's id. The evolution engine picks it up at gen 0 spawn
+    time and routes through ``spawner.spawn_from_parent()`` instead of
+    ``spawn_gen0()``.
+    """
+    if not invite_code_valid(req.invite_code):
+        raise HTTPException(
+            status_code=403,
+            detail="This platform is invite-only. Enter a valid invite code or request one.",
+        )
+
+    # Resolve the parent genome
+    if req.parent_source == "registry":
+        # Search the seed-library run first, then fall back to any run
+        parent = None
+        seed_run = await get_run("seed-library")
+        if seed_run:
+            for gen in seed_run.generations:
+                for sk in gen.skills:
+                    if sk.id == req.parent_id:
+                        parent = sk
+                        break
+                if parent:
+                    break
+        if parent is None:
+            raise HTTPException(
+                status_code=404,
+                detail=f"registry skill {req.parent_id!r} not found",
+            )
+        effective_spec = req.specialization or (
+            parent.frontmatter.get("description", "")[:200]
+            if isinstance(parent.frontmatter, dict)
+            else ""
+        )
+    elif req.parent_source == "upload":
+        parent = get_upload(req.parent_id)
+        if parent is None:
+            raise HTTPException(
+                status_code=404, detail=f"upload {req.parent_id!r} not found or expired"
+            )
+        effective_spec = req.specialization or "User-uploaded Skill (evolved)"
+    elif req.parent_source == "generated":
+        if not req.skill_md_content:
+            raise HTTPException(
+                status_code=400,
+                detail="generated source requires skill_md_content",
+            )
+        parent = SkillGenome(
+            id=str(uuid.uuid4()),
+            generation=0,
+            skill_md_content=req.skill_md_content,
+            supporting_files=req.supporting_files or {},
+            frontmatter={},
+            traits=[],
+            maturity="draft",
+        )
+        effective_spec = req.specialization or "AI-generated skill"
+    else:
+        raise HTTPException(
+            status_code=400,
+            detail=f"parent_source must be 'registry', 'upload', or 'generated', got {req.parent_source!r}",
+        )
+
+    await init_db()
+
+    run = EvolutionRun(
+        id=str(uuid.uuid4()),
+        mode="domain",
+        specialization=effective_spec,
+        population_size=req.population_size,
+        num_generations=req.num_generations,
+        max_budget_usd=req.max_budget_usd,
+        status="pending",
+        created_at=datetime.now(UTC),
+    )
+    await save_run(run)
+
+    # Stash the parent so the engine's gen-0 spawn picks it up
+    registry.stash_parent(run.id, parent)
+
+    # Clear the upload cache so we don't leak memory
+    if req.parent_source == "upload":
+        clear_upload(req.parent_id)
+
+    task = asyncio.create_task(run_evolution(run))
+    registry.set_task(run.id, task)
+
+    def _cleanup(t: asyncio.Task) -> None:
+        registry.clear_task(run.id)
+
+    task.add_done_callback(_cleanup)
+
+    return EvolveResponse(run_id=run.id, ws_url=f"/ws/evolve/{run.id}")