fix(eval): cancel queued provider-eval jobs when a standalone evaluation is deleted

validator/src/eval_backend/api/routes.py

@@ -60,6 +60,7 @@
 from ..services.github import create_pr_submission
 from ..services.github import set_commit_status
 from ..services.artifacts import persist_stored_artifact
+from ..services.queue import cancel_evaluation_jobs
 from ..services.queue import cancel_submission_jobs
 from ..services.queue import enqueue_provider_eval_job
 from ..services.queue import enqueue_submission_job
@@ -1211,6 +1212,7 @@ def admin_delete_evaluation(
                 detail="only standalone provider evaluations can be deleted here",
             )
         run.deleted_at = _utcnow()
+        cancel_evaluation_jobs(session, evaluation_id, reason="evaluation deleted")
         session.commit()
         return _evaluation_to_schema(run)
     except HTTPException:

validator/src/eval_backend/services/queue.py

@@ -99,6 +99,32 @@ def cancel_submission_jobs(
     return jobs
 
 
+def cancel_evaluation_jobs(
+    session: Session,
+    evaluation_id: int,
+    *,
+    reason: str = "evaluation deleted",
+) -> list[JobQueue]:
+    jobs = session.execute(
+        select(JobQueue).where(
+            JobQueue.job_type == "provider_eval",
+            JobQueue.job_id == str(evaluation_id),
+            JobQueue.status == "queued",
+        )
+    ).scalars().all()
+    now = _utcnow()
+    for job in jobs:
+        job.status = "cancelled"
+        job.claimed_by = None
+        job.claimed_at = None
+        job.heartbeat_at = None
+        job.last_error = reason
+        job.updated_at = now
+    if jobs:
+        session.flush()
+    return jobs
+
+
 def enqueue_train_job(
     session: Session,
     train: TrainRun,

validator/tests/test_delete_evaluation.py

@@ -0,0 +1,150 @@
+from __future__ import annotations
+
+from datetime import datetime, timezone
+from unittest.mock import MagicMock
+
+import pytest
+from sqlalchemy.orm import sessionmaker
+
+from eval_backend.api.routes import admin_delete_evaluation
+from eval_backend.models import EvaluationRun, JobQueue, Submission
+from eval_backend.services.queue import enqueue_provider_eval_job
+
+
+def _build_request(session_factory) -> MagicMock:
+    request = MagicMock()
+    request.app.state.session_factory = session_factory
+    return request
+
+
+def _standalone_eval(status: str = "queued") -> EvaluationRun:
+    now = datetime.now(timezone.utc)
+    return EvaluationRun(
+        submission_id=None,
+        benchmark_names_json=["math500"],
+        provider="compatible",
+        models_config="configs/models.openrouter-chutes.yaml",
+        execution_mode="local_cpu",
+        device="cpu",
+        dtype="float32",
+        batch_size=1,
+        max_items=1,
+        status=status,
+        phase=status,
+        created_at=now,
+    )
+
+
+def test_delete_standalone_evaluation_cancels_its_queued_job(validator_engine):
+    session_factory = sessionmaker(
+        bind=validator_engine,
+        autoflush=False,
+        autocommit=False,
+        expire_on_commit=False,
+        future=True,
+    )
+    with session_factory() as session:
+        run = _standalone_eval(status="queued")
+        session.add(run)
+        session.flush()
+        job = enqueue_provider_eval_job(
+            session,
+            run,
+            payload_json={"evaluation_id": run.id, "job_type": "provider_eval"},
+        )
+        evaluation_id = run.id
+        job_id = job.id
+        session.commit()
+
+    response = admin_delete_evaluation(
+        _build_request(session_factory),
+        evaluation_id,
+        user=MagicMock(),
+    )
+
+    assert response.deleted_at is not None
+
+    with session_factory() as session:
+        run = session.get(EvaluationRun, evaluation_id)
+        job = session.get(JobQueue, job_id)
+        assert run.deleted_at is not None
+        # A soft-deleted standalone eval must not stay in the queue: otherwise the
+        # worker would still claim the job, run the provider route, and incur cost.
+        assert job.status == "cancelled"
+        assert job.last_error == "evaluation deleted"
+
+
+def test_delete_evaluation_leaves_other_evaluations_jobs_untouched(validator_engine):
+    session_factory = sessionmaker(
+        bind=validator_engine,
+        autoflush=False,
+        autocommit=False,
+        expire_on_commit=False,
+        future=True,
+    )
+    with session_factory() as session:
+        target = _standalone_eval(status="queued")
+        other = _standalone_eval(status="queued")
+        session.add_all([target, other])
+        session.flush()
+        target_job = enqueue_provider_eval_job(
+            session,
+            target,
+            payload_json={"evaluation_id": target.id, "job_type": "provider_eval"},
+        )
+        other_job = enqueue_provider_eval_job(
+            session,
+            other,
+            payload_json={"evaluation_id": other.id, "job_type": "provider_eval"},
+        )
+        target_id = target.id
+        target_job_id = target_job.id
+        other_job_id = other_job.id
+        session.commit()
+
+    admin_delete_evaluation(
+        _build_request(session_factory),
+        target_id,
+        user=MagicMock(),
+    )
+
+    with session_factory() as session:
+        assert session.get(JobQueue, target_job_id).status == "cancelled"
+        assert session.get(JobQueue, other_job_id).status == "queued"
+
+
+def test_delete_submission_backed_evaluation_is_rejected(validator_engine):
+    session_factory = sessionmaker(
+        bind=validator_engine,
+        autoflush=False,
+        autocommit=False,
+        expire_on_commit=False,
+        future=True,
+    )
+    now = datetime.now(timezone.utc)
+    with session_factory() as session:
+        submission = Submission(
+            id="sub-eval-1",
+            source="upload",
+            miner_id="miner",
+            benchmark_names_json=["math500"],
+            status="completed",
+            created_at=now,
+            updated_at=now,
+        )
+        run = _standalone_eval(status="completed")
+        run.submission_id = submission.id
+        session.add_all([submission, run])
+        session.flush()
+        evaluation_id = run.id
+        session.commit()
+
+    from fastapi import HTTPException
+
+    with pytest.raises(HTTPException) as excinfo:
+        admin_delete_evaluation(
+            _build_request(session_factory),
+            evaluation_id,
+            user=MagicMock(),
+        )
+    assert excinfo.value.status_code == 400