diff --git a/validator/src/eval_backend/api/routes.py b/validator/src/eval_backend/api/routes.py index 8b1b5fc..12570a2 100644 --- a/validator/src/eval_backend/api/routes.py +++ b/validator/src/eval_backend/api/routes.py @@ -60,6 +60,7 @@ from ..services.github import create_pr_submission from ..services.github import set_commit_status from ..services.artifacts import persist_stored_artifact +from ..services.queue import cancel_evaluation_jobs from ..services.queue import cancel_submission_jobs from ..services.queue import enqueue_provider_eval_job from ..services.queue import enqueue_submission_job @@ -1211,6 +1212,7 @@ def admin_delete_evaluation( detail="only standalone provider evaluations can be deleted here", ) run.deleted_at = _utcnow() + cancel_evaluation_jobs(session, evaluation_id, reason="evaluation deleted") session.commit() return _evaluation_to_schema(run) except HTTPException: diff --git a/validator/src/eval_backend/services/queue.py b/validator/src/eval_backend/services/queue.py index 16d795d..4f51a4e 100644 --- a/validator/src/eval_backend/services/queue.py +++ b/validator/src/eval_backend/services/queue.py @@ -99,6 +99,32 @@ def cancel_submission_jobs( return jobs +def cancel_evaluation_jobs( + session: Session, + evaluation_id: int, + *, + reason: str = "evaluation deleted", +) -> list[JobQueue]: + jobs = session.execute( + select(JobQueue).where( + JobQueue.job_type == "provider_eval", + JobQueue.job_id == str(evaluation_id), + JobQueue.status == "queued", + ) + ).scalars().all() + now = _utcnow() + for job in jobs: + job.status = "cancelled" + job.claimed_by = None + job.claimed_at = None + job.heartbeat_at = None + job.last_error = reason + job.updated_at = now + if jobs: + session.flush() + return jobs + + def enqueue_train_job( session: Session, train: TrainRun, diff --git a/validator/tests/test_delete_evaluation.py b/validator/tests/test_delete_evaluation.py new file mode 100644 index 0000000..cc7594f --- /dev/null +++ b/validator/tests/test_delete_evaluation.py @@ -0,0 +1,150 @@ +from __future__ import annotations + +from datetime import datetime, timezone +from unittest.mock import MagicMock + +import pytest +from sqlalchemy.orm import sessionmaker + +from eval_backend.api.routes import admin_delete_evaluation +from eval_backend.models import EvaluationRun, JobQueue, Submission +from eval_backend.services.queue import enqueue_provider_eval_job + + +def _build_request(session_factory) -> MagicMock: + request = MagicMock() + request.app.state.session_factory = session_factory + return request + + +def _standalone_eval(status: str = "queued") -> EvaluationRun: + now = datetime.now(timezone.utc) + return EvaluationRun( + submission_id=None, + benchmark_names_json=["math500"], + provider="compatible", + models_config="configs/models.openrouter-chutes.yaml", + execution_mode="local_cpu", + device="cpu", + dtype="float32", + batch_size=1, + max_items=1, + status=status, + phase=status, + created_at=now, + ) + + +def test_delete_standalone_evaluation_cancels_its_queued_job(validator_engine): + session_factory = sessionmaker( + bind=validator_engine, + autoflush=False, + autocommit=False, + expire_on_commit=False, + future=True, + ) + with session_factory() as session: + run = _standalone_eval(status="queued") + session.add(run) + session.flush() + job = enqueue_provider_eval_job( + session, + run, + payload_json={"evaluation_id": run.id, "job_type": "provider_eval"}, + ) + evaluation_id = run.id + job_id = job.id + session.commit() + + response = admin_delete_evaluation( + _build_request(session_factory), + evaluation_id, + user=MagicMock(), + ) + + assert response.deleted_at is not None + + with session_factory() as session: + run = session.get(EvaluationRun, evaluation_id) + job = session.get(JobQueue, job_id) + assert run.deleted_at is not None + # A soft-deleted standalone eval must not stay in the queue: otherwise the + # worker would still claim the job, run the provider route, and incur cost. + assert job.status == "cancelled" + assert job.last_error == "evaluation deleted" + + +def test_delete_evaluation_leaves_other_evaluations_jobs_untouched(validator_engine): + session_factory = sessionmaker( + bind=validator_engine, + autoflush=False, + autocommit=False, + expire_on_commit=False, + future=True, + ) + with session_factory() as session: + target = _standalone_eval(status="queued") + other = _standalone_eval(status="queued") + session.add_all([target, other]) + session.flush() + target_job = enqueue_provider_eval_job( + session, + target, + payload_json={"evaluation_id": target.id, "job_type": "provider_eval"}, + ) + other_job = enqueue_provider_eval_job( + session, + other, + payload_json={"evaluation_id": other.id, "job_type": "provider_eval"}, + ) + target_id = target.id + target_job_id = target_job.id + other_job_id = other_job.id + session.commit() + + admin_delete_evaluation( + _build_request(session_factory), + target_id, + user=MagicMock(), + ) + + with session_factory() as session: + assert session.get(JobQueue, target_job_id).status == "cancelled" + assert session.get(JobQueue, other_job_id).status == "queued" + + +def test_delete_submission_backed_evaluation_is_rejected(validator_engine): + session_factory = sessionmaker( + bind=validator_engine, + autoflush=False, + autocommit=False, + expire_on_commit=False, + future=True, + ) + now = datetime.now(timezone.utc) + with session_factory() as session: + submission = Submission( + id="sub-eval-1", + source="upload", + miner_id="miner", + benchmark_names_json=["math500"], + status="completed", + created_at=now, + updated_at=now, + ) + run = _standalone_eval(status="completed") + run.submission_id = submission.id + session.add_all([submission, run]) + session.flush() + evaluation_id = run.id + session.commit() + + from fastapi import HTTPException + + with pytest.raises(HTTPException) as excinfo: + admin_delete_evaluation( + _build_request(session_factory), + evaluation_id, + user=MagicMock(), + ) + assert excinfo.value.status_code == 400