diff --git a/validator/src/eval_backend/services/github.py b/validator/src/eval_backend/services/github.py index d97027d..d68de84 100644 --- a/validator/src/eval_backend/services/github.py +++ b/validator/src/eval_backend/services/github.py @@ -227,11 +227,14 @@ def build_submission_summary_markdown( return "\n".join(parts) -def should_promote_submission(score: float | None, threshold: float, king_score: float | None) -> bool: +def should_promote_submission(score: float | None, king_score: float | None, *, is_first: bool = False) -> bool: if score is None: return False - current_king = threshold if king_score is None else king_score - return score >= threshold and score > current_king + if is_first: + return True + if king_score is None: + return True + return score > king_score async def _github_request( @@ -370,7 +373,6 @@ async def publish_submission_result( accepted = should_promote_submission( run.score, settings.github_review_score_threshold, - settings.github_review_score_threshold, ) commit_state = "pending" commit_description = "Evaluation queued" diff --git a/validator/src/eval_backend/worker.py b/validator/src/eval_backend/worker.py index 3497ed3..e1814d4 100644 --- a/validator/src/eval_backend/worker.py +++ b/validator/src/eval_backend/worker.py @@ -239,10 +239,11 @@ def process_once(session_factory, settings: Settings) -> int: job.updated_at = result.run.finished_at or now session.commit() if submission.source == "github_pr": + is_first = runtime.king_score == runtime_settings.github_review_score_threshold accepted = should_promote_submission( result.run.score, - runtime_settings.github_review_score_threshold, runtime.king_score, + is_first=is_first, ) try: import asyncio diff --git a/validator/tests/test_github_submission.py b/validator/tests/test_github_submission.py index 256ac30..9c8fc30 100644 --- a/validator/tests/test_github_submission.py +++ b/validator/tests/test_github_submission.py @@ -117,10 +117,13 @@ def test_github_webhook_enqueues_submission_job(validator_engine) -> None: def test_should_promote_submission_requires_threshold_and_king_score() -> None: - assert should_promote_submission(0.81, 0.8, 0.8) is True - assert should_promote_submission(0.8, 0.8, 0.8) is False - assert should_promote_submission(0.95, 0.8, 0.96) is False - assert should_promote_submission(None, 0.8, 0.8) is False + assert should_promote_submission(0.81, 0.8) is True + assert should_promote_submission(0.8, 0.8) is False + assert should_promote_submission(0.95, 0.96) is False + assert should_promote_submission(None, 0.8) is False + assert should_promote_submission(0.5, 0.8, is_first=True) is True + assert should_promote_submission(0.0, 0.8, is_first=True) is True + assert should_promote_submission(None, 0.8, is_first=True) is False def test_github_webhook_ignores_non_submission_pr(validator_engine) -> None: settings = Settings(