Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 6 additions & 4 deletions validator/src/eval_backend/services/github.py
Original file line number Diff line number Diff line change
Expand Up @@ -227,11 +227,14 @@ def build_submission_summary_markdown(
return "\n".join(parts)


def should_promote_submission(score: float | None, threshold: float, king_score: float | None) -> bool:
def should_promote_submission(score: float | None, king_score: float | None, *, is_first: bool = False) -> bool:
if score is None:
return False
current_king = threshold if king_score is None else king_score
return score >= threshold and score > current_king
if is_first:
return True
if king_score is None:
return True
return score > king_score


async def _github_request(
Expand Down Expand Up @@ -370,7 +373,6 @@ async def publish_submission_result(
accepted = should_promote_submission(
run.score,
settings.github_review_score_threshold,
settings.github_review_score_threshold,
)
commit_state = "pending"
commit_description = "Evaluation queued"
Expand Down
3 changes: 2 additions & 1 deletion validator/src/eval_backend/worker.py
Original file line number Diff line number Diff line change
Expand Up @@ -239,10 +239,11 @@ def process_once(session_factory, settings: Settings) -> int:
job.updated_at = result.run.finished_at or now
session.commit()
if submission.source == "github_pr":
is_first = runtime.king_score == runtime_settings.github_review_score_threshold
accepted = should_promote_submission(
result.run.score,
runtime_settings.github_review_score_threshold,
runtime.king_score,
is_first=is_first,
)
try:
import asyncio
Expand Down
11 changes: 7 additions & 4 deletions validator/tests/test_github_submission.py
Original file line number Diff line number Diff line change
Expand Up @@ -117,10 +117,13 @@ def test_github_webhook_enqueues_submission_job(validator_engine) -> None:


def test_should_promote_submission_requires_threshold_and_king_score() -> None:
assert should_promote_submission(0.81, 0.8, 0.8) is True
assert should_promote_submission(0.8, 0.8, 0.8) is False
assert should_promote_submission(0.95, 0.8, 0.96) is False
assert should_promote_submission(None, 0.8, 0.8) is False
assert should_promote_submission(0.81, 0.8) is True
assert should_promote_submission(0.8, 0.8) is False
assert should_promote_submission(0.95, 0.96) is False
assert should_promote_submission(None, 0.8) is False
assert should_promote_submission(0.5, 0.8, is_first=True) is True
assert should_promote_submission(0.0, 0.8, is_first=True) is True
assert should_promote_submission(None, 0.8, is_first=True) is False

def test_github_webhook_ignores_non_submission_pr(validator_engine) -> None:
settings = Settings(
Expand Down
Loading