From a157d55ae4628f9e8bccd8ca770fb8a82a7c87f3 Mon Sep 17 00:00:00 2001 From: Yuto Uematsu <65487096+Yuto-24@users.noreply.github.com> Date: Thu, 7 May 2026 12:00:14 +0900 Subject: [PATCH 1/5] Enhance formatting and content in README.md Updated README with improved formatting and additional details on requirements and setup. - Add Requirements section - edit code example of Step 0 for easy use --- offline/README.md | 32 +++++++++++++++++++++++--------- 1 file changed, 23 insertions(+), 9 deletions(-) diff --git a/offline/README.md b/offline/README.md index 3aa9456..9fbcc51 100644 --- a/offline/README.md +++ b/offline/README.md @@ -25,6 +25,7 @@ Adding a new tool requires forking the benchmark PRs and collecting the tool's r Each of the 50 benchmark PRs has a set of **golden comments**: real issues that a human reviewer identified, with severity labels (Low / Medium / High / Critical). These are the ground truth. For each tool, the pipeline: + 1. **Extracts** individual issues from the tool's review comments (line-specific comments become candidates directly; general comments are sent to an LLM to extract distinct issues) 2. **Deduplicates** candidates — tools that post the same issue in both a summary comment and as inline comments would otherwise be penalised for the duplicate. An LLM groups candidates that express the same underlying concern; sibling duplicates are not counted as false positives in step 3. 3. **Judges** each candidate against each golden comment using an LLM: "Do these describe the same underlying issue?" @@ -35,6 +36,7 @@ The judge accepts semantic matches — different wording is fine as long as the ### Judge models used Results are stored per judge model so you can compare how different judges score: + - `anthropic_claude-opus-4-5-20251101` - `anthropic_claude-sonnet-4-5-20250929` - `openai_gpt-5.2` @@ -45,21 +47,29 @@ Results are stored per judge model so you can compare how different judges score - **Golden comments are human-curated** but may miss edge cases or disagree with other reviewers. - **LLM judge introduces model-dependent variance** — different judge models may score differently. We mitigate this by using consistent prompts and reporting the judge model used. +## Requirements + +- Write access to GitHub org +- GitHub CLI +- uv + --- ## Setup 1. Install dependencies: -```bash -cd offline -uv sync -``` + + ```bash + cd offline + uv sync + ``` 2. Create `.env` file (see `.env.example`): -```bash -cp .env.example .env -# fill in your tokens -``` + + ```bash + cp .env.example .env + # fill in your tokens + ``` ## Tests @@ -86,7 +96,10 @@ All scripts live in the `code_review_benchmark/` package. Run from the `offline/ Fork benchmark PRs into a GitHub org where the tool under evaluation is installed: ```bash -uv run python -m code_review_benchmark.step0_fork_prs +uv run python -m code_review_benchmark.step0_fork_prs \ +--file golden_comments/${repository_name}.json \ +--org ${your_organization} \ +--name ${AI_tool_name} ``` ### 1. Download PR data @@ -202,6 +215,7 @@ uv run python -m code_review_benchmark.summary_table ``` **Example output:** + ``` Tool cal_dot_com discourse grafana keycloak sentry Total ---------------------------------------------------------------------------------- From 531b3b7e4b089145bae4c39c9e4af2a39d8dc505 Mon Sep 17 00:00:00 2001 From: Yuto-24 Date: Thu, 7 May 2026 12:15:23 +0900 Subject: [PATCH 2/5] load environment variables and update dependencies in pyproject.toml --- offline/code_review_benchmark/step0_fork_prs.py | 5 ++++- offline/pyproject.toml | 6 +++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/offline/code_review_benchmark/step0_fork_prs.py b/offline/code_review_benchmark/step0_fork_prs.py index 3a41feb..1410d33 100644 --- a/offline/code_review_benchmark/step0_fork_prs.py +++ b/offline/code_review_benchmark/step0_fork_prs.py @@ -28,6 +28,8 @@ import sys import tempfile import time +from dotenv import load_dotenv + import requests @@ -277,8 +279,9 @@ def main(): "--token", default=os.environ.get("GITHUB_TOKEN"), help="GitHub token" ) args = parser.parse_args() + load_dotenv(".env", verbose=True) - if not args.token: + if not os.environ.get("GITHUB_TOKEN") and not args.token: print("Error: Set GITHUB_TOKEN or use --token") sys.exit(1) diff --git a/offline/pyproject.toml b/offline/pyproject.toml index 2087167..ef1d9ce 100644 --- a/offline/pyproject.toml +++ b/offline/pyproject.toml @@ -9,10 +9,14 @@ dependencies = [ "matplotlib>=3.8.0", "requests>=2.32.5", "tqdm>=4.66.0", + "pytest>=9.0.3", + "ruff>=0.14.11", + "python-dotenv>=1.2.2", ] -[project.optional-dependencies] +[dependency-groups] dev = [ + "pytest>=8.0.0", "ruff>=0.6.0", ] From 48b43c7f895307a925f0a55d4d9ef4c793c8db51 Mon Sep 17 00:00:00 2001 From: Yuto-24 Date: Thu, 7 May 2026 12:22:24 +0900 Subject: [PATCH 3/5] Import block format --- offline/code_review_benchmark/step0_fork_prs.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/offline/code_review_benchmark/step0_fork_prs.py b/offline/code_review_benchmark/step0_fork_prs.py index 1410d33..9e96a64 100644 --- a/offline/code_review_benchmark/step0_fork_prs.py +++ b/offline/code_review_benchmark/step0_fork_prs.py @@ -28,9 +28,8 @@ import sys import tempfile import time -from dotenv import load_dotenv - +from dotenv import load_dotenv import requests From 1233470bd3485373b00de5ca2158144caea0b43d Mon Sep 17 00:00:00 2001 From: Yuto Uematsu <65487096+Yuto-24@users.noreply.github.com> Date: Thu, 7 May 2026 14:10:54 +0000 Subject: [PATCH 4/5] remove duplicate codes --- offline/code_review_benchmark/step0_fork_prs.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/offline/code_review_benchmark/step0_fork_prs.py b/offline/code_review_benchmark/step0_fork_prs.py index 9e96a64..3f14219 100644 --- a/offline/code_review_benchmark/step0_fork_prs.py +++ b/offline/code_review_benchmark/step0_fork_prs.py @@ -269,6 +269,8 @@ def _load_pr_urls_from_file(path: str) -> list[str]: def main(): """CLI entrypoint: process a single PR or a batch file.""" + load_dotenv(".env", verbose=True) + parser = argparse.ArgumentParser(description="Clone PR(s) to your org for AI review") parser.add_argument("pr_url", nargs="?", help="GitHub PR URL (for single run)") parser.add_argument("--file", help="Path to golden comments JSON to batch process") @@ -278,9 +280,8 @@ def main(): "--token", default=os.environ.get("GITHUB_TOKEN"), help="GitHub token" ) args = parser.parse_args() - load_dotenv(".env", verbose=True) - if not os.environ.get("GITHUB_TOKEN") and not args.token: + if not args.token: print("Error: Set GITHUB_TOKEN or use --token") sys.exit(1) From e60c204d396ff3a34f07ed702714a517ad484504 Mon Sep 17 00:00:00 2001 From: Yuto-24 Date: Fri, 8 May 2026 00:54:24 +0900 Subject: [PATCH 5/5] update code usage and modify output file --- offline/README.md | 58 +++++++++++++++++++++++------------------------ 1 file changed, 28 insertions(+), 30 deletions(-) diff --git a/offline/README.md b/offline/README.md index 9fbcc51..1afb0d2 100644 --- a/offline/README.md +++ b/offline/README.md @@ -4,19 +4,19 @@ Open replication of the code review benchmark used by companies like [Augment](h ## Evaluated tools -| Tool | Type | -|---|---| -| [Augment](https://www.augmentcode.com/) | AI code review | -| [Claude Code](https://claude.ai) | AI assistant | -| [CodeRabbit](https://www.coderabbit.ai/) | AI code review | -| [Codex](https://openai.com/codex) | AI assistant | -| [Cursor Bugbot](https://cursor.com) | AI code review | -| [Gemini](https://gemini.google.com/) | AI assistant | +| Tool | Type | +| ----------------------------------------------------- | -------------- | +| [Augment](https://www.augmentcode.com/) | AI code review | +| [Claude Code](https://claude.ai) | AI assistant | +| [CodeRabbit](https://www.coderabbit.ai/) | AI code review | +| [Codex](https://openai.com/codex) | AI assistant | +| [Cursor Bugbot](https://cursor.com) | AI code review | +| [Gemini](https://gemini.google.com/) | AI assistant | | [GitHub Copilot](https://github.com/features/copilot) | AI code review | -| [Graphite](https://graphite.dev/) | AI code review | -| [Greptile](https://www.greptile.com/) | AI code review | -| [Propel](https://propelauth.com/) | AI code review | -| [Qodo](https://www.qodo.ai/) | AI code review | +| [Graphite](https://graphite.dev/) | AI code review | +| [Greptile](https://www.greptile.com/) | AI code review | +| [Propel](https://propelauth.com/) | AI code review | +| [Qodo](https://www.qodo.ai/) | AI code review | Adding a new tool requires forking the benchmark PRs and collecting the tool's reviews — see Steps 0 and 1 below. @@ -49,7 +49,7 @@ Results are stored per judge model so you can compare how different judges score ## Requirements -- Write access to GitHub org +- Write access to GitHub org - GitHub CLI - uv @@ -59,17 +59,17 @@ Results are stored per judge model so you can compare how different judges score 1. Install dependencies: - ```bash - cd offline - uv sync - ``` + ```bash + cd offline + uv sync + ``` 2. Create `.env` file (see `.env.example`): - ```bash - cp .env.example .env - # fill in your tokens - ``` + ```bash + cp .env.example .env + # fill in your tokens + ``` ## Tests @@ -108,16 +108,16 @@ Aggregate PR reviews from benchmark repos with golden comments: ```bash # Full run (incremental - skips already downloaded) -uv run python -m code_review_benchmark.step1_download_prs --output results/benchmark_data.json +uv run python -m code_review_benchmark.step1_download_prs --output results/benchmark_data.json --org ${your_organization} # Test mode: 1 PR per tool -uv run python -m code_review_benchmark.step1_download_prs --output results/benchmark_data.json --test +uv run python -m code_review_benchmark.step1_download_prs --output results/benchmark_data.json --test --org ${your_organization} # Force refetch all reviews -uv run python -m code_review_benchmark.step1_download_prs --output results/benchmark_data.json --force +uv run python -m code_review_benchmark.step1_download_prs --output results/benchmark_data.json --force --org ${your_organization} # Force refetch for a specific tool -uv run python -m code_review_benchmark.step1_download_prs --output results/benchmark_data.json --force --tool copilot +uv run python -m code_review_benchmark.step1_download_prs --output results/benchmark_data.json --force --tool copilot --org ${your_organization} ``` **Output:** `results/benchmark_data.json` @@ -139,7 +139,7 @@ uv run python -m code_review_benchmark.step2_extract_comments --tool claude --li Line-specific comments become direct candidates. General comments are sent to the LLM to extract individual issues. -**Output:** Updates `results/benchmark_data.json` with `candidates` field per review. +**Output:** `results/{model}/candidates.json` ### 2.5. Deduplicate candidates (recommended) @@ -269,15 +269,13 @@ Source files: `sentry.json`, `grafana.json`, `keycloak.json`, `discourse.json`, "pr_title": "...", "original_url": "...", "source_repo": "sentry", - "golden_comments": [ - {"comment": "...", "severity": "High"} - ], + "golden_comments": [{ "comment": "...", "severity": "High" }], "reviews": [ { "tool": "claude", "pr_url": "https://github.com/code-review-benchmark/...", "review_comments": [ - {"path": "...", "line": 42, "body": "...", "created_at": "..."} + { "path": "...", "line": 42, "body": "...", "created_at": "..." } ], "candidates": ["issue description 1", "issue description 2"] }