Update PR review workflows to use extensions plugin

.github/workflows/pr-review-by-openhands.yml

@@ -39,7 +39,7 @@ jobs:
         runs-on: ubuntu-24.04
         steps:
             - name: Run PR Review
-              uses: OpenHands/software-agent-sdk/.github/actions/pr-review@main
+              uses: OpenHands/extensions/plugins/pr-review@main
               with:
                   llm-model: litellm_proxy/claude-sonnet-4-5-20250929
                   llm-base-url: https://llm-proxy.app.all-hands.dev

.github/workflows/pr-review-evaluation.yml

@@ -0,0 +1,85 @@
+---
+name: PR Review Evaluation
+
+# This workflow evaluates how well PR review comments were addressed.
+# It runs when a PR is closed to assess review effectiveness.
+#
+# Security note: pull_request_target is safe here because:
+# 1. Only triggers on PR close (not on code changes)
+# 2. Does not checkout PR code - only downloads artifacts from trusted workflow runs
+# 3. Runs evaluation scripts from the extensions repo, not from the PR
+
+on:
+    pull_request_target:
+        types: [closed]
+
+permissions:
+    contents: read
+    pull-requests: read
+
+jobs:
+    evaluate:
+        runs-on: ubuntu-24.04
+        env:
+            PR_NUMBER: ${{ github.event.pull_request.number }}
+            REPO_NAME: ${{ github.repository }}
+            PR_MERGED: ${{ github.event.pull_request.merged }}
+
+        steps:
+            - name: Download review trace artifact
+              id: download-trace
+              uses: dawidd6/action-download-artifact@v6
+              continue-on-error: true
+              with:
+                  workflow: pr-review-by-openhands.yml
+                  name: pr-review-trace-${{ github.event.pull_request.number }}
+                  path: trace-info
+                  search_artifacts: true
+                  if_no_artifact_found: warn
+
+            - name: Check if trace file exists
+              id: check-trace
+              run: |
+                  if [ -f "trace-info/laminar_trace_info.json" ]; then
+                    echo "trace_exists=true" >> $GITHUB_OUTPUT
+                    echo "Found trace file for PR #$PR_NUMBER"
+                  else
+                    echo "trace_exists=false" >> $GITHUB_OUTPUT
+                    echo "No trace file found for PR #$PR_NUMBER - skipping evaluation"
+                  fi
+
+            # Always checkout main branch for security - cannot test script changes in PRs
+            - name: Checkout extensions repository
+              if: steps.check-trace.outputs.trace_exists == 'true'
+              uses: actions/checkout@v5
+              with:
+                  repository: OpenHands/extensions
+                  path: extensions
+
+            - name: Set up Python
+              if: steps.check-trace.outputs.trace_exists == 'true'
+              uses: actions/setup-python@v6
+              with:
+                  python-version: '3.12'
+
+            - name: Install dependencies
+              if: steps.check-trace.outputs.trace_exists == 'true'
+              run: pip install lmnr
+
+            - name: Run evaluation
+              if: steps.check-trace.outputs.trace_exists == 'true'
+              env:
+                  # Script expects LMNR_PROJECT_API_KEY; org secret is named LMNR_SKILLS_API_KEY
+                  LMNR_PROJECT_API_KEY: ${{ secrets.LMNR_SKILLS_API_KEY }}
+                  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+              run: |
+                  python extensions/plugins/pr-review/scripts/evaluate_review.py \
+                      --trace-file trace-info/laminar_trace_info.json
+
+            - name: Upload evaluation logs
+              uses: actions/upload-artifact@v5
+              if: always() && steps.check-trace.outputs.trace_exists == 'true'
+              with:
+                  name: pr-review-evaluation-${{ github.event.pull_request.number }}
+                  path: '*.log'
+                  retention-days: 30

.github/workflows/tests.yml

@@ -0,0 +1,25 @@
+name: Tests
+
+on:
+  pull_request:
+    branches: ["*"]
+  push:
+    branches: ["main", "master"]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v7
+        with:
+          enable-cache: true
+          python-version: "3.12"
+
+      - name: Run tests
+        run: uv run --group test pytest tests/
+        env:
+          PYTHONPATH: ${{ github.workspace }}

plugins/pr-review/README.md

@@ -2,6 +2,20 @@
 
 Automated pull request review using OpenHands agents. This plugin provides GitHub workflows that automatically review PRs with detailed, inline code review comments.
 
+## Quick Start
+
+Copy both workflow files to your repository:
+
+```bash
+mkdir -p .github/workflows
+curl -o .github/workflows/pr-review-by-openhands.yml \
+  https://raw.githubusercontent.com/OpenHands/extensions/main/.github/workflows/pr-review-by-openhands.yml
+curl -o .github/workflows/pr-review-evaluation.yml \
+  https://raw.githubusercontent.com/OpenHands/extensions/main/.github/workflows/pr-review-evaluation.yml
+```
+
+Then configure the required secrets (see [Installation](#installation) below).
+
 ## Features
 
 - **Automated PR Reviews**: Triggered when PRs are opened, marked ready, or when a reviewer is requested
@@ -33,16 +47,16 @@ plugins/pr-review/
 
 ## Installation
 
-### 1. Copy the Workflow File
+### 1. Copy the Workflow Files
 
-Copy the workflow file to your repository's `.github/workflows/` directory:
+Copy the workflow files to your repository's `.github/workflows/` directory:
 
 ```bash
-# Option A: Download from GitHub
+mkdir -p .github/workflows
 curl -o .github/workflows/pr-review-by-openhands.yml \
-  https://raw.githubusercontent.com/OpenHands/extensions/main/plugins/pr-review/workflows/pr-review-by-openhands.yml
-
-# Option B: Create manually (see workflow content below)
+  https://raw.githubusercontent.com/OpenHands/extensions/main/.github/workflows/pr-review-by-openhands.yml
+curl -o .github/workflows/pr-review-evaluation.yml \
+  https://raw.githubusercontent.com/OpenHands/extensions/main/.github/workflows/pr-review-evaluation.yml
 ```
 
 ### 2. Configure Secrets
@@ -53,7 +67,7 @@ Add the following secrets in your repository settings (**Settings → Secrets an
 |--------|----------|-------------|
 | `LLM_API_KEY` | Yes | API key for your LLM provider |
 | `GITHUB_TOKEN` | Auto | Provided automatically by GitHub Actions |
-| `LMNR_PROJECT_API_KEY` | No | Laminar API key for observability |
+| `LMNR_SKILLS_API_KEY` | No | Laminar API key (org-level secret; mapped to `LMNR_PROJECT_API_KEY` env var in workflows) |
 
 **Note**: For repositories that need to post review comments from a bot account, use `ALLHANDS_BOT_GITHUB_PAT` instead of `GITHUB_TOKEN`.
 
@@ -157,14 +171,7 @@ One model is randomly selected for each review. When Laminar observability is en
 
 ### Review Evaluation
 
-To evaluate how well reviews were addressed, add the evaluation workflow:
-
-```bash
-curl -o .github/workflows/pr-review-evaluation.yml \
-  https://raw.githubusercontent.com/OpenHands/extensions/main/plugins/pr-review/workflows/pr-review-evaluation.yml
-```
-
-This workflow runs when PRs are closed and:
+The evaluation workflow (`pr-review-evaluation.yml`) runs when PRs are closed and:
 1. Downloads the review trace artifact
 2. Fetches final PR state and comments
 3. Creates an evaluation span in Laminar

plugins/pr-review/action.yml

@@ -77,11 +77,6 @@ runs:
               sudo apt-get update
               sudo apt-get install -y gh
 
-        - name: Install OpenHands dependencies
-          shell: bash
-          run: |
-              uv pip install --system openhands-sdk openhands-tools lmnr
-
         - name: Check required configuration and select model
           id: select-model
           shell: bash
@@ -132,7 +127,8 @@ runs:
               REPO_NAME: ${{ github.repository }}
           run: |
               cd pr-repo
-              uv run python ../extensions/plugins/pr-review/scripts/agent_script.py
+              uv run --with openhands-sdk --with openhands-tools --with lmnr \
+                python ../extensions/plugins/pr-review/scripts/agent_script.py
 
         - name: Upload logs as artifact
           uses: actions/upload-artifact@v4

plugins/pr-review/scripts/evaluate_review.py

@@ -243,14 +243,17 @@ def truncate_text(text: str, max_chars: int = 50000) -> str:
     return text[:max_chars] + f"\n\n... [truncated, {len(text)} total chars]"
 
 
-def load_trace_info() -> dict:
+def load_trace_info(trace_file_path: str | None = None) -> dict:
     """Load trace info from artifact file.
 
+    Args:
+        trace_file_path: Path to trace info JSON file. If None, uses default path.
+
     Returns:
         Dictionary with trace_id, span_context, and other metadata.
         Empty dict if file not found.
     """
-    trace_info_path = Path("laminar_trace_info.json")
+    trace_info_path = Path(trace_file_path) if trace_file_path else Path("laminar_trace_info.json")
 
     if not trace_info_path.exists():
         logger.warning(
@@ -412,8 +415,12 @@ def create_evaluation_span(
     return str(eval_trace_id) if eval_trace_id else None
 
 
-def main():
-    """Run the PR review evaluation."""
+def main(trace_file_path: str | None = None):
+    """Run the PR review evaluation.
+
+    Args:
+        trace_file_path: Optional path to trace info JSON file.
+    """
     logger.info("Starting PR review evaluation...")
 
     pr_number = _get_required_env("PR_NUMBER")
@@ -423,7 +430,7 @@ def main():
     logger.info(f"Evaluating PR #{pr_number} in {repo_name}")
     logger.info(f"PR was merged: {pr_merged}")
 
-    trace_info = load_trace_info()
+    trace_info = load_trace_info(trace_file_path)
     pr_data = fetch_pr_data(repo_name, pr_number)
     eval_trace_id = create_evaluation_span(
         pr_number, repo_name, pr_merged, pr_data, trace_info
@@ -478,8 +485,17 @@ def main():
 
 
 if __name__ == "__main__":
+    import argparse
+
+    parser = argparse.ArgumentParser(description="Evaluate PR review effectiveness")
+    parser.add_argument(
+        "--trace-file",
+        help="Path to trace info JSON file (default: laminar_trace_info.json)",
+    )
+    args = parser.parse_args()
+
     try:
-        main()
+        main(trace_file_path=args.trace_file)
     except Exception as e:
         logger.error(f"Evaluation failed: {e}")
         sys.exit(1)

plugins/pr-review/workflows/pr-review-by-openhands.yml

@@ -8,10 +8,9 @@ on:
     #   2. A draft PR is marked as ready for review, OR
     #   3. A maintainer adds the 'review-this' label, OR
     #   4. A maintainer requests openhands-agent or all-hands-bot as a reviewer
-    # Adding labels and requesting new reviewers requires write access. GitHub may also allow PR authors
-    # to re-request review from a previous reviewer.
+    # Adding labels and requesting reviewers requires write access.
     # The PR code is explicitly checked out for review, but secrets are only accessible
-    # because the workflow runs in the base repository context
+    # because the workflow runs in the base repository context.
     pull_request_target:
         types: [opened, ready_for_review, labeled, review_requested]
 
@@ -27,7 +26,7 @@ jobs:
         #   2. A draft PR is converted to ready for review by a non-first-time contributor, OR
         #   3. 'review-this' label is added, OR
         #   4. openhands-agent or all-hands-bot is requested as a reviewer
-        # Note: FIRST_TIME_CONTRIBUTOR PRs require manual trigger via label/reviewer request
+        # Note: FIRST_TIME_CONTRIBUTOR and NONE PRs require manual trigger via label/reviewer request.
         if: |
             (github.event.action == 'opened' && github.event.pull_request.draft == false && github.event.pull_request.author_association != 'FIRST_TIME_CONTRIBUTOR' && github.event.pull_request.author_association != 'NONE') ||
             (github.event.action == 'ready_for_review' && github.event.pull_request.author_association != 'FIRST_TIME_CONTRIBUTOR' && github.event.pull_request.author_association != 'NONE') ||
@@ -42,13 +41,10 @@ jobs:
             - name: Run PR Review
               uses: OpenHands/extensions/plugins/pr-review@main
               with:
-                  # LLM model(s) to use. Can be comma-separated for A/B testing
-                  # - one model will be randomly selected per review
-                  llm-model: anthropic/claude-sonnet-4-5-20250929
-                  # llm-base-url: https://llm-proxy.app.all-hands.dev
+                  llm-model: litellm_proxy/claude-sonnet-4-5-20250929
+                  llm-base-url: https://llm-proxy.app.all-hands.dev
                   # Review style: roasted (other option: standard)
                   review-style: roasted
                   llm-api-key: ${{ secrets.LLM_API_KEY }}
-                  github-token: ${{ secrets.GITHUB_TOKEN }}
-                  # Optional: Laminar API key for observability
-                  # lmnr-api-key: ${{ secrets.LMNR_PROJECT_API_KEY }}
+                  github-token: ${{ secrets.ALLHANDS_BOT_GITHUB_PAT }}
+                  lmnr-api-key: ${{ secrets.LMNR_SKILLS_API_KEY }}