From 99bb0fcb8225cbc0bfdf1cbf36d8c477f246cbb4 Mon Sep 17 00:00:00 2001 From: openhands Date: Thu, 19 Feb 2026 18:30:32 +0000 Subject: [PATCH 01/17] Update PR review workflows to use extensions plugin Changes: 1. pr-review-by-openhands.yml: Use plugins/pr-review@main instead of software-agent-sdk/.github/actions/pr-review@main 2. pr-review-evaluation.yml: Add stub workflow that calls the reusable workflow for evaluating PR review effectiveness 3. pr-review-evaluation-reusable.yml: Add reusable workflow containing all evaluation logic, which other repos can call with a minimal stub This centralizes all PR review logic in the extensions repo for easier maintenance across the organization. Co-authored-by: openhands --- .github/workflows/pr-review-by-openhands.yml | 2 +- .../pr-review-evaluation-reusable.yml | 120 ++++++++++++++++++ .github/workflows/pr-review-evaluation.yml | 18 +++ 3 files changed, 139 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/pr-review-evaluation-reusable.yml create mode 100644 .github/workflows/pr-review-evaluation.yml diff --git a/.github/workflows/pr-review-by-openhands.yml b/.github/workflows/pr-review-by-openhands.yml index 2516c9b2..9cf05232 100644 --- a/.github/workflows/pr-review-by-openhands.yml +++ b/.github/workflows/pr-review-by-openhands.yml @@ -39,7 +39,7 @@ jobs: runs-on: ubuntu-24.04 steps: - name: Run PR Review - uses: OpenHands/software-agent-sdk/.github/actions/pr-review@main + uses: OpenHands/extensions/plugins/pr-review@main with: llm-model: litellm_proxy/claude-sonnet-4-5-20250929 llm-base-url: https://llm-proxy.app.all-hands.dev diff --git a/.github/workflows/pr-review-evaluation-reusable.yml b/.github/workflows/pr-review-evaluation-reusable.yml new file mode 100644 index 00000000..c4d03d1a --- /dev/null +++ b/.github/workflows/pr-review-evaluation-reusable.yml @@ -0,0 +1,120 @@ +--- +name: PR Review Evaluation (Reusable) + +# Reusable workflow for evaluating PR review effectiveness. +# This workflow is called by other repositories to assess how well +# review comments were addressed when a PR is closed. +# +# Usage in other repos: +# jobs: +# evaluate: +# uses: OpenHands/extensions/.github/workflows/pr-review-evaluation-reusable.yml@main +# secrets: inherit + +on: + workflow_call: + inputs: + pr_number: + description: 'PR number to evaluate' + required: true + type: number + repo_name: + description: 'Repository name (owner/repo format)' + required: true + type: string + pr_merged: + description: 'Whether the PR was merged' + required: true + type: boolean + secrets: + LMNR_SKILLS_API_KEY: + description: 'Laminar API key for observability' + required: false + GITHUB_TOKEN: + description: 'GitHub token for API access' + required: true + +jobs: + evaluate: + runs-on: ubuntu-24.04 + env: + PR_NUMBER: ${{ inputs.pr_number }} + REPO_NAME: ${{ inputs.repo_name }} + PR_MERGED: ${{ inputs.pr_merged }} + + steps: + # Note: actions/download-artifact@v5 only works within the same workflow run. + # We use dawidd6/action-download-artifact to download from a different workflow. + - name: Download review trace artifact + id: download-trace + uses: dawidd6/action-download-artifact@v6 + continue-on-error: true + with: + workflow: pr-review-by-openhands.yml + name: pr-review-trace-${{ inputs.pr_number }} + path: trace-info + search_artifacts: true + if_no_artifact_found: warn + repo: ${{ inputs.repo_name }} + github_token: ${{ secrets.GITHUB_TOKEN }} + + # Check if the trace file actually exists (the artifact download may + # succeed but with no matching artifact, only issuing a warning) + - name: Check if trace file exists + id: check-trace + run: | + if [ -f "trace-info/laminar_trace_info.json" ]; then + echo "trace_exists=true" >> $GITHUB_OUTPUT + echo "Found trace file for PR #$PR_NUMBER" + else + echo "trace_exists=false" >> $GITHUB_OUTPUT + echo "No trace file found for PR #$PR_NUMBER" + echo "This PR may not have been reviewed by the agent, skipping evaluation" + fi + + - name: Checkout extensions repository + if: steps.check-trace.outputs.trace_exists == 'true' + uses: actions/checkout@v5 + with: + repository: OpenHands/extensions + path: extensions + + - name: Set up Python + if: steps.check-trace.outputs.trace_exists == 'true' + uses: actions/setup-python@v6 + with: + python-version: '3.13' + + - name: Install uv + if: steps.check-trace.outputs.trace_exists == 'true' + uses: astral-sh/setup-uv@v7 + with: + enable-cache: true + + - name: Install dependencies + if: steps.check-trace.outputs.trace_exists == 'true' + run: | + # Install lmnr SDK for Laminar integration + uv pip install --system lmnr + + - name: Run evaluation + if: steps.check-trace.outputs.trace_exists == 'true' + env: + LMNR_PROJECT_API_KEY: ${{ secrets.LMNR_SKILLS_API_KEY }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + # Copy trace info to working directory + cp trace-info/laminar_trace_info.json . + + # Run the evaluation script from the extensions plugin + uv run python extensions/plugins/pr-review/scripts/evaluate_review.py + + - name: Upload evaluation logs + uses: actions/upload-artifact@v5 + if: always() && steps.check-trace.outputs.trace_exists == 'true' + with: + name: pr-review-evaluation-${{ inputs.pr_number }} + path: | + *.log + *.json + retention-days: 30 diff --git a/.github/workflows/pr-review-evaluation.yml b/.github/workflows/pr-review-evaluation.yml new file mode 100644 index 00000000..97a083f3 --- /dev/null +++ b/.github/workflows/pr-review-evaluation.yml @@ -0,0 +1,18 @@ +--- +name: PR Review Evaluation + +# This workflow evaluates how well PR review comments were addressed. +# It calls the reusable workflow from OpenHands/extensions. + +on: + pull_request_target: + types: [closed] + +jobs: + evaluate: + uses: OpenHands/extensions/.github/workflows/pr-review-evaluation-reusable.yml@main + with: + pr_number: ${{ github.event.pull_request.number }} + repo_name: ${{ github.repository }} + pr_merged: ${{ github.event.pull_request.merged }} + secrets: inherit From 8aa4dce405b013c13b0ce8368b10d83a92ded492 Mon Sep 17 00:00:00 2001 From: openhands Date: Thu, 19 Feb 2026 18:40:20 +0000 Subject: [PATCH 02/17] Address review feedback - Add security documentation explaining why pull_request_target is safe here - Replace secrets: inherit with explicit secret passing - Use Python 3.12 instead of 3.13 for better compatibility - Use pip directly instead of uv for simplicity Co-authored-by: openhands --- .../pr-review-evaluation-reusable.yml | 22 +++++++++---------- .github/workflows/pr-review-evaluation.yml | 9 +++++++- 2 files changed, 19 insertions(+), 12 deletions(-) diff --git a/.github/workflows/pr-review-evaluation-reusable.yml b/.github/workflows/pr-review-evaluation-reusable.yml index c4d03d1a..c48cc0e5 100644 --- a/.github/workflows/pr-review-evaluation-reusable.yml +++ b/.github/workflows/pr-review-evaluation-reusable.yml @@ -5,11 +5,18 @@ name: PR Review Evaluation (Reusable) # This workflow is called by other repositories to assess how well # review comments were addressed when a PR is closed. # +# Security note: This workflow is safe with pull_request_target because: +# 1. Only triggers on PR close (not on code changes) +# 2. Does not checkout PR code - only downloads artifacts from trusted workflow runs +# 3. Runs evaluation scripts from the extensions repo, not from the PR +# # Usage in other repos: # jobs: # evaluate: # uses: OpenHands/extensions/.github/workflows/pr-review-evaluation-reusable.yml@main -# secrets: inherit +# secrets: +# LMNR_SKILLS_API_KEY: ${{ secrets.LMNR_SKILLS_API_KEY }} +# GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} on: workflow_call: @@ -83,19 +90,12 @@ jobs: if: steps.check-trace.outputs.trace_exists == 'true' uses: actions/setup-python@v6 with: - python-version: '3.13' - - - name: Install uv - if: steps.check-trace.outputs.trace_exists == 'true' - uses: astral-sh/setup-uv@v7 - with: - enable-cache: true + python-version: '3.12' - name: Install dependencies if: steps.check-trace.outputs.trace_exists == 'true' run: | - # Install lmnr SDK for Laminar integration - uv pip install --system lmnr + pip install lmnr - name: Run evaluation if: steps.check-trace.outputs.trace_exists == 'true' @@ -107,7 +107,7 @@ jobs: cp trace-info/laminar_trace_info.json . # Run the evaluation script from the extensions plugin - uv run python extensions/plugins/pr-review/scripts/evaluate_review.py + python extensions/plugins/pr-review/scripts/evaluate_review.py - name: Upload evaluation logs uses: actions/upload-artifact@v5 diff --git a/.github/workflows/pr-review-evaluation.yml b/.github/workflows/pr-review-evaluation.yml index 97a083f3..bef40062 100644 --- a/.github/workflows/pr-review-evaluation.yml +++ b/.github/workflows/pr-review-evaluation.yml @@ -3,6 +3,11 @@ name: PR Review Evaluation # This workflow evaluates how well PR review comments were addressed. # It calls the reusable workflow from OpenHands/extensions. +# +# Security note: pull_request_target is safe here because: +# 1. Only triggers on PR close (not on code changes) +# 2. Does not checkout PR code - only downloads artifacts from trusted workflow runs +# 3. Runs evaluation scripts from the extensions repo, not from the PR on: pull_request_target: @@ -15,4 +20,6 @@ jobs: pr_number: ${{ github.event.pull_request.number }} repo_name: ${{ github.repository }} pr_merged: ${{ github.event.pull_request.merged }} - secrets: inherit + secrets: + LMNR_SKILLS_API_KEY: ${{ secrets.LMNR_SKILLS_API_KEY }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} From d487ec2544a51dc1bacba2ada789f1a4fc69cb2a Mon Sep 17 00:00:00 2001 From: openhands Date: Thu, 19 Feb 2026 18:42:14 +0000 Subject: [PATCH 03/17] Remove reusable workflow, use full workflow directly Simplify by having the complete evaluation workflow in each repo instead of using a reusable workflow pattern. The workflow still references extensions/plugins/pr-review/scripts/evaluate_review.py for the actual evaluation logic. Co-authored-by: openhands --- .../pr-review-evaluation-reusable.yml | 120 ------------------ .github/workflows/pr-review-evaluation.yml | 78 ++++++++++-- 2 files changed, 69 insertions(+), 129 deletions(-) delete mode 100644 .github/workflows/pr-review-evaluation-reusable.yml diff --git a/.github/workflows/pr-review-evaluation-reusable.yml b/.github/workflows/pr-review-evaluation-reusable.yml deleted file mode 100644 index c48cc0e5..00000000 --- a/.github/workflows/pr-review-evaluation-reusable.yml +++ /dev/null @@ -1,120 +0,0 @@ ---- -name: PR Review Evaluation (Reusable) - -# Reusable workflow for evaluating PR review effectiveness. -# This workflow is called by other repositories to assess how well -# review comments were addressed when a PR is closed. -# -# Security note: This workflow is safe with pull_request_target because: -# 1. Only triggers on PR close (not on code changes) -# 2. Does not checkout PR code - only downloads artifacts from trusted workflow runs -# 3. Runs evaluation scripts from the extensions repo, not from the PR -# -# Usage in other repos: -# jobs: -# evaluate: -# uses: OpenHands/extensions/.github/workflows/pr-review-evaluation-reusable.yml@main -# secrets: -# LMNR_SKILLS_API_KEY: ${{ secrets.LMNR_SKILLS_API_KEY }} -# GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - -on: - workflow_call: - inputs: - pr_number: - description: 'PR number to evaluate' - required: true - type: number - repo_name: - description: 'Repository name (owner/repo format)' - required: true - type: string - pr_merged: - description: 'Whether the PR was merged' - required: true - type: boolean - secrets: - LMNR_SKILLS_API_KEY: - description: 'Laminar API key for observability' - required: false - GITHUB_TOKEN: - description: 'GitHub token for API access' - required: true - -jobs: - evaluate: - runs-on: ubuntu-24.04 - env: - PR_NUMBER: ${{ inputs.pr_number }} - REPO_NAME: ${{ inputs.repo_name }} - PR_MERGED: ${{ inputs.pr_merged }} - - steps: - # Note: actions/download-artifact@v5 only works within the same workflow run. - # We use dawidd6/action-download-artifact to download from a different workflow. - - name: Download review trace artifact - id: download-trace - uses: dawidd6/action-download-artifact@v6 - continue-on-error: true - with: - workflow: pr-review-by-openhands.yml - name: pr-review-trace-${{ inputs.pr_number }} - path: trace-info - search_artifacts: true - if_no_artifact_found: warn - repo: ${{ inputs.repo_name }} - github_token: ${{ secrets.GITHUB_TOKEN }} - - # Check if the trace file actually exists (the artifact download may - # succeed but with no matching artifact, only issuing a warning) - - name: Check if trace file exists - id: check-trace - run: | - if [ -f "trace-info/laminar_trace_info.json" ]; then - echo "trace_exists=true" >> $GITHUB_OUTPUT - echo "Found trace file for PR #$PR_NUMBER" - else - echo "trace_exists=false" >> $GITHUB_OUTPUT - echo "No trace file found for PR #$PR_NUMBER" - echo "This PR may not have been reviewed by the agent, skipping evaluation" - fi - - - name: Checkout extensions repository - if: steps.check-trace.outputs.trace_exists == 'true' - uses: actions/checkout@v5 - with: - repository: OpenHands/extensions - path: extensions - - - name: Set up Python - if: steps.check-trace.outputs.trace_exists == 'true' - uses: actions/setup-python@v6 - with: - python-version: '3.12' - - - name: Install dependencies - if: steps.check-trace.outputs.trace_exists == 'true' - run: | - pip install lmnr - - - name: Run evaluation - if: steps.check-trace.outputs.trace_exists == 'true' - env: - LMNR_PROJECT_API_KEY: ${{ secrets.LMNR_SKILLS_API_KEY }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - # Copy trace info to working directory - cp trace-info/laminar_trace_info.json . - - # Run the evaluation script from the extensions plugin - python extensions/plugins/pr-review/scripts/evaluate_review.py - - - name: Upload evaluation logs - uses: actions/upload-artifact@v5 - if: always() && steps.check-trace.outputs.trace_exists == 'true' - with: - name: pr-review-evaluation-${{ inputs.pr_number }} - path: | - *.log - *.json - retention-days: 30 diff --git a/.github/workflows/pr-review-evaluation.yml b/.github/workflows/pr-review-evaluation.yml index bef40062..26f7e604 100644 --- a/.github/workflows/pr-review-evaluation.yml +++ b/.github/workflows/pr-review-evaluation.yml @@ -2,7 +2,7 @@ name: PR Review Evaluation # This workflow evaluates how well PR review comments were addressed. -# It calls the reusable workflow from OpenHands/extensions. +# It runs when a PR is closed to assess review effectiveness. # # Security note: pull_request_target is safe here because: # 1. Only triggers on PR close (not on code changes) @@ -13,13 +13,73 @@ on: pull_request_target: types: [closed] +permissions: + contents: read + pull-requests: read + jobs: evaluate: - uses: OpenHands/extensions/.github/workflows/pr-review-evaluation-reusable.yml@main - with: - pr_number: ${{ github.event.pull_request.number }} - repo_name: ${{ github.repository }} - pr_merged: ${{ github.event.pull_request.merged }} - secrets: - LMNR_SKILLS_API_KEY: ${{ secrets.LMNR_SKILLS_API_KEY }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + runs-on: ubuntu-24.04 + env: + PR_NUMBER: ${{ github.event.pull_request.number }} + REPO_NAME: ${{ github.repository }} + PR_MERGED: ${{ github.event.pull_request.merged }} + + steps: + - name: Download review trace artifact + id: download-trace + uses: dawidd6/action-download-artifact@v6 + continue-on-error: true + with: + workflow: pr-review-by-openhands.yml + name: pr-review-trace-${{ github.event.pull_request.number }} + path: trace-info + search_artifacts: true + if_no_artifact_found: warn + + - name: Check if trace file exists + id: check-trace + run: | + if [ -f "trace-info/laminar_trace_info.json" ]; then + echo "trace_exists=true" >> $GITHUB_OUTPUT + echo "Found trace file for PR #$PR_NUMBER" + else + echo "trace_exists=false" >> $GITHUB_OUTPUT + echo "No trace file found for PR #$PR_NUMBER - skipping evaluation" + fi + + - name: Checkout extensions repository + if: steps.check-trace.outputs.trace_exists == 'true' + uses: actions/checkout@v5 + with: + repository: OpenHands/extensions + path: extensions + + - name: Set up Python + if: steps.check-trace.outputs.trace_exists == 'true' + uses: actions/setup-python@v6 + with: + python-version: '3.12' + + - name: Install dependencies + if: steps.check-trace.outputs.trace_exists == 'true' + run: pip install lmnr + + - name: Run evaluation + if: steps.check-trace.outputs.trace_exists == 'true' + env: + LMNR_PROJECT_API_KEY: ${{ secrets.LMNR_SKILLS_API_KEY }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + cp trace-info/laminar_trace_info.json . + python extensions/plugins/pr-review/scripts/evaluate_review.py + + - name: Upload evaluation logs + uses: actions/upload-artifact@v5 + if: always() && steps.check-trace.outputs.trace_exists == 'true' + with: + name: pr-review-evaluation-${{ github.event.pull_request.number }} + path: | + *.log + *.json + retention-days: 30 From b0af14f97437b3c6c9caa1726ecdd6762316092e Mon Sep 17 00:00:00 2001 From: openhands Date: Thu, 19 Feb 2026 19:20:09 +0000 Subject: [PATCH 04/17] Address review feedback: accept trace file path as argument - Add --trace-file argument to evaluate_review.py to accept path directly - Remove unnecessary file copy in workflow - Only upload *.log files (not *.json) to avoid redundant artifact uploads Co-authored-by: openhands --- .github/workflows/pr-review-evaluation.yml | 9 +++---- plugins/pr-review/scripts/evaluate_review.py | 28 +++++++++++++++----- 2 files changed, 26 insertions(+), 11 deletions(-) diff --git a/.github/workflows/pr-review-evaluation.yml b/.github/workflows/pr-review-evaluation.yml index 26f7e604..17efd288 100644 --- a/.github/workflows/pr-review-evaluation.yml +++ b/.github/workflows/pr-review-evaluation.yml @@ -71,15 +71,14 @@ jobs: LMNR_PROJECT_API_KEY: ${{ secrets.LMNR_SKILLS_API_KEY }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | - cp trace-info/laminar_trace_info.json . - python extensions/plugins/pr-review/scripts/evaluate_review.py + python extensions/plugins/pr-review/scripts/evaluate_review.py \ + --trace-file trace-info/laminar_trace_info.json - name: Upload evaluation logs uses: actions/upload-artifact@v5 if: always() && steps.check-trace.outputs.trace_exists == 'true' with: name: pr-review-evaluation-${{ github.event.pull_request.number }} - path: | - *.log - *.json + path: '*.log' retention-days: 30 + if-no-files-found: ignore diff --git a/plugins/pr-review/scripts/evaluate_review.py b/plugins/pr-review/scripts/evaluate_review.py index 896f5292..cfc59b4b 100644 --- a/plugins/pr-review/scripts/evaluate_review.py +++ b/plugins/pr-review/scripts/evaluate_review.py @@ -243,14 +243,17 @@ def truncate_text(text: str, max_chars: int = 50000) -> str: return text[:max_chars] + f"\n\n... [truncated, {len(text)} total chars]" -def load_trace_info() -> dict: +def load_trace_info(trace_file_path: str | None = None) -> dict: """Load trace info from artifact file. + Args: + trace_file_path: Path to trace info JSON file. If None, uses default path. + Returns: Dictionary with trace_id, span_context, and other metadata. Empty dict if file not found. """ - trace_info_path = Path("laminar_trace_info.json") + trace_info_path = Path(trace_file_path) if trace_file_path else Path("laminar_trace_info.json") if not trace_info_path.exists(): logger.warning( @@ -412,8 +415,12 @@ def create_evaluation_span( return str(eval_trace_id) if eval_trace_id else None -def main(): - """Run the PR review evaluation.""" +def main(trace_file_path: str | None = None): + """Run the PR review evaluation. + + Args: + trace_file_path: Optional path to trace info JSON file. + """ logger.info("Starting PR review evaluation...") pr_number = _get_required_env("PR_NUMBER") @@ -423,7 +430,7 @@ def main(): logger.info(f"Evaluating PR #{pr_number} in {repo_name}") logger.info(f"PR was merged: {pr_merged}") - trace_info = load_trace_info() + trace_info = load_trace_info(trace_file_path) pr_data = fetch_pr_data(repo_name, pr_number) eval_trace_id = create_evaluation_span( pr_number, repo_name, pr_merged, pr_data, trace_info @@ -478,8 +485,17 @@ def main(): if __name__ == "__main__": + import argparse + + parser = argparse.ArgumentParser(description="Evaluate PR review effectiveness") + parser.add_argument( + "--trace-file", + help="Path to trace info JSON file (default: laminar_trace_info.json)", + ) + args = parser.parse_args() + try: - main() + main(trace_file_path=args.trace_file) except Exception as e: logger.error(f"Evaluation failed: {e}") sys.exit(1) From 3874c31f0f2a002a2f01f20ddb5f49e38863280e Mon Sep 17 00:00:00 2001 From: openhands Date: Thu, 19 Feb 2026 19:30:46 +0000 Subject: [PATCH 05/17] Address review feedback: add comments and require logs - Document why checkout always uses main (security) - Document LMNR_PROJECT_API_KEY vs LMNR_SKILLS_API_KEY naming - Remove if-no-files-found: ignore (missing logs should fail) Co-authored-by: openhands --- .github/workflows/pr-review-evaluation.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pr-review-evaluation.yml b/.github/workflows/pr-review-evaluation.yml index 17efd288..60baa50c 100644 --- a/.github/workflows/pr-review-evaluation.yml +++ b/.github/workflows/pr-review-evaluation.yml @@ -48,6 +48,7 @@ jobs: echo "No trace file found for PR #$PR_NUMBER - skipping evaluation" fi + # Always checkout main branch for security - cannot test script changes in PRs - name: Checkout extensions repository if: steps.check-trace.outputs.trace_exists == 'true' uses: actions/checkout@v5 @@ -68,6 +69,7 @@ jobs: - name: Run evaluation if: steps.check-trace.outputs.trace_exists == 'true' env: + # Script expects LMNR_PROJECT_API_KEY; org secret is named LMNR_SKILLS_API_KEY LMNR_PROJECT_API_KEY: ${{ secrets.LMNR_SKILLS_API_KEY }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | @@ -81,4 +83,3 @@ jobs: name: pr-review-evaluation-${{ github.event.pull_request.number }} path: '*.log' retention-days: 30 - if-no-files-found: ignore From f27dd68fbfea0c9d48ccf9880fc3b2ff0dbfeb2d Mon Sep 17 00:00:00 2001 From: openhands Date: Thu, 19 Feb 2026 19:34:32 +0000 Subject: [PATCH 06/17] Use symlinks for workflow templates in plugins directory The actual workflows live in .github/workflows/ (required by GitHub Actions). The plugins/pr-review/workflows/ directory now contains symlinks for easy reference and documentation purposes. Also updated README with Quick Start section. Co-authored-by: openhands --- plugins/pr-review/README.md | 16 ++- .../workflows/pr-review-by-openhands.yml | 55 +-------- .../workflows/pr-review-evaluation.yml | 107 +----------------- 3 files changed, 17 insertions(+), 161 deletions(-) mode change 100644 => 120000 plugins/pr-review/workflows/pr-review-by-openhands.yml mode change 100644 => 120000 plugins/pr-review/workflows/pr-review-evaluation.yml diff --git a/plugins/pr-review/README.md b/plugins/pr-review/README.md index 2520204c..be746342 100644 --- a/plugins/pr-review/README.md +++ b/plugins/pr-review/README.md @@ -2,6 +2,20 @@ Automated pull request review using OpenHands agents. This plugin provides GitHub workflows that automatically review PRs with detailed, inline code review comments. +## Quick Start + +Copy both workflow files to your repository: + +```bash +mkdir -p .github/workflows +curl -o .github/workflows/pr-review-by-openhands.yml \ + https://raw.githubusercontent.com/OpenHands/extensions/main/plugins/pr-review/workflows/pr-review-by-openhands.yml +curl -o .github/workflows/pr-review-evaluation.yml \ + https://raw.githubusercontent.com/OpenHands/extensions/main/plugins/pr-review/workflows/pr-review-evaluation.yml +``` + +Then configure the required secrets (see [Installation](#installation) below). + ## Features - **Automated PR Reviews**: Triggered when PRs are opened, marked ready, or when a reviewer is requested @@ -53,7 +67,7 @@ Add the following secrets in your repository settings (**Settings → Secrets an |--------|----------|-------------| | `LLM_API_KEY` | Yes | API key for your LLM provider | | `GITHUB_TOKEN` | Auto | Provided automatically by GitHub Actions | -| `LMNR_PROJECT_API_KEY` | No | Laminar API key for observability | +| `LMNR_SKILLS_API_KEY` | No | Laminar API key for observability (org-level secret for OpenHands repos) | **Note**: For repositories that need to post review comments from a bot account, use `ALLHANDS_BOT_GITHUB_PAT` instead of `GITHUB_TOKEN`. diff --git a/plugins/pr-review/workflows/pr-review-by-openhands.yml b/plugins/pr-review/workflows/pr-review-by-openhands.yml deleted file mode 100644 index 1efa205d..00000000 --- a/plugins/pr-review/workflows/pr-review-by-openhands.yml +++ /dev/null @@ -1,54 +0,0 @@ ---- -name: PR Review by OpenHands - -on: - # Use pull_request_target to allow fork PRs to access secrets when triggered by maintainers - # Security: This workflow runs when: - # 1. A new PR is opened (non-draft), OR - # 2. A draft PR is marked as ready for review, OR - # 3. A maintainer adds the 'review-this' label, OR - # 4. A maintainer requests openhands-agent or all-hands-bot as a reviewer - # Adding labels and requesting new reviewers requires write access. GitHub may also allow PR authors - # to re-request review from a previous reviewer. - # The PR code is explicitly checked out for review, but secrets are only accessible - # because the workflow runs in the base repository context - pull_request_target: - types: [opened, ready_for_review, labeled, review_requested] - -permissions: - contents: read - pull-requests: write - issues: write - -jobs: - pr-review: - # Run when one of the following conditions is met: - # 1. A new non-draft PR is opened by a non-first-time contributor, OR - # 2. A draft PR is converted to ready for review by a non-first-time contributor, OR - # 3. 'review-this' label is added, OR - # 4. openhands-agent or all-hands-bot is requested as a reviewer - # Note: FIRST_TIME_CONTRIBUTOR PRs require manual trigger via label/reviewer request - if: | - (github.event.action == 'opened' && github.event.pull_request.draft == false && github.event.pull_request.author_association != 'FIRST_TIME_CONTRIBUTOR' && github.event.pull_request.author_association != 'NONE') || - (github.event.action == 'ready_for_review' && github.event.pull_request.author_association != 'FIRST_TIME_CONTRIBUTOR' && github.event.pull_request.author_association != 'NONE') || - github.event.label.name == 'review-this' || - github.event.requested_reviewer.login == 'openhands-agent' || - github.event.requested_reviewer.login == 'all-hands-bot' - concurrency: - group: pr-review-${{ github.event.pull_request.number }} - cancel-in-progress: true - runs-on: ubuntu-24.04 - steps: - - name: Run PR Review - uses: OpenHands/extensions/plugins/pr-review@main - with: - # LLM model(s) to use. Can be comma-separated for A/B testing - # - one model will be randomly selected per review - llm-model: anthropic/claude-sonnet-4-5-20250929 - # llm-base-url: https://llm-proxy.app.all-hands.dev - # Review style: roasted (other option: standard) - review-style: roasted - llm-api-key: ${{ secrets.LLM_API_KEY }} - github-token: ${{ secrets.GITHUB_TOKEN }} - # Optional: Laminar API key for observability - # lmnr-api-key: ${{ secrets.LMNR_PROJECT_API_KEY }} diff --git a/plugins/pr-review/workflows/pr-review-by-openhands.yml b/plugins/pr-review/workflows/pr-review-by-openhands.yml new file mode 120000 index 00000000..849c5929 --- /dev/null +++ b/plugins/pr-review/workflows/pr-review-by-openhands.yml @@ -0,0 +1 @@ +../../../.github/workflows/pr-review-by-openhands.yml \ No newline at end of file diff --git a/plugins/pr-review/workflows/pr-review-evaluation.yml b/plugins/pr-review/workflows/pr-review-evaluation.yml deleted file mode 100644 index 446df722..00000000 --- a/plugins/pr-review/workflows/pr-review-evaluation.yml +++ /dev/null @@ -1,106 +0,0 @@ ---- -name: PR Review Evaluation - -# This workflow runs when a PR is merged or closed to evaluate how well -# the review agent's comments were addressed. -# -# It creates an evaluation trace in Laminar that can be processed by a -# signal to determine review effectiveness. -# -# Prerequisites: -# - PR must have been reviewed by pr-review-by-openhands.yml first -# - Trace info artifact must exist from the review workflow - -on: - pull_request_target: - types: [closed] - -permissions: - contents: read - pull-requests: read - -jobs: - evaluate: - # Only run if: - # 1. This is a merged PR, AND - # 2. The PR was previously reviewed (has the trace artifact) - runs-on: ubuntu-24.04 - env: - PR_NUMBER: ${{ github.event.pull_request.number }} - REPO_NAME: ${{ github.repository }} - PR_MERGED: ${{ github.event.pull_request.merged }} - - steps: - # Note: actions/download-artifact@v5 only works within the same workflow run. - # We use dawidd6/action-download-artifact to download from a different workflow. - - name: Download review trace artifact - id: download-trace - uses: dawidd6/action-download-artifact@v6 - continue-on-error: true - with: - workflow: pr-review-by-openhands.yml - name: pr-review-trace-${{ github.event.pull_request.number }} - path: trace-info - search_artifacts: true - if_no_artifact_found: warn - - # Check if the trace file actually exists (the artifact download may - # succeed but with no matching artifact, only issuing a warning) - - name: Check if trace file exists - id: check-trace - run: | - if [ -f "trace-info/laminar_trace_info.json" ]; then - echo "trace_exists=true" >> $GITHUB_OUTPUT - echo "Found trace file for PR #$PR_NUMBER" - else - echo "trace_exists=false" >> $GITHUB_OUTPUT - echo "No trace file found for PR #$PR_NUMBER" - echo "This PR may not have been reviewed by the agent, skipping evaluation" - fi - - - name: Checkout extensions repository - if: steps.check-trace.outputs.trace_exists == 'true' - uses: actions/checkout@v5 - with: - repository: OpenHands/extensions - path: extensions - - - name: Set up Python - if: steps.check-trace.outputs.trace_exists == 'true' - uses: actions/setup-python@v6 - with: - python-version: '3.13' - - - name: Install uv - if: steps.check-trace.outputs.trace_exists == 'true' - uses: astral-sh/setup-uv@v7 - with: - enable-cache: true - - - name: Install dependencies - if: steps.check-trace.outputs.trace_exists == 'true' - run: | - # Install lmnr SDK for Laminar integration - uv pip install --system lmnr - - - name: Run evaluation - if: steps.check-trace.outputs.trace_exists == 'true' - env: - LMNR_PROJECT_API_KEY: ${{ secrets.LMNR_PROJECT_API_KEY }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - # Copy trace info to working directory - cp trace-info/laminar_trace_info.json . - - # Run the evaluation script - uv run python extensions/plugins/pr-review/scripts/evaluate_review.py - - - name: Upload evaluation logs - uses: actions/upload-artifact@v5 - if: always() && steps.check-trace.outputs.trace_exists == 'true' - with: - name: pr-review-evaluation-${{ github.event.pull_request.number }} - path: | - *.log - *.json - retention-days: 30 diff --git a/plugins/pr-review/workflows/pr-review-evaluation.yml b/plugins/pr-review/workflows/pr-review-evaluation.yml new file mode 120000 index 00000000..b25debc0 --- /dev/null +++ b/plugins/pr-review/workflows/pr-review-evaluation.yml @@ -0,0 +1 @@ +../../../.github/workflows/pr-review-evaluation.yml \ No newline at end of file From 9fc9989916ac287c2373a5e7b3125648af87ae89 Mon Sep 17 00:00:00 2001 From: openhands Date: Thu, 19 Feb 2026 19:42:46 +0000 Subject: [PATCH 07/17] Fix README: use correct URLs for workflow files - Point curl commands to .github/workflows/ (symlinks don't work with raw URLs) - Clarify LMNR secret naming in secrets table - Simplify installation instructions Co-authored-by: openhands --- plugins/pr-review/README.md | 27 ++++++++++----------------- 1 file changed, 10 insertions(+), 17 deletions(-) diff --git a/plugins/pr-review/README.md b/plugins/pr-review/README.md index be746342..4c457fba 100644 --- a/plugins/pr-review/README.md +++ b/plugins/pr-review/README.md @@ -9,9 +9,9 @@ Copy both workflow files to your repository: ```bash mkdir -p .github/workflows curl -o .github/workflows/pr-review-by-openhands.yml \ - https://raw.githubusercontent.com/OpenHands/extensions/main/plugins/pr-review/workflows/pr-review-by-openhands.yml + https://raw.githubusercontent.com/OpenHands/extensions/main/.github/workflows/pr-review-by-openhands.yml curl -o .github/workflows/pr-review-evaluation.yml \ - https://raw.githubusercontent.com/OpenHands/extensions/main/plugins/pr-review/workflows/pr-review-evaluation.yml + https://raw.githubusercontent.com/OpenHands/extensions/main/.github/workflows/pr-review-evaluation.yml ``` Then configure the required secrets (see [Installation](#installation) below). @@ -47,16 +47,16 @@ plugins/pr-review/ ## Installation -### 1. Copy the Workflow File +### 1. Copy the Workflow Files -Copy the workflow file to your repository's `.github/workflows/` directory: +Copy the workflow files to your repository's `.github/workflows/` directory: ```bash -# Option A: Download from GitHub +mkdir -p .github/workflows curl -o .github/workflows/pr-review-by-openhands.yml \ - https://raw.githubusercontent.com/OpenHands/extensions/main/plugins/pr-review/workflows/pr-review-by-openhands.yml - -# Option B: Create manually (see workflow content below) + https://raw.githubusercontent.com/OpenHands/extensions/main/.github/workflows/pr-review-by-openhands.yml +curl -o .github/workflows/pr-review-evaluation.yml \ + https://raw.githubusercontent.com/OpenHands/extensions/main/.github/workflows/pr-review-evaluation.yml ``` ### 2. Configure Secrets @@ -67,7 +67,7 @@ Add the following secrets in your repository settings (**Settings → Secrets an |--------|----------|-------------| | `LLM_API_KEY` | Yes | API key for your LLM provider | | `GITHUB_TOKEN` | Auto | Provided automatically by GitHub Actions | -| `LMNR_SKILLS_API_KEY` | No | Laminar API key for observability (org-level secret for OpenHands repos) | +| `LMNR_SKILLS_API_KEY` | No | Laminar API key (org-level secret; mapped to `LMNR_PROJECT_API_KEY` env var in workflows) | **Note**: For repositories that need to post review comments from a bot account, use `ALLHANDS_BOT_GITHUB_PAT` instead of `GITHUB_TOKEN`. @@ -171,14 +171,7 @@ One model is randomly selected for each review. When Laminar observability is en ### Review Evaluation -To evaluate how well reviews were addressed, add the evaluation workflow: - -```bash -curl -o .github/workflows/pr-review-evaluation.yml \ - https://raw.githubusercontent.com/OpenHands/extensions/main/plugins/pr-review/workflows/pr-review-evaluation.yml -``` - -This workflow runs when PRs are closed and: +The evaluation workflow (`pr-review-evaluation.yml`) runs when PRs are closed and: 1. Downloads the review trace artifact 2. Fetches final PR state and comments 3. Creates an evaluation span in Laminar From 5e74fcc041ce4b8ff18bf347462c46dbf0b18694 Mon Sep 17 00:00:00 2001 From: openhands Date: Thu, 19 Feb 2026 19:54:40 +0000 Subject: [PATCH 08/17] Replace symlinks with copies + add sync test - Replace symlinks in plugins/pr-review/workflows/ with actual file copies (symlinks don't work with GitHub raw URLs) - Add test to ensure workflow copies stay in sync with .github/workflows/ - Add CI workflow to run the sync check Co-authored-by: openhands --- .github/workflows/check-workflow-sync.yml | 18 ++++ .../workflows/pr-review-by-openhands.yml | 51 ++++++++++- .../workflows/pr-review-evaluation.yml | 86 ++++++++++++++++++- tests/test_workflow_sync.py | 50 +++++++++++ 4 files changed, 203 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/check-workflow-sync.yml mode change 120000 => 100644 plugins/pr-review/workflows/pr-review-by-openhands.yml mode change 120000 => 100644 plugins/pr-review/workflows/pr-review-evaluation.yml create mode 100644 tests/test_workflow_sync.py diff --git a/.github/workflows/check-workflow-sync.yml b/.github/workflows/check-workflow-sync.yml new file mode 100644 index 00000000..f6f99081 --- /dev/null +++ b/.github/workflows/check-workflow-sync.yml @@ -0,0 +1,18 @@ +name: Check Workflow Sync + +on: + pull_request: + branches: ["*"] + push: + branches: ["main", "master"] + +jobs: + check-workflow-sync: + name: Verify workflow files are in sync + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Check workflow files are in sync + run: python tests/test_workflow_sync.py diff --git a/plugins/pr-review/workflows/pr-review-by-openhands.yml b/plugins/pr-review/workflows/pr-review-by-openhands.yml deleted file mode 120000 index 849c5929..00000000 --- a/plugins/pr-review/workflows/pr-review-by-openhands.yml +++ /dev/null @@ -1 +0,0 @@ -../../../.github/workflows/pr-review-by-openhands.yml \ No newline at end of file diff --git a/plugins/pr-review/workflows/pr-review-by-openhands.yml b/plugins/pr-review/workflows/pr-review-by-openhands.yml new file mode 100644 index 00000000..9cf05232 --- /dev/null +++ b/plugins/pr-review/workflows/pr-review-by-openhands.yml @@ -0,0 +1,50 @@ +--- +name: PR Review by OpenHands + +on: + # Use pull_request_target to allow fork PRs to access secrets when triggered by maintainers + # Security: This workflow runs when: + # 1. A new PR is opened (non-draft), OR + # 2. A draft PR is marked as ready for review, OR + # 3. A maintainer adds the 'review-this' label, OR + # 4. A maintainer requests openhands-agent or all-hands-bot as a reviewer + # Adding labels and requesting reviewers requires write access. + # The PR code is explicitly checked out for review, but secrets are only accessible + # because the workflow runs in the base repository context. + pull_request_target: + types: [opened, ready_for_review, labeled, review_requested] + +permissions: + contents: read + pull-requests: write + issues: write + +jobs: + pr-review: + # Run when one of the following conditions is met: + # 1. A new non-draft PR is opened by a non-first-time contributor, OR + # 2. A draft PR is converted to ready for review by a non-first-time contributor, OR + # 3. 'review-this' label is added, OR + # 4. openhands-agent or all-hands-bot is requested as a reviewer + # Note: FIRST_TIME_CONTRIBUTOR and NONE PRs require manual trigger via label/reviewer request. + if: | + (github.event.action == 'opened' && github.event.pull_request.draft == false && github.event.pull_request.author_association != 'FIRST_TIME_CONTRIBUTOR' && github.event.pull_request.author_association != 'NONE') || + (github.event.action == 'ready_for_review' && github.event.pull_request.author_association != 'FIRST_TIME_CONTRIBUTOR' && github.event.pull_request.author_association != 'NONE') || + github.event.label.name == 'review-this' || + github.event.requested_reviewer.login == 'openhands-agent' || + github.event.requested_reviewer.login == 'all-hands-bot' + concurrency: + group: pr-review-${{ github.event.pull_request.number }} + cancel-in-progress: true + runs-on: ubuntu-24.04 + steps: + - name: Run PR Review + uses: OpenHands/extensions/plugins/pr-review@main + with: + llm-model: litellm_proxy/claude-sonnet-4-5-20250929 + llm-base-url: https://llm-proxy.app.all-hands.dev + # Review style: roasted (other option: standard) + review-style: roasted + llm-api-key: ${{ secrets.LLM_API_KEY }} + github-token: ${{ secrets.ALLHANDS_BOT_GITHUB_PAT }} + lmnr-api-key: ${{ secrets.LMNR_SKILLS_API_KEY }} diff --git a/plugins/pr-review/workflows/pr-review-evaluation.yml b/plugins/pr-review/workflows/pr-review-evaluation.yml deleted file mode 120000 index b25debc0..00000000 --- a/plugins/pr-review/workflows/pr-review-evaluation.yml +++ /dev/null @@ -1 +0,0 @@ -../../../.github/workflows/pr-review-evaluation.yml \ No newline at end of file diff --git a/plugins/pr-review/workflows/pr-review-evaluation.yml b/plugins/pr-review/workflows/pr-review-evaluation.yml new file mode 100644 index 00000000..60baa50c --- /dev/null +++ b/plugins/pr-review/workflows/pr-review-evaluation.yml @@ -0,0 +1,85 @@ +--- +name: PR Review Evaluation + +# This workflow evaluates how well PR review comments were addressed. +# It runs when a PR is closed to assess review effectiveness. +# +# Security note: pull_request_target is safe here because: +# 1. Only triggers on PR close (not on code changes) +# 2. Does not checkout PR code - only downloads artifacts from trusted workflow runs +# 3. Runs evaluation scripts from the extensions repo, not from the PR + +on: + pull_request_target: + types: [closed] + +permissions: + contents: read + pull-requests: read + +jobs: + evaluate: + runs-on: ubuntu-24.04 + env: + PR_NUMBER: ${{ github.event.pull_request.number }} + REPO_NAME: ${{ github.repository }} + PR_MERGED: ${{ github.event.pull_request.merged }} + + steps: + - name: Download review trace artifact + id: download-trace + uses: dawidd6/action-download-artifact@v6 + continue-on-error: true + with: + workflow: pr-review-by-openhands.yml + name: pr-review-trace-${{ github.event.pull_request.number }} + path: trace-info + search_artifacts: true + if_no_artifact_found: warn + + - name: Check if trace file exists + id: check-trace + run: | + if [ -f "trace-info/laminar_trace_info.json" ]; then + echo "trace_exists=true" >> $GITHUB_OUTPUT + echo "Found trace file for PR #$PR_NUMBER" + else + echo "trace_exists=false" >> $GITHUB_OUTPUT + echo "No trace file found for PR #$PR_NUMBER - skipping evaluation" + fi + + # Always checkout main branch for security - cannot test script changes in PRs + - name: Checkout extensions repository + if: steps.check-trace.outputs.trace_exists == 'true' + uses: actions/checkout@v5 + with: + repository: OpenHands/extensions + path: extensions + + - name: Set up Python + if: steps.check-trace.outputs.trace_exists == 'true' + uses: actions/setup-python@v6 + with: + python-version: '3.12' + + - name: Install dependencies + if: steps.check-trace.outputs.trace_exists == 'true' + run: pip install lmnr + + - name: Run evaluation + if: steps.check-trace.outputs.trace_exists == 'true' + env: + # Script expects LMNR_PROJECT_API_KEY; org secret is named LMNR_SKILLS_API_KEY + LMNR_PROJECT_API_KEY: ${{ secrets.LMNR_SKILLS_API_KEY }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + python extensions/plugins/pr-review/scripts/evaluate_review.py \ + --trace-file trace-info/laminar_trace_info.json + + - name: Upload evaluation logs + uses: actions/upload-artifact@v5 + if: always() && steps.check-trace.outputs.trace_exists == 'true' + with: + name: pr-review-evaluation-${{ github.event.pull_request.number }} + path: '*.log' + retention-days: 30 diff --git a/tests/test_workflow_sync.py b/tests/test_workflow_sync.py new file mode 100644 index 00000000..e9a84e7f --- /dev/null +++ b/tests/test_workflow_sync.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +"""Test that workflow files in .github/workflows/ match any copies elsewhere in the repo.""" + +import os +from pathlib import Path + + +def test_workflow_files_are_in_sync(): + """Ensure workflow files in .github/workflows/ are identical to copies elsewhere.""" + repo_root = Path(__file__).parent.parent + workflows_dir = repo_root / ".github" / "workflows" + + if not workflows_dir.exists(): + return # No workflows directory + + # Find all workflow files in .github/workflows/ + canonical_workflows = {f.name: f for f in workflows_dir.glob("*.yml")} + canonical_workflows.update({f.name: f for f in workflows_dir.glob("*.yaml")}) + + # Find all other yml/yaml files in the repo that might be workflow copies + mismatches = [] + for root, dirs, files in os.walk(repo_root): + # Skip .github/workflows itself and hidden directories + if ".github/workflows" in root or "/.git" in root: + continue + + for filename in files: + if filename in canonical_workflows: + other_path = Path(root) / filename + canonical_path = canonical_workflows[filename] + + canonical_content = canonical_path.read_text() + other_content = other_path.read_text() + + if canonical_content != other_content: + rel_path = other_path.relative_to(repo_root) + mismatches.append( + f"{rel_path} differs from .github/workflows/{filename}" + ) + + if mismatches: + raise AssertionError( + "Workflow files are out of sync:\n" + "\n".join(f" - {m}" for m in mismatches) + + "\n\nRun: cp .github/workflows/ to sync" + ) + + +if __name__ == "__main__": + test_workflow_files_are_in_sync() + print("All workflow files are in sync!") From 9692b23d11eb8e7e96ec96b613010b26c8106792 Mon Sep 17 00:00:00 2001 From: openhands Date: Thu, 19 Feb 2026 19:57:13 +0000 Subject: [PATCH 09/17] Merge workflow sync check into validation workflow No need for a separate workflow - just add step to existing checks. Co-authored-by: openhands --- .github/workflows/check-readme.yml | 9 ++++++--- .github/workflows/check-workflow-sync.yml | 18 ------------------ 2 files changed, 6 insertions(+), 21 deletions(-) delete mode 100644 .github/workflows/check-workflow-sync.yml diff --git a/.github/workflows/check-readme.yml b/.github/workflows/check-readme.yml index f179bda5..af92d2e8 100644 --- a/.github/workflows/check-readme.yml +++ b/.github/workflows/check-readme.yml @@ -1,4 +1,4 @@ -name: Check README.md in Skills +name: Validation Checks on: pull_request: @@ -7,13 +7,16 @@ on: branches: ["main", "master"] jobs: - check-readme: - name: Verify README.md exists in all skill directories + validate: + name: Run validation checks runs-on: ubuntu-latest steps: - name: Checkout repository uses: actions/checkout@v4 + - name: Check workflow files are in sync + run: python tests/test_workflow_sync.py + - name: Check for README.md in each skill directory run: | echo "Checking for README.md in all skill directories..." diff --git a/.github/workflows/check-workflow-sync.yml b/.github/workflows/check-workflow-sync.yml deleted file mode 100644 index f6f99081..00000000 --- a/.github/workflows/check-workflow-sync.yml +++ /dev/null @@ -1,18 +0,0 @@ -name: Check Workflow Sync - -on: - pull_request: - branches: ["*"] - push: - branches: ["main", "master"] - -jobs: - check-workflow-sync: - name: Verify workflow files are in sync - runs-on: ubuntu-latest - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Check workflow files are in sync - run: python tests/test_workflow_sync.py From 297d61e05daa78040eaca33355c5c02da947499f Mon Sep 17 00:00:00 2001 From: openhands Date: Thu, 19 Feb 2026 19:58:20 +0000 Subject: [PATCH 10/17] Replace validation checks with proper tests workflow - Remove check-readme.yml - Add tests.yml that runs pytest on tests/ Co-authored-by: openhands --- .github/workflows/check-readme.yml | 57 ------------------------------ .github/workflows/tests.yml | 25 +++++++++++++ 2 files changed, 25 insertions(+), 57 deletions(-) delete mode 100644 .github/workflows/check-readme.yml create mode 100644 .github/workflows/tests.yml diff --git a/.github/workflows/check-readme.yml b/.github/workflows/check-readme.yml deleted file mode 100644 index af92d2e8..00000000 --- a/.github/workflows/check-readme.yml +++ /dev/null @@ -1,57 +0,0 @@ -name: Validation Checks - -on: - pull_request: - branches: ["*"] - push: - branches: ["main", "master"] - -jobs: - validate: - name: Run validation checks - runs-on: ubuntu-latest - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Check workflow files are in sync - run: python tests/test_workflow_sync.py - - - name: Check for README.md in each skill directory - run: | - echo "Checking for README.md in all skill directories..." - - # Find all immediate subdirectories of ./skills/ - missing_readmes=() - - for dir in ./skills/*/; do - # Skip if not a directory - [ -d "$dir" ] || continue - - # Get the directory name - dirname=$(basename "$dir") - - # Skip hidden directories - [[ "$dirname" == .* ]] && continue - - # Check if README.md exists - if [ ! -f "${dir}README.md" ]; then - missing_readmes+=("$dirname") - fi - done - - # Report results - if [ ${#missing_readmes[@]} -gt 0 ]; then - echo "" - echo "❌ ERROR: The following skill directories are missing README.md:" - echo "" - for skill in "${missing_readmes[@]}"; do - echo " - ./skills/$skill/" - done - echo "" - echo "Please add a README.md file to each of the above directories." - exit 1 - else - echo "" - echo "✅ All skill directories contain a README.md file." - fi diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 00000000..3d9fb1b9 --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,25 @@ +name: Tests + +on: + pull_request: + branches: ["*"] + push: + branches: ["main", "master"] + +jobs: + test: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Install pytest + run: pip install pytest + + - name: Run tests + run: pytest tests/ From 8129b96e60c423ebece5736f687c3b6a96973683 Mon Sep 17 00:00:00 2001 From: openhands Date: Thu, 19 Feb 2026 19:59:15 +0000 Subject: [PATCH 11/17] Add PYTHONPATH for tests to find skills module Co-authored-by: openhands --- .github/workflows/tests.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 3d9fb1b9..c56ef574 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -23,3 +23,5 @@ jobs: - name: Run tests run: pytest tests/ + env: + PYTHONPATH: ${{ github.workspace }} From 4ddbdc9db66463fec15066536fcf92cad38b2910 Mon Sep 17 00:00:00 2001 From: openhands Date: Thu, 19 Feb 2026 19:59:57 +0000 Subject: [PATCH 12/17] Add requests to test dependencies Co-authored-by: openhands --- .github/workflows/tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index c56ef574..b2401fd5 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -18,8 +18,8 @@ jobs: with: python-version: "3.12" - - name: Install pytest - run: pip install pytest + - name: Install dependencies + run: pip install pytest requests - name: Run tests run: pytest tests/ From 87ba88ac904784671191ed27dd46ebc124653e67 Mon Sep 17 00:00:00 2001 From: openhands Date: Thu, 19 Feb 2026 20:01:04 +0000 Subject: [PATCH 13/17] Use uv for tests workflow Co-authored-by: openhands --- .github/workflows/tests.yml | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index b2401fd5..faf9ef69 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -13,15 +13,13 @@ jobs: - name: Checkout repository uses: actions/checkout@v4 - - name: Set up Python - uses: actions/setup-python@v5 + - name: Install uv + uses: astral-sh/setup-uv@v7 with: + enable-cache: true python-version: "3.12" - - name: Install dependencies - run: pip install pytest requests - - name: Run tests - run: pytest tests/ + run: uv run --with pytest --with requests pytest tests/ env: PYTHONPATH: ${{ github.workspace }} From fe609a71966445bf9ee212117f60d219c8820ba8 Mon Sep 17 00:00:00 2001 From: openhands Date: Thu, 19 Feb 2026 20:09:22 +0000 Subject: [PATCH 14/17] Fix workflow sync test to only check known copy locations Only check plugins/*/workflows/ directories, not arbitrary yml files. Co-authored-by: openhands --- tests/test_workflow_sync.py | 47 +++++++++++++++++++------------------ 1 file changed, 24 insertions(+), 23 deletions(-) diff --git a/tests/test_workflow_sync.py b/tests/test_workflow_sync.py index e9a84e7f..d317c3bd 100644 --- a/tests/test_workflow_sync.py +++ b/tests/test_workflow_sync.py @@ -1,46 +1,47 @@ #!/usr/bin/env python3 -"""Test that workflow files in .github/workflows/ match any copies elsewhere in the repo.""" +"""Test that workflow files in .github/workflows/ match copies in plugins/*/workflows/.""" -import os from pathlib import Path +# Known locations where workflow copies are maintained +WORKFLOW_COPY_DIRS = [ + "plugins/pr-review/workflows", +] + + def test_workflow_files_are_in_sync(): - """Ensure workflow files in .github/workflows/ are identical to copies elsewhere.""" + """Ensure workflow files in .github/workflows/ are identical to copies in plugin dirs.""" repo_root = Path(__file__).parent.parent workflows_dir = repo_root / ".github" / "workflows" if not workflows_dir.exists(): return # No workflows directory - # Find all workflow files in .github/workflows/ - canonical_workflows = {f.name: f for f in workflows_dir.glob("*.yml")} - canonical_workflows.update({f.name: f for f in workflows_dir.glob("*.yaml")}) - - # Find all other yml/yaml files in the repo that might be workflow copies mismatches = [] - for root, dirs, files in os.walk(repo_root): - # Skip .github/workflows itself and hidden directories - if ".github/workflows" in root or "/.git" in root: + for copy_dir in WORKFLOW_COPY_DIRS: + copy_path = repo_root / copy_dir + if not copy_path.exists(): continue - for filename in files: - if filename in canonical_workflows: - other_path = Path(root) / filename - canonical_path = canonical_workflows[filename] + for copy_file in copy_path.glob("*.yml"): + canonical_file = workflows_dir / copy_file.name + if not canonical_file.exists(): + continue - canonical_content = canonical_path.read_text() - other_content = other_path.read_text() + canonical_content = canonical_file.read_text() + copy_content = copy_file.read_text() - if canonical_content != other_content: - rel_path = other_path.relative_to(repo_root) - mismatches.append( - f"{rel_path} differs from .github/workflows/{filename}" - ) + if canonical_content != copy_content: + mismatches.append( + f"{copy_dir}/{copy_file.name} differs from " + f".github/workflows/{copy_file.name}" + ) if mismatches: raise AssertionError( - "Workflow files are out of sync:\n" + "\n".join(f" - {m}" for m in mismatches) + "Workflow files are out of sync:\n" + + "\n".join(f" - {m}" for m in mismatches) + "\n\nRun: cp .github/workflows/ to sync" ) From 08e135c42252a21e14dec5786909b4462fe8236f Mon Sep 17 00:00:00 2001 From: openhands Date: Thu, 19 Feb 2026 20:09:50 +0000 Subject: [PATCH 15/17] Auto-discover all plugins/*/workflows directories Co-authored-by: openhands --- tests/test_workflow_sync.py | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/tests/test_workflow_sync.py b/tests/test_workflow_sync.py index d317c3bd..1373bcda 100644 --- a/tests/test_workflow_sync.py +++ b/tests/test_workflow_sync.py @@ -4,12 +4,6 @@ from pathlib import Path -# Known locations where workflow copies are maintained -WORKFLOW_COPY_DIRS = [ - "plugins/pr-review/workflows", -] - - def test_workflow_files_are_in_sync(): """Ensure workflow files in .github/workflows/ are identical to copies in plugin dirs.""" repo_root = Path(__file__).parent.parent @@ -19,9 +13,9 @@ def test_workflow_files_are_in_sync(): return # No workflows directory mismatches = [] - for copy_dir in WORKFLOW_COPY_DIRS: - copy_path = repo_root / copy_dir - if not copy_path.exists(): + # Check all plugins/*/workflows/ directories + for copy_path in repo_root.glob("plugins/*/workflows"): + if not copy_path.is_dir(): continue for copy_file in copy_path.glob("*.yml"): @@ -33,8 +27,9 @@ def test_workflow_files_are_in_sync(): copy_content = copy_file.read_text() if canonical_content != copy_content: + rel_path = copy_path.relative_to(repo_root) mismatches.append( - f"{copy_dir}/{copy_file.name} differs from " + f"{rel_path}/{copy_file.name} differs from " f".github/workflows/{copy_file.name}" ) From 694ffeff58269de95454854e2627865944559185 Mon Sep 17 00:00:00 2001 From: openhands Date: Thu, 19 Feb 2026 21:59:00 +0000 Subject: [PATCH 16/17] Add pyproject.toml with test dependencies Pin pytest and requests versions for reproducible test runs. Co-authored-by: openhands --- .github/workflows/tests.yml | 2 +- pyproject.toml | 11 +++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) create mode 100644 pyproject.toml diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index faf9ef69..308e5572 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -20,6 +20,6 @@ jobs: python-version: "3.12" - name: Run tests - run: uv run --with pytest --with requests pytest tests/ + run: uv run --group test pytest tests/ env: PYTHONPATH: ${{ github.workspace }} diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..a98a3801 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,11 @@ +[project] +name = "extensions" +version = "0.1.0" +description = "OpenHands extensions, plugins, and skills" +requires-python = ">=3.12" + +[dependency-groups] +test = [ + "pytest>=8.0", + "requests>=2.31", +] From 39ade095c363d5c0df5a69e8ccfef6d8a17fa4f3 Mon Sep 17 00:00:00 2001 From: openhands Date: Thu, 19 Feb 2026 22:18:53 +0000 Subject: [PATCH 17/17] Fix PR review action: use uv run --with for dependencies The previous approach used 'uv pip install --system' then 'uv run python' which created a new venv without the installed dependencies. Co-authored-by: openhands --- plugins/pr-review/action.yml | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/plugins/pr-review/action.yml b/plugins/pr-review/action.yml index d4e39352..ba96e839 100644 --- a/plugins/pr-review/action.yml +++ b/plugins/pr-review/action.yml @@ -77,11 +77,6 @@ runs: sudo apt-get update sudo apt-get install -y gh - - name: Install OpenHands dependencies - shell: bash - run: | - uv pip install --system openhands-sdk openhands-tools lmnr - - name: Check required configuration and select model id: select-model shell: bash @@ -132,7 +127,8 @@ runs: REPO_NAME: ${{ github.repository }} run: | cd pr-repo - uv run python ../extensions/plugins/pr-review/scripts/agent_script.py + uv run --with openhands-sdk --with openhands-tools --with lmnr \ + python ../extensions/plugins/pr-review/scripts/agent_script.py - name: Upload logs as artifact uses: actions/upload-artifact@v4