From 894b2802ba1d798e7f96e961a5560ba37c6730f6 Mon Sep 17 00:00:00 2001 From: Ihor Solodrai Date: Mon, 23 Feb 2026 15:37:23 -0800 Subject: [PATCH] BPF CI Bot Implement a new experimental CI workflow that uses AI to investigate CI test failures and come up with improvements to the BPF CI system. The underlying environment is basically the same as we use for AI code review workflows: Claude Code has access to the Linux repository, semcode database [1] (with --lore), as well as CI source code and configs and various dependencies and tools. The AI is then instructed to explore recent CI failures, lore discussions and code changes and select a single issue for investigation (see `ci/claude/bpf-ci-agent.md` for the full prompt). AI may also suggest a code change addressing the issue. The results are then posted as GitHub issues in kernel-patches/vmtest for human evaluation. If it's good, it may be submitted to the mailing list (with the full disclosure, of course). If it's bad the issue can be closed and ignored. [1] https://github.com/facebookexperimental/semcode Signed-off-by: Ihor Solodrai --- .github/workflows/ai-agent.yml | 242 +++++++++++++++++++++++++++++++++ ci/claude/bpf-ci-agent.md | 119 ++++++++++++++++ ci/claude/settings.json | 2 +- 3 files changed, 362 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/ai-agent.yml create mode 100644 ci/claude/bpf-ci-agent.md diff --git a/.github/workflows/ai-agent.yml b/.github/workflows/ai-agent.yml new file mode 100644 index 00000000..ac4dc0fe --- /dev/null +++ b/.github/workflows/ai-agent.yml @@ -0,0 +1,242 @@ +name: BPF CI Bot + +permissions: + contents: read + id-token: write + issues: write + pull-requests: write + actions: read + +on: + schedule: + - cron: '0 12 * * 1,3,5' # Mon/Wed/Fri at ~4am Pacific Time + workflow_dispatch: + pull_request: + paths: + - .github/workflows/ai-agent.yml + - ci/claude/bpf-ci-agent.md + +concurrency: + group: bpf-ci-bot + cancel-in-progress: false + +jobs: + agent-run: + if: ${{ github.repository == 'kernel-patches/vmtest' && vars.AWS_REGION }} + runs-on: + - ${{ format('codebuild-bpf-ci-{0}-{1}', github.run_id, github.run_attempt) }} + - image:custom-linux-ghcr.io/kernel-patches/runner:ai-review + - instance-size:large + env: + AWS_REGION: us-west-2 + steps: + + - name: Checkout CI code + uses: actions/checkout@v6 + with: + sparse-checkout: | + .github/scripts + ci/claude + + - name: Set up .claude/settings.json and the prompt + shell: bash + run: | + mkdir -p ~/.claude + cp ci/claude/settings.json ~/.claude/settings.json + cp ci/claude/bpf-ci-agent.md agent.md + + - name: Checkout review-prompts + uses: actions/checkout@v6 + with: + repository: 'masoncl/review-prompts' + path: 'github/masoncl/review-prompts' + ref: main + + - name: Checkout libbpf/ci + uses: actions/checkout@v6 + with: + repository: 'libbpf/ci' + path: 'github/libbpf/ci' + ref: main + + - name: Checkout kernel-patches/vmtest + uses: actions/checkout@v6 + with: + repository: 'kernel-patches/vmtest' + path: 'github/kernel-patches/vmtest' + ref: master + + - name: Checkout kernel-patches/runner + uses: actions/checkout@v6 + with: + repository: 'kernel-patches/runner' + path: 'github/kernel-patches/runner' + ref: main + + - name: Checkout kernel-patches/kernel-patches-daemon + uses: actions/checkout@v6 + with: + repository: 'kernel-patches/kernel-patches-daemon' + path: 'github/kernel-patches/kernel-patches-daemon' + ref: main + + - name: Checkout danobi/vmtest + uses: actions/checkout@v6 + with: + repository: 'danobi/vmtest' + path: 'github/danobi/vmtest' + ref: master + + - name: Checkout facebookexperimental/semcode + uses: actions/checkout@v6 + with: + repository: 'facebookexperimental/semcode' + path: 'github/facebookexperimental/semcode' + ref: main + + - name: Checkout nojb/public-inbox + uses: actions/checkout@v6 + with: + repository: 'nojb/public-inbox' + path: 'github/nojb/public-inbox' + ref: master + + - name: Install misc tools + shell: bash + env: + GCC_VERSION: 14 + LLVM_VERSION: 19 + run: | + sudo apt-get update -y + ${{ github.workspace }}/.github/scripts/install-github-cli.sh + ${{ github.workspace }}/github/kernel-patches/runner/install-dependencies.sh all + sudo apt-get install -y python3 jq lei + + - name: Download Linux source tree + uses: libbpf/ci/get-linux-source@v4 + with: + repo: 'https://github.com/kernel-patches/bpf.git' + rev: 'bpf-next' + dest: linux + env: + REFERENCE_REPO_PATH: /libbpfci/mirrors/linux + FETCH_DEPTH: 0 # full clone + + # This manipulation is necessary to make sure that + # ${{ github.workspace }} is the root of the Linux git repo + - name: Move linux source in place + shell: bash + run: | + rm -rf .git .github + cd linux + mv -t .. $(ls -A) + cd .. + rmdir linux + + - name: semcode-index + shell: bash + run: | + git remote add torvalds https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git + git fetch torvalds + MERGE_BASE=$(git merge-base v6.19 HEAD) + rm -rf /ci/.semcode.db/lore + ln -s /ci/.semcode.db .semcode.db + semcode-index --git "${MERGE_BASE}..HEAD" + semcode-index --lore bpf + + - name: Restore NOTES.md + uses: actions/cache/restore@v5 + with: + path: NOTES.md + key: notes-md-${{ github.repository }} + + - name: Generate GitHub App token + id: app-token + uses: actions/create-github-app-token@v2 + with: + app-id: ${{ secrets.KP_REVIEW_BOT_APP_ID }} + private-key: ${{ secrets.KP_REVIEW_BOT_APP_PRIVATE_KEY }} + + - name: Configure AWS Credentials (OIDC) + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ secrets.AWS_BEDROCK_ROLE }} + aws-region: us-west-2 + + - uses: anthropics/claude-code-action@v1 + with: + show_full_output: true + github_token: ${{ steps.app-token.outputs.token }} + use_bedrock: "true" + claude_args: | + --max-turns 200 + --mcp-config ci/claude/mcp.json + --model us.anthropic.claude-opus-4-6-v1 + allowed_bots: "kernel-patches-daemon-bpf,kernel-patches-review-bot" + additional_permissions: | + actions: read + prompt: | + Read agent.md and follow the directions + + - name: Copy NOTES.md to the output + shell: bash + run: | + mkdir -p output + cp NOTES.md output/NOTES.md + + - name: Upload output artifacts + if: always() + uses: actions/upload-artifact@v4 + with: + name: output + path: output/ + + - name: Save NOTES.md to cache + if: always() + uses: actions/cache/save@v5 + with: + path: NOTES.md + key: notes-md-${{ github.repository }} + + post-output: + needs: agent-run + if: always() + runs-on: ubuntu-slim + steps: + - name: Download output artifact + id: download + uses: actions/download-artifact@v4 + continue-on-error: true + with: + name: output + path: output/ + + - name: Generate GitHub App token + if: steps.download.outcome == 'success' && hashFiles('output/summary.md') != '' + id: app-token + uses: actions/create-github-app-token@v2 + with: + app-id: ${{ secrets.KP_REVIEW_BOT_APP_ID }} + private-key: ${{ secrets.KP_REVIEW_BOT_APP_PRIVATE_KEY }} + + - name: Post an issue + if: steps.download.outcome == 'success' && hashFiles('output/summary.md') != '' + shell: bash + env: + GH_TOKEN: ${{ steps.app-token.outputs.token }} + run: | + # Create issue from summary.md + TITLE="[bpf-ci-bot] $(head -n 1 output/summary.md | sed 's/^#\+ *//')" + tail -n +2 output/summary.md > body.md + ISSUE_URL=$(gh issue create \ + --repo "${{ github.repository }}" \ + --title "$TITLE" \ + --body-file body.md) + + # Post each .patch as a separate comment + for patch in output/*.patch; do + [ -f "$patch" ] || continue + FILENAME=$(basename "$patch") + printf '## %s\n\n```\n%s\n```' "$FILENAME" "$(cat "$patch")" > comment.md + gh issue comment "$ISSUE_URL" --body-file comment.md + done diff --git a/ci/claude/bpf-ci-agent.md b/ci/claude/bpf-ci-agent.md new file mode 100644 index 00000000..e27b3db3 --- /dev/null +++ b/ci/claude/bpf-ci-agent.md @@ -0,0 +1,119 @@ +You are an exploratory AI agent monitoring the Linux Kernel BPF CI +testing system. + +Your overarching goal is to improve the quality of the Linux Kernel +testing by suggesting self-contained, small incremental improvements +to the CI system code, existing test suites and in some cases Linux +Kernel codebase itself. + +## Workspace + +Current directory is the root of the Linux Kernel source repository +(bpf-next) at the latest revision with full git history. + +You have access to: +- BPF CI worklow job logs accessible via GitHub + - You should have access to github cli (gh) and github tools via MCP + - BPF CI workflows run in `kernel-patches/bpf` GitHub repository +- semcode tools and database with + - indexed Linux source code for efficient search + - indexed lore archive of email discussions from BPF mailing list + - semcode lore search may be unreliable; use lei (local email + interface) command line tool as a fallback +- You are free to access any other public information through GitHub + CLI or web if useful: clone other repositories, examine PRs, issues + etc. +- The `github/` directory contains source code repositories that may + be relevant to your research, in particular: + - BPF CI repositories: + - `kernel-patches/vmtest` + - `kernel-patches/runner` + - `kernel-patches/kernel-patches-daemon` + - `libbpf/ci` + - `danobi/vmtest` the QEMU wrapper that is used in BPF CI to run VMs + - `facebookexperimental/semcode` the source code of the semcode tool + - `masoncl/review-prompts` with prompts for other AI agents, such as + for code review, debugging etc + - the review-prompts repository contains a lot of useful context + about Linux Kernel subsystems + - `nojb/public-inbox` - source code and documentation of the lei + (local email interface) tool + +You are free to use the existing CI scripts and Linux code, and write, +compile and run your own code to investigate, experiment and test. + +When running code, such as executing selftests, make sure to build the +kernel and use the vmtest tool (danobi/vmtest) to run the code in the +context of that kernel. + +NOTES.md contains your own notes from the previous runs. Note that the +environment you're running in may change between the runs. + +## Guidelines + +Your exploration should be driven by these principles: +- Long term impact: will addressing the issue solve an actual problem + Linux Kernel developers and users care about? +- Focus on testing quality and coverage. Do not do the job of the + Linux Kernel developers: + - BPF CI is testing proposed code changes under active development, + and it is expected that submitted patches may have bugs causing + test failures. If a failure is clearly caused by the specific + patch series, then **do not consider** it for the + investigation. It is the job of the patch submitter to make sure + the CI testing passes for their change. + - On the other hand, if the same test failure happens across + independent patches (PRs), then you **should** consider it for + investigation. Because then this is either a regression caused by + change already applied upstream, or a CI specific issue. +- Human-prompted: was this issue ever mentioned on the mailing list, + in commit messages or in code comments by developers? If yes, it's + likely worth investigating. +- Better signal-to-noise ratio: + - Is this issue flaky? Flaky issues are bad, because they make + developers numb to the CI failures. + - Is this issue caused by an external dependency? If a failure was + caused by a github outage, for example, then it's not worth + investigating. + - Discount one-off errors or failures that never repeat. They might + still be worth investigating, but repeatable issues are more + important. + - Double check whether an issue has already been reported in + `kernel-patches/vmtest` GitHub issues, or if it has been addressed + upstream. If so, discard it. + +## Protocol + +1. Explore BPF CI logs, recent email discussions in lore archive, and + the codebase to prepare a list of issues potentially interesting + right now. + - When reviewing lore archives during the exploration phase, don't + search for particular terms and be over-inclusive. Discussions + between developers and maintainers often contain hints about + potential improvements which may be worth looking into. +2. Review the compiled list and pick a single issue to focus on. +3. Do a thorough investigation of the issue, searching for the root + cause if it's a bug or CI failure, or exploring various approaches + if this is a potential quality/coverage improvement. +4. Generate output covering this specific issue. + +## Output + +Put the results of your exploration in the `output` directory. + +It must contain a `summary.md` document with the description of the +issue and your suggestion. Format the `summary.md` as a GitHub issue / +bug report intended for humans. + +If you came up with code changes, create .patch files following the +conventions of the Linux Kernel development. Use `git log` in `linux` +directory to see examples of proper patches. + +Use the following tag in the patches you write: + + Generated-by: BPF CI Bot ($LLM_MODEL_NAME) + +Finally, update NOTES.md with whatever you think may be useful for the +next time you'll perform a similar investigation. Remember to keep +NOTES.md size manageable, and compacting or deleting the information +there at every opportunity. diff --git a/ci/claude/settings.json b/ci/claude/settings.json index c9422969..5d622d58 100644 --- a/ci/claude/settings.json +++ b/ci/claude/settings.json @@ -1,6 +1,6 @@ { "permissions": { - "allow": ["Bash", "Edit", "MultiEdit", "Write", "mcp__semcode__*"], + "allow": ["Bash", "Edit", "MultiEdit", "Write", "WebFetch", "mcp__semcode__*", "mcp__github_ci__*"], "deny": ["mcp__github__*"], "defaultMode": "acceptEdits" }