diff --git a/.github/workflows/ingest.yml b/.github/workflows/ingest.yml index 067d018d73..4eb7247531 100644 --- a/.github/workflows/ingest.yml +++ b/.github/workflows/ingest.yml @@ -7,14 +7,28 @@ on: - cron: "20 8 * * *" workflow_dispatch: -permissions: - contents: write - pull-requests: write +defaults: + run: + shell: bash -xeuo pipefail {0} + +concurrency: + group: ingest + +env: + HOMEBREW_DEVELOPER: 1 + HOMEBREW_NO_AUTO_UPDATE: 1 + HOMEBREW_NO_ENV_HINTS: 1 + +permissions: {} jobs: match: - if: github.ref == 'refs/heads/main' + name: Match advisory candidates + if: github.repository_owner == 'Homebrew' && github.ref == 'refs/heads/main' runs-on: ubuntu-latest + permissions: + contents: write # Push the bot-owned candidate branch. + pull-requests: write # Open a pull request for that branch. # Local Aug 2026 measurement with --no-history and the trimmed # data/repology.json (select_ecosystems + overrides.yml skip applied): # 90820 records in 126.4 min, peak RSS 1.68 GB. @@ -33,15 +47,15 @@ jobs: uses: Homebrew/actions/setup-ruby@fd832223f9f99ebf0244dd20658680e5d4aca049 # 2026.08.03.2 with: bundler-cache: true + portable-ruby: true - name: Runner diagnostics run: free -m && df -h - name: Match advisories + # Step-scoped: setup-homebrew unsets this so brew install can use the API. env: - HOMEBREW_NO_AUTO_UPDATE: "1" - HOMEBREW_NO_INSTALL_FROM_API: "1" - HOMEBREW_DEVELOPER: "1" + HOMEBREW_NO_INSTALL_FROM_API: 1 # --no-history: the FormulaVersions walk for every :fixed hit across # ~8500 formulae would exceed the job limit. New records get # `fixed: ` which reviewers tighten per @@ -54,26 +68,52 @@ jobs: - name: Concatenate advisories run: bundle exec rake advisories:concat - - name: Open pull request + - name: Configure git + uses: Homebrew/actions/git-user-config@fd832223f9f99ebf0244dd20658680e5d4aca049 # 2026.08.03.2 + with: + username: github-actions[bot] + + - name: Commit advisories + id: commit env: - GITHUB_TOKEN: ${{ github.token }} BRANCH: matched-advisories run: | - git config user.name "github-actions[bot]" - git config user.email "41898282+github-actions[bot]@users.noreply.github.com" git add advisories/ data/ if git diff --cached --quiet; then echo "No changes." exit 0 fi - gh auth setup-git --hostname github.com --force - git checkout -b "$BRANCH" + git checkout -b "${BRANCH}" git commit -m "Matched advisory candidates" - git push --force origin "HEAD:refs/heads/$BRANCH" - if [ -n "$(gh pr list --head "$BRANCH" --state open --json number --jq '.[].number')" ]; then + { + echo "branch=${BRANCH}" + echo "committed=true" + } >> "${GITHUB_OUTPUT}" + + - name: Push commits + if: steps.commit.outputs.committed == 'true' + uses: Homebrew/actions/git-try-push@fd832223f9f99ebf0244dd20658680e5d4aca049 # 2026.08.03.2 + with: + token: ${{ github.token }} + branch: ${{ steps.commit.outputs.branch }} + # No lease: rebuilt from main each run, and no remote-tracking ref exists. + force: true + no_lease: true + origin_branch: HEAD + + - name: Open pull request + if: steps.commit.outputs.committed == 'true' + env: + GITHUB_TOKEN: ${{ github.token }} + BRANCH: ${{ steps.commit.outputs.branch }} + run: | + open_pr_number="$(gh pr list --head "${BRANCH}" --state open \ + --json number,isCrossRepository \ + --jq '.[] | select(.isCrossRepository | not) | .number')" + if [[ -n "${open_pr_number}" ]]; then echo "PR already open." else - gh pr create --base main --head "$BRANCH" \ + gh pr create --base main --head "${BRANCH}" \ --title "Matched advisory candidates" \ --body "Automated candidates via \`brew advisory-match --all\`. See [CONTRIBUTING.md](https://github.com/${GITHUB_REPOSITORY}/blob/HEAD/CONTRIBUTING.md#reviewing-matched-candidates) for the review checklist." fi diff --git a/.github/workflows/regenerate.yml b/.github/workflows/regenerate.yml index de70e5940b..70ff623f7c 100644 --- a/.github/workflows/regenerate.yml +++ b/.github/workflows/regenerate.yml @@ -5,14 +5,29 @@ on: - cron: "20 6 * * *" workflow_dispatch: -permissions: - contents: write - pull-requests: write +defaults: + run: + shell: bash -xeuo pipefail {0} + +concurrency: + group: regenerate + +env: + HOMEBREW_DEVELOPER: 1 + HOMEBREW_NO_AUTO_UPDATE: 1 + HOMEBREW_NO_ENV_HINTS: 1 + +permissions: {} jobs: regenerate: - if: github.ref == 'refs/heads/main' + name: Regenerate advisories + if: github.repository_owner == 'Homebrew' && github.ref == 'refs/heads/main' runs-on: macos-latest + timeout-minutes: 180 + permissions: + contents: write # Push the bot-owned regeneration branch. + pull-requests: write # Open a pull request for that branch. steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: @@ -25,6 +40,7 @@ jobs: uses: Homebrew/actions/setup-ruby@fd832223f9f99ebf0244dd20658680e5d4aca049 # 2026.08.03.2 with: bundler-cache: true + portable-ruby: true - name: Run tests run: bundle exec rake spec @@ -33,35 +49,60 @@ jobs: run: bundle exec rake repology:build - name: Regenerate advisories + # Step-scoped: setup-homebrew unsets this so brew install can use the API. env: - HOMEBREW_NO_AUTO_UPDATE: "1" - HOMEBREW_NO_INSTALL_FROM_API: "1" - HOMEBREW_DEVELOPER: "1" + HOMEBREW_NO_INSTALL_FROM_API: 1 run: brew generate-vulns-advisories advisories - name: Concatenate advisories run: bundle exec rake advisories:concat - - name: Open pull request + - name: Configure git + uses: Homebrew/actions/git-user-config@fd832223f9f99ebf0244dd20658680e5d4aca049 # 2026.08.03.2 + with: + username: github-actions[bot] + + - name: Commit advisories + id: commit env: - GITHUB_TOKEN: ${{ github.token }} BRANCH: regenerate-advisories run: | - git config user.name "github-actions[bot]" - git config user.email "41898282+github-actions[bot]@users.noreply.github.com" git add advisories/ data/ if git diff --cached --quiet; then echo "No changes." exit 0 fi - gh auth setup-git --hostname github.com --force - git checkout -b "$BRANCH" + git checkout -b "${BRANCH}" git commit -m "Regenerate advisories" - git push --force origin "HEAD:refs/heads/$BRANCH" - if [ -n "$(gh pr list --head "$BRANCH" --state open --json number --jq '.[].number')" ]; then + { + echo "branch=${BRANCH}" + echo "committed=true" + } >> "${GITHUB_OUTPUT}" + + - name: Push commits + if: steps.commit.outputs.committed == 'true' + uses: Homebrew/actions/git-try-push@fd832223f9f99ebf0244dd20658680e5d4aca049 # 2026.08.03.2 + with: + token: ${{ github.token }} + branch: ${{ steps.commit.outputs.branch }} + # No lease: rebuilt from main each run, and no remote-tracking ref exists. + force: true + no_lease: true + origin_branch: HEAD + + - name: Open pull request + if: steps.commit.outputs.committed == 'true' + env: + GITHUB_TOKEN: ${{ github.token }} + BRANCH: ${{ steps.commit.outputs.branch }} + run: | + open_pr_number="$(gh pr list --head "${BRANCH}" --state open \ + --json number,isCrossRepository \ + --jq '.[] | select(.isCrossRepository | not) | .number')" + if [[ -n "${open_pr_number}" ]]; then echo "PR already open." else - gh pr create --base main --head "$BRANCH" \ + gh pr create --base main --head "${BRANCH}" \ --title "Regenerate advisories" \ --body "Automated regeneration via \`brew generate-vulns-advisories\`." fi diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index d06fc401c3..068cc9ddde 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -2,34 +2,70 @@ name: Validate on: push: + branches: + - main pull_request: + merge_group: -permissions: - contents: read +defaults: + run: + shell: bash -xeuo pipefail {0} + +concurrency: + group: "validate-${{ github.ref }}" + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +permissions: {} jobs: schema: + name: Validate advisory schema + if: github.repository_owner == 'Homebrew' runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: read steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + - name: Set up Homebrew + id: set-up-homebrew + uses: Homebrew/actions/setup-homebrew@fd832223f9f99ebf0244dd20658680e5d4aca049 # 2026.08.03.2 with: - python-version: "3.x" + core: false + cask: false - name: Install check-jsonschema - run: pip install check-jsonschema + uses: Homebrew/actions/cache-homebrew-prefix@fd832223f9f99ebf0244dd20658680e5d4aca049 # 2026.08.03.2 + with: + install: check-jsonschema + workflow-key: validate-schema - name: Fetch OSV schema - run: curl -sSfL -o osv-schema.json https://raw.githubusercontent.com/ossf/osv-schema/main/validation/schema.json + # OSV schema v1.9.0, which includes the Homebrew ecosystem and BREW- id registrations. + run: >- + curl --fail --location --retry 3 --show-error --silent + --output "${RUNNER_TEMP}/osv-schema.json" + https://raw.githubusercontent.com/ossf/osv-schema/f3f826310aeca8e324baabd195632f2229952abe/validation/schema.json - name: Validate advisories against OSV schema - run: check-jsonschema --schemafile osv-schema.json advisories/*.json + run: | + if ! find advisories -type f -name '*.json' -print -quit | grep -q .; then + echo "No advisory files found." + exit 1 + fi + find advisories -type f -name '*.json' -print0 | + xargs -0 -r -n 500 check-jsonschema --schemafile "${RUNNER_TEMP}/osv-schema.json" test: + name: Test Ruby code + if: github.repository_owner == 'Homebrew' runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: read steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: @@ -42,11 +78,7 @@ jobs: uses: Homebrew/actions/setup-ruby@fd832223f9f99ebf0244dd20658680e5d4aca049 # 2026.08.03.2 with: bundler-cache: true + portable-ruby: true - - name: Run tests - run: bundle exec rake spec - - # RuboCop is not run: the org-synced .rubocop.yml sets - # `AllCops: Include: ["**/*.rbi"]` which replaces (not extends) the - # default .rb globs, so it scans nothing in this repository. Fix belongs - # in the Homebrew/.github sync source. + - name: Run tests and style checks + run: bundle exec rake diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 3aaa84d272..f78ea502f8 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -62,6 +62,23 @@ Do not merge a `matched` record over an existing `source: "generated"` record fo Records here describe vulnerabilities in software Homebrew distributes, scoped to the Homebrew formula name and version. They are not a substitute for the upstream project's own advisory; the purpose is to let tools that read `pkg:brew` purls or `Homebrew` OSV queries answer "is this installed formula affected". Casks are out of scope for now. +## Development + +The required Ruby version is recorded in `.ruby-version`. Install the development dependencies and run the same Ruby checks as CI with: + +```sh +bundle install +bundle exec rake +``` + +Workflow changes should also pass Homebrew's standard workflow checks: + +```sh +brew install actionlint shellcheck zizmor +actionlint +zizmor . +``` + ## Code of Conduct This project follows the [Homebrew Code of Conduct](https://github.com/Homebrew/.github/blob/HEAD/CODE_OF_CONDUCT.md). diff --git a/Gemfile b/Gemfile index 6eba972731..0514068e35 100644 --- a/Gemfile +++ b/Gemfile @@ -2,5 +2,8 @@ source "https://rubygems.org" +ruby file: ".ruby-version" + gem "rake" gem "rspec" +gem "rubocop" diff --git a/Gemfile.lock b/Gemfile.lock index dd12279726..4bf6bfe481 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,8 +1,20 @@ GEM remote: https://rubygems.org/ specs: + ast (2.4.3) diff-lcs (1.6.2) + json (2.21.2) + language_server-protocol (3.17.0.6) + lint_roller (1.1.0) + parallel (2.1.0) + parser (3.3.12.0) + ast (~> 2.4.1) + racc + prism (1.9.0) + racc (1.8.1) + rainbow (3.1.1) rake (13.4.2) + regexp_parser (2.12.0) rspec (3.13.2) rspec-core (~> 3.13.0) rspec-expectations (~> 3.13.0) @@ -16,6 +28,24 @@ GEM diff-lcs (>= 1.2.0, < 2.0) rspec-support (~> 3.13.0) rspec-support (3.13.7) + rubocop (1.89.0) + json (~> 2.3) + language_server-protocol (~> 3.17.0.2) + lint_roller (~> 1.1.0) + parallel (>= 1.10) + parser (>= 3.3.0.2) + rainbow (>= 2.2.2, < 4.0) + regexp_parser (>= 2.9.3, < 3.0) + rubocop-ast (>= 1.49.0, < 2.0) + ruby-progressbar (~> 1.7) + unicode-display_width (>= 2.4.0, < 4.0) + rubocop-ast (1.50.0) + parser (>= 3.3.7.2) + prism (~> 1.7) + ruby-progressbar (1.13.0) + unicode-display_width (3.2.0) + unicode-emoji (~> 4.1) + unicode-emoji (4.2.0) PLATFORMS aarch64-linux-gnu @@ -26,15 +56,34 @@ PLATFORMS DEPENDENCIES rake rspec + rubocop CHECKSUMS + ast (2.4.3) sha256=954615157c1d6a382bc27d690d973195e79db7f55e9765ac7c481c60bdb4d383 diff-lcs (1.6.2) sha256=9ae0d2cba7d4df3075fe8cd8602a8604993efc0dfa934cff568969efb1909962 + json (2.21.2) sha256=1f1d3b7cf2b3ba1a69beca0bb6db13d5438b80bff3cd54cdaaa620b9b07c1c6a + language_server-protocol (3.17.0.6) sha256=5ef2c0c138f8267e1bc631d3328347d354f96724b0af22f2c79516120443b7f0 + lint_roller (1.1.0) sha256=2c0c845b632a7d172cb849cc90c1bce937a28c5c8ccccb50dfd46a485003cc87 + parallel (2.1.0) sha256=b35258865c2e31134c5ecb708beaaf6772adf9d5efae28e93e99260877b09356 + parser (3.3.12.0) sha256=21a6d7f755d5a24dfbdc6e6b772e4e879a52e7631a88bc5a3a134606052c9828 + prism (1.9.0) sha256=7b530c6a9f92c24300014919c9dcbc055bf4cdf51ec30aed099b06cd6674ef85 + racc (1.8.1) sha256=4a7f6929691dbec8b5209a0b373bc2614882b55fc5d2e447a21aaa691303d62f + rainbow (3.1.1) sha256=039491aa3a89f42efa1d6dec2fc4e62ede96eb6acd95e52f1ad581182b79bc6a rake (13.4.2) sha256=cb825b2bd5f1f8e91ca37bddb4b9aaf345551b4731da62949be002fa89283701 + regexp_parser (2.12.0) sha256=35a916a1d63190ab5c9009457136ae5f3c0c7512d60291d0d1378ba18ce08ebb rspec (3.13.2) sha256=206284a08ad798e61f86d7ca3e376718d52c0bc944626b2349266f239f820587 rspec-core (3.13.6) sha256=a8823c6411667b60a8bca135364351dda34cd55e44ff94c4be4633b37d828b2d rspec-expectations (3.13.5) sha256=33a4d3a1d95060aea4c94e9f237030a8f9eae5615e9bd85718fe3a09e4b58836 rspec-mocks (3.13.8) sha256=086ad3d3d17533f4237643de0b5c42f04b66348c28bf6b9c2d3f4a3b01af1d47 rspec-support (3.13.7) sha256=0640e5570872aafefd79867901deeeeb40b0c9875a36b983d85f54fb7381c47c + rubocop (1.89.0) sha256=4dee8e3ee9c45e474834efd9e8d6fd031e8331c8dacdff0de4ad65ae0a6faae7 + rubocop-ast (1.50.0) sha256=b9ca88300da0803ee222ad20cdb30494c0a784eed06fdc35d254b06d662788db + ruby-progressbar (1.13.0) sha256=80fc9c47a9b640d6834e0dc7b3c94c9df37f08cb072b7761e4a71e22cff29b33 + unicode-display_width (3.2.0) sha256=0cdd96b5681a5949cdbc2c55e7b420facae74c4aaf9a9815eee1087cb1853c42 + unicode-emoji (4.2.0) sha256=519e69150f75652e40bf736106cfbc8f0f73aa3fb6a65afe62fefa7f80b0f80f + +RUBY VERSION + ruby 4.0.6 BUNDLED WITH 4.0.16 diff --git a/Rakefile b/Rakefile index e22f87fbd5..d396eabd3f 100644 --- a/Rakefile +++ b/Rakefile @@ -1,10 +1,15 @@ # frozen_string_literal: true require "rspec/core/rake_task" +require "rubocop/rake_task" RSpec::Core::RakeTask.new(:spec) -task default: :spec +RuboCop::RakeTask.new(:rubocop) do |task| + task.patterns = ["Gemfile", "Rakefile", "lib/**/*.rb", "spec/**/*.rb"] +end + +task default: [:rubocop, :spec] namespace :repology do desc "Build data/repology.json from the Repology API" diff --git a/lib/advisory_index.rb b/lib/advisory_index.rb index b23a61aaef..8ee3063da3 100644 --- a/lib/advisory_index.rb +++ b/lib/advisory_index.rb @@ -25,7 +25,7 @@ def build(dir) schema_versions = [] skipped = 0 - Dir.glob(File.join(dir, "*.json")).sort.each do |path| + Dir.glob(File.join(dir, "*.json")).each do |path| record = JSON.parse(File.read(path)) name = record.dig("affected", 0, "package", "name") raise "#{path}: missing affected[0].package.name" unless name diff --git a/lib/repology_index.rb b/lib/repology_index.rb index 921461031b..15cff12b44 100644 --- a/lib/repology_index.rb +++ b/lib/repology_index.rb @@ -71,7 +71,7 @@ def self.select_ecosystems(distros) fallback ? { fallback => distros[fallback] } : {} end - OVERRIDES_PATH = File.expand_path("../data/overrides.yml", __dir__) + OVERRIDES_PATH = File.expand_path("../data/overrides.yml", __dir__).freeze # Formula names with `skip: true` in `data/overrides.yml`. Dropped from the # index so `Homebrew::Vulns::Match` never issues distro queries for them. diff --git a/spec/repology_index_spec.rb b/spec/repology_index_spec.rb index 0225ef9097..125a6afaaa 100644 --- a/spec/repology_index_spec.rb +++ b/spec/repology_index_spec.rb @@ -358,14 +358,19 @@ def full_page(base) path = File.join(dir, "repology.json") overrides = File.join(dir, "overrides.yml") File.write(overrides, { "linux-headers@5.15" => { "skip" => true } }.to_yaml) - File.write(path, JSON.generate( + data = { "meta" => { "osv_distros" => %w[Debian Alpine openSUSE FreeBSD] }, "formulae" => { - "curl" => { "Debian" => ["curl"], "Alpine" => ["curl"], "openSUSE" => ["curl"] }, + "curl" => { + "Debian" => ["curl"], + "Alpine" => ["curl"], + "openSUSE" => ["curl"], + }, "cassandra" => { "FreeBSD" => ["cassandra3"], "openSUSE" => ["cassandra"] }, "linux-headers@5.15" => { "Debian" => ["linux"], "Ubuntu" => ["linux"] }, }, - )) + } + File.write(path, JSON.generate(data)) result = described_class.trim(path, overrides_path: overrides) expect(result["formulae"]).to eq( "cassandra" => { "openSUSE" => ["cassandra"] }, diff --git a/spec/workflows_spec.rb b/spec/workflows_spec.rb index 739fc16e8c..29e1152f6d 100644 --- a/spec/workflows_spec.rb +++ b/spec/workflows_spec.rb @@ -13,4 +13,17 @@ expect(concat_step.fetch("run")).to eq "bundle exec rake advisories:concat" end end + + it "validates advisories in bounded batches" do + path = File.expand_path("../.github/workflows/validate.yml", __dir__) + workflow = YAML.safe_load_file(path, aliases: true) + steps = workflow.dig("jobs", "schema", "steps") + validation = steps.find { |step| step["name"] == "Validate advisories against OSV schema" } + run = validation.fetch("run") + + expect(run).to match(/find advisories [^\n]*-print0/) + expect(run).to match(/xargs -0 [^\n]*-n [1-9]\d* check-jsonschema/) + expect(run).to include("--schemafile") + expect(run).not_to include("advisories/*.json") + end end