diff --git a/.github/scripts/addZoektSyncChangelogEntry.sh b/.github/scripts/addZoektSyncChangelogEntry.sh new file mode 100755 index 000000000..85be5f9ac --- /dev/null +++ b/.github/scripts/addZoektSyncChangelogEntry.sh @@ -0,0 +1,127 @@ +#!/usr/bin/env bash + +set -euo pipefail + +pr_number="${1:-}" +changelog="${CHANGELOG_PATH:-CHANGELOG.md}" + +if [[ ! "$pr_number" =~ ^[1-9][0-9]*$ ]]; then + echo "Expected a pull request number, got: $pr_number" >&2 + exit 1 +fi + +pr_url="https://github.com/sourcebot-dev/sourcebot/pull/$pr_number" +entry="- Updated the bundled Zoekt version. [#$pr_number]($pr_url)" + +if grep -Fq "$pr_url" "$changelog"; then + echo "Changelog already links to Sourcebot PR #$pr_number." + exit 0 +fi + +changelog_directory=$(dirname "$changelog") +changelog_basename=$(basename "$changelog") +temporary_file=$(mktemp "$changelog_directory/.${changelog_basename}.XXXXXX") +trap 'rm -f "$temporary_file"' EXIT + +if changelog_mode=$(stat -f '%Lp' "$changelog" 2> /dev/null); then + : +else + changelog_mode=$(stat -c '%a' "$changelog") +fi + +awk -v entry="$entry" ' + function flush_changed_section( last, i) { + last = changed_line_count + while (last > 0 && changed_lines[last] == "") { + last-- + } + for (i = 1; i <= last; i++) { + print changed_lines[i] + } + print entry + print "" + changed_line_count = 0 + } + + $0 == "## [Unreleased]" { + in_unreleased = 1 + last_was_blank = 0 + print + next + } + + in_unreleased && $0 == "### Changed" { + found_changed = 1 + in_changed = 1 + print + next + } + + in_changed && /^##(#)? / { + flush_changed_section() + in_changed = 0 + inserted = 1 + if ($0 ~ /^## /) { + in_unreleased = 0 + } + print + next + } + + in_changed { + changed_lines[++changed_line_count] = $0 + next + } + + in_unreleased && !found_changed && /^### (Deprecated|Removed|Fixed|Security)$/ { + print "### Changed" + print entry + print "" + found_changed = 1 + inserted = 1 + print + next + } + + in_unreleased && !found_changed && /^## / { + print "### Changed" + print entry + print "" + found_changed = 1 + inserted = 1 + in_unreleased = 0 + print + next + } + + { + last_was_blank = ($0 == "") + print + } + + END { + if (in_changed) { + flush_changed_section() + inserted = 1 + } else if (in_unreleased && !found_changed) { + if (!last_was_blank) { + print "" + } + print "### Changed" + print entry + found_changed = 1 + inserted = 1 + } + if (!found_changed || !inserted) { + exit 2 + } + } +' "$changelog" > "$temporary_file" || { + echo "Unable to add an Unreleased Changed entry to $changelog." >&2 + exit 1 +} + +chmod "$changelog_mode" "$temporary_file" +mv "$temporary_file" "$changelog" +trap - EXIT +echo "Added the changelog entry for Sourcebot PR #$pr_number." diff --git a/.github/scripts/testZoektSync.sh b/.github/scripts/testZoektSync.sh new file mode 100755 index 000000000..1170463cb --- /dev/null +++ b/.github/scripts/testZoektSync.sh @@ -0,0 +1,198 @@ +#!/usr/bin/env bash + +set -euo pipefail + +script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +repository_root=$(cd "$script_dir/../.." && pwd) +update_script="$script_dir/updateZoektSubmodule.sh" +changelog_script="$script_dir/addZoektSyncChangelogEntry.sh" +workflow="$repository_root/.github/workflows/syncZoekt.yml" + +fail() { + echo "FAIL: $*" >&2 + exit 1 +} + +assert_contains() { + local file=$1 + local expected=$2 + local description=$3 + + if ! grep -Fq -- "$expected" "$file"; then + fail "$description" + fi +} + +assert_equals() { + local actual=$1 + local expected=$2 + local description=$3 + + if [[ "$actual" != "$expected" ]]; then + fail "$description (expected $expected, got $actual)" + fi +} + +test_root=$(mktemp -d) +trap 'rm -rf "$test_root"' EXIT + +zoekt_remote="$test_root/zoekt.git" +zoekt_upstream="$test_root/zoekt-upstream" +sourcebot_test="$test_root/sourcebot" + +git init --quiet --bare --initial-branch=main "$zoekt_remote" +git init --quiet --initial-branch=main "$zoekt_upstream" +git -C "$zoekt_upstream" config user.name "Zoekt Sync Test" +git -C "$zoekt_upstream" config user.email "zoekt-sync-test@example.com" +git -C "$zoekt_upstream" remote add origin "$zoekt_remote" + +printf '%s\n' first > "$zoekt_upstream/version.txt" +git -C "$zoekt_upstream" add version.txt +git -C "$zoekt_upstream" commit --quiet -m "first" +first_sha=$(git -C "$zoekt_upstream" rev-parse HEAD) +git -C "$zoekt_upstream" push --quiet --set-upstream origin main + +printf '%s\n' second > "$zoekt_upstream/version.txt" +git -C "$zoekt_upstream" commit --quiet -am "second" +second_sha=$(git -C "$zoekt_upstream" rev-parse HEAD) +git -C "$zoekt_upstream" push --quiet origin main + +git init --quiet --initial-branch=main "$sourcebot_test" +git -C "$sourcebot_test" config user.name "Zoekt Sync Test" +git -C "$sourcebot_test" config user.email "zoekt-sync-test@example.com" +git -C "$sourcebot_test" -c protocol.file.allow=always \ + submodule add --quiet "$zoekt_remote" vendor/zoekt +git -C "$sourcebot_test/vendor/zoekt" checkout --quiet --detach "$first_sha" +git -C "$sourcebot_test" add vendor/zoekt +git -C "$sourcebot_test" commit --quiet -m "pin first Zoekt commit" + +( + cd "$sourcebot_test" + "$update_script" "$second_sha" +) +staged_sha=$(git -C "$sourcebot_test" rev-parse :vendor/zoekt) +assert_equals "$staged_sha" "$second_sha" \ + "the updater should stage the requested main-branch commit" +git -C "$sourcebot_test" commit --quiet -m "advance Zoekt" + +( + cd "$sourcebot_test" + "$update_script" "$first_sha" +) +current_sha=$(git -C "$sourcebot_test" rev-parse HEAD:vendor/zoekt) +assert_equals "$current_sha" "$second_sha" \ + "a stale event must not downgrade the Zoekt gitlink" +git -C "$sourcebot_test" diff --quiet || \ + fail "a stale event should leave the worktree unchanged" +git -C "$sourcebot_test" diff --cached --quiet || \ + fail "a stale event should leave the index unchanged" + +git -C "$zoekt_upstream" switch --quiet --detach "$first_sha" +git -C "$zoekt_upstream" switch --quiet -c divergent +printf '%s\n' divergent > "$zoekt_upstream/version.txt" +git -C "$zoekt_upstream" commit --quiet -am "divergent" +divergent_sha=$(git -C "$zoekt_upstream" rev-parse HEAD) +git -C "$zoekt_upstream" push --quiet origin divergent +git -C "$sourcebot_test/vendor/zoekt" fetch --quiet origin divergent + +if ( + cd "$sourcebot_test" + "$update_script" "$divergent_sha" +) 2> "$test_root/divergent-error.txt"; then + fail "the updater should reject a commit outside origin/main" +fi +assert_contains "$test_root/divergent-error.txt" \ + "is not reachable from origin/main" \ + "the non-main error should explain the rejected target" + +git -C "$sourcebot_test/vendor/zoekt" checkout --quiet --detach "$divergent_sha" +git -C "$sourcebot_test" add vendor/zoekt +git -C "$sourcebot_test" commit --quiet -m "pin divergent Zoekt commit" + +if ( + cd "$sourcebot_test" + "$update_script" "$second_sha" +) 2> "$test_root/divergent-history-error.txt"; then + fail "the updater should reject divergent current and target commits" +fi +assert_contains "$test_root/divergent-history-error.txt" \ + "Refusing to move Zoekt between divergent histories" \ + "the divergent-history error should explain the rejected update" + +if ( + cd "$sourcebot_test" + "$update_script" deadbeef +) 2> "$test_root/invalid-error.txt"; then + fail "the updater should reject abbreviated commit SHAs" +fi +assert_contains "$test_root/invalid-error.txt" \ + "Expected a full lowercase Zoekt commit SHA" \ + "the invalid-SHA error should explain the required format" + +changelog_fixture="$test_root/CHANGELOG.md" +cat > "$changelog_fixture" <<'EOF' +# Changelog + +## [Unreleased] + +### Added +- Added something. + +### Removed +- Removed something. + +### Fixed +- Fixed something. + +## [1.0.0] +EOF + +chmod 640 "$changelog_fixture" +CHANGELOG_PATH="$changelog_fixture" "$changelog_script" 42 +assert_contains "$changelog_fixture" \ + "### Changed" \ + "the changelog helper should create the Changed section when absent" +assert_contains "$changelog_fixture" \ + "- Updated the bundled Zoekt version. [#42](https://github.com/sourcebot-dev/sourcebot/pull/42)" \ + "the changelog helper should add the generated PR link" +CHANGELOG_PATH="$changelog_fixture" "$changelog_script" 42 +entry_count=$(grep -Fc "sourcebot/pull/42" "$changelog_fixture") +assert_equals "$entry_count" 1 \ + "the changelog helper should not duplicate an existing PR entry" +if changelog_mode=$(stat -f '%Lp' "$changelog_fixture" 2> /dev/null); then + : +else + changelog_mode=$(stat -c '%a' "$changelog_fixture") +fi +assert_equals "$changelog_mode" 640 \ + "the changelog helper should preserve the target file mode" + +empty_changelog_fixture="$test_root/EMPTY_CHANGELOG.md" +cat > "$empty_changelog_fixture" <<'EOF' +# Changelog + +## [Unreleased] +EOF +CHANGELOG_PATH="$empty_changelog_fixture" "$changelog_script" 43 +assert_contains "$empty_changelog_fixture" \ + "- Updated the bundled Zoekt version. [#43](https://github.com/sourcebot-dev/sourcebot/pull/43)" \ + "the changelog helper should handle an empty Unreleased section at EOF" + +ruby -e 'require "yaml"; YAML.parse_file(ARGV.fetch(0))' "$workflow" +assert_contains "$workflow" \ + "types: [zoekt-pr-merged]" \ + "the receiver should handle only Zoekt merge dispatches" +assert_contains "$workflow" \ + 'uses: actions/create-github-app-token@v2' \ + "the receiver should use a GitHub App token" +assert_contains "$workflow" \ + 'permission-pull-requests: write' \ + "the receiver should restrict its app token to required permissions" +assert_contains "$workflow" \ + '--force-with-lease=' \ + "the receiver should protect updates to its stable automation branch" +assert_contains "$workflow" \ + '.github/scripts/updateZoektSubmodule.sh "$ZOEKT_SHA"' \ + "the receiver should validate and stage the requested Zoekt commit" + +echo "All Zoekt sync tests passed." diff --git a/.github/scripts/updateZoektSubmodule.sh b/.github/scripts/updateZoektSubmodule.sh new file mode 100755 index 000000000..d29218072 --- /dev/null +++ b/.github/scripts/updateZoektSubmodule.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash + +set -euo pipefail + +target_sha="${1:-}" +submodule_path="${ZOEKT_SUBMODULE_PATH:-vendor/zoekt}" +remote="${ZOEKT_REMOTE:-origin}" +branch="${ZOEKT_BRANCH:-main}" + +if [[ ! "$target_sha" =~ ^[0-9a-f]{40}$ ]]; then + echo "Expected a full lowercase Zoekt commit SHA, got: $target_sha" >&2 + exit 1 +fi + +repository_root=$(git rev-parse --show-toplevel) +cd "$repository_root" + +if ! current_sha=$(git rev-parse "HEAD:$submodule_path"); then + echo "Unable to read the $submodule_path gitlink from HEAD." >&2 + exit 1 +fi + +git -C "$submodule_path" fetch --quiet "$remote" "$branch" +remote_head=$(git -C "$submodule_path" rev-parse FETCH_HEAD) + +if ! git -C "$submodule_path" cat-file -e "$target_sha^{commit}"; then + echo "Zoekt commit $target_sha does not exist." >&2 + exit 1 +fi + +if ! git -C "$submodule_path" merge-base --is-ancestor "$target_sha" "$remote_head"; then + echo "Zoekt commit $target_sha is not reachable from $remote/$branch." >&2 + exit 1 +fi + +if git -C "$submodule_path" merge-base --is-ancestor "$target_sha" "$current_sha"; then + echo "Zoekt is already at or ahead of $target_sha." + exit 0 +fi + +if ! git -C "$submodule_path" merge-base --is-ancestor "$current_sha" "$target_sha"; then + echo "Refusing to move Zoekt between divergent histories: $current_sha -> $target_sha." >&2 + exit 1 +fi + +git -C "$submodule_path" checkout --quiet --detach "$target_sha" +git add -- "$submodule_path" +echo "Staged Zoekt update: $current_sha -> $target_sha" diff --git a/.github/workflows/syncZoekt.yml b/.github/workflows/syncZoekt.yml new file mode 100644 index 000000000..3637be25e --- /dev/null +++ b/.github/workflows/syncZoekt.yml @@ -0,0 +1,229 @@ +name: Sync Zoekt Submodule + +on: + repository_dispatch: + types: [zoekt-pr-merged] + workflow_dispatch: + inputs: + zoekt_sha: + description: Full Zoekt commit SHA to sync. + required: true + type: string + zoekt_pr_number: + description: Merged Zoekt pull request number associated with the SHA. + required: true + type: number + +concurrency: + group: sync-zoekt-submodule + cancel-in-progress: false + +permissions: + contents: read + +jobs: + sync: + runs-on: ubuntu-latest + env: + AUTOMATION_BRANCH: automation/sync-zoekt + ZOEKT_SHA: ${{ github.event.client_payload.zoekt_sha || inputs.zoekt_sha }} + ZOEKT_PR_NUMBER: ${{ github.event.client_payload.zoekt_pr_number || inputs.zoekt_pr_number }} + steps: + - name: Generate GitHub App token + id: app-token + uses: actions/create-github-app-token@v2 + with: + app-id: ${{ secrets.SOURCEBOT_SYNC_APP_ID }} + private-key: ${{ secrets.SOURCEBOT_SYNC_APP_PRIVATE_KEY }} + owner: sourcebot-dev + repositories: sourcebot + permission-contents: write + permission-pull-requests: write + + - name: Checkout Sourcebot + uses: actions/checkout@v4 + with: + ref: main + fetch-depth: 0 + persist-credentials: false + + - name: Initialize public Zoekt submodule + run: git submodule update --init --recursive vendor/zoekt + + - name: Configure automation git identity + run: | + git config user.name "sourcebot-sync[bot]" + git config user.email "sourcebot-sync[bot]@users.noreply.github.com" + + - name: Validate dispatch payload + run: | + set -euo pipefail + + if [[ ! "$ZOEKT_SHA" =~ ^[0-9a-f]{40}$ ]]; then + echo "Expected a full lowercase Zoekt commit SHA, got: $ZOEKT_SHA" >&2 + exit 1 + fi + if [[ ! "$ZOEKT_PR_NUMBER" =~ ^[1-9][0-9]*$ ]]; then + echo "Expected a Zoekt pull request number, got: $ZOEKT_PR_NUMBER" >&2 + exit 1 + fi + + - name: Prepare automation branch + id: branch + env: + GH_TOKEN: ${{ steps.app-token.outputs.token }} + run: | + set -euo pipefail + + open_pr_url=$(gh pr list \ + --repo "$GITHUB_REPOSITORY" \ + --state open \ + --head "$AUTOMATION_BRANCH" \ + --json url \ + --jq '.[0].url // ""') + remote_sha=$(git ls-remote \ + --heads origin "refs/heads/$AUTOMATION_BRANCH" \ + | awk '{print $1}') + branch_changed=false + + if [[ -n "$open_pr_url" ]]; then + if [[ -z "$remote_sha" ]]; then + echo "Open sync PR $open_pr_url has no remote branch." >&2 + exit 1 + fi + git fetch origin \ + "refs/heads/$AUTOMATION_BRANCH:refs/remotes/origin/$AUTOMATION_BRANCH" + git switch -C "$AUTOMATION_BRANCH" "origin/$AUTOMATION_BRANCH" + git rebase origin/main + if [[ "$(git rev-parse HEAD)" != "$remote_sha" ]]; then + branch_changed=true + fi + else + git switch -c "$AUTOMATION_BRANCH" + fi + + { + echo "open_pr_url=$open_pr_url" + echo "remote_sha=$remote_sha" + echo "changed=$branch_changed" + } >> "$GITHUB_OUTPUT" + + - name: Update Zoekt submodule + id: update + run: | + set -euo pipefail + + current_sha=$(git rev-parse HEAD:vendor/zoekt) + .github/scripts/updateZoektSubmodule.sh "$ZOEKT_SHA" + if git diff --cached --quiet -- vendor/zoekt; then + { + echo "changed=false" + if [[ "$current_sha" == "$ZOEKT_SHA" ]]; then + echo "target_applied=true" + else + echo "target_applied=false" + fi + } >> "$GITHUB_OUTPUT" + echo "Sourcebot already contains Zoekt $ZOEKT_SHA." >> "$GITHUB_STEP_SUMMARY" + exit 0 + fi + + git commit -m "chore: sync zoekt after upstream PR #$ZOEKT_PR_NUMBER" + { + echo "changed=true" + echo "target_applied=true" + } >> "$GITHUB_OUTPUT" + + - name: Push automation branch + if: steps.update.outputs.changed == 'true' || steps.branch.outputs.changed == 'true' + env: + GH_TOKEN: ${{ steps.app-token.outputs.token }} + REMOTE_SHA: ${{ steps.branch.outputs.remote_sha }} + run: | + set -euo pipefail + + gh auth setup-git + if [[ -n "$REMOTE_SHA" ]]; then + git push \ + --force-with-lease="refs/heads/$AUTOMATION_BRANCH:$REMOTE_SHA" \ + origin "HEAD:refs/heads/$AUTOMATION_BRANCH" + else + git push --set-upstream origin \ + "HEAD:refs/heads/$AUTOMATION_BRANCH" + fi + + - name: Create or update pull request + if: >- + steps.update.outputs.changed == 'true' || + steps.branch.outputs.changed == 'true' || + (steps.update.outputs.target_applied == 'true' && + steps.branch.outputs.open_pr_url != '') + id: pull-request + env: + GH_TOKEN: ${{ steps.app-token.outputs.token }} + OPEN_PR_URL: ${{ steps.branch.outputs.open_pr_url }} + TARGET_APPLIED: ${{ steps.update.outputs.target_applied }} + run: | + set -euo pipefail + + body_file=$(mktemp) + trap 'rm -f "$body_file"' EXIT + cat > "$body_file" <> "$GITHUB_OUTPUT" + + - name: Add changelog entry + if: steps.pull-request.outputs.pr_url != '' + env: + GH_TOKEN: ${{ steps.app-token.outputs.token }} + PR_URL: ${{ steps.pull-request.outputs.pr_url }} + run: | + set -euo pipefail + + pr_number=${PR_URL##*/} + .github/scripts/addZoektSyncChangelogEntry.sh "$pr_number" + if git diff --quiet -- CHANGELOG.md; then + exit 0 + fi + + git add CHANGELOG.md + git commit -m "chore: update changelog for #$pr_number" + gh auth setup-git + git push origin "HEAD:refs/heads/$AUTOMATION_BRANCH" + + - name: Summarize pull request + if: steps.pull-request.outputs.pr_url != '' + env: + PR_URL: ${{ steps.pull-request.outputs.pr_url }} + run: | + synced_sha=$(git rev-parse HEAD:vendor/zoekt) + { + echo "## Zoekt sync" + echo + echo "Updated \`vendor/zoekt\` to \`$synced_sha\`." + echo + echo "Sourcebot pull request: $PR_URL" + } >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index dc203635c..9fee4b6c7 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -18,6 +18,8 @@ jobs: run: .github/scripts/test-vulnerability-triage.sh - name: Test CVE remediation discovery run: .github/scripts/test-cve-remediation.sh + - name: Test Zoekt sync automation + run: .github/scripts/testZoektSync.sh test: runs-on: ubuntu-latest