diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml new file mode 100644 index 000000000..b55cb9e67 --- /dev/null +++ b/.github/workflows/ci.yaml @@ -0,0 +1,60 @@ +name: CI + +on: + pull_request: + push: + branches: [master] + +permissions: + contents: read + +jobs: + build-and-test: + name: Build and test + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + with: + submodules: 'true' + + - uses: actions/setup-go@v7 + with: + go-version-file: go.mod + cache: true + + - name: Build + run: go build ./... + + - name: Test + run: go test -race -count=1 ./... + + # This repo holds several Go modules, each with its own go.mod, and + # `go build ./...` never crosses a module boundary. The two steps above + # therefore cover only the root module; this one covers the rest. + # + # The set of modules is discovered rather than written out, because it + # differs on either side of an upstream sync. Right now this fork has + # codelab, repl, repl/appengine and server, while current upstream has + # codelab, conformance, policy, repl and tools. A hardcoded list breaks + # every time that changes. + # + # Three are skipped because they do not build against unmodified upstream + # cel-go either, so building them would report upstream's breakage as + # ours: + # + # repl - its go.mod replaces cel.dev/expr with ../../cel-spec, a + # sibling checkout that does not exist on a CI runner + # codelab - stale go.mod; `go mod tidy` is needed before it builds + # tools - does not compile against the version of the policy module + # that it pins + - name: Build other Go modules + run: | + skip="repl codelab tools" + for m in $(git ls-files '*/go.mod' | sed 's|/go.mod$||' | sort -u); do + case " $skip " in + *" $m "*) echo "skipping $m"; continue ;; + esac + echo "::group::go build ./... ($m)" + (cd "$m" && go build ./...) + echo "::endgroup::" + done diff --git a/.github/workflows/sync-upstream.sh b/.github/workflows/sync-upstream.sh new file mode 100755 index 000000000..0732a8bef --- /dev/null +++ b/.github/workflows/sync-upstream.sh @@ -0,0 +1,229 @@ +#!/usr/bin/env bash +# Syncs this fork with upstream cel-go. +# Run it locally, or from Actions -> "Sync with upstream" -> Run workflow. +# +# sync-upstream.sh # full sync +# sync-upstream.sh --continue # after resolving conflicts by hand +# sync-upstream.sh rename [--reverse] # just the rename +# +# This fork differs from upstream in exactly one way: every +# `github.com/google/cel-go` import is rewritten to `github.com/authzed/cel-go`. +# Merging upstream directly means both sides edited the same import lines, so it +# conflicts on nearly every file upstream touched -- ~77 conflicts, none real. +# +# So the rename is applied last, and never merged: +# +# 1. un-rename, making our tree match the upstream commit we last synced from +# 2. merge upstream, which now sees no changes on our side +# 3. re-rename +# 4. build and test +# +# Commits, never pushes. The workflow hands the commits to +# peter-evans/create-pull-request. +# +# Merge the resulting PR with a merge commit, NOT a squash. Step 2's merge +# commit is what moves `git merge-base master upstream/master` forward, and that +# is what keeps the next sync clean. +# +# Env: +# PREFER_UPSTREAM=0 stop on conflicts instead of taking upstream's version +# NO_TEST=1 skip `go test ./...` +# UPSTREAM_REMOTE default: upstream +# UPSTREAM_URL default: https://github.com/cel-expr/cel-go.git +# UPSTREAM_BRANCH default: master +# BRANCH branch to commit on; defaults to a new +# sync-upstream- locally, current branch on CI +set -euo pipefail + +cd "$(git rev-parse --show-toplevel)" + +UPSTREAM_PATH="github.com/google/cel-go" +FORK_PATH="github.com/authzed/cel-go" + +UPSTREAM_REMOTE="${UPSTREAM_REMOTE:-upstream}" +UPSTREAM_URL="${UPSTREAM_URL:-https://github.com/cel-expr/cel-go.git}" +UPSTREAM_BRANCH="${UPSTREAM_BRANCH:-master}" +PREFER_UPSTREAM="${PREFER_UPSTREAM:-1}" + +# Skipped when verifying: these do not build against unmodified upstream +# either, so a failure there says nothing about the sync. +SKIP_MODULES="repl codelab tools" + +log() { printf '\n\033[1m==> %s\033[0m\n' "$*"; } +warn() { printf '\033[33m%s\033[0m\n' "$*"; } +die() { printf '\n\033[31merror: %s\033[0m\n' "$*" >&2; exit 1; } + +# The entire fork delta, as one substitution. +rename() { + local from="$UPSTREAM_PATH" to="$FORK_PATH" files count + if [[ "${1:-}" == "--reverse" ]]; then + from="$FORK_PATH" + to="$UPSTREAM_PATH" + fi + + # -I skips binary files, and git grep only looks at tracked ones. + files="$(git grep -I --name-only --fixed-strings -e "$from" -- . || true)" + if [[ -z "$files" ]]; then + echo "rename: no occurrences of ${from}; nothing to do" + return 0 + fi + + count="$(printf '%s\n' "$files" | wc -l | tr -d ' ')" + printf '%s\n' "$files" | tr '\n' '\0' \ + | xargs -0 perl -pi -e "s{\\Q${from}\\E}{${to}}g" + echo "rename: ${from} -> ${to} in ${count} file(s)" +} + +# Take upstream's version, and honor upstream's deletions. Correct for a fork +# whose only intentional change is the rename, which goes back on afterwards. +auto_resolve() { + local unmerged path + unmerged="$(git diff --name-only --diff-filter=U)" + [[ -z "$unmerged" ]] && return 0 + + warn "Auto-resolving in upstream's favor (PREFER_UPSTREAM=1):" + while IFS= read -r path; do + [[ -z "$path" ]] && continue + if git rev-parse -q --verify ":3:$path" >/dev/null 2>&1; then + git checkout --theirs -- "$path" + git add -- "$path" + echo " took upstream: $path" + else + git rm -q --force -- "$path" >/dev/null + echo " upstream deleted: $path" + fi + done <<< "$unmerged" +} + +# Builds every Go module in the repo. The set is discovered, not listed, +# because it changes as upstream adds and drops modules. +verify() { + local m + for m in . $(git ls-files '*/go.mod' | sed 's|/go.mod$||' | sort -u); do + case " $SKIP_MODULES " in + *" $m "*) echo " skipped $m" ; continue ;; + esac + ( cd "$m" && go build ./... ) || die "go build failed in module '$m'" + echo " go build OK $m" + done + if [[ "${NO_TEST:-0}" == "1" ]]; then + echo " go test skipped (NO_TEST=1)" + else + go test ./... >/dev/null || die "go test failed in the root module" + echo " go test OK ." + fi +} + +finish() { + log "Step 3/4: re-applying the rename" + rename + if git diff --quiet && git diff --cached --quiet; then + echo "Nothing to re-rename." + else + git add -A + git commit -qm "Re-apply authzed rename" + fi + + log "Step 4/4: verifying" + verify + + # The fork should be upstream plus the rename and nothing else. + local extra + extra="$(git diff "$UPSTREAM_REMOTE/$UPSTREAM_BRANCH" -- . ':(exclude).github' \ + | grep -E '^[+-]' | grep -vE '^(\+\+\+|---)' | grep -cv 'cel-go' || true)" + echo " non-rename lines vs upstream (excluding .github/): ${extra}" + + log "Sync complete on '$(git rev-parse --abbrev-ref HEAD)'. Nothing was pushed." +} + +case "${1:-}" in +rename) + shift + rename "$@" + exit 0 + ;; +--continue) + if git rev-parse -q --verify MERGE_HEAD >/dev/null; then + if git diff --name-only --diff-filter=U | grep -q .; then + die "still unresolved: $(git diff --name-only --diff-filter=U | tr '\n' ' ')" + fi + git commit -qm "Merge ${UPSTREAM_REMOTE}/${UPSTREAM_BRANCH} into fork" + fi + finish + exit 0 + ;; +"") ;; +*) die "unknown argument '${1}'" ;; +esac + +if git rev-parse -q --verify MERGE_HEAD >/dev/null; then + die "a merge is already in progress; run 'git merge --abort' or use --continue" +fi +if ! git diff --quiet || ! git diff --cached --quiet; then + die "working tree is dirty; commit or stash first" +fi + +log "Step 1/4: fetching ${UPSTREAM_REMOTE}/${UPSTREAM_BRANCH}" +git remote get-url "$UPSTREAM_REMOTE" >/dev/null 2>&1 \ + || git remote add "$UPSTREAM_REMOTE" "$UPSTREAM_URL" +git fetch --quiet "$UPSTREAM_REMOTE" "$UPSTREAM_BRANCH" +UPSTREAM_REF="$UPSTREAM_REMOTE/$UPSTREAM_BRANCH" +UPSTREAM_SHA="$(git rev-parse --short "$UPSTREAM_REF")" +echo "upstream is at ${UPSTREAM_SHA}" + +if git merge-base --is-ancestor "$UPSTREAM_REF" HEAD; then + log "Already up to date with ${UPSTREAM_REF}. Nothing to do." + exit 0 +fi + +# On CI, commit onto the branch that is already checked out. +# create-pull-request only pushes commits as-is when they sit on the base +# branch; otherwise it cherry-picks them, and cherry-pick cannot replay the +# merge commit that step 2 creates. +if [[ -n "${BRANCH:-}" ]]; then + git checkout -q -B "$BRANCH" +elif [[ -z "${GITHUB_ACTIONS:-}" ]]; then + git checkout -q -B "sync-upstream-${UPSTREAM_SHA}" +fi +echo "working on '$(git rev-parse --abbrev-ref HEAD)'" + +BASE="$(git merge-base HEAD "$UPSTREAM_REF")" + +log "Step 2/4: reverting the rename, then merging" +rename --reverse +git add -A +if git diff --cached --quiet; then + echo "Nothing to un-rename." +else + git commit -qm "Revert authzed rename for upstream sync" +fi + +# If un-renaming did not make our tree identical to the merge base, the fork +# carries changes of its own, and those files are the only ones that can +# conflict below. +DRIFT="$(git diff --name-only "$BASE" HEAD)" +if [[ -n "$DRIFT" ]]; then + warn "Fork differs from upstream beyond the rename in $(printf '%s\n' "$DRIFT" | wc -l | tr -d ' ') file(s):" + printf '%s\n' "$DRIFT" | sed 's/^/ /' + echo " -> only these files can conflict." +else + echo "drift check: clean. Fork is pure upstream + rename; the merge cannot conflict." +fi + +if ! git merge --no-edit "$UPSTREAM_REF"; then + if [[ "$PREFER_UPSTREAM" == "1" ]]; then + auto_resolve + git commit -qm "Merge ${UPSTREAM_REF} into fork" + else + warn "Conflicts (real fork divergence, not rename noise):" + git diff --name-only --diff-filter=U | sed 's/^/ /' + cat <<'EOF' + +Resolve them, `git add` the files, then run: + .github/workflows/sync-upstream.sh --continue +EOF + exit 1 + fi +fi + +finish diff --git a/.github/workflows/sync-upstream.yaml b/.github/workflows/sync-upstream.yaml new file mode 100644 index 000000000..b645b1aec --- /dev/null +++ b/.github/workflows/sync-upstream.yaml @@ -0,0 +1,56 @@ +name: Sync with upstream + +on: + workflow_dispatch: + +permissions: + contents: write + pull-requests: write + +jobs: + sync: + runs-on: ubuntu-latest + steps: + - name: Check out fork + uses: actions/checkout@v7 + with: + fetch-depth: 0 # all history + + - uses: actions/setup-go@v7 + with: + go-version-file: go.mod + cache: true + + - name: Sync with upstream and verify + run: .github/workflows/sync-upstream.sh + + - name: Record upstream revision + run: echo "UPSTREAM_SHA=$(git rev-parse --short upstream/master)" >> "$GITHUB_ENV" + + # merge-commit, never squash: the merge commit from the sync is what + # advances `git merge-base master upstream/master`, and that is what keeps + # the next sync conflict-free. + - name: Create Pull Request + id: cpr-sync + uses: peter-evans/create-pull-request@v8 + with: + delete-branch: "true" + token: ${{ secrets.AUTHZEDBOT_REPO_SCOPED_TOKEN }} + branch: sync-upstream + title: "Sync with upstream ${{ env.UPSTREAM_SHA }}" + body: |- + Automatically Generated Pull Request + + Synced with upstream `cel-expr/cel-go@${{ env.UPSTREAM_SHA }}`: + + 1. Reverted the `authzed` import rename + 2. Merged `upstream/master` + 3. Re-applied the rename + 4. Verified `go build` in every Go module and `go test ./...` in the root + + The result is upstream plus the import-path rename and nothing else. + + > [!IMPORTANT] + > Merge this with a **merge commit**, not a squash. The merge commit is + > what advances `git merge-base master upstream/master`, which is what + > keeps the next sync conflict-free.