diff --git a/.github/forks.yml b/.github/forks.yml index 517c077..e5dd37d 100644 --- a/.github/forks.yml +++ b/.github/forks.yml @@ -16,6 +16,13 @@ # `.github/workflows/forward-port.yml` to this fork # so labelled PRs cherry-pick across `branches`. # Default false; opt-in per fork. +# rebase_globs: list of fnmatch globs (e.g. ["19.0-mig-*", "19.0-fix-*"]). +# The digest rebases each matching feature branch onto the +# freshly-synced `upstream_track` base and force-pushes the +# clean ones (PAT push ⇒ the fork's CI re-runs); conflicts +# are reported, never pushed. Absent = skip. Opt-in per +# fork — only set it where you carry not-yet-merged feature +# branches that must track an advancing base. forks: # === OCA forks — primary 18.0 + 19.0 tracking === @@ -23,85 +30,85 @@ forks: branches: ["18.0", "19.0", "20.0"] upstream_org: OCA upstream_track: "19.0" - install_forward_port: true + install_forward_port: false - repo: ledoent/sale-workflow branches: ["18.0", "19.0", "20.0"] upstream_org: OCA upstream_track: "19.0" - install_forward_port: true + install_forward_port: false - repo: ledoent/calendar branches: ["18.0", "19.0", "20.0"] upstream_org: OCA upstream_track: "19.0" - install_forward_port: true + install_forward_port: false - repo: ledoent/mis-builder branches: ["18.0", "19.0", "20.0"] upstream_org: OCA upstream_track: "19.0" - install_forward_port: true + install_forward_port: false - repo: ledoent/account-financial-reporting branches: ["18.0", "19.0", "20.0"] upstream_org: OCA upstream_track: "19.0" - install_forward_port: true + install_forward_port: false - repo: ledoent/account-financial-tools branches: ["18.0", "19.0", "20.0"] upstream_org: OCA upstream_track: "19.0" - install_forward_port: true + install_forward_port: false - repo: ledoent/spreadsheet branches: ["18.0", "19.0", "20.0"] upstream_org: OCA upstream_track: "19.0" - install_forward_port: true + install_forward_port: false - repo: ledoent/dms branches: ["18.0", "19.0", "20.0"] upstream_org: OCA upstream_track: "19.0" - install_forward_port: true + install_forward_port: false - repo: ledoent/bank-statement-import branches: ["18.0", "19.0", "20.0"] upstream_org: OCA upstream_track: "19.0" - install_forward_port: true + install_forward_port: false - repo: ledoent/connector-telephony branches: ["18.0", "19.0", "20.0"] upstream_org: OCA upstream_track: "19.0" - install_forward_port: true + install_forward_port: false - repo: ledoent/e-commerce branches: ["18.0", "19.0", "20.0"] upstream_org: OCA upstream_track: "19.0" - install_forward_port: true + install_forward_port: false - repo: ledoent/social branches: ["18.0", "19.0", "20.0"] upstream_org: OCA upstream_track: "19.0" - install_forward_port: true + install_forward_port: false - repo: ledoent/knowledge branches: ["18.0", "19.0", "20.0"] upstream_org: OCA upstream_track: "19.0" - install_forward_port: true + install_forward_port: false - repo: ledoent/geospatial branches: ["18.0", "19.0", "20.0"] upstream_org: OCA upstream_track: "19.0" - install_forward_port: true + install_forward_port: false - repo: ledoent/OpenUpgrade # NOTE: the fork's default branch `ledoent` is a CI overlay @@ -127,6 +134,10 @@ forks: upstream_org: OCA upstream_track: "19.0" install_forward_port: false + # ~16 not-yet-upstreamed submodule MIGs + a few fixes ride on top of + # OCA 19.0; keep them rebased nightly so each is mergeable when we open + # its PR (1-2/day cadence). + rebase_globs: ["19.0-mig-*", "19.0-fix-*"] - repo: ledoent/report-print-send branches: ["18.0", "19.0", "20.0"] diff --git a/.github/scripts/fork_sync_digest.py b/.github/scripts/fork_sync_digest.py index 2ebdb88..12fce86 100644 --- a/.github/scripts/fork_sync_digest.py +++ b/.github/scripts/fork_sync_digest.py @@ -21,9 +21,13 @@ from __future__ import annotations +import fnmatch import json +import os import re +import subprocess import sys +import tempfile import urllib.parse from datetime import datetime, timedelta, timezone from pathlib import Path @@ -195,6 +199,224 @@ def _compare_with_upstream( return body.get("ahead_by"), body.get("behind_by"), subjects +def check_ledoent_branch(repo: str, upstream_org: str | None, upstream_track: str | None) -> dict | None: + """Test if the ledoent branch of the fork rebases/aggregates cleanly.""" + status, body = gh("GET", f"/repos/{repo}/branches/ledoent") + if status != 200: + return None # No ledoent branch exists on this fork, skip + + print(f" Testing ledoent branch for {repo}...", file=sys.stderr) + base_branch = upstream_track or "18.0" + token = require_token() + repo_name = repo.split("/", 1)[1] + fork_url = f"https://x-access-token:{token}@github.com/{repo}.git" + upstream_org_name = upstream_org or "OCA" + upstream_url = f"https://github.com/{upstream_org_name}/{repo_name}.git" + + check_status = "error" + error_msg = "" + + with tempfile.TemporaryDirectory() as tmpdir: + tmp_path = Path(tmpdir) + repo_dir = tmp_path / "repo" + + try: + # 1. Clone the fork's ledoent branch + subprocess.run( + ["git", "clone", "--branch", "ledoent", "--single-branch", fork_url, "repo"], + cwd=tmpdir, check=True, capture_output=True + ) + # 2. Add upstream remote + subprocess.run( + ["git", "remote", "add", "upstream", upstream_url], + cwd=repo_dir, check=True, capture_output=True + ) + # 3. Fetch target branch + subprocess.run( + ["git", "fetch", "upstream", base_branch], + cwd=repo_dir, check=True, capture_output=True + ) + # 4. Configure dummy identity for merging/rebasing + subprocess.run(["git", "config", "user.name", "github-actions[bot]"], cwd=repo_dir, check=True) + subprocess.run(["git", "config", "user.email", "41898282+github-actions[bot]@users.noreply.github.com"], cwd=repo_dir, check=True) + + repos_yaml = repo_dir / "repos.yaml" + if repos_yaml.exists(): + # Test gitaggregate + res = subprocess.run( + ["gitaggregate", "-c", "repos.yaml"], + cwd=repo_dir, capture_output=True + ) + if res.returncode != 0: + check_status = "conflict" + error_msg = f"gitaggregate failed:\n{res.stderr.decode()}" + else: + check_status = "clean" + else: + # Test rebase + res = subprocess.run( + ["git", "rebase", f"upstream/{base_branch}"], + cwd=repo_dir, capture_output=True + ) + if res.returncode != 0: + subprocess.run(["git", "rebase", "--abort"], cwd=repo_dir, capture_output=True) + check_status = "conflict" + error_msg = f"git rebase upstream/{base_branch} failed with conflicts:\n{res.stderr.decode()}" + else: + check_status = "clean" + except subprocess.CalledProcessError as e: + check_status = "error" + error_msg = f"Git command failed: {e.stderr.decode()}" + except FileNotFoundError: + check_status = "error" + error_msg = "gitaggregate executable not found on runner." + except Exception as e: + check_status = "error" + error_msg = f"Unexpected check error: {str(e)}" + + return { + "repo": repo, + "branch": "ledoent", + "check_type": "rebase_check", + "status": 200 if check_status == "clean" else (409 if check_status == "conflict" else 500), + "check_status": check_status, + "message": error_msg or "Clean", + "skipped": False, + } + + +def _list_matching_branches(repo: str, globs: list[str]) -> list[str]: + """Fork branch names matching any fnmatch glob in `globs` (paginated).""" + out: list[str] = [] + page = 1 + while True: + status, body = gh( + "GET", f"/repos/{repo}/branches?per_page=100&page={page}" + ) + if status != 200 or not isinstance(body, list) or not body: + break + names = [b["name"] for b in body] + out.extend( + n for n in names if any(fnmatch.fnmatch(n, g) for g in globs) + ) + if len(names) < 100: + break + page += 1 + return sorted(set(out)) + + +def _rebase_one( + repo_dir: Path, repo: str, branch: str, base_branch: str, base_ref: str +) -> dict: + """Rebase one fetched feature branch onto base_ref; force-push if clean.""" + + def row(status: int, rebase_status: str, behind, message: str) -> dict: + return { + "repo": repo, + "branch": branch, + "check_type": "feature_rebase", + "status": status, + "rebase_status": rebase_status, + "behind": behind, + "message": message, + "skipped": False, + } + + def git(*args, **kw): + return subprocess.run( + ["git", *args], cwd=repo_dir, capture_output=True, **kw + ) + + fetched = git("fetch", "origin", branch) + if fetched.returncode != 0: + return row(500, "error", None, f"fetch failed: {fetched.stderr.decode()[:200]}") + + cnt = git("rev-list", "--count", f"origin/{branch}..{base_ref}") + behind = int(cnt.stdout.decode().strip() or "0") if cnt.returncode == 0 else None + if behind == 0: + return row(200, "already-current", 0, "up to date with base") + + git("checkout", "--detach", f"origin/{branch}", check=True) + git("clean", "-fdq") + mergebase = git("merge-base", base_ref, f"origin/{branch}").stdout.decode().strip() + # Content-preserving rebase: hooks off (nothing new to lint — the branch + # content is unchanged, only re-parented) and a no-op editor so an empty/ + # message step can't block in the headless runner. + env = {**os.environ, "GIT_EDITOR": "true"} + reb = subprocess.run( + ["git", "-c", "core.hooksPath=/dev/null", "rebase", "--onto", base_ref, mergebase], + cwd=repo_dir, capture_output=True, env=env, + ) + if reb.returncode != 0: + files = git("diff", "--name-only", "--diff-filter=U").stdout.decode().split() + subprocess.run( + ["git", "-c", "core.hooksPath=/dev/null", "rebase", "--abort"], + cwd=repo_dir, capture_output=True, env=env, + ) + return row(409, "conflict", behind, "conflict in: " + ", ".join(files[:8])) + + # PAT-authed push ⇒ re-triggers the fork's tests/pre-commit (a GITHUB_TOKEN + # push would not). --force-with-lease: the clone fetched origin/ + # moments ago, so this guards against a racing push in the same window. + pushed = git("push", "--force-with-lease", "origin", f"HEAD:refs/heads/{branch}") + if pushed.returncode != 0: + return row(500, "error", behind, f"push failed: {pushed.stderr.decode()[:200]}") + return row(200, "rebased", behind, f"rebased onto {base_branch} (+{behind})") + + +def rebase_feature_branches(repo: str, globs: list[str], base_branch: str) -> list[dict]: + """Rebase fork feature branches matching `globs` onto the fork's freshly + synced `base_branch`, force-pushing clean rebases and reporting conflicts. + + Runs AFTER the merge-upstream sync in the same fork iteration, so + `origin/` already reflects upstream. Whole-fork failures + (clone, etc.) collapse to a single error row so the digest still renders. + """ + branches = _list_matching_branches(repo, globs) + if not branches: + return [] + print( + f" Rebasing {len(branches)} feature branch(es) on {repo} onto {base_branch}...", + file=sys.stderr, + ) + token = require_token() + fork_url = f"https://x-access-token:{token}@github.com/{repo}.git" + results: list[dict] = [] + try: + with tempfile.TemporaryDirectory() as tmpdir: + repo_dir = Path(tmpdir) / "repo" + clone = subprocess.run( + ["git", "clone", "--branch", base_branch, "--single-branch", fork_url, "repo"], + cwd=tmpdir, capture_output=True, + ) + if clone.returncode != 0: + return [{ + "repo": repo, "branch": "(clone)", "check_type": "feature_rebase", + "status": 500, "rebase_status": "error", "behind": None, + "message": f"clone failed: {clone.stderr.decode()[:300]}", + "skipped": False, + }] + subprocess.run( + ["git", "config", "user.name", "github-actions[bot]"], + cwd=repo_dir, check=True, capture_output=True, + ) + subprocess.run( + ["git", "config", "user.email", + "41898282+github-actions[bot]@users.noreply.github.com"], + cwd=repo_dir, check=True, capture_output=True, + ) + base_ref = f"origin/{base_branch}" + for b in branches: + results.append(_rebase_one(repo_dir, repo, b, base_branch, base_ref)) + except Exception as e: # noqa: BLE001 — never let one fork crash the digest + results.append({ + "repo": repo, "branch": "(rebase)", "check_type": "feature_rebase", + "status": 500, "rebase_status": "error", "behind": None, + "message": f"unexpected error: {e}", "skipped": False, + }) + return results + + def list_recent_mig_prs( org: str, repo_name: str, track: str, since: datetime ) -> list[dict]: @@ -226,6 +448,7 @@ def main() -> int: Path("forks-parsed.json").write_text(json.dumps(forks, indent=2)) sync_results: list[dict] = [] + rebase_results: list[dict] = [] for f in forks: for branch in f.get("branches", []): res = sync_branch(f["repo"], branch, upstream_org=f.get("upstream_org")) @@ -236,7 +459,28 @@ def main() -> int: if res.get("diverged"): tag = f"{tag} DIVERGED+{res['ahead_by']}" print(f" {res['repo']}@{branch:>10} -> {tag}", file=sys.stderr) + + # Check ledoent branch if it exists on the fork + ledoent_res = check_ledoent_branch(f["repo"], f.get("upstream_org"), f.get("upstream_track")) + if ledoent_res: + sync_results.append(ledoent_res) + print( + f" {ledoent_res['repo']}@ledoent -> CHECK {ledoent_res['check_status'].upper()}", + file=sys.stderr, + ) + + # Rebase opted-in feature branches onto the freshly-synced base. + globs = f.get("rebase_globs") + if globs: + base = f.get("upstream_track") or "19.0" + for r in rebase_feature_branches(f["repo"], globs, base): + rebase_results.append(r) + print( + f" {r['repo']}@{r['branch']} -> REBASE {r['rebase_status'].upper()}", + file=sys.stderr, + ) Path("sync-results.json").write_text(json.dumps(sync_results, indent=2)) + Path("rebase-results.json").write_text(json.dumps(rebase_results, indent=2)) since = datetime.now(timezone.utc) - timedelta(hours=24) mig_buckets: dict[str, list[dict]] = {} diff --git a/.github/scripts/render_digest.py b/.github/scripts/render_digest.py index adc7d8c..a84b1a8 100644 --- a/.github/scripts/render_digest.py +++ b/.github/scripts/render_digest.py @@ -33,25 +33,32 @@ def render( sync_results: list[dict], mig_buckets: dict[str, list[dict]], distribution: list[dict], + rebase_results: list[dict] | None = None, ) -> tuple[str, str, str]: + rebase_results = rebase_results or [] today = datetime.now(timezone.utc).strftime("%Y-%m-%d") + + # Separate regular syncs from ledoent checks + sync_only = [r for r in sync_results if r.get("branch") != "ledoent"] + ledoent_checks = [r for r in sync_results if r.get("branch") == "ledoent"] + sync_fail = [ - r for r in sync_results if r["status"] >= 400 and not r["skipped"] + r for r in sync_only if r["status"] >= 400 and not r["skipped"] ] sync_updated = [ - r for r in sync_results if r.get("merge_type") in ("fast-forward", "merge") + r for r in sync_only if r.get("merge_type") in ("fast-forward", "merge") ] sync_nochange = [ - r for r in sync_results + r for r in sync_only if r.get("merge_type") == "none" and r["status"] < 400 ] - sync_skipped = [r for r in sync_results if r["skipped"]] + sync_skipped = [r for r in sync_only if r["skipped"]] # Divergence = fork branch has commits upstream doesn't, AND those # commits aren't part of the managed overlay (forward-port.yml # chore commit + sync-produced merge commits). Real divergence # means someone pushed work to a series branch — needs manual fix. sync_diverged = [ - r for r in sync_results + r for r in sync_only if r.get("diverged") and not r["skipped"] ] # Managed-overlay branches are 1+ ahead but only by structural @@ -63,11 +70,27 @@ def render( if r.get("managed_overlay") and not r["skipped"] ] + ledoent_fail = [r for r in ledoent_checks if r["status"] >= 400] + ledoent_clean = [r for r in ledoent_checks if r["status"] == 200] + + rebase_done = [r for r in rebase_results if r["rebase_status"] == "rebased"] + rebase_current = [r for r in rebase_results if r["rebase_status"] == "already-current"] + rebase_conflict = [r for r in rebase_results if r["rebase_status"] == "conflict"] + rebase_error = [r for r in rebase_results if r["rebase_status"] == "error"] + dist_fail = [r for r in distribution if r["action"] == "failed"] dist_created = [r for r in distribution if r["action"] == "created"] dist_updated = [r for r in distribution if r["action"] == "updated"] dist_unchanged = [r for r in distribution if r["action"] == "unchanged"] + ledoent_summary = "" + if ledoent_checks: + ledoent_summary = ( + f" · custom branches: {len(ledoent_clean)} clean · " + f'' + f"{len(ledoent_fail)} failed" + ) + lines: list[str] = [ '', @@ -81,6 +104,7 @@ def render( f' · {len(sync_diverged)} diverged' if sync_diverged else "" ) + + ledoent_summary + "

", ] if distribution: @@ -135,6 +159,17 @@ def render( ) lines.append("") + if ledoent_fail: + lines.append('

⚠️ Custom branch (ledoent) check failures

') + lines.append('

The following custom branches failed to rebase or aggregate cleanly against their upstream targets:

") + if dist_fail: lines.append('

⚠️ Distributor failures