From 21c3983f35cf918c1c6736a025dbba10060ef6bb Mon Sep 17 00:00:00 2001 From: Norrie Taylor Date: Wed, 17 Jun 2026 19:53:12 -0700 Subject: [PATCH 1/2] feat(e2e): /e2e staging-repo dispatcher for end-to-end pipeline runs Walking skeleton for #89: drive a synthetic feature through the full SDD lifecycle on a dedicated staging repo, assert terminal state, report cost. - e2e-dispatch.yml: plain Actions orchestrator (no lock). /e2e [scenario] from write-access authors, nightly schedule, workflow_dispatch. Gate -> provision -> seed -> poll -> assert -> cost -> report. - Provision by ref: quick-setup.sh --ref re-pins staging wrappers; cross-repo uses:@ runs the PR's hosted lock. No lock push. - e2e-assert.py: walks staging state via gh api, checks expectations.yml (terminal label, artifact globs, PRs opened/merged). - e2e-setup-staging.sh: one-time staging provisioning, documented, not CI-run. - happy-path-feature fixture; docs/sdd/e2e.md + mkdocs nav entry. Cost is staging Actions minutes + a coarse per-scenario token constant; lock metadata carries no token cost. happy-path-bug, revise-loop-spec, and needs-human-handoff are follow-up fixture-only PRs. Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/e2e-dispatch.yml | 345 ++++++++++++++++++ docs/sdd/e2e.md | 85 +++++ mkdocs.yml | 1 + scripts/e2e-assert.py | 174 +++++++++ scripts/e2e-setup-staging.sh | 135 +++++++ .../e2e/happy-path-feature/expectations.yml | 33 ++ .../fixtures/e2e/happy-path-feature/issue.md | 15 + 7 files changed, 788 insertions(+) create mode 100644 .github/workflows/e2e-dispatch.yml create mode 100644 docs/sdd/e2e.md create mode 100755 scripts/e2e-assert.py create mode 100755 scripts/e2e-setup-staging.sh create mode 100644 tests/fixtures/e2e/happy-path-feature/expectations.yml create mode 100644 tests/fixtures/e2e/happy-path-feature/issue.md diff --git a/.github/workflows/e2e-dispatch.yml b/.github/workflows/e2e-dispatch.yml new file mode 100644 index 0000000..8cb512a --- /dev/null +++ b/.github/workflows/e2e-dispatch.yml @@ -0,0 +1,345 @@ +name: e2e-dispatch + +# End-to-end pipeline test: drive a synthetic feature through the full SDD +# lifecycle on a dedicated staging repo, assert the terminal state, and report +# duration and a coarse cost estimate. See docs/sdd/e2e.md and issue #89. +# +# This is a plain GitHub Actions workflow, NOT a gh-aw source: it orchestrates +# the run with deterministic `gh` calls and a Python assert script, and never +# invokes an agent itself. The agents run on the STAGING repo, dispatched by +# the SDD wrappers that quick-setup installs there, pinned to the spectacles +# ref under test. +# +# Provisioning is by ref, not by lock push: the staging wrappers call the +# hosted locks via `uses: norrietaylor/spectacles/...@`. Re-pinning the +# wrappers to the PR head SHA (which carries the PR's recompiled locks) makes +# the staging run exercise the PR's code. + +on: + # /e2e [scenario] from a write-access author on a spectacles issue or PR. + issue_comment: + types: [created] + # Nightly at a low-traffic hour, default scenario. + schedule: + - cron: '0 7 * * *' + # Manual run with an explicit scenario. + workflow_dispatch: + inputs: + scenario: + description: Scenario fixture under tests/fixtures/e2e/ + required: false + default: happy-path-feature + +permissions: + contents: read + +concurrency: + # Serialize per scenario; never cancel an in-flight E2E run. + group: e2e-dispatch-${{ github.event.inputs.scenario || 'happy-path-feature' }} + cancel-in-progress: false + +jobs: + e2e: + # Token-strict match on `/e2e` for comment events (enumerate boundary chars + # since Actions expressions have no regex); non-comment events always pass + # this job-level gate and are filtered inside the resolve step. + if: | + github.event_name != 'issue_comment' + || github.event.comment.body == '/e2e' + || startsWith(github.event.comment.body, '/e2e ') + || startsWith(github.event.comment.body, '/e2e\n') + || startsWith(github.event.comment.body, '/e2e\r\n') + || startsWith(github.event.comment.body, '/e2e\t') + runs-on: ubuntu-latest + permissions: + contents: read + issues: read + pull-requests: read + steps: + - name: Check out spectacles (fixtures + scripts) + uses: actions/checkout@v4 + + - name: Resolve inputs and gate + id: gate + env: + GH_TOKEN: ${{ github.token }} + EVENT_NAME: ${{ github.event_name }} + COMMENT_BODY: ${{ github.event.comment.body }} + COMMENT_AUTHOR: ${{ github.event.comment.user.login }} + DISPATCH_SCENARIO: ${{ github.event.inputs.scenario }} + ISSUE_NUMBER: ${{ github.event.issue.number }} + IS_PR: ${{ github.event.issue.pull_request != null }} + REPO: ${{ github.repository }} + E2E_DISABLED: ${{ vars.SPECTACLES_E2E_DISABLED }} + STAGING_REPO: ${{ vars.SPECTACLES_E2E_STAGING_REPO }} + run: | + set -euo pipefail + + emit() { echo "$1=$2" >> "$GITHUB_OUTPUT"; } + stop() { emit should_run false; echo "e2e-dispatch: $1"; exit 0; } + + # Off switch: silences the nightly without a workflow edit. + case "${E2E_DISABLED:-}" in + 1|true|TRUE|True) stop "SPECTACLES_E2E_DISABLED is set; skipping." ;; + esac + + if [ -z "${STAGING_REPO:-}" ]; then + stop "SPECTACLES_E2E_STAGING_REPO is unset; nothing to target." + fi + + # Resolve the scenario and the spectacles ref under test. + scenario="happy-path-feature" + spectacles_ref="main" + origin_issue="" + + if [ "$EVENT_NAME" = "issue_comment" ]; then + # Write-access gate: accept write|maintain|admin only. + perm="$(gh api "repos/$REPO/collaborators/$COMMENT_AUTHOR/permission" \ + --jq '.permission' 2>/dev/null || echo none)" + case "$perm" in + write|maintain|admin) : ;; + *) stop "author $COMMENT_AUTHOR lacks write access (perm=$perm)." ;; + esac + + # /e2e [scenario] — second token is the optional scenario. + arg="$(printf '%s' "$COMMENT_BODY" | awk 'NR==1{print $2}')" + [ -n "$arg" ] && scenario="$arg" + origin_issue="$ISSUE_NUMBER" + + if [ "$IS_PR" = "true" ]; then + spectacles_ref="$(gh api "repos/$REPO/pulls/$ISSUE_NUMBER" --jq '.head.sha')" + fi + elif [ "$EVENT_NAME" = "workflow_dispatch" ]; then + [ -n "${DISPATCH_SCENARIO:-}" ] && scenario="$DISPATCH_SCENARIO" + fi + # schedule: defaults stand (happy-path-feature on main). + + fixture_dir="tests/fixtures/e2e/$scenario" + if [ ! -f "$fixture_dir/issue.md" ] || [ ! -f "$fixture_dir/expectations.yml" ]; then + stop "scenario '$scenario' has no fixture at $fixture_dir." + fi + + emit should_run true + emit scenario "$scenario" + emit spectacles_ref "$spectacles_ref" + emit origin_issue "$origin_issue" + emit staging_repo "$STAGING_REPO" + echo "e2e-dispatch: scenario=$scenario ref=$spectacles_ref staging=$STAGING_REPO" + + - name: Layer-0 gate — require green checks on the PR head + if: ${{ steps.gate.outputs.should_run == 'true' && steps.gate.outputs.spectacles_ref != 'main' }} + env: + GH_TOKEN: ${{ github.token }} + REPO: ${{ github.repository }} + SHA: ${{ steps.gate.outputs.spectacles_ref }} + run: | + set -euo pipefail + # The cheap Layer-0 checks (lint.yml jobs) must be green before the + # expensive Layer-1 E2E run provisions staging. A failing or pending + # required check fails fast here. + failing="$(gh api "repos/$REPO/commits/$SHA/check-runs" \ + --jq '[.check_runs[] | select(.conclusion=="failure" or .conclusion=="cancelled" or .conclusion=="timed_out")] | length')" + if [ "$failing" -gt 0 ]; then + echo "e2e-dispatch: $failing Layer-0 check(s) not green on $SHA; refusing to run." >&2 + exit 1 + fi + + - name: Mint App token (spectacles + staging) + id: app-token + if: ${{ steps.gate.outputs.should_run == 'true' }} + uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3 + with: + app-id: ${{ vars.APP_ID }} + private-key: ${{ secrets.APP_PRIVATE_KEY }} + owner: ${{ github.repository_owner }} + # Cover both the spectacles repo (post the summary comment) and the + # staging repo (provision, seed, poll). Names are bare repo names. + repositories: | + ${{ github.event.repository.name }} + ${{ steps.gate.outputs.staging_repo }} + permission-contents: write + permission-issues: write + permission-pull-requests: write + permission-actions: read + permission-administration: write + + - name: Provision staging (pin wrappers to the ref under test) + if: ${{ steps.gate.outputs.should_run == 'true' }} + env: + GH_TOKEN: ${{ steps.app-token.outputs.token }} + STAGING: ${{ steps.gate.outputs.staging_repo }} + REF: ${{ steps.gate.outputs.spectacles_ref }} + run: | + set -euo pipefail + bash scripts/quick-setup.sh --target-repo "$STAGING" --suite sdd --ref "$REF" --direct + + - name: Reset staging (close stale E2E issues) + if: ${{ steps.gate.outputs.should_run == 'true' }} + env: + GH_TOKEN: ${{ steps.app-token.outputs.token }} + STAGING: ${{ steps.gate.outputs.staging_repo }} + run: | + set -euo pipefail + # Close any open tracking issue from a prior run so this run starts + # from a clean slate. Idempotent; a fresh staging repo closes nothing. + gh issue list --repo "$STAGING" --state open --label e2e:seed --json number \ + --jq '.[].number' | while read -r n; do + [ -n "$n" ] && gh issue close "$n" --repo "$STAGING" --comment "Superseded by a new E2E run." + done + + - name: Seed the synthetic tracking issue + id: seed + if: ${{ steps.gate.outputs.should_run == 'true' }} + env: + GH_TOKEN: ${{ steps.app-token.outputs.token }} + STAGING: ${{ steps.gate.outputs.staging_repo }} + SCENARIO: ${{ steps.gate.outputs.scenario }} + run: | + set -euo pipefail + pip install --quiet pyyaml + fixture="tests/fixtures/e2e/$SCENARIO" + + # Seed time bounds the cost/PR queries to this run's activity. + echo "since=$(date -u +%Y-%m-%dT%H:%M:%SZ)" >> "$GITHUB_OUTPUT" + + # The feature issue template applies its labels only in the web UI; + # gh issue create does not, so pass the fixture's seed_labels plus an + # e2e:seed marker explicitly. + labels="$(python -c "import yaml,sys; print(','.join(yaml.safe_load(open(sys.argv[1])).get('seed_labels',[])))" "$fixture/expectations.yml")" + # Ensure the marker label exists, then create the issue. + gh label create e2e:seed --repo "$STAGING" --color ededed \ + --description "Synthetic E2E tracking issue (issue #89)." --force >/dev/null 2>&1 || true + + url="$(gh issue create --repo "$STAGING" \ + --title "feature: e2e $SCENARIO" \ + --body-file "$fixture/issue.md" \ + --label "e2e:seed${labels:+,$labels}")" + number="${url##*/}" + echo "issue=$number" >> "$GITHUB_OUTPUT" + echo "url=$url" >> "$GITHUB_OUTPUT" + echo "e2e-dispatch: seeded $url" + + - name: Poll lifecycle to terminal state + id: poll + if: ${{ steps.gate.outputs.should_run == 'true' }} + env: + GH_TOKEN: ${{ steps.app-token.outputs.token }} + STAGING: ${{ steps.gate.outputs.staging_repo }} + ISSUE: ${{ steps.seed.outputs.issue }} + TIMEOUT_MIN: ${{ vars.SPECTACLES_E2E_TIMEOUT_MIN || '30' }} + run: | + set -euo pipefail + deadline=$(( SECONDS + TIMEOUT_MIN * 60 )) + state="(none)" + while [ "$SECONDS" -lt "$deadline" ]; do + labels="$(gh issue view "$ISSUE" --repo "$STAGING" --json labels --jq '.labels[].name')" + # Terminal on sdd:done (success) or needs-human (escalation halt). + if echo "$labels" | grep -qx "sdd:done"; then state="sdd:done"; break; fi + if echo "$labels" | grep -qx "needs-human"; then state="needs-human"; break; fi + sleep 30 + done + [ "$SECONDS" -ge "$deadline" ] && [ "$state" = "(none)" ] && state="timeout" + echo "final_state=$state" >> "$GITHUB_OUTPUT" + echo "e2e-dispatch: final state = $state" + + - name: Assert terminal state + id: assert + if: ${{ steps.gate.outputs.should_run == 'true' }} + continue-on-error: true + env: + GH_TOKEN: ${{ steps.app-token.outputs.token }} + STAGING: ${{ steps.gate.outputs.staging_repo }} + ISSUE: ${{ steps.seed.outputs.issue }} + SCENARIO: ${{ steps.gate.outputs.scenario }} + SINCE: ${{ steps.seed.outputs.since }} + run: | + set -euo pipefail + pip install --quiet pyyaml + python scripts/e2e-assert.py \ + --repo "$STAGING" --issue "$ISSUE" \ + --expect "tests/fixtures/e2e/$SCENARIO/expectations.yml" \ + --since "$SINCE" | tee assert-output.txt + + - name: Estimate cost + id: cost + if: ${{ steps.gate.outputs.should_run == 'true' }} + env: + GH_TOKEN: ${{ steps.app-token.outputs.token }} + STAGING: ${{ steps.gate.outputs.staging_repo }} + SINCE: ${{ steps.seed.outputs.since }} + run: | + set -euo pipefail + # Coarse estimate: count staging Actions runs started since the seed + # and sum their wall-clock minutes. Token cost is a fixed per-run + # constant until a real usage source is wired (issue #89 follow-up). + runs="$(gh api "repos/$STAGING/actions/runs?created=>$SINCE&per_page=100" \ + --jq '[.workflow_runs[] | {s:.run_started_at, u:.updated_at}]')" + minutes="$(python -c " + import json,sys,datetime + def p(t): return datetime.datetime.fromisoformat(t.replace('Z','+00:00')) + rs=json.loads(sys.argv[1]) + m=sum((p(r['u'])-p(r['s'])).total_seconds() for r in rs if r['s'])/60 + print(f'{m:.1f}')" "$runs")" + echo "actions_minutes=$minutes" >> "$GITHUB_OUTPUT" + echo "token_estimate=~\$5 (coarse per-scenario constant)" >> "$GITHUB_OUTPUT" + + - name: Report summary + if: ${{ steps.gate.outputs.should_run == 'true' }} + env: + GH_TOKEN: ${{ steps.app-token.outputs.token }} + REPO: ${{ github.repository }} + STAGING: ${{ steps.gate.outputs.staging_repo }} + SCENARIO: ${{ steps.gate.outputs.scenario }} + ORIGIN_ISSUE: ${{ steps.gate.outputs.origin_issue }} + STAGING_ISSUE_URL: ${{ steps.seed.outputs.url }} + FINAL_STATE: ${{ steps.poll.outputs.final_state }} + ASSERT_OUTCOME: ${{ steps.assert.outcome }} + ACTIONS_MINUTES: ${{ steps.cost.outputs.actions_minutes }} + TOKEN_ESTIMATE: ${{ steps.cost.outputs.token_estimate }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + run: | + set -euo pipefail + verdict="✅ pass" + [ "$ASSERT_OUTCOME" != "success" ] && verdict="❌ fail" + + { + echo "### E2E \`$SCENARIO\` — $verdict" + echo + echo "- Staging tracking issue: $STAGING_ISSUE_URL" + echo "- Final lifecycle state: \`$FINAL_STATE\`" + echo "- Assertions: $ASSERT_OUTCOME" + echo "- Cost: ${ACTIONS_MINUTES} Actions minutes on staging; tokens $TOKEN_ESTIMATE" + echo "- Dispatcher run: $RUN_URL" + if [ -f assert-output.txt ]; then + echo + echo '```' + cat assert-output.txt + echo '```' + fi + } > summary.md + + cat summary.md >> "$GITHUB_STEP_SUMMARY" + + if [ -n "${ORIGIN_ISSUE:-}" ]; then + # Comment back on the originating spectacles issue/PR. + gh issue comment "$ORIGIN_ISSUE" --repo "$REPO" --body-file summary.md + else + # Schedule/dispatch: open a dated nightly result issue. + gh issue create --repo "$REPO" \ + --title "e2e nightly: $SCENARIO $(date -u +%Y-%m-%d)" \ + --body-file summary.md >/dev/null + fi + + - name: Tag staging for post-mortem on failure + if: ${{ steps.gate.outputs.should_run == 'true' && steps.assert.outcome != 'success' }} + env: + GH_TOKEN: ${{ steps.app-token.outputs.token }} + STAGING: ${{ steps.gate.outputs.staging_repo }} + ISSUE: ${{ steps.seed.outputs.issue }} + run: | + set -euo pipefail + gh label create e2e:failed --repo "$STAGING" --color b60205 \ + --description "E2E assertions failed; left for post-mortem." --force >/dev/null 2>&1 || true + gh issue edit "$ISSUE" --repo "$STAGING" --add-label e2e:failed + echo "::error::E2E scenario assertions failed; staging issue left open for post-mortem." + exit 1 diff --git a/docs/sdd/e2e.md b/docs/sdd/e2e.md new file mode 100644 index 0000000..8df71f3 --- /dev/null +++ b/docs/sdd/e2e.md @@ -0,0 +1,85 @@ +# End-to-end pipeline testing + +`e2e-dispatch` drives a synthetic feature through the full SDD lifecycle on a +dedicated **staging repo**, asserts the terminal state, and reports duration and +a coarse cost estimate. It runs on `/e2e` from a write-access author and nightly +on a schedule. See [#89](https://github.com/norrietaylor/spectacles/issues/89). + +## How it works + +1. A write-access author comments `/e2e [scenario]` on a spectacles issue or PR + (or the nightly schedule fires with the default scenario). +2. The dispatcher installs the SDD suite onto the staging repo with + `quick-setup.sh --ref `, pinning the staging wrappers to the + spectacles ref under test. The staging wrappers call the hosted locks via + `uses: …@`, so the staging run exercises the PR's code — no lock push. +3. It seeds a synthetic tracking issue from + `tests/fixtures/e2e//issue.md` (≤ 50 tokens to bound cost), with + the labels declared in that scenario's `expectations.yml`. +4. It polls the tracking issue's `sdd:*` label until a terminal state + (`sdd:done`, `needs-human`, or timeout — default 30 min). +5. `scripts/e2e-assert.py` checks the staging state against + `expectations.yml` (terminal label, artifact files, PRs opened and merged). +6. It posts a summary back on the originating issue/PR (or a dated nightly + issue): scenario, final state, assertion result, cost, and a link to the + dispatcher run. + +The dispatcher is a plain GitHub Actions workflow (`no` gh-aw agent step); only +the staging agents consume tokens. + +## One-time setup + +Provision the staging repo once with `scripts/e2e-setup-staging.sh`: + +```bash +STAGING_APP_PRIVATE_KEY=… ANTHROPIC_API_KEY=… \ + scripts/e2e-setup-staging.sh --staging norrietaylor/spectacles-staging +``` + +It clears branch protection on the staging default branch, installs the SDD +suite, and sets repo variables/secrets. It then prints the manual steps it +cannot perform (install the GitHub App on the staging repo; set `OTLP_*` +telemetry secrets per ADR 0020). + +## Configuration + +Set these on the **spectacles** repo: + +| Variable | Purpose | Default | +| --- | --- | --- | +| `SPECTACLES_E2E_STAGING_REPO` | Staging repo as `owner/name`. Required. | — | +| `SPECTACLES_E2E_TIMEOUT_MIN` | Lifecycle poll timeout, minutes. | `30` | +| `SPECTACLES_E2E_DISABLED` | `1`/`true` silences the nightly schedule without a workflow edit. | unset | + +## Scenarios + +Each scenario is one directory under `tests/fixtures/e2e/`: + +```text +tests/fixtures/e2e// + issue.md # the synthetic feature body (≤ 50 tokens) + expectations.yml # the assertion contract +``` + +`expectations.yml` keys (all optional): + +| Key | Meaning | +| --- | --- | +| `seed_labels` | Labels applied to the seeded tracking issue. | +| `terminal_label` | Required final `sdd:*` label (`null` = none, e.g. needs-human). | +| `require_label` | A label that must be present at terminal state. | +| `forbid_labels` | Labels that must be absent at terminal state. | +| `artifacts` | Globs; each must match ≥ 1 file on the staging default branch. | +| `prs.opened_min` | Minimum PRs opened since the seed. | +| `prs.all_merged` | Every such PR must be merged. | + +Shipped: `happy-path-feature`. Planned follow-ups: `happy-path-bug`, +`revise-loop-spec`, `needs-human-handoff` — each adds a fixture directory only, +no dispatcher change. + +## Cost + +The summary reports staging Actions minutes (summed from the staging repo's +workflow runs since the seed) plus a fixed per-scenario token estimate. Lock +metadata carries no token cost, so the token figure is coarse until a real +usage source is wired (issue #89 follow-up). A typical scenario is under $5. diff --git a/mkdocs.yml b/mkdocs.yml index f9361d1..09771cd 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -47,6 +47,7 @@ nav: - MCP tools: sdd/mcp-tools.md - The sdd-monitor agent: sdd/sdd-monitor.md - The sdd-status surface: sdd/sdd-status.md + - End-to-end testing: sdd/e2e.md - Specs: - "01: Issue-native SDD": specs/01-spec-issue-native-sdd/01-spec-issue-native-sdd.md not_in_nav: | diff --git a/scripts/e2e-assert.py b/scripts/e2e-assert.py new file mode 100755 index 0000000..aba37f3 --- /dev/null +++ b/scripts/e2e-assert.py @@ -0,0 +1,174 @@ +#!/usr/bin/env python3 +"""Assert a staging repo's terminal state against a scenario's expectations. + +The `e2e-dispatch` workflow seeds a synthetic tracking issue on the staging +repo, drives it through the SDD lifecycle, and polls until the issue reaches a +terminal `sdd:*` label (or `needs-human`, or a timeout). This script reads the +resulting state through the GitHub API and checks it against the scenario's +`expectations.yml` assertion contract: + + - `terminal_label` the tracking issue carries this `sdd:*` label (null = no + terminal lifecycle label is required, e.g. needs-human). + - `require_label` the tracking issue carries this label (e.g. needs-human). + - `forbid_labels` none of these labels are present on the tracking issue. + - `artifacts` each glob matches at least one file on the default branch. + - `prs.opened_min` at least N pull requests were opened since `--since`. + - `prs.all_merged` every such pull request is merged. + +All GitHub reads go through the `gh` CLI (`gh api`), matching the rest of the +repo's scripts — no PyGithub, no raw HTTP. Every failed check is collected and +printed; the script exits 1 if any check fails, 0 when all hold. Assertion +failures are reported, never raised. +""" + +from __future__ import annotations + +import argparse +import json +import re +import subprocess +import sys +from pathlib import Path + +import yaml + + +def gh_api(path: str, *, paginate: bool = False) -> object: + """Call `gh api ` and return the parsed JSON.""" + cmd = ["gh", "api", path] + if paginate: + # --slurp wraps the paginated pages into a single JSON array so a + # single json.loads sees every page. + cmd += ["--paginate", "--slurp"] + proc = subprocess.run(cmd, capture_output=True, text=True) + if proc.returncode != 0: + raise RuntimeError(f"gh api {path} failed: {proc.stderr.strip()}") + return json.loads(proc.stdout) + + +def glob_to_regex(pattern: str) -> re.Pattern[str]: + """Translate a path glob (supporting `**`) to an anchored regex. + + `**` matches any number of path segments (including zero); `*` matches + within a single segment; `?` matches one non-slash character. + """ + out = [] + i = 0 + while i < len(pattern): + c = pattern[i] + if c == "*": + if pattern[i : i + 2] == "**": + # `**/` collapses to "zero or more leading segments"; a bare + # `**` matches the rest of the path across segments. + if pattern[i : i + 3] == "**/": + out.append("(?:.*/)?") + i += 3 + continue + out.append(".*") + i += 2 + continue + out.append("[^/]*") + elif c == "?": + out.append("[^/]") + else: + out.append(re.escape(c)) + i += 1 + return re.compile("^" + "".join(out) + "$") + + +def check_labels(issue: dict, expect: dict, fail) -> None: + labels = {lbl["name"] for lbl in issue.get("labels", [])} + + terminal = expect.get("terminal_label", "__unset__") + if terminal not in ("__unset__", None): + if terminal not in labels: + fail(f"terminal_label: expected {terminal!r}; issue labels are {sorted(labels)}") + + require = expect.get("require_label") + if require and require not in labels: + fail(f"require_label: expected {require!r}; issue labels are {sorted(labels)}") + + for forbidden in expect.get("forbid_labels") or []: + if forbidden in labels: + fail(f"forbid_labels: {forbidden!r} is present on the tracking issue") + + +def check_artifacts(repo: str, expect: dict, fail) -> None: + globs = expect.get("artifacts") or [] + if not globs: + return + meta = gh_api(f"repos/{repo}") + branch = meta["default_branch"] + tree = gh_api(f"repos/{repo}/git/trees/{branch}?recursive=1") + if tree.get("truncated"): + fail("artifacts: default-branch tree was truncated; cannot assert reliably") + paths = [e["path"] for e in tree.get("tree", []) if e.get("type") == "blob"] + for pattern in globs: + rx = glob_to_regex(pattern) + if not any(rx.match(p) for p in paths): + fail(f"artifacts: no file on {branch} matches {pattern!r}") + + +def check_prs(repo: str, expect: dict, since: str | None, fail) -> None: + pr_expect = expect.get("prs") or {} + if not pr_expect: + return + pulls = gh_api(f"repos/{repo}/pulls?state=all&per_page=100", paginate=True) + # --slurp yields a list of pages (each a list); flatten. Without pagination + # gh returns a single list. + if pulls and isinstance(pulls[0], list): + pulls = [pr for page in pulls for pr in page] + if since: + pulls = [pr for pr in pulls if pr["created_at"] >= since] + + opened_min = pr_expect.get("opened_min") + if opened_min is not None and len(pulls) < opened_min: + fail(f"prs.opened_min: expected >= {opened_min}; found {len(pulls)} since {since}") + + if pr_expect.get("all_merged"): + unmerged = [pr["number"] for pr in pulls if not pr.get("merged_at")] + if unmerged: + fail(f"prs.all_merged: PRs not merged: {unmerged}") + + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument("--repo", required=True, help="staging repo as owner/name") + ap.add_argument("--issue", required=True, type=int, help="tracking issue number") + ap.add_argument("--expect", required=True, type=Path, help="path to expectations.yml") + ap.add_argument( + "--since", + default=None, + help="ISO 8601 timestamp; only PRs created at or after this are considered", + ) + args = ap.parse_args() + + expect = yaml.safe_load(args.expect.read_text()) + if not isinstance(expect, dict): + print(f"e2e-assert: {args.expect} is not a mapping", file=sys.stderr) + return 1 + + failures: list[str] = [] + fail = failures.append + + try: + issue = gh_api(f"repos/{args.repo}/issues/{args.issue}") + check_labels(issue, expect, fail) + check_artifacts(args.repo, expect, fail) + check_prs(args.repo, expect, args.since, fail) + except RuntimeError as exc: + print(f"e2e-assert: {exc}", file=sys.stderr) + return 1 + + if failures: + print(f"e2e-assert: {len(failures)} assertion(s) failed for {args.repo}#{args.issue}:") + for f in failures: + print(f" - {f}") + return 1 + + print(f"e2e-assert: all assertions passed for {args.repo}#{args.issue}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/e2e-setup-staging.sh b/scripts/e2e-setup-staging.sh new file mode 100755 index 0000000..093d7a8 --- /dev/null +++ b/scripts/e2e-setup-staging.sh @@ -0,0 +1,135 @@ +#!/usr/bin/env bash +# e2e-setup-staging.sh — one-time provisioning of the E2E staging repo. +# +# The e2e-dispatch workflow (.github/workflows/e2e-dispatch.yml) drives a +# synthetic feature through the full SDD lifecycle on a dedicated staging repo. +# That repo must exist, carry the SDD wrappers and labels, have its default +# branch unprotected (so the agents and the dispatcher can write directly), and +# hold the secrets and variables the agents need. This script provisions all of +# that. It is documented and run by an operator once per staging repo; CI never +# runs it. +# +# It is intentionally idempotent: re-running it re-syncs the wrappers and +# re-applies the variables without error. +# +# Manual prerequisite this script cannot do for you: install the GitHub App +# (the one whose APP_ID/APP_PRIVATE_KEY the suite uses) on the staging repo. +# The script prints a checklist of the manual steps at the end. + +set -euo pipefail + +usage() { + cat <<'EOF' +Usage: e2e-setup-staging.sh --staging / [--ref ] [--dry-run] + + --staging / The dedicated E2E staging repository. + --ref spectacles ref the staging wrappers pin to. The + dispatcher re-pins this per run to the PR head SHA; + this initial value only seeds the install + (default: main). + --dry-run Print what would happen; make no changes. + +Required environment (for the non-dry-run path): + STAGING_APP_PRIVATE_KEY PEM private key for the suite's GitHub App. + ANTHROPIC_API_KEY Model key the sdd-* agents authenticate with. + OTLP_* OTLP endpoint + headers per the ADR 0020 OTEL + mandate (passed through to the agents' wrapper + secret map). Optional if telemetry is disabled. +EOF +} + +staging="" +ref="main" +dry_run=0 + +while [ $# -gt 0 ]; do + case "$1" in + --staging) + [ $# -ge 2 ] || { echo "error: --staging needs a value" >&2; exit 2; } + staging="$2"; shift 2 ;; + --ref) + [ $# -ge 2 ] || { echo "error: --ref needs a value" >&2; exit 2; } + ref="$2"; shift 2 ;; + --dry-run) + dry_run=1; shift ;; + -h|--help) + usage; exit 0 ;; + *) + echo "error: unknown argument: $1" >&2; usage; exit 2 ;; + esac +done + +if [ -z "$staging" ]; then + echo "error: --staging is required" >&2 + usage + exit 2 +fi + +repo_root="$(cd "$(dirname "$0")/.." && pwd)" + +run() { + if [ "$dry_run" -eq 1 ]; then + echo "DRY-RUN: $*" + else + "$@" + fi +} + +echo "e2e-setup-staging: provisioning $staging (ref=$ref, dry_run=$dry_run)" + +# 1. Verify the staging repo exists and is reachable. +if ! gh repo view "$staging" >/dev/null 2>&1; then + echo "error: cannot reach $staging. Create it first (gh repo create $staging --private)." >&2 + exit 1 +fi + +# 2. Turn off branch protection on the default branch. The staging repo is a +# throwaway target; the dispatcher and agents write directly to its default +# branch, so protection would block them. Ignore a 404 (no protection set). +default_branch="$(gh repo view "$staging" --json defaultBranchRef --jq '.defaultBranchRef.name')" +echo "e2e-setup-staging: clearing branch protection on $staging@$default_branch" +if [ "$dry_run" -eq 1 ]; then + echo "DRY-RUN: gh api -X DELETE repos/$staging/branches/$default_branch/protection" +else + gh api -X DELETE "repos/$staging/branches/$default_branch/protection" >/dev/null 2>&1 \ + || echo "e2e-setup-staging: no branch protection to clear (ok)" +fi + +# 3. Install the SDD suite (wrappers + labels + templates) onto the staging +# repo via quick-setup, writing straight to the default branch (--direct, +# safe because protection is now off). +echo "e2e-setup-staging: installing SDD suite via quick-setup" +run bash "$repo_root/scripts/quick-setup.sh" \ + --target-repo "$staging" --suite sdd --ref "$ref" --direct + +# 4. Repo variables the dispatcher and agents read. +echo "e2e-setup-staging: setting repository variables" +run gh variable set SPECTACLES_E2E_STAGING_REPO --repo "$staging" --body "$staging" + +# 5. Secrets. Set only those present in the environment; warn on the rest. +set_secret() { + local name="$1" value="$2" + if [ -z "$value" ]; then + echo "e2e-setup-staging: WARNING $name not in environment; set it manually" + return + fi + run gh secret set "$name" --repo "$staging" --body "$value" +} +echo "e2e-setup-staging: setting secrets from the environment" +set_secret APP_PRIVATE_KEY "${STAGING_APP_PRIVATE_KEY:-}" +set_secret ANTHROPIC_API_KEY "${ANTHROPIC_API_KEY:-}" + +cat < GitHub Apps). + 2. Set APP_ID as a repository variable on $staging if it differs from the + org default: gh variable set APP_ID --repo $staging --body + 3. Set the OTLP_* telemetry secrets per the ADR 0020 OTEL mandate if the + agents emit traces from this repo. + 4. On the spectacles repo, set the dispatcher's variables: + SPECTACLES_E2E_STAGING_REPO = $staging + (optional) SPECTACLES_E2E_TIMEOUT_MIN, SPECTACLES_E2E_DISABLED +EOF diff --git a/tests/fixtures/e2e/happy-path-feature/expectations.yml b/tests/fixtures/e2e/happy-path-feature/expectations.yml new file mode 100644 index 0000000..db6cc9c --- /dev/null +++ b/tests/fixtures/e2e/happy-path-feature/expectations.yml @@ -0,0 +1,33 @@ +# Assertion contract for the happy-path-feature E2E scenario. +# +# scripts/e2e-assert.py reads this file and checks the staging repo's terminal +# state after the e2e-dispatch workflow polls the seeded tracking issue to a +# terminal label. Every key is optional; an absent key is not asserted. + +# Seed labels the dispatcher applies to the synthetic tracking issue. The +# feature issue template applies these in the web UI, but `gh issue create` +# does not honor template labels, so the dispatcher passes them explicitly and +# this list documents the contract. +seed_labels: + - kind:feature + - sdd:spec + +# Required final sdd:* lifecycle label on the tracking issue. The canonical +# full-path flow ends at sdd:done. +terminal_label: sdd:done + +# Glob patterns; each must match at least one file on the staging repo's +# default branch after the implementation PR merges. The spec phase writes a +# spec doc; this scenario does not require an ADR. +artifacts: + - "docs/specs/**/*.md" + +# Pull-request assertions over PRs opened on the staging repo since the seed. +prs: + opened_min: 1 + all_merged: true + +# Labels that must be ABSENT on the tracking issue at terminal state. A +# happy-path run never escalates. +forbid_labels: + - needs-human diff --git a/tests/fixtures/e2e/happy-path-feature/issue.md b/tests/fixtures/e2e/happy-path-feature/issue.md new file mode 100644 index 0000000..5217da5 --- /dev/null +++ b/tests/fixtures/e2e/happy-path-feature/issue.md @@ -0,0 +1,15 @@ +## Summary + +Add a `--version` flag to the sample CLI that prints the package version and exits 0. + +## Problem + +The CLI has no way to report its own version, so operators cannot confirm which build is deployed. + +## Desired outcome + +Running the CLI with `--version` prints the version string on stdout and exits 0; no other flag behavior changes. + +## Scope notes + +Single flag. No config, no new dependencies. From fe0ed6a41b92de2d12831c1db3eb22e4aeb39570 Mon Sep 17 00:00:00 2001 From: Norrie Taylor Date: Wed, 17 Jun 2026 19:57:59 -0700 Subject: [PATCH 2/2] fix(e2e): correct engine secret to CLAUDE_CODE_OAUTH_TOKEN The compiled locks use engine: claude and read CLAUDE_CODE_OAUTH_TOKEN, not ANTHROPIC_API_KEY. Update the staging setup script and docs to match, and add GH_AW_OTEL_ENDPOINT to the optional telemetry secrets. Co-Authored-By: Claude Opus 4.8 (1M context) --- docs/sdd/e2e.md | 7 ++++--- scripts/e2e-setup-staging.sh | 22 ++++++++++++---------- 2 files changed, 16 insertions(+), 13 deletions(-) diff --git a/docs/sdd/e2e.md b/docs/sdd/e2e.md index 8df71f3..9e70718 100644 --- a/docs/sdd/e2e.md +++ b/docs/sdd/e2e.md @@ -32,14 +32,15 @@ the staging agents consume tokens. Provision the staging repo once with `scripts/e2e-setup-staging.sh`: ```bash -STAGING_APP_PRIVATE_KEY=… ANTHROPIC_API_KEY=… \ +STAGING_APP_PRIVATE_KEY=… CLAUDE_CODE_OAUTH_TOKEN=… \ scripts/e2e-setup-staging.sh --staging norrietaylor/spectacles-staging ``` It clears branch protection on the staging default branch, installs the SDD suite, and sets repo variables/secrets. It then prints the manual steps it -cannot perform (install the GitHub App on the staging repo; set `OTLP_*` -telemetry secrets per ADR 0020). +cannot perform (install the GitHub App on the staging repo; set +`GH_AW_OTEL_ENDPOINT` per ADR 0020). The compiled locks use `engine: claude`, +so the engine secret is `CLAUDE_CODE_OAUTH_TOKEN`. ## Configuration diff --git a/scripts/e2e-setup-staging.sh b/scripts/e2e-setup-staging.sh index 093d7a8..5343bf9 100755 --- a/scripts/e2e-setup-staging.sh +++ b/scripts/e2e-setup-staging.sh @@ -31,10 +31,11 @@ Usage: e2e-setup-staging.sh --staging / [--ref ] [--dry-run] Required environment (for the non-dry-run path): STAGING_APP_PRIVATE_KEY PEM private key for the suite's GitHub App. - ANTHROPIC_API_KEY Model key the sdd-* agents authenticate with. - OTLP_* OTLP endpoint + headers per the ADR 0020 OTEL - mandate (passed through to the agents' wrapper - secret map). Optional if telemetry is disabled. + CLAUDE_CODE_OAUTH_TOKEN Engine token the sdd-* agents authenticate with + (engine: claude). This is the secret the compiled + locks read. + GH_AW_OTEL_ENDPOINT OTLP endpoint per the ADR 0020 OTEL mandate. + Optional if telemetry is disabled. EOF } @@ -117,18 +118,19 @@ set_secret() { } echo "e2e-setup-staging: setting secrets from the environment" set_secret APP_PRIVATE_KEY "${STAGING_APP_PRIVATE_KEY:-}" -set_secret ANTHROPIC_API_KEY "${ANTHROPIC_API_KEY:-}" +set_secret CLAUDE_CODE_OAUTH_TOKEN "${CLAUDE_CODE_OAUTH_TOKEN:-}" +set_secret GH_AW_OTEL_ENDPOINT "${GH_AW_OTEL_ENDPOINT:-}" cat < GitHub Apps). - 2. Set APP_ID as a repository variable on $staging if it differs from the - org default: gh variable set APP_ID --repo $staging --body - 3. Set the OTLP_* telemetry secrets per the ADR 0020 OTEL mandate if the - agents emit traces from this repo. + 1. Install the suite's GitHub App on $staging (or install it on all repos). + 2. Set APP_ID as a repository variable on $staging: + gh variable set APP_ID --repo $staging --body + 3. Set GH_AW_OTEL_ENDPOINT and any Distillery secrets per the ADR 0020 OTEL + mandate if the agents emit traces from this repo. 4. On the spectacles repo, set the dispatcher's variables: SPECTACLES_E2E_STAGING_REPO = $staging (optional) SPECTACLES_E2E_TIMEOUT_MIN, SPECTACLES_E2E_DISABLED