Design: run AAQ scraper as an ArgoCD CronJob on the workloads cluster

.dockerignore

@@ -0,0 +1,11 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at https://mozilla.org/MPL/2.0/.
+.git
+.venv
+__pycache__
+*.pyc
+20*/
+docs/
+backfill-reports/
+tests/

.github/workflows/aaq-scraper-image.yml

@@ -0,0 +1,75 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at https://mozilla.org/MPL/2.0/.
+#
+# Build & push the scraper image to the shared ECR via GitHub OIDC.
+# One-time setup (see platform-infrastructure Task B1):
+#   - GitHub Environment "image-aaq-scraper" (protected to main)
+#   - repo variable IMAGE_PUSH_ROLE_ARN = the push-only role ARN
+name: Build scraper image
+
+on:
+  push:
+    branches: [main]
+    paths:
+      - "**.py"
+      - "pyproject.toml"
+      - "uv.lock"
+      - "Dockerfile"
+      - "deploy/entrypoint.sh"
+      - ".github/workflows/aaq-scraper-image.yml"
+  workflow_dispatch:
+    inputs:
+      tag:
+        description: "Explicit image tag (default git-<sha>)"
+        required: false
+
+permissions:
+  contents: read
+  id-token: write
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    environment: image-aaq-scraper
+    env:
+      ECR: 826971876779.dkr.ecr.us-east-1.amazonaws.com
+      REPO: aaq-scraper
+    steps:
+      - uses: actions/checkout@v5
+      - name: Compute tag
+        id: tag
+        env:
+          # Pass the workflow_dispatch input through an env var rather than
+          # interpolating ${{ github.event.* }} directly into the run script
+          # (that is a shell-injection vector, since this job holds ECR-push
+          # OIDC creds). Then validate the tag charset before use.
+          TAG_INPUT: ${{ github.event.inputs.tag }}
+        run: |
+          T="$TAG_INPUT"
+          [ -n "$T" ] || T="git-$(git rev-parse --short HEAD)"
+          case "$T" in
+            ""|*[!A-Za-z0-9._-]*)
+              echo "invalid tag: $T" >&2; exit 1 ;;
+          esac
+          printf 'tag=%s\n' "$T" >> "$GITHUB_OUTPUT"
+      - name: Configure AWS credentials (OIDC)
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          role-to-assume: ${{ vars.IMAGE_PUSH_ROLE_ARN }}
+          aws-region: us-east-1
+      - name: Login to ECR
+        uses: aws-actions/amazon-ecr-login@v2
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+        with:
+          platforms: arm64
+      - name: Set up Buildx
+        uses: docker/setup-buildx-action@v3
+      - name: Build and push
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          platforms: linux/arm64
+          push: true
+          tags: ${{ env.ECR }}/${{ env.REPO }}:${{ steps.tag.outputs.tag }}

.github/workflows/schema-check.yml

@@ -3,9 +3,13 @@
 # file, You can obtain one at https://mozilla.org/MPL/2.0/.
 name: API schema check
 
+# NOTE (2026-07-13, #28): parked pending its own migration to the k8s runner.
+# check_schema.py hits the live SUMO API and is blocked by the same Fastly
+# challenge as the scraper, so the daily schedule is disabled to stop spurious
+# api-blocked issues. Re-enable / migrate after the egress IP is allowlisted.
 on:
-  schedule:
-    - cron: "30 6 * * *" # daily at 06:30 UTC
+  # schedule:
+  #   - cron: "30 6 * * *"   # disabled — see NOTE above
   workflow_dispatch:
 
 permissions:

.gitignore

@@ -11,9 +11,6 @@ poc-sample.json
 /answers*.csv
 /*.json
 
-# Incremental-refresh high-water mark (persisted via Actions cache, not git)
-/.refresh-hwm
-
 # Month-backfill failure log (local record of dates to redo)
 /backfill-failures.txt
 

.python-version

@@ -0,0 +1 @@
+3.12

CLAUDE.md

@@ -8,19 +8,21 @@ Scrape the Mozilla SUMO (support.mozilla.org) "Ask a Question" API for Thunderbi
 Desktop and Thunderbird Android, producing CSVs compatible with the legacy Ruby
 reports in `thunderbird/github-action-thunderbird-aaq`. Since ~June 2026 the API
 sits behind a JavaScript challenge that blocks headless HTTP (issue
-thunderbird/github-action-thunderbird-aaq#34), so we drive a **real browser** to
-pass the challenge and call the JSON API from inside the browser's authenticated
-context.
+thunderbird/github-action-thunderbird-aaq#34); a real-browser (Playwright/
+Chromium) workaround passed the challenge for a while but is now itself
+fingerprinted and blocked (issue #28), so the scraper instead calls the API
+over a plain **`httpx`** HTTP client, and is moving to a Kubernetes CronJob
+deployment with a stable, allowlistable egress IP (issue #27).
 
 ## Core architecture & crucial decisions
 
-- **Browser-passes-challenge approach** (`sumo.py`): `SumoBrowser` launches
-  Chromium (Playwright), loads the site once to acquire challenge cookies, then
-  `fetch_json()` does an **in-page `fetch()`** (`page.evaluate`) so the request
-  reuses the page's cookies/origin. Headless works (locally **and in GitHub
-  Actions** — this is what resolves #34).
-- **Stack: Python + Playwright, managed with `uv`** — use `uv sync` / `uv run`,
-  never pip or raw venv. Deps in `pyproject.toml`.
+- **HTTP-client approach** (`sumo.py`): `SumoBrowser` (name kept for
+  compatibility) drives a plain `httpx` client; historically it launched
+  Chromium via Playwright to pass the JS challenge, but that is now
+  fingerprinted/blocked (#28).
+- **Stack: Python + `httpx`, managed with `uv`** — use `uv sync` / `uv run`,
+  never pip or raw venv. Deps in `pyproject.toml`. (`poc.py` still uses Playwright,
+  which is no longer a project dependency — install it separately to run the PoC.)
 - **`fetch_json` retries** transient failures with exponential backoff: HTTP 429
   (honours `Retry-After`), 5xx, and a 200-but-non-JSON challenge hiccup.
   Non-retryable 4xx fail fast.
@@ -111,8 +113,7 @@ context.
 ## Commands
 
 ```sh
-uv sync
-uv run playwright install chromium                 # one-time
+uv sync                                            # httpx client; no browser install needed
 
 # Questions (single day = same date twice). Add --product thunderbird-android for Android.
 uv run python scrape_questions.py 2026 6 10 2026 6 10 --headless
@@ -184,6 +185,29 @@ vary 2–10s (`--min-delay`/`--max-delay`). Use `--headless` for CI parity.
   `run_refresh` instead.
 - Actions are pinned to Node-24 versions: `actions/checkout@v5`,
   `astral-sh/setup-uv@v8.2.0`.
+- **k8s CronJob deployment (Phase B, in progress; ready-for-cutover, not live):**
+  the scraper is moving off GitHub Actions onto an ArgoCD-managed **Kubernetes
+  CronJob** on the workloads EKS cluster so it egresses via a stable IP Mozilla
+  can allowlist (issue #27) — the k8s manifests/Pulumi/ArgoCD app live in the
+  separate `platform-infrastructure` repo; this repo only builds the image
+  (`Dockerfile`, `.github/workflows/aaq-scraper-image.yml` → shared ECR via
+  OIDC) and ships `deploy/entrypoint.sh` (clone → `run_refresh.py` → commit).
+  Two prerequisites already landed here: (1) `sumo.py` **dropped Playwright**
+  entirely — the Chromium challenge-bypass is itself now fingerprinted and
+  blocked (issue #28) — and drives the same `SumoBrowser`/`fetch_json` public
+  API over a plain **`httpx`** client instead; (2) **`.refresh-hwm` is now
+  tracked in git** (removed from `.gitignore`, no longer the Actions cache)
+  since the pod is stateless and the repo is the durable state. The pod pushes
+  its commits authenticated with a **fine-grained GitHub PAT** sourced from AWS
+  Secrets Manager (synced in by External Secrets Operator) via a **git
+  credential helper** — the token is read from the environment at call time and
+  never appears in argv or a URL. Because the API is still blocked pending the
+  egress-IP allowlist, `.github/workflows/scrape.yml` **stays live** as the
+  production refresh until cutover, and `schema-check.yml`'s daily schedule is
+  **parked** (`workflow_dispatch` only) to stop spurious `api-blocked` issues
+  against the same block. Full design:
+  `docs/superpowers/specs/2026-07-13-k8s-argocd-scraper-deployment-design.md`
+  and `docs/superpowers/plans/2026-07-13-k8s-argocd-scraper-deployment.md`.
 
 ## Notes
 

Dockerfile

@@ -0,0 +1,31 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+# Browser-free image: the old Playwright/Chromium approach is dead (issue #28);
+# this runs the plain-httpx scraper. git + a shell are needed at runtime because
+# the entrypoint clones aaq-scraper, runs the refresh, and commits CSVs back.
+FROM python:3.12-slim
+
+# uv for dependency management (matches local/CI: uv sync / uv run).
+COPY --from=ghcr.io/astral-sh/uv:0.5.11 /uv /usr/local/bin/uv
+
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends git ca-certificates \
+    && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /app
+COPY pyproject.toml uv.lock ./
+RUN uv sync --frozen --no-dev
+
+COPY . .
+
+# Activate venv by adding it to PATH so `python` resolves to the venv's interpreter.
+ENV PATH="/app/.venv/bin:$PATH"
+
+# Non-root. The clone/commit workdir is a writable emptyDir mounted at runtime
+# (the root FS is read-only in the pod), so this user only needs to read /app.
+RUN useradd --create-home --uid 65532 appuser
+USER 65532
+
+ENTRYPOINT ["deploy/entrypoint.sh"]

README.md

@@ -1,14 +1,25 @@
 # aaq-scraper
 
 Until we get an official API that has a way to prevent DOS-ing, we scrape the SUMO
-(support.mozilla.org) API by driving a real browser. Since ~June 2026 the API sits
-behind a JavaScript challenge that blocks headless HTTP (see
-[thunderbird/github-action-thunderbird-aaq#34](https://github.com/thunderbird/github-action-thunderbird-aaq/issues/34));
-a real browser passes the challenge, then we call the JSON API from inside the
-browser's authenticated context.
+(support.mozilla.org) API. Since ~June 2026 the API sits behind a JavaScript
+challenge that blocks headless HTTP (see
+[thunderbird/github-action-thunderbird-aaq#34](https://github.com/thunderbird/github-action-thunderbird-aaq/issues/34)).
+A real-browser (Playwright/Chromium) workaround passed the challenge for a
+while but is now itself fingerprinted and blocked (issue #28), so the scraper
+is a plain **`httpx`** client instead, and is being redeployed as a
+**Kubernetes CronJob** on a cluster with a stable egress IP that Mozilla can
+allowlist (issue #27) — see the design and plan docs under
+[`docs/superpowers/specs/2026-07-13-k8s-argocd-scraper-deployment-design.md`](docs/superpowers/specs/2026-07-13-k8s-argocd-scraper-deployment-design.md)
+and
+[`docs/superpowers/plans/2026-07-13-k8s-argocd-scraper-deployment.md`](docs/superpowers/plans/2026-07-13-k8s-argocd-scraper-deployment.md).
+Until that IP is allowlisted, live API calls remain blocked and the hourly
+GitHub Actions refresh (`scrape.yml`) stays the production path.
 
 ## Proof of concept (Bucket 0)
 
+Historical Bucket-0 script; `playwright` is no longer a project dependency,
+so install it separately (`uv run pip install playwright`) before running.
+
 ```sh
 uv sync
 uv run playwright install chromium

deploy/entrypoint.sh

@@ -0,0 +1,73 @@
+#!/usr/bin/env bash
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at https://mozilla.org/MPL/2.0/.
+#
+# Pod entrypoint: clone aaq-scraper, run the incremental refresh, commit changed
+# CSVs (and the high-water mark) back to the branch. Mirrors the logic that used
+# to live in .github/workflows/scrape.yml (checkout + run_refresh + commit with
+# rebase-onto-latest-main retry). The pod is stateless; the repo IS the state.
+set -euo pipefail
+
+: "${GIT_REPO_URL:?set GIT_REPO_URL}"
+: "${GITHUB_TOKEN:?set GITHUB_TOKEN}"
+GIT_BRANCH="${GIT_BRANCH:-main}"
+GIT_AUTHOR_NAME="${GIT_AUTHOR_NAME:-aaq-scraper-bot}"
+GIT_AUTHOR_EMAIL="${GIT_AUTHOR_EMAIL:-aaq-scraper-bot@thunderbird.net}"
+REFRESH_ARGS="${REFRESH_ARGS:---soft-deadline 40 --max-429-wait 120}"
+
+# $HOME is the writable emptyDir mounted in the pod (Phase B sets HOME=/work);
+# /tmp is on the read-only root filesystem (readOnlyRootFilesystem: true), so
+# mktemp must be pointed at $HOME rather than its default /tmp base.
+WORKDIR="$(mktemp -d "${HOME:-/tmp}/aaq-scraper.XXXXXX")"
+trap 'rm -rf "$WORKDIR"' EXIT
+
+# Supply the PAT via a credential helper that reads $GITHUB_TOKEN from the
+# environment at call time, so only the variable NAME (never its value) ever
+# appears in argv (visible via ps / /proc/<pid>/cmdline). The clone/pull/push
+# all use the plain GIT_REPO_URL -- no token spliced into the URL.
+export GIT_TERMINAL_PROMPT=0
+CRED_HELPER='!f() { echo "username=x-access-token"; echo "password=${GITHUB_TOKEN}"; }; f'
+
+git -c credential.helper="$CRED_HELPER" clone --branch "$GIT_BRANCH" "$GIT_REPO_URL" "$WORKDIR/repo"
+cd "$WORKDIR/repo"
+git config credential.helper "$CRED_HELPER"
+git config user.name "$GIT_AUTHOR_NAME"
+git config user.email "$GIT_AUTHOR_EMAIL"
+
+# Run the refresh. REFRESH_CMD lets tests inject a fake; default is the real one.
+# Preserve the deferral exit code (75) as a non-error signal.
+set +e
+if [ -n "${REFRESH_CMD:-}" ]; then
+  eval "$REFRESH_CMD"
+else
+  uv run python run_refresh.py $REFRESH_ARGS
+fi
+rc=$?
+set -e
+if [ "$rc" -eq 75 ]; then
+  echo "refresh deferred (exit 75); committing whatever completed"
+elif [ "$rc" -ne 0 ]; then
+  echo "refresh failed (exit $rc)"; exit "$rc"
+fi
+
+git add 20*/ .refresh-hwm 2>/dev/null || git add 20*/
+if git diff --cached --quiet; then
+  echo "No changes to commit."
+  exit 0
+fi
+git commit -m "Hourly refresh $(date -u +%Y-%m-%dT%H:%MZ)"
+
+# main also receives pushes from manual backfills, so rebase onto latest and
+# retry with backoff rather than failing on a rejected push.
+for attempt in 1 2 3 4 5; do
+  if ! git pull --rebase origin "$GIT_BRANCH"; then
+    git rebase --abort || true
+    echo "rebase failed (attempt $attempt/5); retrying"; sleep $((attempt * 5)); continue
+  fi
+  if git push origin "$GIT_BRANCH"; then
+    echo "pushed on attempt $attempt"; exit 0
+  fi
+  echo "push rejected (attempt $attempt/5); retrying"; sleep $((attempt * 5))
+done
+echo "ERROR: could not push after 5 attempts"; exit 1