Worktree r2 budget gate

.github/workflows/regen-symbolcache.yml

@@ -15,6 +15,12 @@ name: Regenerate symbol cache
 # to lose an entire night's work -- with segments, each slice is uploaded
 # before the next starts, and a killed pod costs at most one segment.
 #
+# Before anything runs, a budget gate (scripts/r2_budget_gate.sh, in the plan
+# job) checks that month-to-date R2 Class A operations plus this run's upper
+# bound fit in the free tier's 1M/month allowance; if not, the regen jobs are
+# skipped. Requires the CLOUDFLARE_ANALYTICS_TOKEN secret (Account
+# Analytics:Read).
+#
 # Runs on a self-hosted runner (label: cloudindexer). See
 # docs/self-hosted-runner.md for runner prerequisites and setup.
 
@@ -54,16 +60,43 @@ permissions:
 
 jobs:
   plan:
-    # Tiny setup job: expands the `shards` count into the JSON list the matrix
-    # below needs (workflow expressions can't generate ranges).
+    # Setup + budget gate. Expands the `shards` count into the JSON list the
+    # matrix below needs (workflow expressions can't generate ranges), and
+    # runs scripts/r2_budget_gate.sh: regen only starts when month-to-date R2
+    # Class A ops + this run's upper bound fit in the free-tier budget.
+    # Fail-closed — if the gate can't determine usage, this job fails and
+    # regen never runs.
     runs-on:
       group: cloudindexer-self-hosted-32vcpu-64gb
-    timeout-minutes: 15
+    timeout-minutes: 30
     env:
       SHARDS: ${{ inputs.shards || '10' }}
+      JW_REF: main
+      R2_BUCKET: ${{ vars.R2_BUCKET || 'symbolcache' }}
+      REGEN_MODE: ${{ inputs.mode || 'incremental' }}
+      SWEEP_ARGS: ${{ inputs.sweep_args || '--newest 3' }}
+      R2_CLASSA_BUDGET: ${{ vars.R2_CLASSA_BUDGET || '1000000' }}
+      R2_CLASSA_MARGIN: ${{ vars.R2_CLASSA_MARGIN || '10000' }}
+      CLOUDFLARE_ANALYTICS_TOKEN: ${{ secrets.CLOUDFLARE_ANALYTICS_TOKEN }}
+      R2_ACCOUNT_ID: ${{ secrets.R2_ACCOUNT_ID }}
+      # Same env-var-defined rclone remote as the regen job (the gate
+      # downloads index.tar.gz + tombstones.txt.gz to build its done set).
+      RCLONE_CONFIG_R2_TYPE: s3
+      RCLONE_CONFIG_R2_PROVIDER: Cloudflare
+      RCLONE_CONFIG_R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }}
+      RCLONE_CONFIG_R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }}
+      RCLONE_CONFIG_R2_ENDPOINT: https://${{ secrets.R2_ACCOUNT_ID }}.r2.cloudflarestorage.com
+      RCLONE_CONFIG_R2_REGION: auto
+      RCLONE_CONFIG_R2_NO_CHECK_BUCKET: "true"
+      RCLONE_CONFIG_R2_ACL: private
     outputs:
       list: ${{ steps.gen.outputs.list }}
       total: ${{ steps.gen.outputs.total }}
+      proceed: ${{ steps.gate.outputs.proceed }}
+      used: ${{ steps.gate.outputs.used }}
+      planned: ${{ steps.gate.outputs.planned }}
+      pending: ${{ steps.gate.outputs.pending }}
+      budget: ${{ steps.gate.outputs.budget }}
     steps:
       - name: Generate shard list
         id: gen
@@ -77,8 +110,74 @@ jobs:
           echo "list=[$(seq -s, 0 $((SHARDS - 1)))]" >> "$GITHUB_OUTPUT"
           echo "Sweeping in $SHARDS sequential segments"
 
+      - name: Check out cloudindexer
+        uses: actions/checkout@v7
+
+      - name: Verify gate credentials are set
+        run: |
+          set -euo pipefail
+          if [[ -z "${CLOUDFLARE_ANALYTICS_TOKEN:-}" || -z "${R2_ACCOUNT_ID:-}" ]]; then
+            echo "Missing gate secrets (CLOUDFLARE_ANALYTICS_TOKEN / R2_ACCOUNT_ID)" >&2
+            exit 1
+          fi
+
+      - name: Clone latest JuliaWorkspaces.jl
+        run: |
+          set -euo pipefail
+          rm -rf JuliaWorkspaces.jl
+          git clone --depth 1 --branch "$JW_REF" \
+            https://github.com/julia-vscode/JuliaWorkspaces.jl.git JuliaWorkspaces.jl
+          echo "JW_DIR=$PWD/JuliaWorkspaces.jl" >> "$GITHUB_ENV"
+
+      - name: Set up Julia
+        uses: julia-actions/setup-julia@v3
+        with:
+          # Matches the checked-in Manifest (julia_version = 1.12.5).
+          version: "1.12"
+
+      - name: Install rclone and jq
+        run: |
+          set -euo pipefail
+          # Don't use rclone from apt: Ubuntu LTS pins old builds (24.04 ships
+          # 1.60.1 from 2022) that predate most of rclone's Cloudflare R2
+          # compatibility work, and every upload fails with "501
+          # NotImplemented". Install the current stable release instead.
+          export DEBIAN_FRONTEND=noninteractive
+          sudo apt-get update -qq
+          sudo apt-get install -y --no-install-recommends jq curl unzip
+          curl -fsSL https://rclone.org/install.sh | sudo bash
+          rclone version
+          jq --version
+
+      - name: Instantiate package project
+        run: |
+          set -euo pipefail
+          julia --project="$JW_DIR" -e 'using Pkg; Pkg.instantiate()'
+
+      - name: Download General registry
+        run: |
+          set -euo pipefail
+          # Unpacked so RegistryInstance can read it directly (same pattern as
+          # run_cloudindex_docker.sh). Trailing ':' appends the default depots
+          # so Pkg itself doesn't recompile from scratch.
+          REGDEPOT="$RUNNER_TEMP/regdepot"
+          mkdir -p "$REGDEPOT"
+          JULIA_DEPOT_PATH="$REGDEPOT:" JULIA_PKG_UNPACK_REGISTRY=true \
+            julia --startup-file=no -e 'using Pkg; Pkg.Registry.add("General")'
+          test -f "$REGDEPOT/registries/General/Registry.toml"
+          echo "REGISTRY=$REGDEPOT/registries/General" >> "$GITHUB_ENV"
+
+      - name: R2 Class A budget gate
+        id: gate
+        run: |
+          set -euo pipefail
+          REMOTE="r2:$R2_BUCKET" bash scripts/r2_budget_gate.sh
+
   regen:
     needs: plan
+    # Skipped (not failed) when the budget gate says the run wouldn't fit in
+    # the R2 free tier's monthly Class A allowance.
+    if: needs.plan.outputs.proceed == 'true'
     # One sweep segment per matrix job, strictly sequential. Sequencing
     # matters twice over: the regen script assumes single-flight access to the
     # bucket, and each segment downloads an index that already contains the

README.md

@@ -2,3 +2,27 @@
 
 Daily runs of the JuliaWorkspaces.jl based registry indexer. It creates cache files for every new
 package version registered into General and pushes them into an R2 bucket for general availability.
+
+## R2 Class A budget gate
+
+The workflow only starts a regen when it fits in the R2 free tier's monthly
+Class A allowance. Before the sweep, `scripts/r2_budget_gate.sh` (run in the
+`plan` job) queries month-to-date account-wide Class A operations from the
+Cloudflare GraphQL Analytics API and computes an upper bound for the run
+(`2 * pending versions + 2 * shards + margin`). If `used + planned` exceeds
+the budget, the regen jobs are skipped (shown as *skipped*, with a warning
+annotation and a step-summary table). The gate is fail-closed: if usage can't
+be determined, the run doesn't start.
+
+Configuration:
+
+- Secret `CLOUDFLARE_ANALYTICS_TOKEN` (required): Cloudflare API token with
+  *Account Analytics: Read* for the account holding the bucket.
+- Repo variable `R2_CLASSA_BUDGET` (optional, default `1000000`): total
+  monthly budget; set lower for standing headroom.
+- Repo variable `R2_CLASSA_MARGIN` (optional, default `10000`): fixed slack
+  for list pagination, retries, and analytics sampling error. The margin is
+  load-bearing — the 2·P + 2·S terms undercount listing pagination and
+  multipart overhead — so don't set it below ~1000 in production.
+
+Tests: `bash tests/test_r2_budget_gate.sh` (stubs curl/rclone/julia; needs jq).

docs/self-hosted-runner.md

@@ -54,6 +54,7 @@ Set these repository secrets (**Settings → Secrets and variables → Actions**
 | `R2_ACCESS_KEY_ID` | R2 access key ID. |
 | `R2_SECRET_ACCESS_KEY` | R2 secret access key. |
 | `R2_ACCOUNT_ID` | Cloudflare account ID; used to build the S3 endpoint `https://<account>.r2.cloudflarestorage.com`. |
+| `CLOUDFLARE_ANALYTICS_TOKEN` | Cloudflare API token with Account Analytics: Read; used by the budget gate in the plan job. |
 
 The workflow does not write an rclone config file. The regen scripts assume the
 `r2:` remote already exists, so the workflow defines it through
@@ -66,6 +67,8 @@ Optional repository **variable**:
 | Variable | Default | Description |
 | --- | --- | --- |
 | `R2_BUCKET` | `symbolcache` | R2 bucket name; the rclone remote passed to the script is `r2:<R2_BUCKET>`. |
+| `R2_CLASSA_BUDGET` | `1000000` | Monthly Class A op budget for the budget gate. |
+| `R2_CLASSA_MARGIN` | `10000` | Fixed slack added to the gate's planned-ops bound. |
 
 ## Concurrency and container resource limits
 

scripts/r2_budget_gate.sh

@@ -0,0 +1,207 @@
+#!/usr/bin/env bash
+#
+# R2 Class A budget gate for the symbol-cache regen workflow.
+#
+# Decides whether a regen run fits in the R2 free tier's monthly Class A
+# operation budget: queries month-to-date usage (account-wide, UTC calendar
+# month) from Cloudflare's GraphQL Analytics API, computes an upper bound on
+# the operations the run would consume, and reports proceed/skip.
+#
+#   planned = 2*P + 2*SHARDS + margin
+#     P       pending versions (jwcloudindex --dry-run): 1 PutObject per
+#             artifact + roughly 1 ListObjects of destination listing during
+#             `rclone copy` (bounded per-artifact for the uuid dirs it walks)
+#     2*S     per-shard index.tar.gz + tombstones.txt.gz uploads (tombstones
+#             go via `rclone rcat`, which may use multipart: ~3 Class A ops)
+#     margin  absorbs what the terms above undercount: destination-prefix
+#             ListObjects pagination (grows with registry size), rcat
+#             multipart overhead, rclone retries, and analytics sampling
+#             error. Do not set it near zero in production.
+#
+# Proceed iff used + planned <= budget. Fail-closed: any operational failure
+# (analytics API, rclone, dry run) exits nonzero; an over-budget skip is a
+# successful run with proceed=false.
+#
+# Required env:
+#   CLOUDFLARE_ANALYTICS_TOKEN  API token with Account Analytics:Read
+#   R2_ACCOUNT_ID               Cloudflare account tag
+#   REMOTE                      rclone remote + bucket, e.g. r2:symbolcache
+#   REGEN_MODE                  incremental | full
+#   SHARDS                      sweep segment count
+#   JW_DIR                      JuliaWorkspaces.jl checkout (instantiated)
+#   REGISTRY                    unpacked General registry path
+# Optional env:
+#   SWEEP_ARGS                  filters forwarded to the dry run (default: "")
+#   R2_CLASSA_BUDGET            default 1000000
+#   R2_CLASSA_MARGIN            default 10000
+#   WORK                        scratch dir (default: fresh mktemp)
+#
+# Writes proceed/used/planned/pending/budget to $GITHUB_OUTPUT and a summary
+# table to $GITHUB_STEP_SUMMARY when those are set.
+set -euo pipefail
+
+# Class A operations per
+# https://developers.cloudflare.com/r2/pricing/#class-a-operations
+CLASS_A_ACTIONS='["ListBuckets","PutBucket","ListObjects","PutObject","CopyObject","CompleteMultipartUpload","CreateMultipartUpload","LifecycleStorageTierTransition","ListMultipartUploads","UploadPart","UploadPartCopy","ListParts","PutBucketEncryption","PutBucketCors","PutBucketLifecycleConfiguration"]'
+
+# Month-to-date Class A operations, account-wide, UTC calendar month.
+# Echoes a non-negative integer. Zero rows is a valid zero; a transport
+# error, GraphQL error, unmatched account, or malformed sum is a failure.
+query_used() {
+    local start now payload response used
+    start="$(date -u +%Y-%m-01T00:00:00Z)"
+    now="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
+    payload="$(jq -n \
+        --arg account "$R2_ACCOUNT_ID" \
+        --arg start "$start" \
+        --arg now "$now" \
+        --argjson actions "$CLASS_A_ACTIONS" \
+        '{query: "query($account: string!, $start: Time!, $now: Time!, $actions: [string!]) { viewer { accounts(filter: {accountTag: $account}) { r2OperationsAdaptiveGroups(limit: 10000, filter: {datetime_geq: $start, datetime_leq: $now, actionType_in: $actions}) { sum { requests } } } } }",
+          variables: {account: $account, start: $start, now: $now, actions: $actions}}')"
+    response="$(curl -fsS --retry 3 --max-time 60 https://api.cloudflare.com/client/v4/graphql \
+        -H "Authorization: Bearer $CLOUDFLARE_ANALYTICS_TOKEN" \
+        -H "Content-Type: application/json" \
+        --data "$payload")" || {
+        echo "[gate] ERROR: analytics API request failed" >&2
+        return 1
+    }
+    used="$(jq -er '
+        if (.errors // []) != [] then error("graphql errors") else . end
+        | .data.viewer.accounts
+        | if length == 0 then error("no account matched accountTag") else . end
+        | [.[0].r2OperationsAdaptiveGroups[].sum.requests] as $reqs
+        | if ($reqs | all(type == "number")) then ($reqs | add // 0 | round) else error("malformed group: non-numeric sum.requests") end
+    ' <<<"$response")" || {
+        echo "[gate] ERROR: unexpected analytics response: $response" >&2
+        return 1
+    }
+    [[ "$used" =~ ^[0-9]+$ ]] || {
+        echo "[gate] ERROR: non-integer usage value: $used" >&2
+        return 1
+    }
+    echo "$used"
+}
+
+# done.txt = keys to skip, mirroring regen_symbolcache.sh steps 1-2.
+# incremental: successes ∪ tombstones; full: successes only (retry tombstones).
+build_done_set() {
+    local pfx="$STORE_PREFIX" state="$STORE_PREFIX/_state" rc
+    touch "$WORK/successes.txt"
+    rc=0
+    rclone copyto "${REMOTE}/${pfx}/index.tar.gz" "$WORK/index.tar.gz" 2>"$WORK/rclone_index.err" || rc=$?
+    if (( rc == 0 )); then
+        tar -xzO -f "$WORK/index.tar.gz" index.txt > "$WORK/successes.txt" || true
+    elif (( rc == 3 || rc == 4 )); then
+        # rclone: 3 = directory not found, 4 = file not found -> first run
+        echo "[gate] no existing index.tar.gz (first run or empty remote)"
+    else
+        echo "[gate] ERROR: rclone copyto index.tar.gz failed with exit $rc" >&2
+        cat "$WORK/rclone_index.err" >&2
+        return 1
+    fi
+    touch "$WORK/tombstones.txt"
+    rc=0
+    rclone copyto "${REMOTE}/${state}/tombstones.txt.gz" "$WORK/tombstones.txt.gz" 2>"$WORK/rclone_tombstones.err" || rc=$?
+    if (( rc == 0 )); then
+        gzip -dc "$WORK/tombstones.txt.gz" > "$WORK/tombstones.txt" || true
+    elif (( rc == 3 || rc == 4 )); then
+        # rclone: 3 = directory not found, 4 = file not found -> first run
+        echo "[gate] no existing tombstones.txt.gz (first run or empty remote)"
+    else
+        echo "[gate] ERROR: rclone copyto tombstones.txt.gz failed with exit $rc" >&2
+        cat "$WORK/rclone_tombstones.err" >&2
+        return 1
+    fi
+    if [[ "$REGEN_MODE" == "incremental" ]]; then
+        sort -u "$WORK/successes.txt" "$WORK/tombstones.txt" > "$WORK/done.txt"
+    elif [[ "$REGEN_MODE" == "full" ]]; then
+        sort -u "$WORK/successes.txt" > "$WORK/done.txt"
+    else
+        echo "[gate] ERROR: REGEN_MODE must be 'incremental' or 'full', got '$REGEN_MODE'" >&2
+        return 1
+    fi
+}
+
+# Exact pending-version count for the whole run: dry-run worklist size over
+# the full version set (deliberately no --shard — the gate bounds the sum of
+# all segments).
+count_pending() {
+    # SWEEP_ARGS is intentionally word-split, same as the regen job's usage.
+    # shellcheck disable=SC2086
+    julia --project="$JW_DIR" \
+        -e 'using JuliaWorkspaces; exit(JuliaWorkspaces.CloudIndexApp.cli_main(ARGS))' -- \
+        --dry-run --registry "$REGISTRY" --done-set "$WORK/done.txt" \
+        --out "$WORK/worklist.jsonl" ${SWEEP_ARGS:-} >&2 || {
+        echo "[gate] ERROR: dry-run enumeration failed" >&2
+        return 1
+    }
+    wc -l < "$WORK/worklist.jsonl" | tr -d ' '
+}
+
+# Compute the decision and report it to the log, $GITHUB_OUTPUT and
+# $GITHUB_STEP_SUMMARY (each only when set).
+decide_and_report() {
+    local used=$1 pending=$2
+    local budget margin planned proceed
+    budget="${R2_CLASSA_BUDGET:-1000000}"
+    margin="${R2_CLASSA_MARGIN:-10000}"
+    planned=$((2 * pending + 2 * SHARDS + margin))
+    if (( used + planned <= budget )); then
+        proceed=true
+    else
+        proceed=false
+    fi
+
+    echo "[gate] used=$used planned=$planned (pending=$pending shards=$SHARDS margin=$margin) budget=$budget proceed=$proceed"
+    if [[ "$proceed" == false ]]; then
+        echo "::warning title=R2 Class A budget gate::Skipping regen: month-to-date Class A ops ($used) + planned upper bound ($planned) exceed budget ($budget)."
+    fi
+    if [[ -n "${GITHUB_OUTPUT:-}" ]]; then
+        {
+            echo "proceed=$proceed"
+            echo "used=$used"
+            echo "planned=$planned"
+            echo "pending=$pending"
+            echo "budget=$budget"
+        } >> "$GITHUB_OUTPUT"
+    fi
+    if [[ -n "${GITHUB_STEP_SUMMARY:-}" ]]; then
+        {
+            echo "### R2 Class A budget gate"
+            echo ""
+            echo "| used (month-to-date) | planned (upper bound) | budget | pending versions | proceed |"
+            echo "| ---: | ---: | ---: | ---: | :--- |"
+            echo "| $used | $planned | $budget | $pending | $proceed |"
+        } >> "$GITHUB_STEP_SUMMARY"
+    fi
+}
+
+main() {
+    : "${CLOUDFLARE_ANALYTICS_TOKEN:?}" "${R2_ACCOUNT_ID:?}" "${REMOTE:?}" \
+      "${REGEN_MODE:?}" "${SHARDS:?}" "${JW_DIR:?}" "${REGISTRY:?}"
+    [[ "$SHARDS" =~ ^[0-9]+$ ]] || {
+        echo "[gate] ERROR: SHARDS must be an integer, got '$SHARDS'" >&2
+        exit 2
+    }
+    if [[ " ${SWEEP_ARGS:-} " =~ [[:space:]]--shard([[:space:]]|$) ]]; then
+        echo "[gate] ERROR: --shard is not allowed in SWEEP_ARGS: the gate counts the whole run (the regen matrix appends its own --shard, overriding yours, so the count would no longer be an upper bound)" >&2
+        exit 2
+    fi
+    WORK="${WORK:-$(mktemp -d "${TMPDIR:-/tmp}/r2_budget_gate.XXXXXX")}"
+    mkdir -p "$WORK"
+    # shellcheck disable=SC1091
+    source "$JW_DIR/scripts/symbolcache_common.sh"   # provides STORE_PREFIX
+
+    local used pending
+    used="$(query_used)"
+    echo "[gate] month-to-date Class A ops (account-wide): $used"
+    build_done_set
+    echo "[gate] done.txt has $(wc -l < "$WORK/done.txt") entries (mode=$REGEN_MODE)"
+    pending="$(count_pending)"
+    echo "[gate] pending versions this run: $pending"
+    decide_and_report "$used" "$pending"
+}
+
+if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then
+    main "$@"
+fi

tests/fixtures/graphql_empty.json

@@ -0,0 +1 @@
+{"data":{"viewer":{"accounts":[{"r2OperationsAdaptiveGroups":[]}]}},"errors":null}

tests/fixtures/graphql_errors.json

@@ -0,0 +1 @@
+{"data":null,"errors":[{"message":"authentication error"}]}

tests/fixtures/graphql_malformed.json

@@ -0,0 +1 @@
+{"data":{"viewer":{"accounts":[{"r2OperationsAdaptiveGroups":[{"sum":{}}]}]}},"errors":null}

tests/fixtures/graphql_noaccount.json

@@ -0,0 +1 @@
+{"data":{"viewer":{"accounts":[]}},"errors":null}

tests/fixtures/graphql_ok.json

@@ -0,0 +1 @@
+{"data":{"viewer":{"accounts":[{"r2OperationsAdaptiveGroups":[{"sum":{"requests":123456}}]}]}},"errors":null}