diff --git a/.github/workflows/cloud-e2e.yml b/.github/workflows/cloud-e2e.yml index b83fd9db7..b9caaa085 100644 --- a/.github/workflows/cloud-e2e.yml +++ b/.github/workflows/cloud-e2e.yml @@ -18,6 +18,10 @@ on: description: Parallel incognito browsers required: true default: "2" + scenario: + description: Optional single scenario ID + required: false + type: string permissions: contents: read @@ -44,7 +48,15 @@ jobs: GNIPPETS_BASE_URL: ${{ vars.GNIPPETS_BASE_URL }} GNIPPETS_E2E_CONTROL_TOKEN: ${{ secrets.GNIPPETS_E2E_CONTROL_TOKEN }} CAPSOLVER_API_KEY: ${{ secrets.CAPSOLVER_API_KEY }} - run: node ci/run.mjs --pack "${{ inputs.pack }}" --concurrency "${{ inputs.concurrency }}" + E2E_PACK: ${{ inputs.pack }} + E2E_CONCURRENCY: ${{ inputs.concurrency }} + E2E_SCENARIO: ${{ inputs.scenario }} + run: | + args=(--pack "$E2E_PACK" --concurrency "$E2E_CONCURRENCY") + if [[ -n "$E2E_SCENARIO" ]]; then + args+=(--scenario "$E2E_SCENARIO") + fi + node ci/run.mjs "${args[@]}" - name: Upload traces, recordings, and rubrics if: always() diff --git a/ci/lib/webbrain-client.mjs b/ci/lib/webbrain-client.mjs index 27074b883..d17c070b2 100644 --- a/ci/lib/webbrain-client.mjs +++ b/ci/lib/webbrain-client.mjs @@ -146,12 +146,44 @@ export class GnippetsE2EClient { method, headers: { authorization: `Bearer ${this.controlToken}`, + accept: 'application/json', + 'user-agent': 'Mozilla/5.0 (compatible; WebBrainCloudE2E/1.0; +https://webbrain.cloud)', ...(body === undefined ? {} : { 'content-type': 'application/json' }), }, body: body === undefined ? undefined : JSON.stringify(body), }); - const value = await response.json().catch(() => ({})); - if (!response.ok) throw new Error(value.error || `Gnippets E2E returned HTTP ${response.status}.`); + const text = await response.text(); + let value; + try { value = text ? JSON.parse(text) : {}; } catch { value = {}; } + if (!response.ok) { + const header = (name) => String(response.headers?.get?.(name) || 'unknown') + .replace(/\s+/g, ' ') + .slice(0, 160); + let preview = text; + if (this.controlToken) preview = preview.split(this.controlToken).join('[redacted]'); + preview = preview + .replace(/Bearer\s+[^\s<]+/gi, 'Bearer [redacted]') + .replace(/\s+/g, ' ') + .trim() + .slice(0, 240); + const diagnostics = { + server: header('server'), + cf_ray: header('cf-ray'), + cf_mitigated: header('cf-mitigated'), + content_type: header('content-type'), + body_preview: preview, + }; + const summary = Object.entries(diagnostics) + .filter(([, diagnostic]) => diagnostic && diagnostic !== 'unknown') + .map(([name, diagnostic]) => `${name}=${diagnostic}`) + .join('; '); + const error = new Error( + value.error || `Gnippets E2E returned HTTP ${response.status}${summary ? ` (${summary})` : ''}.`, + ); + error.status = response.status; + error.body = diagnostics; + throw error; + } return value; } diff --git a/ci/test.mjs b/ci/test.mjs index 8a9cd3295..7e191c34d 100644 --- a/ci/test.mjs +++ b/ci/test.mjs @@ -4,6 +4,7 @@ import path from 'node:path'; import { fileURLToPath } from 'node:url'; import { gradeScenario, inferStuckAt, renderSummary } from './lib/grader.mjs'; import { buildSessionSettings, resolveCloudRunId, suiteShouldFail } from './lib/suite.mjs'; +import { GnippetsE2EClient } from './lib/webbrain-client.mjs'; const root = path.dirname(fileURLToPath(import.meta.url)); const scenarios = JSON.parse(await fs.readFile(path.join(root, 'catalog', 'scenarios.json'), 'utf8')); @@ -29,6 +30,47 @@ assert.deepEqual( { enabled: true, key: 'captcha-key' }, ); +const diagnosticSecret = 'diagnostic-secret-that-must-not-leak'; +let diagnosticRequest; +const diagnosticClient = new GnippetsE2EClient({ + baseUrl: 'https://gnippets.example', + controlToken: diagnosticSecret, + fetchImpl: async (url, options) => { + diagnosticRequest = { url, options }; + return { + ok: false, + status: 403, + headers: { + get(name) { + return { + server: 'cloudflare', + 'cf-ray': 'fixture-ray-IST', + 'cf-mitigated': 'challenge', + 'content-type': 'text/html', + }[name.toLowerCase()] || null; + }, + }, + async text() { + return `