From b1230c97d256d39e706488e6c65492df388b2825 Mon Sep 17 00:00:00 2001 From: Emre Sokullu Date: Tue, 21 Jul 2026 13:25:24 +0300 Subject: [PATCH 1/2] Diagnose Gnippets E2E HTTP failures --- .github/workflows/cloud-e2e.yml | 14 ++++++++++++- ci/lib/webbrain-client.mjs | 34 +++++++++++++++++++++++++++++-- ci/test.mjs | 36 +++++++++++++++++++++++++++++++++ 3 files changed, 81 insertions(+), 3 deletions(-) diff --git a/.github/workflows/cloud-e2e.yml b/.github/workflows/cloud-e2e.yml index b83fd9db7..b9caaa085 100644 --- a/.github/workflows/cloud-e2e.yml +++ b/.github/workflows/cloud-e2e.yml @@ -18,6 +18,10 @@ on: description: Parallel incognito browsers required: true default: "2" + scenario: + description: Optional single scenario ID + required: false + type: string permissions: contents: read @@ -44,7 +48,15 @@ jobs: GNIPPETS_BASE_URL: ${{ vars.GNIPPETS_BASE_URL }} GNIPPETS_E2E_CONTROL_TOKEN: ${{ secrets.GNIPPETS_E2E_CONTROL_TOKEN }} CAPSOLVER_API_KEY: ${{ secrets.CAPSOLVER_API_KEY }} - run: node ci/run.mjs --pack "${{ inputs.pack }}" --concurrency "${{ inputs.concurrency }}" + E2E_PACK: ${{ inputs.pack }} + E2E_CONCURRENCY: ${{ inputs.concurrency }} + E2E_SCENARIO: ${{ inputs.scenario }} + run: | + args=(--pack "$E2E_PACK" --concurrency "$E2E_CONCURRENCY") + if [[ -n "$E2E_SCENARIO" ]]; then + args+=(--scenario "$E2E_SCENARIO") + fi + node ci/run.mjs "${args[@]}" - name: Upload traces, recordings, and rubrics if: always() diff --git a/ci/lib/webbrain-client.mjs b/ci/lib/webbrain-client.mjs index 27074b883..513dc47e5 100644 --- a/ci/lib/webbrain-client.mjs +++ b/ci/lib/webbrain-client.mjs @@ -150,8 +150,38 @@ export class GnippetsE2EClient { }, body: body === undefined ? undefined : JSON.stringify(body), }); - const value = await response.json().catch(() => ({})); - if (!response.ok) throw new Error(value.error || `Gnippets E2E returned HTTP ${response.status}.`); + const text = await response.text(); + let value; + try { value = text ? JSON.parse(text) : {}; } catch { value = {}; } + if (!response.ok) { + const header = (name) => String(response.headers?.get?.(name) || 'unknown') + .replace(/\s+/g, ' ') + .slice(0, 160); + let preview = text; + if (this.controlToken) preview = preview.split(this.controlToken).join('[redacted]'); + preview = preview + .replace(/Bearer\s+[^\s<]+/gi, 'Bearer [redacted]') + .replace(/\s+/g, ' ') + .trim() + .slice(0, 240); + const diagnostics = { + server: header('server'), + cf_ray: header('cf-ray'), + cf_mitigated: header('cf-mitigated'), + content_type: header('content-type'), + body_preview: preview, + }; + const summary = Object.entries(diagnostics) + .filter(([, diagnostic]) => diagnostic && diagnostic !== 'unknown') + .map(([name, diagnostic]) => `${name}=${diagnostic}`) + .join('; '); + const error = new Error( + value.error || `Gnippets E2E returned HTTP ${response.status}${summary ? ` (${summary})` : ''}.`, + ); + error.status = response.status; + error.body = diagnostics; + throw error; + } return value; } diff --git a/ci/test.mjs b/ci/test.mjs index 8a9cd3295..9c2d8b0bc 100644 --- a/ci/test.mjs +++ b/ci/test.mjs @@ -4,6 +4,7 @@ import path from 'node:path'; import { fileURLToPath } from 'node:url'; import { gradeScenario, inferStuckAt, renderSummary } from './lib/grader.mjs'; import { buildSessionSettings, resolveCloudRunId, suiteShouldFail } from './lib/suite.mjs'; +import { GnippetsE2EClient } from './lib/webbrain-client.mjs'; const root = path.dirname(fileURLToPath(import.meta.url)); const scenarios = JSON.parse(await fs.readFile(path.join(root, 'catalog', 'scenarios.json'), 'utf8')); @@ -29,6 +30,41 @@ assert.deepEqual( { enabled: true, key: 'captcha-key' }, ); +const diagnosticSecret = 'diagnostic-secret-that-must-not-leak'; +const diagnosticClient = new GnippetsE2EClient({ + baseUrl: 'https://gnippets.example', + controlToken: diagnosticSecret, + fetchImpl: async () => ({ + ok: false, + status: 403, + headers: { + get(name) { + return { + server: 'cloudflare', + 'cf-ray': 'fixture-ray-IST', + 'cf-mitigated': 'challenge', + 'content-type': 'text/html', + }[name.toLowerCase()] || null; + }, + }, + async text() { + return `Attention RequiredBearer ${diagnosticSecret}`; + }, + }), +}); +await assert.rejects( + diagnosticClient.createRun('fixture'), + (error) => { + assert.equal(error.status, 403); + assert.equal(error.body.server, 'cloudflare'); + assert.equal(error.body.cf_mitigated, 'challenge'); + assert.match(error.message, /fixture-ray-IST/); + assert.match(error.message, /Attention Required/); + assert.doesNotMatch(error.message, new RegExp(diagnosticSecret)); + return true; + }, +); + const mountainScenario = scenarios.find((scenario) => scenario.id === 'wikipedia-table-extraction'); const invalidMountainHeights = gradeScenario({ scenario: mountainScenario, From bb4e46f0722346519d27c6d219e60e6cb3e866a5 Mon Sep 17 00:00:00 2001 From: Emre Sokullu Date: Tue, 21 Jul 2026 13:27:44 +0300 Subject: [PATCH 2/2] Identify WebBrain E2E control requests --- ci/lib/webbrain-client.mjs | 2 ++ ci/test.mjs | 38 ++++++++++++++++++++++---------------- 2 files changed, 24 insertions(+), 16 deletions(-) diff --git a/ci/lib/webbrain-client.mjs b/ci/lib/webbrain-client.mjs index 513dc47e5..d17c070b2 100644 --- a/ci/lib/webbrain-client.mjs +++ b/ci/lib/webbrain-client.mjs @@ -146,6 +146,8 @@ export class GnippetsE2EClient { method, headers: { authorization: `Bearer ${this.controlToken}`, + accept: 'application/json', + 'user-agent': 'Mozilla/5.0 (compatible; WebBrainCloudE2E/1.0; +https://webbrain.cloud)', ...(body === undefined ? {} : { 'content-type': 'application/json' }), }, body: body === undefined ? undefined : JSON.stringify(body), diff --git a/ci/test.mjs b/ci/test.mjs index 9c2d8b0bc..7e191c34d 100644 --- a/ci/test.mjs +++ b/ci/test.mjs @@ -31,26 +31,30 @@ assert.deepEqual( ); const diagnosticSecret = 'diagnostic-secret-that-must-not-leak'; +let diagnosticRequest; const diagnosticClient = new GnippetsE2EClient({ baseUrl: 'https://gnippets.example', controlToken: diagnosticSecret, - fetchImpl: async () => ({ - ok: false, - status: 403, - headers: { - get(name) { - return { - server: 'cloudflare', - 'cf-ray': 'fixture-ray-IST', - 'cf-mitigated': 'challenge', - 'content-type': 'text/html', - }[name.toLowerCase()] || null; + fetchImpl: async (url, options) => { + diagnosticRequest = { url, options }; + return { + ok: false, + status: 403, + headers: { + get(name) { + return { + server: 'cloudflare', + 'cf-ray': 'fixture-ray-IST', + 'cf-mitigated': 'challenge', + 'content-type': 'text/html', + }[name.toLowerCase()] || null; + }, }, - }, - async text() { - return `Attention RequiredBearer ${diagnosticSecret}`; - }, - }), + async text() { + return `Attention RequiredBearer ${diagnosticSecret}`; + }, + }; + }, }); await assert.rejects( diagnosticClient.createRun('fixture'), @@ -64,6 +68,8 @@ await assert.rejects( return true; }, ); +assert.equal(diagnosticRequest.options.headers.accept, 'application/json'); +assert.match(diagnosticRequest.options.headers['user-agent'], /WebBrainCloudE2E/); const mountainScenario = scenarios.find((scenario) => scenario.id === 'wikipedia-table-extraction'); const invalidMountainHeights = gradeScenario({