From 9a020f446c7a098101eeb77e52c7b5b07f12daa2 Mon Sep 17 00:00:00 2001 From: Jonathan Brennan Date: Thu, 9 Jul 2026 21:04:11 -0500 Subject: [PATCH 1/8] Raise usage e2e ingestion poll and per-test timeouts to reduce cold-start flake --- e2e/helpers/usage.ts | 4 ++-- e2e/specs/usage.spec.ts | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/e2e/helpers/usage.ts b/e2e/helpers/usage.ts index c08d30ab711..cfa0a33ba13 100644 --- a/e2e/helpers/usage.ts +++ b/e2e/helpers/usage.ts @@ -90,7 +90,7 @@ export function createUsageHelper( return page.getByRole('link').filter({ hasText: operationName }).count(); }, - { timeout: 60_000, intervals: [2_000] }, + { timeout: 90_000, intervals: [2_000] }, ) .toBeGreaterThan(0); }, @@ -102,7 +102,7 @@ export function createUsageHelper( return page.getByText(version, { exact: true }).count(); }, - { timeout: 60_000, intervals: [2_000] }, + { timeout: 90_000, intervals: [2_000] }, ) .toBeGreaterThan(0); }, diff --git a/e2e/specs/usage.spec.ts b/e2e/specs/usage.spec.ts index 2a19dc2b51a..52637c249bf 100644 --- a/e2e/specs/usage.spec.ts +++ b/e2e/specs/usage.spec.ts @@ -7,7 +7,7 @@ import type { UsageHelper } from '../helpers/usage'; // Each test waits on ClickHouse ingestion through several stacked polls (see helpers/usage.ts). // Keep the timeout above the sum of those poll budgets so a slow-but-succeeding poll isn't cut // off mid-wait by the per-test deadline. -test.describe.configure({ mode: 'serial', timeout: 180_000 }); +test.describe.configure({ mode: 'serial', timeout: 300_000 }); type UsageReport = { size: number; From a7a9a1617f64c5c17f882e0b0f880a6bf8652ab5 Mon Sep 17 00:00:00 2001 From: Jonathan Brennan Date: Fri, 10 Jul 2026 08:25:52 -0500 Subject: [PATCH 2/8] Flush ClickHouse async inserts fast in e2e so usage reports are visible within poll window --- docker/docker-compose.end2end.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docker/docker-compose.end2end.yml b/docker/docker-compose.end2end.yml index 1227a185b34..a444e28b6de 100644 --- a/docker/docker-compose.end2end.yml +++ b/docker/docker-compose.end2end.yml @@ -47,6 +47,12 @@ services: ports: - '6379:6379' + usage-ingestor: + environment: + # Flush ClickHouse async inserts fast so a report is visible in Insights within the + # e2e poll window (prod keeps the default 30s batching). + CLICKHOUSE_ASYNC_INSERT_BUSY_TIMEOUT_MS: '1000' + server: environment: SUPERTOKENS_RATE_LIMIT: '0' From 2ca3fe71a730d87c7f050e5c212f8494a2159c05 Mon Sep 17 00:00:00 2001 From: Jonathan Brennan Date: Fri, 10 Jul 2026 09:15:54 -0500 Subject: [PATCH 3/8] Surface OIDC signinup response body on parse failure to diagnose flaky auth test --- integration-tests/testkit/oidc-integration.ts | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/integration-tests/testkit/oidc-integration.ts b/integration-tests/testkit/oidc-integration.ts index d77e3533694..45d753b5516 100644 --- a/integration-tests/testkit/oidc-integration.ts +++ b/integration-tests/testkit/oidc-integration.ts @@ -299,7 +299,7 @@ export async function createOIDCIntegration(args: { const rawBody = await result.json(); - const body = z + const parsed = z .object({ user: z.object({ id: z.string(), @@ -311,7 +311,16 @@ export async function createOIDCIntegration(args: { ), }), }) - .parse(rawBody); + .safeParse(rawBody); + + if (!parsed.success) { + // SuperTokens answers 200 with a non-OK status (e.g. SIGN_IN_UP_NOT_ALLOWED) and no + // `user`. Surface the actual body so a flaky sign-in is diagnosable instead of a bare + // ZodError that hides why it failed. + throw new Error('Unexpected signinup response: ' + JSON.stringify(rawBody)); + } + + const body = parsed.data; const cookies = setCookie.parse(result.headers.getSetCookie()); return { accessToken: cookies.find(c => c.name === 'sAccessToken')?.value ?? '', From a7fb16a9e1aad62f2c4db6e54f619a1170318886 Mon Sep 17 00:00:00 2001 From: Jonathan Brennan Date: Fri, 10 Jul 2026 13:48:21 -0500 Subject: [PATCH 4/8] Lower e2e ClickHouse async insert flush to 100ms --- docker/docker-compose.end2end.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/docker-compose.end2end.yml b/docker/docker-compose.end2end.yml index a444e28b6de..367c656c8b2 100644 --- a/docker/docker-compose.end2end.yml +++ b/docker/docker-compose.end2end.yml @@ -51,7 +51,7 @@ services: environment: # Flush ClickHouse async inserts fast so a report is visible in Insights within the # e2e poll window (prod keeps the default 30s batching). - CLICKHOUSE_ASYNC_INSERT_BUSY_TIMEOUT_MS: '1000' + CLICKHOUSE_ASYNC_INSERT_BUSY_TIMEOUT_MS: '100' server: environment: From daa64dde7ac3c78e1e2cf554ead7e3bcccd77c9d Mon Sep 17 00:00:00 2001 From: Jonathan Brennan Date: Fri, 10 Jul 2026 13:50:07 -0500 Subject: [PATCH 5/8] Tighten usage e2e poll and flush timeouts now that async-insert flush is the real fix --- e2e/helpers/usage.ts | 4 ++-- e2e/specs/usage.spec.ts | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/e2e/helpers/usage.ts b/e2e/helpers/usage.ts index cfa0a33ba13..66aaedc99e8 100644 --- a/e2e/helpers/usage.ts +++ b/e2e/helpers/usage.ts @@ -90,7 +90,7 @@ export function createUsageHelper( return page.getByRole('link').filter({ hasText: operationName }).count(); }, - { timeout: 90_000, intervals: [2_000] }, + { timeout: 30_000, intervals: [2_000] }, ) .toBeGreaterThan(0); }, @@ -102,7 +102,7 @@ export function createUsageHelper( return page.getByText(version, { exact: true }).count(); }, - { timeout: 90_000, intervals: [2_000] }, + { timeout: 30_000, intervals: [2_000] }, ) .toBeGreaterThan(0); }, diff --git a/e2e/specs/usage.spec.ts b/e2e/specs/usage.spec.ts index 52637c249bf..a1ca1eafef6 100644 --- a/e2e/specs/usage.spec.ts +++ b/e2e/specs/usage.spec.ts @@ -7,7 +7,7 @@ import type { UsageHelper } from '../helpers/usage'; // Each test waits on ClickHouse ingestion through several stacked polls (see helpers/usage.ts). // Keep the timeout above the sum of those poll budgets so a slow-but-succeeding poll isn't cut // off mid-wait by the per-test deadline. -test.describe.configure({ mode: 'serial', timeout: 300_000 }); +test.describe.configure({ mode: 'serial', timeout: 120_000 }); type UsageReport = { size: number; From e1e82eb567fc727fe98f5ead8263021c86dd368a Mon Sep 17 00:00:00 2001 From: Jonathan Brennan Date: Sat, 11 Jul 2026 19:24:03 -0500 Subject: [PATCH 6/8] Force CH async insert flush in usage e2e for deterministic read-after-write --- docker/docker-compose.end2end.yml | 11 ++++++----- e2e/helpers/usage.ts | 26 ++++++++++++++++++++++++++ 2 files changed, 32 insertions(+), 5 deletions(-) diff --git a/docker/docker-compose.end2end.yml b/docker/docker-compose.end2end.yml index 367c656c8b2..2a342c3f97c 100644 --- a/docker/docker-compose.end2end.yml +++ b/docker/docker-compose.end2end.yml @@ -47,11 +47,12 @@ services: ports: - '6379:6379' - usage-ingestor: - environment: - # Flush ClickHouse async inserts fast so a report is visible in Insights within the - # e2e poll window (prod keeps the default 30s batching). - CLICKHOUSE_ASYNC_INSERT_BUSY_TIMEOUT_MS: '100' + clickhouse: + # Expose the HTTP interface so the usage e2e helper can force a synchronous flush of the + # async insert queue (SYSTEM FLUSH ASYNC INSERT QUEUE), making ingested reports queryable + # immediately instead of racing the non-deterministic async-insert flush timer. + ports: + - '8123:8123' server: environment: diff --git a/e2e/helpers/usage.ts b/e2e/helpers/usage.ts index 66aaedc99e8..a4c218eb17c 100644 --- a/e2e/helpers/usage.ts +++ b/e2e/helpers/usage.ts @@ -21,6 +21,30 @@ export function createUsageHelper( (typeof baseURL === 'string' && baseURL.startsWith('http://localhost:3000')); const usageEndpoint = isLocal ? 'http://localhost:4001' : 'http://localhost:8081'; + // The usage-ingestor writes to ClickHouse with async_insert + wait_for_async_insert=0, so a + // report isn't queryable until the async buffer flushes, and that flush timing is + // non-deterministic under CI load. Force a synchronous flush before each read so the polls + // below are deterministic instead of racing the flush timer. This is the approach ClickHouse + // itself uses for async-insert read-after-write in tests. + const clickhouseEndpoint = 'http://localhost:8123'; + const clickhouseUser = process.env.CLICKHOUSE_USER ?? 'clickhouse'; + const clickhousePassword = process.env.CLICKHOUSE_PASSWORD ?? 'wowverysecuremuchsecret'; + + async function flushAsyncInserts() { + const response = await request.post(clickhouseEndpoint, { + params: { query: 'SYSTEM FLUSH ASYNC INSERT QUEUE' }, + headers: { + 'X-ClickHouse-User': clickhouseUser, + 'X-ClickHouse-Key': clickhousePassword, + }, + }); + if (!response.ok()) { + throw new Error( + `ClickHouse async insert flush failed (${response.status()}): ${await response.text()}`, + ); + } + } + async function reloadInsightsPage() { await page.reload({ waitUntil: 'domcontentloaded' }); await page.locator('#root > *').first().waitFor({ state: 'attached' }); @@ -86,6 +110,7 @@ export function createUsageHelper( await expect .poll( async () => { + await flushAsyncInserts(); await reloadInsightsPage(); return page.getByRole('link').filter({ hasText: operationName }).count(); @@ -98,6 +123,7 @@ export function createUsageHelper( await expect .poll( async () => { + await flushAsyncInserts(); await reloadInsightsPage(); return page.getByText(version, { exact: true }).count(); From b7162b10bae764039ca9b0bfdee27ea9892ae577 Mon Sep 17 00:00:00 2001 From: Jonathan Brennan Date: Sat, 11 Jul 2026 20:02:45 -0500 Subject: [PATCH 7/8] Fix flush to send SQL in request body so it actually runs --- e2e/helpers/usage.ts | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/e2e/helpers/usage.ts b/e2e/helpers/usage.ts index a4c218eb17c..4ec751c3069 100644 --- a/e2e/helpers/usage.ts +++ b/e2e/helpers/usage.ts @@ -29,14 +29,15 @@ export function createUsageHelper( const clickhouseEndpoint = 'http://localhost:8123'; const clickhouseUser = process.env.CLICKHOUSE_USER ?? 'clickhouse'; const clickhousePassword = process.env.CLICKHOUSE_PASSWORD ?? 'wowverysecuremuchsecret'; + const clickhouseAuth = Buffer.from(`${clickhouseUser}:${clickhousePassword}`).toString('base64'); async function flushAsyncInserts() { + // Send the SQL in the request body with Basic auth, mirroring + // integration-tests/testkit/clickhouse.ts. Passing the query as a URL param POSTs to the + // server root, which just returns "Ok." without running anything. const response = await request.post(clickhouseEndpoint, { - params: { query: 'SYSTEM FLUSH ASYNC INSERT QUEUE' }, - headers: { - 'X-ClickHouse-User': clickhouseUser, - 'X-ClickHouse-Key': clickhousePassword, - }, + data: 'SYSTEM FLUSH ASYNC INSERT QUEUE', + headers: { Authorization: `Basic ${clickhouseAuth}` }, }); if (!response.ok()) { throw new Error( From d7c0b7a235008f53bfd8b4050f4afcd98d5ad16c Mon Sep 17 00:00:00 2001 From: Jonathan Brennan Date: Mon, 13 Jul 2026 16:57:32 -0500 Subject: [PATCH 8/8] use wait_for_async_insert to fix flaky Insights tests --- docker/docker-compose.end2end.yml | 12 ++++----- e2e/helpers/usage.ts | 27 ------------------- .../usage-ingestor/src/environment.ts | 6 +++++ .../services/usage-ingestor/src/writer.ts | 3 ++- 4 files changed, 14 insertions(+), 34 deletions(-) diff --git a/docker/docker-compose.end2end.yml b/docker/docker-compose.end2end.yml index 2a342c3f97c..45338f5fe0a 100644 --- a/docker/docker-compose.end2end.yml +++ b/docker/docker-compose.end2end.yml @@ -47,12 +47,12 @@ services: ports: - '6379:6379' - clickhouse: - # Expose the HTTP interface so the usage e2e helper can force a synchronous flush of the - # async insert queue (SYSTEM FLUSH ASYNC INSERT QUEUE), making ingested reports queryable - # immediately instead of racing the non-deterministic async-insert flush timer. - ports: - - '8123:8123' + usage-ingestor: + environment: + # Wait for the async insert to flush before acking, so a reported operation is queryable in + # Insights immediately instead of racing ClickHouse's non-deterministic async-insert timer + # (the source of usage e2e flakiness). Prod keeps the default fire-and-forget behavior. + CLICKHOUSE_WAIT_FOR_ASYNC_INSERT: '1' server: environment: diff --git a/e2e/helpers/usage.ts b/e2e/helpers/usage.ts index 4ec751c3069..66aaedc99e8 100644 --- a/e2e/helpers/usage.ts +++ b/e2e/helpers/usage.ts @@ -21,31 +21,6 @@ export function createUsageHelper( (typeof baseURL === 'string' && baseURL.startsWith('http://localhost:3000')); const usageEndpoint = isLocal ? 'http://localhost:4001' : 'http://localhost:8081'; - // The usage-ingestor writes to ClickHouse with async_insert + wait_for_async_insert=0, so a - // report isn't queryable until the async buffer flushes, and that flush timing is - // non-deterministic under CI load. Force a synchronous flush before each read so the polls - // below are deterministic instead of racing the flush timer. This is the approach ClickHouse - // itself uses for async-insert read-after-write in tests. - const clickhouseEndpoint = 'http://localhost:8123'; - const clickhouseUser = process.env.CLICKHOUSE_USER ?? 'clickhouse'; - const clickhousePassword = process.env.CLICKHOUSE_PASSWORD ?? 'wowverysecuremuchsecret'; - const clickhouseAuth = Buffer.from(`${clickhouseUser}:${clickhousePassword}`).toString('base64'); - - async function flushAsyncInserts() { - // Send the SQL in the request body with Basic auth, mirroring - // integration-tests/testkit/clickhouse.ts. Passing the query as a URL param POSTs to the - // server root, which just returns "Ok." without running anything. - const response = await request.post(clickhouseEndpoint, { - data: 'SYSTEM FLUSH ASYNC INSERT QUEUE', - headers: { Authorization: `Basic ${clickhouseAuth}` }, - }); - if (!response.ok()) { - throw new Error( - `ClickHouse async insert flush failed (${response.status()}): ${await response.text()}`, - ); - } - } - async function reloadInsightsPage() { await page.reload({ waitUntil: 'domcontentloaded' }); await page.locator('#root > *').first().waitFor({ state: 'attached' }); @@ -111,7 +86,6 @@ export function createUsageHelper( await expect .poll( async () => { - await flushAsyncInserts(); await reloadInsightsPage(); return page.getByRole('link').filter({ hasText: operationName }).count(); @@ -124,7 +98,6 @@ export function createUsageHelper( await expect .poll( async () => { - await flushAsyncInserts(); await reloadInsightsPage(); return page.getByText(version, { exact: true }).count(); diff --git a/packages/services/usage-ingestor/src/environment.ts b/packages/services/usage-ingestor/src/environment.ts index 99ea345b30a..17709962815 100644 --- a/packages/services/usage-ingestor/src/environment.ts +++ b/packages/services/usage-ingestor/src/environment.ts @@ -81,6 +81,11 @@ const ClickHouseModel = zod.object({ CLICKHOUSE_PASSWORD: zod.string(), CLICKHOUSE_ASYNC_INSERT_BUSY_TIMEOUT_MS: emptyString(NumberFromString.optional()), CLICKHOUSE_ASYNC_INSERT_MAX_DATA_SIZE: emptyString(NumberFromString.optional()), + // Whether ClickHouse waits for the async insert to flush before acking. Defaults to 0 + // (fire-and-forget) for throughput; e2e sets it to 1 so reports are queryable immediately. + CLICKHOUSE_WAIT_FOR_ASYNC_INSERT: emptyString( + zod.union([zod.literal('0'), zod.literal('1')]).optional(), + ), }); const PrometheusModel = zod.object({ @@ -238,6 +243,7 @@ export const env = { password: clickhouse.CLICKHOUSE_PASSWORD, async_insert_busy_timeout_ms: clickhouse.CLICKHOUSE_ASYNC_INSERT_BUSY_TIMEOUT_MS ?? 30_000, async_insert_max_data_size: clickhouse.CLICKHOUSE_ASYNC_INSERT_MAX_DATA_SIZE ?? 200_000_000, + wait_for_async_insert: clickhouse.CLICKHOUSE_WAIT_FOR_ASYNC_INSERT === '1' ? 1 : 0, }, heartbeat: base.HEARTBEAT_ENDPOINT ? { endpoint: base.HEARTBEAT_ENDPOINT } : null, sentry: sentry.SENTRY === '1' ? { dsn: sentry.SENTRY_DSN } : null, diff --git a/packages/services/usage-ingestor/src/writer.ts b/packages/services/usage-ingestor/src/writer.ts index ebaabd9fbc7..cc584c6de41 100644 --- a/packages/services/usage-ingestor/src/writer.ts +++ b/packages/services/usage-ingestor/src/writer.ts @@ -20,6 +20,7 @@ export interface ClickHouseConfig { password: string; async_insert_busy_timeout_ms: number; async_insert_max_data_size: number; + wait_for_async_insert: number; } const operationsFields = operationsOrder.join(', '); @@ -153,7 +154,7 @@ async function writeCsv( searchParams: { query, async_insert: 1, - wait_for_async_insert: 0, + wait_for_async_insert: config.wait_for_async_insert, async_insert_busy_timeout_ms: config.async_insert_busy_timeout_ms, async_insert_max_data_size: config.async_insert_max_data_size, },