From e02f897eb44c1faaa7d8c7a05e14711cce161827 Mon Sep 17 00:00:00 2001 From: jdolle <1841898+jdolle@users.noreply.github.com> Date: Tue, 7 Jul 2026 13:46:05 -0700 Subject: [PATCH 1/9] Use the optimized operations_by_target_hourly table in the usage estimator --- .changeset/spicy-symbols-find.md | 7 +++++++ .../services/commerce/src/usage-estimator/estimator.ts | 8 +++++++- 2 files changed, 14 insertions(+), 1 deletion(-) create mode 100644 .changeset/spicy-symbols-find.md diff --git a/.changeset/spicy-symbols-find.md b/.changeset/spicy-symbols-find.md new file mode 100644 index 00000000000..efbaca8d179 --- /dev/null +++ b/.changeset/spicy-symbols-find.md @@ -0,0 +1,7 @@ +--- +'hive': minor +--- + +Use the optimized operations_by_target_hourly table in the usage estimator if the start time is +after when the table was created. This table has dramatically better performance given the +selection, filters, and grouping. diff --git a/packages/services/commerce/src/usage-estimator/estimator.ts b/packages/services/commerce/src/usage-estimator/estimator.ts index bb85540c922..7ecdce69f51 100644 --- a/packages/services/commerce/src/usage-estimator/estimator.ts +++ b/packages/services/commerce/src/usage-estimator/estimator.ts @@ -50,6 +50,12 @@ export function createEstimator(config: { }, }); + const byTargetTableCreationDate = new Date('2027-06-11T00:00:00'); + const tableName = + input.startTime > byTargetTableCreationDate + ? 'operations_by_target_hourly' + : 'operations_hourly'; + const result = await clickhouse.query<{ total: string; target: string; @@ -58,7 +64,7 @@ export function createEstimator(config: { SELECT target, sum(total) as total - FROM operations_hourly + FROM ${tableName} ${filter} GROUP BY target `, From 16d9592cc712b79ee58ded55e72034e4eb4571aa Mon Sep 17 00:00:00 2001 From: jdolle <1841898+jdolle@users.noreply.github.com> Date: Tue, 7 Jul 2026 14:19:19 -0700 Subject: [PATCH 2/9] Take a new environment variable to configure when this cutover happened --- deployment/services/commerce.ts | 1 + integration-tests/docker-compose.integration.yaml | 1 + packages/services/commerce/.env.template | 1 + packages/services/commerce/src/environment.ts | 2 ++ .../services/commerce/src/usage-estimator/estimator.ts | 8 ++++---- 5 files changed, 9 insertions(+), 4 deletions(-) diff --git a/deployment/services/commerce.ts b/deployment/services/commerce.ts index 4fd2a12c717..0da2612108a 100644 --- a/deployment/services/commerce.ts +++ b/deployment/services/commerce.ts @@ -50,6 +50,7 @@ export function deployCommerce({ livenessProbe: '/_health', startupProbe: '/_health', env: { + OPERATIONS_BY_TARGET_TABLE_CREATED_AT: '2027-06-11T00:00:00Z', ...environment.envVars, SENTRY: sentry.enabled ? '1' : '0', WEB_APP_URL: `https://${environment.appDns}/`, diff --git a/integration-tests/docker-compose.integration.yaml b/integration-tests/docker-compose.integration.yaml index 412cf9739b8..2d3b7577fbd 100644 --- a/integration-tests/docker-compose.integration.yaml +++ b/integration-tests/docker-compose.integration.yaml @@ -137,6 +137,7 @@ services: CLICKHOUSE_PASSWORD: '${CLICKHOUSE_PASSWORD}' STRIPE_SECRET_KEY: empty PORT: 3009 + OPERATIONS_BY_TARGET_TABLE_CREATED_AT: '${OPERATIONS_BY_TARGET_TABLE_CREATED_AT}' # Overrides only to `docker-compose.community.yaml` from now on: server: diff --git a/packages/services/commerce/.env.template b/packages/services/commerce/.env.template index 7cfdf50b05a..4e57fcd38f7 100644 --- a/packages/services/commerce/.env.template +++ b/packages/services/commerce/.env.template @@ -14,3 +14,4 @@ POSTGRES_PORT=5432 POSTGRES_DB=registry WEB_APP_URL="http://localhost:3000" STRIPE_SECRET_KEY="empty" +OPERATIONS_BY_TARGET_TABLE_CREATED_AT="2027-06-11T00:00:00Z" \ No newline at end of file diff --git a/packages/services/commerce/src/environment.ts b/packages/services/commerce/src/environment.ts index c3226287d0e..57b64c76200 100644 --- a/packages/services/commerce/src/environment.ts +++ b/packages/services/commerce/src/environment.ts @@ -82,6 +82,7 @@ const PostgresModel = zod.object({ const HiveServicesModel = zod.object({ WEB_APP_URL: emptyString(zod.string().url().optional()), + OPERATIONS_BY_TARGET_TABLE_CREATED_AT: zod.coerce.date().optional().default(new Date()), }); const RateLimitModel = zod.object({ @@ -183,6 +184,7 @@ export const env = { : null, hiveServices: { webAppUrl: hiveServices.WEB_APP_URL, + operationsByTargetTableCreatedAt: hiveServices.OPERATIONS_BY_TARGET_TABLE_CREATED_AT, }, rateLimit: { limitCacheUpdateIntervalMs: rateLimit.LIMIT_CACHE_UPDATE_INTERVAL_MS ?? 60_000, diff --git a/packages/services/commerce/src/usage-estimator/estimator.ts b/packages/services/commerce/src/usage-estimator/estimator.ts index 7ecdce69f51..0bc1727a2c8 100644 --- a/packages/services/commerce/src/usage-estimator/estimator.ts +++ b/packages/services/commerce/src/usage-estimator/estimator.ts @@ -1,5 +1,6 @@ import { ClickHouse, HttpClient, OperationsReader, sql } from '@hive/api'; import type { ServiceLogger } from '@hive/service-common'; +import { env } from '../environment'; import { clickHouseElapsedDuration, clickHouseReadDuration } from './metrics'; export type UsageEstimator = ReturnType; @@ -50,11 +51,10 @@ export function createEstimator(config: { }, }); - const byTargetTableCreationDate = new Date('2027-06-11T00:00:00'); const tableName = - input.startTime > byTargetTableCreationDate - ? 'operations_by_target_hourly' - : 'operations_hourly'; + input.startTime > env.hiveServices.operationsByTargetTableCreatedAt + ? sql`operations_by_target_hourly` + : sql`operations_hourly`; const result = await clickhouse.query<{ total: string; From 8552b38b41c27c32c2669b419da43a0f55bea6c8 Mon Sep 17 00:00:00 2001 From: jdolle <1841898+jdolle@users.noreply.github.com> Date: Tue, 7 Jul 2026 21:14:09 -0700 Subject: [PATCH 3/9] Remove unnecessary env var in integration test docker file --- integration-tests/docker-compose.integration.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/integration-tests/docker-compose.integration.yaml b/integration-tests/docker-compose.integration.yaml index 2d3b7577fbd..412cf9739b8 100644 --- a/integration-tests/docker-compose.integration.yaml +++ b/integration-tests/docker-compose.integration.yaml @@ -137,7 +137,6 @@ services: CLICKHOUSE_PASSWORD: '${CLICKHOUSE_PASSWORD}' STRIPE_SECRET_KEY: empty PORT: 3009 - OPERATIONS_BY_TARGET_TABLE_CREATED_AT: '${OPERATIONS_BY_TARGET_TABLE_CREATED_AT}' # Overrides only to `docker-compose.community.yaml` from now on: server: From 0a741255e52071ada04e9b23e59580b04e960094 Mon Sep 17 00:00:00 2001 From: jdolle <1841898+jdolle@users.noreply.github.com> Date: Tue, 7 Jul 2026 21:28:41 -0700 Subject: [PATCH 4/9] Update deployment/services/commerce.ts Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> --- deployment/services/commerce.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deployment/services/commerce.ts b/deployment/services/commerce.ts index 0da2612108a..357f1b4e72c 100644 --- a/deployment/services/commerce.ts +++ b/deployment/services/commerce.ts @@ -50,7 +50,7 @@ export function deployCommerce({ livenessProbe: '/_health', startupProbe: '/_health', env: { - OPERATIONS_BY_TARGET_TABLE_CREATED_AT: '2027-06-11T00:00:00Z', + OPERATIONS_BY_TARGET_TABLE_CREATED_AT: '2026-06-11T00:00:00Z', ...environment.envVars, SENTRY: sentry.enabled ? '1' : '0', WEB_APP_URL: `https://${environment.appDns}/`, From 92544c5d3414fd5e6e51de6f59f6fbff432b5881 Mon Sep 17 00:00:00 2001 From: jdolle <1841898+jdolle@users.noreply.github.com> Date: Tue, 7 Jul 2026 21:29:44 -0700 Subject: [PATCH 5/9] fix env --- packages/services/commerce/.env.template | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/services/commerce/.env.template b/packages/services/commerce/.env.template index 4e57fcd38f7..97f95ed531d 100644 --- a/packages/services/commerce/.env.template +++ b/packages/services/commerce/.env.template @@ -14,4 +14,4 @@ POSTGRES_PORT=5432 POSTGRES_DB=registry WEB_APP_URL="http://localhost:3000" STRIPE_SECRET_KEY="empty" -OPERATIONS_BY_TARGET_TABLE_CREATED_AT="2027-06-11T00:00:00Z" \ No newline at end of file +OPERATIONS_BY_TARGET_TABLE_CREATED_AT="2026-06-11T00:00:00Z" \ No newline at end of file From e9ed038b9fe832618bd7c32cdd6ae0af251d7e44 Mon Sep 17 00:00:00 2001 From: jdolle <1841898+jdolle@users.noreply.github.com> Date: Thu, 9 Jul 2026 07:58:25 -0700 Subject: [PATCH 6/9] Add 30 day safeguard to data --- packages/services/commerce/src/usage-estimator/estimator.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/services/commerce/src/usage-estimator/estimator.ts b/packages/services/commerce/src/usage-estimator/estimator.ts index 0bc1727a2c8..e42f6738d46 100644 --- a/packages/services/commerce/src/usage-estimator/estimator.ts +++ b/packages/services/commerce/src/usage-estimator/estimator.ts @@ -50,9 +50,11 @@ export function createEstimator(config: { to: input.endTime, }, }); - + /** This table is truncated at 30 days worth of data. Make sure not to request beyond that. */ + const thirtyDaysInMs = 30 * 24 * 60 * 60 * 1_000; const tableName = - input.startTime > env.hiveServices.operationsByTargetTableCreatedAt + input.startTime > env.hiveServices.operationsByTargetTableCreatedAt && + input.startTime.getTime() > Date.now() - thirtyDaysInMs ? sql`operations_by_target_hourly` : sql`operations_hourly`; From 38129cf27d06a1769f14bdeb869a2c97820145a2 Mon Sep 17 00:00:00 2001 From: jdolle <1841898+jdolle@users.noreply.github.com> Date: Thu, 9 Jul 2026 13:18:27 -0700 Subject: [PATCH 7/9] Use monthly org totals table instead of hourly target table for usage estimator --- .changeset/spicy-symbols-find.md | 6 ++-- deployment/services/commerce.ts | 1 - packages/services/commerce/.env.template | 3 +- packages/services/commerce/src/environment.ts | 2 -- .../commerce/src/rate-limit/limiter.ts | 10 +++---- .../commerce/src/usage-estimator/estimator.ts | 29 +++++++++++++------ .../commerce/src/usage-estimator/index.ts | 26 +++++++++++++++-- 7 files changed, 53 insertions(+), 24 deletions(-) diff --git a/.changeset/spicy-symbols-find.md b/.changeset/spicy-symbols-find.md index efbaca8d179..fff7c26959d 100644 --- a/.changeset/spicy-symbols-find.md +++ b/.changeset/spicy-symbols-find.md @@ -2,6 +2,6 @@ 'hive': minor --- -Use the optimized operations_by_target_hourly table in the usage estimator if the start time is -after when the table was created. This table has dramatically better performance given the -selection, filters, and grouping. +Use the monthly table in the usage estimator + +This table has dramatically better performance than operations_hourly. diff --git a/deployment/services/commerce.ts b/deployment/services/commerce.ts index 357f1b4e72c..4fd2a12c717 100644 --- a/deployment/services/commerce.ts +++ b/deployment/services/commerce.ts @@ -50,7 +50,6 @@ export function deployCommerce({ livenessProbe: '/_health', startupProbe: '/_health', env: { - OPERATIONS_BY_TARGET_TABLE_CREATED_AT: '2026-06-11T00:00:00Z', ...environment.envVars, SENTRY: sentry.enabled ? '1' : '0', WEB_APP_URL: `https://${environment.appDns}/`, diff --git a/packages/services/commerce/.env.template b/packages/services/commerce/.env.template index 97f95ed531d..e9469cfa19c 100644 --- a/packages/services/commerce/.env.template +++ b/packages/services/commerce/.env.template @@ -13,5 +13,4 @@ POSTGRES_HOST=localhost POSTGRES_PORT=5432 POSTGRES_DB=registry WEB_APP_URL="http://localhost:3000" -STRIPE_SECRET_KEY="empty" -OPERATIONS_BY_TARGET_TABLE_CREATED_AT="2026-06-11T00:00:00Z" \ No newline at end of file +STRIPE_SECRET_KEY="empty" \ No newline at end of file diff --git a/packages/services/commerce/src/environment.ts b/packages/services/commerce/src/environment.ts index 57b64c76200..c3226287d0e 100644 --- a/packages/services/commerce/src/environment.ts +++ b/packages/services/commerce/src/environment.ts @@ -82,7 +82,6 @@ const PostgresModel = zod.object({ const HiveServicesModel = zod.object({ WEB_APP_URL: emptyString(zod.string().url().optional()), - OPERATIONS_BY_TARGET_TABLE_CREATED_AT: zod.coerce.date().optional().default(new Date()), }); const RateLimitModel = zod.object({ @@ -184,7 +183,6 @@ export const env = { : null, hiveServices: { webAppUrl: hiveServices.WEB_APP_URL, - operationsByTargetTableCreatedAt: hiveServices.OPERATIONS_BY_TARGET_TABLE_CREATED_AT, }, rateLimit: { limitCacheUpdateIntervalMs: rateLimit.LIMIT_CACHE_UPDATE_INTERVAL_MS ?? 60_000, diff --git a/packages/services/commerce/src/rate-limit/limiter.ts b/packages/services/commerce/src/rate-limit/limiter.ts index 77785b9beb5..a46344909a8 100644 --- a/packages/services/commerce/src/rate-limit/limiter.ts +++ b/packages/services/commerce/src/rate-limit/limiter.ts @@ -77,7 +77,10 @@ export function createRateLimiter(config: { const [records, operations] = await Promise.all([ storage.getGetOrganizationsAndTargetsWithLimitInfo(), - usageEstimator.estimateOperationsForAllTargets(timeWindow), + usageEstimator.estimateCollectedOperationsForAllOrganizations({ + month: now.getUTCMonth() + 1, + year: now.getUTCFullYear(), + }), ]); const totalTargets = records.reduce((acc, record) => acc + record.targets.length, 0); @@ -113,10 +116,7 @@ export function createRateLimiter(config: { } const orgRecord = newCachedResult.get(record.organization)!; - - for (const target of record.targets) { - orgRecord.operations.current += operations[target] || 0; - } + orgRecord.operations.current = operations[record.organization]; } newCachedResult.forEach((orgRecord, orgId) => { diff --git a/packages/services/commerce/src/usage-estimator/estimator.ts b/packages/services/commerce/src/usage-estimator/estimator.ts index e42f6738d46..7a3d6edf244 100644 --- a/packages/services/commerce/src/usage-estimator/estimator.ts +++ b/packages/services/commerce/src/usage-estimator/estimator.ts @@ -1,6 +1,5 @@ import { ClickHouse, HttpClient, OperationsReader, sql } from '@hive/api'; import type { ServiceLogger } from '@hive/service-common'; -import { env } from '../environment'; import { clickHouseElapsedDuration, clickHouseReadDuration } from './metrics'; export type UsageEstimator = ReturnType; @@ -50,13 +49,6 @@ export function createEstimator(config: { to: input.endTime, }, }); - /** This table is truncated at 30 days worth of data. Make sure not to request beyond that. */ - const thirtyDaysInMs = 30 * 24 * 60 * 60 * 1_000; - const tableName = - input.startTime > env.hiveServices.operationsByTargetTableCreatedAt && - input.startTime.getTime() > Date.now() - thirtyDaysInMs - ? sql`operations_by_target_hourly` - : sql`operations_hourly`; const result = await clickhouse.query<{ total: string; @@ -66,7 +58,7 @@ export function createEstimator(config: { SELECT target, sum(total) as total - FROM ${tableName} + FROM operations_hourly ${filter} GROUP BY target `, @@ -96,5 +88,24 @@ export function createEstimator(config: { timeout: 15_000, }); }, + async estimateCollectedOperationsForAllOrganizations(input: { month: number; year: number }) { + const startOfMonth = `${input.year}-${String(input.month).padStart(2, '0')}-01`; + const result = await clickhouse.query<{ + total: string; + organization: string; + }>({ + query: sql` + SELECT + sum(total) as total, + organization + FROM monthly_overview + PREWHERE date=${startOfMonth} + GROUP BY organization + `, + queryId: 'usage_estimator_count_operations', + timeout: 15_000, + }); + return Object.fromEntries(result.data.map(item => [item.organization, parseInt(item.total)])); + }, }; } diff --git a/packages/services/commerce/src/usage-estimator/index.ts b/packages/services/commerce/src/usage-estimator/index.ts index 77e4b661430..8b68de24244 100644 --- a/packages/services/commerce/src/usage-estimator/index.ts +++ b/packages/services/commerce/src/usage-estimator/index.ts @@ -17,8 +17,8 @@ export const usageEstimatorRouter = router({ const now = new Date(); const rangeValidation = z .number() - .min(now.getFullYear() - 1) - .max(new Date(now.setMonth(now.getMonth() + 1)).getFullYear()); + .min(now.getUTCFullYear() - 1) + .max(new Date(now.setUTCMonth(now.getUTCMonth() + 1)).getUTCFullYear()); const year = rangeValidation.parse(input.year); const estimationResponse = @@ -53,6 +53,28 @@ export const usageEstimatorRouter = router({ endTime: new Date(input.endTime), }); }), + estimateCollectedOperationsForAllOrganizations: publicProcedure + .input( + z + .object({ + month: z.number().min(1).max(12), + year: z.any(), + }) + .required(), + ) + .query(async ({ ctx, input }) => { + const now = new Date(); + const rangeValidation = z + .number() + .min(now.getUTCFullYear() - 1) + .max(new Date(now.setUTCMonth(now.getUTCMonth() + 1)).getUTCFullYear()); + const year = rangeValidation.parse(input.year); + + return await ctx.usageEstimator.estimateCollectedOperationsForAllOrganizations({ + month: input.month, + year, + }); + }), }); export type UsageEstimatorRouter = typeof usageEstimatorRouter; From 674f3306da5351a67a6d7e3cb53d3ce2264401c7 Mon Sep 17 00:00:00 2001 From: jdolle <1841898+jdolle@users.noreply.github.com> Date: Thu, 9 Jul 2026 13:20:27 -0700 Subject: [PATCH 8/9] revert extra space --- packages/services/commerce/.env.template | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/services/commerce/.env.template b/packages/services/commerce/.env.template index e9469cfa19c..7cfdf50b05a 100644 --- a/packages/services/commerce/.env.template +++ b/packages/services/commerce/.env.template @@ -13,4 +13,4 @@ POSTGRES_HOST=localhost POSTGRES_PORT=5432 POSTGRES_DB=registry WEB_APP_URL="http://localhost:3000" -STRIPE_SECRET_KEY="empty" \ No newline at end of file +STRIPE_SECRET_KEY="empty" From 3483fa54e8434f400c95b62707752440bd0aef80 Mon Sep 17 00:00:00 2001 From: jdolle <1841898+jdolle@users.noreply.github.com> Date: Thu, 16 Jul 2026 07:53:28 -0700 Subject: [PATCH 9/9] Apply suggestion from @n1ru4l Co-authored-by: Laurin --- packages/services/commerce/src/rate-limit/limiter.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/services/commerce/src/rate-limit/limiter.ts b/packages/services/commerce/src/rate-limit/limiter.ts index a46344909a8..6f1b4361b66 100644 --- a/packages/services/commerce/src/rate-limit/limiter.ts +++ b/packages/services/commerce/src/rate-limit/limiter.ts @@ -116,7 +116,7 @@ export function createRateLimiter(config: { } const orgRecord = newCachedResult.get(record.organization)!; - orgRecord.operations.current = operations[record.organization]; + orgRecord.operations.current = operations[record.organization] ?? 0; } newCachedResult.forEach((orgRecord, orgId) => {