From ad0501e6ecde1c267437698e4ca5d9806270a366 Mon Sep 17 00:00:00 2001 From: jdolle <1841898+jdolle@users.noreply.github.com> Date: Wed, 1 Jul 2026 20:37:27 -0700 Subject: [PATCH 1/4] Improve schema check and push latency for conditional breaking changes --- .changeset/cute-otters-dance.md | 6 ++++++ .../operations/providers/operations-reader.ts | 19 +++++++++++++------ 2 files changed, 19 insertions(+), 6 deletions(-) create mode 100644 .changeset/cute-otters-dance.md diff --git a/.changeset/cute-otters-dance.md b/.changeset/cute-otters-dance.md new file mode 100644 index 0000000000..353cde3a17 --- /dev/null +++ b/.changeset/cute-otters-dance.md @@ -0,0 +1,6 @@ +--- +'hive': patch +--- + +Improve performance of schema checks by using a table with an ordering that more closely aligns with +the query for total request count for a target" diff --git a/packages/services/api/src/modules/operations/providers/operations-reader.ts b/packages/services/api/src/modules/operations/providers/operations-reader.ts index 3319d63c76..430654a352 100644 --- a/packages/services/api/src/modules/operations/providers/operations-reader.ts +++ b/packages/services/api/src/modules/operations/providers/operations-reader.ts @@ -1013,6 +1013,13 @@ export class OperationsReader { .tuple([z.object({ amountOfRequests: z.string() })]) .transform(data => ensureNumber(data[0].amountOfRequests)); + /** + * This is using clients_daily instead of operations_daily because the + * order by is more favorable in clients_daily since the client_name is + * right after target. However, this could be better without storing + * much more data, by creating another materialized view ordered by + * target, timestamp, then client_name. + */ return await this.clickHouse .query({ queryId: 'getTotalCountForSchemaCoordinates', @@ -1021,14 +1028,14 @@ export class OperationsReader { SUM("result"."total") AS "amountOfRequests" FROM ( SELECT - SUM("operations_daily"."total") AS "total" + SUM("clients_daily"."total") AS "total" FROM - "operations_daily" + "clients_daily" PREWHERE - "operations_daily"."target" IN (${sql.array(args.targetIds, 'String')}) - AND "operations_daily"."timestamp" >= toDateTime(${formatDate(args.period.from)}, 'UTC') - AND "operations_daily"."timestamp" <= toDateTime(${formatDate(args.period.to)}, 'UTC') - ${args.excludedClients ? sql`AND "operations_daily"."client_name" NOT IN (${sql.array(args.excludedClients, 'String')})` : sql``} + "clients_daily"."target" IN (${sql.array(args.targetIds, 'String')}) + AND "clients_daily"."timestamp" >= toDateTime(${formatDate(args.period.from)}, 'UTC') + AND "clients_daily"."timestamp" <= toDateTime(${formatDate(args.period.to)}, 'UTC') + ${args.excludedClients ? sql`AND "clients_daily"."client_name" NOT IN (${sql.array(args.excludedClients, 'String')})` : sql``} UNION ALL From a13d68c9f038c90814130dcc7de7c489f05a2ae8 Mon Sep 17 00:00:00 2001 From: jdolle <1841898+jdolle@users.noreply.github.com> Date: Wed, 1 Jul 2026 21:07:05 -0700 Subject: [PATCH 2/4] Use operations_by_target_hourly --- .../operations/providers/operations-reader.ts | 52 ++++++++++++++++--- 1 file changed, 45 insertions(+), 7 deletions(-) diff --git a/packages/services/api/src/modules/operations/providers/operations-reader.ts b/packages/services/api/src/modules/operations/providers/operations-reader.ts index 430654a352..986702fadc 100644 --- a/packages/services/api/src/modules/operations/providers/operations-reader.ts +++ b/packages/services/api/src/modules/operations/providers/operations-reader.ts @@ -1020,6 +1020,46 @@ export class OperationsReader { * much more data, by creating another materialized view ordered by * target, timestamp, then client_name. */ + if (args.excludedClients) { + return await this.clickHouse + .query({ + queryId: 'getTotalCountForSchemaCoordinatesExcludingClients', + query: sql` + SELECT + SUM("result"."total") AS "amountOfRequests" + FROM ( + SELECT + SUM("clients_daily"."total") AS "total" + FROM + "clients_daily" + PREWHERE + "clients_daily"."target" IN (${sql.array(args.targetIds, 'String')}) + AND "clients_daily"."timestamp" >= toDateTime(${formatDate(args.period.from)}, 'UTC') + AND "clients_daily"."timestamp" <= toDateTime(${formatDate(args.period.to)}, 'UTC') + AND "clients_daily"."client_name" NOT IN (${sql.array(args.excludedClients, 'String')}) + + UNION ALL + + SELECT + SUM("subscription_operations_daily"."total") AS "total" + FROM + "subscription_operations_daily" + PREWHERE + "subscription_operations_daily"."target" IN (${sql.array(args.targetIds, 'String')}) + AND "subscription_operations_daily"."timestamp" >= toDateTime(${formatDate(args.period.from)}, 'UTC') + AND "subscription_operations_daily"."timestamp" <= toDateTime(${formatDate(args.period.to)}, 'UTC') + AND "subscription_operations_daily"."client_name" NOT IN (${sql.array(args.excludedClients, 'String')}) + ) AS "result" + `, + timeout: 10_000, + }) + .then(result => TotalCountModel.parse(result.data)); + } + + /** + * This uses the "operations_by_target_hourly" table because there is no daily + * and even with 24 hours in a day, the table's order by will more than compensate. + */ return await this.clickHouse .query({ queryId: 'getTotalCountForSchemaCoordinates', @@ -1028,14 +1068,13 @@ export class OperationsReader { SUM("result"."total") AS "amountOfRequests" FROM ( SELECT - SUM("clients_daily"."total") AS "total" + SUM("operations_by_target_hourly"."total") AS "total" FROM - "clients_daily" + "operations_by_target_hourly" PREWHERE - "clients_daily"."target" IN (${sql.array(args.targetIds, 'String')}) - AND "clients_daily"."timestamp" >= toDateTime(${formatDate(args.period.from)}, 'UTC') - AND "clients_daily"."timestamp" <= toDateTime(${formatDate(args.period.to)}, 'UTC') - ${args.excludedClients ? sql`AND "clients_daily"."client_name" NOT IN (${sql.array(args.excludedClients, 'String')})` : sql``} + "operations_by_target_hourly"."target" IN (${sql.array(args.targetIds, 'String')}) + AND "operations_by_target_hourly"."timestamp" >= toDateTime(${formatDate(args.period.from)}, 'UTC') + AND "operations_by_target_hourly"."timestamp" <= toDateTime(${formatDate(args.period.to)}, 'UTC') UNION ALL @@ -1047,7 +1086,6 @@ export class OperationsReader { "subscription_operations_daily"."target" IN (${sql.array(args.targetIds, 'String')}) AND "subscription_operations_daily"."timestamp" >= toDateTime(${formatDate(args.period.from)}, 'UTC') AND "subscription_operations_daily"."timestamp" <= toDateTime(${formatDate(args.period.to)}, 'UTC') - ${args.excludedClients ? sql`AND "subscription_operations_daily"."client_name" NOT IN (${sql.array(args.excludedClients, 'String')})` : sql``} ) AS "result" `, timeout: 10_000, From 95ae4490f86600b34f2546f8b9bd9cf7e9522aca Mon Sep 17 00:00:00 2001 From: jdolle <1841898+jdolle@users.noreply.github.com> Date: Wed, 1 Jul 2026 21:09:00 -0700 Subject: [PATCH 3/4] Changeset --- .changeset/cute-otters-dance.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.changeset/cute-otters-dance.md b/.changeset/cute-otters-dance.md index 353cde3a17..6903a44e1c 100644 --- a/.changeset/cute-otters-dance.md +++ b/.changeset/cute-otters-dance.md @@ -2,5 +2,4 @@ 'hive': patch --- -Improve performance of schema checks by using a table with an ordering that more closely aligns with -the query for total request count for a target" +Improve performance of schema checks using conditional breaking changes by using a table with an ordering that more closely aligns with the query for total request count for a target" From 1465bb9f4282cd69d2f047df0878fab8847681bc Mon Sep 17 00:00:00 2001 From: jdolle <1841898+jdolle@users.noreply.github.com> Date: Thu, 2 Jul 2026 17:33:03 -0700 Subject: [PATCH 4/4] Fix condition and typo --- .changeset/cute-otters-dance.md | 2 +- .../api/src/modules/operations/providers/operations-reader.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.changeset/cute-otters-dance.md b/.changeset/cute-otters-dance.md index 6903a44e1c..07531c9c3c 100644 --- a/.changeset/cute-otters-dance.md +++ b/.changeset/cute-otters-dance.md @@ -2,4 +2,4 @@ 'hive': patch --- -Improve performance of schema checks using conditional breaking changes by using a table with an ordering that more closely aligns with the query for total request count for a target" +Improve performance of schema checks using conditional breaking changes by using a table with an ordering that more closely aligns with the query for total request count for a target diff --git a/packages/services/api/src/modules/operations/providers/operations-reader.ts b/packages/services/api/src/modules/operations/providers/operations-reader.ts index 986702fadc..1d1a361b82 100644 --- a/packages/services/api/src/modules/operations/providers/operations-reader.ts +++ b/packages/services/api/src/modules/operations/providers/operations-reader.ts @@ -1020,7 +1020,7 @@ export class OperationsReader { * much more data, by creating another materialized view ordered by * target, timestamp, then client_name. */ - if (args.excludedClients) { + if (args.excludedClients && args.excludedClients.length > 0) { return await this.clickHouse .query({ queryId: 'getTotalCountForSchemaCoordinatesExcludingClients',