diff --git a/.changeset/cute-otters-dance.md b/.changeset/cute-otters-dance.md new file mode 100644 index 0000000000..07531c9c3c --- /dev/null +++ b/.changeset/cute-otters-dance.md @@ -0,0 +1,5 @@ +--- +'hive': patch +--- + +Improve performance of schema checks using conditional breaking changes by using a table with an ordering that more closely aligns with the query for total request count for a target diff --git a/packages/services/api/src/modules/operations/providers/operations-reader.ts b/packages/services/api/src/modules/operations/providers/operations-reader.ts index 3319d63c76..1d1a361b82 100644 --- a/packages/services/api/src/modules/operations/providers/operations-reader.ts +++ b/packages/services/api/src/modules/operations/providers/operations-reader.ts @@ -1013,6 +1013,53 @@ export class OperationsReader { .tuple([z.object({ amountOfRequests: z.string() })]) .transform(data => ensureNumber(data[0].amountOfRequests)); + /** + * This is using clients_daily instead of operations_daily because the + * order by is more favorable in clients_daily since the client_name is + * right after target. However, this could be better without storing + * much more data, by creating another materialized view ordered by + * target, timestamp, then client_name. + */ + if (args.excludedClients && args.excludedClients.length > 0) { + return await this.clickHouse + .query({ + queryId: 'getTotalCountForSchemaCoordinatesExcludingClients', + query: sql` + SELECT + SUM("result"."total") AS "amountOfRequests" + FROM ( + SELECT + SUM("clients_daily"."total") AS "total" + FROM + "clients_daily" + PREWHERE + "clients_daily"."target" IN (${sql.array(args.targetIds, 'String')}) + AND "clients_daily"."timestamp" >= toDateTime(${formatDate(args.period.from)}, 'UTC') + AND "clients_daily"."timestamp" <= toDateTime(${formatDate(args.period.to)}, 'UTC') + AND "clients_daily"."client_name" NOT IN (${sql.array(args.excludedClients, 'String')}) + + UNION ALL + + SELECT + SUM("subscription_operations_daily"."total") AS "total" + FROM + "subscription_operations_daily" + PREWHERE + "subscription_operations_daily"."target" IN (${sql.array(args.targetIds, 'String')}) + AND "subscription_operations_daily"."timestamp" >= toDateTime(${formatDate(args.period.from)}, 'UTC') + AND "subscription_operations_daily"."timestamp" <= toDateTime(${formatDate(args.period.to)}, 'UTC') + AND "subscription_operations_daily"."client_name" NOT IN (${sql.array(args.excludedClients, 'String')}) + ) AS "result" + `, + timeout: 10_000, + }) + .then(result => TotalCountModel.parse(result.data)); + } + + /** + * This uses the "operations_by_target_hourly" table because there is no daily + * and even with 24 hours in a day, the table's order by will more than compensate. + */ return await this.clickHouse .query({ queryId: 'getTotalCountForSchemaCoordinates', @@ -1021,14 +1068,13 @@ export class OperationsReader { SUM("result"."total") AS "amountOfRequests" FROM ( SELECT - SUM("operations_daily"."total") AS "total" + SUM("operations_by_target_hourly"."total") AS "total" FROM - "operations_daily" + "operations_by_target_hourly" PREWHERE - "operations_daily"."target" IN (${sql.array(args.targetIds, 'String')}) - AND "operations_daily"."timestamp" >= toDateTime(${formatDate(args.period.from)}, 'UTC') - AND "operations_daily"."timestamp" <= toDateTime(${formatDate(args.period.to)}, 'UTC') - ${args.excludedClients ? sql`AND "operations_daily"."client_name" NOT IN (${sql.array(args.excludedClients, 'String')})` : sql``} + "operations_by_target_hourly"."target" IN (${sql.array(args.targetIds, 'String')}) + AND "operations_by_target_hourly"."timestamp" >= toDateTime(${formatDate(args.period.from)}, 'UTC') + AND "operations_by_target_hourly"."timestamp" <= toDateTime(${formatDate(args.period.to)}, 'UTC') UNION ALL @@ -1040,7 +1086,6 @@ export class OperationsReader { "subscription_operations_daily"."target" IN (${sql.array(args.targetIds, 'String')}) AND "subscription_operations_daily"."timestamp" >= toDateTime(${formatDate(args.period.from)}, 'UTC') AND "subscription_operations_daily"."timestamp" <= toDateTime(${formatDate(args.period.to)}, 'UTC') - ${args.excludedClients ? sql`AND "subscription_operations_daily"."client_name" NOT IN (${sql.array(args.excludedClients, 'String')})` : sql``} ) AS "result" `, timeout: 10_000,