Usage estimator performance

.changeset/spicy-symbols-find.md

@@ -0,0 +1,7 @@
+---
+'hive': minor
+---
+
+Use the monthly table in the usage estimator 
+
+This table has dramatically better performance than operations_hourly.

packages/services/commerce/src/rate-limit/limiter.ts

@@ -77,7 +77,10 @@ export function createRateLimiter(config: {
 
     const [records, operations] = await Promise.all([
       storage.getGetOrganizationsAndTargetsWithLimitInfo(),
-      usageEstimator.estimateOperationsForAllTargets(timeWindow),
+      usageEstimator.estimateCollectedOperationsForAllOrganizations({
+        month: now.getUTCMonth() + 1,
+        year: now.getUTCFullYear(),
+      }),
     ]);
 
     const totalTargets = records.reduce((acc, record) => acc + record.targets.length, 0);
@@ -113,10 +116,7 @@ export function createRateLimiter(config: {
       }
 
       const orgRecord = newCachedResult.get(record.organization)!;
-
-      for (const target of record.targets) {
-        orgRecord.operations.current += operations[target] || 0;
-      }
+      orgRecord.operations.current = operations[record.organization] ?? 0;
     }
 
     newCachedResult.forEach((orgRecord, orgId) => {

packages/services/commerce/src/usage-estimator/estimator.ts

@@ -88,5 +88,24 @@ export function createEstimator(config: {
         timeout: 15_000,
       });
     },
+    async estimateCollectedOperationsForAllOrganizations(input: { month: number; year: number }) {
+      const startOfMonth = `${input.year}-${String(input.month).padStart(2, '0')}-01`;
+      const result = await clickhouse.query<{
+        total: string;
+        organization: string;
+      }>({
+        query: sql`
+          SELECT
+            sum(total) as total,
+            organization
+          FROM monthly_overview
+          PREWHERE date=${startOfMonth}
+          GROUP BY organization
+        `,
+        queryId: 'usage_estimator_count_operations',
+        timeout: 15_000,
+      });
+      return Object.fromEntries(result.data.map(item => [item.organization, parseInt(item.total)]));
+    },
   };
 }

packages/services/commerce/src/usage-estimator/index.ts

@@ -17,8 +17,8 @@ export const usageEstimatorRouter = router({
       const now = new Date();
       const rangeValidation = z
         .number()
-        .min(now.getFullYear() - 1)
-        .max(new Date(now.setMonth(now.getMonth() + 1)).getFullYear());
+        .min(now.getUTCFullYear() - 1)
+        .max(new Date(now.setUTCMonth(now.getUTCMonth() + 1)).getUTCFullYear());
       const year = rangeValidation.parse(input.year);
 
       const estimationResponse =
@@ -53,6 +53,28 @@ export const usageEstimatorRouter = router({
         endTime: new Date(input.endTime),
       });
     }),
+  estimateCollectedOperationsForAllOrganizations: publicProcedure
+    .input(
+      z
+        .object({
+          month: z.number().min(1).max(12),
+          year: z.any(),
+        })
+        .required(),
+    )
+    .query(async ({ ctx, input }) => {
+      const now = new Date();
+      const rangeValidation = z
+        .number()
+        .min(now.getUTCFullYear() - 1)
+        .max(new Date(now.setUTCMonth(now.getUTCMonth() + 1)).getUTCFullYear());
+      const year = rangeValidation.parse(input.year);
+
+      return await ctx.usageEstimator.estimateCollectedOperationsForAllOrganizations({
+        month: input.month,
+        year,
+      });
+    }),
 });
 
 export type UsageEstimatorRouter = typeof usageEstimatorRouter;