Billing - optimize usageEvent CH table (#20019)

- Update usageEvent clickhouse table, partitioning, indexing and
projection (auto materialized view) to optimize credit usage queries
- Add caching for available credits and billing subscription


To do in next PR: deprecate enforceCapUsage cron. Bonus : real-time on
billingSubscription

packages/twenty-server/src/engine/core-modules/usage/services/usage-analytics.service.ts

@@ -3,7 +3,7 @@
 import { Injectable } from '@nestjs/common';
 
 import { ClickHouseService } from 'src/database/clickHouse/clickHouse.service';
-import { formatDateForClickHouse } from 'src/database/clickHouse/clickHouse.util';
+import { formatDateTimeForClickHouse } from 'src/database/clickHouse/clickHouse.util';
 import { fillUsageTimeSeriesGaps } from 'src/engine/core-modules/usage/utils/fill-usage-time-series-gaps.util';
 import { toDisplayCredits } from 'src/engine/core-modules/usage/utils/to-display-credits.util';
 import { toDollars } from 'src/engine/core-modules/usage/utils/to-dollars.util';
@@ -75,8 +75,8 @@ export class UsageAnalyticsService {
     `;
 
     const rows = await this.clickHouseService.select<BreakdownRowMicro>(query, {
-      periodStart: formatDateForClickHouse(params.periodStart),
-      periodEnd: formatDateForClickHouse(params.periodEnd),
+      periodStart: formatDateTimeForClickHouse(params.periodStart),
+      periodEnd: formatDateTimeForClickHouse(params.periodEnd),
       operationTypes: aiOperationTypes,
     });
 
@@ -174,8 +174,8 @@ export class UsageAnalyticsService {
 
     const rows = await this.clickHouseService.select<BreakdownRowMicro>(query, {
       workspaceId,
-      periodStart: formatDateForClickHouse(periodStart),
-      periodEnd: formatDateForClickHouse(periodEnd),
+      periodStart: formatDateTimeForClickHouse(periodStart),
+      periodEnd: formatDateTimeForClickHouse(periodEnd),
       ...(operationTypes && operationTypes.length > 0
         ? { operationTypes }
         : {}),
@@ -222,8 +222,8 @@ export class UsageAnalyticsService {
       query,
       {
         workspaceId,
-        periodStart: formatDateForClickHouse(periodStart),
-        periodEnd: formatDateForClickHouse(periodEnd),
+        periodStart: formatDateTimeForClickHouse(periodStart),
+        periodEnd: formatDateTimeForClickHouse(periodEnd),
         ...(operationTypes && operationTypes.length > 0
           ? { operationTypes }
           : {}),

packages/twenty-server/src/engine/core-modules/usage/services/usage-event-writer.service.ts

@@ -3,9 +3,9 @@
 import { Injectable, Logger } from '@nestjs/common';
 
 import { ClickHouseService } from 'src/database/clickHouse/clickHouse.service';
-import { formatDateForClickHouse } from 'src/database/clickHouse/clickHouse.util';
-import { type UsageEvent } from 'src/engine/core-modules/usage/types/usage-event.type';
+import { formatDateTimeForClickHouse } from 'src/database/clickHouse/clickHouse.util';
 import { TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service';
+import { type UsageEvent } from 'src/engine/core-modules/usage/types/usage-event.type';
 
 @Injectable()
 export class UsageEventWriterService {
@@ -21,11 +21,14 @@ export class UsageEventWriterService {
       return;
     }
 
-    const now = formatDateForClickHouse(new Date());
+    const now = formatDateTimeForClickHouse(new Date());
 
     const rows = usageEvents.map((usageEvent) => ({
       timestamp: now,
       workspaceId,
+      periodStart: usageEvent.periodStart
+        ? formatDateTimeForClickHouse(usageEvent.periodStart)
+        : undefined,
       userWorkspaceId: usageEvent.userWorkspaceId ?? '',
       resourceType: usageEvent.resourceType,
       operationType: usageEvent.operationType,

packages/twenty-server/src/engine/core-modules/usage/types/usage-event.type.ts

@@ -1,7 +1,7 @@
 /* @license Enterprise */
 
-import { type UsageResourceType } from 'src/engine/core-modules/usage/enums/usage-resource-type.enum';
 import { type UsageOperationType } from 'src/engine/core-modules/usage/enums/usage-operation-type.enum';
+import { type UsageResourceType } from 'src/engine/core-modules/usage/enums/usage-resource-type.enum';
 import { type UsageUnit } from 'src/engine/core-modules/usage/enums/usage-unit.enum';
 
 export type UsageEvent = {
@@ -10,6 +10,7 @@ export type UsageEvent = {
   creditsUsedMicro: number;
   quantity: number;
   unit: UsageUnit;
+  periodStart?: Date;
   resourceId?: string | null;
   resourceContext?: string | null;
   userWorkspaceId?: string | null;