Billing - optimize usageEvent CH table (#20019)

- Update usageEvent clickhouse table, partitioning, indexing and
projection (auto materialized view) to optimize credit usage queries
- Add caching for available credits and billing subscription


To do in next PR: deprecate enforceCapUsage cron. Bonus : real-time on
billingSubscription

packages/twenty-server/src/database/clickHouse/clickHouse.util.ts

@@ -3,9 +3,12 @@
  * ClickHouse expects: YYYY-MM-DD HH:mm:ss.SSS (no 'T' separator, no 'Z' suffix)
  * JavaScript toISOString() returns: YYYY-MM-DDTHH:mm:ss.SSSZ
  */
-export const formatDateForClickHouse = (date: Date | string): string => {
+export const formatDateTimeForClickHouse = (date: Date | string): string => {
   const iso = typeof date === 'string' ? date : date.toISOString();
 
   // Extract date (YYYY-MM-DD) and time with milliseconds (HH:mm:ss.SSS)
   return `${iso.slice(0, 10)} ${iso.slice(11, 23)}`;
 };
+
+export const formatDateForClickHouse = (date: Date): string =>
+  date.toISOString().slice(0, 10);

packages/twenty-server/src/database/clickHouse/migrations/006-update-usage-event-table.sql

@@ -0,0 +1,53 @@
+CREATE TABLE
+    IF NOT EXISTS usageEvent_v2 (
+        `timestamp` DateTime64 (3) NOT NULL,
+        `workspaceId` String NOT NULL,
+        `periodStart` DateTime64 (3),
+        `userWorkspaceId` String DEFAULT '',
+        `resourceType` LowCardinality (String) NOT NULL,
+        `operationType` LowCardinality (String) NOT NULL,
+        `quantity` Int64 NOT NULL DEFAULT 0,
+        `unit` LowCardinality (String) NOT NULL DEFAULT 'CREDIT',
+        `creditsUsedMicro` Int64 NOT NULL DEFAULT 0,
+        `resourceId` String DEFAULT '',
+        `resourceContext` String DEFAULT '',
+        `metadata` JSON
+    ) ENGINE = MergeTree
+PARTITION BY
+    toYYYYMM (timestamp)
+ORDER BY
+    (workspaceId, timestamp, userWorkspaceId)
+PRIMARY KEY (workspaceId, timestamp)
+TTL toDateTime(timestamp) + INTERVAL 3 YEAR DELETE;
+
+INSERT INTO
+    usageEvent_v2 (timestamp, workspaceId, userWorkspaceId, resourceType, operationType, quantity, unit, creditsUsedMicro, resourceId, resourceContext, metadata)
+SELECT
+    timestamp,
+    workspaceId,
+    userWorkspaceId,
+    resourceType,
+    operationType,
+    quantity,
+    unit,
+    creditsUsedMicro,
+    resourceId,
+    resourceContext,
+    metadata
+FROM
+    usageEvent;
+
+RENAME TABLE usageEvent TO usageEvent_old,
+usageEvent_v2 TO usageEvent;
+
+DROP TABLE IF EXISTS usageEvent_old;
+
+ALTER TABLE usageEvent ADD PROJECTION IF NOT EXISTS billing_by_workspace_period (
+    SELECT
+        workspaceId,
+        periodStart,
+        sum(creditsUsedMicro) AS totalCreditsUsedMicro
+    GROUP BY
+        workspaceId, periodStart
+);
+

packages/twenty-server/src/database/clickHouse/seeds/fixtures.ts

@@ -4,7 +4,7 @@ import { OBJECT_RECORD_UPDATED_EVENT } from 'src/engine/core-modules/audit/utils
 import { CUSTOM_DOMAIN_ACTIVATED_EVENT } from 'src/engine/core-modules/audit/utils/events/workspace-event/custom-domain/custom-domain-activated';
 import { CUSTOM_DOMAIN_DEACTIVATED_EVENT } from 'src/engine/core-modules/audit/utils/events/workspace-event/custom-domain/custom-domain-deactivated';
 import { type GenericTrackEvent } from 'src/engine/core-modules/audit/utils/events/workspace-event/track';
-import { formatDateForClickHouse } from 'src/database/clickHouse/clickHouse.util';
+import { formatDateTimeForClickHouse } from 'src/database/clickHouse/clickHouse.util';
 import { SEED_APPLE_WORKSPACE_ID } from 'src/engine/workspace-manager/dev-seeder/core/constants/seeder-workspaces.constant';
 import { USER_WORKSPACE_DATA_SEED_IDS } from 'src/engine/workspace-manager/dev-seeder/core/utils/seed-user-workspaces.util';
 
@@ -17,6 +17,7 @@ export type ObjectEventFixture = GenericTrackEvent & {
 export type UsageEventFixture = {
   timestamp: string;
   workspaceId: string;
+  periodStart: string;
   userWorkspaceId: string;
   resourceType: string;
   operationType: string;
@@ -215,8 +216,17 @@ const buildUsageEventFixtures = (): UsageEventFixture[] => {
             : '';
 
           fixtures.push({
-            timestamp: formatDateForClickHouse(eventDate),
+            timestamp: formatDateTimeForClickHouse(eventDate),
             workspaceId: SEED_APPLE_WORKSPACE_ID,
+            periodStart: formatDateTimeForClickHouse(
+              new Date(
+                Date.UTC(
+                  eventDate.getUTCFullYear(),
+                  eventDate.getUTCMonth(),
+                  1,
+                ),
+              ),
+            ),
             userWorkspaceId: users[userIdx],
             resourceType: op.resourceType,
             operationType: op.operationType,