Add prometheus exporter (#17392)

Create a metric endpoint and expose prometheus gauge for event stream
count.

packages/twenty-server/src/engine/core-modules/cache-storage/services/cache-storage.service.ts

@@ -206,6 +206,38 @@ export class CacheStorageService {
     } while (cursor !== 0);
   }
 
+  async scanAndCountSetMembers(scanPattern: string): Promise<number> {
+    if (!this.isRedisCache()) {
+      throw new Error(
+        'scanAndCountSetMembers is only supported with Redis cache',
+      );
+    }
+
+    const redisClient = (this.cache as RedisCache).store.client;
+    let cursor = 0;
+    let totalCount = 0;
+
+    do {
+      const result = await redisClient.scan(cursor, {
+        MATCH: `${this.namespace}:${scanPattern}`,
+        COUNT: 100,
+      });
+
+      cursor = result.cursor;
+      const keys = result.keys;
+
+      if (keys.length > 0) {
+        const counts = await Promise.all(
+          keys.map((key) => redisClient.sCard(key)),
+        );
+
+        totalCount += counts.reduce((sum, count) => sum + count, 0);
+      }
+    } while (cursor !== 0);
+
+    return totalCount;
+  }
+
   async acquireLock(key: string, ttl = 1000): Promise<boolean> {
     if (!this.isRedisCache()) {
       throw new Error('acquireLock is only supported with Redis cache');

packages/twenty-server/src/engine/core-modules/metrics/metrics.service.ts

@@ -1,14 +1,39 @@
 import { Injectable } from '@nestjs/common';
 
-import { metrics, type Attributes } from '@opentelemetry/api';
+import {
+  metrics,
+  type Attributes,
+  type Meter,
+  type MetricOptions,
+  type ObservableGauge,
+  type ObservableResult,
+} from '@opentelemetry/api';
 
 import { MetricsCacheService } from 'src/engine/core-modules/metrics/metrics-cache.service';
 import { type MetricsKeys } from 'src/engine/core-modules/metrics/types/metrics-keys.type';
 
+const METER_NAME = 'twenty-server';
+
 @Injectable()
 export class MetricsService {
   constructor(private readonly metricsCacheService: MetricsCacheService) {}
 
+  getMeter(): Meter {
+    return metrics.getMeter(METER_NAME);
+  }
+
+  createObservableGauge(
+    name: string,
+    options: MetricOptions,
+    callback: (observableResult: ObservableResult) => void | Promise<void>,
+  ): ObservableGauge {
+    const gauge = this.getMeter().createObservableGauge(name, options);
+
+    gauge.addCallback(callback);
+
+    return gauge;
+  }
+
   async incrementCounter({
     key,
     eventId,
@@ -20,9 +45,7 @@ export class MetricsService {
     attributes?: Attributes;
     shouldStoreInCache?: boolean;
   }) {
-    //TODO : Define meter name usage in monitoring
-    const meter = metrics.getMeter('twenty-server');
-    const counter = meter.createCounter(key);
+    const counter = this.getMeter().createCounter(key);
 
     counter.add(1, attributes);
 
@@ -42,9 +65,7 @@ export class MetricsService {
     attributes?: Attributes;
     shouldStoreInCache?: boolean;
   }) {
-    //TODO : Define meter name usage in monitoring
-    const meter = metrics.getMeter('twenty-server');
-    const counter = meter.createCounter(key);
+    const counter = this.getMeter().createCounter(key);
 
     counter.add(eventIds.length, attributes);
 

packages/twenty-server/src/engine/core-modules/metrics/types/meter-driver.type.ts

@@ -1,4 +1,5 @@
 export enum MeterDriver {
   OpenTelemetry = 'opentelemetry',
   Console = 'console',
+  Prometheus = 'prometheus',
 }