From 51e3eddb290a96f2d8eadd8afd1417dfa8aa07a8 Mon Sep 17 00:00:00 2001
From: Thomas Trompette <thomas.trompette@sfr.fr>
Date: Fri, 5 Jun 2026 18:04:54 +0200
Subject: [PATCH] Add job queue latency histogram metric (#21260)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

## Summary
- Records the time each job spends waiting in the queue (enqueue →
processing start) as an OpenTelemetry histogram
- Metric is broken down by `queue` and `job_name` attributes, enabling
per-queue p50/p95/p99 latency analysis
- Uses BullMQ's native `job.timestamp` for accurate measurement

## Test plan
- [x] Deploy to staging and verify `job/latency-ms` metric appears in
ClickHouse `otel_metrics_histogram` table
- [x] Confirm Grafana dashboard can query the histogram data
---
 .../core-modules/message-queue/drivers/bullmq.driver.ts  | 9 +++++++++
 .../core-modules/metrics/types/metrics-keys.type.ts      | 1 +
 2 files changed, 10 insertions(+)

diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/bullmq.driver.ts b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/bullmq.driver.ts
index abfb79b2b4..13b6e4782f 100644
--- a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/bullmq.driver.ts
+++ b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/bullmq.driver.ts
@@ -114,6 +114,15 @@ export class BullMQDriver
         Sentry.withIsolationScope(async () => {
           applyWorkspaceSentryContextFromJobData(job.data);
 
+          const queueLatency = Math.max(0, Date.now() - job.timestamp);
+
+          this.metricsService.recordHistogram({
+            key: MetricsKeys.JobLatencyMs,
+            value: queueLatency,
+            unit: 'ms',
+            attributes: { queue: queueName, job_name: job.name },
+          });
+
           // TODO: Correctly support for job.id
           const timeStart = performance.now();
           const workspaceId = job.data?.workspaceId;
diff --git a/packages/twenty-server/src/engine/core-modules/metrics/types/metrics-keys.type.ts b/packages/twenty-server/src/engine/core-modules/metrics/types/metrics-keys.type.ts
index a34d9e714c..3435d0c41d 100644
--- a/packages/twenty-server/src/engine/core-modules/metrics/types/metrics-keys.type.ts
+++ b/packages/twenty-server/src/engine/core-modules/metrics/types/metrics-keys.type.ts
@@ -40,6 +40,7 @@ export enum MetricsKeys {
   JobCompleted = 'job/completed',
   JobFailed = 'job/failed',
   JobWaiting = 'job/waiting',
+  JobLatencyMs = 'job/latency-ms',
   AiChatTurnLatencyMs = 'ai-chat/turn-latency-ms',
   AiChatStepLatencyMs = 'ai-chat/step-latency-ms',
   AiChatTtftMs = 'ai-chat/ttft-ms',