Add job queue latency histogram metric (#21260)

## Summary
- Records the time each job spends waiting in the queue (enqueue →
processing start) as an OpenTelemetry histogram
- Metric is broken down by `queue` and `job_name` attributes, enabling
per-queue p50/p95/p99 latency analysis
- Uses BullMQ's native `job.timestamp` for accurate measurement

## Test plan
- [x] Deploy to staging and verify `job/latency-ms` metric appears in
ClickHouse `otel_metrics_histogram` table
- [x] Confirm Grafana dashboard can query the histogram data
This commit is contained in:
Thomas Trompette
2026-06-05 18:04:54 +02:00
committed by GitHub
parent 27db8bbae4
commit 51e3eddb29
2 changed files with 10 additions and 0 deletions
@@ -114,6 +114,15 @@ export class BullMQDriver
Sentry.withIsolationScope(async () => {
applyWorkspaceSentryContextFromJobData(job.data);
const queueLatency = Math.max(0, Date.now() - job.timestamp);
this.metricsService.recordHistogram({
key: MetricsKeys.JobLatencyMs,
value: queueLatency,
unit: 'ms',
attributes: { queue: queueName, job_name: job.name },
});
// TODO: Correctly support for job.id
const timeStart = performance.now();
const workspaceId = job.data?.workspaceId;
@@ -40,6 +40,7 @@ export enum MetricsKeys {
JobCompleted = 'job/completed',
JobFailed = 'job/failed',
JobWaiting = 'job/waiting',
JobLatencyMs = 'job/latency-ms',
AiChatTurnLatencyMs = 'ai-chat/turn-latency-ms',
AiChatStepLatencyMs = 'ai-chat/step-latency-ms',
AiChatTtftMs = 'ai-chat/ttft-ms',