Add job queue latency histogram metric (#21260)
## Summary - Records the time each job spends waiting in the queue (enqueue → processing start) as an OpenTelemetry histogram - Metric is broken down by `queue` and `job_name` attributes, enabling per-queue p50/p95/p99 latency analysis - Uses BullMQ's native `job.timestamp` for accurate measurement ## Test plan - [x] Deploy to staging and verify `job/latency-ms` metric appears in ClickHouse `otel_metrics_histogram` table - [x] Confirm Grafana dashboard can query the histogram data
This commit is contained in:
@@ -114,6 +114,15 @@ export class BullMQDriver
|
||||
Sentry.withIsolationScope(async () => {
|
||||
applyWorkspaceSentryContextFromJobData(job.data);
|
||||
|
||||
const queueLatency = Math.max(0, Date.now() - job.timestamp);
|
||||
|
||||
this.metricsService.recordHistogram({
|
||||
key: MetricsKeys.JobLatencyMs,
|
||||
value: queueLatency,
|
||||
unit: 'ms',
|
||||
attributes: { queue: queueName, job_name: job.name },
|
||||
});
|
||||
|
||||
// TODO: Correctly support for job.id
|
||||
const timeStart = performance.now();
|
||||
const workspaceId = job.data?.workspaceId;
|
||||
|
||||
@@ -40,6 +40,7 @@ export enum MetricsKeys {
|
||||
JobCompleted = 'job/completed',
|
||||
JobFailed = 'job/failed',
|
||||
JobWaiting = 'job/waiting',
|
||||
JobLatencyMs = 'job/latency-ms',
|
||||
AiChatTurnLatencyMs = 'ai-chat/turn-latency-ms',
|
||||
AiChatStepLatencyMs = 'ai-chat/step-latency-ms',
|
||||
AiChatTtftMs = 'ai-chat/ttft-ms',
|
||||
|
||||
Reference in New Issue
Block a user