fix(server): deduplicate @opentelemetry/api to fix NoopMeterProvider (#20231)

## Summary

**All OTel metrics in twenty-server have been silently dropped since
April 30.**

### Root cause

PR #20149 (`bump @sentry/profiling-node 10.27→10.51`) pulled in
`@sentry/node@10.51.0`, which declares `@opentelemetry/api: ^1.9.1` as a
**dependency** (not peer). Yarn installed it as a **nested** copy at
`1.9.1`, while the hoisted copy stayed at `1.9.0`.

At startup in `instrument.ts`:
1. `Sentry.init()` uses the **nested `1.9.1`** to register `trace`,
`propagation`, `context` on the OTel global → global version becomes
**`1.9.1`**
2. `setGlobalMeterProvider()` uses the **hoisted `1.9.0`** →
`registerGlobal` sees version mismatch (`1.9.1` ≠ `1.9.0`) → **silently
returns `false`**
3. Global stays `NoopMeterProvider` → every counter, gauge, and
histogram in the server is a no-op

### What this PR does

1. **Reverts three troubleshooting PRs** that are no longer needed now
that the root cause is identified:
   - #20230 — heartbeat gauge
   - #20228 — OTLP export lifecycle logs
- #20221 — Sentry revert to 10.27 (which never actually downgraded in
`yarn.lock` since `^10.27.0` resolved to `10.51.0`)

2. **Fixes the root cause**:
- Root Yarn resolution pinning `@opentelemetry/api` to `1.9.1` → single
copy in the entire tree, Sentry and Twenty share the same instance
- Named import in `instrument.ts` (`import { metrics as otelMetrics }`
instead of default import) as defense-in-depth against CJS interop
issues

### Verified on dev cluster

Exec'd into the running pod and confirmed:
- `@sentry/node` nests `@opentelemetry/api@1.9.1`, hoisted is `1.9.0`
- `Sentry.init()` → global version `1.9.1` → `setGlobalMeterProvider`
with VERSION `1.9.0` → returns `false` → `NoopMeterProvider`
- Same-version registration returns `true` → `MeterProvider` ✓

## Test plan
- [ ] CI passes (lint, typecheck, build)
- [ ] Deploy to dev cluster and verify metrics flow to collector
- [ ] Confirm `node_modules/@opentelemetry/api/package.json` shows
`1.9.1` with no nested copy under `@sentry/`

---------

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
Charles Bochet
2026-05-04 15:15:00 +02:00
committed by GitHub
parent 596ce32bd6
commit 3c7c62c79f
10 changed files with 147 additions and 164 deletions
+2 -1
View File
@@ -31,7 +31,8 @@
"@lingui/core": "5.1.2",
"@types/qs": "6.9.16",
"@wyw-in-js/transform@npm:0.6.0": "patch:@wyw-in-js/transform@npm%3A0.7.0#~/.yarn/patches/@wyw-in-js-transform-npm-0.7.0-ba641dc99f.patch",
"@wyw-in-js/transform@npm:0.7.0": "patch:@wyw-in-js/transform@npm%3A0.7.0#~/.yarn/patches/@wyw-in-js-transform-npm-0.7.0-ba641dc99f.patch"
"@wyw-in-js/transform@npm:0.7.0": "patch:@wyw-in-js/transform@npm%3A0.7.0#~/.yarn/patches/@wyw-in-js-transform-npm-0.7.0-ba641dc99f.patch",
"@opentelemetry/api": "1.9.1"
},
"version": "0.2.1",
"nx": {},
+1 -1
View File
@@ -57,7 +57,7 @@
"@react-email/components": "^0.5.3",
"@react-pdf/renderer": "^4.1.6",
"@scalar/api-reference-react": "^0.4.36",
"@sentry/react": "^10.27.0",
"@sentry/react": "^10.51.0",
"@sniptt/guards": "^0.2.0",
"@tiptap/core": "3.4.2",
"@tiptap/extension-bold": "3.4.2",
+3 -3
View File
@@ -78,9 +78,9 @@
"@ptc-org/nestjs-query-graphql": "patch:@ptc-org/nestjs-query-graphql@4.2.0#./patches/@ptc-org+nestjs-query-graphql+4.2.0.patch",
"@ptc-org/nestjs-query-typeorm": "4.2.1-alpha.2",
"@react-email/render": "^1.2.3",
"@sentry/nestjs": "^10.27.0",
"@sentry/node": "^10.27.0",
"@sentry/profiling-node": "^10.27.0",
"@sentry/nestjs": "^10.51.0",
"@sentry/node": "^10.51.0",
"@sentry/profiling-node": "^10.51.0",
"@sniptt/guards": "0.2.0",
"addressparser": "1.0.1",
"ai": "6.0.97",
@@ -13,6 +13,7 @@ import { buildApiKeyAuthContext } from 'src/engine/core-modules/auth/utils/build
import { buildApplicationAuthContext } from 'src/engine/core-modules/auth/utils/build-application-auth-context.util';
import { buildPendingActivationUserAuthContext } from 'src/engine/core-modules/auth/utils/build-pending-activation-user-auth-context.util';
import { buildUserAuthContext } from 'src/engine/core-modules/auth/utils/build-user-auth-context.util';
import { applyWorkspaceSentryContext } from 'src/engine/core-modules/sentry/utils/apply-workspace-sentry-context.util';
@Injectable()
export class WorkspaceAuthContextMiddleware implements NestMiddleware {
@@ -25,6 +26,8 @@ export class WorkspaceAuthContextMiddleware implements NestMiddleware {
const authContext = this.buildAuthContext(req);
applyWorkspaceSentryContext(authContext);
withWorkspaceAuthContext(authContext, () => {
next();
});
@@ -4,6 +4,7 @@ import {
type OnModuleInit,
} from '@nestjs/common';
import * as Sentry from '@sentry/node';
import {
type JobsOptions,
MetricsTime,
@@ -28,6 +29,7 @@ import { type MessageQueue } from 'src/engine/core-modules/message-queue/message
import { getJobKey } from 'src/engine/core-modules/message-queue/utils/get-job-key.util';
import { type MetricsService } from 'src/engine/core-modules/metrics/metrics.service';
import { MetricsKeys } from 'src/engine/core-modules/metrics/types/metrics-keys.type';
import { applyWorkspaceSentryContextFromJobData } from 'src/engine/core-modules/sentry/utils/apply-workspace-sentry-context-from-job-data.util';
export type BullMQDriverOptions = QueueOptions;
@@ -108,25 +110,28 @@ export class BullMQDriver
this.workerMap[queueName] = new Worker(
queueName,
async (job) => {
// TODO: Correctly support for job.id
const timeStart = performance.now();
const workspaceId = job.data?.workspaceId;
const workspaceSuffix = workspaceId
? ` [workspace=${workspaceId}]`
: '';
async (job) =>
Sentry.withIsolationScope(async () => {
applyWorkspaceSentryContextFromJobData(job.data);
this.logger.log(
`Processing job ${job.id} with name ${job.name} on queue ${queueName}${workspaceSuffix}`,
);
await handler({ data: job.data, id: job.id ?? '', name: job.name });
const timeEnd = performance.now();
const executionTime = timeEnd - timeStart;
// TODO: Correctly support for job.id
const timeStart = performance.now();
const workspaceId = job.data?.workspaceId;
const workspaceSuffix = workspaceId
? ` [workspace=${workspaceId}]`
: '';
this.logger.log(
`Job ${job.id} with name ${job.name} processed on queue ${queueName} in ${executionTime.toFixed(2)}ms${workspaceSuffix}`,
);
},
this.logger.log(
`Processing job ${job.id} with name ${job.name} on queue ${queueName}${workspaceSuffix}`,
);
await handler({ data: job.data, id: job.id ?? '', name: job.name });
const timeEnd = performance.now();
const executionTime = timeEnd - timeStart;
this.logger.log(
`Job ${job.id} with name ${job.name} processed on queue ${queueName} in ${executionTime.toFixed(2)}ms${workspaceSuffix}`,
);
}),
workerOptions,
);
@@ -0,0 +1,25 @@
import { applyWorkspaceSentryFields } from 'src/engine/core-modules/sentry/utils/apply-workspace-sentry-fields.util';
export const applyWorkspaceSentryContextFromJobData = (
jobData: unknown,
): void => {
if (typeof jobData !== 'object' || jobData === null) {
return;
}
const workspaceId = (jobData as { workspaceId?: unknown }).workspaceId;
const userWorkspaceId = (jobData as { userWorkspaceId?: unknown })
.userWorkspaceId;
if (typeof workspaceId !== 'string' || workspaceId.length === 0) {
return;
}
applyWorkspaceSentryFields({
workspaceId,
userWorkspaceId:
typeof userWorkspaceId === 'string' && userWorkspaceId.length > 0
? userWorkspaceId
: undefined,
});
};
@@ -0,0 +1,27 @@
import { type WorkspaceAuthContext } from 'src/engine/core-modules/auth/types/workspace-auth-context.type';
import { applyWorkspaceSentryFields } from 'src/engine/core-modules/sentry/utils/apply-workspace-sentry-fields.util';
export const applyWorkspaceSentryContext = (
authContext: WorkspaceAuthContext,
): void => {
const workspaceId = authContext.workspace?.id;
if (!workspaceId) {
return;
}
switch (authContext.type) {
case 'user':
case 'pendingActivationUser':
applyWorkspaceSentryFields({
workspaceId,
userWorkspaceId: authContext.userWorkspaceId,
});
return;
case 'apiKey':
case 'application':
case 'system':
applyWorkspaceSentryFields({ workspaceId });
return;
}
};
@@ -0,0 +1,26 @@
import * as Sentry from '@sentry/node';
type WorkspaceSentryFields = {
workspaceId: string;
userWorkspaceId?: string;
};
export const applyWorkspaceSentryFields = (
fields: WorkspaceSentryFields,
): void => {
Sentry.setUser({
id: fields.userWorkspaceId ?? fields.workspaceId,
});
Sentry.setTag('twenty.workspace.id', fields.workspaceId);
if (fields.userWorkspaceId) {
Sentry.setTag('twenty.user_workspace.id', fields.userWorkspaceId);
}
Sentry.setContext('twenty', {
workspace_id: fields.workspaceId,
...(fields.userWorkspaceId && {
user_workspace_id: fields.userWorkspaceId,
}),
});
};
+29 -126
View File
@@ -1,7 +1,6 @@
import process from 'process';
import { ExportResultCode } from '@opentelemetry/core';
import opentelemetry from '@opentelemetry/api';
import { metrics as otelMetrics } from '@opentelemetry/api';
import { OTLPMetricExporter } from '@opentelemetry/exporter-metrics-otlp-http';
import { PrometheusExporter } from '@opentelemetry/exporter-prometheus';
import {
@@ -9,8 +8,6 @@ import {
ConsoleMetricExporter,
MeterProvider,
PeriodicExportingMetricReader,
type PushMetricExporter,
type ResourceMetrics,
} from '@opentelemetry/sdk-metrics';
import * as Sentry from '@sentry/node';
import { nodeProfilingIntegration } from '@sentry/profiling-node';
@@ -27,91 +24,6 @@ const meterDrivers = parseArrayEnvVar(
[],
);
const OTLP_METRICS_LOG_PREFIX = '[Twenty OTEL metrics]';
const formatOtlEndpointForLog = (rawUrl: string | undefined): string => {
if (!rawUrl?.trim()) {
return 'unset';
}
try {
const parsed = new URL(rawUrl);
return `${parsed.protocol}//${parsed.host}${parsed.pathname}`;
} catch {
return 'invalid_url';
}
};
const serializeOtlExportFailure = (error: unknown): string => {
if (error === undefined || error === null) {
return 'none';
}
if (error instanceof Error) {
const base = `${error.name}: ${error.message}`;
const maybeAggregate = error as Error & { errors?: unknown[] };
if (
Array.isArray(maybeAggregate.errors) &&
maybeAggregate.errors.length > 0
) {
const causes = maybeAggregate.errors.map((cause: unknown) =>
serializeOtlExportFailure(cause),
);
return `${base} | causes=[${causes.join(' | ')}]`;
}
return base;
}
try {
return JSON.stringify(error);
} catch {
return String(error);
}
};
let otlpMetricsFirstExportSuccessLogged = false;
let otlpMetricsFirstExportAttemptLogged = false;
const wrapOtlMetricExporterWithProcessLogs = (
inner: OTLPMetricExporter,
otlpEndpointForLog: string,
): PushMetricExporter => ({
export(metrics: ResourceMetrics, resultCallback) {
if (!otlpMetricsFirstExportAttemptLogged) {
otlpMetricsFirstExportAttemptLogged = true;
console.log(
`${OTLP_METRICS_LOG_PREFIX} first periodic export attempt | endpoint=${otlpEndpointForLog}`,
);
}
const totalMetricData = metrics.scopeMetrics.reduce(
(accumulator, scopeMetric) => accumulator + scopeMetric.metrics.length,
0,
);
inner.export(metrics, (result) => {
if (result.code === ExportResultCode.SUCCESS) {
if (!otlpMetricsFirstExportSuccessLogged) {
otlpMetricsFirstExportSuccessLogged = true;
// One-time line so prod logs stay usable; further success is silent.
console.log(
`${OTLP_METRICS_LOG_PREFIX} first export ok | endpoint=${otlpEndpointForLog} | metricDataPoints=${totalMetricData}`,
);
}
} else {
console.warn(
`${OTLP_METRICS_LOG_PREFIX} export failed | endpoint=${otlpEndpointForLog} | metricDataPoints=${totalMetricData} | code=${result.code} | error=${serializeOtlExportFailure(result.error)}`,
);
}
resultCallback(result);
});
},
forceFlush: () => inner.forceFlush(),
shutdown: () => inner.shutdown(),
selectAggregationTemporality: (instrumentType) =>
inner.selectAggregationTemporality(instrumentType),
selectAggregation: (instrumentType) =>
inner.selectAggregation(instrumentType),
});
if (process.env.EXCEPTION_HANDLER_DRIVER === ExceptionHandlerDriver.SENTRY) {
Sentry.init({
environment: process.env.SENTRY_ENVIRONMENT,
@@ -133,24 +45,34 @@ if (process.env.EXCEPTION_HANDLER_DRIVER === ExceptionHandlerDriver.SENTRY) {
profilesSampleRate: 0.3,
sendDefaultPii: true,
debug: process.env.NODE_ENV === NodeEnvironment.DEVELOPMENT,
beforeSendSpan: (span) => {
const twentyContext = Sentry.getIsolationScope().getScopeData().contexts
?.twenty as
| {
workspace_id?: string;
user_workspace_id?: string;
}
| undefined;
if (!twentyContext?.workspace_id) {
return span;
}
span.data = {
...span.data,
'twenty.workspace.id': twentyContext.workspace_id,
...(twentyContext.user_workspace_id && {
'twenty.user_workspace.id': twentyContext.user_workspace_id,
}),
};
return span;
},
});
}
// Meter setup
const otlpCollectorMetricsUrl = process.env.OTLP_COLLECTOR_METRICS_ENDPOINT_URL;
const otlpEndpointForLog = formatOtlEndpointForLog(otlpCollectorMetricsUrl);
if (meterDrivers.includes(MeterDriver.OpenTelemetry)) {
console.log(
`${OTLP_METRICS_LOG_PREFIX} OTLP reader enabled | exportIntervalMs=10000 | endpoint=${otlpEndpointForLog} | meterDrivers=${meterDrivers.join(',')}`,
);
} else {
console.log(
`${OTLP_METRICS_LOG_PREFIX} OTLP reader disabled | parsedMeterDrivers=${JSON.stringify(meterDrivers)} | rawMETER_DRIVER=${JSON.stringify(process.env.METER_DRIVER ?? '')}`,
);
}
const prometheusExporter = meterDrivers.includes(MeterDriver.Prometheus)
? new PrometheusExporter({ port: 9464 })
: null;
@@ -168,13 +90,10 @@ const meterProvider = new MeterProvider({
...(meterDrivers.includes(MeterDriver.OpenTelemetry)
? [
new PeriodicExportingMetricReader({
exporter: wrapOtlMetricExporterWithProcessLogs(
new OTLPMetricExporter({
url: otlpCollectorMetricsUrl,
temporalityPreference: AggregationTemporality.DELTA,
}),
otlpEndpointForLog,
),
exporter: new OTLPMetricExporter({
url: process.env.OTLP_COLLECTOR_METRICS_ENDPOINT_URL,
temporalityPreference: AggregationTemporality.DELTA,
}),
exportIntervalMillis: 10000,
}),
]
@@ -183,20 +102,4 @@ const meterProvider = new MeterProvider({
],
});
opentelemetry.metrics.setGlobalMeterProvider(meterProvider);
// Always-on gauge so the OTLP exporter fires on every collection tick,
// even when the process is otherwise idle. This guarantees the process-log
// wrapper above can prove connectivity (first export ok / export failed).
if (meterDrivers.includes(MeterDriver.OpenTelemetry)) {
const heartbeatMeter = opentelemetry.metrics.getMeter(
'twenty-server-heartbeat',
);
const heartbeat = heartbeatMeter.createObservableGauge('twenty.heartbeat', {
description: 'Always-on gauge (1) to prove OTLP export pipeline',
});
heartbeat.addCallback((observableResult) => {
observableResult.observe(1);
});
}
otelMetrics.setGlobalMeterProvider(meterProvider);
+9 -16
View File
@@ -14370,14 +14370,7 @@ __metadata:
languageName: node
linkType: hard
"@opentelemetry/api@npm:1.9.0, @opentelemetry/api@npm:^1.3.0, @opentelemetry/api@npm:^1.9.0":
version: 1.9.0
resolution: "@opentelemetry/api@npm:1.9.0"
checksum: 10c0/9aae2fe6e8a3a3eeb6c1fdef78e1939cf05a0f37f8a4fae4d6bf2e09eb1e06f966ece85805626e01ba5fab48072b94f19b835449e58b6d26720ee19a58298add
languageName: node
linkType: hard
"@opentelemetry/api@npm:^1.9.1":
"@opentelemetry/api@npm:1.9.1":
version: 1.9.1
resolution: "@opentelemetry/api@npm:1.9.1"
checksum: 10c0/c608485fc8b5a91e1f7e05e843b45b509307456b31cd2ad365933d90813e40ebfedf179f1451c762037e82d7c76aa8500e95d2da3609f640a1206cde5322cd14
@@ -20408,7 +20401,7 @@ __metadata:
languageName: node
linkType: hard
"@sentry/nestjs@npm:^10.27.0":
"@sentry/nestjs@npm:^10.51.0":
version: 10.51.0
resolution: "@sentry/nestjs@npm:10.51.0"
dependencies:
@@ -20457,7 +20450,7 @@ __metadata:
languageName: node
linkType: hard
"@sentry/node@npm:10.51.0, @sentry/node@npm:^10.27.0":
"@sentry/node@npm:10.51.0, @sentry/node@npm:^10.51.0":
version: 10.51.0
resolution: "@sentry/node@npm:10.51.0"
dependencies:
@@ -20510,7 +20503,7 @@ __metadata:
languageName: node
linkType: hard
"@sentry/profiling-node@npm:^10.27.0":
"@sentry/profiling-node@npm:^10.51.0":
version: 10.51.0
resolution: "@sentry/profiling-node@npm:10.51.0"
dependencies:
@@ -20523,7 +20516,7 @@ __metadata:
languageName: node
linkType: hard
"@sentry/react@npm:^10.27.0":
"@sentry/react@npm:^10.51.0":
version: 10.51.0
resolution: "@sentry/react@npm:10.51.0"
dependencies:
@@ -57848,7 +57841,7 @@ __metadata:
"@react-email/components": "npm:^0.5.3"
"@react-pdf/renderer": "npm:^4.1.6"
"@scalar/api-reference-react": "npm:^0.4.36"
"@sentry/react": "npm:^10.27.0"
"@sentry/react": "npm:^10.51.0"
"@sniptt/guards": "npm:^0.2.0"
"@storybook-community/storybook-addon-cookie": "npm:^5.0.0"
"@storybook/addon-coverage": "npm:^3.0.0"
@@ -58096,9 +58089,9 @@ __metadata:
"@ptc-org/nestjs-query-graphql": "patch:@ptc-org/nestjs-query-graphql@4.2.0#./patches/@ptc-org+nestjs-query-graphql+4.2.0.patch"
"@ptc-org/nestjs-query-typeorm": "npm:4.2.1-alpha.2"
"@react-email/render": "npm:^1.2.3"
"@sentry/nestjs": "npm:^10.27.0"
"@sentry/node": "npm:^10.27.0"
"@sentry/profiling-node": "npm:^10.27.0"
"@sentry/nestjs": "npm:^10.51.0"
"@sentry/node": "npm:^10.51.0"
"@sentry/profiling-node": "npm:^10.51.0"
"@sniptt/guards": "npm:0.2.0"
"@swc/cli": "npm:^0.7.10"
"@swc/core": "npm:^1.15.11"