diff --git a/package.json b/package.json index 032e97a8d7..f0159cdba9 100644 --- a/package.json +++ b/package.json @@ -31,7 +31,8 @@ "@lingui/core": "5.1.2", "@types/qs": "6.9.16", "@wyw-in-js/transform@npm:0.6.0": "patch:@wyw-in-js/transform@npm%3A0.7.0#~/.yarn/patches/@wyw-in-js-transform-npm-0.7.0-ba641dc99f.patch", - "@wyw-in-js/transform@npm:0.7.0": "patch:@wyw-in-js/transform@npm%3A0.7.0#~/.yarn/patches/@wyw-in-js-transform-npm-0.7.0-ba641dc99f.patch" + "@wyw-in-js/transform@npm:0.7.0": "patch:@wyw-in-js/transform@npm%3A0.7.0#~/.yarn/patches/@wyw-in-js-transform-npm-0.7.0-ba641dc99f.patch", + "@opentelemetry/api": "1.9.1" }, "version": "0.2.1", "nx": {}, diff --git a/packages/twenty-front/package.json b/packages/twenty-front/package.json index 00ad214758..0cc2b67ed1 100644 --- a/packages/twenty-front/package.json +++ b/packages/twenty-front/package.json @@ -57,7 +57,7 @@ "@react-email/components": "^0.5.3", "@react-pdf/renderer": "^4.1.6", "@scalar/api-reference-react": "^0.4.36", - "@sentry/react": "^10.27.0", + "@sentry/react": "^10.51.0", "@sniptt/guards": "^0.2.0", "@tiptap/core": "3.4.2", "@tiptap/extension-bold": "3.4.2", diff --git a/packages/twenty-server/package.json b/packages/twenty-server/package.json index a5ed9ba6ac..68bcf9620b 100644 --- a/packages/twenty-server/package.json +++ b/packages/twenty-server/package.json @@ -78,9 +78,9 @@ "@ptc-org/nestjs-query-graphql": "patch:@ptc-org/nestjs-query-graphql@4.2.0#./patches/@ptc-org+nestjs-query-graphql+4.2.0.patch", "@ptc-org/nestjs-query-typeorm": "4.2.1-alpha.2", "@react-email/render": "^1.2.3", - "@sentry/nestjs": "^10.27.0", - "@sentry/node": "^10.27.0", - "@sentry/profiling-node": "^10.27.0", + "@sentry/nestjs": "^10.51.0", + "@sentry/node": "^10.51.0", + "@sentry/profiling-node": "^10.51.0", "@sniptt/guards": "0.2.0", "addressparser": "1.0.1", "ai": "6.0.97", diff --git a/packages/twenty-server/src/engine/core-modules/auth/middlewares/workspace-auth-context.middleware.ts b/packages/twenty-server/src/engine/core-modules/auth/middlewares/workspace-auth-context.middleware.ts index c446be9c32..23c87cd7d8 100644 --- a/packages/twenty-server/src/engine/core-modules/auth/middlewares/workspace-auth-context.middleware.ts +++ b/packages/twenty-server/src/engine/core-modules/auth/middlewares/workspace-auth-context.middleware.ts @@ -13,6 +13,7 @@ import { buildApiKeyAuthContext } from 'src/engine/core-modules/auth/utils/build import { buildApplicationAuthContext } from 'src/engine/core-modules/auth/utils/build-application-auth-context.util'; import { buildPendingActivationUserAuthContext } from 'src/engine/core-modules/auth/utils/build-pending-activation-user-auth-context.util'; import { buildUserAuthContext } from 'src/engine/core-modules/auth/utils/build-user-auth-context.util'; +import { applyWorkspaceSentryContext } from 'src/engine/core-modules/sentry/utils/apply-workspace-sentry-context.util'; @Injectable() export class WorkspaceAuthContextMiddleware implements NestMiddleware { @@ -25,6 +26,8 @@ export class WorkspaceAuthContextMiddleware implements NestMiddleware { const authContext = this.buildAuthContext(req); + applyWorkspaceSentryContext(authContext); + withWorkspaceAuthContext(authContext, () => { next(); }); diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/bullmq.driver.ts b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/bullmq.driver.ts index 0f94e52594..dedb66ac09 100644 --- a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/bullmq.driver.ts +++ b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/bullmq.driver.ts @@ -4,6 +4,7 @@ import { type OnModuleInit, } from '@nestjs/common'; +import * as Sentry from '@sentry/node'; import { type JobsOptions, MetricsTime, @@ -28,6 +29,7 @@ import { type MessageQueue } from 'src/engine/core-modules/message-queue/message import { getJobKey } from 'src/engine/core-modules/message-queue/utils/get-job-key.util'; import { type MetricsService } from 'src/engine/core-modules/metrics/metrics.service'; import { MetricsKeys } from 'src/engine/core-modules/metrics/types/metrics-keys.type'; +import { applyWorkspaceSentryContextFromJobData } from 'src/engine/core-modules/sentry/utils/apply-workspace-sentry-context-from-job-data.util'; export type BullMQDriverOptions = QueueOptions; @@ -108,25 +110,28 @@ export class BullMQDriver this.workerMap[queueName] = new Worker( queueName, - async (job) => { - // TODO: Correctly support for job.id - const timeStart = performance.now(); - const workspaceId = job.data?.workspaceId; - const workspaceSuffix = workspaceId - ? ` [workspace=${workspaceId}]` - : ''; + async (job) => + Sentry.withIsolationScope(async () => { + applyWorkspaceSentryContextFromJobData(job.data); - this.logger.log( - `Processing job ${job.id} with name ${job.name} on queue ${queueName}${workspaceSuffix}`, - ); - await handler({ data: job.data, id: job.id ?? '', name: job.name }); - const timeEnd = performance.now(); - const executionTime = timeEnd - timeStart; + // TODO: Correctly support for job.id + const timeStart = performance.now(); + const workspaceId = job.data?.workspaceId; + const workspaceSuffix = workspaceId + ? ` [workspace=${workspaceId}]` + : ''; - this.logger.log( - `Job ${job.id} with name ${job.name} processed on queue ${queueName} in ${executionTime.toFixed(2)}ms${workspaceSuffix}`, - ); - }, + this.logger.log( + `Processing job ${job.id} with name ${job.name} on queue ${queueName}${workspaceSuffix}`, + ); + await handler({ data: job.data, id: job.id ?? '', name: job.name }); + const timeEnd = performance.now(); + const executionTime = timeEnd - timeStart; + + this.logger.log( + `Job ${job.id} with name ${job.name} processed on queue ${queueName} in ${executionTime.toFixed(2)}ms${workspaceSuffix}`, + ); + }), workerOptions, ); diff --git a/packages/twenty-server/src/engine/core-modules/sentry/utils/apply-workspace-sentry-context-from-job-data.util.ts b/packages/twenty-server/src/engine/core-modules/sentry/utils/apply-workspace-sentry-context-from-job-data.util.ts new file mode 100644 index 0000000000..0d78ba3515 --- /dev/null +++ b/packages/twenty-server/src/engine/core-modules/sentry/utils/apply-workspace-sentry-context-from-job-data.util.ts @@ -0,0 +1,25 @@ +import { applyWorkspaceSentryFields } from 'src/engine/core-modules/sentry/utils/apply-workspace-sentry-fields.util'; + +export const applyWorkspaceSentryContextFromJobData = ( + jobData: unknown, +): void => { + if (typeof jobData !== 'object' || jobData === null) { + return; + } + + const workspaceId = (jobData as { workspaceId?: unknown }).workspaceId; + const userWorkspaceId = (jobData as { userWorkspaceId?: unknown }) + .userWorkspaceId; + + if (typeof workspaceId !== 'string' || workspaceId.length === 0) { + return; + } + + applyWorkspaceSentryFields({ + workspaceId, + userWorkspaceId: + typeof userWorkspaceId === 'string' && userWorkspaceId.length > 0 + ? userWorkspaceId + : undefined, + }); +}; diff --git a/packages/twenty-server/src/engine/core-modules/sentry/utils/apply-workspace-sentry-context.util.ts b/packages/twenty-server/src/engine/core-modules/sentry/utils/apply-workspace-sentry-context.util.ts new file mode 100644 index 0000000000..3c9ea343d8 --- /dev/null +++ b/packages/twenty-server/src/engine/core-modules/sentry/utils/apply-workspace-sentry-context.util.ts @@ -0,0 +1,27 @@ +import { type WorkspaceAuthContext } from 'src/engine/core-modules/auth/types/workspace-auth-context.type'; +import { applyWorkspaceSentryFields } from 'src/engine/core-modules/sentry/utils/apply-workspace-sentry-fields.util'; + +export const applyWorkspaceSentryContext = ( + authContext: WorkspaceAuthContext, +): void => { + const workspaceId = authContext.workspace?.id; + + if (!workspaceId) { + return; + } + + switch (authContext.type) { + case 'user': + case 'pendingActivationUser': + applyWorkspaceSentryFields({ + workspaceId, + userWorkspaceId: authContext.userWorkspaceId, + }); + return; + case 'apiKey': + case 'application': + case 'system': + applyWorkspaceSentryFields({ workspaceId }); + return; + } +}; diff --git a/packages/twenty-server/src/engine/core-modules/sentry/utils/apply-workspace-sentry-fields.util.ts b/packages/twenty-server/src/engine/core-modules/sentry/utils/apply-workspace-sentry-fields.util.ts new file mode 100644 index 0000000000..a65e17d3d5 --- /dev/null +++ b/packages/twenty-server/src/engine/core-modules/sentry/utils/apply-workspace-sentry-fields.util.ts @@ -0,0 +1,26 @@ +import * as Sentry from '@sentry/node'; + +type WorkspaceSentryFields = { + workspaceId: string; + userWorkspaceId?: string; +}; + +export const applyWorkspaceSentryFields = ( + fields: WorkspaceSentryFields, +): void => { + Sentry.setUser({ + id: fields.userWorkspaceId ?? fields.workspaceId, + }); + + Sentry.setTag('twenty.workspace.id', fields.workspaceId); + if (fields.userWorkspaceId) { + Sentry.setTag('twenty.user_workspace.id', fields.userWorkspaceId); + } + + Sentry.setContext('twenty', { + workspace_id: fields.workspaceId, + ...(fields.userWorkspaceId && { + user_workspace_id: fields.userWorkspaceId, + }), + }); +}; diff --git a/packages/twenty-server/src/instrument.ts b/packages/twenty-server/src/instrument.ts index 83ee3da9bb..bcdb07da45 100644 --- a/packages/twenty-server/src/instrument.ts +++ b/packages/twenty-server/src/instrument.ts @@ -1,7 +1,6 @@ import process from 'process'; -import { ExportResultCode } from '@opentelemetry/core'; -import opentelemetry from '@opentelemetry/api'; +import { metrics as otelMetrics } from '@opentelemetry/api'; import { OTLPMetricExporter } from '@opentelemetry/exporter-metrics-otlp-http'; import { PrometheusExporter } from '@opentelemetry/exporter-prometheus'; import { @@ -9,8 +8,6 @@ import { ConsoleMetricExporter, MeterProvider, PeriodicExportingMetricReader, - type PushMetricExporter, - type ResourceMetrics, } from '@opentelemetry/sdk-metrics'; import * as Sentry from '@sentry/node'; import { nodeProfilingIntegration } from '@sentry/profiling-node'; @@ -27,91 +24,6 @@ const meterDrivers = parseArrayEnvVar( [], ); -const OTLP_METRICS_LOG_PREFIX = '[Twenty OTEL metrics]'; - -const formatOtlEndpointForLog = (rawUrl: string | undefined): string => { - if (!rawUrl?.trim()) { - return 'unset'; - } - try { - const parsed = new URL(rawUrl); - - return `${parsed.protocol}//${parsed.host}${parsed.pathname}`; - } catch { - return 'invalid_url'; - } -}; - -const serializeOtlExportFailure = (error: unknown): string => { - if (error === undefined || error === null) { - return 'none'; - } - if (error instanceof Error) { - const base = `${error.name}: ${error.message}`; - const maybeAggregate = error as Error & { errors?: unknown[] }; - if ( - Array.isArray(maybeAggregate.errors) && - maybeAggregate.errors.length > 0 - ) { - const causes = maybeAggregate.errors.map((cause: unknown) => - serializeOtlExportFailure(cause), - ); - - return `${base} | causes=[${causes.join(' | ')}]`; - } - - return base; - } - try { - return JSON.stringify(error); - } catch { - return String(error); - } -}; - -let otlpMetricsFirstExportSuccessLogged = false; -let otlpMetricsFirstExportAttemptLogged = false; - -const wrapOtlMetricExporterWithProcessLogs = ( - inner: OTLPMetricExporter, - otlpEndpointForLog: string, -): PushMetricExporter => ({ - export(metrics: ResourceMetrics, resultCallback) { - if (!otlpMetricsFirstExportAttemptLogged) { - otlpMetricsFirstExportAttemptLogged = true; - console.log( - `${OTLP_METRICS_LOG_PREFIX} first periodic export attempt | endpoint=${otlpEndpointForLog}`, - ); - } - const totalMetricData = metrics.scopeMetrics.reduce( - (accumulator, scopeMetric) => accumulator + scopeMetric.metrics.length, - 0, - ); - inner.export(metrics, (result) => { - if (result.code === ExportResultCode.SUCCESS) { - if (!otlpMetricsFirstExportSuccessLogged) { - otlpMetricsFirstExportSuccessLogged = true; - // One-time line so prod logs stay usable; further success is silent. - console.log( - `${OTLP_METRICS_LOG_PREFIX} first export ok | endpoint=${otlpEndpointForLog} | metricDataPoints=${totalMetricData}`, - ); - } - } else { - console.warn( - `${OTLP_METRICS_LOG_PREFIX} export failed | endpoint=${otlpEndpointForLog} | metricDataPoints=${totalMetricData} | code=${result.code} | error=${serializeOtlExportFailure(result.error)}`, - ); - } - resultCallback(result); - }); - }, - forceFlush: () => inner.forceFlush(), - shutdown: () => inner.shutdown(), - selectAggregationTemporality: (instrumentType) => - inner.selectAggregationTemporality(instrumentType), - selectAggregation: (instrumentType) => - inner.selectAggregation(instrumentType), -}); - if (process.env.EXCEPTION_HANDLER_DRIVER === ExceptionHandlerDriver.SENTRY) { Sentry.init({ environment: process.env.SENTRY_ENVIRONMENT, @@ -133,24 +45,34 @@ if (process.env.EXCEPTION_HANDLER_DRIVER === ExceptionHandlerDriver.SENTRY) { profilesSampleRate: 0.3, sendDefaultPii: true, debug: process.env.NODE_ENV === NodeEnvironment.DEVELOPMENT, + beforeSendSpan: (span) => { + const twentyContext = Sentry.getIsolationScope().getScopeData().contexts + ?.twenty as + | { + workspace_id?: string; + user_workspace_id?: string; + } + | undefined; + + if (!twentyContext?.workspace_id) { + return span; + } + + span.data = { + ...span.data, + 'twenty.workspace.id': twentyContext.workspace_id, + ...(twentyContext.user_workspace_id && { + 'twenty.user_workspace.id': twentyContext.user_workspace_id, + }), + }; + + return span; + }, }); } // Meter setup -const otlpCollectorMetricsUrl = process.env.OTLP_COLLECTOR_METRICS_ENDPOINT_URL; -const otlpEndpointForLog = formatOtlEndpointForLog(otlpCollectorMetricsUrl); - -if (meterDrivers.includes(MeterDriver.OpenTelemetry)) { - console.log( - `${OTLP_METRICS_LOG_PREFIX} OTLP reader enabled | exportIntervalMs=10000 | endpoint=${otlpEndpointForLog} | meterDrivers=${meterDrivers.join(',')}`, - ); -} else { - console.log( - `${OTLP_METRICS_LOG_PREFIX} OTLP reader disabled | parsedMeterDrivers=${JSON.stringify(meterDrivers)} | rawMETER_DRIVER=${JSON.stringify(process.env.METER_DRIVER ?? '')}`, - ); -} - const prometheusExporter = meterDrivers.includes(MeterDriver.Prometheus) ? new PrometheusExporter({ port: 9464 }) : null; @@ -168,13 +90,10 @@ const meterProvider = new MeterProvider({ ...(meterDrivers.includes(MeterDriver.OpenTelemetry) ? [ new PeriodicExportingMetricReader({ - exporter: wrapOtlMetricExporterWithProcessLogs( - new OTLPMetricExporter({ - url: otlpCollectorMetricsUrl, - temporalityPreference: AggregationTemporality.DELTA, - }), - otlpEndpointForLog, - ), + exporter: new OTLPMetricExporter({ + url: process.env.OTLP_COLLECTOR_METRICS_ENDPOINT_URL, + temporalityPreference: AggregationTemporality.DELTA, + }), exportIntervalMillis: 10000, }), ] @@ -183,20 +102,4 @@ const meterProvider = new MeterProvider({ ], }); -opentelemetry.metrics.setGlobalMeterProvider(meterProvider); - -// Always-on gauge so the OTLP exporter fires on every collection tick, -// even when the process is otherwise idle. This guarantees the process-log -// wrapper above can prove connectivity (first export ok / export failed). -if (meterDrivers.includes(MeterDriver.OpenTelemetry)) { - const heartbeatMeter = opentelemetry.metrics.getMeter( - 'twenty-server-heartbeat', - ); - const heartbeat = heartbeatMeter.createObservableGauge('twenty.heartbeat', { - description: 'Always-on gauge (1) to prove OTLP export pipeline', - }); - - heartbeat.addCallback((observableResult) => { - observableResult.observe(1); - }); -} +otelMetrics.setGlobalMeterProvider(meterProvider); diff --git a/yarn.lock b/yarn.lock index d96a118835..4425fc16a0 100644 --- a/yarn.lock +++ b/yarn.lock @@ -14370,14 +14370,7 @@ __metadata: languageName: node linkType: hard -"@opentelemetry/api@npm:1.9.0, @opentelemetry/api@npm:^1.3.0, @opentelemetry/api@npm:^1.9.0": - version: 1.9.0 - resolution: "@opentelemetry/api@npm:1.9.0" - checksum: 10c0/9aae2fe6e8a3a3eeb6c1fdef78e1939cf05a0f37f8a4fae4d6bf2e09eb1e06f966ece85805626e01ba5fab48072b94f19b835449e58b6d26720ee19a58298add - languageName: node - linkType: hard - -"@opentelemetry/api@npm:^1.9.1": +"@opentelemetry/api@npm:1.9.1": version: 1.9.1 resolution: "@opentelemetry/api@npm:1.9.1" checksum: 10c0/c608485fc8b5a91e1f7e05e843b45b509307456b31cd2ad365933d90813e40ebfedf179f1451c762037e82d7c76aa8500e95d2da3609f640a1206cde5322cd14 @@ -20408,7 +20401,7 @@ __metadata: languageName: node linkType: hard -"@sentry/nestjs@npm:^10.27.0": +"@sentry/nestjs@npm:^10.51.0": version: 10.51.0 resolution: "@sentry/nestjs@npm:10.51.0" dependencies: @@ -20457,7 +20450,7 @@ __metadata: languageName: node linkType: hard -"@sentry/node@npm:10.51.0, @sentry/node@npm:^10.27.0": +"@sentry/node@npm:10.51.0, @sentry/node@npm:^10.51.0": version: 10.51.0 resolution: "@sentry/node@npm:10.51.0" dependencies: @@ -20510,7 +20503,7 @@ __metadata: languageName: node linkType: hard -"@sentry/profiling-node@npm:^10.27.0": +"@sentry/profiling-node@npm:^10.51.0": version: 10.51.0 resolution: "@sentry/profiling-node@npm:10.51.0" dependencies: @@ -20523,7 +20516,7 @@ __metadata: languageName: node linkType: hard -"@sentry/react@npm:^10.27.0": +"@sentry/react@npm:^10.51.0": version: 10.51.0 resolution: "@sentry/react@npm:10.51.0" dependencies: @@ -57848,7 +57841,7 @@ __metadata: "@react-email/components": "npm:^0.5.3" "@react-pdf/renderer": "npm:^4.1.6" "@scalar/api-reference-react": "npm:^0.4.36" - "@sentry/react": "npm:^10.27.0" + "@sentry/react": "npm:^10.51.0" "@sniptt/guards": "npm:^0.2.0" "@storybook-community/storybook-addon-cookie": "npm:^5.0.0" "@storybook/addon-coverage": "npm:^3.0.0" @@ -58096,9 +58089,9 @@ __metadata: "@ptc-org/nestjs-query-graphql": "patch:@ptc-org/nestjs-query-graphql@4.2.0#./patches/@ptc-org+nestjs-query-graphql+4.2.0.patch" "@ptc-org/nestjs-query-typeorm": "npm:4.2.1-alpha.2" "@react-email/render": "npm:^1.2.3" - "@sentry/nestjs": "npm:^10.27.0" - "@sentry/node": "npm:^10.27.0" - "@sentry/profiling-node": "npm:^10.27.0" + "@sentry/nestjs": "npm:^10.51.0" + "@sentry/node": "npm:^10.51.0" + "@sentry/profiling-node": "npm:^10.51.0" "@sniptt/guards": "npm:0.2.0" "@swc/cli": "npm:^0.7.10" "@swc/core": "npm:^1.15.11"