From 024b379e1a590e8952960689fe4a45c210aa41e0 Mon Sep 17 00:00:00 2001 From: Weiko Date: Tue, 21 Jul 2026 16:02:58 +0200 Subject: [PATCH] Instrument Node runtime and workspace cache metrics (#23107) ## Context High API tail latency can come from either downstream work or the Node.js process itself being unable to schedule work. Existing traces expose database and HTTP spans, but they do not provide a continuous event-loop signal or identify time spent rebuilding individual workspace metadata cache entries. ## What changed - Enable Sentry's built-in Node runtime integration with a 30-second collection interval. - Collect only event-loop delay p99, event-loop delay max, and event-loop utilization. CPU, memory, p50, and uptime metrics remain disabled because existing infrastructure telemetry already covers those areas. - Add a parent span around workspace metadata cache invalidation and recomputation. - Add child spans around cache-provider computation, including the cache key, recomputation strategy, and whether the provider uses local data only. ## Telemetry scope - Cache hits do not create spans. - Cache spans use `onlyIfParent`, so they are recorded only inside an already-sampled trace. - Runtime metrics are three low-cardinality values every 30 seconds per server process. - This does not add Prometheus histograms, per-cache-key metric labels, database pool gauges, or a custom runtime collector. - No cache behavior or invalidation semantics change. This should let us distinguish event-loop stalls from downstream latency, then identify which cache provider contributes to a slow cache rebuild without materially increasing telemetry volume. --- .../__tests__/workspace-cache.service.spec.ts | 32 ++++++++++++++ .../services/workspace-cache.service.ts | 42 +++++++++++++++---- packages/twenty-server/src/instrument.ts | 14 +++++++ 3 files changed, 79 insertions(+), 9 deletions(-) diff --git a/packages/twenty-server/src/engine/workspace-cache/services/__tests__/workspace-cache.service.spec.ts b/packages/twenty-server/src/engine/workspace-cache/services/__tests__/workspace-cache.service.spec.ts index cd0462c65d..ef391dfb34 100644 --- a/packages/twenty-server/src/engine/workspace-cache/services/__tests__/workspace-cache.service.spec.ts +++ b/packages/twenty-server/src/engine/workspace-cache/services/__tests__/workspace-cache.service.spec.ts @@ -1,6 +1,8 @@ import { DiscoveryService, Reflector } from '@nestjs/core'; import { Test, type TestingModule } from '@nestjs/testing'; +import * as Sentry from '@sentry/node'; + import { WorkspaceCacheProvider } from 'src/engine/workspace-cache/interfaces/workspace-cache-provider.service'; import { type CacheStorageService } from 'src/engine/core-modules/cache-storage/services/cache-storage.service'; @@ -13,6 +15,10 @@ import { } from 'src/engine/workspace-cache/decorators/workspace-cache.decorator'; import { WorkspaceCacheService } from 'src/engine/workspace-cache/services/workspace-cache.service'; +jest.mock('@sentry/node', () => ({ + startSpan: jest.fn(), +})); + const WORKSPACE_ID = '20202020-0000-4000-8000-000000000000'; class MockFeatureFlagsCacheProvider extends WorkspaceCacheProvider<{ @@ -48,6 +54,9 @@ describe('WorkspaceCacheService', () => { beforeEach(async () => { jest.useFakeTimers(); + jest + .mocked(Sentry.startSpan) + .mockImplementation((_options, callback) => callback({} as never)); mockProvider = new MockFeatureFlagsCacheProvider(); @@ -186,6 +195,19 @@ describe('WorkspaceCacheService', () => { }); expect(mockProvider.computeForCache).toHaveBeenCalledWith(WORKSPACE_ID); expect(cacheStorageService.mset).toHaveBeenCalled(); + expect(Sentry.startSpan).toHaveBeenCalledWith( + { + name: 'compute workspace metadata cache entry from provider', + op: 'cache.recompute', + onlyIfParent: true, + attributes: { + 'cache.key_name': 'featureFlagsMap', + 'cache.recompute.strategy': 'recover', + 'cache.local_data_only': false, + }, + }, + expect.any(Function), + ); }); it('should return data from redis when available', async () => { @@ -203,6 +225,7 @@ describe('WorkspaceCacheService', () => { ]); expect(result).toEqual({ featureFlagsMap: cachedData }); + expect(Sentry.startSpan).not.toHaveBeenCalled(); }); it('should use local cache when within TTL staleness window', async () => { @@ -295,6 +318,15 @@ describe('WorkspaceCacheService', () => { ]); expect(mockProvider.computeForCache).toHaveBeenCalledWith(WORKSPACE_ID); expect(cacheStorageService.mset).toHaveBeenCalled(); + expect(Sentry.startSpan).toHaveBeenCalledWith( + { + name: 'invalidate and recompute workspace metadata cache', + op: 'cache.invalidate', + onlyIfParent: true, + attributes: { 'cache.key_count': 1 }, + }, + expect.any(Function), + ); }); it('should invalidate multiple cache keys at once', async () => { diff --git a/packages/twenty-server/src/engine/workspace-cache/services/workspace-cache.service.ts b/packages/twenty-server/src/engine/workspace-cache/services/workspace-cache.service.ts index 1344bbf48b..f5339163c2 100644 --- a/packages/twenty-server/src/engine/workspace-cache/services/workspace-cache.service.ts +++ b/packages/twenty-server/src/engine/workspace-cache/services/workspace-cache.service.ts @@ -1,6 +1,7 @@ import { Injectable, Logger, OnModuleInit } from '@nestjs/common'; import { DiscoveryService, Reflector } from '@nestjs/core'; +import * as Sentry from '@sentry/node'; import crypto from 'crypto'; import { isDefined, isValidUuid } from 'twenty-shared/utils'; @@ -185,16 +186,26 @@ export class WorkspaceCacheService implements OnModuleInit { workspaceId: string, cacheKeyNames: WorkspaceCacheKeyName[], ): Promise { - await this.memoizer.clearKeys(`${workspaceId}-`); + return Sentry.startSpan( + { + name: 'invalidate and recompute workspace metadata cache', + op: 'cache.invalidate', + onlyIfParent: true, + attributes: { 'cache.key_count': cacheKeyNames.length }, + }, + async () => { + await this.memoizer.clearKeys(`${workspaceId}-`); - await this.flush(workspaceId, cacheKeyNames); - await this.recomputeDataFromProvider(workspaceId, cacheKeyNames, { - strategy: 'mint', - }); + await this.flush(workspaceId, cacheKeyNames); + await this.recomputeDataFromProvider(workspaceId, cacheKeyNames, { + strategy: 'mint', + }); - // Clear memoizer again after recomputation to evict any stale entries - // cached by concurrent getOrRecompute calls during the flush window. - await this.memoizer.clearKeys(`${workspaceId}-`); + // Clear memoizer again after recomputation to evict any stale entries + // cached by concurrent getOrRecompute calls during the flush window. + await this.memoizer.clearKeys(`${workspaceId}-`); + }, + ); } public async getCacheHashes( @@ -366,7 +377,20 @@ export class WorkspaceCacheService implements OnModuleInit { const computePromises = cacheKeyNames.map(async (keyName) => { const provider = this.getProviderOrThrow(keyName); - const data = await provider.computeForCache(workspaceId); + const isLocalDataOnly = this.localDataOnlyKeys.has(keyName); + const data = await Sentry.startSpan( + { + name: 'compute workspace metadata cache entry from provider', + op: 'cache.recompute', + onlyIfParent: true, + attributes: { + 'cache.key_name': keyName, + 'cache.recompute.strategy': hashResolution.strategy, + 'cache.local_data_only': isLocalDataOnly, + }, + }, + () => provider.computeForCache(workspaceId), + ); if (hashResolution.strategy === 'mint') { return { keyName, data, hash: crypto.randomUUID(), isAdopted: false }; diff --git a/packages/twenty-server/src/instrument.ts b/packages/twenty-server/src/instrument.ts index bcdb07da45..f367c747e8 100644 --- a/packages/twenty-server/src/instrument.ts +++ b/packages/twenty-server/src/instrument.ts @@ -35,6 +35,20 @@ if (process.env.EXCEPTION_HANDLER_DRIVER === ExceptionHandlerDriver.SENTRY) { Sentry.expressIntegration(), Sentry.graphqlIntegration(), Sentry.postgresIntegration(), + Sentry.nodeRuntimeMetricsIntegration({ + collectionIntervalMs: 30_000, + collect: { + cpuUtilization: false, + memHeapUsed: false, + memHeapTotal: false, + memRss: false, + eventLoopDelayP50: false, + eventLoopDelayP99: true, + eventLoopDelayMax: true, + eventLoopUtilization: true, + uptime: false, + }, + }), Sentry.vercelAIIntegration({ recordInputs: true, recordOutputs: true,