Instrument Node runtime and workspace cache metrics (#23107)

## Context

High API tail latency can come from either downstream work or the
Node.js process itself being unable to schedule work. Existing traces
expose database and HTTP spans, but they do not provide a continuous
event-loop signal or identify time spent rebuilding individual workspace
metadata cache entries.

## What changed

- Enable Sentry's built-in Node runtime integration with a 30-second
collection interval.
- Collect only event-loop delay p99, event-loop delay max, and
event-loop utilization. CPU, memory, p50, and uptime metrics remain
disabled because existing infrastructure telemetry already covers those
areas.
- Add a parent span around workspace metadata cache invalidation and
recomputation.
- Add child spans around cache-provider computation, including the cache
key, recomputation strategy, and whether the provider uses local data
only.

## Telemetry scope

- Cache hits do not create spans.
- Cache spans use `onlyIfParent`, so they are recorded only inside an
already-sampled trace.
- Runtime metrics are three low-cardinality values every 30 seconds per
server process.
- This does not add Prometheus histograms, per-cache-key metric labels,
database pool gauges, or a custom runtime collector.
- No cache behavior or invalidation semantics change.

This should let us distinguish event-loop stalls from downstream
latency, then identify which cache provider contributes to a slow cache
rebuild without materially increasing telemetry volume.
This commit is contained in:
Weiko
2026-07-21 16:02:58 +02:00
committed by GitHub
parent 64891c561d
commit 024b379e1a
3 changed files with 79 additions and 9 deletions
@@ -1,6 +1,8 @@
import { DiscoveryService, Reflector } from '@nestjs/core';
import { Test, type TestingModule } from '@nestjs/testing';
import * as Sentry from '@sentry/node';
import { WorkspaceCacheProvider } from 'src/engine/workspace-cache/interfaces/workspace-cache-provider.service';
import { type CacheStorageService } from 'src/engine/core-modules/cache-storage/services/cache-storage.service';
@@ -13,6 +15,10 @@ import {
} from 'src/engine/workspace-cache/decorators/workspace-cache.decorator';
import { WorkspaceCacheService } from 'src/engine/workspace-cache/services/workspace-cache.service';
jest.mock('@sentry/node', () => ({
startSpan: jest.fn(),
}));
const WORKSPACE_ID = '20202020-0000-4000-8000-000000000000';
class MockFeatureFlagsCacheProvider extends WorkspaceCacheProvider<{
@@ -48,6 +54,9 @@ describe('WorkspaceCacheService', () => {
beforeEach(async () => {
jest.useFakeTimers();
jest
.mocked(Sentry.startSpan)
.mockImplementation((_options, callback) => callback({} as never));
mockProvider = new MockFeatureFlagsCacheProvider();
@@ -186,6 +195,19 @@ describe('WorkspaceCacheService', () => {
});
expect(mockProvider.computeForCache).toHaveBeenCalledWith(WORKSPACE_ID);
expect(cacheStorageService.mset).toHaveBeenCalled();
expect(Sentry.startSpan).toHaveBeenCalledWith(
{
name: 'compute workspace metadata cache entry from provider',
op: 'cache.recompute',
onlyIfParent: true,
attributes: {
'cache.key_name': 'featureFlagsMap',
'cache.recompute.strategy': 'recover',
'cache.local_data_only': false,
},
},
expect.any(Function),
);
});
it('should return data from redis when available', async () => {
@@ -203,6 +225,7 @@ describe('WorkspaceCacheService', () => {
]);
expect(result).toEqual({ featureFlagsMap: cachedData });
expect(Sentry.startSpan).not.toHaveBeenCalled();
});
it('should use local cache when within TTL staleness window', async () => {
@@ -295,6 +318,15 @@ describe('WorkspaceCacheService', () => {
]);
expect(mockProvider.computeForCache).toHaveBeenCalledWith(WORKSPACE_ID);
expect(cacheStorageService.mset).toHaveBeenCalled();
expect(Sentry.startSpan).toHaveBeenCalledWith(
{
name: 'invalidate and recompute workspace metadata cache',
op: 'cache.invalidate',
onlyIfParent: true,
attributes: { 'cache.key_count': 1 },
},
expect.any(Function),
);
});
it('should invalidate multiple cache keys at once', async () => {
@@ -1,6 +1,7 @@
import { Injectable, Logger, OnModuleInit } from '@nestjs/common';
import { DiscoveryService, Reflector } from '@nestjs/core';
import * as Sentry from '@sentry/node';
import crypto from 'crypto';
import { isDefined, isValidUuid } from 'twenty-shared/utils';
@@ -185,16 +186,26 @@ export class WorkspaceCacheService implements OnModuleInit {
workspaceId: string,
cacheKeyNames: WorkspaceCacheKeyName[],
): Promise<void> {
await this.memoizer.clearKeys(`${workspaceId}-`);
return Sentry.startSpan(
{
name: 'invalidate and recompute workspace metadata cache',
op: 'cache.invalidate',
onlyIfParent: true,
attributes: { 'cache.key_count': cacheKeyNames.length },
},
async () => {
await this.memoizer.clearKeys(`${workspaceId}-`);
await this.flush(workspaceId, cacheKeyNames);
await this.recomputeDataFromProvider(workspaceId, cacheKeyNames, {
strategy: 'mint',
});
await this.flush(workspaceId, cacheKeyNames);
await this.recomputeDataFromProvider(workspaceId, cacheKeyNames, {
strategy: 'mint',
});
// Clear memoizer again after recomputation to evict any stale entries
// cached by concurrent getOrRecompute calls during the flush window.
await this.memoizer.clearKeys(`${workspaceId}-`);
// Clear memoizer again after recomputation to evict any stale entries
// cached by concurrent getOrRecompute calls during the flush window.
await this.memoizer.clearKeys(`${workspaceId}-`);
},
);
}
public async getCacheHashes(
@@ -366,7 +377,20 @@ export class WorkspaceCacheService implements OnModuleInit {
const computePromises = cacheKeyNames.map(async (keyName) => {
const provider = this.getProviderOrThrow(keyName);
const data = await provider.computeForCache(workspaceId);
const isLocalDataOnly = this.localDataOnlyKeys.has(keyName);
const data = await Sentry.startSpan(
{
name: 'compute workspace metadata cache entry from provider',
op: 'cache.recompute',
onlyIfParent: true,
attributes: {
'cache.key_name': keyName,
'cache.recompute.strategy': hashResolution.strategy,
'cache.local_data_only': isLocalDataOnly,
},
},
() => provider.computeForCache(workspaceId),
);
if (hashResolution.strategy === 'mint') {
return { keyName, data, hash: crypto.randomUUID(), isAdopted: false };
+14
View File
@@ -35,6 +35,20 @@ if (process.env.EXCEPTION_HANDLER_DRIVER === ExceptionHandlerDriver.SENTRY) {
Sentry.expressIntegration(),
Sentry.graphqlIntegration(),
Sentry.postgresIntegration(),
Sentry.nodeRuntimeMetricsIntegration({
collectionIntervalMs: 30_000,
collect: {
cpuUtilization: false,
memHeapUsed: false,
memHeapTotal: false,
memRss: false,
eventLoopDelayP50: false,
eventLoopDelayP99: true,
eventLoopDelayMax: true,
eventLoopUtilization: true,
uptime: false,
},
}),
Sentry.vercelAIIntegration({
recordInputs: true,
recordOutputs: true,