Files
twenty/packages/twenty-server/src/engine/subscriptions/event-stream.resolver.ts
T
Charles Bochet 1c3ae92c04 fix(server): prevent SSE stream teardown errors from crashing all pods (#21395)
## Context

In prod-eu, **all `twenty-server` API pods crash simultaneously**
several times per hour (then restart in lockstep) since ~Jun 2. Each
crash is an **unhandled promise rejection** in SSE event-stream teardown
— `exitCode=1`, identical stack on every one of the 7 pods:

```
Error: Failed to acquire lock for key: workspace:<id>:activeStreams
  at CacheLockService.withLock (cache-lock.service.ts:53)
  at async EventStreamService.destroyEventStream (event-stream.service.ts:75)
  at async cleanup (wrap-async-iterator-with-lifecycle.ts)
  at async Object.return (wrap-async-iterator-with-lifecycle.ts)
  at async Object.cancel (graphql-yoga/.../result-processor/sse.js:68)
```

### Mechanism

1. A busy workspace's SSE clients reconnect (no client backoff), so
connect/disconnect contend on a **single per-workspace Redis lock**
`workspace:<id>:activeStreams`.
2. Under contention `CacheLockService.withLock` exhausts its retries and
**throws**.
3. In the `destroyEventStream` teardown path the throw escapes
`wrapAsyncIteratorWithLifecycle`'s `cleanup()` — `return()` does `try {
await cleanup() } finally { … }` and does **not** catch a cleanup throw.
4. graphql-yoga invokes this from `cancel()` as a **fire-and-forget**
`Promise.all` on connection abort. With **no global `unhandledRejection`
handler**, Node's default policy terminates the process with **exit code
1**.
5. The crash drops all that pod's SSE clients → they reconnect to
surviving pods → contention moves there → the whole fleet crashes
together → restarts → reconnect storm → repeats (~14 min period,
matching the metrics).

## Changes

Crash-stopping hotfix (defense in depth). Does **not** change the
locking design or client reconnect behavior — see follow-ups.

- **`wrapAsyncIteratorWithLifecycle`**: `onCleanup()` is now best-effort
— wrapped in try/catch so teardown can never reject out of
`next()`/`return()`/`throw()`. The original iterator error is still
rethrown unchanged. Adds an `onCleanupError` hook so the swallowed error
is still reported.
- **`EventStreamResolver`**: wires `onCleanupError` to
`ExceptionHandlerService.captureExceptions` (→ Sentry) with workspace +
channel context, so these failures stay visible.
- **`main.ts`**: registers a global `process.on('unhandledRejection')`
that reports via `ExceptionHandlerService` (Sentry) instead of letting
Node terminate. Registering the listener also suppresses Node's default
process-termination. Non-`Error` reasons are formatted with
`util.inspect` (per Copilot review) so Sentry gets a readable message
rather than `[object Object]`.

## Verification

The fix was checked against the exact crash path — a wrapped iterator
whose `onCleanup` rejects:

- `return()` (graphql-yoga's `cancel()` path) **resolves** instead of
rejecting → no unhandled rejection.
- `next()` on a completed stream **resolves** despite a rejecting
cleanup.
- A genuine iterator error still surfaces on `next()` (cleanup failure
doesn't mask it).
- `onCleanupError` receives the original `Error`.

All four pass. `nx lint twenty-server` + `oxfmt` clean; no new `tsc`
errors in the changed files.

## Follow-ups (not in this PR)

- Remove the unnecessary `withLock` around the already-atomic Redis
`SADD`/`SREM` in `event-stream.service.ts` (the contention source).
- Restore exponential backoff + jitter on the frontend SSE reconnect
(regressed in #21061) to stop the thundering herd.
- Infra: make `/healthz` a meaningful liveness signal and add
`maxUnavailable` + a PodDisruptionBudget so pods can't all die together.
2026-06-10 10:05:55 +00:00

253 lines
8.9 KiB
TypeScript

import { UseFilters, UseGuards, UsePipes } from '@nestjs/common';
import { Args, Mutation, Subscription } from '@nestjs/graphql';
import { isDefined } from 'twenty-shared/utils';
import { MetadataResolver } from 'src/engine/api/graphql/graphql-config/decorators/metadata-resolver.decorator';
import { type ApiKeyEntity } from 'src/engine/core-modules/api-key/api-key.entity';
import { ExceptionHandlerService } from 'src/engine/core-modules/exception-handler/exception-handler.service';
import { PreventNestToAutoLogGraphqlErrorsFilter } from 'src/engine/core-modules/graphql/filters/prevent-nest-to-auto-log-graphql-errors.filter';
import { ResolverValidationPipe } from 'src/engine/core-modules/graphql/pipes/resolver-validation.pipe';
import { type AuthContextUser } from 'src/engine/core-modules/auth/types/auth-context.type';
import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity';
import { AuthApiKey } from 'src/engine/decorators/auth/auth-api-key.decorator';
import { AuthUserWorkspaceId } from 'src/engine/decorators/auth/auth-user-workspace-id.decorator';
import { AuthUser } from 'src/engine/decorators/auth/auth-user.decorator';
import { AuthWorkspace } from 'src/engine/decorators/auth/auth-workspace.decorator';
import { NoPermissionGuard } from 'src/engine/guards/no-permission.guard';
import { UserAuthGuard } from 'src/engine/guards/user-auth.guard';
import { WorkspaceAuthGuard } from 'src/engine/guards/workspace-auth.guard';
import { APPLICATION_KEEPALIVE_INTERVAL_MS } from 'src/engine/subscriptions/constants/application-keepalive-interval-ms.constant';
import { EVENT_STREAM_TTL_MS } from 'src/engine/subscriptions/constants/event-stream-ttl.constant';
import { AddQuerySubscriptionInput } from 'src/engine/subscriptions/dtos/add-query-subscription.input';
import { EventSubscriptionDTO } from 'src/engine/subscriptions/dtos/event-subscription.dto';
import { RemoveQueryFromEventStreamInput } from 'src/engine/subscriptions/dtos/remove-query-subscription.input';
import { EventStreamExceptionFilter } from 'src/engine/subscriptions/event-stream-exception.filter';
import {
EventStreamException,
EventStreamExceptionCode,
} from 'src/engine/subscriptions/event-stream.exception';
import { EventStreamService } from 'src/engine/subscriptions/event-stream.service';
import { SubscriptionService } from 'src/engine/subscriptions/subscription.service';
import { type EventStreamPayload } from 'src/engine/subscriptions/types/event-stream-payload.type';
import { eventStreamIdToChannelId } from 'src/engine/subscriptions/utils/get-channel-id-from-event-stream-id';
import { wrapAsyncIteratorWithLifecycle } from 'src/engine/subscriptions/utils/wrap-async-iterator-with-lifecycle';
@MetadataResolver()
@UseGuards(WorkspaceAuthGuard, UserAuthGuard, NoPermissionGuard)
@UsePipes(ResolverValidationPipe)
@UseFilters(EventStreamExceptionFilter, PreventNestToAutoLogGraphqlErrorsFilter)
export class EventStreamResolver {
constructor(
private readonly subscriptionService: SubscriptionService,
private readonly eventStreamService: EventStreamService,
private readonly exceptionHandlerService: ExceptionHandlerService,
) {}
@Subscription(() => EventSubscriptionDTO, {
nullable: true,
resolve: (
payload: EventStreamPayload,
variables: { eventStreamId: string },
) => {
return {
eventStreamId: variables.eventStreamId,
objectRecordEventsWithQueryIds: payload.objectRecordEventsWithQueryIds,
metadataEvents: payload.metadataEvents,
};
},
})
async onEventSubscription(
@Args('eventStreamId') eventStreamId: string,
@AuthWorkspace() workspace: WorkspaceEntity,
@AuthUser({ allowUndefined: true }) user: AuthContextUser | undefined,
@AuthUserWorkspaceId({ allowUndefined: true })
userWorkspaceId: string | undefined,
@AuthApiKey() apiKey: ApiKeyEntity | undefined,
) {
const eventStreamChannelId = eventStreamIdToChannelId(eventStreamId);
const existingStreamData = await this.eventStreamService.getStreamData(
workspace.id,
eventStreamChannelId,
);
if (isDefined(existingStreamData)) {
const isAuthorized = await this.eventStreamService.isAuthorized({
streamData: existingStreamData,
authContext: {
userWorkspaceId,
apiKeyId: apiKey?.id,
},
});
if (!isAuthorized) {
throw new EventStreamException(
'Event stream already exists',
EventStreamExceptionCode.EVENT_STREAM_ALREADY_EXISTS,
);
}
await this.eventStreamService.destroyEventStream({
workspaceId: workspace.id,
eventStreamChannelId,
});
}
await this.eventStreamService.createEventStream({
workspaceId: workspace.id,
eventStreamChannelId,
authContext: {
userId: user?.id,
userWorkspaceId,
apiKeyId: apiKey?.id,
},
});
let iterator: AsyncIterableIterator<EventStreamPayload>;
try {
iterator = await this.subscriptionService.subscribeToEventStream({
workspaceId: workspace.id,
eventStreamChannelId,
});
} catch (error) {
await this.eventStreamService.destroyEventStream({
workspaceId: workspace.id,
eventStreamChannelId,
});
throw error;
}
let lastTtlRefreshAt = 0;
return wrapAsyncIteratorWithLifecycle(iterator, {
initialValue: {
objectRecordEventsWithQueryIds: [],
metadataEvents: [],
},
onHeartbeat: async () => {
const now = Date.now();
if (now - lastTtlRefreshAt > EVENT_STREAM_TTL_MS / 5) {
lastTtlRefreshAt = now;
await this.eventStreamService.refreshEventStreamTTL({
workspaceId: workspace.id,
eventStreamChannelId,
});
}
await this.subscriptionService.publishToEventStream({
workspaceId: workspace.id,
eventStreamChannelId,
payload: {
objectRecordEventsWithQueryIds: [],
metadataEvents: [],
},
});
return true;
},
heartbeatIntervalMs: APPLICATION_KEEPALIVE_INTERVAL_MS,
onCleanup: () =>
this.eventStreamService.destroyEventStream({
workspaceId: workspace.id,
eventStreamChannelId,
}),
onCleanupError: (error) =>
this.exceptionHandlerService.captureExceptions([error], {
workspace: { id: workspace.id },
additionalData: { eventStreamChannelId },
}),
});
}
@Mutation(() => Boolean)
async addQueryToEventStream(
@Args('input') input: AddQuerySubscriptionInput,
@AuthWorkspace() workspace: WorkspaceEntity,
@AuthUser({ allowUndefined: true }) user: AuthContextUser | undefined,
@AuthUserWorkspaceId({ allowUndefined: true })
userWorkspaceId: string | undefined,
@AuthApiKey() apiKey: ApiKeyEntity | undefined,
): Promise<boolean> {
const eventStreamChannelId = eventStreamIdToChannelId(input.eventStreamId);
const streamData = await this.eventStreamService.getStreamData(
workspace.id,
eventStreamChannelId,
);
if (!isDefined(streamData)) {
return false;
}
const isAuthorized = await this.eventStreamService.isAuthorized({
streamData,
authContext: {
userWorkspaceId,
apiKeyId: apiKey?.id,
},
});
if (!isAuthorized) {
throw new EventStreamException(
'You are not authorized to add a query to this event stream',
EventStreamExceptionCode.NOT_AUTHORIZED,
);
}
await this.eventStreamService.addQuery({
workspaceId: workspace.id,
eventStreamChannelId,
queryId: input.queryId,
operationSignature: input.operationSignature,
});
return true;
}
@Mutation(() => Boolean)
async removeQueryFromEventStream(
@Args('input') input: RemoveQueryFromEventStreamInput,
@AuthWorkspace() workspace: WorkspaceEntity,
@AuthUser({ allowUndefined: true }) user: AuthContextUser | undefined,
@AuthUserWorkspaceId({ allowUndefined: true })
userWorkspaceId: string | undefined,
@AuthApiKey() apiKey: ApiKeyEntity | undefined,
): Promise<boolean> {
const eventStreamChannelId = eventStreamIdToChannelId(input.eventStreamId);
const streamData = await this.eventStreamService.getStreamData(
workspace.id,
eventStreamChannelId,
);
if (!isDefined(streamData)) {
return false;
}
const isAuthorized = await this.eventStreamService.isAuthorized({
streamData,
authContext: {
userWorkspaceId,
apiKeyId: apiKey?.id,
},
});
if (!isAuthorized) {
throw new EventStreamException(
'You are not authorized to remove a query from this event stream',
EventStreamExceptionCode.NOT_AUTHORIZED,
);
}
await this.eventStreamService.removeQuery({
workspaceId: workspace.id,
eventStreamChannelId,
queryId: input.queryId,
});
return true;
}
}