Files
twenty/packages/twenty-server/src/engine/subscriptions/event-stream.resolver.ts
T
Thomas Trompette f4380f89a8 fix: SSE event stream reconnection after idle connection death (#21061)
The SSE event stream could silently die from network partitions, NAT
table flushes, browser tab throttling, or server restarts. When this
happened:
1. The `error` callback only called `captureException` — no reconnection
was triggered
2. The `complete` callback was `() => {}` — a cleanly terminated stream
left the client permanently broken
3. No mechanism existed to detect a silently dead connection where no
FIN/RST was received

## Summary

- **Fix `error`/`complete` callbacks**: The `graphql-sse` subscription's
`error` callback only reported to Sentry, and `complete` was a no-op.
Both now set `shouldDestroyEventStreamState = true` to trigger the
destroy-recreate lifecycle, ensuring detected transport failures and
clean stream terminations lead to automatic reconnection.
- **Add server-side keepalive**: The existing heartbeat timer now runs
every 30s (instead of 6min) and publishes empty events through the Redis
pub/sub channel in addition to refreshing the Redis TTL (throttled to
~6min). Unlike GraphQL Yoga's opaque SSE comment pings, these are real
subscription events that flow through the client's `next`/`message`
handlers.
- **Add client-side keepalive monitor (`SSEKeepAliveEffect`)**: Tracks
the timestamp of the last received event. If no event arrives within 90
seconds (3x the keepalive interval), it clears query listeners and
triggers a stream destroy-recreate cycle.

## Test plan

- [x] Start the app, verify SSE events flow normally (workflow runs
update in real-time)
- [x] Leave the app idle for >90 seconds, then trigger a workflow run —
verify the stream auto-reconnects and events are delivered
- [x] Kill the server, restart it, verify the frontend recovers its
event stream
- [x] Verify keepalive events (empty
`objectRecordEventsWithQueryIds`/`metadataEvents`) appear in browser
network tab every ~30s
- [x] Verify no regressions in SSE-dependent features (record updates,
metadata changes, workflow run visualization)
2026-06-01 08:09:22 +00:00

246 lines
8.5 KiB
TypeScript

import { UseFilters, UseGuards, UsePipes } from '@nestjs/common';
import { Args, Mutation, Subscription } from '@nestjs/graphql';
import { isDefined } from 'twenty-shared/utils';
import { MetadataResolver } from 'src/engine/api/graphql/graphql-config/decorators/metadata-resolver.decorator';
import { type ApiKeyEntity } from 'src/engine/core-modules/api-key/api-key.entity';
import { PreventNestToAutoLogGraphqlErrorsFilter } from 'src/engine/core-modules/graphql/filters/prevent-nest-to-auto-log-graphql-errors.filter';
import { ResolverValidationPipe } from 'src/engine/core-modules/graphql/pipes/resolver-validation.pipe';
import { type AuthContextUser } from 'src/engine/core-modules/auth/types/auth-context.type';
import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity';
import { AuthApiKey } from 'src/engine/decorators/auth/auth-api-key.decorator';
import { AuthUserWorkspaceId } from 'src/engine/decorators/auth/auth-user-workspace-id.decorator';
import { AuthUser } from 'src/engine/decorators/auth/auth-user.decorator';
import { AuthWorkspace } from 'src/engine/decorators/auth/auth-workspace.decorator';
import { NoPermissionGuard } from 'src/engine/guards/no-permission.guard';
import { UserAuthGuard } from 'src/engine/guards/user-auth.guard';
import { WorkspaceAuthGuard } from 'src/engine/guards/workspace-auth.guard';
import { APPLICATION_KEEPALIVE_INTERVAL_MS } from 'src/engine/subscriptions/constants/application-keepalive-interval-ms.constant';
import { EVENT_STREAM_TTL_MS } from 'src/engine/subscriptions/constants/event-stream-ttl.constant';
import { AddQuerySubscriptionInput } from 'src/engine/subscriptions/dtos/add-query-subscription.input';
import { EventSubscriptionDTO } from 'src/engine/subscriptions/dtos/event-subscription.dto';
import { RemoveQueryFromEventStreamInput } from 'src/engine/subscriptions/dtos/remove-query-subscription.input';
import { EventStreamExceptionFilter } from 'src/engine/subscriptions/event-stream-exception.filter';
import {
EventStreamException,
EventStreamExceptionCode,
} from 'src/engine/subscriptions/event-stream.exception';
import { EventStreamService } from 'src/engine/subscriptions/event-stream.service';
import { SubscriptionService } from 'src/engine/subscriptions/subscription.service';
import { type EventStreamPayload } from 'src/engine/subscriptions/types/event-stream-payload.type';
import { eventStreamIdToChannelId } from 'src/engine/subscriptions/utils/get-channel-id-from-event-stream-id';
import { wrapAsyncIteratorWithLifecycle } from 'src/engine/subscriptions/utils/wrap-async-iterator-with-lifecycle';
@MetadataResolver()
@UseGuards(WorkspaceAuthGuard, UserAuthGuard, NoPermissionGuard)
@UsePipes(ResolverValidationPipe)
@UseFilters(EventStreamExceptionFilter, PreventNestToAutoLogGraphqlErrorsFilter)
export class EventStreamResolver {
constructor(
private readonly subscriptionService: SubscriptionService,
private readonly eventStreamService: EventStreamService,
) {}
@Subscription(() => EventSubscriptionDTO, {
nullable: true,
resolve: (
payload: EventStreamPayload,
variables: { eventStreamId: string },
) => {
return {
eventStreamId: variables.eventStreamId,
objectRecordEventsWithQueryIds: payload.objectRecordEventsWithQueryIds,
metadataEvents: payload.metadataEvents,
};
},
})
async onEventSubscription(
@Args('eventStreamId') eventStreamId: string,
@AuthWorkspace() workspace: WorkspaceEntity,
@AuthUser({ allowUndefined: true }) user: AuthContextUser | undefined,
@AuthUserWorkspaceId({ allowUndefined: true })
userWorkspaceId: string | undefined,
@AuthApiKey() apiKey: ApiKeyEntity | undefined,
) {
const eventStreamChannelId = eventStreamIdToChannelId(eventStreamId);
const existingStreamData = await this.eventStreamService.getStreamData(
workspace.id,
eventStreamChannelId,
);
if (isDefined(existingStreamData)) {
const isAuthorized = await this.eventStreamService.isAuthorized({
streamData: existingStreamData,
authContext: {
userWorkspaceId,
apiKeyId: apiKey?.id,
},
});
if (!isAuthorized) {
throw new EventStreamException(
'Event stream already exists',
EventStreamExceptionCode.EVENT_STREAM_ALREADY_EXISTS,
);
}
await this.eventStreamService.destroyEventStream({
workspaceId: workspace.id,
eventStreamChannelId,
});
}
await this.eventStreamService.createEventStream({
workspaceId: workspace.id,
eventStreamChannelId,
authContext: {
userId: user?.id,
userWorkspaceId,
apiKeyId: apiKey?.id,
},
});
let iterator: AsyncIterableIterator<EventStreamPayload>;
try {
iterator = await this.subscriptionService.subscribeToEventStream({
workspaceId: workspace.id,
eventStreamChannelId,
});
} catch (error) {
await this.eventStreamService.destroyEventStream({
workspaceId: workspace.id,
eventStreamChannelId,
});
throw error;
}
let lastTtlRefreshAt = 0;
return wrapAsyncIteratorWithLifecycle(iterator, {
initialValue: {
objectRecordEventsWithQueryIds: [],
metadataEvents: [],
},
onHeartbeat: async () => {
const now = Date.now();
if (now - lastTtlRefreshAt > EVENT_STREAM_TTL_MS / 5) {
lastTtlRefreshAt = now;
await this.eventStreamService.refreshEventStreamTTL({
workspaceId: workspace.id,
eventStreamChannelId,
});
}
await this.subscriptionService.publishToEventStream({
workspaceId: workspace.id,
eventStreamChannelId,
payload: {
objectRecordEventsWithQueryIds: [],
metadataEvents: [],
},
});
return true;
},
heartbeatIntervalMs: APPLICATION_KEEPALIVE_INTERVAL_MS,
onCleanup: () =>
this.eventStreamService.destroyEventStream({
workspaceId: workspace.id,
eventStreamChannelId,
}),
});
}
@Mutation(() => Boolean)
async addQueryToEventStream(
@Args('input') input: AddQuerySubscriptionInput,
@AuthWorkspace() workspace: WorkspaceEntity,
@AuthUser({ allowUndefined: true }) user: AuthContextUser | undefined,
@AuthUserWorkspaceId({ allowUndefined: true })
userWorkspaceId: string | undefined,
@AuthApiKey() apiKey: ApiKeyEntity | undefined,
): Promise<boolean> {
const eventStreamChannelId = eventStreamIdToChannelId(input.eventStreamId);
const streamData = await this.eventStreamService.getStreamData(
workspace.id,
eventStreamChannelId,
);
if (!isDefined(streamData)) {
return false;
}
const isAuthorized = await this.eventStreamService.isAuthorized({
streamData,
authContext: {
userWorkspaceId,
apiKeyId: apiKey?.id,
},
});
if (!isAuthorized) {
throw new EventStreamException(
'You are not authorized to add a query to this event stream',
EventStreamExceptionCode.NOT_AUTHORIZED,
);
}
await this.eventStreamService.addQuery({
workspaceId: workspace.id,
eventStreamChannelId,
queryId: input.queryId,
operationSignature: input.operationSignature,
});
return true;
}
@Mutation(() => Boolean)
async removeQueryFromEventStream(
@Args('input') input: RemoveQueryFromEventStreamInput,
@AuthWorkspace() workspace: WorkspaceEntity,
@AuthUser({ allowUndefined: true }) user: AuthContextUser | undefined,
@AuthUserWorkspaceId({ allowUndefined: true })
userWorkspaceId: string | undefined,
@AuthApiKey() apiKey: ApiKeyEntity | undefined,
): Promise<boolean> {
const eventStreamChannelId = eventStreamIdToChannelId(input.eventStreamId);
const streamData = await this.eventStreamService.getStreamData(
workspace.id,
eventStreamChannelId,
);
if (!isDefined(streamData)) {
return false;
}
const isAuthorized = await this.eventStreamService.isAuthorized({
streamData,
authContext: {
userWorkspaceId,
apiKeyId: apiKey?.id,
},
});
if (!isAuthorized) {
throw new EventStreamException(
'You are not authorized to remove a query from this event stream',
EventStreamExceptionCode.NOT_AUTHORIZED,
);
}
await this.eventStreamService.removeQuery({
workspaceId: workspace.id,
eventStreamChannelId,
queryId: input.queryId,
});
return true;
}
}