f4380f89a8
The SSE event stream could silently die from network partitions, NAT
table flushes, browser tab throttling, or server restarts. When this
happened:
1. The `error` callback only called `captureException` — no reconnection
was triggered
2. The `complete` callback was `() => {}` — a cleanly terminated stream
left the client permanently broken
3. No mechanism existed to detect a silently dead connection where no
FIN/RST was received
## Summary
- **Fix `error`/`complete` callbacks**: The `graphql-sse` subscription's
`error` callback only reported to Sentry, and `complete` was a no-op.
Both now set `shouldDestroyEventStreamState = true` to trigger the
destroy-recreate lifecycle, ensuring detected transport failures and
clean stream terminations lead to automatic reconnection.
- **Add server-side keepalive**: The existing heartbeat timer now runs
every 30s (instead of 6min) and publishes empty events through the Redis
pub/sub channel in addition to refreshing the Redis TTL (throttled to
~6min). Unlike GraphQL Yoga's opaque SSE comment pings, these are real
subscription events that flow through the client's `next`/`message`
handlers.
- **Add client-side keepalive monitor (`SSEKeepAliveEffect`)**: Tracks
the timestamp of the last received event. If no event arrives within 90
seconds (3x the keepalive interval), it clears query listeners and
triggers a stream destroy-recreate cycle.
## Test plan
- [x] Start the app, verify SSE events flow normally (workflow runs
update in real-time)
- [x] Leave the app idle for >90 seconds, then trigger a workflow run —
verify the stream auto-reconnects and events are delivered
- [x] Kill the server, restart it, verify the frontend recovers its
event stream
- [x] Verify keepalive events (empty
`objectRecordEventsWithQueryIds`/`metadataEvents`) appear in browser
network tab every ~30s
- [x] Verify no regressions in SSE-dependent features (record updates,
metadata changes, workflow run visualization)
246 lines
8.5 KiB
TypeScript
246 lines
8.5 KiB
TypeScript
import { UseFilters, UseGuards, UsePipes } from '@nestjs/common';
|
|
import { Args, Mutation, Subscription } from '@nestjs/graphql';
|
|
|
|
import { isDefined } from 'twenty-shared/utils';
|
|
|
|
import { MetadataResolver } from 'src/engine/api/graphql/graphql-config/decorators/metadata-resolver.decorator';
|
|
import { type ApiKeyEntity } from 'src/engine/core-modules/api-key/api-key.entity';
|
|
import { PreventNestToAutoLogGraphqlErrorsFilter } from 'src/engine/core-modules/graphql/filters/prevent-nest-to-auto-log-graphql-errors.filter';
|
|
import { ResolverValidationPipe } from 'src/engine/core-modules/graphql/pipes/resolver-validation.pipe';
|
|
import { type AuthContextUser } from 'src/engine/core-modules/auth/types/auth-context.type';
|
|
import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity';
|
|
import { AuthApiKey } from 'src/engine/decorators/auth/auth-api-key.decorator';
|
|
import { AuthUserWorkspaceId } from 'src/engine/decorators/auth/auth-user-workspace-id.decorator';
|
|
import { AuthUser } from 'src/engine/decorators/auth/auth-user.decorator';
|
|
import { AuthWorkspace } from 'src/engine/decorators/auth/auth-workspace.decorator';
|
|
import { NoPermissionGuard } from 'src/engine/guards/no-permission.guard';
|
|
import { UserAuthGuard } from 'src/engine/guards/user-auth.guard';
|
|
import { WorkspaceAuthGuard } from 'src/engine/guards/workspace-auth.guard';
|
|
import { APPLICATION_KEEPALIVE_INTERVAL_MS } from 'src/engine/subscriptions/constants/application-keepalive-interval-ms.constant';
|
|
import { EVENT_STREAM_TTL_MS } from 'src/engine/subscriptions/constants/event-stream-ttl.constant';
|
|
import { AddQuerySubscriptionInput } from 'src/engine/subscriptions/dtos/add-query-subscription.input';
|
|
import { EventSubscriptionDTO } from 'src/engine/subscriptions/dtos/event-subscription.dto';
|
|
import { RemoveQueryFromEventStreamInput } from 'src/engine/subscriptions/dtos/remove-query-subscription.input';
|
|
import { EventStreamExceptionFilter } from 'src/engine/subscriptions/event-stream-exception.filter';
|
|
import {
|
|
EventStreamException,
|
|
EventStreamExceptionCode,
|
|
} from 'src/engine/subscriptions/event-stream.exception';
|
|
import { EventStreamService } from 'src/engine/subscriptions/event-stream.service';
|
|
import { SubscriptionService } from 'src/engine/subscriptions/subscription.service';
|
|
import { type EventStreamPayload } from 'src/engine/subscriptions/types/event-stream-payload.type';
|
|
import { eventStreamIdToChannelId } from 'src/engine/subscriptions/utils/get-channel-id-from-event-stream-id';
|
|
import { wrapAsyncIteratorWithLifecycle } from 'src/engine/subscriptions/utils/wrap-async-iterator-with-lifecycle';
|
|
|
|
@MetadataResolver()
|
|
@UseGuards(WorkspaceAuthGuard, UserAuthGuard, NoPermissionGuard)
|
|
@UsePipes(ResolverValidationPipe)
|
|
@UseFilters(EventStreamExceptionFilter, PreventNestToAutoLogGraphqlErrorsFilter)
|
|
export class EventStreamResolver {
|
|
constructor(
|
|
private readonly subscriptionService: SubscriptionService,
|
|
private readonly eventStreamService: EventStreamService,
|
|
) {}
|
|
|
|
@Subscription(() => EventSubscriptionDTO, {
|
|
nullable: true,
|
|
resolve: (
|
|
payload: EventStreamPayload,
|
|
variables: { eventStreamId: string },
|
|
) => {
|
|
return {
|
|
eventStreamId: variables.eventStreamId,
|
|
objectRecordEventsWithQueryIds: payload.objectRecordEventsWithQueryIds,
|
|
metadataEvents: payload.metadataEvents,
|
|
};
|
|
},
|
|
})
|
|
async onEventSubscription(
|
|
@Args('eventStreamId') eventStreamId: string,
|
|
@AuthWorkspace() workspace: WorkspaceEntity,
|
|
@AuthUser({ allowUndefined: true }) user: AuthContextUser | undefined,
|
|
@AuthUserWorkspaceId({ allowUndefined: true })
|
|
userWorkspaceId: string | undefined,
|
|
@AuthApiKey() apiKey: ApiKeyEntity | undefined,
|
|
) {
|
|
const eventStreamChannelId = eventStreamIdToChannelId(eventStreamId);
|
|
|
|
const existingStreamData = await this.eventStreamService.getStreamData(
|
|
workspace.id,
|
|
eventStreamChannelId,
|
|
);
|
|
|
|
if (isDefined(existingStreamData)) {
|
|
const isAuthorized = await this.eventStreamService.isAuthorized({
|
|
streamData: existingStreamData,
|
|
authContext: {
|
|
userWorkspaceId,
|
|
apiKeyId: apiKey?.id,
|
|
},
|
|
});
|
|
|
|
if (!isAuthorized) {
|
|
throw new EventStreamException(
|
|
'Event stream already exists',
|
|
EventStreamExceptionCode.EVENT_STREAM_ALREADY_EXISTS,
|
|
);
|
|
}
|
|
|
|
await this.eventStreamService.destroyEventStream({
|
|
workspaceId: workspace.id,
|
|
eventStreamChannelId,
|
|
});
|
|
}
|
|
|
|
await this.eventStreamService.createEventStream({
|
|
workspaceId: workspace.id,
|
|
eventStreamChannelId,
|
|
authContext: {
|
|
userId: user?.id,
|
|
userWorkspaceId,
|
|
apiKeyId: apiKey?.id,
|
|
},
|
|
});
|
|
|
|
let iterator: AsyncIterableIterator<EventStreamPayload>;
|
|
|
|
try {
|
|
iterator = await this.subscriptionService.subscribeToEventStream({
|
|
workspaceId: workspace.id,
|
|
eventStreamChannelId,
|
|
});
|
|
} catch (error) {
|
|
await this.eventStreamService.destroyEventStream({
|
|
workspaceId: workspace.id,
|
|
eventStreamChannelId,
|
|
});
|
|
throw error;
|
|
}
|
|
|
|
let lastTtlRefreshAt = 0;
|
|
|
|
return wrapAsyncIteratorWithLifecycle(iterator, {
|
|
initialValue: {
|
|
objectRecordEventsWithQueryIds: [],
|
|
metadataEvents: [],
|
|
},
|
|
onHeartbeat: async () => {
|
|
const now = Date.now();
|
|
|
|
if (now - lastTtlRefreshAt > EVENT_STREAM_TTL_MS / 5) {
|
|
lastTtlRefreshAt = now;
|
|
await this.eventStreamService.refreshEventStreamTTL({
|
|
workspaceId: workspace.id,
|
|
eventStreamChannelId,
|
|
});
|
|
}
|
|
|
|
await this.subscriptionService.publishToEventStream({
|
|
workspaceId: workspace.id,
|
|
eventStreamChannelId,
|
|
payload: {
|
|
objectRecordEventsWithQueryIds: [],
|
|
metadataEvents: [],
|
|
},
|
|
});
|
|
|
|
return true;
|
|
},
|
|
heartbeatIntervalMs: APPLICATION_KEEPALIVE_INTERVAL_MS,
|
|
onCleanup: () =>
|
|
this.eventStreamService.destroyEventStream({
|
|
workspaceId: workspace.id,
|
|
eventStreamChannelId,
|
|
}),
|
|
});
|
|
}
|
|
|
|
@Mutation(() => Boolean)
|
|
async addQueryToEventStream(
|
|
@Args('input') input: AddQuerySubscriptionInput,
|
|
@AuthWorkspace() workspace: WorkspaceEntity,
|
|
@AuthUser({ allowUndefined: true }) user: AuthContextUser | undefined,
|
|
@AuthUserWorkspaceId({ allowUndefined: true })
|
|
userWorkspaceId: string | undefined,
|
|
@AuthApiKey() apiKey: ApiKeyEntity | undefined,
|
|
): Promise<boolean> {
|
|
const eventStreamChannelId = eventStreamIdToChannelId(input.eventStreamId);
|
|
const streamData = await this.eventStreamService.getStreamData(
|
|
workspace.id,
|
|
eventStreamChannelId,
|
|
);
|
|
|
|
if (!isDefined(streamData)) {
|
|
return false;
|
|
}
|
|
|
|
const isAuthorized = await this.eventStreamService.isAuthorized({
|
|
streamData,
|
|
authContext: {
|
|
userWorkspaceId,
|
|
apiKeyId: apiKey?.id,
|
|
},
|
|
});
|
|
|
|
if (!isAuthorized) {
|
|
throw new EventStreamException(
|
|
'You are not authorized to add a query to this event stream',
|
|
EventStreamExceptionCode.NOT_AUTHORIZED,
|
|
);
|
|
}
|
|
|
|
await this.eventStreamService.addQuery({
|
|
workspaceId: workspace.id,
|
|
eventStreamChannelId,
|
|
queryId: input.queryId,
|
|
operationSignature: input.operationSignature,
|
|
});
|
|
|
|
return true;
|
|
}
|
|
|
|
@Mutation(() => Boolean)
|
|
async removeQueryFromEventStream(
|
|
@Args('input') input: RemoveQueryFromEventStreamInput,
|
|
@AuthWorkspace() workspace: WorkspaceEntity,
|
|
@AuthUser({ allowUndefined: true }) user: AuthContextUser | undefined,
|
|
@AuthUserWorkspaceId({ allowUndefined: true })
|
|
userWorkspaceId: string | undefined,
|
|
@AuthApiKey() apiKey: ApiKeyEntity | undefined,
|
|
): Promise<boolean> {
|
|
const eventStreamChannelId = eventStreamIdToChannelId(input.eventStreamId);
|
|
|
|
const streamData = await this.eventStreamService.getStreamData(
|
|
workspace.id,
|
|
eventStreamChannelId,
|
|
);
|
|
|
|
if (!isDefined(streamData)) {
|
|
return false;
|
|
}
|
|
|
|
const isAuthorized = await this.eventStreamService.isAuthorized({
|
|
streamData,
|
|
authContext: {
|
|
userWorkspaceId,
|
|
apiKeyId: apiKey?.id,
|
|
},
|
|
});
|
|
|
|
if (!isAuthorized) {
|
|
throw new EventStreamException(
|
|
'You are not authorized to remove a query from this event stream',
|
|
EventStreamExceptionCode.NOT_AUTHORIZED,
|
|
);
|
|
}
|
|
|
|
await this.eventStreamService.removeQuery({
|
|
workspaceId: workspace.id,
|
|
eventStreamChannelId,
|
|
queryId: input.queryId,
|
|
});
|
|
|
|
return true;
|
|
}
|
|
}
|