fix: SSE event stream reconnection after idle connection death (#21061)

The SSE event stream could silently die from network partitions, NAT
table flushes, browser tab throttling, or server restarts. When this
happened:
1. The `error` callback only called `captureException` — no reconnection
was triggered
2. The `complete` callback was `() => {}` — a cleanly terminated stream
left the client permanently broken
3. No mechanism existed to detect a silently dead connection where no
FIN/RST was received

## Summary

- **Fix `error`/`complete` callbacks**: The `graphql-sse` subscription's
`error` callback only reported to Sentry, and `complete` was a no-op.
Both now set `shouldDestroyEventStreamState = true` to trigger the
destroy-recreate lifecycle, ensuring detected transport failures and
clean stream terminations lead to automatic reconnection.
- **Add server-side keepalive**: The existing heartbeat timer now runs
every 30s (instead of 6min) and publishes empty events through the Redis
pub/sub channel in addition to refreshing the Redis TTL (throttled to
~6min). Unlike GraphQL Yoga's opaque SSE comment pings, these are real
subscription events that flow through the client's `next`/`message`
handlers.
- **Add client-side keepalive monitor (`SSEKeepAliveEffect`)**: Tracks
the timestamp of the last received event. If no event arrives within 90
seconds (3x the keepalive interval), it clears query listeners and
triggers a stream destroy-recreate cycle.

## Test plan

- [x] Start the app, verify SSE events flow normally (workflow runs
update in real-time)
- [x] Leave the app idle for >90 seconds, then trigger a workflow run —
verify the stream auto-reconnects and events are delivered
- [x] Kill the server, restart it, verify the frontend recovers its
event stream
- [x] Verify keepalive events (empty
`objectRecordEventsWithQueryIds`/`metadataEvents`) appear in browser
network tab every ~30s
- [x] Verify no regressions in SSE-dependent features (record updates,
metadata changes, workflow run visualization)
This commit is contained in:
Thomas Trompette
2026-06-01 10:09:22 +02:00
committed by GitHub
parent e430e4ea0a
commit f4380f89a8
8 changed files with 89 additions and 5 deletions
@@ -0,0 +1,43 @@
import { SSE_LIVENESS_CHECK_INTERVAL_IN_MS } from '@/sse-db-event/constants/SseLivenessCheckIntervalInMs';
import { SSE_LIVENESS_TIMEOUT_IN_MS } from '@/sse-db-event/constants/SseLivenessTimeoutInMs';
import { activeQueryListenersState } from '@/sse-db-event/states/activeQueryListenersState';
import { lastSseEventReceivedTimestampState } from '@/sse-db-event/states/lastSseEventReceivedTimestampState';
import { shouldDestroyEventStreamState } from '@/sse-db-event/states/shouldDestroyEventStreamState';
import { sseEventStreamIdState } from '@/sse-db-event/states/sseEventStreamIdState';
import { sseEventStreamReadyState } from '@/sse-db-event/states/sseEventStreamReadyState';
import { useAtomStateValue } from '@/ui/utilities/state/jotai/hooks/useAtomStateValue';
import { isNonEmptyString } from '@sniptt/guards';
import { useStore } from 'jotai';
import { useEffect } from 'react';
import { isDefined } from 'twenty-shared/utils';
export const SSEKeepAliveEffect = () => {
const store = useStore();
const sseEventStreamReady = useAtomStateValue(sseEventStreamReadyState);
const sseEventStreamId = useAtomStateValue(sseEventStreamIdState);
useEffect(() => {
if (!sseEventStreamReady || !isNonEmptyString(sseEventStreamId)) {
return;
}
const interval = setInterval(() => {
const lastTimestamp = store.get(lastSseEventReceivedTimestampState.atom);
if (!isDefined(lastTimestamp)) {
return;
}
const timeSinceLastEvent = Date.now() - lastTimestamp;
if (timeSinceLastEvent > SSE_LIVENESS_TIMEOUT_IN_MS) {
store.set(activeQueryListenersState.atom, []);
store.set(shouldDestroyEventStreamState.atom, true);
}
}, SSE_LIVENESS_CHECK_INTERVAL_IN_MS);
return () => clearInterval(interval);
}, [sseEventStreamReady, sseEventStreamId, store]);
return null;
};
@@ -1,6 +1,7 @@
import { MetadataStoreSSEEffect } from '@/metadata-store/effect-components/MetadataStoreSSEEffect';
import { SSEClientEffect } from '@/sse-db-event/components/SSEClientEffect';
import { SSEEventStreamEffect } from '@/sse-db-event/components/SSEEventStreamEffect';
import { SSEKeepAliveEffect } from '@/sse-db-event/components/SSEKeepAliveEffect';
import { SSEQuerySubscribeEffect } from '@/sse-db-event/components/SSEQuerySubscribeEffect';
import { type ReactNode } from 'react';
@@ -14,6 +15,7 @@ export const SSEProvider = ({ children }: SSEProviderProps) => {
<SSEClientEffect />
<SSEEventStreamEffect />
<SSEQuerySubscribeEffect />
<SSEKeepAliveEffect />
<MetadataStoreSSEEffect />
{children}
</>
@@ -0,0 +1 @@
export const SSE_LIVENESS_CHECK_INTERVAL_IN_MS = 30_000;
@@ -0,0 +1 @@
export const SSE_LIVENESS_TIMEOUT_IN_MS = 90_000;
@@ -5,6 +5,7 @@ import { useTriggerOptimisticEffectFromSseEvents } from '@/sse-db-event/hooks/us
import { disposeFunctionForEventStreamState } from '@/sse-db-event/states/disposeFunctionByEventStreamMapState';
import { isCreatingSseEventStreamState } from '@/sse-db-event/states/isCreatingSseEventStreamState';
import { isDestroyingEventStreamState } from '@/sse-db-event/states/isDestroyingEventStreamState';
import { lastSseEventReceivedTimestampState } from '@/sse-db-event/states/lastSseEventReceivedTimestampState';
import { shouldDestroyEventStreamState } from '@/sse-db-event/states/shouldDestroyEventStreamState';
import { sseClientState } from '@/sse-db-event/states/sseClientState';
import { sseEventStreamIdState } from '@/sse-db-event/states/sseEventStreamIdState';
@@ -81,6 +82,8 @@ export const useTriggerEventStreamCreation = () => {
onEventSubscription: EventSubscription;
}>,
) => {
store.set(lastSseEventReceivedTimestampState.atom, Date.now());
if (isDefined(value?.errors) && Array.isArray(value.errors)) {
const extensions = getGraphqlErrorExtensionsFromError(
value.errors[0],
@@ -132,8 +135,11 @@ export const useTriggerEventStreamCreation = () => {
},
error: (error) => {
captureException(error);
store.set(shouldDestroyEventStreamState.atom, true);
},
complete: () => {
store.set(shouldDestroyEventStreamState.atom, true);
},
complete: () => {},
},
{
message: ({ data, event }) => {
@@ -143,6 +149,8 @@ export const useTriggerEventStreamCreation = () => {
try {
if (event === 'next') {
store.set(lastSseEventReceivedTimestampState.atom, Date.now());
if (isDefined(result?.errors)) {
const extensions = getGraphqlErrorExtensionsFromError(
result.errors[0],
@@ -0,0 +1,8 @@
import { createAtomState } from '@/ui/utilities/state/jotai/utils/createAtomState';
export const lastSseEventReceivedTimestampState = createAtomState<
number | null
>({
key: 'lastSseEventReceivedTimestampState',
defaultValue: null,
});
@@ -0,0 +1 @@
export const APPLICATION_KEEPALIVE_INTERVAL_MS = 30 * 1_000; // 30 seconds
@@ -16,6 +16,7 @@ import { AuthWorkspace } from 'src/engine/decorators/auth/auth-workspace.decorat
import { NoPermissionGuard } from 'src/engine/guards/no-permission.guard';
import { UserAuthGuard } from 'src/engine/guards/user-auth.guard';
import { WorkspaceAuthGuard } from 'src/engine/guards/workspace-auth.guard';
import { APPLICATION_KEEPALIVE_INTERVAL_MS } from 'src/engine/subscriptions/constants/application-keepalive-interval-ms.constant';
import { EVENT_STREAM_TTL_MS } from 'src/engine/subscriptions/constants/event-stream-ttl.constant';
import { AddQuerySubscriptionInput } from 'src/engine/subscriptions/dtos/add-query-subscription.input';
import { EventSubscriptionDTO } from 'src/engine/subscriptions/dtos/event-subscription.dto';
@@ -116,17 +117,36 @@ export class EventStreamResolver {
throw error;
}
let lastTtlRefreshAt = 0;
return wrapAsyncIteratorWithLifecycle(iterator, {
initialValue: {
objectRecordEventsWithQueryIds: [],
metadataEvents: [],
},
onHeartbeat: () =>
this.eventStreamService.refreshEventStreamTTL({
onHeartbeat: async () => {
const now = Date.now();
if (now - lastTtlRefreshAt > EVENT_STREAM_TTL_MS / 5) {
lastTtlRefreshAt = now;
await this.eventStreamService.refreshEventStreamTTL({
workspaceId: workspace.id,
eventStreamChannelId,
});
}
await this.subscriptionService.publishToEventStream({
workspaceId: workspace.id,
eventStreamChannelId,
}),
heartbeatIntervalMs: EVENT_STREAM_TTL_MS / 5,
payload: {
objectRecordEventsWithQueryIds: [],
metadataEvents: [],
},
});
return true;
},
heartbeatIntervalMs: APPLICATION_KEEPALIVE_INTERVAL_MS,
onCleanup: () =>
this.eventStreamService.destroyEventStream({
workspaceId: workspace.id,