fix: SSE event stream reconnection after idle connection death (#21061)
The SSE event stream could silently die from network partitions, NAT
table flushes, browser tab throttling, or server restarts. When this
happened:
1. The `error` callback only called `captureException` — no reconnection
was triggered
2. The `complete` callback was `() => {}` — a cleanly terminated stream
left the client permanently broken
3. No mechanism existed to detect a silently dead connection where no
FIN/RST was received
## Summary
- **Fix `error`/`complete` callbacks**: The `graphql-sse` subscription's
`error` callback only reported to Sentry, and `complete` was a no-op.
Both now set `shouldDestroyEventStreamState = true` to trigger the
destroy-recreate lifecycle, ensuring detected transport failures and
clean stream terminations lead to automatic reconnection.
- **Add server-side keepalive**: The existing heartbeat timer now runs
every 30s (instead of 6min) and publishes empty events through the Redis
pub/sub channel in addition to refreshing the Redis TTL (throttled to
~6min). Unlike GraphQL Yoga's opaque SSE comment pings, these are real
subscription events that flow through the client's `next`/`message`
handlers.
- **Add client-side keepalive monitor (`SSEKeepAliveEffect`)**: Tracks
the timestamp of the last received event. If no event arrives within 90
seconds (3x the keepalive interval), it clears query listeners and
triggers a stream destroy-recreate cycle.
## Test plan
- [x] Start the app, verify SSE events flow normally (workflow runs
update in real-time)
- [x] Leave the app idle for >90 seconds, then trigger a workflow run —
verify the stream auto-reconnects and events are delivered
- [x] Kill the server, restart it, verify the frontend recovers its
event stream
- [x] Verify keepalive events (empty
`objectRecordEventsWithQueryIds`/`metadataEvents`) appear in browser
network tab every ~30s
- [x] Verify no regressions in SSE-dependent features (record updates,
metadata changes, workflow run visualization)
This commit is contained in:
@@ -0,0 +1,43 @@
|
||||
import { SSE_LIVENESS_CHECK_INTERVAL_IN_MS } from '@/sse-db-event/constants/SseLivenessCheckIntervalInMs';
|
||||
import { SSE_LIVENESS_TIMEOUT_IN_MS } from '@/sse-db-event/constants/SseLivenessTimeoutInMs';
|
||||
import { activeQueryListenersState } from '@/sse-db-event/states/activeQueryListenersState';
|
||||
import { lastSseEventReceivedTimestampState } from '@/sse-db-event/states/lastSseEventReceivedTimestampState';
|
||||
import { shouldDestroyEventStreamState } from '@/sse-db-event/states/shouldDestroyEventStreamState';
|
||||
import { sseEventStreamIdState } from '@/sse-db-event/states/sseEventStreamIdState';
|
||||
import { sseEventStreamReadyState } from '@/sse-db-event/states/sseEventStreamReadyState';
|
||||
import { useAtomStateValue } from '@/ui/utilities/state/jotai/hooks/useAtomStateValue';
|
||||
import { isNonEmptyString } from '@sniptt/guards';
|
||||
import { useStore } from 'jotai';
|
||||
import { useEffect } from 'react';
|
||||
import { isDefined } from 'twenty-shared/utils';
|
||||
|
||||
export const SSEKeepAliveEffect = () => {
|
||||
const store = useStore();
|
||||
const sseEventStreamReady = useAtomStateValue(sseEventStreamReadyState);
|
||||
const sseEventStreamId = useAtomStateValue(sseEventStreamIdState);
|
||||
|
||||
useEffect(() => {
|
||||
if (!sseEventStreamReady || !isNonEmptyString(sseEventStreamId)) {
|
||||
return;
|
||||
}
|
||||
|
||||
const interval = setInterval(() => {
|
||||
const lastTimestamp = store.get(lastSseEventReceivedTimestampState.atom);
|
||||
|
||||
if (!isDefined(lastTimestamp)) {
|
||||
return;
|
||||
}
|
||||
|
||||
const timeSinceLastEvent = Date.now() - lastTimestamp;
|
||||
|
||||
if (timeSinceLastEvent > SSE_LIVENESS_TIMEOUT_IN_MS) {
|
||||
store.set(activeQueryListenersState.atom, []);
|
||||
store.set(shouldDestroyEventStreamState.atom, true);
|
||||
}
|
||||
}, SSE_LIVENESS_CHECK_INTERVAL_IN_MS);
|
||||
|
||||
return () => clearInterval(interval);
|
||||
}, [sseEventStreamReady, sseEventStreamId, store]);
|
||||
|
||||
return null;
|
||||
};
|
||||
@@ -1,6 +1,7 @@
|
||||
import { MetadataStoreSSEEffect } from '@/metadata-store/effect-components/MetadataStoreSSEEffect';
|
||||
import { SSEClientEffect } from '@/sse-db-event/components/SSEClientEffect';
|
||||
import { SSEEventStreamEffect } from '@/sse-db-event/components/SSEEventStreamEffect';
|
||||
import { SSEKeepAliveEffect } from '@/sse-db-event/components/SSEKeepAliveEffect';
|
||||
import { SSEQuerySubscribeEffect } from '@/sse-db-event/components/SSEQuerySubscribeEffect';
|
||||
import { type ReactNode } from 'react';
|
||||
|
||||
@@ -14,6 +15,7 @@ export const SSEProvider = ({ children }: SSEProviderProps) => {
|
||||
<SSEClientEffect />
|
||||
<SSEEventStreamEffect />
|
||||
<SSEQuerySubscribeEffect />
|
||||
<SSEKeepAliveEffect />
|
||||
<MetadataStoreSSEEffect />
|
||||
{children}
|
||||
</>
|
||||
|
||||
+1
@@ -0,0 +1 @@
|
||||
export const SSE_LIVENESS_CHECK_INTERVAL_IN_MS = 30_000;
|
||||
@@ -0,0 +1 @@
|
||||
export const SSE_LIVENESS_TIMEOUT_IN_MS = 90_000;
|
||||
+9
-1
@@ -5,6 +5,7 @@ import { useTriggerOptimisticEffectFromSseEvents } from '@/sse-db-event/hooks/us
|
||||
import { disposeFunctionForEventStreamState } from '@/sse-db-event/states/disposeFunctionByEventStreamMapState';
|
||||
import { isCreatingSseEventStreamState } from '@/sse-db-event/states/isCreatingSseEventStreamState';
|
||||
import { isDestroyingEventStreamState } from '@/sse-db-event/states/isDestroyingEventStreamState';
|
||||
import { lastSseEventReceivedTimestampState } from '@/sse-db-event/states/lastSseEventReceivedTimestampState';
|
||||
import { shouldDestroyEventStreamState } from '@/sse-db-event/states/shouldDestroyEventStreamState';
|
||||
import { sseClientState } from '@/sse-db-event/states/sseClientState';
|
||||
import { sseEventStreamIdState } from '@/sse-db-event/states/sseEventStreamIdState';
|
||||
@@ -81,6 +82,8 @@ export const useTriggerEventStreamCreation = () => {
|
||||
onEventSubscription: EventSubscription;
|
||||
}>,
|
||||
) => {
|
||||
store.set(lastSseEventReceivedTimestampState.atom, Date.now());
|
||||
|
||||
if (isDefined(value?.errors) && Array.isArray(value.errors)) {
|
||||
const extensions = getGraphqlErrorExtensionsFromError(
|
||||
value.errors[0],
|
||||
@@ -132,8 +135,11 @@ export const useTriggerEventStreamCreation = () => {
|
||||
},
|
||||
error: (error) => {
|
||||
captureException(error);
|
||||
store.set(shouldDestroyEventStreamState.atom, true);
|
||||
},
|
||||
complete: () => {
|
||||
store.set(shouldDestroyEventStreamState.atom, true);
|
||||
},
|
||||
complete: () => {},
|
||||
},
|
||||
{
|
||||
message: ({ data, event }) => {
|
||||
@@ -143,6 +149,8 @@ export const useTriggerEventStreamCreation = () => {
|
||||
|
||||
try {
|
||||
if (event === 'next') {
|
||||
store.set(lastSseEventReceivedTimestampState.atom, Date.now());
|
||||
|
||||
if (isDefined(result?.errors)) {
|
||||
const extensions = getGraphqlErrorExtensionsFromError(
|
||||
result.errors[0],
|
||||
|
||||
+8
@@ -0,0 +1,8 @@
|
||||
import { createAtomState } from '@/ui/utilities/state/jotai/utils/createAtomState';
|
||||
|
||||
export const lastSseEventReceivedTimestampState = createAtomState<
|
||||
number | null
|
||||
>({
|
||||
key: 'lastSseEventReceivedTimestampState',
|
||||
defaultValue: null,
|
||||
});
|
||||
+1
@@ -0,0 +1 @@
|
||||
export const APPLICATION_KEEPALIVE_INTERVAL_MS = 30 * 1_000; // 30 seconds
|
||||
@@ -16,6 +16,7 @@ import { AuthWorkspace } from 'src/engine/decorators/auth/auth-workspace.decorat
|
||||
import { NoPermissionGuard } from 'src/engine/guards/no-permission.guard';
|
||||
import { UserAuthGuard } from 'src/engine/guards/user-auth.guard';
|
||||
import { WorkspaceAuthGuard } from 'src/engine/guards/workspace-auth.guard';
|
||||
import { APPLICATION_KEEPALIVE_INTERVAL_MS } from 'src/engine/subscriptions/constants/application-keepalive-interval-ms.constant';
|
||||
import { EVENT_STREAM_TTL_MS } from 'src/engine/subscriptions/constants/event-stream-ttl.constant';
|
||||
import { AddQuerySubscriptionInput } from 'src/engine/subscriptions/dtos/add-query-subscription.input';
|
||||
import { EventSubscriptionDTO } from 'src/engine/subscriptions/dtos/event-subscription.dto';
|
||||
@@ -116,17 +117,36 @@ export class EventStreamResolver {
|
||||
throw error;
|
||||
}
|
||||
|
||||
let lastTtlRefreshAt = 0;
|
||||
|
||||
return wrapAsyncIteratorWithLifecycle(iterator, {
|
||||
initialValue: {
|
||||
objectRecordEventsWithQueryIds: [],
|
||||
metadataEvents: [],
|
||||
},
|
||||
onHeartbeat: () =>
|
||||
this.eventStreamService.refreshEventStreamTTL({
|
||||
onHeartbeat: async () => {
|
||||
const now = Date.now();
|
||||
|
||||
if (now - lastTtlRefreshAt > EVENT_STREAM_TTL_MS / 5) {
|
||||
lastTtlRefreshAt = now;
|
||||
await this.eventStreamService.refreshEventStreamTTL({
|
||||
workspaceId: workspace.id,
|
||||
eventStreamChannelId,
|
||||
});
|
||||
}
|
||||
|
||||
await this.subscriptionService.publishToEventStream({
|
||||
workspaceId: workspace.id,
|
||||
eventStreamChannelId,
|
||||
}),
|
||||
heartbeatIntervalMs: EVENT_STREAM_TTL_MS / 5,
|
||||
payload: {
|
||||
objectRecordEventsWithQueryIds: [],
|
||||
metadataEvents: [],
|
||||
},
|
||||
});
|
||||
|
||||
return true;
|
||||
},
|
||||
heartbeatIntervalMs: APPLICATION_KEEPALIVE_INTERVAL_MS,
|
||||
onCleanup: () =>
|
||||
this.eventStreamService.destroyEventStream({
|
||||
workspaceId: workspace.id,
|
||||
|
||||
Reference in New Issue
Block a user