From f263edd1222d7d4bc83f7f1d1c79bb4ab3c81058 Mon Sep 17 00:00:00 2001 From: nitin <142569587+ehconitin@users.noreply.github.com> Date: Mon, 29 Jun 2026 20:44:06 +0530 Subject: [PATCH] Add call recorder oom diagnosis logs (#22315) Review in cubic --- .../flows/handle-recall-webhook.util.ts | 115 +++++++++++++++++- .../flows/ingest-call-recording-media.util.ts | 52 ++++++-- 2 files changed, 159 insertions(+), 8 deletions(-) diff --git a/packages/twenty-apps/public/call-recorder/src/logic-functions/flows/handle-recall-webhook.util.ts b/packages/twenty-apps/public/call-recorder/src/logic-functions/flows/handle-recall-webhook.util.ts index f1a41a9125..f92120473c 100644 --- a/packages/twenty-apps/public/call-recorder/src/logic-functions/flows/handle-recall-webhook.util.ts +++ b/packages/twenty-apps/public/call-recorder/src/logic-functions/flows/handle-recall-webhook.util.ts @@ -106,6 +106,19 @@ const handleRecallStatusEvent = async ({ }; } + const shouldLogTerminalDiagnostics = isRecallRecordingDoneSignal({ + event, + statusCode, + }); + + if (shouldLogTerminalDiagnostics) { + logRecallWebhookPhase({ + phase: 'match-start', + webhookEvent, + callRecordingStatus, + }); + } + const callRecording = await findMatchingCallRecording({ client, webhookEvent, @@ -144,12 +157,28 @@ const handleRecallStatusEvent = async ({ ...buildRecordingTimestampsUpdate({ webhookEvent, callRecording }), }; - if (isRecallRecordingDoneSignal({ event, statusCode })) { + if (shouldLogTerminalDiagnostics) { + logRecallWebhookPhase({ + phase: 'terminal-start', + webhookEvent, + callRecording, + callRecordingStatus, + }); + const externalRecordingIdResolution = await resolveExternalRecordingId({ callRecording, webhookEvent, }); + logRecallWebhookPhase({ + phase: 'recording-id-resolved', + webhookEvent, + callRecording, + externalRecordingId: externalRecordingIdResolution.externalRecordingId, + providerLookupFailed: externalRecordingIdResolution.providerLookupFailed, + callRecordingStatus, + }); + Object.assign( updateData, await buildTranscriptArtifactUpdate({ @@ -158,6 +187,15 @@ const handleRecallStatusEvent = async ({ }), ); + logRecallWebhookPhase({ + phase: 'transcript-complete', + webhookEvent, + callRecording, + externalRecordingId: externalRecordingIdResolution.externalRecordingId, + updateData, + callRecordingStatus, + }); + Object.assign( updateData, await buildMediaIngestionUpdate({ @@ -186,6 +224,18 @@ const handleRecallStatusEvent = async ({ updateData, }); + if (shouldLogTerminalDiagnostics) { + logRecallWebhookPhase({ + phase: 'terminal-complete', + webhookEvent, + callRecording, + updateData, + callRecordingStatus: completesIngestion + ? CallRecordingStatus.COMPLETED + : (updateData.status ?? callRecordingStatus), + }); + } + return { status: 'updated', event, @@ -196,6 +246,69 @@ const handleRecallStatusEvent = async ({ }; }; +const logRecallWebhookPhase = ({ + phase, + webhookEvent, + callRecording, + callRecordingStatus, + externalRecordingId, + providerLookupFailed, + updateData, +}: { + phase: string; + webhookEvent: RecallWebhookEvent; + callRecording?: MatchedCallRecording; + callRecordingStatus?: string; + externalRecordingId?: string; + providerLookupFailed?: boolean; + updateData?: CallRecordingUpdateFields; +}) => { + console.log( + [ + `[call-recorder] recall-webhook phase=${phase}`, + `event=${webhookEvent.event}`, + `statusCode=${webhookEvent.statusCode ?? 'n/a'}`, + `callRecordingId=${callRecording?.id ?? webhookEvent.callRecordingIdFromMetadata ?? 'n/a'}`, + `externalBotId=${webhookEvent.externalBotId ?? 'n/a'}`, + `externalRecordingId=${externalRecordingId ?? webhookEvent.externalRecordingId ?? callRecording?.externalRecordingId ?? 'n/a'}`, + `callRecordingStatus=${callRecordingStatus ?? 'n/a'}`, + `currentStatus=${callRecording?.status ?? 'n/a'}`, + `hasTranscript=${hasReachableTranscript(callRecording?.transcript)}`, + `hasAudio=${isNonEmptyArray(callRecording?.audio)}`, + `hasVideo=${isNonEmptyArray(callRecording?.video)}`, + `updates=${formatUpdateDataKeys(updateData)}`, + `providerLookupFailed=${providerLookupFailed ?? false}`, + formatMemoryUsageForLog(), + ].join(' '), + ); +}; + +const formatUpdateDataKeys = ( + updateData: CallRecordingUpdateFields | undefined, +): string => { + if (isUndefined(updateData)) { + return 'none'; + } + + const updateDataKeys = Object.keys(updateData); + + return updateDataKeys.length === 0 ? 'none' : updateDataKeys.join(','); +}; + +const formatMemoryUsageForLog = (): string => { + const memoryUsage = process.memoryUsage(); + + return [ + `rssMegaBytes=${formatBytesAsMegaBytes(memoryUsage.rss)}`, + `heapUsedMegaBytes=${formatBytesAsMegaBytes(memoryUsage.heapUsed)}`, + `externalMegaBytes=${formatBytesAsMegaBytes(memoryUsage.external)}`, + `arrayBuffersMegaBytes=${formatBytesAsMegaBytes(memoryUsage.arrayBuffers)}`, + ].join(' '); +}; + +const formatBytesAsMegaBytes = (bytes: number): string => + (bytes / 1024 / 1024).toFixed(1); + const findMatchingCallRecording = async ({ client, webhookEvent, diff --git a/packages/twenty-apps/public/call-recorder/src/logic-functions/flows/ingest-call-recording-media.util.ts b/packages/twenty-apps/public/call-recorder/src/logic-functions/flows/ingest-call-recording-media.util.ts index 517cb00e29..7b016fbd72 100644 --- a/packages/twenty-apps/public/call-recorder/src/logic-functions/flows/ingest-call-recording-media.util.ts +++ b/packages/twenty-apps/public/call-recorder/src/logic-functions/flows/ingest-call-recording-media.util.ts @@ -91,7 +91,16 @@ const ingestMediaArtifact = async ({ fieldMetadataUniversalIdentifier: string; }): Promise => { try { - const { buffer, contentType } = await downloadMediaFile(url); + const { buffer, contentType } = await downloadMediaFile({ + callRecordingId, + fileName, + url, + }); + + console.log( + `[call-recorder] media-ingestion phase=artifact-upload-start callRecordingId=${callRecordingId} fileName=${fileName} downloadedBytes=${buffer.byteLength} contentType=${contentType} ${formatMemoryUsageForLog()}`, + ); + const uploadedFile = await metadataClient.uploadFile( buffer, fileName, @@ -109,20 +118,49 @@ const ingestMediaArtifact = async ({ } }; -const downloadMediaFile = async ( - url: string, -): Promise<{ buffer: Buffer; contentType: string }> => { +const downloadMediaFile = async ({ + callRecordingId, + fileName, + url, +}: { + callRecordingId: string; + fileName: string; + url: string; +}): Promise<{ buffer: Buffer; contentType: string }> => { const response = await fetch(url, { signal: AbortSignal.timeout(MEDIA_DOWNLOAD_TIMEOUT_MS), }); + const contentType = + response.headers.get('content-type') ?? 'application/octet-stream'; + const contentLength = response.headers.get('content-length') ?? 'unknown'; + + console.log( + `[call-recorder] media-ingestion phase=artifact-download-response callRecordingId=${callRecordingId} fileName=${fileName} responseStatus=${response.status} contentLengthBytes=${contentLength} contentType=${contentType} ${formatMemoryUsageForLog()}`, + ); if (!response.ok) { throw new Error(`download failed with status ${response.status}`); } + const arrayBuffer = await response.arrayBuffer(); + const buffer = Buffer.from(arrayBuffer); + return { - buffer: Buffer.from(await response.arrayBuffer()), - contentType: - response.headers.get('content-type') ?? 'application/octet-stream', + buffer, + contentType, }; }; + +const formatMemoryUsageForLog = (): string => { + const memoryUsage = process.memoryUsage(); + + return [ + `rssMegaBytes=${formatBytesAsMegaBytes(memoryUsage.rss)}`, + `heapUsedMegaBytes=${formatBytesAsMegaBytes(memoryUsage.heapUsed)}`, + `externalMegaBytes=${formatBytesAsMegaBytes(memoryUsage.external)}`, + `arrayBuffersMegaBytes=${formatBytesAsMegaBytes(memoryUsage.arrayBuffers)}`, + ].join(' '); +}; + +const formatBytesAsMegaBytes = (bytes: number): string => + (bytes / 1024 / 1024).toFixed(1);