feat(call-recorder): cap media file size during ingestion to avoid OOM (#22463)

Media ingestion buffered whole Recall files in memory; long recordings
OOM'd the logic function.

- Caps the size of ingested media files, configurable via the
`CALL_RECORDER_MAX_MEDIA_FILE_SIZE_MB` app variable (default 80 MB).
- Downloads stream chunk by chunk and stop at the cap; a Content-Length
above the cap skips the download without reading the body.
- A skipped file is recorded as `video_file_too_large` /
`audio_file_too_large` in `callRecorderFailureReason`; the completion
gate treats a marked file as resolved, so the recording still completes
and bills with its remaining artifacts.
- A real failure reason always wins over the size markers when the
recording fails.

Deferred: the cap is a stopgap until core supports streaming uploads
(TODO in code).

<!-- This is an auto-generated description by cubic. -->
<a
href="https://cubic.dev/pr/twentyhq/twenty/pull/22463?utm_source=github"
target="_blank" rel="noopener noreferrer"
data-no-image-dialog="true"><picture><source
media="(prefers-color-scheme: dark)"
srcset="https://www.cubic.dev/buttons/review-in-cubic-dark.svg"><source
media="(prefers-color-scheme: light)"
srcset="https://www.cubic.dev/buttons/review-in-cubic-light.svg"><img
alt="Review in cubic"
src="https://www.cubic.dev/buttons/review-in-cubic-dark.svg"></picture></a>
<!-- End of auto-generated description by cubic. -->
This commit is contained in:
nitin
2026-07-02 20:55:47 +05:30
committed by GitHub
parent 6f64be5751
commit bc1a4cb3fb
19 changed files with 855 additions and 49 deletions
@@ -1,6 +1,6 @@
{
"name": "@twentyhq/call-recorder",
"version": "1.0.4",
"version": "1.0.5",
"license": "MIT",
"engines": {
"node": "^24.5.0",
@@ -14,6 +14,7 @@ import { CALL_RECORDER_BOT_IMAGE_BACKGROUND_ENV_VAR_NAME } from 'src/logic-funct
import { CALL_RECORDER_EVERYONE_LEFT_TIMEOUT_SECONDS } from 'src/logic-functions/constants/call-recorder-everyone-left-timeout-seconds';
import { CALL_RECORDER_EVERYONE_LEFT_TIMEOUT_SECONDS_ENV_VAR_NAME } from 'src/logic-functions/constants/call-recorder-everyone-left-timeout-seconds-env-var-name';
import { CALL_RECORDER_JOIN_EARLY_MINUTES_ENV_VAR_NAME } from 'src/logic-functions/constants/call-recorder-join-early-minutes-env-var-name';
import { CALL_RECORDER_MAX_MEDIA_FILE_SIZE_MB_ENV_VAR_NAME } from 'src/logic-functions/constants/call-recorder-max-media-file-size-mb-env-var-name';
import { CALL_RECORDER_NAME_ENV_VAR_NAME } from 'src/logic-functions/constants/call-recorder-name-env-var-name';
import { CALL_RECORDER_NOONE_JOINED_TIMEOUT_SECONDS } from 'src/logic-functions/constants/call-recorder-noone-joined-timeout-seconds';
import { CALL_RECORDER_NOONE_JOINED_TIMEOUT_SECONDS_ENV_VAR_NAME } from 'src/logic-functions/constants/call-recorder-noone-joined-timeout-seconds-env-var-name';
@@ -23,6 +24,7 @@ import { CALL_RECORDER_WAITING_ROOM_TIMEOUT_SECONDS } from 'src/logic-functions/
import { CALL_RECORDER_WAITING_ROOM_TIMEOUT_SECONDS_ENV_VAR_NAME } from 'src/logic-functions/constants/call-recorder-waiting-room-timeout-seconds-env-var-name';
import { DEFAULT_CALL_RECORDER_BOT_IMAGE_BACKGROUND } from 'src/logic-functions/constants/default-call-recorder-bot-image-background';
import { DEFAULT_CALL_RECORDER_JOIN_EARLY_MINUTES } from 'src/logic-functions/constants/default-call-recorder-join-early-minutes';
import { DEFAULT_CALL_RECORDER_MAX_MEDIA_FILE_SIZE_MB } from 'src/logic-functions/constants/default-call-recorder-max-media-file-size-mb';
import { DEFAULT_CALL_RECORDER_NAME } from 'src/logic-functions/constants/default-call-recorder-name';
import { DEFAULT_CALL_RECORDER_RECORDING_RETENTION_HOURS } from 'src/logic-functions/constants/default-call-recorder-recording-retention-hours';
import { DEFAULT_CALL_RECORDER_USE_WORKSPACE_LOGO } from 'src/logic-functions/constants/default-call-recorder-use-workspace-logo';
@@ -109,6 +111,10 @@ export default defineApplication({
description: `How many hours Recall.ai retains recording media after processing. Defaults to ${DEFAULT_CALL_RECORDER_RECORDING_RETENTION_HOURS} hours (6 days and 22 hours) to stay below Recall.ai's 7-day free storage window. Values above 168 hours may incur Recall.ai storage charges.`,
isSecret: false,
},
[CALL_RECORDER_MAX_MEDIA_FILE_SIZE_MB_ENV_VAR_NAME]: {
description: `Maximum size in megabytes for a single recording media file (video or audio) ingested from Recall.ai. Larger files are skipped and noted in the call recording failure reason; the recording still completes with its remaining artifacts. Defaults to ${DEFAULT_CALL_RECORDER_MAX_MEDIA_FILE_SIZE_MB} MB to keep media ingestion within the logic function memory limit.`,
isSecret: false,
},
[RECALL_WEBHOOK_SECRET_ENV_VAR_NAME]: {
description:
'Recall.ai webhook signing secret (whsec_...). Set by the server admin from the Recall webhook endpoint settings; used to verify the Svix signature of incoming Recall webhook deliveries.',
@@ -0,0 +1,2 @@
export const CALL_RECORDER_MAX_MEDIA_FILE_SIZE_MB_ENV_VAR_NAME =
'CALL_RECORDER_MAX_MEDIA_FILE_SIZE_MB';
@@ -0,0 +1 @@
export const DEFAULT_CALL_RECORDER_MAX_MEDIA_FILE_SIZE_MB = 80;
@@ -0,0 +1,23 @@
import { CALL_RECORDER_MAX_MEDIA_FILE_SIZE_MB_ENV_VAR_NAME } from 'src/logic-functions/constants/call-recorder-max-media-file-size-mb-env-var-name';
import { DEFAULT_CALL_RECORDER_MAX_MEDIA_FILE_SIZE_MB } from 'src/logic-functions/constants/default-call-recorder-max-media-file-size-mb';
import { getApplicationVariableValue } from 'src/logic-functions/utils/get-application-variable-value.util';
import { isNonEmptyString } from 'src/logic-functions/utils/is-non-empty-string.util';
const BYTES_PER_MEGABYTE = 1024 * 1024;
export const getMaxMediaFileSizeBytes = (): number => {
const configuredMaxMediaFileSizeMb = getApplicationVariableValue(
CALL_RECORDER_MAX_MEDIA_FILE_SIZE_MB_ENV_VAR_NAME,
);
const maxMediaFileSizeMb = isNonEmptyString(configuredMaxMediaFileSizeMb)
? Number(configuredMaxMediaFileSizeMb.trim())
: NaN;
const resolvedMaxMediaFileSizeMb =
Number.isFinite(maxMediaFileSizeMb) && maxMediaFileSizeMb > 0
? maxMediaFileSizeMb
: DEFAULT_CALL_RECORDER_MAX_MEDIA_FILE_SIZE_MB;
return resolvedMaxMediaFileSizeMb * BYTES_PER_MEGABYTE;
};
@@ -0,0 +1,2 @@
export const VIDEO_FILE_TOO_LARGE_FAILURE_REASON = 'video_file_too_large';
export const AUDIO_FILE_TOO_LARGE_FAILURE_REASON = 'audio_file_too_large';
@@ -13,6 +13,7 @@ describe('isCallRecordingIngestionComplete', () => {
transcript: TRANSCRIPT_CONTENT,
audio: AUDIO_VALUE,
video: VIDEO_VALUE,
callRecorderFailureReason: undefined,
}),
).toBe(true);
});
@@ -26,6 +27,7 @@ describe('isCallRecordingIngestionComplete', () => {
},
audio: AUDIO_VALUE,
video: VIDEO_VALUE,
callRecorderFailureReason: undefined,
}),
).toBe(false);
});
@@ -36,6 +38,7 @@ describe('isCallRecordingIngestionComplete', () => {
transcript: null,
audio: AUDIO_VALUE,
video: VIDEO_VALUE,
callRecorderFailureReason: undefined,
}),
).toBe(false);
});
@@ -46,6 +49,7 @@ describe('isCallRecordingIngestionComplete', () => {
transcript: TRANSCRIPT_CONTENT,
audio: undefined,
video: VIDEO_VALUE,
callRecorderFailureReason: undefined,
}),
).toBe(false);
expect(
@@ -53,6 +57,56 @@ describe('isCallRecordingIngestionComplete', () => {
transcript: TRANSCRIPT_CONTENT,
audio: AUDIO_VALUE,
video: [],
callRecorderFailureReason: undefined,
}),
).toBe(false);
});
it('treats a media file skipped for size as resolved', () => {
expect(
isCallRecordingIngestionComplete({
transcript: TRANSCRIPT_CONTENT,
audio: AUDIO_VALUE,
video: undefined,
callRecorderFailureReason: 'video_file_too_large',
}),
).toBe(true);
expect(
isCallRecordingIngestionComplete({
transcript: TRANSCRIPT_CONTENT,
audio: undefined,
video: VIDEO_VALUE,
callRecorderFailureReason: 'audio_file_too_large',
}),
).toBe(true);
expect(
isCallRecordingIngestionComplete({
transcript: TRANSCRIPT_CONTENT,
audio: undefined,
video: undefined,
callRecorderFailureReason: 'video_file_too_large,audio_file_too_large',
}),
).toBe(true);
});
it('does not let a size marker excuse the other missing artifact', () => {
expect(
isCallRecordingIngestionComplete({
transcript: TRANSCRIPT_CONTENT,
audio: undefined,
video: VIDEO_VALUE,
callRecorderFailureReason: 'video_file_too_large',
}),
).toBe(false);
});
it('ignores unrelated failure reasons', () => {
expect(
isCallRecordingIngestionComplete({
transcript: TRANSCRIPT_CONTENT,
audio: AUDIO_VALUE,
video: undefined,
callRecorderFailureReason: 'recall_bot_not_found',
}),
).toBe(false);
});
@@ -0,0 +1,46 @@
import { describe, expect, it } from 'vitest';
import { parseMediaFileTooLargeMarkers } from 'src/logic-functions/domain/parse-media-file-too-large-markers.util';
describe('parseMediaFileTooLargeMarkers', () => {
it('parses single-artifact markers', () => {
expect(parseMediaFileTooLargeMarkers('video_file_too_large')).toEqual({
audioFileTooLarge: false,
videoFileTooLarge: true,
});
expect(parseMediaFileTooLargeMarkers('audio_file_too_large')).toEqual({
audioFileTooLarge: true,
videoFileTooLarge: false,
});
});
it('parses comma-joined markers for both artifacts', () => {
expect(
parseMediaFileTooLargeMarkers(
'video_file_too_large,audio_file_too_large',
),
).toEqual({
audioFileTooLarge: true,
videoFileTooLarge: true,
});
});
it('reports no markers for unset or unrelated reasons', () => {
expect(parseMediaFileTooLargeMarkers(undefined)).toEqual({
audioFileTooLarge: false,
videoFileTooLarge: false,
});
expect(parseMediaFileTooLargeMarkers(null)).toEqual({
audioFileTooLarge: false,
videoFileTooLarge: false,
});
expect(parseMediaFileTooLargeMarkers('')).toEqual({
audioFileTooLarge: false,
videoFileTooLarge: false,
});
expect(parseMediaFileTooLargeMarkers('recall_bot_not_found')).toEqual({
audioFileTooLarge: false,
videoFileTooLarge: false,
});
});
});
@@ -66,6 +66,37 @@ describe('shouldCompleteCallRecordingIngestion', () => {
).toBe(false);
});
it('completes when a missing media file is marked too large', () => {
expect(
shouldCompleteCallRecordingIngestion({
current: {
status: CallRecordingStatus.PROCESSING,
startedAt: '2026-06-10T09:00:00.000Z',
endedAt: '2026-06-10T10:00:00.000Z',
transcript: filledTranscript,
audio: filledAudio,
},
updateData: {
callRecorderFailureReason: 'video_file_too_large',
},
}),
).toBe(true);
expect(
shouldCompleteCallRecordingIngestion({
current: {
status: CallRecordingStatus.PROCESSING,
startedAt: '2026-06-10T09:00:00.000Z',
endedAt: '2026-06-10T10:00:00.000Z',
transcript: filledTranscript,
audio: filledAudio,
callRecorderFailureReason: 'video_file_too_large',
},
updateData: {},
}),
).toBe(true);
});
it('does not complete a persisted failed recording', () => {
expect(
shouldCompleteCallRecordingIngestion({
@@ -1,19 +1,28 @@
import { isNonEmptyArray, isNull, isUndefined } from '@sniptt/guards';
import { type FilesFieldValue } from 'src/logic-functions/types/files-field-value.type';
import { parseMediaFileTooLargeMarkers } from 'src/logic-functions/domain/parse-media-file-too-large-markers.util';
import { parseTranscriptMarker } from 'src/logic-functions/domain/parse-transcript-marker.util';
export const isCallRecordingIngestionComplete = ({
transcript,
audio,
video,
callRecorderFailureReason,
}: {
transcript: unknown;
audio: FilesFieldValue | undefined;
video: FilesFieldValue | undefined;
}): boolean =>
!isNull(transcript) &&
!isUndefined(transcript) &&
isUndefined(parseTranscriptMarker(transcript)) &&
isNonEmptyArray(audio) &&
isNonEmptyArray(video);
callRecorderFailureReason: string | null | undefined;
}): boolean => {
const { audioFileTooLarge, videoFileTooLarge } =
parseMediaFileTooLargeMarkers(callRecorderFailureReason);
return (
!isNull(transcript) &&
!isUndefined(transcript) &&
isUndefined(parseTranscriptMarker(transcript)) &&
(isNonEmptyArray(audio) || audioFileTooLarge) &&
(isNonEmptyArray(video) || videoFileTooLarge)
);
};
@@ -0,0 +1,27 @@
import {
AUDIO_FILE_TOO_LARGE_FAILURE_REASON,
VIDEO_FILE_TOO_LARGE_FAILURE_REASON,
} from 'src/logic-functions/constants/media-file-too-large-failure-reasons';
import { isNonEmptyString } from 'src/logic-functions/utils/is-non-empty-string.util';
export type MediaFileTooLargeMarkers = {
audioFileTooLarge: boolean;
videoFileTooLarge: boolean;
};
export const parseMediaFileTooLargeMarkers = (
callRecorderFailureReason: string | null | undefined,
): MediaFileTooLargeMarkers => {
const failureReasons = isNonEmptyString(callRecorderFailureReason)
? callRecorderFailureReason.split(',').map((reason) => reason.trim())
: [];
return {
audioFileTooLarge: failureReasons.includes(
AUDIO_FILE_TOO_LARGE_FAILURE_REASON,
),
videoFileTooLarge: failureReasons.includes(
VIDEO_FILE_TOO_LARGE_FAILURE_REASON,
),
};
};
@@ -15,6 +15,7 @@ export const shouldCompleteCallRecordingIngestion = ({
transcript?: unknown;
audio?: FilesFieldValue;
video?: FilesFieldValue;
callRecorderFailureReason?: string | null;
};
updateData: CallRecordingUpdateFields;
}): boolean =>
@@ -29,4 +30,6 @@ export const shouldCompleteCallRecordingIngestion = ({
transcript: updateData.transcript ?? current.transcript,
audio: updateData.audio ?? current.audio,
video: updateData.video ?? current.video,
callRecorderFailureReason:
updateData.callRecorderFailureReason ?? current.callRecorderFailureReason,
});
@@ -279,6 +279,163 @@ describe('convergeDivergedCallRecordings', () => {
});
});
it('completes and charges when the missing video is marked too large', async () => {
getRecallBotMock.mockResolvedValue({
ok: true,
bot: {
status_changes: [
{ code: 'done', created_at: '2026-06-09T14:05:00.000Z' },
],
recordings: [
{
id: 'recall-recording-1',
started_at: '2026-06-09T13:02:00.000Z',
completed_at: '2026-06-09T14:00:00.000Z',
},
],
},
});
ingestCallRecordingMediaMock.mockResolvedValue({
audio: [{ fileId: 'file-audio-1', label: 'audio.mp3' }],
callRecorderFailureReason: 'video_file_too_large',
});
const client = buildClient([
buildStuckRecordingNode({
status: 'PROCESSING',
startedAt: '2026-06-09T13:02:00.000Z',
endedAt: '2026-06-09T14:00:00.000Z',
externalRecordingId: 'recall-recording-1',
transcript: [{ participant: { id: 1 }, words: [] }],
}),
]);
const result = await convergeDivergedCallRecordings({
client: client as unknown as CoreApiClient,
now: NOW,
});
expect(client.mutations).toEqual([
{
id: 'call-recording-1',
data: {
audio: [{ fileId: 'file-audio-1', label: 'audio.mp3' }],
callRecorderFailureReason: 'video_file_too_large',
},
},
{
id: 'call-recording-1',
data: { status: 'COMPLETED' },
},
]);
expect(chargeCompletedCallRecordingMock).toHaveBeenCalledWith({
callRecordingId: 'call-recording-1',
startedAt: '2026-06-09T13:02:00.000Z',
endedAt: '2026-06-09T14:00:00.000Z',
});
expect(result.updatedCallRecordingIds).toEqual(['call-recording-1']);
});
it('completes from a persisted size marker once the transcript lands', async () => {
getRecallBotMock.mockResolvedValue({
ok: true,
bot: {
status_changes: [
{ code: 'done', created_at: '2026-06-09T14:05:00.000Z' },
],
recordings: [
{
id: 'recall-recording-1',
started_at: '2026-06-09T13:02:00.000Z',
completed_at: '2026-06-09T14:00:00.000Z',
},
],
},
});
const client = buildClient([
buildStuckRecordingNode({
status: 'PROCESSING',
startedAt: '2026-06-09T13:02:00.000Z',
endedAt: '2026-06-09T14:00:00.000Z',
externalRecordingId: 'recall-recording-1',
callRecorderFailureReason: 'video_file_too_large',
audio: [{ fileId: 'file-audio-1' }],
transcript: [{ participant: { id: 1 }, words: [] }],
}),
]);
const result = await convergeDivergedCallRecordings({
client: client as unknown as CoreApiClient,
now: NOW,
});
expect(ingestCallRecordingMediaMock).toHaveBeenCalledWith({
callRecordingId: 'call-recording-1',
externalRecordingId: 'recall-recording-1',
hasAudio: true,
hasVideo: false,
});
expect(client.mutations).toEqual([
{
id: 'call-recording-1',
data: { status: 'COMPLETED' },
},
]);
expect(chargeCompletedCallRecordingMock).toHaveBeenCalledWith({
callRecordingId: 'call-recording-1',
startedAt: '2026-06-09T13:02:00.000Z',
endedAt: '2026-06-09T14:00:00.000Z',
});
expect(result.updatedCallRecordingIds).toEqual(['call-recording-1']);
});
it('keeps the real failure reason over the size marker when the bot failed', async () => {
getRecallBotMock.mockResolvedValue({
ok: true,
bot: {
status_changes: [
{ code: 'fatal', created_at: '2026-06-09T14:05:00.000Z' },
],
recordings: [
{
id: 'recall-recording-1',
started_at: '2026-06-09T13:02:00.000Z',
completed_at: '2026-06-09T14:00:00.000Z',
},
],
},
});
ingestCallRecordingMediaMock.mockResolvedValue({
audio: [{ fileId: 'file-audio-1', label: 'audio.mp3' }],
callRecorderFailureReason: 'video_file_too_large',
});
const client = buildClient([
buildStuckRecordingNode({
status: 'PROCESSING',
startedAt: '2026-06-09T13:02:00.000Z',
endedAt: '2026-06-09T14:00:00.000Z',
externalRecordingId: 'recall-recording-1',
transcript: [{ participant: { id: 1 }, words: [] }],
}),
]);
await convergeDivergedCallRecordings({
client: client as unknown as CoreApiClient,
now: NOW,
});
expect(client.mutations).toEqual([
{
id: 'call-recording-1',
data: {
status: 'FAILED',
callRecorderFailureReason: 'fatal',
audio: [{ fileId: 'file-audio-1', label: 'audio.mp3' }],
},
},
]);
expect(chargeCompletedCallRecordingMock).not.toHaveBeenCalled();
});
it('skips records whose meeting has not started yet', async () => {
const client = buildClient([
buildStuckRecordingNode({
@@ -936,6 +936,110 @@ describe('handleRecallWebhook', () => {
});
});
it('completes and keeps the size marker when a media file is too large', async () => {
getRecallBotMock.mockResolvedValue({
ok: true,
bot: { id: 'recall-bot-1' },
});
ingestCallRecordingMediaMock.mockResolvedValue({
audio: [{ fileId: 'file-audio-1', label: 'audio.mp3' }],
callRecorderFailureReason: 'video_file_too_large',
});
const client = new FakeCoreApiClient([
{
id: 'call-recording-1',
status: 'PROCESSING',
externalBotId: 'recall-bot-1',
externalRecordingId: 'recall-recording-1',
startedAt: '2026-01-01T13:02:00.000Z',
endedAt: '2026-01-01T14:05:00.000Z',
transcript: [{ participant: { id: 1 }, words: [] }],
},
]);
const result = await handleRecallWebhook({
client: client as unknown as CoreApiClient,
body: buildRecordingDoneWebhookBody(),
});
expect(client.mutations).toEqual([
{
id: 'call-recording-1',
data: {
externalBotId: 'recall-bot-1',
externalRecordingId: 'recall-recording-1',
audio: [{ fileId: 'file-audio-1', label: 'audio.mp3' }],
callRecorderFailureReason: 'video_file_too_large',
},
},
{
id: 'call-recording-1',
data: { status: 'COMPLETED' },
},
]);
expect(chargeCompletedCallRecordingMock).toHaveBeenCalledWith({
callRecordingId: 'call-recording-1',
startedAt: '2026-01-01T13:02:00.000Z',
endedAt: '2026-01-01T14:05:00.000Z',
});
expect(result).toEqual({
status: 'updated',
event: 'recording.done',
callRecordingId: 'call-recording-1',
callRecordingStatus: 'COMPLETED',
});
});
it('keeps the real failure reason over the size marker on recording.failed', async () => {
getRecallBotMock.mockResolvedValue({
ok: true,
bot: { id: 'recall-bot-1' },
});
ingestCallRecordingMediaMock.mockResolvedValue({
audio: [{ fileId: 'file-audio-1', label: 'audio.mp3' }],
callRecorderFailureReason: 'video_file_too_large',
});
const client = new FakeCoreApiClient([
{
id: 'call-recording-1',
status: 'PROCESSING',
externalBotId: 'recall-bot-1',
externalRecordingId: 'recall-recording-1',
startedAt: '2026-01-01T13:02:00.000Z',
endedAt: '2026-01-01T14:05:00.000Z',
transcript: [{ participant: { id: 1 }, words: [] }],
},
]);
const result = await handleRecallWebhook({
client: client as unknown as CoreApiClient,
body: {
...buildRecordingDoneWebhookBody(),
event: 'recording.failed',
},
});
expect(client.mutations).toEqual([
{
id: 'call-recording-1',
data: {
externalBotId: 'recall-bot-1',
externalRecordingId: 'recall-recording-1',
status: 'FAILED',
callRecorderFailureReason: 'recording.failed',
audio: [{ fileId: 'file-audio-1', label: 'audio.mp3' }],
},
},
]);
expect(chargeCompletedCallRecordingMock).not.toHaveBeenCalled();
expect(result).toEqual({
status: 'updated',
event: 'recording.failed',
callRecordingId: 'call-recording-1',
callRecordingStatus: 'FAILED',
});
});
it('stays PROCESSING on recording.done while artifacts are missing', async () => {
getRecallBotMock.mockResolvedValue({
ok: true,
@@ -1,5 +1,6 @@
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import { CALL_RECORDER_MAX_MEDIA_FILE_SIZE_MB_ENV_VAR_NAME } from 'src/logic-functions/constants/call-recorder-max-media-file-size-mb-env-var-name';
import { ingestCallRecordingMedia } from 'src/logic-functions/flows/ingest-call-recording-media.util';
const uploadFileMock = vi.hoisted(() => vi.fn());
@@ -15,19 +16,66 @@ vi.mock('src/logic-functions/recall-api/get-recall-recording.util', () => ({
getRecallRecording: getRecallRecordingMock,
}));
const VIDEO_URL = 'https://media.example.com/video.mp4';
const AUDIO_URL = 'https://media.example.com/audio.mp3';
const RECORDING_WITH_MEDIA = {
id: 'recall-recording-1',
media_shortcuts: {
video_mixed: { download_url: 'https://media.example.com/video.mp4' },
audio_mixed: { download_url: 'https://media.example.com/audio.mp3' },
video_mixed: { download_url: VIDEO_URL },
audio_mixed: { download_url: AUDIO_URL },
},
};
const buildFetchResponse = () => ({
ok: true,
headers: { get: () => 'video/mp4' },
arrayBuffer: async () => new ArrayBuffer(8),
});
const buildBodyStream = (chunks: Uint8Array[]): ReadableStream<Uint8Array> =>
new ReadableStream<Uint8Array>({
start(controller) {
for (const chunk of chunks) {
controller.enqueue(chunk);
}
controller.close();
},
});
const buildFetchResponse = ({
contentType = 'video/mp4',
contentLengthBytes,
body,
}: {
contentType?: string;
contentLengthBytes?: number;
body?: unknown;
} = {}) => {
const headers = new Map<string, string>([['content-type', contentType]]);
if (contentLengthBytes !== undefined) {
headers.set('content-length', String(contentLengthBytes));
}
return {
ok: true,
status: 200,
headers: {
get: (name: string) => headers.get(name.toLowerCase()) ?? null,
},
body: body ?? buildBodyStream([new Uint8Array(8)]),
};
};
const stubFetchByUrl = (responsesByUrl: Record<string, unknown>) => {
vi.stubGlobal(
'fetch',
vi.fn().mockImplementation((url: string) => {
const response = responsesByUrl[url];
if (response === undefined) {
throw new Error(`Unhandled fetch url in test: ${url}`);
}
return Promise.resolve(response);
}),
);
};
describe('ingestCallRecordingMedia', () => {
beforeEach(() => {
@@ -38,11 +86,19 @@ describe('ingestCallRecordingMedia', () => {
ok: true,
recording: RECORDING_WITH_MEDIA,
});
vi.stubGlobal('fetch', vi.fn().mockResolvedValue(buildFetchResponse()));
vi.stubGlobal(
'fetch',
vi
.fn()
.mockImplementation(() =>
Promise.resolve(buildFetchResponse({ contentLengthBytes: 8 })),
),
);
});
afterEach(() => {
vi.unstubAllGlobals();
vi.unstubAllEnvs();
});
it('downloads and uploads every missing artifact', async () => {
@@ -150,4 +206,154 @@ describe('ingestCallRecordingMedia', () => {
expect.stringContaining('recording boom'),
);
});
it('skips an oversized file without reading its body and records the reason', async () => {
const cancelMock = vi.fn().mockResolvedValue(undefined);
stubFetchByUrl({
[VIDEO_URL]: buildFetchResponse({
contentLengthBytes: 200 * 1024 * 1024,
body: { cancel: cancelMock },
}),
[AUDIO_URL]: buildFetchResponse({ contentLengthBytes: 8 }),
});
uploadFileMock.mockResolvedValue({ id: 'file-audio-1' });
const updateFields = await ingestCallRecordingMedia({
callRecordingId: 'call-recording-1',
externalRecordingId: 'recall-recording-1',
hasAudio: false,
hasVideo: false,
});
expect(updateFields).toEqual({
audio: [{ fileId: 'file-audio-1', label: 'audio.mp3' }],
callRecorderFailureReason: 'video_file_too_large',
});
expect(uploadFileMock).toHaveBeenCalledTimes(1);
expect(cancelMock).toHaveBeenCalled();
expect(console.warn).toHaveBeenCalledWith(
expect.stringContaining('artifact-too-large'),
);
});
it('records both markers when video and audio exceed the cap', async () => {
stubFetchByUrl({
[VIDEO_URL]: buildFetchResponse({
contentLengthBytes: 200 * 1024 * 1024,
}),
[AUDIO_URL]: buildFetchResponse({
contentLengthBytes: 120 * 1024 * 1024,
}),
});
const updateFields = await ingestCallRecordingMedia({
callRecordingId: 'call-recording-1',
externalRecordingId: 'recall-recording-1',
hasAudio: false,
hasVideo: false,
});
expect(updateFields).toEqual({
callRecorderFailureReason: 'video_file_too_large,audio_file_too_large',
});
expect(uploadFileMock).not.toHaveBeenCalled();
});
it('honors the cap configured through the environment', async () => {
vi.stubEnv(CALL_RECORDER_MAX_MEDIA_FILE_SIZE_MB_ENV_VAR_NAME, '1');
stubFetchByUrl({
[VIDEO_URL]: buildFetchResponse({
contentLengthBytes: 2 * 1024 * 1024,
}),
[AUDIO_URL]: buildFetchResponse({ contentLengthBytes: 8 }),
});
uploadFileMock.mockResolvedValue({ id: 'file-audio-1' });
const updateFields = await ingestCallRecordingMedia({
callRecordingId: 'call-recording-1',
externalRecordingId: 'recall-recording-1',
hasAudio: false,
hasVideo: false,
});
expect(updateFields).toEqual({
audio: [{ fileId: 'file-audio-1', label: 'audio.mp3' }],
callRecorderFailureReason: 'video_file_too_large',
});
});
it('falls back to the default cap when the configured value is invalid', async () => {
vi.stubEnv(
CALL_RECORDER_MAX_MEDIA_FILE_SIZE_MB_ENV_VAR_NAME,
'not-a-number',
);
uploadFileMock.mockResolvedValue({ id: 'file-video-1' });
stubFetchByUrl({
[VIDEO_URL]: buildFetchResponse({
contentLengthBytes: 2 * 1024 * 1024,
}),
});
const updateFields = await ingestCallRecordingMedia({
callRecordingId: 'call-recording-1',
externalRecordingId: 'recall-recording-1',
hasAudio: true,
hasVideo: false,
});
expect(updateFields).toEqual({
video: [{ fileId: 'file-video-1', label: 'video.mp4' }],
});
});
it('enforces the cap while reading a response without content-length', async () => {
vi.stubEnv(CALL_RECORDER_MAX_MEDIA_FILE_SIZE_MB_ENV_VAR_NAME, '1');
stubFetchByUrl({
[VIDEO_URL]: buildFetchResponse({
body: buildBodyStream([
new Uint8Array(700_000),
new Uint8Array(700_000),
]),
}),
});
const updateFields = await ingestCallRecordingMedia({
callRecordingId: 'call-recording-1',
externalRecordingId: 'recall-recording-1',
hasAudio: true,
hasVideo: false,
});
expect(updateFields).toEqual({
callRecorderFailureReason: 'video_file_too_large',
});
expect(uploadFileMock).not.toHaveBeenCalled();
});
it('ingests a response without content-length when it stays within the cap', async () => {
uploadFileMock.mockResolvedValue({ id: 'file-video-1' });
stubFetchByUrl({
[VIDEO_URL]: buildFetchResponse({
body: buildBodyStream([new Uint8Array([1, 2, 3]), new Uint8Array([4])]),
}),
});
const updateFields = await ingestCallRecordingMedia({
callRecordingId: 'call-recording-1',
externalRecordingId: 'recall-recording-1',
hasAudio: true,
hasVideo: false,
});
expect(updateFields).toEqual({
video: [{ fileId: 'file-video-1', label: 'video.mp4' }],
});
expect(uploadFileMock).toHaveBeenCalledWith(
Buffer.from([1, 2, 3, 4]),
'video.mp4',
'video/mp4',
expect.any(String),
);
});
});
@@ -35,6 +35,7 @@ type DivergedCallRecordingCandidate = {
endedAt: string | undefined;
externalBotId: string | undefined;
externalRecordingId: string | undefined;
callRecorderFailureReason: string | undefined;
transcript: unknown;
audio: FilesFieldValue | undefined;
video: FilesFieldValue | undefined;
@@ -50,6 +51,7 @@ type DivergedCallRecordingNode = {
endedAt?: string | null;
externalBotId?: string | null;
externalRecordingId?: string | null;
callRecorderFailureReason?: string | null;
transcript?: unknown;
audio?: FilesFieldValue | null;
video?: FilesFieldValue | null;
@@ -150,6 +152,7 @@ const fetchDivergedCallRecordingCandidates = async (
endedAt: true,
externalBotId: true,
externalRecordingId: true,
callRecorderFailureReason: true,
transcript: true,
audio: { fileId: true },
video: { fileId: true },
@@ -180,6 +183,9 @@ const fetchDivergedCallRecordingCandidates = async (
externalRecordingId: isNonEmptyString(node.externalRecordingId)
? node.externalRecordingId
: undefined,
callRecorderFailureReason: isNonEmptyString(node.callRecorderFailureReason)
? node.callRecorderFailureReason
: undefined,
transcript: node.transcript ?? undefined,
audio: node.audio ?? undefined,
video: node.video ?? undefined,
@@ -267,15 +273,18 @@ const convergeCallRecording = async ({
result.requestedTranscriptCallRecordingIds.push(candidate.id);
}
Object.assign(
updateData,
await ingestCallRecordingMedia({
callRecordingId: candidate.id,
externalRecordingId,
hasAudio: isNonEmptyArray(candidate.audio),
hasVideo: isNonEmptyArray(candidate.video),
}),
);
const mediaIngestionUpdate = await ingestCallRecordingMedia({
callRecordingId: candidate.id,
externalRecordingId,
hasAudio: isNonEmptyArray(candidate.audio),
hasVideo: isNonEmptyArray(candidate.video),
});
if (updateData.status === CallRecordingStatus.FAILED) {
delete mediaIngestionUpdate.callRecorderFailureReason;
}
Object.assign(updateData, mediaIngestionUpdate);
}
const terminalArtifactGateFailureUpdate =
@@ -30,6 +30,7 @@ type MatchedCallRecording = {
startedAt?: string;
endedAt?: string;
externalRecordingId?: string;
callRecorderFailureReason?: string;
transcript?: unknown;
audio?: FilesFieldValue;
video?: FilesFieldValue;
@@ -196,13 +197,16 @@ const handleRecallStatusEvent = async ({
callRecordingStatus,
});
Object.assign(
updateData,
await buildMediaIngestionUpdate({
callRecording,
externalRecordingId: externalRecordingIdResolution.externalRecordingId,
}),
);
const mediaIngestionUpdate = await buildMediaIngestionUpdate({
callRecording,
externalRecordingId: externalRecordingIdResolution.externalRecordingId,
});
if (updateData.status === CallRecordingStatus.FAILED) {
delete mediaIngestionUpdate.callRecorderFailureReason;
}
Object.assign(updateData, mediaIngestionUpdate);
const terminalArtifactGateFailureUpdate =
buildTerminalArtifactGateFailureUpdate({
@@ -348,6 +352,7 @@ const findCallRecordingByFilter = async (
startedAt: true,
endedAt: true,
externalRecordingId: true,
callRecorderFailureReason: true,
transcript: true,
audio: { fileId: true },
video: { fileId: true },
@@ -368,6 +373,7 @@ const findCallRecordingByFilter = async (
startedAt: getString(node.startedAt),
endedAt: getString(node.endedAt),
externalRecordingId: getString(node.externalRecordingId),
callRecorderFailureReason: getString(node.callRecorderFailureReason),
transcript: node.transcript ?? undefined,
audio: node.audio ?? undefined,
video: node.video ?? undefined,
@@ -539,7 +545,12 @@ const buildMediaIngestionUpdate = async ({
}: {
callRecording: MatchedCallRecording;
externalRecordingId: string | undefined;
}): Promise<Pick<CallRecordingUpdateFields, 'audio' | 'video'>> => {
}): Promise<
Pick<
CallRecordingUpdateFields,
'audio' | 'video' | 'callRecorderFailureReason'
>
> => {
const hasAudio = isNonEmptyArray(callRecording.audio);
const hasVideo = isNonEmptyArray(callRecording.video);
@@ -1,18 +1,33 @@
import { isUndefined } from '@sniptt/guards';
import { isNull, isUndefined } from '@sniptt/guards';
import { MetadataApiClient } from 'twenty-client-sdk/metadata';
import { CALL_RECORDING_AUDIO_FIELD_UNIVERSAL_IDENTIFIER } from 'src/constants/call-recording-audio-field-universal-identifier';
import { CALL_RECORDING_VIDEO_FIELD_UNIVERSAL_IDENTIFIER } from 'src/constants/call-recording-video-field-universal-identifier';
import { getMaxMediaFileSizeBytes } from 'src/logic-functions/constants/get-max-media-file-size-bytes';
import {
AUDIO_FILE_TOO_LARGE_FAILURE_REASON,
VIDEO_FILE_TOO_LARGE_FAILURE_REASON,
} from 'src/logic-functions/constants/media-file-too-large-failure-reasons';
import { extractRecallMediaUrls } from 'src/logic-functions/recall-api/extract-recall-media-urls.util';
import { getRecallRecording } from 'src/logic-functions/recall-api/get-recall-recording.util';
import { type CallRecordingMediaFile } from 'src/logic-functions/types/call-recording-media-file.type';
import { type CallRecordingUpdateFields } from 'src/logic-functions/types/call-recording-update-fields.type';
import { isNonEmptyString } from 'src/logic-functions/utils/is-non-empty-string.util';
type CallRecordingMediaUpdateFields = Pick<
CallRecordingUpdateFields,
'audio' | 'video'
'audio' | 'video' | 'callRecorderFailureReason'
>;
type IngestMediaArtifactResult =
| { outcome: 'ingested'; files: CallRecordingMediaFile[] }
| { outcome: 'too-large' }
| { outcome: 'failed' };
type DownloadMediaFileResult =
| { outcome: 'downloaded'; buffer: Buffer; contentType: string }
| { outcome: 'too-large'; sizeBytes: number | undefined };
const MEDIA_DOWNLOAD_TIMEOUT_MS = 120_000;
export const ingestCallRecordingMedia = async ({
@@ -42,7 +57,10 @@ export const ingestCallRecordingMedia = async ({
const mediaUrls = extractRecallMediaUrls(recordingResult.recording);
const metadataClient = new MetadataApiClient();
// TODO: drop the size cap once core streams uploads without buffering whole files in memory.
const maxMediaFileSizeBytes = getMaxMediaFileSizeBytes();
const updateFields: CallRecordingMediaUpdateFields = {};
const tooLargeFailureReasons: string[] = [];
if (!hasVideo && !isUndefined(mediaUrls.videoUrl)) {
const video = await ingestMediaArtifact({
@@ -52,10 +70,15 @@ export const ingestCallRecordingMedia = async ({
fileName: 'video.mp4',
fieldMetadataUniversalIdentifier:
CALL_RECORDING_VIDEO_FIELD_UNIVERSAL_IDENTIFIER,
maxMediaFileSizeBytes,
});
if (!isUndefined(video)) {
updateFields.video = video;
if (video.outcome === 'ingested') {
updateFields.video = video.files;
}
if (video.outcome === 'too-large') {
tooLargeFailureReasons.push(VIDEO_FILE_TOO_LARGE_FAILURE_REASON);
}
}
@@ -67,11 +90,20 @@ export const ingestCallRecordingMedia = async ({
fileName: 'audio.mp3',
fieldMetadataUniversalIdentifier:
CALL_RECORDING_AUDIO_FIELD_UNIVERSAL_IDENTIFIER,
maxMediaFileSizeBytes,
});
if (!isUndefined(audio)) {
updateFields.audio = audio;
if (audio.outcome === 'ingested') {
updateFields.audio = audio.files;
}
if (audio.outcome === 'too-large') {
tooLargeFailureReasons.push(AUDIO_FILE_TOO_LARGE_FAILURE_REASON);
}
}
if (tooLargeFailureReasons.length > 0) {
updateFields.callRecorderFailureReason = tooLargeFailureReasons.join(',');
}
return updateFields;
@@ -83,20 +115,33 @@ const ingestMediaArtifact = async ({
url,
fileName,
fieldMetadataUniversalIdentifier,
maxMediaFileSizeBytes,
}: {
callRecordingId: string;
metadataClient: InstanceType<typeof MetadataApiClient>;
url: string;
fileName: string;
fieldMetadataUniversalIdentifier: string;
}): Promise<CallRecordingMediaFile[] | undefined> => {
maxMediaFileSizeBytes: number;
}): Promise<IngestMediaArtifactResult> => {
try {
const { buffer, contentType } = await downloadMediaFile({
const downloadResult = await downloadMediaFile({
callRecordingId,
fileName,
url,
maxMediaFileSizeBytes,
});
if (downloadResult.outcome === 'too-large') {
console.warn(
`[call-recorder] media-ingestion phase=artifact-too-large callRecordingId=${callRecordingId} fileName=${fileName} sizeBytes=${downloadResult.sizeBytes ?? 'unknown'} maxMediaFileSizeBytes=${maxMediaFileSizeBytes}`,
);
return { outcome: 'too-large' };
}
const { buffer, contentType } = downloadResult;
console.log(
`[call-recorder] media-ingestion phase=artifact-upload-start callRecordingId=${callRecordingId} fileName=${fileName} downloadedBytes=${buffer.byteLength} contentType=${contentType} ${formatMemoryUsageForLog()}`,
);
@@ -108,13 +153,16 @@ const ingestMediaArtifact = async ({
fieldMetadataUniversalIdentifier,
);
return [{ fileId: uploadedFile.id, label: fileName }];
return {
outcome: 'ingested',
files: [{ fileId: uploadedFile.id, label: fileName }],
};
} catch (error) {
console.warn(
`[call-recorder] failed to ingest ${fileName} for call recording ${callRecordingId}: ${error instanceof Error ? error.message : String(error)}`,
);
return undefined;
return { outcome: 'failed' };
}
};
@@ -122,35 +170,102 @@ const downloadMediaFile = async ({
callRecordingId,
fileName,
url,
maxMediaFileSizeBytes,
}: {
callRecordingId: string;
fileName: string;
url: string;
}): Promise<{ buffer: Buffer; contentType: string }> => {
maxMediaFileSizeBytes: number;
}): Promise<DownloadMediaFileResult> => {
const response = await fetch(url, {
signal: AbortSignal.timeout(MEDIA_DOWNLOAD_TIMEOUT_MS),
});
const contentType =
response.headers.get('content-type') ?? 'application/octet-stream';
const contentLength = response.headers.get('content-length') ?? 'unknown';
const contentLengthBytes = parseContentLengthBytes(
response.headers.get('content-length'),
);
console.log(
`[call-recorder] media-ingestion phase=artifact-download-response callRecordingId=${callRecordingId} fileName=${fileName} responseStatus=${response.status} contentLengthBytes=${contentLength} contentType=${contentType} ${formatMemoryUsageForLog()}`,
`[call-recorder] media-ingestion phase=artifact-download-response callRecordingId=${callRecordingId} fileName=${fileName} responseStatus=${response.status} contentLengthBytes=${contentLengthBytes ?? 'unknown'} contentType=${contentType} ${formatMemoryUsageForLog()}`,
);
if (!response.ok) {
throw new Error(`download failed with status ${response.status}`);
}
const arrayBuffer = await response.arrayBuffer();
const buffer = Buffer.from(arrayBuffer);
if (
!isUndefined(contentLengthBytes) &&
contentLengthBytes > maxMediaFileSizeBytes
) {
await response.body?.cancel();
return { outcome: 'too-large', sizeBytes: contentLengthBytes };
}
if (isNull(response.body)) {
throw new Error('download returned no body');
}
return readBodyWithinSizeCap({
body: response.body,
contentType,
maxMediaFileSizeBytes,
});
};
const readBodyWithinSizeCap = async ({
body,
contentType,
maxMediaFileSizeBytes,
}: {
body: ReadableStream<Uint8Array>;
contentType: string;
maxMediaFileSizeBytes: number;
}): Promise<DownloadMediaFileResult> => {
const reader = body.getReader();
const chunks: Uint8Array[] = [];
let downloadedBytes = 0;
for (;;) {
const { done, value } = await reader.read();
if (done) {
break;
}
downloadedBytes += value.byteLength;
if (downloadedBytes > maxMediaFileSizeBytes) {
await reader.cancel();
return { outcome: 'too-large', sizeBytes: undefined };
}
chunks.push(value);
}
return {
buffer,
outcome: 'downloaded',
buffer: Buffer.concat(chunks),
contentType,
};
};
const parseContentLengthBytes = (
headerValue: string | null,
): number | undefined => {
if (!isNonEmptyString(headerValue)) {
return undefined;
}
const parsedBytes = Number(headerValue.trim());
return Number.isFinite(parsedBytes) && parsedBytes >= 0
? parsedBytes
: undefined;
};
const formatMemoryUsageForLog = (): string => {
const memoryUsage = process.memoryUsage();
@@ -13,6 +13,7 @@ type PersistCallRecordingProgressCurrent = {
transcript?: unknown;
audio?: FilesFieldValue;
video?: FilesFieldValue;
callRecorderFailureReason?: string | null;
};
export const persistCallRecordingProgress = async (
@@ -42,7 +43,6 @@ export const persistCallRecordingProgress = async (
const nonStatusUpdate: CallRecordingUpdateFields = { ...updateData };
delete nonStatusUpdate.status;
delete nonStatusUpdate.callRecorderFailureReason;
if (Object.keys(nonStatusUpdate).length > 0) {
await updateCallRecording(client, { id, data: nonStatusUpdate });