diff --git a/packages/twenty-apps/public/call-recorder/src/__tests__/call-recorder-lifecycle.integration-test.ts b/packages/twenty-apps/public/call-recorder/src/__tests__/call-recorder-lifecycle.integration-test.ts new file mode 100644 index 0000000000..d90e31bb8d --- /dev/null +++ b/packages/twenty-apps/public/call-recorder/src/__tests__/call-recorder-lifecycle.integration-test.ts @@ -0,0 +1,858 @@ +import { randomUUID } from 'crypto'; + +import { CoreApiClient } from 'twenty-client-sdk/core'; +import { afterEach, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest'; + +import { cancelCallRecordingRequest } from 'src/logic-functions/flows/cancel-call-recording-request.util'; +import { reconcileCallRecorderForCalendarEventIds } from 'src/logic-functions/flows/reconcile-call-recorder.util'; +import { retryFailedRecallCancellations } from 'src/logic-functions/flows/retry-failed-recall-cancellations.util'; +import { scheduleRecallBotsForPendingCallRecordings } from 'src/logic-functions/flows/schedule-recall-bots-for-pending-call-recordings.util'; +import { processRecallWebhookHandler } from 'src/logic-functions/process-recall-webhook'; + +// --------------------------------------------------------------------------- +// Call Recorder end-to-end behavior against a live Twenty server. +// +// The app is installed on the test server by the vitest global setup, and all +// reads and writes go through the real API into the test database. Only the +// externals are mocked: +// - the Recall API (a fetch interceptor that replays the same bot for a +// repeated Idempotency-Key, like the real API), +// - the trigger transports: webhook deliveries invoke the webhook logic +// function handler directly, and cron / database-event triggers invoke +// the flows they dispatch. +// +// Every scenario then asserts the resulting CallRecording rows in the DB. +// +// The suite issues a few hundred API requests; if the test server runs with +// the default API_RATE_LIMITING_LONG_LIMIT of 100 requests per minute, +// raise it (e.g. to 100000) or the runs trip the limiter. +// --------------------------------------------------------------------------- + +const WORKSPACE_API_KEY_ENV = 'TWENTY_API_KEY'; +const RECALL_BASE_URL = 'https://us-west-2.recall.ai/api/v1'; +const FUNCTIONS_URL = 'https://call-recorder-functions.test'; +const ARTIFACT_IMPORT_ROUTE = '/call-recorder/import-call-recording-artifacts'; +const RESTRICTED_TITLE_PLACEHOLDER = + 'FIELD_RESTRICTED_ADDITIONAL_PERMISSIONS_REQUIRED'; + +// The app's generated client only covers the objects the app uses, but test +// seeding also needs calendarChannelEventAssociations (see below); this hits +// the workspace GraphQL API directly with the test API key. +const workspaceGraphql = async ( + query: string, + variables: Record = {}, +): Promise => { + const response = await fetch(`${process.env.TWENTY_API_URL}/graphql`, { + method: 'POST', + headers: { + Authorization: `Bearer ${process.env[WORKSPACE_API_KEY_ENV]}`, + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ query, variables }), + }); + const payload = await response.json(); + + if (payload.errors !== undefined) { + throw new Error( + `Workspace GraphQL request failed: ${JSON.stringify(payload.errors)}`, + ); + } + + return payload.data; +}; + +// Calendar events are only readable when a calendar channel with +// SHARE_EVERYTHING visibility claims them, exactly like events coming from a +// real calendar sync. The dev seeds provide such a channel; it is found by +// following a fully visible seeded event to its channel association. +const discoverShareEverythingChannelId = async (): Promise => { + const eventsData = await workspaceGraphql( + `query { calendarEvents(first: 30) { edges { node { id title } } } }`, + ); + const visibleEvent = eventsData.calendarEvents.edges + .map((edge: any) => edge.node) + .find( + (node: any) => + node.title !== null && node.title !== RESTRICTED_TITLE_PLACEHOLDER, + ); + + if (visibleEvent === undefined) { + throw new Error( + 'No fully visible seeded calendar event found; run the dev seeds before the integration tests', + ); + } + + const associationsData = await workspaceGraphql( + `query ($calendarEventId: UUID) { + calendarChannelEventAssociations( + filter: { calendarEventId: { eq: $calendarEventId } } + first: 1 + ) { edges { node { calendarChannelId } } } + }`, + { calendarEventId: visibleEvent.id }, + ); + const channelId = + associationsData.calendarChannelEventAssociations.edges[0]?.node + ?.calendarChannelId; + + if (channelId === undefined) { + throw new Error('Seeded visible calendar event has no channel association'); + } + + return channelId; +}; + +const inOneHour = () => new Date(Date.now() + 60 * 60 * 1000).toISOString(); +const inTwoHours = () => new Date(Date.now() + 2 * 60 * 60 * 1000).toISOString(); +const hoursAgo = (hours: number) => + new Date(Date.now() - hours * 60 * 60 * 1000).toISOString(); + +// --------------------------------------------------------------------------- +// Recall API fake, installed as a fetch interceptor. Twenty API traffic is +// never intercepted: the shared client is built before the interceptor and +// unknown URLs fall through to the real fetch. +// --------------------------------------------------------------------------- + +type FakeRecallBot = { + id: string; + metadata: Record; + statusCode: string; +}; + +class FakeRecallApi { + bots = new Map(); + botIdByIdempotencyKey = new Map(); + deletedBotIds: string[] = []; + listRequestCount = 0; + artifactImportRequests: object[] = []; + failNextDelete = false; + + seedBot(bot: FakeRecallBot): void { + this.bots.set(bot.id, bot); + } + + botForCallRecording(callRecordingId: string): FakeRecallBot | undefined { + return [...this.bots.values()].find( + (bot) => bot.metadata.twentyCallRecordingId === callRecordingId, + ); + } + + handle(requestUrl: string, requestInit?: any): Response | undefined { + const method: string = requestInit?.method ?? 'GET'; + + if (requestUrl.startsWith(`${FUNCTIONS_URL}${ARTIFACT_IMPORT_ROUTE}`)) { + this.artifactImportRequests.push(JSON.parse(requestInit?.body ?? '{}')); + + return jsonResponse(200, {}); + } + + if (!requestUrl.startsWith(RECALL_BASE_URL)) { + return undefined; + } + + if (method === 'POST' && requestUrl === `${RECALL_BASE_URL}/bot/`) { + return this.createBot(requestInit); + } + + if (method === 'GET' && requestUrl.startsWith(`${RECALL_BASE_URL}/bot/?`)) { + this.listRequestCount += 1; + + return jsonResponse(200, { + next: null, + results: [...this.bots.values()].map((bot) => ({ + id: bot.id, + metadata: bot.metadata, + status: { code: bot.statusCode }, + })), + }); + } + + const botIdMatch = requestUrl.match(/\/bot\/([^/]+)\/$/); + + if (method === 'DELETE' && botIdMatch !== null) { + if (this.failNextDelete) { + this.failNextDelete = false; + + return jsonResponse(400, {}); + } + + this.bots.delete(botIdMatch[1]); + this.deletedBotIds.push(botIdMatch[1]); + + return new Response(null, { status: 204 }); + } + + throw new Error(`Unhandled Recall API request: ${method} ${requestUrl}`); + } + + private createBot(requestInit: any): Response { + const idempotencyKey: string | undefined = + requestInit?.headers?.['Idempotency-Key']; + const alreadyCreatedBotId = + idempotencyKey === undefined + ? undefined + : this.botIdByIdempotencyKey.get(idempotencyKey); + + if (alreadyCreatedBotId !== undefined) { + return jsonResponse(200, { id: alreadyCreatedBotId }); + } + + const body = JSON.parse(requestInit?.body ?? '{}'); + const bot: FakeRecallBot = { + id: `recall-bot-${randomUUID()}`, + metadata: body.metadata ?? {}, + statusCode: 'ready', + }; + + this.bots.set(bot.id, bot); + + if (idempotencyKey !== undefined) { + this.botIdByIdempotencyKey.set(idempotencyKey, bot.id); + } + + return jsonResponse(201, { id: bot.id }); + } +} + +const jsonResponse = (status: number, body: object): Response => + new Response(JSON.stringify(body), { status }); + +// --------------------------------------------------------------------------- +// Recall webhook payloads, mirroring the shapes Recall actually delivers. +// --------------------------------------------------------------------------- + +const buildBotMetadata = (callRecordingId: string, workspaceId: string) => ({ + twentyWorkspaceId: workspaceId, + twentyCallRecordingId: callRecordingId, +}); + +const buildBotStatusChangeWebhook = ({ + botId, + metadata, + statusCode, + statusTimestamp, +}: { + botId: string; + metadata: Record; + statusCode: string; + statusTimestamp?: string; +}) => ({ + event: 'bot.status_change', + data: { + bot_id: botId, + status: { + code: statusCode, + created_at: statusTimestamp ?? new Date().toISOString(), + }, + bot: { id: botId, metadata }, + }, +}); + +const buildRecordingDoneWebhook = ({ + botId, + metadata, + startedAt, + completedAt, +}: { + botId: string; + metadata: Record; + startedAt: string; + completedAt: string; +}) => ({ + event: 'recording.done', + data: { + bot: { id: botId, metadata }, + recording: { + id: 'recall-recording-1', + started_at: startedAt, + completed_at: completedAt, + }, + }, +}); + +const buildTranscriptDoneWebhook = ({ + botId, + metadata, +}: { + botId: string; + metadata: Record; +}) => ({ + event: 'transcript.done', + data: { + bot: { id: botId, metadata }, + transcript: { id: 'recall-transcript-1' }, + }, +}); + +// --------------------------------------------------------------------------- +// Test workspace helpers: real rows in the test database, destroyed after +// each scenario. +// --------------------------------------------------------------------------- + +describe('call recorder app lifecycle (integration)', () => { + // Built before the fetch interceptor is installed so the shared client's + // Twenty API traffic always uses the real fetch. + let client: CoreApiClient; + let workspaceId: string; + let shareEverythingChannelId: string; + let recall: FakeRecallApi; + const createdCalendarEventIds: string[] = []; + const createdCallRecordingIds: string[] = []; + const createdAssociationIds: string[] = []; + + const readWorkspaceIdFromApiKey = (): string => { + const apiKey = process.env[WORKSPACE_API_KEY_ENV] ?? ''; + const payload = JSON.parse( + Buffer.from(apiKey.split('.')[1] ?? '', 'base64url').toString('utf8'), + ); + + return payload.workspaceId; + }; + + beforeAll(async () => { + client = new CoreApiClient(); + workspaceId = readWorkspaceIdFromApiKey(); + shareEverythingChannelId = await discoverShareEverythingChannelId(); + }); + + beforeEach(() => { + recall = new FakeRecallApi(); + + const realFetch = globalThis.fetch; + + vi.stubGlobal( + 'fetch', + (requestUrl: any, requestInit?: any): Promise => { + const intercepted = + typeof requestUrl === 'string' + ? recall.handle(requestUrl, requestInit) + : undefined; + + return intercepted !== undefined + ? Promise.resolve(intercepted) + : realFetch(requestUrl, requestInit); + }, + ); + vi.stubEnv('RECALL_API_KEY', 'recall-api-key'); + vi.stubEnv('RECALL_REGION', 'us-west-2'); + vi.stubEnv('CALL_RECORDER_USE_WORKSPACE_LOGO', 'false'); + vi.stubEnv('TWENTY_FUNCTIONS_URL', FUNCTIONS_URL); + // Logic functions normally run with an app access token; the workspace + // API key is a token with the same workspaceId claim. + vi.stubEnv( + 'TWENTY_APP_ACCESS_TOKEN', + process.env[WORKSPACE_API_KEY_ENV] ?? '', + ); + }); + + afterEach(async () => { + vi.unstubAllGlobals(); + vi.unstubAllEnvs(); + + await destroyCreatedRows(); + }); + + const destroyCreatedRows = async (): Promise => { + const callRecordingsForCreatedCalendarEvents = + createdCalendarEventIds.length === 0 + ? [] + : await findCallRecordings({ + calendarEventId: { in: createdCalendarEventIds }, + }); + const callRecordingIds = [ + ...new Set([ + ...createdCallRecordingIds, + ...callRecordingsForCreatedCalendarEvents.map(({ id }) => id), + ]), + ]; + + for (const callRecordingId of callRecordingIds) { + await client + .mutation({ + destroyCallRecording: { __args: { id: callRecordingId }, id: true }, + }) + .catch(() => {}); + } + + for (const associationId of createdAssociationIds) { + await workspaceGraphql( + `mutation ($id: UUID!) { + destroyCalendarChannelEventAssociation(id: $id) { id } + }`, + { id: associationId }, + ).catch(() => {}); + } + + for (const calendarEventId of createdCalendarEventIds) { + await client + .mutation({ + destroyCalendarEvent: { __args: { id: calendarEventId }, id: true }, + }) + .catch(() => {}); + } + + createdCalendarEventIds.length = 0; + createdCallRecordingIds.length = 0; + createdAssociationIds.length = 0; + }; + + const createCalendarEvent = async ( + overrides: Record = {}, + ): Promise => { + const calendarEventId = randomUUID(); + + await client.mutation({ + createCalendarEvent: { + __args: { + data: { + id: calendarEventId, + title: 'Customer Sync (call recorder integration test)', + startsAt: inOneHour(), + endsAt: inTwoHours(), + iCalUid: `call-recorder-test-${calendarEventId}`, + conferenceLink: { + primaryLinkUrl: `https://meet.example.com/${calendarEventId}`, + }, + callRecorderPreference: 'ON', + ...overrides, + }, + }, + id: true, + }, + }); + createdCalendarEventIds.push(calendarEventId); + + // Without a SHARE_EVERYTHING channel association the event would be + // invisible to every query, like an event no calendar sync produced. + const associationData = await workspaceGraphql( + `mutation ($data: CalendarChannelEventAssociationCreateInput!) { + createCalendarChannelEventAssociation(data: $data) { id } + }`, + { + data: { + calendarChannelId: shareEverythingChannelId, + calendarEventId, + eventExternalId: `call-recorder-test-${calendarEventId}`, + }, + }, + ); + + createdAssociationIds.push( + associationData.createCalendarChannelEventAssociation.id, + ); + + return calendarEventId; + }; + + const createPendingCallRecording = async ({ + calendarEventId, + ...overrides + }: Record & { calendarEventId: string }): Promise => { + const callRecordingId = randomUUID(); + + await client.mutation({ + createCallRecording: { + __args: { + data: { + id: callRecordingId, + title: 'Customer Sync (call recorder integration test)', + status: 'SCHEDULED', + recordingRequestStatus: 'REQUESTED', + calendarEventId, + ...overrides, + }, + }, + id: true, + }, + }); + createdCallRecordingIds.push(callRecordingId); + + return callRecordingId; + }; + + const findCallRecordings = async ( + filter: Record, + ): Promise>> => { + const result = await client.query({ + callRecordings: { + __args: { filter, first: 50 }, + edges: { + node: { + id: true, + status: true, + recordingRequestStatus: true, + calendarEventId: true, + externalBotId: true, + externalRecordingId: true, + botScheduleAttemptedAt: true, + botScheduleIdempotencyKey: true, + callRecorderFailureReason: true, + startedAt: true, + endedAt: true, + }, + }, + }, + }); + + return (result.callRecordings?.edges ?? []).map( + (edge: any) => edge.node, + ); + }; + + const fetchCallRecording = async ( + callRecordingId: string, + ): Promise> => { + const callRecording = ( + await findCallRecordings({ id: { eq: callRecordingId } }) + )[0]; + + expect(callRecording).toBeDefined(); + + return callRecording; + }; + + // Mocked database-event trigger: runs the reconciliation the + // calendarEvent.* trigger dispatches, then returns the recording it wrote + // to the DB together with its live Recall bot. + const scheduleRecordingThroughCalendarReconciliation = async (): Promise<{ + calendarEventId: string; + callRecordingId: string; + botId: string; + metadata: Record; + }> => { + const calendarEventId = await createCalendarEvent(); + + await reconcileCallRecorderForCalendarEventIds({ + client, + calendarEventIds: [calendarEventId], + }); + + const callRecording = ( + await findCallRecordings({ calendarEventId: { in: [calendarEventId] } }) + )[0]; + + expect(callRecording).toBeDefined(); + expect(callRecording.externalBotId).toBeTruthy(); + + return { + calendarEventId, + callRecordingId: callRecording.id, + botId: callRecording.externalBotId, + metadata: buildBotMetadata(callRecording.id, workspaceId), + }; + }; + + // Mocked webhook trigger: invokes the webhook logic function handler with + // the payload Recall would have delivered. + const deliverRecallWebhook = (body: object) => + processRecallWebhookHandler(body); + + // Mocked cron trigger: runs the flows the recovery cron dispatches. + const runPendingRecoveryCron = () => + scheduleRecallBotsForPendingCallRecordings({ client, now: new Date() }); + const runCancellationRetryCron = () => + retryFailedRecallCancellations({ client, now: new Date() }); + + describe('scheduling from calendar changes', () => { + it('creates a recording and schedules a Recall bot for a meeting with recording enabled', async () => { + const { callRecordingId, botId } = + await scheduleRecordingThroughCalendarReconciliation(); + + const callRecording = await fetchCallRecording(callRecordingId); + + expect(callRecording.status).toBe('SCHEDULED'); + expect(callRecording.recordingRequestStatus).toBe('REQUESTED'); + expect(callRecording.botScheduleAttemptedAt).toBeTruthy(); + expect(callRecording.botScheduleIdempotencyKey).toBeTruthy(); + expect(recall.bots.get(botId)?.metadata).toEqual( + buildBotMetadata(callRecordingId, workspaceId), + ); + }); + + it('creates nothing for a meeting without a conference link', async () => { + const calendarEventId = await createCalendarEvent({ + conferenceLink: { primaryLinkUrl: '' }, + }); + + await reconcileCallRecorderForCalendarEventIds({ + client, + calendarEventIds: [calendarEventId], + }); + + expect( + await findCallRecordings({ calendarEventId: { in: [calendarEventId] } }), + ).toEqual([]); + }); + }); + + describe('Recall webhook lifecycle', () => { + it('moves the recording through joining, recording, and processing', async () => { + const { callRecordingId, botId, metadata } = + await scheduleRecordingThroughCalendarReconciliation(); + + await deliverRecallWebhook( + buildBotStatusChangeWebhook({ + botId, + metadata, + statusCode: 'joining_call', + }), + ); + expect((await fetchCallRecording(callRecordingId)).status).toBe( + 'JOINING', + ); + + const recordingStartedAt = new Date().toISOString(); + + await deliverRecallWebhook( + buildBotStatusChangeWebhook({ + botId, + metadata, + statusCode: 'in_call_recording', + statusTimestamp: recordingStartedAt, + }), + ); + const recordingCallRecording = await fetchCallRecording(callRecordingId); + + expect(recordingCallRecording.status).toBe('RECORDING'); + expect(recordingCallRecording.startedAt).toBeTruthy(); + + const recordingEndedAt = new Date().toISOString(); + + await deliverRecallWebhook( + buildRecordingDoneWebhook({ + botId, + metadata, + startedAt: recordingStartedAt, + completedAt: recordingEndedAt, + }), + ); + const processedCallRecording = await fetchCallRecording(callRecordingId); + + expect(processedCallRecording.status).toBe('PROCESSING'); + expect(processedCallRecording.externalRecordingId).toBe( + 'recall-recording-1', + ); + expect(processedCallRecording.endedAt).toBeTruthy(); + // recording.done hands media and transcript work to the artifact + // import route. + expect(recall.artifactImportRequests).toHaveLength(1); + expect(recall.artifactImportRequests[0]).toMatchObject({ + callRecordingId, + }); + }); + + it('queues another artifact import when the transcript finishes later', async () => { + const { callRecordingId, botId, metadata } = + await scheduleRecordingThroughCalendarReconciliation(); + + await deliverRecallWebhook( + buildRecordingDoneWebhook({ + botId, + metadata, + startedAt: new Date().toISOString(), + completedAt: new Date().toISOString(), + }), + ); + await deliverRecallWebhook( + buildTranscriptDoneWebhook({ botId, metadata }), + ); + + expect(recall.artifactImportRequests).toHaveLength(2); + expect(recall.artifactImportRequests[1]).toMatchObject({ + callRecordingId, + }); + }); + + it('never moves the status backwards on late webhook deliveries', async () => { + const { callRecordingId, botId, metadata } = + await scheduleRecordingThroughCalendarReconciliation(); + + await deliverRecallWebhook( + buildRecordingDoneWebhook({ + botId, + metadata, + startedAt: new Date().toISOString(), + completedAt: new Date().toISOString(), + }), + ); + + const lateJoiningResult = await deliverRecallWebhook( + buildBotStatusChangeWebhook({ + botId, + metadata, + statusCode: 'joining_call', + }), + ); + + expect(lateJoiningResult.status).toBe('skipped'); + expect((await fetchCallRecording(callRecordingId)).status).toBe( + 'PROCESSING', + ); + }); + + it('marks the recording failed when the bot dies fatally', async () => { + const { callRecordingId, botId, metadata } = + await scheduleRecordingThroughCalendarReconciliation(); + + await deliverRecallWebhook( + buildBotStatusChangeWebhook({ botId, metadata, statusCode: 'fatal' }), + ); + + const callRecording = await fetchCallRecording(callRecordingId); + + expect(callRecording.status).toBe('FAILED'); + expect(callRecording.callRecorderFailureReason).toBe('fatal'); + }); + + it('ignores webhooks that match no known recording', async () => { + const { callRecordingId } = + await scheduleRecordingThroughCalendarReconciliation(); + + const result = await deliverRecallWebhook( + buildBotStatusChangeWebhook({ + botId: 'recall-bot-from-another-app', + metadata: buildBotMetadata(randomUUID(), workspaceId), + statusCode: 'joining_call', + }), + ); + + expect(result.status).toBe('skipped'); + expect((await fetchCallRecording(callRecordingId)).status).toBe( + 'SCHEDULED', + ); + }); + }); + + describe('cancellation', () => { + it('cancels the request and deletes the Recall bot', async () => { + const { callRecordingId, botId } = + await scheduleRecordingThroughCalendarReconciliation(); + + await cancelCallRecordingRequest({ + client, + callRecording: { id: callRecordingId, externalBotId: botId }, + }); + + const callRecording = await fetchCallRecording(callRecordingId); + + expect(callRecording.recordingRequestStatus).toBe('CANCELED'); + // The API stores a cleared TEXT field as an empty string. + expect(callRecording.externalBotId).toBeFalsy(); + expect(recall.deletedBotIds).toEqual([botId]); + }); + + it('retries a failed Recall cancellation on the next cron run', async () => { + const { callRecordingId, botId } = + await scheduleRecordingThroughCalendarReconciliation(); + + recall.failNextDelete = true; + await cancelCallRecordingRequest({ + client, + callRecording: { id: callRecordingId, externalBotId: botId }, + }); + + // The Recall half failed, so the bot id must survive for the retry. + expect((await fetchCallRecording(callRecordingId)).externalBotId).toBe( + botId, + ); + + await runCancellationRetryCron(); + + expect( + (await fetchCallRecording(callRecordingId)).externalBotId, + ).toBeFalsy(); + expect(recall.deletedBotIds).toContain(botId); + }); + }); + + describe('crash recovery cron', () => { + it('schedules a bot for a recording created without one, with zero Recall list reads', async () => { + const calendarEventId = await createCalendarEvent(); + const callRecordingId = await createPendingCallRecording({ + calendarEventId, + }); + + await runPendingRecoveryCron(); + + const callRecording = await fetchCallRecording(callRecordingId); + + expect(callRecording.externalBotId).toBeTruthy(); + expect(recall.botForCallRecording(callRecordingId)?.id).toBe( + callRecording.externalBotId, + ); + expect(recall.listRequestCount).toBe(0); + }); + + it('re-sends the creation after a lost write-back and lands on the same bot', async () => { + const calendarEventId = await createCalendarEvent(); + const callRecordingId = await createPendingCallRecording({ + calendarEventId, + }); + + // First recovery run creates the bot and records the attempt. + await runPendingRecoveryCron(); + const firstBotId = (await fetchCallRecording(callRecordingId)) + .externalBotId; + + // Simulate the id write-back getting lost after the POST reached + // Recall. + await client.mutation({ + updateCallRecording: { + __args: { id: callRecordingId, data: { externalBotId: null } }, + id: true, + }, + }); + + await runPendingRecoveryCron(); + + // Recall dedupes the repeated idempotency key: the very same bot is + // written back, without any list request. + expect((await fetchCallRecording(callRecordingId)).externalBotId).toBe( + firstBotId, + ); + expect(recall.listRequestCount).toBe(0); + }); + + it('attaches an existing bot found by lookup when the recorded attempt drifted', async () => { + const calendarEventId = await createCalendarEvent(); + const callRecordingId = await createPendingCallRecording({ + calendarEventId, + botScheduleAttemptedAt: hoursAgo(1), + botScheduleIdempotencyKey: 'key-from-before-the-meeting-moved', + }); + + recall.seedBot({ + id: 'recall-bot-from-crashed-run', + metadata: buildBotMetadata(callRecordingId, workspaceId), + statusCode: 'ready', + }); + + await runPendingRecoveryCron(); + + expect((await fetchCallRecording(callRecordingId)).externalBotId).toBe( + 'recall-bot-from-crashed-run', + ); + expect(recall.listRequestCount).toBe(1); + }); + + it('fails a recording whose meeting ended before any bot creation was attempted', async () => { + const calendarEventId = await createCalendarEvent({ + startsAt: hoursAgo(3), + endsAt: hoursAgo(2), + }); + const callRecordingId = await createPendingCallRecording({ + calendarEventId, + }); + + await runPendingRecoveryCron(); + + const callRecording = await fetchCallRecording(callRecordingId); + + expect(callRecording.status).toBe('FAILED'); + expect(callRecording.callRecorderFailureReason).toBe( + 'bot_never_scheduled', + ); + expect(recall.botForCallRecording(callRecordingId)).toBeUndefined(); + }); + }); +}); diff --git a/packages/twenty-apps/public/call-recorder/src/constants/bot-schedule-attempted-at-field-universal-identifier.ts b/packages/twenty-apps/public/call-recorder/src/constants/bot-schedule-attempted-at-field-universal-identifier.ts new file mode 100644 index 0000000000..0fc768d532 --- /dev/null +++ b/packages/twenty-apps/public/call-recorder/src/constants/bot-schedule-attempted-at-field-universal-identifier.ts @@ -0,0 +1,2 @@ +export const BOT_SCHEDULE_ATTEMPTED_AT_FIELD_UNIVERSAL_IDENTIFIER = + 'ca1d0179-f611-46bd-ba8a-b503ea9d024e'; diff --git a/packages/twenty-apps/public/call-recorder/src/constants/bot-schedule-idempotency-key-field-universal-identifier.ts b/packages/twenty-apps/public/call-recorder/src/constants/bot-schedule-idempotency-key-field-universal-identifier.ts new file mode 100644 index 0000000000..a1e694e897 --- /dev/null +++ b/packages/twenty-apps/public/call-recorder/src/constants/bot-schedule-idempotency-key-field-universal-identifier.ts @@ -0,0 +1,2 @@ +export const BOT_SCHEDULE_IDEMPOTENCY_KEY_FIELD_UNIVERSAL_IDENTIFIER = + '6f00ea4d-1c5a-47ac-8c18-baacf5ee7c3f'; diff --git a/packages/twenty-apps/public/call-recorder/src/constants/schedule-recall-bot-on-call-recording-update-logic-function-universal-identifier.ts b/packages/twenty-apps/public/call-recorder/src/constants/schedule-recall-bot-on-call-recording-update-logic-function-universal-identifier.ts new file mode 100644 index 0000000000..1ddd4102bd --- /dev/null +++ b/packages/twenty-apps/public/call-recorder/src/constants/schedule-recall-bot-on-call-recording-update-logic-function-universal-identifier.ts @@ -0,0 +1,2 @@ +export const SCHEDULE_RECALL_BOT_ON_CALL_RECORDING_UPDATE_LOGIC_FUNCTION_UNIVERSAL_IDENTIFIER = + 'fad4b1b5-8b6e-48c7-a8c7-3aecae2672c1'; diff --git a/packages/twenty-apps/public/call-recorder/src/fields/bot-schedule-attempted-at-on-call-recording.field.ts b/packages/twenty-apps/public/call-recorder/src/fields/bot-schedule-attempted-at-on-call-recording.field.ts new file mode 100644 index 0000000000..4966e968fd --- /dev/null +++ b/packages/twenty-apps/public/call-recorder/src/fields/bot-schedule-attempted-at-on-call-recording.field.ts @@ -0,0 +1,21 @@ +import { + defineField, + FieldType, + STANDARD_OBJECT_UNIVERSAL_IDENTIFIERS, +} from 'twenty-sdk/define'; + +import { BOT_SCHEDULE_ATTEMPTED_AT_FIELD_UNIVERSAL_IDENTIFIER } from 'src/constants/bot-schedule-attempted-at-field-universal-identifier'; + +export default defineField({ + universalIdentifier: BOT_SCHEDULE_ATTEMPTED_AT_FIELD_UNIVERSAL_IDENTIFIER, + objectUniversalIdentifier: + STANDARD_OBJECT_UNIVERSAL_IDENTIFIERS.callRecording.universalIdentifier, + type: FieldType.DATE_TIME, + name: 'botScheduleAttemptedAt', + label: 'Bot Schedule Attempted At', + description: + 'Set right before a Recall bot creation request is sent; recovery uses it to tell rows that never reached Recall from rows whose creation outcome is unknown.', + icon: 'IconClockPlay', + isNullable: true, + isUIEditable: false, +}); diff --git a/packages/twenty-apps/public/call-recorder/src/fields/bot-schedule-idempotency-key-on-call-recording.field.ts b/packages/twenty-apps/public/call-recorder/src/fields/bot-schedule-idempotency-key-on-call-recording.field.ts new file mode 100644 index 0000000000..8957d54a48 --- /dev/null +++ b/packages/twenty-apps/public/call-recorder/src/fields/bot-schedule-idempotency-key-on-call-recording.field.ts @@ -0,0 +1,21 @@ +import { + defineField, + FieldType, + STANDARD_OBJECT_UNIVERSAL_IDENTIFIERS, +} from 'twenty-sdk/define'; + +import { BOT_SCHEDULE_IDEMPOTENCY_KEY_FIELD_UNIVERSAL_IDENTIFIER } from 'src/constants/bot-schedule-idempotency-key-field-universal-identifier'; + +export default defineField({ + universalIdentifier: BOT_SCHEDULE_IDEMPOTENCY_KEY_FIELD_UNIVERSAL_IDENTIFIER, + objectUniversalIdentifier: + STANDARD_OBJECT_UNIVERSAL_IDENTIFIERS.callRecording.universalIdentifier, + type: FieldType.TEXT, + name: 'botScheduleIdempotencyKey', + label: 'Bot Schedule Idempotency Key', + description: + 'Idempotency key of the last Recall bot creation attempt; when the scheduling inputs still hash to this key, recovery can safely re-send the creation instead of listing bots.', + icon: 'IconKey', + isNullable: true, + isUIEditable: false, +}); diff --git a/packages/twenty-apps/public/call-recorder/src/logic-functions/__tests__/schedule-recall-bot-on-call-recording-update.test.ts b/packages/twenty-apps/public/call-recorder/src/logic-functions/__tests__/schedule-recall-bot-on-call-recording-update.test.ts new file mode 100644 index 0000000000..d124f07539 --- /dev/null +++ b/packages/twenty-apps/public/call-recorder/src/logic-functions/__tests__/schedule-recall-bot-on-call-recording-update.test.ts @@ -0,0 +1,309 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +import { scheduleRecallBotOnCallRecordingUpdateHandler } from 'src/logic-functions/schedule-recall-bot-on-call-recording-update'; + +const queryMock = vi.hoisted(() => vi.fn()); +const mutationMock = vi.hoisted(() => vi.fn()); + +vi.mock('twenty-client-sdk/core', () => ({ + CoreApiClient: class { + query = queryMock; + mutation = mutationMock; + }, +})); + +const fetchMock = vi.fn(); + +const NOW = new Date('2026-01-01T12:00:00.000Z'); +const WORKSPACE_ID = '123e4567-e89b-12d3-a456-426614174000'; +const RECALL_CREATE_BOT_URL = 'https://us-west-2.recall.ai/api/v1/bot/'; + +const buildAccessToken = (payload: Record): string => + [ + Buffer.from(JSON.stringify({ alg: 'none' })).toString('base64url'), + Buffer.from(JSON.stringify(payload)).toString('base64url'), + 'signature', + ].join('.'); + +const buildConnection = (nodes: Node[]) => ({ + pageInfo: { hasNextPage: false, endCursor: undefined }, + edges: nodes.map((node) => ({ node })), +}); + +type HandlerEvent = Parameters< + typeof scheduleRecallBotOnCallRecordingUpdateHandler +>[0]; + +const buildUpdateEvent = (overrides: Partial = {}): HandlerEvent => + ({ + name: 'callRecording.updated', + recordId: 'call-recording-1', + properties: { + updatedFields: ['externalBotId'], + before: { + id: 'call-recording-1', + status: 'SCHEDULED', + recordingRequestStatus: 'REQUESTED', + externalBotId: 'recall-bot-vanished', + }, + after: { + id: 'call-recording-1', + status: 'SCHEDULED', + recordingRequestStatus: 'REQUESTED', + externalBotId: null, + }, + }, + ...overrides, + }) as HandlerEvent; + +const stubPendingCallRecordingQueries = () => { + queryMock.mockImplementation(async (query: any) => { + if (query.callRecordings !== undefined) { + return { + callRecordings: buildConnection([ + { + id: 'call-recording-1', + status: 'SCHEDULED', + recordingRequestStatus: 'REQUESTED', + calendarEventId: 'calendar-event-1', + externalBotId: null, + botScheduleAttemptedAt: null, + }, + ]), + }; + } + + if (query.calendarEvents !== undefined) { + return { + calendarEvents: buildConnection([ + { + id: 'calendar-event-1', + startsAt: '2026-01-01T13:00:00.000Z', + endsAt: '2026-01-01T14:00:00.000Z', + iCalUid: 'calendar-event-uid', + conferenceLink: { + primaryLinkUrl: 'https://meet.example.com/customer-sync', + }, + }, + ]), + }; + } + + throw new Error(`Unhandled query: ${JSON.stringify(query)}`); + }); +}; + +describe('scheduleRecallBotOnCallRecordingUpdateHandler', () => { + beforeEach(() => { + vi.useFakeTimers(); + vi.setSystemTime(NOW); + vi.spyOn(console, 'warn').mockImplementation(() => {}); + vi.stubGlobal('fetch', fetchMock); + vi.stubEnv('RECALL_API_KEY', 'recall-api-key'); + vi.stubEnv('RECALL_REGION', 'us-west-2'); + vi.stubEnv('CALL_RECORDER_USE_WORKSPACE_LOGO', 'false'); + vi.stubEnv( + 'TWENTY_APP_ACCESS_TOKEN', + buildAccessToken({ workspaceId: WORKSPACE_ID }), + ); + queryMock.mockReset(); + mutationMock.mockReset(); + mutationMock.mockImplementation(async (mutation: any) => ({ + updateCallRecording: { id: mutation.updateCallRecording.__args.id }, + })); + fetchMock.mockReset(); + fetchMock.mockImplementation(async (requestUrl: string) => { + if (requestUrl === RECALL_CREATE_BOT_URL) { + return new Response(JSON.stringify({ id: 'recall-bot-new' }), { + status: 201, + }); + } + + throw new Error(`Unhandled fetch in test: ${requestUrl}`); + }); + }); + + afterEach(() => { + vi.unstubAllGlobals(); + vi.unstubAllEnvs(); + vi.useRealTimers(); + vi.restoreAllMocks(); + }); + + it('schedules a bot when an update clears the bot id of a requested recording', async () => { + stubPendingCallRecordingQueries(); + + const result = await scheduleRecallBotOnCallRecordingUpdateHandler( + buildUpdateEvent(), + ); + + expect(result).toEqual({ + callRecordingId: 'call-recording-1', + result: { status: 'scheduled' }, + }); + expect(fetchMock).toHaveBeenCalledTimes(1); + expect(fetchMock).toHaveBeenCalledWith( + RECALL_CREATE_BOT_URL, + expect.objectContaining({ method: 'POST' }), + ); + }); + + it('skips events that are not call recording updates', async () => { + const result = await scheduleRecallBotOnCallRecordingUpdateHandler( + buildUpdateEvent({ name: 'callRecording.created' } as HandlerEvent), + ); + + expect(result).toEqual({ + skipped: true, + reason: 'not a call recording update', + }); + expect(queryMock).not.toHaveBeenCalled(); + expect(fetchMock).not.toHaveBeenCalled(); + }); + + it('skips its own scheduling-progress writes so it cannot re-trigger itself', async () => { + const result = await scheduleRecallBotOnCallRecordingUpdateHandler( + buildUpdateEvent({ + properties: { + updatedFields: [ + 'botScheduleAttemptedAt', + 'botScheduleIdempotencyKey', + ], + before: {}, + after: { + id: 'call-recording-1', + status: 'SCHEDULED', + recordingRequestStatus: 'REQUESTED', + externalBotId: null, + }, + }, + } as unknown as HandlerEvent), + ); + + expect(result).toEqual({ + skipped: true, + reason: 'no pending-transition field changed', + }); + expect(queryMock).not.toHaveBeenCalled(); + expect(fetchMock).not.toHaveBeenCalled(); + }); + + it('skips updates that leave the recording with a bot attached', async () => { + const result = await scheduleRecallBotOnCallRecordingUpdateHandler( + buildUpdateEvent({ + properties: { + updatedFields: ['externalBotId'], + before: {}, + after: { + id: 'call-recording-1', + status: 'SCHEDULED', + recordingRequestStatus: 'REQUESTED', + externalBotId: 'recall-bot-1', + }, + }, + } as unknown as HandlerEvent), + ); + + expect(result).toEqual({ + skipped: true, + reason: 'call recording is not pending', + }); + expect(queryMock).not.toHaveBeenCalled(); + expect(fetchMock).not.toHaveBeenCalled(); + }); + + it('defers rows with an ambiguous prior attempt to the recovery cron instead of listing bots', async () => { + queryMock.mockImplementationOnce(async () => ({ + callRecordings: buildConnection([ + { + id: 'call-recording-1', + status: 'SCHEDULED', + recordingRequestStatus: 'REQUESTED', + calendarEventId: 'calendar-event-1', + externalBotId: null, + botScheduleAttemptedAt: '2026-01-01T11:55:00.000Z', + botScheduleIdempotencyKey: 'stale-key-from-moved-meeting', + }, + ]), + })); + queryMock.mockImplementationOnce(async () => ({ + calendarEvents: buildConnection([ + { + id: 'calendar-event-1', + startsAt: '2026-01-01T13:00:00.000Z', + endsAt: '2026-01-01T14:00:00.000Z', + conferenceLink: { + primaryLinkUrl: 'https://meet.example.com/customer-sync', + }, + }, + ]), + })); + + const result = await scheduleRecallBotOnCallRecordingUpdateHandler( + buildUpdateEvent(), + ); + + expect(result).toEqual({ + callRecordingId: 'call-recording-1', + result: { + status: 'deferred', + reason: 'ambiguous prior attempt; the recovery cron will reconcile it', + }, + }); + expect(fetchMock).not.toHaveBeenCalled(); + }); + + it('skips slim payloads whose diff shows the bot id was written back', async () => { + const result = await scheduleRecallBotOnCallRecordingUpdateHandler( + buildUpdateEvent({ + properties: { + updatedFields: ['externalBotId'], + diff: { + externalBotId: { before: null, after: 'recall-bot-1' }, + }, + }, + } as unknown as HandlerEvent), + ); + + expect(result).toEqual({ + skipped: true, + reason: 'call recording is not pending', + }); + expect(queryMock).not.toHaveBeenCalled(); + expect(fetchMock).not.toHaveBeenCalled(); + }); + + it('does not schedule anything when the meeting already ended', async () => { + stubPendingCallRecordingQueries(); + queryMock.mockImplementationOnce(async () => ({ + callRecordings: buildConnection([ + { + id: 'call-recording-1', + status: 'SCHEDULED', + recordingRequestStatus: 'REQUESTED', + calendarEventId: 'calendar-event-1', + externalBotId: null, + }, + ]), + })); + queryMock.mockImplementationOnce(async () => ({ + calendarEvents: buildConnection([ + { + id: 'calendar-event-1', + startsAt: '2026-01-01T10:00:00.000Z', + endsAt: '2026-01-01T11:00:00.000Z', + }, + ]), + })); + + const result = await scheduleRecallBotOnCallRecordingUpdateHandler( + buildUpdateEvent(), + ); + + expect(result).toEqual({ + callRecordingId: 'call-recording-1', + result: { status: 'skipped', reason: 'meeting already ended' }, + }); + expect(fetchMock).not.toHaveBeenCalled(); + }); +}); diff --git a/packages/twenty-apps/public/call-recorder/src/logic-functions/constants/active-recall-bot-statuses.ts b/packages/twenty-apps/public/call-recorder/src/logic-functions/constants/active-recall-bot-statuses.ts new file mode 100644 index 0000000000..d0c5c25dd8 --- /dev/null +++ b/packages/twenty-apps/public/call-recorder/src/logic-functions/constants/active-recall-bot-statuses.ts @@ -0,0 +1,9 @@ +export const ACTIVE_RECALL_BOT_STATUSES = [ + 'ready', + 'joining_call', + 'in_waiting_room', + 'in_call_not_recording', + 'recording_permission_allowed', + 'recording_permission_denied', + 'in_call_recording', +]; diff --git a/packages/twenty-apps/public/call-recorder/src/logic-functions/constants/pending-call-recording-requests-cron-pattern.ts b/packages/twenty-apps/public/call-recorder/src/logic-functions/constants/pending-call-recording-requests-cron-pattern.ts index 0e6bdcafa7..95018c8c6b 100644 --- a/packages/twenty-apps/public/call-recorder/src/logic-functions/constants/pending-call-recording-requests-cron-pattern.ts +++ b/packages/twenty-apps/public/call-recorder/src/logic-functions/constants/pending-call-recording-requests-cron-pattern.ts @@ -1 +1,3 @@ -export const PENDING_CALL_RECORDING_REQUESTS_CRON_PATTERN = '*/5 * * * *'; +// The callRecording.updated trigger resumes most stuck rows within seconds; +// this cron is the backstop for crashed creations and missed events. +export const PENDING_CALL_RECORDING_REQUESTS_CRON_PATTERN = '*/15 * * * *'; diff --git a/packages/twenty-apps/public/call-recorder/src/logic-functions/data/find-call-recordings-by-filter.util.ts b/packages/twenty-apps/public/call-recorder/src/logic-functions/data/find-call-recordings-by-filter.util.ts index 799b2a75cb..a44e14c313 100644 --- a/packages/twenty-apps/public/call-recorder/src/logic-functions/data/find-call-recordings-by-filter.util.ts +++ b/packages/twenty-apps/public/call-recorder/src/logic-functions/data/find-call-recordings-by-filter.util.ts @@ -21,6 +21,8 @@ type CallRecordingNode = { endedAt?: string | null; calendarEventId?: string | null; externalBotId?: string | null; + botScheduleAttemptedAt?: string | null; + botScheduleIdempotencyKey?: string | null; externalRecordingId?: string | null; callRecorderFailureReason?: string | null; }; @@ -54,6 +56,8 @@ export const findCallRecordingsByFilter = async ( endedAt: true, calendarEventId: true, externalBotId: true, + botScheduleAttemptedAt: true, + botScheduleIdempotencyKey: true, externalRecordingId: true, callRecorderFailureReason: true, }, @@ -80,6 +84,12 @@ export const findCallRecordingsByFilter = async ( endedAt: callRecording.endedAt ?? undefined, calendarEventId: callRecording.calendarEventId ?? undefined, externalBotId: normalizeOptionalString(callRecording.externalBotId), + botScheduleAttemptedAt: normalizeOptionalString( + callRecording.botScheduleAttemptedAt, + ), + botScheduleIdempotencyKey: normalizeOptionalString( + callRecording.botScheduleIdempotencyKey, + ), externalRecordingId: normalizeOptionalString( callRecording.externalRecordingId, ), diff --git a/packages/twenty-apps/public/call-recorder/src/logic-functions/domain/can-reschedule-call-recording-without-recall-lookup.util.ts b/packages/twenty-apps/public/call-recorder/src/logic-functions/domain/can-reschedule-call-recording-without-recall-lookup.util.ts new file mode 100644 index 0000000000..989fa64668 --- /dev/null +++ b/packages/twenty-apps/public/call-recorder/src/logic-functions/domain/can-reschedule-call-recording-without-recall-lookup.util.ts @@ -0,0 +1,54 @@ +import { isUndefined } from '@sniptt/guards'; + +import { type CalendarEventRecord } from 'src/logic-functions/types/calendar-event-record.type'; +import { type CallRecordingRecord } from 'src/logic-functions/types/call-recording-record.type'; +import { hasUnchangedBotScheduleIdempotencyKey } from 'src/logic-functions/domain/has-unchanged-bot-schedule-idempotency-key.util'; + +// Recall does not document how long idempotency keys are retained (24h is +// the industry minimum), so re-sends are only trusted while the recorded +// attempt is clearly fresh; a stale key would create a twin bot instead of +// deduping. Recovery normally runs within minutes of the attempt. +const IDEMPOTENT_RESEND_WINDOW_HOURS = 12; + +// Rows without a schedule-attempt marker never reached Recall, so no bot can +// exist for them. Rows whose stored idempotency key still matches the current +// scheduling inputs can re-send the creation and let Recall dedupe it. +export const canRescheduleCallRecordingWithoutRecallLookup = ({ + callRecording, + calendarEvent, + workspaceId, + now, +}: { + callRecording: CallRecordingRecord; + calendarEvent: CalendarEventRecord; + workspaceId: string | undefined; + now: Date; +}): boolean => + isUndefined(callRecording.botScheduleAttemptedAt) || + (isWithinIdempotentResendWindow(callRecording.botScheduleAttemptedAt, now) && + !isUndefined(workspaceId) && + hasUnchangedBotScheduleIdempotencyKey({ + callRecording, + calendarEvent, + workspaceId, + })); + +const isWithinIdempotentResendWindow = ( + botScheduleAttemptedAt: string, + now: Date, +): boolean => { + const attemptedTime = new Date(botScheduleAttemptedAt).getTime(); + + if (Number.isNaN(attemptedTime)) { + return false; + } + + const elapsedMilliseconds = now.getTime() - attemptedTime; + + // A future timestamp means clock skew or corrupt data; treat it as + // untrustworthy rather than fresh. + return ( + elapsedMilliseconds >= 0 && + elapsedMilliseconds < IDEMPOTENT_RESEND_WINDOW_HOURS * 60 * 60 * 1000 + ); +}; diff --git a/packages/twenty-apps/public/call-recorder/src/logic-functions/domain/has-unchanged-bot-schedule-idempotency-key.util.ts b/packages/twenty-apps/public/call-recorder/src/logic-functions/domain/has-unchanged-bot-schedule-idempotency-key.util.ts new file mode 100644 index 0000000000..0314cb973a --- /dev/null +++ b/packages/twenty-apps/public/call-recorder/src/logic-functions/domain/has-unchanged-bot-schedule-idempotency-key.util.ts @@ -0,0 +1,45 @@ +import { isUndefined } from '@sniptt/guards'; + +import { type CalendarEventRecord } from 'src/logic-functions/types/calendar-event-record.type'; +import { type CallRecordingRecord } from 'src/logic-functions/types/call-recording-record.type'; +import { buildRecallRoutingMetadata } from 'src/logic-functions/domain/build-recall-routing-metadata.util'; +import { computeRecallBotJoinAt } from 'src/logic-functions/domain/compute-recall-bot-join-at.util'; +import { computeRecallBotCreationIdempotencyKey } from 'src/logic-functions/recall-api/schedule-recall-bot.util'; + +// True when re-sending the bot creation would carry the same idempotency key +// as the recorded attempt, so Recall dedupes it instead of creating a twin. +// A moved meeting, changed conference link, or changed join-early setting +// drifts the key and recovery must fall back to a bot lookup. +export const hasUnchangedBotScheduleIdempotencyKey = ({ + callRecording, + calendarEvent, + workspaceId, +}: { + callRecording: CallRecordingRecord; + calendarEvent: CalendarEventRecord; + workspaceId: string; +}): boolean => { + const storedIdempotencyKey = callRecording.botScheduleIdempotencyKey; + const meetingUrl = calendarEvent.conferenceLinkUrl; + const meetingStartsAt = calendarEvent.startsAt; + + if ( + isUndefined(storedIdempotencyKey) || + isUndefined(meetingUrl) || + isUndefined(meetingStartsAt) + ) { + return false; + } + + return ( + storedIdempotencyKey === + computeRecallBotCreationIdempotencyKey({ + meetingUrl, + joinAt: computeRecallBotJoinAt(meetingStartsAt), + metadata: buildRecallRoutingMetadata({ + callRecordingId: callRecording.id, + workspaceId, + }), + }) + ); +}; diff --git a/packages/twenty-apps/public/call-recorder/src/logic-functions/flows/__tests__/reconcile-call-recorder.test.ts b/packages/twenty-apps/public/call-recorder/src/logic-functions/flows/__tests__/reconcile-call-recorder.test.ts index ecdd91004a..a07a6e848f 100644 --- a/packages/twenty-apps/public/call-recorder/src/logic-functions/flows/__tests__/reconcile-call-recorder.test.ts +++ b/packages/twenty-apps/public/call-recorder/src/logic-functions/flows/__tests__/reconcile-call-recorder.test.ts @@ -281,6 +281,8 @@ describe('reconcileCallRecorderForCalendarEventIds', () => { recordingRequestStatus: 'REQUESTED', calendarEventId: 'calendar-event-1', externalBotId: 'recall-bot-1', + botScheduleAttemptedAt: NOW.toISOString(), + botScheduleIdempotencyKey: expect.any(String), }, ]); expect(recallBotCreateCalls()).toHaveLength(1); diff --git a/packages/twenty-apps/public/call-recorder/src/logic-functions/flows/__tests__/retry-failed-recall-cancellations.test.ts b/packages/twenty-apps/public/call-recorder/src/logic-functions/flows/__tests__/retry-failed-recall-cancellations.test.ts index 4eb0ad7edf..04bf0f2623 100644 --- a/packages/twenty-apps/public/call-recorder/src/logic-functions/flows/__tests__/retry-failed-recall-cancellations.test.ts +++ b/packages/twenty-apps/public/call-recorder/src/logic-functions/flows/__tests__/retry-failed-recall-cancellations.test.ts @@ -304,8 +304,8 @@ describe('retryFailedRecallCancellations', () => { expect(listRequestParameters.get('metadata__twentyWorkspaceId')).toBe( WORKSPACE_ID, ); - expect(listRequestParameters.get('metadata__twentyCallRecordingId')).toBe( - 'call-recording-1', + expect(listRequestParameters.has('metadata__twentyCallRecordingId')).toBe( + false, ); expect(fetchMock).toHaveBeenCalledWith( `${BASE_URL}/bot/recall-bot-recovered/`, @@ -338,7 +338,15 @@ describe('retryFailedRecallCancellations', () => { ...buildJsonResponse(200), json: async () => ({ next: null, - results: [{ id: 'recall-bot-recovered' }], + results: [ + { + id: 'recall-bot-recovered', + metadata: { + twentyWorkspaceId: WORKSPACE_ID, + twentyCallRecordingId: 'call-recording-1', + }, + }, + ], }), }) .mockResolvedValueOnce(buildJsonResponse(400)) @@ -422,7 +430,15 @@ describe('retryFailedRecallCancellations', () => { ...buildJsonResponse(200), json: async () => ({ next: null, - results: [{ id: 'recall-bot-recovered' }], + results: [ + { + id: 'recall-bot-recovered', + metadata: { + twentyWorkspaceId: WORKSPACE_ID, + twentyCallRecordingId: 'call-recording-1', + }, + }, + ], }), }; } @@ -467,7 +483,15 @@ describe('retryFailedRecallCancellations', () => { ...buildJsonResponse(200), json: async () => ({ next: null, - results: [{ id: 'recall-bot-recovered' }], + results: [ + { + id: 'recall-bot-recovered', + metadata: { + twentyWorkspaceId: WORKSPACE_ID, + twentyCallRecordingId: 'call-recording-1', + }, + }, + ], }), }; } @@ -490,6 +514,74 @@ describe('retryFailedRecallCancellations', () => { ]); }); + it('does not claim a listed bot for a cancellation whose recovery window elapsed', async () => { + const client = new FakeCoreApiClient([ + { + id: 'call-recording-recent', + recordingRequestStatus: 'CANCELED', + status: 'SCHEDULED', + createdAt: '2026-01-02T11:30:00.000Z', + calendarEventId: null, + externalBotId: null, + }, + { + id: 'call-recording-aged', + recordingRequestStatus: 'CANCELED', + status: 'SCHEDULED', + createdAt: '2026-01-01T11:00:00.000Z', + calendarEventId: null, + externalBotId: null, + }, + ]); + fetchMock.mockImplementation( + async (requestUrl: string, requestInit?: { method?: string }) => { + if (requestInit?.method === 'DELETE') { + return buildJsonResponse(204); + } + + if (requestUrl.startsWith(`${BASE_URL}/bot/?`)) { + return { + ...buildJsonResponse(200), + json: async () => ({ + next: null, + results: [ + { + id: 'recall-bot-recent', + metadata: { + twentyWorkspaceId: WORKSPACE_ID, + twentyCallRecordingId: 'call-recording-recent', + }, + }, + { + id: 'recall-bot-aged', + metadata: { + twentyWorkspaceId: WORKSPACE_ID, + twentyCallRecordingId: 'call-recording-aged', + }, + }, + ], + }), + }; + } + + throw new Error(`Unhandled fetch: ${requestUrl}`); + }, + ); + + const result = await retryFailedRecallCancellations({ + client: client as unknown as CoreApiClient, + now: new Date('2026-01-02T12:00:00.000Z'), + }); + + expect(result.canceledExternalBotCallRecordingIds).toEqual([ + 'call-recording-recent', + ]); + expect(fetchMock).not.toHaveBeenCalledWith( + `${BASE_URL}/bot/recall-bot-aged/`, + expect.objectContaining({ method: 'DELETE' }), + ); + }); + it('does not repeatedly look up botless cancellations after their meeting ended', async () => { const client = new FakeCoreApiClient( [ diff --git a/packages/twenty-apps/public/call-recorder/src/logic-functions/flows/__tests__/schedule-recall-bots-for-pending-call-recordings.test.ts b/packages/twenty-apps/public/call-recorder/src/logic-functions/flows/__tests__/schedule-recall-bots-for-pending-call-recordings.test.ts index a21626bbdb..86a57da587 100644 --- a/packages/twenty-apps/public/call-recorder/src/logic-functions/flows/__tests__/schedule-recall-bots-for-pending-call-recordings.test.ts +++ b/packages/twenty-apps/public/call-recorder/src/logic-functions/flows/__tests__/schedule-recall-bots-for-pending-call-recordings.test.ts @@ -1,7 +1,9 @@ import { type CoreApiClient } from 'twenty-client-sdk/core'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { computeRecallBotJoinAt } from 'src/logic-functions/domain/compute-recall-bot-join-at.util'; import { scheduleRecallBotsForPendingCallRecordings } from 'src/logic-functions/flows/schedule-recall-bots-for-pending-call-recordings.util'; +import { computeRecallBotCreationIdempotencyKey } from 'src/logic-functions/recall-api/schedule-recall-bot.util'; const NOW = new Date('2026-01-01T12:00:00.000Z'); const WORKSPACE_ID = '123e4567-e89b-12d3-a456-426614174000'; @@ -28,6 +30,9 @@ type CallRecordingNode = { recordingRequestStatus?: string | null; calendarEventId?: string | null; externalBotId?: string | null; + botScheduleAttemptedAt?: string | null; + botScheduleIdempotencyKey?: string | null; + callRecorderFailureReason?: string | null; }; type CalendarEventNode = { @@ -216,6 +221,11 @@ describe('scheduleRecallBotsForPendingCallRecordings', () => { expect(result.scheduledCallRecordingIds).toEqual(['call-recording-1']); expect(result.attachedCallRecordingIds).toEqual([]); + // A row that never attempted a bot creation needs no Recall lookup. + expect(listBotRequestUrls()).toHaveLength(0); + expect(client.callRecordings[0].botScheduleAttemptedAt).toBe( + NOW.toISOString(), + ); expect(createBotCalls()).toHaveLength(1); const [requestUrl, requestInit] = createBotCalls()[0]; expect(requestUrl).toBe(RECALL_CREATE_BOT_URL); @@ -246,7 +256,11 @@ describe('scheduleRecallBotsForPendingCallRecordings', () => { ], }); const client = new FakeCoreApiClient({ - callRecordings: [buildPendingCallRecording()], + callRecordings: [ + buildPendingCallRecording({ + botScheduleAttemptedAt: '2026-01-01T11:55:00.000Z', + }), + ], calendarEvents: [buildCalendarEvent()], }); @@ -262,8 +276,8 @@ describe('scheduleRecallBotsForPendingCallRecordings', () => { expect(lookupParameters.get('metadata__twentyWorkspaceId')).toBe( WORKSPACE_ID, ); - expect(lookupParameters.get('metadata__twentyCallRecordingId')).toBe( - 'call-recording-1', + expect(lookupParameters.has('metadata__twentyCallRecordingId')).toBe( + false, ); expect(lookupParameters.has('join_at_after')).toBe(false); expect(lookupParameters.has('join_at_before')).toBe(false); @@ -279,10 +293,54 @@ describe('scheduleRecallBotsForPendingCallRecordings', () => { expect(client.callRecordings[0].externalBotId).toBe('recall-bot-existing'); }); + it('looks up existing bots once for the whole run instead of per recording', async () => { + stubRecallApi({ + listedBots: [ + { + id: 'recall-bot-existing', + metadata: { + twentyWorkspaceId: WORKSPACE_ID, + twentyCallRecordingId: 'call-recording-1', + }, + }, + ], + }); + const client = new FakeCoreApiClient({ + callRecordings: [ + buildPendingCallRecording({ + botScheduleAttemptedAt: '2026-01-01T11:55:00.000Z', + }), + buildPendingCallRecording({ + id: 'call-recording-2', + calendarEventId: 'calendar-event-2', + botScheduleAttemptedAt: '2026-01-01T11:55:00.000Z', + }), + ], + calendarEvents: [ + buildCalendarEvent(), + buildCalendarEvent({ id: 'calendar-event-2' }), + ], + }); + + const result = await scheduleRecallBotsForPendingCallRecordings({ + client: client as unknown as CoreApiClient, + now: NOW, + }); + + expect(listBotRequestUrls()).toHaveLength(1); + expect(result.attachedCallRecordingIds).toEqual(['call-recording-1']); + expect(result.scheduledCallRecordingIds).toEqual(['call-recording-2']); + expect(createBotCalls()).toHaveLength(1); + }); + it('defers scheduling when the existing-bot lookup fails so no duplicate bot is created', async () => { stubRecallApi({ listStatus: 400 }); const client = new FakeCoreApiClient({ - callRecordings: [buildPendingCallRecording()], + callRecordings: [ + buildPendingCallRecording({ + botScheduleAttemptedAt: '2026-01-01T11:55:00.000Z', + }), + ], calendarEvents: [buildCalendarEvent()], }); @@ -297,6 +355,121 @@ describe('scheduleRecallBotsForPendingCallRecordings', () => { expect(client.callRecordings[0].externalBotId).toBeNull(); }); + it('re-sends the creation without any Recall lookup when the stored idempotency key still matches', async () => { + const unchangedIdempotencyKey = computeRecallBotCreationIdempotencyKey({ + meetingUrl: 'https://meet.example.com/customer-sync', + joinAt: computeRecallBotJoinAt(UPCOMING_STARTS_AT), + metadata: { + twentyWorkspaceId: WORKSPACE_ID, + twentyCallRecordingId: 'call-recording-1', + }, + }); + const client = new FakeCoreApiClient({ + callRecordings: [ + buildPendingCallRecording({ + botScheduleAttemptedAt: '2026-01-01T11:55:00.000Z', + botScheduleIdempotencyKey: unchangedIdempotencyKey, + }), + ], + calendarEvents: [buildCalendarEvent()], + }); + + const result = await scheduleRecallBotsForPendingCallRecordings({ + client: client as unknown as CoreApiClient, + now: NOW, + }); + + expect(listBotRequestUrls()).toHaveLength(0); + expect(result.scheduledCallRecordingIds).toEqual(['call-recording-1']); + const [, requestInit] = createBotCalls()[0]; + expect(requestInit.headers).toMatchObject({ + 'Idempotency-Key': unchangedIdempotencyKey, + }); + expect(client.callRecordings[0].externalBotId).toBe('recall-bot-1'); + // The first attempt timestamp survives the re-send so repeated unknown + // outcomes age out of the resend window. + expect(client.callRecordings[0].botScheduleAttemptedAt).toBe( + '2026-01-01T11:55:00.000Z', + ); + }); + + it('falls back to the Recall lookup when the recorded attempt is too old to trust its idempotency key', async () => { + const unchangedIdempotencyKey = computeRecallBotCreationIdempotencyKey({ + meetingUrl: 'https://meet.example.com/customer-sync', + joinAt: computeRecallBotJoinAt(UPCOMING_STARTS_AT), + metadata: { + twentyWorkspaceId: WORKSPACE_ID, + twentyCallRecordingId: 'call-recording-1', + }, + }); + const client = new FakeCoreApiClient({ + callRecordings: [ + buildPendingCallRecording({ + botScheduleAttemptedAt: '2025-12-30T12:00:00.000Z', + botScheduleIdempotencyKey: unchangedIdempotencyKey, + }), + ], + calendarEvents: [buildCalendarEvent()], + }); + + const result = await scheduleRecallBotsForPendingCallRecordings({ + client: client as unknown as CoreApiClient, + now: NOW, + }); + + expect(listBotRequestUrls()).toHaveLength(1); + expect(result.scheduledCallRecordingIds).toEqual(['call-recording-1']); + }); + + it('falls back to the Recall lookup when the meeting moved since the recorded attempt', async () => { + const staleIdempotencyKey = computeRecallBotCreationIdempotencyKey({ + meetingUrl: 'https://meet.example.com/customer-sync', + joinAt: computeRecallBotJoinAt('2026-01-01T09:00:00.000Z'), + metadata: { + twentyWorkspaceId: WORKSPACE_ID, + twentyCallRecordingId: 'call-recording-1', + }, + }); + const client = new FakeCoreApiClient({ + callRecordings: [ + buildPendingCallRecording({ + botScheduleAttemptedAt: '2026-01-01T11:55:00.000Z', + botScheduleIdempotencyKey: staleIdempotencyKey, + }), + ], + calendarEvents: [buildCalendarEvent()], + }); + + const result = await scheduleRecallBotsForPendingCallRecordings({ + client: client as unknown as CoreApiClient, + now: NOW, + }); + + expect(listBotRequestUrls()).toHaveLength(1); + expect(result.scheduledCallRecordingIds).toEqual(['call-recording-1']); + }); + + it('re-schedules an ambiguous recording when the lookup confirms no bot exists', async () => { + const client = new FakeCoreApiClient({ + callRecordings: [ + buildPendingCallRecording({ + botScheduleAttemptedAt: '2026-01-01T11:55:00.000Z', + }), + ], + calendarEvents: [buildCalendarEvent()], + }); + + const result = await scheduleRecallBotsForPendingCallRecordings({ + client: client as unknown as CoreApiClient, + now: NOW, + }); + + expect(listBotRequestUrls()).toHaveLength(1); + expect(result.scheduledCallRecordingIds).toEqual(['call-recording-1']); + expect(createBotCalls()).toHaveLength(1); + expect(client.callRecordings[0].externalBotId).toBe('recall-bot-1'); + }); + it('does not report a recording as scheduled when Recall scheduling fails', async () => { stubRecallApi({ createBotStatus: 500 }); const client = new FakeCoreApiClient({ @@ -318,7 +491,7 @@ describe('scheduleRecallBotsForPendingCallRecordings', () => { expect(client.callRecordings[0].externalBotId).toBeNull(); }); - it('skips a recording whose meeting has already ended', async () => { + it('marks a recording failed when its meeting ended before any bot was scheduled', async () => { const client = new FakeCoreApiClient({ callRecordings: [buildPendingCallRecording()], calendarEvents: [ @@ -335,7 +508,81 @@ describe('scheduleRecallBotsForPendingCallRecordings', () => { }); expect(result.scheduledCallRecordingIds).toEqual([]); + expect(result.markedFailedCallRecordingIds).toEqual(['call-recording-1']); expect(fetchMock).not.toHaveBeenCalled(); + expect(client.callRecordings[0].status).toBe('FAILED'); + expect(client.callRecordings[0].callRecorderFailureReason).toBe( + 'bot_never_scheduled', + ); + }); + + it('keeps an ended recording with an unresolved attempt pending while convergence may still resolve it', async () => { + const client = new FakeCoreApiClient({ + callRecordings: [ + buildPendingCallRecording({ + botScheduleAttemptedAt: '2026-01-01T09:55:00.000Z', + }), + ], + calendarEvents: [ + buildCalendarEvent({ + startsAt: PAST_STARTS_AT, + endsAt: PAST_ENDS_AT, + }), + ], + }); + + const result = await scheduleRecallBotsForPendingCallRecordings({ + client: client as unknown as CoreApiClient, + now: NOW, + }); + + expect(result.markedFailedCallRecordingIds).toEqual([]); + expect(fetchMock).not.toHaveBeenCalled(); + expect(client.callRecordings[0].status).toBe('SCHEDULED'); + }); + + it('fails an ended recording with an unresolved attempt once the convergence lookback has passed', async () => { + const client = new FakeCoreApiClient({ + callRecordings: [ + buildPendingCallRecording({ + botScheduleAttemptedAt: '2025-12-20T09:55:00.000Z', + }), + ], + calendarEvents: [ + buildCalendarEvent({ + startsAt: '2025-12-20T10:00:00.000Z', + endsAt: '2025-12-20T11:00:00.000Z', + }), + ], + }); + + const result = await scheduleRecallBotsForPendingCallRecordings({ + client: client as unknown as CoreApiClient, + now: NOW, + }); + + expect(result.markedFailedCallRecordingIds).toEqual(['call-recording-1']); + expect(fetchMock).not.toHaveBeenCalled(); + expect(client.callRecordings[0].status).toBe('FAILED'); + expect(client.callRecordings[0].callRecorderFailureReason).toBe( + 'bot_schedule_outcome_unknown', + ); + }); + + it('leaves a recording untouched when its calendar event is missing', async () => { + const client = new FakeCoreApiClient({ + callRecordings: [buildPendingCallRecording()], + calendarEvents: [], + }); + + const result = await scheduleRecallBotsForPendingCallRecordings({ + client: client as unknown as CoreApiClient, + now: NOW, + }); + + expect(result.markedFailedCallRecordingIds).toEqual([]); + expect(fetchMock).not.toHaveBeenCalled(); + expect(client.callRecordings[0].status).toBe('SCHEDULED'); }); it('does nothing when every scheduled recording already has a bot', async () => { diff --git a/packages/twenty-apps/public/call-recorder/src/logic-functions/flows/attach-existing-recall-bot-to-call-recording.util.ts b/packages/twenty-apps/public/call-recorder/src/logic-functions/flows/attach-existing-recall-bot-to-call-recording.util.ts deleted file mode 100644 index 4e628d956a..0000000000 --- a/packages/twenty-apps/public/call-recorder/src/logic-functions/flows/attach-existing-recall-bot-to-call-recording.util.ts +++ /dev/null @@ -1,44 +0,0 @@ -import { isUndefined } from '@sniptt/guards'; -import { type CoreApiClient } from 'twenty-client-sdk/core'; - -import { type CallRecordingRecord } from 'src/logic-functions/types/call-recording-record.type'; -import { findScheduledRecallBotIdForCallRecording } from 'src/logic-functions/recall-api/find-scheduled-recall-bot-id-for-call-recording.util'; -import { getCurrentWorkspaceId } from 'src/logic-functions/data/get-current-workspace-id.util'; -import { updateCallRecording } from 'src/logic-functions/data/update-call-recording.util'; - -export type AttachExistingRecallBotToCallRecordingResult = - | { status: 'attached'; externalBotId: string } - | { status: 'no-existing-bot' } - | { status: 'lookup-failed' }; - -// A run that POSTed a bot but died before the id write-back leaves the bot claimable by metadata; attaching it instead of re-POSTing prevents duplicate bots. -export const attachExistingRecallBotToCallRecording = async ( - client: CoreApiClient, - { callRecording }: { callRecording: CallRecordingRecord }, -): Promise => { - const workspaceId = getCurrentWorkspaceId(); - - if (isUndefined(workspaceId)) { - return { status: 'no-existing-bot' }; - } - - const findResult = await findScheduledRecallBotIdForCallRecording({ - callRecordingId: callRecording.id, - workspaceId, - }); - - if (!findResult.ok) { - return { status: 'lookup-failed' }; - } - - if (isUndefined(findResult.externalBotId)) { - return { status: 'no-existing-bot' }; - } - - await updateCallRecording(client, { - id: callRecording.id, - data: { externalBotId: findResult.externalBotId }, - }); - - return { status: 'attached', externalBotId: findResult.externalBotId }; -}; diff --git a/packages/twenty-apps/public/call-recorder/src/logic-functions/flows/reschedule-call-recording-bot.util.ts b/packages/twenty-apps/public/call-recorder/src/logic-functions/flows/reschedule-call-recording-bot.util.ts index 3ec904cf49..a8134fed1b 100644 --- a/packages/twenty-apps/public/call-recorder/src/logic-functions/flows/reschedule-call-recording-bot.util.ts +++ b/packages/twenty-apps/public/call-recorder/src/logic-functions/flows/reschedule-call-recording-bot.util.ts @@ -53,11 +53,18 @@ export const rescheduleCallRecordingBot = async ( return; } - // The bot vanished externally; drop the id so the stale-state cron re-creates it as the single writer. + // The bot vanished externally; drop the id so recovery re-creates it as the + // single writer. The recorded attempt state is resolved (its bot is + // confirmed gone), so clearing it lets recovery schedule directly instead + // of treating the row as an ambiguous attempt. if (rescheduleResult.status === RECALL_BOT_NOT_FOUND_STATUS) { await updateCallRecording(client, { id: callRecording.id, - data: { externalBotId: null }, + data: { + externalBotId: null, + botScheduleAttemptedAt: null, + botScheduleIdempotencyKey: null, + }, }); return; diff --git a/packages/twenty-apps/public/call-recorder/src/logic-functions/flows/resume-pending-call-recording.util.ts b/packages/twenty-apps/public/call-recorder/src/logic-functions/flows/resume-pending-call-recording.util.ts new file mode 100644 index 0000000000..618db44d3a --- /dev/null +++ b/packages/twenty-apps/public/call-recorder/src/logic-functions/flows/resume-pending-call-recording.util.ts @@ -0,0 +1,112 @@ +import { isUndefined } from '@sniptt/guards'; +import { type CoreApiClient } from 'twenty-client-sdk/core'; + +import { CallRecordingRequestStatus } from 'src/logic-functions/constants/call-recording-request-status'; +import { CallRecordingStatus } from 'src/logic-functions/constants/call-recording-status'; +import { type CalendarEventRecord } from 'src/logic-functions/types/calendar-event-record.type'; +import { type CallRecordingRecord } from 'src/logic-functions/types/call-recording-record.type'; +import { canRescheduleCallRecordingWithoutRecallLookup } from 'src/logic-functions/domain/can-reschedule-call-recording-without-recall-lookup.util'; +import { fetchCalendarEventsByIds } from 'src/logic-functions/data/fetch-calendar-events-by-ids.util'; +import { findCallRecordingsByIds } from 'src/logic-functions/data/find-call-recordings-by-ids.util'; +import { getCurrentWorkspaceId } from 'src/logic-functions/data/get-current-workspace-id.util'; +import { hasMeetingEnded } from 'src/logic-functions/domain/has-meeting-ended.util'; +import { scheduleRecallBotForCallRecording } from 'src/logic-functions/flows/schedule-recall-bot-for-call-recording.util'; + +export type ResumePendingCallRecordingResult = + | { status: 'scheduled' } + | { status: 'skipped'; reason: string } + | { status: 'deferred'; reason: string }; + +// Single-recording variant of the recovery sweep: finishes bot scheduling for +// one row that transitioned back to pending. Deferred outcomes are retried by +// the queue and ultimately by the recovery cron. +export const resumePendingCallRecording = async ({ + client, + callRecordingId, + now, +}: { + client: CoreApiClient; + callRecordingId: string; + now: Date; +}): Promise => { + const callRecording = ( + await findCallRecordingsByIds(client, [callRecordingId]) + )[0]; + + if (isUndefined(callRecording)) { + return { status: 'skipped', reason: 'call recording not found' }; + } + + if ( + callRecording.recordingRequestStatus !== + CallRecordingRequestStatus.REQUESTED || + callRecording.status !== CallRecordingStatus.SCHEDULED || + !isUndefined(callRecording.externalBotId) + ) { + return { status: 'skipped', reason: 'call recording is not pending' }; + } + + if (isUndefined(callRecording.calendarEventId)) { + return { status: 'skipped', reason: 'no calendar event attached' }; + } + + const calendarEvent = ( + await fetchCalendarEventsByIds(client, [callRecording.calendarEventId]) + )[0]; + + if (isUndefined(calendarEvent)) { + return { status: 'skipped', reason: 'calendar event not found' }; + } + + if ( + hasMeetingEnded({ + startsAt: calendarEvent.startsAt, + endsAt: calendarEvent.endsAt, + now, + }) + ) { + // The recovery cron owns failing rows whose meeting is over. + return { status: 'skipped', reason: 'meeting already ended' }; + } + + if ( + canRescheduleCallRecordingWithoutRecallLookup({ + callRecording, + calendarEvent, + workspaceId: getCurrentWorkspaceId(), + now, + }) + ) { + return scheduleBot({ client, callRecording, calendarEvent }); + } + + // Ambiguous attempts need a Recall bot lookup; doing one per event could + // burst past the shared list budget, so the recovery cron owns them and + // amortizes a single lookup across all ambiguous rows. + return { + status: 'deferred', + reason: 'ambiguous prior attempt; the recovery cron will reconcile it', + }; +}; + +const scheduleBot = async ({ + client, + callRecording, + calendarEvent, +}: { + client: CoreApiClient; + callRecording: CallRecordingRecord; + calendarEvent: CalendarEventRecord; +}): Promise => { + const didScheduleRecallBot = await scheduleRecallBotForCallRecording( + client, + { + callRecording, + calendarEvent, + }, + ); + + return didScheduleRecallBot + ? { status: 'scheduled' } + : { status: 'deferred', reason: 'Recall bot scheduling failed' }; +}; diff --git a/packages/twenty-apps/public/call-recorder/src/logic-functions/flows/retry-failed-recall-cancellations.util.ts b/packages/twenty-apps/public/call-recorder/src/logic-functions/flows/retry-failed-recall-cancellations.util.ts index 7451718fcc..b3de1a8b34 100644 --- a/packages/twenty-apps/public/call-recorder/src/logic-functions/flows/retry-failed-recall-cancellations.util.ts +++ b/packages/twenty-apps/public/call-recorder/src/logic-functions/flows/retry-failed-recall-cancellations.util.ts @@ -7,10 +7,9 @@ import { NON_TERMINAL_CALL_RECORDING_STATUSES } from 'src/logic-functions/consta import { fetchCalendarEventsByIds } from 'src/logic-functions/data/fetch-calendar-events-by-ids.util'; import { findCallRecordingsByFilter } from 'src/logic-functions/data/find-call-recordings-by-filter.util'; import { findCallRecordingsByIds } from 'src/logic-functions/data/find-call-recordings-by-ids.util'; -import { getCurrentWorkspaceId } from 'src/logic-functions/data/get-current-workspace-id.util'; import { replaceCanceledCallRecordingExternalBotId } from 'src/logic-functions/data/replace-canceled-call-recording-external-bot-id.util'; import { cancelOrEjectRecallBot } from 'src/logic-functions/recall-api/cancel-or-eject-recall-bot.util'; -import { findScheduledRecallBotIdForCallRecording } from 'src/logic-functions/recall-api/find-scheduled-recall-bot-id-for-call-recording.util'; +import { findScheduledRecallBotIdsByCallRecordingId } from 'src/logic-functions/recall-api/find-scheduled-recall-bot-ids-by-call-recording-id.util'; import { type CalendarEventRecord } from 'src/logic-functions/types/calendar-event-record.type'; import { type CallRecordingRecord } from 'src/logic-functions/types/call-recording-record.type'; import { getUniqueSortedIds } from 'src/logic-functions/utils/get-unique-sorted-ids.util'; @@ -49,17 +48,32 @@ export const retryFailedRecallCancellations = async ({ ) ).map((calendarEvent) => [calendarEvent.id, calendarEvent]), ); + const recoverableCallRecordingIds = new Set( + canceledCallRecordings + .filter( + (callRecording) => + isUndefined(callRecording.externalBotId) && + isWithinCanceledBotRecoveryWindow({ + callRecording, + calendarEvent: isUndefined(callRecording.calendarEventId) + ? undefined + : calendarEventsById.get(callRecording.calendarEventId), + now, + }), + ) + .map((callRecording) => callRecording.id), + ); + const externalBotIdByCallRecordingId = + await lookupRecoverableExternalBotIds(recoverableCallRecordingIds); const canceledExternalBotCallRecordingIds: string[] = []; for (const callRecording of canceledCallRecordings) { - const calendarEvent = isUndefined(callRecording.calendarEventId) - ? undefined - : calendarEventsById.get(callRecording.calendarEventId); const externalBotId = await recoverRecallBotIdForCanceledCallRecording({ client, callRecording, - calendarEvent, - now, + listedExternalBotId: recoverableCallRecordingIds.has(callRecording.id) + ? externalBotIdByCallRecordingId?.get(callRecording.id) + : undefined, }); if (isUndefined(externalBotId)) { @@ -98,21 +112,31 @@ export const retryFailedRecallCancellations = async ({ return { canceledExternalBotCallRecordingIds }; }; -const recoverRecallBotIdForCanceledCallRecording = async ({ - client, +// One workspace-wide list request covers every recoverable row; undefined +// means the lookup failed and recovery must wait for the next run. +const lookupRecoverableExternalBotIds = async ( + recoverableCallRecordingIds: Set, +): Promise | undefined> => { + if (recoverableCallRecordingIds.size === 0) { + return new Map(); + } + + const lookupResult = await findScheduledRecallBotIdsByCallRecordingId(); + + return lookupResult.ok + ? lookupResult.externalBotIdByCallRecordingId + : undefined; +}; + +const isWithinCanceledBotRecoveryWindow = ({ callRecording, calendarEvent, now, }: { - client: CoreApiClient; callRecording: CallRecordingRecord; calendarEvent: CalendarEventRecord | undefined; now: Date; -}): Promise => { - if (!isUndefined(callRecording.externalBotId)) { - return callRecording.externalBotId; - } - +}): boolean => { if ( !isUndefined(calendarEvent) && hasMeetingEnded({ @@ -122,50 +146,15 @@ const recoverRecallBotIdForCanceledCallRecording = async ({ startGraceHours: CANCELED_BOT_RECOVERY_AFTER_START_HOURS, }) ) { - return undefined; + return false; } // Recovery only closes the crash window right after cancellation; once a row ages out the daily cleanup sweep owns it, so stop the per-run Recall lookup instead of listing forever (notably for rows whose calendar event was deleted and can no longer bound the retry). // updatedAt tracks the cancellation write, so a request scheduled far ahead but canceled recently still gets its window; createdAt would age it out from scheduling time. - if ( - hasCanceledRecoveryWindowElapsed({ - canceledAt: callRecording.updatedAt ?? callRecording.createdAt, - now, - }) - ) { - return undefined; - } - - const currentWorkspaceId = getCurrentWorkspaceId(); - - if (isUndefined(currentWorkspaceId)) { - return undefined; - } - - const scheduledRecallBotLookupResult = - await findScheduledRecallBotIdForCallRecording({ - callRecordingId: callRecording.id, - workspaceId: currentWorkspaceId, - }); - - if ( - !scheduledRecallBotLookupResult.ok || - isUndefined(scheduledRecallBotLookupResult.externalBotId) - ) { - return undefined; - } - - const externalBotId = scheduledRecallBotLookupResult.externalBotId; - const didClaimRecoveredBot = await replaceCanceledCallRecordingExternalBotId( - client, - { - id: callRecording.id, - expectedExternalBotId: null, - nextExternalBotId: externalBotId, - }, - ); - - return didClaimRecoveredBot ? externalBotId : undefined; + return !hasCanceledRecoveryWindowElapsed({ + canceledAt: callRecording.updatedAt ?? callRecording.createdAt, + now, + }); }; const hasCanceledRecoveryWindowElapsed = ({ @@ -187,3 +176,32 @@ const hasCanceledRecoveryWindowElapsed = ({ now.getTime() ); }; + +const recoverRecallBotIdForCanceledCallRecording = async ({ + client, + callRecording, + listedExternalBotId, +}: { + client: CoreApiClient; + callRecording: CallRecordingRecord; + listedExternalBotId: string | undefined; +}): Promise => { + if (!isUndefined(callRecording.externalBotId)) { + return callRecording.externalBotId; + } + + if (isUndefined(listedExternalBotId)) { + return undefined; + } + + const didClaimRecoveredBot = await replaceCanceledCallRecordingExternalBotId( + client, + { + id: callRecording.id, + expectedExternalBotId: null, + nextExternalBotId: listedExternalBotId, + }, + ); + + return didClaimRecoveredBot ? listedExternalBotId : undefined; +}; diff --git a/packages/twenty-apps/public/call-recorder/src/logic-functions/flows/schedule-recall-bot-for-call-recording.util.ts b/packages/twenty-apps/public/call-recorder/src/logic-functions/flows/schedule-recall-bot-for-call-recording.util.ts index 08974ced45..6719047f7d 100644 --- a/packages/twenty-apps/public/call-recorder/src/logic-functions/flows/schedule-recall-bot-for-call-recording.util.ts +++ b/packages/twenty-apps/public/call-recorder/src/logic-functions/flows/schedule-recall-bot-for-call-recording.util.ts @@ -2,13 +2,17 @@ import { isUndefined } from '@sniptt/guards'; import { type CoreApiClient } from 'twenty-client-sdk/core'; import { CallRecordingRequestStatus } from 'src/logic-functions/constants/call-recording-request-status'; +import { CallRecordingStatus } from 'src/logic-functions/constants/call-recording-status'; import { type MeetingRecording } from 'src/logic-functions/types/meeting-recording.type'; import { buildRecallBotAutomaticVideoOutput } from 'src/logic-functions/domain/build-recall-bot-automatic-video-output.util'; import { buildRecallRoutingMetadata } from 'src/logic-functions/domain/build-recall-routing-metadata.util'; import { computeRecallBotJoinAt } from 'src/logic-functions/domain/compute-recall-bot-join-at.util'; import { findCallRecordingsByIds } from 'src/logic-functions/data/find-call-recordings-by-ids.util'; import { getCurrentWorkspaceId } from 'src/logic-functions/data/get-current-workspace-id.util'; -import { scheduleRecallBot } from 'src/logic-functions/recall-api/schedule-recall-bot.util'; +import { + computeRecallBotCreationIdempotencyKey, + scheduleRecallBot, +} from 'src/logic-functions/recall-api/schedule-recall-bot.util'; import { updateCallRecording } from 'src/logic-functions/data/update-call-recording.util'; // The sole place a Recall bot is created. Only the deterministic-create winner and the stale-state cron call it, so one writer per meeting POSTs exactly one bot. @@ -33,6 +37,7 @@ export const scheduleRecallBotForCallRecording = async ( isUndefined(freshCallRecording) || freshCallRecording.recordingRequestStatus !== CallRecordingRequestStatus.REQUESTED || + freshCallRecording.status !== CallRecordingStatus.SCHEDULED || !isUndefined(freshCallRecording.externalBotId) ) { return false; @@ -49,15 +54,42 @@ export const scheduleRecallBotForCallRecording = async ( } const automaticVideoOutput = await buildRecallBotAutomaticVideoOutput(); + const metadata = buildRecallRoutingMetadata({ + callRecordingId: callRecording.id, + workspaceId, + }); + const idempotencyKey = computeRecallBotCreationIdempotencyKey({ + meetingUrl, + joinAt, + metadata, + }); + + // Persisted before the POST so a crash leaves proof that a bot creation may + // have reached Recall; while the stored key still matches the scheduling + // inputs, recovery can re-send the creation idempotently instead of asking + // Recall whether a bot already exists. Re-sends of the same key keep the + // first attempt's timestamp so repeated unknown outcomes age out of the + // resend window instead of staying trusted forever. + const recordedAttemptTimestamp = + freshCallRecording.botScheduleIdempotencyKey === idempotencyKey + ? freshCallRecording.botScheduleAttemptedAt + : undefined; + + await updateCallRecording(client, { + id: callRecording.id, + data: { + botScheduleAttemptedAt: + recordedAttemptTimestamp ?? new Date().toISOString(), + botScheduleIdempotencyKey: idempotencyKey, + }, + }); const scheduleResult = await scheduleRecallBot({ meetingUrl, joinAt, - metadata: buildRecallRoutingMetadata({ - callRecordingId: callRecording.id, - workspaceId, - }), + metadata, automaticVideoOutput, + idempotencyKey, }); if (!scheduleResult.ok) { diff --git a/packages/twenty-apps/public/call-recorder/src/logic-functions/flows/schedule-recall-bots-for-pending-call-recordings.util.ts b/packages/twenty-apps/public/call-recorder/src/logic-functions/flows/schedule-recall-bots-for-pending-call-recordings.util.ts index bf2bada651..6f005ee39e 100644 --- a/packages/twenty-apps/public/call-recorder/src/logic-functions/flows/schedule-recall-bots-for-pending-call-recordings.util.ts +++ b/packages/twenty-apps/public/call-recorder/src/logic-functions/flows/schedule-recall-bots-for-pending-call-recordings.util.ts @@ -1,16 +1,36 @@ import { isUndefined } from '@sniptt/guards'; import { type CoreApiClient } from 'twenty-client-sdk/core'; +import { CallRecordingStatus } from 'src/logic-functions/constants/call-recording-status'; +import { type CalendarEventRecord } from 'src/logic-functions/types/calendar-event-record.type'; +import { type CallRecordingRecord } from 'src/logic-functions/types/call-recording-record.type'; +import { canRescheduleCallRecordingWithoutRecallLookup } from 'src/logic-functions/domain/can-reschedule-call-recording-without-recall-lookup.util'; +import { getCurrentWorkspaceId } from 'src/logic-functions/data/get-current-workspace-id.util'; import { hasMeetingEnded } from 'src/logic-functions/domain/has-meeting-ended.util'; -import { attachExistingRecallBotToCallRecording } from 'src/logic-functions/flows/attach-existing-recall-bot-to-call-recording.util'; import { scheduleRecallBotForCallRecording } from 'src/logic-functions/flows/schedule-recall-bot-for-call-recording.util'; import { fetchCalendarEventsByIds } from 'src/logic-functions/data/fetch-calendar-events-by-ids.util'; import { findOpenScheduledCallRecordings } from 'src/logic-functions/data/find-open-scheduled-call-recordings.util'; +import { findScheduledRecallBotIdsByCallRecordingId } from 'src/logic-functions/recall-api/find-scheduled-recall-bot-ids-by-call-recording-id.util'; import { getUniqueSortedIds } from 'src/logic-functions/utils/get-unique-sorted-ids.util'; +import { updateCallRecording } from 'src/logic-functions/data/update-call-recording.util'; + +export const BOT_NEVER_SCHEDULED_FAILURE_REASON = 'bot_never_scheduled'; +export const BOT_SCHEDULE_OUTCOME_UNKNOWN_FAILURE_REASON = + 'bot_schedule_outcome_unknown'; + +// Mirrors the stale-state convergence lookback: past it no automatic pull +// pass will resolve the row anymore, so keeping it pending only wastes runs. +const UNRESOLVED_ATTEMPT_MAX_AGE_DAYS = 7; export type ScheduleRecallBotsForPendingCallRecordingsResult = { attachedCallRecordingIds: string[]; scheduledCallRecordingIds: string[]; + markedFailedCallRecordingIds: string[]; +}; + +type ResumableCallRecording = { + callRecording: CallRecordingRecord; + calendarEvent: CalendarEventRecord; }; // Resumes a CallRecording inserted before its Recall bot was scheduled. @@ -21,12 +41,17 @@ export const scheduleRecallBotsForPendingCallRecordings = async ({ client: CoreApiClient; now: Date; }): Promise => { + const result: ScheduleRecallBotsForPendingCallRecordingsResult = { + attachedCallRecordingIds: [], + scheduledCallRecordingIds: [], + markedFailedCallRecordingIds: [], + }; const pendingCallRecordings = ( await findOpenScheduledCallRecordings(client) ).filter((callRecording) => isUndefined(callRecording.externalBotId)); if (pendingCallRecordings.length === 0) { - return { attachedCallRecordingIds: [], scheduledCallRecordingIds: [] }; + return result; } const calendarEventsById = new Map( @@ -41,51 +66,222 @@ export const scheduleRecallBotsForPendingCallRecordings = async ({ ) ).map((calendarEvent) => [calendarEvent.id, calendarEvent]), ); - const attachedCallRecordingIds: string[] = []; - const scheduledCallRecordingIds: string[] = []; + const resumableCallRecordings: ResumableCallRecording[] = []; for (const callRecording of pendingCallRecordings) { const calendarEvent = isUndefined(callRecording.calendarEventId) ? undefined : calendarEventsById.get(callRecording.calendarEventId); + if (isUndefined(calendarEvent)) { + continue; + } + if ( - isUndefined(calendarEvent) || hasMeetingEnded({ startsAt: calendarEvent.startsAt, endsAt: calendarEvent.endsAt, now, }) ) { - continue; - } - - const attachResult = await attachExistingRecallBotToCallRecording(client, { - callRecording, - }); - - if (attachResult.status === 'attached') { - attachedCallRecordingIds.push(callRecording.id); - continue; - } - - // A failed lookup can hide an existing bot; creating one now could duplicate it, so defer to the next run. - if (attachResult.status === 'lookup-failed') { - continue; - } - - const didScheduleRecallBot = await scheduleRecallBotForCallRecording( - client, - { + await resolveEndedPendingCallRecording({ + client, callRecording, calendarEvent, - }, - ); - - if (didScheduleRecallBot) { - scheduledCallRecordingIds.push(callRecording.id); + now, + result, + }); + continue; } + + resumableCallRecordings.push({ callRecording, calendarEvent }); } - return { attachedCallRecordingIds, scheduledCallRecordingIds }; + if (resumableCallRecordings.length === 0) { + return result; + } + + // Rows without a schedule-attempt marker never reached Recall, so no bot + // can exist for them. Rows whose stored idempotency key still matches the + // current scheduling inputs can re-send the creation and let Recall dedupe + // it. Only attempts whose inputs drifted since the attempt pay for a + // Recall lookup. + const workspaceId = getCurrentWorkspaceId(); + const ambiguousCallRecordings = resumableCallRecordings.filter( + ({ callRecording, calendarEvent }) => + !canRescheduleCallRecordingWithoutRecallLookup({ + callRecording, + calendarEvent, + workspaceId, + now, + }), + ); + const unambiguousCallRecordings = resumableCallRecordings.filter( + ({ callRecording, calendarEvent }) => + canRescheduleCallRecordingWithoutRecallLookup({ + callRecording, + calendarEvent, + workspaceId, + now, + }), + ); + + for (const { callRecording, calendarEvent } of unambiguousCallRecordings) { + await scheduleBotForResumableCallRecording({ + client, + callRecording, + calendarEvent, + result, + }); + } + + if (ambiguousCallRecordings.length === 0) { + return result; + } + + // A run that POSTed a bot but died before the id write-back leaves the bot + // claimable by metadata; one workspace-wide lookup finds them all without a + // per-recording list call. + const lookupResult = await findScheduledRecallBotIdsByCallRecordingId(); + + // A failed lookup can hide existing bots; creating one now could duplicate + // them, so defer to the next run. + if (!lookupResult.ok) { + return result; + } + + for (const { callRecording, calendarEvent } of ambiguousCallRecordings) { + const existingExternalBotId = + lookupResult.externalBotIdByCallRecordingId.get(callRecording.id); + + if (!isUndefined(existingExternalBotId)) { + await updateCallRecording(client, { + id: callRecording.id, + data: { externalBotId: existingExternalBotId }, + }); + result.attachedCallRecordingIds.push(callRecording.id); + continue; + } + + await scheduleBotForResumableCallRecording({ + client, + callRecording, + calendarEvent, + result, + }); + } + + return result; +}; + +const scheduleBotForResumableCallRecording = async ({ + client, + callRecording, + calendarEvent, + result, +}: { + client: CoreApiClient; + callRecording: CallRecordingRecord; + calendarEvent: CalendarEventRecord; + result: ScheduleRecallBotsForPendingCallRecordingsResult; +}): Promise => { + const didScheduleRecallBot = await scheduleRecallBotForCallRecording( + client, + { + callRecording, + calendarEvent, + }, + ); + + if (didScheduleRecallBot) { + result.scheduledCallRecordingIds.push(callRecording.id); + } +}; + +// Only an absent attempt marker proves no POST reached Recall; a marked row +// may have a bot that joined and recorded before the id write-back was lost, +// so it keeps its recovery chance until the convergence lookback has passed. +const resolveEndedPendingCallRecording = async ({ + client, + callRecording, + calendarEvent, + now, + result, +}: { + client: CoreApiClient; + callRecording: CallRecordingRecord; + calendarEvent: CalendarEventRecord; + now: Date; + result: ScheduleRecallBotsForPendingCallRecordingsResult; +}): Promise => { + if (isUndefined(callRecording.botScheduleAttemptedAt)) { + await markCallRecordingFailed({ + client, + callRecording, + failureReason: BOT_NEVER_SCHEDULED_FAILURE_REASON, + logMessage: `call recording ${callRecording.id} never got a Recall bot and its meeting has ended; marking it failed`, + }); + result.markedFailedCallRecordingIds.push(callRecording.id); + + return; + } + + if (!hasUnresolvedAttemptAgedOut({ calendarEvent, now })) { + console.warn( + `[call-recorder] call recording ${callRecording.id} has an unresolved Recall bot creation attempt and its meeting has ended; waiting for convergence`, + ); + + return; + } + + await markCallRecordingFailed({ + client, + callRecording, + failureReason: BOT_SCHEDULE_OUTCOME_UNKNOWN_FAILURE_REASON, + logMessage: `call recording ${callRecording.id} has an unresolved Recall bot creation attempt older than the convergence lookback; marking it failed`, + }); + result.markedFailedCallRecordingIds.push(callRecording.id); +}; + +const hasUnresolvedAttemptAgedOut = ({ + calendarEvent, + now, +}: { + calendarEvent: CalendarEventRecord; + now: Date; +}): boolean => { + // Mirrors hasMeetingEnded: an unparseable end time falls back to the start + // time so these rows still age out of the pending sweep eventually. + const meetingEndTime = [calendarEvent.endsAt, calendarEvent.startsAt] + .filter((candidate) => !isUndefined(candidate)) + .map((candidate) => new Date(candidate).getTime()) + .find((candidateTime) => !Number.isNaN(candidateTime)); + + return ( + !isUndefined(meetingEndTime) && + meetingEndTime + UNRESOLVED_ATTEMPT_MAX_AGE_DAYS * 24 * 60 * 60 * 1000 <= + now.getTime() + ); +}; + +const markCallRecordingFailed = async ({ + client, + callRecording, + failureReason, + logMessage, +}: { + client: CoreApiClient; + callRecording: CallRecordingRecord; + failureReason: string; + logMessage: string; +}): Promise => { + console.warn(`[call-recorder] ${logMessage}`); + + await updateCallRecording(client, { + id: callRecording.id, + data: { + status: CallRecordingStatus.FAILED, + callRecorderFailureReason: failureReason, + }, + }); }; diff --git a/packages/twenty-apps/public/call-recorder/src/logic-functions/recall-api/find-scheduled-recall-bot-id-for-call-recording.util.ts b/packages/twenty-apps/public/call-recorder/src/logic-functions/recall-api/find-scheduled-recall-bot-id-for-call-recording.util.ts deleted file mode 100644 index 99bd41ea83..0000000000 --- a/packages/twenty-apps/public/call-recorder/src/logic-functions/recall-api/find-scheduled-recall-bot-id-for-call-recording.util.ts +++ /dev/null @@ -1,41 +0,0 @@ -import { listScheduledRecallBots } from 'src/logic-functions/recall-api/list-scheduled-recall-bots.util'; - -const ACTIVE_RECALL_BOT_STATUSES = [ - 'ready', - 'joining_call', - 'in_waiting_room', - 'in_call_not_recording', - 'recording_permission_allowed', - 'recording_permission_denied', - 'in_call_recording', -]; - -export type FindScheduledRecallBotIdResult = - | { ok: true; externalBotId: string | undefined } - | { ok: false }; - -export const findScheduledRecallBotIdForCallRecording = async ({ - callRecordingId, - workspaceId, -}: { - callRecordingId: string; - workspaceId: string; -}): Promise => { - const listResult = await listScheduledRecallBots({ - metadata: { - twentyWorkspaceId: workspaceId, - twentyCallRecordingId: callRecordingId, - }, - statuses: ACTIVE_RECALL_BOT_STATUSES, - }); - - if (!listResult.ok) { - console.warn( - `[call-recorder] failed to look up existing Recall bot for call recording ${callRecordingId}: ${listResult.errorMessage}`, - ); - - return { ok: false }; - } - - return { ok: true, externalBotId: listResult.bots[0]?.id }; -}; diff --git a/packages/twenty-apps/public/call-recorder/src/logic-functions/recall-api/find-scheduled-recall-bot-ids-by-call-recording-id.util.ts b/packages/twenty-apps/public/call-recorder/src/logic-functions/recall-api/find-scheduled-recall-bot-ids-by-call-recording-id.util.ts new file mode 100644 index 0000000000..3d6372dba6 --- /dev/null +++ b/packages/twenty-apps/public/call-recorder/src/logic-functions/recall-api/find-scheduled-recall-bot-ids-by-call-recording-id.util.ts @@ -0,0 +1,62 @@ +import { isUndefined } from '@sniptt/guards'; + +import { ACTIVE_RECALL_BOT_STATUSES } from 'src/logic-functions/constants/active-recall-bot-statuses'; +import { getCurrentWorkspaceId } from 'src/logic-functions/data/get-current-workspace-id.util'; +import { listScheduledRecallBots } from 'src/logic-functions/recall-api/list-scheduled-recall-bots.util'; +import { isNonEmptyString } from 'src/logic-functions/utils/is-non-empty-string.util'; + +export type FindScheduledRecallBotIdsByCallRecordingIdResult = + | { ok: true; externalBotIdByCallRecordingId: Map } + | { ok: false }; + +// One workspace-wide list request covers every pending recording; Recall's +// list endpoint has the tightest rate budget, so per-recording lookups must +// not fan out. +export const findScheduledRecallBotIdsByCallRecordingId = + async (): Promise => { + const workspaceId = getCurrentWorkspaceId(); + + if (isUndefined(workspaceId)) { + return { ok: true, externalBotIdByCallRecordingId: new Map() }; + } + + const listResult = await listScheduledRecallBots({ + metadata: { twentyWorkspaceId: workspaceId }, + statuses: ACTIVE_RECALL_BOT_STATUSES, + }); + + if (!listResult.ok) { + console.warn( + `[call-recorder] failed to look up existing Recall bots for pending call recordings: ${listResult.errorMessage}`, + ); + + return { ok: false }; + } + + // A truncated list can hide existing bots; callers treat a map miss as + // permission to create, so an incomplete map must read as a failed lookup. + if (listResult.truncated) { + console.warn( + '[call-recorder] Recall bot list was truncated; deferring bot recovery to the next run', + ); + + return { ok: false }; + } + + const externalBotIdByCallRecordingId = new Map(); + + for (const bot of listResult.bots) { + const callRecordingId = bot.metadata.twentyCallRecordingId; + + if ( + !isNonEmptyString(callRecordingId) || + externalBotIdByCallRecordingId.has(callRecordingId) + ) { + continue; + } + + externalBotIdByCallRecordingId.set(callRecordingId, bot.id); + } + + return { ok: true, externalBotIdByCallRecordingId }; + }; diff --git a/packages/twenty-apps/public/call-recorder/src/logic-functions/recall-api/schedule-recall-bot.util.ts b/packages/twenty-apps/public/call-recorder/src/logic-functions/recall-api/schedule-recall-bot.util.ts index 71dfe546c8..2f1ab4baea 100644 --- a/packages/twenty-apps/public/call-recorder/src/logic-functions/recall-api/schedule-recall-bot.util.ts +++ b/packages/twenty-apps/public/call-recorder/src/logic-functions/recall-api/schedule-recall-bot.util.ts @@ -20,6 +20,7 @@ export type ScheduleRecallBotArgs = { joinAt: string; metadata: RecallRoutingMetadata; automaticVideoOutput?: RecallBotAutomaticVideoOutput; + idempotencyKey?: string; }; export const scheduleRecallBot = async ({ @@ -27,6 +28,11 @@ export const scheduleRecallBot = async ({ joinAt, metadata, automaticVideoOutput, + idempotencyKey = computeRecallBotCreationIdempotencyKey({ + meetingUrl, + joinAt, + metadata, + }), }: ScheduleRecallBotArgs): Promise => { const configResult = getRecallApiConfig(); @@ -35,11 +41,6 @@ export const scheduleRecallBot = async ({ } const automaticLeave = getRecallBotAutomaticLeave(); - const idempotencyKey = computeRecallBotCreationIdempotencyKey({ - meetingUrl, - joinAt, - metadata, - }); const result = await recallBotApiRequest({ config: configResult.config, @@ -82,7 +83,7 @@ export const scheduleRecallBot = async ({ }; }; -const computeRecallBotCreationIdempotencyKey = ({ +export const computeRecallBotCreationIdempotencyKey = ({ meetingUrl, joinAt, metadata, diff --git a/packages/twenty-apps/public/call-recorder/src/logic-functions/schedule-recall-bot-on-call-recording-update.ts b/packages/twenty-apps/public/call-recorder/src/logic-functions/schedule-recall-bot-on-call-recording-update.ts new file mode 100644 index 0000000000..78c7026129 --- /dev/null +++ b/packages/twenty-apps/public/call-recorder/src/logic-functions/schedule-recall-bot-on-call-recording-update.ts @@ -0,0 +1,129 @@ +import { isUndefined } from '@sniptt/guards'; +import { CoreApiClient } from 'twenty-client-sdk/core'; +import { + defineLogicFunction, + type DatabaseEventPayload, + type ObjectRecordBaseEvent, +} from 'twenty-sdk/define'; + +import { SCHEDULE_RECALL_BOT_ON_CALL_RECORDING_UPDATE_LOGIC_FUNCTION_UNIVERSAL_IDENTIFIER } from 'src/constants/schedule-recall-bot-on-call-recording-update-logic-function-universal-identifier'; +import { CallRecordingRequestStatus } from 'src/logic-functions/constants/call-recording-request-status'; +import { CallRecordingStatus } from 'src/logic-functions/constants/call-recording-status'; +import { + resumePendingCallRecording, + type ResumePendingCallRecordingResult, +} from 'src/logic-functions/flows/resume-pending-call-recording.util'; + +const CALL_RECORDING_OBJECT_NAME = 'callRecording'; + +// Only these fields can transition a row into the pending shape; ignoring the +// rest keeps this trigger from re-firing on its own scheduling-progress +// writes (botScheduleAttemptedAt, botScheduleIdempotencyKey, externalBotId +// write-back) and on webhook artifact updates. +const PENDING_TRANSITION_FIELDS = [ + 'recordingRequestStatus', + 'status', + 'externalBotId', + 'calendarEventId', +]; + +type CallRecordingForDatabaseEvent = { + id: string; + status?: string | null; + recordingRequestStatus?: string | null; + externalBotId?: string | null; +}; + +type CallRecordingDatabaseEvent = DatabaseEventPayload< + ObjectRecordBaseEvent +>; + +// Created rows are scheduled inline by the run that inserts them; reacting to +// creations here would race that run into duplicate Recall creates. This +// trigger only resumes rows that fall back to pending later (a bot cleared +// after vanishing at Recall, a canceled request re-requested, a failed row +// reset by calendar reconciliation), which the recovery cron would otherwise +// pick up minutes later. +export const scheduleRecallBotOnCallRecordingUpdateHandler = async ( + event: CallRecordingDatabaseEvent, +): Promise< + | { skipped: true; reason: string } + | { callRecordingId: string; result: ResumePendingCallRecordingResult } +> => { + const [objectName, action] = event.name.split('.'); + + if (objectName !== CALL_RECORDING_OBJECT_NAME || action !== 'updated') { + return { skipped: true, reason: 'not a call recording update' }; + } + + const updatedFields = event.properties.updatedFields ?? []; + + if ( + !updatedFields.some((updatedField) => + PENDING_TRANSITION_FIELDS.includes(updatedField), + ) + ) { + return { skipped: true, reason: 'no pending-transition field changed' }; + } + + if (contradictsPendingCallRecording(event.properties)) { + return { skipped: true, reason: 'call recording is not pending' }; + } + + const result = await resumePendingCallRecording({ + client: new CoreApiClient(), + callRecordingId: event.recordId, + now: new Date(), + }); + + return { callRecordingId: event.recordId, result }; +}; + +// Values can come from a full `after` snapshot or, on slim payloads, from the +// per-field diff; any known value that contradicts the pending shape lets the +// trigger skip without the authoritative fetch. Unknown values stay ambiguous +// and the resume flow re-fetches to decide. +const contradictsPendingCallRecording = ( + properties: CallRecordingDatabaseEvent['properties'], +): boolean => { + const knownValues = properties.after ?? resolveDiffAfterValues(properties); + + return ( + (!isUndefined(knownValues.recordingRequestStatus) && + knownValues.recordingRequestStatus !== + CallRecordingRequestStatus.REQUESTED) || + (!isUndefined(knownValues.status) && + knownValues.status !== CallRecordingStatus.SCHEDULED) || + (!isUndefined(knownValues.externalBotId) && + knownValues.externalBotId !== null) + ); +}; + +const resolveDiffAfterValues = ( + properties: CallRecordingDatabaseEvent['properties'], +): Partial => { + const diff = properties.diff ?? {}; + + return { + ...(isUndefined(diff.status) ? {} : { status: diff.status.after }), + ...(isUndefined(diff.recordingRequestStatus) + ? {} + : { recordingRequestStatus: diff.recordingRequestStatus.after }), + ...(isUndefined(diff.externalBotId) + ? {} + : { externalBotId: diff.externalBotId.after }), + }; +}; + +export default defineLogicFunction({ + universalIdentifier: + SCHEDULE_RECALL_BOT_ON_CALL_RECORDING_UPDATE_LOGIC_FUNCTION_UNIVERSAL_IDENTIFIER, + name: 'schedule-recall-bot-on-call-recording-update', + description: + 'Resumes Recall bot scheduling as soon as a call recording transitions back to pending instead of waiting for the recovery cron.', + timeoutSeconds: 60, + handler: scheduleRecallBotOnCallRecordingUpdateHandler, + databaseEventTriggerSettings: { + eventName: `${CALL_RECORDING_OBJECT_NAME}.updated`, + }, +}); diff --git a/packages/twenty-apps/public/call-recorder/src/logic-functions/types/call-recording-record.type.ts b/packages/twenty-apps/public/call-recorder/src/logic-functions/types/call-recording-record.type.ts index 0cf11c58fd..8d43271108 100644 --- a/packages/twenty-apps/public/call-recorder/src/logic-functions/types/call-recording-record.type.ts +++ b/packages/twenty-apps/public/call-recorder/src/logic-functions/types/call-recording-record.type.ts @@ -12,6 +12,8 @@ export type CallRecordingRecord = { endedAt?: string; calendarEventId?: string; externalBotId?: string; + botScheduleAttemptedAt?: string; + botScheduleIdempotencyKey?: string; externalRecordingId?: string; callRecorderFailureReason?: string; }; diff --git a/packages/twenty-apps/public/call-recorder/src/logic-functions/types/call-recording-update-fields.type.ts b/packages/twenty-apps/public/call-recorder/src/logic-functions/types/call-recording-update-fields.type.ts index a465f3e824..03eabc041c 100644 --- a/packages/twenty-apps/public/call-recorder/src/logic-functions/types/call-recording-update-fields.type.ts +++ b/packages/twenty-apps/public/call-recorder/src/logic-functions/types/call-recording-update-fields.type.ts @@ -13,6 +13,9 @@ export type CallRecordingUpdateFields = Partial<{ calendarEventId: string; // null clears stale app-owned state on cancel/eject or reschedule. externalBotId: string | null; + // null clears attempt state once its outcome is resolved (bot confirmed gone). + botScheduleAttemptedAt: string | null; + botScheduleIdempotencyKey: string | null; externalRecordingId: string; callRecorderFailureReason: string | null; transcript: Record;