From 6cd069ce40c47827660cea2103794e2312fdf8ef Mon Sep 17 00:00:00 2001 From: neo773 <62795688+neo773@users.noreply.github.com> Date: Tue, 19 May 2026 16:08:34 +0530 Subject: [PATCH] messaging minor perf improvement (#20687) This PR adds two changes 1. Pass `lite:true` to `ExecuteInWorkspaceContextOptions` introduced in https://github.com/twentyhq/twenty/pull/18376 2. Remove redundant gmail alias call, it adds 300ms every cron job, we only do it once now when user connects, realistically I don't see people changing their aliases every day you only set it up once actual real diff is small, it's just prettier format contributing to diff Objective decrease total time take per job --- .../engine/core-modules/auth/auth.module.ts | 2 + .../auth/services/google-apis.service.spec.ts | 7 + .../auth/services/google-apis.service.ts | 16 + .../services/microsoft-apis.service.spec.ts | 7 + .../auth/services/microsoft-apis.service.ts | 18 + ...ocklist-item-delete-calendar-events.job.ts | 250 ++++----- .../blocklist-reimport-calendar-events.job.ts | 92 ++-- .../calendar-event-cleaner.service.ts | 152 +++--- ...lendar-trigger-event-list-fetch.command.ts | 89 ++-- .../jobs/calendar-event-list-fetch.job.ts | 56 +- .../jobs/calendar-events-import.job.ts | 56 +- .../jobs/calendar-ongoing-stale.job.ts | 94 ++-- ...ar-relaunch-failed-calendar-channel.job.ts | 54 +- .../calendar-events-import.service.ts | 248 ++++----- .../services/calendar-fetch-events.service.ts | 150 +++--- .../services/calendar-save-events.service.ts | 494 +++++++++--------- .../calendar-event-participant.service.ts | 231 ++++---- ...-events-visibility-restrictions.service.ts | 1 + .../calendar-channel-sync-status.service.ts | 250 +++++---- .../google-email-alias-manager.service.ts | 27 +- ...ging-blocklist-item-delete-messages.job.ts | 279 +++++----- ...ssaging-blocklist-reimport-messages.job.ts | 106 ++-- ...essages-visibility-restrictions.service.ts | 1 + .../message-channel-sync-status.service.ts | 315 ++++++----- .../messaging-reset-channel.command.ts | 66 +-- .../messaging-message-cleaner.service.ts | 406 +++++++------- .../services/sync-message-folders.service.ts | 1 + ...ging-trigger-message-list-fetch.command.ts | 88 ++-- .../services/inbound-email-import.service.ts | 20 +- .../jobs/messaging-message-list-fetch.job.ts | 100 ++-- .../jobs/messaging-messages-import.job.ts | 66 +-- .../jobs/messaging-ongoing-stale.job.ts | 92 ++-- ...ing-relaunch-failed-message-channel.job.ts | 54 +- .../messaging-messages-import.service.spec.ts | 11 +- .../services/messaging-cursor.service.ts | 68 +-- ...essaging-delete-folder-messages.service.ts | 140 ++--- ...ing-delete-group-email-messages.service.ts | 1 + ...ging-message-folder-association.service.ts | 98 ++-- .../messaging-message-list-fetch.service.ts | 407 ++++++++------- .../services/messaging-message.service.ts | 1 + .../messaging-messages-import.service.ts | 374 ++++++------- ...essaging-process-folder-actions.service.ts | 1 + ...ing-process-group-email-actions.service.ts | 1 + ...es-and-enqueue-contact-creation.service.ts | 1 + .../messaging-message-participant.service.ts | 102 ++-- ...channel-sync-status-monitoring.cron.job.ts | 1 + 46 files changed, 2685 insertions(+), 2409 deletions(-) diff --git a/packages/twenty-server/src/engine/core-modules/auth/auth.module.ts b/packages/twenty-server/src/engine/core-modules/auth/auth.module.ts index 45c35da445..48cfe8c5e2 100644 --- a/packages/twenty-server/src/engine/core-modules/auth/auth.module.ts +++ b/packages/twenty-server/src/engine/core-modules/auth/auth.module.ts @@ -69,6 +69,7 @@ import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadat import { PermissionsModule } from 'src/engine/metadata-modules/permissions/permissions.module'; import { WorkspaceCacheModule } from 'src/engine/workspace-cache/workspace-cache.module'; import { CalendarChannelSyncStatusService } from 'src/modules/calendar/common/services/calendar-channel-sync-status.service'; +import { EmailAliasManagerModule } from 'src/modules/connected-account/email-alias-manager/email-alias-manager.module'; import { ConnectedAccountModule } from 'src/modules/connected-account/connected-account.module'; import { MessagingCommonModule } from 'src/modules/messaging/common/messaging-common.module'; import { MessagingFolderSyncManagerModule } from 'src/modules/messaging/message-folder-manager/messaging-folder-sync-manager.module'; @@ -127,6 +128,7 @@ import { JwtAuthStrategy } from './strategies/jwt.auth.strategy'; EnterpriseModule, FileModule, ConnectedAccountTokenEncryptionModule, + EmailAliasManagerModule, ], controllers: [ GoogleAuthController, diff --git a/packages/twenty-server/src/engine/core-modules/auth/services/google-apis.service.spec.ts b/packages/twenty-server/src/engine/core-modules/auth/services/google-apis.service.spec.ts index c5d2c58b3c..999f55a384 100644 --- a/packages/twenty-server/src/engine/core-modules/auth/services/google-apis.service.spec.ts +++ b/packages/twenty-server/src/engine/core-modules/auth/services/google-apis.service.spec.ts @@ -26,6 +26,7 @@ import { MessageChannelEntity } from 'src/engine/metadata-modules/message-channe import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity'; import { GlobalWorkspaceOrmManager } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager'; import { CalendarChannelSyncStatusService } from 'src/modules/calendar/common/services/calendar-channel-sync-status.service'; +import { EmailAliasManagerService } from 'src/modules/connected-account/email-alias-manager/services/email-alias-manager.service'; import { AccountsToReconnectService } from 'src/modules/connected-account/services/accounts-to-reconnect.service'; import { MessageChannelSyncStatusService } from 'src/modules/messaging/common/services/message-channel-sync-status.service'; import { SyncMessageFoldersService } from 'src/modules/messaging/message-folder-manager/services/sync-message-folders.service'; @@ -195,6 +196,12 @@ describe('GoogleAPIsService', () => { syncMessageFolders: jest.fn().mockResolvedValue([]), }, }, + { + provide: EmailAliasManagerService, + useValue: { + refreshHandleAliases: jest.fn().mockResolvedValue([]), + }, + }, { provide: getRepositoryToken(ConnectedAccountEntity), useValue: mockConnectedAccountRepository, diff --git a/packages/twenty-server/src/engine/core-modules/auth/services/google-apis.service.ts b/packages/twenty-server/src/engine/core-modules/auth/services/google-apis.service.ts index 0b55a87bb0..9dcab5fac9 100644 --- a/packages/twenty-server/src/engine/core-modules/auth/services/google-apis.service.ts +++ b/packages/twenty-server/src/engine/core-modules/auth/services/google-apis.service.ts @@ -38,6 +38,7 @@ import { type CalendarEventListFetchJobData, } from 'src/modules/calendar/calendar-event-import-manager/jobs/calendar-event-list-fetch.job'; import { CalendarChannelSyncStatusService } from 'src/modules/calendar/common/services/calendar-channel-sync-status.service'; +import { EmailAliasManagerService } from 'src/modules/connected-account/email-alias-manager/services/email-alias-manager.service'; import { AccountsToReconnectService } from 'src/modules/connected-account/services/accounts-to-reconnect.service'; import { MessageChannelSyncStatusService } from 'src/modules/messaging/common/services/message-channel-sync-status.service'; @@ -66,6 +67,7 @@ export class GoogleAPIsService { private readonly googleAPIScopesService: GoogleAPIScopesService, private readonly googleApisServiceAvailabilityService: GoogleApisServiceAvailabilityService, private readonly syncMessageFoldersService: SyncMessageFoldersService, + private readonly emailAliasManagerService: EmailAliasManagerService, @InjectRepository(ConnectedAccountEntity) private readonly connectedAccountRepository: Repository, @InjectRepository(UserWorkspaceEntity) @@ -238,6 +240,20 @@ export class GoogleAPIsService { }, ); + if (isMessagingEnabled && isMessagingAvailable) { + const connectedAccountForAliases = + await this.connectedAccountRepository.findOne({ + where: { id: newOrExistingConnectedAccountId, workspaceId }, + }); + + if (isDefined(connectedAccountForAliases)) { + await this.emailAliasManagerService.refreshHandleAliases( + connectedAccountForAliases, + workspaceId, + ); + } + } + if ( isMessagingEnabled && isMessagingAvailable && diff --git a/packages/twenty-server/src/engine/core-modules/auth/services/microsoft-apis.service.spec.ts b/packages/twenty-server/src/engine/core-modules/auth/services/microsoft-apis.service.spec.ts index 896993f994..e15284189c 100644 --- a/packages/twenty-server/src/engine/core-modules/auth/services/microsoft-apis.service.spec.ts +++ b/packages/twenty-server/src/engine/core-modules/auth/services/microsoft-apis.service.spec.ts @@ -23,6 +23,7 @@ import { MessageChannelEntity } from 'src/engine/metadata-modules/message-channe import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity'; import { GlobalWorkspaceOrmManager } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager'; import { CalendarChannelSyncStatusService } from 'src/modules/calendar/common/services/calendar-channel-sync-status.service'; +import { EmailAliasManagerService } from 'src/modules/connected-account/email-alias-manager/services/email-alias-manager.service'; import { AccountsToReconnectService } from 'src/modules/connected-account/services/accounts-to-reconnect.service'; import { MessageChannelSyncStatusService } from 'src/modules/messaging/common/services/message-channel-sync-status.service'; import { SyncMessageFoldersService } from 'src/modules/messaging/message-folder-manager/services/sync-message-folders.service'; @@ -185,6 +186,12 @@ describe('MicrosoftAPIsService', () => { syncMessageFolders: jest.fn().mockResolvedValue([]), }, }, + { + provide: EmailAliasManagerService, + useValue: { + refreshHandleAliases: jest.fn().mockResolvedValue([]), + }, + }, ], }).compile(); diff --git a/packages/twenty-server/src/engine/core-modules/auth/services/microsoft-apis.service.ts b/packages/twenty-server/src/engine/core-modules/auth/services/microsoft-apis.service.ts index c3dd9297ed..eb96e017a0 100644 --- a/packages/twenty-server/src/engine/core-modules/auth/services/microsoft-apis.service.ts +++ b/packages/twenty-server/src/engine/core-modules/auth/services/microsoft-apis.service.ts @@ -36,6 +36,7 @@ import { type CalendarEventListFetchJobData, } from 'src/modules/calendar/calendar-event-import-manager/jobs/calendar-event-list-fetch.job'; import { CalendarChannelSyncStatusService } from 'src/modules/calendar/common/services/calendar-channel-sync-status.service'; +import { EmailAliasManagerService } from 'src/modules/connected-account/email-alias-manager/services/email-alias-manager.service'; import { AccountsToReconnectService } from 'src/modules/connected-account/services/accounts-to-reconnect.service'; import { MessageChannelSyncStatusService } from 'src/modules/messaging/common/services/message-channel-sync-status.service'; @@ -62,6 +63,7 @@ export class MicrosoftAPIsService { private readonly updateConnectedAccountOnReconnectService: UpdateConnectedAccountOnReconnectService, private readonly twentyConfigService: TwentyConfigService, private readonly syncMessageFoldersService: SyncMessageFoldersService, + private readonly emailAliasManagerService: EmailAliasManagerService, @InjectRepository(ConnectedAccountEntity) private readonly connectedAccountRepository: Repository, @InjectRepository(UserWorkspaceEntity) @@ -216,6 +218,22 @@ export class MicrosoftAPIsService { }, ); + if ( + this.twentyConfigService.get('MESSAGING_PROVIDER_MICROSOFT_ENABLED') + ) { + const connectedAccountForAliases = + await this.connectedAccountRepository.findOne({ + where: { id: newOrExistingConnectedAccountId, workspaceId }, + }); + + if (isDefined(connectedAccountForAliases)) { + await this.emailAliasManagerService.refreshHandleAliases( + connectedAccountForAliases, + workspaceId, + ); + } + } + if ( this.twentyConfigService.get( 'MESSAGING_PROVIDER_MICROSOFT_ENABLED', diff --git a/packages/twenty-server/src/modules/calendar/blocklist-manager/jobs/blocklist-item-delete-calendar-events.job.ts b/packages/twenty-server/src/modules/calendar/blocklist-manager/jobs/blocklist-item-delete-calendar-events.job.ts index e9baeec97a..7fb30dbbb4 100644 --- a/packages/twenty-server/src/modules/calendar/blocklist-manager/jobs/blocklist-item-delete-calendar-events.job.ts +++ b/packages/twenty-server/src/modules/calendar/blocklist-manager/jobs/blocklist-item-delete-calendar-events.job.ts @@ -41,145 +41,149 @@ export class BlocklistItemDeleteCalendarEventsJob { const authContext = buildSystemAuthContext(workspaceId); - await this.globalWorkspaceOrmManager.executeInWorkspaceContext(async () => { - const blocklistItemIds = data.events.map( - (eventPayload) => eventPayload.recordId, - ); - - const blocklistRepository = - await this.globalWorkspaceOrmManager.getRepository( - workspaceId, - 'blocklist', + await this.globalWorkspaceOrmManager.executeInWorkspaceContext( + async () => { + const blocklistItemIds = data.events.map( + (eventPayload) => eventPayload.recordId, ); - const blocklist = await blocklistRepository.find({ - where: { - id: Any(blocklistItemIds), - }, - }); - - const handlesToDeleteByWorkspaceMemberIdMap = blocklist.reduce( - (acc, blocklistItem) => { - const { handle, workspaceMemberId } = blocklistItem; - - if (!acc.has(workspaceMemberId)) { - acc.set(workspaceMemberId, []); - } - - if (!isDefined(handle)) { - return acc; - } - - acc.get(workspaceMemberId)?.push(handle); - - return acc; - }, - new Map(), - ); - - const calendarChannelEventAssociationRepository = - await this.globalWorkspaceOrmManager.getRepository( - workspaceId, - 'calendarChannelEventAssociation', - ); - - const workspaceMemberRepository = - await this.globalWorkspaceOrmManager.getRepository( - workspaceId, - 'workspaceMember', - { shouldBypassPermissionChecks: true }, - ); - - for (const workspaceMemberId of handlesToDeleteByWorkspaceMemberIdMap.keys()) { - const handles = - handlesToDeleteByWorkspaceMemberIdMap.get(workspaceMemberId); - - if (!handles) { - continue; - } - - const workspaceMember = await workspaceMemberRepository.findOne({ - where: { id: workspaceMemberId }, - }); - - if (!workspaceMember) { - continue; - } - - const userWorkspace = await this.userWorkspaceRepository.findOne({ - where: { userId: workspaceMember.userId, workspaceId }, - select: ['id'], - }); - - if (!userWorkspace) { - continue; - } - - const calendarChannels = await this.calendarChannelRepository.find({ - select: { - id: true, - handle: true, - connectedAccount: { - handleAliases: true, - }, - }, - where: { - connectedAccount: { userWorkspaceId: userWorkspace.id }, + const blocklistRepository = + await this.globalWorkspaceOrmManager.getRepository( workspaceId, + 'blocklist', + ); + + const blocklist = await blocklistRepository.find({ + where: { + id: Any(blocklistItemIds), }, - relations: ['connectedAccount'], }); - for (const calendarChannel of calendarChannels) { - const calendarChannelHandles = [calendarChannel.handle]; + const handlesToDeleteByWorkspaceMemberIdMap = blocklist.reduce( + (acc, blocklistItem) => { + const { handle, workspaceMemberId } = blocklistItem; - if (calendarChannel.connectedAccount.handleAliases) { - const rawAliases = calendarChannel.connectedAccount - .handleAliases as string | string[]; + if (!acc.has(workspaceMemberId)) { + acc.set(workspaceMemberId, []); + } - const aliasList = Array.isArray(rawAliases) - ? rawAliases - : rawAliases.split(',').map((alias: string) => alias.trim()); + if (!isDefined(handle)) { + return acc; + } - calendarChannelHandles.push(...aliasList); - } + acc.get(workspaceMemberId)?.push(handle); - const handleConditions = handles.map((handle) => { - const isHandleDomain = handle.startsWith('@'); + return acc; + }, + new Map(), + ); - return isHandleDomain - ? { - handle: And( - Or(ILike(`%${handle}`), ILike(`%.${handle.slice(1)}`)), - Not(In(calendarChannelHandles)), - ), - } - : { handle }; - }); + const calendarChannelEventAssociationRepository = + await this.globalWorkspaceOrmManager.getRepository( + workspaceId, + 'calendarChannelEventAssociation', + ); - const calendarEventsAssociationsToDelete = - await calendarChannelEventAssociationRepository.find({ - where: { - calendarChannelId: calendarChannel.id, - calendarEvent: { - calendarEventParticipants: handleConditions, - }, - }, - }); + const workspaceMemberRepository = + await this.globalWorkspaceOrmManager.getRepository( + workspaceId, + 'workspaceMember', + { shouldBypassPermissionChecks: true }, + ); - if (calendarEventsAssociationsToDelete.length === 0) { + for (const workspaceMemberId of handlesToDeleteByWorkspaceMemberIdMap.keys()) { + const handles = + handlesToDeleteByWorkspaceMemberIdMap.get(workspaceMemberId); + + if (!handles) { continue; } - await calendarChannelEventAssociationRepository.delete( - calendarEventsAssociationsToDelete.map(({ id }) => id), - ); - } - } + const workspaceMember = await workspaceMemberRepository.findOne({ + where: { id: workspaceMemberId }, + }); - await this.calendarEventCleanerService.cleanWorkspaceCalendarEvents( - workspaceId, - ); - }, authContext); + if (!workspaceMember) { + continue; + } + + const userWorkspace = await this.userWorkspaceRepository.findOne({ + where: { userId: workspaceMember.userId, workspaceId }, + select: ['id'], + }); + + if (!userWorkspace) { + continue; + } + + const calendarChannels = await this.calendarChannelRepository.find({ + select: { + id: true, + handle: true, + connectedAccount: { + handleAliases: true, + }, + }, + where: { + connectedAccount: { userWorkspaceId: userWorkspace.id }, + workspaceId, + }, + relations: ['connectedAccount'], + }); + + for (const calendarChannel of calendarChannels) { + const calendarChannelHandles = [calendarChannel.handle]; + + if (calendarChannel.connectedAccount.handleAliases) { + const rawAliases = calendarChannel.connectedAccount + .handleAliases as string | string[]; + + const aliasList = Array.isArray(rawAliases) + ? rawAliases + : rawAliases.split(',').map((alias: string) => alias.trim()); + + calendarChannelHandles.push(...aliasList); + } + + const handleConditions = handles.map((handle) => { + const isHandleDomain = handle.startsWith('@'); + + return isHandleDomain + ? { + handle: And( + Or(ILike(`%${handle}`), ILike(`%.${handle.slice(1)}`)), + Not(In(calendarChannelHandles)), + ), + } + : { handle }; + }); + + const calendarEventsAssociationsToDelete = + await calendarChannelEventAssociationRepository.find({ + where: { + calendarChannelId: calendarChannel.id, + calendarEvent: { + calendarEventParticipants: handleConditions, + }, + }, + }); + + if (calendarEventsAssociationsToDelete.length === 0) { + continue; + } + + await calendarChannelEventAssociationRepository.delete( + calendarEventsAssociationsToDelete.map(({ id }) => id), + ); + } + } + + await this.calendarEventCleanerService.cleanWorkspaceCalendarEvents( + workspaceId, + ); + }, + authContext, + { lite: true }, + ); } } diff --git a/packages/twenty-server/src/modules/calendar/blocklist-manager/jobs/blocklist-reimport-calendar-events.job.ts b/packages/twenty-server/src/modules/calendar/blocklist-manager/jobs/blocklist-reimport-calendar-events.job.ts index 4a19eb3b73..e472a0c64c 100644 --- a/packages/twenty-server/src/modules/calendar/blocklist-manager/jobs/blocklist-reimport-calendar-events.job.ts +++ b/packages/twenty-server/src/modules/calendar/blocklist-manager/jobs/blocklist-reimport-calendar-events.job.ts @@ -41,51 +41,55 @@ export class BlocklistReimportCalendarEventsJob { const authContext = buildSystemAuthContext(workspaceId); - await this.globalWorkspaceOrmManager.executeInWorkspaceContext(async () => { - const workspaceMemberRepository = - await this.globalWorkspaceOrmManager.getRepository( - workspaceId, - 'workspaceMember', - { shouldBypassPermissionChecks: true }, - ); - - for (const eventPayload of data.events) { - const workspaceMemberId = - eventPayload.properties.before.workspaceMemberId; - - const workspaceMember = await workspaceMemberRepository.findOne({ - where: { id: workspaceMemberId }, - }); - - if (!isDefined(workspaceMember)) { - continue; - } - - const userWorkspace = await this.userWorkspaceRepository.findOne({ - where: { userId: workspaceMember.userId, workspaceId }, - select: ['id'], - }); - - if (!isDefined(userWorkspace)) { - continue; - } - - const calendarChannels = await this.calendarChannelRepository.find({ - select: ['id'], - where: { - connectedAccount: { userWorkspaceId: userWorkspace.id }, - syncStage: Not( - CalendarChannelSyncStage.CALENDAR_EVENT_LIST_FETCH_PENDING, - ), + await this.globalWorkspaceOrmManager.executeInWorkspaceContext( + async () => { + const workspaceMemberRepository = + await this.globalWorkspaceOrmManager.getRepository( workspaceId, - }, - }); + 'workspaceMember', + { shouldBypassPermissionChecks: true }, + ); - await this.calendarChannelSyncStatusService.resetAndMarkAsCalendarEventListFetchPending( - calendarChannels.map((calendarChannel) => calendarChannel.id), - workspaceId, - ); - } - }, authContext); + for (const eventPayload of data.events) { + const workspaceMemberId = + eventPayload.properties.before.workspaceMemberId; + + const workspaceMember = await workspaceMemberRepository.findOne({ + where: { id: workspaceMemberId }, + }); + + if (!isDefined(workspaceMember)) { + continue; + } + + const userWorkspace = await this.userWorkspaceRepository.findOne({ + where: { userId: workspaceMember.userId, workspaceId }, + select: ['id'], + }); + + if (!isDefined(userWorkspace)) { + continue; + } + + const calendarChannels = await this.calendarChannelRepository.find({ + select: ['id'], + where: { + connectedAccount: { userWorkspaceId: userWorkspace.id }, + syncStage: Not( + CalendarChannelSyncStage.CALENDAR_EVENT_LIST_FETCH_PENDING, + ), + workspaceId, + }, + }); + + await this.calendarChannelSyncStatusService.resetAndMarkAsCalendarEventListFetchPending( + calendarChannels.map((calendarChannel) => calendarChannel.id), + workspaceId, + ); + } + }, + authContext, + { lite: true }, + ); } } diff --git a/packages/twenty-server/src/modules/calendar/calendar-event-cleaner/services/calendar-event-cleaner.service.ts b/packages/twenty-server/src/modules/calendar/calendar-event-cleaner/services/calendar-event-cleaner.service.ts index 169fa10527..aa2570bd88 100644 --- a/packages/twenty-server/src/modules/calendar/calendar-event-cleaner/services/calendar-event-cleaner.service.ts +++ b/packages/twenty-server/src/modules/calendar/calendar-event-cleaner/services/calendar-event-cleaner.service.ts @@ -24,90 +24,98 @@ export class CalendarEventCleanerService { }) { const authContext = buildSystemAuthContext(workspaceId); - await this.globalWorkspaceOrmManager.executeInWorkspaceContext(async () => { - const calendarChannelEventAssociationRepository = - await this.globalWorkspaceOrmManager.getRepository( - workspaceId, - 'calendarChannelEventAssociation', - ); + await this.globalWorkspaceOrmManager.executeInWorkspaceContext( + async () => { + const calendarChannelEventAssociationRepository = + await this.globalWorkspaceOrmManager.getRepository( + workspaceId, + 'calendarChannelEventAssociation', + ); - const workspaceDataSource = - await this.globalWorkspaceOrmManager.getGlobalWorkspaceDataSource(); + const workspaceDataSource = + await this.globalWorkspaceOrmManager.getGlobalWorkspaceDataSource(); - await workspaceDataSource.transaction(async (manager) => { - const transactionManager = manager as WorkspaceEntityManager; + await workspaceDataSource.transaction(async (manager) => { + const transactionManager = manager as WorkspaceEntityManager; - await deleteUsingPagination( - workspaceId, - 500, - async ( - limit: number, - offset: number, - _workspaceId: string, - transactionManager?: WorkspaceEntityManager, - ) => { - const associations = - await calendarChannelEventAssociationRepository.find( - { - where: { calendarChannelId }, - take: limit, - skip: offset, - }, + await deleteUsingPagination( + workspaceId, + 500, + async ( + limit: number, + offset: number, + _workspaceId: string, + transactionManager?: WorkspaceEntityManager, + ) => { + const associations = + await calendarChannelEventAssociationRepository.find( + { + where: { calendarChannelId }, + take: limit, + skip: offset, + }, + transactionManager, + ); + + return associations.map(({ id }) => id); + }, + async ( + ids: string[], + workspaceId: string, + transactionManager?: WorkspaceEntityManager, + ) => { + this.logger.log( + `WorkspaceId: ${workspaceId} Deleting ${ids.length} calendar channel event associations for channel ${calendarChannelId}`, + ); + await calendarChannelEventAssociationRepository.delete( + ids, transactionManager, ); - - return associations.map(({ id }) => id); - }, - async ( - ids: string[], - workspaceId: string, - transactionManager?: WorkspaceEntityManager, - ) => { - this.logger.log( - `WorkspaceId: ${workspaceId} Deleting ${ids.length} calendar channel event associations for channel ${calendarChannelId}`, - ); - await calendarChannelEventAssociationRepository.delete( - ids, - transactionManager, - ); - }, - transactionManager, - ); - }); - }, authContext); + }, + transactionManager, + ); + }); + }, + authContext, + { lite: true }, + ); } public async cleanWorkspaceCalendarEvents(workspaceId: string) { const authContext = buildSystemAuthContext(workspaceId); - await this.globalWorkspaceOrmManager.executeInWorkspaceContext(async () => { - const calendarEventRepository = - await this.globalWorkspaceOrmManager.getRepository( + await this.globalWorkspaceOrmManager.executeInWorkspaceContext( + async () => { + const calendarEventRepository = + await this.globalWorkspaceOrmManager.getRepository( + workspaceId, + 'calendarEvent', + ); + + await deleteUsingPagination( workspaceId, - 'calendarEvent', - ); - - await deleteUsingPagination( - workspaceId, - 500, - async (limit, offset) => { - const nonAssociatedCalendarEvents = - await calendarEventRepository.find({ - where: { - calendarChannelEventAssociations: { - id: IsNull(), + 500, + async (limit, offset) => { + const nonAssociatedCalendarEvents = + await calendarEventRepository.find({ + where: { + calendarChannelEventAssociations: { + id: IsNull(), + }, }, - }, - take: limit, - skip: offset, - }); + take: limit, + skip: offset, + }); - return nonAssociatedCalendarEvents.map(({ id }) => id); - }, - async (ids) => { - await calendarEventRepository.delete({ id: Any(ids) }); - }, - ); - }, authContext); + return nonAssociatedCalendarEvents.map(({ id }) => id); + }, + async (ids) => { + await calendarEventRepository.delete({ id: Any(ids) }); + }, + ); + }, + authContext, + { lite: true }, + ); } } diff --git a/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/commands/calendar-trigger-event-list-fetch.command.ts b/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/commands/calendar-trigger-event-list-fetch.command.ts index e69ca9ade0..3e7effa064 100644 --- a/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/commands/calendar-trigger-event-list-fetch.command.ts +++ b/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/commands/calendar-trigger-event-list-fetch.command.ts @@ -53,55 +53,60 @@ export class CalendarTriggerEventListFetchCommand extends CommandRunner { const authContext = buildSystemAuthContext(workspaceId); - await this.globalWorkspaceOrmManager.executeInWorkspaceContext(async () => { - const calendarChannels = await this.calendarChannelRepository.find({ - where: { - isSyncEnabled: true, - syncStage: CalendarChannelSyncStage.CALENDAR_EVENT_LIST_FETCH_PENDING, - ...(calendarChannelId ? { id: calendarChannelId } : {}), - workspaceId, - }, - }); - - if (calendarChannels.length === 0) { - this.logger.warn( - 'No calendar channels found with CALENDAR_EVENT_LIST_FETCH_PENDING status', - ); - - return; - } - - this.logger.log( - `Found ${calendarChannels.length} calendar channel(s) to process`, - ); - - for (const calendarChannel of calendarChannels) { - await this.calendarChannelRepository.update( - { id: calendarChannel.id, workspaceId }, - { + await this.globalWorkspaceOrmManager.executeInWorkspaceContext( + async () => { + const calendarChannels = await this.calendarChannelRepository.find({ + where: { + isSyncEnabled: true, syncStage: - CalendarChannelSyncStage.CALENDAR_EVENT_LIST_FETCH_SCHEDULED, - syncStageStartedAt: new Date().toISOString(), - }, - ); - - await this.messageQueueService.add( - CalendarEventListFetchJob.name, - { - calendarChannelId: calendarChannel.id, + CalendarChannelSyncStage.CALENDAR_EVENT_LIST_FETCH_PENDING, + ...(calendarChannelId ? { id: calendarChannelId } : {}), workspaceId, }, - ); + }); + + if (calendarChannels.length === 0) { + this.logger.warn( + 'No calendar channels found with CALENDAR_EVENT_LIST_FETCH_PENDING status', + ); + + return; + } this.logger.log( - `Triggered fetch for calendar channel ${calendarChannel.id}`, + `Found ${calendarChannels.length} calendar channel(s) to process`, ); - } - this.logger.log( - `Successfully triggered ${calendarChannels.length} calendar event list fetch job(s)`, - ); - }, authContext); + for (const calendarChannel of calendarChannels) { + await this.calendarChannelRepository.update( + { id: calendarChannel.id, workspaceId }, + { + syncStage: + CalendarChannelSyncStage.CALENDAR_EVENT_LIST_FETCH_SCHEDULED, + syncStageStartedAt: new Date().toISOString(), + }, + ); + + await this.messageQueueService.add( + CalendarEventListFetchJob.name, + { + calendarChannelId: calendarChannel.id, + workspaceId, + }, + ); + + this.logger.log( + `Triggered fetch for calendar channel ${calendarChannel.id}`, + ); + } + + this.logger.log( + `Successfully triggered ${calendarChannels.length} calendar event list fetch job(s)`, + ); + }, + authContext, + { lite: true }, + ); } @Option({ diff --git a/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/jobs/calendar-event-list-fetch.job.ts b/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/jobs/calendar-event-list-fetch.job.ts index 2271432210..3541735076 100644 --- a/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/jobs/calendar-event-list-fetch.job.ts +++ b/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/jobs/calendar-event-list-fetch.job.ts @@ -35,32 +35,36 @@ export class CalendarEventListFetchJob { const authContext = buildSystemAuthContext(workspaceId); - await this.globalWorkspaceOrmManager.executeInWorkspaceContext(async () => { - const calendarChannel = await this.calendarChannelRepository.findOne({ - where: { - id: calendarChannelId, - isSyncEnabled: true, + await this.globalWorkspaceOrmManager.executeInWorkspaceContext( + async () => { + const calendarChannel = await this.calendarChannelRepository.findOne({ + where: { + id: calendarChannelId, + isSyncEnabled: true, + workspaceId, + }, + relations: ['connectedAccount'], + }); + + if (!calendarChannel) { + return; + } + + if ( + calendarChannel.syncStage !== + CalendarChannelSyncStage.CALENDAR_EVENT_LIST_FETCH_SCHEDULED + ) { + return; + } + + await this.calendarFetchEventsService.fetchCalendarEvents( + calendarChannel as unknown as CalendarChannelEntity, + calendarChannel.connectedAccount, workspaceId, - }, - relations: ['connectedAccount'], - }); - - if (!calendarChannel) { - return; - } - - if ( - calendarChannel.syncStage !== - CalendarChannelSyncStage.CALENDAR_EVENT_LIST_FETCH_SCHEDULED - ) { - return; - } - - await this.calendarFetchEventsService.fetchCalendarEvents( - calendarChannel as unknown as CalendarChannelEntity, - calendarChannel.connectedAccount, - workspaceId, - ); - }, authContext); + ); + }, + authContext, + { lite: true }, + ); } } diff --git a/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/jobs/calendar-events-import.job.ts b/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/jobs/calendar-events-import.job.ts index 929aafa482..7ddf2d69e5 100644 --- a/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/jobs/calendar-events-import.job.ts +++ b/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/jobs/calendar-events-import.job.ts @@ -35,32 +35,36 @@ export class CalendarEventsImportJob { const authContext = buildSystemAuthContext(workspaceId); - await this.globalWorkspaceOrmManager.executeInWorkspaceContext(async () => { - const calendarChannel = await this.calendarChannelRepository.findOne({ - where: { - id: calendarChannelId, - isSyncEnabled: true, + await this.globalWorkspaceOrmManager.executeInWorkspaceContext( + async () => { + const calendarChannel = await this.calendarChannelRepository.findOne({ + where: { + id: calendarChannelId, + isSyncEnabled: true, + workspaceId, + }, + relations: ['connectedAccount'], + }); + + if (!calendarChannel?.isSyncEnabled) { + return; + } + + if ( + calendarChannel.syncStage !== + CalendarChannelSyncStage.CALENDAR_EVENTS_IMPORT_SCHEDULED + ) { + return; + } + + await this.calendarEventsImportService.processCalendarEventsImport( + calendarChannel as unknown as CalendarChannelEntity, + calendarChannel.connectedAccount, workspaceId, - }, - relations: ['connectedAccount'], - }); - - if (!calendarChannel?.isSyncEnabled) { - return; - } - - if ( - calendarChannel.syncStage !== - CalendarChannelSyncStage.CALENDAR_EVENTS_IMPORT_SCHEDULED - ) { - return; - } - - await this.calendarEventsImportService.processCalendarEventsImport( - calendarChannel as unknown as CalendarChannelEntity, - calendarChannel.connectedAccount, - workspaceId, - ); - }, authContext); + ); + }, + authContext, + { lite: true }, + ); } } diff --git a/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/jobs/calendar-ongoing-stale.job.ts b/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/jobs/calendar-ongoing-stale.job.ts index 30f11f7f7c..62b6180ad1 100644 --- a/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/jobs/calendar-ongoing-stale.job.ts +++ b/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/jobs/calendar-ongoing-stale.job.ts @@ -36,54 +36,58 @@ export class CalendarOngoingStaleJob { const authContext = buildSystemAuthContext(workspaceId); - await this.globalWorkspaceOrmManager.executeInWorkspaceContext(async () => { - const calendarChannels = await this.calendarChannelRepository.find({ - where: { - syncStage: In([ - CalendarChannelSyncStage.CALENDAR_EVENTS_IMPORT_ONGOING, - CalendarChannelSyncStage.CALENDAR_EVENT_LIST_FETCH_ONGOING, - CalendarChannelSyncStage.CALENDAR_EVENTS_IMPORT_SCHEDULED, - CalendarChannelSyncStage.CALENDAR_EVENT_LIST_FETCH_SCHEDULED, - ]), - workspaceId, - }, - }); - - for (const calendarChannel of calendarChannels) { - const syncStageStartedAt = calendarChannel.syncStageStartedAt; - - if (isSyncStale(syncStageStartedAt?.toISOString() ?? null)) { - await this.calendarChannelSyncStatusService.resetSyncStageStartedAt( - [calendarChannel.id], + await this.globalWorkspaceOrmManager.executeInWorkspaceContext( + async () => { + const calendarChannels = await this.calendarChannelRepository.find({ + where: { + syncStage: In([ + CalendarChannelSyncStage.CALENDAR_EVENTS_IMPORT_ONGOING, + CalendarChannelSyncStage.CALENDAR_EVENT_LIST_FETCH_ONGOING, + CalendarChannelSyncStage.CALENDAR_EVENTS_IMPORT_SCHEDULED, + CalendarChannelSyncStage.CALENDAR_EVENT_LIST_FETCH_SCHEDULED, + ]), workspaceId, - ); + }, + }); - switch (calendarChannel.syncStage) { - case CalendarChannelSyncStage.CALENDAR_EVENT_LIST_FETCH_ONGOING: - case CalendarChannelSyncStage.CALENDAR_EVENT_LIST_FETCH_SCHEDULED: - this.logger.log( - `Sync for calendar channel ${calendarChannel.id} and workspace ${workspaceId} is stale. Setting sync stage to CALENDAR_EVENT_LIST_FETCH_PENDING`, - ); - await this.calendarChannelSyncStatusService.markAsCalendarEventListFetchPending( - [calendarChannel.id], - workspaceId, - ); - break; - case CalendarChannelSyncStage.CALENDAR_EVENTS_IMPORT_ONGOING: - case CalendarChannelSyncStage.CALENDAR_EVENTS_IMPORT_SCHEDULED: - this.logger.log( - `Sync for calendar channel ${calendarChannel.id} and workspace ${workspaceId} is stale. Setting sync stage to CALENDAR_EVENTS_IMPORT_PENDING`, - ); - await this.calendarChannelSyncStatusService.markAsCalendarEventsImportPending( - [calendarChannel.id], - workspaceId, - ); - break; - default: - break; + for (const calendarChannel of calendarChannels) { + const syncStageStartedAt = calendarChannel.syncStageStartedAt; + + if (isSyncStale(syncStageStartedAt?.toISOString() ?? null)) { + await this.calendarChannelSyncStatusService.resetSyncStageStartedAt( + [calendarChannel.id], + workspaceId, + ); + + switch (calendarChannel.syncStage) { + case CalendarChannelSyncStage.CALENDAR_EVENT_LIST_FETCH_ONGOING: + case CalendarChannelSyncStage.CALENDAR_EVENT_LIST_FETCH_SCHEDULED: + this.logger.log( + `Sync for calendar channel ${calendarChannel.id} and workspace ${workspaceId} is stale. Setting sync stage to CALENDAR_EVENT_LIST_FETCH_PENDING`, + ); + await this.calendarChannelSyncStatusService.markAsCalendarEventListFetchPending( + [calendarChannel.id], + workspaceId, + ); + break; + case CalendarChannelSyncStage.CALENDAR_EVENTS_IMPORT_ONGOING: + case CalendarChannelSyncStage.CALENDAR_EVENTS_IMPORT_SCHEDULED: + this.logger.log( + `Sync for calendar channel ${calendarChannel.id} and workspace ${workspaceId} is stale. Setting sync stage to CALENDAR_EVENTS_IMPORT_PENDING`, + ); + await this.calendarChannelSyncStatusService.markAsCalendarEventsImportPending( + [calendarChannel.id], + workspaceId, + ); + break; + default: + break; + } } } - } - }, authContext); + }, + authContext, + { lite: true }, + ); } } diff --git a/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/jobs/calendar-relaunch-failed-calendar-channel.job.ts b/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/jobs/calendar-relaunch-failed-calendar-channel.job.ts index 3a110de506..75c3fb61ab 100644 --- a/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/jobs/calendar-relaunch-failed-calendar-channel.job.ts +++ b/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/jobs/calendar-relaunch-failed-calendar-channel.job.ts @@ -36,31 +36,37 @@ export class CalendarRelaunchFailedCalendarChannelJob { const authContext = buildSystemAuthContext(workspaceId); - await this.globalWorkspaceOrmManager.executeInWorkspaceContext(async () => { - const calendarChannel = await this.calendarChannelRepository.findOne({ - where: { - id: calendarChannelId, - workspaceId, - }, - }); + await this.globalWorkspaceOrmManager.executeInWorkspaceContext( + async () => { + const calendarChannel = await this.calendarChannelRepository.findOne({ + where: { + id: calendarChannelId, + workspaceId, + }, + }); - if ( - !calendarChannel || - calendarChannel.syncStage !== CalendarChannelSyncStage.FAILED || - calendarChannel.syncStatus !== CalendarChannelSyncStatus.FAILED_UNKNOWN - ) { - return; - } + if ( + !calendarChannel || + calendarChannel.syncStage !== CalendarChannelSyncStage.FAILED || + calendarChannel.syncStatus !== + CalendarChannelSyncStatus.FAILED_UNKNOWN + ) { + return; + } - await this.calendarChannelRepository.update( - { id: calendarChannelId, workspaceId }, - { - syncStage: CalendarChannelSyncStage.CALENDAR_EVENT_LIST_FETCH_PENDING, - syncStatus: CalendarChannelSyncStatus.ACTIVE, - throttleFailureCount: 0, - syncStageStartedAt: null, - }, - ); - }, authContext); + await this.calendarChannelRepository.update( + { id: calendarChannelId, workspaceId }, + { + syncStage: + CalendarChannelSyncStage.CALENDAR_EVENT_LIST_FETCH_PENDING, + syncStatus: CalendarChannelSyncStatus.ACTIVE, + throttleFailureCount: 0, + syncStageStartedAt: null, + }, + ); + }, + authContext, + { lite: true }, + ); } } diff --git a/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/services/calendar-events-import.service.ts b/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/services/calendar-events-import.service.ts index 524a5f6db9..d8c053742a 100644 --- a/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/services/calendar-events-import.service.ts +++ b/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/services/calendar-events-import.service.ts @@ -65,142 +65,146 @@ export class CalendarEventsImportService { const authContext = buildSystemAuthContext(workspaceId); - await this.globalWorkspaceOrmManager.executeInWorkspaceContext(async () => { - let calendarEvents: FetchedCalendarEvent[] = []; + await this.globalWorkspaceOrmManager.executeInWorkspaceContext( + async () => { + let calendarEvents: FetchedCalendarEvent[] = []; - try { - if (fetchedCalendarEvents) { - calendarEvents = fetchedCalendarEvents; - } else { - const eventIdsToFetch: string[] = await this.cacheStorage.setPop( - `calendar-events-to-import:${workspaceId}:${calendarChannel.id}`, - CALENDAR_EVENT_IMPORT_BATCH_SIZE, - ); + try { + if (fetchedCalendarEvents) { + calendarEvents = fetchedCalendarEvents; + } else { + const eventIdsToFetch: string[] = await this.cacheStorage.setPop( + `calendar-events-to-import:${workspaceId}:${calendarChannel.id}`, + CALENDAR_EVENT_IMPORT_BATCH_SIZE, + ); - if (!eventIdsToFetch || eventIdsToFetch.length === 0) { - await this.calendarChannelSyncStatusService.markAsCompletedAndMarkAsCalendarEventListFetchPending( + if (!eventIdsToFetch || eventIdsToFetch.length === 0) { + await this.calendarChannelSyncStatusService.markAsCompletedAndMarkAsCalendarEventListFetchPending( + [calendarChannel.id], + workspaceId, + ); + + return; + } + + switch (connectedAccount.provider) { + case 'microsoft': + calendarEvents = + await this.microsoftCalendarImportEventService.getCalendarEvents( + connectedAccount, + eventIdsToFetch, + ); + break; + default: + break; + } + } + + if (!calendarEvents || calendarEvents?.length === 0) { + await this.calendarChannelSyncStatusService.markAsCalendarEventListFetchPending( [calendarChannel.id], workspaceId, ); - - return; } - switch (connectedAccount.provider) { - case 'microsoft': - calendarEvents = - await this.microsoftCalendarImportEventService.getCalendarEvents( - connectedAccount, - eventIdsToFetch, - ); - break; - default: - break; - } - } + const userWorkspace = await this.userWorkspaceRepository.findOne({ + where: { + id: (connectedAccount as unknown as { userWorkspaceId: string }) + .userWorkspaceId, + }, + }); - if (!calendarEvents || calendarEvents?.length === 0) { - await this.calendarChannelSyncStatusService.markAsCalendarEventListFetchPending( + const workspaceMemberRepository = + await this.globalWorkspaceOrmManager.getRepository( + workspaceId, + 'workspaceMember', + { shouldBypassPermissionChecks: true }, + ); + + const workspaceMember = userWorkspace + ? await workspaceMemberRepository.findOne({ + where: { userId: userWorkspace.userId }, + }) + : null; + + const blocklist = workspaceMember + ? await this.blocklistRepository.getByWorkspaceMemberId( + workspaceMember.id, + workspaceId, + ) + : []; + + if (!isDefined(connectedAccount.handleAliases)) { + connectedAccount.handleAliases = + await this.emailAliasManagerService.refreshHandleAliases( + connectedAccount, + workspaceId, + ); + } + + if ( + !isDefined(connectedAccount.handleAliases) || + !isDefined(calendarChannel.handle) + ) { + throw new CalendarEventImportDriverException( + 'Calendar channel handle or Handle aliases are required', + CalendarEventImportDriverExceptionCode.CHANNEL_MISCONFIGURED, + ); + } + + const { filteredEvents, cancelledEvents } = + filterEventsAndReturnCancelledEvents( + [calendarChannel.handle, ...connectedAccount.handleAliases], + calendarEvents, + blocklist.map((blocklist) => blocklist.handle ?? ''), + ); + + const cancelledEventExternalIds = cancelledEvents.map( + (event) => event.id, + ); + + const BATCH_SIZE = 1000; + + for (let i = 0; i < filteredEvents.length; i = i + BATCH_SIZE) { + const eventsBatch = filteredEvents.slice(i, i + BATCH_SIZE); + + await this.calendarSaveEventsService.saveCalendarEventsAndEnqueueContactCreationJob( + eventsBatch, + calendarChannel, + connectedAccount, + workspaceId, + ); + } + const calendarChannelEventAssociationRepository = + await this.globalWorkspaceOrmManager.getRepository( + workspaceId, + 'calendarChannelEventAssociation', + ); + + await calendarChannelEventAssociationRepository.delete({ + eventExternalId: Any(cancelledEventExternalIds), + calendarChannelId: calendarChannel.id, + }); + + await this.calendarEventCleanerService.cleanWorkspaceCalendarEvents( + workspaceId, + ); + + await this.calendarChannelSyncStatusService.markAsCompletedAndMarkAsCalendarEventListFetchPending( [calendarChannel.id], workspaceId, ); - } - - const userWorkspace = await this.userWorkspaceRepository.findOne({ - where: { - id: (connectedAccount as unknown as { userWorkspaceId: string }) - .userWorkspaceId, - }, - }); - - const workspaceMemberRepository = - await this.globalWorkspaceOrmManager.getRepository( - workspaceId, - 'workspaceMember', - { shouldBypassPermissionChecks: true }, - ); - - const workspaceMember = userWorkspace - ? await workspaceMemberRepository.findOne({ - where: { userId: userWorkspace.userId }, - }) - : null; - - const blocklist = workspaceMember - ? await this.blocklistRepository.getByWorkspaceMemberId( - workspaceMember.id, - workspaceId, - ) - : []; - - const refreshedHandleAliases = - await this.emailAliasManagerService.refreshHandleAliases( - connectedAccount, - workspaceId, - ); - - connectedAccount.handleAliases = refreshedHandleAliases; - - if ( - !isDefined(connectedAccount.handleAliases) || - !isDefined(calendarChannel.handle) - ) { - throw new CalendarEventImportDriverException( - 'Calendar channel handle or Handle aliases are required', - CalendarEventImportDriverExceptionCode.CHANNEL_MISCONFIGURED, - ); - } - - const { filteredEvents, cancelledEvents } = - filterEventsAndReturnCancelledEvents( - [calendarChannel.handle, ...connectedAccount.handleAliases], - calendarEvents, - blocklist.map((blocklist) => blocklist.handle ?? ''), - ); - - const cancelledEventExternalIds = cancelledEvents.map( - (event) => event.id, - ); - - const BATCH_SIZE = 1000; - - for (let i = 0; i < filteredEvents.length; i = i + BATCH_SIZE) { - const eventsBatch = filteredEvents.slice(i, i + BATCH_SIZE); - - await this.calendarSaveEventsService.saveCalendarEventsAndEnqueueContactCreationJob( - eventsBatch, + } catch (error) { + await this.calendarEventImportErrorHandlerService.handleDriverException( + error, + CalendarEventImportSyncStep.CALENDAR_EVENTS_IMPORT, calendarChannel, - connectedAccount, workspaceId, ); } - const calendarChannelEventAssociationRepository = - await this.globalWorkspaceOrmManager.getRepository( - workspaceId, - 'calendarChannelEventAssociation', - ); - - await calendarChannelEventAssociationRepository.delete({ - eventExternalId: Any(cancelledEventExternalIds), - calendarChannelId: calendarChannel.id, - }); - - await this.calendarEventCleanerService.cleanWorkspaceCalendarEvents( - workspaceId, - ); - - await this.calendarChannelSyncStatusService.markAsCompletedAndMarkAsCalendarEventListFetchPending( - [calendarChannel.id], - workspaceId, - ); - } catch (error) { - await this.calendarEventImportErrorHandlerService.handleDriverException( - error, - CalendarEventImportSyncStep.CALENDAR_EVENTS_IMPORT, - calendarChannel, - workspaceId, - ); - } - }, authContext); + }, + authContext, + { lite: true }, + ); } } diff --git a/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/services/calendar-fetch-events.service.ts b/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/services/calendar-fetch-events.service.ts index a914333d43..2083d952c2 100644 --- a/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/services/calendar-fetch-events.service.ts +++ b/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/services/calendar-fetch-events.service.ts @@ -55,38 +55,52 @@ export class CalendarFetchEventsService { const authContext = buildSystemAuthContext(workspaceId); - await this.globalWorkspaceOrmManager.executeInWorkspaceContext(async () => { - try { - const { accessToken, refreshToken } = - await this.calendarAccountAuthenticationService.validateAndRefreshConnectedAccountAuthentication( - { - connectedAccount, + await this.globalWorkspaceOrmManager.executeInWorkspaceContext( + async () => { + try { + const { accessToken, refreshToken } = + await this.calendarAccountAuthenticationService.validateAndRefreshConnectedAccountAuthentication( + { + connectedAccount, + workspaceId, + calendarChannelId: calendarChannel.id, + }, + ); + + const connectedAccountWithFreshTokens = { + ...connectedAccount, + accessToken, + refreshToken, + }; + + const getCalendarEventsResponse = + await this.getCalendarEventsService.getCalendarEvents( + connectedAccountWithFreshTokens, + calendarChannel.syncCursor || undefined, + ); + + const hasFullEvents = getCalendarEventsResponse.fullEvents; + + const calendarEvents = hasFullEvents + ? getCalendarEventsResponse.calendarEvents + : null; + const calendarEventIds = getCalendarEventsResponse.calendarEventIds; + const nextSyncCursor = getCalendarEventsResponse.nextSyncCursor; + + if (!calendarEvents || calendarEvents?.length === 0) { + await this.calendarChannelRepository.update( + { id: calendarChannel.id, workspaceId }, + { + syncCursor: nextSyncCursor, + }, + ); + + await this.calendarChannelSyncStatusService.markAsCalendarEventListFetchPending( + [calendarChannel.id], workspaceId, - calendarChannelId: calendarChannel.id, - }, - ); + ); + } - const connectedAccountWithFreshTokens = { - ...connectedAccount, - accessToken, - refreshToken, - }; - - const getCalendarEventsResponse = - await this.getCalendarEventsService.getCalendarEvents( - connectedAccountWithFreshTokens, - calendarChannel.syncCursor || undefined, - ); - - const hasFullEvents = getCalendarEventsResponse.fullEvents; - - const calendarEvents = hasFullEvents - ? getCalendarEventsResponse.calendarEvents - : null; - const calendarEventIds = getCalendarEventsResponse.calendarEventIds; - const nextSyncCursor = getCalendarEventsResponse.nextSyncCursor; - - if (!calendarEvents || calendarEvents?.length === 0) { await this.calendarChannelRepository.update( { id: calendarChannel.id, workspaceId }, { @@ -94,53 +108,43 @@ export class CalendarFetchEventsService { }, ); - await this.calendarChannelSyncStatusService.markAsCalendarEventListFetchPending( - [calendarChannel.id], - workspaceId, + if (hasFullEvents && calendarEvents) { + await this.calendarEventsImportService.processCalendarEventsImport( + calendarChannel, + connectedAccount, + workspaceId, + calendarEvents, + ); + } else if (!hasFullEvents && calendarEventIds) { + await this.cacheStorage.setAdd( + `calendar-events-to-import:${workspaceId}:${calendarChannel.id}`, + calendarEventIds, + ); + + await this.calendarChannelSyncStatusService.markAsCalendarEventsImportPending( + [calendarChannel.id], + workspaceId, + ); + } else { + throw new CalendarEventImportDriverException( + "Expected 'calendarEvents' or 'calendarEventIds' to be present", + CalendarEventImportDriverExceptionCode.UNKNOWN, + ); + } + } catch (error) { + this.logger.error( + `WorkspaceId: ${workspaceId}, CalendarChannelId: ${calendarChannel.id} - Calendar event fetch error: ${error.message}`, ); - } - - await this.calendarChannelRepository.update( - { id: calendarChannel.id, workspaceId }, - { - syncCursor: nextSyncCursor, - }, - ); - - if (hasFullEvents && calendarEvents) { - await this.calendarEventsImportService.processCalendarEventsImport( + await this.calendarEventImportErrorHandlerService.handleDriverException( + error, + CalendarEventImportSyncStep.CALENDAR_EVENT_LIST_FETCH, calendarChannel, - connectedAccount, workspaceId, - calendarEvents, - ); - } else if (!hasFullEvents && calendarEventIds) { - await this.cacheStorage.setAdd( - `calendar-events-to-import:${workspaceId}:${calendarChannel.id}`, - calendarEventIds, - ); - - await this.calendarChannelSyncStatusService.markAsCalendarEventsImportPending( - [calendarChannel.id], - workspaceId, - ); - } else { - throw new CalendarEventImportDriverException( - "Expected 'calendarEvents' or 'calendarEventIds' to be present", - CalendarEventImportDriverExceptionCode.UNKNOWN, ); } - } catch (error) { - this.logger.error( - `WorkspaceId: ${workspaceId}, CalendarChannelId: ${calendarChannel.id} - Calendar event fetch error: ${error.message}`, - ); - await this.calendarEventImportErrorHandlerService.handleDriverException( - error, - CalendarEventImportSyncStep.CALENDAR_EVENT_LIST_FETCH, - calendarChannel, - workspaceId, - ); - } - }, authContext); + }, + authContext, + { lite: true }, + ); } } diff --git a/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/services/calendar-save-events.service.ts b/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/services/calendar-save-events.service.ts index 17731a92e4..4a3ff602ea 100644 --- a/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/services/calendar-save-events.service.ts +++ b/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/services/calendar-save-events.service.ts @@ -34,284 +34,290 @@ export class CalendarSaveEventsService { ): Promise { const authContext = buildSystemAuthContext(workspaceId); - await this.globalWorkspaceOrmManager.executeInWorkspaceContext(async () => { - const calendarEventRepository = - await this.globalWorkspaceOrmManager.getRepository( - workspaceId, - 'calendarEvent', - ); + await this.globalWorkspaceOrmManager.executeInWorkspaceContext( + async () => { + const calendarEventRepository = + await this.globalWorkspaceOrmManager.getRepository( + workspaceId, + 'calendarEvent', + ); - const calendarChannelEventAssociationRepository = - await this.globalWorkspaceOrmManager.getRepository( - workspaceId, - 'calendarChannelEventAssociation', - ); + const calendarChannelEventAssociationRepository = + await this.globalWorkspaceOrmManager.getRepository( + workspaceId, + 'calendarChannelEventAssociation', + ); - const workspaceDataSource = - await this.globalWorkspaceOrmManager.getGlobalWorkspaceDataSource(); + const workspaceDataSource = + await this.globalWorkspaceOrmManager.getGlobalWorkspaceDataSource(); - await workspaceDataSource.transaction( - async (transactionManager: WorkspaceEntityManager) => { - const existingAssociations = - await calendarChannelEventAssociationRepository.find( - { - where: { - eventExternalId: Any( - fetchedCalendarEvents.map((event) => event.id), - ), - calendarChannelId: calendarChannel.id, + await workspaceDataSource.transaction( + async (transactionManager: WorkspaceEntityManager) => { + const existingAssociations = + await calendarChannelEventAssociationRepository.find( + { + where: { + eventExternalId: Any( + fetchedCalendarEvents.map((event) => event.id), + ), + calendarChannelId: calendarChannel.id, + }, }, - }, - transactionManager, - ); - - const existingCalendarEventIdByExternalId = new Map( - existingAssociations.map((association) => [ - association.eventExternalId, - association.calendarEventId, - ]), - ); - - const existingAssociationIdByExternalId = new Map( - existingAssociations.map((association) => [ - association.eventExternalId, - association.id, - ]), - ); - - const fetchedCalendarEventsWithDBEvents: FetchedCalendarEventWithDBEvent[] = - fetchedCalendarEvents.map( - (event): FetchedCalendarEventWithDBEvent => { - const existingCalendarEventId = - existingCalendarEventIdByExternalId.get(event.id); - - return { - fetchedCalendarEvent: event, - existingCalendarEvent: existingCalendarEventId - ? ({ - id: existingCalendarEventId, - } as CalendarEventWorkspaceEntity) - : null, - newlyCreatedCalendarEvent: null, - }; - }, - ); - - const newCalendarEventIdByExternalId = new Map(); - - const newCalendarEventsToInsert = fetchedCalendarEventsWithDBEvents - .filter( - ({ existingCalendarEvent }) => existingCalendarEvent === null, - ) - .map(({ fetchedCalendarEvent }) => { - const calendarEventId = uuid(); - - newCalendarEventIdByExternalId.set( - fetchedCalendarEvent.id, - calendarEventId, + transactionManager, ); - return { - id: calendarEventId, - iCalUid: fetchedCalendarEvent.iCalUid, - title: fetchedCalendarEvent.title, - description: fetchedCalendarEvent.description, - startsAt: fetchedCalendarEvent.startsAt, - endsAt: fetchedCalendarEvent.endsAt, - location: fetchedCalendarEvent.location, - isFullDay: fetchedCalendarEvent.isFullDay, - isCanceled: fetchedCalendarEvent.isCanceled, - conferenceSolution: fetchedCalendarEvent.conferenceSolution, - conferenceLink: { - primaryLinkLabel: fetchedCalendarEvent.conferenceLinkLabel, - primaryLinkUrl: fetchedCalendarEvent.conferenceLinkUrl, - secondaryLinks: [], - }, - externalCreatedAt: fetchedCalendarEvent.externalCreatedAt, - externalUpdatedAt: fetchedCalendarEvent.externalUpdatedAt, - }; - }); - - if (newCalendarEventsToInsert.length > 0) { - await calendarEventRepository.insert( - newCalendarEventsToInsert, - transactionManager, + const existingCalendarEventIdByExternalId = new Map( + existingAssociations.map((association) => [ + association.eventExternalId, + association.calendarEventId, + ]), ); - } - const fetchedCalendarEventsWithDBEventsEnrichedWithSavedEvents: FetchedCalendarEventWithDBEvent[] = - fetchedCalendarEventsWithDBEvents.map( - ({ fetchedCalendarEvent, existingCalendarEvent }) => { - const savedCalendarEventId = newCalendarEventIdByExternalId.get( + const existingAssociationIdByExternalId = new Map( + existingAssociations.map((association) => [ + association.eventExternalId, + association.id, + ]), + ); + + const fetchedCalendarEventsWithDBEvents: FetchedCalendarEventWithDBEvent[] = + fetchedCalendarEvents.map( + (event): FetchedCalendarEventWithDBEvent => { + const existingCalendarEventId = + existingCalendarEventIdByExternalId.get(event.id); + + return { + fetchedCalendarEvent: event, + existingCalendarEvent: existingCalendarEventId + ? ({ + id: existingCalendarEventId, + } as CalendarEventWorkspaceEntity) + : null, + newlyCreatedCalendarEvent: null, + }; + }, + ); + + const newCalendarEventIdByExternalId = new Map(); + + const newCalendarEventsToInsert = fetchedCalendarEventsWithDBEvents + .filter( + ({ existingCalendarEvent }) => existingCalendarEvent === null, + ) + .map(({ fetchedCalendarEvent }) => { + const calendarEventId = uuid(); + + newCalendarEventIdByExternalId.set( fetchedCalendarEvent.id, + calendarEventId, ); return { - fetchedCalendarEvent, - existingCalendarEvent, - newlyCreatedCalendarEvent: savedCalendarEventId - ? ({ - id: savedCalendarEventId, - } as CalendarEventWorkspaceEntity) - : null, - }; - }, - ); - - const existingEventsToUpdate = - fetchedCalendarEventsWithDBEventsEnrichedWithSavedEvents - .filter( - ({ existingCalendarEvent }) => existingCalendarEvent !== null, - ) - .map(({ fetchedCalendarEvent, existingCalendarEvent }) => { - if (!existingCalendarEvent) { - throw new Error( - `Existing calendar event with iCalUid ${fetchedCalendarEvent.iCalUid} not found - should never happen`, - ); - } - - return { - criteria: existingCalendarEvent.id, - partialEntity: { - iCalUid: fetchedCalendarEvent.iCalUid, - title: fetchedCalendarEvent.title, - description: fetchedCalendarEvent.description, - startsAt: fetchedCalendarEvent.startsAt, - endsAt: fetchedCalendarEvent.endsAt, - location: fetchedCalendarEvent.location, - isFullDay: fetchedCalendarEvent.isFullDay, - isCanceled: fetchedCalendarEvent.isCanceled, - conferenceSolution: fetchedCalendarEvent.conferenceSolution, - conferenceLink: { - primaryLinkLabel: - fetchedCalendarEvent.conferenceLinkLabel, - primaryLinkUrl: fetchedCalendarEvent.conferenceLinkUrl, - secondaryLinks: [], - }, - externalCreatedAt: fetchedCalendarEvent.externalCreatedAt, - externalUpdatedAt: fetchedCalendarEvent.externalUpdatedAt, + id: calendarEventId, + iCalUid: fetchedCalendarEvent.iCalUid, + title: fetchedCalendarEvent.title, + description: fetchedCalendarEvent.description, + startsAt: fetchedCalendarEvent.startsAt, + endsAt: fetchedCalendarEvent.endsAt, + location: fetchedCalendarEvent.location, + isFullDay: fetchedCalendarEvent.isFullDay, + isCanceled: fetchedCalendarEvent.isCanceled, + conferenceSolution: fetchedCalendarEvent.conferenceSolution, + conferenceLink: { + primaryLinkLabel: fetchedCalendarEvent.conferenceLinkLabel, + primaryLinkUrl: fetchedCalendarEvent.conferenceLinkUrl, + secondaryLinks: [], }, + externalCreatedAt: fetchedCalendarEvent.externalCreatedAt, + externalUpdatedAt: fetchedCalendarEvent.externalUpdatedAt, }; }); - if (existingEventsToUpdate.length > 0) { - await calendarEventRepository.updateMany( - existingEventsToUpdate, - transactionManager, - ); - } + if (newCalendarEventsToInsert.length > 0) { + await calendarEventRepository.insert( + newCalendarEventsToInsert, + transactionManager, + ); + } - const calendarChannelEventAssociationsToSave: Pick< - CalendarChannelEventAssociationWorkspaceEntity, - | 'calendarEventId' - | 'eventExternalId' - | 'calendarChannelId' - | 'recurringEventExternalId' - >[] = fetchedCalendarEventsWithDBEventsEnrichedWithSavedEvents - .filter( - ({ newlyCreatedCalendarEvent }) => - newlyCreatedCalendarEvent !== null, - ) - .map(({ fetchedCalendarEvent, newlyCreatedCalendarEvent }) => { - if (!newlyCreatedCalendarEvent?.id) { - throw new Error( - `Calendar event id not found for event with iCalUid ${fetchedCalendarEvent.iCalUid} - should never happen`, - ); - } + const fetchedCalendarEventsWithDBEventsEnrichedWithSavedEvents: FetchedCalendarEventWithDBEvent[] = + fetchedCalendarEventsWithDBEvents.map( + ({ fetchedCalendarEvent, existingCalendarEvent }) => { + const savedCalendarEventId = + newCalendarEventIdByExternalId.get(fetchedCalendarEvent.id); - return { - calendarEventId: newlyCreatedCalendarEvent.id, - eventExternalId: fetchedCalendarEvent.id, - calendarChannelId: calendarChannel.id, - recurringEventExternalId: - fetchedCalendarEvent.recurringEventExternalId ?? '', - }; - }); - - if (calendarChannelEventAssociationsToSave.length > 0) { - await calendarChannelEventAssociationRepository.insert( - calendarChannelEventAssociationsToSave, - transactionManager, - ); - } - - const existingAssociationsToUpdate = - fetchedCalendarEventsWithDBEventsEnrichedWithSavedEvents - .filter( - ({ existingCalendarEvent }) => existingCalendarEvent !== null, - ) - .map(({ fetchedCalendarEvent }) => ({ - criteria: existingAssociationIdByExternalId.get( - fetchedCalendarEvent.id, - )!, - partialEntity: { - recurringEventExternalId: - fetchedCalendarEvent.recurringEventExternalId ?? '', + return { + fetchedCalendarEvent, + existingCalendarEvent, + newlyCreatedCalendarEvent: savedCalendarEventId + ? ({ + id: savedCalendarEventId, + } as CalendarEventWorkspaceEntity) + : null, + }; }, - })); + ); - if (existingAssociationsToUpdate.length > 0) { - await calendarChannelEventAssociationRepository.updateMany( - existingAssociationsToUpdate, - transactionManager, - ); - } + const existingEventsToUpdate = + fetchedCalendarEventsWithDBEventsEnrichedWithSavedEvents + .filter( + ({ existingCalendarEvent }) => existingCalendarEvent !== null, + ) + .map(({ fetchedCalendarEvent, existingCalendarEvent }) => { + if (!existingCalendarEvent) { + throw new Error( + `Existing calendar event with iCalUid ${fetchedCalendarEvent.iCalUid} not found - should never happen`, + ); + } - const participantsToCreate = - fetchedCalendarEventsWithDBEventsEnrichedWithSavedEvents + return { + criteria: existingCalendarEvent.id, + partialEntity: { + iCalUid: fetchedCalendarEvent.iCalUid, + title: fetchedCalendarEvent.title, + description: fetchedCalendarEvent.description, + startsAt: fetchedCalendarEvent.startsAt, + endsAt: fetchedCalendarEvent.endsAt, + location: fetchedCalendarEvent.location, + isFullDay: fetchedCalendarEvent.isFullDay, + isCanceled: fetchedCalendarEvent.isCanceled, + conferenceSolution: + fetchedCalendarEvent.conferenceSolution, + conferenceLink: { + primaryLinkLabel: + fetchedCalendarEvent.conferenceLinkLabel, + primaryLinkUrl: fetchedCalendarEvent.conferenceLinkUrl, + secondaryLinks: [], + }, + externalCreatedAt: fetchedCalendarEvent.externalCreatedAt, + externalUpdatedAt: fetchedCalendarEvent.externalUpdatedAt, + }, + }; + }); + + if (existingEventsToUpdate.length > 0) { + await calendarEventRepository.updateMany( + existingEventsToUpdate, + transactionManager, + ); + } + + const calendarChannelEventAssociationsToSave: Pick< + CalendarChannelEventAssociationWorkspaceEntity, + | 'calendarEventId' + | 'eventExternalId' + | 'calendarChannelId' + | 'recurringEventExternalId' + >[] = fetchedCalendarEventsWithDBEventsEnrichedWithSavedEvents .filter( ({ newlyCreatedCalendarEvent }) => newlyCreatedCalendarEvent !== null, ) - .flatMap( - ({ newlyCreatedCalendarEvent, fetchedCalendarEvent }) => { - if (!newlyCreatedCalendarEvent?.id) { + .map(({ fetchedCalendarEvent, newlyCreatedCalendarEvent }) => { + if (!newlyCreatedCalendarEvent?.id) { + throw new Error( + `Calendar event id not found for event with iCalUid ${fetchedCalendarEvent.iCalUid} - should never happen`, + ); + } + + return { + calendarEventId: newlyCreatedCalendarEvent.id, + eventExternalId: fetchedCalendarEvent.id, + calendarChannelId: calendarChannel.id, + recurringEventExternalId: + fetchedCalendarEvent.recurringEventExternalId ?? '', + }; + }); + + if (calendarChannelEventAssociationsToSave.length > 0) { + await calendarChannelEventAssociationRepository.insert( + calendarChannelEventAssociationsToSave, + transactionManager, + ); + } + + const existingAssociationsToUpdate = + fetchedCalendarEventsWithDBEventsEnrichedWithSavedEvents + .filter( + ({ existingCalendarEvent }) => existingCalendarEvent !== null, + ) + .map(({ fetchedCalendarEvent }) => ({ + criteria: existingAssociationIdByExternalId.get( + fetchedCalendarEvent.id, + )!, + partialEntity: { + recurringEventExternalId: + fetchedCalendarEvent.recurringEventExternalId ?? '', + }, + })); + + if (existingAssociationsToUpdate.length > 0) { + await calendarChannelEventAssociationRepository.updateMany( + existingAssociationsToUpdate, + transactionManager, + ); + } + + const participantsToCreate = + fetchedCalendarEventsWithDBEventsEnrichedWithSavedEvents + .filter( + ({ newlyCreatedCalendarEvent }) => + newlyCreatedCalendarEvent !== null, + ) + .flatMap( + ({ newlyCreatedCalendarEvent, fetchedCalendarEvent }) => { + if (!newlyCreatedCalendarEvent?.id) { + throw new Error( + `Newly created calendar event with iCalUid ${fetchedCalendarEvent.iCalUid} not found - should never happen`, + ); + } + + return fetchedCalendarEvent.participants.map( + (participant) => ({ + ...participant, + calendarEventId: newlyCreatedCalendarEvent.id, + }), + ); + }, + ); + + // todo: we should prevent duplicate rows on calendarEventAssociation by creating + // an index on calendarChannelId and calendarEventId + const participantsToUpdate = + fetchedCalendarEventsWithDBEventsEnrichedWithSavedEvents + .filter( + ({ existingCalendarEvent }) => existingCalendarEvent !== null, + ) + .flatMap(({ fetchedCalendarEvent, existingCalendarEvent }) => { + if (!existingCalendarEvent?.id) { throw new Error( - `Newly created calendar event with iCalUid ${fetchedCalendarEvent.iCalUid} not found - should never happen`, + `Existing calendar event with iCalUid ${fetchedCalendarEvent.iCalUid} not found - should never happen`, ); } return fetchedCalendarEvent.participants.map( (participant) => ({ ...participant, - calendarEventId: newlyCreatedCalendarEvent.id, + calendarEventId: existingCalendarEvent.id, }), ); - }, - ); + }); - // todo: we should prevent duplicate rows on calendarEventAssociation by creating - // an index on calendarChannelId and calendarEventId - const participantsToUpdate = - fetchedCalendarEventsWithDBEventsEnrichedWithSavedEvents - .filter( - ({ existingCalendarEvent }) => existingCalendarEvent !== null, - ) - .flatMap(({ fetchedCalendarEvent, existingCalendarEvent }) => { - if (!existingCalendarEvent?.id) { - throw new Error( - `Existing calendar event with iCalUid ${fetchedCalendarEvent.iCalUid} not found - should never happen`, - ); - } - - return fetchedCalendarEvent.participants.map((participant) => ({ - ...participant, - calendarEventId: existingCalendarEvent.id, - })); - }); - - await this.calendarEventParticipantService.upsertAndDeleteCalendarEventParticipants( - { - participantsToCreate, - participantsToUpdate, - transactionManager, - calendarChannel, - connectedAccount, - workspaceId, - }, - ); - }, - ); - }, authContext); + await this.calendarEventParticipantService.upsertAndDeleteCalendarEventParticipants( + { + participantsToCreate, + participantsToUpdate, + transactionManager, + calendarChannel, + connectedAccount, + workspaceId, + }, + ); + }, + ); + }, + authContext, + { lite: true }, + ); } } diff --git a/packages/twenty-server/src/modules/calendar/calendar-event-participant-manager/services/calendar-event-participant.service.ts b/packages/twenty-server/src/modules/calendar/calendar-event-participant-manager/services/calendar-event-participant.service.ts index d3e5899f68..ce0f8334b8 100644 --- a/packages/twenty-server/src/modules/calendar/calendar-event-participant-manager/services/calendar-event-participant.service.ts +++ b/packages/twenty-server/src/modules/calendar/calendar-event-participant-manager/services/calendar-event-participant.service.ts @@ -58,127 +58,132 @@ export class CalendarEventParticipantService { }): Promise { const authContext = buildSystemAuthContext(workspaceId); - await this.globalWorkspaceOrmManager.executeInWorkspaceContext(async () => { - const chunkedParticipantsToUpdate = chunk(participantsToUpdate, 200); + await this.globalWorkspaceOrmManager.executeInWorkspaceContext( + async () => { + const chunkedParticipantsToUpdate = chunk(participantsToUpdate, 200); - const calendarEventParticipantRepository = - await this.globalWorkspaceOrmManager.getRepository( - workspaceId, - 'calendarEventParticipant', - ); + const calendarEventParticipantRepository = + await this.globalWorkspaceOrmManager.getRepository( + workspaceId, + 'calendarEventParticipant', + ); - for (const participantsToUpdateChunk of chunkedParticipantsToUpdate) { - const existingCalendarEventParticipants = - await calendarEventParticipantRepository.find({ - where: { - calendarEventId: Any( - participantsToUpdateChunk - .map((participant) => participant.calendarEventId) - .filter(isDefined), + for (const participantsToUpdateChunk of chunkedParticipantsToUpdate) { + const existingCalendarEventParticipants = + await calendarEventParticipantRepository.find({ + where: { + calendarEventId: Any( + participantsToUpdateChunk + .map((participant) => participant.calendarEventId) + .filter(isDefined), + ), + }, + }); + + const { + calendarEventParticipantsToUpdate, + newCalendarEventParticipants, + } = participantsToUpdateChunk.reduce<{ + calendarEventParticipantsToUpdate: FetchedCalendarEventParticipantWithCalendarEventIdAndExistingId[]; + newCalendarEventParticipants: FetchedCalendarEventParticipantWithCalendarEventId[]; + }>( + (acc, calendarEventParticipant) => { + const existingCalendarEventParticipant = + existingCalendarEventParticipants.find( + (existingCalendarEventParticipant) => + existingCalendarEventParticipant.handle === + calendarEventParticipant.handle && + existingCalendarEventParticipant.calendarEventId === + calendarEventParticipant.calendarEventId, + ); + + if (existingCalendarEventParticipant) { + acc.calendarEventParticipantsToUpdate.push({ + ...calendarEventParticipant, + id: existingCalendarEventParticipant.id, + }); + } else { + acc.newCalendarEventParticipants.push(calendarEventParticipant); + } + + return acc; + }, + { + calendarEventParticipantsToUpdate: [], + newCalendarEventParticipants: [], + }, + ); + + const calendarEventParticipantsToDelete = differenceWith( + existingCalendarEventParticipants, + participantsToUpdateChunk, + (existingCalendarEventParticipant, participantToUpdate) => + existingCalendarEventParticipant.handle === + participantToUpdate.handle && + existingCalendarEventParticipant.calendarEventId === + participantToUpdate.calendarEventId, + ); + + await calendarEventParticipantRepository.delete( + { + id: Any( + calendarEventParticipantsToDelete.map( + (calendarEventParticipant) => calendarEventParticipant.id, + ), ), }, - }); - - const { - calendarEventParticipantsToUpdate, - newCalendarEventParticipants, - } = participantsToUpdateChunk.reduce<{ - calendarEventParticipantsToUpdate: FetchedCalendarEventParticipantWithCalendarEventIdAndExistingId[]; - newCalendarEventParticipants: FetchedCalendarEventParticipantWithCalendarEventId[]; - }>( - (acc, calendarEventParticipant) => { - const existingCalendarEventParticipant = - existingCalendarEventParticipants.find( - (existingCalendarEventParticipant) => - existingCalendarEventParticipant.handle === - calendarEventParticipant.handle && - existingCalendarEventParticipant.calendarEventId === - calendarEventParticipant.calendarEventId, - ); - - if (existingCalendarEventParticipant) { - acc.calendarEventParticipantsToUpdate.push({ - ...calendarEventParticipant, - id: existingCalendarEventParticipant.id, - }); - } else { - acc.newCalendarEventParticipants.push(calendarEventParticipant); - } - - return acc; - }, - { - calendarEventParticipantsToUpdate: [], - newCalendarEventParticipants: [], - }, - ); - - const calendarEventParticipantsToDelete = differenceWith( - existingCalendarEventParticipants, - participantsToUpdateChunk, - (existingCalendarEventParticipant, participantToUpdate) => - existingCalendarEventParticipant.handle === - participantToUpdate.handle && - existingCalendarEventParticipant.calendarEventId === - participantToUpdate.calendarEventId, - ); - - await calendarEventParticipantRepository.delete( - { - id: Any( - calendarEventParticipantsToDelete.map( - (calendarEventParticipant) => calendarEventParticipant.id, - ), - ), - }, - transactionManager, - ); - - await calendarEventParticipantRepository.updateMany( - calendarEventParticipantsToUpdate.map((participant) => ({ - criteria: participant.id, - partialEntity: participant, - })), - transactionManager, - ); - participantsToCreate.push(...newCalendarEventParticipants); - } - - const chunkedParticipantsToCreate = chunk(participantsToCreate, 200); - const savedParticipants: CalendarEventParticipantWorkspaceEntity[] = []; - - for (const participantsToCreateChunk of chunkedParticipantsToCreate) { - const savedParticipantsChunk = - await calendarEventParticipantRepository.insert( - participantsToCreateChunk, transactionManager, ); - savedParticipants.push(...savedParticipantsChunk.raw); - } - - if (calendarChannel.isContactAutoCreationEnabled) { - await this.messageQueueService.add( - CreateCompanyAndContactJob.name, - { - workspaceId, - connectedAccount, - contactsToCreate: savedParticipants.map((participant) => ({ - handle: participant.handle ?? '', - displayName: participant.displayName ?? participant.handle ?? '', + await calendarEventParticipantRepository.updateMany( + calendarEventParticipantsToUpdate.map((participant) => ({ + criteria: participant.id, + partialEntity: participant, })), - source: FieldActorSource.CALENDAR, - }, - ); - } + transactionManager, + ); + participantsToCreate.push(...newCalendarEventParticipants); + } - await this.matchParticipantService.matchParticipants({ - participants: savedParticipants, - objectMetadataName: 'calendarEventParticipant', - transactionManager, - matchWith: 'workspaceMemberAndPerson', - workspaceId, - }); - }, authContext); + const chunkedParticipantsToCreate = chunk(participantsToCreate, 200); + const savedParticipants: CalendarEventParticipantWorkspaceEntity[] = []; + + for (const participantsToCreateChunk of chunkedParticipantsToCreate) { + const savedParticipantsChunk = + await calendarEventParticipantRepository.insert( + participantsToCreateChunk, + transactionManager, + ); + + savedParticipants.push(...savedParticipantsChunk.raw); + } + + if (calendarChannel.isContactAutoCreationEnabled) { + await this.messageQueueService.add( + CreateCompanyAndContactJob.name, + { + workspaceId, + connectedAccount, + contactsToCreate: savedParticipants.map((participant) => ({ + handle: participant.handle ?? '', + displayName: + participant.displayName ?? participant.handle ?? '', + })), + source: FieldActorSource.CALENDAR, + }, + ); + } + + await this.matchParticipantService.matchParticipants({ + participants: savedParticipants, + objectMetadataName: 'calendarEventParticipant', + transactionManager, + matchWith: 'workspaceMemberAndPerson', + workspaceId, + }); + }, + authContext, + { lite: true }, + ); } } diff --git a/packages/twenty-server/src/modules/calendar/common/query-hooks/calendar-event/services/apply-calendar-events-visibility-restrictions.service.ts b/packages/twenty-server/src/modules/calendar/common/query-hooks/calendar-event/services/apply-calendar-events-visibility-restrictions.service.ts index 2cf829b802..c620a8a7f0 100644 --- a/packages/twenty-server/src/modules/calendar/common/query-hooks/calendar-event/services/apply-calendar-events-visibility-restrictions.service.ts +++ b/packages/twenty-server/src/modules/calendar/common/query-hooks/calendar-event/services/apply-calendar-events-visibility-restrictions.service.ts @@ -138,6 +138,7 @@ export class ApplyCalendarEventsVisibilityRestrictionsService { return calendarEvents; }, authContext, + { lite: true }, ); } } diff --git a/packages/twenty-server/src/modules/calendar/common/services/calendar-channel-sync-status.service.ts b/packages/twenty-server/src/modules/calendar/common/services/calendar-channel-sync-status.service.ts index a9c08f64d5..d8f2e07a01 100644 --- a/packages/twenty-server/src/modules/calendar/common/services/calendar-channel-sync-status.service.ts +++ b/packages/twenty-server/src/modules/calendar/common/services/calendar-channel-sync-status.service.ts @@ -46,15 +46,22 @@ export class CalendarChannelSyncStatusService { const authContext = buildSystemAuthContext(workspaceId); - await this.globalWorkspaceOrmManager.executeInWorkspaceContext(async () => { - await this.calendarChannelRepository.update( - { id: In(calendarChannelIds), workspaceId }, - { - syncStage: CalendarChannelSyncStage.CALENDAR_EVENT_LIST_FETCH_PENDING, - ...(!preserveSyncStageStartedAt ? { syncStageStartedAt: null } : {}), - }, - ); - }, authContext); + await this.globalWorkspaceOrmManager.executeInWorkspaceContext( + async () => { + await this.calendarChannelRepository.update( + { id: In(calendarChannelIds), workspaceId }, + { + syncStage: + CalendarChannelSyncStage.CALENDAR_EVENT_LIST_FETCH_PENDING, + ...(!preserveSyncStageStartedAt + ? { syncStageStartedAt: null } + : {}), + }, + ); + }, + authContext, + { lite: true }, + ); } public async markAsCalendarEventListFetchOngoing( @@ -67,16 +74,21 @@ export class CalendarChannelSyncStatusService { const authContext = buildSystemAuthContext(workspaceId); - await this.globalWorkspaceOrmManager.executeInWorkspaceContext(async () => { - await this.calendarChannelRepository.update( - { id: In(calendarChannelIds), workspaceId }, - { - syncStage: CalendarChannelSyncStage.CALENDAR_EVENT_LIST_FETCH_ONGOING, - syncStatus: CalendarChannelSyncStatus.ONGOING, - syncStageStartedAt: new Date().toISOString(), - }, - ); - }, authContext); + await this.globalWorkspaceOrmManager.executeInWorkspaceContext( + async () => { + await this.calendarChannelRepository.update( + { id: In(calendarChannelIds), workspaceId }, + { + syncStage: + CalendarChannelSyncStage.CALENDAR_EVENT_LIST_FETCH_ONGOING, + syncStatus: CalendarChannelSyncStatus.ONGOING, + syncStageStartedAt: new Date().toISOString(), + }, + ); + }, + authContext, + { lite: true }, + ); } public async resetAndMarkAsCalendarEventListFetchPending( @@ -95,16 +107,20 @@ export class CalendarChannelSyncStatusService { const authContext = buildSystemAuthContext(workspaceId); - await this.globalWorkspaceOrmManager.executeInWorkspaceContext(async () => { - await this.calendarChannelRepository.update( - { id: In(calendarChannelIds), workspaceId }, - { - syncCursor: '', - syncStageStartedAt: null, - throttleFailureCount: 0, - }, - ); - }, authContext); + await this.globalWorkspaceOrmManager.executeInWorkspaceContext( + async () => { + await this.calendarChannelRepository.update( + { id: In(calendarChannelIds), workspaceId }, + { + syncCursor: '', + syncStageStartedAt: null, + throttleFailureCount: 0, + }, + ); + }, + authContext, + { lite: true }, + ); await this.markAsCalendarEventListFetchPending( calendarChannelIds, @@ -122,14 +138,18 @@ export class CalendarChannelSyncStatusService { const authContext = buildSystemAuthContext(workspaceId); - await this.globalWorkspaceOrmManager.executeInWorkspaceContext(async () => { - await this.calendarChannelRepository.update( - { id: In(calendarChannelIds), workspaceId }, - { - syncStageStartedAt: null, - }, - ); - }, authContext); + await this.globalWorkspaceOrmManager.executeInWorkspaceContext( + async () => { + await this.calendarChannelRepository.update( + { id: In(calendarChannelIds), workspaceId }, + { + syncStageStartedAt: null, + }, + ); + }, + authContext, + { lite: true }, + ); } public async markAsCalendarEventsImportPending( @@ -143,15 +163,21 @@ export class CalendarChannelSyncStatusService { const authContext = buildSystemAuthContext(workspaceId); - await this.globalWorkspaceOrmManager.executeInWorkspaceContext(async () => { - await this.calendarChannelRepository.update( - { id: In(calendarChannelIds), workspaceId }, - { - syncStage: CalendarChannelSyncStage.CALENDAR_EVENTS_IMPORT_PENDING, - ...(!preserveSyncStageStartedAt ? { syncStageStartedAt: null } : {}), - }, - ); - }, authContext); + await this.globalWorkspaceOrmManager.executeInWorkspaceContext( + async () => { + await this.calendarChannelRepository.update( + { id: In(calendarChannelIds), workspaceId }, + { + syncStage: CalendarChannelSyncStage.CALENDAR_EVENTS_IMPORT_PENDING, + ...(!preserveSyncStageStartedAt + ? { syncStageStartedAt: null } + : {}), + }, + ); + }, + authContext, + { lite: true }, + ); } public async markAsCalendarEventsImportOngoing( @@ -164,16 +190,20 @@ export class CalendarChannelSyncStatusService { const authContext = buildSystemAuthContext(workspaceId); - await this.globalWorkspaceOrmManager.executeInWorkspaceContext(async () => { - await this.calendarChannelRepository.update( - { id: In(calendarChannelIds), workspaceId }, - { - syncStage: CalendarChannelSyncStage.CALENDAR_EVENTS_IMPORT_ONGOING, - syncStatus: CalendarChannelSyncStatus.ONGOING, - syncStageStartedAt: new Date().toISOString(), - }, - ); - }, authContext); + await this.globalWorkspaceOrmManager.executeInWorkspaceContext( + async () => { + await this.calendarChannelRepository.update( + { id: In(calendarChannelIds), workspaceId }, + { + syncStage: CalendarChannelSyncStage.CALENDAR_EVENTS_IMPORT_ONGOING, + syncStatus: CalendarChannelSyncStatus.ONGOING, + syncStageStartedAt: new Date().toISOString(), + }, + ); + }, + authContext, + { lite: true }, + ); } public async markAsCompletedAndMarkAsCalendarEventListFetchPending( @@ -186,18 +216,23 @@ export class CalendarChannelSyncStatusService { const authContext = buildSystemAuthContext(workspaceId); - await this.globalWorkspaceOrmManager.executeInWorkspaceContext(async () => { - await this.calendarChannelRepository.update( - { id: In(calendarChannelIds), workspaceId }, - { - syncStage: CalendarChannelSyncStage.CALENDAR_EVENT_LIST_FETCH_PENDING, - syncStatus: CalendarChannelSyncStatus.ACTIVE, - throttleFailureCount: 0, - syncStageStartedAt: null, - syncedAt: new Date().toISOString(), - }, - ); - }, authContext); + await this.globalWorkspaceOrmManager.executeInWorkspaceContext( + async () => { + await this.calendarChannelRepository.update( + { id: In(calendarChannelIds), workspaceId }, + { + syncStage: + CalendarChannelSyncStage.CALENDAR_EVENT_LIST_FETCH_PENDING, + syncStatus: CalendarChannelSyncStatus.ACTIVE, + throttleFailureCount: 0, + syncStageStartedAt: null, + syncedAt: new Date().toISOString(), + }, + ); + }, + authContext, + { lite: true }, + ); await this.markAsCalendarEventListFetchPending( calendarChannelIds, @@ -226,15 +261,19 @@ export class CalendarChannelSyncStatusService { const authContext = buildSystemAuthContext(workspaceId); - await this.globalWorkspaceOrmManager.executeInWorkspaceContext(async () => { - await this.calendarChannelRepository.update( - { id: In(calendarChannelIds), workspaceId }, - { - syncStatus: CalendarChannelSyncStatus.FAILED_UNKNOWN, - syncStage: CalendarChannelSyncStage.FAILED, - }, - ); - }, authContext); + await this.globalWorkspaceOrmManager.executeInWorkspaceContext( + async () => { + await this.calendarChannelRepository.update( + { id: In(calendarChannelIds), workspaceId }, + { + syncStatus: CalendarChannelSyncStatus.FAILED_UNKNOWN, + syncStage: CalendarChannelSyncStage.FAILED, + }, + ); + }, + authContext, + { lite: true }, + ); await this.metricsService.batchIncrementCounter({ key: MetricsKeys.CalendarEventSyncJobFailedUnknown, @@ -258,36 +297,41 @@ export class CalendarChannelSyncStatusService { const authContext = buildSystemAuthContext(workspaceId); - await this.globalWorkspaceOrmManager.executeInWorkspaceContext(async () => { - await this.calendarChannelRepository.update( - { id: In(calendarChannelIds), workspaceId }, - { - syncStatus: CalendarChannelSyncStatus.FAILED_INSUFFICIENT_PERMISSIONS, - syncStage: CalendarChannelSyncStage.FAILED, - }, - ); + await this.globalWorkspaceOrmManager.executeInWorkspaceContext( + async () => { + await this.calendarChannelRepository.update( + { id: In(calendarChannelIds), workspaceId }, + { + syncStatus: + CalendarChannelSyncStatus.FAILED_INSUFFICIENT_PERMISSIONS, + syncStage: CalendarChannelSyncStage.FAILED, + }, + ); - const calendarChannels = await this.calendarChannelRepository.find({ - select: ['id', 'connectedAccountId'], - where: { id: Any(calendarChannelIds), workspaceId }, - }); + const calendarChannels = await this.calendarChannelRepository.find({ + select: ['id', 'connectedAccountId'], + where: { id: Any(calendarChannelIds), workspaceId }, + }); - const connectedAccountIds = calendarChannels.map( - (calendarChannel) => calendarChannel.connectedAccountId, - ); + const connectedAccountIds = calendarChannels.map( + (calendarChannel) => calendarChannel.connectedAccountId, + ); - await this.connectedAccountRepository.update( - { id: Any(connectedAccountIds), workspaceId }, - { - authFailedAt: new Date(), - }, - ); + await this.connectedAccountRepository.update( + { id: Any(connectedAccountIds), workspaceId }, + { + authFailedAt: new Date(), + }, + ); - await this.addToAccountsToReconnect( - calendarChannels.map((calendarChannel) => calendarChannel.id), - workspaceId, - ); - }, authContext); + await this.addToAccountsToReconnect( + calendarChannels.map((calendarChannel) => calendarChannel.id), + workspaceId, + ); + }, + authContext, + { lite: true }, + ); await this.metricsService.batchIncrementCounter({ key: MetricsKeys.CalendarEventSyncJobFailedInsufficientPermissions, diff --git a/packages/twenty-server/src/modules/connected-account/email-alias-manager/drivers/google/services/google-email-alias-manager.service.ts b/packages/twenty-server/src/modules/connected-account/email-alias-manager/drivers/google/services/google-email-alias-manager.service.ts index f9e30b9c1c..4eed709c7d 100644 --- a/packages/twenty-server/src/modules/connected-account/email-alias-manager/drivers/google/services/google-email-alias-manager.service.ts +++ b/packages/twenty-server/src/modules/connected-account/email-alias-manager/drivers/google/services/google-email-alias-manager.service.ts @@ -19,31 +19,22 @@ export class GoogleEmailAliasManagerService { connectedAccount, ); - const peopleClient = google.people({ + const gmailClient = google.gmail({ version: 'v1', auth: oAuth2Client, }); - const emailsResponse = await peopleClient.people - .get({ - resourceName: 'people/me', - personFields: 'emailAddresses', - }) + const sendAsResponse = await gmailClient.users.settings.sendAs + .list({ userId: 'me' }) .catch((error) => { throw this.gmailEmailAliasErrorHandlerService.handleError(error); }); - const emailAddresses = emailsResponse.data.emailAddresses; - - const handleAliases = - emailAddresses - ?.filter((emailAddress) => { - return emailAddress.metadata?.primary !== true; - }) - .map((emailAddress) => { - return emailAddress.value || ''; - }) || []; - - return handleAliases; + return ( + sendAsResponse.data.sendAs + ?.filter((alias) => alias.isPrimary !== true) + .map((alias) => alias.sendAsEmail || '') + .filter((email) => email.length > 0) ?? [] + ); } } diff --git a/packages/twenty-server/src/modules/messaging/blocklist-manager/jobs/messaging-blocklist-item-delete-messages.job.ts b/packages/twenty-server/src/modules/messaging/blocklist-manager/jobs/messaging-blocklist-item-delete-messages.job.ts index b183067ed6..80524aa850 100644 --- a/packages/twenty-server/src/modules/messaging/blocklist-manager/jobs/messaging-blocklist-item-delete-messages.job.ts +++ b/packages/twenty-server/src/modules/messaging/blocklist-manager/jobs/messaging-blocklist-item-delete-messages.job.ts @@ -46,159 +46,164 @@ export class BlocklistItemDeleteMessagesJob { const authContext = buildSystemAuthContext(workspaceId); - await this.globalWorkspaceOrmManager.executeInWorkspaceContext(async () => { - const blocklistItemIds = data.events.map( - (eventPayload) => eventPayload.recordId, - ); - - const blocklistRepository = - await this.globalWorkspaceOrmManager.getRepository( - workspaceId, - 'blocklist', + await this.globalWorkspaceOrmManager.executeInWorkspaceContext( + async () => { + const blocklistItemIds = data.events.map( + (eventPayload) => eventPayload.recordId, ); - const blocklist = await blocklistRepository.find({ - where: { - id: Any(blocklistItemIds), - }, - }); - - const handlesToDeleteByWorkspaceMemberIdMap = blocklist.reduce( - (acc, blocklistItem) => { - const { handle, workspaceMemberId } = blocklistItem; - - if (!acc.has(workspaceMemberId)) { - acc.set(workspaceMemberId, []); - } - - if (!isDefined(handle)) { - return acc; - } - - acc.get(workspaceMemberId)?.push(handle); - - return acc; - }, - new Map(), - ); - - const messageChannelMessageAssociationRepository = - await this.globalWorkspaceOrmManager.getRepository( - workspaceId, - 'messageChannelMessageAssociation', - ); - - const workspaceMemberRepository = - await this.globalWorkspaceOrmManager.getRepository( - workspaceId, - 'workspaceMember', - { shouldBypassPermissionChecks: true }, - ); - - for (const workspaceMemberId of handlesToDeleteByWorkspaceMemberIdMap.keys()) { - const handles = - handlesToDeleteByWorkspaceMemberIdMap.get(workspaceMemberId); - - if (!handles) { - continue; - } - - const rolesToDelete = [ - MessageParticipantRole.FROM, - MessageParticipantRole.TO, - ] as const; - - const workspaceMember = await workspaceMemberRepository.findOne({ - where: { id: workspaceMemberId }, - }); - - if (!workspaceMember) { - continue; - } - - const userWorkspace = await this.userWorkspaceRepository.findOne({ - where: { userId: workspaceMember.userId, workspaceId }, - }); - - if (!userWorkspace) { - continue; - } - - const connectedAccounts = await this.connectedAccountRepository.find({ - where: { userWorkspaceId: userWorkspace.id, workspaceId }, - }); - - const connectedAccountIds = connectedAccounts.map((ca) => ca.id); - - if (connectedAccountIds.length === 0) { - continue; - } - - const messageChannels = await this.messageChannelRepository.find({ - select: { - id: true, - handle: true, - connectedAccount: { - handleAliases: true, - }, - }, - where: { - connectedAccountId: In(connectedAccountIds), + const blocklistRepository = + await this.globalWorkspaceOrmManager.getRepository( workspaceId, + 'blocklist', + ); + + const blocklist = await blocklistRepository.find({ + where: { + id: Any(blocklistItemIds), }, - relations: { connectedAccount: true }, }); - for (const messageChannel of messageChannels) { - const messageChannelHandles = [messageChannel.handle]; + const handlesToDeleteByWorkspaceMemberIdMap = blocklist.reduce( + (acc, blocklistItem) => { + const { handle, workspaceMemberId } = blocklistItem; - const handleAliases = messageChannel.connectedAccount?.handleAliases; + if (!acc.has(workspaceMemberId)) { + acc.set(workspaceMemberId, []); + } - if (isDefined(handleAliases)) { - const aliasList: string[] = Array.isArray(handleAliases) - ? handleAliases - : (handleAliases as string).split(','); + if (!isDefined(handle)) { + return acc; + } - messageChannelHandles.push(...aliasList); - } + acc.get(workspaceMemberId)?.push(handle); - const handleConditions = handles.map((handle) => { - const isHandleDomain = handle.startsWith('@'); + return acc; + }, + new Map(), + ); - return isHandleDomain - ? { - handle: And( - Or(ILike(`%${handle}`), ILike(`%.${handle.slice(1)}`)), - Not(In(messageChannelHandles)), - ), - role: In(rolesToDelete), - } - : { handle, role: In(rolesToDelete) }; - }); + const messageChannelMessageAssociationRepository = + await this.globalWorkspaceOrmManager.getRepository( + workspaceId, + 'messageChannelMessageAssociation', + ); - const messageChannelMessageAssociationsToDelete = - await messageChannelMessageAssociationRepository.find({ - where: { - messageChannelId: messageChannel.id, - message: { - messageParticipants: handleConditions, - }, - }, - }); + const workspaceMemberRepository = + await this.globalWorkspaceOrmManager.getRepository( + workspaceId, + 'workspaceMember', + { shouldBypassPermissionChecks: true }, + ); - if (messageChannelMessageAssociationsToDelete.length === 0) { + for (const workspaceMemberId of handlesToDeleteByWorkspaceMemberIdMap.keys()) { + const handles = + handlesToDeleteByWorkspaceMemberIdMap.get(workspaceMemberId); + + if (!handles) { continue; } - await messageChannelMessageAssociationRepository.delete( - messageChannelMessageAssociationsToDelete.map(({ id }) => id), - ); - } - } + const rolesToDelete = [ + MessageParticipantRole.FROM, + MessageParticipantRole.TO, + ] as const; - await this.threadCleanerService.cleanOrphanMessagesAndThreads( - workspaceId, - ); - }, authContext); + const workspaceMember = await workspaceMemberRepository.findOne({ + where: { id: workspaceMemberId }, + }); + + if (!workspaceMember) { + continue; + } + + const userWorkspace = await this.userWorkspaceRepository.findOne({ + where: { userId: workspaceMember.userId, workspaceId }, + }); + + if (!userWorkspace) { + continue; + } + + const connectedAccounts = await this.connectedAccountRepository.find({ + where: { userWorkspaceId: userWorkspace.id, workspaceId }, + }); + + const connectedAccountIds = connectedAccounts.map((ca) => ca.id); + + if (connectedAccountIds.length === 0) { + continue; + } + + const messageChannels = await this.messageChannelRepository.find({ + select: { + id: true, + handle: true, + connectedAccount: { + handleAliases: true, + }, + }, + where: { + connectedAccountId: In(connectedAccountIds), + workspaceId, + }, + relations: { connectedAccount: true }, + }); + + for (const messageChannel of messageChannels) { + const messageChannelHandles = [messageChannel.handle]; + + const handleAliases = + messageChannel.connectedAccount?.handleAliases; + + if (isDefined(handleAliases)) { + const aliasList: string[] = Array.isArray(handleAliases) + ? handleAliases + : (handleAliases as string).split(','); + + messageChannelHandles.push(...aliasList); + } + + const handleConditions = handles.map((handle) => { + const isHandleDomain = handle.startsWith('@'); + + return isHandleDomain + ? { + handle: And( + Or(ILike(`%${handle}`), ILike(`%.${handle.slice(1)}`)), + Not(In(messageChannelHandles)), + ), + role: In(rolesToDelete), + } + : { handle, role: In(rolesToDelete) }; + }); + + const messageChannelMessageAssociationsToDelete = + await messageChannelMessageAssociationRepository.find({ + where: { + messageChannelId: messageChannel.id, + message: { + messageParticipants: handleConditions, + }, + }, + }); + + if (messageChannelMessageAssociationsToDelete.length === 0) { + continue; + } + + await messageChannelMessageAssociationRepository.delete( + messageChannelMessageAssociationsToDelete.map(({ id }) => id), + ); + } + } + + await this.threadCleanerService.cleanOrphanMessagesAndThreads( + workspaceId, + ); + }, + authContext, + { lite: true }, + ); } } diff --git a/packages/twenty-server/src/modules/messaging/blocklist-manager/jobs/messaging-blocklist-reimport-messages.job.ts b/packages/twenty-server/src/modules/messaging/blocklist-manager/jobs/messaging-blocklist-reimport-messages.job.ts index 03d6582115..710d34e624 100644 --- a/packages/twenty-server/src/modules/messaging/blocklist-manager/jobs/messaging-blocklist-reimport-messages.job.ts +++ b/packages/twenty-server/src/modules/messaging/blocklist-manager/jobs/messaging-blocklist-reimport-messages.job.ts @@ -44,57 +44,63 @@ export class BlocklistReimportMessagesJob { const authContext = buildSystemAuthContext(workspaceId); - await this.globalWorkspaceOrmManager.executeInWorkspaceContext(async () => { - const workspaceMemberRepository = - await this.globalWorkspaceOrmManager.getRepository( - workspaceId, - 'workspaceMember', - { shouldBypassPermissionChecks: true }, - ); - - for (const eventPayload of data.events) { - const workspaceMemberId = - eventPayload.properties.before.workspaceMemberId; - - const workspaceMember = await workspaceMemberRepository.findOne({ - where: { id: workspaceMemberId }, - }); - - if (!workspaceMember) { - continue; - } - - const userWorkspace = await this.userWorkspaceRepository.findOne({ - where: { userId: workspaceMember.userId, workspaceId }, - }); - - if (!userWorkspace) { - continue; - } - - const connectedAccounts = await this.connectedAccountRepository.find({ - where: { userWorkspaceId: userWorkspace.id, workspaceId }, - }); - - const connectedAccountIds = connectedAccounts.map((ca) => ca.id); - - if (connectedAccountIds.length === 0) { - continue; - } - - const messageChannels = await this.messageChannelRepository.find({ - where: { - connectedAccountId: In(connectedAccountIds), - syncStage: Not(MessageChannelSyncStage.MESSAGE_LIST_FETCH_PENDING), + await this.globalWorkspaceOrmManager.executeInWorkspaceContext( + async () => { + const workspaceMemberRepository = + await this.globalWorkspaceOrmManager.getRepository( workspaceId, - }, - }); + 'workspaceMember', + { shouldBypassPermissionChecks: true }, + ); - await this.messagingChannelSyncStatusService.resetAndMarkAsMessagesListFetchPending( - messageChannels.map((messageChannel) => messageChannel.id), - workspaceId, - ); - } - }, authContext); + for (const eventPayload of data.events) { + const workspaceMemberId = + eventPayload.properties.before.workspaceMemberId; + + const workspaceMember = await workspaceMemberRepository.findOne({ + where: { id: workspaceMemberId }, + }); + + if (!workspaceMember) { + continue; + } + + const userWorkspace = await this.userWorkspaceRepository.findOne({ + where: { userId: workspaceMember.userId, workspaceId }, + }); + + if (!userWorkspace) { + continue; + } + + const connectedAccounts = await this.connectedAccountRepository.find({ + where: { userWorkspaceId: userWorkspace.id, workspaceId }, + }); + + const connectedAccountIds = connectedAccounts.map((ca) => ca.id); + + if (connectedAccountIds.length === 0) { + continue; + } + + const messageChannels = await this.messageChannelRepository.find({ + where: { + connectedAccountId: In(connectedAccountIds), + syncStage: Not( + MessageChannelSyncStage.MESSAGE_LIST_FETCH_PENDING, + ), + workspaceId, + }, + }); + + await this.messagingChannelSyncStatusService.resetAndMarkAsMessagesListFetchPending( + messageChannels.map((messageChannel) => messageChannel.id), + workspaceId, + ); + } + }, + authContext, + { lite: true }, + ); } } diff --git a/packages/twenty-server/src/modules/messaging/common/query-hooks/message/apply-messages-visibility-restrictions.service.ts b/packages/twenty-server/src/modules/messaging/common/query-hooks/message/apply-messages-visibility-restrictions.service.ts index 1b6481d993..7a674c53cd 100644 --- a/packages/twenty-server/src/modules/messaging/common/query-hooks/message/apply-messages-visibility-restrictions.service.ts +++ b/packages/twenty-server/src/modules/messaging/common/query-hooks/message/apply-messages-visibility-restrictions.service.ts @@ -157,6 +157,7 @@ export class ApplyMessagesVisibilityRestrictionsService { return messages; }, authContext, + { lite: true }, ); } } diff --git a/packages/twenty-server/src/modules/messaging/common/services/message-channel-sync-status.service.ts b/packages/twenty-server/src/modules/messaging/common/services/message-channel-sync-status.service.ts index d1ae372c56..fed6c5c6d4 100644 --- a/packages/twenty-server/src/modules/messaging/common/services/message-channel-sync-status.service.ts +++ b/packages/twenty-server/src/modules/messaging/common/services/message-channel-sync-status.service.ts @@ -53,15 +53,21 @@ export class MessageChannelSyncStatusService { const authContext = buildSystemAuthContext(workspaceId); - await this.globalWorkspaceOrmManager.executeInWorkspaceContext(async () => { - await this.messageChannelRepository.update( - { id: In(messageChannelIds), workspaceId }, - { - syncStage: MessageChannelSyncStage.MESSAGE_LIST_FETCH_PENDING, - ...(!preserveSyncStageStartedAt ? { syncStageStartedAt: null } : {}), - }, - ); - }, authContext); + await this.globalWorkspaceOrmManager.executeInWorkspaceContext( + async () => { + await this.messageChannelRepository.update( + { id: In(messageChannelIds), workspaceId }, + { + syncStage: MessageChannelSyncStage.MESSAGE_LIST_FETCH_PENDING, + ...(!preserveSyncStageStartedAt + ? { syncStageStartedAt: null } + : {}), + }, + ); + }, + authContext, + { lite: true }, + ); } public async markAsMessagesImportPending( @@ -75,15 +81,21 @@ export class MessageChannelSyncStatusService { const authContext = buildSystemAuthContext(workspaceId); - await this.globalWorkspaceOrmManager.executeInWorkspaceContext(async () => { - await this.messageChannelRepository.update( - { id: In(messageChannelIds), workspaceId }, - { - syncStage: MessageChannelSyncStage.MESSAGES_IMPORT_PENDING, - ...(!preserveSyncStageStartedAt ? { syncStageStartedAt: null } : {}), - }, - ); - }, authContext); + await this.globalWorkspaceOrmManager.executeInWorkspaceContext( + async () => { + await this.messageChannelRepository.update( + { id: In(messageChannelIds), workspaceId }, + { + syncStage: MessageChannelSyncStage.MESSAGES_IMPORT_PENDING, + ...(!preserveSyncStageStartedAt + ? { syncStageStartedAt: null } + : {}), + }, + ); + }, + authContext, + { lite: true }, + ); } public async resetAndMarkAsMessagesListFetchPending( @@ -102,26 +114,31 @@ export class MessageChannelSyncStatusService { const authContext = buildSystemAuthContext(workspaceId); - await this.globalWorkspaceOrmManager.executeInWorkspaceContext(async () => { - await this.messageChannelRepository.update( - { id: In(messageChannelIds), workspaceId }, - { - syncCursor: '', - syncStageStartedAt: null, - throttleFailureCount: 0, - throttleRetryAfter: null, - pendingGroupEmailsAction: MessageChannelPendingGroupEmailsAction.NONE, - }, - ); + await this.globalWorkspaceOrmManager.executeInWorkspaceContext( + async () => { + await this.messageChannelRepository.update( + { id: In(messageChannelIds), workspaceId }, + { + syncCursor: '', + syncStageStartedAt: null, + throttleFailureCount: 0, + throttleRetryAfter: null, + pendingGroupEmailsAction: + MessageChannelPendingGroupEmailsAction.NONE, + }, + ); - await this.messageFolderRepository.update( - { messageChannelId: In(messageChannelIds), workspaceId }, - { - syncCursor: '', - pendingSyncAction: MessageFolderPendingSyncAction.NONE, - }, - ); - }, authContext); + await this.messageFolderRepository.update( + { messageChannelId: In(messageChannelIds), workspaceId }, + { + syncCursor: '', + pendingSyncAction: MessageFolderPendingSyncAction.NONE, + }, + ); + }, + authContext, + { lite: true }, + ); await this.markAsMessagesListFetchPending(messageChannelIds, workspaceId); } @@ -136,12 +153,16 @@ export class MessageChannelSyncStatusService { const authContext = buildSystemAuthContext(workspaceId); - await this.globalWorkspaceOrmManager.executeInWorkspaceContext(async () => { - await this.messageChannelRepository.update( - { id: In(messageChannelIds), workspaceId }, - { syncStageStartedAt: null }, - ); - }, authContext); + await this.globalWorkspaceOrmManager.executeInWorkspaceContext( + async () => { + await this.messageChannelRepository.update( + { id: In(messageChannelIds), workspaceId }, + { syncStageStartedAt: null }, + ); + }, + authContext, + { lite: true }, + ); } public async markAsMessagesListFetchScheduled( @@ -154,16 +175,20 @@ export class MessageChannelSyncStatusService { const authContext = buildSystemAuthContext(workspaceId); - await this.globalWorkspaceOrmManager.executeInWorkspaceContext(async () => { - await this.messageChannelRepository.update( - { id: In(messageChannelIds), workspaceId }, - { - syncStage: MessageChannelSyncStage.MESSAGE_LIST_FETCH_SCHEDULED, - syncStatus: MessageChannelSyncStatus.ONGOING, - syncStageStartedAt: new Date().toISOString(), - }, - ); - }, authContext); + await this.globalWorkspaceOrmManager.executeInWorkspaceContext( + async () => { + await this.messageChannelRepository.update( + { id: In(messageChannelIds), workspaceId }, + { + syncStage: MessageChannelSyncStage.MESSAGE_LIST_FETCH_SCHEDULED, + syncStatus: MessageChannelSyncStatus.ONGOING, + syncStageStartedAt: new Date().toISOString(), + }, + ); + }, + authContext, + { lite: true }, + ); } public async markAsMessagesListFetchOngoing( @@ -176,16 +201,20 @@ export class MessageChannelSyncStatusService { const authContext = buildSystemAuthContext(workspaceId); - await this.globalWorkspaceOrmManager.executeInWorkspaceContext(async () => { - await this.messageChannelRepository.update( - { id: In(messageChannelIds), workspaceId }, - { - syncStage: MessageChannelSyncStage.MESSAGE_LIST_FETCH_ONGOING, - syncStatus: MessageChannelSyncStatus.ONGOING, - syncStageStartedAt: new Date().toISOString(), - }, - ); - }, authContext); + await this.globalWorkspaceOrmManager.executeInWorkspaceContext( + async () => { + await this.messageChannelRepository.update( + { id: In(messageChannelIds), workspaceId }, + { + syncStage: MessageChannelSyncStage.MESSAGE_LIST_FETCH_ONGOING, + syncStatus: MessageChannelSyncStatus.ONGOING, + syncStageStartedAt: new Date().toISOString(), + }, + ); + }, + authContext, + { lite: true }, + ); } public async markAsCompletedAndMarkAsMessagesListFetchPending( @@ -198,19 +227,23 @@ export class MessageChannelSyncStatusService { const authContext = buildSystemAuthContext(workspaceId); - await this.globalWorkspaceOrmManager.executeInWorkspaceContext(async () => { - await this.messageChannelRepository.update( - { id: In(messageChannelIds), workspaceId }, - { - syncStatus: MessageChannelSyncStatus.ACTIVE, - syncStage: MessageChannelSyncStage.MESSAGE_LIST_FETCH_PENDING, - throttleFailureCount: 0, - throttleRetryAfter: null, - syncStageStartedAt: null, - syncedAt: new Date().toISOString(), - }, - ); - }, authContext); + await this.globalWorkspaceOrmManager.executeInWorkspaceContext( + async () => { + await this.messageChannelRepository.update( + { id: In(messageChannelIds), workspaceId }, + { + syncStatus: MessageChannelSyncStatus.ACTIVE, + syncStage: MessageChannelSyncStage.MESSAGE_LIST_FETCH_PENDING, + throttleFailureCount: 0, + throttleRetryAfter: null, + syncStageStartedAt: null, + syncedAt: new Date().toISOString(), + }, + ); + }, + authContext, + { lite: true }, + ); await this.metricsService.batchIncrementCounter({ key: MetricsKeys.MessageChannelSyncJobActive, @@ -228,14 +261,18 @@ export class MessageChannelSyncStatusService { const authContext = buildSystemAuthContext(workspaceId); - await this.globalWorkspaceOrmManager.executeInWorkspaceContext(async () => { - await this.messageChannelRepository.update( - { id: In(messageChannelIds), workspaceId }, - { - syncStage: MessageChannelSyncStage.MESSAGES_IMPORT_SCHEDULED, - }, - ); - }, authContext); + await this.globalWorkspaceOrmManager.executeInWorkspaceContext( + async () => { + await this.messageChannelRepository.update( + { id: In(messageChannelIds), workspaceId }, + { + syncStage: MessageChannelSyncStage.MESSAGES_IMPORT_SCHEDULED, + }, + ); + }, + authContext, + { lite: true }, + ); } public async markAsMessagesImportOngoing( @@ -248,16 +285,20 @@ export class MessageChannelSyncStatusService { const authContext = buildSystemAuthContext(workspaceId); - await this.globalWorkspaceOrmManager.executeInWorkspaceContext(async () => { - await this.messageChannelRepository.update( - { id: In(messageChannelIds), workspaceId }, - { - syncStage: MessageChannelSyncStage.MESSAGES_IMPORT_ONGOING, - syncStatus: MessageChannelSyncStatus.ONGOING, - syncStageStartedAt: new Date().toISOString(), - }, - ); - }, authContext); + await this.globalWorkspaceOrmManager.executeInWorkspaceContext( + async () => { + await this.messageChannelRepository.update( + { id: In(messageChannelIds), workspaceId }, + { + syncStage: MessageChannelSyncStage.MESSAGES_IMPORT_ONGOING, + syncStatus: MessageChannelSyncStatus.ONGOING, + syncStageStartedAt: new Date().toISOString(), + }, + ); + }, + authContext, + { lite: true }, + ); } public async markAsFailed( @@ -273,50 +314,56 @@ export class MessageChannelSyncStatusService { const authContext = buildSystemAuthContext(workspaceId); - await this.globalWorkspaceOrmManager.executeInWorkspaceContext(async () => { - await this.messageChannelRepository.update( - { id: In(messageChannelIds), workspaceId }, - { - syncStage: MessageChannelSyncStage.FAILED, - syncStatus: syncStatus, - throttleRetryAfter: null, - }, - ); - - const metricsKey = - syncStatus === MessageChannelSyncStatus.FAILED_INSUFFICIENT_PERMISSIONS - ? MetricsKeys.MessageChannelSyncJobFailedInsufficientPermissions - : MetricsKeys.MessageChannelSyncJobFailedUnknown; - - await this.metricsService.batchIncrementCounter({ - key: metricsKey, - eventIds: messageChannelIds, - }); - - if ( - syncStatus === MessageChannelSyncStatus.FAILED_INSUFFICIENT_PERMISSIONS - ) { - const messageChannels = await this.messageChannelRepository.find({ - where: { id: In(messageChannelIds), workspaceId }, - }); - - const connectedAccountIds = messageChannels.map( - (messageChannel) => messageChannel.connectedAccountId, - ); - - await this.connectedAccountRepository.update( - { id: Any(connectedAccountIds), workspaceId }, + await this.globalWorkspaceOrmManager.executeInWorkspaceContext( + async () => { + await this.messageChannelRepository.update( + { id: In(messageChannelIds), workspaceId }, { - authFailedAt: new Date(), + syncStage: MessageChannelSyncStage.FAILED, + syncStatus: syncStatus, + throttleRetryAfter: null, }, ); - await this.addToAccountsToReconnect( - messageChannels.map((messageChannel) => messageChannel.id), - workspaceId, - ); - } - }, authContext); + const metricsKey = + syncStatus === + MessageChannelSyncStatus.FAILED_INSUFFICIENT_PERMISSIONS + ? MetricsKeys.MessageChannelSyncJobFailedInsufficientPermissions + : MetricsKeys.MessageChannelSyncJobFailedUnknown; + + await this.metricsService.batchIncrementCounter({ + key: metricsKey, + eventIds: messageChannelIds, + }); + + if ( + syncStatus === + MessageChannelSyncStatus.FAILED_INSUFFICIENT_PERMISSIONS + ) { + const messageChannels = await this.messageChannelRepository.find({ + where: { id: In(messageChannelIds), workspaceId }, + }); + + const connectedAccountIds = messageChannels.map( + (messageChannel) => messageChannel.connectedAccountId, + ); + + await this.connectedAccountRepository.update( + { id: Any(connectedAccountIds), workspaceId }, + { + authFailedAt: new Date(), + }, + ); + + await this.addToAccountsToReconnect( + messageChannels.map((messageChannel) => messageChannel.id), + workspaceId, + ); + } + }, + authContext, + { lite: true }, + ); } private async addToAccountsToReconnect( diff --git a/packages/twenty-server/src/modules/messaging/message-cleaner/commands/messaging-reset-channel.command.ts b/packages/twenty-server/src/modules/messaging/message-cleaner/commands/messaging-reset-channel.command.ts index 5263fbbf1d..a4cd71f8ba 100644 --- a/packages/twenty-server/src/modules/messaging/message-cleaner/commands/messaging-reset-channel.command.ts +++ b/packages/twenty-server/src/modules/messaging/message-cleaner/commands/messaging-reset-channel.command.ts @@ -42,44 +42,48 @@ export class MessagingResetChannelCommand extends CommandRunner { const authContext = buildSystemAuthContext(workspaceId); - await this.globalWorkspaceOrmManager.executeInWorkspaceContext(async () => { - this.logger.log( - `No message channel ID provided, resetting all message channels in workspace ${workspaceId}`, - ); - - const messageChannels = await this.messageChannelRepository.find({ - where: { - ...(isDefined(messageChannelId) ? { id: messageChannelId } : {}), - workspaceId, - }, - }); - - if (messageChannels.length === 0) { + await this.globalWorkspaceOrmManager.executeInWorkspaceContext( + async () => { this.logger.log( - `No message channels found in workspace ${workspaceId}`, + `No message channel ID provided, resetting all message channels in workspace ${workspaceId}`, ); - return; - } + const messageChannels = await this.messageChannelRepository.find({ + where: { + ...(isDefined(messageChannelId) ? { id: messageChannelId } : {}), + workspaceId, + }, + }); - this.logger.log( - `Found ${messageChannels.length} message channels to reset`, - ); + if (messageChannels.length === 0) { + this.logger.log( + `No message channels found in workspace ${workspaceId}`, + ); - for (const messageChannel of messageChannels) { - await this.messagingChannelSyncStatusService.resetAndMarkAsMessagesListFetchPending( - [messageChannel.id], - workspaceId, + return; + } + + this.logger.log( + `Found ${messageChannels.length} message channels to reset`, ); - await this.messagingMessageCleanerService.cleanOrphanMessagesAndThreads( - workspaceId, - ); - } - this.logger.log( - `Successfully reset all ${messageChannels.length} message channels in workspace ${workspaceId}`, - ); - }, authContext); + for (const messageChannel of messageChannels) { + await this.messagingChannelSyncStatusService.resetAndMarkAsMessagesListFetchPending( + [messageChannel.id], + workspaceId, + ); + await this.messagingMessageCleanerService.cleanOrphanMessagesAndThreads( + workspaceId, + ); + } + + this.logger.log( + `Successfully reset all ${messageChannels.length} message channels in workspace ${workspaceId}`, + ); + }, + authContext, + { lite: true }, + ); } @Option({ diff --git a/packages/twenty-server/src/modules/messaging/message-cleaner/services/messaging-message-cleaner.service.ts b/packages/twenty-server/src/modules/messaging/message-cleaner/services/messaging-message-cleaner.service.ts index 7b2f6ce43c..c83dbf3117 100644 --- a/packages/twenty-server/src/modules/messaging/message-cleaner/services/messaging-message-cleaner.service.ts +++ b/packages/twenty-server/src/modules/messaging/message-cleaner/services/messaging-message-cleaner.service.ts @@ -29,95 +29,99 @@ export class MessagingMessageCleanerService { }) { const authContext = buildSystemAuthContext(workspaceId); - await this.globalWorkspaceOrmManager.executeInWorkspaceContext(async () => { - const messageRepository = - await this.globalWorkspaceOrmManager.getRepository( - workspaceId, - 'message', - ); + await this.globalWorkspaceOrmManager.executeInWorkspaceContext( + async () => { + const messageRepository = + await this.globalWorkspaceOrmManager.getRepository( + workspaceId, + 'message', + ); - const messageChannelMessageAssociationRepository = - await this.globalWorkspaceOrmManager.getRepository( - workspaceId, - 'messageChannelMessageAssociation', - ); + const messageChannelMessageAssociationRepository = + await this.globalWorkspaceOrmManager.getRepository( + workspaceId, + 'messageChannelMessageAssociation', + ); - const messageThreadRepository = - await this.globalWorkspaceOrmManager.getRepository( - workspaceId, - 'messageThread', - ); + const messageThreadRepository = + await this.globalWorkspaceOrmManager.getRepository( + workspaceId, + 'messageThread', + ); - const messageExternalIdsChunks = chunk(messageExternalIds, 500); + const messageExternalIdsChunks = chunk(messageExternalIds, 500); - for (const messageExternalIdsChunk of messageExternalIdsChunks) { - const messageChannelMessageAssociationsToDelete = - await messageChannelMessageAssociationRepository.find({ + for (const messageExternalIdsChunk of messageExternalIdsChunks) { + const messageChannelMessageAssociationsToDelete = + await messageChannelMessageAssociationRepository.find({ + where: { + messageExternalId: In(messageExternalIdsChunk), + messageChannelId, + }, + }); + + if (messageChannelMessageAssociationsToDelete.length <= 0) { + continue; + } + + await messageChannelMessageAssociationRepository.delete( + messageChannelMessageAssociationsToDelete.map(({ id }) => id), + ); + + this.logger.log( + `WorkspaceId: ${workspaceId} Deleting ${messageChannelMessageAssociationsToDelete.length} message channel message associations`, + ); + + const orphanMessages = await messageRepository.find({ where: { - messageExternalId: In(messageExternalIdsChunk), - messageChannelId, + id: In( + messageChannelMessageAssociationsToDelete.map( + ({ messageId }) => messageId, + ), + ), + messageChannelMessageAssociations: { + id: IsNull(), + }, }, }); - if (messageChannelMessageAssociationsToDelete.length <= 0) { - continue; - } + if (orphanMessages.length <= 0) { + continue; + } - await messageChannelMessageAssociationRepository.delete( - messageChannelMessageAssociationsToDelete.map(({ id }) => id), - ); + this.logger.debug( + `WorkspaceId: ${workspaceId} Deleting ${orphanMessages.length} orphan messages`, + ); - this.logger.log( - `WorkspaceId: ${workspaceId} Deleting ${messageChannelMessageAssociationsToDelete.length} message channel message associations`, - ); + await messageRepository.delete(orphanMessages.map(({ id }) => id)); - const orphanMessages = await messageRepository.find({ - where: { - id: In( - messageChannelMessageAssociationsToDelete.map( - ({ messageId }) => messageId, + const orphanMessageThreads = await messageThreadRepository.find({ + where: { + id: In( + orphanMessages.map(({ messageThreadId }) => messageThreadId), ), - ), - messageChannelMessageAssociations: { - id: IsNull(), + messages: { + id: IsNull(), + }, }, - }, - }); + }); - if (orphanMessages.length <= 0) { - continue; + if (orphanMessageThreads.length <= 0) { + continue; + } + + this.logger.debug( + `WorkspaceId: ${workspaceId} Deleting ${orphanMessageThreads.length} orphan message threads`, + ); + + await messageThreadRepository.delete( + orphanMessageThreads.map(({ id }) => id), + ); } - - this.logger.debug( - `WorkspaceId: ${workspaceId} Deleting ${orphanMessages.length} orphan messages`, - ); - - await messageRepository.delete(orphanMessages.map(({ id }) => id)); - - const orphanMessageThreads = await messageThreadRepository.find({ - where: { - id: In( - orphanMessages.map(({ messageThreadId }) => messageThreadId), - ), - messages: { - id: IsNull(), - }, - }, - }); - - if (orphanMessageThreads.length <= 0) { - continue; - } - - this.logger.debug( - `WorkspaceId: ${workspaceId} Deleting ${orphanMessageThreads.length} orphan message threads`, - ); - - await messageThreadRepository.delete( - orphanMessageThreads.map(({ id }) => id), - ); - } - }, authContext); + }, + authContext, + { lite: true }, + ); } async deleteMessageChannelMessageAssociationsByChannelId({ @@ -129,80 +133,20 @@ export class MessagingMessageCleanerService { }) { const authContext = buildSystemAuthContext(workspaceId); - await this.globalWorkspaceOrmManager.executeInWorkspaceContext(async () => { - const messageChannelMessageAssociationRepository = - await this.globalWorkspaceOrmManager.getRepository( - workspaceId, - 'messageChannelMessageAssociation', - ); + await this.globalWorkspaceOrmManager.executeInWorkspaceContext( + async () => { + const messageChannelMessageAssociationRepository = + await this.globalWorkspaceOrmManager.getRepository( + workspaceId, + 'messageChannelMessageAssociation', + ); - const workspaceDataSource = - await this.globalWorkspaceOrmManager.getGlobalWorkspaceDataSource(); + const workspaceDataSource = + await this.globalWorkspaceOrmManager.getGlobalWorkspaceDataSource(); - await workspaceDataSource.transaction(async (manager) => { - const transactionManager = manager as WorkspaceEntityManager; + await workspaceDataSource.transaction(async (manager) => { + const transactionManager = manager as WorkspaceEntityManager; - await deleteUsingPagination( - workspaceId, - 500, - async ( - limit: number, - offset: number, - _workspaceId: string, - transactionManager?: WorkspaceEntityManager, - ) => { - const associations = - await messageChannelMessageAssociationRepository.find( - { - where: { messageChannelId }, - take: limit, - skip: offset, - }, - transactionManager, - ); - - return associations.map(({ id }) => id); - }, - async ( - ids: string[], - workspaceId: string, - transactionManager?: WorkspaceEntityManager, - ) => { - this.logger.log( - `WorkspaceId: ${workspaceId} Deleting ${ids.length} message channel message associations for channel ${messageChannelId}`, - ); - await messageChannelMessageAssociationRepository.delete( - ids, - transactionManager, - ); - }, - transactionManager, - ); - }); - }, authContext); - } - - public async cleanOrphanMessagesAndThreads(workspaceId: string) { - const authContext = buildSystemAuthContext(workspaceId); - - await this.globalWorkspaceOrmManager.executeInWorkspaceContext(async () => { - const messageThreadRepository = - await this.globalWorkspaceOrmManager.getRepository( - workspaceId, - 'messageThread', - ); - - const messageRepository = - await this.globalWorkspaceOrmManager.getRepository( - workspaceId, - 'message', - ); - - const workspaceDataSource = - await this.globalWorkspaceOrmManager.getGlobalWorkspaceDataSource(); - - await workspaceDataSource.transaction( - async (transactionManager: WorkspaceEntityManager) => { await deleteUsingPagination( workspaceId, 500, @@ -210,72 +154,140 @@ export class MessagingMessageCleanerService { limit: number, offset: number, _workspaceId: string, - transactionManager: WorkspaceEntityManager, + transactionManager?: WorkspaceEntityManager, ) => { - const nonAssociatedMessages = await messageRepository.find( - { - where: { - messageChannelMessageAssociations: { - id: IsNull(), - }, + const associations = + await messageChannelMessageAssociationRepository.find( + { + where: { messageChannelId }, + take: limit, + skip: offset, }, - take: limit, - skip: offset, - relations: ['messageChannelMessageAssociations'], - }, - transactionManager, - ); + transactionManager, + ); - return nonAssociatedMessages.map(({ id }) => id); + return associations.map(({ id }) => id); }, async ( ids: string[], workspaceId: string, transactionManager?: WorkspaceEntityManager, ) => { - this.logger.debug( - `WorkspaceId: ${workspaceId} Deleting ${ids.length} messages from message cleaner`, + this.logger.log( + `WorkspaceId: ${workspaceId} Deleting ${ids.length} message channel message associations for channel ${messageChannelId}`, ); - await messageRepository.delete(ids, transactionManager); - }, - transactionManager, - ); - - await deleteUsingPagination( - workspaceId, - 500, - async ( - limit: number, - offset: number, - _workspaceId: string, - transactionManager?: WorkspaceEntityManager, - ) => { - const orphanThreads = await messageThreadRepository.find( - { - where: { - messages: { - id: IsNull(), - }, - }, - take: limit, - skip: offset, - }, + await messageChannelMessageAssociationRepository.delete( + ids, transactionManager, ); - - return orphanThreads.map(({ id }) => id); - }, - async ( - ids: string[], - _workspaceId: string, - transactionManager?: WorkspaceEntityManager, - ) => { - await messageThreadRepository.delete(ids, transactionManager); }, transactionManager, ); - }, - ); - }, authContext); + }); + }, + authContext, + { lite: true }, + ); + } + + public async cleanOrphanMessagesAndThreads(workspaceId: string) { + const authContext = buildSystemAuthContext(workspaceId); + + await this.globalWorkspaceOrmManager.executeInWorkspaceContext( + async () => { + const messageThreadRepository = + await this.globalWorkspaceOrmManager.getRepository( + workspaceId, + 'messageThread', + ); + + const messageRepository = + await this.globalWorkspaceOrmManager.getRepository( + workspaceId, + 'message', + ); + + const workspaceDataSource = + await this.globalWorkspaceOrmManager.getGlobalWorkspaceDataSource(); + + await workspaceDataSource.transaction( + async (transactionManager: WorkspaceEntityManager) => { + await deleteUsingPagination( + workspaceId, + 500, + async ( + limit: number, + offset: number, + _workspaceId: string, + transactionManager: WorkspaceEntityManager, + ) => { + const nonAssociatedMessages = await messageRepository.find( + { + where: { + messageChannelMessageAssociations: { + id: IsNull(), + }, + }, + take: limit, + skip: offset, + relations: ['messageChannelMessageAssociations'], + }, + transactionManager, + ); + + return nonAssociatedMessages.map(({ id }) => id); + }, + async ( + ids: string[], + workspaceId: string, + transactionManager?: WorkspaceEntityManager, + ) => { + this.logger.debug( + `WorkspaceId: ${workspaceId} Deleting ${ids.length} messages from message cleaner`, + ); + await messageRepository.delete(ids, transactionManager); + }, + transactionManager, + ); + + await deleteUsingPagination( + workspaceId, + 500, + async ( + limit: number, + offset: number, + _workspaceId: string, + transactionManager?: WorkspaceEntityManager, + ) => { + const orphanThreads = await messageThreadRepository.find( + { + where: { + messages: { + id: IsNull(), + }, + }, + take: limit, + skip: offset, + }, + transactionManager, + ); + + return orphanThreads.map(({ id }) => id); + }, + async ( + ids: string[], + _workspaceId: string, + transactionManager?: WorkspaceEntityManager, + ) => { + await messageThreadRepository.delete(ids, transactionManager); + }, + transactionManager, + ); + }, + ); + }, + authContext, + { lite: true }, + ); } } diff --git a/packages/twenty-server/src/modules/messaging/message-folder-manager/services/sync-message-folders.service.ts b/packages/twenty-server/src/modules/messaging/message-folder-manager/services/sync-message-folders.service.ts index e72866b98c..c7753f45b9 100644 --- a/packages/twenty-server/src/modules/messaging/message-folder-manager/services/sync-message-folders.service.ts +++ b/packages/twenty-server/src/modules/messaging/message-folder-manager/services/sync-message-folders.service.ts @@ -188,6 +188,7 @@ export class SyncMessageFoldersService { return [...updatedExistingFolders, ...createdFolders]; }, authContext, + { lite: true }, ); } } diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/commands/messaging-trigger-message-list-fetch.command.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/commands/messaging-trigger-message-list-fetch.command.ts index 442df878e5..102c035319 100644 --- a/packages/twenty-server/src/modules/messaging/message-import-manager/commands/messaging-trigger-message-list-fetch.command.ts +++ b/packages/twenty-server/src/modules/messaging/message-import-manager/commands/messaging-trigger-message-list-fetch.command.ts @@ -53,54 +53,58 @@ export class MessagingTriggerMessageListFetchCommand extends CommandRunner { const authContext = buildSystemAuthContext(workspaceId); - await this.globalWorkspaceOrmManager.executeInWorkspaceContext(async () => { - const messageChannels = await this.messageChannelRepository.find({ - where: { - isSyncEnabled: true, - syncStage: MessageChannelSyncStage.MESSAGE_LIST_FETCH_PENDING, - ...(messageChannelId ? { id: messageChannelId } : {}), - workspaceId, - }, - }); - - if (messageChannels.length === 0) { - this.logger.warn( - 'No message channels found with MESSAGE_LIST_FETCH_PENDING status', - ); - - return; - } - - this.logger.log( - `Found ${messageChannels.length} message channel(s) to process`, - ); - - for (const messageChannel of messageChannels) { - await this.messageChannelRepository.update( - { id: messageChannel.id, workspaceId }, - { - syncStage: MessageChannelSyncStage.MESSAGE_LIST_FETCH_SCHEDULED, - syncStageStartedAt: new Date().toISOString(), - }, - ); - - await this.messageQueueService.add( - MessagingMessageListFetchJob.name, - { - messageChannelId: messageChannel.id, + await this.globalWorkspaceOrmManager.executeInWorkspaceContext( + async () => { + const messageChannels = await this.messageChannelRepository.find({ + where: { + isSyncEnabled: true, + syncStage: MessageChannelSyncStage.MESSAGE_LIST_FETCH_PENDING, + ...(messageChannelId ? { id: messageChannelId } : {}), workspaceId, }, - ); + }); + + if (messageChannels.length === 0) { + this.logger.warn( + 'No message channels found with MESSAGE_LIST_FETCH_PENDING status', + ); + + return; + } this.logger.log( - `Triggered fetch for message channel ${messageChannel.id}`, + `Found ${messageChannels.length} message channel(s) to process`, ); - } - this.logger.log( - `Successfully triggered ${messageChannels.length} message list fetch job(s)`, - ); - }, authContext); + for (const messageChannel of messageChannels) { + await this.messageChannelRepository.update( + { id: messageChannel.id, workspaceId }, + { + syncStage: MessageChannelSyncStage.MESSAGE_LIST_FETCH_SCHEDULED, + syncStageStartedAt: new Date().toISOString(), + }, + ); + + await this.messageQueueService.add( + MessagingMessageListFetchJob.name, + { + messageChannelId: messageChannel.id, + workspaceId, + }, + ); + + this.logger.log( + `Triggered fetch for message channel ${messageChannel.id}`, + ); + } + + this.logger.log( + `Successfully triggered ${messageChannels.length} message list fetch job(s)`, + ); + }, + authContext, + { lite: true }, + ); } @Option({ diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/inbound-email/services/inbound-email-import.service.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/inbound-email/services/inbound-email-import.service.ts index 0e5bc87b30..715cf8d0aa 100644 --- a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/inbound-email/services/inbound-email-import.service.ts +++ b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/inbound-email/services/inbound-email-import.service.ts @@ -94,14 +94,18 @@ export class InboundEmailImportService { ); } - await this.globalWorkspaceOrmManager.executeInWorkspaceContext(async () => { - await this.messagingSaveMessagesAndEnqueueContactCreationService.saveMessagesAndEnqueueContactCreation( - [parsedInboundMessage.message], - messageChannel, - connectedAccount, - workspaceId, - ); - }, buildSystemAuthContext(workspaceId)); + await this.globalWorkspaceOrmManager.executeInWorkspaceContext( + async () => { + await this.messagingSaveMessagesAndEnqueueContactCreationService.saveMessagesAndEnqueueContactCreation( + [parsedInboundMessage.message], + messageChannel, + connectedAccount, + workspaceId, + ); + }, + buildSystemAuthContext(workspaceId), + { lite: true }, + ); await this.inboundEmailStorageService.deleteRawMessage(s3Key); diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/jobs/messaging-message-list-fetch.job.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/jobs/messaging-message-list-fetch.job.ts index 0cc6f868d8..329df84ce1 100644 --- a/packages/twenty-server/src/modules/messaging/message-import-manager/jobs/messaging-message-list-fetch.job.ts +++ b/packages/twenty-server/src/modules/messaging/message-import-manager/jobs/messaging-message-list-fetch.job.ts @@ -48,59 +48,63 @@ export class MessagingMessageListFetchJob { const authContext = buildSystemAuthContext(workspaceId); - await this.globalWorkspaceOrmManager.executeInWorkspaceContext(async () => { - const messageChannel = await this.messageChannelRepository.findOne({ - where: { - id: messageChannelId, - workspaceId, - }, - relations: { connectedAccount: true, messageFolders: true }, - }); - - if (!messageChannel) { - await this.messagingMonitoringService.track({ - eventName: 'message_list_fetch_job.error.message_channel_not_found', - messageChannelId, - workspaceId, + await this.globalWorkspaceOrmManager.executeInWorkspaceContext( + async () => { + const messageChannel = await this.messageChannelRepository.findOne({ + where: { + id: messageChannelId, + workspaceId, + }, + relations: { connectedAccount: true, messageFolders: true }, }); - return; - } + if (!messageChannel) { + await this.messagingMonitoringService.track({ + eventName: 'message_list_fetch_job.error.message_channel_not_found', + messageChannelId, + workspaceId, + }); - if ( - messageChannel.syncStage !== - MessageChannelSyncStage.MESSAGE_LIST_FETCH_SCHEDULED - ) { - return; - } + return; + } - try { - await this.messagingMonitoringService.track({ - eventName: 'message_list_fetch.started', - workspaceId, - connectedAccountId: messageChannel.connectedAccount.id, - messageChannelId: messageChannel.id, - }); + if ( + messageChannel.syncStage !== + MessageChannelSyncStage.MESSAGE_LIST_FETCH_SCHEDULED + ) { + return; + } - await this.messagingMessageListFetchService.processMessageListFetch( - messageChannel, - workspaceId, - ); + try { + await this.messagingMonitoringService.track({ + eventName: 'message_list_fetch.started', + workspaceId, + connectedAccountId: messageChannel.connectedAccount.id, + messageChannelId: messageChannel.id, + }); - await this.messagingMonitoringService.track({ - eventName: 'message_list_fetch.completed', - workspaceId, - connectedAccountId: messageChannel.connectedAccount.id, - messageChannelId: messageChannel.id, - }); - } catch (error) { - await this.messageImportErrorHandlerService.handleDriverException( - error, - MessageImportSyncStep.MESSAGE_LIST_FETCH, - messageChannel, - workspaceId, - ); - } - }, authContext); + await this.messagingMessageListFetchService.processMessageListFetch( + messageChannel, + workspaceId, + ); + + await this.messagingMonitoringService.track({ + eventName: 'message_list_fetch.completed', + workspaceId, + connectedAccountId: messageChannel.connectedAccount.id, + messageChannelId: messageChannel.id, + }); + } catch (error) { + await this.messageImportErrorHandlerService.handleDriverException( + error, + MessageImportSyncStep.MESSAGE_LIST_FETCH, + messageChannel, + workspaceId, + ); + } + }, + authContext, + { lite: true }, + ); } } diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/jobs/messaging-messages-import.job.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/jobs/messaging-messages-import.job.ts index a8a30d91cc..8cdcfc4d7f 100644 --- a/packages/twenty-server/src/modules/messaging/message-import-manager/jobs/messaging-messages-import.job.ts +++ b/packages/twenty-server/src/modules/messaging/message-import-manager/jobs/messaging-messages-import.job.ts @@ -43,41 +43,45 @@ export class MessagingMessagesImportJob { const authContext = buildSystemAuthContext(workspaceId); - await this.globalWorkspaceOrmManager.executeInWorkspaceContext(async () => { - const messageChannel = await this.messageChannelRepository.findOne({ - where: { - id: messageChannelId, - workspaceId, - }, - relations: { connectedAccount: true, messageFolders: true }, - }); - - if (!messageChannel) { - await this.messagingMonitoringService.track({ - eventName: 'messages_import.error.message_channel_not_found', - messageChannelId, - workspaceId, + await this.globalWorkspaceOrmManager.executeInWorkspaceContext( + async () => { + const messageChannel = await this.messageChannelRepository.findOne({ + where: { + id: messageChannelId, + workspaceId, + }, + relations: { connectedAccount: true, messageFolders: true }, }); - return; - } + if (!messageChannel) { + await this.messagingMonitoringService.track({ + eventName: 'messages_import.error.message_channel_not_found', + messageChannelId, + workspaceId, + }); - if (!messageChannel?.isSyncEnabled) { - return; - } + return; + } - if ( - messageChannel.syncStage !== - MessageChannelSyncStage.MESSAGES_IMPORT_SCHEDULED - ) { - return; - } + if (!messageChannel?.isSyncEnabled) { + return; + } - await this.messagingMessagesImportService.processMessageBatchImport( - messageChannel, - messageChannel.connectedAccount, - workspaceId, - ); - }, authContext); + if ( + messageChannel.syncStage !== + MessageChannelSyncStage.MESSAGES_IMPORT_SCHEDULED + ) { + return; + } + + await this.messagingMessagesImportService.processMessageBatchImport( + messageChannel, + messageChannel.connectedAccount, + workspaceId, + ); + }, + authContext, + { lite: true }, + ); } } diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/jobs/messaging-ongoing-stale.job.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/jobs/messaging-ongoing-stale.job.ts index 25021a2d05..0a906bfe90 100644 --- a/packages/twenty-server/src/modules/messaging/message-import-manager/jobs/messaging-ongoing-stale.job.ts +++ b/packages/twenty-server/src/modules/messaging/message-import-manager/jobs/messaging-ongoing-stale.job.ts @@ -37,52 +37,58 @@ export class MessagingOngoingStaleJob { const authContext = buildSystemAuthContext(workspaceId); - await this.globalWorkspaceOrmManager.executeInWorkspaceContext(async () => { - const messageChannels = await this.messageChannelRepository.find({ - where: { - syncStage: In([ - MessageChannelSyncStage.MESSAGES_IMPORT_ONGOING, - MessageChannelSyncStage.MESSAGE_LIST_FETCH_ONGOING, - MessageChannelSyncStage.MESSAGES_IMPORT_SCHEDULED, - MessageChannelSyncStage.MESSAGE_LIST_FETCH_SCHEDULED, - ]), - workspaceId, - }, - }); - - for (const messageChannel of messageChannels) { - if (isSyncStale(toIsoStringOrNull(messageChannel.syncStageStartedAt))) { - await this.messageChannelSyncStatusService.resetSyncStageStartedAt( - [messageChannel.id], + await this.globalWorkspaceOrmManager.executeInWorkspaceContext( + async () => { + const messageChannels = await this.messageChannelRepository.find({ + where: { + syncStage: In([ + MessageChannelSyncStage.MESSAGES_IMPORT_ONGOING, + MessageChannelSyncStage.MESSAGE_LIST_FETCH_ONGOING, + MessageChannelSyncStage.MESSAGES_IMPORT_SCHEDULED, + MessageChannelSyncStage.MESSAGE_LIST_FETCH_SCHEDULED, + ]), workspaceId, - ); + }, + }); - switch (messageChannel.syncStage) { - case MessageChannelSyncStage.MESSAGE_LIST_FETCH_ONGOING: - case MessageChannelSyncStage.MESSAGE_LIST_FETCH_SCHEDULED: - this.logger.log( - `Sync for message channel ${messageChannel.id} and workspace ${workspaceId} is stale. Setting sync stage to MESSAGE_LIST_FETCH_PENDING`, - ); - await this.messageChannelSyncStatusService.markAsMessagesListFetchPending( - [messageChannel.id], - workspaceId, - ); - break; - case MessageChannelSyncStage.MESSAGES_IMPORT_ONGOING: - case MessageChannelSyncStage.MESSAGES_IMPORT_SCHEDULED: - this.logger.log( - `Sync for message channel ${messageChannel.id} and workspace ${workspaceId} is stale. Setting sync stage to MESSAGES_IMPORT_PENDING`, - ); - await this.messageChannelSyncStatusService.markAsMessagesImportPending( - [messageChannel.id], - workspaceId, - ); - break; - default: - break; + for (const messageChannel of messageChannels) { + if ( + isSyncStale(toIsoStringOrNull(messageChannel.syncStageStartedAt)) + ) { + await this.messageChannelSyncStatusService.resetSyncStageStartedAt( + [messageChannel.id], + workspaceId, + ); + + switch (messageChannel.syncStage) { + case MessageChannelSyncStage.MESSAGE_LIST_FETCH_ONGOING: + case MessageChannelSyncStage.MESSAGE_LIST_FETCH_SCHEDULED: + this.logger.log( + `Sync for message channel ${messageChannel.id} and workspace ${workspaceId} is stale. Setting sync stage to MESSAGE_LIST_FETCH_PENDING`, + ); + await this.messageChannelSyncStatusService.markAsMessagesListFetchPending( + [messageChannel.id], + workspaceId, + ); + break; + case MessageChannelSyncStage.MESSAGES_IMPORT_ONGOING: + case MessageChannelSyncStage.MESSAGES_IMPORT_SCHEDULED: + this.logger.log( + `Sync for message channel ${messageChannel.id} and workspace ${workspaceId} is stale. Setting sync stage to MESSAGES_IMPORT_PENDING`, + ); + await this.messageChannelSyncStatusService.markAsMessagesImportPending( + [messageChannel.id], + workspaceId, + ); + break; + default: + break; + } } } - } - }, authContext); + }, + authContext, + { lite: true }, + ); } } diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/jobs/messaging-relaunch-failed-message-channel.job.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/jobs/messaging-relaunch-failed-message-channel.job.ts index 437a0e374a..c23da7352d 100644 --- a/packages/twenty-server/src/modules/messaging/message-import-manager/jobs/messaging-relaunch-failed-message-channel.job.ts +++ b/packages/twenty-server/src/modules/messaging/message-import-manager/jobs/messaging-relaunch-failed-message-channel.job.ts @@ -36,32 +36,36 @@ export class MessagingRelaunchFailedMessageChannelJob { const authContext = buildSystemAuthContext(workspaceId); - await this.globalWorkspaceOrmManager.executeInWorkspaceContext(async () => { - const messageChannel = await this.messageChannelRepository.findOne({ - where: { - id: messageChannelId, - workspaceId, - }, - }); + await this.globalWorkspaceOrmManager.executeInWorkspaceContext( + async () => { + const messageChannel = await this.messageChannelRepository.findOne({ + where: { + id: messageChannelId, + workspaceId, + }, + }); - if ( - !messageChannel || - messageChannel.syncStage !== MessageChannelSyncStage.FAILED || - messageChannel.syncStatus !== MessageChannelSyncStatus.FAILED_UNKNOWN - ) { - return; - } + if ( + !messageChannel || + messageChannel.syncStage !== MessageChannelSyncStage.FAILED || + messageChannel.syncStatus !== MessageChannelSyncStatus.FAILED_UNKNOWN + ) { + return; + } - await this.messageChannelRepository.update( - { id: messageChannelId, workspaceId }, - { - syncStage: MessageChannelSyncStage.MESSAGE_LIST_FETCH_PENDING, - syncStatus: MessageChannelSyncStatus.ACTIVE, - throttleFailureCount: 0, - throttleRetryAfter: null, - syncStageStartedAt: null, - }, - ); - }, authContext); + await this.messageChannelRepository.update( + { id: messageChannelId, workspaceId }, + { + syncStage: MessageChannelSyncStage.MESSAGE_LIST_FETCH_PENDING, + syncStatus: MessageChannelSyncStatus.ACTIVE, + throttleFailureCount: 0, + throttleRetryAfter: null, + syncStageStartedAt: null, + }, + ); + }, + authContext, + { lite: true }, + ); } } diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/services/__tests__/messaging-messages-import.service.spec.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/services/__tests__/messaging-messages-import.service.spec.ts index 250661323a..1bd84533a3 100644 --- a/packages/twenty-server/src/modules/messaging/message-import-manager/services/__tests__/messaging-messages-import.service.spec.ts +++ b/packages/twenty-server/src/modules/messaging/message-import-manager/services/__tests__/messaging-messages-import.service.spec.ts @@ -269,14 +269,9 @@ describe('MessagingMessagesImportService', () => { connectedAccountRefreshTokensService.refreshAndSaveTokens, ).toHaveBeenCalledWith(mockConnectedAccount, workspaceId); - expect(emailAliasManagerService.refreshHandleAliases).toHaveBeenCalledWith( - { - ...mockConnectedAccount, - accessToken: 'new-access-token', - refreshToken: 'new-refresh-token', - }, - workspaceId, - ); + expect( + emailAliasManagerService.refreshHandleAliases, + ).not.toHaveBeenCalled(); expect(messagingGetMessagesService.getMessages).toHaveBeenCalledWith( ['message-id-1', 'message-id-2'], { diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-cursor.service.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-cursor.service.ts index bc60700ab6..3181e40fca 100644 --- a/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-cursor.service.ts +++ b/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-cursor.service.ts @@ -26,37 +26,41 @@ export class MessagingCursorService { ) { const authContext = buildSystemAuthContext(workspaceId); - await this.globalWorkspaceOrmManager.executeInWorkspaceContext(async () => { - if (!folderId) { - await this.messageChannelRepository.update( - { id: messageChannel.id, workspaceId }, - { - throttleFailureCount: 0, - throttleRetryAfter: null, - syncStageStartedAt: null, - syncCursor: - !messageChannel.syncCursor || - nextSyncCursor > messageChannel.syncCursor - ? nextSyncCursor - : messageChannel.syncCursor, - }, - ); - } else { - await this.messageFolderRepository.update( - { id: folderId, workspaceId }, - { - syncCursor: nextSyncCursor, - }, - ); - await this.messageChannelRepository.update( - { id: messageChannel.id, workspaceId }, - { - throttleFailureCount: 0, - throttleRetryAfter: null, - syncStageStartedAt: null, - }, - ); - } - }, authContext); + await this.globalWorkspaceOrmManager.executeInWorkspaceContext( + async () => { + if (!folderId) { + await this.messageChannelRepository.update( + { id: messageChannel.id, workspaceId }, + { + throttleFailureCount: 0, + throttleRetryAfter: null, + syncStageStartedAt: null, + syncCursor: + !messageChannel.syncCursor || + nextSyncCursor > messageChannel.syncCursor + ? nextSyncCursor + : messageChannel.syncCursor, + }, + ); + } else { + await this.messageFolderRepository.update( + { id: folderId, workspaceId }, + { + syncCursor: nextSyncCursor, + }, + ); + await this.messageChannelRepository.update( + { id: messageChannel.id, workspaceId }, + { + throttleFailureCount: 0, + throttleRetryAfter: null, + syncStageStartedAt: null, + }, + ); + } + }, + authContext, + { lite: true }, + ); } } diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-delete-folder-messages.service.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-delete-folder-messages.service.ts index 920eaec05a..dbd07051d1 100644 --- a/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-delete-folder-messages.service.ts +++ b/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-delete-folder-messages.service.ts @@ -37,77 +37,81 @@ export class MessagingDeleteFolderMessagesService { let totalDeletedCount = 0; - await this.globalWorkspaceOrmManager.executeInWorkspaceContext(async () => { - const messageFolderAssociationRepository = - await this.globalWorkspaceOrmManager.getRepository( - workspaceId, - 'messageChannelMessageAssociationMessageFolder', - ); - - const messageChannelMessageAssociationRepository = - await this.globalWorkspaceOrmManager.getRepository( - workspaceId, - 'messageChannelMessageAssociation', - ); - - let hasMoreData = true; - - while (hasMoreData) { - const folderAssociations = - await messageFolderAssociationRepository.find({ - where: { - messageFolderId: messageFolder.id, - }, - take: BATCH_SIZE, - }); - - if (folderAssociations.length === 0) { - hasMoreData = false; - continue; - } - - const folderAssociationIds = folderAssociations.map( - (folderAssociation) => folderAssociation.id, - ); - - const messageChannelMessageAssociationIds = folderAssociations.map( - (folderAssociation) => - folderAssociation.messageChannelMessageAssociationId, - ); - - const associations = - await messageChannelMessageAssociationRepository.find({ - where: { - id: In(messageChannelMessageAssociationIds), - messageChannelId: messageChannel.id, - }, - }); - - const messageExternalIds = associations - .map((association) => association.messageExternalId) - .filter(isDefined); - - this.logger.log( - `WorkspaceId: ${workspaceId}, MessageChannelId: ${messageChannel.id}, FolderId: ${messageFolder.id} - Deleting ${messageExternalIds.length} messages`, - ); - - if (messageExternalIds.length > 0) { - await this.messagingMessageCleanerService.deleteMessagesChannelMessageAssociationsAndRelatedOrphans( - { - workspaceId, - messageExternalIds, - messageChannelId: messageChannel.id, - }, + await this.globalWorkspaceOrmManager.executeInWorkspaceContext( + async () => { + const messageFolderAssociationRepository = + await this.globalWorkspaceOrmManager.getRepository( + workspaceId, + 'messageChannelMessageAssociationMessageFolder', ); - totalDeletedCount += messageExternalIds.length; - } + const messageChannelMessageAssociationRepository = + await this.globalWorkspaceOrmManager.getRepository( + workspaceId, + 'messageChannelMessageAssociation', + ); - await messageFolderAssociationRepository.delete({ - id: In(folderAssociationIds), - }); - } - }, authContext); + let hasMoreData = true; + + while (hasMoreData) { + const folderAssociations = + await messageFolderAssociationRepository.find({ + where: { + messageFolderId: messageFolder.id, + }, + take: BATCH_SIZE, + }); + + if (folderAssociations.length === 0) { + hasMoreData = false; + continue; + } + + const folderAssociationIds = folderAssociations.map( + (folderAssociation) => folderAssociation.id, + ); + + const messageChannelMessageAssociationIds = folderAssociations.map( + (folderAssociation) => + folderAssociation.messageChannelMessageAssociationId, + ); + + const associations = + await messageChannelMessageAssociationRepository.find({ + where: { + id: In(messageChannelMessageAssociationIds), + messageChannelId: messageChannel.id, + }, + }); + + const messageExternalIds = associations + .map((association) => association.messageExternalId) + .filter(isDefined); + + this.logger.log( + `WorkspaceId: ${workspaceId}, MessageChannelId: ${messageChannel.id}, FolderId: ${messageFolder.id} - Deleting ${messageExternalIds.length} messages`, + ); + + if (messageExternalIds.length > 0) { + await this.messagingMessageCleanerService.deleteMessagesChannelMessageAssociationsAndRelatedOrphans( + { + workspaceId, + messageExternalIds, + messageChannelId: messageChannel.id, + }, + ); + + totalDeletedCount += messageExternalIds.length; + } + + await messageFolderAssociationRepository.delete({ + id: In(folderAssociationIds), + }); + } + }, + authContext, + { lite: true }, + ); this.logger.log( `WorkspaceId: ${workspaceId}, MessageChannelId: ${messageChannel.id}, FolderId: ${messageFolder.id} - Completed deleting ${totalDeletedCount} messages from folder: ${messageFolder.name}`, diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-delete-group-email-messages.service.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-delete-group-email-messages.service.ts index ec3dcf0e04..6ad0dc35e9 100644 --- a/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-delete-group-email-messages.service.ts +++ b/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-delete-group-email-messages.service.ts @@ -146,6 +146,7 @@ export class MessagingDeleteGroupEmailMessagesService { return totalDeletedCount; }, authContext, + { lite: true }, ); } } diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-message-folder-association.service.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-message-folder-association.service.ts index 3950e9dbd3..a52f4dd4de 100644 --- a/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-message-folder-association.service.ts +++ b/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-message-folder-association.service.ts @@ -29,57 +29,61 @@ export class MessagingMessageFolderAssociationService { const authContext = buildSystemAuthContext(workspaceId); - await this.globalWorkspaceOrmManager.executeInWorkspaceContext(async () => { - const repository = - await this.globalWorkspaceOrmManager.getRepository( - workspaceId, - 'messageChannelMessageAssociationMessageFolder', + await this.globalWorkspaceOrmManager.executeInWorkspaceContext( + async () => { + const repository = + await this.globalWorkspaceOrmManager.getRepository( + workspaceId, + 'messageChannelMessageAssociationMessageFolder', + ); + + const records = associations.flatMap((association) => + association.messageFolderIds.map((folderId) => ({ + messageChannelMessageAssociationId: + association.messageChannelMessageAssociationId, + messageFolderId: folderId, + })), ); - const records = associations.flatMap((association) => - association.messageFolderIds.map((folderId) => ({ - messageChannelMessageAssociationId: - association.messageChannelMessageAssociationId, - messageFolderId: folderId, - })), - ); + if (records.length === 0) { + return; + } - if (records.length === 0) { - return; - } - - const associationIds = [ - ...new Set( - records.map((record) => record.messageChannelMessageAssociationId), - ), - ]; - - const existingRecords = await repository.find( - { - where: { - messageChannelMessageAssociationId: In(associationIds), - }, - }, - transactionManager, - ); - - const existingKeys = new Set( - existingRecords.map( - (record) => - `${record.messageChannelMessageAssociationId}:${record.messageFolderId}`, - ), - ); - - const recordsToInsert = records.filter( - (record) => - !existingKeys.has( - `${record.messageChannelMessageAssociationId}:${record.messageFolderId}`, + const associationIds = [ + ...new Set( + records.map((record) => record.messageChannelMessageAssociationId), ), - ); + ]; - if (recordsToInsert.length > 0) { - await repository.insert(recordsToInsert, transactionManager); - } - }, authContext); + const existingRecords = await repository.find( + { + where: { + messageChannelMessageAssociationId: In(associationIds), + }, + }, + transactionManager, + ); + + const existingKeys = new Set( + existingRecords.map( + (record) => + `${record.messageChannelMessageAssociationId}:${record.messageFolderId}`, + ), + ); + + const recordsToInsert = records.filter( + (record) => + !existingKeys.has( + `${record.messageChannelMessageAssociationId}:${record.messageFolderId}`, + ), + ); + + if (recordsToInsert.length > 0) { + await repository.insert(recordsToInsert, transactionManager); + } + }, + authContext, + { lite: true }, + ); } } diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-message-list-fetch.service.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-message-list-fetch.service.ts index 0ac53bf32f..187f254b7c 100644 --- a/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-message-list-fetch.service.ts +++ b/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-message-list-fetch.service.ts @@ -61,241 +61,246 @@ export class MessagingMessageListFetchService { ) { const authContext = buildSystemAuthContext(workspaceId); - await this.globalWorkspaceOrmManager.executeInWorkspaceContext(async () => { - try { - const pendingGroupEmailActionsProcessed = - await this.processPendingGroupEmailActions( - messageChannel, - workspaceId, - ); - - const pendingFolderActionsProcessed = - await this.processPendingFolderActions(messageChannel, workspaceId); - - await this.messageChannelSyncStatusService.markAsMessagesListFetchOngoing( - [messageChannel.id], - workspaceId, - ); - - this.logger.log( - `WorkspaceId: ${workspaceId}, MessageChannelId: ${messageChannel.id} - Processing message list fetch`, - ); - - const freshMessageChannel = - pendingGroupEmailActionsProcessed || pendingFolderActionsProcessed - ? await this.messageChannelRepository.findOne({ - where: { - id: messageChannel.id, - workspaceId, - }, - relations: { connectedAccount: true, messageFolders: true }, - }) - : messageChannel; - - if (!isDefined(freshMessageChannel)) { - this.logger.error( - `WorkspaceId: ${workspaceId}, MessageChannelId: ${messageChannel.id} - Message channel not found`, - ); - - return; - } - - const { accessToken, refreshToken } = - await this.messagingAccountAuthenticationService.validateAndRefreshConnectedAccountAuthentication( - { - connectedAccount: freshMessageChannel.connectedAccount, + await this.globalWorkspaceOrmManager.executeInWorkspaceContext( + async () => { + try { + const pendingGroupEmailActionsProcessed = + await this.processPendingGroupEmailActions( + messageChannel, workspaceId, - messageChannelId: freshMessageChannel.id, - }, - ); + ); - const messageChannelWithFreshTokens = { - ...freshMessageChannel, - connectedAccount: { - ...freshMessageChannel.connectedAccount, - accessToken, - refreshToken, - }, - }; + const pendingFolderActionsProcessed = + await this.processPendingFolderActions(messageChannel, workspaceId); - const messageFolders = - await this.syncMessageFoldersService.syncMessageFolders({ - messageChannel: messageChannelWithFreshTokens, + await this.messageChannelSyncStatusService.markAsMessagesListFetchOngoing( + [messageChannel.id], workspaceId, - }); - - const messageFoldersToSync = messageFolders.filter( - (folder) => - folder.pendingSyncAction === MessageFolderPendingSyncAction.NONE, - ); - - const messageLists = - await this.messagingGetMessageListService.getMessageLists( - messageChannelWithFreshTokens, - messageFoldersToSync, ); - await this.cacheStorage.del( - `messages-to-import:${workspaceId}:${freshMessageChannel.id}`, - ); - - const messageExternalIds = messageLists.flatMap( - (messageList) => messageList.messageExternalIds, - ); - - const messageExternalIdsToDelete = messageLists.flatMap( - (messageList) => messageList.messageExternalIdsToDelete, - ); - - const isFullSync = - messageLists.every( - (messageList) => !isNonEmptyString(messageList.previousSyncCursor), - ) && !isNonEmptyString(freshMessageChannel.syncCursor); - - let totalMessagesToImportCount = 0; - - this.logger.log( - `WorkspaceId: ${workspaceId}, MessageChannelId: ${freshMessageChannel.id} - Is full sync: ${isFullSync}, toImportCount: ${messageExternalIds.length}, toDeleteCount: ${messageExternalIdsToDelete.length}`, - ); - - const messageChannelMessageAssociationRepository = - await this.globalWorkspaceOrmManager.getRepository( - workspaceId, - 'messageChannelMessageAssociation', + this.logger.log( + `WorkspaceId: ${workspaceId}, MessageChannelId: ${messageChannel.id} - Processing message list fetch`, ); - const messageExternalIdsChunks = chunk(messageExternalIds, 200); + const freshMessageChannel = + pendingGroupEmailActionsProcessed || pendingFolderActionsProcessed + ? await this.messageChannelRepository.findOne({ + where: { + id: messageChannel.id, + workspaceId, + }, + relations: { connectedAccount: true, messageFolders: true }, + }) + : messageChannel; - for (const [ - index, - messageExternalIdsChunk, - ] of messageExternalIdsChunks.entries()) { - const existingMessageChannelMessageAssociations = - await messageChannelMessageAssociationRepository.find({ - where: { + if (!isDefined(freshMessageChannel)) { + this.logger.error( + `WorkspaceId: ${workspaceId}, MessageChannelId: ${messageChannel.id} - Message channel not found`, + ); + + return; + } + + const { accessToken, refreshToken } = + await this.messagingAccountAuthenticationService.validateAndRefreshConnectedAccountAuthentication( + { + connectedAccount: freshMessageChannel.connectedAccount, + workspaceId, messageChannelId: freshMessageChannel.id, - messageExternalId: In(messageExternalIdsChunk), }, + ); + + const messageChannelWithFreshTokens = { + ...freshMessageChannel, + connectedAccount: { + ...freshMessageChannel.connectedAccount, + accessToken, + refreshToken, + }, + }; + + const messageFolders = + await this.syncMessageFoldersService.syncMessageFolders({ + messageChannel: messageChannelWithFreshTokens, + workspaceId, }); - const existingMessageChannelMessageAssociationsExternalIds = - existingMessageChannelMessageAssociations.map( + const messageFoldersToSync = messageFolders.filter( + (folder) => + folder.pendingSyncAction === MessageFolderPendingSyncAction.NONE, + ); + + const messageLists = + await this.messagingGetMessageListService.getMessageLists( + messageChannelWithFreshTokens, + messageFoldersToSync, + ); + + await this.cacheStorage.del( + `messages-to-import:${workspaceId}:${freshMessageChannel.id}`, + ); + + const messageExternalIds = messageLists.flatMap( + (messageList) => messageList.messageExternalIds, + ); + + const messageExternalIdsToDelete = messageLists.flatMap( + (messageList) => messageList.messageExternalIdsToDelete, + ); + + const isFullSync = + messageLists.every( + (messageList) => + !isNonEmptyString(messageList.previousSyncCursor), + ) && !isNonEmptyString(freshMessageChannel.syncCursor); + + let totalMessagesToImportCount = 0; + + this.logger.log( + `WorkspaceId: ${workspaceId}, MessageChannelId: ${freshMessageChannel.id} - Is full sync: ${isFullSync}, toImportCount: ${messageExternalIds.length}, toDeleteCount: ${messageExternalIdsToDelete.length}`, + ); + + const messageChannelMessageAssociationRepository = + await this.globalWorkspaceOrmManager.getRepository( + workspaceId, + 'messageChannelMessageAssociation', + ); + + const messageExternalIdsChunks = chunk(messageExternalIds, 200); + + for (const [ + index, + messageExternalIdsChunk, + ] of messageExternalIdsChunks.entries()) { + const existingMessageChannelMessageAssociations = + await messageChannelMessageAssociationRepository.find({ + where: { + messageChannelId: freshMessageChannel.id, + messageExternalId: In(messageExternalIdsChunk), + }, + }); + + const existingMessageChannelMessageAssociationsExternalIds = + existingMessageChannelMessageAssociations.map( + (messageChannelMessageAssociation) => + messageChannelMessageAssociation.messageExternalId, + ); + + const messageExternalIdsToImport = messageExternalIdsChunk.filter( + (messageExternalId) => + !existingMessageChannelMessageAssociationsExternalIds.includes( + messageExternalId, + ), + ); + + if (messageExternalIdsToImport.length) { + this.logger.debug( + `messageChannelId: ${freshMessageChannel.id} Adding ${messageExternalIdsToImport.length} message external ids to import in batch ${index + 1}`, + ); + + totalMessagesToImportCount += messageExternalIdsToImport.length; + + await this.cacheStorage.setAdd( + `messages-to-import:${workspaceId}:${messageChannelWithFreshTokens.id}`, + messageExternalIdsToImport, + ONE_WEEK_IN_MILLISECONDS, + ); + } + } + + for (const messageList of messageLists) { + const { nextSyncCursor, folderId } = messageList; + + await this.messagingCursorService.updateCursor( + messageChannelWithFreshTokens, + nextSyncCursor, + workspaceId, + folderId, + ); + } + + const fullSyncMessageChannelMessageAssociationsToDelete = isFullSync + ? await this.computeFullSyncMessageChannelMessageAssociationsToDelete( + freshMessageChannel, + messageExternalIds, + workspaceId, + ) + : []; + + const allMessageExternalIdsToDelete = [ + ...messageExternalIdsToDelete, + ...fullSyncMessageChannelMessageAssociationsToDelete.map( (messageChannelMessageAssociation) => messageChannelMessageAssociation.messageExternalId, + ), + ]; + + if (allMessageExternalIdsToDelete.length) { + this.logger.log( + `WorkspaceId: ${workspaceId}, MessageChannelId: ${freshMessageChannel.id} - Deleting ${allMessageExternalIdsToDelete.length} message channel message associations`, ); - const messageExternalIdsToImport = messageExternalIdsChunk.filter( - (messageExternalId) => - !existingMessageChannelMessageAssociationsExternalIds.includes( - messageExternalId, - ), - ); + const toDeleteChunks = chunk(allMessageExternalIdsToDelete, 200); - if (messageExternalIdsToImport.length) { - this.logger.debug( - `messageChannelId: ${freshMessageChannel.id} Adding ${messageExternalIdsToImport.length} message external ids to import in batch ${index + 1}`, - ); + for (const [index, toDeleteChunk] of toDeleteChunks.entries()) { + this.logger.debug( + `messageChannelId: ${freshMessageChannel.id} Deleting ${toDeleteChunk.length} message channel message associations in batch ${index + 1}`, + ); - totalMessagesToImportCount += messageExternalIdsToImport.length; - - await this.cacheStorage.setAdd( - `messages-to-import:${workspaceId}:${messageChannelWithFreshTokens.id}`, - messageExternalIdsToImport, - ONE_WEEK_IN_MILLISECONDS, - ); + await this.messagingMessageCleanerService.deleteMessagesChannelMessageAssociationsAndRelatedOrphans( + { + workspaceId, + messageExternalIds: toDeleteChunk.filter( + (messageExternalId) => isNonEmptyString(messageExternalId), + ), + messageChannelId: messageChannelWithFreshTokens.id, + }, + ); + } } - } - for (const messageList of messageLists) { - const { nextSyncCursor, folderId } = messageList; - - await this.messagingCursorService.updateCursor( - messageChannelWithFreshTokens, - nextSyncCursor, - workspaceId, - folderId, - ); - } - - const fullSyncMessageChannelMessageAssociationsToDelete = isFullSync - ? await this.computeFullSyncMessageChannelMessageAssociationsToDelete( - freshMessageChannel, - messageExternalIds, - workspaceId, - ) - : []; - - const allMessageExternalIdsToDelete = [ - ...messageExternalIdsToDelete, - ...fullSyncMessageChannelMessageAssociationsToDelete.map( - (messageChannelMessageAssociation) => - messageChannelMessageAssociation.messageExternalId, - ), - ]; - - if (allMessageExternalIdsToDelete.length) { this.logger.log( - `WorkspaceId: ${workspaceId}, MessageChannelId: ${freshMessageChannel.id} - Deleting ${allMessageExternalIdsToDelete.length} message channel message associations`, + `WorkspaceId: ${workspaceId}, MessageChannelId: ${freshMessageChannel.id} - Total messages to import count: ${totalMessagesToImportCount}`, ); - const toDeleteChunks = chunk(allMessageExternalIdsToDelete, 200); - - for (const [index, toDeleteChunk] of toDeleteChunks.entries()) { - this.logger.debug( - `messageChannelId: ${freshMessageChannel.id} Deleting ${toDeleteChunk.length} message channel message associations in batch ${index + 1}`, + if (totalMessagesToImportCount === 0) { + await this.messageChannelSyncStatusService.markAsCompletedAndMarkAsMessagesListFetchPending( + [messageChannelWithFreshTokens.id], + workspaceId, ); - await this.messagingMessageCleanerService.deleteMessagesChannelMessageAssociationsAndRelatedOrphans( - { - workspaceId, - messageExternalIds: toDeleteChunk.filter((messageExternalId) => - isNonEmptyString(messageExternalId), - ), - messageChannelId: messageChannelWithFreshTokens.id, - }, - ); + return; } - } - this.logger.log( - `WorkspaceId: ${workspaceId}, MessageChannelId: ${freshMessageChannel.id} - Total messages to import count: ${totalMessagesToImportCount}`, - ); + this.logger.debug( + `messageChannelId: ${freshMessageChannel.id} Scheduling direct messages import`, + ); - if (totalMessagesToImportCount === 0) { - await this.messageChannelSyncStatusService.markAsCompletedAndMarkAsMessagesListFetchPending( + await this.messageChannelSyncStatusService.markAsMessagesImportScheduled( [messageChannelWithFreshTokens.id], workspaceId, ); - return; + await this.messagingMessagesImportService.processMessageBatchImport( + { + ...messageChannelWithFreshTokens, + syncStage: MessageChannelSyncStage.MESSAGES_IMPORT_SCHEDULED, + }, + messageChannelWithFreshTokens.connectedAccount, + workspaceId, + ); + } catch (error) { + await this.messageImportErrorHandlerService.handleDriverException( + error, + MessageImportSyncStep.MESSAGE_LIST_FETCH, + messageChannel, + workspaceId, + ); } - - this.logger.debug( - `messageChannelId: ${freshMessageChannel.id} Scheduling direct messages import`, - ); - - await this.messageChannelSyncStatusService.markAsMessagesImportScheduled( - [messageChannelWithFreshTokens.id], - workspaceId, - ); - - await this.messagingMessagesImportService.processMessageBatchImport( - { - ...messageChannelWithFreshTokens, - syncStage: MessageChannelSyncStage.MESSAGES_IMPORT_SCHEDULED, - }, - messageChannelWithFreshTokens.connectedAccount, - workspaceId, - ); - } catch (error) { - await this.messageImportErrorHandlerService.handleDriverException( - error, - MessageImportSyncStep.MESSAGE_LIST_FETCH, - messageChannel, - workspaceId, - ); - } - }, authContext); + }, + authContext, + { lite: true }, + ); } private async processPendingGroupEmailActions( diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-message.service.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-message.service.ts index 1b8951f3a5..e71ce58120 100644 --- a/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-message.service.ts +++ b/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-message.service.ts @@ -311,6 +311,7 @@ export class MessagingMessageService { }; }, authContext, + { lite: true }, ); } diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-messages-import.service.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-messages-import.service.ts index c26dce8f1f..61d3169dd5 100644 --- a/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-messages-import.service.ts +++ b/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-messages-import.service.ts @@ -74,58 +74,200 @@ export class MessagingMessagesImportService { const authContext = buildSystemAuthContext(workspaceId); - await this.globalWorkspaceOrmManager.executeInWorkspaceContext(async () => { - try { - if ( - messageChannel.syncStage !== - MessageChannelSyncStage.MESSAGES_IMPORT_SCHEDULED - ) { - return; - } + await this.globalWorkspaceOrmManager.executeInWorkspaceContext( + async () => { + try { + if ( + messageChannel.syncStage !== + MessageChannelSyncStage.MESSAGES_IMPORT_SCHEDULED + ) { + return; + } - await this.messagingMonitoringService.track({ - eventName: 'messages_import.started', - workspaceId, - connectedAccountId: messageChannel.connectedAccountId, - messageChannelId: messageChannel.id, - }); + await this.messagingMonitoringService.track({ + eventName: 'messages_import.started', + workspaceId, + connectedAccountId: messageChannel.connectedAccountId, + messageChannelId: messageChannel.id, + }); - await this.messageChannelSyncStatusService.markAsMessagesImportOngoing( - [messageChannel.id], - workspaceId, - ); - - const { accessToken, refreshToken } = - await this.messagingAccountAuthenticationService.validateAndRefreshConnectedAccountAuthentication( - { - connectedAccount, - workspaceId, - messageChannelId: messageChannel.id, - }, - ); - - const connectedAccountWithFreshTokens = { - ...connectedAccount, - accessToken, - refreshToken, - }; - - const refreshedHandleAliases = - await this.emailAliasManagerService.refreshHandleAliases( - connectedAccountWithFreshTokens, + await this.messageChannelSyncStatusService.markAsMessagesImportOngoing( + [messageChannel.id], workspaceId, ); - connectedAccountWithFreshTokens.handleAliases = refreshedHandleAliases; + const { accessToken, refreshToken } = + await this.messagingAccountAuthenticationService.validateAndRefreshConnectedAccountAuthentication( + { + connectedAccount, + workspaceId, + messageChannelId: messageChannel.id, + }, + ); - messageIdsToFetch = await this.cacheStorage.setPop( - `messages-to-import:${workspaceId}:${messageChannel.id}`, - messagesGetBatchSize, - ); + const connectedAccountWithFreshTokens = { + ...connectedAccount, + accessToken, + refreshToken, + }; - if (!messageIdsToFetch?.length) { - await this.messageChannelSyncStatusService.markAsCompletedAndMarkAsMessagesListFetchPending( - [messageChannel.id], + if (!isDefined(connectedAccountWithFreshTokens.handleAliases)) { + connectedAccountWithFreshTokens.handleAliases = + await this.emailAliasManagerService.refreshHandleAliases( + connectedAccountWithFreshTokens, + workspaceId, + ); + } + + messageIdsToFetch = await this.cacheStorage.setPop( + `messages-to-import:${workspaceId}:${messageChannel.id}`, + messagesGetBatchSize, + ); + + if (!messageIdsToFetch?.length) { + await this.messageChannelSyncStatusService.markAsCompletedAndMarkAsMessagesListFetchPending( + [messageChannel.id], + workspaceId, + ); + + return await this.trackMessageImportCompleted( + messageChannel, + workspaceId, + ); + } + + const allMessages = + await this.messagingGetMessagesService.getMessages( + messageIdsToFetch, + connectedAccountWithFreshTokens, + messageChannel, + ); + + // Map external folder IDs to internal folder IDs + const messageFolders = messageChannel.messageFolders ?? []; + const foldersWithExternalId = messageFolders.filter( + (folder): folder is typeof folder & { externalId: string } => + isDefined(folder.externalId), + ); + + const folderExternalToInternalMap = new Map( + foldersWithExternalId.map((folder) => [ + folder.externalId, + folder.id, + ]), + ); + + for (const message of allMessages) { + const externalFolderIds = message.messageFolderExternalIds ?? []; + + message.messageFolderIds = externalFolderIds + .map((externalId) => folderExternalToInternalMap.get(externalId)) + .filter(isDefined); + } + + const userWorkspace = await this.userWorkspaceRepository.findOne({ + where: { + id: connectedAccountWithFreshTokens.userWorkspaceId, + }, + }); + + const workspaceMemberRepository = + await this.globalWorkspaceOrmManager.getRepository( + workspaceId, + 'workspaceMember', + { shouldBypassPermissionChecks: true }, + ); + + const workspaceMember = userWorkspace + ? await workspaceMemberRepository.findOne({ + where: { userId: userWorkspace.userId }, + }) + : null; + + const blocklist = workspaceMember + ? await this.blocklistRepository.getByWorkspaceMemberId( + workspaceMember.id, + workspaceId, + ) + : []; + + if (!isDefined(messageChannel.handle)) { + throw new MessageImportDriverException( + 'Message channel handle is required', + MessageImportDriverExceptionCode.CHANNEL_MISCONFIGURED, + ); + } + + if (!isDefined(connectedAccountWithFreshTokens.handleAliases)) { + throw new MessageImportDriverException( + 'Message channel handle is required', + MessageImportDriverExceptionCode.CHANNEL_MISCONFIGURED, + ); + } + + const workspace = await this.workspaceRepository.findOne({ + where: { id: workspaceId }, + select: ['id', 'isInternalMessagesImportEnabled'], + }); + + const messagesToSave = filterEmails( + messageChannel.handle, + [...connectedAccountWithFreshTokens.handleAliases], + allMessages, + blocklist + .map((blocklistItem) => blocklistItem.handle) + .filter(isDefined), + messageChannel.excludeGroupEmails, + workspace?.isInternalMessagesImportEnabled ?? false, + ); + + if (messagesToSave.length > 0) { + await this.saveMessagesAndEnqueueContactCreationService.saveMessagesAndEnqueueContactCreation( + messagesToSave, + messageChannel, + connectedAccountWithFreshTokens, + workspaceId, + ); + } + + if (messageIdsToFetch.length < messagesGetBatchSize) { + await this.messageChannelSyncStatusService.markAsCompletedAndMarkAsMessagesListFetchPending( + [messageChannel.id], + workspaceId, + ); + } else { + await this.messageChannelSyncStatusService.markAsMessagesImportPending( + [messageChannel.id], + workspaceId, + ); + } + + await this.messageChannelRepository.update( + { id: messageChannel.id, workspaceId }, + { + throttleFailureCount: 0, + throttleRetryAfter: null, + syncStageStartedAt: null, + }, + ); + + return await this.trackMessageImportCompleted( + messageChannel, + workspaceId, + ); + } catch (error) { + this.logger.error( + `WorkspaceId: ${workspaceId}, MessageChannelId: ${messageChannel.id} - Error (${error.code}) importing messages: ${error.message}`, + ); + await this.cacheStorage.setAdd( + `messages-to-import:${workspaceId}:${messageChannel.id}`, + messageIdsToFetch, + ); + + await this.messageImportErrorHandlerService.handleDriverException( + error, + MessageImportSyncStep.MESSAGES_IMPORT_ONGOING, + messageChannel, workspaceId, ); @@ -134,144 +276,10 @@ export class MessagingMessagesImportService { workspaceId, ); } - - const allMessages = await this.messagingGetMessagesService.getMessages( - messageIdsToFetch, - connectedAccountWithFreshTokens, - messageChannel, - ); - - // Map external folder IDs to internal folder IDs - const messageFolders = messageChannel.messageFolders ?? []; - const foldersWithExternalId = messageFolders.filter( - (folder): folder is typeof folder & { externalId: string } => - isDefined(folder.externalId), - ); - - const folderExternalToInternalMap = new Map( - foldersWithExternalId.map((folder) => [folder.externalId, folder.id]), - ); - - for (const message of allMessages) { - const externalFolderIds = message.messageFolderExternalIds ?? []; - - message.messageFolderIds = externalFolderIds - .map((externalId) => folderExternalToInternalMap.get(externalId)) - .filter(isDefined); - } - - const userWorkspace = await this.userWorkspaceRepository.findOne({ - where: { - id: connectedAccountWithFreshTokens.userWorkspaceId, - }, - }); - - const workspaceMemberRepository = - await this.globalWorkspaceOrmManager.getRepository( - workspaceId, - 'workspaceMember', - { shouldBypassPermissionChecks: true }, - ); - - const workspaceMember = userWorkspace - ? await workspaceMemberRepository.findOne({ - where: { userId: userWorkspace.userId }, - }) - : null; - - const blocklist = workspaceMember - ? await this.blocklistRepository.getByWorkspaceMemberId( - workspaceMember.id, - workspaceId, - ) - : []; - - if (!isDefined(messageChannel.handle)) { - throw new MessageImportDriverException( - 'Message channel handle is required', - MessageImportDriverExceptionCode.CHANNEL_MISCONFIGURED, - ); - } - - if (!isDefined(connectedAccountWithFreshTokens.handleAliases)) { - throw new MessageImportDriverException( - 'Message channel handle is required', - MessageImportDriverExceptionCode.CHANNEL_MISCONFIGURED, - ); - } - - const workspace = await this.workspaceRepository.findOne({ - where: { id: workspaceId }, - select: ['id', 'isInternalMessagesImportEnabled'], - }); - - const messagesToSave = filterEmails( - messageChannel.handle, - [...connectedAccountWithFreshTokens.handleAliases], - allMessages, - blocklist - .map((blocklistItem) => blocklistItem.handle) - .filter(isDefined), - messageChannel.excludeGroupEmails, - workspace?.isInternalMessagesImportEnabled ?? false, - ); - - if (messagesToSave.length > 0) { - await this.saveMessagesAndEnqueueContactCreationService.saveMessagesAndEnqueueContactCreation( - messagesToSave, - messageChannel, - connectedAccountWithFreshTokens, - workspaceId, - ); - } - - if (messageIdsToFetch.length < messagesGetBatchSize) { - await this.messageChannelSyncStatusService.markAsCompletedAndMarkAsMessagesListFetchPending( - [messageChannel.id], - workspaceId, - ); - } else { - await this.messageChannelSyncStatusService.markAsMessagesImportPending( - [messageChannel.id], - workspaceId, - ); - } - - await this.messageChannelRepository.update( - { id: messageChannel.id, workspaceId }, - { - throttleFailureCount: 0, - throttleRetryAfter: null, - syncStageStartedAt: null, - }, - ); - - return await this.trackMessageImportCompleted( - messageChannel, - workspaceId, - ); - } catch (error) { - this.logger.error( - `WorkspaceId: ${workspaceId}, MessageChannelId: ${messageChannel.id} - Error (${error.code}) importing messages: ${error.message}`, - ); - await this.cacheStorage.setAdd( - `messages-to-import:${workspaceId}:${messageChannel.id}`, - messageIdsToFetch, - ); - - await this.messageImportErrorHandlerService.handleDriverException( - error, - MessageImportSyncStep.MESSAGES_IMPORT_ONGOING, - messageChannel, - workspaceId, - ); - - return await this.trackMessageImportCompleted( - messageChannel, - workspaceId, - ); - } - }, authContext); + }, + authContext, + { lite: true }, + ); } private async trackMessageImportCompleted( diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-process-folder-actions.service.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-process-folder-actions.service.ts index 3f5c7f512b..3ca45eb6d6 100644 --- a/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-process-folder-actions.service.ts +++ b/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-process-folder-actions.service.ts @@ -114,6 +114,7 @@ export class MessagingProcessFolderActionsService { } }, authContext, + { lite: true }, ); } } diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-process-group-email-actions.service.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-process-group-email-actions.service.ts index fa8b5ab557..1d84416e59 100644 --- a/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-process-group-email-actions.service.ts +++ b/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-process-group-email-actions.service.ts @@ -79,6 +79,7 @@ export class MessagingProcessGroupEmailActionsService { } }, authContext, + { lite: true }, ); await this.messageChannelRepository.update( diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-save-messages-and-enqueue-contact-creation.service.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-save-messages-and-enqueue-contact-creation.service.ts index 5fbf7e4e22..abbd649965 100644 --- a/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-save-messages-and-enqueue-contact-creation.service.ts +++ b/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-save-messages-and-enqueue-contact-creation.service.ts @@ -157,6 +157,7 @@ export class MessagingSaveMessagesAndEnqueueContactCreationService { ); }, authContext, + { lite: true }, ); if ( diff --git a/packages/twenty-server/src/modules/messaging/message-participant-manager/services/messaging-message-participant.service.ts b/packages/twenty-server/src/modules/messaging/message-participant-manager/services/messaging-message-participant.service.ts index 040faa7961..142ffcefc4 100644 --- a/packages/twenty-server/src/modules/messaging/message-participant-manager/services/messaging-message-participant.service.ts +++ b/packages/twenty-server/src/modules/messaging/message-participant-manager/services/messaging-message-participant.service.ts @@ -23,57 +23,61 @@ export class MessagingMessageParticipantService { ): Promise { const authContext = buildSystemAuthContext(workspaceId); - await this.globalWorkspaceOrmManager.executeInWorkspaceContext(async () => { - const messageParticipantRepository = - await this.globalWorkspaceOrmManager.getRepository( - workspaceId, - 'messageParticipant', + await this.globalWorkspaceOrmManager.executeInWorkspaceContext( + async () => { + const messageParticipantRepository = + await this.globalWorkspaceOrmManager.getRepository( + workspaceId, + 'messageParticipant', + ); + + const existingParticipantsBasedOnMessageIds = + await messageParticipantRepository.find({ + where: { + messageId: In( + participants.map((participant) => participant.messageId), + ), + }, + }); + + const participantsToCreate: Pick< + MessageParticipantWorkspaceEntity, + 'messageId' | 'handle' | 'displayName' | 'role' + >[] = participants + .filter( + (participant) => + !existingParticipantsBasedOnMessageIds.find( + (existingParticipant) => + existingParticipant.messageId === participant.messageId && + existingParticipant.handle === participant.handle && + existingParticipant.displayName === participant.displayName && + existingParticipant.role === participant.role, + ), + ) + .map((participant) => { + return { + messageId: participant.messageId, + handle: participant.handle, + displayName: participant.displayName, + role: participant.role, + }; + }); + + const createdParticipants = await messageParticipantRepository.insert( + participantsToCreate, + transactionManager, ); - const existingParticipantsBasedOnMessageIds = - await messageParticipantRepository.find({ - where: { - messageId: In( - participants.map((participant) => participant.messageId), - ), - }, + await this.matchParticipantService.matchParticipants({ + participants: createdParticipants.raw ?? [], + objectMetadataName: 'messageParticipant', + transactionManager, + matchWith: 'workspaceMemberAndPerson', + workspaceId, }); - - const participantsToCreate: Pick< - MessageParticipantWorkspaceEntity, - 'messageId' | 'handle' | 'displayName' | 'role' - >[] = participants - .filter( - (participant) => - !existingParticipantsBasedOnMessageIds.find( - (existingParticipant) => - existingParticipant.messageId === participant.messageId && - existingParticipant.handle === participant.handle && - existingParticipant.displayName === participant.displayName && - existingParticipant.role === participant.role, - ), - ) - .map((participant) => { - return { - messageId: participant.messageId, - handle: participant.handle, - displayName: participant.displayName, - role: participant.role, - }; - }); - - const createdParticipants = await messageParticipantRepository.insert( - participantsToCreate, - transactionManager, - ); - - await this.matchParticipantService.matchParticipants({ - participants: createdParticipants.raw ?? [], - objectMetadataName: 'messageParticipant', - transactionManager, - matchWith: 'workspaceMemberAndPerson', - workspaceId, - }); - }, authContext); + }, + authContext, + { lite: true }, + ); } } diff --git a/packages/twenty-server/src/modules/messaging/monitoring/crons/jobs/messaging-message-channel-sync-status-monitoring.cron.job.ts b/packages/twenty-server/src/modules/messaging/monitoring/crons/jobs/messaging-message-channel-sync-status-monitoring.cron.job.ts index 9eff83845d..23bbc3c017 100644 --- a/packages/twenty-server/src/modules/messaging/monitoring/crons/jobs/messaging-message-channel-sync-status-monitoring.cron.job.ts +++ b/packages/twenty-server/src/modules/messaging/monitoring/crons/jobs/messaging-message-channel-sync-status-monitoring.cron.job.ts @@ -74,6 +74,7 @@ export class MessagingMessageChannelSyncStatusMonitoringCronJob { } }, authContext, + { lite: true }, ); } catch (error) { this.exceptionHandlerService.captureExceptions([error], {