From f9677122b68e0f6fd6d278c4b7cd8e514aa64292 Mon Sep 17 00:00:00 2001 From: Charles Bochet Date: Sun, 7 Sep 2025 20:36:28 +0200 Subject: [PATCH] Improve Messaging Gmail experience (#14342) In this PR, I'm solving several issues: 1) We were not checking if the currentWorkspaceMember was owning the message in the thread. It kind of worked before because the case of shared threads (with shared threads visibility restriction) was not happening that often. It seems that the bug has always been there 2) Re-implement orphan messages and threads deletion on messageChannel deletion. We used to brutally look for all orphans, we disabled it last week because it was too heavy on db. I've re-implemented it more carefully and "surgically" 3) Gmail sync was not handling folder synced correctly. It was leveraging labelIds which it shouldn't do (this is a AND AND parameter) in full sync 4) Added a command to clean orphan message threads manually if needed. Usually this is done when you remove a messageChannel, or change blocklist rules but it can be useful to have it to debug --- .../src/command/command.module.ts | 2 + .../services/timeline-messaging.service.ts | 78 ++++--------- ...ging-blocklist-item-delete-messages.job.ts | 2 +- ...message-clearner-remove-orphans.command.ts | 39 +++++++ ...-connected-account-deletion-cleanup.job.ts | 4 +- .../messaging-message-cleaner.module.ts | 6 +- .../messaging-message-cleaner.service.ts | 107 ++++++++++++++++-- .../gmail/gmail-get-all-folders.service.ts | 37 +----- .../services/sync-message-folders.service.ts | 3 +- ...ssaging-gmail-default-not-synced-labels.ts | 10 ++ .../messaging-gmail-excluded-categories.ts | 8 -- .../gmail-get-message-list.service.ts | 67 ++++------- .../gmail-default-message-category.type.ts | 6 - ...mail-category-excude-search-filter.spec.ts | 27 ----- .../compute-gmail-category-label-id.spec.ts | 9 -- ...ompute-gmail-exclude-search-filter.spec.ts | 69 +++++++++++ ...mail-category-excude-search-filter.util.ts | 3 - .../compute-gmail-category-label-id.util.ts | 2 - ...ompute-gmail-exclude-search-filter.util.ts | 12 ++ .../map-gmail-default-folder-to-category.ts | 37 ------ .../services/imap-get-message-list.service.ts | 7 +- .../microsoft-get-message-list.service.ts | 6 +- .../messaging-message-list-fetch.service.ts | 37 +++--- 23 files changed, 317 insertions(+), 261 deletions(-) create mode 100644 packages/twenty-server/src/modules/messaging/message-cleaner/commands/messaging-message-clearner-remove-orphans.command.ts create mode 100644 packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/constants/messaging-gmail-default-not-synced-labels.ts delete mode 100644 packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/constants/messaging-gmail-excluded-categories.ts delete mode 100644 packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/types/gmail-default-message-category.type.ts delete mode 100644 packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/utils/__tests__/compute-gmail-category-excude-search-filter.spec.ts delete mode 100644 packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/utils/__tests__/compute-gmail-category-label-id.spec.ts create mode 100644 packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/utils/__tests__/compute-gmail-exclude-search-filter.spec.ts delete mode 100644 packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/utils/compute-gmail-category-excude-search-filter.util.ts delete mode 100644 packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/utils/compute-gmail-category-label-id.util.ts create mode 100644 packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/utils/compute-gmail-exclude-search-filter.util.ts delete mode 100644 packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/utils/map-gmail-default-folder-to-category.ts diff --git a/packages/twenty-server/src/command/command.module.ts b/packages/twenty-server/src/command/command.module.ts index 2c99a18f47..6cd6fe2cec 100644 --- a/packages/twenty-server/src/command/command.module.ts +++ b/packages/twenty-server/src/command/command.module.ts @@ -8,6 +8,7 @@ import { WorkspaceCleanerModule } from 'src/engine/workspace-manager/workspace-c import { WorkspaceHealthCommandModule } from 'src/engine/workspace-manager/workspace-health/commands/workspace-health-command.module'; import { WorkspaceMigrationRunnerCommandsModule } from 'src/engine/workspace-manager/workspace-migration-runner/commands/workspace-migration-runner-commands.module'; import { WorkspaceSyncMetadataCommandsModule } from 'src/engine/workspace-manager/workspace-sync-metadata/commands/workspace-sync-metadata-commands.module'; +import { MessagingMessageCleanerModule } from 'src/modules/messaging/message-cleaner/messaging-message-cleaner.module'; @Module({ imports: [ @@ -16,6 +17,7 @@ import { WorkspaceSyncMetadataCommandsModule } from 'src/engine/workspace-manage DatabaseCommandModule, WorkspaceCleanerModule, WorkspaceHealthCommandModule, + MessagingMessageCleanerModule, WorkspaceMigrationRunnerCommandsModule, ObjectMetadataModule, FieldMetadataModule, diff --git a/packages/twenty-server/src/engine/core-modules/messaging/services/timeline-messaging.service.ts b/packages/twenty-server/src/engine/core-modules/messaging/services/timeline-messaging.service.ts index c739385622..4e68d3c1cc 100644 --- a/packages/twenty-server/src/engine/core-modules/messaging/services/timeline-messaging.service.ts +++ b/packages/twenty-server/src/engine/core-modules/messaging/services/timeline-messaging.service.ts @@ -1,6 +1,6 @@ import { Injectable } from '@nestjs/common'; -import { In, Not } from 'typeorm'; +import { In } from 'typeorm'; import { type TimelineThread } from 'src/engine/core-modules/messaging/dtos/timeline-thread.dto'; import { TwentyORMManager } from 'src/engine/twenty-orm/twenty-orm.manager'; @@ -94,6 +94,7 @@ export class TimelineMessagingService { await this.twentyORMManager.getRepository( 'messageParticipant', ); + const threadParticipants = await messageParticipantRepository .createQueryBuilder() .select('messageParticipant') @@ -183,32 +184,11 @@ export class TimelineMessagingService { 'messageThread', ); - const threadsWithoutWorkspaceMember = await messageThreadRepository.find({ - select: { - id: true, - }, - where: { - id: In(messageThreadIds), - messages: { - messageChannelMessageAssociations: { - messageChannel: { - connectedAccount: { - accountOwnerId: Not(workspaceMemberId), - }, - }, - }, - }, - }, - }); - - const threadIdsWithoutWorkspaceMember = threadsWithoutWorkspaceMember.map( - (thread) => thread.id, - ); - const threadVisibility = await messageThreadRepository .createQueryBuilder() .select('messageThread.id', 'id') .addSelect('messageChannel.visibility', 'visibility') + .addSelect('connectedAccount.accountOwnerId', 'accountOwnerId') .leftJoin('messageThread.messages', 'message') .leftJoin( 'message.messageChannelMessageAssociations', @@ -218,46 +198,34 @@ export class TimelineMessagingService { 'messageChannelMessageAssociation.messageChannel', 'messageChannel', ) + .leftJoin('messageChannel.connectedAccount', 'connectedAccount') .where('messageThread.id = ANY(:messageThreadIds)', { - messageThreadIds: threadIdsWithoutWorkspaceMember, + messageThreadIds: messageThreadIds, }) .getRawMany(); const visibilityValues = Object.values(MessageChannelVisibility); - const threadVisibilityByThreadIdForWhichWorkspaceMemberIsNotOwner: - | { - [key: string]: MessageChannelVisibility; - } - | undefined = threadVisibility?.reduce( - (threadVisibilityAcc, threadVisibility) => { - threadVisibilityAcc[threadVisibility.id] = - visibilityValues[ - Math.max( - visibilityValues.indexOf(threadVisibility.visibility), - visibilityValues.indexOf( - threadVisibilityAcc[threadVisibility.id] ?? - MessageChannelVisibility.METADATA, - ), - ) - ]; - - return threadVisibilityAcc; - }, - {}, - ); - const threadVisibilityByThreadId: { [key: string]: MessageChannelVisibility; - } = messageThreadIds.reduce((threadVisibilityAcc, messageThreadId) => { - // If the workspace member is not the owner of the thread, use the visibility value from the query - // @ts-expect-error legacy noImplicitAny - threadVisibilityAcc[messageThreadId] = - threadIdsWithoutWorkspaceMember.includes(messageThreadId) - ? (threadVisibilityByThreadIdForWhichWorkspaceMemberIsNotOwner?.[ - messageThreadId - ] ?? MessageChannelVisibility.METADATA) - : MessageChannelVisibility.SHARE_EVERYTHING; + } = threadVisibility.reduce((threadVisibilityAcc, threadVisibility) => { + if (threadVisibility.accountOwnerId === workspaceMemberId) { + threadVisibilityAcc[threadVisibility.id] = + MessageChannelVisibility.SHARE_EVERYTHING; + + return threadVisibilityAcc; + } + + threadVisibilityAcc[threadVisibility.id] = + visibilityValues[ + Math.max( + visibilityValues.indexOf(threadVisibility.visibility), + visibilityValues.indexOf( + threadVisibilityAcc[threadVisibility.id] ?? + MessageChannelVisibility.METADATA, + ), + ) + ]; return threadVisibilityAcc; }, {}); diff --git a/packages/twenty-server/src/modules/messaging/blocklist-manager/jobs/messaging-blocklist-item-delete-messages.job.ts b/packages/twenty-server/src/modules/messaging/blocklist-manager/jobs/messaging-blocklist-item-delete-messages.job.ts index 2ce8f97e76..c4cb444486 100644 --- a/packages/twenty-server/src/modules/messaging/blocklist-manager/jobs/messaging-blocklist-item-delete-messages.job.ts +++ b/packages/twenty-server/src/modules/messaging/blocklist-manager/jobs/messaging-blocklist-item-delete-messages.job.ts @@ -140,6 +140,6 @@ export class BlocklistItemDeleteMessagesJob { } } - await this.threadCleanerService.cleanWorkspaceThreads(workspaceId); + await this.threadCleanerService.cleanOrphanMessagesAndThreads(workspaceId); } } diff --git a/packages/twenty-server/src/modules/messaging/message-cleaner/commands/messaging-message-clearner-remove-orphans.command.ts b/packages/twenty-server/src/modules/messaging/message-cleaner/commands/messaging-message-clearner-remove-orphans.command.ts new file mode 100644 index 0000000000..c1a73cdc0e --- /dev/null +++ b/packages/twenty-server/src/modules/messaging/message-cleaner/commands/messaging-message-clearner-remove-orphans.command.ts @@ -0,0 +1,39 @@ +import { InjectRepository } from '@nestjs/typeorm'; + +import { Command } from 'nest-commander'; +import { Repository } from 'typeorm'; + +import { + ActiveOrSuspendedWorkspacesMigrationCommandRunner, + type RunOnWorkspaceArgs, +} from 'src/database/commands/command-runners/active-or-suspended-workspaces-migration.command-runner'; +import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity'; +import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager'; +import { MessagingMessageCleanerService } from 'src/modules/messaging/message-cleaner/services/messaging-message-cleaner.service'; + +@Command({ + name: 'messaging:message-cleaner-remove-orphans', + description: 'Remove orphan message and threads from messaging', +}) +export class MessagingMessageCleanerRemoveOrphansCommand extends ActiveOrSuspendedWorkspacesMigrationCommandRunner { + constructor( + @InjectRepository(Workspace) + protected readonly workspaceRepository: Repository, + protected readonly twentyORMGlobalManager: TwentyORMGlobalManager, + private readonly messagingMessageCleanerService: MessagingMessageCleanerService, + ) { + super(workspaceRepository, twentyORMGlobalManager); + } + + override async runOnWorkspace({ + workspaceId, + }: RunOnWorkspaceArgs): Promise { + try { + await this.messagingMessageCleanerService.cleanOrphanMessagesAndThreads( + workspaceId, + ); + } catch (error) { + this.logger.error('Error while deleting workflowRun', error); + } + } +} diff --git a/packages/twenty-server/src/modules/messaging/message-cleaner/jobs/messaging-connected-account-deletion-cleanup.job.ts b/packages/twenty-server/src/modules/messaging/message-cleaner/jobs/messaging-connected-account-deletion-cleanup.job.ts index f564256b54..6291f49ee8 100644 --- a/packages/twenty-server/src/modules/messaging/message-cleaner/jobs/messaging-connected-account-deletion-cleanup.job.ts +++ b/packages/twenty-server/src/modules/messaging/message-cleaner/jobs/messaging-connected-account-deletion-cleanup.job.ts @@ -23,6 +23,8 @@ export class MessagingConnectedAccountDeletionCleanupJob { async handle( data: MessagingConnectedAccountDeletionCleanupJobData, ): Promise { - await this.messageCleanerService.cleanWorkspaceThreads(data.workspaceId); + await this.messageCleanerService.cleanOrphanMessagesAndThreads( + data.workspaceId, + ); } } diff --git a/packages/twenty-server/src/modules/messaging/message-cleaner/messaging-message-cleaner.module.ts b/packages/twenty-server/src/modules/messaging/message-cleaner/messaging-message-cleaner.module.ts index 7b25fa456a..fe3254da99 100644 --- a/packages/twenty-server/src/modules/messaging/message-cleaner/messaging-message-cleaner.module.ts +++ b/packages/twenty-server/src/modules/messaging/message-cleaner/messaging-message-cleaner.module.ts @@ -1,15 +1,19 @@ import { Module } from '@nestjs/common'; +import { TypeOrmModule } from '@nestjs/typeorm'; +import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity'; +import { MessagingMessageCleanerRemoveOrphansCommand } from 'src/modules/messaging/message-cleaner/commands/messaging-message-clearner-remove-orphans.command'; import { MessagingConnectedAccountDeletionCleanupJob } from 'src/modules/messaging/message-cleaner/jobs/messaging-connected-account-deletion-cleanup.job'; import { MessagingMessageCleanerConnectedAccountListener } from 'src/modules/messaging/message-cleaner/listeners/messaging-message-cleaner-connected-account.listener'; import { MessagingMessageCleanerService } from 'src/modules/messaging/message-cleaner/services/messaging-message-cleaner.service'; @Module({ - imports: [], + imports: [TypeOrmModule.forFeature([Workspace])], providers: [ MessagingMessageCleanerService, MessagingConnectedAccountDeletionCleanupJob, MessagingMessageCleanerConnectedAccountListener, + MessagingMessageCleanerRemoveOrphansCommand, ], exports: [MessagingMessageCleanerService], }) diff --git a/packages/twenty-server/src/modules/messaging/message-cleaner/services/messaging-message-cleaner.service.ts b/packages/twenty-server/src/modules/messaging/message-cleaner/services/messaging-message-cleaner.service.ts index 1c7f0d9f98..636c71723e 100644 --- a/packages/twenty-server/src/modules/messaging/message-cleaner/services/messaging-message-cleaner.service.ts +++ b/packages/twenty-server/src/modules/messaging/message-cleaner/services/messaging-message-cleaner.service.ts @@ -1,9 +1,11 @@ import { Injectable, Logger } from '@nestjs/common'; -import { IsNull } from 'typeorm'; +import chunk from 'lodash.chunk'; +import { In, IsNull } from 'typeorm'; import { type WorkspaceEntityManager } from 'src/engine/twenty-orm/entity-manager/workspace-entity-manager'; -import { TwentyORMManager } from 'src/engine/twenty-orm/twenty-orm.manager'; +import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager'; +import { MessageChannelMessageAssociationWorkspaceEntity } from 'src/modules/messaging/common/standard-objects/message-channel-message-association.workspace-entity'; import { type MessageThreadWorkspaceEntity } from 'src/modules/messaging/common/standard-objects/message-thread.workspace-entity'; import { type MessageWorkspaceEntity } from 'src/modules/messaging/common/standard-objects/message.workspace-entity'; import { deleteUsingPagination } from 'src/modules/messaging/message-cleaner/utils/delete-using-pagination.util'; @@ -11,20 +13,111 @@ import { deleteUsingPagination } from 'src/modules/messaging/message-cleaner/uti @Injectable() export class MessagingMessageCleanerService { private readonly logger = new Logger(MessagingMessageCleanerService.name); - constructor(private readonly twentyORMManager: TwentyORMManager) {} + constructor( + private readonly twentyORMGlobalManager: TwentyORMGlobalManager, + ) {} + + async deleteMessagesChannelMessageAssociationsAndRelatedOrphans({ + workspaceId, + messageExternalIds, + messageChannelId, + }: { + workspaceId: string; + messageExternalIds: string[]; + messageChannelId: string; + }) { + const messageRepository = + await this.twentyORMGlobalManager.getRepositoryForWorkspace( + workspaceId, + 'message', + ); + + const messageChannelMessageAssociationRepository = + await this.twentyORMGlobalManager.getRepositoryForWorkspace( + workspaceId, + 'messageChannelMessageAssociation', + ); - public async cleanWorkspaceThreads(workspaceId: string) { const messageThreadRepository = - await this.twentyORMManager.getRepository( + await this.twentyORMGlobalManager.getRepositoryForWorkspace( + workspaceId, + 'messageThread', + ); + + const messageExternalIdsChunks = chunk(messageExternalIds, 500); + + for (const messageExternalIdsChunk of messageExternalIdsChunks) { + const messageChannelMessageAssociationsToDelete = + await messageChannelMessageAssociationRepository.find({ + where: { + messageExternalId: In(messageExternalIdsChunk), + messageChannelId, + }, + }); + + await messageChannelMessageAssociationRepository.delete( + messageChannelMessageAssociationsToDelete.map(({ id }) => id), + ); + + this.logger.log( + `WorkspaceId: ${workspaceId} Deleting ${messageChannelMessageAssociationsToDelete.length} message channel message associations`, + ); + + const orphanMessages = await messageRepository.find({ + where: { + id: In( + messageChannelMessageAssociationsToDelete.map( + ({ messageId }) => messageId, + ), + ), + messageChannelMessageAssociations: { + id: IsNull(), + }, + }, + }); + + this.logger.log( + `WorkspaceId: ${workspaceId} Deleting ${orphanMessages.length} orphan messages`, + ); + + await messageRepository.delete(orphanMessages.map(({ id }) => id)); + + const orphanMessageThreads = await messageThreadRepository.find({ + where: { + id: In(orphanMessages.map(({ messageThreadId }) => messageThreadId)), + messages: { + id: IsNull(), + }, + }, + }); + + this.logger.log( + `WorkspaceId: ${workspaceId} Deleting ${orphanMessageThreads.length} orphan message threads`, + ); + + await messageThreadRepository.delete( + orphanMessageThreads.map(({ id }) => id), + ); + } + } + + public async cleanOrphanMessagesAndThreads(workspaceId: string) { + const messageThreadRepository = + await this.twentyORMGlobalManager.getRepositoryForWorkspace( + workspaceId, 'messageThread', ); const messageRepository = - await this.twentyORMManager.getRepository( + await this.twentyORMGlobalManager.getRepositoryForWorkspace( + workspaceId, 'message', ); - const workspaceDataSource = await this.twentyORMManager.getDatasource(); + const workspaceDataSource = + await this.twentyORMGlobalManager.getDataSourceForWorkspace({ + workspaceId, + }); await workspaceDataSource.transaction( async (transactionManager: WorkspaceEntityManager) => { diff --git a/packages/twenty-server/src/modules/messaging/message-folder-manager/drivers/gmail/gmail-get-all-folders.service.ts b/packages/twenty-server/src/modules/messaging/message-folder-manager/drivers/gmail/gmail-get-all-folders.service.ts index 6a6376c47e..3b3d8f4875 100644 --- a/packages/twenty-server/src/modules/messaging/message-folder-manager/drivers/gmail/gmail-get-all-folders.service.ts +++ b/packages/twenty-server/src/modules/messaging/message-folder-manager/drivers/gmail/gmail-get-all-folders.service.ts @@ -1,18 +1,14 @@ import { Injectable, Logger } from '@nestjs/common'; -import { gmail_v1 } from 'googleapis'; -import { isDefined } from 'twenty-shared/utils'; - import { MessageFolder, MessageFolderDriver, } from 'src/modules/messaging/message-folder-manager/interfaces/message-folder-driver.interface'; import { type ConnectedAccountWorkspaceEntity } from 'src/modules/connected-account/standard-objects/connected-account.workspace-entity'; -import { MESSAGING_GMAIL_EXCLUDED_CATEGORIES } from 'src/modules/messaging/message-import-manager/drivers/gmail/constants/messaging-gmail-excluded-categories'; +import { MESSAGING_GMAIL_DEFAULT_NOT_SYNCED_LABELS } from 'src/modules/messaging/message-import-manager/drivers/gmail/constants/messaging-gmail-default-not-synced-labels'; import { GmailClientProvider } from 'src/modules/messaging/message-import-manager/drivers/gmail/providers/gmail-client.provider'; import { GmailHandleErrorService } from 'src/modules/messaging/message-import-manager/drivers/gmail/services/gmail-handle-error.service'; -import { computeGmailCategoryLabelId } from 'src/modules/messaging/message-import-manager/drivers/gmail/utils/compute-gmail-category-label-id.util'; @Injectable() export class GmailGetAllFoldersService implements MessageFolderDriver { @@ -23,24 +19,8 @@ export class GmailGetAllFoldersService implements MessageFolderDriver { private readonly gmailHandleErrorService: GmailHandleErrorService, ) {} - private isExcludedCategoryFolder(labelId: string): boolean { - const excludedCategoryIds = MESSAGING_GMAIL_EXCLUDED_CATEGORIES.map( - (category) => computeGmailCategoryLabelId(category), - ); - - return excludedCategoryIds.includes(labelId); - } - - private isIncludedFolder(label: gmail_v1.Schema$Label): boolean { - if (!isDefined(label.id)) { - return false; - } - - const isTargetSystemFolder = - label.type === 'system' && (label.id === 'INBOX' || label.id === 'SENT'); - const isUserFolder = label.type === 'user'; - - return isTargetSystemFolder || isUserFolder; + private isSyncedByDefault(labelId: string): boolean { + return !MESSAGING_GMAIL_DEFAULT_NOT_SYNCED_LABELS.includes(labelId); } async getAllMessageFolders( @@ -74,21 +54,12 @@ export class GmailGetAllFoldersService implements MessageFolderDriver { continue; } - if (this.isExcludedCategoryFolder(label.id)) { - continue; - } - - if (!this.isIncludedFolder(label)) { - continue; - } - const isSentFolder = label.id === 'SENT'; - const isSyncedByDefault = label.id === 'INBOX' || label.id === 'SENT'; folders.push({ externalId: label.id, name: label.name, - isSynced: isSyncedByDefault, + isSynced: this.isSyncedByDefault(label.id), isSentFolder, }); } diff --git a/packages/twenty-server/src/modules/messaging/message-folder-manager/services/sync-message-folders.service.ts b/packages/twenty-server/src/modules/messaging/message-folder-manager/services/sync-message-folders.service.ts index f02fb279b1..24f50a100e 100644 --- a/packages/twenty-server/src/modules/messaging/message-folder-manager/services/sync-message-folders.service.ts +++ b/packages/twenty-server/src/modules/messaging/message-folder-manager/services/sync-message-folders.service.ts @@ -69,6 +69,7 @@ export class SyncMessageFoldersService { messageFolderRepository, }); + // TODO: we should delete folders that are not in the list anymore for (const folder of folders) { const existingFolder = this.findExistingFolderInMap( existingFolderMap, @@ -80,8 +81,6 @@ export class SyncMessageFoldersService { existingFolder.id, { name: folder.name, - isSynced: folder.isSynced, - isSentFolder: folder.isSentFolder, externalId: folder.externalId, }, manager, diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/constants/messaging-gmail-default-not-synced-labels.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/constants/messaging-gmail-default-not-synced-labels.ts new file mode 100644 index 0000000000..5e88beafcb --- /dev/null +++ b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/constants/messaging-gmail-default-not-synced-labels.ts @@ -0,0 +1,10 @@ +export const MESSAGING_GMAIL_DEFAULT_NOT_SYNCED_LABELS = [ + 'CATEGORY_PROMOTIONS', + 'CATEGORY_SOCIAL', + 'CATEGORY_FORUMS', + 'CATEGORY_UPDATES', + 'TRASH', + 'SPAM', + 'DRAFT', + 'CHAT', +]; diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/constants/messaging-gmail-excluded-categories.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/constants/messaging-gmail-excluded-categories.ts deleted file mode 100644 index d795c7aa8e..0000000000 --- a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/constants/messaging-gmail-excluded-categories.ts +++ /dev/null @@ -1,8 +0,0 @@ -import { GmailDefaultMessageCategory } from 'src/modules/messaging/message-import-manager/drivers/gmail/types/gmail-default-message-category.type'; - -export const MESSAGING_GMAIL_EXCLUDED_CATEGORIES = [ - GmailDefaultMessageCategory.promotions, - GmailDefaultMessageCategory.forums, - GmailDefaultMessageCategory.social, - GmailDefaultMessageCategory.updates, -]; diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/services/gmail-get-message-list.service.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/services/gmail-get-message-list.service.ts index e9bdfb0670..f52f17e264 100644 --- a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/services/gmail-get-message-list.service.ts +++ b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/services/gmail-get-message-list.service.ts @@ -10,14 +10,11 @@ import { MessageImportDriverException, MessageImportDriverExceptionCode, } from 'src/modules/messaging/message-import-manager/drivers/exceptions/message-import-driver.exception'; -import { MESSAGING_GMAIL_EXCLUDED_CATEGORIES } from 'src/modules/messaging/message-import-manager/drivers/gmail/constants/messaging-gmail-excluded-categories'; import { MESSAGING_GMAIL_USERS_MESSAGES_LIST_MAX_RESULT } from 'src/modules/messaging/message-import-manager/drivers/gmail/constants/messaging-gmail-users-messages-list-max-result.constant'; import { GmailClientProvider } from 'src/modules/messaging/message-import-manager/drivers/gmail/providers/gmail-client.provider'; import { GmailGetHistoryService } from 'src/modules/messaging/message-import-manager/drivers/gmail/services/gmail-get-history.service'; import { GmailHandleErrorService } from 'src/modules/messaging/message-import-manager/drivers/gmail/services/gmail-handle-error.service'; -import { computeGmailCategoryExcludeSearchFilter } from 'src/modules/messaging/message-import-manager/drivers/gmail/utils/compute-gmail-category-excude-search-filter.util'; -import { computeGmailCategoryLabelId } from 'src/modules/messaging/message-import-manager/drivers/gmail/utils/compute-gmail-category-label-id.util'; -import { mapGmailDefaultFolderToCategoryOrUndefined } from 'src/modules/messaging/message-import-manager/drivers/gmail/utils/map-gmail-default-folder-to-category'; +import { computeGmailExcludeSearchFilter } from 'src/modules/messaging/message-import-manager/drivers/gmail/utils/compute-gmail-exclude-search-filter.util'; import { type GetMessageListsArgs } from 'src/modules/messaging/message-import-manager/types/get-message-lists-args.type'; import { type GetMessageListsResponse } from 'src/modules/messaging/message-import-manager/types/get-message-lists-response.type'; import { assertNotNull } from 'src/utils/assert'; @@ -48,7 +45,9 @@ export class GmailGetMessageListService { let hasMoreMessages = true; const messageExternalIds: string[] = []; - const excludedCategories = this.computeExcludedCategories(messageFolders); + + const excludedSearchFilter = + computeGmailExcludeSearchFilter(messageFolders); while (hasMoreMessages) { const messageList = await gmailClient.users.messages @@ -56,8 +55,7 @@ export class GmailGetMessageListService { userId: 'me', maxResults: MESSAGING_GMAIL_USERS_MESSAGES_LIST_MAX_RESULT, pageToken, - q: computeGmailCategoryExcludeSearchFilter(excludedCategories), - labelIds: this.getCustomLabelIds(messageFolders), + q: excludedSearchFilter, }) .catch((error) => { this.logger.error( @@ -157,7 +155,7 @@ export class GmailGetMessageListService { const { messagesAdded, messagesDeleted } = await this.gmailGetHistoryService.getMessageIdsFromHistory(history); - const messageIdsToFilter = await this.getEmailIdsFromExcludedCategories( + const messageIdsToFilter = await this.getEmailIdsFromExcludedFolders( gmailClient, messageChannel.syncCursor, messageFolders, @@ -185,36 +183,30 @@ export class GmailGetMessageListService { ]; } - private computeExcludedCategories( - messageFolders: Pick[], - ) { - const includedDefaultCategories = messageFolders - .map((messageFolder) => - mapGmailDefaultFolderToCategoryOrUndefined(messageFolder.name), - ) - .filter(isDefined); - - return MESSAGING_GMAIL_EXCLUDED_CATEGORIES.filter( - (excludedCategory) => - !includedDefaultCategories.includes(excludedCategory), - ); - } - - private async getEmailIdsFromExcludedCategories( + private async getEmailIdsFromExcludedFolders( gmailClient: gmailV1.Gmail, lastSyncHistoryId: string, - messageFolders: Pick[], + messageFolders: Pick< + MessageFolderWorkspaceEntity, + 'name' | 'externalId' | 'isSynced' + >[], ): Promise { const emailIds: string[] = []; - const excludedCategories = this.computeExcludedCategories(messageFolders); + const toBeExcludedFolders = messageFolders.filter( + (folder) => !folder.isSynced, + ); + + for (const folder of toBeExcludedFolders) { + if (!isDefined(folder.externalId)) { + continue; + } - for (const category of excludedCategories) { const { history } = await this.gmailGetHistoryService.getHistory( gmailClient, lastSyncHistoryId, ['messageAdded'], - computeGmailCategoryLabelId(category), + folder.externalId, ); const emailIdsFromCategory = history @@ -229,23 +221,4 @@ export class GmailGetMessageListService { return emailIds; } - - private getCustomLabelIds( - messageFolders: Pick< - MessageFolderWorkspaceEntity, - 'name' | 'externalId' | 'isSynced' - >[], - ): string[] | undefined { - const customLabelIds = messageFolders - .filter( - (folder) => - folder.externalId && - folder.isSynced && - !mapGmailDefaultFolderToCategoryOrUndefined(folder.name), - ) - .map((folder) => folder.externalId) - .filter((id): id is string => !!id); - - return customLabelIds.length > 0 ? customLabelIds : undefined; - } } diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/types/gmail-default-message-category.type.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/types/gmail-default-message-category.type.ts deleted file mode 100644 index aaa90891b1..0000000000 --- a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/types/gmail-default-message-category.type.ts +++ /dev/null @@ -1,6 +0,0 @@ -export enum GmailDefaultMessageCategory { - promotions = 'promotions', - social = 'social', - forums = 'forums', - updates = 'updates', -} diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/utils/__tests__/compute-gmail-category-excude-search-filter.spec.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/utils/__tests__/compute-gmail-category-excude-search-filter.spec.ts deleted file mode 100644 index 7f1635d2a9..0000000000 --- a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/utils/__tests__/compute-gmail-category-excude-search-filter.spec.ts +++ /dev/null @@ -1,27 +0,0 @@ -import { computeGmailCategoryExcludeSearchFilter } from 'src/modules/messaging/message-import-manager/drivers/gmail/utils/compute-gmail-category-excude-search-filter.util'; - -describe('computeGmailCategoryExcludeSearchFilter', () => { - it('should return correct exclude search filter with empty category array', () => { - const result = computeGmailCategoryExcludeSearchFilter([]); - - expect(result).toBe(''); - }); - - it('should return correct exclude search filter with one category', () => { - const result = computeGmailCategoryExcludeSearchFilter(['CATEGORY1']); - - expect(result).toBe('-category:CATEGORY1'); - }); - - it('should return correct exclude search filter with multiple categories', () => { - const result = computeGmailCategoryExcludeSearchFilter([ - 'CATEGORY1', - 'CATEGORY2', - 'CATEGORY3', - ]); - - expect(result).toBe( - '-category:CATEGORY1 -category:CATEGORY2 -category:CATEGORY3', - ); - }); -}); diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/utils/__tests__/compute-gmail-category-label-id.spec.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/utils/__tests__/compute-gmail-category-label-id.spec.ts deleted file mode 100644 index 91d6634949..0000000000 --- a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/utils/__tests__/compute-gmail-category-label-id.spec.ts +++ /dev/null @@ -1,9 +0,0 @@ -import { computeGmailCategoryLabelId } from 'src/modules/messaging/message-import-manager/drivers/gmail/utils/compute-gmail-category-label-id.util'; - -describe('computeGmailCategoryLabelId', () => { - it('should return correct category label id', () => { - const result = computeGmailCategoryLabelId('CATEGORY1'); - - expect(result).toBe('CATEGORY_CATEGORY1'); - }); -}); diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/utils/__tests__/compute-gmail-exclude-search-filter.spec.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/utils/__tests__/compute-gmail-exclude-search-filter.spec.ts new file mode 100644 index 0000000000..8fc24d95fa --- /dev/null +++ b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/utils/__tests__/compute-gmail-exclude-search-filter.spec.ts @@ -0,0 +1,69 @@ +import { computeGmailExcludeSearchFilter } from 'src/modules/messaging/message-import-manager/drivers/gmail/utils/compute-gmail-exclude-search-filter.util'; + +describe('computeGmailExcludeSearchFilter', () => { + it('should return correct exclude search filter with empty label array', () => { + const result = computeGmailExcludeSearchFilter([]); + + expect(result).toBe(''); + }); + + it('should return correct exclude search filter with one label', () => { + const result = computeGmailExcludeSearchFilter([ + { + externalId: 'LABEL1', + isSynced: false, + }, + ]); + + expect(result).toBe('-label:LABEL1'); + }); + + it('should return correct exclude search filter with multiple categories', () => { + const result = computeGmailExcludeSearchFilter([ + { + externalId: 'LABEL1', + isSynced: false, + }, + { + externalId: 'LABEL2', + isSynced: false, + }, + { + externalId: 'LABEL3', + isSynced: false, + }, + ]); + + expect(result).toBe('-label:LABEL1 -label:LABEL2 -label:LABEL3'); + }); + + it('should return correct exclude search filter with one label that is synced', () => { + const result = computeGmailExcludeSearchFilter([ + { + externalId: 'LABEL1', + isSynced: true, + }, + ]); + + expect(result).toBe(''); + }); + + it('should return correct exclude search filter with multiple categories that are synced', () => { + const result = computeGmailExcludeSearchFilter([ + { + externalId: 'LABEL1', + isSynced: true, + }, + { + externalId: 'LABEL2', + isSynced: false, + }, + { + externalId: 'LABEL3', + isSynced: true, + }, + ]); + + expect(result).toBe('-label:LABEL2'); + }); +}); diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/utils/compute-gmail-category-excude-search-filter.util.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/utils/compute-gmail-category-excude-search-filter.util.ts deleted file mode 100644 index e8deb1140d..0000000000 --- a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/utils/compute-gmail-category-excude-search-filter.util.ts +++ /dev/null @@ -1,3 +0,0 @@ -export const computeGmailCategoryExcludeSearchFilter = ( - excludedCategories: string[], -) => excludedCategories.map((category) => `-category:${category}`).join(' '); diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/utils/compute-gmail-category-label-id.util.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/utils/compute-gmail-category-label-id.util.ts deleted file mode 100644 index 7c3de6e2a6..0000000000 --- a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/utils/compute-gmail-category-label-id.util.ts +++ /dev/null @@ -1,2 +0,0 @@ -export const computeGmailCategoryLabelId = (category: string) => - `CATEGORY_${category.toUpperCase()}`; diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/utils/compute-gmail-exclude-search-filter.util.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/utils/compute-gmail-exclude-search-filter.util.ts new file mode 100644 index 0000000000..2af6e8dd01 --- /dev/null +++ b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/utils/compute-gmail-exclude-search-filter.util.ts @@ -0,0 +1,12 @@ +import { type MessageFolderWorkspaceEntity } from 'src/modules/messaging/common/standard-objects/message-folder.workspace-entity'; + +export const computeGmailExcludeSearchFilter = ( + messageFolders: Pick< + MessageFolderWorkspaceEntity, + 'externalId' | 'isSynced' + >[], +) => + messageFolders + .filter((folder) => !folder.isSynced) + .map((folder) => `-label:${folder.externalId}`) + .join(' '); diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/utils/map-gmail-default-folder-to-category.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/utils/map-gmail-default-folder-to-category.ts deleted file mode 100644 index 6f93cbb706..0000000000 --- a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/utils/map-gmail-default-folder-to-category.ts +++ /dev/null @@ -1,37 +0,0 @@ -import { isDefined } from 'twenty-shared/utils'; - -import { GmailDefaultMessageCategory } from 'src/modules/messaging/message-import-manager/drivers/gmail/types/gmail-default-message-category.type'; -import { GmailDefaultMessageFolder } from 'src/modules/messaging/message-import-manager/drivers/gmail/types/gmail-default-message-folder.type'; - -const DEFAULT_FOLDER_NAME_TO_CATEGORY_MAPPING = [ - { - folderName: GmailDefaultMessageFolder.CATEGORY_FORUMS, - category: GmailDefaultMessageCategory.forums, - }, - { - folderName: GmailDefaultMessageFolder.CATEGORY_PROMOTIONS, - category: GmailDefaultMessageCategory.promotions, - }, - { - folderName: GmailDefaultMessageFolder.CATEGORY_SOCIAL, - category: GmailDefaultMessageCategory.social, - }, - { - folderName: GmailDefaultMessageFolder.CATEGORY_UPDATES, - category: GmailDefaultMessageCategory.updates, - }, -]; - -export const mapGmailDefaultFolderToCategoryOrUndefined = ( - messageFolderName: string, -) => { - const mapping = DEFAULT_FOLDER_NAME_TO_CATEGORY_MAPPING.find( - ({ folderName }) => folderName === messageFolderName, - ); - - if (!isDefined(mapping)) { - return undefined; - } - - return mapping.category; -}; diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/imap/services/imap-get-message-list.service.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/imap/services/imap-get-message-list.service.ts index 288834d288..0fc0fa83e8 100644 --- a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/imap/services/imap-get-message-list.service.ts +++ b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/imap/services/imap-get-message-list.service.ts @@ -33,12 +33,15 @@ export class ImapGetMessageListService { messageFolders, }: GetMessageListsArgs): Promise { let client: ImapFlow | null = null; + const syncedMessageFolders = messageFolders.filter( + (folder) => folder.isSynced, + ); try { client = await this.imapClientProvider.getClient(connectedAccount); const result: GetMessageListsResponse = []; - for (const folder of messageFolders) { + for (const folder of syncedMessageFolders) { this.logger.log(`Processing folder: ${folder.name}`); try { @@ -76,7 +79,7 @@ export class ImapGetMessageListService { this.imapHandleErrorService.handleImapMessageListFetchError(error); - return messageFolders.map((folder) => ({ + return syncedMessageFolders.map((folder) => ({ messageExternalIds: [], nextSyncCursor: folder.syncCursor || '', previousSyncCursor: folder.syncCursor, diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-get-message-list.service.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-get-message-list.service.ts index 6f06a9151b..27277bca3c 100644 --- a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-get-message-list.service.ts +++ b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-get-message-list.service.ts @@ -47,7 +47,11 @@ export class MicrosoftGetMessageListService { ); } - for (const folder of messageFolders) { + const syncedMessageFolders = messageFolders.filter( + (folder) => folder.isSynced, + ); + + for (const folder of syncedMessageFolders) { const response = await this.getMessageList(connectedAccount, folder); result.push({ diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-message-list-fetch.service.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-message-list-fetch.service.ts index e3ea690644..75958a4a25 100644 --- a/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-message-list-fetch.service.ts +++ b/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-message-list-fetch.service.ts @@ -91,17 +91,16 @@ export class MessagingMessageListFetchService { 'messageFolder', ); - const messageFoldersToSync = await messageFolderRepository.find({ + const messageFolders = await messageFolderRepository.find({ where: { messageChannelId: messageChannel.id, - isSynced: true, }, }); const messageLists = await this.messagingGetMessageListService.getMessageLists( messageChannelWithFreshTokens, - messageFoldersToSync, + messageFolders, ); await this.cacheStorage.del( @@ -124,7 +123,11 @@ export class MessagingMessageListFetchService { let totalMessagesToImportCount = 0; this.logger.log( - `messageChannelId: ${messageChannel.id} Is full sync: ${isFullSync} and toImportCount: ${messageExternalIds.length}, toDeleteCount: ${messageExternalIdsToDelete.length}`, + `messageChannelId: ${messageChannel.id} Is full sync: ${isFullSync} and toImportCount: ${messageExternalIds.length}, toDeleteCount: ${messageExternalIdsToDelete.length}, cursors: ${messageLists.map( + (messageList) => { + messageList.nextSyncCursor; + }, + )}`, ); const messageChannelMessageAssociationRepository = @@ -207,26 +210,22 @@ export class MessagingMessageListFetchService { const toDeleteChunks = chunk(allMessageExternalIdsToDelete, 200); for (const [index, toDeleteChunk] of toDeleteChunks.entries()) { - await messageChannelMessageAssociationRepository.delete({ - messageChannelId: messageChannelWithFreshTokens.id, - messageExternalId: In(toDeleteChunk), - }); - this.logger.log( - `messageChannelId: ${messageChannel.id} Deleted ${toDeleteChunk.length} message channel message associations in batch ${index + 1}`, + `messageChannelId: ${messageChannel.id} Deleting ${toDeleteChunk.length} message channel message associations in batch ${index + 1}`, + ); + + await this.messagingMessageCleanerService.deleteMessagesChannelMessageAssociationsAndRelatedOrphans( + { + workspaceId, + messageExternalIds: toDeleteChunk.filter((messageExternalId) => + isNonEmptyString(messageExternalId), + ), + messageChannelId: messageChannelWithFreshTokens.id, + }, ); } } - this.logger.log( - `messageChannelId: ${messageChannel.id} launching workspace thread cleanup`, - ); - - // TODO: Re-enable this after we have a way to clean up threads without impacting the message import - // await this.messagingMessageCleanerService.cleanWorkspaceThreads( - // workspaceId, - // ); - this.logger.log( `messageChannelId: ${messageChannel.id} Total messages to import count: ${totalMessagesToImportCount}`, );