diff --git a/packages/twenty-server/src/command/command.module.ts b/packages/twenty-server/src/command/command.module.ts index 2c99a18f47..6cd6fe2cec 100644 --- a/packages/twenty-server/src/command/command.module.ts +++ b/packages/twenty-server/src/command/command.module.ts @@ -8,6 +8,7 @@ import { WorkspaceCleanerModule } from 'src/engine/workspace-manager/workspace-c import { WorkspaceHealthCommandModule } from 'src/engine/workspace-manager/workspace-health/commands/workspace-health-command.module'; import { WorkspaceMigrationRunnerCommandsModule } from 'src/engine/workspace-manager/workspace-migration-runner/commands/workspace-migration-runner-commands.module'; import { WorkspaceSyncMetadataCommandsModule } from 'src/engine/workspace-manager/workspace-sync-metadata/commands/workspace-sync-metadata-commands.module'; +import { MessagingMessageCleanerModule } from 'src/modules/messaging/message-cleaner/messaging-message-cleaner.module'; @Module({ imports: [ @@ -16,6 +17,7 @@ import { WorkspaceSyncMetadataCommandsModule } from 'src/engine/workspace-manage DatabaseCommandModule, WorkspaceCleanerModule, WorkspaceHealthCommandModule, + MessagingMessageCleanerModule, WorkspaceMigrationRunnerCommandsModule, ObjectMetadataModule, FieldMetadataModule, diff --git a/packages/twenty-server/src/engine/core-modules/messaging/services/timeline-messaging.service.ts b/packages/twenty-server/src/engine/core-modules/messaging/services/timeline-messaging.service.ts index c739385622..4e68d3c1cc 100644 --- a/packages/twenty-server/src/engine/core-modules/messaging/services/timeline-messaging.service.ts +++ b/packages/twenty-server/src/engine/core-modules/messaging/services/timeline-messaging.service.ts @@ -1,6 +1,6 @@ import { Injectable } from '@nestjs/common'; -import { In, Not } from 'typeorm'; +import { In } from 'typeorm'; import { type TimelineThread } from 'src/engine/core-modules/messaging/dtos/timeline-thread.dto'; import { TwentyORMManager } from 'src/engine/twenty-orm/twenty-orm.manager'; @@ -94,6 +94,7 @@ export class TimelineMessagingService { await this.twentyORMManager.getRepository( 'messageParticipant', ); + const threadParticipants = await messageParticipantRepository .createQueryBuilder() .select('messageParticipant') @@ -183,32 +184,11 @@ export class TimelineMessagingService { 'messageThread', ); - const threadsWithoutWorkspaceMember = await messageThreadRepository.find({ - select: { - id: true, - }, - where: { - id: In(messageThreadIds), - messages: { - messageChannelMessageAssociations: { - messageChannel: { - connectedAccount: { - accountOwnerId: Not(workspaceMemberId), - }, - }, - }, - }, - }, - }); - - const threadIdsWithoutWorkspaceMember = threadsWithoutWorkspaceMember.map( - (thread) => thread.id, - ); - const threadVisibility = await messageThreadRepository .createQueryBuilder() .select('messageThread.id', 'id') .addSelect('messageChannel.visibility', 'visibility') + .addSelect('connectedAccount.accountOwnerId', 'accountOwnerId') .leftJoin('messageThread.messages', 'message') .leftJoin( 'message.messageChannelMessageAssociations', @@ -218,46 +198,34 @@ export class TimelineMessagingService { 'messageChannelMessageAssociation.messageChannel', 'messageChannel', ) + .leftJoin('messageChannel.connectedAccount', 'connectedAccount') .where('messageThread.id = ANY(:messageThreadIds)', { - messageThreadIds: threadIdsWithoutWorkspaceMember, + messageThreadIds: messageThreadIds, }) .getRawMany(); const visibilityValues = Object.values(MessageChannelVisibility); - const threadVisibilityByThreadIdForWhichWorkspaceMemberIsNotOwner: - | { - [key: string]: MessageChannelVisibility; - } - | undefined = threadVisibility?.reduce( - (threadVisibilityAcc, threadVisibility) => { - threadVisibilityAcc[threadVisibility.id] = - visibilityValues[ - Math.max( - visibilityValues.indexOf(threadVisibility.visibility), - visibilityValues.indexOf( - threadVisibilityAcc[threadVisibility.id] ?? - MessageChannelVisibility.METADATA, - ), - ) - ]; - - return threadVisibilityAcc; - }, - {}, - ); - const threadVisibilityByThreadId: { [key: string]: MessageChannelVisibility; - } = messageThreadIds.reduce((threadVisibilityAcc, messageThreadId) => { - // If the workspace member is not the owner of the thread, use the visibility value from the query - // @ts-expect-error legacy noImplicitAny - threadVisibilityAcc[messageThreadId] = - threadIdsWithoutWorkspaceMember.includes(messageThreadId) - ? (threadVisibilityByThreadIdForWhichWorkspaceMemberIsNotOwner?.[ - messageThreadId - ] ?? MessageChannelVisibility.METADATA) - : MessageChannelVisibility.SHARE_EVERYTHING; + } = threadVisibility.reduce((threadVisibilityAcc, threadVisibility) => { + if (threadVisibility.accountOwnerId === workspaceMemberId) { + threadVisibilityAcc[threadVisibility.id] = + MessageChannelVisibility.SHARE_EVERYTHING; + + return threadVisibilityAcc; + } + + threadVisibilityAcc[threadVisibility.id] = + visibilityValues[ + Math.max( + visibilityValues.indexOf(threadVisibility.visibility), + visibilityValues.indexOf( + threadVisibilityAcc[threadVisibility.id] ?? + MessageChannelVisibility.METADATA, + ), + ) + ]; return threadVisibilityAcc; }, {}); diff --git a/packages/twenty-server/src/modules/messaging/blocklist-manager/jobs/messaging-blocklist-item-delete-messages.job.ts b/packages/twenty-server/src/modules/messaging/blocklist-manager/jobs/messaging-blocklist-item-delete-messages.job.ts index 2ce8f97e76..c4cb444486 100644 --- a/packages/twenty-server/src/modules/messaging/blocklist-manager/jobs/messaging-blocklist-item-delete-messages.job.ts +++ b/packages/twenty-server/src/modules/messaging/blocklist-manager/jobs/messaging-blocklist-item-delete-messages.job.ts @@ -140,6 +140,6 @@ export class BlocklistItemDeleteMessagesJob { } } - await this.threadCleanerService.cleanWorkspaceThreads(workspaceId); + await this.threadCleanerService.cleanOrphanMessagesAndThreads(workspaceId); } } diff --git a/packages/twenty-server/src/modules/messaging/message-cleaner/commands/messaging-message-clearner-remove-orphans.command.ts b/packages/twenty-server/src/modules/messaging/message-cleaner/commands/messaging-message-clearner-remove-orphans.command.ts new file mode 100644 index 0000000000..c1a73cdc0e --- /dev/null +++ b/packages/twenty-server/src/modules/messaging/message-cleaner/commands/messaging-message-clearner-remove-orphans.command.ts @@ -0,0 +1,39 @@ +import { InjectRepository } from '@nestjs/typeorm'; + +import { Command } from 'nest-commander'; +import { Repository } from 'typeorm'; + +import { + ActiveOrSuspendedWorkspacesMigrationCommandRunner, + type RunOnWorkspaceArgs, +} from 'src/database/commands/command-runners/active-or-suspended-workspaces-migration.command-runner'; +import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity'; +import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager'; +import { MessagingMessageCleanerService } from 'src/modules/messaging/message-cleaner/services/messaging-message-cleaner.service'; + +@Command({ + name: 'messaging:message-cleaner-remove-orphans', + description: 'Remove orphan message and threads from messaging', +}) +export class MessagingMessageCleanerRemoveOrphansCommand extends ActiveOrSuspendedWorkspacesMigrationCommandRunner { + constructor( + @InjectRepository(Workspace) + protected readonly workspaceRepository: Repository, + protected readonly twentyORMGlobalManager: TwentyORMGlobalManager, + private readonly messagingMessageCleanerService: MessagingMessageCleanerService, + ) { + super(workspaceRepository, twentyORMGlobalManager); + } + + override async runOnWorkspace({ + workspaceId, + }: RunOnWorkspaceArgs): Promise { + try { + await this.messagingMessageCleanerService.cleanOrphanMessagesAndThreads( + workspaceId, + ); + } catch (error) { + this.logger.error('Error while deleting workflowRun', error); + } + } +} diff --git a/packages/twenty-server/src/modules/messaging/message-cleaner/jobs/messaging-connected-account-deletion-cleanup.job.ts b/packages/twenty-server/src/modules/messaging/message-cleaner/jobs/messaging-connected-account-deletion-cleanup.job.ts index f564256b54..6291f49ee8 100644 --- a/packages/twenty-server/src/modules/messaging/message-cleaner/jobs/messaging-connected-account-deletion-cleanup.job.ts +++ b/packages/twenty-server/src/modules/messaging/message-cleaner/jobs/messaging-connected-account-deletion-cleanup.job.ts @@ -23,6 +23,8 @@ export class MessagingConnectedAccountDeletionCleanupJob { async handle( data: MessagingConnectedAccountDeletionCleanupJobData, ): Promise { - await this.messageCleanerService.cleanWorkspaceThreads(data.workspaceId); + await this.messageCleanerService.cleanOrphanMessagesAndThreads( + data.workspaceId, + ); } } diff --git a/packages/twenty-server/src/modules/messaging/message-cleaner/messaging-message-cleaner.module.ts b/packages/twenty-server/src/modules/messaging/message-cleaner/messaging-message-cleaner.module.ts index 7b25fa456a..fe3254da99 100644 --- a/packages/twenty-server/src/modules/messaging/message-cleaner/messaging-message-cleaner.module.ts +++ b/packages/twenty-server/src/modules/messaging/message-cleaner/messaging-message-cleaner.module.ts @@ -1,15 +1,19 @@ import { Module } from '@nestjs/common'; +import { TypeOrmModule } from '@nestjs/typeorm'; +import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity'; +import { MessagingMessageCleanerRemoveOrphansCommand } from 'src/modules/messaging/message-cleaner/commands/messaging-message-clearner-remove-orphans.command'; import { MessagingConnectedAccountDeletionCleanupJob } from 'src/modules/messaging/message-cleaner/jobs/messaging-connected-account-deletion-cleanup.job'; import { MessagingMessageCleanerConnectedAccountListener } from 'src/modules/messaging/message-cleaner/listeners/messaging-message-cleaner-connected-account.listener'; import { MessagingMessageCleanerService } from 'src/modules/messaging/message-cleaner/services/messaging-message-cleaner.service'; @Module({ - imports: [], + imports: [TypeOrmModule.forFeature([Workspace])], providers: [ MessagingMessageCleanerService, MessagingConnectedAccountDeletionCleanupJob, MessagingMessageCleanerConnectedAccountListener, + MessagingMessageCleanerRemoveOrphansCommand, ], exports: [MessagingMessageCleanerService], }) diff --git a/packages/twenty-server/src/modules/messaging/message-cleaner/services/messaging-message-cleaner.service.ts b/packages/twenty-server/src/modules/messaging/message-cleaner/services/messaging-message-cleaner.service.ts index 1c7f0d9f98..636c71723e 100644 --- a/packages/twenty-server/src/modules/messaging/message-cleaner/services/messaging-message-cleaner.service.ts +++ b/packages/twenty-server/src/modules/messaging/message-cleaner/services/messaging-message-cleaner.service.ts @@ -1,9 +1,11 @@ import { Injectable, Logger } from '@nestjs/common'; -import { IsNull } from 'typeorm'; +import chunk from 'lodash.chunk'; +import { In, IsNull } from 'typeorm'; import { type WorkspaceEntityManager } from 'src/engine/twenty-orm/entity-manager/workspace-entity-manager'; -import { TwentyORMManager } from 'src/engine/twenty-orm/twenty-orm.manager'; +import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager'; +import { MessageChannelMessageAssociationWorkspaceEntity } from 'src/modules/messaging/common/standard-objects/message-channel-message-association.workspace-entity'; import { type MessageThreadWorkspaceEntity } from 'src/modules/messaging/common/standard-objects/message-thread.workspace-entity'; import { type MessageWorkspaceEntity } from 'src/modules/messaging/common/standard-objects/message.workspace-entity'; import { deleteUsingPagination } from 'src/modules/messaging/message-cleaner/utils/delete-using-pagination.util'; @@ -11,20 +13,111 @@ import { deleteUsingPagination } from 'src/modules/messaging/message-cleaner/uti @Injectable() export class MessagingMessageCleanerService { private readonly logger = new Logger(MessagingMessageCleanerService.name); - constructor(private readonly twentyORMManager: TwentyORMManager) {} + constructor( + private readonly twentyORMGlobalManager: TwentyORMGlobalManager, + ) {} + + async deleteMessagesChannelMessageAssociationsAndRelatedOrphans({ + workspaceId, + messageExternalIds, + messageChannelId, + }: { + workspaceId: string; + messageExternalIds: string[]; + messageChannelId: string; + }) { + const messageRepository = + await this.twentyORMGlobalManager.getRepositoryForWorkspace( + workspaceId, + 'message', + ); + + const messageChannelMessageAssociationRepository = + await this.twentyORMGlobalManager.getRepositoryForWorkspace( + workspaceId, + 'messageChannelMessageAssociation', + ); - public async cleanWorkspaceThreads(workspaceId: string) { const messageThreadRepository = - await this.twentyORMManager.getRepository( + await this.twentyORMGlobalManager.getRepositoryForWorkspace( + workspaceId, + 'messageThread', + ); + + const messageExternalIdsChunks = chunk(messageExternalIds, 500); + + for (const messageExternalIdsChunk of messageExternalIdsChunks) { + const messageChannelMessageAssociationsToDelete = + await messageChannelMessageAssociationRepository.find({ + where: { + messageExternalId: In(messageExternalIdsChunk), + messageChannelId, + }, + }); + + await messageChannelMessageAssociationRepository.delete( + messageChannelMessageAssociationsToDelete.map(({ id }) => id), + ); + + this.logger.log( + `WorkspaceId: ${workspaceId} Deleting ${messageChannelMessageAssociationsToDelete.length} message channel message associations`, + ); + + const orphanMessages = await messageRepository.find({ + where: { + id: In( + messageChannelMessageAssociationsToDelete.map( + ({ messageId }) => messageId, + ), + ), + messageChannelMessageAssociations: { + id: IsNull(), + }, + }, + }); + + this.logger.log( + `WorkspaceId: ${workspaceId} Deleting ${orphanMessages.length} orphan messages`, + ); + + await messageRepository.delete(orphanMessages.map(({ id }) => id)); + + const orphanMessageThreads = await messageThreadRepository.find({ + where: { + id: In(orphanMessages.map(({ messageThreadId }) => messageThreadId)), + messages: { + id: IsNull(), + }, + }, + }); + + this.logger.log( + `WorkspaceId: ${workspaceId} Deleting ${orphanMessageThreads.length} orphan message threads`, + ); + + await messageThreadRepository.delete( + orphanMessageThreads.map(({ id }) => id), + ); + } + } + + public async cleanOrphanMessagesAndThreads(workspaceId: string) { + const messageThreadRepository = + await this.twentyORMGlobalManager.getRepositoryForWorkspace( + workspaceId, 'messageThread', ); const messageRepository = - await this.twentyORMManager.getRepository( + await this.twentyORMGlobalManager.getRepositoryForWorkspace( + workspaceId, 'message', ); - const workspaceDataSource = await this.twentyORMManager.getDatasource(); + const workspaceDataSource = + await this.twentyORMGlobalManager.getDataSourceForWorkspace({ + workspaceId, + }); await workspaceDataSource.transaction( async (transactionManager: WorkspaceEntityManager) => { diff --git a/packages/twenty-server/src/modules/messaging/message-folder-manager/drivers/gmail/gmail-get-all-folders.service.ts b/packages/twenty-server/src/modules/messaging/message-folder-manager/drivers/gmail/gmail-get-all-folders.service.ts index 6a6376c47e..3b3d8f4875 100644 --- a/packages/twenty-server/src/modules/messaging/message-folder-manager/drivers/gmail/gmail-get-all-folders.service.ts +++ b/packages/twenty-server/src/modules/messaging/message-folder-manager/drivers/gmail/gmail-get-all-folders.service.ts @@ -1,18 +1,14 @@ import { Injectable, Logger } from '@nestjs/common'; -import { gmail_v1 } from 'googleapis'; -import { isDefined } from 'twenty-shared/utils'; - import { MessageFolder, MessageFolderDriver, } from 'src/modules/messaging/message-folder-manager/interfaces/message-folder-driver.interface'; import { type ConnectedAccountWorkspaceEntity } from 'src/modules/connected-account/standard-objects/connected-account.workspace-entity'; -import { MESSAGING_GMAIL_EXCLUDED_CATEGORIES } from 'src/modules/messaging/message-import-manager/drivers/gmail/constants/messaging-gmail-excluded-categories'; +import { MESSAGING_GMAIL_DEFAULT_NOT_SYNCED_LABELS } from 'src/modules/messaging/message-import-manager/drivers/gmail/constants/messaging-gmail-default-not-synced-labels'; import { GmailClientProvider } from 'src/modules/messaging/message-import-manager/drivers/gmail/providers/gmail-client.provider'; import { GmailHandleErrorService } from 'src/modules/messaging/message-import-manager/drivers/gmail/services/gmail-handle-error.service'; -import { computeGmailCategoryLabelId } from 'src/modules/messaging/message-import-manager/drivers/gmail/utils/compute-gmail-category-label-id.util'; @Injectable() export class GmailGetAllFoldersService implements MessageFolderDriver { @@ -23,24 +19,8 @@ export class GmailGetAllFoldersService implements MessageFolderDriver { private readonly gmailHandleErrorService: GmailHandleErrorService, ) {} - private isExcludedCategoryFolder(labelId: string): boolean { - const excludedCategoryIds = MESSAGING_GMAIL_EXCLUDED_CATEGORIES.map( - (category) => computeGmailCategoryLabelId(category), - ); - - return excludedCategoryIds.includes(labelId); - } - - private isIncludedFolder(label: gmail_v1.Schema$Label): boolean { - if (!isDefined(label.id)) { - return false; - } - - const isTargetSystemFolder = - label.type === 'system' && (label.id === 'INBOX' || label.id === 'SENT'); - const isUserFolder = label.type === 'user'; - - return isTargetSystemFolder || isUserFolder; + private isSyncedByDefault(labelId: string): boolean { + return !MESSAGING_GMAIL_DEFAULT_NOT_SYNCED_LABELS.includes(labelId); } async getAllMessageFolders( @@ -74,21 +54,12 @@ export class GmailGetAllFoldersService implements MessageFolderDriver { continue; } - if (this.isExcludedCategoryFolder(label.id)) { - continue; - } - - if (!this.isIncludedFolder(label)) { - continue; - } - const isSentFolder = label.id === 'SENT'; - const isSyncedByDefault = label.id === 'INBOX' || label.id === 'SENT'; folders.push({ externalId: label.id, name: label.name, - isSynced: isSyncedByDefault, + isSynced: this.isSyncedByDefault(label.id), isSentFolder, }); } diff --git a/packages/twenty-server/src/modules/messaging/message-folder-manager/services/sync-message-folders.service.ts b/packages/twenty-server/src/modules/messaging/message-folder-manager/services/sync-message-folders.service.ts index f02fb279b1..24f50a100e 100644 --- a/packages/twenty-server/src/modules/messaging/message-folder-manager/services/sync-message-folders.service.ts +++ b/packages/twenty-server/src/modules/messaging/message-folder-manager/services/sync-message-folders.service.ts @@ -69,6 +69,7 @@ export class SyncMessageFoldersService { messageFolderRepository, }); + // TODO: we should delete folders that are not in the list anymore for (const folder of folders) { const existingFolder = this.findExistingFolderInMap( existingFolderMap, @@ -80,8 +81,6 @@ export class SyncMessageFoldersService { existingFolder.id, { name: folder.name, - isSynced: folder.isSynced, - isSentFolder: folder.isSentFolder, externalId: folder.externalId, }, manager, diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/constants/messaging-gmail-default-not-synced-labels.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/constants/messaging-gmail-default-not-synced-labels.ts new file mode 100644 index 0000000000..5e88beafcb --- /dev/null +++ b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/constants/messaging-gmail-default-not-synced-labels.ts @@ -0,0 +1,10 @@ +export const MESSAGING_GMAIL_DEFAULT_NOT_SYNCED_LABELS = [ + 'CATEGORY_PROMOTIONS', + 'CATEGORY_SOCIAL', + 'CATEGORY_FORUMS', + 'CATEGORY_UPDATES', + 'TRASH', + 'SPAM', + 'DRAFT', + 'CHAT', +]; diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/constants/messaging-gmail-excluded-categories.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/constants/messaging-gmail-excluded-categories.ts deleted file mode 100644 index d795c7aa8e..0000000000 --- a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/constants/messaging-gmail-excluded-categories.ts +++ /dev/null @@ -1,8 +0,0 @@ -import { GmailDefaultMessageCategory } from 'src/modules/messaging/message-import-manager/drivers/gmail/types/gmail-default-message-category.type'; - -export const MESSAGING_GMAIL_EXCLUDED_CATEGORIES = [ - GmailDefaultMessageCategory.promotions, - GmailDefaultMessageCategory.forums, - GmailDefaultMessageCategory.social, - GmailDefaultMessageCategory.updates, -]; diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/services/gmail-get-message-list.service.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/services/gmail-get-message-list.service.ts index e9bdfb0670..f52f17e264 100644 --- a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/services/gmail-get-message-list.service.ts +++ b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/services/gmail-get-message-list.service.ts @@ -10,14 +10,11 @@ import { MessageImportDriverException, MessageImportDriverExceptionCode, } from 'src/modules/messaging/message-import-manager/drivers/exceptions/message-import-driver.exception'; -import { MESSAGING_GMAIL_EXCLUDED_CATEGORIES } from 'src/modules/messaging/message-import-manager/drivers/gmail/constants/messaging-gmail-excluded-categories'; import { MESSAGING_GMAIL_USERS_MESSAGES_LIST_MAX_RESULT } from 'src/modules/messaging/message-import-manager/drivers/gmail/constants/messaging-gmail-users-messages-list-max-result.constant'; import { GmailClientProvider } from 'src/modules/messaging/message-import-manager/drivers/gmail/providers/gmail-client.provider'; import { GmailGetHistoryService } from 'src/modules/messaging/message-import-manager/drivers/gmail/services/gmail-get-history.service'; import { GmailHandleErrorService } from 'src/modules/messaging/message-import-manager/drivers/gmail/services/gmail-handle-error.service'; -import { computeGmailCategoryExcludeSearchFilter } from 'src/modules/messaging/message-import-manager/drivers/gmail/utils/compute-gmail-category-excude-search-filter.util'; -import { computeGmailCategoryLabelId } from 'src/modules/messaging/message-import-manager/drivers/gmail/utils/compute-gmail-category-label-id.util'; -import { mapGmailDefaultFolderToCategoryOrUndefined } from 'src/modules/messaging/message-import-manager/drivers/gmail/utils/map-gmail-default-folder-to-category'; +import { computeGmailExcludeSearchFilter } from 'src/modules/messaging/message-import-manager/drivers/gmail/utils/compute-gmail-exclude-search-filter.util'; import { type GetMessageListsArgs } from 'src/modules/messaging/message-import-manager/types/get-message-lists-args.type'; import { type GetMessageListsResponse } from 'src/modules/messaging/message-import-manager/types/get-message-lists-response.type'; import { assertNotNull } from 'src/utils/assert'; @@ -48,7 +45,9 @@ export class GmailGetMessageListService { let hasMoreMessages = true; const messageExternalIds: string[] = []; - const excludedCategories = this.computeExcludedCategories(messageFolders); + + const excludedSearchFilter = + computeGmailExcludeSearchFilter(messageFolders); while (hasMoreMessages) { const messageList = await gmailClient.users.messages @@ -56,8 +55,7 @@ export class GmailGetMessageListService { userId: 'me', maxResults: MESSAGING_GMAIL_USERS_MESSAGES_LIST_MAX_RESULT, pageToken, - q: computeGmailCategoryExcludeSearchFilter(excludedCategories), - labelIds: this.getCustomLabelIds(messageFolders), + q: excludedSearchFilter, }) .catch((error) => { this.logger.error( @@ -157,7 +155,7 @@ export class GmailGetMessageListService { const { messagesAdded, messagesDeleted } = await this.gmailGetHistoryService.getMessageIdsFromHistory(history); - const messageIdsToFilter = await this.getEmailIdsFromExcludedCategories( + const messageIdsToFilter = await this.getEmailIdsFromExcludedFolders( gmailClient, messageChannel.syncCursor, messageFolders, @@ -185,36 +183,30 @@ export class GmailGetMessageListService { ]; } - private computeExcludedCategories( - messageFolders: Pick[], - ) { - const includedDefaultCategories = messageFolders - .map((messageFolder) => - mapGmailDefaultFolderToCategoryOrUndefined(messageFolder.name), - ) - .filter(isDefined); - - return MESSAGING_GMAIL_EXCLUDED_CATEGORIES.filter( - (excludedCategory) => - !includedDefaultCategories.includes(excludedCategory), - ); - } - - private async getEmailIdsFromExcludedCategories( + private async getEmailIdsFromExcludedFolders( gmailClient: gmailV1.Gmail, lastSyncHistoryId: string, - messageFolders: Pick[], + messageFolders: Pick< + MessageFolderWorkspaceEntity, + 'name' | 'externalId' | 'isSynced' + >[], ): Promise { const emailIds: string[] = []; - const excludedCategories = this.computeExcludedCategories(messageFolders); + const toBeExcludedFolders = messageFolders.filter( + (folder) => !folder.isSynced, + ); + + for (const folder of toBeExcludedFolders) { + if (!isDefined(folder.externalId)) { + continue; + } - for (const category of excludedCategories) { const { history } = await this.gmailGetHistoryService.getHistory( gmailClient, lastSyncHistoryId, ['messageAdded'], - computeGmailCategoryLabelId(category), + folder.externalId, ); const emailIdsFromCategory = history @@ -229,23 +221,4 @@ export class GmailGetMessageListService { return emailIds; } - - private getCustomLabelIds( - messageFolders: Pick< - MessageFolderWorkspaceEntity, - 'name' | 'externalId' | 'isSynced' - >[], - ): string[] | undefined { - const customLabelIds = messageFolders - .filter( - (folder) => - folder.externalId && - folder.isSynced && - !mapGmailDefaultFolderToCategoryOrUndefined(folder.name), - ) - .map((folder) => folder.externalId) - .filter((id): id is string => !!id); - - return customLabelIds.length > 0 ? customLabelIds : undefined; - } } diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/types/gmail-default-message-category.type.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/types/gmail-default-message-category.type.ts deleted file mode 100644 index aaa90891b1..0000000000 --- a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/types/gmail-default-message-category.type.ts +++ /dev/null @@ -1,6 +0,0 @@ -export enum GmailDefaultMessageCategory { - promotions = 'promotions', - social = 'social', - forums = 'forums', - updates = 'updates', -} diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/utils/__tests__/compute-gmail-category-excude-search-filter.spec.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/utils/__tests__/compute-gmail-category-excude-search-filter.spec.ts deleted file mode 100644 index 7f1635d2a9..0000000000 --- a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/utils/__tests__/compute-gmail-category-excude-search-filter.spec.ts +++ /dev/null @@ -1,27 +0,0 @@ -import { computeGmailCategoryExcludeSearchFilter } from 'src/modules/messaging/message-import-manager/drivers/gmail/utils/compute-gmail-category-excude-search-filter.util'; - -describe('computeGmailCategoryExcludeSearchFilter', () => { - it('should return correct exclude search filter with empty category array', () => { - const result = computeGmailCategoryExcludeSearchFilter([]); - - expect(result).toBe(''); - }); - - it('should return correct exclude search filter with one category', () => { - const result = computeGmailCategoryExcludeSearchFilter(['CATEGORY1']); - - expect(result).toBe('-category:CATEGORY1'); - }); - - it('should return correct exclude search filter with multiple categories', () => { - const result = computeGmailCategoryExcludeSearchFilter([ - 'CATEGORY1', - 'CATEGORY2', - 'CATEGORY3', - ]); - - expect(result).toBe( - '-category:CATEGORY1 -category:CATEGORY2 -category:CATEGORY3', - ); - }); -}); diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/utils/__tests__/compute-gmail-category-label-id.spec.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/utils/__tests__/compute-gmail-category-label-id.spec.ts deleted file mode 100644 index 91d6634949..0000000000 --- a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/utils/__tests__/compute-gmail-category-label-id.spec.ts +++ /dev/null @@ -1,9 +0,0 @@ -import { computeGmailCategoryLabelId } from 'src/modules/messaging/message-import-manager/drivers/gmail/utils/compute-gmail-category-label-id.util'; - -describe('computeGmailCategoryLabelId', () => { - it('should return correct category label id', () => { - const result = computeGmailCategoryLabelId('CATEGORY1'); - - expect(result).toBe('CATEGORY_CATEGORY1'); - }); -}); diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/utils/__tests__/compute-gmail-exclude-search-filter.spec.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/utils/__tests__/compute-gmail-exclude-search-filter.spec.ts new file mode 100644 index 0000000000..8fc24d95fa --- /dev/null +++ b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/utils/__tests__/compute-gmail-exclude-search-filter.spec.ts @@ -0,0 +1,69 @@ +import { computeGmailExcludeSearchFilter } from 'src/modules/messaging/message-import-manager/drivers/gmail/utils/compute-gmail-exclude-search-filter.util'; + +describe('computeGmailExcludeSearchFilter', () => { + it('should return correct exclude search filter with empty label array', () => { + const result = computeGmailExcludeSearchFilter([]); + + expect(result).toBe(''); + }); + + it('should return correct exclude search filter with one label', () => { + const result = computeGmailExcludeSearchFilter([ + { + externalId: 'LABEL1', + isSynced: false, + }, + ]); + + expect(result).toBe('-label:LABEL1'); + }); + + it('should return correct exclude search filter with multiple categories', () => { + const result = computeGmailExcludeSearchFilter([ + { + externalId: 'LABEL1', + isSynced: false, + }, + { + externalId: 'LABEL2', + isSynced: false, + }, + { + externalId: 'LABEL3', + isSynced: false, + }, + ]); + + expect(result).toBe('-label:LABEL1 -label:LABEL2 -label:LABEL3'); + }); + + it('should return correct exclude search filter with one label that is synced', () => { + const result = computeGmailExcludeSearchFilter([ + { + externalId: 'LABEL1', + isSynced: true, + }, + ]); + + expect(result).toBe(''); + }); + + it('should return correct exclude search filter with multiple categories that are synced', () => { + const result = computeGmailExcludeSearchFilter([ + { + externalId: 'LABEL1', + isSynced: true, + }, + { + externalId: 'LABEL2', + isSynced: false, + }, + { + externalId: 'LABEL3', + isSynced: true, + }, + ]); + + expect(result).toBe('-label:LABEL2'); + }); +}); diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/utils/compute-gmail-category-excude-search-filter.util.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/utils/compute-gmail-category-excude-search-filter.util.ts deleted file mode 100644 index e8deb1140d..0000000000 --- a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/utils/compute-gmail-category-excude-search-filter.util.ts +++ /dev/null @@ -1,3 +0,0 @@ -export const computeGmailCategoryExcludeSearchFilter = ( - excludedCategories: string[], -) => excludedCategories.map((category) => `-category:${category}`).join(' '); diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/utils/compute-gmail-category-label-id.util.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/utils/compute-gmail-category-label-id.util.ts deleted file mode 100644 index 7c3de6e2a6..0000000000 --- a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/utils/compute-gmail-category-label-id.util.ts +++ /dev/null @@ -1,2 +0,0 @@ -export const computeGmailCategoryLabelId = (category: string) => - `CATEGORY_${category.toUpperCase()}`; diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/utils/compute-gmail-exclude-search-filter.util.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/utils/compute-gmail-exclude-search-filter.util.ts new file mode 100644 index 0000000000..2af6e8dd01 --- /dev/null +++ b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/utils/compute-gmail-exclude-search-filter.util.ts @@ -0,0 +1,12 @@ +import { type MessageFolderWorkspaceEntity } from 'src/modules/messaging/common/standard-objects/message-folder.workspace-entity'; + +export const computeGmailExcludeSearchFilter = ( + messageFolders: Pick< + MessageFolderWorkspaceEntity, + 'externalId' | 'isSynced' + >[], +) => + messageFolders + .filter((folder) => !folder.isSynced) + .map((folder) => `-label:${folder.externalId}`) + .join(' '); diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/utils/map-gmail-default-folder-to-category.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/utils/map-gmail-default-folder-to-category.ts deleted file mode 100644 index 6f93cbb706..0000000000 --- a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/utils/map-gmail-default-folder-to-category.ts +++ /dev/null @@ -1,37 +0,0 @@ -import { isDefined } from 'twenty-shared/utils'; - -import { GmailDefaultMessageCategory } from 'src/modules/messaging/message-import-manager/drivers/gmail/types/gmail-default-message-category.type'; -import { GmailDefaultMessageFolder } from 'src/modules/messaging/message-import-manager/drivers/gmail/types/gmail-default-message-folder.type'; - -const DEFAULT_FOLDER_NAME_TO_CATEGORY_MAPPING = [ - { - folderName: GmailDefaultMessageFolder.CATEGORY_FORUMS, - category: GmailDefaultMessageCategory.forums, - }, - { - folderName: GmailDefaultMessageFolder.CATEGORY_PROMOTIONS, - category: GmailDefaultMessageCategory.promotions, - }, - { - folderName: GmailDefaultMessageFolder.CATEGORY_SOCIAL, - category: GmailDefaultMessageCategory.social, - }, - { - folderName: GmailDefaultMessageFolder.CATEGORY_UPDATES, - category: GmailDefaultMessageCategory.updates, - }, -]; - -export const mapGmailDefaultFolderToCategoryOrUndefined = ( - messageFolderName: string, -) => { - const mapping = DEFAULT_FOLDER_NAME_TO_CATEGORY_MAPPING.find( - ({ folderName }) => folderName === messageFolderName, - ); - - if (!isDefined(mapping)) { - return undefined; - } - - return mapping.category; -}; diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/imap/services/imap-get-message-list.service.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/imap/services/imap-get-message-list.service.ts index 288834d288..0fc0fa83e8 100644 --- a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/imap/services/imap-get-message-list.service.ts +++ b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/imap/services/imap-get-message-list.service.ts @@ -33,12 +33,15 @@ export class ImapGetMessageListService { messageFolders, }: GetMessageListsArgs): Promise { let client: ImapFlow | null = null; + const syncedMessageFolders = messageFolders.filter( + (folder) => folder.isSynced, + ); try { client = await this.imapClientProvider.getClient(connectedAccount); const result: GetMessageListsResponse = []; - for (const folder of messageFolders) { + for (const folder of syncedMessageFolders) { this.logger.log(`Processing folder: ${folder.name}`); try { @@ -76,7 +79,7 @@ export class ImapGetMessageListService { this.imapHandleErrorService.handleImapMessageListFetchError(error); - return messageFolders.map((folder) => ({ + return syncedMessageFolders.map((folder) => ({ messageExternalIds: [], nextSyncCursor: folder.syncCursor || '', previousSyncCursor: folder.syncCursor, diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-get-message-list.service.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-get-message-list.service.ts index 6f06a9151b..27277bca3c 100644 --- a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-get-message-list.service.ts +++ b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-get-message-list.service.ts @@ -47,7 +47,11 @@ export class MicrosoftGetMessageListService { ); } - for (const folder of messageFolders) { + const syncedMessageFolders = messageFolders.filter( + (folder) => folder.isSynced, + ); + + for (const folder of syncedMessageFolders) { const response = await this.getMessageList(connectedAccount, folder); result.push({ diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-message-list-fetch.service.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-message-list-fetch.service.ts index e3ea690644..75958a4a25 100644 --- a/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-message-list-fetch.service.ts +++ b/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-message-list-fetch.service.ts @@ -91,17 +91,16 @@ export class MessagingMessageListFetchService { 'messageFolder', ); - const messageFoldersToSync = await messageFolderRepository.find({ + const messageFolders = await messageFolderRepository.find({ where: { messageChannelId: messageChannel.id, - isSynced: true, }, }); const messageLists = await this.messagingGetMessageListService.getMessageLists( messageChannelWithFreshTokens, - messageFoldersToSync, + messageFolders, ); await this.cacheStorage.del( @@ -124,7 +123,11 @@ export class MessagingMessageListFetchService { let totalMessagesToImportCount = 0; this.logger.log( - `messageChannelId: ${messageChannel.id} Is full sync: ${isFullSync} and toImportCount: ${messageExternalIds.length}, toDeleteCount: ${messageExternalIdsToDelete.length}`, + `messageChannelId: ${messageChannel.id} Is full sync: ${isFullSync} and toImportCount: ${messageExternalIds.length}, toDeleteCount: ${messageExternalIdsToDelete.length}, cursors: ${messageLists.map( + (messageList) => { + messageList.nextSyncCursor; + }, + )}`, ); const messageChannelMessageAssociationRepository = @@ -207,26 +210,22 @@ export class MessagingMessageListFetchService { const toDeleteChunks = chunk(allMessageExternalIdsToDelete, 200); for (const [index, toDeleteChunk] of toDeleteChunks.entries()) { - await messageChannelMessageAssociationRepository.delete({ - messageChannelId: messageChannelWithFreshTokens.id, - messageExternalId: In(toDeleteChunk), - }); - this.logger.log( - `messageChannelId: ${messageChannel.id} Deleted ${toDeleteChunk.length} message channel message associations in batch ${index + 1}`, + `messageChannelId: ${messageChannel.id} Deleting ${toDeleteChunk.length} message channel message associations in batch ${index + 1}`, + ); + + await this.messagingMessageCleanerService.deleteMessagesChannelMessageAssociationsAndRelatedOrphans( + { + workspaceId, + messageExternalIds: toDeleteChunk.filter((messageExternalId) => + isNonEmptyString(messageExternalId), + ), + messageChannelId: messageChannelWithFreshTokens.id, + }, ); } } - this.logger.log( - `messageChannelId: ${messageChannel.id} launching workspace thread cleanup`, - ); - - // TODO: Re-enable this after we have a way to clean up threads without impacting the message import - // await this.messagingMessageCleanerService.cleanWorkspaceThreads( - // workspaceId, - // ); - this.logger.log( `messageChannelId: ${messageChannel.id} Total messages to import count: ${totalMessagesToImportCount}`, );