diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/services/gmail-get-message-list.service.spec.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/services/gmail-get-message-list.service.spec.ts index e71aefb734..50eb76521d 100644 --- a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/services/gmail-get-message-list.service.spec.ts +++ b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/services/gmail-get-message-list.service.spec.ts @@ -3,11 +3,28 @@ import { Test, type TestingModule } from '@nestjs/testing'; import { google } from 'googleapis'; import { ConnectedAccountProvider } from 'twenty-shared/types'; +import { type MessageFolder } from 'src/modules/messaging/message-folder-manager/interfaces/message-folder-driver.interface'; + import { OAuth2ClientManagerService } from 'src/modules/connected-account/oauth2-client-manager/services/oauth2-client-manager.service'; import { type ConnectedAccountWorkspaceEntity } from 'src/modules/connected-account/standard-objects/connected-account.workspace-entity'; +import { MessageFolderImportPolicy } from 'src/modules/messaging/common/standard-objects/message-channel.workspace-entity'; +import { MessageFolderPendingSyncAction } from 'src/modules/messaging/common/standard-objects/message-folder.workspace-entity'; import { GmailGetHistoryService } from 'src/modules/messaging/message-import-manager/drivers/gmail/services/gmail-get-history.service'; import { GmailGetMessageListService } from 'src/modules/messaging/message-import-manager/drivers/gmail/services/gmail-get-message-list.service'; import { GmailMessageListFetchErrorHandler } from 'src/modules/messaging/message-import-manager/drivers/gmail/services/gmail-message-list-fetch-error-handler.service'; +import { computeGmailExcludeSearchFilter } from 'src/modules/messaging/message-import-manager/drivers/gmail/utils/compute-gmail-exclude-search-filter.util'; + +const createMockFolder = ( + overrides: Partial & + Pick, +): MessageFolder => ({ + id: `folder-${overrides.externalId}`, + syncCursor: null, + isSentFolder: false, + parentFolderId: null, + pendingSyncAction: MessageFolderPendingSyncAction.NONE, + ...overrides, +}); describe('GmailGetMessageListService', () => { let service: GmailGetMessageListService; @@ -25,7 +42,7 @@ describe('GmailGetMessageListService', () => { id: 'connected-account-id', provider: ConnectedAccountProvider.GOOGLE, accessToken: 'access-token', - refreshToken: 'refresh-token', // dummy value for testing + refreshToken: 'refresh-token', handle: 'test@gmail.com', connectionParameters: {}, }; @@ -90,9 +107,19 @@ describe('GmailGetMessageListService', () => { ).mockResolvedValue({}); const result = await service.getMessageLists({ - messageChannel: { syncCursor: '', id: 'my-id' }, + messageChannel: { + syncCursor: '', + id: 'my-id', + messageFolderImportPolicy: MessageFolderImportPolicy.SELECTED_FOLDERS, + }, connectedAccount: mockConnectedAccount, - messageFolders: [], + messageFolders: [ + createMockFolder({ + name: 'INBOX', + externalId: 'INBOX', + isSynced: true, + }), + ], }); expect(result[0].messageExternalIds).toHaveLength(0); @@ -143,9 +170,19 @@ describe('GmailGetMessageListService', () => { ).mockResolvedValue({}); const result = await service.getMessageLists({ - messageChannel: { syncCursor: '', id: 'my-id' }, + messageChannel: { + syncCursor: '', + id: 'my-id', + messageFolderImportPolicy: MessageFolderImportPolicy.SELECTED_FOLDERS, + }, connectedAccount: mockConnectedAccount, - messageFolders: [], + messageFolders: [ + createMockFolder({ + name: 'INBOX', + externalId: 'INBOX', + isSynced: true, + }), + ], }); expect(result[0].messageExternalIds).toHaveLength(5); @@ -196,9 +233,19 @@ describe('GmailGetMessageListService', () => { ).mockResolvedValue({}); const result = await service.getMessageLists({ - messageChannel: { syncCursor: '', id: 'my-id' }, + messageChannel: { + syncCursor: '', + id: 'my-id', + messageFolderImportPolicy: MessageFolderImportPolicy.SELECTED_FOLDERS, + }, connectedAccount: mockConnectedAccount, - messageFolders: [], + messageFolders: [ + createMockFolder({ + name: 'INBOX', + externalId: 'INBOX', + isSynced: true, + }), + ], }); expect(result[0].messageExternalIds).toHaveLength(3); @@ -230,13 +277,352 @@ describe('GmailGetMessageListService', () => { ).mockResolvedValue({}); const result = await service.getMessageLists({ - messageChannel: { syncCursor: '', id: 'my-id' }, + messageChannel: { + syncCursor: '', + id: 'my-id', + messageFolderImportPolicy: MessageFolderImportPolicy.SELECTED_FOLDERS, + }, connectedAccount: mockConnectedAccount, - messageFolders: [], + messageFolders: [ + createMockFolder({ + name: 'INBOX', + externalId: 'INBOX', + isSynced: true, + }), + ], }); expect(result[0].messageExternalIds).toHaveLength(0); expect(mockGmailClient.users.messages.list).toHaveBeenCalledTimes(1); }); + + it('should return empty array when no folders have isSynced=true with SELECTED_FOLDERS policy', async () => { + const mockGmailClient = { + users: { + messages: { + list: jest.fn(), + }, + }, + }; + + jest.spyOn(google, 'gmail').mockReturnValue(mockGmailClient as never); + + ( + oAuth2ClientManagerService.getGoogleOAuth2Client as jest.Mock + ).mockResolvedValue({}); + + const result = await service.getMessageLists({ + messageChannel: { + syncCursor: '', + id: 'my-id', + messageFolderImportPolicy: MessageFolderImportPolicy.SELECTED_FOLDERS, + }, + connectedAccount: mockConnectedAccount, + messageFolders: [ + createMockFolder({ + name: 'Personal', + externalId: 'Label_personal', + isSynced: false, + }), + createMockFolder({ + name: 'Work', + externalId: 'Label_work', + isSynced: false, + }), + ], + }); + + expect(result).toEqual([]); + expect(mockGmailClient.users.messages.list).not.toHaveBeenCalled(); + }); + }); + + describe('initial sync folder filtering', () => { + it('should build Gmail query with -label exclusions for non-synced folders', async () => { + const mockGmailClient = { + users: { + messages: { + list: jest.fn().mockResolvedValue({ + data: { messages: [{ id: 'msg-1' }], nextPageToken: undefined }, + }), + get: jest.fn().mockResolvedValue({ + data: { historyId: '12345' }, + }), + }, + }, + }; + + jest.spyOn(google, 'gmail').mockReturnValue(mockGmailClient as never); + ( + oAuth2ClientManagerService.getGoogleOAuth2Client as jest.Mock + ).mockResolvedValue({}); + + await service.getMessageLists({ + messageChannel: { + syncCursor: '', + id: 'channel-1', + messageFolderImportPolicy: MessageFolderImportPolicy.SELECTED_FOLDERS, + }, + connectedAccount: mockConnectedAccount, + messageFolders: [ + createMockFolder({ + name: 'INBOX', + externalId: 'INBOX', + isSynced: true, + }), + createMockFolder({ + name: 'Work', + externalId: 'Label_work', + isSynced: true, + }), + createMockFolder({ + name: 'Personal', + externalId: 'Label_personal', + isSynced: false, + }), + createMockFolder({ + name: 'Newsletters', + externalId: 'Label_newsletters', + isSynced: false, + }), + ], + }); + + const expectedQuery = computeGmailExcludeSearchFilter([ + { + externalId: 'Label_personal', + name: 'Personal', + isSynced: false, + parentFolderId: null, + }, + { + externalId: 'Label_newsletters', + name: 'Newsletters', + isSynced: false, + parentFolderId: null, + }, + ]); + + expect(mockGmailClient.users.messages.list).toHaveBeenCalledWith( + expect.objectContaining({ + q: expectedQuery, + }), + ); + }); + + it('should not include exclusion filter when ALL_FOLDERS policy is set', async () => { + const mockGmailClient = { + users: { + messages: { + list: jest.fn().mockResolvedValue({ + data: { messages: [{ id: 'msg-1' }], nextPageToken: undefined }, + }), + get: jest.fn().mockResolvedValue({ + data: { historyId: '12345' }, + }), + }, + }, + }; + + jest.spyOn(google, 'gmail').mockReturnValue(mockGmailClient as never); + ( + oAuth2ClientManagerService.getGoogleOAuth2Client as jest.Mock + ).mockResolvedValue({}); + + await service.getMessageLists({ + messageChannel: { + syncCursor: '', + id: 'channel-1', + messageFolderImportPolicy: MessageFolderImportPolicy.ALL_FOLDERS, + }, + connectedAccount: mockConnectedAccount, + messageFolders: [ + createMockFolder({ + name: 'INBOX', + externalId: 'INBOX', + isSynced: true, + }), + createMockFolder({ + name: 'Personal', + externalId: 'Label_personal', + isSynced: false, + }), + ], + }); + + expect(mockGmailClient.users.messages.list).toHaveBeenCalledWith( + expect.objectContaining({ q: '' }), + ); + }); + }); + + describe('incremental sync folder filtering', () => { + it('should filter out messages from disabled folders during incremental sync', async () => { + const mockHistoryService = { + getHistory: jest.fn(), + getMessageIdsFromHistory: jest.fn(), + }; + + const module: TestingModule = await Test.createTestingModule({ + providers: [ + GmailGetMessageListService, + { + provide: OAuth2ClientManagerService, + useValue: { + getGoogleOAuth2Client: jest.fn().mockResolvedValue({}), + }, + }, + { + provide: GmailGetHistoryService, + useValue: mockHistoryService, + }, + { + provide: GmailMessageListFetchErrorHandler, + useValue: { handleError: jest.fn() }, + }, + ], + }).compile(); + + const testService = module.get( + GmailGetMessageListService, + ); + + mockHistoryService.getHistory.mockImplementation( + (_client, _cursor, _types, labelId) => { + if (labelId === 'Label_personal') { + return Promise.resolve({ + history: [ + { messagesAdded: [{ message: { id: 'personal-msg' } }] }, + ], + historyId: 'new-cursor', + }); + } + if (labelId === undefined) { + return Promise.resolve({ + history: [{ messagesAdded: [{ message: { id: 'inbox-msg' } }] }], + historyId: 'new-cursor', + }); + } + + return Promise.resolve({ history: [], historyId: 'new-cursor' }); + }, + ); + + mockHistoryService.getMessageIdsFromHistory.mockResolvedValue({ + messagesAdded: ['inbox-msg', 'personal-msg'], + messagesDeleted: [], + }); + + jest.spyOn(google, 'gmail').mockReturnValue({} as never); + + const result = await testService.getMessageLists({ + messageChannel: { + syncCursor: 'old-cursor', + id: 'channel-1', + messageFolderImportPolicy: MessageFolderImportPolicy.SELECTED_FOLDERS, + }, + connectedAccount: mockConnectedAccount, + messageFolders: [ + createMockFolder({ + name: 'INBOX', + externalId: 'INBOX', + isSynced: true, + }), + createMockFolder({ + name: 'Personal', + externalId: 'Label_personal', + isSynced: false, + }), + ], + }); + + expect(result[0].messageExternalIds).toEqual(['inbox-msg']); + + const allHistoryCalls = mockHistoryService.getHistory.mock.calls; + + expect(allHistoryCalls[0]).toHaveLength(2); + expect(allHistoryCalls[0][1]).toBe('old-cursor'); + + const labelIdsQueried = allHistoryCalls + .slice(1) + .map((call) => call[3]) + .filter(Boolean); + + expect(labelIdsQueried).toContain('Label_personal'); + expect(labelIdsQueried).toHaveLength(1); + + // 1 main history call + 1 excluded folder call + expect(mockHistoryService.getHistory).toHaveBeenCalledTimes(2); + }); + + it('should skip per-folder filtering when ALL_FOLDERS policy is set', async () => { + const mockHistoryService = { + getHistory: jest.fn(), + getMessageIdsFromHistory: jest.fn(), + }; + + const module: TestingModule = await Test.createTestingModule({ + providers: [ + GmailGetMessageListService, + { + provide: OAuth2ClientManagerService, + useValue: { + getGoogleOAuth2Client: jest.fn().mockResolvedValue({}), + }, + }, + { + provide: GmailGetHistoryService, + useValue: mockHistoryService, + }, + { + provide: GmailMessageListFetchErrorHandler, + useValue: { handleError: jest.fn() }, + }, + ], + }).compile(); + + const testService = module.get( + GmailGetMessageListService, + ); + + mockHistoryService.getHistory.mockResolvedValue({ + history: [], + historyId: 'new-cursor', + }); + + mockHistoryService.getMessageIdsFromHistory.mockResolvedValue({ + messagesAdded: ['inbox-msg', 'personal-msg'], + messagesDeleted: [], + }); + + jest.spyOn(google, 'gmail').mockReturnValue({} as never); + + const result = await testService.getMessageLists({ + messageChannel: { + syncCursor: 'old-cursor', + id: 'channel-1', + messageFolderImportPolicy: MessageFolderImportPolicy.ALL_FOLDERS, + }, + connectedAccount: mockConnectedAccount, + messageFolders: [ + createMockFolder({ + name: 'INBOX', + externalId: 'INBOX', + isSynced: true, + }), + createMockFolder({ + name: 'Personal', + externalId: 'Label_personal', + isSynced: false, + }), + ], + }); + + expect(result[0].messageExternalIds).toEqual([ + 'inbox-msg', + 'personal-msg', + ]); + expect(mockHistoryService.getHistory).toHaveBeenCalledTimes(1); + }); }); }); diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/services/gmail-get-message-list.service.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/services/gmail-get-message-list.service.ts index 833ed0b190..bb5c6c6d4f 100644 --- a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/services/gmail-get-message-list.service.ts +++ b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/services/gmail-get-message-list.service.ts @@ -7,6 +7,10 @@ import { isDefined } from 'twenty-shared/utils'; import { OAuth2ClientManagerService } from 'src/modules/connected-account/oauth2-client-manager/services/oauth2-client-manager.service'; import { type ConnectedAccountWorkspaceEntity } from 'src/modules/connected-account/standard-objects/connected-account.workspace-entity'; +import { + MessageChannelWorkspaceEntity, + MessageFolderImportPolicy, +} from 'src/modules/messaging/common/standard-objects/message-channel.workspace-entity'; import { type MessageFolderWorkspaceEntity } from 'src/modules/messaging/common/standard-objects/message-folder.workspace-entity'; import { MessageImportDriverException, @@ -38,8 +42,12 @@ export class GmailGetMessageListService { >, messageFolders: Pick< MessageFolderWorkspaceEntity, - 'name' | 'externalId' | 'isSynced' + 'name' | 'externalId' | 'isSynced' | 'parentFolderId' >[], + messageChannel: Pick< + MessageChannelWorkspaceEntity, + 'messageFolderImportPolicy' + >, ): Promise { const oAuth2Client = await this.oAuth2ClientManagerService.getGoogleOAuth2Client( @@ -56,7 +64,10 @@ export class GmailGetMessageListService { const messageExternalIds: string[] = []; const excludedSearchFilter = - computeGmailExcludeSearchFilter(messageFolders); + messageChannel.messageFolderImportPolicy === + MessageFolderImportPolicy.SELECTED_FOLDERS + ? computeGmailExcludeSearchFilter(messageFolders) + : ''; while (hasMoreMessages) { const messageList = await gmailClient.users.messages @@ -145,6 +156,21 @@ export class GmailGetMessageListService { connectedAccount, messageFolders, }: GetMessageListsArgs): Promise { + if ( + messageChannel.messageFolderImportPolicy === + MessageFolderImportPolicy.SELECTED_FOLDERS + ) { + const foldersToSync = messageFolders.filter((folder) => folder.isSynced); + + if (foldersToSync.length === 0) { + this.logger.warn( + `Connected account ${connectedAccount.id} Message Channel: ${messageChannel.id}: No folders to process`, + ); + + return []; + } + } + const oAuth2Client = await this.oAuth2ClientManagerService.getGoogleOAuth2Client( connectedAccount, @@ -155,7 +181,11 @@ export class GmailGetMessageListService { }); if (!isNonEmptyString(messageChannel.syncCursor)) { - return this.getMessageListWithoutCursor(connectedAccount, messageFolders); + return this.getMessageListWithoutCursor( + connectedAccount, + messageFolders, + messageChannel, + ); } const { history, historyId: nextSyncCursor } = @@ -167,11 +197,15 @@ export class GmailGetMessageListService { const { messagesAdded, messagesDeleted } = await this.gmailGetHistoryService.getMessageIdsFromHistory(history); - const messageIdsToFilter = await this.getEmailIdsFromExcludedFolders( - connectedAccount, - messageChannel.syncCursor, - messageFolders, - ); + const messageIdsToFilter = + messageChannel.messageFolderImportPolicy === + MessageFolderImportPolicy.SELECTED_FOLDERS + ? await this.getEmailIdsFromExcludedFolders( + connectedAccount, + messageChannel.syncCursor, + messageFolders, + ) + : []; const messagesAddedFiltered = messagesAdded.filter( (messageId) => !messageIdsToFilter.includes(messageId), diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/utils/__tests__/build-gmail-label-search-name.spec.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/utils/__tests__/build-gmail-label-search-name.spec.ts new file mode 100644 index 0000000000..d7e037c7ae --- /dev/null +++ b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/utils/__tests__/build-gmail-label-search-name.spec.ts @@ -0,0 +1,60 @@ +import { buildGmailLabelSearchName } from 'src/modules/messaging/message-import-manager/drivers/gmail/utils/build-gmail-label-search-name.util'; + +describe('buildGmailLabelSearchName', () => { + it('should return null for folder without name', () => { + const result = buildGmailLabelSearchName( + { externalId: 'Label_1', name: '', parentFolderId: null }, + [], + ); + + expect(result).toBeNull(); + }); + + it('should convert spaces to hyphens and lowercase', () => { + const folder = { + externalId: 'Label_1', + name: 'My Custom Folder', + parentFolderId: null, + }; + + const result = buildGmailLabelSearchName(folder, [folder]); + + expect(result).toBe('my-custom-folder'); + }); + + it('should build full path for nested folder', () => { + const parent = { + externalId: 'Label_parent', + name: 'Work', + parentFolderId: null, + }; + const child = { + externalId: 'Label_child', + name: 'Projects', + parentFolderId: 'Label_parent', + }; + const allFolders = [parent, child]; + + const result = buildGmailLabelSearchName(child, allFolders); + + expect(result).toBe('work-projects'); + }); + + it('should handle deeply nested folders', () => { + const grandparent = { + externalId: 'L1', + name: 'Work', + parentFolderId: null, + }; + const parent = { externalId: 'L2', name: 'Projects', parentFolderId: 'L1' }; + const child = { externalId: 'L3', name: 'Active', parentFolderId: 'L2' }; + + const result = buildGmailLabelSearchName(child, [ + grandparent, + parent, + child, + ]); + + expect(result).toBe('work-projects-active'); + }); +}); diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/utils/__tests__/compute-gmail-exclude-search-filter.spec.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/utils/__tests__/compute-gmail-exclude-search-filter.spec.ts index 8fc24d95fa..81713df200 100644 --- a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/utils/__tests__/compute-gmail-exclude-search-filter.spec.ts +++ b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/utils/__tests__/compute-gmail-exclude-search-filter.spec.ts @@ -1,69 +1,146 @@ import { computeGmailExcludeSearchFilter } from 'src/modules/messaging/message-import-manager/drivers/gmail/utils/compute-gmail-exclude-search-filter.util'; describe('computeGmailExcludeSearchFilter', () => { - it('should return correct exclude search filter with empty label array', () => { + it('should return empty string with empty folder array', () => { const result = computeGmailExcludeSearchFilter([]); expect(result).toBe(''); }); - it('should return correct exclude search filter with one label', () => { + it('should return correct exclude filter for one unsynced folder', () => { const result = computeGmailExcludeSearchFilter([ { - externalId: 'LABEL1', + externalId: 'Label_123', + name: 'Custom Folder', isSynced: false, + parentFolderId: null, }, ]); - expect(result).toBe('-label:LABEL1'); + expect(result).toBe('-label:custom-folder'); }); - it('should return correct exclude search filter with multiple categories', () => { + it('should return correct exclude filter for multiple unsynced folders', () => { const result = computeGmailExcludeSearchFilter([ { - externalId: 'LABEL1', + externalId: 'Label_1', + name: 'Folder One', isSynced: false, + parentFolderId: null, }, { - externalId: 'LABEL2', - isSynced: false, - }, - { - externalId: 'LABEL3', + externalId: 'Label_2', + name: 'Folder Two', isSynced: false, + parentFolderId: null, }, ]); - expect(result).toBe('-label:LABEL1 -label:LABEL2 -label:LABEL3'); + expect(result).toBe('-label:folder-one -label:folder-two'); }); - it('should return correct exclude search filter with one label that is synced', () => { + it('should return empty string when all folders are synced', () => { const result = computeGmailExcludeSearchFilter([ { - externalId: 'LABEL1', + externalId: 'Label_1', + name: 'Synced Folder', isSynced: true, + parentFolderId: null, }, ]); expect(result).toBe(''); }); - it('should return correct exclude search filter with multiple categories that are synced', () => { + it('should only exclude unsynced folders', () => { const result = computeGmailExcludeSearchFilter([ { - externalId: 'LABEL1', + externalId: 'Label_1', + name: 'Synced', isSynced: true, + parentFolderId: null, }, { - externalId: 'LABEL2', + externalId: 'Label_2', + name: 'Not Synced', isSynced: false, + parentFolderId: null, }, { - externalId: 'LABEL3', + externalId: 'Label_3', + name: 'Also Synced', isSynced: true, + parentFolderId: null, }, ]); - expect(result).toBe('-label:LABEL2'); + expect(result).toBe('-label:not-synced'); + }); + + it('should handle nested folders with parent path', () => { + const folders = [ + { + externalId: 'Label_parent', + name: 'Parent Folder', + isSynced: true, + parentFolderId: null, + }, + { + externalId: 'Label_child', + name: 'Child Folder', + isSynced: false, + parentFolderId: 'Label_parent', + }, + ]; + + const result = computeGmailExcludeSearchFilter(folders); + + expect(result).toBe('-label:parent-folder-child-folder'); + }); + + it('should handle deeply nested folders', () => { + const folders = [ + { + externalId: 'Label_grandparent', + name: 'Level One', + isSynced: true, + parentFolderId: null, + }, + { + externalId: 'Label_parent', + name: 'Level Two', + isSynced: true, + parentFolderId: 'Label_grandparent', + }, + { + externalId: 'Label_child', + name: 'Level Three', + isSynced: false, + parentFolderId: 'Label_parent', + }, + ]; + + const result = computeGmailExcludeSearchFilter(folders); + + expect(result).toBe('-label:level-one-level-two-level-three'); + }); + + it('should skip folders without names', () => { + const result = computeGmailExcludeSearchFilter([ + { + externalId: 'Label_1', + name: '', + isSynced: false, + parentFolderId: null, + }, + { + externalId: 'Label_2', + name: 'Valid Folder', + isSynced: false, + parentFolderId: null, + }, + ]); + + expect(result).toBe('-label:valid-folder'); }); }); diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/utils/build-gmail-label-search-name.util.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/utils/build-gmail-label-search-name.util.ts new file mode 100644 index 0000000000..10535d1cc4 --- /dev/null +++ b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/utils/build-gmail-label-search-name.util.ts @@ -0,0 +1,44 @@ +import { type MessageFolderWorkspaceEntity } from 'src/modules/messaging/common/standard-objects/message-folder.workspace-entity'; + +const MAXIMUM_GMAIL_FOLDER_DEPTH = 50; + +type FolderInput = Pick< + MessageFolderWorkspaceEntity, + 'externalId' | 'name' | 'parentFolderId' +>; + +export const buildGmailLabelSearchName = ( + folder: FolderInput, + allFolders: FolderInput[], +): string | null => { + if (!folder.name) { + return null; + } + + const folderMap = new Map( + allFolders + .filter((folder) => folder.externalId) + .map((folder) => [folder.externalId, folder]), + ); + + const pathParts: string[] = []; + let current: FolderInput | undefined = folder; + let depth = 0; + + while (current?.name && depth < MAXIMUM_GMAIL_FOLDER_DEPTH) { + pathParts.unshift(current.name); + current = current.parentFolderId + ? folderMap.get(current.parentFolderId) + : undefined; + depth++; + } + + if (depth >= MAXIMUM_GMAIL_FOLDER_DEPTH) { + return null; + } + + return pathParts + .join('/') + .replace(/[\s/]+/g, '-') + .toLowerCase(); +}; diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/utils/compute-gmail-exclude-search-filter.util.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/utils/compute-gmail-exclude-search-filter.util.ts index 2af6e8dd01..05d4403094 100644 --- a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/utils/compute-gmail-exclude-search-filter.util.ts +++ b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/utils/compute-gmail-exclude-search-filter.util.ts @@ -1,12 +1,17 @@ +import { isDefined } from 'twenty-shared/utils'; + import { type MessageFolderWorkspaceEntity } from 'src/modules/messaging/common/standard-objects/message-folder.workspace-entity'; +import { buildGmailLabelSearchName } from 'src/modules/messaging/message-import-manager/drivers/gmail/utils/build-gmail-label-search-name.util'; export const computeGmailExcludeSearchFilter = ( messageFolders: Pick< MessageFolderWorkspaceEntity, - 'externalId' | 'isSynced' + 'externalId' | 'isSynced' | 'name' | 'parentFolderId' >[], ) => messageFolders .filter((folder) => !folder.isSynced) - .map((folder) => `-label:${folder.externalId}`) + .map((folder) => buildGmailLabelSearchName(folder, messageFolders)) + .filter(isDefined) + .map((name) => `-label:${name}`) .join(' '); diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/imap/services/imap-get-message-list.service.spec.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/imap/services/imap-get-message-list.service.spec.ts new file mode 100644 index 0000000000..a2d89963b6 --- /dev/null +++ b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/imap/services/imap-get-message-list.service.spec.ts @@ -0,0 +1,227 @@ +import { Test, type TestingModule } from '@nestjs/testing'; + +import { ConnectedAccountProvider } from 'twenty-shared/types'; + +import { type MessageFolder } from 'src/modules/messaging/message-folder-manager/interfaces/message-folder-driver.interface'; + +import { type ConnectedAccountWorkspaceEntity } from 'src/modules/connected-account/standard-objects/connected-account.workspace-entity'; +import { MessageFolderImportPolicy } from 'src/modules/messaging/common/standard-objects/message-channel.workspace-entity'; +import { MessageFolderPendingSyncAction } from 'src/modules/messaging/common/standard-objects/message-folder.workspace-entity'; +import { ImapClientProvider } from 'src/modules/messaging/message-import-manager/drivers/imap/providers/imap-client.provider'; +import { ImapGetMessageListService } from 'src/modules/messaging/message-import-manager/drivers/imap/services/imap-get-message-list.service'; +import { ImapMessageListFetchErrorHandler } from 'src/modules/messaging/message-import-manager/drivers/imap/services/imap-message-list-fetch-error-handler.service'; +import { ImapSyncService } from 'src/modules/messaging/message-import-manager/drivers/imap/services/imap-sync.service'; + +const createMockFolder = ( + overrides: Partial & + Pick, +): MessageFolder => ({ + id: `folder-${overrides.externalId}`, + syncCursor: null, + isSentFolder: false, + parentFolderId: null, + pendingSyncAction: MessageFolderPendingSyncAction.NONE, + ...overrides, +}); + +describe('ImapGetMessageListService', () => { + let service: ImapGetMessageListService; + let imapClientProvider: ImapClientProvider; + + const mockConnectedAccount: Pick< + ConnectedAccountWorkspaceEntity, + | 'provider' + | 'accessToken' + | 'refreshToken' + | 'id' + | 'handle' + | 'connectionParameters' + > = { + id: 'connected-account-id', + provider: ConnectedAccountProvider.IMAP_SMTP_CALDAV, + accessToken: 'access-token', + refreshToken: 'refresh-token', + handle: 'test@example.com', + connectionParameters: {}, + }; + + const mockImapClient = { + getMailboxLock: jest.fn().mockResolvedValue({ release: jest.fn() }), + mailbox: { + uidValidity: 12345, + uidNext: 100, + highestModseq: '1000', + }, + capabilities: new Set(['CONDSTORE']), + status: jest.fn().mockResolvedValue({ + uidValidity: 12345, + uidNext: 100, + highestModseq: '1000', + }), + }; + + beforeEach(async () => { + const module: TestingModule = await Test.createTestingModule({ + providers: [ + ImapGetMessageListService, + { + provide: ImapClientProvider, + useValue: { + getClient: jest.fn().mockResolvedValue(mockImapClient), + closeClient: jest.fn().mockResolvedValue(undefined), + }, + }, + { + provide: ImapSyncService, + useValue: { + syncFolder: jest.fn().mockResolvedValue({ messageUids: [1, 2, 3] }), + }, + }, + { + provide: ImapMessageListFetchErrorHandler, + useValue: { + handleError: jest.fn(), + }, + }, + ], + }).compile(); + + service = module.get(ImapGetMessageListService); + imapClientProvider = module.get(ImapClientProvider); + }); + + afterEach(() => { + jest.clearAllMocks(); + }); + + describe('folder filtering based on import policy', () => { + it('should only process synced folders when SELECTED_FOLDERS policy is set', async () => { + const syncedFolder = createMockFolder({ + name: 'INBOX', + externalId: 'INBOX:1', + isSynced: true, + }); + + const nonSyncedFolder = createMockFolder({ + name: 'Personal', + externalId: 'Personal:1', + isSynced: false, + }); + + const result = await service.getMessageLists({ + connectedAccount: mockConnectedAccount, + messageChannel: { + syncCursor: '', + id: 'channel-1', + messageFolderImportPolicy: MessageFolderImportPolicy.SELECTED_FOLDERS, + }, + messageFolders: [syncedFolder, nonSyncedFolder], + }); + + expect(result).toHaveLength(1); + expect(result[0].folderId).toBe(syncedFolder.id); + }); + + it('should process all folders when ALL_FOLDERS policy is set', async () => { + const syncedFolder = createMockFolder({ + name: 'INBOX', + externalId: 'INBOX:1', + isSynced: true, + }); + + const nonSyncedFolder = createMockFolder({ + name: 'Personal', + externalId: 'Personal:1', + isSynced: false, + }); + + const result = await service.getMessageLists({ + connectedAccount: mockConnectedAccount, + messageChannel: { + syncCursor: '', + id: 'channel-1', + messageFolderImportPolicy: MessageFolderImportPolicy.ALL_FOLDERS, + }, + messageFolders: [syncedFolder, nonSyncedFolder], + }); + + expect(result).toHaveLength(2); + expect(result.map((r) => r.folderId)).toEqual([ + syncedFolder.id, + nonSyncedFolder.id, + ]); + }); + + it('should return empty array when SELECTED_FOLDERS policy and no folders are synced', async () => { + const nonSyncedFolder1 = createMockFolder({ + name: 'Personal', + externalId: 'Personal:1', + isSynced: false, + }); + + const nonSyncedFolder2 = createMockFolder({ + name: 'Work', + externalId: 'Work:1', + isSynced: false, + }); + + const result = await service.getMessageLists({ + connectedAccount: mockConnectedAccount, + messageChannel: { + syncCursor: '', + id: 'channel-1', + messageFolderImportPolicy: MessageFolderImportPolicy.SELECTED_FOLDERS, + }, + messageFolders: [nonSyncedFolder1, nonSyncedFolder2], + }); + + expect(result).toHaveLength(0); + }); + + it('should process all non-synced folders when ALL_FOLDERS policy is set', async () => { + const nonSyncedFolder1 = createMockFolder({ + name: 'Personal', + externalId: 'Personal:1', + isSynced: false, + }); + + const nonSyncedFolder2 = createMockFolder({ + name: 'Work', + externalId: 'Work:1', + isSynced: false, + }); + + const result = await service.getMessageLists({ + connectedAccount: mockConnectedAccount, + messageChannel: { + syncCursor: '', + id: 'channel-1', + messageFolderImportPolicy: MessageFolderImportPolicy.ALL_FOLDERS, + }, + messageFolders: [nonSyncedFolder1, nonSyncedFolder2], + }); + + expect(result).toHaveLength(2); + }); + + it('should always close the IMAP client regardless of policy', async () => { + const folder = createMockFolder({ + name: 'INBOX', + externalId: 'INBOX:1', + isSynced: true, + }); + + await service.getMessageLists({ + connectedAccount: mockConnectedAccount, + messageChannel: { + syncCursor: '', + id: 'channel-1', + messageFolderImportPolicy: MessageFolderImportPolicy.ALL_FOLDERS, + }, + messageFolders: [folder], + }); + + expect(imapClientProvider.closeClient).toHaveBeenCalledTimes(1); + }); + }); +}); diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/imap/services/imap-get-message-list.service.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/imap/services/imap-get-message-list.service.ts index a4698cd5a9..51ebd36883 100644 --- a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/imap/services/imap-get-message-list.service.ts +++ b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/imap/services/imap-get-message-list.service.ts @@ -5,6 +5,7 @@ import { isDefined } from 'twenty-shared/utils'; import { MessageFolder } from 'src/modules/messaging/message-folder-manager/interfaces/message-folder-driver.interface'; +import { MessageFolderImportPolicy } from 'src/modules/messaging/common/standard-objects/message-channel.workspace-entity'; import { MessageImportDriverException, MessageImportDriverExceptionCode, @@ -35,13 +36,28 @@ export class ImapGetMessageListService { async getMessageLists({ connectedAccount, messageFolders, + messageChannel, }: GetMessageListsArgs): Promise { + const foldersToProcess = + messageChannel.messageFolderImportPolicy === + MessageFolderImportPolicy.SELECTED_FOLDERS + ? messageFolders.filter((folder) => folder.isSynced) + : messageFolders; + + if (foldersToProcess.length === 0) { + this.logger.warn( + `Connected account ${connectedAccount.id}: No folders to process`, + ); + + return []; + } + const client = await this.imapClientProvider.getClient(connectedAccount); try { const results: GetMessageListsResponse = []; - for (const folder of messageFolders) { + for (const folder of foldersToProcess) { const response = await this.getMessageList(client, folder); results.push({ ...response, folderId: folder.id }); diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-get-message-list.service.dev.spec.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-get-message-list.service.dev.spec.ts index 2fc97bf457..e857f35b03 100644 --- a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-get-message-list.service.dev.spec.ts +++ b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-get-message-list.service.dev.spec.ts @@ -7,7 +7,10 @@ import { TwentyConfigModule } from 'src/engine/core-modules/twenty-config/twenty import { MicrosoftOAuth2ClientManagerService } from 'src/modules/connected-account/oauth2-client-manager/drivers/microsoft/microsoft-oauth2-client-manager.service'; import { OAuth2ClientManagerService } from 'src/modules/connected-account/oauth2-client-manager/services/oauth2-client-manager.service'; import { type ConnectedAccountWorkspaceEntity } from 'src/modules/connected-account/standard-objects/connected-account.workspace-entity'; -import { MessageChannelWorkspaceEntity } from 'src/modules/messaging/common/standard-objects/message-channel.workspace-entity'; +import { + MessageChannelWorkspaceEntity, + MessageFolderImportPolicy, +} from 'src/modules/messaging/common/standard-objects/message-channel.workspace-entity'; import { MessageFolderPendingSyncAction, MessageFolderWorkspaceEntity, @@ -41,10 +44,11 @@ const mockConnectedAccount: Pick< const mockMessageChannel: Pick< MessageChannelWorkspaceEntity, - 'id' | 'syncCursor' + 'id' | 'syncCursor' | 'messageFolderImportPolicy' > = { id: 'message-channel-id', syncCursor: '', // Should be empty for Microsoft as cursors are stored at the folder level + messageFolderImportPolicy: MessageFolderImportPolicy.SELECTED_FOLDERS, }; xdescribe('Microsoft dev tests : get message list service', () => { @@ -149,6 +153,7 @@ xdescribe('Microsoft dev tests : get message list service', () => { messageChannel: { id: 'message-channel-id', syncCursor: '', + messageFolderImportPolicy: MessageFolderImportPolicy.SELECTED_FOLDERS, }, connectedAccount: mockConnectedAccount, messageFolders: [ @@ -202,18 +207,24 @@ xdescribe('Microsoft dev tests : get message list service for folders', () => { messageChannelNoFolders.id = 'message-channel-0'; messageChannelNoFolders.messageFolders = []; messageChannelNoFolders.syncCursor = ''; + messageChannelNoFolders.messageFolderImportPolicy = + MessageFolderImportPolicy.SELECTED_FOLDERS; const messageChannelMicrosoftOneFolder = new MessageChannelWorkspaceEntity(); messageChannelMicrosoftOneFolder.id = 'message-channel-1'; messageChannelMicrosoftOneFolder.messageFolders = [inboxFolder]; messageChannelMicrosoftOneFolder.syncCursor = ''; + messageChannelMicrosoftOneFolder.messageFolderImportPolicy = + MessageFolderImportPolicy.SELECTED_FOLDERS; const messageChannelMicrosoft = new MessageChannelWorkspaceEntity(); messageChannelMicrosoft.id = 'message-channel-2'; messageChannelMicrosoft.messageFolders = [inboxFolder, sentFolder]; messageChannelMicrosoft.syncCursor = ''; + messageChannelMicrosoft.messageFolderImportPolicy = + MessageFolderImportPolicy.SELECTED_FOLDERS; beforeEach(async () => { const module: TestingModule = await Test.createTestingModule({ diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-get-message-list.service.spec.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-get-message-list.service.spec.ts new file mode 100644 index 0000000000..ec52e19be9 --- /dev/null +++ b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-get-message-list.service.spec.ts @@ -0,0 +1,233 @@ +import { Test, type TestingModule } from '@nestjs/testing'; + +import { ConnectedAccountProvider } from 'twenty-shared/types'; + +import { type MessageFolder } from 'src/modules/messaging/message-folder-manager/interfaces/message-folder-driver.interface'; + +import { OAuth2ClientManagerService } from 'src/modules/connected-account/oauth2-client-manager/services/oauth2-client-manager.service'; +import { type ConnectedAccountWorkspaceEntity } from 'src/modules/connected-account/standard-objects/connected-account.workspace-entity'; +import { MessageFolderImportPolicy } from 'src/modules/messaging/common/standard-objects/message-channel.workspace-entity'; +import { MessageFolderPendingSyncAction } from 'src/modules/messaging/common/standard-objects/message-folder.workspace-entity'; +import { MicrosoftGetMessageListService } from 'src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-get-message-list.service'; +import { MicrosoftMessageListFetchErrorHandler } from 'src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-message-list-fetch-error-handler.service'; + +const createMockFolder = ( + overrides: Partial & + Pick, +): MessageFolder => ({ + id: `folder-${overrides.externalId}`, + syncCursor: null, + isSentFolder: false, + parentFolderId: null, + pendingSyncAction: MessageFolderPendingSyncAction.NONE, + ...overrides, +}); + +describe('MicrosoftGetMessageListService', () => { + let service: MicrosoftGetMessageListService; + let oAuth2ClientManagerService: OAuth2ClientManagerService; + + const mockConnectedAccount: Pick< + ConnectedAccountWorkspaceEntity, + | 'provider' + | 'accessToken' + | 'refreshToken' + | 'id' + | 'handle' + | 'connectionParameters' + > = { + id: 'connected-account-id', + provider: ConnectedAccountProvider.MICROSOFT, + accessToken: 'access-token', + refreshToken: 'refresh-token', + handle: 'test@outlook.com', + connectionParameters: {}, + }; + + const createMockMicrosoftClient = () => ({ + api: jest.fn().mockReturnThis(), + version: jest.fn().mockReturnThis(), + headers: jest.fn().mockReturnThis(), + get: jest.fn().mockResolvedValue({ + value: [{ id: 'msg-1' }, { id: 'msg-2' }], + '@odata.deltaLink': 'https://graph.microsoft.com/delta?token=abc', + }), + }); + + beforeEach(async () => { + const module: TestingModule = await Test.createTestingModule({ + providers: [ + MicrosoftGetMessageListService, + { + provide: OAuth2ClientManagerService, + useValue: { + getMicrosoftOAuth2Client: jest.fn(), + }, + }, + { + provide: MicrosoftMessageListFetchErrorHandler, + useValue: { + handleError: jest.fn(), + }, + }, + ], + }).compile(); + + service = module.get( + MicrosoftGetMessageListService, + ); + oAuth2ClientManagerService = module.get( + OAuth2ClientManagerService, + ); + }); + + afterEach(() => { + jest.clearAllMocks(); + }); + + describe('folder filtering based on import policy', () => { + it('should only process synced folders when SELECTED_FOLDERS policy is set', async () => { + const mockClient = createMockMicrosoftClient(); + + ( + oAuth2ClientManagerService.getMicrosoftOAuth2Client as jest.Mock + ).mockResolvedValue(mockClient); + + const syncedFolder = createMockFolder({ + name: 'Inbox', + externalId: 'inbox-id', + isSynced: true, + }); + + const nonSyncedFolder = createMockFolder({ + name: 'Personal', + externalId: 'personal-id', + isSynced: false, + }); + + const result = await service.getMessageLists({ + connectedAccount: mockConnectedAccount, + messageChannel: { + syncCursor: '', + id: 'channel-1', + messageFolderImportPolicy: MessageFolderImportPolicy.SELECTED_FOLDERS, + }, + messageFolders: [syncedFolder, nonSyncedFolder], + }); + + expect(result).toHaveLength(1); + expect(result[0].folderId).toBe(syncedFolder.id); + expect( + oAuth2ClientManagerService.getMicrosoftOAuth2Client, + ).toHaveBeenCalledTimes(1); + }); + + it('should process all folders when ALL_FOLDERS policy is set', async () => { + const mockClient = createMockMicrosoftClient(); + + ( + oAuth2ClientManagerService.getMicrosoftOAuth2Client as jest.Mock + ).mockResolvedValue(mockClient); + + const syncedFolder = createMockFolder({ + name: 'Inbox', + externalId: 'inbox-id', + isSynced: true, + }); + + const nonSyncedFolder = createMockFolder({ + name: 'Personal', + externalId: 'personal-id', + isSynced: false, + }); + + const result = await service.getMessageLists({ + connectedAccount: mockConnectedAccount, + messageChannel: { + syncCursor: '', + id: 'channel-1', + messageFolderImportPolicy: MessageFolderImportPolicy.ALL_FOLDERS, + }, + messageFolders: [syncedFolder, nonSyncedFolder], + }); + + expect(result).toHaveLength(2); + expect(result.map((r) => r.folderId)).toEqual([ + syncedFolder.id, + nonSyncedFolder.id, + ]); + }); + + it('should return empty array when SELECTED_FOLDERS policy and no folders are synced', async () => { + const nonSyncedFolder1 = createMockFolder({ + name: 'Personal', + externalId: 'personal-id', + isSynced: false, + }); + + const nonSyncedFolder2 = createMockFolder({ + name: 'Work', + externalId: 'work-id', + isSynced: false, + }); + + const result = await service.getMessageLists({ + connectedAccount: mockConnectedAccount, + messageChannel: { + syncCursor: '', + id: 'channel-1', + messageFolderImportPolicy: MessageFolderImportPolicy.SELECTED_FOLDERS, + }, + messageFolders: [nonSyncedFolder1, nonSyncedFolder2], + }); + + expect(result).toEqual([]); + }); + + it('should process all non-synced folders when ALL_FOLDERS policy is set', async () => { + const mockClient = createMockMicrosoftClient(); + + ( + oAuth2ClientManagerService.getMicrosoftOAuth2Client as jest.Mock + ).mockResolvedValue(mockClient); + + const nonSyncedFolder1 = createMockFolder({ + name: 'Personal', + externalId: 'personal-id', + isSynced: false, + }); + + const nonSyncedFolder2 = createMockFolder({ + name: 'Work', + externalId: 'work-id', + isSynced: false, + }); + + const result = await service.getMessageLists({ + connectedAccount: mockConnectedAccount, + messageChannel: { + syncCursor: '', + id: 'channel-1', + messageFolderImportPolicy: MessageFolderImportPolicy.ALL_FOLDERS, + }, + messageFolders: [nonSyncedFolder1, nonSyncedFolder2], + }); + + expect(result).toHaveLength(2); + }); + + it('should return empty array when ALL_FOLDERS policy but messageFolders array is empty', async () => { + const result = await service.getMessageLists({ + connectedAccount: mockConnectedAccount, + messageChannel: { + syncCursor: '', + id: 'channel-1', + messageFolderImportPolicy: MessageFolderImportPolicy.ALL_FOLDERS, + }, + messageFolders: [], + }); + + expect(result).toEqual([]); + }); + }); +}); diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-get-message-list.service.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-get-message-list.service.ts index 82f7332a4a..6840c341ee 100644 --- a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-get-message-list.service.ts +++ b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-get-message-list.service.ts @@ -10,11 +10,8 @@ import pLimit from 'p-limit'; import { OAuth2ClientManagerService } from 'src/modules/connected-account/oauth2-client-manager/services/oauth2-client-manager.service'; import { type ConnectedAccountWorkspaceEntity } from 'src/modules/connected-account/standard-objects/connected-account.workspace-entity'; +import { MessageFolderImportPolicy } from 'src/modules/messaging/common/standard-objects/message-channel.workspace-entity'; import { type MessageFolderWorkspaceEntity } from 'src/modules/messaging/common/standard-objects/message-folder.workspace-entity'; -import { - MessageImportDriverException, - MessageImportDriverExceptionCode, -} from 'src/modules/messaging/message-import-manager/drivers/exceptions/message-import-driver.exception'; import { MicrosoftMessageListFetchErrorHandler } from 'src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-message-list-fetch-error-handler.service'; import { type GetMessageListsArgs } from 'src/modules/messaging/message-import-manager/types/get-message-lists-args.type'; import { @@ -41,17 +38,24 @@ export class MicrosoftGetMessageListService { connectedAccount, messageFolders, }: GetMessageListsArgs): Promise { - if (messageFolders.length === 0) { - throw new MessageImportDriverException( - `Message channel ${messageChannel.id} has no message folders`, - MessageImportDriverExceptionCode.NOT_FOUND, + const foldersToProcess = + messageChannel.messageFolderImportPolicy === + MessageFolderImportPolicy.SELECTED_FOLDERS + ? messageFolders.filter((folder) => folder.isSynced) + : messageFolders; + + if (foldersToProcess.length === 0) { + this.logger.warn( + `Connected account ${connectedAccount.id}: Message Channel: ${messageChannel.id}: No folders to process`, ); + + return []; } const limit = pLimit(FOLDER_PROCESSING_CONCURRENCY); const results = await Promise.all( - messageFolders.map((folder) => + foldersToProcess.map((folder) => limit(async () => { const response = await this.getMessageList(connectedAccount, folder); diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/services/__tests__/messaging-message-list-fetch.service.spec.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/services/__tests__/messaging-message-list-fetch.service.spec.ts index 73b9c81ade..285917a14e 100644 --- a/packages/twenty-server/src/modules/messaging/message-import-manager/services/__tests__/messaging-message-list-fetch.service.spec.ts +++ b/packages/twenty-server/src/modules/messaging/message-import-manager/services/__tests__/messaging-message-list-fetch.service.spec.ts @@ -6,7 +6,10 @@ import { CacheStorageService } from 'src/engine/core-modules/cache-storage/servi import { CacheStorageNamespace } from 'src/engine/core-modules/cache-storage/types/cache-storage-namespace.enum'; import { GlobalWorkspaceOrmManager } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager'; import { MessageChannelSyncStatusService } from 'src/modules/messaging/common/services/message-channel-sync-status.service'; -import { type MessageChannelWorkspaceEntity } from 'src/modules/messaging/common/standard-objects/message-channel.workspace-entity'; +import { + MessageFolderImportPolicy, + type MessageChannelWorkspaceEntity, +} from 'src/modules/messaging/common/standard-objects/message-channel.workspace-entity'; import { MessageFolderPendingSyncAction, type MessageFolderWorkspaceEntity, @@ -54,6 +57,7 @@ describe('MessagingMessageListFetchService', () => { messageChannelId: 'microsoft-message-channel-id', } as MessageFolderWorkspaceEntity, ], + messageFolderImportPolicy: MessageFolderImportPolicy.SELECTED_FOLDERS, } as MessageChannelWorkspaceEntity; mockGoogleMessageChannel = { @@ -68,6 +72,7 @@ describe('MessagingMessageListFetchService', () => { }, syncCursor: 'google-sync-cursor', messageFolders: [], + messageFolderImportPolicy: MessageFolderImportPolicy.SELECTED_FOLDERS, } as unknown as MessageChannelWorkspaceEntity; }); diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-message-list-fetch.service.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-message-list-fetch.service.ts index 28b511f263..da3e50e40f 100644 --- a/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-message-list-fetch.service.ts +++ b/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-message-list-fetch.service.ts @@ -16,7 +16,6 @@ import { MessageChannelPendingGroupEmailsAction, MessageChannelSyncStage, MessageChannelWorkspaceEntity, - MessageFolderImportPolicy, } from 'src/modules/messaging/common/standard-objects/message-channel.workspace-entity'; import { MessageFolderPendingSyncAction } from 'src/modules/messaging/common/standard-objects/message-folder.workspace-entity'; import { MessagingMessageCleanerService } from 'src/modules/messaging/message-cleaner/services/messaging-message-cleaner.service'; @@ -129,12 +128,7 @@ export class MessagingMessageListFetchService { workspaceId, }); - const messageFoldersToSync = ( - messageChannelWithFreshTokens.messageFolderImportPolicy === - MessageFolderImportPolicy.ALL_FOLDERS - ? messageFolders - : messageFolders.filter((folder) => folder.isSynced) - ).filter( + const messageFoldersToSync = messageFolders.filter( (folder) => folder.pendingSyncAction === MessageFolderPendingSyncAction.NONE, ); diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/types/get-message-lists-args.type.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/types/get-message-lists-args.type.ts index 20fcef8144..3c58548e71 100644 --- a/packages/twenty-server/src/modules/messaging/message-import-manager/types/get-message-lists-args.type.ts +++ b/packages/twenty-server/src/modules/messaging/message-import-manager/types/get-message-lists-args.type.ts @@ -4,7 +4,10 @@ import { type ConnectedAccountWorkspaceEntity } from 'src/modules/connected-acco import { type MessageChannelWorkspaceEntity } from 'src/modules/messaging/common/standard-objects/message-channel.workspace-entity'; export type GetMessageListsArgs = { - messageChannel: Pick; + messageChannel: Pick< + MessageChannelWorkspaceEntity, + 'syncCursor' | 'id' | 'messageFolderImportPolicy' + >; connectedAccount: Pick< ConnectedAccountWorkspaceEntity, | 'provider'