fix gmail message import policy (#17272)
Gmail sync was importing messages from folders users had explicitly disabled because the folder filtering logic lived in the parent service but the Gmail driver wasn't aware of the import policy when building its API queries. Moved messageFolderImportPolicy down to the driver level so Gmail can - Build proper -label: exclusions during initial sync - Added `buildGmailLabelSearchName` as our syntax for nested folders was wrong - Filter out messages from disabled folders during incremental sync via history API
This commit is contained in:
+395
-9
@@ -3,11 +3,28 @@ import { Test, type TestingModule } from '@nestjs/testing';
|
||||
import { google } from 'googleapis';
|
||||
import { ConnectedAccountProvider } from 'twenty-shared/types';
|
||||
|
||||
import { type MessageFolder } from 'src/modules/messaging/message-folder-manager/interfaces/message-folder-driver.interface';
|
||||
|
||||
import { OAuth2ClientManagerService } from 'src/modules/connected-account/oauth2-client-manager/services/oauth2-client-manager.service';
|
||||
import { type ConnectedAccountWorkspaceEntity } from 'src/modules/connected-account/standard-objects/connected-account.workspace-entity';
|
||||
import { MessageFolderImportPolicy } from 'src/modules/messaging/common/standard-objects/message-channel.workspace-entity';
|
||||
import { MessageFolderPendingSyncAction } from 'src/modules/messaging/common/standard-objects/message-folder.workspace-entity';
|
||||
import { GmailGetHistoryService } from 'src/modules/messaging/message-import-manager/drivers/gmail/services/gmail-get-history.service';
|
||||
import { GmailGetMessageListService } from 'src/modules/messaging/message-import-manager/drivers/gmail/services/gmail-get-message-list.service';
|
||||
import { GmailMessageListFetchErrorHandler } from 'src/modules/messaging/message-import-manager/drivers/gmail/services/gmail-message-list-fetch-error-handler.service';
|
||||
import { computeGmailExcludeSearchFilter } from 'src/modules/messaging/message-import-manager/drivers/gmail/utils/compute-gmail-exclude-search-filter.util';
|
||||
|
||||
const createMockFolder = (
|
||||
overrides: Partial<MessageFolder> &
|
||||
Pick<MessageFolder, 'name' | 'externalId' | 'isSynced'>,
|
||||
): MessageFolder => ({
|
||||
id: `folder-${overrides.externalId}`,
|
||||
syncCursor: null,
|
||||
isSentFolder: false,
|
||||
parentFolderId: null,
|
||||
pendingSyncAction: MessageFolderPendingSyncAction.NONE,
|
||||
...overrides,
|
||||
});
|
||||
|
||||
describe('GmailGetMessageListService', () => {
|
||||
let service: GmailGetMessageListService;
|
||||
@@ -25,7 +42,7 @@ describe('GmailGetMessageListService', () => {
|
||||
id: 'connected-account-id',
|
||||
provider: ConnectedAccountProvider.GOOGLE,
|
||||
accessToken: 'access-token',
|
||||
refreshToken: 'refresh-token', // dummy value for testing
|
||||
refreshToken: 'refresh-token',
|
||||
handle: 'test@gmail.com',
|
||||
connectionParameters: {},
|
||||
};
|
||||
@@ -90,9 +107,19 @@ describe('GmailGetMessageListService', () => {
|
||||
).mockResolvedValue({});
|
||||
|
||||
const result = await service.getMessageLists({
|
||||
messageChannel: { syncCursor: '', id: 'my-id' },
|
||||
messageChannel: {
|
||||
syncCursor: '',
|
||||
id: 'my-id',
|
||||
messageFolderImportPolicy: MessageFolderImportPolicy.SELECTED_FOLDERS,
|
||||
},
|
||||
connectedAccount: mockConnectedAccount,
|
||||
messageFolders: [],
|
||||
messageFolders: [
|
||||
createMockFolder({
|
||||
name: 'INBOX',
|
||||
externalId: 'INBOX',
|
||||
isSynced: true,
|
||||
}),
|
||||
],
|
||||
});
|
||||
|
||||
expect(result[0].messageExternalIds).toHaveLength(0);
|
||||
@@ -143,9 +170,19 @@ describe('GmailGetMessageListService', () => {
|
||||
).mockResolvedValue({});
|
||||
|
||||
const result = await service.getMessageLists({
|
||||
messageChannel: { syncCursor: '', id: 'my-id' },
|
||||
messageChannel: {
|
||||
syncCursor: '',
|
||||
id: 'my-id',
|
||||
messageFolderImportPolicy: MessageFolderImportPolicy.SELECTED_FOLDERS,
|
||||
},
|
||||
connectedAccount: mockConnectedAccount,
|
||||
messageFolders: [],
|
||||
messageFolders: [
|
||||
createMockFolder({
|
||||
name: 'INBOX',
|
||||
externalId: 'INBOX',
|
||||
isSynced: true,
|
||||
}),
|
||||
],
|
||||
});
|
||||
|
||||
expect(result[0].messageExternalIds).toHaveLength(5);
|
||||
@@ -196,9 +233,19 @@ describe('GmailGetMessageListService', () => {
|
||||
).mockResolvedValue({});
|
||||
|
||||
const result = await service.getMessageLists({
|
||||
messageChannel: { syncCursor: '', id: 'my-id' },
|
||||
messageChannel: {
|
||||
syncCursor: '',
|
||||
id: 'my-id',
|
||||
messageFolderImportPolicy: MessageFolderImportPolicy.SELECTED_FOLDERS,
|
||||
},
|
||||
connectedAccount: mockConnectedAccount,
|
||||
messageFolders: [],
|
||||
messageFolders: [
|
||||
createMockFolder({
|
||||
name: 'INBOX',
|
||||
externalId: 'INBOX',
|
||||
isSynced: true,
|
||||
}),
|
||||
],
|
||||
});
|
||||
|
||||
expect(result[0].messageExternalIds).toHaveLength(3);
|
||||
@@ -230,13 +277,352 @@ describe('GmailGetMessageListService', () => {
|
||||
).mockResolvedValue({});
|
||||
|
||||
const result = await service.getMessageLists({
|
||||
messageChannel: { syncCursor: '', id: 'my-id' },
|
||||
messageChannel: {
|
||||
syncCursor: '',
|
||||
id: 'my-id',
|
||||
messageFolderImportPolicy: MessageFolderImportPolicy.SELECTED_FOLDERS,
|
||||
},
|
||||
connectedAccount: mockConnectedAccount,
|
||||
messageFolders: [],
|
||||
messageFolders: [
|
||||
createMockFolder({
|
||||
name: 'INBOX',
|
||||
externalId: 'INBOX',
|
||||
isSynced: true,
|
||||
}),
|
||||
],
|
||||
});
|
||||
|
||||
expect(result[0].messageExternalIds).toHaveLength(0);
|
||||
expect(mockGmailClient.users.messages.list).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('should return empty array when no folders have isSynced=true with SELECTED_FOLDERS policy', async () => {
|
||||
const mockGmailClient = {
|
||||
users: {
|
||||
messages: {
|
||||
list: jest.fn(),
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
jest.spyOn(google, 'gmail').mockReturnValue(mockGmailClient as never);
|
||||
|
||||
(
|
||||
oAuth2ClientManagerService.getGoogleOAuth2Client as jest.Mock
|
||||
).mockResolvedValue({});
|
||||
|
||||
const result = await service.getMessageLists({
|
||||
messageChannel: {
|
||||
syncCursor: '',
|
||||
id: 'my-id',
|
||||
messageFolderImportPolicy: MessageFolderImportPolicy.SELECTED_FOLDERS,
|
||||
},
|
||||
connectedAccount: mockConnectedAccount,
|
||||
messageFolders: [
|
||||
createMockFolder({
|
||||
name: 'Personal',
|
||||
externalId: 'Label_personal',
|
||||
isSynced: false,
|
||||
}),
|
||||
createMockFolder({
|
||||
name: 'Work',
|
||||
externalId: 'Label_work',
|
||||
isSynced: false,
|
||||
}),
|
||||
],
|
||||
});
|
||||
|
||||
expect(result).toEqual([]);
|
||||
expect(mockGmailClient.users.messages.list).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
describe('initial sync folder filtering', () => {
|
||||
it('should build Gmail query with -label exclusions for non-synced folders', async () => {
|
||||
const mockGmailClient = {
|
||||
users: {
|
||||
messages: {
|
||||
list: jest.fn().mockResolvedValue({
|
||||
data: { messages: [{ id: 'msg-1' }], nextPageToken: undefined },
|
||||
}),
|
||||
get: jest.fn().mockResolvedValue({
|
||||
data: { historyId: '12345' },
|
||||
}),
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
jest.spyOn(google, 'gmail').mockReturnValue(mockGmailClient as never);
|
||||
(
|
||||
oAuth2ClientManagerService.getGoogleOAuth2Client as jest.Mock
|
||||
).mockResolvedValue({});
|
||||
|
||||
await service.getMessageLists({
|
||||
messageChannel: {
|
||||
syncCursor: '',
|
||||
id: 'channel-1',
|
||||
messageFolderImportPolicy: MessageFolderImportPolicy.SELECTED_FOLDERS,
|
||||
},
|
||||
connectedAccount: mockConnectedAccount,
|
||||
messageFolders: [
|
||||
createMockFolder({
|
||||
name: 'INBOX',
|
||||
externalId: 'INBOX',
|
||||
isSynced: true,
|
||||
}),
|
||||
createMockFolder({
|
||||
name: 'Work',
|
||||
externalId: 'Label_work',
|
||||
isSynced: true,
|
||||
}),
|
||||
createMockFolder({
|
||||
name: 'Personal',
|
||||
externalId: 'Label_personal',
|
||||
isSynced: false,
|
||||
}),
|
||||
createMockFolder({
|
||||
name: 'Newsletters',
|
||||
externalId: 'Label_newsletters',
|
||||
isSynced: false,
|
||||
}),
|
||||
],
|
||||
});
|
||||
|
||||
const expectedQuery = computeGmailExcludeSearchFilter([
|
||||
{
|
||||
externalId: 'Label_personal',
|
||||
name: 'Personal',
|
||||
isSynced: false,
|
||||
parentFolderId: null,
|
||||
},
|
||||
{
|
||||
externalId: 'Label_newsletters',
|
||||
name: 'Newsletters',
|
||||
isSynced: false,
|
||||
parentFolderId: null,
|
||||
},
|
||||
]);
|
||||
|
||||
expect(mockGmailClient.users.messages.list).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
q: expectedQuery,
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it('should not include exclusion filter when ALL_FOLDERS policy is set', async () => {
|
||||
const mockGmailClient = {
|
||||
users: {
|
||||
messages: {
|
||||
list: jest.fn().mockResolvedValue({
|
||||
data: { messages: [{ id: 'msg-1' }], nextPageToken: undefined },
|
||||
}),
|
||||
get: jest.fn().mockResolvedValue({
|
||||
data: { historyId: '12345' },
|
||||
}),
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
jest.spyOn(google, 'gmail').mockReturnValue(mockGmailClient as never);
|
||||
(
|
||||
oAuth2ClientManagerService.getGoogleOAuth2Client as jest.Mock
|
||||
).mockResolvedValue({});
|
||||
|
||||
await service.getMessageLists({
|
||||
messageChannel: {
|
||||
syncCursor: '',
|
||||
id: 'channel-1',
|
||||
messageFolderImportPolicy: MessageFolderImportPolicy.ALL_FOLDERS,
|
||||
},
|
||||
connectedAccount: mockConnectedAccount,
|
||||
messageFolders: [
|
||||
createMockFolder({
|
||||
name: 'INBOX',
|
||||
externalId: 'INBOX',
|
||||
isSynced: true,
|
||||
}),
|
||||
createMockFolder({
|
||||
name: 'Personal',
|
||||
externalId: 'Label_personal',
|
||||
isSynced: false,
|
||||
}),
|
||||
],
|
||||
});
|
||||
|
||||
expect(mockGmailClient.users.messages.list).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ q: '' }),
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('incremental sync folder filtering', () => {
|
||||
it('should filter out messages from disabled folders during incremental sync', async () => {
|
||||
const mockHistoryService = {
|
||||
getHistory: jest.fn(),
|
||||
getMessageIdsFromHistory: jest.fn(),
|
||||
};
|
||||
|
||||
const module: TestingModule = await Test.createTestingModule({
|
||||
providers: [
|
||||
GmailGetMessageListService,
|
||||
{
|
||||
provide: OAuth2ClientManagerService,
|
||||
useValue: {
|
||||
getGoogleOAuth2Client: jest.fn().mockResolvedValue({}),
|
||||
},
|
||||
},
|
||||
{
|
||||
provide: GmailGetHistoryService,
|
||||
useValue: mockHistoryService,
|
||||
},
|
||||
{
|
||||
provide: GmailMessageListFetchErrorHandler,
|
||||
useValue: { handleError: jest.fn() },
|
||||
},
|
||||
],
|
||||
}).compile();
|
||||
|
||||
const testService = module.get<GmailGetMessageListService>(
|
||||
GmailGetMessageListService,
|
||||
);
|
||||
|
||||
mockHistoryService.getHistory.mockImplementation(
|
||||
(_client, _cursor, _types, labelId) => {
|
||||
if (labelId === 'Label_personal') {
|
||||
return Promise.resolve({
|
||||
history: [
|
||||
{ messagesAdded: [{ message: { id: 'personal-msg' } }] },
|
||||
],
|
||||
historyId: 'new-cursor',
|
||||
});
|
||||
}
|
||||
if (labelId === undefined) {
|
||||
return Promise.resolve({
|
||||
history: [{ messagesAdded: [{ message: { id: 'inbox-msg' } }] }],
|
||||
historyId: 'new-cursor',
|
||||
});
|
||||
}
|
||||
|
||||
return Promise.resolve({ history: [], historyId: 'new-cursor' });
|
||||
},
|
||||
);
|
||||
|
||||
mockHistoryService.getMessageIdsFromHistory.mockResolvedValue({
|
||||
messagesAdded: ['inbox-msg', 'personal-msg'],
|
||||
messagesDeleted: [],
|
||||
});
|
||||
|
||||
jest.spyOn(google, 'gmail').mockReturnValue({} as never);
|
||||
|
||||
const result = await testService.getMessageLists({
|
||||
messageChannel: {
|
||||
syncCursor: 'old-cursor',
|
||||
id: 'channel-1',
|
||||
messageFolderImportPolicy: MessageFolderImportPolicy.SELECTED_FOLDERS,
|
||||
},
|
||||
connectedAccount: mockConnectedAccount,
|
||||
messageFolders: [
|
||||
createMockFolder({
|
||||
name: 'INBOX',
|
||||
externalId: 'INBOX',
|
||||
isSynced: true,
|
||||
}),
|
||||
createMockFolder({
|
||||
name: 'Personal',
|
||||
externalId: 'Label_personal',
|
||||
isSynced: false,
|
||||
}),
|
||||
],
|
||||
});
|
||||
|
||||
expect(result[0].messageExternalIds).toEqual(['inbox-msg']);
|
||||
|
||||
const allHistoryCalls = mockHistoryService.getHistory.mock.calls;
|
||||
|
||||
expect(allHistoryCalls[0]).toHaveLength(2);
|
||||
expect(allHistoryCalls[0][1]).toBe('old-cursor');
|
||||
|
||||
const labelIdsQueried = allHistoryCalls
|
||||
.slice(1)
|
||||
.map((call) => call[3])
|
||||
.filter(Boolean);
|
||||
|
||||
expect(labelIdsQueried).toContain('Label_personal');
|
||||
expect(labelIdsQueried).toHaveLength(1);
|
||||
|
||||
// 1 main history call + 1 excluded folder call
|
||||
expect(mockHistoryService.getHistory).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it('should skip per-folder filtering when ALL_FOLDERS policy is set', async () => {
|
||||
const mockHistoryService = {
|
||||
getHistory: jest.fn(),
|
||||
getMessageIdsFromHistory: jest.fn(),
|
||||
};
|
||||
|
||||
const module: TestingModule = await Test.createTestingModule({
|
||||
providers: [
|
||||
GmailGetMessageListService,
|
||||
{
|
||||
provide: OAuth2ClientManagerService,
|
||||
useValue: {
|
||||
getGoogleOAuth2Client: jest.fn().mockResolvedValue({}),
|
||||
},
|
||||
},
|
||||
{
|
||||
provide: GmailGetHistoryService,
|
||||
useValue: mockHistoryService,
|
||||
},
|
||||
{
|
||||
provide: GmailMessageListFetchErrorHandler,
|
||||
useValue: { handleError: jest.fn() },
|
||||
},
|
||||
],
|
||||
}).compile();
|
||||
|
||||
const testService = module.get<GmailGetMessageListService>(
|
||||
GmailGetMessageListService,
|
||||
);
|
||||
|
||||
mockHistoryService.getHistory.mockResolvedValue({
|
||||
history: [],
|
||||
historyId: 'new-cursor',
|
||||
});
|
||||
|
||||
mockHistoryService.getMessageIdsFromHistory.mockResolvedValue({
|
||||
messagesAdded: ['inbox-msg', 'personal-msg'],
|
||||
messagesDeleted: [],
|
||||
});
|
||||
|
||||
jest.spyOn(google, 'gmail').mockReturnValue({} as never);
|
||||
|
||||
const result = await testService.getMessageLists({
|
||||
messageChannel: {
|
||||
syncCursor: 'old-cursor',
|
||||
id: 'channel-1',
|
||||
messageFolderImportPolicy: MessageFolderImportPolicy.ALL_FOLDERS,
|
||||
},
|
||||
connectedAccount: mockConnectedAccount,
|
||||
messageFolders: [
|
||||
createMockFolder({
|
||||
name: 'INBOX',
|
||||
externalId: 'INBOX',
|
||||
isSynced: true,
|
||||
}),
|
||||
createMockFolder({
|
||||
name: 'Personal',
|
||||
externalId: 'Label_personal',
|
||||
isSynced: false,
|
||||
}),
|
||||
],
|
||||
});
|
||||
|
||||
expect(result[0].messageExternalIds).toEqual([
|
||||
'inbox-msg',
|
||||
'personal-msg',
|
||||
]);
|
||||
expect(mockHistoryService.getHistory).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
+42
-8
@@ -7,6 +7,10 @@ import { isDefined } from 'twenty-shared/utils';
|
||||
|
||||
import { OAuth2ClientManagerService } from 'src/modules/connected-account/oauth2-client-manager/services/oauth2-client-manager.service';
|
||||
import { type ConnectedAccountWorkspaceEntity } from 'src/modules/connected-account/standard-objects/connected-account.workspace-entity';
|
||||
import {
|
||||
MessageChannelWorkspaceEntity,
|
||||
MessageFolderImportPolicy,
|
||||
} from 'src/modules/messaging/common/standard-objects/message-channel.workspace-entity';
|
||||
import { type MessageFolderWorkspaceEntity } from 'src/modules/messaging/common/standard-objects/message-folder.workspace-entity';
|
||||
import {
|
||||
MessageImportDriverException,
|
||||
@@ -38,8 +42,12 @@ export class GmailGetMessageListService {
|
||||
>,
|
||||
messageFolders: Pick<
|
||||
MessageFolderWorkspaceEntity,
|
||||
'name' | 'externalId' | 'isSynced'
|
||||
'name' | 'externalId' | 'isSynced' | 'parentFolderId'
|
||||
>[],
|
||||
messageChannel: Pick<
|
||||
MessageChannelWorkspaceEntity,
|
||||
'messageFolderImportPolicy'
|
||||
>,
|
||||
): Promise<GetMessageListsResponse> {
|
||||
const oAuth2Client =
|
||||
await this.oAuth2ClientManagerService.getGoogleOAuth2Client(
|
||||
@@ -56,7 +64,10 @@ export class GmailGetMessageListService {
|
||||
const messageExternalIds: string[] = [];
|
||||
|
||||
const excludedSearchFilter =
|
||||
computeGmailExcludeSearchFilter(messageFolders);
|
||||
messageChannel.messageFolderImportPolicy ===
|
||||
MessageFolderImportPolicy.SELECTED_FOLDERS
|
||||
? computeGmailExcludeSearchFilter(messageFolders)
|
||||
: '';
|
||||
|
||||
while (hasMoreMessages) {
|
||||
const messageList = await gmailClient.users.messages
|
||||
@@ -145,6 +156,21 @@ export class GmailGetMessageListService {
|
||||
connectedAccount,
|
||||
messageFolders,
|
||||
}: GetMessageListsArgs): Promise<GetMessageListsResponse> {
|
||||
if (
|
||||
messageChannel.messageFolderImportPolicy ===
|
||||
MessageFolderImportPolicy.SELECTED_FOLDERS
|
||||
) {
|
||||
const foldersToSync = messageFolders.filter((folder) => folder.isSynced);
|
||||
|
||||
if (foldersToSync.length === 0) {
|
||||
this.logger.warn(
|
||||
`Connected account ${connectedAccount.id} Message Channel: ${messageChannel.id}: No folders to process`,
|
||||
);
|
||||
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
const oAuth2Client =
|
||||
await this.oAuth2ClientManagerService.getGoogleOAuth2Client(
|
||||
connectedAccount,
|
||||
@@ -155,7 +181,11 @@ export class GmailGetMessageListService {
|
||||
});
|
||||
|
||||
if (!isNonEmptyString(messageChannel.syncCursor)) {
|
||||
return this.getMessageListWithoutCursor(connectedAccount, messageFolders);
|
||||
return this.getMessageListWithoutCursor(
|
||||
connectedAccount,
|
||||
messageFolders,
|
||||
messageChannel,
|
||||
);
|
||||
}
|
||||
|
||||
const { history, historyId: nextSyncCursor } =
|
||||
@@ -167,11 +197,15 @@ export class GmailGetMessageListService {
|
||||
const { messagesAdded, messagesDeleted } =
|
||||
await this.gmailGetHistoryService.getMessageIdsFromHistory(history);
|
||||
|
||||
const messageIdsToFilter = await this.getEmailIdsFromExcludedFolders(
|
||||
connectedAccount,
|
||||
messageChannel.syncCursor,
|
||||
messageFolders,
|
||||
);
|
||||
const messageIdsToFilter =
|
||||
messageChannel.messageFolderImportPolicy ===
|
||||
MessageFolderImportPolicy.SELECTED_FOLDERS
|
||||
? await this.getEmailIdsFromExcludedFolders(
|
||||
connectedAccount,
|
||||
messageChannel.syncCursor,
|
||||
messageFolders,
|
||||
)
|
||||
: [];
|
||||
|
||||
const messagesAddedFiltered = messagesAdded.filter(
|
||||
(messageId) => !messageIdsToFilter.includes(messageId),
|
||||
|
||||
+60
@@ -0,0 +1,60 @@
|
||||
import { buildGmailLabelSearchName } from 'src/modules/messaging/message-import-manager/drivers/gmail/utils/build-gmail-label-search-name.util';
|
||||
|
||||
describe('buildGmailLabelSearchName', () => {
|
||||
it('should return null for folder without name', () => {
|
||||
const result = buildGmailLabelSearchName(
|
||||
{ externalId: 'Label_1', name: '', parentFolderId: null },
|
||||
[],
|
||||
);
|
||||
|
||||
expect(result).toBeNull();
|
||||
});
|
||||
|
||||
it('should convert spaces to hyphens and lowercase', () => {
|
||||
const folder = {
|
||||
externalId: 'Label_1',
|
||||
name: 'My Custom Folder',
|
||||
parentFolderId: null,
|
||||
};
|
||||
|
||||
const result = buildGmailLabelSearchName(folder, [folder]);
|
||||
|
||||
expect(result).toBe('my-custom-folder');
|
||||
});
|
||||
|
||||
it('should build full path for nested folder', () => {
|
||||
const parent = {
|
||||
externalId: 'Label_parent',
|
||||
name: 'Work',
|
||||
parentFolderId: null,
|
||||
};
|
||||
const child = {
|
||||
externalId: 'Label_child',
|
||||
name: 'Projects',
|
||||
parentFolderId: 'Label_parent',
|
||||
};
|
||||
const allFolders = [parent, child];
|
||||
|
||||
const result = buildGmailLabelSearchName(child, allFolders);
|
||||
|
||||
expect(result).toBe('work-projects');
|
||||
});
|
||||
|
||||
it('should handle deeply nested folders', () => {
|
||||
const grandparent = {
|
||||
externalId: 'L1',
|
||||
name: 'Work',
|
||||
parentFolderId: null,
|
||||
};
|
||||
const parent = { externalId: 'L2', name: 'Projects', parentFolderId: 'L1' };
|
||||
const child = { externalId: 'L3', name: 'Active', parentFolderId: 'L2' };
|
||||
|
||||
const result = buildGmailLabelSearchName(child, [
|
||||
grandparent,
|
||||
parent,
|
||||
child,
|
||||
]);
|
||||
|
||||
expect(result).toBe('work-projects-active');
|
||||
});
|
||||
});
|
||||
+96
-19
@@ -1,69 +1,146 @@
|
||||
import { computeGmailExcludeSearchFilter } from 'src/modules/messaging/message-import-manager/drivers/gmail/utils/compute-gmail-exclude-search-filter.util';
|
||||
|
||||
describe('computeGmailExcludeSearchFilter', () => {
|
||||
it('should return correct exclude search filter with empty label array', () => {
|
||||
it('should return empty string with empty folder array', () => {
|
||||
const result = computeGmailExcludeSearchFilter([]);
|
||||
|
||||
expect(result).toBe('');
|
||||
});
|
||||
|
||||
it('should return correct exclude search filter with one label', () => {
|
||||
it('should return correct exclude filter for one unsynced folder', () => {
|
||||
const result = computeGmailExcludeSearchFilter([
|
||||
{
|
||||
externalId: 'LABEL1',
|
||||
externalId: 'Label_123',
|
||||
name: 'Custom Folder',
|
||||
isSynced: false,
|
||||
parentFolderId: null,
|
||||
},
|
||||
]);
|
||||
|
||||
expect(result).toBe('-label:LABEL1');
|
||||
expect(result).toBe('-label:custom-folder');
|
||||
});
|
||||
|
||||
it('should return correct exclude search filter with multiple categories', () => {
|
||||
it('should return correct exclude filter for multiple unsynced folders', () => {
|
||||
const result = computeGmailExcludeSearchFilter([
|
||||
{
|
||||
externalId: 'LABEL1',
|
||||
externalId: 'Label_1',
|
||||
name: 'Folder One',
|
||||
isSynced: false,
|
||||
parentFolderId: null,
|
||||
},
|
||||
{
|
||||
externalId: 'LABEL2',
|
||||
isSynced: false,
|
||||
},
|
||||
{
|
||||
externalId: 'LABEL3',
|
||||
externalId: 'Label_2',
|
||||
name: 'Folder Two',
|
||||
isSynced: false,
|
||||
parentFolderId: null,
|
||||
},
|
||||
]);
|
||||
|
||||
expect(result).toBe('-label:LABEL1 -label:LABEL2 -label:LABEL3');
|
||||
expect(result).toBe('-label:folder-one -label:folder-two');
|
||||
});
|
||||
|
||||
it('should return correct exclude search filter with one label that is synced', () => {
|
||||
it('should return empty string when all folders are synced', () => {
|
||||
const result = computeGmailExcludeSearchFilter([
|
||||
{
|
||||
externalId: 'LABEL1',
|
||||
externalId: 'Label_1',
|
||||
name: 'Synced Folder',
|
||||
isSynced: true,
|
||||
parentFolderId: null,
|
||||
},
|
||||
]);
|
||||
|
||||
expect(result).toBe('');
|
||||
});
|
||||
|
||||
it('should return correct exclude search filter with multiple categories that are synced', () => {
|
||||
it('should only exclude unsynced folders', () => {
|
||||
const result = computeGmailExcludeSearchFilter([
|
||||
{
|
||||
externalId: 'LABEL1',
|
||||
externalId: 'Label_1',
|
||||
name: 'Synced',
|
||||
isSynced: true,
|
||||
parentFolderId: null,
|
||||
},
|
||||
{
|
||||
externalId: 'LABEL2',
|
||||
externalId: 'Label_2',
|
||||
name: 'Not Synced',
|
||||
isSynced: false,
|
||||
parentFolderId: null,
|
||||
},
|
||||
{
|
||||
externalId: 'LABEL3',
|
||||
externalId: 'Label_3',
|
||||
name: 'Also Synced',
|
||||
isSynced: true,
|
||||
parentFolderId: null,
|
||||
},
|
||||
]);
|
||||
|
||||
expect(result).toBe('-label:LABEL2');
|
||||
expect(result).toBe('-label:not-synced');
|
||||
});
|
||||
|
||||
it('should handle nested folders with parent path', () => {
|
||||
const folders = [
|
||||
{
|
||||
externalId: 'Label_parent',
|
||||
name: 'Parent Folder',
|
||||
isSynced: true,
|
||||
parentFolderId: null,
|
||||
},
|
||||
{
|
||||
externalId: 'Label_child',
|
||||
name: 'Child Folder',
|
||||
isSynced: false,
|
||||
parentFolderId: 'Label_parent',
|
||||
},
|
||||
];
|
||||
|
||||
const result = computeGmailExcludeSearchFilter(folders);
|
||||
|
||||
expect(result).toBe('-label:parent-folder-child-folder');
|
||||
});
|
||||
|
||||
it('should handle deeply nested folders', () => {
|
||||
const folders = [
|
||||
{
|
||||
externalId: 'Label_grandparent',
|
||||
name: 'Level One',
|
||||
isSynced: true,
|
||||
parentFolderId: null,
|
||||
},
|
||||
{
|
||||
externalId: 'Label_parent',
|
||||
name: 'Level Two',
|
||||
isSynced: true,
|
||||
parentFolderId: 'Label_grandparent',
|
||||
},
|
||||
{
|
||||
externalId: 'Label_child',
|
||||
name: 'Level Three',
|
||||
isSynced: false,
|
||||
parentFolderId: 'Label_parent',
|
||||
},
|
||||
];
|
||||
|
||||
const result = computeGmailExcludeSearchFilter(folders);
|
||||
|
||||
expect(result).toBe('-label:level-one-level-two-level-three');
|
||||
});
|
||||
|
||||
it('should skip folders without names', () => {
|
||||
const result = computeGmailExcludeSearchFilter([
|
||||
{
|
||||
externalId: 'Label_1',
|
||||
name: '',
|
||||
isSynced: false,
|
||||
parentFolderId: null,
|
||||
},
|
||||
{
|
||||
externalId: 'Label_2',
|
||||
name: 'Valid Folder',
|
||||
isSynced: false,
|
||||
parentFolderId: null,
|
||||
},
|
||||
]);
|
||||
|
||||
expect(result).toBe('-label:valid-folder');
|
||||
});
|
||||
});
|
||||
|
||||
+44
@@ -0,0 +1,44 @@
|
||||
import { type MessageFolderWorkspaceEntity } from 'src/modules/messaging/common/standard-objects/message-folder.workspace-entity';
|
||||
|
||||
const MAXIMUM_GMAIL_FOLDER_DEPTH = 50;
|
||||
|
||||
type FolderInput = Pick<
|
||||
MessageFolderWorkspaceEntity,
|
||||
'externalId' | 'name' | 'parentFolderId'
|
||||
>;
|
||||
|
||||
export const buildGmailLabelSearchName = (
|
||||
folder: FolderInput,
|
||||
allFolders: FolderInput[],
|
||||
): string | null => {
|
||||
if (!folder.name) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const folderMap = new Map(
|
||||
allFolders
|
||||
.filter((folder) => folder.externalId)
|
||||
.map((folder) => [folder.externalId, folder]),
|
||||
);
|
||||
|
||||
const pathParts: string[] = [];
|
||||
let current: FolderInput | undefined = folder;
|
||||
let depth = 0;
|
||||
|
||||
while (current?.name && depth < MAXIMUM_GMAIL_FOLDER_DEPTH) {
|
||||
pathParts.unshift(current.name);
|
||||
current = current.parentFolderId
|
||||
? folderMap.get(current.parentFolderId)
|
||||
: undefined;
|
||||
depth++;
|
||||
}
|
||||
|
||||
if (depth >= MAXIMUM_GMAIL_FOLDER_DEPTH) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return pathParts
|
||||
.join('/')
|
||||
.replace(/[\s/]+/g, '-')
|
||||
.toLowerCase();
|
||||
};
|
||||
+7
-2
@@ -1,12 +1,17 @@
|
||||
import { isDefined } from 'twenty-shared/utils';
|
||||
|
||||
import { type MessageFolderWorkspaceEntity } from 'src/modules/messaging/common/standard-objects/message-folder.workspace-entity';
|
||||
import { buildGmailLabelSearchName } from 'src/modules/messaging/message-import-manager/drivers/gmail/utils/build-gmail-label-search-name.util';
|
||||
|
||||
export const computeGmailExcludeSearchFilter = (
|
||||
messageFolders: Pick<
|
||||
MessageFolderWorkspaceEntity,
|
||||
'externalId' | 'isSynced'
|
||||
'externalId' | 'isSynced' | 'name' | 'parentFolderId'
|
||||
>[],
|
||||
) =>
|
||||
messageFolders
|
||||
.filter((folder) => !folder.isSynced)
|
||||
.map((folder) => `-label:${folder.externalId}`)
|
||||
.map((folder) => buildGmailLabelSearchName(folder, messageFolders))
|
||||
.filter(isDefined)
|
||||
.map((name) => `-label:${name}`)
|
||||
.join(' ');
|
||||
|
||||
+227
@@ -0,0 +1,227 @@
|
||||
import { Test, type TestingModule } from '@nestjs/testing';
|
||||
|
||||
import { ConnectedAccountProvider } from 'twenty-shared/types';
|
||||
|
||||
import { type MessageFolder } from 'src/modules/messaging/message-folder-manager/interfaces/message-folder-driver.interface';
|
||||
|
||||
import { type ConnectedAccountWorkspaceEntity } from 'src/modules/connected-account/standard-objects/connected-account.workspace-entity';
|
||||
import { MessageFolderImportPolicy } from 'src/modules/messaging/common/standard-objects/message-channel.workspace-entity';
|
||||
import { MessageFolderPendingSyncAction } from 'src/modules/messaging/common/standard-objects/message-folder.workspace-entity';
|
||||
import { ImapClientProvider } from 'src/modules/messaging/message-import-manager/drivers/imap/providers/imap-client.provider';
|
||||
import { ImapGetMessageListService } from 'src/modules/messaging/message-import-manager/drivers/imap/services/imap-get-message-list.service';
|
||||
import { ImapMessageListFetchErrorHandler } from 'src/modules/messaging/message-import-manager/drivers/imap/services/imap-message-list-fetch-error-handler.service';
|
||||
import { ImapSyncService } from 'src/modules/messaging/message-import-manager/drivers/imap/services/imap-sync.service';
|
||||
|
||||
const createMockFolder = (
|
||||
overrides: Partial<MessageFolder> &
|
||||
Pick<MessageFolder, 'name' | 'externalId' | 'isSynced'>,
|
||||
): MessageFolder => ({
|
||||
id: `folder-${overrides.externalId}`,
|
||||
syncCursor: null,
|
||||
isSentFolder: false,
|
||||
parentFolderId: null,
|
||||
pendingSyncAction: MessageFolderPendingSyncAction.NONE,
|
||||
...overrides,
|
||||
});
|
||||
|
||||
describe('ImapGetMessageListService', () => {
|
||||
let service: ImapGetMessageListService;
|
||||
let imapClientProvider: ImapClientProvider;
|
||||
|
||||
const mockConnectedAccount: Pick<
|
||||
ConnectedAccountWorkspaceEntity,
|
||||
| 'provider'
|
||||
| 'accessToken'
|
||||
| 'refreshToken'
|
||||
| 'id'
|
||||
| 'handle'
|
||||
| 'connectionParameters'
|
||||
> = {
|
||||
id: 'connected-account-id',
|
||||
provider: ConnectedAccountProvider.IMAP_SMTP_CALDAV,
|
||||
accessToken: 'access-token',
|
||||
refreshToken: 'refresh-token',
|
||||
handle: 'test@example.com',
|
||||
connectionParameters: {},
|
||||
};
|
||||
|
||||
const mockImapClient = {
|
||||
getMailboxLock: jest.fn().mockResolvedValue({ release: jest.fn() }),
|
||||
mailbox: {
|
||||
uidValidity: 12345,
|
||||
uidNext: 100,
|
||||
highestModseq: '1000',
|
||||
},
|
||||
capabilities: new Set(['CONDSTORE']),
|
||||
status: jest.fn().mockResolvedValue({
|
||||
uidValidity: 12345,
|
||||
uidNext: 100,
|
||||
highestModseq: '1000',
|
||||
}),
|
||||
};
|
||||
|
||||
beforeEach(async () => {
|
||||
const module: TestingModule = await Test.createTestingModule({
|
||||
providers: [
|
||||
ImapGetMessageListService,
|
||||
{
|
||||
provide: ImapClientProvider,
|
||||
useValue: {
|
||||
getClient: jest.fn().mockResolvedValue(mockImapClient),
|
||||
closeClient: jest.fn().mockResolvedValue(undefined),
|
||||
},
|
||||
},
|
||||
{
|
||||
provide: ImapSyncService,
|
||||
useValue: {
|
||||
syncFolder: jest.fn().mockResolvedValue({ messageUids: [1, 2, 3] }),
|
||||
},
|
||||
},
|
||||
{
|
||||
provide: ImapMessageListFetchErrorHandler,
|
||||
useValue: {
|
||||
handleError: jest.fn(),
|
||||
},
|
||||
},
|
||||
],
|
||||
}).compile();
|
||||
|
||||
service = module.get<ImapGetMessageListService>(ImapGetMessageListService);
|
||||
imapClientProvider = module.get<ImapClientProvider>(ImapClientProvider);
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
jest.clearAllMocks();
|
||||
});
|
||||
|
||||
describe('folder filtering based on import policy', () => {
|
||||
it('should only process synced folders when SELECTED_FOLDERS policy is set', async () => {
|
||||
const syncedFolder = createMockFolder({
|
||||
name: 'INBOX',
|
||||
externalId: 'INBOX:1',
|
||||
isSynced: true,
|
||||
});
|
||||
|
||||
const nonSyncedFolder = createMockFolder({
|
||||
name: 'Personal',
|
||||
externalId: 'Personal:1',
|
||||
isSynced: false,
|
||||
});
|
||||
|
||||
const result = await service.getMessageLists({
|
||||
connectedAccount: mockConnectedAccount,
|
||||
messageChannel: {
|
||||
syncCursor: '',
|
||||
id: 'channel-1',
|
||||
messageFolderImportPolicy: MessageFolderImportPolicy.SELECTED_FOLDERS,
|
||||
},
|
||||
messageFolders: [syncedFolder, nonSyncedFolder],
|
||||
});
|
||||
|
||||
expect(result).toHaveLength(1);
|
||||
expect(result[0].folderId).toBe(syncedFolder.id);
|
||||
});
|
||||
|
||||
it('should process all folders when ALL_FOLDERS policy is set', async () => {
|
||||
const syncedFolder = createMockFolder({
|
||||
name: 'INBOX',
|
||||
externalId: 'INBOX:1',
|
||||
isSynced: true,
|
||||
});
|
||||
|
||||
const nonSyncedFolder = createMockFolder({
|
||||
name: 'Personal',
|
||||
externalId: 'Personal:1',
|
||||
isSynced: false,
|
||||
});
|
||||
|
||||
const result = await service.getMessageLists({
|
||||
connectedAccount: mockConnectedAccount,
|
||||
messageChannel: {
|
||||
syncCursor: '',
|
||||
id: 'channel-1',
|
||||
messageFolderImportPolicy: MessageFolderImportPolicy.ALL_FOLDERS,
|
||||
},
|
||||
messageFolders: [syncedFolder, nonSyncedFolder],
|
||||
});
|
||||
|
||||
expect(result).toHaveLength(2);
|
||||
expect(result.map((r) => r.folderId)).toEqual([
|
||||
syncedFolder.id,
|
||||
nonSyncedFolder.id,
|
||||
]);
|
||||
});
|
||||
|
||||
it('should return empty array when SELECTED_FOLDERS policy and no folders are synced', async () => {
|
||||
const nonSyncedFolder1 = createMockFolder({
|
||||
name: 'Personal',
|
||||
externalId: 'Personal:1',
|
||||
isSynced: false,
|
||||
});
|
||||
|
||||
const nonSyncedFolder2 = createMockFolder({
|
||||
name: 'Work',
|
||||
externalId: 'Work:1',
|
||||
isSynced: false,
|
||||
});
|
||||
|
||||
const result = await service.getMessageLists({
|
||||
connectedAccount: mockConnectedAccount,
|
||||
messageChannel: {
|
||||
syncCursor: '',
|
||||
id: 'channel-1',
|
||||
messageFolderImportPolicy: MessageFolderImportPolicy.SELECTED_FOLDERS,
|
||||
},
|
||||
messageFolders: [nonSyncedFolder1, nonSyncedFolder2],
|
||||
});
|
||||
|
||||
expect(result).toHaveLength(0);
|
||||
});
|
||||
|
||||
it('should process all non-synced folders when ALL_FOLDERS policy is set', async () => {
|
||||
const nonSyncedFolder1 = createMockFolder({
|
||||
name: 'Personal',
|
||||
externalId: 'Personal:1',
|
||||
isSynced: false,
|
||||
});
|
||||
|
||||
const nonSyncedFolder2 = createMockFolder({
|
||||
name: 'Work',
|
||||
externalId: 'Work:1',
|
||||
isSynced: false,
|
||||
});
|
||||
|
||||
const result = await service.getMessageLists({
|
||||
connectedAccount: mockConnectedAccount,
|
||||
messageChannel: {
|
||||
syncCursor: '',
|
||||
id: 'channel-1',
|
||||
messageFolderImportPolicy: MessageFolderImportPolicy.ALL_FOLDERS,
|
||||
},
|
||||
messageFolders: [nonSyncedFolder1, nonSyncedFolder2],
|
||||
});
|
||||
|
||||
expect(result).toHaveLength(2);
|
||||
});
|
||||
|
||||
it('should always close the IMAP client regardless of policy', async () => {
|
||||
const folder = createMockFolder({
|
||||
name: 'INBOX',
|
||||
externalId: 'INBOX:1',
|
||||
isSynced: true,
|
||||
});
|
||||
|
||||
await service.getMessageLists({
|
||||
connectedAccount: mockConnectedAccount,
|
||||
messageChannel: {
|
||||
syncCursor: '',
|
||||
id: 'channel-1',
|
||||
messageFolderImportPolicy: MessageFolderImportPolicy.ALL_FOLDERS,
|
||||
},
|
||||
messageFolders: [folder],
|
||||
});
|
||||
|
||||
expect(imapClientProvider.closeClient).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
});
|
||||
});
|
||||
+17
-1
@@ -5,6 +5,7 @@ import { isDefined } from 'twenty-shared/utils';
|
||||
|
||||
import { MessageFolder } from 'src/modules/messaging/message-folder-manager/interfaces/message-folder-driver.interface';
|
||||
|
||||
import { MessageFolderImportPolicy } from 'src/modules/messaging/common/standard-objects/message-channel.workspace-entity';
|
||||
import {
|
||||
MessageImportDriverException,
|
||||
MessageImportDriverExceptionCode,
|
||||
@@ -35,13 +36,28 @@ export class ImapGetMessageListService {
|
||||
async getMessageLists({
|
||||
connectedAccount,
|
||||
messageFolders,
|
||||
messageChannel,
|
||||
}: GetMessageListsArgs): Promise<GetMessageListsResponse> {
|
||||
const foldersToProcess =
|
||||
messageChannel.messageFolderImportPolicy ===
|
||||
MessageFolderImportPolicy.SELECTED_FOLDERS
|
||||
? messageFolders.filter((folder) => folder.isSynced)
|
||||
: messageFolders;
|
||||
|
||||
if (foldersToProcess.length === 0) {
|
||||
this.logger.warn(
|
||||
`Connected account ${connectedAccount.id}: No folders to process`,
|
||||
);
|
||||
|
||||
return [];
|
||||
}
|
||||
|
||||
const client = await this.imapClientProvider.getClient(connectedAccount);
|
||||
|
||||
try {
|
||||
const results: GetMessageListsResponse = [];
|
||||
|
||||
for (const folder of messageFolders) {
|
||||
for (const folder of foldersToProcess) {
|
||||
const response = await this.getMessageList(client, folder);
|
||||
|
||||
results.push({ ...response, folderId: folder.id });
|
||||
|
||||
+13
-2
@@ -7,7 +7,10 @@ import { TwentyConfigModule } from 'src/engine/core-modules/twenty-config/twenty
|
||||
import { MicrosoftOAuth2ClientManagerService } from 'src/modules/connected-account/oauth2-client-manager/drivers/microsoft/microsoft-oauth2-client-manager.service';
|
||||
import { OAuth2ClientManagerService } from 'src/modules/connected-account/oauth2-client-manager/services/oauth2-client-manager.service';
|
||||
import { type ConnectedAccountWorkspaceEntity } from 'src/modules/connected-account/standard-objects/connected-account.workspace-entity';
|
||||
import { MessageChannelWorkspaceEntity } from 'src/modules/messaging/common/standard-objects/message-channel.workspace-entity';
|
||||
import {
|
||||
MessageChannelWorkspaceEntity,
|
||||
MessageFolderImportPolicy,
|
||||
} from 'src/modules/messaging/common/standard-objects/message-channel.workspace-entity';
|
||||
import {
|
||||
MessageFolderPendingSyncAction,
|
||||
MessageFolderWorkspaceEntity,
|
||||
@@ -41,10 +44,11 @@ const mockConnectedAccount: Pick<
|
||||
|
||||
const mockMessageChannel: Pick<
|
||||
MessageChannelWorkspaceEntity,
|
||||
'id' | 'syncCursor'
|
||||
'id' | 'syncCursor' | 'messageFolderImportPolicy'
|
||||
> = {
|
||||
id: 'message-channel-id',
|
||||
syncCursor: '', // Should be empty for Microsoft as cursors are stored at the folder level
|
||||
messageFolderImportPolicy: MessageFolderImportPolicy.SELECTED_FOLDERS,
|
||||
};
|
||||
|
||||
xdescribe('Microsoft dev tests : get message list service', () => {
|
||||
@@ -149,6 +153,7 @@ xdescribe('Microsoft dev tests : get message list service', () => {
|
||||
messageChannel: {
|
||||
id: 'message-channel-id',
|
||||
syncCursor: '',
|
||||
messageFolderImportPolicy: MessageFolderImportPolicy.SELECTED_FOLDERS,
|
||||
},
|
||||
connectedAccount: mockConnectedAccount,
|
||||
messageFolders: [
|
||||
@@ -202,18 +207,24 @@ xdescribe('Microsoft dev tests : get message list service for folders', () => {
|
||||
messageChannelNoFolders.id = 'message-channel-0';
|
||||
messageChannelNoFolders.messageFolders = [];
|
||||
messageChannelNoFolders.syncCursor = '';
|
||||
messageChannelNoFolders.messageFolderImportPolicy =
|
||||
MessageFolderImportPolicy.SELECTED_FOLDERS;
|
||||
|
||||
const messageChannelMicrosoftOneFolder = new MessageChannelWorkspaceEntity();
|
||||
|
||||
messageChannelMicrosoftOneFolder.id = 'message-channel-1';
|
||||
messageChannelMicrosoftOneFolder.messageFolders = [inboxFolder];
|
||||
messageChannelMicrosoftOneFolder.syncCursor = '';
|
||||
messageChannelMicrosoftOneFolder.messageFolderImportPolicy =
|
||||
MessageFolderImportPolicy.SELECTED_FOLDERS;
|
||||
|
||||
const messageChannelMicrosoft = new MessageChannelWorkspaceEntity();
|
||||
|
||||
messageChannelMicrosoft.id = 'message-channel-2';
|
||||
messageChannelMicrosoft.messageFolders = [inboxFolder, sentFolder];
|
||||
messageChannelMicrosoft.syncCursor = '';
|
||||
messageChannelMicrosoft.messageFolderImportPolicy =
|
||||
MessageFolderImportPolicy.SELECTED_FOLDERS;
|
||||
|
||||
beforeEach(async () => {
|
||||
const module: TestingModule = await Test.createTestingModule({
|
||||
|
||||
+233
@@ -0,0 +1,233 @@
|
||||
import { Test, type TestingModule } from '@nestjs/testing';
|
||||
|
||||
import { ConnectedAccountProvider } from 'twenty-shared/types';
|
||||
|
||||
import { type MessageFolder } from 'src/modules/messaging/message-folder-manager/interfaces/message-folder-driver.interface';
|
||||
|
||||
import { OAuth2ClientManagerService } from 'src/modules/connected-account/oauth2-client-manager/services/oauth2-client-manager.service';
|
||||
import { type ConnectedAccountWorkspaceEntity } from 'src/modules/connected-account/standard-objects/connected-account.workspace-entity';
|
||||
import { MessageFolderImportPolicy } from 'src/modules/messaging/common/standard-objects/message-channel.workspace-entity';
|
||||
import { MessageFolderPendingSyncAction } from 'src/modules/messaging/common/standard-objects/message-folder.workspace-entity';
|
||||
import { MicrosoftGetMessageListService } from 'src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-get-message-list.service';
|
||||
import { MicrosoftMessageListFetchErrorHandler } from 'src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-message-list-fetch-error-handler.service';
|
||||
|
||||
const createMockFolder = (
|
||||
overrides: Partial<MessageFolder> &
|
||||
Pick<MessageFolder, 'name' | 'externalId' | 'isSynced'>,
|
||||
): MessageFolder => ({
|
||||
id: `folder-${overrides.externalId}`,
|
||||
syncCursor: null,
|
||||
isSentFolder: false,
|
||||
parentFolderId: null,
|
||||
pendingSyncAction: MessageFolderPendingSyncAction.NONE,
|
||||
...overrides,
|
||||
});
|
||||
|
||||
describe('MicrosoftGetMessageListService', () => {
|
||||
let service: MicrosoftGetMessageListService;
|
||||
let oAuth2ClientManagerService: OAuth2ClientManagerService;
|
||||
|
||||
const mockConnectedAccount: Pick<
|
||||
ConnectedAccountWorkspaceEntity,
|
||||
| 'provider'
|
||||
| 'accessToken'
|
||||
| 'refreshToken'
|
||||
| 'id'
|
||||
| 'handle'
|
||||
| 'connectionParameters'
|
||||
> = {
|
||||
id: 'connected-account-id',
|
||||
provider: ConnectedAccountProvider.MICROSOFT,
|
||||
accessToken: 'access-token',
|
||||
refreshToken: 'refresh-token',
|
||||
handle: 'test@outlook.com',
|
||||
connectionParameters: {},
|
||||
};
|
||||
|
||||
const createMockMicrosoftClient = () => ({
|
||||
api: jest.fn().mockReturnThis(),
|
||||
version: jest.fn().mockReturnThis(),
|
||||
headers: jest.fn().mockReturnThis(),
|
||||
get: jest.fn().mockResolvedValue({
|
||||
value: [{ id: 'msg-1' }, { id: 'msg-2' }],
|
||||
'@odata.deltaLink': 'https://graph.microsoft.com/delta?token=abc',
|
||||
}),
|
||||
});
|
||||
|
||||
beforeEach(async () => {
|
||||
const module: TestingModule = await Test.createTestingModule({
|
||||
providers: [
|
||||
MicrosoftGetMessageListService,
|
||||
{
|
||||
provide: OAuth2ClientManagerService,
|
||||
useValue: {
|
||||
getMicrosoftOAuth2Client: jest.fn(),
|
||||
},
|
||||
},
|
||||
{
|
||||
provide: MicrosoftMessageListFetchErrorHandler,
|
||||
useValue: {
|
||||
handleError: jest.fn(),
|
||||
},
|
||||
},
|
||||
],
|
||||
}).compile();
|
||||
|
||||
service = module.get<MicrosoftGetMessageListService>(
|
||||
MicrosoftGetMessageListService,
|
||||
);
|
||||
oAuth2ClientManagerService = module.get<OAuth2ClientManagerService>(
|
||||
OAuth2ClientManagerService,
|
||||
);
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
jest.clearAllMocks();
|
||||
});
|
||||
|
||||
describe('folder filtering based on import policy', () => {
|
||||
it('should only process synced folders when SELECTED_FOLDERS policy is set', async () => {
|
||||
const mockClient = createMockMicrosoftClient();
|
||||
|
||||
(
|
||||
oAuth2ClientManagerService.getMicrosoftOAuth2Client as jest.Mock
|
||||
).mockResolvedValue(mockClient);
|
||||
|
||||
const syncedFolder = createMockFolder({
|
||||
name: 'Inbox',
|
||||
externalId: 'inbox-id',
|
||||
isSynced: true,
|
||||
});
|
||||
|
||||
const nonSyncedFolder = createMockFolder({
|
||||
name: 'Personal',
|
||||
externalId: 'personal-id',
|
||||
isSynced: false,
|
||||
});
|
||||
|
||||
const result = await service.getMessageLists({
|
||||
connectedAccount: mockConnectedAccount,
|
||||
messageChannel: {
|
||||
syncCursor: '',
|
||||
id: 'channel-1',
|
||||
messageFolderImportPolicy: MessageFolderImportPolicy.SELECTED_FOLDERS,
|
||||
},
|
||||
messageFolders: [syncedFolder, nonSyncedFolder],
|
||||
});
|
||||
|
||||
expect(result).toHaveLength(1);
|
||||
expect(result[0].folderId).toBe(syncedFolder.id);
|
||||
expect(
|
||||
oAuth2ClientManagerService.getMicrosoftOAuth2Client,
|
||||
).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('should process all folders when ALL_FOLDERS policy is set', async () => {
|
||||
const mockClient = createMockMicrosoftClient();
|
||||
|
||||
(
|
||||
oAuth2ClientManagerService.getMicrosoftOAuth2Client as jest.Mock
|
||||
).mockResolvedValue(mockClient);
|
||||
|
||||
const syncedFolder = createMockFolder({
|
||||
name: 'Inbox',
|
||||
externalId: 'inbox-id',
|
||||
isSynced: true,
|
||||
});
|
||||
|
||||
const nonSyncedFolder = createMockFolder({
|
||||
name: 'Personal',
|
||||
externalId: 'personal-id',
|
||||
isSynced: false,
|
||||
});
|
||||
|
||||
const result = await service.getMessageLists({
|
||||
connectedAccount: mockConnectedAccount,
|
||||
messageChannel: {
|
||||
syncCursor: '',
|
||||
id: 'channel-1',
|
||||
messageFolderImportPolicy: MessageFolderImportPolicy.ALL_FOLDERS,
|
||||
},
|
||||
messageFolders: [syncedFolder, nonSyncedFolder],
|
||||
});
|
||||
|
||||
expect(result).toHaveLength(2);
|
||||
expect(result.map((r) => r.folderId)).toEqual([
|
||||
syncedFolder.id,
|
||||
nonSyncedFolder.id,
|
||||
]);
|
||||
});
|
||||
|
||||
it('should return empty array when SELECTED_FOLDERS policy and no folders are synced', async () => {
|
||||
const nonSyncedFolder1 = createMockFolder({
|
||||
name: 'Personal',
|
||||
externalId: 'personal-id',
|
||||
isSynced: false,
|
||||
});
|
||||
|
||||
const nonSyncedFolder2 = createMockFolder({
|
||||
name: 'Work',
|
||||
externalId: 'work-id',
|
||||
isSynced: false,
|
||||
});
|
||||
|
||||
const result = await service.getMessageLists({
|
||||
connectedAccount: mockConnectedAccount,
|
||||
messageChannel: {
|
||||
syncCursor: '',
|
||||
id: 'channel-1',
|
||||
messageFolderImportPolicy: MessageFolderImportPolicy.SELECTED_FOLDERS,
|
||||
},
|
||||
messageFolders: [nonSyncedFolder1, nonSyncedFolder2],
|
||||
});
|
||||
|
||||
expect(result).toEqual([]);
|
||||
});
|
||||
|
||||
it('should process all non-synced folders when ALL_FOLDERS policy is set', async () => {
|
||||
const mockClient = createMockMicrosoftClient();
|
||||
|
||||
(
|
||||
oAuth2ClientManagerService.getMicrosoftOAuth2Client as jest.Mock
|
||||
).mockResolvedValue(mockClient);
|
||||
|
||||
const nonSyncedFolder1 = createMockFolder({
|
||||
name: 'Personal',
|
||||
externalId: 'personal-id',
|
||||
isSynced: false,
|
||||
});
|
||||
|
||||
const nonSyncedFolder2 = createMockFolder({
|
||||
name: 'Work',
|
||||
externalId: 'work-id',
|
||||
isSynced: false,
|
||||
});
|
||||
|
||||
const result = await service.getMessageLists({
|
||||
connectedAccount: mockConnectedAccount,
|
||||
messageChannel: {
|
||||
syncCursor: '',
|
||||
id: 'channel-1',
|
||||
messageFolderImportPolicy: MessageFolderImportPolicy.ALL_FOLDERS,
|
||||
},
|
||||
messageFolders: [nonSyncedFolder1, nonSyncedFolder2],
|
||||
});
|
||||
|
||||
expect(result).toHaveLength(2);
|
||||
});
|
||||
|
||||
it('should return empty array when ALL_FOLDERS policy but messageFolders array is empty', async () => {
|
||||
const result = await service.getMessageLists({
|
||||
connectedAccount: mockConnectedAccount,
|
||||
messageChannel: {
|
||||
syncCursor: '',
|
||||
id: 'channel-1',
|
||||
messageFolderImportPolicy: MessageFolderImportPolicy.ALL_FOLDERS,
|
||||
},
|
||||
messageFolders: [],
|
||||
});
|
||||
|
||||
expect(result).toEqual([]);
|
||||
});
|
||||
});
|
||||
});
|
||||
+13
-9
@@ -10,11 +10,8 @@ import pLimit from 'p-limit';
|
||||
|
||||
import { OAuth2ClientManagerService } from 'src/modules/connected-account/oauth2-client-manager/services/oauth2-client-manager.service';
|
||||
import { type ConnectedAccountWorkspaceEntity } from 'src/modules/connected-account/standard-objects/connected-account.workspace-entity';
|
||||
import { MessageFolderImportPolicy } from 'src/modules/messaging/common/standard-objects/message-channel.workspace-entity';
|
||||
import { type MessageFolderWorkspaceEntity } from 'src/modules/messaging/common/standard-objects/message-folder.workspace-entity';
|
||||
import {
|
||||
MessageImportDriverException,
|
||||
MessageImportDriverExceptionCode,
|
||||
} from 'src/modules/messaging/message-import-manager/drivers/exceptions/message-import-driver.exception';
|
||||
import { MicrosoftMessageListFetchErrorHandler } from 'src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-message-list-fetch-error-handler.service';
|
||||
import { type GetMessageListsArgs } from 'src/modules/messaging/message-import-manager/types/get-message-lists-args.type';
|
||||
import {
|
||||
@@ -41,17 +38,24 @@ export class MicrosoftGetMessageListService {
|
||||
connectedAccount,
|
||||
messageFolders,
|
||||
}: GetMessageListsArgs): Promise<GetMessageListsResponse> {
|
||||
if (messageFolders.length === 0) {
|
||||
throw new MessageImportDriverException(
|
||||
`Message channel ${messageChannel.id} has no message folders`,
|
||||
MessageImportDriverExceptionCode.NOT_FOUND,
|
||||
const foldersToProcess =
|
||||
messageChannel.messageFolderImportPolicy ===
|
||||
MessageFolderImportPolicy.SELECTED_FOLDERS
|
||||
? messageFolders.filter((folder) => folder.isSynced)
|
||||
: messageFolders;
|
||||
|
||||
if (foldersToProcess.length === 0) {
|
||||
this.logger.warn(
|
||||
`Connected account ${connectedAccount.id}: Message Channel: ${messageChannel.id}: No folders to process`,
|
||||
);
|
||||
|
||||
return [];
|
||||
}
|
||||
|
||||
const limit = pLimit(FOLDER_PROCESSING_CONCURRENCY);
|
||||
|
||||
const results = await Promise.all(
|
||||
messageFolders.map((folder) =>
|
||||
foldersToProcess.map((folder) =>
|
||||
limit(async () => {
|
||||
const response = await this.getMessageList(connectedAccount, folder);
|
||||
|
||||
|
||||
+6
-1
@@ -6,7 +6,10 @@ import { CacheStorageService } from 'src/engine/core-modules/cache-storage/servi
|
||||
import { CacheStorageNamespace } from 'src/engine/core-modules/cache-storage/types/cache-storage-namespace.enum';
|
||||
import { GlobalWorkspaceOrmManager } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager';
|
||||
import { MessageChannelSyncStatusService } from 'src/modules/messaging/common/services/message-channel-sync-status.service';
|
||||
import { type MessageChannelWorkspaceEntity } from 'src/modules/messaging/common/standard-objects/message-channel.workspace-entity';
|
||||
import {
|
||||
MessageFolderImportPolicy,
|
||||
type MessageChannelWorkspaceEntity,
|
||||
} from 'src/modules/messaging/common/standard-objects/message-channel.workspace-entity';
|
||||
import {
|
||||
MessageFolderPendingSyncAction,
|
||||
type MessageFolderWorkspaceEntity,
|
||||
@@ -54,6 +57,7 @@ describe('MessagingMessageListFetchService', () => {
|
||||
messageChannelId: 'microsoft-message-channel-id',
|
||||
} as MessageFolderWorkspaceEntity,
|
||||
],
|
||||
messageFolderImportPolicy: MessageFolderImportPolicy.SELECTED_FOLDERS,
|
||||
} as MessageChannelWorkspaceEntity;
|
||||
|
||||
mockGoogleMessageChannel = {
|
||||
@@ -68,6 +72,7 @@ describe('MessagingMessageListFetchService', () => {
|
||||
},
|
||||
syncCursor: 'google-sync-cursor',
|
||||
messageFolders: [],
|
||||
messageFolderImportPolicy: MessageFolderImportPolicy.SELECTED_FOLDERS,
|
||||
} as unknown as MessageChannelWorkspaceEntity;
|
||||
});
|
||||
|
||||
|
||||
+1
-7
@@ -16,7 +16,6 @@ import {
|
||||
MessageChannelPendingGroupEmailsAction,
|
||||
MessageChannelSyncStage,
|
||||
MessageChannelWorkspaceEntity,
|
||||
MessageFolderImportPolicy,
|
||||
} from 'src/modules/messaging/common/standard-objects/message-channel.workspace-entity';
|
||||
import { MessageFolderPendingSyncAction } from 'src/modules/messaging/common/standard-objects/message-folder.workspace-entity';
|
||||
import { MessagingMessageCleanerService } from 'src/modules/messaging/message-cleaner/services/messaging-message-cleaner.service';
|
||||
@@ -129,12 +128,7 @@ export class MessagingMessageListFetchService {
|
||||
workspaceId,
|
||||
});
|
||||
|
||||
const messageFoldersToSync = (
|
||||
messageChannelWithFreshTokens.messageFolderImportPolicy ===
|
||||
MessageFolderImportPolicy.ALL_FOLDERS
|
||||
? messageFolders
|
||||
: messageFolders.filter((folder) => folder.isSynced)
|
||||
).filter(
|
||||
const messageFoldersToSync = messageFolders.filter(
|
||||
(folder) =>
|
||||
folder.pendingSyncAction === MessageFolderPendingSyncAction.NONE,
|
||||
);
|
||||
|
||||
+4
-1
@@ -4,7 +4,10 @@ import { type ConnectedAccountWorkspaceEntity } from 'src/modules/connected-acco
|
||||
import { type MessageChannelWorkspaceEntity } from 'src/modules/messaging/common/standard-objects/message-channel.workspace-entity';
|
||||
|
||||
export type GetMessageListsArgs = {
|
||||
messageChannel: Pick<MessageChannelWorkspaceEntity, 'syncCursor' | 'id'>;
|
||||
messageChannel: Pick<
|
||||
MessageChannelWorkspaceEntity,
|
||||
'syncCursor' | 'id' | 'messageFolderImportPolicy'
|
||||
>;
|
||||
connectedAccount: Pick<
|
||||
ConnectedAccountWorkspaceEntity,
|
||||
| 'provider'
|
||||
|
||||
Reference in New Issue
Block a user