houseassassin/twenty
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

exclude Gmail category labels only for system folders (#17640)

Browse Source 
Previous code in `MESSAGING_GMAIL_DEFAULT_NOT_SYNCED_LABELS` was
problematic as we grouped `category` labels along with `system folders`
labels together.

This fixes partially missing emails issue by splitting it and not
applying category exclusion when querying for a singular custom label.

Also removes old approach of getting message label_id's association from
additional network call overhead to local utility
`filterGmailMessagesByFolderPolicy`
This commit is contained in:
neo773
2026-02-04 23:27:48 +05:30
committed by GitHub
parent 64700a00b0
commit 752359335e
20 changed files with 464 additions and 279 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/twenty-server/src/modules/messaging/message-folder-manager/drivers/gmail/services/gmail-get-all-folders.service.ts
+2 -2
View File
@@ -14,7 +14,7 @@ import { GmailFoldersErrorHandlerService } from 'src/modules/messaging/message-f
import { extractGmailFolderName } from 'src/modules/messaging/message-folder-manager/drivers/gmail/utils/extract-gmail-folder-name.util';
import { getGmailFolderParentId } from 'src/modules/messaging/message-folder-manager/drivers/gmail/utils/get-gmail-folder-parent-id.util';
import { shouldSyncFolderByDefault } from 'src/modules/messaging/message-folder-manager/utils/should-sync-folder-by-default.util';
import { MESSAGING_GMAIL_DEFAULT_NOT_SYNCED_LABELS } from 'src/modules/messaging/message-import-manager/drivers/gmail/constants/messaging-gmail-default-not-synced-labels';
import { MESSAGING_GMAIL_DEFAULT_EXCLUDED_LABELS } from 'src/modules/messaging/message-import-manager/drivers/gmail/constants/messaging-gmail-default-excluded-labels.constant';
@Injectable()
export class GmailGetAllFoldersService implements MessageFolderDriver {
@@ -77,7 +77,7 @@ export class GmailGetAllFoldersService implements MessageFolderDriver {
continue;
}
if (MESSAGING_GMAIL_DEFAULT_NOT_SYNCED_LABELS.includes(label.id)) {
if (MESSAGING_GMAIL_DEFAULT_EXCLUDED_LABELS.includes(label.id)) {
continue;
}
packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/constants/messaging-gmail-default-excluded-labels.constant.ts
+7
View File
@@ -0,0 +1,7 @@
import { MESSAGING_GMAIL_EXCLUDED_CATEGORY_LABELS } from './messaging-gmail-excluded-category-labels.constant';
import { MESSAGING_GMAIL_EXCLUDED_SYSTEM_LABELS } from './messaging-gmail-excluded-system-labels.constant';
export const MESSAGING_GMAIL_DEFAULT_EXCLUDED_LABELS = [
...MESSAGING_GMAIL_EXCLUDED_CATEGORY_LABELS,
...MESSAGING_GMAIL_EXCLUDED_SYSTEM_LABELS,
];
packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/constants/messaging-gmail-default-not-synced-labels.ts
-10
View File
@@ -1,10 +0,0 @@
export const MESSAGING_GMAIL_DEFAULT_NOT_SYNCED_LABELS = [
'CATEGORY_PROMOTIONS',
'CATEGORY_SOCIAL',
'CATEGORY_FORUMS',
'CATEGORY_UPDATES',
'TRASH',
'SPAM',
'DRAFT',
'CHAT',
];
packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/constants/messaging-gmail-excluded-category-labels.constant.ts
+6
View File
@@ -0,0 +1,6 @@
export const MESSAGING_GMAIL_EXCLUDED_CATEGORY_LABELS = [
'CATEGORY_PROMOTIONS',
'CATEGORY_SOCIAL',
'CATEGORY_FORUMS',
'CATEGORY_UPDATES',
];
packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/constants/messaging-gmail-excluded-system-labels.constant.ts
+6
View File
@@ -0,0 +1,6 @@
export const MESSAGING_GMAIL_EXCLUDED_SYSTEM_LABELS = [
'TRASH',
'SPAM',
'DRAFT',
'CHAT',
];
packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/constants/messaging-gmail-folders-with-category-exclusions.constant.ts
+5
View File
@@ -0,0 +1,5 @@
export const MESSAGING_GMAIL_FOLDERS_WITH_CATEGORY_EXCLUSIONS = [
'INBOX',
'IMPORTANT',
'SENT',
];
packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/services/gmail-get-message-list.service.spec.ts
-170
View File
@@ -449,174 +449,4 @@ describe('GmailGetMessageListService', () => {
expect(callArgs.q).not.toContain('label:inbox');
});
});
describe('incremental sync folder filtering', () => {
it('should filter out messages from disabled folders during incremental sync', async () => {
const mockHistoryService = {
getHistory: jest.fn(),
getMessageIdsFromHistory: jest.fn(),
};
const module: TestingModule = await Test.createTestingModule({
providers: [
GmailGetMessageListService,
{
provide: OAuth2ClientManagerService,
useValue: {
getGoogleOAuth2Client: jest.fn().mockResolvedValue({}),
},
},
{
provide: GmailGetHistoryService,
useValue: mockHistoryService,
},
{
provide: GmailMessageListFetchErrorHandler,
useValue: { handleError: jest.fn() },
},
],
}).compile();
const testService = module.get<GmailGetMessageListService>(
GmailGetMessageListService,
);
mockHistoryService.getHistory.mockImplementation(
(_client, _cursor, _types, labelId) => {
if (labelId === 'Label_personal') {
return Promise.resolve({
history: [
{ messagesAdded: [{ message: { id: 'personal-msg' } }] },
],
historyId: 'new-cursor',
});
}
if (labelId === undefined) {
return Promise.resolve({
history: [{ messagesAdded: [{ message: { id: 'inbox-msg' } }] }],
historyId: 'new-cursor',
});
}
return Promise.resolve({ history: [], historyId: 'new-cursor' });
},
);
mockHistoryService.getMessageIdsFromHistory.mockResolvedValue({
messagesAdded: ['inbox-msg', 'personal-msg'],
messagesDeleted: [],
});
jest.spyOn(google, 'gmail').mockReturnValue({} as never);
const result = await testService.getMessageLists({
messageChannel: {
syncCursor: 'old-cursor',
id: 'channel-1',
messageFolderImportPolicy: MessageFolderImportPolicy.SELECTED_FOLDERS,
},
connectedAccount: mockConnectedAccount,
messageFolders: [
createMockFolder({
name: 'INBOX',
externalId: 'INBOX',
isSynced: true,
}),
createMockFolder({
name: 'Personal',
externalId: 'Label_personal',
isSynced: false,
}),
],
});
expect(result[0].messageExternalIds).toEqual(['inbox-msg']);
const allHistoryCalls = mockHistoryService.getHistory.mock.calls;
expect(allHistoryCalls[0]).toHaveLength(2);
expect(allHistoryCalls[0][1]).toBe('old-cursor');
const labelIdsQueried = allHistoryCalls
.slice(1)
.map((call) => call[3])
.filter(Boolean);
expect(labelIdsQueried).toContain('Label_personal');
expect(labelIdsQueried).toHaveLength(1);
// 1 main history call + 1 excluded folder call
expect(mockHistoryService.getHistory).toHaveBeenCalledTimes(2);
});
it('should skip per-folder filtering when ALL_FOLDERS policy is set', async () => {
const mockHistoryService = {
getHistory: jest.fn(),
getMessageIdsFromHistory: jest.fn(),
};
const module: TestingModule = await Test.createTestingModule({
providers: [
GmailGetMessageListService,
{
provide: OAuth2ClientManagerService,
useValue: {
getGoogleOAuth2Client: jest.fn().mockResolvedValue({}),
},
},
{
provide: GmailGetHistoryService,
useValue: mockHistoryService,
},
{
provide: GmailMessageListFetchErrorHandler,
useValue: { handleError: jest.fn() },
},
],
}).compile();
const testService = module.get<GmailGetMessageListService>(
GmailGetMessageListService,
);
mockHistoryService.getHistory.mockResolvedValue({
history: [],
historyId: 'new-cursor',
});
mockHistoryService.getMessageIdsFromHistory.mockResolvedValue({
messagesAdded: ['inbox-msg', 'personal-msg'],
messagesDeleted: [],
});
jest.spyOn(google, 'gmail').mockReturnValue({} as never);
const result = await testService.getMessageLists({
messageChannel: {
syncCursor: 'old-cursor',
id: 'channel-1',
messageFolderImportPolicy: MessageFolderImportPolicy.ALL_FOLDERS,
},
connectedAccount: mockConnectedAccount,
messageFolders: [
createMockFolder({
name: 'INBOX',
externalId: 'INBOX',
isSynced: true,
}),
createMockFolder({
name: 'Personal',
externalId: 'Label_personal',
isSynced: false,
}),
],
});
expect(result[0].messageExternalIds).toEqual([
'inbox-msg',
'personal-msg',
]);
expect(mockHistoryService.getHistory).toHaveBeenCalledTimes(1);
});
});
});
packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/services/gmail-get-message-list.service.ts
+1 -80
View File
@@ -1,9 +1,7 @@
import { Injectable, Logger } from '@nestjs/common';
import { batchFetchImplementation } from '@jrmdayn/googleapis-batcher';
import { isNonEmptyString } from '@sniptt/guards';
import { google } from 'googleapis';
import { isDefined } from 'twenty-shared/utils';
import { OAuth2ClientManagerService } from 'src/modules/connected-account/oauth2-client-manager/services/oauth2-client-manager.service';
import { type ConnectedAccountWorkspaceEntity } from 'src/modules/connected-account/standard-objects/connected-account.workspace-entity';
@@ -22,9 +20,6 @@ import { GmailMessageListFetchErrorHandler } from 'src/modules/messaging/message
import { computeGmailExcludeSearchFilter } from 'src/modules/messaging/message-import-manager/drivers/gmail/utils/compute-gmail-exclude-search-filter.util';
import { type GetMessageListsArgs } from 'src/modules/messaging/message-import-manager/types/get-message-lists-args.type';
import { type GetMessageListsResponse } from 'src/modules/messaging/message-import-manager/types/get-message-lists-response.type';
import { assertNotNull } from 'src/utils/assert';
const GMAIL_BATCH_REQUEST_MAX_SIZE = 50;
@Injectable()
export class GmailGetMessageListService {
@@ -196,20 +191,6 @@ export class GmailGetMessageListService {
const { messagesAdded, messagesDeleted } =
await this.gmailGetHistoryService.getMessageIdsFromHistory(history);
const messageIdsToFilter =
messageChannel.messageFolderImportPolicy ===
MessageFolderImportPolicy.SELECTED_FOLDERS
? await this.getEmailIdsFromExcludedFolders(
connectedAccount,
messageChannel.syncCursor,
messageFolders,
)
: [];
const messagesAddedFiltered = messagesAdded.filter(
(messageId) => !messageIdsToFilter.includes(messageId),
);
if (!nextSyncCursor) {
throw new MessageImportDriverException(
`No nextSyncCursor found for connected account ${connectedAccount.id}`,
@@ -219,7 +200,7 @@ export class GmailGetMessageListService {
return [
{
messageExternalIds: messagesAddedFiltered,
messageExternalIds: messagesAdded,
messageExternalIdsToDelete: messagesDeleted,
previousSyncCursor: messageChannel.syncCursor,
nextSyncCursor,
@@ -227,64 +208,4 @@ export class GmailGetMessageListService {
},
];
}
private async getEmailIdsFromExcludedFolders(
connectedAccount: Pick<
ConnectedAccountWorkspaceEntity,
'provider' | 'accessToken' | 'refreshToken' | 'id' | 'handle'
>,
lastSyncHistoryId: string,
messageFolders: Pick<
MessageFolderWorkspaceEntity,
'name' | 'externalId' | 'isSynced'
>[],
): Promise<string[]> {
const toBeExcludedFolders = messageFolders.filter(
(folder) => !folder.isSynced && isDefined(folder.externalId),
);
if (toBeExcludedFolders.length === 0) {
return [];
}
const oAuth2Client =
await this.oAuth2ClientManagerService.getGoogleOAuth2Client(
connectedAccount,
);
const batchedFetchImplementation = batchFetchImplementation({
maxBatchSize: GMAIL_BATCH_REQUEST_MAX_SIZE,
});
const batchedGmailClient = google.gmail({
version: 'v1',
auth: oAuth2Client,
fetchImplementation: batchedFetchImplementation,
});
const historyPromises = toBeExcludedFolders.map((folder) =>
this.gmailGetHistoryService.getHistory(
batchedGmailClient,
lastSyncHistoryId,
['messageAdded'],
folder.externalId!,
),
);
const historyResults = await Promise.all(historyPromises);
const emailIds: string[] = [];
for (const { history } of historyResults) {
const emailIdsFromCategory = history
.map((historyItem) => historyItem.messagesAdded)
.flat()
.map((message) => message?.message?.id)
.filter((id) => id)
.filter(assertNotNull);
emailIds.push(...emailIdsFromCategory);
}
return emailIds;
}
}
packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/services/gmail-get-messages.service.ts
+12 -1
View File
@@ -6,7 +6,9 @@ import { isDefined } from 'twenty-shared/utils';
import { OAuth2ClientManagerService } from 'src/modules/connected-account/oauth2-client-manager/services/oauth2-client-manager.service';
import { type ConnectedAccountWorkspaceEntity } from 'src/modules/connected-account/standard-objects/connected-account.workspace-entity';
import { MessageChannelWorkspaceEntity } from 'src/modules/messaging/common/standard-objects/message-channel.workspace-entity';
import { GmailMessagesImportErrorHandler } from 'src/modules/messaging/message-import-manager/drivers/gmail/services/gmail-messages-import-error-handler.service';
import { filterGmailMessagesByFolderPolicy } from 'src/modules/messaging/message-import-manager/drivers/gmail/utils/filter-gmail-messages-by-folder-policy.util';
import { parseAndFormatGmailMessage } from 'src/modules/messaging/message-import-manager/drivers/gmail/utils/parse-and-format-gmail-message.util';
import { type MessageWithParticipants } from 'src/modules/messaging/message-import-manager/types/message';
@@ -30,6 +32,10 @@ export class GmailGetMessagesService {
| 'handle'
| 'handleAliases'
>,
messageChannel: Pick<
MessageChannelWorkspaceEntity,
'messageFolders' | 'messageFolderImportPolicy'
>,
): Promise<MessageWithParticipants[]> {
const oAuth2Client =
await this.oAuth2ClientManagerService.getGoogleOAuth2Client(
@@ -73,6 +79,11 @@ export class GmailGetMessagesService {
})
.filter(isDefined);
return messages;
const filteredMessages = filterGmailMessagesByFolderPolicy(
messages,
messageChannel,
);
return filteredMessages;
}
}
packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/utils/__tests__/compute-gmail-exclude-search-filter.spec.ts
+74 -3
View File
@@ -19,7 +19,7 @@ describe('computeGmailExcludeSearchFilter', () => {
expect(result).toBe('');
});
it('builds positive OR query for selected folders', () => {
it('builds positive OR query for selected user labels without category exclusions', () => {
const result = computeGmailExcludeSearchFilter(
[
{
@@ -45,7 +45,66 @@ describe('computeGmailExcludeSearchFilter', () => {
);
expect(result).toContain('(label:crm OR label:twenty-visible)');
expect(result).not.toContain('-label:inbox');
expect(result).toContain('-label:spam');
expect(result).toContain('-label:trash');
expect(result).not.toContain('-category:promotions');
expect(result).not.toContain('-category:updates');
});
it('includes category exclusions when INBOX is selected', () => {
const result = computeGmailExcludeSearchFilter(
[
{
externalId: 'INBOX',
name: 'INBOX',
isSynced: true,
parentFolderId: null,
},
{
externalId: 'Label_1',
name: 'CRM',
isSynced: false,
parentFolderId: null,
},
],
MessageFolderImportPolicy.SELECTED_FOLDERS,
);
expect(result).toContain('label:inbox');
expect(result).toContain('-category:promotions');
expect(result).toContain('-category:social');
expect(result).toContain('-label:spam');
});
it('excludes category filters when user label and INBOX are both selected', () => {
const result = computeGmailExcludeSearchFilter(
[
{
externalId: 'INBOX',
name: 'INBOX',
isSynced: true,
parentFolderId: null,
},
{
externalId: 'Label_1',
name: 'CRM',
isSynced: true,
parentFolderId: null,
},
{
externalId: 'SENT',
name: 'SENT',
isSynced: false,
parentFolderId: null,
},
],
MessageFolderImportPolicy.SELECTED_FOLDERS,
);
expect(result).toContain('label:crm');
expect(result).toContain('label:inbox');
expect(result).not.toContain('-category:promotions');
expect(result).toContain('-label:spam');
});
it('returns only default exclusions when all folders are synced', () => {
@@ -116,7 +175,7 @@ describe('computeGmailExcludeSearchFilter', () => {
});
});
it('uses correct Gmail search syntax for labels and categories', () => {
it('uses -label: syntax for system exclusions', () => {
const result = computeGmailExcludeSearchFilter(
[
{
@@ -131,7 +190,19 @@ describe('computeGmailExcludeSearchFilter', () => {
expect(result).toContain('-label:trash');
expect(result).toContain('-label:spam');
expect(result).toContain('-label:draft');
expect(result).toContain('-label:chat');
});
it('uses -category: syntax for category exclusions in ALL_FOLDERS mode', () => {
const result = computeGmailExcludeSearchFilter(
[],
MessageFolderImportPolicy.ALL_FOLDERS,
);
expect(result).toContain('-category:promotions');
expect(result).toContain('-category:social');
expect(result).toContain('-category:forums');
expect(result).toContain('-category:updates');
});
});
packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/utils/__tests__/filter-out-gmail-excluded-folders.util.spec.ts
+243
View File
@@ -0,0 +1,243 @@
import { MessageFolderImportPolicy } from 'src/modules/messaging/common/standard-objects/message-channel.workspace-entity';
import { type MessageFolderWorkspaceEntity } from 'src/modules/messaging/common/standard-objects/message-folder.workspace-entity';
import { filterGmailMessagesByFolderPolicy } from 'src/modules/messaging/message-import-manager/drivers/gmail/utils/filter-gmail-messages-by-folder-policy.util';
import { type MessageWithParticipants } from 'src/modules/messaging/message-import-manager/types/message';
const createMessage = (
externalId: string,
labelIds: string[],
): MessageWithParticipants =>
({ externalId, labelIds }) as MessageWithParticipants;
const createFolder = (
externalId: string,
isSynced: boolean,
): MessageFolderWorkspaceEntity =>
({ externalId, isSynced }) as MessageFolderWorkspaceEntity;
describe('filterGmailMessagesByFolderPolicy', () => {
describe('ALL_FOLDERS policy', () => {
it('bypasses all filtering', () => {
const messages = [
createMessage('1', ['SPAM', 'CATEGORY_PROMOTIONS']),
createMessage('2', ['TRASH']),
createMessage('3', ['INBOX', 'CATEGORY_SOCIAL']),
];
const result = filterGmailMessagesByFolderPolicy(messages, {
messageFolders: [createFolder('INBOX', true)],
messageFolderImportPolicy: MessageFolderImportPolicy.ALL_FOLDERS,
});
expect(result).toHaveLength(3);
});
});
describe('only custom labels synced', () => {
const CRM_LABEL = 'Label_CRM';
const DEALS_LABEL = 'Label_Deals';
it('includes message with synced label even if it also has non-synced labels', () => {
const messages = [
createMessage('1', [
CRM_LABEL,
DEALS_LABEL,
'IMPORTANT',
'CATEGORY_PERSONAL',
'INBOX',
]),
createMessage('2', ['IMPORTANT', 'CATEGORY_PERSONAL', 'INBOX']),
createMessage('3', ['SENT']),
];
const result = filterGmailMessagesByFolderPolicy(messages, {
messageFolders: [
createFolder('INBOX', false),
createFolder('SENT', false),
createFolder('IMPORTANT', false),
createFolder(CRM_LABEL, true),
createFolder(DEALS_LABEL, true),
],
messageFolderImportPolicy: MessageFolderImportPolicy.SELECTED_FOLDERS,
});
expect(result.map((m) => m.externalId)).toEqual(['1']);
});
it('ignores all category labels for custom folders', () => {
const messages = [
createMessage('1', [CRM_LABEL, 'CATEGORY_PROMOTIONS']),
createMessage('2', [CRM_LABEL, 'CATEGORY_SOCIAL']),
createMessage('3', [CRM_LABEL, 'CATEGORY_FORUMS']),
createMessage('4', [CRM_LABEL, 'CATEGORY_UPDATES']),
createMessage('5', [CRM_LABEL, 'CATEGORY_PERSONAL']),
];
const result = filterGmailMessagesByFolderPolicy(messages, {
messageFolders: [createFolder(CRM_LABEL, true)],
messageFolderImportPolicy: MessageFolderImportPolicy.SELECTED_FOLDERS,
});
expect(result).toHaveLength(5);
});
});
describe('only system folders synced (INBOX/SENT/IMPORTANT)', () => {
it('excludes promotional/social/forums/updates from INBOX', () => {
const messages = [
createMessage('1', ['INBOX']),
createMessage('2', ['INBOX', 'CATEGORY_PROMOTIONS']),
createMessage('3', ['INBOX', 'CATEGORY_SOCIAL']),
createMessage('4', ['INBOX', 'CATEGORY_FORUMS']),
createMessage('5', ['INBOX', 'CATEGORY_UPDATES']),
];
const result = filterGmailMessagesByFolderPolicy(messages, {
messageFolders: [createFolder('INBOX', true)],
messageFolderImportPolicy: MessageFolderImportPolicy.SELECTED_FOLDERS,
});
expect(result.map((m) => m.externalId)).toEqual(['1']);
});
it('does NOT exclude CATEGORY_PERSONAL (intentionally allowed)', () => {
const messages = [
createMessage('1', ['INBOX', 'CATEGORY_PERSONAL']),
createMessage('2', ['INBOX', 'CATEGORY_PROMOTIONS']),
];
const result = filterGmailMessagesByFolderPolicy(messages, {
messageFolders: [createFolder('INBOX', true)],
messageFolderImportPolicy: MessageFolderImportPolicy.SELECTED_FOLDERS,
});
expect(result.map((m) => m.externalId)).toEqual(['1']);
});
it('applies category exclusions to SENT folder', () => {
const messages = [
createMessage('1', ['SENT']),
createMessage('2', ['SENT', 'CATEGORY_PROMOTIONS']),
];
const result = filterGmailMessagesByFolderPolicy(messages, {
messageFolders: [createFolder('SENT', true)],
messageFolderImportPolicy: MessageFolderImportPolicy.SELECTED_FOLDERS,
});
expect(result.map((m) => m.externalId)).toEqual(['1']);
});
it('applies category exclusions to IMPORTANT folder', () => {
const messages = [
createMessage('1', ['IMPORTANT']),
createMessage('2', ['IMPORTANT', 'CATEGORY_SOCIAL']),
];
const result = filterGmailMessagesByFolderPolicy(messages, {
messageFolders: [createFolder('IMPORTANT', true)],
messageFolderImportPolicy: MessageFolderImportPolicy.SELECTED_FOLDERS,
});
expect(result.map((m) => m.externalId)).toEqual(['1']);
});
});
describe('STARRED folder synced (not a category-exclusion folder)', () => {
it('does NOT apply category exclusions to STARRED', () => {
const messages = [
createMessage('1', ['STARRED']),
createMessage('2', ['STARRED', 'CATEGORY_PROMOTIONS']),
createMessage('3', ['STARRED', 'CATEGORY_SOCIAL']),
];
const result = filterGmailMessagesByFolderPolicy(messages, {
messageFolders: [createFolder('STARRED', true)],
messageFolderImportPolicy: MessageFolderImportPolicy.SELECTED_FOLDERS,
});
expect(result).toHaveLength(3);
});
});
describe('mixed: custom label + system folder synced', () => {
const SALES_LABEL = 'Label_Sales';
it('includes promo email if in custom label, excludes if only in INBOX', () => {
const messages = [
createMessage('1', ['INBOX', 'CATEGORY_PROMOTIONS']),
createMessage('2', ['INBOX', 'CATEGORY_PROMOTIONS', SALES_LABEL]),
createMessage('3', ['INBOX']),
];
const result = filterGmailMessagesByFolderPolicy(messages, {
messageFolders: [
createFolder('INBOX', true),
createFolder(SALES_LABEL, true),
],
messageFolderImportPolicy: MessageFolderImportPolicy.SELECTED_FOLDERS,
});
expect(result.map((m) => m.externalId)).toEqual(['2', '3']);
});
it('requires message to be in at least one synced folder', () => {
const messages = [
createMessage('1', ['TRASH']),
createMessage('2', [SALES_LABEL]),
createMessage('3', ['INBOX', 'SPAM']),
];
const result = filterGmailMessagesByFolderPolicy(messages, {
messageFolders: [
createFolder('INBOX', true),
createFolder(SALES_LABEL, true),
createFolder('TRASH', false),
createFolder('SPAM', false),
],
messageFolderImportPolicy: MessageFolderImportPolicy.SELECTED_FOLDERS,
});
expect(result.map((m) => m.externalId)).toEqual(['2', '3']);
});
});
describe('edge cases', () => {
it('excludes messages not in any synced folder', () => {
const messages = [
createMessage('1', ['TRASH']),
createMessage('2', ['SPAM']),
createMessage('3', ['DRAFT']),
];
const result = filterGmailMessagesByFolderPolicy(messages, {
messageFolders: [createFolder('INBOX', true)],
messageFolderImportPolicy: MessageFolderImportPolicy.SELECTED_FOLDERS,
});
expect(result).toHaveLength(0);
});
it('handles messages with empty labelIds', () => {
const messages = [createMessage('1', [])];
const result = filterGmailMessagesByFolderPolicy(messages, {
messageFolders: [createFolder('INBOX', true)],
messageFolderImportPolicy: MessageFolderImportPolicy.SELECTED_FOLDERS,
});
expect(result).toHaveLength(0);
});
it('handles empty messageFolders array', () => {
const messages = [createMessage('1', ['INBOX'])];
const result = filterGmailMessagesByFolderPolicy(messages, {
messageFolders: [],
messageFolderImportPolicy: MessageFolderImportPolicy.SELECTED_FOLDERS,
});
expect(result).toHaveLength(0);
});
});
});
packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/utils/compute-gmail-exclude-search-filter.util.ts
+25 -7
View File
@@ -2,7 +2,9 @@ import { isDefined } from 'twenty-shared/utils';
import { MessageFolderImportPolicy } from 'src/modules/messaging/common/standard-objects/message-channel.workspace-entity';
import { type MessageFolderWorkspaceEntity } from 'src/modules/messaging/common/standard-objects/message-folder.workspace-entity';
import { MESSAGING_GMAIL_DEFAULT_NOT_SYNCED_LABELS } from 'src/modules/messaging/message-import-manager/drivers/gmail/constants/messaging-gmail-default-not-synced-labels';
import { MESSAGING_GMAIL_DEFAULT_EXCLUDED_LABELS } from 'src/modules/messaging/message-import-manager/drivers/gmail/constants/messaging-gmail-default-excluded-labels.constant';
import { MESSAGING_GMAIL_EXCLUDED_SYSTEM_LABELS } from 'src/modules/messaging/message-import-manager/drivers/gmail/constants/messaging-gmail-excluded-system-labels.constant';
import { MESSAGING_GMAIL_FOLDERS_WITH_CATEGORY_EXCLUSIONS } from 'src/modules/messaging/message-import-manager/drivers/gmail/constants/messaging-gmail-folders-with-category-exclusions.constant';
import { buildGmailLabelSearchName } from 'src/modules/messaging/message-import-manager/drivers/gmail/utils/build-gmail-label-search-name.util';
import { computeGmailDefaultNotSyncedLabelsSearchFilter } from 'src/modules/messaging/message-import-manager/drivers/gmail/utils/compute-gmail-default-not-synced-labels-search-filter';
@@ -13,24 +15,29 @@ export const computeGmailExcludeSearchFilter = (
>[],
messageFolderImportPolicy: MessageFolderImportPolicy,
): string => {
const defaultExclusions = MESSAGING_GMAIL_DEFAULT_NOT_SYNCED_LABELS.map(
const allExclusions = MESSAGING_GMAIL_DEFAULT_EXCLUDED_LABELS.map(
computeGmailDefaultNotSyncedLabelsSearchFilter,
).join(' ');
const systemExclusions = MESSAGING_GMAIL_EXCLUDED_SYSTEM_LABELS.map(
computeGmailDefaultNotSyncedLabelsSearchFilter,
).join(' ');
if (messageFolderImportPolicy === MessageFolderImportPolicy.ALL_FOLDERS) {
return defaultExclusions;
return allExclusions;
}
const syncedFolders = messageFolders.filter((folder) => folder.isSynced);
const allFoldersSynced =
messageFolders.length > 0 &&
messageFolders.every((folder) => folder.isSynced);
if (allFoldersSynced) {
return defaultExclusions;
return allExclusions;
}
const labelNamesToInclude = messageFolders
.filter((folder) => folder.isSynced)
const labelNamesToInclude = syncedFolders
.map((folder) => buildGmailLabelSearchName(folder, messageFolders))
.filter(isDefined);
@@ -43,5 +50,16 @@ export const computeGmailExcludeSearchFilter = (
? `label:${labelNamesToInclude[0]}`
: `(${labelNamesToInclude.map((name) => `label:${name}`).join(' OR ')})`;
return `${inclusionQuery} ${defaultExclusions}`;
const hasCustomLabelSelected = syncedFolders.some(
(folder) =>
!MESSAGING_GMAIL_FOLDERS_WITH_CATEGORY_EXCLUSIONS.includes(
folder.externalId ?? '',
),
);
if (hasCustomLabelSelected) {
return `${inclusionQuery} ${systemExclusions}`;
}
return `${inclusionQuery} ${allExclusions}`;
};
packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/utils/filter-gmail-messages-by-folder-policy.util.ts
+53
View File
@@ -0,0 +1,53 @@
import {
type MessageChannelWorkspaceEntity,
MessageFolderImportPolicy,
} from 'src/modules/messaging/common/standard-objects/message-channel.workspace-entity';
import { MESSAGING_GMAIL_EXCLUDED_CATEGORY_LABELS } from 'src/modules/messaging/message-import-manager/drivers/gmail/constants/messaging-gmail-excluded-category-labels.constant';
import { MESSAGING_GMAIL_FOLDERS_WITH_CATEGORY_EXCLUSIONS } from 'src/modules/messaging/message-import-manager/drivers/gmail/constants/messaging-gmail-folders-with-category-exclusions.constant';
import { type MessageWithParticipants } from 'src/modules/messaging/message-import-manager/types/message';
export const filterGmailMessagesByFolderPolicy = (
messages: MessageWithParticipants[],
messageChannel: Pick<
MessageChannelWorkspaceEntity,
'messageFolders' | 'messageFolderImportPolicy'
>,
): MessageWithParticipants[] => {
const { messageFolders, messageFolderImportPolicy } = messageChannel;
if (messageFolderImportPolicy === MessageFolderImportPolicy.ALL_FOLDERS) {
return messages;
}
const syncedFolderExternalIds = (messageFolders ?? [])
.filter((folder) => folder.isSynced && folder.externalId)
.map((folder) => folder.externalId);
return messages.filter((message) => {
const messageLabelIds = message.labelIds ?? [];
const messageIsInAtLeastOneSyncedFolder = messageLabelIds.some((labelId) =>
syncedFolderExternalIds.includes(labelId),
);
if (!messageIsInAtLeastOneSyncedFolder) {
return false;
}
const messageIsInSyncedCustomFolder = messageLabelIds.some(
(labelId) =>
syncedFolderExternalIds.includes(labelId) &&
!MESSAGING_GMAIL_FOLDERS_WITH_CATEGORY_EXCLUSIONS.includes(labelId),
);
if (messageIsInSyncedCustomFolder) {
return true;
}
const messageHasExcludedCategoryLabel = messageLabelIds.some((labelId) =>
MESSAGING_GMAIL_EXCLUDED_CATEGORY_LABELS.includes(labelId),
);
return !messageHasExcludedCategoryLabel;
});
};
packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/utils/parse-and-format-gmail-message.util.ts
+2
View File
@@ -29,6 +29,7 @@ export const parseAndFormatGmailMessage = (
text,
attachments,
deliveredTo,
labelIds,
} = parseGmailMessage(message);
if (
@@ -83,5 +84,6 @@ export const parseAndFormatGmailMessage = (
participants,
text: sanitizeString(textWithoutReplyQuotations),
attachments,
labelIds,
};
};
packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/utils/parse-gmail-message.util.ts
+2
View File
@@ -19,6 +19,7 @@ export const parseGmailMessage = (message: gmail_v1.Schema$Message) => {
const threadId = message.threadId;
const historyId = message.historyId;
const internalDate = message.internalDate;
const labelIds = message.labelIds ?? [];
assert(id, 'ID is missing');
assert(historyId, 'History-ID is missing');
@@ -45,5 +46,6 @@ export const parseGmailMessage = (message: gmail_v1.Schema$Message) => {
bcc: rawBcc ? safeParseEmailAddressAddress(rawBcc) : undefined,
text,
attachments,
labelIds,
};
};
packages/twenty-server/src/modules/messaging/message-import-manager/jobs/messaging-messages-import.job.ts
+1 -1
View File
@@ -54,7 +54,7 @@ export class MessagingMessagesImportJob {
where: {
id: messageChannelId,
},
relations: ['connectedAccount'],
relations: ['connectedAccount', 'messageFolders'],
});
if (!messageChannel) {
packages/twenty-server/src/modules/messaging/message-import-manager/services/__tests__/messaging-messages-import.service.spec.ts
+17 -5
View File
@@ -13,6 +13,7 @@ import { type ConnectedAccountWorkspaceEntity } from 'src/modules/connected-acco
import { MessageChannelSyncStatusService } from 'src/modules/messaging/common/services/message-channel-sync-status.service';
import {
MessageChannelSyncStage,
MessageFolderImportPolicy,
type MessageChannelWorkspaceEntity,
} from 'src/modules/messaging/common/standard-objects/message-channel.workspace-entity';
import { MESSAGING_GMAIL_USERS_MESSAGES_GET_BATCH_SIZE } from 'src/modules/messaging/message-import-manager/drivers/gmail/constants/messaging-gmail-users-messages-get-batch-size.constant';
@@ -32,7 +33,15 @@ describe('MessagingMessagesImportService', () => {
let saveMessagesService: MessagingSaveMessagesAndEnqueueContactCreationService;
const workspaceId = 'workspace-id';
let mockMessageChannel: MessageChannelWorkspaceEntity;
let mockMessageChannel: Pick<
MessageChannelWorkspaceEntity,
| 'id'
| 'syncStage'
| 'connectedAccountId'
| 'handle'
| 'messageFolders'
| 'messageFolderImportPolicy'
>;
let mockConnectedAccount: ConnectedAccountWorkspaceEntity;
let providersBase: Provider[];
@@ -52,7 +61,9 @@ describe('MessagingMessagesImportService', () => {
syncStage: MessageChannelSyncStage.MESSAGES_IMPORT_SCHEDULED,
connectedAccountId: mockConnectedAccount.id,
handle: 'test@gmail.com',
} as MessageChannelWorkspaceEntity;
messageFolders: [],
messageFolderImportPolicy: MessageFolderImportPolicy.ALL_FOLDERS,
};
providersBase = [
MessagingMessagesImportService,
@@ -201,7 +212,7 @@ describe('MessagingMessagesImportService', () => {
expect(
service.processMessageBatchImport(
mockMessageChannel,
mockMessageChannel as MessageChannelWorkspaceEntity,
mockConnectedAccount,
workspaceId,
),
@@ -210,7 +221,7 @@ describe('MessagingMessagesImportService', () => {
it('should process message batch import successfully', async () => {
await service.processMessageBatchImport(
mockMessageChannel,
mockMessageChannel as MessageChannelWorkspaceEntity,
mockConnectedAccount,
workspaceId,
);
@@ -237,6 +248,7 @@ describe('MessagingMessagesImportService', () => {
accessToken: 'new-access-token',
refreshToken: 'new-refresh-token',
},
mockMessageChannel,
);
expect(
saveMessagesService.saveMessagesAndEnqueueContactCreation,
@@ -293,7 +305,7 @@ describe('MessagingMessagesImportService', () => {
);
await service.processMessageBatchImport(
mockMessageChannel,
mockMessageChannel as MessageChannelWorkspaceEntity,
mockConnectedAccount,
workspaceId,
);
packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-get-messages.service.ts
+6
View File
@@ -3,6 +3,7 @@ import { Injectable } from '@nestjs/common';
import { ConnectedAccountProvider } from 'twenty-shared/types';
import { type ConnectedAccountWorkspaceEntity } from 'src/modules/connected-account/standard-objects/connected-account.workspace-entity';
import { type MessageChannelWorkspaceEntity } from 'src/modules/messaging/common/standard-objects/message-channel.workspace-entity';
import {
MessageImportDriverException,
MessageImportDriverExceptionCode,
@@ -35,12 +36,17 @@ export class MessagingGetMessagesService {
| 'accountOwnerId'
| 'connectionParameters'
>,
messageChannel: Pick<
MessageChannelWorkspaceEntity,
'messageFolders' | 'messageFolderImportPolicy'
>,
): Promise<GetMessagesResponse> {
switch (connectedAccount.provider) {
case ConnectedAccountProvider.GOOGLE:
return this.gmailGetMessagesService.getMessages(
messageIds,
connectedAccount,
messageChannel,
);
case ConnectedAccountProvider.MICROSOFT:
return this.microsoftGetMessagesService.getMessages(
packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-messages-import.service.ts
+1
View File
@@ -121,6 +121,7 @@ export class MessagingMessagesImportService {
const allMessages = await this.messagingGetMessagesService.getMessages(
messageIdsToFetch,
connectedAccountWithFreshTokens,
messageChannel,
);
const blocklist = await this.blocklistRepository.getByWorkspaceMemberId(
packages/twenty-server/src/modules/messaging/message-import-manager/types/message.ts
+1
View File
@@ -19,6 +19,7 @@ export type Message = Omit<
externalId: string;
messageThreadExternalId: string;
direction: MessageDirection;
labelIds?: string[];
};
export type MessageAttachment = {