From 0489d39f2b9f4a24e237dfb022cfd662f966a021 Mon Sep 17 00:00:00 2001 From: neo773 <62795688+neo773@users.noreply.github.com> Date: Wed, 22 Jul 2026 17:17:45 +0530 Subject: [PATCH] fix(messaging): stop group and bulk emails from creating contacts (#23137) Group addresses were only checked against the message sender, so anything arriving as reply-to or cc slipped through and became a contact. Now checked per participant at contact creation. The group word list can't keep up with real senders (posts-recap@, showinfo@, follow-suggestions@), so bulk-mail headers back it up: `List-Unsubscribe, List-Id, Precedence, Auto-Submitted` (Industry standard headers) Review in cubic --- .../parse-and-format-gmail-message.util.ts | 2 + .../gmail/utils/parse-gmail-message.util.ts | 6 +++ .../services/imap-get-messages.service.ts | 4 ++ .../services/inbound-email-parser.service.ts | 4 ++ .../microsoft-fetch-by-batch.service.ts | 19 ++++++- .../microsoft-get-messages.interface.ts | 1 + .../microsoft-get-messages.service.spec.ts | 3 ++ .../microsoft-get-messages.service.ts | 1 + ...d-enqueue-contact-creation.service.spec.ts | 48 ++++++++++++++++++ ...es-and-enqueue-contact-creation.service.ts | 6 +++ .../message-import-manager/types/message.ts | 6 +++ .../__tests__/filter-emails.util.spec.ts | 32 ++++++++++++ .../utils/__tests__/is-bulk-mail.util.spec.ts | 50 +++++++++++++++++++ .../utils/filter-emails.util.ts | 5 ++ .../utils/is-bulk-mail.util.ts | 26 ++++++++++ 15 files changed, 212 insertions(+), 1 deletion(-) create mode 100644 packages/twenty-server/src/modules/messaging/message-import-manager/utils/__tests__/is-bulk-mail.util.spec.ts create mode 100644 packages/twenty-server/src/modules/messaging/message-import-manager/utils/is-bulk-mail.util.ts diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/utils/parse-and-format-gmail-message.util.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/utils/parse-and-format-gmail-message.util.ts index ff45f68336..fec60205f9 100644 --- a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/utils/parse-and-format-gmail-message.util.ts +++ b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/utils/parse-and-format-gmail-message.util.ts @@ -31,6 +31,7 @@ export const parseAndFormatGmailMessage = ( attachments, deliveredTo, labelIds, + messageHeaders, } = parseGmailMessage(message); const isDraft = (labelIds ?? []).includes('DRAFT'); @@ -86,5 +87,6 @@ export const parseAndFormatGmailMessage = ( messageFolderExternalIds: labelIds, labelIds, isDraft, + messageHeaders, }; }; diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/utils/parse-gmail-message.util.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/utils/parse-gmail-message.util.ts index 486ec9dd6a..029cb1e798 100644 --- a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/utils/parse-gmail-message.util.ts +++ b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/utils/parse-gmail-message.util.ts @@ -1,6 +1,7 @@ import assert from 'assert'; import { type gmail_v1 } from 'googleapis'; +import { isDefined } from 'twenty-shared/utils'; import { getAttachmentData } from 'src/modules/messaging/message-import-manager/drivers/gmail/utils/get-attachment-data.util'; import { getBodyData } from 'src/modules/messaging/message-import-manager/drivers/gmail/utils/get-body-data.util'; @@ -22,6 +23,10 @@ export const parseGmailMessage = (message: gmail_v1.Schema$Message) => { const historyId = message.historyId; const internalDate = message.internalDate; const labelIds = message.labelIds ?? []; + const messageHeaders = (message.payload?.headers ?? []).flatMap( + ({ name, value }) => + isDefined(name) && isDefined(value) ? [{ name, value }] : [], + ); assert(id, 'ID is missing'); assert(historyId, 'History-ID is missing'); @@ -54,5 +59,6 @@ export const parseGmailMessage = (message: gmail_v1.Schema$Message) => { isHtml, attachments, labelIds, + messageHeaders, }; }; diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/imap/services/imap-get-messages.service.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/imap/services/imap-get-messages.service.ts index c8138e2b59..6720745c12 100644 --- a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/imap/services/imap-get-messages.service.ts +++ b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/imap/services/imap-get-messages.service.ts @@ -187,6 +187,10 @@ export class ImapGetMessagesService { participants: extractParticipantsFromParsedEmail(parsed), messageFolderExternalIds: [folderExternalId], isDraft: flags?.has('\\Draft') ?? false, + messageHeaders: parsed.headers.map(({ key, value }) => ({ + name: key, + value, + })), }; } } diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/inbound-email/services/inbound-email-parser.service.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/inbound-email/services/inbound-email-parser.service.ts index 02e4898701..2e2b2a8f8e 100644 --- a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/inbound-email/services/inbound-email-parser.service.ts +++ b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/inbound-email/services/inbound-email-parser.service.ts @@ -36,6 +36,10 @@ export class InboundEmailParserService { attachments: [], participants: extractParticipantsFromParsedEmail(parsedEmail), isDraft: false, + messageHeaders: parsedEmail.headers.map(({ key, value }) => ({ + name: key, + value, + })), }; } } diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-fetch-by-batch.service.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-fetch-by-batch.service.ts index 475e95b33e..77929e6534 100644 --- a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-fetch-by-batch.service.ts +++ b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-fetch-by-batch.service.ts @@ -17,6 +17,23 @@ export class MicrosoftFetchByBatchService { messageIdsByBatch: string[][]; batchResponses: MicrosoftGraphBatchResponse[]; }> { + const selectedFields = [ + 'id', + 'subject', + 'body', + 'receivedDateTime', + 'internetMessageId', + 'internetMessageHeaders', + 'conversationId', + 'parentFolderId', + 'isDraft', + 'from', + 'replyTo', + 'toRecipients', + 'ccRecipients', + 'bccRecipients', + ].join(','); + const batchLimit = 20; const batchResponses: MicrosoftGraphBatchResponse[] = []; const messageIdsByBatch: string[][] = []; @@ -33,7 +50,7 @@ export class MicrosoftFetchByBatchService { const batchRequests = batchMessageIds.map((messageId, index) => ({ id: (index + 1).toString(), method: 'GET', - url: `/me/messages/${messageId}`, + url: `/me/messages/${messageId}?$select=${selectedFields}`, headers: { 'Content-Type': 'application/json', Prefer: 'outlook.body-content-type="text"', diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-get-messages.interface.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-get-messages.interface.ts index 4c4abaa9ff..6215ccdf04 100644 --- a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-get-messages.interface.ts +++ b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-get-messages.interface.ts @@ -19,6 +19,7 @@ export interface MicrosoftGraphBatchResponse { sentDateTime?: string; hasAttachments?: boolean; internetMessageId?: string; + internetMessageHeaders?: { name: string; value: string }[]; subject?: string; bodyPreview?: string; importance?: string; diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-get-messages.service.spec.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-get-messages.service.spec.ts index 426ccaccef..4182136620 100644 --- a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-get-messages.service.spec.ts +++ b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-get-messages.service.spec.ts @@ -104,6 +104,7 @@ describe('Microsoft get messages service', () => { messageFolderExternalIds: responseExample1.body.parentFolderId ? [responseExample1.body.parentFolderId] : [], + messageHeaders: [], }); const responseExample2 = @@ -151,6 +152,7 @@ describe('Microsoft get messages service', () => { messageFolderExternalIds: responseExample2.body.parentFolderId ? [responseExample2.body.parentFolderId] : [], + messageHeaders: [], }); }); @@ -195,6 +197,7 @@ describe('Microsoft get messages service', () => { messageFolderExternalIds: responseExample.body.parentFolderId ? [responseExample.body.parentFolderId] : [], + messageHeaders: [], }); }); }); diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-get-messages.service.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-get-messages.service.ts index ff67a241e0..d694def53e 100644 --- a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-get-messages.service.ts +++ b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-get-messages.service.ts @@ -164,6 +164,7 @@ export class MicrosoftGetMessagesService { ? [response.parentFolderId] : [], isDraft: response.isDraft ?? false, + messageHeaders: response.internetMessageHeaders ?? [], }; }); diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-save-messages-and-enqueue-contact-creation.service.spec.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-save-messages-and-enqueue-contact-creation.service.spec.ts index 71bf3a6f6d..3b4f9294bd 100644 --- a/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-save-messages-and-enqueue-contact-creation.service.spec.ts +++ b/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-save-messages-and-enqueue-contact-creation.service.spec.ts @@ -305,6 +305,54 @@ describe('MessagingSaveMessagesAndEnqueueContactCreationService', () => { }, ); }); + it('should not create contacts for non-sender group email participants when excludeGroupEmails is enabled', async () => { + await service.saveMessagesAndEnqueueContactCreation( + [ + { + ...mockMessages[1], + participants: [ + { + role: MessageParticipantRole.FROM, + handle: 'tim@apple.com', + displayName: 'Tim', + }, + { + role: MessageParticipantRole.REPLY_TO, + handle: 'no-reply@mail.instagram.com', + displayName: 'Instagram', + }, + { + role: MessageParticipantRole.CC, + handle: 'support@company.com', + displayName: 'Support', + }, + ], + }, + ], + mockMessageChannel, + mockConnectedAccount, + workspaceId, + ); + + expect(messageQueueService.add).toHaveBeenCalledWith( + CreateCompanyAndContactJob.name, + { + workspaceId, + connectedAccount: mockConnectedAccount, + source: FieldActorSource.EMAIL, + contactsToCreate: [ + { + handle: 'tim@apple.com', + displayName: 'Tim', + role: MessageParticipantRole.FROM, + shouldCreateContact: true, + messageId: 'db-message-id-2', + }, + ], + }, + ); + }); + it('should not create contact if the participant is the connected account', async () => { const mockMessagesWithConnectedAccount = [ { diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-save-messages-and-enqueue-contact-creation.service.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-save-messages-and-enqueue-contact-creation.service.ts index cd0e4bd876..2eb120ee56 100644 --- a/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-save-messages-and-enqueue-contact-creation.service.ts +++ b/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-save-messages-and-enqueue-contact-creation.service.ts @@ -29,6 +29,7 @@ import { } from 'src/modules/messaging/message-import-manager/services/messaging-message-folder-association.service'; import { MessagingMessageService } from 'src/modules/messaging/message-import-manager/services/messaging-message.service'; import { type MessageWithParticipants } from 'src/modules/messaging/message-import-manager/types/message'; +import { isGroupEmail } from 'src/modules/messaging/message-import-manager/utils/is-group-email'; import { MessagingMessageParticipantService } from 'src/modules/messaging/message-participant-manager/services/messaging-message-participant.service'; import { isWorkEmail } from 'src/utils/is-work-email'; @@ -103,6 +104,10 @@ export class MessagingSaveMessagesAndEnqueueContactCreationService { messageChannel.excludeNonProfessionalEmails && !isWorkEmail(participant.handle); + const isExcludedByGroupEmails = + messageChannel.excludeGroupEmails && + isGroupEmail(participant.handle); + // Drafts are outgoing, so don't turn recipients of an // unsent email into CRM contacts. const shouldCreateContact = @@ -110,6 +115,7 @@ export class MessagingSaveMessagesAndEnqueueContactCreationService { !!participant.handle && !isParticipantConnectedAccount && !isExcludedByNonProfessionalEmails && + !isExcludedByGroupEmails && (messageChannel.contactAutoCreationPolicy === MessageChannelContactAutoCreationPolicy.SENT_AND_RECEIVED || (messageChannel.contactAutoCreationPolicy === diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/types/message.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/types/message.ts index 044c1606b1..d20232b625 100644 --- a/packages/twenty-server/src/modules/messaging/message-import-manager/types/message.ts +++ b/packages/twenty-server/src/modules/messaging/message-import-manager/types/message.ts @@ -26,6 +26,12 @@ export type Message = Omit< messageFolderIds?: string[]; messageFolderExternalIds?: string[]; labelIds?: string[]; + messageHeaders?: MessageHeader[]; +}; + +export type MessageHeader = { + name: string; + value: string; }; export type MessageAttachment = { diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/utils/__tests__/filter-emails.util.spec.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/utils/__tests__/filter-emails.util.spec.ts index bb7c941e1b..dc5fa58aea 100644 --- a/packages/twenty-server/src/modules/messaging/message-import-manager/utils/__tests__/filter-emails.util.spec.ts +++ b/packages/twenty-server/src/modules/messaging/message-import-manager/utils/__tests__/filter-emails.util.spec.ts @@ -139,6 +139,38 @@ describe('filterEmails', () => { expect(result[0].externalId).toBe('regular-message'); }); + it('should filter out bulk mail whose sender does not look like a group address', () => { + const primaryHandle = 'user@example.com'; + const messages: MessageWithParticipants[] = [ + { + externalId: 'newsletter-message', + subject: 'Your weekly recap', + receivedAt: new Date('2025-01-09T09:54:37.000Z'), + text: 'Recap', + headerMessageId: '', + messageThreadExternalId: 'thread-1', + direction: MessageDirection.INCOMING, + participants: [ + { + role: MessageParticipantRole.FROM, + handle: 'posts-recap@mail.instagram.com', + displayName: 'Instagram', + }, + ], + attachments: [], + isDraft: false, + messageHeaders: [ + { name: 'List-Unsubscribe', value: '' }, + ], + }, + ]; + + expect(filterEmails(primaryHandle, [], messages, [])).toEqual([]); + expect(filterEmails(primaryHandle, [], messages, [], false)).toEqual( + messages, + ); + }); + it('should not filter out group emails when excludeGroupEmails is false', () => { const primaryHandle = 'user@example.com'; const messages: MessageWithParticipants[] = [ diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/utils/__tests__/is-bulk-mail.util.spec.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/utils/__tests__/is-bulk-mail.util.spec.ts new file mode 100644 index 0000000000..2c11c4927c --- /dev/null +++ b/packages/twenty-server/src/modules/messaging/message-import-manager/utils/__tests__/is-bulk-mail.util.spec.ts @@ -0,0 +1,50 @@ +import { isBulkMail } from 'src/modules/messaging/message-import-manager/utils/is-bulk-mail.util'; + +describe('isBulkMail', () => { + it('should detect a List-Unsubscribe header regardless of casing', () => { + expect( + isBulkMail([ + { name: 'List-Unsubscribe', value: '' }, + ]), + ).toBe(true); + + expect( + isBulkMail([ + { name: 'list-unsubscribe', value: '' }, + ]), + ).toBe(true); + }); + + it('should detect a List-Id header', () => { + expect( + isBulkMail([{ name: 'List-Id', value: '' }]), + ).toBe(true); + }); + + it('should detect bulk precedence values only', () => { + expect(isBulkMail([{ name: 'Precedence', value: 'bulk' }])).toBe(true); + expect(isBulkMail([{ name: 'Precedence', value: ' List ' }])).toBe(true); + expect(isBulkMail([{ name: 'Precedence', value: 'urgent' }])).toBe(false); + }); + + it('should treat Auto-Submitted as bulk unless it is "no"', () => { + expect( + isBulkMail([{ name: 'Auto-Submitted', value: 'auto-generated' }]), + ).toBe(true); + expect(isBulkMail([{ name: 'Auto-Submitted', value: 'no' }])).toBe(false); + }); + + it('should not flag a regular message', () => { + expect( + isBulkMail([ + { name: 'From', value: 'tim@apple.com' }, + { name: 'Subject', value: 'Lunch' }, + { name: 'List-Unsubscribe', value: '' }, + ]), + ).toBe(false); + }); + + it('should not flag a message without headers', () => { + expect(isBulkMail([])).toBe(false); + }); +}); diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/utils/filter-emails.util.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/utils/filter-emails.util.ts index 5ab28dba91..c6bae229bd 100644 --- a/packages/twenty-server/src/modules/messaging/message-import-manager/utils/filter-emails.util.ts +++ b/packages/twenty-server/src/modules/messaging/message-import-manager/utils/filter-emails.util.ts @@ -5,6 +5,7 @@ import { type MessageWithParticipants } from 'src/modules/messaging/message-impo import { filterOutBlocklistedMessages } from 'src/modules/messaging/message-import-manager/utils/filter-out-blocklisted-messages.util'; import { filterOutIcsAttachments } from 'src/modules/messaging/message-import-manager/utils/filter-out-ics-attachments.util'; import { filterOutInternals } from 'src/modules/messaging/message-import-manager/utils/filter-out-internals.util'; +import { isBulkMail } from 'src/modules/messaging/message-import-manager/utils/is-bulk-mail.util'; import { isGroupEmail } from 'src/modules/messaging/message-import-manager/utils/is-group-email'; import { isMessageSenderMatchingHandles } from 'src/modules/messaging/message-import-manager/utils/is-message-sender-matching-handles.util'; import { isWorkEmail } from 'src/utils/is-work-email'; @@ -45,6 +46,10 @@ export const filterEmails = ( return true; } + if (isBulkMail(message.messageHeaders ?? [])) { + return false; + } + const senderHandle = message.participants?.find( (participant) => participant.role === MessageParticipantRole.FROM, )?.handle; diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/utils/is-bulk-mail.util.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/utils/is-bulk-mail.util.ts new file mode 100644 index 0000000000..e11545a96d --- /dev/null +++ b/packages/twenty-server/src/modules/messaging/message-import-manager/utils/is-bulk-mail.util.ts @@ -0,0 +1,26 @@ +import { isNonEmptyString } from '@sniptt/guards'; + +import { type MessageHeader } from 'src/modules/messaging/message-import-manager/types/message'; + +const BULK_LIST_HEADER_NAMES = ['list-unsubscribe', 'list-id']; +const BULK_PRECEDENCE_VALUES = ['bulk', 'list', 'junk']; + +export const isBulkMail = (headers: MessageHeader[]): boolean => + headers.some(({ name, value }) => { + const headerName = name.toLowerCase(); + const headerValue = value.trim().toLowerCase(); + + if (BULK_LIST_HEADER_NAMES.includes(headerName)) { + return isNonEmptyString(headerValue); + } + + if (headerName === 'precedence') { + return BULK_PRECEDENCE_VALUES.includes(headerValue); + } + + if (headerName === 'auto-submitted') { + return isNonEmptyString(headerValue) && headerValue !== 'no'; + } + + return false; + });