diff --git a/packages/twenty-server/@types/planer.d.ts b/packages/twenty-server/@types/planer.d.ts new file mode 100644 index 0000000000..21a9f97005 --- /dev/null +++ b/packages/twenty-server/@types/planer.d.ts @@ -0,0 +1,19 @@ +declare module 'planer' { + export function extractFrom( + msgBody: string, + contentType?: 'text/plain' | 'text/html', + dom?: Document, + ): string; + + export function extractFromPlain(msgBody: string): string; + + export function extractFromHtml(msgBody: string, dom?: Document): string; + + export function markMessageLines(lines: string[]): string; + + export function processMarkedLines( + lines: string[], + markers: string, + returnFlags?: Record, + ): string[]; +} diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/utils/parse-and-format-gmail-message.util.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/utils/parse-and-format-gmail-message.util.ts index 2b0a44b02f..3e1d31debd 100644 --- a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/utils/parse-and-format-gmail-message.util.ts +++ b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/utils/parse-and-format-gmail-message.util.ts @@ -1,5 +1,4 @@ import { type gmail_v1 as gmailV1 } from 'googleapis'; -// @ts-expect-error legacy noImplicitAny import planer from 'planer'; import { type ConnectedAccountWorkspaceEntity } from 'src/modules/connected-account/standard-objects/connected-account.workspace-entity'; diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/imap/messaging-imap-driver.module.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/imap/messaging-imap-driver.module.ts index ab1dde7cf2..b37e40f486 100644 --- a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/imap/messaging-imap-driver.module.ts +++ b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/imap/messaging-imap-driver.module.ts @@ -11,6 +11,7 @@ import { EmailAliasManagerModule } from 'src/modules/connected-account/email-ali import { MessagingCommonModule } from 'src/modules/messaging/common/messaging-common.module'; import { ImapClientProvider } from 'src/modules/messaging/message-import-manager/drivers/imap/providers/imap-client.provider'; import { ImapFetchByBatchService } from 'src/modules/messaging/message-import-manager/drivers/imap/services/imap-fetch-by-batch.service'; +import { ImapFindSentFolderService } from 'src/modules/messaging/message-import-manager/drivers/imap/services/imap-find-sent-folder.service'; import { ImapGetMessageListService } from 'src/modules/messaging/message-import-manager/drivers/imap/services/imap-get-message-list.service'; import { ImapGetMessagesService } from 'src/modules/messaging/message-import-manager/drivers/imap/services/imap-get-messages.service'; import { ImapHandleErrorService } from 'src/modules/messaging/message-import-manager/drivers/imap/services/imap-handle-error.service'; @@ -37,6 +38,7 @@ import { MessageParticipantManagerModule } from 'src/modules/messaging/message-p ImapHandleErrorService, ImapMessageLocatorService, ImapMessageProcessorService, + ImapFindSentFolderService, ], exports: [ ImapGetMessagesService, diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/imap/services/imap-find-sent-folder.service.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/imap/services/imap-find-sent-folder.service.ts new file mode 100644 index 0000000000..a4a2c8a15d --- /dev/null +++ b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/imap/services/imap-find-sent-folder.service.ts @@ -0,0 +1,167 @@ +import { Injectable, Logger } from '@nestjs/common'; + +import { isNumber } from 'class-validator'; +import { ListResponse, type ImapFlow } from 'imapflow'; + +/** + * Service to find sent folder using IMAP special-use flags + * + * This service uses IMAP special-use extension (RFC 6154) to identify + * the sent folder by looking for the \Sent flag rather than relying on + * folder names which can vary across providers and locales. + * + * Falls back to regex-based detection if special-use flags are not available. + * The regex pattern is inspired by imapsync's comprehensive folder mapping. + */ +@Injectable() +export class ImapFindSentFolderService { + private readonly logger = new Logger(ImapFindSentFolderService.name); + + public async findSentFolder(client: ImapFlow): Promise { + try { + const list = await client.list(); + + this.logger.debug( + `Available folders: ${list.map((item) => item.path).join(', ')}`, + ); + + const specialUseSentFolder = await this.findSentFolderBySpecialUse( + client, + list, + ); + + if (specialUseSentFolder) { + return specialUseSentFolder; + } + + const candidateSentFolder = await this.findSentFolderByRegexCandidates( + client, + list, + ); + + if (candidateSentFolder) { + return candidateSentFolder; + } + + this.logger.warn( + 'No sent folder found. Only inbox messages will be imported.', + ); + + return null; + } catch (error) { + this.logger.warn(`Error listing folders: ${error.message}`); + + return null; + } + } + + private async findSentFolderBySpecialUse( + client: ImapFlow, + list: ListResponse[], + ): Promise { + for (const folder of list) { + if (folder.specialUse && folder.specialUse.includes('\\Sent')) { + this.logger.log( + `Found sent folder via special-use flag: ${folder.path}`, + ); + + const messageCount = await this.getFolderMessageCount( + client, + folder.path, + ); + + if (messageCount > 0) { + return folder.path; + } + + this.logger.warn( + `Special-use sent folder "${folder.path}" is empty, checking other candidates`, + ); + + break; + } + } + + return null; + } + + private async findSentFolderByRegexCandidates( + client: ImapFlow, + list: ListResponse[], + ): Promise { + const regexCandidateFolders = this.getSentFolderCandidatesByRegex(list); + + for (const folder of regexCandidateFolders) { + const messageCount = await this.getFolderMessageCount(client, folder); + + if (messageCount > 0) { + this.logger.log(`Selected sent folder via pattern match: ${folder}`); + + return folder; + } + } + + if (regexCandidateFolders.length > 0) { + this.logger.log( + `Using first regex candidate sent folder: ${regexCandidateFolders[0]} (no messages found in any regex candidate)`, + ); + + return regexCandidateFolders[0]; + } + + return null; + } + + private getSentFolderCandidatesByRegex(list: ListResponse[]): string[] { + // Comprehensive regex pattern for legacy IMAP servers + // Source: https://imapsync.lamiral.info/FAQ.d/FAQ.Folders_Mapping.txt + // Based on imapsync's regextrans2 examples (originally "Sent|Sent Messages|Gesendet") + // Extended with additional common localizations for broader provider/language support + const sentFolderPattern = + /^(.*\/)?(sent|sent[\s_-]?(items|mail|messages|elements)?|envoy[éê]s?|[ée]l[ée]ments[\s_-]?envoy[éê]s|gesendet|gesendete[\s_-]?elemente|enviados?|elementos[\s_-]?enviados|itens[\s_-]?enviados|posta[\s_-]?inviata|inviati|보낸편지함|\[gmail\]\/sent[\s_-]?mail)$/i; + + const regexCandidateFolders = []; + + for (const folder of list) { + if (sentFolderPattern.test(folder.path)) { + this.logger.debug( + `Found potential sent folder via pattern match: ${folder.path}`, + ); + regexCandidateFolders.push(folder.path); + } + } + + return regexCandidateFolders; + } + + private async getFolderMessageCount( + client: ImapFlow, + folderPath: string, + ): Promise { + try { + const lock = await client.getMailboxLock(folderPath); + + try { + const status = await client.status(folderPath, { + messages: true, + }); + + const messageCount = status?.messages; + + this.logger.debug( + `Folder "${folderPath}" has ${messageCount} messages`, + ); + + return isNumber(messageCount) ? messageCount : 0; + } finally { + lock.release(); + } + } catch (error) { + this.logger.warn( + `Error checking folder "${folderPath}": ${error.message}`, + ); + + return 0; + } + } +} diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/imap/services/imap-get-message-list.service.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/imap/services/imap-get-message-list.service.ts index f5e3af8a38..9651a6a5cd 100644 --- a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/imap/services/imap-get-message-list.service.ts +++ b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/imap/services/imap-get-message-list.service.ts @@ -1,12 +1,12 @@ import { Injectable, Logger } from '@nestjs/common'; -import { type ImapFlow } from 'imapflow'; +import { FetchQueryObject, type ImapFlow } from 'imapflow'; import { type MessageFolderWorkspaceEntity } from 'src/modules/messaging/common/standard-objects/message-folder.workspace-entity'; import { ImapClientProvider } from 'src/modules/messaging/message-import-manager/drivers/imap/providers/imap-client.provider'; +import { ImapFindSentFolderService } from 'src/modules/messaging/message-import-manager/drivers/imap/services/imap-find-sent-folder.service'; import { ImapHandleErrorService } from 'src/modules/messaging/message-import-manager/drivers/imap/services/imap-handle-error.service'; import { MessageFolderName } from 'src/modules/messaging/message-import-manager/drivers/imap/types/folders'; -import { findSentMailbox } from 'src/modules/messaging/message-import-manager/drivers/imap/utils/find-sent-mailbox.util'; import { type GetMessageListsArgs } from 'src/modules/messaging/message-import-manager/types/get-message-lists-args.type'; import { type GetMessageListsResponse, @@ -19,6 +19,7 @@ export class ImapGetMessageListService { constructor( private readonly imapClientProvider: ImapClientProvider, + private readonly imapFindSentFolderService: ImapFindSentFolderService, private readonly imapHandleErrorService: ImapHandleErrorService, ) {} @@ -33,16 +34,18 @@ export class ImapGetMessageListService { const result: GetMessageListsResponse = []; for (const folder of messageFolders) { - const mailboxName = await this.getMailboxName(client, folder.name); + this.logger.log(`Processing folder: ${folder.name}`); + const folderName = await this.getFolderName(client, folder.name); - if (!mailboxName) { + if (!folderName) { + this.logger.warn(`No folder name found for folder: ${folder.name}`); continue; } try { const response = await this.getMessageList( client, - mailboxName, + folderName, folder, ); @@ -52,7 +55,7 @@ export class ImapGetMessageListService { }); } catch (error) { this.logger.warn( - `Error fetching from folder ${folder.name} (${mailboxName}): ${error.message}. Continuing with other folders.`, + `Error fetching from folder ${folder.name} (${folderName}): ${error.message}. Continuing with other folders.`, ); result.push({ @@ -90,12 +93,12 @@ export class ImapGetMessageListService { public async getMessageList( client: ImapFlow, - mailbox: string, + folder: string, messageFolder: Pick, ): Promise { - const messages = await this.getMessagesFromMailbox( + const messages = await this.getMessagesFromFolder( client, - mailbox, + folder, messageFolder.syncCursor, ); @@ -115,7 +118,7 @@ export class ImapGetMessageListService { }; } - private async getMailboxName( + private async getFolderName( client: ImapFlow, folderName: string, ): Promise { @@ -124,60 +127,74 @@ export class ImapGetMessageListService { } if (folderName === MessageFolderName.SENT_ITEMS) { - const sentMailbox = await findSentMailbox(client, this.logger); + const sentFolder = + await this.imapFindSentFolderService.findSentFolder(client); - if (!sentMailbox) { + if (!sentFolder) { this.logger.warn('SENT folder not found, skipping'); return null; } - return sentMailbox; + return sentFolder; } return folderName; } - private async getMessagesFromMailbox( + private async getMessagesFromFolder( client: ImapFlow, - mailbox: string, + folder: string, cursor?: string, ): Promise<{ id: string; uid: string }[]> { let lock; try { - lock = await client.getMailboxLock(mailbox); + lock = await client.getMailboxLock(folder); - let searchOptions = {}; + const supportsUidPlus = client.capabilities.has('UIDPLUS'); + let sequence = '1:*'; - if (cursor) { + if (cursor && supportsUidPlus) { const cursorUid = parseInt(cursor); if (!isNaN(cursorUid)) { - searchOptions = { - uid: `${cursorUid + 1}:*`, - }; + sequence = `${cursorUid + 1}:*`; } } const messages: { id: string; uid: string }[] = []; - for await (const message of client.fetch(searchOptions, { + this.logger.log( + `Fetching from folder: ${folder} with sequence: ${sequence} (UIDPLUS: ${supportsUidPlus})`, + ); + + const fetchOptions: FetchQueryObject = { envelope: true, - uid: true, - })) { - if (message.envelope?.messageId && message.uid) { + }; + + if (supportsUidPlus) { + fetchOptions.uid = true; + } + + for await (const message of client.fetch(sequence, fetchOptions)) { + if (message.envelope?.messageId) { messages.push({ id: message.envelope.messageId, - uid: message.uid.toString(), + uid: + supportsUidPlus && message.uid + ? message.uid.toString() + : message.seq?.toString() || '0', }); } } + this.logger.log(`Found ${messages.length} messages in folder: ${folder}`); + return messages; } catch (error) { this.logger.error( - `Error fetching from mailbox ${mailbox}: ${error.message}`, + `Error fetching from folder ${folder}: ${error.message}`, error.stack, ); diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/imap/services/imap-get-messages.service.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/imap/services/imap-get-messages.service.ts index db67c84fcb..29a2f062f1 100644 --- a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/imap/services/imap-get-messages.service.ts +++ b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/imap/services/imap-get-messages.service.ts @@ -1,14 +1,13 @@ import { Injectable, Logger } from '@nestjs/common'; import { type AddressObject, type ParsedMail } from 'mailparser'; -// @ts-expect-error legacy noImplicitAny -import planer from 'planer'; import { isDefined } from 'twenty-shared/utils'; import { type ConnectedAccountWorkspaceEntity } from 'src/modules/connected-account/standard-objects/connected-account.workspace-entity'; import { computeMessageDirection } from 'src/modules/messaging/message-import-manager/drivers/gmail/utils/compute-message-direction.util'; import { ImapFetchByBatchService } from 'src/modules/messaging/message-import-manager/drivers/imap/services/imap-fetch-by-batch.service'; import { type MessageFetchResult } from 'src/modules/messaging/message-import-manager/drivers/imap/services/imap-message-processor.service'; +import { extractTextWithoutReplyQuotations } from 'src/modules/messaging/message-import-manager/drivers/imap/utils/extract-message-text.util'; import { type EmailAddress } from 'src/modules/messaging/message-import-manager/types/email-address'; import { type MessageWithParticipants } from 'src/modules/messaging/message-import-manager/types/message'; import { formatAddressObjectAsParticipants } from 'src/modules/messaging/message-import-manager/utils/format-address-object-as-participants.util'; @@ -71,8 +70,8 @@ export class ImapGetMessagesService { ): MessageWithParticipants[] { const messages = batchResults.map((result) => { if (!result.parsed) { - this.logger.debug( - `Message ${result.messageId} could not be parsed - likely not found in current mailboxes`, + this.logger.warn( + `Message ${result.messageId} could not be parsed - likely not found in current folders`, ); return undefined; @@ -85,7 +84,13 @@ export class ImapGetMessagesService { ); }); - return messages.filter(isDefined); + const validMessages = messages.filter(isDefined); + + this.logger.log( + `Successfully parsed ${validMessages.length} out of ${batchResults.length} messages`, + ); + + return validMessages; } private createMessageFromParsedMail( @@ -107,9 +112,8 @@ export class ImapGetMessagesService { const fromHandle = fromAddresses.length > 0 ? fromAddresses[0].address : ''; - const textWithoutReplyQuotations = parsed.text - ? planer.extractFrom(parsed.text, 'text/plain') - : ''; + const textWithoutReplyQuotations = + extractTextWithoutReplyQuotations(parsed); const direction = computeMessageDirection(fromHandle, connectedAccount); const text = sanitizeString(textWithoutReplyQuotations); @@ -135,7 +139,7 @@ export class ImapGetMessagesService { const threadRoot = references[0].trim(); if (threadRoot && threadRoot.length > 0) { - return this.normalizeMessageId(threadRoot); + return threadRoot; } } @@ -146,12 +150,12 @@ export class ImapGetMessagesService { : String(inReplyTo).trim(); if (cleanInReplyTo && cleanInReplyTo.length > 0) { - return this.normalizeMessageId(cleanInReplyTo); + return cleanInReplyTo; } } if (messageId) { - return this.normalizeMessageId(messageId); + return messageId.trim(); } const timestamp = Date.now(); @@ -160,20 +164,6 @@ export class ImapGetMessagesService { return `thread-${timestamp}-${randomSuffix}`; } - private normalizeMessageId(messageId: string): string { - const trimmedMessageId = messageId.trim(); - - if ( - trimmedMessageId.includes('@') && - !trimmedMessageId.startsWith('<') && - !trimmedMessageId.endsWith('>') - ) { - return `<${trimmedMessageId}>`; - } - - return trimmedMessageId; - } - private extractAllParticipants(parsed: ParsedMail) { const fromAddresses = this.extractAddresses( parsed.from as AddressObject | undefined, diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/imap/services/imap-message-locator.service.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/imap/services/imap-message-locator.service.ts index 8a0d0b70b4..c3049e2103 100644 --- a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/imap/services/imap-message-locator.service.ts +++ b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/imap/services/imap-message-locator.service.ts @@ -1,103 +1,125 @@ import { Injectable, Logger } from '@nestjs/common'; -import { type ImapFlow } from 'imapflow'; +import { FetchMessageObject, type ImapFlow } from 'imapflow'; -import { findSentMailbox } from 'src/modules/messaging/message-import-manager/drivers/imap/utils/find-sent-mailbox.util'; +import { ImapFindSentFolderService } from 'src/modules/messaging/message-import-manager/drivers/imap/services/imap-find-sent-folder.service'; export type MessageLocation = { messageId: string; - sequence: number; - mailbox: string; + uid: number; + folder: string; }; @Injectable() export class ImapMessageLocatorService { private readonly logger = new Logger(ImapMessageLocatorService.name); + private static readonly FETCH_BATCH_SIZE = 50; - private static readonly IMAP_SEARCH_BATCH_SIZE = 50; + constructor( + private readonly imapFindSentFolderService: ImapFindSentFolderService, + ) {} async locateAllMessages( messageIds: string[], client: ImapFlow, ): Promise> { const locations = new Map(); - const mailboxes = await this.getMailboxesToSearch(client); + const folders = await this.getFoldersToSearch(client); + const messageIdSet = new Set(messageIds); - for (const mailbox of mailboxes) { - try { - const lock = await client.getMailboxLock(mailbox); - - try { - const searchBatches = this.chunkArray( - messageIds.filter((id) => !locations.has(id)), - ImapMessageLocatorService.IMAP_SEARCH_BATCH_SIZE, - ); - - for (const batch of searchBatches) { - await this.locateMessagesInMailbox( - batch, - mailbox, - client, - locations, - ); - } - } finally { - lock.release(); - } - } catch (error) { - this.logger.warn( - `Error searching mailbox ${mailbox}: ${error.message}`, - ); - } + for (const folder of folders) { + await this.searchFolderForMessages( + folder, + client, + messageIdSet, + locations, + ); } return locations; } - private async locateMessagesInMailbox( - messageIds: string[], - mailbox: string, + private async searchFolderForMessages( + folder: string, client: ImapFlow, + messageIdSet: Set, locations: Map, ): Promise { + let lock; + try { - const orConditions = messageIds.map((id) => ({ - header: { 'message-id': id }, - })); - const searchResults = await client.search({ or: orConditions }); + lock = await client.getMailboxLock(folder); + const uids = await client.search({ all: true }); - if (searchResults.length === 0) return; - - const fetchResults = client.fetch( - searchResults.map((r) => r.toString()).join(','), - { envelope: true }, + await this.processBatchedMessages( + uids, + folder, + client, + messageIdSet, + locations, ); - - for await (const message of fetchResults) { - const messageId = message.envelope?.messageId; - - if (messageId && messageIds.includes(messageId)) { - locations.set(messageId, { - messageId, - sequence: message.seq, - mailbox, - }); - } - } } catch (error) { - this.logger.debug(`Batch search failed in ${mailbox}: ${error.message}`); + this.logger.warn(`Error searching folder ${folder}: ${error.message}`); + } finally { + lock?.release(); } } - private async getMailboxesToSearch(client: ImapFlow): Promise { - const mailboxes = ['INBOX']; - const sentFolder = await findSentMailbox(client, this.logger); + private async processBatchedMessages( + uids: number[], + folder: string, + client: ImapFlow, + messageIdSet: Set, + locations: Map, + ): Promise { + const batches = this.chunkArray( + uids, + ImapMessageLocatorService.FETCH_BATCH_SIZE, + ); - if (sentFolder) { - mailboxes.push(sentFolder); + for (const batchUids of batches) { + const fetchResults = client.fetch(batchUids.join(','), { + envelope: true, + }); + + for await (const message of fetchResults) { + this.processMessage(message, folder, messageIdSet, locations); + } + } + } + + private processMessage( + message: FetchMessageObject, + folder: string, + messageIdSet: Set, + locations: Map, + ): void { + const envelopeMessageId = message.envelope?.messageId; + + if (envelopeMessageId && messageIdSet.has(envelopeMessageId)) { + locations.set(envelopeMessageId, { + messageId: envelopeMessageId, + uid: message.uid, + folder, + }); + } + } + + private async getFoldersToSearch(client: ImapFlow): Promise { + const folders = ['INBOX']; + + try { + const sentFolder = + await this.imapFindSentFolderService.findSentFolder(client); + + if (sentFolder && sentFolder !== 'INBOX') { + folders.push(sentFolder); + } + } catch (error) { + this.logger.warn(`Failed to find sent folder: ${error.message}`); } - return mailboxes; + return folders; } private chunkArray(array: T[], chunkSize: number): T[][] { diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/imap/services/imap-message-processor.service.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/imap/services/imap-message-processor.service.ts index 4767dfd3e1..aee01252fa 100644 --- a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/imap/services/imap-message-processor.service.ts +++ b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/imap/services/imap-message-processor.service.ts @@ -31,34 +31,34 @@ export class ImapMessageProcessorService { const results: MessageFetchResult[] = []; - const messagesByMailbox = new Map(); + const messagesByFolder = new Map(); const notFoundIds: string[] = []; for (const messageId of messageIds) { const location = messageLocations.get(messageId); if (location) { - const locations = messagesByMailbox.get(location.mailbox) || []; + const locations = messagesByFolder.get(location.folder) || []; locations.push(location); - messagesByMailbox.set(location.mailbox, locations); + messagesByFolder.set(location.folder, locations); } else { notFoundIds.push(messageId); } } - const fetchPromises = Array.from(messagesByMailbox.entries()).map( - ([mailbox, locations]) => - this.fetchMessagesFromMailbox(locations, client, mailbox), + const fetchPromises = Array.from(messagesByFolder.entries()).map( + ([folder, locations]) => + this.fetchMessagesFromFolder(locations, client, folder), ); - const mailboxResults = await Promise.allSettled(fetchPromises); + const folderResults = await Promise.allSettled(fetchPromises); - for (const result of mailboxResults) { + for (const result of folderResults) { if (result.status === 'fulfilled') { results.push(...result.value); } else { - this.logger.error(`Mailbox batch fetch failed: ${result.reason}`); + this.logger.error(`Folder batch fetch failed: ${result.reason}`); } } @@ -73,24 +73,24 @@ export class ImapMessageProcessorService { return results; } - private async fetchMessagesFromMailbox( + private async fetchMessagesFromFolder( messageLocations: MessageLocation[], client: ImapFlow, - mailbox: string, + folder: string, ): Promise { if (!messageLocations.length) return []; try { - const lock = await client.getMailboxLock(mailbox); + const lock = await client.getMailboxLock(folder); try { - return await this.fetchMessagesWithSequences(messageLocations, client); + return await this.fetchMessagesWithUids(messageLocations, client); } finally { lock.release(); } } catch (error) { this.logger.error( - `Failed to fetch messages from mailbox ${mailbox}: ${error.message}`, + `Failed to fetch messages from folder ${folder}: ${error.message}`, ); return messageLocations.map((location) => @@ -99,7 +99,7 @@ export class ImapMessageProcessorService { } } - private async fetchMessagesWithSequences( + private async fetchMessagesWithUids( messageLocations: MessageLocation[], client: ImapFlow, ): Promise { @@ -107,10 +107,11 @@ export class ImapMessageProcessorService { const results: MessageFetchResult[] = []; try { - const sequences = messageLocations.map((loc) => loc.sequence.toString()); - const sequenceSet = sequences.join(','); + const uids = messageLocations.map((loc) => loc.uid.toString()); + const uidSet = uids.join(','); - const fetchResults = client.fetch(sequenceSet, { + const fetchResults = client.fetch(uidSet, { + uid: true, source: true, envelope: true, }); @@ -118,11 +119,11 @@ export class ImapMessageProcessorService { const messagesData = new Map(); for await (const message of fetchResults) { - messagesData.set(message.seq, message); + messagesData.set(message.uid, message); } for (const location of messageLocations) { - const messageData = messagesData.get(location.sequence); + const messageData = messagesData.get(location.uid); if (messageData) { const result = await this.processMessageData( diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/imap/utils/__tests__/extract-message-text.util.spec.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/imap/utils/__tests__/extract-message-text.util.spec.ts new file mode 100644 index 0000000000..7ddafe7873 --- /dev/null +++ b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/imap/utils/__tests__/extract-message-text.util.spec.ts @@ -0,0 +1,188 @@ +import { type ParsedMail } from 'mailparser'; + +import { extractTextWithoutReplyQuotations } from 'src/modules/messaging/message-import-manager/drivers/imap/utils/extract-message-text.util'; + +describe('extractTextWithoutReplyQuotations', () => { + it('should extract text from plain text emails with lot of reply quotations', () => { + const parsed: ParsedMail = { + text: `Hi John, + +Thank you for contacting Developer Support, this is Erica again. I hope you are having a good day. + +I understand that you are unable to contact finance. Despite your account being expired, you should still be able to contact our finance team. + +Follow the link below the link for contacting our finance team. + +https://idmsa.apple.com/IDMSWebAuth/signin.html?path=/contact/finance/ + +Best Regards, + +Erica +Developer Support + +>On Mar 26, 2025 at 6:59 PM, zef wrote: +> +>Just bumping this incase you missed my last message +> +>On Thu, Mar 20, 2025 at 5:50 AM zef wrote: +> +>> About that I can’t contact the finance department as I’m no longer a +>> member it doesn’t let me choose it on the contact page. Says “Permission +>> denied" +>> +>> So this was my last hope and resort +>> +>> On Thu, Mar 20, 2025 at 5:30 AM Apple Support +>> wrote: +>> +>>> Hi Uzef, +>>> +>>> Thank you for contacting Developer Support, my name is Erica and I would +>>> be happy to assist you. +>>> +>>> I understand that you are contacting us regarding a balance in your +>>> account and requesting to verify your eligibility for a payout. +>>> +>>> The finance team specializes in tax, banking, and payment questions. +>>> +>>> Visit Contact Us About Financial Information +>>> to submit your questions. +>>> For payment questions, include the Transaction ID or Consolidated Credit +>>> Identifier (CII). You'll receive an automated email with a follow-up number. +>>> +>>> Note that the finance team supports only requests in English. +>>> +>>> If you have additional questions related to this request, please refer to +>>> case number 123. +>>> +>>> Best Regards, +>>> +>>> Erica +>>> +>>> Developer Support +>>> +>>> On Mar 19, 2025 at 1:07 AM, wrote: +>>> +>>> Product Name : Apple Developer Support +>>> +>>> Support Category : Membership and Account +>>> +>>> Support Topic : Other Membership or Account Questions +>>> +>>> Additional Details : +>>> +>>> Message: +>>> +>>> Hi, +>>> +>>> I was a member of the Apple Developer Program some time ago. +>>> +>>> I'm no longer a member so it won't let me specifically select "Payments" +>>> page when contacting so I'm using this. +>>> +>>> During my period I checked my account has around $40 in revenue which +>>> meets the minimum threshold for a payout, however I never received one. +>>> +>>> I was hoping you could look into it and see if I'm eligible to get the +>>> payout? +>>> +>>> Thanks +>>> +>>> -John +`, + attachments: [], + headers: new Map(), + headerLines: [], + html: false, + }; + + const result = extractTextWithoutReplyQuotations(parsed); + + expect(result).toBe(`Hi John, + +Thank you for contacting Developer Support, this is Erica again. I hope you are having a good day. + +I understand that you are unable to contact finance. Despite your account being expired, you should still be able to contact our finance team. + +Follow the link below the link for contacting our finance team. + +https://idmsa.apple.com/IDMSWebAuth/signin.html?path=/contact/finance/ + +Best Regards, + +Erica +Developer Support`); + }); + + it('should handle email with reply quotations (Titan email style)', () => { + const parsed: ParsedMail = { + text: `just a follow up + + + On Aug 18 2025, at 4:06 pm, neo@titanemailtest.xyz wrote: + + + + Dear Colleagues,This is a reminder that the updated security policy goes into effect starting next Monday. All employees must reset their corporate VPN credentials and enable two-factor authentication by then. Please reach out to the IT helpdesk if you experience any issues during the setup. Regards, IT Department + + `, + attachments: [], + headers: new Map(), + headerLines: [], + html: false, + }; + + const result = extractTextWithoutReplyQuotations(parsed); + + expect(result).toBe('just a follow up'); + }); + + it('should handle html email with reply quotations', () => { + const parsed: ParsedMail = { + html: `
just a follow up

+
+
+ On Aug 18 2025, at 4:06 pm, neo@titanemailtest.xyz wrote: +
+
+
+
Dear Colleagues,

This is a reminder that the updated security policy goes into effect starting next Monday.  
All employees must reset their corporate VPN credentials and enable two-factor authentication by then.  

Please reach out to the IT helpdesk if you experience any issues during the setup.  

Regards,  
IT Department

+
+
`, + attachments: [], + headers: new Map(), + headerLines: [], + }; + + const result = extractTextWithoutReplyQuotations(parsed); + + expect(result).toBe('just a follow up'); + }); + + it('should return empty string when no text or html content', () => { + const parsed: ParsedMail = { + attachments: [], + headers: new Map(), + headerLines: [], + html: false, + }; + + const result = extractTextWithoutReplyQuotations(parsed); + + expect(result).toBe(''); + }); + + it('should prefer text over html when both are available', () => { + const parsed: ParsedMail = { + text: 'Plain text content\n\nOn 2023-01-01, user@example.com wrote:\n> Reply', + html: '

HTML content

', + attachments: [], + headers: new Map(), + headerLines: [], + }; + + const result = extractTextWithoutReplyQuotations(parsed); + + expect(result).toBe('Plain text content'); + }); +}); diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/imap/utils/extract-message-text.util.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/imap/utils/extract-message-text.util.ts new file mode 100644 index 0000000000..e2be81ed49 --- /dev/null +++ b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/imap/utils/extract-message-text.util.ts @@ -0,0 +1,31 @@ +import DOMPurify from 'dompurify'; +import { JSDOM } from 'jsdom'; +import { type ParsedMail } from 'mailparser'; +import * as planer from 'planer'; + +export const extractTextWithoutReplyQuotations = ( + parsed: ParsedMail, +): string => { + if (parsed.text) { + return planer.extractFrom(parsed.text, 'text/plain'); + } + + if (parsed.html) { + const window = new JSDOM('').window; + const purify = DOMPurify(window); + const sanitizedHtml = purify.sanitize(parsed.html); + const dom = new JSDOM(sanitizedHtml, { runScripts: 'outside-only' }); + + const cleanedHtml = planer.extractFromHtml( + sanitizedHtml, + dom.window.document, + ); + + const textContent = new JSDOM(cleanedHtml, { runScripts: 'outside-only' }) + .window.document.body?.textContent; + + return textContent ?? ''; + } + + return ''; +}; diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/imap/utils/find-sent-mailbox.util.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/imap/utils/find-sent-mailbox.util.ts deleted file mode 100644 index 017de71f8d..0000000000 --- a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/imap/utils/find-sent-mailbox.util.ts +++ /dev/null @@ -1,59 +0,0 @@ -import { type Logger } from '@nestjs/common'; - -import { type ImapFlow } from 'imapflow'; - -/** - * Find sent folder using IMAP special-use flags - * - * This function uses IMAP special-use extension (RFC 6154) to identify - * the sent folder by looking for the \Sent flag rather than relying on - * folder names which can vary across providers and locales. - * - * Falls back to regex-based detection if special-use flags are not available. - * The regex pattern is inspired by imapsync's comprehensive folder mapping. - */ -export async function findSentMailbox( - client: ImapFlow, - logger: Logger, -): Promise { - try { - const list = await client.list(); - - logger.debug( - `Available folders: ${list.map((item) => item.path).join(', ')}`, - ); - - for (const folder of list) { - if (folder.specialUse && folder.specialUse.includes('\\Sent')) { - logger.log(`Found sent folder via special-use flag: ${folder.path}`); - - return folder.path; - } - } - - // Fallback: comprehensive regex pattern for legacy IMAP servers - // Source: https://imapsync.lamiral.info/FAQ.d/FAQ.Folders_Mapping.txt - // Based on imapsync's regextrans2 examples (originally "Sent|Sent Messages|Gesendet") - // Extended with additional common localizations for broader provider/language support - const sentFolderPattern = - /^(.*\/)?(sent|sent[\s_-]?(items|mail|messages|elements)?|envoy[éê]s?|[ée]l[ée]ments[\s_-]?envoy[éê]s|gesendet|gesendete[\s_-]?elemente|enviados?|elementos[\s_-]?enviados|itens[\s_-]?enviados|posta[\s_-]?inviata|inviati|보낸편지함|\[gmail\]\/sent[\s_-]?mail)$/i; - - const availableFolders = list.map((item) => item.path); - - for (const folder of availableFolders) { - if (sentFolderPattern.test(folder)) { - logger.log(`Found sent folder via pattern match: ${folder}`); - - return folder; - } - } - - logger.warn('No sent folder found. Only inbox messages will be imported.'); - - return null; - } catch (error) { - logger.warn(`Error listing mailboxes: ${error.message}`); - - return null; - } -}