IMAP FIxes (#13973)

- Added `ImapFindSentMailboxService` to replace utility function it fixes an edge case where special flag `Sent` folder may have zero messages, we fall back to regex based approach to find other candidates - Fixed edge case where some servers may not return text content for body it parses the HTML in that case - Some other enhancements with capability based detection Tested with Stalwart server and Titan email /closes #13884
2025-08-19 22:52:13 +05:30
parent d63ede5aca
commit c582666aeb
11 changed files with 572 additions and 195 deletions
@@ -0,0 +1,19 @@
+declare module 'planer' {
+  export function extractFrom(
+    msgBody: string,
+    contentType?: 'text/plain' | 'text/html',
+    dom?: Document,
+  ): string;
+
+  export function extractFromPlain(msgBody: string): string;
+
+  export function extractFromHtml(msgBody: string, dom?: Document): string;
+
+  export function markMessageLines(lines: string[]): string;
+
+  export function processMarkedLines(
+    lines: string[],
+    markers: string,
+    returnFlags?: Record<string, unknown>,
+  ): string[];
+}
@@ -1,5 +1,4 @@
 import { type gmail_v1 as gmailV1 } from 'googleapis';
-// @ts-expect-error legacy noImplicitAny
 import planer from 'planer';

 import { type ConnectedAccountWorkspaceEntity } from 'src/modules/connected-account/standard-objects/connected-account.workspace-entity';
@@ -11,6 +11,7 @@ import { EmailAliasManagerModule } from 'src/modules/connected-account/email-ali
 import { MessagingCommonModule } from 'src/modules/messaging/common/messaging-common.module';
 import { ImapClientProvider } from 'src/modules/messaging/message-import-manager/drivers/imap/providers/imap-client.provider';
 import { ImapFetchByBatchService } from 'src/modules/messaging/message-import-manager/drivers/imap/services/imap-fetch-by-batch.service';
+import { ImapFindSentFolderService } from 'src/modules/messaging/message-import-manager/drivers/imap/services/imap-find-sent-folder.service';
 import { ImapGetMessageListService } from 'src/modules/messaging/message-import-manager/drivers/imap/services/imap-get-message-list.service';
 import { ImapGetMessagesService } from 'src/modules/messaging/message-import-manager/drivers/imap/services/imap-get-messages.service';
 import { ImapHandleErrorService } from 'src/modules/messaging/message-import-manager/drivers/imap/services/imap-handle-error.service';
@@ -37,6 +38,7 @@ import { MessageParticipantManagerModule } from 'src/modules/messaging/message-p
    ImapHandleErrorService,
    ImapMessageLocatorService,
    ImapMessageProcessorService,
+    ImapFindSentFolderService,
  ],
  exports: [
    ImapGetMessagesService,
@@ -0,0 +1,167 @@
+import { Injectable, Logger } from '@nestjs/common';
+
+import { isNumber } from 'class-validator';
+import { ListResponse, type ImapFlow } from 'imapflow';
+
+/**
+ * Service to find sent folder using IMAP special-use flags
+ *
+ * This service uses IMAP special-use extension (RFC 6154) to identify
+ * the sent folder by looking for the \Sent flag rather than relying on
+ * folder names which can vary across providers and locales.
+ *
+ * Falls back to regex-based detection if special-use flags are not available.
+ * The regex pattern is inspired by imapsync's comprehensive folder mapping.
+ */
+@Injectable()
+export class ImapFindSentFolderService {
+  private readonly logger = new Logger(ImapFindSentFolderService.name);
+
+  public async findSentFolder(client: ImapFlow): Promise<string | null> {
+    try {
+      const list = await client.list();
+
+      this.logger.debug(
+        `Available folders: ${list.map((item) => item.path).join(', ')}`,
+      );
+
+      const specialUseSentFolder = await this.findSentFolderBySpecialUse(
+        client,
+        list,
+      );
+
+      if (specialUseSentFolder) {
+        return specialUseSentFolder;
+      }
+
+      const candidateSentFolder = await this.findSentFolderByRegexCandidates(
+        client,
+        list,
+      );
+
+      if (candidateSentFolder) {
+        return candidateSentFolder;
+      }
+
+      this.logger.warn(
+        'No sent folder found. Only inbox messages will be imported.',
+      );
+
+      return null;
+    } catch (error) {
+      this.logger.warn(`Error listing folders: ${error.message}`);
+
+      return null;
+    }
+  }
+
+  private async findSentFolderBySpecialUse(
+    client: ImapFlow,
+    list: ListResponse[],
+  ): Promise<string | null> {
+    for (const folder of list) {
+      if (folder.specialUse && folder.specialUse.includes('\\Sent')) {
+        this.logger.log(
+          `Found sent folder via special-use flag: ${folder.path}`,
+        );
+
+        const messageCount = await this.getFolderMessageCount(
+          client,
+          folder.path,
+        );
+
+        if (messageCount > 0) {
+          return folder.path;
+        }
+
+        this.logger.warn(
+          `Special-use sent folder "${folder.path}" is empty, checking other candidates`,
+        );
+
+        break;
+      }
+    }
+
+    return null;
+  }
+
+  private async findSentFolderByRegexCandidates(
+    client: ImapFlow,
+    list: ListResponse[],
+  ): Promise<string | null> {
+    const regexCandidateFolders = this.getSentFolderCandidatesByRegex(list);
+
+    for (const folder of regexCandidateFolders) {
+      const messageCount = await this.getFolderMessageCount(client, folder);
+
+      if (messageCount > 0) {
+        this.logger.log(`Selected sent folder via pattern match: ${folder}`);
+
+        return folder;
+      }
+    }
+
+    if (regexCandidateFolders.length > 0) {
+      this.logger.log(
+        `Using first regex candidate sent folder: ${regexCandidateFolders[0]} (no messages found in any regex candidate)`,
+      );
+
+      return regexCandidateFolders[0];
+    }
+
+    return null;
+  }
+
+  private getSentFolderCandidatesByRegex(list: ListResponse[]): string[] {
+    // Comprehensive regex pattern for legacy IMAP servers
+    // Source: https://imapsync.lamiral.info/FAQ.d/FAQ.Folders_Mapping.txt
+    // Based on imapsync's regextrans2 examples (originally "Sent|Sent Messages|Gesendet")
+    // Extended with additional common localizations for broader provider/language support
+    const sentFolderPattern =
+      /^(.*\/)?(sent|sent[\s_-]?(items|mail|messages|elements)?|envoy[éê]s?|[ée]l[ée]ments[\s_-]?envoy[éê]s|gesendet|gesendete[\s_-]?elemente|enviados?|elementos[\s_-]?enviados|itens[\s_-]?enviados|posta[\s_-]?inviata|inviati|보낸편지함|\[gmail\]\/sent[\s_-]?mail)$/i;
+
+    const regexCandidateFolders = [];
+
+    for (const folder of list) {
+      if (sentFolderPattern.test(folder.path)) {
+        this.logger.debug(
+          `Found potential sent folder via pattern match: ${folder.path}`,
+        );
+        regexCandidateFolders.push(folder.path);
+      }
+    }
+
+    return regexCandidateFolders;
+  }
+
+  private async getFolderMessageCount(
+    client: ImapFlow,
+    folderPath: string,
+  ): Promise<number> {
+    try {
+      const lock = await client.getMailboxLock(folderPath);
+
+      try {
+        const status = await client.status(folderPath, {
+          messages: true,
+        });
+
+        const messageCount = status?.messages;
+
+        this.logger.debug(
+          `Folder "${folderPath}" has ${messageCount} messages`,
+        );
+
+        return isNumber(messageCount) ? messageCount : 0;
+      } finally {
+        lock.release();
+      }
+    } catch (error) {
+      this.logger.warn(
+        `Error checking folder "${folderPath}": ${error.message}`,
+      );
+
+      return 0;
+    }
+  }
+}
@@ -1,12 +1,12 @@
 import { Injectable, Logger } from '@nestjs/common';

-import { type ImapFlow } from 'imapflow';
+import { FetchQueryObject, type ImapFlow } from 'imapflow';

 import { type MessageFolderWorkspaceEntity } from 'src/modules/messaging/common/standard-objects/message-folder.workspace-entity';
 import { ImapClientProvider } from 'src/modules/messaging/message-import-manager/drivers/imap/providers/imap-client.provider';
+import { ImapFindSentFolderService } from 'src/modules/messaging/message-import-manager/drivers/imap/services/imap-find-sent-folder.service';
 import { ImapHandleErrorService } from 'src/modules/messaging/message-import-manager/drivers/imap/services/imap-handle-error.service';
 import { MessageFolderName } from 'src/modules/messaging/message-import-manager/drivers/imap/types/folders';
-import { findSentMailbox } from 'src/modules/messaging/message-import-manager/drivers/imap/utils/find-sent-mailbox.util';
 import { type GetMessageListsArgs } from 'src/modules/messaging/message-import-manager/types/get-message-lists-args.type';
 import {
  type GetMessageListsResponse,
@@ -19,6 +19,7 @@ export class ImapGetMessageListService {

  constructor(
    private readonly imapClientProvider: ImapClientProvider,
+    private readonly imapFindSentFolderService: ImapFindSentFolderService,
    private readonly imapHandleErrorService: ImapHandleErrorService,
  ) {}

@@ -33,16 +34,18 @@ export class ImapGetMessageListService {
      const result: GetMessageListsResponse = [];

      for (const folder of messageFolders) {
-        const mailboxName = await this.getMailboxName(client, folder.name);
+        this.logger.log(`Processing folder: ${folder.name}`);
+        const folderName = await this.getFolderName(client, folder.name);

-        if (!mailboxName) {
+        if (!folderName) {
+          this.logger.warn(`No folder name found for folder: ${folder.name}`);
          continue;
        }

        try {
          const response = await this.getMessageList(
            client,
-            mailboxName,
+            folderName,
            folder,
          );

@@ -52,7 +55,7 @@ export class ImapGetMessageListService {
          });
        } catch (error) {
          this.logger.warn(
-            `Error fetching from folder ${folder.name} (${mailboxName}): ${error.message}. Continuing with other folders.`,
+            `Error fetching from folder ${folder.name} (${folderName}): ${error.message}. Continuing with other folders.`,
          );

          result.push({
@@ -90,12 +93,12 @@ export class ImapGetMessageListService {

  public async getMessageList(
    client: ImapFlow,
-    mailbox: string,
+    folder: string,
    messageFolder: Pick<MessageFolderWorkspaceEntity, 'syncCursor'>,
  ): Promise<GetOneMessageListResponse> {
-    const messages = await this.getMessagesFromMailbox(
+    const messages = await this.getMessagesFromFolder(
      client,
-      mailbox,
+      folder,
      messageFolder.syncCursor,
    );

@@ -115,7 +118,7 @@ export class ImapGetMessageListService {
    };
  }

-  private async getMailboxName(
+  private async getFolderName(
    client: ImapFlow,
    folderName: string,
  ): Promise<string | null> {
@@ -124,60 +127,74 @@ export class ImapGetMessageListService {
    }

    if (folderName === MessageFolderName.SENT_ITEMS) {
-      const sentMailbox = await findSentMailbox(client, this.logger);
+      const sentFolder =
+        await this.imapFindSentFolderService.findSentFolder(client);

-      if (!sentMailbox) {
+      if (!sentFolder) {
        this.logger.warn('SENT folder not found, skipping');

        return null;
      }

-      return sentMailbox;
+      return sentFolder;
    }

    return folderName;
  }

-  private async getMessagesFromMailbox(
+  private async getMessagesFromFolder(
    client: ImapFlow,
-    mailbox: string,
+    folder: string,
    cursor?: string,
  ): Promise<{ id: string; uid: string }[]> {
    let lock;

    try {
-      lock = await client.getMailboxLock(mailbox);
+      lock = await client.getMailboxLock(folder);

-      let searchOptions = {};
+      const supportsUidPlus = client.capabilities.has('UIDPLUS');
+      let sequence = '1:*';

-      if (cursor) {
+      if (cursor && supportsUidPlus) {
        const cursorUid = parseInt(cursor);

        if (!isNaN(cursorUid)) {
-          searchOptions = {
-            uid: `${cursorUid + 1}:*`,
-          };
+          sequence = `${cursorUid + 1}:*`;
        }
      }

      const messages: { id: string; uid: string }[] = [];

-      for await (const message of client.fetch(searchOptions, {
+      this.logger.log(
+        `Fetching from folder: ${folder} with sequence: ${sequence} (UIDPLUS: ${supportsUidPlus})`,
+      );
+
+      const fetchOptions: FetchQueryObject = {
        envelope: true,
-        uid: true,
-      })) {
-        if (message.envelope?.messageId && message.uid) {
+      };
+
+      if (supportsUidPlus) {
+        fetchOptions.uid = true;
+      }
+
+      for await (const message of client.fetch(sequence, fetchOptions)) {
+        if (message.envelope?.messageId) {
          messages.push({
            id: message.envelope.messageId,
-            uid: message.uid.toString(),
+            uid:
+              supportsUidPlus && message.uid
+                ? message.uid.toString()
+                : message.seq?.toString() || '0',
          });
        }
      }

+      this.logger.log(`Found ${messages.length} messages in folder: ${folder}`);
+
      return messages;
    } catch (error) {
      this.logger.error(
-        `Error fetching from mailbox ${mailbox}: ${error.message}`,
+        `Error fetching from folder ${folder}: ${error.message}`,
        error.stack,
      );

@@ -1,14 +1,13 @@
 import { Injectable, Logger } from '@nestjs/common';

 import { type AddressObject, type ParsedMail } from 'mailparser';
-// @ts-expect-error legacy noImplicitAny
-import planer from 'planer';
 import { isDefined } from 'twenty-shared/utils';

 import { type ConnectedAccountWorkspaceEntity } from 'src/modules/connected-account/standard-objects/connected-account.workspace-entity';
 import { computeMessageDirection } from 'src/modules/messaging/message-import-manager/drivers/gmail/utils/compute-message-direction.util';
 import { ImapFetchByBatchService } from 'src/modules/messaging/message-import-manager/drivers/imap/services/imap-fetch-by-batch.service';
 import { type MessageFetchResult } from 'src/modules/messaging/message-import-manager/drivers/imap/services/imap-message-processor.service';
+import { extractTextWithoutReplyQuotations } from 'src/modules/messaging/message-import-manager/drivers/imap/utils/extract-message-text.util';
 import { type EmailAddress } from 'src/modules/messaging/message-import-manager/types/email-address';
 import { type MessageWithParticipants } from 'src/modules/messaging/message-import-manager/types/message';
 import { formatAddressObjectAsParticipants } from 'src/modules/messaging/message-import-manager/utils/format-address-object-as-participants.util';
@@ -71,8 +70,8 @@ export class ImapGetMessagesService {
  ): MessageWithParticipants[] {
    const messages = batchResults.map((result) => {
      if (!result.parsed) {
-        this.logger.debug(
-          `Message ${result.messageId} could not be parsed - likely not found in current mailboxes`,
+        this.logger.warn(
+          `Message ${result.messageId} could not be parsed - likely not found in current folders`,
        );

        return undefined;
@@ -85,7 +84,13 @@ export class ImapGetMessagesService {
      );
    });

-    return messages.filter(isDefined);
+    const validMessages = messages.filter(isDefined);
+
+    this.logger.log(
+      `Successfully parsed ${validMessages.length} out of ${batchResults.length} messages`,
+    );
+
+    return validMessages;
  }

  private createMessageFromParsedMail(
@@ -107,9 +112,8 @@ export class ImapGetMessagesService {

    const fromHandle = fromAddresses.length > 0 ? fromAddresses[0].address : '';

-    const textWithoutReplyQuotations = parsed.text
-      ? planer.extractFrom(parsed.text, 'text/plain')
-      : '';
+    const textWithoutReplyQuotations =
+      extractTextWithoutReplyQuotations(parsed);

    const direction = computeMessageDirection(fromHandle, connectedAccount);
    const text = sanitizeString(textWithoutReplyQuotations);
@@ -135,7 +139,7 @@ export class ImapGetMessagesService {
      const threadRoot = references[0].trim();

      if (threadRoot && threadRoot.length > 0) {
-        return this.normalizeMessageId(threadRoot);
+        return threadRoot;
      }
    }

@@ -146,12 +150,12 @@ export class ImapGetMessagesService {
          : String(inReplyTo).trim();

      if (cleanInReplyTo && cleanInReplyTo.length > 0) {
-        return this.normalizeMessageId(cleanInReplyTo);
+        return cleanInReplyTo;
      }
    }

    if (messageId) {
-      return this.normalizeMessageId(messageId);
+      return messageId.trim();
    }

    const timestamp = Date.now();
@@ -160,20 +164,6 @@ export class ImapGetMessagesService {
    return `thread-${timestamp}-${randomSuffix}`;
  }

-  private normalizeMessageId(messageId: string): string {
-    const trimmedMessageId = messageId.trim();
-
-    if (
-      trimmedMessageId.includes('@') &&
-      !trimmedMessageId.startsWith('<') &&
-      !trimmedMessageId.endsWith('>')
-    ) {
-      return `<${trimmedMessageId}>`;
-    }
-
-    return trimmedMessageId;
-  }
-
  private extractAllParticipants(parsed: ParsedMail) {
    const fromAddresses = this.extractAddresses(
      parsed.from as AddressObject | undefined,
@@ -1,103 +1,125 @@
 import { Injectable, Logger } from '@nestjs/common';

-import { type ImapFlow } from 'imapflow';
+import { FetchMessageObject, type ImapFlow } from 'imapflow';

-import { findSentMailbox } from 'src/modules/messaging/message-import-manager/drivers/imap/utils/find-sent-mailbox.util';
+import { ImapFindSentFolderService } from 'src/modules/messaging/message-import-manager/drivers/imap/services/imap-find-sent-folder.service';

 export type MessageLocation = {
  messageId: string;
-  sequence: number;
-  mailbox: string;
+  uid: number;
+  folder: string;
 };

@Injectable()
 export class ImapMessageLocatorService {
  private readonly logger = new Logger(ImapMessageLocatorService.name);
+  private static readonly FETCH_BATCH_SIZE = 50;

-  private static readonly IMAP_SEARCH_BATCH_SIZE = 50;
+  constructor(
+    private readonly imapFindSentFolderService: ImapFindSentFolderService,
+  ) {}

  async locateAllMessages(
    messageIds: string[],
    client: ImapFlow,
  ): Promise<Map<string, MessageLocation>> {
    const locations = new Map<string, MessageLocation>();
-    const mailboxes = await this.getMailboxesToSearch(client);
+    const folders = await this.getFoldersToSearch(client);
+    const messageIdSet = new Set(messageIds);

-    for (const mailbox of mailboxes) {
-      try {
-        const lock = await client.getMailboxLock(mailbox);
-
-        try {
-          const searchBatches = this.chunkArray(
-            messageIds.filter((id) => !locations.has(id)),
-            ImapMessageLocatorService.IMAP_SEARCH_BATCH_SIZE,
-          );
-
-          for (const batch of searchBatches) {
-            await this.locateMessagesInMailbox(
-              batch,
-              mailbox,
-              client,
-              locations,
-            );
-          }
-        } finally {
-          lock.release();
-        }
-      } catch (error) {
-        this.logger.warn(
-          `Error searching mailbox ${mailbox}: ${error.message}`,
-        );
-      }
+    for (const folder of folders) {
+      await this.searchFolderForMessages(
+        folder,
+        client,
+        messageIdSet,
+        locations,
+      );
    }

    return locations;
  }

-  private async locateMessagesInMailbox(
-    messageIds: string[],
-    mailbox: string,
+  private async searchFolderForMessages(
+    folder: string,
    client: ImapFlow,
+    messageIdSet: Set<string>,
    locations: Map<string, MessageLocation>,
  ): Promise<void> {
+    let lock;
+
    try {
-      const orConditions = messageIds.map((id) => ({
-        header: { 'message-id': id },
-      }));
-      const searchResults = await client.search({ or: orConditions });
+      lock = await client.getMailboxLock(folder);
+      const uids = await client.search({ all: true });

-      if (searchResults.length === 0) return;
-
-      const fetchResults = client.fetch(
-        searchResults.map((r) => r.toString()).join(','),
-        { envelope: true },
+      await this.processBatchedMessages(
+        uids,
+        folder,
+        client,
+        messageIdSet,
+        locations,
      );
-
-      for await (const message of fetchResults) {
-        const messageId = message.envelope?.messageId;
-
-        if (messageId && messageIds.includes(messageId)) {
-          locations.set(messageId, {
-            messageId,
-            sequence: message.seq,
-            mailbox,
-          });
-        }
-      }
    } catch (error) {
-      this.logger.debug(`Batch search failed in ${mailbox}: ${error.message}`);
+      this.logger.warn(`Error searching folder ${folder}: ${error.message}`);
+    } finally {
+      lock?.release();
    }
  }

-  private async getMailboxesToSearch(client: ImapFlow): Promise<string[]> {
-    const mailboxes = ['INBOX'];
-    const sentFolder = await findSentMailbox(client, this.logger);
+  private async processBatchedMessages(
+    uids: number[],
+    folder: string,
+    client: ImapFlow,
+    messageIdSet: Set<string>,
+    locations: Map<string, MessageLocation>,
+  ): Promise<void> {
+    const batches = this.chunkArray(
+      uids,
+      ImapMessageLocatorService.FETCH_BATCH_SIZE,
+    );

-    if (sentFolder) {
-      mailboxes.push(sentFolder);
+    for (const batchUids of batches) {
+      const fetchResults = client.fetch(batchUids.join(','), {
+        envelope: true,
+      });
+
+      for await (const message of fetchResults) {
+        this.processMessage(message, folder, messageIdSet, locations);
+      }
+    }
+  }
+
+  private processMessage(
+    message: FetchMessageObject,
+    folder: string,
+    messageIdSet: Set<string>,
+    locations: Map<string, MessageLocation>,
+  ): void {
+    const envelopeMessageId = message.envelope?.messageId;
+
+    if (envelopeMessageId && messageIdSet.has(envelopeMessageId)) {
+      locations.set(envelopeMessageId, {
+        messageId: envelopeMessageId,
+        uid: message.uid,
+        folder,
+      });
+    }
+  }
+
+  private async getFoldersToSearch(client: ImapFlow): Promise<string[]> {
+    const folders = ['INBOX'];
+
+    try {
+      const sentFolder =
+        await this.imapFindSentFolderService.findSentFolder(client);
+
+      if (sentFolder && sentFolder !== 'INBOX') {
+        folders.push(sentFolder);
+      }
+    } catch (error) {
+      this.logger.warn(`Failed to find sent folder: ${error.message}`);
    }

-    return mailboxes;
+    return folders;
  }

  private chunkArray<T>(array: T[], chunkSize: number): T[][] {
@@ -31,34 +31,34 @@ export class ImapMessageProcessorService {

    const results: MessageFetchResult[] = [];

-    const messagesByMailbox = new Map<string, MessageLocation[]>();
+    const messagesByFolder = new Map<string, MessageLocation[]>();
    const notFoundIds: string[] = [];

    for (const messageId of messageIds) {
      const location = messageLocations.get(messageId);

      if (location) {
-        const locations = messagesByMailbox.get(location.mailbox) || [];
+        const locations = messagesByFolder.get(location.folder) || [];

        locations.push(location);
-        messagesByMailbox.set(location.mailbox, locations);
+        messagesByFolder.set(location.folder, locations);
      } else {
        notFoundIds.push(messageId);
      }
    }

-    const fetchPromises = Array.from(messagesByMailbox.entries()).map(
-      ([mailbox, locations]) =>
-        this.fetchMessagesFromMailbox(locations, client, mailbox),
+    const fetchPromises = Array.from(messagesByFolder.entries()).map(
+      ([folder, locations]) =>
+        this.fetchMessagesFromFolder(locations, client, folder),
    );

-    const mailboxResults = await Promise.allSettled(fetchPromises);
+    const folderResults = await Promise.allSettled(fetchPromises);

-    for (const result of mailboxResults) {
+    for (const result of folderResults) {
      if (result.status === 'fulfilled') {
        results.push(...result.value);
      } else {
-        this.logger.error(`Mailbox batch fetch failed: ${result.reason}`);
+        this.logger.error(`Folder batch fetch failed: ${result.reason}`);
      }
    }

@@ -73,24 +73,24 @@ export class ImapMessageProcessorService {
    return results;
  }

-  private async fetchMessagesFromMailbox(
+  private async fetchMessagesFromFolder(
    messageLocations: MessageLocation[],
    client: ImapFlow,
-    mailbox: string,
+    folder: string,
  ): Promise<MessageFetchResult[]> {
    if (!messageLocations.length) return [];

    try {
-      const lock = await client.getMailboxLock(mailbox);
+      const lock = await client.getMailboxLock(folder);

      try {
-        return await this.fetchMessagesWithSequences(messageLocations, client);
+        return await this.fetchMessagesWithUids(messageLocations, client);
      } finally {
        lock.release();
      }
    } catch (error) {
      this.logger.error(
-        `Failed to fetch messages from mailbox ${mailbox}: ${error.message}`,
+        `Failed to fetch messages from folder ${folder}: ${error.message}`,
      );

      return messageLocations.map((location) =>
@@ -99,7 +99,7 @@ export class ImapMessageProcessorService {
    }
  }

-  private async fetchMessagesWithSequences(
+  private async fetchMessagesWithUids(
    messageLocations: MessageLocation[],
    client: ImapFlow,
  ): Promise<MessageFetchResult[]> {
@@ -107,10 +107,11 @@ export class ImapMessageProcessorService {
    const results: MessageFetchResult[] = [];

    try {
-      const sequences = messageLocations.map((loc) => loc.sequence.toString());
-      const sequenceSet = sequences.join(',');
+      const uids = messageLocations.map((loc) => loc.uid.toString());
+      const uidSet = uids.join(',');

-      const fetchResults = client.fetch(sequenceSet, {
+      const fetchResults = client.fetch(uidSet, {
+        uid: true,
        source: true,
        envelope: true,
      });
@@ -118,11 +119,11 @@ export class ImapMessageProcessorService {
      const messagesData = new Map<number, FetchMessageObject>();

      for await (const message of fetchResults) {
-        messagesData.set(message.seq, message);
+        messagesData.set(message.uid, message);
      }

      for (const location of messageLocations) {
-        const messageData = messagesData.get(location.sequence);
+        const messageData = messagesData.get(location.uid);

        if (messageData) {
          const result = await this.processMessageData(
@@ -0,0 +1,188 @@
+import { type ParsedMail } from 'mailparser';
+
+import { extractTextWithoutReplyQuotations } from 'src/modules/messaging/message-import-manager/drivers/imap/utils/extract-message-text.util';
+
+describe('extractTextWithoutReplyQuotations', () => {
+  it('should extract text from plain text emails with lot of reply quotations', () => {
+    const parsed: ParsedMail = {
+      text: `Hi John,
+
+Thank you for contacting Developer Support, this is Erica again. I hope you are having a good day. 
+
+I understand that you are unable to contact finance. Despite your account being expired, you should still be able to contact our finance team. 
+
+Follow the link below the link for contacting our finance team. 
+
+https://idmsa.apple.com/IDMSWebAuth/signin.html?path=/contact/finance/
+
+Best Regards,
+
+Erica 
+Developer Support
+
+>On Mar 26, 2025 at 6:59 PM, zef<john@gmail.com> wrote:
+>
+>Just bumping this incase you missed my last message
+>
+>On Thu, Mar 20, 2025 at 5:50 AM zef <john@gmail.com> wrote:
+>
+>> About that I can’t contact the finance department as I’m no longer a
+>> member it doesn’t let me choose it on the contact page. Says “Permission
+>> denied"
+>>
+>> So this was my last hope and resort
+>>
+>> On Thu, Mar 20, 2025 at 5:30 AM Apple Support <devprograms@apple.com>
+>> wrote:
+>>
+>>> Hi Uzef,
+>>>
+>>> Thank you for contacting Developer Support, my name is Erica and I would
+>>> be happy to assist you.
+>>>
+>>> I understand that you are contacting us regarding a balance in your
+>>> account and requesting to verify your eligibility for a payout.
+>>>
+>>> The finance team specializes in tax, banking, and payment questions.
+>>>
+>>> Visit Contact Us About Financial Information
+>>>  to submit your questions.
+>>> For payment questions, include the Transaction ID or Consolidated Credit
+>>> Identifier (CII). You'll receive an automated email with a follow-up number.
+>>>
+>>> Note that the finance team supports only requests in English.
+>>>
+>>> If you have additional questions related to this request, please refer to
+>>> case number 123.
+>>>
+>>> Best Regards,
+>>>
+>>> Erica
+>>>
+>>> Developer Support
+>>>
+>>> On Mar 19, 2025 at 1:07 AM, <john@gmail.com> wrote:
+>>>
+>>> Product Name :  Apple Developer Support
+>>>
+>>> Support Category :  Membership and Account
+>>>
+>>> Support Topic :  Other Membership or Account Questions
+>>>
+>>> Additional Details :
+>>>
+>>> Message:
+>>>
+>>> Hi,
+>>>
+>>> I was a member of the Apple Developer Program some time ago.
+>>>
+>>> I'm no longer a member so it won't let me specifically select "Payments"
+>>> page when contacting so I'm using this.
+>>>
+>>> During my period I checked my account has around $40 in revenue which
+>>> meets the minimum threshold for a payout, however I never received one.
+>>>
+>>> I was hoping you could look into it and see if I'm eligible to get the
+>>> payout?
+>>>
+>>> Thanks
+>>>
+>>> -John
+`,
+      attachments: [],
+      headers: new Map(),
+      headerLines: [],
+      html: false,
+    };
+
+    const result = extractTextWithoutReplyQuotations(parsed);
+
+    expect(result).toBe(`Hi John,
+
+Thank you for contacting Developer Support, this is Erica again. I hope you are having a good day. 
+
+I understand that you are unable to contact finance. Despite your account being expired, you should still be able to contact our finance team. 
+
+Follow the link below the link for contacting our finance team. 
+
+https://idmsa.apple.com/IDMSWebAuth/signin.html?path=/contact/finance/
+
+Best Regards,
+
+Erica 
+Developer Support`);
+  });
+
+  it('should handle email with reply quotations (Titan email style)', () => {
+    const parsed: ParsedMail = {
+      text: `just a follow up
+        
+        
+          On Aug 18 2025, at 4:06 pm, neo@titanemailtest.xyz wrote:
+          
+        
+        
+           Dear Colleagues,This is a reminder that the updated security policy goes into effect starting next Monday.  All employees must reset their corporate VPN credentials and enable two-factor authentication by then.  Please reach out to the IT helpdesk if you experience any issues during the setup.  Regards,  IT Department
+        
+      `,
+      attachments: [],
+      headers: new Map(),
+      headerLines: [],
+      html: false,
+    };
+
+    const result = extractTextWithoutReplyQuotations(parsed);
+
+    expect(result).toBe('just a follow up');
+  });
+
+  it('should handle html email with reply quotations', () => {
+    const parsed: ParsedMail = {
+      html: `<div fr-original-style="" style="user-select: inherit; scrollbar-color: var(--scrollbar-active-color) #0000; box-sizing: border-box;">just a follow up</div><br fr-original-style="" style="user-select: inherit; scrollbar-color: var(--scrollbar-active-color) #0000; box-sizing: border-box;"><img class="flm-open" width="0" height="0" style="border:0;width:0;height:0;" data-open-tracking-src="{{track-read-receipt}}"><div class="fr-inner gmail_quote flockmail-quote flockmail-quote-id-<186307386731076608.0.v2@titan.email>">
+        <br>
+        <div dir="ltr">
+          On Aug 18 2025, at 4:06 pm, neo@titanemailtest.xyz wrote:
+          <br>
+        </div>
+        <blockquote class="gmail_quote" style="margin:0 0 0 .8ex;border-left:1px #ccc solid;padding:initial;padding-left:1ex;color:inherit">
+           <div id="isPasted" fr-original-style="" style="display:block;user-select:inherit;scrollbar-color:var(--scrollbar-active-color) #0000;box-sizing:border-box">Dear Colleagues,</div><div fr-original-style="" style="display:block;user-select:inherit;scrollbar-color:var(--scrollbar-active-color) #0000;box-sizing:border-box"><br fr-original-style="" style="user-select:inherit;scrollbar-color:var(--scrollbar-active-color) #0000;box-sizing:border-box"></div><div fr-original-style="" style="display:block;user-select:inherit;scrollbar-color:var(--scrollbar-active-color) #0000;box-sizing:border-box">This is a reminder that the updated security policy goes into effect starting next Monday. &nbsp;</div><div fr-original-style="" style="display:block;user-select:inherit;scrollbar-color:var(--scrollbar-active-color) #0000;box-sizing:border-box">All employees must reset their corporate VPN credentials and enable two-factor authentication by then. &nbsp;</div><div fr-original-style="" style="display:block;user-select:inherit;scrollbar-color:var(--scrollbar-active-color) #0000;box-sizing:border-box"><br fr-original-style="" style="user-select:inherit;scrollbar-color:var(--scrollbar-active-color) #0000;box-sizing:border-box"></div><div fr-original-style="" style="display:block;user-select:inherit;scrollbar-color:var(--scrollbar-active-color) #0000;box-sizing:border-box">Please reach out to the IT helpdesk if you experience any issues during the setup. &nbsp;</div><div fr-original-style="" style="display:block;user-select:inherit;scrollbar-color:var(--scrollbar-active-color) #0000;box-sizing:border-box"><br fr-original-style="" style="user-select:inherit;scrollbar-color:var(--scrollbar-active-color) #0000;box-sizing:border-box"></div><div fr-original-style="" style="display:block;user-select:inherit;scrollbar-color:var(--scrollbar-active-color) #0000;box-sizing:border-box">Regards, &nbsp;</div><div fr-original-style="" style="display:block;user-select:inherit;scrollbar-color:var(--scrollbar-active-color) #0000;box-sizing:border-box">IT Department</div><div fr-original-style="" style="display:block;user-select:inherit;scrollbar-color:var(--scrollbar-active-color) #0000;box-sizing:border-box"><br fr-original-style="" style="user-select:inherit;scrollbar-color:var(--scrollbar-active-color) #0000;box-sizing:border-box"></div>
+        </blockquote>
+      </div>`,
+      attachments: [],
+      headers: new Map(),
+      headerLines: [],
+    };
+
+    const result = extractTextWithoutReplyQuotations(parsed);
+
+    expect(result).toBe('just a follow up');
+  });
+
+  it('should return empty string when no text or html content', () => {
+    const parsed: ParsedMail = {
+      attachments: [],
+      headers: new Map(),
+      headerLines: [],
+      html: false,
+    };
+
+    const result = extractTextWithoutReplyQuotations(parsed);
+
+    expect(result).toBe('');
+  });
+
+  it('should prefer text over html when both are available', () => {
+    const parsed: ParsedMail = {
+      text: 'Plain text content\n\nOn 2023-01-01, user@example.com wrote:\n> Reply',
+      html: '<html><body><p>HTML content</p></body></html>',
+      attachments: [],
+      headers: new Map(),
+      headerLines: [],
+    };
+
+    const result = extractTextWithoutReplyQuotations(parsed);
+
+    expect(result).toBe('Plain text content');
+  });
+});
@@ -0,0 +1,31 @@
+import DOMPurify from 'dompurify';
+import { JSDOM } from 'jsdom';
+import { type ParsedMail } from 'mailparser';
+import * as planer from 'planer';
+
+export const extractTextWithoutReplyQuotations = (
+  parsed: ParsedMail,
+): string => {
+  if (parsed.text) {
+    return planer.extractFrom(parsed.text, 'text/plain');
+  }
+
+  if (parsed.html) {
+    const window = new JSDOM('').window;
+    const purify = DOMPurify(window);
+    const sanitizedHtml = purify.sanitize(parsed.html);
+    const dom = new JSDOM(sanitizedHtml, { runScripts: 'outside-only' });
+
+    const cleanedHtml = planer.extractFromHtml(
+      sanitizedHtml,
+      dom.window.document,
+    );
+
+    const textContent = new JSDOM(cleanedHtml, { runScripts: 'outside-only' })
+      .window.document.body?.textContent;
+
+    return textContent ?? '';
+  }
+
+  return '';
+};
@@ -1,59 +0,0 @@
-import { type Logger } from '@nestjs/common';
-
-import { type ImapFlow } from 'imapflow';
-
-/**
- * Find sent folder using IMAP special-use flags
- *
- * This function uses IMAP special-use extension (RFC 6154) to identify
- * the sent folder by looking for the \Sent flag rather than relying on
- * folder names which can vary across providers and locales.
- *
- * Falls back to regex-based detection if special-use flags are not available.
- * The regex pattern is inspired by imapsync's comprehensive folder mapping.
- */
-export async function findSentMailbox(
-  client: ImapFlow,
-  logger: Logger,
-): Promise<string | null> {
-  try {
-    const list = await client.list();
-
-    logger.debug(
-      `Available folders: ${list.map((item) => item.path).join(', ')}`,
-    );
-
-    for (const folder of list) {
-      if (folder.specialUse && folder.specialUse.includes('\\Sent')) {
-        logger.log(`Found sent folder via special-use flag: ${folder.path}`);
-
-        return folder.path;
-      }
-    }
-
-    // Fallback: comprehensive regex pattern for legacy IMAP servers
-    // Source: https://imapsync.lamiral.info/FAQ.d/FAQ.Folders_Mapping.txt
-    // Based on imapsync's regextrans2 examples (originally "Sent|Sent Messages|Gesendet")
-    // Extended with additional common localizations for broader provider/language support
-    const sentFolderPattern =
-      /^(.*\/)?(sent|sent[\s_-]?(items|mail|messages|elements)?|envoy[éê]s?|[ée]l[ée]ments[\s_-]?envoy[éê]s|gesendet|gesendete[\s_-]?elemente|enviados?|elementos[\s_-]?enviados|itens[\s_-]?enviados|posta[\s_-]?inviata|inviati|보낸편지함|\[gmail\]\/sent[\s_-]?mail)$/i;
-
-    const availableFolders = list.map((item) => item.path);
-
-    for (const folder of availableFolders) {
-      if (sentFolderPattern.test(folder)) {
-        logger.log(`Found sent folder via pattern match: ${folder}`);
-
-        return folder;
-      }
-    }
-
-    logger.warn('No sent folder found. Only inbox messages will be imported.');
-
-    return null;
-  } catch (error) {
-    logger.warn(`Error listing mailboxes: ${error.message}`);
-
-    return null;
-  }
-}