6256 refactor messaging module to remove all provider specific code and put it inside the drivers folders (#6721)

Closes #6256 Closes #6257 + Create custom exceptions --------- Co-authored-by: Charles Bochet <charles@twenty.com>
2024-08-27 18:14:45 +02:00
parent eb49cb2d08
commit 81fa3f0c41
62 changed files with 1540 additions and 1360 deletions
@@ -0,0 +1,17 @@
+import { CustomException } from 'src/utils/custom-exception';
+
+export class MessageImportDriverException extends CustomException {
+  code: MessageImportDriverExceptionCode;
+  constructor(message: string, code: MessageImportDriverExceptionCode) {
+    super(message, code);
+  }
+}
+
+export enum MessageImportDriverExceptionCode {
+  NOT_FOUND = 'NOT_FOUND',
+  TEMPORARY_ERROR = 'TEMPORARY_ERROR',
+  INSUFFICIENT_PERMISSIONS = 'INSUFFICIENT_PERMISSIONS',
+  UNKNOWN = 'UNKNOWN',
+  UNKNOWN_NETWORK_ERROR = 'UNKNOWN_NETWORK_ERROR',
+  NO_NEXT_SYNC_CURSOR = 'NO_NEXT_SYNC_CURSOR',
+}
@@ -12,11 +12,12 @@ import { EmailAliasManagerModule } from 'src/modules/connected-account/email-ali
 import { OAuth2ClientManagerModule } from 'src/modules/connected-account/oauth2-client-manager/oauth2-client-manager.module';
 import { ConnectedAccountWorkspaceEntity } from 'src/modules/connected-account/standard-objects/connected-account.workspace-entity';
 import { MessagingCommonModule } from 'src/modules/messaging/common/messaging-common.module';
-import { MessagingGmailClientProvider } from 'src/modules/messaging/message-import-manager/drivers/gmail/providers/messaging-gmail-client.provider';
-import { MessagingGmailFetchByBatchesService } from 'src/modules/messaging/message-import-manager/drivers/gmail/services/messaging-gmail-fetch-by-batch.service';
-import { MessagingGmailFetchMessagesByBatchesService } from 'src/modules/messaging/message-import-manager/drivers/gmail/services/messaging-gmail-fetch-messages-by-batches.service';
-import { MessagingGmailFetchMessageIdsToExcludeService } from 'src/modules/messaging/message-import-manager/drivers/gmail/services/messaging-gmail-fetch-messages-ids-to-exclude.service';
-import { MessagingGmailHistoryService } from 'src/modules/messaging/message-import-manager/drivers/gmail/services/messaging-gmail-history.service';
+import { GmailClientProvider } from 'src/modules/messaging/message-import-manager/drivers/gmail/providers/gmail-client.provider';
+import { GmailFetchByBatchService } from 'src/modules/messaging/message-import-manager/drivers/gmail/services/gmail-fetch-by-batch.service';
+import { GmailGetHistoryService } from 'src/modules/messaging/message-import-manager/drivers/gmail/services/gmail-get-history.service';
+import { GmailGetMessageListService } from 'src/modules/messaging/message-import-manager/drivers/gmail/services/gmail-get-message-list.service';
+import { GmailGetMessagesService } from 'src/modules/messaging/message-import-manager/drivers/gmail/services/gmail-get-messages.service';
+import { GmailHandleErrorService } from 'src/modules/messaging/message-import-manager/drivers/gmail/services/gmail-handle-error.service';
 import { MessageParticipantManagerModule } from 'src/modules/messaging/message-participant-manager/message-participant-manager.module';

@Module({
@@ -38,18 +39,13 @@ import { MessageParticipantManagerModule } from 'src/modules/messaging/message-p
    MessageParticipantManagerModule,
  ],
  providers: [
-    MessagingGmailClientProvider,
-    MessagingGmailHistoryService,
-    MessagingGmailFetchByBatchesService,
-    MessagingGmailFetchMessagesByBatchesService,
-    MessagingGmailFetchMessageIdsToExcludeService,
-  ],
-  exports: [
-    MessagingGmailClientProvider,
-    MessagingGmailHistoryService,
-    MessagingGmailFetchByBatchesService,
-    MessagingGmailFetchMessagesByBatchesService,
-    MessagingGmailFetchMessageIdsToExcludeService,
+    GmailClientProvider,
+    GmailGetHistoryService,
+    GmailFetchByBatchService,
+    GmailGetMessagesService,
+    GmailGetMessageListService,
+    GmailHandleErrorService,
  ],
+  exports: [GmailGetMessagesService, GmailGetMessageListService],
 })
 export class MessagingGmailDriverModule {}
@@ -6,7 +6,7 @@ import { OAuth2ClientManagerService } from 'src/modules/connected-account/oauth2
 import { ConnectedAccountWorkspaceEntity } from 'src/modules/connected-account/standard-objects/connected-account.workspace-entity';

@Injectable()
-export class MessagingGmailClientProvider {
+export class GmailClientProvider {
  constructor(
    private readonly oAuth2ClientManagerService: OAuth2ClientManagerService,
  ) {}
@@ -3,12 +3,12 @@ import { Injectable } from '@nestjs/common';

 import { AxiosResponse } from 'axios';

-import { GmailMessageParsedResponse } from 'src/modules/messaging/message-import-manager/drivers/gmail/types/gmail-message-parsed-response';
+import { GmailMessageParsedResponse } from 'src/modules/messaging/message-import-manager/drivers/gmail/types/gmail-message-parsed-response.type';
+import { createQueriesFromMessageIds } from 'src/modules/messaging/message-import-manager/drivers/gmail/utils/create-queries-from-message-ids.util';
 import { BatchQueries } from 'src/modules/messaging/message-import-manager/types/batch-queries';
-import { createQueriesFromMessageIds } from 'src/modules/messaging/message-import-manager/utils/create-queries-from-message-ids.util';

@Injectable()
-export class MessagingGmailFetchByBatchesService {
+export class GmailFetchByBatchService {
  constructor(private readonly httpService: HttpService) {}

  async fetchAllByBatches(
@@ -1,14 +1,15 @@
 import { Injectable } from '@nestjs/common';

-import { GaxiosResponse } from 'gaxios';
 import { gmail_v1 } from 'googleapis';

 import { MESSAGING_GMAIL_USERS_HISTORY_MAX_RESULT } from 'src/modules/messaging/message-import-manager/drivers/gmail/constants/messaging-gmail-users-history-max-result.constant';
-import { GmailError } from 'src/modules/messaging/message-import-manager/services/messaging-error-handling.service';
+import { GmailHandleErrorService } from 'src/modules/messaging/message-import-manager/drivers/gmail/services/gmail-handle-error.service';

@Injectable()
-export class MessagingGmailHistoryService {
-  constructor() {}
+export class GmailGetHistoryService {
+  constructor(
+    private readonly gmailHandleErrorService: GmailHandleErrorService,
+  ) {}

  public async getHistory(
    gmailClient: gmail_v1.Gmail,
@@ -18,34 +19,33 @@ export class MessagingGmailHistoryService {
  ): Promise<{
    history: gmail_v1.Schema$History[];
    historyId?: string | null;
-    error?: GmailError;
  }> {
    const fullHistory: gmail_v1.Schema$History[] = [];
    let pageToken: string | undefined;
    let hasMoreMessages = true;
    let nextHistoryId: string | undefined;
-    let response: GaxiosResponse<gmail_v1.Schema$ListHistoryResponse>;

    while (hasMoreMessages) {
-      try {
-        response = await gmailClient.users.history.list({
+      const response = await gmailClient.users.history
+        .list({
          userId: 'me',
          maxResults: MESSAGING_GMAIL_USERS_HISTORY_MAX_RESULT,
          pageToken,
          startHistoryId: lastSyncHistoryId,
          historyTypes: historyTypes || ['messageAdded', 'messageDeleted'],
          labelId,
+        })
+        .catch((error) => {
+          this.gmailHandleErrorService.handleError(error);
+
+          return {
+            data: {
+              history: [],
+              historyId: lastSyncHistoryId,
+              nextPageToken: undefined,
+            },
+          };
        });
-      } catch (error) {
-        return {
-          history: [],
-          error: {
-            code: error.response?.status,
-            reason: error.response?.data?.error,
-          },
-          historyId: lastSyncHistoryId,
-        };
-      }

      nextHistoryId = response?.data?.historyId ?? undefined;

@@ -0,0 +1,168 @@
+import { Injectable } from '@nestjs/common';
+
+import { gmail_v1 as gmailV1 } from 'googleapis';
+
+import { ConnectedAccountWorkspaceEntity } from 'src/modules/connected-account/standard-objects/connected-account.workspace-entity';
+import {
+  MessageImportDriverException,
+  MessageImportDriverExceptionCode,
+} from 'src/modules/messaging/message-import-manager/drivers/exceptions/message-import-driver.exception';
+import { MESSAGING_GMAIL_EXCLUDED_CATEGORIES } from 'src/modules/messaging/message-import-manager/drivers/gmail/constants/messaging-gmail-excluded-categories';
+import { MESSAGING_GMAIL_USERS_MESSAGES_LIST_MAX_RESULT } from 'src/modules/messaging/message-import-manager/drivers/gmail/constants/messaging-gmail-users-messages-list-max-result.constant';
+import { GmailClientProvider } from 'src/modules/messaging/message-import-manager/drivers/gmail/providers/gmail-client.provider';
+import { GmailGetHistoryService } from 'src/modules/messaging/message-import-manager/drivers/gmail/services/gmail-get-history.service';
+import { GmailHandleErrorService } from 'src/modules/messaging/message-import-manager/drivers/gmail/services/gmail-handle-error.service';
+import { computeGmailCategoryExcludeSearchFilter } from 'src/modules/messaging/message-import-manager/drivers/gmail/utils/compute-gmail-category-excude-search-filter.util';
+import { computeGmailCategoryLabelId } from 'src/modules/messaging/message-import-manager/drivers/gmail/utils/compute-gmail-category-label-id.util';
+import {
+  GetFullMessageListResponse,
+  GetPartialMessageListResponse,
+} from 'src/modules/messaging/message-import-manager/services/messaging-get-message-list.service';
+import { assertNotNull } from 'src/utils/assert';
+
+@Injectable()
+export class GmailGetMessageListService {
+  constructor(
+    private readonly gmailClientProvider: GmailClientProvider,
+    private readonly gmailGetHistoryService: GmailGetHistoryService,
+    private readonly gmailHandleErrorService: GmailHandleErrorService,
+  ) {}
+
+  public async getFullMessageList(
+    connectedAccount: Pick<
+      ConnectedAccountWorkspaceEntity,
+      'provider' | 'refreshToken' | 'id'
+    >,
+  ): Promise<GetFullMessageListResponse> {
+    const gmailClient =
+      await this.gmailClientProvider.getGmailClient(connectedAccount);
+
+    let pageToken: string | undefined;
+    let hasMoreMessages = true;
+    let firstMessageExternalId: string | undefined;
+    const messageExternalIds: string[] = [];
+
+    while (hasMoreMessages) {
+      const messageList = await gmailClient.users.messages
+        .list({
+          userId: 'me',
+          maxResults: MESSAGING_GMAIL_USERS_MESSAGES_LIST_MAX_RESULT,
+          pageToken,
+          q: computeGmailCategoryExcludeSearchFilter(
+            MESSAGING_GMAIL_EXCLUDED_CATEGORIES,
+          ),
+        })
+        .catch((error) => {
+          this.gmailHandleErrorService.handleError(error);
+
+          return {
+            data: {
+              messages: [],
+              nextPageToken: undefined,
+            },
+          };
+        });
+
+      pageToken = messageList.data.nextPageToken ?? undefined;
+      hasMoreMessages = !!pageToken;
+
+      const { messages } = messageList.data;
+
+      if (!messages || messages.length === 0) {
+        break;
+      }
+
+      if (!firstMessageExternalId) {
+        firstMessageExternalId = messageList.data.messages?.[0].id ?? undefined;
+      }
+
+      messageExternalIds.push(...messages.map((message) => message.id));
+    }
+
+    const firstMessageContent = await gmailClient.users.messages
+      .get({
+        userId: 'me',
+        id: firstMessageExternalId,
+      })
+      .catch((error) => {
+        this.gmailHandleErrorService.handleError(error);
+      });
+
+    const nextSyncCursor = firstMessageContent?.data?.historyId;
+
+    if (!nextSyncCursor) {
+      throw new MessageImportDriverException(
+        `No historyId found for message ${firstMessageExternalId} for connected account ${connectedAccount.id}`,
+        MessageImportDriverExceptionCode.NO_NEXT_SYNC_CURSOR,
+      );
+    }
+
+    return { messageExternalIds, nextSyncCursor };
+  }
+
+  public async getPartialMessageList(
+    connectedAccount: Pick<
+      ConnectedAccountWorkspaceEntity,
+      'provider' | 'refreshToken' | 'id'
+    >,
+    syncCursor: string,
+  ): Promise<GetPartialMessageListResponse> {
+    const gmailClient =
+      await this.gmailClientProvider.getGmailClient(connectedAccount);
+
+    const { history, historyId: nextSyncCursor } =
+      await this.gmailGetHistoryService.getHistory(gmailClient, syncCursor);
+
+    const { messagesAdded, messagesDeleted } =
+      await this.gmailGetHistoryService.getMessageIdsFromHistory(history);
+
+    const messageIdsToFilter = await this.getEmailIdsFromExcludedCategories(
+      gmailClient,
+      syncCursor,
+    );
+
+    const messagesAddedFiltered = messagesAdded.filter(
+      (messageId) => !messageIdsToFilter.includes(messageId),
+    );
+
+    if (!nextSyncCursor) {
+      throw new MessageImportDriverException(
+        `No nextSyncCursor found for connected account ${connectedAccount.id}`,
+        MessageImportDriverExceptionCode.NO_NEXT_SYNC_CURSOR,
+      );
+    }
+
+    return {
+      messageExternalIds: messagesAddedFiltered,
+      messageExternalIdsToDelete: messagesDeleted,
+      nextSyncCursor,
+    };
+  }
+
+  private async getEmailIdsFromExcludedCategories(
+    gmailClient: gmailV1.Gmail,
+    lastSyncHistoryId: string,
+  ): Promise<string[]> {
+    const emailIds: string[] = [];
+
+    for (const category of MESSAGING_GMAIL_EXCLUDED_CATEGORIES) {
+      const { history } = await this.gmailGetHistoryService.getHistory(
+        gmailClient,
+        lastSyncHistoryId,
+        ['messageAdded'],
+        computeGmailCategoryLabelId(category),
+      );
+
+      const emailIdsFromCategory = history
+        .map((history) => history.messagesAdded)
+        .flat()
+        .map((message) => message?.message?.id)
+        .filter((id) => id)
+        .filter(assertNotNull);
+
+      emailIds.push(...emailIdsFromCategory);
+    }
+
+    return emailIds;
+  }
+}
@@ -0,0 +1,98 @@
+import { Injectable, Logger } from '@nestjs/common';
+
+import { AxiosResponse } from 'axios';
+import { gmail_v1 as gmailV1 } from 'googleapis';
+
+import { ConnectedAccountWorkspaceEntity } from 'src/modules/connected-account/standard-objects/connected-account.workspace-entity';
+import { GmailFetchByBatchService } from 'src/modules/messaging/message-import-manager/drivers/gmail/services/gmail-fetch-by-batch.service';
+import { GmailHandleErrorService } from 'src/modules/messaging/message-import-manager/drivers/gmail/services/gmail-handle-error.service';
+import { parseAndFormatGmailMessage } from 'src/modules/messaging/message-import-manager/drivers/gmail/utils/parse-and-format-gmail-message.util';
+import { MessageWithParticipants } from 'src/modules/messaging/message-import-manager/types/message';
+import { isDefined } from 'src/utils/is-defined';
+
+@Injectable()
+export class GmailGetMessagesService {
+  private readonly logger = new Logger(GmailGetMessagesService.name);
+
+  constructor(
+    private readonly fetchByBatchesService: GmailFetchByBatchService,
+    private readonly gmailHandleErrorService: GmailHandleErrorService,
+  ) {}
+
+  async getMessages(
+    messageIds: string[],
+    connectedAccount: Pick<
+      ConnectedAccountWorkspaceEntity,
+      'accessToken' | 'refreshToken' | 'id' | 'handle' | 'handleAliases'
+    >,
+    workspaceId: string,
+  ): Promise<MessageWithParticipants[]> {
+    let startTime = Date.now();
+
+    const { messageIdsByBatch, batchResponses } =
+      await this.fetchByBatchesService.fetchAllByBatches(
+        messageIds,
+        connectedAccount.accessToken,
+        'batch_gmail_messages',
+      );
+    let endTime = Date.now();
+
+    this.logger.log(
+      `Messaging import for workspace ${workspaceId} and account ${connectedAccount.id} fetching ${
+        messageIds.length
+      } messages in ${endTime - startTime}ms`,
+    );
+
+    startTime = Date.now();
+
+    const messages = batchResponses.flatMap((response, index) => {
+      return this.formatBatchResponseAsMessage(
+        messageIdsByBatch[index],
+        response,
+        connectedAccount,
+      );
+    });
+
+    endTime = Date.now();
+
+    this.logger.log(
+      `Messaging import for workspace ${workspaceId} and account ${connectedAccount.id} formatting ${
+        messageIds.length
+      } messages in ${endTime - startTime}ms`,
+    );
+
+    return messages;
+  }
+
+  private formatBatchResponseAsMessage(
+    messageIds: string[],
+    responseCollection: AxiosResponse<any, any>,
+    connectedAccount: Pick<
+      ConnectedAccountWorkspaceEntity,
+      'handle' | 'handleAliases'
+    >,
+  ): MessageWithParticipants[] {
+    const parsedResponses =
+      this.fetchByBatchesService.parseBatch(responseCollection);
+
+    const messages = parsedResponses.map((response, index) => {
+      if ('error' in response) {
+        if (response.error.code === 404) {
+          return null;
+        }
+
+        this.gmailHandleErrorService.handleError(
+          response.error,
+          messageIds[index],
+        );
+      }
+
+      return parseAndFormatGmailMessage(
+        response as gmailV1.Schema$Message,
+        connectedAccount,
+      );
+    });
+
+    return messages.filter(isDefined);
+  }
+}
@@ -0,0 +1,33 @@
+import { Injectable } from '@nestjs/common';
+
+import { parseGaxiosError } from 'src/modules/messaging/message-import-manager/drivers/gmail/utils/parse-gaxios-error.util';
+import { parseGmailError } from 'src/modules/messaging/message-import-manager/drivers/gmail/utils/parse-gmail-error.util';
+
+@Injectable()
+export class GmailHandleErrorService {
+  constructor() {}
+
+  public handleError(error: any, messageExternalId?: string): void {
+    if (
+      error.code &&
+      [
+        'ECONNRESET',
+        'ENOTFOUND',
+        'ECONNABORTED',
+        'ETIMEDOUT',
+        'ERR_NETWORK',
+      ].includes(error.code)
+    ) {
+      throw parseGaxiosError(error);
+    }
+    if (error.response?.status !== 410) {
+      const gmailError = {
+        code: error.response?.status,
+        reason: `${error.response?.data?.error?.errors?.[0].reason || error.response?.data?.error || ''}`,
+        message: `${error.response?.data?.error?.errors?.[0].message || error.response?.data?.error_description || ''}${messageExternalId ? ` for message with externalId: ${messageExternalId}` : ''}`,
+      };
+
+      throw parseGmailError(gmailError);
+    }
+  }
+}
@@ -1,263 +0,0 @@
-import { Injectable, Logger } from '@nestjs/common';
-
-import addressparser from 'addressparser';
-import { AxiosResponse } from 'axios';
-import { gmail_v1 } from 'googleapis';
-import planer from 'planer';
-
-import { MessagingGmailFetchByBatchesService } from 'src/modules/messaging/message-import-manager/drivers/gmail/services/messaging-gmail-fetch-by-batch.service';
-import { GmailMessage } from 'src/modules/messaging/message-import-manager/drivers/gmail/types/gmail-message';
-import { formatAddressObjectAsParticipants } from 'src/modules/messaging/message-import-manager/utils/format-address-object-as-participants.util';
-import { assert, assertNotNull } from 'src/utils/assert';
-
-@Injectable()
-export class MessagingGmailFetchMessagesByBatchesService {
-  private readonly logger = new Logger(
-    MessagingGmailFetchMessagesByBatchesService.name,
-  );
-
-  constructor(
-    private readonly fetchByBatchesService: MessagingGmailFetchByBatchesService,
-  ) {}
-
-  async fetchAllMessages(
-    messageIds: string[],
-    accessToken: string,
-    connectedAccountId: string,
-    workspaceId: string,
-  ): Promise<GmailMessage[]> {
-    let startTime = Date.now();
-
-    const { messageIdsByBatch, batchResponses } =
-      await this.fetchByBatchesService.fetchAllByBatches(
-        messageIds,
-        accessToken,
-        'batch_gmail_messages',
-      );
-    let endTime = Date.now();
-
-    this.logger.log(
-      `Messaging import for workspace ${workspaceId} and account ${connectedAccountId} fetching ${
-        messageIds.length
-      } messages in ${endTime - startTime}ms`,
-    );
-
-    startTime = Date.now();
-
-    const formattedResponse = this.formatBatchResponsesAsGmailMessages(
-      messageIdsByBatch,
-      batchResponses,
-      workspaceId,
-      connectedAccountId,
-    );
-
-    endTime = Date.now();
-
-    this.logger.log(
-      `Messaging import for workspace ${workspaceId} and account ${connectedAccountId} formatting ${
-        messageIds.length
-      } messages in ${endTime - startTime}ms`,
-    );
-
-    return formattedResponse;
-  }
-
-  private formatBatchResponseAsGmailMessage(
-    messageIds: string[],
-    responseCollection: AxiosResponse<any, any>,
-    workspaceId: string,
-    connectedAccountId: string,
-  ): GmailMessage[] {
-    const parsedResponses =
-      this.fetchByBatchesService.parseBatch(responseCollection);
-
-    const sanitizeString = (str: string) => {
-      return str.replace(/\0/g, '');
-    };
-
-    const formattedResponse = parsedResponses.map((response, index) => {
-      if ('error' in response) {
-        if (response.error.code === 404) {
-          return null;
-        }
-
-        throw { ...response.error, messageId: messageIds[index] };
-      }
-
-      const {
-        historyId,
-        id,
-        threadId,
-        internalDate,
-        subject,
-        from,
-        to,
-        cc,
-        bcc,
-        headerMessageId,
-        text,
-        attachments,
-        deliveredTo,
-      } = this.parseGmailMessage(response);
-
-      if (!from) {
-        this.logger.log(
-          `From value is missing while importing message #${id} in workspace ${workspaceId} and account ${connectedAccountId}`,
-        );
-
-        return null;
-      }
-
-      if (!to && !deliveredTo && !bcc && !cc) {
-        this.logger.log(
-          `To, Delivered-To, Bcc or Cc value is missing while importing message #${id} in workspace ${workspaceId} and account ${connectedAccountId}`,
-        );
-
-        return null;
-      }
-
-      if (!headerMessageId) {
-        this.logger.log(
-          `Message-ID is missing while importing message #${id} in workspace ${workspaceId} and account ${connectedAccountId}`,
-        );
-
-        return null;
-      }
-
-      if (!threadId) {
-        this.logger.log(
-          `Thread Id is missing while importing message #${id} in workspace ${workspaceId} and account ${connectedAccountId}`,
-        );
-
-        return null;
-      }
-
-      const participants = [
-        ...formatAddressObjectAsParticipants(from, 'from'),
-        ...formatAddressObjectAsParticipants(to ?? deliveredTo, 'to'),
-        ...formatAddressObjectAsParticipants(cc, 'cc'),
-        ...formatAddressObjectAsParticipants(bcc, 'bcc'),
-      ];
-
-      let textWithoutReplyQuotations = text;
-
-      if (text) {
-        textWithoutReplyQuotations = planer.extractFrom(text, 'text/plain');
-      }
-
-      const messageFromGmail: GmailMessage = {
-        historyId,
-        externalId: id,
-        headerMessageId,
-        subject: subject || '',
-        messageThreadExternalId: threadId,
-        internalDate,
-        fromHandle: from[0].address || '',
-        fromDisplayName: from[0].name || '',
-        participants,
-        text: sanitizeString(textWithoutReplyQuotations || ''),
-        attachments,
-      };
-
-      return messageFromGmail;
-    });
-
-    const filteredMessages = formattedResponse.filter((message) =>
-      assertNotNull(message),
-    ) as GmailMessage[];
-
-    return filteredMessages;
-  }
-
-  private formatBatchResponsesAsGmailMessages(
-    messageIdsByBatch: string[][],
-    batchResponses: AxiosResponse<any, any>[],
-    workspaceId: string,
-    connectedAccountId: string,
-  ): GmailMessage[] {
-    const messageBatches = batchResponses.map((response, index) => {
-      return this.formatBatchResponseAsGmailMessage(
-        messageIdsByBatch[index],
-        response,
-        workspaceId,
-        connectedAccountId,
-      );
-    });
-
-    return messageBatches.flat();
-  }
-
-  private parseGmailMessage(message: gmail_v1.Schema$Message) {
-    const subject = this.getPropertyFromHeaders(message, 'Subject');
-    const rawFrom = this.getPropertyFromHeaders(message, 'From');
-    const rawTo = this.getPropertyFromHeaders(message, 'To');
-    const rawDeliveredTo = this.getPropertyFromHeaders(message, 'Delivered-To');
-    const rawCc = this.getPropertyFromHeaders(message, 'Cc');
-    const rawBcc = this.getPropertyFromHeaders(message, 'Bcc');
-    const messageId = this.getPropertyFromHeaders(message, 'Message-ID');
-    const id = message.id;
-    const threadId = message.threadId;
-    const historyId = message.historyId;
-    const internalDate = message.internalDate;
-
-    assert(id, 'ID is missing');
-    assert(historyId, 'History-ID is missing');
-    assert(internalDate, 'Internal date is missing');
-
-    const bodyData = this.getBodyData(message);
-    const text = bodyData ? Buffer.from(bodyData, 'base64').toString() : '';
-
-    const attachments = this.getAttachmentData(message);
-
-    return {
-      id,
-      headerMessageId: messageId,
-      threadId,
-      historyId,
-      internalDate,
-      subject,
-      from: rawFrom ? addressparser(rawFrom) : undefined,
-      deliveredTo: rawDeliveredTo ? addressparser(rawDeliveredTo) : undefined,
-      to: rawTo ? addressparser(rawTo) : undefined,
-      cc: rawCc ? addressparser(rawCc) : undefined,
-      bcc: rawBcc ? addressparser(rawBcc) : undefined,
-      text,
-      attachments,
-    };
-  }
-
-  private getBodyData(message: gmail_v1.Schema$Message) {
-    const firstPart = message.payload?.parts?.[0];
-
-    if (firstPart?.mimeType === 'text/plain') {
-      return firstPart?.body?.data;
-    }
-
-    return firstPart?.parts?.find((part) => part.mimeType === 'text/plain')
-      ?.body?.data;
-  }
-
-  private getAttachmentData(message: gmail_v1.Schema$Message) {
-    return (
-      message.payload?.parts
-        ?.filter((part) => part.filename && part.body?.attachmentId)
-        .map((part) => ({
-          filename: part.filename || '',
-          id: part.body?.attachmentId || '',
-          mimeType: part.mimeType || '',
-          size: part.body?.size || 0,
-        })) || []
-    );
-  }
-
-  private getPropertyFromHeaders(
-    message: gmail_v1.Schema$Message,
-    property: string,
-  ) {
-    const header = message.payload?.headers?.find(
-      (header) => header.name?.toLowerCase() === property.toLowerCase(),
-    );
-
-    return header?.value;
-  }
-}
@@ -1,46 +0,0 @@
-import { Injectable } from '@nestjs/common';
-
-import { gmail_v1 } from 'googleapis';
-
-import { MESSAGING_GMAIL_EXCLUDED_CATEGORIES } from 'src/modules/messaging/message-import-manager/drivers/gmail/constants/messaging-gmail-excluded-categories';
-import { MessagingGmailHistoryService } from 'src/modules/messaging/message-import-manager/drivers/gmail/services/messaging-gmail-history.service';
-import { computeGmailCategoryLabelId } from 'src/modules/messaging/message-import-manager/drivers/gmail/utils/compute-gmail-category-label-id';
-import { assertNotNull } from 'src/utils/assert';
-
-@Injectable()
-export class MessagingGmailFetchMessageIdsToExcludeService {
-  constructor(
-    private readonly gmailGetHistoryService: MessagingGmailHistoryService,
-  ) {}
-
-  public async fetchEmailIdsToExcludeOrThrow(
-    gmailClient: gmail_v1.Gmail,
-    lastSyncHistoryId: string,
-  ): Promise<string[]> {
-    const emailIds: string[] = [];
-
-    for (const category of MESSAGING_GMAIL_EXCLUDED_CATEGORIES) {
-      const { history, error } = await this.gmailGetHistoryService.getHistory(
-        gmailClient,
-        lastSyncHistoryId,
-        ['messageAdded'],
-        computeGmailCategoryLabelId(category),
-      );
-
-      if (error) {
-        throw error;
-      }
-
-      const emailIdsFromCategory = history
-        .map((history) => history.messagesAdded)
-        .flat()
-        .map((message) => message?.message?.id)
-        .filter((id) => id)
-        .filter(assertNotNull);
-
-      emailIds.push(...emailIdsFromCategory);
-    }
-
-    return emailIds;
-  }
-}
@@ -1,4 +1,4 @@
-import { computeGmailCategoryExcludeSearchFilter } from 'src/modules/messaging/message-import-manager/drivers/gmail/utils/compute-gmail-category-excude-search-filter';
+import { computeGmailCategoryExcludeSearchFilter } from 'src/modules/messaging/message-import-manager/drivers/gmail/utils/compute-gmail-category-excude-search-filter.util';

 describe('computeGmailCategoryExcludeSearchFilter', () => {
  it('should return correct exclude search filter with empty category array', () => {
@@ -1,4 +1,4 @@
-import { computeGmailCategoryLabelId } from 'src/modules/messaging/message-import-manager/drivers/gmail/utils/compute-gmail-category-label-id';
+import { computeGmailCategoryLabelId } from 'src/modules/messaging/message-import-manager/drivers/gmail/utils/compute-gmail-category-label-id.util';

 describe('computeGmailCategoryLabelId', () => {
  it('should return correct category label id', () => {
@@ -0,0 +1,13 @@
+import { ConnectedAccountWorkspaceEntity } from 'src/modules/connected-account/standard-objects/connected-account.workspace-entity';
+
+export const computeMessageDirection = (
+  fromHandle: string,
+  connectedAccount: Pick<
+    ConnectedAccountWorkspaceEntity,
+    'handle' | 'handleAliases'
+  >,
+): 'outgoing' | 'incoming' =>
+  connectedAccount.handle === fromHandle ||
+  connectedAccount.handleAliases?.includes(fromHandle)
+    ? 'outgoing'
+    : 'incoming';
@@ -0,0 +1,9 @@
+import { MessageQuery } from 'src/modules/messaging/message-import-manager/types/message-or-thread-query';
+
+export const createQueriesFromMessageIds = (
+  messageExternalIds: string[],
+): MessageQuery[] => {
+  return messageExternalIds.map((messageId) => ({
+    uri: '/gmail/v1/users/me/messages/' + messageId + '?format=FULL',
+  }));
+};
@@ -0,0 +1,14 @@
+import { gmail_v1 as gmailV1 } from 'googleapis';
+
+export const getAttachmentData = (message: gmailV1.Schema$Message) => {
+  return (
+    message.payload?.parts
+      ?.filter((part) => part.filename && part.body?.attachmentId)
+      .map((part) => ({
+        filename: part.filename ?? '',
+        id: part.body?.attachmentId ?? '',
+        mimeType: part.mimeType ?? '',
+        size: part.body?.size ?? 0,
+      })) ?? []
+  );
+};
@@ -0,0 +1,12 @@
+import { gmail_v1 as gmailV1 } from 'googleapis';
+
+export const getBodyData = (message: gmailV1.Schema$Message) => {
+  const firstPart = message.payload?.parts?.[0];
+
+  if (firstPart?.mimeType === 'text/plain') {
+    return firstPart?.body?.data;
+  }
+
+  return firstPart?.parts?.find((part) => part.mimeType === 'text/plain')?.body
+    ?.data;
+};
@@ -0,0 +1,12 @@
+import { gmail_v1 as gmailV1 } from 'googleapis';
+
+export const getPropertyFromHeaders = (
+  message: gmailV1.Schema$Message,
+  property: string,
+) => {
+  const header = message.payload?.headers?.find(
+    (header) => header.name?.toLowerCase() === property.toLowerCase(),
+  );
+
+  return header?.value;
+};
@@ -0,0 +1,64 @@
+import { gmail_v1 as gmailV1 } from 'googleapis';
+import planer from 'planer';
+
+import { ConnectedAccountWorkspaceEntity } from 'src/modules/connected-account/standard-objects/connected-account.workspace-entity';
+import { computeMessageDirection } from 'src/modules/messaging/message-import-manager/drivers/gmail/utils/compute-message-direction.util';
+import { parseGmailMessage } from 'src/modules/messaging/message-import-manager/drivers/gmail/utils/parse-gmail-message.util';
+import { sanitizeString } from 'src/modules/messaging/message-import-manager/drivers/gmail/utils/sanitize-string.util';
+import { MessageWithParticipants } from 'src/modules/messaging/message-import-manager/types/message';
+import { formatAddressObjectAsParticipants } from 'src/modules/messaging/message-import-manager/utils/format-address-object-as-participants.util';
+
+export const parseAndFormatGmailMessage = (
+  message: gmailV1.Schema$Message,
+  connectedAccount: Pick<
+    ConnectedAccountWorkspaceEntity,
+    'handle' | 'handleAliases'
+  >,
+): MessageWithParticipants | null => {
+  const {
+    id,
+    threadId,
+    internalDate,
+    subject,
+    from,
+    to,
+    cc,
+    bcc,
+    headerMessageId,
+    text,
+    attachments,
+    deliveredTo,
+  } = parseGmailMessage(message);
+
+  if (
+    !from ||
+    (!to && !deliveredTo && !bcc && !cc) ||
+    !headerMessageId ||
+    !threadId
+  ) {
+    return null;
+  }
+
+  const participants = [
+    ...formatAddressObjectAsParticipants(from, 'from'),
+    ...formatAddressObjectAsParticipants(to ?? deliveredTo, 'to'),
+    ...formatAddressObjectAsParticipants(cc, 'cc'),
+    ...formatAddressObjectAsParticipants(bcc, 'bcc'),
+  ];
+
+  const textWithoutReplyQuotations = text
+    ? planer.extractFrom(text, 'text/plain')
+    : '';
+
+  return {
+    externalId: id,
+    headerMessageId,
+    subject: subject || '',
+    messageThreadExternalId: threadId,
+    receivedAt: new Date(parseInt(internalDate)),
+    direction: computeMessageDirection(from[0].address || '', connectedAccount),
+    participants,
+    text: sanitizeString(textWithoutReplyQuotations),
+    attachments,
+  };
+};
@@ -0,0 +1,30 @@
+import { GaxiosError } from 'gaxios';
+
+import {
+  MessageImportDriverException,
+  MessageImportDriverExceptionCode,
+} from 'src/modules/messaging/message-import-manager/drivers/exceptions/message-import-driver.exception';
+
+export const parseGaxiosError = (
+  error: GaxiosError,
+): MessageImportDriverException => {
+  const { code } = error;
+
+  switch (code) {
+    case 'ECONNRESET':
+    case 'ENOTFOUND':
+    case 'ECONNABORTED':
+    case 'ETIMEDOUT':
+    case 'ERR_NETWORK':
+      return new MessageImportDriverException(
+        error.message,
+        MessageImportDriverExceptionCode.TEMPORARY_ERROR,
+      );
+
+    default:
+      return new MessageImportDriverException(
+        error.message,
+        MessageImportDriverExceptionCode.UNKNOWN,
+      );
+  }
+};
@@ -0,0 +1,88 @@
+import {
+  MessageImportDriverException,
+  MessageImportDriverExceptionCode,
+} from 'src/modules/messaging/message-import-manager/drivers/exceptions/message-import-driver.exception';
+
+export const parseGmailError = (error: {
+  code?: number;
+  reason: string;
+  message: string;
+}): MessageImportDriverException => {
+  const { code, reason, message } = error;
+
+  switch (code) {
+    case 400:
+      if (reason === 'invalid_grant') {
+        return new MessageImportDriverException(
+          message,
+          MessageImportDriverExceptionCode.INSUFFICIENT_PERMISSIONS,
+        );
+      }
+      if (reason === 'failedPrecondition') {
+        return new MessageImportDriverException(
+          message,
+          MessageImportDriverExceptionCode.TEMPORARY_ERROR,
+        );
+      }
+
+      return new MessageImportDriverException(
+        message,
+        MessageImportDriverExceptionCode.UNKNOWN,
+      );
+
+    case 404:
+      return new MessageImportDriverException(
+        message,
+        MessageImportDriverExceptionCode.NOT_FOUND,
+      );
+
+    case 429:
+      return new MessageImportDriverException(
+        message,
+        MessageImportDriverExceptionCode.TEMPORARY_ERROR,
+      );
+
+    case 403:
+      if (
+        reason === 'rateLimitExceeded' ||
+        reason === 'userRateLimitExceeded'
+      ) {
+        return new MessageImportDriverException(
+          message,
+          MessageImportDriverExceptionCode.TEMPORARY_ERROR,
+        );
+      } else {
+        return new MessageImportDriverException(
+          message,
+          MessageImportDriverExceptionCode.INSUFFICIENT_PERMISSIONS,
+        );
+      }
+
+    case 401:
+      return new MessageImportDriverException(
+        message,
+        MessageImportDriverExceptionCode.INSUFFICIENT_PERMISSIONS,
+      );
+
+    case 500:
+      if (reason === 'backendError') {
+        return new MessageImportDriverException(
+          message,
+          MessageImportDriverExceptionCode.TEMPORARY_ERROR,
+        );
+      } else {
+        return new MessageImportDriverException(
+          message,
+          MessageImportDriverExceptionCode.UNKNOWN,
+        );
+      }
+
+    default:
+      break;
+  }
+
+  return new MessageImportDriverException(
+    message,
+    MessageImportDriverExceptionCode.UNKNOWN,
+  );
+};
@@ -0,0 +1,47 @@
+import assert from 'assert';
+
+import addressparser from 'addressparser';
+import { gmail_v1 } from 'googleapis';
+
+import { getAttachmentData } from 'src/modules/messaging/message-import-manager/drivers/gmail/utils/get-attachment-data.util';
+import { getBodyData } from 'src/modules/messaging/message-import-manager/drivers/gmail/utils/get-body-data.util';
+import { getPropertyFromHeaders } from 'src/modules/messaging/message-import-manager/drivers/gmail/utils/get-property-from-headers.util';
+
+export const parseGmailMessage = (message: gmail_v1.Schema$Message) => {
+  const subject = getPropertyFromHeaders(message, 'Subject');
+  const rawFrom = getPropertyFromHeaders(message, 'From');
+  const rawTo = getPropertyFromHeaders(message, 'To');
+  const rawDeliveredTo = getPropertyFromHeaders(message, 'Delivered-To');
+  const rawCc = getPropertyFromHeaders(message, 'Cc');
+  const rawBcc = getPropertyFromHeaders(message, 'Bcc');
+  const messageId = getPropertyFromHeaders(message, 'Message-ID');
+  const id = message.id;
+  const threadId = message.threadId;
+  const historyId = message.historyId;
+  const internalDate = message.internalDate;
+
+  assert(id, 'ID is missing');
+  assert(historyId, 'History-ID is missing');
+  assert(internalDate, 'Internal date is missing');
+
+  const bodyData = getBodyData(message);
+  const text = bodyData ? Buffer.from(bodyData, 'base64').toString() : '';
+
+  const attachments = getAttachmentData(message);
+
+  return {
+    id,
+    headerMessageId: messageId,
+    threadId,
+    historyId,
+    internalDate,
+    subject,
+    from: rawFrom ? addressparser(rawFrom) : undefined,
+    deliveredTo: rawDeliveredTo ? addressparser(rawDeliveredTo) : undefined,
+    to: rawTo ? addressparser(rawTo) : undefined,
+    cc: rawCc ? addressparser(rawCc) : undefined,
+    bcc: rawBcc ? addressparser(rawBcc) : undefined,
+    text,
+    attachments,
+  };
+};
@@ -0,0 +1,3 @@
+export const sanitizeString = (str: string) => {
+  return str.replace(/\0/g, '');
+};