From fda3eabb5f3e54a84e9a18279b49aa51c2254fd8 Mon Sep 17 00:00:00 2001 From: neo773 <62795688+neo773@users.noreply.github.com> Date: Wed, 8 Apr 2026 20:17:04 +0530 Subject: [PATCH] introduce MESSAGING_MESSAGES_GET_BATCH_SIZE as config variable (#19455) makes`MESSAGING_MESSAGES_GET_BATCH_SIZE` configurable for self hosters /closes #19147 --- .../core-modules/twenty-config/config-variables.ts | 10 ++++++++++ ...gmail-users-messages-get-batch-size.constant.ts | 1 - .../messaging-messages-import.service.spec.ts | 10 ++++++++-- .../services/messaging-messages-import.service.ts | 14 ++++++++------ 4 files changed, 26 insertions(+), 9 deletions(-) delete mode 100644 packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/constants/messaging-gmail-users-messages-get-batch-size.constant.ts diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/config-variables.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/config-variables.ts index 346517abbb..cc878ca994 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/config-variables.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/config-variables.ts @@ -245,6 +245,16 @@ export class ConfigVariables { }) MESSAGING_PROVIDER_MICROSOFT_ENABLED = false; + @ConfigVariablesMetadata({ + group: ConfigVariablesGroup.ADVANCED_SETTINGS, + description: + 'Number of messages fetched per batch during message import, adjust incase of rate limiting caused by Gmail, Outlook or IMAP', + type: ConfigVariableType.NUMBER, + }) + @CastToPositiveNumber() + @IsOptional() + MESSAGING_MESSAGES_GET_BATCH_SIZE = 400; + @ConfigVariablesMetadata({ group: ConfigVariablesGroup.MICROSOFT_AUTH, description: 'Enable or disable the Microsoft Calendar integration', diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/constants/messaging-gmail-users-messages-get-batch-size.constant.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/constants/messaging-gmail-users-messages-get-batch-size.constant.ts deleted file mode 100644 index a8a4f4abfc..0000000000 --- a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/constants/messaging-gmail-users-messages-get-batch-size.constant.ts +++ /dev/null @@ -1 +0,0 @@ -export const MESSAGING_GMAIL_USERS_MESSAGES_GET_BATCH_SIZE = 400; diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/services/__tests__/messaging-messages-import.service.spec.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/services/__tests__/messaging-messages-import.service.spec.ts index 02ab4e1889..3d9fca638e 100644 --- a/packages/twenty-server/src/modules/messaging/message-import-manager/services/__tests__/messaging-messages-import.service.spec.ts +++ b/packages/twenty-server/src/modules/messaging/message-import-manager/services/__tests__/messaging-messages-import.service.spec.ts @@ -15,7 +15,7 @@ import { EmailAliasManagerService } from 'src/modules/connected-account/email-al import { ConnectedAccountRefreshTokensService } from 'src/modules/connected-account/refresh-tokens-manager/services/connected-account-refresh-tokens.service'; import { type ConnectedAccountEntity } from 'src/engine/metadata-modules/connected-account/entities/connected-account.entity'; import { MessageChannelSyncStatusService } from 'src/modules/messaging/common/services/message-channel-sync-status.service'; -import { MESSAGING_GMAIL_USERS_MESSAGES_GET_BATCH_SIZE } from 'src/modules/messaging/message-import-manager/drivers/gmail/constants/messaging-gmail-users-messages-get-batch-size.constant'; +import { TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service'; import { MessagingAccountAuthenticationService } from 'src/modules/messaging/message-import-manager/services/messaging-account-authentication.service'; import { MessagingGetMessagesService } from 'src/modules/messaging/message-import-manager/services/messaging-get-messages.service'; import { MessageImportExceptionHandlerService } from 'src/modules/messaging/message-import-manager/services/messaging-import-exception-handler.service'; @@ -183,6 +183,12 @@ describe('MessagingMessagesImportService', () => { findOne: jest.fn().mockResolvedValue({ userId: 'user-id' }), }, }, + { + provide: TwentyConfigService, + useValue: { + get: jest.fn().mockReturnValue(400), + }, + }, ]; const module: TestingModule = await Test.createTestingModule({ providers: [ @@ -281,7 +287,7 @@ describe('MessagingMessagesImportService', () => { it('should process message batch import of more than MESSAGING_GMAIL_USERS_MESSAGES_GET_BATCH_SIZE successfully', async () => { const arrayMessagesBig = Array.from( - { length: MESSAGING_GMAIL_USERS_MESSAGES_GET_BATCH_SIZE + 1 }, + { length: 401 }, (_, index) => `message-id-${index + 1}`, ); diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-messages-import.service.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-messages-import.service.ts index 281ecb33a6..acd4f8d21c 100644 --- a/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-messages-import.service.ts +++ b/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-messages-import.service.ts @@ -22,7 +22,7 @@ import { MessageImportDriverException, MessageImportDriverExceptionCode, } from 'src/modules/messaging/message-import-manager/drivers/exceptions/message-import-driver.exception'; -import { MESSAGING_GMAIL_USERS_MESSAGES_GET_BATCH_SIZE } from 'src/modules/messaging/message-import-manager/drivers/gmail/constants/messaging-gmail-users-messages-get-batch-size.constant'; +import { TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service'; import { MessagingAccountAuthenticationService } from 'src/modules/messaging/message-import-manager/services/messaging-account-authentication.service'; import { MessagingGetMessagesService } from 'src/modules/messaging/message-import-manager/services/messaging-get-messages.service'; import { @@ -55,6 +55,7 @@ export class MessagingMessagesImportService { private readonly messagingAccountAuthenticationService: MessagingAccountAuthenticationService, @InjectRepository(UserWorkspaceEntity) private readonly userWorkspaceRepository: Repository, + private readonly twentyConfigService: TwentyConfigService, ) {} async processMessageBatchImport( @@ -64,6 +65,10 @@ export class MessagingMessagesImportService { ) { let messageIdsToFetch: string[] = []; + const messagesGetBatchSize = this.twentyConfigService.get( + 'MESSAGING_MESSAGES_GET_BATCH_SIZE', + ); + const authContext = buildSystemAuthContext(workspaceId); await this.globalWorkspaceOrmManager.executeInWorkspaceContext(async () => { @@ -112,7 +117,7 @@ export class MessagingMessagesImportService { messageIdsToFetch = await this.cacheStorage.setPop( `messages-to-import:${workspaceId}:${messageChannel.id}`, - MESSAGING_GMAIL_USERS_MESSAGES_GET_BATCH_SIZE, + messagesGetBatchSize, ); if (!messageIdsToFetch?.length) { @@ -210,10 +215,7 @@ export class MessagingMessagesImportService { ); } - if ( - messageIdsToFetch.length < - MESSAGING_GMAIL_USERS_MESSAGES_GET_BATCH_SIZE - ) { + if (messageIdsToFetch.length < messagesGetBatchSize) { await this.messageChannelSyncStatusService.markAsCompletedAndMarkAsMessagesListFetchPending( [messageChannel.id], workspaceId,