Respect Gmail retry-after in messaging throttle (#17850)

Gmail 429/403 rate-limit responses include an explicit retry-after
timestamp, usually ~15 minutes out.

The exponential backoff starts at 1 minute, so the channel burns through
all 5 retry attempts before the window actually closes and gets marked
as permanently failed.

Adds throttleRetryAfter to the message channel and uses max(backoff,
retryAfter) in isThrottled().
This commit is contained in:
neo773
2026-02-13 22:50:37 +05:30
committed by GitHub
parent 84afbb4d2c
commit d54b713264
21 changed files with 1235 additions and 866 deletions
@@ -0,0 +1,120 @@
import { InjectRepository } from '@nestjs/typeorm';
import { Command } from 'nest-commander';
import { STANDARD_OBJECTS } from 'twenty-shared/metadata';
import { FieldMetadataType } from 'twenty-shared/types';
import { Repository } from 'typeorm';
import { ActiveOrSuspendedWorkspacesMigrationCommandRunner } from 'src/database/commands/command-runners/active-or-suspended-workspaces-migration.command-runner';
import { RunOnWorkspaceArgs } from 'src/database/commands/command-runners/workspaces-migration.command-runner';
import { ApplicationService } from 'src/engine/core-modules/application/services/application.service';
import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity';
import { DataSourceService } from 'src/engine/metadata-modules/data-source/data-source.service';
import { FieldMetadataService } from 'src/engine/metadata-modules/field-metadata/services/field-metadata.service';
import { findFlatEntityByUniversalIdentifier } from 'src/engine/metadata-modules/flat-entity/utils/find-flat-entity-by-universal-identifier.util';
import { type FlatObjectMetadata } from 'src/engine/metadata-modules/flat-object-metadata/types/flat-object-metadata.type';
import { GlobalWorkspaceOrmManager } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager';
import { WorkspaceCacheService } from 'src/engine/workspace-cache/services/workspace-cache.service';
@Command({
name: 'upgrade:1-18:backfill-message-channel-throttle-retry-after',
description:
'Backfill throttleRetryAfter field on messageChannel standard object',
})
export class BackfillMessageChannelThrottleRetryAfterCommand extends ActiveOrSuspendedWorkspacesMigrationCommandRunner {
constructor(
@InjectRepository(WorkspaceEntity)
protected readonly workspaceRepository: Repository<WorkspaceEntity>,
protected readonly twentyORMGlobalManager: GlobalWorkspaceOrmManager,
protected readonly dataSourceService: DataSourceService,
private readonly applicationService: ApplicationService,
private readonly fieldMetadataService: FieldMetadataService,
private readonly workspaceCacheService: WorkspaceCacheService,
) {
super(workspaceRepository, twentyORMGlobalManager, dataSourceService);
}
override async runOnWorkspace({
workspaceId,
options,
}: RunOnWorkspaceArgs): Promise<void> {
this.logger.log(
`Backfilling throttleRetryAfter field for workspace ${workspaceId}`,
);
const { flatObjectMetadataMaps } =
await this.workspaceCacheService.getOrRecompute(workspaceId, [
'flatObjectMetadataMaps',
]);
const messageChannelObjectMetadata =
findFlatEntityByUniversalIdentifier<FlatObjectMetadata>({
flatEntityMaps: flatObjectMetadataMaps,
universalIdentifier:
STANDARD_OBJECTS.messageChannel.universalIdentifier,
});
if (!messageChannelObjectMetadata) {
this.logger.log(
`MessageChannel object metadata not found for workspace ${workspaceId}. Skipping.`,
);
return;
}
const { flatFieldMetadataMaps } =
await this.workspaceCacheService.getOrRecompute(workspaceId, [
'flatFieldMetadataMaps',
]);
const existingField = findFlatEntityByUniversalIdentifier({
flatEntityMaps: flatFieldMetadataMaps,
universalIdentifier:
STANDARD_OBJECTS.messageChannel.fields.throttleRetryAfter
.universalIdentifier,
});
if (existingField) {
this.logger.log(
`throttleRetryAfter field already exists for workspace ${workspaceId}. Skipping.`,
);
return;
}
if (options.dryRun) {
this.logger.log(
`Would have created throttleRetryAfter field for workspace ${workspaceId}. Skipping (dry run).`,
);
return;
}
const { twentyStandardFlatApplication } =
await this.applicationService.findWorkspaceTwentyStandardAndCustomApplicationOrThrow(
{ workspaceId },
);
await this.fieldMetadataService.createOneField({
createFieldInput: {
name: 'throttleRetryAfter',
type: FieldMetadataType.DATE_TIME,
label: 'Throttle Retry After',
description: 'Throttle Retry After',
icon: 'IconClock',
isNullable: true,
isUIReadOnly: true,
objectMetadataId: messageChannelObjectMetadata.id,
universalIdentifier:
STANDARD_OBJECTS.messageChannel.fields.throttleRetryAfter
.universalIdentifier,
},
workspaceId,
ownerFlatApplication: twentyStandardFlatApplication,
});
this.logger.log(
`Successfully created throttleRetryAfter field for workspace ${workspaceId}`,
);
}
}
@@ -2,6 +2,7 @@ import { Module } from '@nestjs/common';
import { TypeOrmModule } from '@nestjs/typeorm';
import { BackfillFileSizeAndMimeTypeCommand } from 'src/database/commands/upgrade-version-command/1-18/1-18-backfill-file-size-and-mime-type.command';
import { BackfillMessageChannelThrottleRetryAfterCommand } from 'src/database/commands/upgrade-version-command/1-18/1-18-backfill-message-channel-throttle-retry-after.command';
import { MigrateActivityRichTextAttachmentFileIdsCommand } from 'src/database/commands/upgrade-version-command/1-18/1-18-migrate-activity-rich-text-attachment-file-ids.command';
import { MigrateAttachmentFilesCommand } from 'src/database/commands/upgrade-version-command/1-18/1-18-migrate-attachment-files.command';
import { MigratePersonAvatarFilesCommand } from 'src/database/commands/upgrade-version-command/1-18/1-18-migrate-person-avatar-files.command';
@@ -40,12 +41,14 @@ import { PersonWorkspaceEntity } from 'src/modules/person/standard-objects/perso
MigrateAttachmentFilesCommand,
BackfillFileSizeAndMimeTypeCommand,
MigrateActivityRichTextAttachmentFileIdsCommand,
BackfillMessageChannelThrottleRetryAfterCommand,
],
exports: [
MigratePersonAvatarFilesCommand,
MigrateAttachmentFilesCommand,
BackfillFileSizeAndMimeTypeCommand,
MigrateActivityRichTextAttachmentFileIdsCommand,
BackfillMessageChannelThrottleRetryAfterCommand,
],
})
export class V1_18_UpgradeVersionCommandModule {}
@@ -20,6 +20,7 @@ import { MigrateNoteTargetToMorphRelationsCommand } from 'src/database/commands/
import { MigrateTaskTargetToMorphRelationsCommand } from 'src/database/commands/upgrade-version-command/1-17/1-17-migrate-task-target-to-morph-relations.command';
import { MigrateWorkflowCodeStepsCommand } from 'src/database/commands/upgrade-version-command/1-17/1-17-migrate-workflow-code-steps.command';
import { BackfillFileSizeAndMimeTypeCommand } from 'src/database/commands/upgrade-version-command/1-18/1-18-backfill-file-size-and-mime-type.command';
import { BackfillMessageChannelThrottleRetryAfterCommand } from 'src/database/commands/upgrade-version-command/1-18/1-18-backfill-message-channel-throttle-retry-after.command';
import { MigrateActivityRichTextAttachmentFileIdsCommand } from 'src/database/commands/upgrade-version-command/1-18/1-18-migrate-activity-rich-text-attachment-file-ids.command';
import { MigrateAttachmentFilesCommand } from 'src/database/commands/upgrade-version-command/1-18/1-18-migrate-attachment-files.command';
import { MigratePersonAvatarFilesCommand } from 'src/database/commands/upgrade-version-command/1-18/1-18-migrate-person-avatar-files.command';
@@ -59,6 +60,7 @@ export class UpgradeCommand extends UpgradeCommandRunner {
protected readonly backfillFileSizeAndMimeTypeCommand: BackfillFileSizeAndMimeTypeCommand,
protected readonly migrateAttachmentFilesCommand: MigrateAttachmentFilesCommand,
protected readonly migrateActivityRichTextAttachmentFileIdsCommand: MigrateActivityRichTextAttachmentFileIdsCommand,
protected readonly backfillMessageChannelThrottleRetryAfterCommand: BackfillMessageChannelThrottleRetryAfterCommand,
) {
super(
workspaceRepository,
@@ -89,6 +91,7 @@ export class UpgradeCommand extends UpgradeCommandRunner {
this.migrateAttachmentFilesCommand,
this.migrateActivityRichTextAttachmentFileIdsCommand,
this.backfillFileSizeAndMimeTypeCommand,
this.backfillMessageChannelThrottleRetryAfterCommand,
];
this.allCommands = {
@@ -517,6 +517,23 @@ export const buildMessageChannelStandardFlatFieldMetadatas = ({
twentyStandardApplicationId,
now,
}),
throttleRetryAfter: createStandardFieldFlatMetadata({
objectName,
workspaceId,
context: {
fieldName: 'throttleRetryAfter',
type: FieldMetadataType.DATE_TIME,
label: 'Throttle Retry After',
description: 'Throttle Retry After',
icon: 'IconClock',
isNullable: true,
isUIReadOnly: true,
},
standardObjectMetadataRelatedEntityIds,
dependencyFlatEntityMaps,
twentyStandardApplicationId,
now,
}),
connectedAccount: createStandardRelationFieldFlatMetadata({
objectName,
workspaceId,
@@ -0,0 +1,52 @@
import { isThrottled } from 'src/modules/connected-account/utils/is-throttled';
describe('isThrottled', () => {
it('should not throttle when no sync stage is active', () => {
expect(isThrottled(null, 3)).toBe(false);
});
it('should not throttle when there have been no failures', () => {
expect(isThrottled(new Date().toISOString(), 0)).toBe(false);
});
it('should keep throttling when retryAfter is in the future even though exponential backoff has expired', () => {
const tenMinutesAgo = new Date(Date.now() - 10 * 60 * 1000).toISOString();
const fiveMinutesFromNow = new Date(
Date.now() + 5 * 60 * 1000,
).toISOString();
expect(isThrottled(tenMinutesAgo, 1, fiveMinutesFromNow)).toBe(true);
});
it('should fall back to exponential backoff when retryAfter is not provided', () => {
const justNow = new Date().toISOString();
expect(isThrottled(justNow, 1)).toBe(true);
});
it('should fall back to exponential backoff when retryAfter is in the past', () => {
const justNow = new Date().toISOString();
const fiveMinutesAgo = new Date(Date.now() - 5 * 60 * 1000).toISOString();
expect(isThrottled(justNow, 1, fiveMinutesAgo)).toBe(true);
});
it('should fall back to exponential backoff when retryAfter is explicitly null', () => {
const justNow = new Date().toISOString();
expect(isThrottled(justNow, 1, null)).toBe(true);
});
it('should fall back to exponential backoff when retryAfter is an invalid date string', () => {
const justNow = new Date().toISOString();
expect(isThrottled(justNow, 1, 'not-a-date')).toBe(true);
});
it('should use exponential backoff when both backoff is active and retryAfter is in the past', () => {
const justNow = new Date().toISOString();
const oneMinuteAgo = new Date(Date.now() - 60 * 1000).toISOString();
expect(isThrottled(justNow, 1, oneMinuteAgo)).toBe(true);
});
});
@@ -1,8 +1,12 @@
import { isDefined } from 'twenty-shared/utils';
import { MESSAGING_THROTTLE_DURATION } from 'src/modules/messaging/message-import-manager/constants/messaging-throttle-duration';
import { isValidDate } from 'src/utils/date/isValidDate';
export const isThrottled = (
syncStageStartedAt: string | null,
throttleFailureCount: number,
throttleRetryAfter?: string | null,
): boolean => {
if (!syncStageStartedAt) {
return false;
@@ -12,10 +16,25 @@ export const isThrottled = (
return false;
}
return (
computeThrottlePauseUntil(syncStageStartedAt, throttleFailureCount) >
new Date()
const now = new Date();
const exponentialBackoffUntil = computeThrottlePauseUntil(
syncStageStartedAt,
throttleFailureCount,
);
const retryAfterCandidate = isDefined(throttleRetryAfter)
? new Date(throttleRetryAfter)
: null;
const retryAfterDate = isValidDate(retryAfterCandidate)
? retryAfterCandidate
: null;
const effectiveUntil =
isDefined(retryAfterDate) && retryAfterDate > exponentialBackoffUntil
? retryAfterDate
: exponentialBackoffUntil;
return effectiveUntil > now;
};
const computeThrottlePauseUntil = (
@@ -116,6 +116,7 @@ export class MessageChannelSyncStatusService {
syncCursor: '',
syncStageStartedAt: null,
throttleFailureCount: 0,
throttleRetryAfter: null,
pendingGroupEmailsAction: MessageChannelPendingGroupEmailsAction.NONE,
});
@@ -225,6 +226,7 @@ export class MessageChannelSyncStatusService {
syncStatus: MessageChannelSyncStatus.ACTIVE,
syncStage: MessageChannelSyncStage.MESSAGE_LIST_FETCH_PENDING,
throttleFailureCount: 0,
throttleRetryAfter: null,
syncStageStartedAt: null,
syncedAt: new Date().toISOString(),
});
@@ -307,6 +309,7 @@ export class MessageChannelSyncStatusService {
await messageChannelRepository.update(messageChannelIds, {
syncStage: MessageChannelSyncStage.FAILED,
syncStatus: syncStatus,
throttleRetryAfter: null,
});
const metricsKey =
@@ -98,6 +98,7 @@ export class MessageChannelWorkspaceEntity extends BaseWorkspaceEntity {
syncStage: MessageChannelSyncStage;
syncStageStartedAt: string | null;
throttleFailureCount: number;
throttleRetryAfter: string | null;
connectedAccount: EntityRelation<ConnectedAccountWorkspaceEntity>;
connectedAccountId: string;
messageChannelMessageAssociations: EntityRelation<
@@ -3,6 +3,7 @@ import { type MessageNetworkExceptionCode } from 'src/modules/messaging/message-
export class MessageImportDriverException extends Error {
code: MessageImportDriverExceptionCode | MessageNetworkExceptionCode;
cause?: Error;
throttleRetryAfter?: Date;
context?: {
messageChannelId?: string;
workspaceId?: string;
@@ -14,6 +15,7 @@ export class MessageImportDriverException extends Error {
code: MessageImportDriverExceptionCode | MessageNetworkExceptionCode,
options?: {
cause?: Error;
throttleRetryAfter?: Date;
context?: {
messageChannelId?: string;
workspaceId?: string;
@@ -25,6 +27,7 @@ export class MessageImportDriverException extends Error {
this.name = 'MessageImportDriverException';
this.code = code;
this.cause = options?.cause;
this.throttleRetryAfter = options?.throttleRetryAfter;
this.context = options?.context;
if (options?.cause?.stack) {
@@ -50,9 +50,11 @@ const ERROR_DEFINITIONS: Record<number, Record<string, ErrorConfig>> = {
export const getGmailApiError = ({
code,
reason,
message,
}: {
code: number;
reason?: string;
message?: string;
}): GaxiosError => {
const statusMap = ERROR_DEFINITIONS[code];
@@ -62,8 +64,10 @@ export const getGmailApiError = ({
const config = statusMap[reason || ''] ?? statusMap.default;
const errorMessage = message ?? config.message;
return new GaxiosError(
config.message,
errorMessage,
{ url: 'https://gmail.googleapis.com/mocks' },
{
status: code,
@@ -71,10 +75,10 @@ export const getGmailApiError = ({
data: {
error: {
code,
message: config.message,
message: errorMessage,
errors: [
{
message: config.message,
message: errorMessage,
reason: config.reason,
},
],
@@ -0,0 +1,40 @@
import { parseGmailErrorRetryAfter } from 'src/modules/messaging/message-import-manager/drivers/gmail/utils/parse-gmail-error-retry-after.util';
describe('parseGmailErrorRetryAfter', () => {
it('should extract the retry-after date from a Gmail 429 error message', () => {
const fifteenMinutesFromNow = new Date(Date.now() + 15 * 60 * 1000);
const message = `User-rate limit exceeded. Retry after ${fifteenMinutesFromNow.toISOString()}`;
const result = parseGmailErrorRetryAfter(message);
expect(result).toBeInstanceOf(Date);
expect(result!.getTime()).toBeCloseTo(fifteenMinutesFromNow.getTime(), -3);
});
it('should return undefined when the message contains no retry-after timestamp', () => {
expect(
parseGmailErrorRetryAfter('Too Many Concurrent Requests'),
).toBeUndefined();
});
it('should return undefined when the retry-after timestamp has already passed', () => {
const fiveMinutesAgo = new Date(Date.now() - 5 * 60 * 1000);
const message = `User-rate limit exceeded. Retry after ${fiveMinutesAgo.toISOString()}`;
expect(parseGmailErrorRetryAfter(message)).toBeUndefined();
});
it('should return undefined for an empty string', () => {
expect(parseGmailErrorRetryAfter('')).toBeUndefined();
});
it('should match case variations of "Retry after"', () => {
const fifteenMinutesFromNow = new Date(Date.now() + 15 * 60 * 1000);
const message = `User-rate limit exceeded. retry after ${fifteenMinutesFromNow.toISOString()}`;
const result = parseGmailErrorRetryAfter(message);
expect(result).toBeInstanceOf(Date);
expect(result!.getTime()).toBeCloseTo(fifteenMinutesFromNow.getTime(), -3);
});
});
@@ -232,4 +232,48 @@ describe('parseGmailApiError', () => {
MessageImportDriverExceptionCode.SYNC_CURSOR_ERROR,
);
});
it('should populate retryAfter on the exception when a 429 contains a retry-after timestamp', () => {
const fifteenMinutesFromNow = new Date(Date.now() + 15 * 60 * 1000);
const error = getGmailApiError({
code: 429,
message: `User-rate limit exceeded. Retry after ${fifteenMinutesFromNow.toISOString()}`,
});
const exception = parseGmailApiError(error);
expect(exception.throttleRetryAfter).toBeInstanceOf(Date);
expect(exception.throttleRetryAfter!.getTime()).toBeCloseTo(
fifteenMinutesFromNow.getTime(),
-3,
);
});
it('should leave retryAfter undefined on 429 without a retry-after timestamp', () => {
const error = getGmailApiError({ code: 429 });
const exception = parseGmailApiError(error);
expect(exception.throttleRetryAfter).toBeUndefined();
});
it('should populate retryAfter on the exception when a 403 rateLimitExceeded contains a retry-after timestamp', () => {
const fifteenMinutesFromNow = new Date(Date.now() + 15 * 60 * 1000);
const error = getGmailApiError({
code: 403,
reason: 'rateLimit',
message: `Rate Limit Exceeded. Retry after ${fifteenMinutesFromNow.toISOString()}`,
});
const exception = parseGmailApiError(error);
expect(exception.code).toBe(
MessageImportDriverExceptionCode.TEMPORARY_ERROR,
);
expect(exception.throttleRetryAfter).toBeInstanceOf(Date);
expect(exception.throttleRetryAfter!.getTime()).toBeCloseTo(
fifteenMinutesFromNow.getTime(),
-3,
);
});
});
@@ -4,6 +4,7 @@ import {
MessageImportDriverException,
MessageImportDriverExceptionCode,
} from 'src/modules/messaging/message-import-manager/drivers/exceptions/message-import-driver.exception';
import { parseGmailErrorRetryAfter } from 'src/modules/messaging/message-import-manager/drivers/gmail/utils/parse-gmail-error-retry-after.util';
export const parseGmailApiError = (
error: GaxiosError,
@@ -57,6 +58,9 @@ export const parseGmailApiError = (
return new MessageImportDriverException(
gmailApiError.message,
MessageImportDriverExceptionCode.TEMPORARY_ERROR,
{
throttleRetryAfter: parseGmailErrorRetryAfter(gmailApiError.message),
},
);
case 403:
@@ -68,6 +72,11 @@ export const parseGmailApiError = (
return new MessageImportDriverException(
gmailApiError.message,
MessageImportDriverExceptionCode.TEMPORARY_ERROR,
{
throttleRetryAfter: parseGmailErrorRetryAfter(
gmailApiError.message,
),
},
);
}
if (gmailApiError.reason === 'domainPolicy') {
@@ -0,0 +1,24 @@
const RETRY_AFTER_REGEX =
/Retry after (\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?Z)/i;
export const parseGmailErrorRetryAfter = (
message: string,
): Date | undefined => {
const match = message.match(RETRY_AFTER_REGEX);
if (!match) {
return undefined;
}
const retryAfter = new Date(match[1]);
if (isNaN(retryAfter.getTime())) {
return undefined;
}
if (retryAfter <= new Date()) {
return undefined;
}
return retryAfter;
};
@@ -84,6 +84,7 @@ export class MessagingMessageListFetchJob {
isThrottled(
messageChannel.syncStageStartedAt,
messageChannel.throttleFailureCount,
messageChannel.throttleRetryAfter,
)
) {
await this.messageChannelSyncStatusService.markAsMessagesListFetchPending(
@@ -82,6 +82,7 @@ export class MessagingMessagesImportJob {
isThrottled(
messageChannel.syncStageStartedAt,
messageChannel.throttleFailureCount,
messageChannel.throttleRetryAfter,
)
) {
await this.messageChannelSyncStatusService.markAsMessagesImportPending(
@@ -38,6 +38,7 @@ export class MessagingCursorService {
},
{
throttleFailureCount: 0,
throttleRetryAfter: null,
syncStageStartedAt: null,
syncCursor:
!messageChannel.syncCursor ||
@@ -61,6 +62,7 @@ export class MessagingCursorService {
},
{
throttleFailureCount: 0,
throttleRetryAfter: null,
syncStageStartedAt: null,
},
);
@@ -1,5 +1,7 @@
import { Injectable } from '@nestjs/common';
import { isDefined } from 'twenty-shared/utils';
import { ExceptionHandlerService } from 'src/engine/core-modules/exception-handler/exception-handler.service';
import {
type TwentyORMException,
@@ -166,6 +168,20 @@ export class MessageImportExceptionHandlerService {
undefined,
['throttleFailureCount', 'id'],
);
const throttleRetryAfter =
exception instanceof MessageImportDriverException
? exception.throttleRetryAfter
: undefined;
await messageChannelRepository.update(
{ id: messageChannel.id },
{
throttleRetryAfter: isDefined(throttleRetryAfter)
? throttleRetryAfter.toISOString()
: null,
},
);
}, authContext);
switch (syncStep) {
@@ -189,6 +189,7 @@ export class MessagingMessagesImportService {
},
{
throttleFailureCount: 0,
throttleRetryAfter: null,
syncStageStartedAt: null,
},
);
@@ -821,6 +821,9 @@ export const STANDARD_OBJECTS = {
throttleFailureCount: {
universalIdentifier: '20202020-0291-42be-9ad0-d578a51684ab',
},
throttleRetryAfter: {
universalIdentifier: '20202020-a1e3-4d7f-b5c2-9f8e6d4c3b2a',
},
},
indexes: {
connectedAccountIdIndex: {