Respect Gmail retry-after in messaging throttle (#17850)
Gmail 429/403 rate-limit responses include an explicit retry-after timestamp, usually ~15 minutes out. The exponential backoff starts at 1 minute, so the channel burns through all 5 retry attempts before the window actually closes and gets marked as permanently failed. Adds throttleRetryAfter to the message channel and uses max(backoff, retryAfter) in isThrottled().
This commit is contained in:
+120
@@ -0,0 +1,120 @@
|
||||
import { InjectRepository } from '@nestjs/typeorm';
|
||||
|
||||
import { Command } from 'nest-commander';
|
||||
import { STANDARD_OBJECTS } from 'twenty-shared/metadata';
|
||||
import { FieldMetadataType } from 'twenty-shared/types';
|
||||
import { Repository } from 'typeorm';
|
||||
|
||||
import { ActiveOrSuspendedWorkspacesMigrationCommandRunner } from 'src/database/commands/command-runners/active-or-suspended-workspaces-migration.command-runner';
|
||||
import { RunOnWorkspaceArgs } from 'src/database/commands/command-runners/workspaces-migration.command-runner';
|
||||
import { ApplicationService } from 'src/engine/core-modules/application/services/application.service';
|
||||
import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity';
|
||||
import { DataSourceService } from 'src/engine/metadata-modules/data-source/data-source.service';
|
||||
import { FieldMetadataService } from 'src/engine/metadata-modules/field-metadata/services/field-metadata.service';
|
||||
import { findFlatEntityByUniversalIdentifier } from 'src/engine/metadata-modules/flat-entity/utils/find-flat-entity-by-universal-identifier.util';
|
||||
import { type FlatObjectMetadata } from 'src/engine/metadata-modules/flat-object-metadata/types/flat-object-metadata.type';
|
||||
import { GlobalWorkspaceOrmManager } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager';
|
||||
import { WorkspaceCacheService } from 'src/engine/workspace-cache/services/workspace-cache.service';
|
||||
|
||||
@Command({
|
||||
name: 'upgrade:1-18:backfill-message-channel-throttle-retry-after',
|
||||
description:
|
||||
'Backfill throttleRetryAfter field on messageChannel standard object',
|
||||
})
|
||||
export class BackfillMessageChannelThrottleRetryAfterCommand extends ActiveOrSuspendedWorkspacesMigrationCommandRunner {
|
||||
constructor(
|
||||
@InjectRepository(WorkspaceEntity)
|
||||
protected readonly workspaceRepository: Repository<WorkspaceEntity>,
|
||||
protected readonly twentyORMGlobalManager: GlobalWorkspaceOrmManager,
|
||||
protected readonly dataSourceService: DataSourceService,
|
||||
private readonly applicationService: ApplicationService,
|
||||
private readonly fieldMetadataService: FieldMetadataService,
|
||||
private readonly workspaceCacheService: WorkspaceCacheService,
|
||||
) {
|
||||
super(workspaceRepository, twentyORMGlobalManager, dataSourceService);
|
||||
}
|
||||
|
||||
override async runOnWorkspace({
|
||||
workspaceId,
|
||||
options,
|
||||
}: RunOnWorkspaceArgs): Promise<void> {
|
||||
this.logger.log(
|
||||
`Backfilling throttleRetryAfter field for workspace ${workspaceId}`,
|
||||
);
|
||||
|
||||
const { flatObjectMetadataMaps } =
|
||||
await this.workspaceCacheService.getOrRecompute(workspaceId, [
|
||||
'flatObjectMetadataMaps',
|
||||
]);
|
||||
|
||||
const messageChannelObjectMetadata =
|
||||
findFlatEntityByUniversalIdentifier<FlatObjectMetadata>({
|
||||
flatEntityMaps: flatObjectMetadataMaps,
|
||||
universalIdentifier:
|
||||
STANDARD_OBJECTS.messageChannel.universalIdentifier,
|
||||
});
|
||||
|
||||
if (!messageChannelObjectMetadata) {
|
||||
this.logger.log(
|
||||
`MessageChannel object metadata not found for workspace ${workspaceId}. Skipping.`,
|
||||
);
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
const { flatFieldMetadataMaps } =
|
||||
await this.workspaceCacheService.getOrRecompute(workspaceId, [
|
||||
'flatFieldMetadataMaps',
|
||||
]);
|
||||
|
||||
const existingField = findFlatEntityByUniversalIdentifier({
|
||||
flatEntityMaps: flatFieldMetadataMaps,
|
||||
universalIdentifier:
|
||||
STANDARD_OBJECTS.messageChannel.fields.throttleRetryAfter
|
||||
.universalIdentifier,
|
||||
});
|
||||
|
||||
if (existingField) {
|
||||
this.logger.log(
|
||||
`throttleRetryAfter field already exists for workspace ${workspaceId}. Skipping.`,
|
||||
);
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
if (options.dryRun) {
|
||||
this.logger.log(
|
||||
`Would have created throttleRetryAfter field for workspace ${workspaceId}. Skipping (dry run).`,
|
||||
);
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
const { twentyStandardFlatApplication } =
|
||||
await this.applicationService.findWorkspaceTwentyStandardAndCustomApplicationOrThrow(
|
||||
{ workspaceId },
|
||||
);
|
||||
|
||||
await this.fieldMetadataService.createOneField({
|
||||
createFieldInput: {
|
||||
name: 'throttleRetryAfter',
|
||||
type: FieldMetadataType.DATE_TIME,
|
||||
label: 'Throttle Retry After',
|
||||
description: 'Throttle Retry After',
|
||||
icon: 'IconClock',
|
||||
isNullable: true,
|
||||
isUIReadOnly: true,
|
||||
objectMetadataId: messageChannelObjectMetadata.id,
|
||||
universalIdentifier:
|
||||
STANDARD_OBJECTS.messageChannel.fields.throttleRetryAfter
|
||||
.universalIdentifier,
|
||||
},
|
||||
workspaceId,
|
||||
ownerFlatApplication: twentyStandardFlatApplication,
|
||||
});
|
||||
|
||||
this.logger.log(
|
||||
`Successfully created throttleRetryAfter field for workspace ${workspaceId}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
+3
@@ -2,6 +2,7 @@ import { Module } from '@nestjs/common';
|
||||
import { TypeOrmModule } from '@nestjs/typeorm';
|
||||
|
||||
import { BackfillFileSizeAndMimeTypeCommand } from 'src/database/commands/upgrade-version-command/1-18/1-18-backfill-file-size-and-mime-type.command';
|
||||
import { BackfillMessageChannelThrottleRetryAfterCommand } from 'src/database/commands/upgrade-version-command/1-18/1-18-backfill-message-channel-throttle-retry-after.command';
|
||||
import { MigrateActivityRichTextAttachmentFileIdsCommand } from 'src/database/commands/upgrade-version-command/1-18/1-18-migrate-activity-rich-text-attachment-file-ids.command';
|
||||
import { MigrateAttachmentFilesCommand } from 'src/database/commands/upgrade-version-command/1-18/1-18-migrate-attachment-files.command';
|
||||
import { MigratePersonAvatarFilesCommand } from 'src/database/commands/upgrade-version-command/1-18/1-18-migrate-person-avatar-files.command';
|
||||
@@ -40,12 +41,14 @@ import { PersonWorkspaceEntity } from 'src/modules/person/standard-objects/perso
|
||||
MigrateAttachmentFilesCommand,
|
||||
BackfillFileSizeAndMimeTypeCommand,
|
||||
MigrateActivityRichTextAttachmentFileIdsCommand,
|
||||
BackfillMessageChannelThrottleRetryAfterCommand,
|
||||
],
|
||||
exports: [
|
||||
MigratePersonAvatarFilesCommand,
|
||||
MigrateAttachmentFilesCommand,
|
||||
BackfillFileSizeAndMimeTypeCommand,
|
||||
MigrateActivityRichTextAttachmentFileIdsCommand,
|
||||
BackfillMessageChannelThrottleRetryAfterCommand,
|
||||
],
|
||||
})
|
||||
export class V1_18_UpgradeVersionCommandModule {}
|
||||
|
||||
+3
@@ -20,6 +20,7 @@ import { MigrateNoteTargetToMorphRelationsCommand } from 'src/database/commands/
|
||||
import { MigrateTaskTargetToMorphRelationsCommand } from 'src/database/commands/upgrade-version-command/1-17/1-17-migrate-task-target-to-morph-relations.command';
|
||||
import { MigrateWorkflowCodeStepsCommand } from 'src/database/commands/upgrade-version-command/1-17/1-17-migrate-workflow-code-steps.command';
|
||||
import { BackfillFileSizeAndMimeTypeCommand } from 'src/database/commands/upgrade-version-command/1-18/1-18-backfill-file-size-and-mime-type.command';
|
||||
import { BackfillMessageChannelThrottleRetryAfterCommand } from 'src/database/commands/upgrade-version-command/1-18/1-18-backfill-message-channel-throttle-retry-after.command';
|
||||
import { MigrateActivityRichTextAttachmentFileIdsCommand } from 'src/database/commands/upgrade-version-command/1-18/1-18-migrate-activity-rich-text-attachment-file-ids.command';
|
||||
import { MigrateAttachmentFilesCommand } from 'src/database/commands/upgrade-version-command/1-18/1-18-migrate-attachment-files.command';
|
||||
import { MigratePersonAvatarFilesCommand } from 'src/database/commands/upgrade-version-command/1-18/1-18-migrate-person-avatar-files.command';
|
||||
@@ -59,6 +60,7 @@ export class UpgradeCommand extends UpgradeCommandRunner {
|
||||
protected readonly backfillFileSizeAndMimeTypeCommand: BackfillFileSizeAndMimeTypeCommand,
|
||||
protected readonly migrateAttachmentFilesCommand: MigrateAttachmentFilesCommand,
|
||||
protected readonly migrateActivityRichTextAttachmentFileIdsCommand: MigrateActivityRichTextAttachmentFileIdsCommand,
|
||||
protected readonly backfillMessageChannelThrottleRetryAfterCommand: BackfillMessageChannelThrottleRetryAfterCommand,
|
||||
) {
|
||||
super(
|
||||
workspaceRepository,
|
||||
@@ -89,6 +91,7 @@ export class UpgradeCommand extends UpgradeCommandRunner {
|
||||
this.migrateAttachmentFilesCommand,
|
||||
this.migrateActivityRichTextAttachmentFileIdsCommand,
|
||||
this.backfillFileSizeAndMimeTypeCommand,
|
||||
this.backfillMessageChannelThrottleRetryAfterCommand,
|
||||
];
|
||||
|
||||
this.allCommands = {
|
||||
|
||||
+863
-860
File diff suppressed because it is too large
Load Diff
+17
@@ -517,6 +517,23 @@ export const buildMessageChannelStandardFlatFieldMetadatas = ({
|
||||
twentyStandardApplicationId,
|
||||
now,
|
||||
}),
|
||||
throttleRetryAfter: createStandardFieldFlatMetadata({
|
||||
objectName,
|
||||
workspaceId,
|
||||
context: {
|
||||
fieldName: 'throttleRetryAfter',
|
||||
type: FieldMetadataType.DATE_TIME,
|
||||
label: 'Throttle Retry After',
|
||||
description: 'Throttle Retry After',
|
||||
icon: 'IconClock',
|
||||
isNullable: true,
|
||||
isUIReadOnly: true,
|
||||
},
|
||||
standardObjectMetadataRelatedEntityIds,
|
||||
dependencyFlatEntityMaps,
|
||||
twentyStandardApplicationId,
|
||||
now,
|
||||
}),
|
||||
connectedAccount: createStandardRelationFieldFlatMetadata({
|
||||
objectName,
|
||||
workspaceId,
|
||||
|
||||
+52
@@ -0,0 +1,52 @@
|
||||
import { isThrottled } from 'src/modules/connected-account/utils/is-throttled';
|
||||
|
||||
describe('isThrottled', () => {
|
||||
it('should not throttle when no sync stage is active', () => {
|
||||
expect(isThrottled(null, 3)).toBe(false);
|
||||
});
|
||||
|
||||
it('should not throttle when there have been no failures', () => {
|
||||
expect(isThrottled(new Date().toISOString(), 0)).toBe(false);
|
||||
});
|
||||
|
||||
it('should keep throttling when retryAfter is in the future even though exponential backoff has expired', () => {
|
||||
const tenMinutesAgo = new Date(Date.now() - 10 * 60 * 1000).toISOString();
|
||||
const fiveMinutesFromNow = new Date(
|
||||
Date.now() + 5 * 60 * 1000,
|
||||
).toISOString();
|
||||
|
||||
expect(isThrottled(tenMinutesAgo, 1, fiveMinutesFromNow)).toBe(true);
|
||||
});
|
||||
|
||||
it('should fall back to exponential backoff when retryAfter is not provided', () => {
|
||||
const justNow = new Date().toISOString();
|
||||
|
||||
expect(isThrottled(justNow, 1)).toBe(true);
|
||||
});
|
||||
|
||||
it('should fall back to exponential backoff when retryAfter is in the past', () => {
|
||||
const justNow = new Date().toISOString();
|
||||
const fiveMinutesAgo = new Date(Date.now() - 5 * 60 * 1000).toISOString();
|
||||
|
||||
expect(isThrottled(justNow, 1, fiveMinutesAgo)).toBe(true);
|
||||
});
|
||||
|
||||
it('should fall back to exponential backoff when retryAfter is explicitly null', () => {
|
||||
const justNow = new Date().toISOString();
|
||||
|
||||
expect(isThrottled(justNow, 1, null)).toBe(true);
|
||||
});
|
||||
|
||||
it('should fall back to exponential backoff when retryAfter is an invalid date string', () => {
|
||||
const justNow = new Date().toISOString();
|
||||
|
||||
expect(isThrottled(justNow, 1, 'not-a-date')).toBe(true);
|
||||
});
|
||||
|
||||
it('should use exponential backoff when both backoff is active and retryAfter is in the past', () => {
|
||||
const justNow = new Date().toISOString();
|
||||
const oneMinuteAgo = new Date(Date.now() - 60 * 1000).toISOString();
|
||||
|
||||
expect(isThrottled(justNow, 1, oneMinuteAgo)).toBe(true);
|
||||
});
|
||||
});
|
||||
@@ -1,8 +1,12 @@
|
||||
import { isDefined } from 'twenty-shared/utils';
|
||||
|
||||
import { MESSAGING_THROTTLE_DURATION } from 'src/modules/messaging/message-import-manager/constants/messaging-throttle-duration';
|
||||
import { isValidDate } from 'src/utils/date/isValidDate';
|
||||
|
||||
export const isThrottled = (
|
||||
syncStageStartedAt: string | null,
|
||||
throttleFailureCount: number,
|
||||
throttleRetryAfter?: string | null,
|
||||
): boolean => {
|
||||
if (!syncStageStartedAt) {
|
||||
return false;
|
||||
@@ -12,10 +16,25 @@ export const isThrottled = (
|
||||
return false;
|
||||
}
|
||||
|
||||
return (
|
||||
computeThrottlePauseUntil(syncStageStartedAt, throttleFailureCount) >
|
||||
new Date()
|
||||
const now = new Date();
|
||||
|
||||
const exponentialBackoffUntil = computeThrottlePauseUntil(
|
||||
syncStageStartedAt,
|
||||
throttleFailureCount,
|
||||
);
|
||||
const retryAfterCandidate = isDefined(throttleRetryAfter)
|
||||
? new Date(throttleRetryAfter)
|
||||
: null;
|
||||
const retryAfterDate = isValidDate(retryAfterCandidate)
|
||||
? retryAfterCandidate
|
||||
: null;
|
||||
|
||||
const effectiveUntil =
|
||||
isDefined(retryAfterDate) && retryAfterDate > exponentialBackoffUntil
|
||||
? retryAfterDate
|
||||
: exponentialBackoffUntil;
|
||||
|
||||
return effectiveUntil > now;
|
||||
};
|
||||
|
||||
const computeThrottlePauseUntil = (
|
||||
|
||||
+3
@@ -116,6 +116,7 @@ export class MessageChannelSyncStatusService {
|
||||
syncCursor: '',
|
||||
syncStageStartedAt: null,
|
||||
throttleFailureCount: 0,
|
||||
throttleRetryAfter: null,
|
||||
pendingGroupEmailsAction: MessageChannelPendingGroupEmailsAction.NONE,
|
||||
});
|
||||
|
||||
@@ -225,6 +226,7 @@ export class MessageChannelSyncStatusService {
|
||||
syncStatus: MessageChannelSyncStatus.ACTIVE,
|
||||
syncStage: MessageChannelSyncStage.MESSAGE_LIST_FETCH_PENDING,
|
||||
throttleFailureCount: 0,
|
||||
throttleRetryAfter: null,
|
||||
syncStageStartedAt: null,
|
||||
syncedAt: new Date().toISOString(),
|
||||
});
|
||||
@@ -307,6 +309,7 @@ export class MessageChannelSyncStatusService {
|
||||
await messageChannelRepository.update(messageChannelIds, {
|
||||
syncStage: MessageChannelSyncStage.FAILED,
|
||||
syncStatus: syncStatus,
|
||||
throttleRetryAfter: null,
|
||||
});
|
||||
|
||||
const metricsKey =
|
||||
|
||||
+1
@@ -98,6 +98,7 @@ export class MessageChannelWorkspaceEntity extends BaseWorkspaceEntity {
|
||||
syncStage: MessageChannelSyncStage;
|
||||
syncStageStartedAt: string | null;
|
||||
throttleFailureCount: number;
|
||||
throttleRetryAfter: string | null;
|
||||
connectedAccount: EntityRelation<ConnectedAccountWorkspaceEntity>;
|
||||
connectedAccountId: string;
|
||||
messageChannelMessageAssociations: EntityRelation<
|
||||
|
||||
+3
@@ -3,6 +3,7 @@ import { type MessageNetworkExceptionCode } from 'src/modules/messaging/message-
|
||||
export class MessageImportDriverException extends Error {
|
||||
code: MessageImportDriverExceptionCode | MessageNetworkExceptionCode;
|
||||
cause?: Error;
|
||||
throttleRetryAfter?: Date;
|
||||
context?: {
|
||||
messageChannelId?: string;
|
||||
workspaceId?: string;
|
||||
@@ -14,6 +15,7 @@ export class MessageImportDriverException extends Error {
|
||||
code: MessageImportDriverExceptionCode | MessageNetworkExceptionCode,
|
||||
options?: {
|
||||
cause?: Error;
|
||||
throttleRetryAfter?: Date;
|
||||
context?: {
|
||||
messageChannelId?: string;
|
||||
workspaceId?: string;
|
||||
@@ -25,6 +27,7 @@ export class MessageImportDriverException extends Error {
|
||||
this.name = 'MessageImportDriverException';
|
||||
this.code = code;
|
||||
this.cause = options?.cause;
|
||||
this.throttleRetryAfter = options?.throttleRetryAfter;
|
||||
this.context = options?.context;
|
||||
|
||||
if (options?.cause?.stack) {
|
||||
|
||||
+7
-3
@@ -50,9 +50,11 @@ const ERROR_DEFINITIONS: Record<number, Record<string, ErrorConfig>> = {
|
||||
export const getGmailApiError = ({
|
||||
code,
|
||||
reason,
|
||||
message,
|
||||
}: {
|
||||
code: number;
|
||||
reason?: string;
|
||||
message?: string;
|
||||
}): GaxiosError => {
|
||||
const statusMap = ERROR_DEFINITIONS[code];
|
||||
|
||||
@@ -62,8 +64,10 @@ export const getGmailApiError = ({
|
||||
|
||||
const config = statusMap[reason || ''] ?? statusMap.default;
|
||||
|
||||
const errorMessage = message ?? config.message;
|
||||
|
||||
return new GaxiosError(
|
||||
config.message,
|
||||
errorMessage,
|
||||
{ url: 'https://gmail.googleapis.com/mocks' },
|
||||
{
|
||||
status: code,
|
||||
@@ -71,10 +75,10 @@ export const getGmailApiError = ({
|
||||
data: {
|
||||
error: {
|
||||
code,
|
||||
message: config.message,
|
||||
message: errorMessage,
|
||||
errors: [
|
||||
{
|
||||
message: config.message,
|
||||
message: errorMessage,
|
||||
reason: config.reason,
|
||||
},
|
||||
],
|
||||
|
||||
+40
@@ -0,0 +1,40 @@
|
||||
import { parseGmailErrorRetryAfter } from 'src/modules/messaging/message-import-manager/drivers/gmail/utils/parse-gmail-error-retry-after.util';
|
||||
|
||||
describe('parseGmailErrorRetryAfter', () => {
|
||||
it('should extract the retry-after date from a Gmail 429 error message', () => {
|
||||
const fifteenMinutesFromNow = new Date(Date.now() + 15 * 60 * 1000);
|
||||
const message = `User-rate limit exceeded. Retry after ${fifteenMinutesFromNow.toISOString()}`;
|
||||
|
||||
const result = parseGmailErrorRetryAfter(message);
|
||||
|
||||
expect(result).toBeInstanceOf(Date);
|
||||
expect(result!.getTime()).toBeCloseTo(fifteenMinutesFromNow.getTime(), -3);
|
||||
});
|
||||
|
||||
it('should return undefined when the message contains no retry-after timestamp', () => {
|
||||
expect(
|
||||
parseGmailErrorRetryAfter('Too Many Concurrent Requests'),
|
||||
).toBeUndefined();
|
||||
});
|
||||
|
||||
it('should return undefined when the retry-after timestamp has already passed', () => {
|
||||
const fiveMinutesAgo = new Date(Date.now() - 5 * 60 * 1000);
|
||||
const message = `User-rate limit exceeded. Retry after ${fiveMinutesAgo.toISOString()}`;
|
||||
|
||||
expect(parseGmailErrorRetryAfter(message)).toBeUndefined();
|
||||
});
|
||||
|
||||
it('should return undefined for an empty string', () => {
|
||||
expect(parseGmailErrorRetryAfter('')).toBeUndefined();
|
||||
});
|
||||
|
||||
it('should match case variations of "Retry after"', () => {
|
||||
const fifteenMinutesFromNow = new Date(Date.now() + 15 * 60 * 1000);
|
||||
const message = `User-rate limit exceeded. retry after ${fifteenMinutesFromNow.toISOString()}`;
|
||||
|
||||
const result = parseGmailErrorRetryAfter(message);
|
||||
|
||||
expect(result).toBeInstanceOf(Date);
|
||||
expect(result!.getTime()).toBeCloseTo(fifteenMinutesFromNow.getTime(), -3);
|
||||
});
|
||||
});
|
||||
+44
@@ -232,4 +232,48 @@ describe('parseGmailApiError', () => {
|
||||
MessageImportDriverExceptionCode.SYNC_CURSOR_ERROR,
|
||||
);
|
||||
});
|
||||
|
||||
it('should populate retryAfter on the exception when a 429 contains a retry-after timestamp', () => {
|
||||
const fifteenMinutesFromNow = new Date(Date.now() + 15 * 60 * 1000);
|
||||
const error = getGmailApiError({
|
||||
code: 429,
|
||||
message: `User-rate limit exceeded. Retry after ${fifteenMinutesFromNow.toISOString()}`,
|
||||
});
|
||||
|
||||
const exception = parseGmailApiError(error);
|
||||
|
||||
expect(exception.throttleRetryAfter).toBeInstanceOf(Date);
|
||||
expect(exception.throttleRetryAfter!.getTime()).toBeCloseTo(
|
||||
fifteenMinutesFromNow.getTime(),
|
||||
-3,
|
||||
);
|
||||
});
|
||||
|
||||
it('should leave retryAfter undefined on 429 without a retry-after timestamp', () => {
|
||||
const error = getGmailApiError({ code: 429 });
|
||||
|
||||
const exception = parseGmailApiError(error);
|
||||
|
||||
expect(exception.throttleRetryAfter).toBeUndefined();
|
||||
});
|
||||
|
||||
it('should populate retryAfter on the exception when a 403 rateLimitExceeded contains a retry-after timestamp', () => {
|
||||
const fifteenMinutesFromNow = new Date(Date.now() + 15 * 60 * 1000);
|
||||
const error = getGmailApiError({
|
||||
code: 403,
|
||||
reason: 'rateLimit',
|
||||
message: `Rate Limit Exceeded. Retry after ${fifteenMinutesFromNow.toISOString()}`,
|
||||
});
|
||||
|
||||
const exception = parseGmailApiError(error);
|
||||
|
||||
expect(exception.code).toBe(
|
||||
MessageImportDriverExceptionCode.TEMPORARY_ERROR,
|
||||
);
|
||||
expect(exception.throttleRetryAfter).toBeInstanceOf(Date);
|
||||
expect(exception.throttleRetryAfter!.getTime()).toBeCloseTo(
|
||||
fifteenMinutesFromNow.getTime(),
|
||||
-3,
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
+9
@@ -4,6 +4,7 @@ import {
|
||||
MessageImportDriverException,
|
||||
MessageImportDriverExceptionCode,
|
||||
} from 'src/modules/messaging/message-import-manager/drivers/exceptions/message-import-driver.exception';
|
||||
import { parseGmailErrorRetryAfter } from 'src/modules/messaging/message-import-manager/drivers/gmail/utils/parse-gmail-error-retry-after.util';
|
||||
|
||||
export const parseGmailApiError = (
|
||||
error: GaxiosError,
|
||||
@@ -57,6 +58,9 @@ export const parseGmailApiError = (
|
||||
return new MessageImportDriverException(
|
||||
gmailApiError.message,
|
||||
MessageImportDriverExceptionCode.TEMPORARY_ERROR,
|
||||
{
|
||||
throttleRetryAfter: parseGmailErrorRetryAfter(gmailApiError.message),
|
||||
},
|
||||
);
|
||||
|
||||
case 403:
|
||||
@@ -68,6 +72,11 @@ export const parseGmailApiError = (
|
||||
return new MessageImportDriverException(
|
||||
gmailApiError.message,
|
||||
MessageImportDriverExceptionCode.TEMPORARY_ERROR,
|
||||
{
|
||||
throttleRetryAfter: parseGmailErrorRetryAfter(
|
||||
gmailApiError.message,
|
||||
),
|
||||
},
|
||||
);
|
||||
}
|
||||
if (gmailApiError.reason === 'domainPolicy') {
|
||||
|
||||
+24
@@ -0,0 +1,24 @@
|
||||
const RETRY_AFTER_REGEX =
|
||||
/Retry after (\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?Z)/i;
|
||||
|
||||
export const parseGmailErrorRetryAfter = (
|
||||
message: string,
|
||||
): Date | undefined => {
|
||||
const match = message.match(RETRY_AFTER_REGEX);
|
||||
|
||||
if (!match) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
const retryAfter = new Date(match[1]);
|
||||
|
||||
if (isNaN(retryAfter.getTime())) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
if (retryAfter <= new Date()) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
return retryAfter;
|
||||
};
|
||||
+1
@@ -84,6 +84,7 @@ export class MessagingMessageListFetchJob {
|
||||
isThrottled(
|
||||
messageChannel.syncStageStartedAt,
|
||||
messageChannel.throttleFailureCount,
|
||||
messageChannel.throttleRetryAfter,
|
||||
)
|
||||
) {
|
||||
await this.messageChannelSyncStatusService.markAsMessagesListFetchPending(
|
||||
|
||||
+1
@@ -82,6 +82,7 @@ export class MessagingMessagesImportJob {
|
||||
isThrottled(
|
||||
messageChannel.syncStageStartedAt,
|
||||
messageChannel.throttleFailureCount,
|
||||
messageChannel.throttleRetryAfter,
|
||||
)
|
||||
) {
|
||||
await this.messageChannelSyncStatusService.markAsMessagesImportPending(
|
||||
|
||||
+2
@@ -38,6 +38,7 @@ export class MessagingCursorService {
|
||||
},
|
||||
{
|
||||
throttleFailureCount: 0,
|
||||
throttleRetryAfter: null,
|
||||
syncStageStartedAt: null,
|
||||
syncCursor:
|
||||
!messageChannel.syncCursor ||
|
||||
@@ -61,6 +62,7 @@ export class MessagingCursorService {
|
||||
},
|
||||
{
|
||||
throttleFailureCount: 0,
|
||||
throttleRetryAfter: null,
|
||||
syncStageStartedAt: null,
|
||||
},
|
||||
);
|
||||
|
||||
+16
@@ -1,5 +1,7 @@
|
||||
import { Injectable } from '@nestjs/common';
|
||||
|
||||
import { isDefined } from 'twenty-shared/utils';
|
||||
|
||||
import { ExceptionHandlerService } from 'src/engine/core-modules/exception-handler/exception-handler.service';
|
||||
import {
|
||||
type TwentyORMException,
|
||||
@@ -166,6 +168,20 @@ export class MessageImportExceptionHandlerService {
|
||||
undefined,
|
||||
['throttleFailureCount', 'id'],
|
||||
);
|
||||
|
||||
const throttleRetryAfter =
|
||||
exception instanceof MessageImportDriverException
|
||||
? exception.throttleRetryAfter
|
||||
: undefined;
|
||||
|
||||
await messageChannelRepository.update(
|
||||
{ id: messageChannel.id },
|
||||
{
|
||||
throttleRetryAfter: isDefined(throttleRetryAfter)
|
||||
? throttleRetryAfter.toISOString()
|
||||
: null,
|
||||
},
|
||||
);
|
||||
}, authContext);
|
||||
|
||||
switch (syncStep) {
|
||||
|
||||
+1
@@ -189,6 +189,7 @@ export class MessagingMessagesImportService {
|
||||
},
|
||||
{
|
||||
throttleFailureCount: 0,
|
||||
throttleRetryAfter: null,
|
||||
syncStageStartedAt: null,
|
||||
},
|
||||
);
|
||||
|
||||
@@ -821,6 +821,9 @@ export const STANDARD_OBJECTS = {
|
||||
throttleFailureCount: {
|
||||
universalIdentifier: '20202020-0291-42be-9ad0-d578a51684ab',
|
||||
},
|
||||
throttleRetryAfter: {
|
||||
universalIdentifier: '20202020-a1e3-4d7f-b5c2-9f8e6d4c3b2a',
|
||||
},
|
||||
},
|
||||
indexes: {
|
||||
connectedAccountIdIndex: {
|
||||
|
||||
Reference in New Issue
Block a user