Respect Gmail retry-after in messaging throttle (#17850)

Gmail 429/403 rate-limit responses include an explicit retry-after
timestamp, usually ~15 minutes out.

The exponential backoff starts at 1 minute, so the channel burns through
all 5 retry attempts before the window actually closes and gets marked
as permanently failed.

Adds throttleRetryAfter to the message channel and uses max(backoff,
retryAfter) in isThrottled().

packages/twenty-server/src/database/commands/upgrade-version-command/1-18/1-18-backfill-message-channel-throttle-retry-after.command.ts

@@ -0,0 +1,120 @@
+import { InjectRepository } from '@nestjs/typeorm';
+
+import { Command } from 'nest-commander';
+import { STANDARD_OBJECTS } from 'twenty-shared/metadata';
+import { FieldMetadataType } from 'twenty-shared/types';
+import { Repository } from 'typeorm';
+
+import { ActiveOrSuspendedWorkspacesMigrationCommandRunner } from 'src/database/commands/command-runners/active-or-suspended-workspaces-migration.command-runner';
+import { RunOnWorkspaceArgs } from 'src/database/commands/command-runners/workspaces-migration.command-runner';
+import { ApplicationService } from 'src/engine/core-modules/application/services/application.service';
+import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity';
+import { DataSourceService } from 'src/engine/metadata-modules/data-source/data-source.service';
+import { FieldMetadataService } from 'src/engine/metadata-modules/field-metadata/services/field-metadata.service';
+import { findFlatEntityByUniversalIdentifier } from 'src/engine/metadata-modules/flat-entity/utils/find-flat-entity-by-universal-identifier.util';
+import { type FlatObjectMetadata } from 'src/engine/metadata-modules/flat-object-metadata/types/flat-object-metadata.type';
+import { GlobalWorkspaceOrmManager } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager';
+import { WorkspaceCacheService } from 'src/engine/workspace-cache/services/workspace-cache.service';
+
+@Command({
+  name: 'upgrade:1-18:backfill-message-channel-throttle-retry-after',
+  description:
+    'Backfill throttleRetryAfter field on messageChannel standard object',
+})
+export class BackfillMessageChannelThrottleRetryAfterCommand extends ActiveOrSuspendedWorkspacesMigrationCommandRunner {
+  constructor(
+    @InjectRepository(WorkspaceEntity)
+    protected readonly workspaceRepository: Repository<WorkspaceEntity>,
+    protected readonly twentyORMGlobalManager: GlobalWorkspaceOrmManager,
+    protected readonly dataSourceService: DataSourceService,
+    private readonly applicationService: ApplicationService,
+    private readonly fieldMetadataService: FieldMetadataService,
+    private readonly workspaceCacheService: WorkspaceCacheService,
+  ) {
+    super(workspaceRepository, twentyORMGlobalManager, dataSourceService);
+  }
+
+  override async runOnWorkspace({
+    workspaceId,
+    options,
+  }: RunOnWorkspaceArgs): Promise<void> {
+    this.logger.log(
+      `Backfilling throttleRetryAfter field for workspace ${workspaceId}`,
+    );
+
+    const { flatObjectMetadataMaps } =
+      await this.workspaceCacheService.getOrRecompute(workspaceId, [
+        'flatObjectMetadataMaps',
+      ]);
+
+    const messageChannelObjectMetadata =
+      findFlatEntityByUniversalIdentifier<FlatObjectMetadata>({
+        flatEntityMaps: flatObjectMetadataMaps,
+        universalIdentifier:
+          STANDARD_OBJECTS.messageChannel.universalIdentifier,
+      });
+
+    if (!messageChannelObjectMetadata) {
+      this.logger.log(
+        `MessageChannel object metadata not found for workspace ${workspaceId}. Skipping.`,
+      );
+
+      return;
+    }
+
+    const { flatFieldMetadataMaps } =
+      await this.workspaceCacheService.getOrRecompute(workspaceId, [
+        'flatFieldMetadataMaps',
+      ]);
+
+    const existingField = findFlatEntityByUniversalIdentifier({
+      flatEntityMaps: flatFieldMetadataMaps,
+      universalIdentifier:
+        STANDARD_OBJECTS.messageChannel.fields.throttleRetryAfter
+          .universalIdentifier,
+    });
+
+    if (existingField) {
+      this.logger.log(
+        `throttleRetryAfter field already exists for workspace ${workspaceId}. Skipping.`,
+      );
+
+      return;
+    }
+
+    if (options.dryRun) {
+      this.logger.log(
+        `Would have created throttleRetryAfter field for workspace ${workspaceId}. Skipping (dry run).`,
+      );
+
+      return;
+    }
+
+    const { twentyStandardFlatApplication } =
+      await this.applicationService.findWorkspaceTwentyStandardAndCustomApplicationOrThrow(
+        { workspaceId },
+      );
+
+    await this.fieldMetadataService.createOneField({
+      createFieldInput: {
+        name: 'throttleRetryAfter',
+        type: FieldMetadataType.DATE_TIME,
+        label: 'Throttle Retry After',
+        description: 'Throttle Retry After',
+        icon: 'IconClock',
+        isNullable: true,
+        isUIReadOnly: true,
+        objectMetadataId: messageChannelObjectMetadata.id,
+        universalIdentifier:
+          STANDARD_OBJECTS.messageChannel.fields.throttleRetryAfter
+            .universalIdentifier,
+      },
+      workspaceId,
+      ownerFlatApplication: twentyStandardFlatApplication,
+    });
+
+    this.logger.log(
+      `Successfully created throttleRetryAfter field for workspace ${workspaceId}`,
+    );
+  }
+}

packages/twenty-server/src/database/commands/upgrade-version-command/1-18/1-18-upgrade-version-command.module.ts

@@ -2,6 +2,7 @@ import { Module } from '@nestjs/common';
 import { TypeOrmModule } from '@nestjs/typeorm';
 
 import { BackfillFileSizeAndMimeTypeCommand } from 'src/database/commands/upgrade-version-command/1-18/1-18-backfill-file-size-and-mime-type.command';
+import { BackfillMessageChannelThrottleRetryAfterCommand } from 'src/database/commands/upgrade-version-command/1-18/1-18-backfill-message-channel-throttle-retry-after.command';
 import { MigrateActivityRichTextAttachmentFileIdsCommand } from 'src/database/commands/upgrade-version-command/1-18/1-18-migrate-activity-rich-text-attachment-file-ids.command';
 import { MigrateAttachmentFilesCommand } from 'src/database/commands/upgrade-version-command/1-18/1-18-migrate-attachment-files.command';
 import { MigratePersonAvatarFilesCommand } from 'src/database/commands/upgrade-version-command/1-18/1-18-migrate-person-avatar-files.command';
@@ -40,12 +41,14 @@ import { PersonWorkspaceEntity } from 'src/modules/person/standard-objects/perso
     MigrateAttachmentFilesCommand,
     BackfillFileSizeAndMimeTypeCommand,
     MigrateActivityRichTextAttachmentFileIdsCommand,
+    BackfillMessageChannelThrottleRetryAfterCommand,
   ],
   exports: [
     MigratePersonAvatarFilesCommand,
     MigrateAttachmentFilesCommand,
     BackfillFileSizeAndMimeTypeCommand,
     MigrateActivityRichTextAttachmentFileIdsCommand,
+    BackfillMessageChannelThrottleRetryAfterCommand,
   ],
 })
 export class V1_18_UpgradeVersionCommandModule {}

packages/twenty-server/src/database/commands/upgrade-version-command/upgrade.command.ts

@@ -20,6 +20,7 @@ import { MigrateNoteTargetToMorphRelationsCommand } from 'src/database/commands/
 import { MigrateTaskTargetToMorphRelationsCommand } from 'src/database/commands/upgrade-version-command/1-17/1-17-migrate-task-target-to-morph-relations.command';
 import { MigrateWorkflowCodeStepsCommand } from 'src/database/commands/upgrade-version-command/1-17/1-17-migrate-workflow-code-steps.command';
 import { BackfillFileSizeAndMimeTypeCommand } from 'src/database/commands/upgrade-version-command/1-18/1-18-backfill-file-size-and-mime-type.command';
+import { BackfillMessageChannelThrottleRetryAfterCommand } from 'src/database/commands/upgrade-version-command/1-18/1-18-backfill-message-channel-throttle-retry-after.command';
 import { MigrateActivityRichTextAttachmentFileIdsCommand } from 'src/database/commands/upgrade-version-command/1-18/1-18-migrate-activity-rich-text-attachment-file-ids.command';
 import { MigrateAttachmentFilesCommand } from 'src/database/commands/upgrade-version-command/1-18/1-18-migrate-attachment-files.command';
 import { MigratePersonAvatarFilesCommand } from 'src/database/commands/upgrade-version-command/1-18/1-18-migrate-person-avatar-files.command';
@@ -59,6 +60,7 @@ export class UpgradeCommand extends UpgradeCommandRunner {
     protected readonly backfillFileSizeAndMimeTypeCommand: BackfillFileSizeAndMimeTypeCommand,
     protected readonly migrateAttachmentFilesCommand: MigrateAttachmentFilesCommand,
     protected readonly migrateActivityRichTextAttachmentFileIdsCommand: MigrateActivityRichTextAttachmentFileIdsCommand,
+    protected readonly backfillMessageChannelThrottleRetryAfterCommand: BackfillMessageChannelThrottleRetryAfterCommand,
   ) {
     super(
       workspaceRepository,
@@ -89,6 +91,7 @@ export class UpgradeCommand extends UpgradeCommandRunner {
       this.migrateAttachmentFilesCommand,
       this.migrateActivityRichTextAttachmentFileIdsCommand,
       this.backfillFileSizeAndMimeTypeCommand,
+      this.backfillMessageChannelThrottleRetryAfterCommand,
     ];
 
     this.allCommands = {