Improve workflow queue cron reliability (#13818)

Improve workflow enqueue cron : instead of relying on the cache to know
how many workflows we can enqueue, query the DB. Then set the cache and
process the not started workflows.

Also adding a second cron that will look for workflows enqueued one hour
ago or more and put these back in the not started status. This will
allow the first cron to start these again.

packages/twenty-server/src/database/commands/cron-register-all.command.ts

@@ -2,6 +2,7 @@ import { Logger } from '@nestjs/common';
 
 import { Command, CommandRunner } from 'nest-commander';
 
+import { CheckCustomDomainValidRecordsCronCommand } from 'src/engine/core-modules/domain-manager/crons/commands/check-custom-domain-valid-records.cron.command';
 import { CleanupOrphanedFilesCronCommand } from 'src/engine/core-modules/file/crons/commands/cleanup-orphaned-files.cron.command';
 import { CalendarEventListFetchCronCommand } from 'src/modules/calendar/calendar-event-import-manager/crons/commands/calendar-event-list-fetch.cron.command';
 import { CalendarEventsImportCronCommand } from 'src/modules/calendar/calendar-event-import-manager/crons/commands/calendar-import.cron.command';
@@ -9,8 +10,9 @@ import { CalendarOngoingStaleCronCommand } from 'src/modules/calendar/calendar-e
 import { MessagingMessageListFetchCronCommand } from 'src/modules/messaging/message-import-manager/crons/commands/messaging-message-list-fetch.cron.command';
 import { MessagingMessagesImportCronCommand } from 'src/modules/messaging/message-import-manager/crons/commands/messaging-messages-import.cron.command';
 import { MessagingOngoingStaleCronCommand } from 'src/modules/messaging/message-import-manager/crons/commands/messaging-ongoing-stale.cron.command';
-import { CronTriggerCronCommand } from 'src/modules/workflow/workflow-trigger/automated-trigger/crons/commands/cron-trigger.cron.command';
-import { CheckCustomDomainValidRecordsCronCommand } from 'src/engine/core-modules/domain-manager/crons/commands/check-custom-domain-valid-records.cron.command';
+import { WorkflowHandleStaledRunsCronCommand } from 'src/modules/workflow/workflow-runner/workflow-run-queue/cron/command/workflow-handle-staled-runs.cron.command';
+import { WorkflowRunEnqueueCronCommand } from 'src/modules/workflow/workflow-runner/workflow-run-queue/cron/command/workflow-run-enqueue.cron.command';
+import { WorkflowCronTriggerCronCommand } from 'src/modules/workflow/workflow-trigger/automated-trigger/crons/commands/workflow-cron-trigger.cron.command';
 
 @Command({
   name: 'cron:register:all',
@@ -26,9 +28,11 @@ export class CronRegisterAllCommand extends CommandRunner {
     private readonly calendarEventListFetchCronCommand: CalendarEventListFetchCronCommand,
     private readonly calendarEventsImportCronCommand: CalendarEventsImportCronCommand,
     private readonly calendarOngoingStaleCronCommand: CalendarOngoingStaleCronCommand,
-    private readonly cronTriggerCronCommand: CronTriggerCronCommand,
+    private readonly workflowCronTriggerCronCommand: WorkflowCronTriggerCronCommand,
     private readonly cleanupOrphanedFilesCronCommand: CleanupOrphanedFilesCronCommand,
     private readonly checkCustomDomainValidRecordsCronCommand: CheckCustomDomainValidRecordsCronCommand,
+    private readonly workflowRunEnqueueCronCommand: WorkflowRunEnqueueCronCommand,
+    private readonly workflowHandleStaledRunsCronCommand: WorkflowHandleStaledRunsCronCommand,
   ) {
     super();
   }
@@ -61,7 +65,6 @@ export class CronRegisterAllCommand extends CommandRunner {
         name: 'CalendarOngoingStale',
         command: this.calendarOngoingStaleCronCommand,
       },
-      { name: 'CronTrigger', command: this.cronTriggerCronCommand },
       {
         name: 'CleanupOrphanedFiles',
         command: this.cleanupOrphanedFilesCronCommand,
@@ -70,6 +73,18 @@ export class CronRegisterAllCommand extends CommandRunner {
         name: 'CheckCustomDomainValidRecords',
         command: this.checkCustomDomainValidRecordsCronCommand,
       },
+      {
+        name: 'WorkflowCronTrigger',
+        command: this.workflowCronTriggerCronCommand,
+      },
+      {
+        name: 'WorkflowRunEnqueue',
+        command: this.workflowRunEnqueueCronCommand,
+      },
+      {
+        name: 'WorkflowHandleStaledRuns',
+        command: this.workflowHandleStaledRunsCronCommand,
+      },
     ];
 
     let successCount = 0;

packages/twenty-server/src/database/commands/database-command.module.ts

@@ -7,6 +7,7 @@ import { UpgradeVersionCommandModule } from 'src/database/commands/upgrade-versi
 import { MigrateViewsToCoreCommand } from 'src/database/commands/views-migration/migrate-views-to-core.command';
 import { TypeORMModule } from 'src/database/typeorm/typeorm.module';
 import { ApiKeyModule } from 'src/engine/core-modules/api-key/api-key.module';
+import { DomainManagerModule } from 'src/engine/core-modules/domain-manager/domain-manager.module';
 import { FeatureFlagModule } from 'src/engine/core-modules/feature-flag/feature-flag.module';
 import { FileModule } from 'src/engine/core-modules/file/file.module';
 import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity';
@@ -18,24 +19,22 @@ import { DevSeederModule } from 'src/engine/workspace-manager/dev-seeder/dev-see
 import { WorkspaceManagerModule } from 'src/engine/workspace-manager/workspace-manager.module';
 import { CalendarEventImportManagerModule } from 'src/modules/calendar/calendar-event-import-manager/calendar-event-import-manager.module';
 import { MessagingImportManagerModule } from 'src/modules/messaging/message-import-manager/messaging-import-manager.module';
+import { WorkflowRunQueueModule } from 'src/modules/workflow/workflow-runner/workflow-run-queue/workflow-run-queue.module';
 import { AutomatedTriggerModule } from 'src/modules/workflow/workflow-trigger/automated-trigger/automated-trigger.module';
-import { DomainManagerModule } from 'src/engine/core-modules/domain-manager/domain-manager.module';
 
 import { DataSeedWorkspaceCommand } from './data-seed-dev-workspace.command';
 
 @Module({
   imports: [
     UpgradeVersionCommandModule,
-
     TypeOrmModule.forFeature([Workspace], 'core'),
-
     // Cron command dependencies
     MessagingImportManagerModule,
     CalendarEventImportManagerModule,
     AutomatedTriggerModule,
     FileModule,
     DomainManagerModule,
-
+    WorkflowRunQueueModule,
     // Data seeding dependencies
     TypeORMModule,
     FieldMetadataModule,