Add a limit to workflow queue per workspace (#12908)

- new status `ENQUEUED` added. With a command to backfill - counter in cache per workspace, managed by a new service [workflow-run-queue.workspace-service.ts](https://github.com/twentyhq/twenty/compare/tt-improve-workflow-run-queueing?expand=1#diff-1e2de2a48cd482a3bd7e8dedf1150a19d0b200afbd9282181a24ecddddb56927) - cron added that will run every minute to look for not started workflows Here is the new flow: - When executing a workflow, we check if the queue is not full. If not, run is created as `ENQUEUED` and the run workflow job is triggered as usual. If full, create the run as NOT_STARTED and do not trigger the job - Cron will look for NOT_STARTED workflows and queue some if there is some place again - Only MANUAL and Form submit skip the queue limit
2025-06-30 14:27:57 +02:00
parent fcb869fdd9
commit 8272e5dfd0
22 changed files with 501 additions and 32 deletions
@@ -0,0 +1,112 @@
+import { InjectRepository } from '@nestjs/typeorm';
+
+import { Command } from 'nest-commander';
+import { Repository } from 'typeorm';
+
+import {
+  ActiveOrSuspendedWorkspacesMigrationCommandRunner,
+  RunOnWorkspaceArgs,
+} from 'src/database/commands/command-runners/active-or-suspended-workspaces-migration.command-runner';
+import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity';
+import { FieldMetadataEntity } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity';
+import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager';
+import { WorkspaceDataSourceService } from 'src/engine/workspace-datasource/workspace-datasource.service';
+import { WORKFLOW_RUN_STANDARD_FIELD_IDS } from 'src/engine/workspace-manager/workspace-sync-metadata/constants/standard-field-ids';
+import { WorkflowRunStatus } from 'src/modules/workflow/common/standard-objects/workflow-run.workspace-entity';
+
+@Command({
+  name: 'upgrade:1-2:add-enqueued-status-to-workflow-run',
+  description: 'Add enqueued status to workflow run',
+})
+export class AddEnqueuedStatusToWorkflowRunCommand extends ActiveOrSuspendedWorkspacesMigrationCommandRunner {
+  constructor(
+    @InjectRepository(Workspace, 'core')
+    protected readonly workspaceRepository: Repository<Workspace>,
+    protected readonly twentyORMGlobalManager: TwentyORMGlobalManager,
+    @InjectRepository(FieldMetadataEntity, 'core')
+    private readonly fieldMetadataRepository: Repository<FieldMetadataEntity>,
+    private readonly workspaceDataSourceService: WorkspaceDataSourceService,
+  ) {
+    super(workspaceRepository, twentyORMGlobalManager);
+  }
+
+  override async runOnWorkspace({
+    workspaceId,
+    options,
+  }: RunOnWorkspaceArgs): Promise<void> {
+    this.logger.log(
+      `Adding enqueued status to workflow run for workspace ${workspaceId}`,
+    );
+
+    const workflowRunStatusFieldMetadata =
+      await this.fieldMetadataRepository.findOne({
+        where: {
+          standardId: WORKFLOW_RUN_STANDARD_FIELD_IDS.status,
+        },
+      });
+
+    if (!workflowRunStatusFieldMetadata) {
+      this.logger.error(
+        `Workflow run status field metadata not found for workspace ${workspaceId}`,
+      );
+
+      return;
+    }
+
+    const workflowRunStatusFieldMetadataOptions =
+      workflowRunStatusFieldMetadata.options;
+
+    // check if enqueued status is already in the field metadata options
+    if (
+      workflowRunStatusFieldMetadataOptions.some(
+        (option) => option.value === WorkflowRunStatus.ENQUEUED,
+      )
+    ) {
+      this.logger.log(
+        `Workflow run status field metadata options already contain enqueued status for workspace ${workspaceId}`,
+      );
+    } else if (options.dryRun) {
+      this.logger.log(
+        `Would add enqueued status to workflow run status field metadata for workspace ${workspaceId}`,
+      );
+    } else {
+      workflowRunStatusFieldMetadataOptions.push({
+        value: WorkflowRunStatus.ENQUEUED,
+        label: 'Enqueued',
+        position: 4,
+        color: 'blue',
+      });
+
+      await this.fieldMetadataRepository.save(workflowRunStatusFieldMetadata);
+
+      this.logger.log(
+        `Enqueued status added to workflow run status field metadata for workspace ${workspaceId}`,
+      );
+    }
+
+    const schemaName =
+      this.workspaceDataSourceService.getSchemaName(workspaceId);
+
+    const mainDataSource =
+      await this.workspaceDataSourceService.connectToMainDataSource();
+
+    if (options.dryRun) {
+      this.logger.log(
+        `Would try to add enqueued status to workflow run status enum for workspace ${workspaceId}`,
+      );
+    } else {
+      try {
+        await mainDataSource.query(
+          `ALTER TYPE ${schemaName}."workflowRun_status_enum" ADD VALUE 'ENQUEUED'`,
+        );
+        this.logger.log(
+          `Enqueued status added to workflow run status enum for workspace ${workspaceId}`,
+        );
+      } catch (error) {
+        this.logger.error(
+          `Error adding enqueued status to workflow run status enum for workspace ${workspaceId}: ${error}`,
+        );
+      }
+    }
+  }
+}
@@ -0,0 +1,17 @@
+import { Module } from '@nestjs/common';
+import { TypeOrmModule } from '@nestjs/typeorm';
+
+import { AddEnqueuedStatusToWorkflowRunCommand } from 'src/database/commands/upgrade-version-command/1-2/1-2-add-enqueued-status-to-workflow-run.command';
+import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity';
+import { FieldMetadataEntity } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity';
+import { WorkspaceDataSourceModule } from 'src/engine/workspace-datasource/workspace-datasource.module';
+
+@Module({
+  imports: [
+    TypeOrmModule.forFeature([Workspace, FieldMetadataEntity], 'core'),
+    WorkspaceDataSourceModule,
+  ],
+  providers: [AddEnqueuedStatusToWorkflowRunCommand],
+  exports: [AddEnqueuedStatusToWorkflowRunCommand],
+})
+export class V1_2_UpgradeVersionCommandModule {}