Improve workflow throttling logic (#16260)

- if >5000 workflows per hour, new ones should failed
- if >100 workflow per min, new ones should be set as not started.
Except manual trigger
- when enqueued, we check if there a not started workflows that may be
queued. If yes, we call the associated job

---------

Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com>

packages/twenty-server/src/modules/workflow/workflow-executor/workflow-actions/delay/jobs/resume-delayed-workflow.job.ts

@@ -17,7 +17,6 @@ import {
 } from 'src/modules/workflow/workflow-runner/exceptions/workflow-run.exception';
 import { RunWorkflowJob } from 'src/modules/workflow/workflow-runner/jobs/run-workflow.job';
 import { type RunWorkflowJobData } from 'src/modules/workflow/workflow-runner/types/run-workflow-job-data.type';
-import { WorkflowRunQueueWorkspaceService } from 'src/modules/workflow/workflow-runner/workflow-run-queue/workspace-services/workflow-run-queue.workspace-service';
 import { WorkflowRunWorkspaceService } from 'src/modules/workflow/workflow-runner/workflow-run/workflow-run.workspace-service';
 
 @Processor({
@@ -29,7 +28,6 @@ export class ResumeDelayedWorkflowJob {
     @InjectMessageQueue(MessageQueue.workflowQueue)
     private readonly messageQueueService: MessageQueueService,
     private readonly workflowRunWorkspaceService: WorkflowRunWorkspaceService,
-    private readonly workflowRunQueueWorkspaceService: WorkflowRunQueueWorkspaceService,
   ) {}
 
   @Process(RESUME_DELAYED_WORKFLOW_JOB_NAME)
@@ -89,10 +87,6 @@ export class ResumeDelayedWorkflowJob {
           lastExecutedStepId: stepId,
         },
       );
-
-      await this.workflowRunQueueWorkspaceService.increaseWorkflowRunQueuedCount(
-        workspaceId,
-      );
     } catch (error) {
       await this.workflowRunWorkspaceService.endWorkflowRun({
         workflowRunId,

packages/twenty-server/src/modules/workflow/workflow-executor/workflow-executor.module.ts

@@ -16,24 +16,22 @@ import { IteratorActionModule } from 'src/modules/workflow/workflow-executor/wor
 import { RecordCRUDActionModule } from 'src/modules/workflow/workflow-executor/workflow-actions/record-crud/record-crud-action.module';
 import { ToolExecutorWorkflowAction } from 'src/modules/workflow/workflow-executor/workflow-actions/tool-executor-workflow-action';
 import { WorkflowExecutorWorkspaceService } from 'src/modules/workflow/workflow-executor/workspace-services/workflow-executor.workspace-service';
-import { WorkflowRunQueueModule } from 'src/modules/workflow/workflow-runner/workflow-run-queue/workflow-run-queue.module';
 import { WorkflowRunModule } from 'src/modules/workflow/workflow-runner/workflow-run/workflow-run.module';
 
 @Module({
   imports: [
     WorkflowCommonModule,
+    WorkflowRunModule,
     CodeActionModule,
     DelayActionModule,
     RecordCRUDActionModule,
     FormActionModule,
-    WorkflowRunModule,
     BillingModule,
     FilterActionModule,
     IteratorActionModule,
     AiAgentActionModule,
     EmptyActionModule,
     FeatureFlagModule,
-    WorkflowRunQueueModule,
     ToolModule,
   ],
   providers: [

packages/twenty-server/src/modules/workflow/workflow-executor/workspace-services/__tests__/workflow-executor.workspace-service.spec.ts

@@ -15,7 +15,6 @@ import {
   WorkflowActionType,
 } from 'src/modules/workflow/workflow-executor/workflow-actions/types/workflow-action.type';
 import { WorkflowExecutorWorkspaceService } from 'src/modules/workflow/workflow-executor/workspace-services/workflow-executor.workspace-service';
-import { WorkflowRunQueueWorkspaceService } from 'src/modules/workflow/workflow-runner/workflow-run-queue/workspace-services/workflow-run-queue.workspace-service';
 import { WorkflowRunWorkspaceService } from 'src/modules/workflow/workflow-runner/workflow-run/workflow-run.workspace-service';
 
 jest.mock(
@@ -61,10 +60,6 @@ describe('WorkflowExecutorWorkspaceService', () => {
     add: jest.fn(),
   };
 
-  const mockWorkflowRunQueueWorkspaceService = {
-    increaseWorkflowRunQueuedCount: jest.fn(),
-  };
-
   beforeEach(async () => {
     jest.clearAllMocks();
 
@@ -93,10 +88,6 @@ describe('WorkflowExecutorWorkspaceService', () => {
           provide: `MESSAGE_QUEUE_${MessageQueue.workflowQueue}`,
           useValue: mockMessageQueueService,
         },
-        {
-          provide: WorkflowRunQueueWorkspaceService,
-          useValue: mockWorkflowRunQueueWorkspaceService,
-        },
       ],
     }).compile();
 
@@ -383,10 +374,6 @@ describe('WorkflowExecutorWorkspaceService', () => {
         },
       );
 
-      expect(
-        mockWorkflowRunQueueWorkspaceService.increaseWorkflowRunQueuedCount,
-      ).toHaveBeenCalledWith(mockWorkspaceId);
-
       // Should not execute the next step (step-2) in the same job
       expect(workflowActionFactory.get).toHaveBeenCalledTimes(1);
       expect(workflowActionFactory.get).toHaveBeenCalledWith(

packages/twenty-server/src/modules/workflow/workflow-executor/workspace-services/workflow-executor.workspace-service.ts

@@ -36,7 +36,6 @@ import { WorkflowIteratorResult } from 'src/modules/workflow/workflow-executor/w
 import { WorkflowAction } from 'src/modules/workflow/workflow-executor/workflow-actions/types/workflow-action.type';
 import { RUN_WORKFLOW_JOB_NAME } from 'src/modules/workflow/workflow-runner/constants/run-workflow-job-name';
 import { type RunWorkflowJobData } from 'src/modules/workflow/workflow-runner/types/run-workflow-job-data.type';
-import { WorkflowRunQueueWorkspaceService } from 'src/modules/workflow/workflow-runner/workflow-run-queue/workspace-services/workflow-run-queue.workspace-service';
 import { WorkflowRunWorkspaceService } from 'src/modules/workflow/workflow-runner/workflow-run/workflow-run.workspace-service';
 
 const MAX_EXECUTED_STEPS_COUNT = 20;
@@ -50,7 +49,6 @@ export class WorkflowExecutorWorkspaceService {
     private readonly billingService: BillingService,
     @InjectMessageQueue(MessageQueue.workflowQueue)
     private readonly messageQueueService: MessageQueueService,
-    private readonly workflowRunQueueWorkspaceService: WorkflowRunQueueWorkspaceService,
   ) {}
 
   async executeFromSteps({
@@ -418,8 +416,5 @@ export class WorkflowExecutorWorkspaceService {
         lastExecutedStepId,
       },
     );
-    await this.workflowRunQueueWorkspaceService.increaseWorkflowRunQueuedCount(
-      workspaceId,
-    );
   }
 }