diff --git a/packages/twenty-server/src/modules/workflow/workflow-executor/types/workflow-executor-input.ts b/packages/twenty-server/src/modules/workflow/workflow-executor/types/workflow-executor-input.ts index 3721c8ecb8..4226627680 100644 --- a/packages/twenty-server/src/modules/workflow/workflow-executor/types/workflow-executor-input.ts +++ b/packages/twenty-server/src/modules/workflow/workflow-executor/types/workflow-executor-input.ts @@ -3,6 +3,7 @@ export type WorkflowExecutorInput = { workflowRunId: string; workspaceId: string; shouldComputeWorkflowRunStatus?: boolean; + executedStepsCount?: number; }; export type WorkflowBranchExecutorInput = { @@ -10,4 +11,5 @@ export type WorkflowBranchExecutorInput = { attemptCount?: number; workflowRunId: string; workspaceId: string; + executedStepsCount?: number; }; diff --git a/packages/twenty-server/src/modules/workflow/workflow-executor/utils/__tests__/should-execute-child-step.util.spec.ts b/packages/twenty-server/src/modules/workflow/workflow-executor/utils/__tests__/should-execute-child-step.util.spec.ts new file mode 100644 index 0000000000..ca63d45639 --- /dev/null +++ b/packages/twenty-server/src/modules/workflow/workflow-executor/utils/__tests__/should-execute-child-step.util.spec.ts @@ -0,0 +1,492 @@ +import { StepStatus } from 'twenty-shared/workflow'; + +import { shouldExecuteChildStep } from 'src/modules/workflow/workflow-executor/utils/should-execute-child-step.util'; +import { + type WorkflowAction, + WorkflowActionType, +} from 'src/modules/workflow/workflow-executor/workflow-actions/types/workflow-action.type'; + +describe('shouldExecuteChildStep', () => { + const parentSteps = [ + { + id: 'parent-1', + type: WorkflowActionType.CODE, + settings: { + errorHandlingOptions: { + continueOnFailure: { value: false }, + retryOnFailure: { value: false }, + }, + outputSchema: {}, + }, + nextStepIds: [], + } as unknown as WorkflowAction, + { + id: 'parent-2', + type: WorkflowActionType.SEND_EMAIL, + settings: { + errorHandlingOptions: { + continueOnFailure: { value: false }, + retryOnFailure: { value: false }, + }, + outputSchema: {}, + }, + nextStepIds: [], + } as unknown as WorkflowAction, + ]; + + it('should return true when there are no parent steps', () => { + const result = shouldExecuteChildStep({ + parentSteps: [], + stepInfos: {}, + }); + + expect(result).toBe(true); + }); + + it('should return true when at least one parent succeeded and all parents are completed', () => { + const stepInfos = { + 'parent-1': { + status: StepStatus.SUCCESS, + }, + 'parent-2': { + status: StepStatus.SUCCESS, + }, + }; + + const result = shouldExecuteChildStep({ + parentSteps, + stepInfos, + }); + + expect(result).toBe(true); + }); + + it('should return true when one parent succeeded and others are stopped', () => { + const stepInfos = { + 'parent-1': { + status: StepStatus.SUCCESS, + }, + 'parent-2': { + status: StepStatus.STOPPED, + }, + }; + + const result = shouldExecuteChildStep({ + parentSteps, + stepInfos, + }); + + expect(result).toBe(true); + }); + + it('should return true when one parent succeeded and others are skipped', () => { + const stepInfos = { + 'parent-1': { + status: StepStatus.SUCCESS, + }, + 'parent-2': { + status: StepStatus.SKIPPED, + }, + }; + + const result = shouldExecuteChildStep({ + parentSteps, + stepInfos, + }); + + expect(result).toBe(true); + }); + + it('should return true when one parent succeeded and others are mix of stopped and skipped', () => { + const multiParentSteps = [ + { + id: 'parent-1', + type: WorkflowActionType.CODE, + settings: { + errorHandlingOptions: { + continueOnFailure: { value: false }, + retryOnFailure: { value: false }, + }, + outputSchema: {}, + }, + nextStepIds: [], + } as unknown as WorkflowAction, + { + id: 'parent-2', + type: WorkflowActionType.SEND_EMAIL, + settings: { + errorHandlingOptions: { + continueOnFailure: { value: false }, + retryOnFailure: { value: false }, + }, + outputSchema: {}, + }, + nextStepIds: [], + } as unknown as WorkflowAction, + { + id: 'parent-3', + type: WorkflowActionType.CODE, + settings: { + errorHandlingOptions: { + continueOnFailure: { value: false }, + retryOnFailure: { value: false }, + }, + outputSchema: {}, + }, + nextStepIds: [], + } as unknown as WorkflowAction, + ]; + + const stepInfos = { + 'parent-1': { + status: StepStatus.SUCCESS, + }, + 'parent-2': { + status: StepStatus.STOPPED, + }, + 'parent-3': { + status: StepStatus.SKIPPED, + }, + }; + + const result = shouldExecuteChildStep({ + parentSteps: multiParentSteps, + stepInfos, + }); + + expect(result).toBe(true); + }); + + it('should return false when all parents are completed but none succeeded', () => { + const stepInfos = { + 'parent-1': { + status: StepStatus.STOPPED, + }, + 'parent-2': { + status: StepStatus.SKIPPED, + }, + }; + + const result = shouldExecuteChildStep({ + parentSteps, + stepInfos, + }); + + expect(result).toBe(false); + }); + + it('should return false when one parent succeeded but another is still running', () => { + const stepInfos = { + 'parent-1': { + status: StepStatus.SUCCESS, + }, + 'parent-2': { + status: StepStatus.RUNNING, + }, + }; + + const result = shouldExecuteChildStep({ + parentSteps, + stepInfos, + }); + + expect(result).toBe(false); + }); + + it('should return false when one parent succeeded but another has not started', () => { + const stepInfos = { + 'parent-1': { + status: StepStatus.SUCCESS, + }, + 'parent-2': { + status: StepStatus.NOT_STARTED, + }, + }; + + const result = shouldExecuteChildStep({ + parentSteps, + stepInfos, + }); + + expect(result).toBe(false); + }); + + it('should return false when one parent succeeded but another is pending', () => { + const stepInfos = { + 'parent-1': { + status: StepStatus.SUCCESS, + }, + 'parent-2': { + status: StepStatus.PENDING, + }, + }; + + const result = shouldExecuteChildStep({ + parentSteps, + stepInfos, + }); + + expect(result).toBe(false); + }); + + it('should return false when one parent succeeded but another has failed', () => { + const stepInfos = { + 'parent-1': { + status: StepStatus.SUCCESS, + }, + 'parent-2': { + status: StepStatus.FAILED, + }, + }; + + const result = shouldExecuteChildStep({ + parentSteps, + stepInfos, + }); + + expect(result).toBe(false); + }); + + it('should return false when all parents are still running', () => { + const stepInfos = { + 'parent-1': { + status: StepStatus.RUNNING, + }, + 'parent-2': { + status: StepStatus.RUNNING, + }, + }; + + const result = shouldExecuteChildStep({ + parentSteps, + stepInfos, + }); + + expect(result).toBe(false); + }); + + it('should return false when all parents have not started', () => { + const stepInfos = { + 'parent-1': { + status: StepStatus.NOT_STARTED, + }, + 'parent-2': { + status: StepStatus.NOT_STARTED, + }, + }; + + const result = shouldExecuteChildStep({ + parentSteps, + stepInfos, + }); + + expect(result).toBe(false); + }); + + it('should return false when no parent has succeeded even though all are completed', () => { + const stepInfos = { + 'parent-1': { + status: StepStatus.SKIPPED, + }, + 'parent-2': { + status: StepStatus.STOPPED, + }, + }; + + const result = shouldExecuteChildStep({ + parentSteps, + stepInfos, + }); + + expect(result).toBe(false); + }); + + it('should handle single parent step successfully', () => { + const singleParent = [ + { + id: 'parent-1', + type: WorkflowActionType.CODE, + settings: { + errorHandlingOptions: { + continueOnFailure: { value: false }, + retryOnFailure: { value: false }, + }, + outputSchema: {}, + }, + nextStepIds: [], + } as unknown as WorkflowAction, + ]; + + const stepInfos = { + 'parent-1': { + status: StepStatus.SUCCESS, + }, + }; + + const result = shouldExecuteChildStep({ + parentSteps: singleParent, + stepInfos, + }); + + expect(result).toBe(true); + }); + + it('should return false when single parent has stopped without success', () => { + const singleParent = [ + { + id: 'parent-1', + type: WorkflowActionType.CODE, + settings: { + errorHandlingOptions: { + continueOnFailure: { value: false }, + retryOnFailure: { value: false }, + }, + outputSchema: {}, + }, + nextStepIds: [], + } as unknown as WorkflowAction, + ]; + + const stepInfos = { + 'parent-1': { + status: StepStatus.STOPPED, + }, + }; + + const result = shouldExecuteChildStep({ + parentSteps: singleParent, + stepInfos, + }); + + expect(result).toBe(false); + }); + + it('should handle missing step info gracefully', () => { + const stepInfos = { + 'parent-1': { + status: StepStatus.SUCCESS, + }, + }; + + const result = shouldExecuteChildStep({ + parentSteps, + stepInfos, + }); + + expect(result).toBe(false); + }); + + it('should work correctly with multiple successful parents', () => { + const multiParentSteps = [ + { + id: 'parent-1', + type: WorkflowActionType.CODE, + settings: { + errorHandlingOptions: { + continueOnFailure: { value: false }, + retryOnFailure: { value: false }, + }, + outputSchema: {}, + }, + nextStepIds: [], + } as unknown as WorkflowAction, + { + id: 'parent-2', + type: WorkflowActionType.SEND_EMAIL, + settings: { + errorHandlingOptions: { + continueOnFailure: { value: false }, + retryOnFailure: { value: false }, + }, + outputSchema: {}, + }, + nextStepIds: [], + } as unknown as WorkflowAction, + { + id: 'parent-3', + type: WorkflowActionType.CODE, + settings: { + errorHandlingOptions: { + continueOnFailure: { value: false }, + retryOnFailure: { value: false }, + }, + outputSchema: {}, + }, + nextStepIds: [], + } as unknown as WorkflowAction, + ]; + + const stepInfos = { + 'parent-1': { status: StepStatus.SUCCESS }, + 'parent-2': { status: StepStatus.SUCCESS }, + 'parent-3': { status: StepStatus.SUCCESS }, + }; + + const result = shouldExecuteChildStep({ + parentSteps: multiParentSteps, + stepInfos, + }); + + expect(result).toBe(true); + }); + + it('should return false with large number of parents when none succeeded', () => { + const manyParentSteps = Array.from({ length: 25 }, (_, i) => ({ + id: `parent-${i + 1}`, + type: WorkflowActionType.CODE, + settings: { + errorHandlingOptions: { + continueOnFailure: { value: false }, + retryOnFailure: { value: false }, + }, + outputSchema: {}, + }, + nextStepIds: [], + })) as unknown as WorkflowAction[]; + + // All parents stopped or skipped, none succeeded + const stepInfos = Object.fromEntries( + manyParentSteps.map((step, i) => [ + step.id, + { status: i % 2 === 0 ? StepStatus.STOPPED : StepStatus.SKIPPED }, + ]), + ); + + const result = shouldExecuteChildStep({ + parentSteps: manyParentSteps, + stepInfos, + }); + + expect(result).toBe(false); + }); + + it('should return false with large number of parents when not all completed', () => { + const manyParentSteps = Array.from({ length: 25 }, (_, i) => ({ + id: `parent-${i + 1}`, + type: WorkflowActionType.CODE, + settings: { + errorHandlingOptions: { + continueOnFailure: { value: false }, + retryOnFailure: { value: false }, + }, + outputSchema: {}, + }, + nextStepIds: [], + })) as unknown as WorkflowAction[]; + + // First parent succeeded, rest are still running + const stepInfos = Object.fromEntries( + manyParentSteps.map((step, i) => [ + step.id, + { status: i === 0 ? StepStatus.SUCCESS : StepStatus.RUNNING }, + ]), + ); + + const result = shouldExecuteChildStep({ + parentSteps: manyParentSteps, + stepInfos, + }); + + expect(result).toBe(false); + }); +}); diff --git a/packages/twenty-server/src/modules/workflow/workflow-executor/utils/should-execute-child-step.util.ts b/packages/twenty-server/src/modules/workflow/workflow-executor/utils/should-execute-child-step.util.ts new file mode 100644 index 0000000000..718835bc44 --- /dev/null +++ b/packages/twenty-server/src/modules/workflow/workflow-executor/utils/should-execute-child-step.util.ts @@ -0,0 +1,27 @@ +import { StepStatus, type WorkflowRunStepInfos } from 'twenty-shared/workflow'; + +import { type WorkflowAction } from 'src/modules/workflow/workflow-executor/workflow-actions/types/workflow-action.type'; + +export const shouldExecuteChildStep = ({ + parentSteps, + stepInfos, +}: { + parentSteps: WorkflowAction[]; + stepInfos: WorkflowRunStepInfos; +}) => { + if (parentSteps.length === 0) { + return true; + } + const hasSuccessfulParentStep = parentSteps.some( + (parentStep) => stepInfos[parentStep.id]?.status === StepStatus.SUCCESS, + ); + + const areAllParentsCompleted = parentSteps.every( + (parentStep) => + stepInfos[parentStep.id]?.status === StepStatus.SUCCESS || + stepInfos[parentStep.id]?.status === StepStatus.STOPPED || + stepInfos[parentStep.id]?.status === StepStatus.SKIPPED, + ); + + return hasSuccessfulParentStep && areAllParentsCompleted; +}; diff --git a/packages/twenty-server/src/modules/workflow/workflow-executor/utils/should-execute-step.util.ts b/packages/twenty-server/src/modules/workflow/workflow-executor/utils/should-execute-step.util.ts index cf5ce656c8..a5ddd90d25 100644 --- a/packages/twenty-server/src/modules/workflow/workflow-executor/utils/should-execute-step.util.ts +++ b/packages/twenty-server/src/modules/workflow/workflow-executor/utils/should-execute-step.util.ts @@ -1,7 +1,8 @@ import { isDefined } from 'twenty-shared/utils'; -import { StepStatus, type WorkflowRunStepInfos } from 'twenty-shared/workflow'; +import { type WorkflowRunStepInfos } from 'twenty-shared/workflow'; import { WorkflowRunStatus } from 'src/modules/workflow/common/standard-objects/workflow-run.workspace-entity'; +import { shouldExecuteChildStep } from 'src/modules/workflow/workflow-executor/utils/should-execute-child-step.util'; import { stepHasBeenStarted } from 'src/modules/workflow/workflow-executor/utils/step-has-been-started.util'; import { isWorkflowIteratorAction } from 'src/modules/workflow/workflow-executor/workflow-actions/iterator/guards/is-workflow-iterator-action.guard'; import { shouldExecuteIteratorStep } from 'src/modules/workflow/workflow-executor/workflow-actions/iterator/utils/should-execute-iterator-step.util'; @@ -39,20 +40,8 @@ export const shouldExecuteStep = ({ isDefined(parentStep) && parentStep.nextStepIds?.includes(step.id), ); - if (parentSteps.length === 0) { - return true; - } - - const hasSuccessfulParentStep = parentSteps.some( - (parentStep) => stepInfos[parentStep.id]?.status === StepStatus.SUCCESS, - ); - - const areAllParentsCompleted = parentSteps.every( - (parentStep) => - stepInfos[parentStep.id]?.status === StepStatus.SUCCESS || - stepInfos[parentStep.id]?.status === StepStatus.STOPPED || - stepInfos[parentStep.id]?.status === StepStatus.SKIPPED, - ); - - return hasSuccessfulParentStep && areAllParentsCompleted; + return shouldExecuteChildStep({ + parentSteps, + stepInfos, + }); }; diff --git a/packages/twenty-server/src/modules/workflow/workflow-executor/workflow-actions/iterator/utils/should-execute-iterator-step.util.ts b/packages/twenty-server/src/modules/workflow/workflow-executor/workflow-actions/iterator/utils/should-execute-iterator-step.util.ts index 2206562d29..bb12b3603b 100644 --- a/packages/twenty-server/src/modules/workflow/workflow-executor/workflow-actions/iterator/utils/should-execute-iterator-step.util.ts +++ b/packages/twenty-server/src/modules/workflow/workflow-executor/workflow-actions/iterator/utils/should-execute-iterator-step.util.ts @@ -1,6 +1,7 @@ import { isDefined } from 'twenty-shared/utils'; -import { StepStatus, type WorkflowRunStepInfos } from 'twenty-shared/workflow'; +import { type WorkflowRunStepInfos } from 'twenty-shared/workflow'; +import { shouldExecuteChildStep } from 'src/modules/workflow/workflow-executor/utils/should-execute-child-step.util'; import { stepHasBeenStarted } from 'src/modules/workflow/workflow-executor/utils/step-has-been-started.util'; import { getAllStepIdsInLoop } from 'src/modules/workflow/workflow-executor/workflow-actions/iterator/utils/get-all-step-ids-in-loop.util'; import { @@ -40,20 +41,8 @@ export const shouldExecuteIteratorStep = ({ ? stepsTargetingIterator : parentSteps; - if (stepsToCheck.length === 0) { - return true; - } - - const hasSuccessfulParentStep = stepsToCheck.some( - (parentStep) => stepInfos[parentStep.id]?.status === StepStatus.SUCCESS, - ); - - const hasFailedNorNotStartedOrRunningParentStep = stepsToCheck.some( - (parentStep) => - stepInfos[parentStep.id]?.status === StepStatus.FAILED || - stepInfos[parentStep.id]?.status === StepStatus.NOT_STARTED || - stepInfos[parentStep.id]?.status === StepStatus.RUNNING, - ); - - return hasSuccessfulParentStep && !hasFailedNorNotStartedOrRunningParentStep; + return shouldExecuteChildStep({ + parentSteps: stepsToCheck, + stepInfos, + }); }; diff --git a/packages/twenty-server/src/modules/workflow/workflow-executor/workflow-executor.module.ts b/packages/twenty-server/src/modules/workflow/workflow-executor/workflow-executor.module.ts index f47b9efc39..33ae01a4fc 100644 --- a/packages/twenty-server/src/modules/workflow/workflow-executor/workflow-executor.module.ts +++ b/packages/twenty-server/src/modules/workflow/workflow-executor/workflow-executor.module.ts @@ -15,6 +15,7 @@ import { IteratorActionModule } from 'src/modules/workflow/workflow-executor/wor import { RecordCRUDActionModule } from 'src/modules/workflow/workflow-executor/workflow-actions/record-crud/record-crud-action.module'; import { ToolExecutorWorkflowAction } from 'src/modules/workflow/workflow-executor/workflow-actions/tool-executor-workflow-action'; import { WorkflowExecutorWorkspaceService } from 'src/modules/workflow/workflow-executor/workspace-services/workflow-executor.workspace-service'; +import { WorkflowRunQueueModule } from 'src/modules/workflow/workflow-runner/workflow-run-queue/workflow-run-queue.module'; import { WorkflowRunModule } from 'src/modules/workflow/workflow-runner/workflow-run/workflow-run.module'; @Module({ @@ -30,6 +31,7 @@ import { WorkflowRunModule } from 'src/modules/workflow/workflow-runner/workflow AiAgentActionModule, EmptyActionModule, FeatureFlagModule, + WorkflowRunQueueModule, AiModule, ], providers: [ diff --git a/packages/twenty-server/src/modules/workflow/workflow-executor/workspace-services/__tests__/workflow-executor.workspace-service.spec.ts b/packages/twenty-server/src/modules/workflow/workflow-executor/workspace-services/__tests__/workflow-executor.workspace-service.spec.ts index bc57bc6be7..f1a53374be 100644 --- a/packages/twenty-server/src/modules/workflow/workflow-executor/workspace-services/__tests__/workflow-executor.workspace-service.spec.ts +++ b/packages/twenty-server/src/modules/workflow/workflow-executor/workspace-services/__tests__/workflow-executor.workspace-service.spec.ts @@ -6,6 +6,7 @@ import { BILLING_FEATURE_USED } from 'src/engine/core-modules/billing/constants/ import { BILLING_WORKFLOW_EXECUTION_ERROR_MESSAGE } from 'src/engine/core-modules/billing/constants/billing-workflow-execution-error-message.constant'; import { BillingMeterEventName } from 'src/engine/core-modules/billing/enums/billing-meter-event-names'; import { BillingService } from 'src/engine/core-modules/billing/services/billing.service'; +import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants'; import { WorkspaceEventEmitter } from 'src/engine/workspace-event-emitter/workspace-event-emitter'; import { WorkflowActionFactory } from 'src/modules/workflow/workflow-executor/factories/workflow-action.factory'; import { shouldExecuteStep } from 'src/modules/workflow/workflow-executor/utils/should-execute-step.util'; @@ -14,6 +15,7 @@ import { WorkflowActionType, } from 'src/modules/workflow/workflow-executor/workflow-actions/types/workflow-action.type'; import { WorkflowExecutorWorkspaceService } from 'src/modules/workflow/workflow-executor/workspace-services/workflow-executor.workspace-service'; +import { WorkflowRunQueueWorkspaceService } from 'src/modules/workflow/workflow-runner/workflow-run-queue/workspace-services/workflow-run-queue.workspace-service'; import { WorkflowRunWorkspaceService } from 'src/modules/workflow/workflow-runner/workflow-run/workflow-run.workspace-service'; jest.mock( @@ -55,6 +57,14 @@ describe('WorkflowExecutorWorkspaceService', () => { canBillMeteredProduct: jest.fn().mockReturnValue(true), }; + const mockMessageQueueService = { + add: jest.fn(), + }; + + const mockWorkflowRunQueueWorkspaceService = { + increaseWorkflowRunQueuedCount: jest.fn(), + }; + beforeEach(async () => { jest.clearAllMocks(); @@ -79,6 +89,14 @@ describe('WorkflowExecutorWorkspaceService', () => { provide: BillingService, useValue: mockBillingService, }, + { + provide: `MESSAGE_QUEUE_${MessageQueue.workflowQueue}`, + useValue: mockMessageQueueService, + }, + { + provide: WorkflowRunQueueWorkspaceService, + useValue: mockWorkflowRunQueueWorkspaceService, + }, ], }).compile(); @@ -335,6 +353,40 @@ describe('WorkflowExecutorWorkspaceService', () => { expect(workflowActionFactory.get).not.toHaveBeenCalled(); }); + + it('should queue another job when max executed step count is reached', async () => { + const mockStepResult = { + result: { stepOutput: 'success' }, + }; + + mockWorkflowExecutor.execute.mockResolvedValueOnce(mockStepResult); + + await service.executeFromSteps({ + workflowRunId: mockWorkflowRunId, + stepIds: ['step-1'], + workspaceId: mockWorkspaceId, + executedStepsCount: 21, // exceeds MAX_EXECUTED_STEPS_COUNT (20) + }); + + expect(mockMessageQueueService.add).toHaveBeenCalledWith( + 'RunWorkflowJob', + { + workspaceId: mockWorkspaceId, + workflowRunId: mockWorkflowRunId, + lastExecutedStepId: 'step-1', + }, + ); + + expect( + mockWorkflowRunQueueWorkspaceService.increaseWorkflowRunQueuedCount, + ).toHaveBeenCalledWith(mockWorkspaceId); + + // Should not execute the next step (step-2) in the same job + expect(workflowActionFactory.get).toHaveBeenCalledTimes(1); + expect(workflowActionFactory.get).toHaveBeenCalledWith( + WorkflowActionType.CODE, + ); + }); }); describe('sendWorkflowNodeRunEvent', () => { diff --git a/packages/twenty-server/src/modules/workflow/workflow-executor/workspace-services/workflow-executor.workspace-service.ts b/packages/twenty-server/src/modules/workflow/workflow-executor/workspace-services/workflow-executor.workspace-service.ts index 4c28da3f4c..086f1ca043 100644 --- a/packages/twenty-server/src/modules/workflow/workflow-executor/workspace-services/workflow-executor.workspace-service.ts +++ b/packages/twenty-server/src/modules/workflow/workflow-executor/workspace-services/workflow-executor.workspace-service.ts @@ -15,6 +15,9 @@ import { BillingMeterEventName } from 'src/engine/core-modules/billing/enums/bil import { BillingProductKey } from 'src/engine/core-modules/billing/enums/billing-product-key.enum'; import { BillingService } from 'src/engine/core-modules/billing/services/billing.service'; import { type BillingUsageEvent } from 'src/engine/core-modules/billing/types/billing-usage-event.type'; +import { InjectMessageQueue } from 'src/engine/core-modules/message-queue/decorators/message-queue.decorator'; +import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants'; +import { MessageQueueService } from 'src/engine/core-modules/message-queue/services/message-queue.service'; import { WorkspaceEventEmitter } from 'src/engine/workspace-event-emitter/workspace-event-emitter'; import { WorkflowRunStatus } from 'src/modules/workflow/common/standard-objects/workflow-run.workspace-entity'; import { WorkflowActionFactory } from 'src/modules/workflow/workflow-executor/factories/workflow-action.factory'; @@ -30,8 +33,13 @@ import { workflowShouldKeepRunning } from 'src/modules/workflow/workflow-executo import { isWorkflowIteratorAction } from 'src/modules/workflow/workflow-executor/workflow-actions/iterator/guards/is-workflow-iterator-action.guard'; import { WorkflowIteratorResult } from 'src/modules/workflow/workflow-executor/workflow-actions/iterator/types/workflow-iterator-result.type'; import { WorkflowAction } from 'src/modules/workflow/workflow-executor/workflow-actions/types/workflow-action.type'; +import { RUN_WORKFLOW_JOB_NAME } from 'src/modules/workflow/workflow-runner/constants/run-workflow-job-name'; +import { type RunWorkflowJobData } from 'src/modules/workflow/workflow-runner/types/run-workflow-job-data.type'; +import { WorkflowRunQueueWorkspaceService } from 'src/modules/workflow/workflow-runner/workflow-run-queue/workspace-services/workflow-run-queue.workspace-service'; import { WorkflowRunWorkspaceService } from 'src/modules/workflow/workflow-runner/workflow-run/workflow-run.workspace-service'; +const MAX_EXECUTED_STEPS_COUNT = 20; + @Injectable() export class WorkflowExecutorWorkspaceService { constructor( @@ -39,6 +47,9 @@ export class WorkflowExecutorWorkspaceService { private readonly workspaceEventEmitter: WorkspaceEventEmitter, private readonly workflowRunWorkspaceService: WorkflowRunWorkspaceService, private readonly billingService: BillingService, + @InjectMessageQueue(MessageQueue.workflowQueue) + private readonly messageQueueService: MessageQueueService, + private readonly workflowRunQueueWorkspaceService: WorkflowRunQueueWorkspaceService, ) {} async executeFromSteps({ @@ -46,6 +57,7 @@ export class WorkflowExecutorWorkspaceService { workflowRunId, workspaceId, shouldComputeWorkflowRunStatus = true, + executedStepsCount = 0, }: WorkflowExecutorInput) { await Promise.all( stepIds.map(async (stepIdToExecute) => { @@ -53,6 +65,7 @@ export class WorkflowExecutorWorkspaceService { stepId: stepIdToExecute, workflowRunId, workspaceId, + executedStepsCount, }); }), ); @@ -69,6 +82,7 @@ export class WorkflowExecutorWorkspaceService { stepId, workflowRunId, workspaceId, + executedStepsCount, }: WorkflowBranchExecutorInput) { const workflowRun = await this.workflowRunWorkspaceService.getWorkflowRunOrFail({ @@ -137,21 +151,60 @@ export class WorkflowExecutorWorkspaceService { workspaceId, }); - if (shouldProcessNextSteps) { - const nextStepIdsToExecute = await this.getNextStepIdsToExecute({ - executedStep: stepToExecute, - executedStepResult: actionOutput, + if (!shouldProcessNextSteps) { + return; + } + + const shouldRunAnotherJob = + executedStepsCount && executedStepsCount > MAX_EXECUTED_STEPS_COUNT; + + if (shouldRunAnotherJob) { + await this.continueExecutionFromStepInAnotherJob({ + lastExecutedStepId: stepId, + workflowRunId, + workspaceId, }); - if (isDefined(nextStepIdsToExecute) && nextStepIdsToExecute.length > 0) { - await this.executeFromSteps({ - stepIds: nextStepIdsToExecute, - workflowRunId, - workspaceId, - shouldComputeWorkflowRunStatus: false, - }); + return; + } + + const nextStepIdsToExecute = await this.getNextStepIdsToExecute({ + executedStep: stepToExecute, + executedStepResult: actionOutput, + }); + + if (isDefined(nextStepIdsToExecute) && nextStepIdsToExecute.length > 0) { + await this.executeFromSteps({ + stepIds: nextStepIdsToExecute, + workflowRunId, + workspaceId, + shouldComputeWorkflowRunStatus: false, + executedStepsCount: (executedStepsCount ?? 0) + 1, + }); + } + } + + async getNextStepIdsToExecute({ + executedStep, + executedStepResult, + }: { + executedStep: WorkflowAction; + executedStepResult: WorkflowActionOutput; + }): Promise { + const isIteratorStep = isWorkflowIteratorAction(executedStep); + + if (isIteratorStep) { + const iteratorStepResult = + executedStepResult.result as WorkflowIteratorResult; + + if (!iteratorStepResult.hasProcessedAllItems) { + return isString(executedStep.settings.input.initialLoopStepIds) + ? JSON.parse(executedStep.settings.input.initialLoopStepIds) + : executedStep.settings.input.initialLoopStepIds; } } + + return executedStep.nextStepIds; } private async computeWorkflowRunStatus({ @@ -216,29 +269,6 @@ export class WorkflowExecutorWorkspaceService { ); } - private async getNextStepIdsToExecute({ - executedStep, - executedStepResult, - }: { - executedStep: WorkflowAction; - executedStepResult: WorkflowActionOutput; - }) { - const isIteratorStep = isWorkflowIteratorAction(executedStep); - - if (isIteratorStep) { - const iteratorStepResult = - executedStepResult.result as WorkflowIteratorResult; - - if (!iteratorStepResult.hasProcessedAllItems) { - return isString(executedStep.settings.input.initialLoopStepIds) - ? JSON.parse(executedStep.settings.input.initialLoopStepIds) - : executedStep.settings.input.initialLoopStepIds; - } - } - - return executedStep.nextStepIds; - } - private async processStepExecutionResult({ actionOutput, stepId, @@ -351,4 +381,26 @@ export class WorkflowExecutorWorkspaceService { }; } } + + private async continueExecutionFromStepInAnotherJob({ + lastExecutedStepId, + workflowRunId, + workspaceId, + }: { + lastExecutedStepId: string; + workflowRunId: string; + workspaceId: string; + }) { + await this.messageQueueService.add( + RUN_WORKFLOW_JOB_NAME, + { + workspaceId, + workflowRunId, + lastExecutedStepId, + }, + ); + await this.workflowRunQueueWorkspaceService.increaseWorkflowRunQueuedCount( + workspaceId, + ); + } } diff --git a/packages/twenty-server/src/modules/workflow/workflow-runner/constants/run-workflow-job-name.ts b/packages/twenty-server/src/modules/workflow/workflow-runner/constants/run-workflow-job-name.ts new file mode 100644 index 0000000000..a550df3f8a --- /dev/null +++ b/packages/twenty-server/src/modules/workflow/workflow-runner/constants/run-workflow-job-name.ts @@ -0,0 +1 @@ +export const RUN_WORKFLOW_JOB_NAME = 'RunWorkflowJob'; diff --git a/packages/twenty-server/src/modules/workflow/workflow-runner/jobs/run-workflow.job.ts b/packages/twenty-server/src/modules/workflow/workflow-runner/jobs/run-workflow.job.ts index 26778836bd..4d917fabaf 100644 --- a/packages/twenty-server/src/modules/workflow/workflow-runner/jobs/run-workflow.job.ts +++ b/packages/twenty-server/src/modules/workflow/workflow-runner/jobs/run-workflow.job.ts @@ -12,20 +12,16 @@ import { TwentyConfigService } from 'src/engine/core-modules/twenty-config/twent import { WorkflowRunStatus } from 'src/modules/workflow/common/standard-objects/workflow-run.workspace-entity'; import { WorkflowCommonWorkspaceService } from 'src/modules/workflow/common/workspace-services/workflow-common.workspace-service'; import { WorkflowExecutorWorkspaceService } from 'src/modules/workflow/workflow-executor/workspace-services/workflow-executor.workspace-service'; +import { RUN_WORKFLOW_JOB_NAME } from 'src/modules/workflow/workflow-runner/constants/run-workflow-job-name'; import { WorkflowRunException, WorkflowRunExceptionCode, } from 'src/modules/workflow/workflow-runner/exceptions/workflow-run.exception'; +import { type RunWorkflowJobData } from 'src/modules/workflow/workflow-runner/types/run-workflow-job-data.type'; import { WorkflowRunQueueWorkspaceService } from 'src/modules/workflow/workflow-runner/workflow-run-queue/workspace-services/workflow-run-queue.workspace-service'; import { WorkflowRunWorkspaceService } from 'src/modules/workflow/workflow-runner/workflow-run/workflow-run.workspace-service'; import { WorkflowTriggerType } from 'src/modules/workflow/workflow-trigger/types/workflow-trigger.type'; -export type RunWorkflowJobData = { - workspaceId: string; - workflowRunId: string; - lastExecutedStepId?: string; -}; - @Processor({ queueName: MessageQueue.workflowQueue, scope: Scope.REQUEST }) export class RunWorkflowJob { constructor( @@ -38,7 +34,7 @@ export class RunWorkflowJob { private readonly workflowRunQueueWorkspaceService: WorkflowRunQueueWorkspaceService, ) {} - @Process(RunWorkflowJob.name) + @Process(RUN_WORKFLOW_JOB_NAME) async handle({ workflowRunId, lastExecutedStepId, @@ -148,10 +144,16 @@ export class RunWorkflowJob { ); } - if ( - !isDefined(lastExecutedStep.nextStepIds) || - lastExecutedStep.nextStepIds.length === 0 - ) { + const lastExecutedStepResult = + workflowRun.state?.stepInfos[lastExecutedStepId]?.result; + + const nextStepIdsToExecute = + await this.workflowExecutorWorkspaceService.getNextStepIdsToExecute({ + executedStep: lastExecutedStep, + executedStepResult: lastExecutedStepResult, + }); + + if (!isDefined(nextStepIdsToExecute) || nextStepIdsToExecute.length === 0) { await this.workflowRunWorkspaceService.endWorkflowRun({ workflowRunId, workspaceId, @@ -162,7 +164,7 @@ export class RunWorkflowJob { } await this.workflowExecutorWorkspaceService.executeFromSteps({ - stepIds: lastExecutedStep.nextStepIds, + stepIds: nextStepIdsToExecute, workflowRunId, workspaceId, }); diff --git a/packages/twenty-server/src/modules/workflow/workflow-runner/types/run-workflow-job-data.type.ts b/packages/twenty-server/src/modules/workflow/workflow-runner/types/run-workflow-job-data.type.ts new file mode 100644 index 0000000000..38884f55e5 --- /dev/null +++ b/packages/twenty-server/src/modules/workflow/workflow-runner/types/run-workflow-job-data.type.ts @@ -0,0 +1,5 @@ +export type RunWorkflowJobData = { + workspaceId: string; + workflowRunId: string; + lastExecutedStepId?: string; +}; diff --git a/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run-queue/workspace-services/workflow-run-enqueue.workspace-service.ts b/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run-queue/workspace-services/workflow-run-enqueue.workspace-service.ts index 550acb6fb5..9b28d9819d 100644 --- a/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run-queue/workspace-services/workflow-run-enqueue.workspace-service.ts +++ b/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run-queue/workspace-services/workflow-run-enqueue.workspace-service.ts @@ -10,10 +10,8 @@ import { WorkflowRunStatus, WorkflowRunWorkspaceEntity, } from 'src/modules/workflow/common/standard-objects/workflow-run.workspace-entity'; -import { - RunWorkflowJob, - RunWorkflowJobData, -} from 'src/modules/workflow/workflow-runner/jobs/run-workflow.job'; +import { RunWorkflowJob } from 'src/modules/workflow/workflow-runner/jobs/run-workflow.job'; +import { type RunWorkflowJobData } from 'src/modules/workflow/workflow-runner/types/run-workflow-job-data.type'; import { WorkflowRunQueueWorkspaceService } from 'src/modules/workflow/workflow-runner/workflow-run-queue/workspace-services/workflow-run-queue.workspace-service'; @Injectable() diff --git a/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run/workflow-run.workspace-service.ts b/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run/workflow-run.workspace-service.ts index ffc3ca240c..8e700965e1 100644 --- a/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run/workflow-run.workspace-service.ts +++ b/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run/workflow-run.workspace-service.ts @@ -198,41 +198,22 @@ export class WorkflowRunWorkspaceService { workspaceId, }); - const updatedStepInfos = Object.entries( - workflowRunToUpdate.state?.stepInfos ?? {}, - ) - .map(([stepId, step]) => { - if ( - step.status === StepStatus.RUNNING || - step.status === StepStatus.PENDING - ) { - return { - [stepId]: { - ...step, - status: StepStatus.FAILED, - error: 'Workflow has been ended before this step was completed', - }, - }; - } + let updatedStepInfos = {}; + const shouldUpdateStepInfos = status === WorkflowRunStatus.FAILED; - return { - [stepId]: step, - }; - }) - .reduce((acc, current) => { - return { - ...acc, - ...current, - }; - }, {}); + if (shouldUpdateStepInfos) { + updatedStepInfos = this.markRunningStepsAsFailed({ + stepInfosToUpdate: workflowRunToUpdate.state?.stepInfos ?? {}, + }); + } const partialUpdate = { status, endedAt: new Date().toISOString(), state: { ...workflowRunToUpdate.state, - stepInfos: updatedStepInfos, workflowRunError: error, + ...(shouldUpdateStepInfos && { stepInfos: updatedStepInfos }), }, }; @@ -459,4 +440,36 @@ export class WorkflowRunWorkspaceService { ['id'], ); } + + private markRunningStepsAsFailed({ + stepInfosToUpdate, + }: { + stepInfosToUpdate: Record; + }) { + return Object.entries(stepInfosToUpdate ?? {}) + .map(([stepId, step]) => { + if ( + step.status === StepStatus.RUNNING || + step.status === StepStatus.PENDING + ) { + return { + [stepId]: { + ...step, + status: StepStatus.FAILED, + error: 'Workflow has been ended before this step was completed', + }, + }; + } + + return { + [stepId]: step, + }; + }) + .reduce((acc, current) => { + return { + ...acc, + ...current, + }; + }, {}); + } } diff --git a/packages/twenty-server/src/modules/workflow/workflow-runner/workspace-services/workflow-runner.workspace-service.ts b/packages/twenty-server/src/modules/workflow/workflow-runner/workspace-services/workflow-runner.workspace-service.ts index b0629bae93..be529c550a 100644 --- a/packages/twenty-server/src/modules/workflow/workflow-runner/workspace-services/workflow-runner.workspace-service.ts +++ b/packages/twenty-server/src/modules/workflow/workflow-runner/workspace-services/workflow-runner.workspace-service.ts @@ -17,10 +17,8 @@ import { WorkflowRunStatus } from 'src/modules/workflow/common/standard-objects/ import { WorkflowCommonWorkspaceService } from 'src/modules/workflow/common/workspace-services/workflow-common.workspace-service'; import { WorkflowVersionStepOperationsWorkspaceService } from 'src/modules/workflow/workflow-builder/workflow-version-step/workflow-version-step-operations.workspace-service'; import { isWorkflowFormAction } from 'src/modules/workflow/workflow-executor/workflow-actions/form/guards/is-workflow-form-action.guard'; -import { - RunWorkflowJob, - RunWorkflowJobData, -} from 'src/modules/workflow/workflow-runner/jobs/run-workflow.job'; +import { RunWorkflowJob } from 'src/modules/workflow/workflow-runner/jobs/run-workflow.job'; +import { type RunWorkflowJobData } from 'src/modules/workflow/workflow-runner/types/run-workflow-job-data.type'; import { WorkflowRunQueueWorkspaceService } from 'src/modules/workflow/workflow-runner/workflow-run-queue/workspace-services/workflow-run-queue.workspace-service'; import { WorkflowRunWorkspaceService } from 'src/modules/workflow/workflow-runner/workflow-run/workflow-run.workspace-service'; import { WorkflowTriggerType } from 'src/modules/workflow/workflow-trigger/types/workflow-trigger.type'; @@ -176,7 +174,7 @@ export class WorkflowRunnerWorkspaceService { }); } - private async enqueueWorkflowRun( + async enqueueWorkflowRun( workspaceId: string, workflowRunId: string, lastExecutedStepId?: string,