Improve workflow perfs (#18376)

Workflow crons take a few minutes to run. Loading each repo takes ~200
to 300ms locally. Adding a lite mode so it takes less than 100ms.
Also doing batch promises.

Finally, cleaning runs timeout when there are too many. Doing batches as
well.

packages/twenty-server/src/modules/workflow/workflow-executor/workflow-actions/iterator/__tests__/iterator-action.workflow-action.spec.ts

@@ -192,9 +192,9 @@ describe('IteratorWorkflowAction', () => {
         },
       } as any;
 
-      workflowRunWorkspaceService.getWorkflowRunOrFail
-        .mockResolvedValueOnce(mockStepInfo)
-        .mockResolvedValueOnce(mockStepInfo);
+      workflowRunWorkspaceService.getWorkflowRunOrFail.mockResolvedValueOnce(
+        mockStepInfo,
+      );
 
       const result = await service.execute(input);
 
@@ -243,9 +243,9 @@ describe('IteratorWorkflowAction', () => {
         },
       } as any;
 
-      workflowRunWorkspaceService.getWorkflowRunOrFail
-        .mockResolvedValueOnce(mockStepInfo)
-        .mockResolvedValueOnce(mockStepInfo);
+      workflowRunWorkspaceService.getWorkflowRunOrFail.mockResolvedValueOnce(
+        mockStepInfo,
+      );
 
       const result = await service.execute(input);
 

packages/twenty-server/src/modules/workflow/workflow-executor/workflow-actions/iterator/iterator.workflow-action.ts

@@ -113,6 +113,7 @@ export class IteratorWorkflowAction implements WorkflowActionInterface {
         workflowRunId: runInfo.workflowRunId,
         workspaceId: runInfo.workspaceId,
         steps,
+        stepInfos,
       });
     }
 
@@ -129,6 +130,7 @@ export class IteratorWorkflowAction implements WorkflowActionInterface {
     workflowRunId,
     workspaceId,
     steps,
+    stepInfos,
   }: {
     iteratorStepId: string;
     initialLoopStepIds: string[];
@@ -136,17 +138,10 @@ export class IteratorWorkflowAction implements WorkflowActionInterface {
     workflowRunId: string;
     workspaceId: string;
     steps: WorkflowAction[];
+    stepInfos: Record<string, WorkflowRunStepInfo>;
   }) {
     let stepInfosToUpdate: Record<string, WorkflowRunStepInfo> = {};
 
-    const workflowRunToUpdate =
-      await this.workflowRunWorkspaceService.getWorkflowRunOrFail({
-        workflowRunId,
-        workspaceId,
-      });
-
-    const stepInfos = workflowRunToUpdate.state.stepInfos;
-
     if (!hasProcessedAllItems) {
       const subStepsInfos = await this.buildSubStepInfosReset({
         iteratorStepId,

packages/twenty-server/src/modules/workflow/workflow-executor/workspace-services/workflow-executor.workspace-service.ts

@@ -490,7 +490,7 @@ export class WorkflowExecutorWorkspaceService {
     }
   }
 
-  private async skipAndFailSafelyStepsThenContinue({
+  async skipAndFailSafelyStepsThenContinue({
     stepIdsToSkip,
     stepIdsToFailSafely,
     steps,
@@ -505,39 +505,41 @@ export class WorkflowExecutorWorkspaceService {
     workspaceId: string;
     executedStepsCount: number;
   }) {
-    const stepsToSkip = stepIdsToSkip.map((stepId) => ({
-      stepId,
-      status: StepStatus.SKIPPED,
-    }));
-    const stepsToFailSafely = stepIdsToFailSafely.map((stepId) => ({
-      stepId,
-      status: StepStatus.FAILED_SAFELY,
-    }));
-    const stepsToProcess = [...stepsToSkip, ...stepsToFailSafely];
+    const stepInfos: Record<string, WorkflowRunStepInfo> = {};
 
-    await Promise.all(
-      stepsToProcess.map(async ({ stepId, status }) => {
-        await this.workflowRunWorkspaceService.updateWorkflowRunStepInfo({
-          stepId,
-          stepInfo: { status },
-          workflowRunId,
-          workspaceId,
-        });
+    for (const stepId of stepIdsToSkip) {
+      stepInfos[stepId] = { status: StepStatus.SKIPPED };
+    }
 
-        const step = steps.find((step) => step.id === stepId);
-        const stepNextStepIds = step?.nextStepIds ?? [];
+    for (const stepId of stepIdsToFailSafely) {
+      stepInfos[stepId] = { status: StepStatus.FAILED_SAFELY };
+    }
 
-        if (stepNextStepIds.length > 0) {
-          await this.executeFromSteps({
-            stepIds: stepNextStepIds,
-            workflowRunId,
-            workspaceId,
-            shouldComputeWorkflowRunStatus: false,
-            executedStepsCount,
-          });
-        }
-      }),
-    );
+    await this.workflowRunWorkspaceService.updateWorkflowRunStepInfos({
+      stepInfos,
+      workflowRunId,
+      workspaceId,
+    });
+
+    const nextStepIds = new Set<string>();
+
+    for (const stepId of [...stepIdsToSkip, ...stepIdsToFailSafely]) {
+      const step = steps.find((step) => step.id === stepId);
+
+      for (const nextStepId of step?.nextStepIds ?? []) {
+        nextStepIds.add(nextStepId);
+      }
+    }
+
+    if (nextStepIds.size > 0) {
+      await this.executeFromSteps({
+        stepIds: Array.from(nextStepIds),
+        workflowRunId,
+        workspaceId,
+        shouldComputeWorkflowRunStatus: false,
+        executedStepsCount,
+      });
+    }
   }
 
   private async continueExecutionFromStepInAnotherJob({