houseassassin/twenty
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

22 branches data migration (#13006)

Browse Source 
This PR does not produce any functional change

First step of the workflow branch feature

- add gather `workflowRun.output` and `workflowRun.context` into one
column `workflowRun.runContext`
- add a command to fill `runContext` from `output` and `context` in
existing records
- maintain `runContext` up to date during workflow runs
This commit is contained in:
martmull
2025-07-03 10:18:33 +02:00
committed by GitHub
parent 29dbd3fc25
commit 51cb35a27a
11 changed files with 387 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/twenty-server/src/modules/workflow/common/standard-objects/workflow-run.workspace-entity.ts
+20
View File
@@ -31,6 +31,7 @@ import { WorkflowWorkspaceEntity } from 'src/modules/workflow/common/standard-ob
import { WorkflowExecutorOutput } from 'src/modules/workflow/workflow-executor/types/workflow-executor-output.type';
import { WorkflowAction } from 'src/modules/workflow/workflow-executor/workflow-actions/types/workflow-action.type';
import { WorkflowTrigger } from 'src/modules/workflow/workflow-trigger/types/workflow-trigger.type';
import { WorkflowRunStepInfo } from 'src/modules/workflow/workflow-executor/types/workflow-run-step-info.type';
export enum WorkflowRunStatus {
NOT_STARTED = 'NOT_STARTED',
@@ -54,6 +55,15 @@ export type WorkflowRunOutput = {
error?: string;
};
export type WorkflowRunState = {
flow: {
trigger: WorkflowTrigger;
steps: WorkflowAction[];
};
stepInfos: Record<string, WorkflowRunStepInfo>;
workflowRunError?: string;
};
const NAME_FIELD_NAME = 'name';
export const SEARCH_FIELDS_FOR_WORKFLOW_RUNS: FieldTypeAndNameMetadata[] = [
@@ -172,6 +182,16 @@ export class WorkflowRunWorkspaceEntity extends BaseWorkspaceEntity {
// eslint-disable-next-line @typescript-eslint/no-explicit-any
context: Record<string, any> | null;
@WorkspaceField({
standardId: WORKFLOW_RUN_STANDARD_FIELD_IDS.state,
type: FieldMetadataType.RAW_JSON,
label: msg`State`,
description: msg`State of the workflow run`,
icon: 'IconHierarchy2',
})
@WorkspaceIsNullable()
state: WorkflowRunState | null;
@WorkspaceField({
standardId: WORKFLOW_RUN_STANDARD_FIELD_IDS.position,
type: FieldMetadataType.POSITION,
packages/twenty-server/src/modules/workflow/workflow-builder/workflow-step/workflow-version-step.workspace-service.ts
+2
View File
@@ -30,6 +30,7 @@ import {
} from 'src/modules/workflow/workflow-executor/workflow-actions/types/workflow-action.type';
import { WorkflowRunWorkspaceService } from 'src/modules/workflow/workflow-runner/workflow-run/workflow-run.workspace-service';
import { WorkflowRunnerWorkspaceService } from 'src/modules/workflow/workflow-runner/workspace-services/workflow-runner.workspace-service';
import { StepStatus } from 'src/modules/workflow/workflow-executor/types/workflow-run-step-info.type';
const TRIGGER_STEP_ID = 'trigger';
@@ -334,6 +335,7 @@ export class WorkflowVersionStepWorkspaceService {
workflowRunId,
stepOutput: newStepOutput,
context: updatedContext,
stepStatus: StepStatus.SUCCESS,
});
await this.workflowRunnerWorkspaceService.resume({
packages/twenty-server/src/modules/workflow/workflow-executor/types/workflow-run-step-info.type.ts
+13
View File
@@ -0,0 +1,13 @@
export enum StepStatus {
NOT_STARTED = 'NOT_STARTED',
RUNNING = 'RUNNING',
SUCCESS = 'SUCCESS',
FAILED = 'FAILED',
PENDING = 'PENDING',
}
export type WorkflowRunStepInfo = {
result?: object;
error?: string;
status: StepStatus;
};
packages/twenty-server/src/modules/workflow/workflow-executor/workspace-services/__tests__/workflow-executor.workspace-service.spec.ts
+94 -1
View File
@@ -13,6 +13,7 @@ import {
} from 'src/modules/workflow/workflow-executor/workflow-actions/types/workflow-action.type';
import { WorkflowExecutorWorkspaceService } from 'src/modules/workflow/workflow-executor/workspace-services/workflow-executor.workspace-service';
import { WorkflowRunWorkspaceService } from 'src/modules/workflow/workflow-runner/workflow-run/workflow-run.workspace-service';
import { StepStatus } from 'src/modules/workflow/workflow-executor/types/workflow-run-step-info.type';
describe('WorkflowExecutorWorkspaceService', () => {
let service: WorkflowExecutorWorkspaceService;
@@ -169,9 +170,27 @@ describe('WorkflowExecutorWorkspaceService', () => {
],
'workspace-id',
);
expect(
workflowRunWorkspaceService.saveWorkflowRunState,
).toHaveBeenCalledTimes(4);
expect(
workflowRunWorkspaceService.saveWorkflowRunState,
).toHaveBeenCalledWith({
workflowRunId: mockWorkflowRunId,
stepOutput: {
id: 'step-1',
output: {},
},
context: {
data: 'some-data',
},
workspaceId: 'workspace-id',
stepStatus: StepStatus.RUNNING,
});
expect(
workflowRunWorkspaceService.saveWorkflowRunState,
).toHaveBeenNthCalledWith(2, {
workflowRunId: mockWorkflowRunId,
stepOutput: {
id: 'step-1',
@@ -179,10 +198,14 @@ describe('WorkflowExecutorWorkspaceService', () => {
},
context: {
data: 'some-data',
'step-1': { stepOutput: 'success' },
'step-1': {
stepOutput: 'success',
},
},
workspaceId: 'workspace-id',
stepStatus: StepStatus.SUCCESS,
});
expect(result).toEqual({ result: { success: true } });
// execute second step
@@ -207,9 +230,24 @@ describe('WorkflowExecutorWorkspaceService', () => {
error: 'Step execution failed',
});
expect(workspaceEventEmitter.emitCustomBatchEvent).not.toHaveBeenCalled();
expect(
workflowRunWorkspaceService.saveWorkflowRunState,
).toHaveBeenCalledTimes(2);
expect(
workflowRunWorkspaceService.saveWorkflowRunState,
).toHaveBeenCalledWith({
workflowRunId: mockWorkflowRunId,
stepOutput: {
id: 'step-1',
output: {},
},
context: mockContext,
workspaceId: 'workspace-id',
stepStatus: StepStatus.RUNNING,
});
expect(
workflowRunWorkspaceService.saveWorkflowRunState,
).toHaveBeenNthCalledWith(2, {
workflowRunId: mockWorkflowRunId,
stepOutput: {
id: 'step-1',
@@ -219,6 +257,7 @@ describe('WorkflowExecutorWorkspaceService', () => {
},
context: mockContext,
workspaceId: 'workspace-id',
stepStatus: StepStatus.FAILED,
});
});
@@ -237,9 +276,24 @@ describe('WorkflowExecutorWorkspaceService', () => {
});
expect(result).toEqual(mockPendingEvent);
expect(
workflowRunWorkspaceService.saveWorkflowRunState,
).toHaveBeenCalledTimes(2);
expect(
workflowRunWorkspaceService.saveWorkflowRunState,
).toHaveBeenCalledWith({
workflowRunId: mockWorkflowRunId,
stepOutput: {
id: 'step-1',
output: {},
},
context: mockContext,
workspaceId: 'workspace-id',
stepStatus: StepStatus.RUNNING,
});
expect(
workflowRunWorkspaceService.saveWorkflowRunState,
).toHaveBeenNthCalledWith(2, {
workflowRunId: mockWorkflowRunId,
stepOutput: {
id: 'step-1',
@@ -247,6 +301,7 @@ describe('WorkflowExecutorWorkspaceService', () => {
},
context: mockContext,
workspaceId: 'workspace-id',
stepStatus: StepStatus.PENDING,
});
// No recursive call to execute should happen
@@ -291,10 +346,27 @@ describe('WorkflowExecutorWorkspaceService', () => {
context: mockContext,
});
expect(
workflowRunWorkspaceService.saveWorkflowRunState,
).toHaveBeenCalledTimes(4);
// execute first step
expect(
workflowRunWorkspaceService.saveWorkflowRunState,
).toHaveBeenCalledWith({
workflowRunId: mockWorkflowRunId,
stepOutput: {
id: 'step-1',
output: {},
},
context: mockContext,
workspaceId: 'workspace-id',
stepStatus: StepStatus.RUNNING,
});
expect(
workflowRunWorkspaceService.saveWorkflowRunState,
).toHaveBeenNthCalledWith(2, {
workflowRunId: mockWorkflowRunId,
stepOutput: {
id: 'step-1',
@@ -304,6 +376,7 @@ describe('WorkflowExecutorWorkspaceService', () => {
},
context: mockContext,
workspaceId: 'workspace-id',
stepStatus: StepStatus.FAILED,
});
expect(result).toEqual({ result: { success: true } });
@@ -378,9 +451,24 @@ describe('WorkflowExecutorWorkspaceService', () => {
// Should not retry anymore
expect(workflowExecutorFactory.get).toHaveBeenCalledTimes(1);
expect(
workflowRunWorkspaceService.saveWorkflowRunState,
).toHaveBeenCalledTimes(2);
expect(
workflowRunWorkspaceService.saveWorkflowRunState,
).toHaveBeenCalledWith({
workflowRunId: mockWorkflowRunId,
stepOutput: {
id: 'step-1',
output: {},
},
context: mockContext,
workspaceId: 'workspace-id',
stepStatus: StepStatus.RUNNING,
});
expect(
workflowRunWorkspaceService.saveWorkflowRunState,
).toHaveBeenNthCalledWith(2, {
workflowRunId: mockWorkflowRunId,
stepOutput: {
id: 'step-1',
@@ -388,6 +476,7 @@ describe('WorkflowExecutorWorkspaceService', () => {
},
context: mockContext,
workspaceId: 'workspace-id',
stepStatus: StepStatus.FAILED,
});
expect(result).toEqual(errorOutput);
});
@@ -404,6 +493,9 @@ describe('WorkflowExecutorWorkspaceService', () => {
});
expect(workflowExecutorFactory.get).toHaveBeenCalledTimes(1);
expect(
workflowRunWorkspaceService.saveWorkflowRunState,
).toHaveBeenCalledTimes(1);
expect(
workflowRunWorkspaceService.saveWorkflowRunState,
).toHaveBeenCalledWith({
@@ -416,6 +508,7 @@ describe('WorkflowExecutorWorkspaceService', () => {
},
context: mockContext,
workspaceId: 'workspace-id',
stepStatus: StepStatus.FAILED,
});
expect(result).toEqual({
error: BILLING_WORKFLOW_EXECUTION_ERROR_MESSAGE,
packages/twenty-server/src/modules/workflow/workflow-executor/workspace-services/workflow-executor.workspace-service.ts
+21 -1
View File
@@ -25,6 +25,7 @@ import {
WorkflowTriggerException,
WorkflowTriggerExceptionCode,
} from 'src/modules/workflow/workflow-trigger/exceptions/workflow-trigger.exception';
import { StepStatus } from 'src/modules/workflow/workflow-executor/types/workflow-run-step-info.type';
const MAX_RETRIES_ON_FAILURE = 3;
@@ -87,11 +88,23 @@ export class WorkflowExecutorWorkspaceService implements WorkflowExecutor {
output: billingOutput,
},
context,
stepStatus: StepStatus.FAILED,
});
return billingOutput;
}
await this.workflowRunWorkspaceService.saveWorkflowRunState({
workflowRunId,
stepOutput: {
id: step.id,
output: {},
},
context,
workspaceId,
stepStatus: StepStatus.RUNNING,
});
try {
actionOutput = await workflowExecutor.execute({
currentStepId,
@@ -121,13 +134,16 @@ export class WorkflowExecutorWorkspaceService implements WorkflowExecutor {
stepOutput,
context,
workspaceId,
stepStatus: StepStatus.PENDING,
});
return actionOutput;
}
const actionOutputSuccess = isDefined(actionOutput.result);
const shouldContinue =
isDefined(actionOutput.result) ||
actionOutputSuccess ||
step.settings.errorHandlingOptions.continueOnFailure.value;
if (shouldContinue) {
@@ -143,6 +159,9 @@ export class WorkflowExecutorWorkspaceService implements WorkflowExecutor {
stepOutput,
context: updatedContext,
workspaceId,
stepStatus: isDefined(actionOutput.result)
? StepStatus.SUCCESS
: StepStatus.FAILED,
});
if (!isDefined(step.nextStepIds?.[0])) {
@@ -176,6 +195,7 @@ export class WorkflowExecutorWorkspaceService implements WorkflowExecutor {
stepOutput,
context,
workspaceId,
stepStatus: StepStatus.FAILED,
});
return actionOutput;
packages/twenty-server/src/modules/workflow/workflow-runner/jobs/run-workflow.job.ts
+1
View File
@@ -118,6 +118,7 @@ export class RunWorkflowJob {
},
},
},
payload: triggerPayload,
});
await this.throttleExecution(workflowVersion.workflowId);
packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run/workflow-run.workspace-service.ts
+72 -3
View File
@@ -3,6 +3,7 @@ import { InjectRepository } from '@nestjs/typeorm';
import { Repository } from 'typeorm';
import { v4 } from 'uuid';
import { isDefined } from 'twenty-shared/utils';
import { DatabaseEventAction } from 'src/engine/api/graphql/graphql-query-runner/enums/database-event-action';
import { objectRecordChangedValues } from 'src/engine/core-modules/event-emitter/utils/object-record-changed-values';
@@ -16,6 +17,7 @@ import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.
import { WorkspaceEventEmitter } from 'src/engine/workspace-event-emitter/workspace-event-emitter';
import {
StepOutput,
WorkflowRunState,
WorkflowRunOutput,
WorkflowRunStatus,
WorkflowRunWorkspaceEntity,
@@ -26,6 +28,8 @@ import {
WorkflowRunException,
WorkflowRunExceptionCode,
} from 'src/modules/workflow/workflow-runner/exceptions/workflow-run.exception';
import { StepStatus } from 'src/modules/workflow/workflow-executor/types/workflow-run-step-info.type';
import { WorkflowVersionWorkspaceEntity } from 'src/modules/workflow/common/standard-objects/workflow-version.workspace-entity';
@Injectable()
export class WorkflowRunWorkspaceService {
@@ -120,6 +124,7 @@ export class WorkflowRunWorkspaceService {
workflowId: workflow.id,
status,
position,
state: this.getInitState(workflowVersion),
context,
});
@@ -132,10 +137,12 @@ export class WorkflowRunWorkspaceService {
workflowRunId,
workspaceId,
output,
payload,
}: {
workflowRunId: string;
workspaceId: string;
output: WorkflowRunOutput;
payload: object;
}) {
const workflowRunRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace<WorkflowRunWorkspaceEntity>(
@@ -169,6 +176,17 @@ export class WorkflowRunWorkspaceService {
status: WorkflowRunStatus.RUNNING,
startedAt: new Date().toISOString(),
output,
state: {
...workflowRunToUpdate.state,
stepInfos: {
...workflowRunToUpdate.state?.stepInfos,
trigger: {
...workflowRunToUpdate.state?.stepInfos.trigger,
status: StepStatus.SUCCESS,
result: payload,
},
},
},
};
await workflowRunRepository.update(workflowRunToUpdate.id, partialUpdate);
@@ -215,13 +233,17 @@ export class WorkflowRunWorkspaceService {
...(workflowRunToUpdate.output ?? {}),
error,
},
state: {
...workflowRunToUpdate.state,
workflowRunError: error,
},
};
await workflowRunRepository.update(workflowRunToUpdate.id, partialUpdate);
await this.emitWorkflowRunUpdatedEvent({
workflowRunBefore: workflowRunToUpdate,
updatedFields: ['status', 'endedAt', 'output'],
updatedFields: ['status', 'endedAt', 'output', 'state'],
});
await this.metricsService.incrementCounter({
@@ -238,12 +260,14 @@ export class WorkflowRunWorkspaceService {
stepOutput,
workspaceId,
context,
stepStatus,
}: {
workflowRunId: string;
stepOutput: StepOutput;
workspaceId: string;
// eslint-disable-next-line @typescript-eslint/no-explicit-any
context: Record<string, any>;
stepStatus: StepStatus;
}) {
const workflowRunRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace<WorkflowRunWorkspaceEntity>(
@@ -274,6 +298,17 @@ export class WorkflowRunWorkspaceService {
[stepOutput.id]: stepOutput.output,
},
},
state: {
...workflowRunToUpdate.state,
stepInfos: {
...workflowRunToUpdate.state?.stepInfos,
[stepOutput.id]: {
result: stepOutput.output?.result,
error: stepOutput.output?.error,
status: stepStatus,
},
},
},
context,
};
@@ -281,7 +316,7 @@ export class WorkflowRunWorkspaceService {
await this.emitWorkflowRunUpdatedEvent({
workflowRunBefore: workflowRunToUpdate,
updatedFields: ['context', 'output'],
updatedFields: ['context', 'output', 'state'],
});
}
@@ -334,13 +369,20 @@ export class WorkflowRunWorkspaceService {
steps: updatedSteps,
},
},
state: {
...workflowRunToUpdate.state,
flow: {
...(workflowRunToUpdate.state?.flow ?? {}),
steps: updatedSteps,
},
},
};
await workflowRunRepository.update(workflowRunToUpdate.id, partialUpdate);
await this.emitWorkflowRunUpdatedEvent({
workflowRunBefore: workflowRunToUpdate,
updatedFields: ['output'],
updatedFields: ['output', 'state'],
});
}
@@ -441,4 +483,31 @@ export class WorkflowRunWorkspaceService {
workspaceId,
});
}
private getInitState(
workflowVersion: WorkflowVersionWorkspaceEntity,
): WorkflowRunState | undefined {
if (
!isDefined(workflowVersion.trigger) ||
!isDefined(workflowVersion.steps)
) {
return undefined;
}
return {
flow: {
trigger: workflowVersion.trigger,
steps: workflowVersion.steps,
},
stepInfos: {
trigger: { status: StepStatus.NOT_STARTED },
...Object.fromEntries(
workflowVersion.steps.map((step) => [
step.id,
{ status: StepStatus.NOT_STARTED },
]),
),
},
};
}
}