58907b733c
## Summary
### Why
1. Sending the code to the lambda (~1Mb usually) is heavy on network and
results to a constant traffic of ~30Mb/s on AWS which results into TB of
network data every month
2. eval(1MB of code) is not that fast, it's heavy on memory and CPU on
lambda side
### High level
Adds two execution modes for logic functions, gated behind the new
`IS_LOGIC_FUNCTION_PREBUILT_MODE_ENABLED` workspace feature flag (off
everywhere by default):
- **LIVE** (current behavior, preserved bit-for-bit): the compiled
bundle is read from object storage and shipped in every Lambda invoke
payload. Used for fast iteration in the workflow editor / Settings test
runs.
- **PREBUILT** (new): the bundle is installed onto the per-function
Lambda alongside the unified executor, and invocations carry only `{
params, env, handlerName }` — saving JSON payload egress and warm-start
`import()` cost on every call.
### Key design choices
- **Unified Lambda handler** (`constants/executor/index.mjs`) dispatches
at runtime: `event.code` present ? LIVE (write to `/tmp`, dynamic
import) : `import('./prebuilt-logic-function.mjs')`. Both code paths
always coexist on the deployment package, so the same Lambda can serve
either mode without redeploying.
- **Install runs inside the `validateBuildAndRun` migration pipeline**,
not at execute time. `Create/UpdateLogicFunctionActionHandlerService`
calls `driver.installPrebuiltBundle` when `executionMode` flips
LIVE?PREBUILT or `checksum` changes while PREBUILT, gated on
`isBuildUpToDate=true` and a fresh checksum.
- **Strict execute, no reconciliation**:
`LogicFunctionExecutorService.execute` resolves `effectiveExecutionMode`
(caller override > feature flag > entity column). For PREBUILT it asks
the driver `getInstalledBundleChecksum` (Lambda `twenty:bundle-checksum`
tag for AWS, sidecar file locally) and throws
`LOGIC_FUNCTION_PREBUILT_BUNDLE_NOT_INSTALLED` on mismatch.
- **Feature flag gates every side effect**: with the flag off the
executor forces LIVE, the action-handler install hooks bail before AWS,
and workflow activation does not flip the mode. Rollback is just turning
the flag off.
### Lifecycle
- New workflow CODE step ? `LIVE`, no install.
- Workflow activated ? build + activation flips `executionMode=PREBUILT`
? action-handler installs the bundle + sets the Lambda tag.
- Draft from active version ? duplicated logic function reset to `LIVE`.
- App install ? manifest converter sets `PREBUILT`, create-action
handler installs.
- Test runs (`executeOneFromSource`, workflow editor) pass
`executionMode=LIVE` explicitly.
### Observability
`[lambda-timing]` log lines now include `effectiveExecutionMode` and
`payloadBytes`; the action handler logs `install_duration_ms` for each
install.
## Test plan
- [x] `npx nx typecheck twenty-server` ? passes
- [x] `npx oxlint --type-aware` on all changed files ? 0 warnings, 0
errors
- [x] `npx nx test twenty-server` ? 588 suites / 5009 tests pass (no
regressions vs main)
- [x] New unit suite `flat-logic-function-validator.service.spec.ts` ?
9/9
- [x] Existing
`workflow-version-step-operations.workspace-service.spec.ts` ? 8/8
(verified the new token-based DI avoids a circular-import regression)
- [x] Snapshot for
`ALL_UNIVERSAL_FLAT_ENTITY_PROPERTIES_TO_COMPARE_AND_STRINGIFY` updated
to include `executionMode`
- [x] Integration suite `logic-function-execution.integration-spec.ts`
extended to assert `executionMode=LIVE` on newly-created functions and
continues to exercise the LIVE happy path
- [ ] Manual staging rollout: flip
`IS_LOGIC_FUNCTION_PREBUILT_MODE_ENABLED` per workspace, observe
`[lambda-timing]` `payloadBytes` drop + `install_duration_ms`, then ramp
in prod.
299 lines
9.3 KiB
TypeScript
299 lines
9.3 KiB
TypeScript
import { createOneLogicFunction } from 'test/integration/metadata/suites/logic-function/utils/create-logic-function.util';
|
|
import { deleteLogicFunction } from 'test/integration/metadata/suites/logic-function/utils/delete-logic-function.util';
|
|
import { executeLogicFunction } from 'test/integration/metadata/suites/logic-function/utils/execute-logic-function.util';
|
|
import { updateLogicFunctionSource } from 'test/integration/metadata/suites/logic-function/utils/update-logic-function-source.util';
|
|
|
|
import { LogicFunctionExecutionStatus } from 'src/engine/metadata-modules/logic-function/dtos/logic-function-execution-result.dto';
|
|
import { LogicFunctionExecutionMode } from 'src/engine/metadata-modules/logic-function/logic-function.entity';
|
|
|
|
// Default template function code that matches the expected behavior
|
|
const DEFAULT_TEMPLATE_FUNCTION_CODE = `export const main = async (params: { a: string; b: number }): Promise<object> => {
|
|
return { message: \`Toto: \${params.a} and \${params.b}\` };
|
|
};`;
|
|
|
|
// Test function using external packages from default layer (lodash.groupby)
|
|
const EXTERNAL_PACKAGES_FUNCTION_CODE = `import groupBy from 'lodash.groupby';
|
|
|
|
export const main = async (params: { items: Array<{ category: string; name: string }> }): Promise<object> => {
|
|
const grouped = groupBy(params.items, 'category');
|
|
return {
|
|
grouped,
|
|
categories: Object.keys(grouped),
|
|
};
|
|
};`;
|
|
|
|
// Test function that throws an error
|
|
const ERROR_FUNCTION_CODE = `export const main = async (params: { shouldFail: boolean }): Promise<object> => {
|
|
if (params.shouldFail) {
|
|
throw new Error('Intentional test error');
|
|
}
|
|
return { success: true };
|
|
};`;
|
|
|
|
describe('Logic Function Execution', () => {
|
|
const createdFunctionIds: string[] = [];
|
|
|
|
afterAll(async () => {
|
|
// Clean up all created functions
|
|
for (const functionId of createdFunctionIds) {
|
|
try {
|
|
await deleteLogicFunction({
|
|
input: { id: functionId },
|
|
expectToFail: false,
|
|
});
|
|
} catch {
|
|
// Ignore cleanup errors
|
|
}
|
|
}
|
|
});
|
|
|
|
it('should execute the default logic function template', async () => {
|
|
// Create the function with default template code
|
|
const { data: createData } = await createOneLogicFunction({
|
|
input: {
|
|
name: 'Test Default Function',
|
|
},
|
|
expectToFail: false,
|
|
});
|
|
|
|
const functionId = createData?.createOneLogicFunction?.id;
|
|
|
|
expect(functionId).toBeDefined();
|
|
expect(createData?.createOneLogicFunction?.executionMode).toBe(
|
|
LogicFunctionExecutionMode.LIVE,
|
|
);
|
|
createdFunctionIds.push(functionId);
|
|
|
|
await updateLogicFunctionSource({
|
|
input: {
|
|
id: createData.createOneLogicFunction.id,
|
|
update: {
|
|
sourceHandlerCode: DEFAULT_TEMPLATE_FUNCTION_CODE,
|
|
},
|
|
},
|
|
expectToFail: false,
|
|
});
|
|
|
|
// Execute with the default template's expected params: { a: string, b: number }
|
|
const { data: executeData } = await executeLogicFunction({
|
|
input: {
|
|
id: functionId,
|
|
payload: { a: 'hello', b: 42 },
|
|
},
|
|
expectToFail: false,
|
|
});
|
|
|
|
const result = executeData?.executeOneLogicFunction;
|
|
|
|
if (result?.status !== LogicFunctionExecutionStatus.SUCCESS) {
|
|
throw new Error(JSON.stringify(result?.error, null, 2));
|
|
}
|
|
|
|
expect(result?.status).toBe(LogicFunctionExecutionStatus.SUCCESS);
|
|
expect(result?.data).toMatchObject({
|
|
message: 'Toto: hello and 42',
|
|
});
|
|
expect(result?.duration).toBeGreaterThan(0);
|
|
});
|
|
|
|
it('should execute a function with external packages (lodash.groupby)', async () => {
|
|
// Create the function with the external packages code
|
|
const { data: createData } = await createOneLogicFunction({
|
|
input: {
|
|
name: 'External Packages Test',
|
|
},
|
|
expectToFail: false,
|
|
});
|
|
|
|
const functionId = createData?.createOneLogicFunction?.id;
|
|
|
|
expect(functionId).toBeDefined();
|
|
createdFunctionIds.push(functionId);
|
|
|
|
await updateLogicFunctionSource({
|
|
input: {
|
|
id: createData.createOneLogicFunction.id,
|
|
update: {
|
|
sourceHandlerCode: EXTERNAL_PACKAGES_FUNCTION_CODE,
|
|
},
|
|
},
|
|
expectToFail: false,
|
|
});
|
|
|
|
// Execute the function with items to group
|
|
const { data: executeData } = await executeLogicFunction({
|
|
input: {
|
|
id: functionId,
|
|
payload: {
|
|
items: [
|
|
{ category: 'fruit', name: 'apple' },
|
|
{ category: 'vegetable', name: 'carrot' },
|
|
{ category: 'fruit', name: 'banana' },
|
|
],
|
|
},
|
|
},
|
|
expectToFail: false,
|
|
});
|
|
|
|
const result = executeData?.executeOneLogicFunction;
|
|
|
|
if (result?.status !== LogicFunctionExecutionStatus.SUCCESS) {
|
|
throw new Error(JSON.stringify(result?.error, null, 2));
|
|
}
|
|
|
|
expect(result?.status).toBe(LogicFunctionExecutionStatus.SUCCESS);
|
|
|
|
const data = result?.data as unknown as {
|
|
grouped: Record<string, Array<{ category: string; name: string }>>;
|
|
categories: string[];
|
|
};
|
|
|
|
expect(data?.grouped).toMatchObject({
|
|
fruit: [
|
|
{ category: 'fruit', name: 'apple' },
|
|
{ category: 'fruit', name: 'banana' },
|
|
],
|
|
vegetable: [{ category: 'vegetable', name: 'carrot' }],
|
|
});
|
|
expect(data?.categories).toEqual(
|
|
expect.arrayContaining(['fruit', 'vegetable']),
|
|
);
|
|
});
|
|
|
|
it('should create logic function without source', async () => {
|
|
// Create the function with default template code
|
|
const { data: createData } = await createOneLogicFunction({
|
|
input: {
|
|
name: 'Test Default Function',
|
|
},
|
|
expectToFail: false,
|
|
});
|
|
|
|
const functionId = createData?.createOneLogicFunction?.id;
|
|
|
|
expect(functionId).toBeDefined();
|
|
createdFunctionIds.push(functionId);
|
|
|
|
// Execute with the default template's expected params: { a: string, b: number }
|
|
const { data: executeData } = await executeLogicFunction({
|
|
input: {
|
|
id: functionId,
|
|
payload: { a: 'hello', b: 42 },
|
|
},
|
|
expectToFail: false,
|
|
});
|
|
|
|
const result = executeData?.executeOneLogicFunction;
|
|
|
|
if (result?.status !== LogicFunctionExecutionStatus.SUCCESS) {
|
|
throw new Error(JSON.stringify(result?.error, null, 2));
|
|
}
|
|
|
|
expect(result?.status).toBe(LogicFunctionExecutionStatus.SUCCESS);
|
|
expect(result?.data).toMatchObject({
|
|
message: 'Hello, input: hello and 42',
|
|
});
|
|
expect(result?.duration).toBeGreaterThan(0);
|
|
});
|
|
|
|
it('should create logic function with source', async () => {
|
|
// Create the function with default template code
|
|
const { data: createData } = await createOneLogicFunction({
|
|
input: {
|
|
name: 'Test Default Function',
|
|
source: {
|
|
sourceHandlerCode: DEFAULT_TEMPLATE_FUNCTION_CODE,
|
|
handlerName: 'main',
|
|
},
|
|
},
|
|
expectToFail: false,
|
|
});
|
|
|
|
const functionId = createData?.createOneLogicFunction?.id;
|
|
|
|
expect(functionId).toBeDefined();
|
|
createdFunctionIds.push(functionId);
|
|
|
|
// Execute with the default template's expected params: { a: string, b: number }
|
|
const { data: executeData } = await executeLogicFunction({
|
|
input: {
|
|
id: functionId,
|
|
payload: { a: 'hello', b: 42 },
|
|
},
|
|
expectToFail: false,
|
|
});
|
|
|
|
const result = executeData?.executeOneLogicFunction;
|
|
|
|
if (result?.status !== LogicFunctionExecutionStatus.SUCCESS) {
|
|
throw new Error(JSON.stringify(result?.error, null, 2));
|
|
}
|
|
|
|
expect(result?.status).toBe(LogicFunctionExecutionStatus.SUCCESS);
|
|
expect(result?.data).toMatchObject({
|
|
message: 'Toto: hello and 42',
|
|
});
|
|
expect(result?.duration).toBeGreaterThan(0);
|
|
});
|
|
|
|
it('should handle errors thrown by logic functions', async () => {
|
|
// Create the function with error-throwing code
|
|
const { data: createData } = await createOneLogicFunction({
|
|
input: {
|
|
name: 'Error Test Function',
|
|
},
|
|
expectToFail: false,
|
|
});
|
|
|
|
const functionId = createData?.createOneLogicFunction?.id;
|
|
|
|
expect(functionId).toBeDefined();
|
|
createdFunctionIds.push(functionId);
|
|
|
|
await updateLogicFunctionSource({
|
|
input: {
|
|
id: createData.createOneLogicFunction.id,
|
|
update: {
|
|
sourceHandlerCode: ERROR_FUNCTION_CODE,
|
|
},
|
|
},
|
|
expectToFail: false,
|
|
});
|
|
|
|
// Execute with shouldFail = false (should succeed)
|
|
const { data: successData } = await executeLogicFunction({
|
|
input: {
|
|
id: functionId,
|
|
payload: { shouldFail: false },
|
|
},
|
|
expectToFail: false,
|
|
});
|
|
|
|
expect(successData?.executeOneLogicFunction?.status).toBe(
|
|
LogicFunctionExecutionStatus.SUCCESS,
|
|
);
|
|
expect(successData?.executeOneLogicFunction?.data).toMatchObject({
|
|
success: true,
|
|
});
|
|
|
|
// Execute with shouldFail = true (should return error status)
|
|
const { data: errorData } = await executeLogicFunction({
|
|
input: {
|
|
id: functionId,
|
|
payload: { shouldFail: true },
|
|
},
|
|
expectToFail: false, // The GraphQL call succeeds, but the function execution fails
|
|
});
|
|
|
|
const errorResult = errorData?.executeOneLogicFunction;
|
|
|
|
expect(errorResult?.status).toBe(LogicFunctionExecutionStatus.ERROR);
|
|
expect(errorResult?.error).toMatchObject({
|
|
errorType: 'UnhandledError',
|
|
errorMessage: expect.stringContaining('Intentional test error'),
|
|
});
|
|
expect(errorResult?.data).toBeNull();
|
|
});
|
|
|
|
});
|