feat(logic-function): add LIVE / PREBUILT execution modes (#20873)
## Summary
### Why
1. Sending the code to the lambda (~1Mb usually) is heavy on network and
results to a constant traffic of ~30Mb/s on AWS which results into TB of
network data every month
2. eval(1MB of code) is not that fast, it's heavy on memory and CPU on
lambda side
### High level
Adds two execution modes for logic functions, gated behind the new
`IS_LOGIC_FUNCTION_PREBUILT_MODE_ENABLED` workspace feature flag (off
everywhere by default):
- **LIVE** (current behavior, preserved bit-for-bit): the compiled
bundle is read from object storage and shipped in every Lambda invoke
payload. Used for fast iteration in the workflow editor / Settings test
runs.
- **PREBUILT** (new): the bundle is installed onto the per-function
Lambda alongside the unified executor, and invocations carry only `{
params, env, handlerName }` — saving JSON payload egress and warm-start
`import()` cost on every call.
### Key design choices
- **Unified Lambda handler** (`constants/executor/index.mjs`) dispatches
at runtime: `event.code` present ? LIVE (write to `/tmp`, dynamic
import) : `import('./prebuilt-logic-function.mjs')`. Both code paths
always coexist on the deployment package, so the same Lambda can serve
either mode without redeploying.
- **Install runs inside the `validateBuildAndRun` migration pipeline**,
not at execute time. `Create/UpdateLogicFunctionActionHandlerService`
calls `driver.installPrebuiltBundle` when `executionMode` flips
LIVE?PREBUILT or `checksum` changes while PREBUILT, gated on
`isBuildUpToDate=true` and a fresh checksum.
- **Strict execute, no reconciliation**:
`LogicFunctionExecutorService.execute` resolves `effectiveExecutionMode`
(caller override > feature flag > entity column). For PREBUILT it asks
the driver `getInstalledBundleChecksum` (Lambda `twenty:bundle-checksum`
tag for AWS, sidecar file locally) and throws
`LOGIC_FUNCTION_PREBUILT_BUNDLE_NOT_INSTALLED` on mismatch.
- **Feature flag gates every side effect**: with the flag off the
executor forces LIVE, the action-handler install hooks bail before AWS,
and workflow activation does not flip the mode. Rollback is just turning
the flag off.
### Lifecycle
- New workflow CODE step ? `LIVE`, no install.
- Workflow activated ? build + activation flips `executionMode=PREBUILT`
? action-handler installs the bundle + sets the Lambda tag.
- Draft from active version ? duplicated logic function reset to `LIVE`.
- App install ? manifest converter sets `PREBUILT`, create-action
handler installs.
- Test runs (`executeOneFromSource`, workflow editor) pass
`executionMode=LIVE` explicitly.
### Observability
`[lambda-timing]` log lines now include `effectiveExecutionMode` and
`payloadBytes`; the action handler logs `install_duration_ms` for each
install.
## Test plan
- [x] `npx nx typecheck twenty-server` ? passes
- [x] `npx oxlint --type-aware` on all changed files ? 0 warnings, 0
errors
- [x] `npx nx test twenty-server` ? 588 suites / 5009 tests pass (no
regressions vs main)
- [x] New unit suite `flat-logic-function-validator.service.spec.ts` ?
9/9
- [x] Existing
`workflow-version-step-operations.workspace-service.spec.ts` ? 8/8
(verified the new token-based DI avoids a circular-import regression)
- [x] Snapshot for
`ALL_UNIVERSAL_FLAT_ENTITY_PROPERTIES_TO_COMPARE_AND_STRINGIFY` updated
to include `executionMode`
- [x] Integration suite `logic-function-execution.integration-spec.ts`
extended to assert `executionMode=LIVE` on newly-created functions and
continues to exercise the LIVE happy path
- [ ] Manual staging rollout: flip
`IS_LOGIC_FUNCTION_PREBUILT_MODE_ENABLED` per workspace, observe
`[lambda-timing]` `payloadBytes` drop + `install_duration_ms`, then ramp
in prod.
This commit is contained in:
+16
-1
@@ -1,4 +1,9 @@
|
||||
import { Field, HideField, ObjectType } from '@nestjs/graphql';
|
||||
import {
|
||||
Field,
|
||||
HideField,
|
||||
ObjectType,
|
||||
registerEnumType,
|
||||
} from '@nestjs/graphql';
|
||||
|
||||
import {
|
||||
Authorize,
|
||||
@@ -7,6 +12,7 @@ import {
|
||||
} from '@ptc-org/nestjs-query-graphql';
|
||||
import {
|
||||
IsDateString,
|
||||
IsEnum,
|
||||
IsNotEmpty,
|
||||
IsNumber,
|
||||
IsObject,
|
||||
@@ -24,6 +30,11 @@ import {
|
||||
} from 'twenty-shared/application';
|
||||
|
||||
import { UUIDScalarType } from 'src/engine/api/graphql/workspace-schema-builder/graphql-types/scalars';
|
||||
import { LogicFunctionExecutionMode } from 'src/engine/metadata-modules/logic-function/logic-function.entity';
|
||||
|
||||
registerEnumType(LogicFunctionExecutionMode, {
|
||||
name: 'LogicFunctionExecutionMode',
|
||||
});
|
||||
|
||||
@ObjectType('LogicFunction')
|
||||
@Authorize({
|
||||
@@ -59,6 +70,10 @@ export class LogicFunctionDTO {
|
||||
@Field()
|
||||
timeoutSeconds: number;
|
||||
|
||||
@IsEnum(LogicFunctionExecutionMode)
|
||||
@Field(() => LogicFunctionExecutionMode)
|
||||
executionMode: LogicFunctionExecutionMode;
|
||||
|
||||
@IsString()
|
||||
@Field()
|
||||
sourceHandlerPath: string;
|
||||
|
||||
+13
@@ -26,6 +26,11 @@ export enum LogicFunctionRuntime {
|
||||
NODE22 = 'nodejs22.x',
|
||||
}
|
||||
|
||||
export enum LogicFunctionExecutionMode {
|
||||
LIVE = 'LIVE',
|
||||
PREBUILT = 'PREBUILT',
|
||||
}
|
||||
|
||||
@Entity('logicFunction')
|
||||
@Index('IDX_LOGIC_FUNCTION_ID_DELETED_AT', ['id', 'deletedAt'])
|
||||
export class LogicFunctionEntity
|
||||
@@ -63,6 +68,14 @@ export class LogicFunctionEntity
|
||||
@Column({ nullable: false, type: 'boolean', default: true })
|
||||
isBuildUpToDate: boolean;
|
||||
|
||||
@Column({
|
||||
type: 'enum',
|
||||
enum: LogicFunctionExecutionMode,
|
||||
default: LogicFunctionExecutionMode.LIVE,
|
||||
nullable: false,
|
||||
})
|
||||
executionMode: LogicFunctionExecutionMode;
|
||||
|
||||
@Column({ nullable: true, type: 'jsonb' })
|
||||
cronTriggerSettings: JsonbProperty<CronTriggerSettings> | null;
|
||||
|
||||
|
||||
+3
@@ -19,6 +19,7 @@ export enum LogicFunctionExceptionCode {
|
||||
LOGIC_FUNCTION_DISABLED = 'LOGIC_FUNCTION_DISABLED',
|
||||
LOGIC_FUNCTION_INVALID_SEED_PROJECT = 'LOGIC_FUNCTION_INVALID_SEED_PROJECT',
|
||||
INVALID_LOGIC_FUNCTION_INPUT = 'INVALID_LOGIC_FUNCTION_INPUT',
|
||||
LOGIC_FUNCTION_PREBUILT_BUNDLE_NOT_INSTALLED = 'LOGIC_FUNCTION_PREBUILT_BUNDLE_NOT_INSTALLED',
|
||||
}
|
||||
|
||||
const getLogicFunctionExceptionUserFriendlyMessage = (
|
||||
@@ -53,6 +54,8 @@ const getLogicFunctionExceptionUserFriendlyMessage = (
|
||||
return msg`Invalid seed project configuration.`;
|
||||
case LogicFunctionExceptionCode.INVALID_LOGIC_FUNCTION_INPUT:
|
||||
return msg`Invalid logic function input.`;
|
||||
case LogicFunctionExceptionCode.LOGIC_FUNCTION_PREBUILT_BUNDLE_NOT_INSTALLED:
|
||||
return msg`Prebuilt bundle is not installed on the function. Rebuild and try again.`;
|
||||
default:
|
||||
assertUnreachable(code);
|
||||
}
|
||||
|
||||
+4
-1
@@ -46,6 +46,9 @@ import { WorkspaceMigrationModule } from 'src/engine/workspace-manager/workspace
|
||||
LogicFunctionResolver,
|
||||
WorkspaceFlatLogicFunctionMapCacheService,
|
||||
],
|
||||
exports: [LogicFunctionFromSourceService],
|
||||
exports: [
|
||||
LogicFunctionFromSourceService,
|
||||
LogicFunctionFromSourceHelperService,
|
||||
],
|
||||
})
|
||||
export class LogicFunctionModule {}
|
||||
|
||||
+3
@@ -14,6 +14,7 @@ import { findFlatEntityByIdInFlatEntityMapsOrThrow } from 'src/engine/metadata-m
|
||||
import { CreateLogicFunctionFromSourceInput } from 'src/engine/metadata-modules/logic-function/dtos/create-logic-function-from-source.input';
|
||||
import { LogicFunctionExecutionResultDTO } from 'src/engine/metadata-modules/logic-function/dtos/logic-function-execution-result.dto';
|
||||
import { LogicFunctionDTO } from 'src/engine/metadata-modules/logic-function/dtos/logic-function.dto';
|
||||
import { LogicFunctionExecutionMode } from 'src/engine/metadata-modules/logic-function/logic-function.entity';
|
||||
import {
|
||||
LogicFunctionException,
|
||||
LogicFunctionExceptionCode,
|
||||
@@ -169,6 +170,7 @@ export class LogicFunctionFromSourceService {
|
||||
timeoutSeconds: existingLogicFunction.timeoutSeconds,
|
||||
isBuildUpToDate: existingLogicFunction.isBuildUpToDate,
|
||||
checksum: existingLogicFunction.checksum,
|
||||
executionMode: LogicFunctionExecutionMode.LIVE,
|
||||
handlerName: existingLogicFunction.handlerName,
|
||||
sourceHandlerPath: toSourceHandlerPath,
|
||||
builtHandlerPath: toBuiltHandlerPath,
|
||||
@@ -365,6 +367,7 @@ export class LogicFunctionFromSourceService {
|
||||
logicFunctionId: id,
|
||||
workspaceId,
|
||||
payload,
|
||||
executionMode: LogicFunctionExecutionMode.LIVE,
|
||||
});
|
||||
|
||||
return {
|
||||
|
||||
+5
-1
@@ -1,6 +1,9 @@
|
||||
import { v4 } from 'uuid';
|
||||
|
||||
import { LogicFunctionRuntime } from 'src/engine/metadata-modules/logic-function/logic-function.entity';
|
||||
import {
|
||||
LogicFunctionExecutionMode,
|
||||
LogicFunctionRuntime,
|
||||
} from 'src/engine/metadata-modules/logic-function/logic-function.entity';
|
||||
import { type UniversalFlatLogicFunction } from 'src/engine/workspace-manager/workspace-migration/universal-flat-entity/types/universal-flat-logic-function.type';
|
||||
|
||||
type AutoGeneratedFields = 'createdAt' | 'updatedAt' | 'deletedAt' | 'runtime';
|
||||
@@ -40,6 +43,7 @@ export const buildUniversalFlatLogicFunctionToCreate = (
|
||||
timeoutSeconds: input.timeoutSeconds ?? 300,
|
||||
checksum: input.checksum ?? null,
|
||||
isBuildUpToDate: input.isBuildUpToDate,
|
||||
executionMode: input.executionMode ?? LogicFunctionExecutionMode.LIVE,
|
||||
handlerName: input.handlerName,
|
||||
sourceHandlerPath: input.sourceHandlerPath,
|
||||
builtHandlerPath: input.builtHandlerPath,
|
||||
|
||||
+5
-1
@@ -1,7 +1,10 @@
|
||||
import { trimAndRemoveDuplicatedWhitespacesFromObjectStringProperties } from 'twenty-shared/utils';
|
||||
import { v4 } from 'uuid';
|
||||
|
||||
import { LogicFunctionRuntime } from 'src/engine/metadata-modules/logic-function/logic-function.entity';
|
||||
import {
|
||||
LogicFunctionExecutionMode,
|
||||
LogicFunctionRuntime,
|
||||
} from 'src/engine/metadata-modules/logic-function/logic-function.entity';
|
||||
import { type CreateLogicFunctionFromSourceInput } from 'src/engine/metadata-modules/logic-function/dtos/create-logic-function-from-source.input';
|
||||
import { type UniversalFlatLogicFunction } from 'src/engine/workspace-manager/workspace-migration/universal-flat-entity/types/universal-flat-logic-function.type';
|
||||
|
||||
@@ -43,6 +46,7 @@ export const fromCreateLogicFunctionFromSourceInputToUniversalFlatLogicFunctionT
|
||||
timeoutSeconds: createLogicFunctionFromSourceInput.timeoutSeconds ?? 300,
|
||||
checksum,
|
||||
isBuildUpToDate,
|
||||
executionMode: LogicFunctionExecutionMode.LIVE,
|
||||
handlerName,
|
||||
sourceHandlerPath,
|
||||
builtHandlerPath,
|
||||
|
||||
+1
@@ -13,6 +13,7 @@ export const fromFlatLogicFunctionToLogicFunctionDto = ({
|
||||
description: flatLogicFunction.description ?? undefined,
|
||||
runtime: flatLogicFunction.runtime,
|
||||
timeoutSeconds: flatLogicFunction.timeoutSeconds,
|
||||
executionMode: flatLogicFunction.executionMode,
|
||||
sourceHandlerPath: flatLogicFunction.sourceHandlerPath,
|
||||
handlerName: flatLogicFunction.handlerName,
|
||||
applicationId: flatLogicFunction.applicationId ?? undefined,
|
||||
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
import { isNonEmptyString } from '@sniptt/guards';
|
||||
|
||||
import { LogicFunctionExecutionMode } from 'src/engine/metadata-modules/logic-function/logic-function.entity';
|
||||
import { type FlatLogicFunction } from 'src/engine/metadata-modules/logic-function/types/flat-logic-function.type';
|
||||
|
||||
export type LogicFunctionPrebuiltStateFields = Pick<
|
||||
FlatLogicFunction,
|
||||
'executionMode' | 'isBuildUpToDate' | 'checksum'
|
||||
>;
|
||||
|
||||
export const isLogicFunctionReadyForPrebuiltInstall = (
|
||||
flatLogicFunction: LogicFunctionPrebuiltStateFields,
|
||||
): boolean =>
|
||||
flatLogicFunction.executionMode === LogicFunctionExecutionMode.PREBUILT &&
|
||||
flatLogicFunction.isBuildUpToDate === true &&
|
||||
isNonEmptyString(flatLogicFunction.checksum);
|
||||
+1
@@ -36,6 +36,7 @@ export const logicFunctionGraphQLApiExceptionHandler = (error: any) => {
|
||||
case LogicFunctionExceptionCode.INVALID_LOGIC_FUNCTION_INPUT:
|
||||
throw new UserInputError(error);
|
||||
case LogicFunctionExceptionCode.LOGIC_FUNCTION_DISABLED:
|
||||
case LogicFunctionExceptionCode.LOGIC_FUNCTION_PREBUILT_BUNDLE_NOT_INSTALLED:
|
||||
throw new ForbiddenError(error);
|
||||
default: {
|
||||
return assertUnreachable(error.code);
|
||||
|
||||
+27
@@ -0,0 +1,27 @@
|
||||
import { LogicFunctionExecutionMode } from 'src/engine/metadata-modules/logic-function/logic-function.entity';
|
||||
import {
|
||||
isLogicFunctionReadyForPrebuiltInstall,
|
||||
type LogicFunctionPrebuiltStateFields,
|
||||
} from 'src/engine/metadata-modules/logic-function/utils/is-logic-function-ready-for-prebuilt-install.util';
|
||||
|
||||
export const shouldReinstallLogicFunctionPrebuiltBundle = ({
|
||||
existingLogicFunction,
|
||||
newLogicFunction,
|
||||
}: {
|
||||
existingLogicFunction: LogicFunctionPrebuiltStateFields;
|
||||
newLogicFunction: LogicFunctionPrebuiltStateFields;
|
||||
}): boolean => {
|
||||
if (!isLogicFunctionReadyForPrebuiltInstall(newLogicFunction)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if (
|
||||
existingLogicFunction.executionMode ===
|
||||
LogicFunctionExecutionMode.PREBUILT &&
|
||||
existingLogicFunction.checksum === newLogicFunction.checksum
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
};
|
||||
Reference in New Issue
Block a user