feat(logic-function): add LIVE / PREBUILT execution modes (#20873)

## Summary

### Why

1. Sending the code to the lambda (~1Mb usually) is heavy on network and
results to a constant traffic of ~30Mb/s on AWS which results into TB of
network data every month
2. eval(1MB of code) is not that fast, it's heavy on memory and CPU on
lambda side

### High level

Adds two execution modes for logic functions, gated behind the new
`IS_LOGIC_FUNCTION_PREBUILT_MODE_ENABLED` workspace feature flag (off
everywhere by default):

- **LIVE** (current behavior, preserved bit-for-bit): the compiled
bundle is read from object storage and shipped in every Lambda invoke
payload. Used for fast iteration in the workflow editor / Settings test
runs.
- **PREBUILT** (new): the bundle is installed onto the per-function
Lambda alongside the unified executor, and invocations carry only `{
params, env, handlerName }` — saving JSON payload egress and warm-start
`import()` cost on every call.

### Key design choices

- **Unified Lambda handler** (`constants/executor/index.mjs`) dispatches
at runtime: `event.code` present ? LIVE (write to `/tmp`, dynamic
import) : `import('./prebuilt-logic-function.mjs')`. Both code paths
always coexist on the deployment package, so the same Lambda can serve
either mode without redeploying.
- **Install runs inside the `validateBuildAndRun` migration pipeline**,
not at execute time. `Create/UpdateLogicFunctionActionHandlerService`
calls `driver.installPrebuiltBundle` when `executionMode` flips
LIVE?PREBUILT or `checksum` changes while PREBUILT, gated on
`isBuildUpToDate=true` and a fresh checksum.
- **Strict execute, no reconciliation**:
`LogicFunctionExecutorService.execute` resolves `effectiveExecutionMode`
(caller override > feature flag > entity column). For PREBUILT it asks
the driver `getInstalledBundleChecksum` (Lambda `twenty:bundle-checksum`
tag for AWS, sidecar file locally) and throws
`LOGIC_FUNCTION_PREBUILT_BUNDLE_NOT_INSTALLED` on mismatch.
- **Feature flag gates every side effect**: with the flag off the
executor forces LIVE, the action-handler install hooks bail before AWS,
and workflow activation does not flip the mode. Rollback is just turning
the flag off.

### Lifecycle

- New workflow CODE step ? `LIVE`, no install.
- Workflow activated ? build + activation flips `executionMode=PREBUILT`
? action-handler installs the bundle + sets the Lambda tag.
- Draft from active version ? duplicated logic function reset to `LIVE`.
- App install ? manifest converter sets `PREBUILT`, create-action
handler installs.
- Test runs (`executeOneFromSource`, workflow editor) pass
`executionMode=LIVE` explicitly.

### Observability

`[lambda-timing]` log lines now include `effectiveExecutionMode` and
`payloadBytes`; the action handler logs `install_duration_ms` for each
install.

## Test plan

- [x] `npx nx typecheck twenty-server` ? passes
- [x] `npx oxlint --type-aware` on all changed files ? 0 warnings, 0
errors
- [x] `npx nx test twenty-server` ? 588 suites / 5009 tests pass (no
regressions vs main)
- [x] New unit suite `flat-logic-function-validator.service.spec.ts` ?
9/9
- [x] Existing
`workflow-version-step-operations.workspace-service.spec.ts` ? 8/8
(verified the new token-based DI avoids a circular-import regression)
- [x] Snapshot for
`ALL_UNIVERSAL_FLAT_ENTITY_PROPERTIES_TO_COMPARE_AND_STRINGIFY` updated
to include `executionMode`
- [x] Integration suite `logic-function-execution.integration-spec.ts`
extended to assert `executionMode=LIVE` on newly-created functions and
continues to exercise the LIVE happy path
- [ ] Manual staging rollout: flip
`IS_LOGIC_FUNCTION_PREBUILT_MODE_ENABLED` per workspace, observe
`[lambda-timing]` `payloadBytes` drop + `install_duration_ms`, then ramp
in prod.
This commit is contained in:
Charles Bochet
2026-06-02 13:14:39 +02:00
committed by GitHub
parent 1ae00d6753
commit 58907b733c
44 changed files with 2212 additions and 951 deletions
@@ -441,6 +441,7 @@ type LogicFunction {
description: String
runtime: String!
timeoutSeconds: Float!
executionMode: LogicFunctionExecutionMode!
sourceHandlerPath: String!
handlerName: String!
cronTriggerSettings: JSON
@@ -454,6 +455,11 @@ type LogicFunction {
updatedAt: DateTime!
}
enum LogicFunctionExecutionMode {
LIVE
PREBUILT
}
type StandardOverrides {
label: String
description: String
@@ -1779,6 +1785,7 @@ enum FeatureFlagKey {
IS_EMAIL_GROUP_ENABLED
IS_JUNCTION_RELATIONS_ENABLED
IS_REST_METADATA_API_NEW_FORMAT_DIRECT
IS_LOGIC_FUNCTION_PREBUILT_MODE_ENABLED
IS_SETTINGS_DISCOVERY_HERO_ENABLED
}
@@ -326,6 +326,7 @@ export interface LogicFunction {
description?: Scalars['String']
runtime: Scalars['String']
timeoutSeconds: Scalars['Float']
executionMode: LogicFunctionExecutionMode
sourceHandlerPath: Scalars['String']
handlerName: Scalars['String']
cronTriggerSettings?: Scalars['JSON']
@@ -340,6 +341,8 @@ export interface LogicFunction {
__typename: 'LogicFunction'
}
export type LogicFunctionExecutionMode = 'LIVE' | 'PREBUILT'
export interface StandardOverrides {
label?: Scalars['String']
description?: Scalars['String']
@@ -1411,7 +1414,7 @@ export interface FeatureFlag {
__typename: 'FeatureFlag'
}
export type FeatureFlagKey = 'IS_UNIQUE_INDEXES_ENABLED' | 'IS_JSON_FILTER_ENABLED' | 'IS_MARKETPLACE_SETTING_TAB_VISIBLE' | 'IS_PUBLIC_DOMAIN_ENABLED' | 'IS_EMAIL_GROUP_ENABLED' | 'IS_JUNCTION_RELATIONS_ENABLED' | 'IS_REST_METADATA_API_NEW_FORMAT_DIRECT' | 'IS_SETTINGS_DISCOVERY_HERO_ENABLED'
export type FeatureFlagKey = 'IS_UNIQUE_INDEXES_ENABLED' | 'IS_JSON_FILTER_ENABLED' | 'IS_MARKETPLACE_SETTING_TAB_VISIBLE' | 'IS_PUBLIC_DOMAIN_ENABLED' | 'IS_EMAIL_GROUP_ENABLED' | 'IS_JUNCTION_RELATIONS_ENABLED' | 'IS_REST_METADATA_API_NEW_FORMAT_DIRECT' | 'IS_LOGIC_FUNCTION_PREBUILT_MODE_ENABLED' | 'IS_SETTINGS_DISCOVERY_HERO_ENABLED'
export interface WorkspaceUrls {
customUrl?: Scalars['String']
@@ -3204,6 +3207,7 @@ export interface LogicFunctionGenqlSelection{
description?: boolean | number
runtime?: boolean | number
timeoutSeconds?: boolean | number
executionMode?: boolean | number
sourceHandlerPath?: boolean | number
handlerName?: boolean | number
cronTriggerSettings?: boolean | number
@@ -8422,6 +8426,11 @@ export const enumCommandMenuItemAvailabilityType = {
FALLBACK: 'FALLBACK' as const
}
export const enumLogicFunctionExecutionMode = {
LIVE: 'LIVE' as const,
PREBUILT: 'PREBUILT' as const
}
export const enumFieldMetadataType = {
ACTOR: 'ACTOR' as const,
ADDRESS: 'ADDRESS' as const,
@@ -8733,6 +8742,7 @@ export const enumFeatureFlagKey = {
IS_EMAIL_GROUP_ENABLED: 'IS_EMAIL_GROUP_ENABLED' as const,
IS_JUNCTION_RELATIONS_ENABLED: 'IS_JUNCTION_RELATIONS_ENABLED' as const,
IS_REST_METADATA_API_NEW_FORMAT_DIRECT: 'IS_REST_METADATA_API_NEW_FORMAT_DIRECT' as const,
IS_LOGIC_FUNCTION_PREBUILT_MODE_ENABLED: 'IS_LOGIC_FUNCTION_PREBUILT_MODE_ENABLED' as const,
IS_SETTINGS_DISCOVERY_HERO_ENABLED: 'IS_SETTINGS_DISCOVERY_HERO_ENABLED' as const
}
File diff suppressed because it is too large Load Diff
@@ -286,6 +286,7 @@ export enum FeatureFlagKey {
IS_EMAIL_GROUP_ENABLED = 'IS_EMAIL_GROUP_ENABLED',
IS_JSON_FILTER_ENABLED = 'IS_JSON_FILTER_ENABLED',
IS_JUNCTION_RELATIONS_ENABLED = 'IS_JUNCTION_RELATIONS_ENABLED',
IS_LOGIC_FUNCTION_PREBUILT_MODE_ENABLED = 'IS_LOGIC_FUNCTION_PREBUILT_MODE_ENABLED',
IS_MARKETPLACE_SETTING_TAB_VISIBLE = 'IS_MARKETPLACE_SETTING_TAB_VISIBLE',
IS_PUBLIC_DOMAIN_ENABLED = 'IS_PUBLIC_DOMAIN_ENABLED',
IS_REST_METADATA_API_NEW_FORMAT_DIRECT = 'IS_REST_METADATA_API_NEW_FORMAT_DIRECT',
File diff suppressed because one or more lines are too long
@@ -7,6 +7,7 @@ export const LOGIC_FUNCTION_FRAGMENT = gql`
description
runtime
timeoutSeconds
executionMode
sourceHandlerPath
handlerName
cronTriggerSettings
@@ -0,0 +1,25 @@
import { type QueryRunner } from 'typeorm';
import { RegisteredInstanceCommand } from 'src/engine/core-modules/upgrade/decorators/registered-instance-command.decorator';
import { type FastInstanceCommand } from 'src/engine/core-modules/upgrade/interfaces/fast-instance-command.interface';
@RegisteredInstanceCommand('2.9.0', 1799000030000)
export class AddLogicFunctionExecutionModeFastInstanceCommand implements FastInstanceCommand {
public async up(queryRunner: QueryRunner): Promise<void> {
await queryRunner.query(
`CREATE TYPE "core"."logicFunction_executionmode_enum" AS ENUM('LIVE', 'PREBUILT')`,
);
await queryRunner.query(
`ALTER TABLE "core"."logicFunction" ADD "executionMode" "core"."logicFunction_executionmode_enum" NOT NULL DEFAULT 'LIVE'`,
);
}
public async down(queryRunner: QueryRunner): Promise<void> {
await queryRunner.query(
`ALTER TABLE "core"."logicFunction" DROP COLUMN "executionMode"`,
);
await queryRunner.query(
`DROP TYPE "core"."logicFunction_executionmode_enum"`,
);
}
}
@@ -56,6 +56,7 @@ import { EncryptConnectionParametersSlowInstanceCommand } from 'src/database/com
import { AddSubFieldNameToIndexFieldMetadataFastInstanceCommand } from 'src/database/commands/upgrade-version-command/2-8/2-8-instance-command-fast-1798200000000-add-sub-field-name-to-index-field-metadata';
import { DropFieldMetadataIsUniqueColumnFastInstanceCommand } from 'src/database/commands/upgrade-version-command/2-8/2-8-instance-command-fast-1798300000000-drop-field-metadata-is-unique-column';
import { EmailingDomainTenantStatusAndGlobalUniquenessFastInstanceCommand } from 'src/database/commands/upgrade-version-command/2-9/2-9-instance-command-fast-1799000020000-emailing-domain-tenant-status-and-global-uniqueness';
import { AddLogicFunctionExecutionModeFastInstanceCommand } from 'src/database/commands/upgrade-version-command/2-9/2-9-instance-command-fast-1799000030000-add-logic-function-execution-mode';
import { EncryptNonSecretApplicationVariableSlowInstanceCommand } from 'src/database/commands/upgrade-version-command/2-9/2-9-instance-command-slow-1798400000000-encrypt-non-secret-application-variable';
import { MigrateAiModelPreferencesSlowInstanceCommand } from 'src/database/commands/upgrade-version-command/2-9/2-9-instance-command-slow-1799000010000-migrate-ai-model-preferences';
@@ -116,6 +117,7 @@ export const INSTANCE_COMMANDS = [
FinalizeRolePermissionFlagCutoverFastInstanceCommand,
AddSubFieldNameToIndexFieldMetadataFastInstanceCommand,
DropFieldMetadataIsUniqueColumnFastInstanceCommand,
AddLogicFunctionExecutionModeFastInstanceCommand,
MigrateAiModelPreferencesSlowInstanceCommand,
EncryptNonSecretApplicationVariableSlowInstanceCommand,
];
@@ -2,7 +2,10 @@ import { parse } from 'path';
import { type LogicFunctionManifest } from 'twenty-shared/application';
import { LogicFunctionRuntime } from 'src/engine/metadata-modules/logic-function/logic-function.entity';
import {
LogicFunctionExecutionMode,
LogicFunctionRuntime,
} from 'src/engine/metadata-modules/logic-function/logic-function.entity';
import { type UniversalFlatLogicFunction } from 'src/engine/workspace-manager/workspace-migration/universal-flat-entity/types/universal-flat-logic-function.type';
export const fromLogicFunctionManifestToUniversalFlatLogicFunction = ({
@@ -37,6 +40,7 @@ export const fromLogicFunctionManifestToUniversalFlatLogicFunction = ({
workflowActionTriggerSettings:
logicFunctionManifest.workflowActionTriggerSettings ?? null,
isBuildUpToDate: true,
executionMode: LogicFunctionExecutionMode.PREBUILT,
createdAt: now,
updatedAt: now,
deletedAt: null,
@@ -4,6 +4,29 @@ import { createHash } from 'crypto';
export const handler = async (event) => {
const { code, params, env, handlerName } = event;
// oxlint-disable-next-line no-undef
const oldProcessEnv = { ...process.env };
try {
// oxlint-disable-next-line no-undef
process.env = { ...process.env, ...(env ?? {}) };
const mainFile = code
? await loadFromPayload(code)
: await import('./prebuilt-logic-function.mjs');
const handlerFn = handlerName
.split('.')
.reduce((obj, key) => obj[key], mainFile);
return await handlerFn(params);
} finally {
// oxlint-disable-next-line no-undef
process.env = oldProcessEnv;
}
};
const loadFromPayload = async (code) => {
// Use a content-hash filename so identical code is written once per
// warm container and Node's ESM module cache can short-circuit subsequent
// `import(mainPath)` calls. With random names, every warm invocation
@@ -23,31 +46,13 @@ export const handler = async (event) => {
const codeHash = createHash('sha256').update(code).digest('hex');
const mainPath = `/tmp/${codeHash}.mjs`;
// oxlint-disable-next-line no-undef
const oldProcessEnv = { ...process.env };
try {
try {
await fs.access(mainPath);
} catch {
await fs.writeFile(mainPath, code, 'utf8');
}
// oxlint-disable-next-line no-undef
process.env = { ...process.env, ...(env ?? {}) };
const mainFile = await import(mainPath);
const handlerFn = handlerName
.split('.')
.reduce((obj, key) => obj[key], mainFile);
return await handlerFn(params);
} finally {
// Intentionally NOT removing `mainPath` here so subsequent warm
// invocations hit Node's ESM cache (see comment above).
// oxlint-disable-next-line no-undef
process.env = oldProcessEnv;
await fs.access(mainPath);
} catch {
await fs.writeFile(mainPath, code, 'utf8');
}
// Intentionally NOT removing `mainPath` after import so subsequent warm
// invocations hit Node's ESM cache (see comment above).
return import(mainPath);
};
@@ -0,0 +1,2 @@
export const LOGIC_FUNCTION_DRIVER_FACTORY_TOKEN =
'LOGIC_FUNCTION_DRIVER_FACTORY';
@@ -27,4 +27,15 @@ export class DisabledDriver implements LogicFunctionDriver {
LogicFunctionExceptionCode.LOGIC_FUNCTION_DISABLED,
);
}
async installPrebuiltBundle(): Promise<void> {
throw new LogicFunctionException(
'Logic function prebuilt install is disabled. Set LOGIC_FUNCTION_TYPE to LOCAL or LAMBDA to enable.',
LogicFunctionExceptionCode.LOGIC_FUNCTION_DISABLED,
);
}
async getInstalledBundleChecksum(): Promise<string | null> {
return null;
}
}
@@ -18,6 +18,8 @@ import {
LogType,
PublishLayerVersionCommand,
ResourceNotFoundException,
TagResourceCommand,
UpdateFunctionCodeCommand,
UpdateFunctionConfigurationCommand,
waitUntilFunctionActiveV2,
waitUntilFunctionUpdatedV2,
@@ -32,6 +34,7 @@ import {
type LogicFunctionDriver,
type LogicFunctionExecuteParams,
type LogicFunctionExecuteResult,
type LogicFunctionInstallPrebuiltBundleParams,
type LogicFunctionTranspileParams,
type LogicFunctionTranspileResult,
} from 'src/engine/core-modules/logic-function/logic-function-drivers/interfaces/logic-function-driver.interface';
@@ -50,12 +53,16 @@ import { TemporaryDirManager } from 'src/engine/core-modules/logic-function/logi
import { type LogicFunctionResourceService } from 'src/engine/core-modules/logic-function/logic-function-resource/logic-function-resource.service';
import { type SdkClientArchiveService } from 'src/engine/core-modules/sdk-client/sdk-client-archive.service';
import { LogicFunctionExecutionStatus } from 'src/engine/metadata-modules/logic-function/dtos/logic-function-execution-result.dto';
import { LogicFunctionRuntime } from 'src/engine/metadata-modules/logic-function/logic-function.entity';
import {
LogicFunctionExecutionMode,
LogicFunctionRuntime,
} from 'src/engine/metadata-modules/logic-function/logic-function.entity';
import {
LogicFunctionException,
LogicFunctionExceptionCode,
} from 'src/engine/metadata-modules/logic-function/logic-function.exception';
import { type FlatLogicFunction } from 'src/engine/metadata-modules/logic-function/types/flat-logic-function.type';
import { isLogicFunctionReadyForPrebuiltInstall } from 'src/engine/metadata-modules/logic-function/utils/is-logic-function-ready-for-prebuilt-install.util';
enum LambdaExecutionPhase {
BUILD = 'build',
@@ -88,8 +95,16 @@ const BUILDER_HANDLER_PATH = resolve(
),
);
const LAMBDA_PREBUILT_BUNDLE_CHECKSUM_TAG = 'twenty:bundle-checksum';
const PREBUILT_BUNDLE_FILE_NAME = 'prebuilt-logic-function.mjs';
const PREBUILT_INSTALL_LOCK_TTL_MS = 180_000;
const PREBUILT_INSTALL_LOCK_RETRY_MS = 1_000;
const PREBUILT_INSTALL_LOCK_MAX_RETRIES = 180;
type LambdaDriverExecutorPayload = {
code: string;
code?: string;
params: object;
env: Record<string, string>;
handlerName: string;
@@ -1164,7 +1179,20 @@ export class LambdaDriver implements LogicFunctionDriver {
payload,
env,
timeoutMs = 900_000,
forceExecutionMode,
}: LogicFunctionExecuteParams): Promise<LogicFunctionExecuteResult> {
const executionMode = forceExecutionMode ?? flatLogicFunction.executionMode;
if (
executionMode === LogicFunctionExecutionMode.PREBUILT &&
!isLogicFunctionReadyForPrebuiltInstall(flatLogicFunction)
) {
throw new LogicFunctionException(
`Cannot run logic function '${flatLogicFunction.id}' in PREBUILT mode: bundle is not installed`,
LogicFunctionExceptionCode.LOGIC_FUNCTION_PREBUILT_BUNDLE_NOT_INSTALLED,
);
}
let currentPhase: LambdaExecutionPhase = LambdaExecutionPhase.BUILD;
let buildExecutorMs = 0;
let getBuiltCodeMs = 0;
@@ -1180,26 +1208,28 @@ export class LambdaDriver implements LogicFunctionDriver {
buildExecutorMs = Date.now() - buildStart;
currentPhase = LambdaExecutionPhase.FETCH_CODE;
const fetchStart = Date.now();
const compiledCode = await this.logicFunctionResourceService.getBuiltCode(
{
workspaceId: flatLogicFunction.workspaceId,
applicationUniversalIdentifier,
builtHandlerPath: flatLogicFunction.builtHandlerPath,
},
);
getBuiltCodeMs = Date.now() - fetchStart;
currentPhase = LambdaExecutionPhase.INVOKE;
const invokeFlowStart = Date.now();
const executorPayload: LambdaDriverExecutorPayload = {
params: payload,
code: compiledCode,
env: env ?? {},
handlerName: flatLogicFunction.handlerName,
};
if (executionMode === LogicFunctionExecutionMode.LIVE) {
const fetchStart = Date.now();
executorPayload.code =
await this.logicFunctionResourceService.getBuiltCode({
workspaceId: flatLogicFunction.workspaceId,
applicationUniversalIdentifier,
builtHandlerPath: flatLogicFunction.builtHandlerPath,
});
getBuiltCodeMs = Date.now() - fetchStart;
}
currentPhase = LambdaExecutionPhase.INVOKE;
const payloadString = JSON.stringify(executorPayload);
const params: InvokeCommandInput = {
FunctionName: flatLogicFunction.id,
@@ -1228,10 +1258,10 @@ export class LambdaDriver implements LogicFunctionDriver {
coldStart,
} = this.parseLambdaLogResult(result.LogResult);
const duration = Date.now() - fetchStart;
const duration = Date.now() - invokeFlowStart;
this.logger.log(
`[lambda-timing] fnId=${flatLogicFunction.id} totalMs=${Date.now() - buildStart} buildExecutorMs=${buildExecutorMs} getBuiltCodeMs=${getBuiltCodeMs} payloadBytes=${Buffer.byteLength(payloadString, 'utf8')} invokeSendMs=${invokeSendMs} reportDurationMs=${reportDurationMs ?? 'n/a'} billedMs=${billedDurationMs ?? 'n/a'} initDurationMs=${initDurationMs ?? 'n/a'} coldStart=${coldStart}`,
`[lambda-timing] fnId=${flatLogicFunction.id} executionMode=${executionMode} totalMs=${Date.now() - buildStart} buildExecutorMs=${buildExecutorMs} getBuiltCodeMs=${getBuiltCodeMs} payloadBytes=${Buffer.byteLength(payloadString, 'utf8')} invokeSendMs=${invokeSendMs} reportDurationMs=${reportDurationMs ?? 'n/a'} billedMs=${billedDurationMs ?? 'n/a'} initDurationMs=${initDurationMs ?? 'n/a'} coldStart=${coldStart}`,
);
if (result.FunctionError) {
@@ -1287,4 +1317,109 @@ export class LambdaDriver implements LogicFunctionDriver {
);
}
}
private getPrebuiltInstallLockKey(flatLogicFunction: FlatLogicFunction) {
return `lambda-install:${flatLogicFunction.id}`;
}
async installPrebuiltBundle({
flatLogicFunction,
flatApplication,
applicationUniversalIdentifier,
}: LogicFunctionInstallPrebuiltBundleParams): Promise<void> {
if (!isNonEmptyString(flatLogicFunction.checksum)) {
throw new LogicFunctionException(
`Cannot install prebuilt bundle for function '${flatLogicFunction.id}' without a checksum`,
LogicFunctionExceptionCode.LOGIC_FUNCTION_PREBUILT_BUNDLE_NOT_INSTALLED,
);
}
const checksum = flatLogicFunction.checksum;
await this.buildLambdaExecutor({
flatLogicFunction,
flatApplication,
applicationUniversalIdentifier,
});
await this.cacheLockService.withLock(
async () => {
const compiledCode =
await this.logicFunctionResourceService.getBuiltCode({
workspaceId: flatLogicFunction.workspaceId,
applicationUniversalIdentifier,
builtHandlerPath: flatLogicFunction.builtHandlerPath,
});
const temporaryDirManager = new TemporaryDirManager();
const { sourceTemporaryDir, lambdaZipPath } =
await temporaryDirManager.init();
try {
await copyExecutor(sourceTemporaryDir);
await fs.writeFile(
join(sourceTemporaryDir, PREBUILT_BUNDLE_FILE_NAME),
compiledCode,
'utf8',
);
await createZipFile(sourceTemporaryDir, lambdaZipPath);
const lambdaClient = await this.getLambdaClient();
const updateResult = await lambdaClient.send(
new UpdateFunctionCodeCommand({
FunctionName: flatLogicFunction.id,
ZipFile: await fs.readFile(lambdaZipPath),
}),
);
await this.waitFunctionUpdated(flatLogicFunction.id);
const functionArn = updateResult.FunctionArn;
if (!isNonEmptyString(functionArn)) {
throw new LogicFunctionException(
`UpdateFunctionCode did not return a FunctionArn for '${flatLogicFunction.id}'`,
LogicFunctionExceptionCode.LOGIC_FUNCTION_PREBUILT_BUNDLE_NOT_INSTALLED,
);
}
await lambdaClient.send(
new TagResourceCommand({
Resource: functionArn,
Tags: {
[LAMBDA_PREBUILT_BUNDLE_CHECKSUM_TAG]: checksum,
},
}),
);
} catch (error) {
this.logger.error(
`Failed to install prebuilt bundle for function ${flatLogicFunction.id}: ${error instanceof Error ? error.message : String(error)}`,
error instanceof Error ? error.stack : undefined,
);
throw error;
} finally {
await temporaryDirManager.clean();
}
},
this.getPrebuiltInstallLockKey(flatLogicFunction),
{
ttl: PREBUILT_INSTALL_LOCK_TTL_MS,
ms: PREBUILT_INSTALL_LOCK_RETRY_MS,
maxRetries: PREBUILT_INSTALL_LOCK_MAX_RETRIES,
},
);
}
async getInstalledBundleChecksum(
flatLogicFunction: FlatLogicFunction,
): Promise<string | null> {
const lambdaExecutor = await this.getLambdaExecutor(flatLogicFunction);
if (!isDefined(lambdaExecutor)) {
return null;
}
return lambdaExecutor.Tags?.[LAMBDA_PREBUILT_BUNDLE_CHECKSUM_TAG] ?? null;
}
}
@@ -9,10 +9,12 @@ import {
type LogicFunctionExecuteParams,
type LogicFunctionExecuteResult,
type LogicFunctionDriver,
type LogicFunctionInstallPrebuiltBundleParams,
type LogicFunctionTranspileParams,
type LogicFunctionTranspileResult,
} from 'src/engine/core-modules/logic-function/logic-function-drivers/interfaces/logic-function-driver.interface';
import { isNonEmptyString } from '@sniptt/guards';
import { type FlatApplication } from 'src/engine/core-modules/application/types/flat-application.type';
import { type CacheLockService } from 'src/engine/core-modules/cache-lock/cache-lock.service';
import { LOGIC_FUNCTION_EXECUTOR_TMPDIR_FOLDER } from 'src/engine/core-modules/logic-function/logic-function-drivers/constants/logic-function-executor-tmpdir-folder';
@@ -20,6 +22,13 @@ import { ConsoleListener } from 'src/engine/core-modules/logic-function/logic-fu
import { TemporaryDirManager } from 'src/engine/core-modules/logic-function/logic-function-drivers/utils/temporary-dir-manager';
import { HANDLER_NAME_REGEX } from 'src/engine/metadata-modules/logic-function/constants/handler.contant';
import { LogicFunctionExecutionStatus } from 'src/engine/metadata-modules/logic-function/dtos/logic-function-execution-result.dto';
import { LogicFunctionExecutionMode } from 'src/engine/metadata-modules/logic-function/logic-function.entity';
import {
LogicFunctionException,
LogicFunctionExceptionCode,
} from 'src/engine/metadata-modules/logic-function/logic-function.exception';
import { type FlatLogicFunction } from 'src/engine/metadata-modules/logic-function/types/flat-logic-function.type';
import { isLogicFunctionReadyForPrebuiltInstall } from 'src/engine/metadata-modules/logic-function/utils/is-logic-function-ready-for-prebuilt-install.util';
import { copyYarnEngineAndBuildDependencies } from 'src/engine/core-modules/application/application-package/utils/copy-yarn-engine-and-build-dependencies';
import type { LogicFunctionResourceService } from 'src/engine/core-modules/logic-function/logic-function-resource/logic-function-resource.service';
import type { SdkClientArchiveService } from 'src/engine/core-modules/sdk-client/sdk-client-archive.service';
@@ -30,6 +39,13 @@ const LAYER_BUILD_LOCK_RETRY_MS = 500;
const LAYER_BUILD_LOCK_MAX_RETRIES = 240;
const LAYER_BUILD_READY_SENTINEL = '.twenty-layer-ready';
const PREBUILT_BUNDLE_FILE_NAME = 'prebuilt-logic-function.mjs';
const PREBUILT_CHECKSUM_FILE_NAME = 'prebuilt-bundle.checksum';
const PREBUILT_INSTALL_LOCK_TTL_MS = 180_000;
const PREBUILT_INSTALL_LOCK_RETRY_MS = 1_000;
const PREBUILT_INSTALL_LOCK_MAX_RETRIES = 180;
export interface LocalDriverOptions {
logicFunctionResourceService: LogicFunctionResourceService;
sdkClientArchiveService: SdkClientArchiveService;
@@ -283,7 +299,20 @@ export class LocalDriver implements LogicFunctionDriver {
payload,
env,
timeoutMs = 900_000,
forceExecutionMode,
}: LogicFunctionExecuteParams): Promise<LogicFunctionExecuteResult> {
const executionMode = forceExecutionMode ?? flatLogicFunction.executionMode;
if (
executionMode === LogicFunctionExecutionMode.PREBUILT &&
!isLogicFunctionReadyForPrebuiltInstall(flatLogicFunction)
) {
throw new LogicFunctionException(
`Cannot run logic function '${flatLogicFunction.id}' in PREBUILT mode: bundle is not installed`,
LogicFunctionExceptionCode.LOGIC_FUNCTION_PREBUILT_BUNDLE_NOT_INSTALLED,
);
}
await this.createLayerIfNotExist({
flatApplication,
applicationUniversalIdentifier,
@@ -300,20 +329,25 @@ export class LocalDriver implements LogicFunctionDriver {
try {
const { sourceTemporaryDir } = await temporaryDirManager.init();
const inMemoryBuiltHandlerPath =
await this.logicFunctionResourceService.copyBuiltCodeInMemory({
workspaceId: flatLogicFunction.workspaceId,
applicationUniversalIdentifier,
builtHandlerPath: flatLogicFunction.builtHandlerPath,
inMemoryDestinationPath: sourceTemporaryDir,
});
await this.assembleNodeModules({
sourceTemporaryDir,
flatApplication,
applicationUniversalIdentifier,
});
const inMemoryBuiltHandlerPath =
executionMode === LogicFunctionExecutionMode.PREBUILT
? await this.copyPrebuiltBundleIntoExecutionDir({
flatLogicFunction,
sourceTemporaryDir,
})
: await this.logicFunctionResourceService.copyBuiltCodeInMemory({
workspaceId: flatLogicFunction.workspaceId,
applicationUniversalIdentifier,
builtHandlerPath: flatLogicFunction.builtHandlerPath,
inMemoryDestinationPath: sourceTemporaryDir,
});
let logs = '';
const consoleListener = new ConsoleListener();
@@ -561,4 +595,114 @@ export class LocalDriver implements LogicFunctionDriver {
child.on('close', () => clearTimeout(t));
});
}
private getPrebuiltBundleDir(flatLogicFunction: FlatLogicFunction): string {
return join(
LOGIC_FUNCTION_EXECUTOR_TMPDIR_FOLDER,
'prebuilt',
flatLogicFunction.id,
);
}
private getInstalledBundlePath(flatLogicFunction: FlatLogicFunction): string {
return join(
this.getPrebuiltBundleDir(flatLogicFunction),
PREBUILT_BUNDLE_FILE_NAME,
);
}
private async copyPrebuiltBundleIntoExecutionDir({
flatLogicFunction,
sourceTemporaryDir,
}: {
flatLogicFunction: FlatLogicFunction;
sourceTemporaryDir: string;
}): Promise<string> {
const installedBundlePath = this.getInstalledBundlePath(flatLogicFunction);
const localBundlePath = join(sourceTemporaryDir, PREBUILT_BUNDLE_FILE_NAME);
await fs.copyFile(installedBundlePath, localBundlePath);
return localBundlePath;
}
private getInstalledChecksumPath(
flatLogicFunction: FlatLogicFunction,
): string {
return join(
this.getPrebuiltBundleDir(flatLogicFunction),
PREBUILT_CHECKSUM_FILE_NAME,
);
}
private getPrebuiltInstallLockKey(flatLogicFunction: FlatLogicFunction) {
return `local-install:${flatLogicFunction.id}`;
}
async installPrebuiltBundle({
flatLogicFunction,
applicationUniversalIdentifier,
}: LogicFunctionInstallPrebuiltBundleParams): Promise<void> {
if (!isNonEmptyString(flatLogicFunction.checksum)) {
throw new LogicFunctionException(
`Cannot install prebuilt bundle for function '${flatLogicFunction.id}' without a checksum`,
LogicFunctionExceptionCode.LOGIC_FUNCTION_PREBUILT_BUNDLE_NOT_INSTALLED,
);
}
const checksum = flatLogicFunction.checksum;
await this.cacheLockService.withLock(
async () => {
const prebuiltDir = this.getPrebuiltBundleDir(flatLogicFunction);
await fs.mkdir(prebuiltDir, { recursive: true });
await this.logicFunctionResourceService.copyBuiltCodeInMemory({
workspaceId: flatLogicFunction.workspaceId,
applicationUniversalIdentifier,
builtHandlerPath: flatLogicFunction.builtHandlerPath,
inMemoryDestinationPath: prebuiltDir,
});
const downloadedPath = join(
prebuiltDir,
flatLogicFunction.builtHandlerPath,
);
const targetPath = this.getInstalledBundlePath(flatLogicFunction);
if (downloadedPath !== targetPath) {
await fs.mkdir(dirname(targetPath), { recursive: true });
await fs.rename(downloadedPath, targetPath);
}
await fs.writeFile(
this.getInstalledChecksumPath(flatLogicFunction),
checksum,
'utf8',
);
},
this.getPrebuiltInstallLockKey(flatLogicFunction),
{
ttl: PREBUILT_INSTALL_LOCK_TTL_MS,
ms: PREBUILT_INSTALL_LOCK_RETRY_MS,
maxRetries: PREBUILT_INSTALL_LOCK_MAX_RETRIES,
},
);
}
async getInstalledBundleChecksum(
flatLogicFunction: FlatLogicFunction,
): Promise<string | null> {
try {
const checksum = await fs.readFile(
this.getInstalledChecksumPath(flatLogicFunction),
'utf8',
);
return checksum.trim() || null;
} catch {
return null;
}
}
}
@@ -1,5 +1,6 @@
import { type FlatApplication } from 'src/engine/core-modules/application/types/flat-application.type';
import { type LogicFunctionExecutionStatus } from 'src/engine/metadata-modules/logic-function/dtos/logic-function-execution-result.dto';
import { type LogicFunctionExecutionMode } from 'src/engine/metadata-modules/logic-function/logic-function.entity';
import { type FlatLogicFunction } from 'src/engine/metadata-modules/logic-function/types/flat-logic-function.type';
export type LogicFunctionExecuteError = {
@@ -23,6 +24,13 @@ export type LogicFunctionExecuteParams = {
payload: object;
env?: Record<string, string>;
timeoutMs?: number;
forceExecutionMode?: LogicFunctionExecutionMode;
};
export type LogicFunctionInstallPrebuiltBundleParams = {
flatLogicFunction: FlatLogicFunction;
flatApplication: FlatApplication;
applicationUniversalIdentifier: string;
};
export type LogicFunctionTranspileParams = {
@@ -43,6 +51,12 @@ export interface LogicFunctionDriver {
transpile(
params: LogicFunctionTranspileParams,
): Promise<LogicFunctionTranspileResult>;
installPrebuiltBundle(
params: LogicFunctionInstallPrebuiltBundleParams,
): Promise<void>;
getInstalledBundleChecksum(
flatLogicFunction: FlatLogicFunction,
): Promise<string | null>;
}
export enum LogicFunctionDriverType {
@@ -5,6 +5,7 @@ import { ApplicationRegistrationVariableEntity } from 'src/engine/core-modules/a
import { AuditModule } from 'src/engine/core-modules/audit/audit.module';
import { TokenModule } from 'src/engine/core-modules/auth/token/token.module';
import { BillingModule } from 'src/engine/core-modules/billing/billing.module';
import { FeatureFlagModule } from 'src/engine/core-modules/feature-flag/feature-flag.module';
import { LogicFunctionExecutorService } from 'src/engine/core-modules/logic-function/logic-function-executor/logic-function-executor.service';
import { SecretEncryptionModule } from 'src/engine/core-modules/secret-encryption/secret-encryption.module';
import { ThrottlerModule } from 'src/engine/core-modules/throttler/throttler.module';
@@ -20,6 +21,7 @@ import { WorkspaceCacheModule } from 'src/engine/workspace-cache/workspace-cache
SubscriptionsModule,
WorkspaceCacheModule,
BillingModule,
FeatureFlagModule,
TypeOrmModule.forFeature([ApplicationRegistrationVariableEntity]),
],
providers: [LogicFunctionExecutorService],
@@ -6,6 +6,7 @@ import {
DEFAULT_API_URL_NAME,
DEFAULT_APP_ACCESS_TOKEN_NAME,
} from 'twenty-shared/application';
import { FeatureFlagKey } from 'twenty-shared/types';
import { isDefined } from 'twenty-shared/utils';
import { Not, Repository } from 'typeorm';
import { v4 } from 'uuid';
@@ -26,6 +27,7 @@ import { LOGIC_FUNCTION_EXECUTED_EVENT } from 'src/engine/core-modules/audit/uti
import { ApplicationTokenService } from 'src/engine/core-modules/auth/token/services/application-token.service';
import { BillingUsageService } from 'src/engine/core-modules/billing/services/billing-usage.service';
import { BillingService } from 'src/engine/core-modules/billing/services/billing.service';
import { FeatureFlagService } from 'src/engine/core-modules/feature-flag/services/feature-flag.service';
import { LogicFunctionDriverFactory } from 'src/engine/core-modules/logic-function/logic-function-drivers/logic-function-driver.factory';
import { buildEnvVar } from 'src/engine/core-modules/logic-function/logic-function-executor/utils/build-env-var';
import { SecretEncryptionService } from 'src/engine/core-modules/secret-encryption/secret-encryption.service';
@@ -37,6 +39,11 @@ import { UsageResourceType } from 'src/engine/core-modules/usage/enums/usage-res
import { UsageUnit } from 'src/engine/core-modules/usage/enums/usage-unit.enum';
import { type UsageEvent } from 'src/engine/core-modules/usage/types/usage-event.type';
import { findFlatEntityByIdInFlatEntityMaps } from 'src/engine/metadata-modules/flat-entity/utils/find-flat-entity-by-id-in-flat-entity-maps.util';
import { LogicFunctionExecutionMode } from 'src/engine/metadata-modules/logic-function/logic-function.entity';
import {
LogicFunctionException,
LogicFunctionExceptionCode,
} from 'src/engine/metadata-modules/logic-function/logic-function.exception';
import { FlatLogicFunction } from 'src/engine/metadata-modules/logic-function/types/flat-logic-function.type';
import { SubscriptionChannel } from 'src/engine/subscriptions/enums/subscription-channel.enum';
import { SubscriptionService } from 'src/engine/subscriptions/subscription.service';
@@ -76,6 +83,7 @@ export class LogicFunctionExecutorService {
private readonly workspaceEventEmitter: WorkspaceEventEmitter,
private readonly billingService: BillingService,
private readonly billingUsageService: BillingUsageService,
private readonly featureFlagService: FeatureFlagService,
@InjectRepository(ApplicationRegistrationVariableEntity)
private readonly applicationRegistrationVariableRepository: Repository<ApplicationRegistrationVariableEntity>,
) {}
@@ -86,12 +94,14 @@ export class LogicFunctionExecutorService {
payload,
userId,
userWorkspaceId,
executionMode,
}: {
logicFunctionId: string;
workspaceId: string;
payload: object;
userId?: string;
userWorkspaceId?: string;
executionMode?: LogicFunctionExecutionMode;
}): Promise<LogicFunctionExecuteResult> {
await this.throttleExecution(workspaceId);
@@ -111,6 +121,19 @@ export class LogicFunctionExecutorService {
const driver = this.logicFunctionDriverFactory.getCurrentDriver();
const effectiveExecutionMode = await this.resolveEffectiveExecutionMode({
workspaceId,
flatLogicFunction,
callerOverride: executionMode,
});
if (effectiveExecutionMode === LogicFunctionExecutionMode.PREBUILT) {
await this.assertPrebuiltBundleInstalled({
driver,
flatLogicFunction,
});
}
let resultLogicFunction: LogicFunctionExecuteResult;
try {
@@ -121,13 +144,15 @@ export class LogicFunctionExecutorService {
payload,
env: envVariables,
timeoutMs: flatLogicFunction.timeoutSeconds * 1_000,
forceExecutionMode: effectiveExecutionMode,
});
} catch (error) {
this.logger.error(
`Logic function execution failed: ` +
`functionId=${logicFunctionId}, ` +
`workspaceId=${workspaceId}, ` +
`driver=${driver.constructor.name}: ` +
`driver=${driver.constructor.name}, ` +
`mode=${effectiveExecutionMode}: ` +
`${error instanceof Error ? error.message : String(error)}`,
error instanceof Error ? error.stack : undefined,
);
@@ -144,6 +169,52 @@ export class LogicFunctionExecutorService {
return resultLogicFunction;
}
private async resolveEffectiveExecutionMode({
workspaceId,
flatLogicFunction,
callerOverride,
}: {
workspaceId: string;
flatLogicFunction: FlatLogicFunction;
callerOverride?: LogicFunctionExecutionMode;
}): Promise<LogicFunctionExecutionMode> {
if (isDefined(callerOverride)) {
return callerOverride;
}
const isPrebuiltModeEnabled =
await this.featureFlagService.isFeatureEnabled(
FeatureFlagKey.IS_LOGIC_FUNCTION_PREBUILT_MODE_ENABLED,
workspaceId,
);
if (!isPrebuiltModeEnabled) {
return LogicFunctionExecutionMode.LIVE;
}
return flatLogicFunction.executionMode ?? LogicFunctionExecutionMode.LIVE;
}
private async assertPrebuiltBundleInstalled({
driver,
flatLogicFunction,
}: {
driver: ReturnType<LogicFunctionDriverFactory['getCurrentDriver']>;
flatLogicFunction: FlatLogicFunction;
}): Promise<void> {
const installedChecksum =
await driver.getInstalledBundleChecksum(flatLogicFunction);
if (installedChecksum !== flatLogicFunction.checksum) {
throw new LogicFunctionException(
`Prebuilt bundle is not installed for function '${flatLogicFunction.id}' ` +
`(installed=${installedChecksum ?? 'none'}, expected=${flatLogicFunction.checksum ?? 'none'}). ` +
`Rebuild and try again.`,
LogicFunctionExceptionCode.LOGIC_FUNCTION_PREBUILT_BUNDLE_NOT_INSTALLED,
);
}
}
async transpile(
params: LogicFunctionTranspileParams,
): Promise<LogicFunctionTranspileResult> {
@@ -1,6 +1,7 @@
import { type DynamicModule, Global, Module } from '@nestjs/common';
import { CacheLockModule } from 'src/engine/core-modules/cache-lock/cache-lock.module';
import { LOGIC_FUNCTION_DRIVER_FACTORY_TOKEN } from 'src/engine/core-modules/logic-function/logic-function-drivers/constants/logic-function-driver-factory.token';
import { LogicFunctionDriverFactory } from 'src/engine/core-modules/logic-function/logic-function-drivers/logic-function-driver.factory';
import { LogicFunctionResourceModule } from 'src/engine/core-modules/logic-function/logic-function-resource/logic-function-resource.module';
import { LogicFunctionTriggerModule } from 'src/engine/core-modules/logic-function/logic-function-trigger/logic-function-trigger.module';
@@ -24,9 +25,16 @@ export class LogicFunctionModule {
SdkClientModule,
WorkspaceCacheModule,
],
providers: [LogicFunctionDriverFactory],
providers: [
LogicFunctionDriverFactory,
{
provide: LOGIC_FUNCTION_DRIVER_FACTORY_TOKEN,
useExisting: LogicFunctionDriverFactory,
},
],
exports: [
LogicFunctionDriverFactory,
LOGIC_FUNCTION_DRIVER_FACTORY_TOKEN,
LogicFunctionResourceModule,
LogicFunctionTriggerModule,
LogicFunctionExecutorModule,
@@ -123,6 +123,7 @@ exports[`ALL_UNIVERSAL_FLAT_ENTITY_PROPERTIES_TO_COMPARE_AND_STRINGIFY should ma
"sourceHandlerPath",
"handlerName",
"isBuildUpToDate",
"executionMode",
"deletedAt",
"cronTriggerSettings",
"databaseEventTriggerSettings",
@@ -587,6 +587,11 @@ export const ALL_ENTITY_PROPERTIES_CONFIGURATION_BY_METADATA_NAME = {
toStringify: false,
universalProperty: undefined,
},
executionMode: {
toCompare: true,
toStringify: false,
universalProperty: undefined,
},
deletedAt: {
toCompare: true,
toStringify: false,
@@ -1,4 +1,9 @@
import { Field, HideField, ObjectType } from '@nestjs/graphql';
import {
Field,
HideField,
ObjectType,
registerEnumType,
} from '@nestjs/graphql';
import {
Authorize,
@@ -7,6 +12,7 @@ import {
} from '@ptc-org/nestjs-query-graphql';
import {
IsDateString,
IsEnum,
IsNotEmpty,
IsNumber,
IsObject,
@@ -24,6 +30,11 @@ import {
} from 'twenty-shared/application';
import { UUIDScalarType } from 'src/engine/api/graphql/workspace-schema-builder/graphql-types/scalars';
import { LogicFunctionExecutionMode } from 'src/engine/metadata-modules/logic-function/logic-function.entity';
registerEnumType(LogicFunctionExecutionMode, {
name: 'LogicFunctionExecutionMode',
});
@ObjectType('LogicFunction')
@Authorize({
@@ -59,6 +70,10 @@ export class LogicFunctionDTO {
@Field()
timeoutSeconds: number;
@IsEnum(LogicFunctionExecutionMode)
@Field(() => LogicFunctionExecutionMode)
executionMode: LogicFunctionExecutionMode;
@IsString()
@Field()
sourceHandlerPath: string;
@@ -26,6 +26,11 @@ export enum LogicFunctionRuntime {
NODE22 = 'nodejs22.x',
}
export enum LogicFunctionExecutionMode {
LIVE = 'LIVE',
PREBUILT = 'PREBUILT',
}
@Entity('logicFunction')
@Index('IDX_LOGIC_FUNCTION_ID_DELETED_AT', ['id', 'deletedAt'])
export class LogicFunctionEntity
@@ -63,6 +68,14 @@ export class LogicFunctionEntity
@Column({ nullable: false, type: 'boolean', default: true })
isBuildUpToDate: boolean;
@Column({
type: 'enum',
enum: LogicFunctionExecutionMode,
default: LogicFunctionExecutionMode.LIVE,
nullable: false,
})
executionMode: LogicFunctionExecutionMode;
@Column({ nullable: true, type: 'jsonb' })
cronTriggerSettings: JsonbProperty<CronTriggerSettings> | null;
@@ -19,6 +19,7 @@ export enum LogicFunctionExceptionCode {
LOGIC_FUNCTION_DISABLED = 'LOGIC_FUNCTION_DISABLED',
LOGIC_FUNCTION_INVALID_SEED_PROJECT = 'LOGIC_FUNCTION_INVALID_SEED_PROJECT',
INVALID_LOGIC_FUNCTION_INPUT = 'INVALID_LOGIC_FUNCTION_INPUT',
LOGIC_FUNCTION_PREBUILT_BUNDLE_NOT_INSTALLED = 'LOGIC_FUNCTION_PREBUILT_BUNDLE_NOT_INSTALLED',
}
const getLogicFunctionExceptionUserFriendlyMessage = (
@@ -53,6 +54,8 @@ const getLogicFunctionExceptionUserFriendlyMessage = (
return msg`Invalid seed project configuration.`;
case LogicFunctionExceptionCode.INVALID_LOGIC_FUNCTION_INPUT:
return msg`Invalid logic function input.`;
case LogicFunctionExceptionCode.LOGIC_FUNCTION_PREBUILT_BUNDLE_NOT_INSTALLED:
return msg`Prebuilt bundle is not installed on the function. Rebuild and try again.`;
default:
assertUnreachable(code);
}
@@ -46,6 +46,9 @@ import { WorkspaceMigrationModule } from 'src/engine/workspace-manager/workspace
LogicFunctionResolver,
WorkspaceFlatLogicFunctionMapCacheService,
],
exports: [LogicFunctionFromSourceService],
exports: [
LogicFunctionFromSourceService,
LogicFunctionFromSourceHelperService,
],
})
export class LogicFunctionModule {}
@@ -14,6 +14,7 @@ import { findFlatEntityByIdInFlatEntityMapsOrThrow } from 'src/engine/metadata-m
import { CreateLogicFunctionFromSourceInput } from 'src/engine/metadata-modules/logic-function/dtos/create-logic-function-from-source.input';
import { LogicFunctionExecutionResultDTO } from 'src/engine/metadata-modules/logic-function/dtos/logic-function-execution-result.dto';
import { LogicFunctionDTO } from 'src/engine/metadata-modules/logic-function/dtos/logic-function.dto';
import { LogicFunctionExecutionMode } from 'src/engine/metadata-modules/logic-function/logic-function.entity';
import {
LogicFunctionException,
LogicFunctionExceptionCode,
@@ -169,6 +170,7 @@ export class LogicFunctionFromSourceService {
timeoutSeconds: existingLogicFunction.timeoutSeconds,
isBuildUpToDate: existingLogicFunction.isBuildUpToDate,
checksum: existingLogicFunction.checksum,
executionMode: LogicFunctionExecutionMode.LIVE,
handlerName: existingLogicFunction.handlerName,
sourceHandlerPath: toSourceHandlerPath,
builtHandlerPath: toBuiltHandlerPath,
@@ -365,6 +367,7 @@ export class LogicFunctionFromSourceService {
logicFunctionId: id,
workspaceId,
payload,
executionMode: LogicFunctionExecutionMode.LIVE,
});
return {
@@ -1,6 +1,9 @@
import { v4 } from 'uuid';
import { LogicFunctionRuntime } from 'src/engine/metadata-modules/logic-function/logic-function.entity';
import {
LogicFunctionExecutionMode,
LogicFunctionRuntime,
} from 'src/engine/metadata-modules/logic-function/logic-function.entity';
import { type UniversalFlatLogicFunction } from 'src/engine/workspace-manager/workspace-migration/universal-flat-entity/types/universal-flat-logic-function.type';
type AutoGeneratedFields = 'createdAt' | 'updatedAt' | 'deletedAt' | 'runtime';
@@ -40,6 +43,7 @@ export const buildUniversalFlatLogicFunctionToCreate = (
timeoutSeconds: input.timeoutSeconds ?? 300,
checksum: input.checksum ?? null,
isBuildUpToDate: input.isBuildUpToDate,
executionMode: input.executionMode ?? LogicFunctionExecutionMode.LIVE,
handlerName: input.handlerName,
sourceHandlerPath: input.sourceHandlerPath,
builtHandlerPath: input.builtHandlerPath,
@@ -1,7 +1,10 @@
import { trimAndRemoveDuplicatedWhitespacesFromObjectStringProperties } from 'twenty-shared/utils';
import { v4 } from 'uuid';
import { LogicFunctionRuntime } from 'src/engine/metadata-modules/logic-function/logic-function.entity';
import {
LogicFunctionExecutionMode,
LogicFunctionRuntime,
} from 'src/engine/metadata-modules/logic-function/logic-function.entity';
import { type CreateLogicFunctionFromSourceInput } from 'src/engine/metadata-modules/logic-function/dtos/create-logic-function-from-source.input';
import { type UniversalFlatLogicFunction } from 'src/engine/workspace-manager/workspace-migration/universal-flat-entity/types/universal-flat-logic-function.type';
@@ -43,6 +46,7 @@ export const fromCreateLogicFunctionFromSourceInputToUniversalFlatLogicFunctionT
timeoutSeconds: createLogicFunctionFromSourceInput.timeoutSeconds ?? 300,
checksum,
isBuildUpToDate,
executionMode: LogicFunctionExecutionMode.LIVE,
handlerName,
sourceHandlerPath,
builtHandlerPath,
@@ -13,6 +13,7 @@ export const fromFlatLogicFunctionToLogicFunctionDto = ({
description: flatLogicFunction.description ?? undefined,
runtime: flatLogicFunction.runtime,
timeoutSeconds: flatLogicFunction.timeoutSeconds,
executionMode: flatLogicFunction.executionMode,
sourceHandlerPath: flatLogicFunction.sourceHandlerPath,
handlerName: flatLogicFunction.handlerName,
applicationId: flatLogicFunction.applicationId ?? undefined,
@@ -0,0 +1,16 @@
import { isNonEmptyString } from '@sniptt/guards';
import { LogicFunctionExecutionMode } from 'src/engine/metadata-modules/logic-function/logic-function.entity';
import { type FlatLogicFunction } from 'src/engine/metadata-modules/logic-function/types/flat-logic-function.type';
export type LogicFunctionPrebuiltStateFields = Pick<
FlatLogicFunction,
'executionMode' | 'isBuildUpToDate' | 'checksum'
>;
export const isLogicFunctionReadyForPrebuiltInstall = (
flatLogicFunction: LogicFunctionPrebuiltStateFields,
): boolean =>
flatLogicFunction.executionMode === LogicFunctionExecutionMode.PREBUILT &&
flatLogicFunction.isBuildUpToDate === true &&
isNonEmptyString(flatLogicFunction.checksum);
@@ -36,6 +36,7 @@ export const logicFunctionGraphQLApiExceptionHandler = (error: any) => {
case LogicFunctionExceptionCode.INVALID_LOGIC_FUNCTION_INPUT:
throw new UserInputError(error);
case LogicFunctionExceptionCode.LOGIC_FUNCTION_DISABLED:
case LogicFunctionExceptionCode.LOGIC_FUNCTION_PREBUILT_BUNDLE_NOT_INSTALLED:
throw new ForbiddenError(error);
default: {
return assertUnreachable(error.code);
@@ -0,0 +1,27 @@
import { LogicFunctionExecutionMode } from 'src/engine/metadata-modules/logic-function/logic-function.entity';
import {
isLogicFunctionReadyForPrebuiltInstall,
type LogicFunctionPrebuiltStateFields,
} from 'src/engine/metadata-modules/logic-function/utils/is-logic-function-ready-for-prebuilt-install.util';
export const shouldReinstallLogicFunctionPrebuiltBundle = ({
existingLogicFunction,
newLogicFunction,
}: {
existingLogicFunction: LogicFunctionPrebuiltStateFields;
newLogicFunction: LogicFunctionPrebuiltStateFields;
}): boolean => {
if (!isLogicFunctionReadyForPrebuiltInstall(newLogicFunction)) {
return false;
}
if (
existingLogicFunction.executionMode ===
LogicFunctionExecutionMode.PREBUILT &&
existingLogicFunction.checksum === newLogicFunction.checksum
) {
return false;
}
return true;
};
@@ -237,6 +237,7 @@ describe('WorkspaceEntityManager', () => {
IS_EMAIL_GROUP_ENABLED: false,
IS_JUNCTION_RELATIONS_ENABLED: false,
IS_REST_METADATA_API_NEW_FORMAT_DIRECT: false,
IS_LOGIC_FUNCTION_PREBUILT_MODE_ENABLED: false,
IS_SETTINGS_DISCOVERY_HERO_ENABLED: false,
},
userWorkspaceRoleMap: {},
@@ -7,7 +7,9 @@ import { isDefined } from 'twenty-shared/utils';
import { validateFilePath } from 'src/engine/core-modules/file-storage/utils/validate-file-path.util';
import { findFlatEntityByUniversalIdentifier } from 'src/engine/metadata-modules/flat-entity/utils/find-flat-entity-by-universal-identifier.util';
import { LogicFunctionExecutionMode } from 'src/engine/metadata-modules/logic-function/logic-function.entity';
import { LogicFunctionExceptionCode } from 'src/engine/metadata-modules/logic-function/logic-function.exception';
import { isLogicFunctionReadyForPrebuiltInstall } from 'src/engine/metadata-modules/logic-function/utils/is-logic-function-ready-for-prebuilt-install.util';
import { type FailedFlatEntityValidation } from 'src/engine/workspace-manager/workspace-migration/workspace-migration-builder/builders/types/failed-flat-entity-validation.type';
import { getEmptyFlatEntityValidationError } from 'src/engine/workspace-manager/workspace-migration/workspace-migration-builder/builders/utils/get-flat-entity-validation-error.util';
import { type FlatEntityUpdateValidationArgs } from 'src/engine/workspace-manager/workspace-migration/workspace-migration-builder/types/universal-flat-entity-update-validation-args.type';
@@ -79,6 +81,31 @@ export class FlatLogicFunctionValidatorService {
}
}
const mergedPrebuiltState = {
executionMode:
flatEntityUpdate.executionMode ??
existingFlatLogicFunction.executionMode,
isBuildUpToDate:
flatEntityUpdate.isBuildUpToDate ??
existingFlatLogicFunction.isBuildUpToDate,
checksum:
flatEntityUpdate.checksum !== undefined
? flatEntityUpdate.checksum
: existingFlatLogicFunction.checksum,
};
if (
mergedPrebuiltState.executionMode ===
LogicFunctionExecutionMode.PREBUILT &&
!isLogicFunctionReadyForPrebuiltInstall(mergedPrebuiltState)
) {
validationResult.errors.push({
code: LogicFunctionExceptionCode.INVALID_LOGIC_FUNCTION_INPUT,
message: t`Logic function cannot be in PREBUILT mode without a fresh build and a checksum`,
userFriendlyMessage: msg`Logic function cannot be in PREBUILT mode without a fresh build and a checksum`,
});
}
return validationResult;
}
@@ -175,6 +202,18 @@ export class FlatLogicFunctionValidatorService {
}
}
if (
flatLogicFunctionToValidate.executionMode ===
LogicFunctionExecutionMode.PREBUILT &&
!isLogicFunctionReadyForPrebuiltInstall(flatLogicFunctionToValidate)
) {
validationResult.errors.push({
code: LogicFunctionExceptionCode.INVALID_LOGIC_FUNCTION_INPUT,
message: t`Logic function cannot be in PREBUILT mode without a fresh build and a checksum`,
userFriendlyMessage: msg`Logic function cannot be in PREBUILT mode without a fresh build and a checksum`,
});
}
return validationResult;
}
}
@@ -1,9 +1,13 @@
import { Injectable } from '@nestjs/common';
import { Inject, Injectable } from '@nestjs/common';
import { v4 } from 'uuid';
import { WorkspaceMigrationRunnerActionHandler } from 'src/engine/workspace-manager/workspace-migration/workspace-migration-runner/interfaces/workspace-migration-runner-action-handler-service.interface';
import { LOGIC_FUNCTION_DRIVER_FACTORY_TOKEN } from 'src/engine/core-modules/logic-function/logic-function-drivers/constants/logic-function-driver-factory.token';
import { isLogicFunctionReadyForPrebuiltInstall } from 'src/engine/metadata-modules/logic-function/utils/is-logic-function-ready-for-prebuilt-install.util';
import type { LogicFunctionDriverFactory } from 'src/engine/core-modules/logic-function/logic-function-drivers/logic-function-driver.factory';
import { getUniversalFlatEntityEmptyForeignKeyAggregators } from 'src/engine/workspace-manager/workspace-migration/universal-flat-entity/utils/reset-universal-flat-entity-foreign-key-aggregators.util';
import {
FlatCreateLogicFunctionAction,
@@ -19,6 +23,13 @@ export class CreateLogicFunctionActionHandlerService extends WorkspaceMigrationR
'create',
'logicFunction',
) {
constructor(
@Inject(LOGIC_FUNCTION_DRIVER_FACTORY_TOKEN)
private readonly logicFunctionDriverFactory: LogicFunctionDriverFactory,
) {
super();
}
override async transpileUniversalActionToFlatAction({
action,
flatApplication,
@@ -44,13 +55,42 @@ export class CreateLogicFunctionActionHandlerService extends WorkspaceMigrationR
async executeForMetadata(
context: WorkspaceMigrationActionRunnerContext<FlatCreateLogicFunctionAction>,
): Promise<void> {
const { flatAction, queryRunner } = context;
const { flatAction, queryRunner, flatApplication } = context;
const { flatEntity: logicFunction } = flatAction;
await this.insertFlatEntitiesInRepository({
queryRunner,
flatEntities: [logicFunction],
});
if (isLogicFunctionReadyForPrebuiltInstall(logicFunction)) {
const driver = this.logicFunctionDriverFactory.getCurrentDriver();
const installStart = Date.now();
try {
await driver.installPrebuiltBundle({
flatLogicFunction: logicFunction,
flatApplication,
applicationUniversalIdentifier: flatApplication.universalIdentifier,
});
this.logger.log(
`[lambda-timing] event=install_prebuilt fnId=${logicFunction.id} ` +
`reason=app_install install_duration_ms=${Date.now() - installStart}`,
CreateLogicFunctionActionHandlerService.name,
);
} catch (error) {
this.logger.error(
`Failed to install prebuilt bundle on app-install for function ${logicFunction.id} ` +
`after ${Date.now() - installStart}ms: ` +
`${error instanceof Error ? error.message : String(error)}`,
CreateLogicFunctionActionHandlerService.name,
error instanceof Error ? error.stack : undefined,
);
throw error;
}
}
}
async rollbackForMetadata(
@@ -1,4 +1,4 @@
import { Injectable } from '@nestjs/common';
import { Inject, Injectable } from '@nestjs/common';
import { FileFolder } from 'twenty-shared/types';
import { isDefined } from 'twenty-shared/utils';
@@ -6,9 +6,17 @@ import { isDefined } from 'twenty-shared/utils';
import { WorkspaceMigrationRunnerActionHandler } from 'src/engine/workspace-manager/workspace-migration/workspace-migration-runner/interfaces/workspace-migration-runner-action-handler-service.interface';
import { FileStorageService } from 'src/engine/core-modules/file-storage/file-storage.service';
import { LOGIC_FUNCTION_DRIVER_FACTORY_TOKEN } from 'src/engine/core-modules/logic-function/logic-function-drivers/constants/logic-function-driver-factory.token';
import { findFlatEntityByIdInFlatEntityMapsOrThrow } from 'src/engine/metadata-modules/flat-entity/utils/find-flat-entity-by-id-in-flat-entity-maps-or-throw.util';
import type { LogicFunctionDriverFactory } from 'src/engine/core-modules/logic-function/logic-function-drivers/logic-function-driver.factory';
import { findFlatEntityByUniversalIdentifierOrThrow } from 'src/engine/metadata-modules/flat-entity/utils/find-flat-entity-by-universal-identifier-or-throw.util';
import { LogicFunctionEntity } from 'src/engine/metadata-modules/logic-function/logic-function.entity';
import {
LogicFunctionEntity,
LogicFunctionExecutionMode,
} from 'src/engine/metadata-modules/logic-function/logic-function.entity';
import { type FlatLogicFunction } from 'src/engine/metadata-modules/logic-function/types/flat-logic-function.type';
import { shouldReinstallLogicFunctionPrebuiltBundle } from 'src/engine/metadata-modules/logic-function/utils/should-reinstall-logic-function-prebuilt-bundle.util';
import { resolveUniversalUpdateRelationIdentifiersToIds } from 'src/engine/workspace-manager/workspace-migration/universal-flat-entity/utils/resolve-universal-update-relation-identifiers-to-ids.util';
import {
FlatUpdateLogicFunctionAction,
@@ -24,7 +32,11 @@ export class UpdateLogicFunctionActionHandlerService extends WorkspaceMigrationR
'update',
'logicFunction',
) {
constructor(private readonly fileStorageService: FileStorageService) {
constructor(
private readonly fileStorageService: FileStorageService,
@Inject(LOGIC_FUNCTION_DRIVER_FACTORY_TOKEN)
private readonly logicFunctionDriverFactory: LogicFunctionDriverFactory,
) {
super();
}
@@ -64,10 +76,11 @@ export class UpdateLogicFunctionActionHandlerService extends WorkspaceMigrationR
} = context;
const { entityId, update } = flatAction;
const existingLogicFunction = findFlatEntityByIdInFlatEntityMapsOrThrow({
flatEntityMaps: allFlatEntityMaps.flatLogicFunctionMaps,
flatEntityId: entityId,
});
const existingLogicFunction =
findFlatEntityByIdInFlatEntityMapsOrThrow<FlatLogicFunction>({
flatEntityMaps: allFlatEntityMaps.flatLogicFunctionMaps,
flatEntityId: entityId,
});
const applicationUniversalIdentifier = flatApplication.universalIdentifier;
@@ -85,6 +98,13 @@ export class UpdateLogicFunctionActionHandlerService extends WorkspaceMigrationR
update as Parameters<typeof logicFunctionRepository.update>[1],
);
await this.installPrebuiltBundleIfNeeded({
existingLogicFunction,
update,
applicationUniversalIdentifier,
context,
});
if (builtPathChanged) {
await this.fileStorageService.deleteFile({
workspaceId,
@@ -94,4 +114,71 @@ export class UpdateLogicFunctionActionHandlerService extends WorkspaceMigrationR
});
}
}
private async installPrebuiltBundleIfNeeded({
existingLogicFunction,
update,
applicationUniversalIdentifier,
context,
}: {
existingLogicFunction: FlatLogicFunction;
update: FlatUpdateLogicFunctionAction['update'];
applicationUniversalIdentifier: string;
context: WorkspaceMigrationActionRunnerContext<FlatUpdateLogicFunctionAction>;
}): Promise<void> {
const newLogicFunction: FlatLogicFunction = {
...existingLogicFunction,
...update,
executionMode:
update.executionMode ?? existingLogicFunction.executionMode,
isBuildUpToDate:
update.isBuildUpToDate ?? existingLogicFunction.isBuildUpToDate,
checksum: update.checksum ?? existingLogicFunction.checksum,
};
if (
!shouldReinstallLogicFunctionPrebuiltBundle({
existingLogicFunction,
newLogicFunction,
})
) {
return;
}
const becamePrebuilt =
existingLogicFunction.executionMode !==
LogicFunctionExecutionMode.PREBUILT;
const driver = this.logicFunctionDriverFactory.getCurrentDriver();
const installStart = Date.now();
try {
await driver.installPrebuiltBundle({
flatLogicFunction: {
...newLogicFunction,
executionMode: LogicFunctionExecutionMode.PREBUILT,
isBuildUpToDate: true,
},
flatApplication: context.flatApplication,
applicationUniversalIdentifier,
});
this.logger.log(
`[lambda-timing] event=install_prebuilt fnId=${existingLogicFunction.id} ` +
`reason=${becamePrebuilt ? 'mode_flip' : 'checksum_change'} ` +
`install_duration_ms=${Date.now() - installStart}`,
UpdateLogicFunctionActionHandlerService.name,
);
} catch (error) {
this.logger.error(
`Failed to install prebuilt bundle for function ${existingLogicFunction.id} ` +
`after ${Date.now() - installStart}ms: ` +
`${error instanceof Error ? error.message : String(error)}`,
UpdateLogicFunctionActionHandlerService.name,
error instanceof Error ? error.stack : undefined,
);
throw error;
}
}
}
@@ -1,6 +1,7 @@
import { Module } from '@nestjs/common';
import { ApplicationModule } from 'src/engine/core-modules/application/application.module';
import { FeatureFlagModule } from 'src/engine/core-modules/feature-flag/feature-flag.module';
import { WorkspaceManyOrAllFlatEntityMapsCacheModule } from 'src/engine/metadata-modules/flat-entity/services/workspace-many-or-all-flat-entity-maps-cache.module';
import { LogicFunctionModule } from 'src/engine/metadata-modules/logic-function/logic-function.module';
@@ -11,6 +12,7 @@ import { CodeStepBuildService } from './services/code-step-build.service';
ApplicationModule,
WorkspaceManyOrAllFlatEntityMapsCacheModule,
LogicFunctionModule,
FeatureFlagModule,
],
providers: [CodeStepBuildService],
exports: [CodeStepBuildService],
@@ -1,21 +1,31 @@
import { Injectable } from '@nestjs/common';
import { Injectable, Logger } from '@nestjs/common';
import { SEED_WORKFLOW_ACTION_TRIGGER_SETTINGS } from 'twenty-shared/logic-function';
import { FeatureFlagKey } from 'twenty-shared/types';
import { isDefined } from 'twenty-shared/utils';
import { FeatureFlagService } from 'src/engine/core-modules/feature-flag/services/feature-flag.service';
import { WorkspaceManyOrAllFlatEntityMapsCacheService } from 'src/engine/metadata-modules/flat-entity/services/workspace-many-or-all-flat-entity-maps-cache.service';
import { findFlatEntityByIdInFlatEntityMaps } from 'src/engine/metadata-modules/flat-entity/utils/find-flat-entity-by-id-in-flat-entity-maps.util';
import { LogicFunctionExecutionMode } from 'src/engine/metadata-modules/logic-function/logic-function.entity';
import { LogicFunctionFromSourceHelperService } from 'src/engine/metadata-modules/logic-function/services/logic-function-from-source-helper.service';
import { LogicFunctionFromSourceService } from 'src/engine/metadata-modules/logic-function/services/logic-function-from-source.service';
import { extractCodeStepLogicFunctionIdsFromWorkflowSteps } from 'src/modules/workflow/workflow-builder/workflow-version-step/code-step/utils/extract-code-step-logic-function-ids-from-workflow-steps.util';
import { type WorkflowAction } from 'src/modules/workflow/workflow-executor/workflow-actions/types/workflow-action.type';
import {
WorkflowActionType,
type WorkflowAction,
} from 'src/modules/workflow/workflow-executor/workflow-actions/types/workflow-action.type';
WorkflowTriggerException,
WorkflowTriggerExceptionCode,
} from 'src/modules/workflow/workflow-trigger/exceptions/workflow-trigger.exception';
@Injectable()
export class CodeStepBuildService {
private readonly logger = new Logger(CodeStepBuildService.name);
constructor(
private readonly workspaceManyOrAllFlatEntityMapsCacheService: WorkspaceManyOrAllFlatEntityMapsCacheService,
private readonly logicFunctionFromSourceService: LogicFunctionFromSourceService,
private readonly logicFunctionFromSourceHelperService: LogicFunctionFromSourceHelperService,
private readonly featureFlagService: FeatureFlagService,
) {}
async createCodeStepLogicFunction({
@@ -56,21 +66,10 @@ export class CodeStepBuildService {
workspaceId: string;
steps: WorkflowAction[];
}): Promise<void> {
const codeSteps = steps.filter(
(
step,
): step is WorkflowAction & {
type: typeof WorkflowActionType.CODE;
settings: { input: { logicFunctionId: string } };
} =>
step.type === WorkflowActionType.CODE &&
isDefined(
(step.settings?.input as { logicFunctionId?: string })
?.logicFunctionId,
),
);
const logicFunctionIds =
extractCodeStepLogicFunctionIdsFromWorkflowSteps(steps);
if (codeSteps.length === 0) {
if (logicFunctionIds.length === 0) {
return;
}
@@ -82,8 +81,7 @@ export class CodeStepBuildService {
},
);
for (const step of codeSteps) {
const logicFunctionId = step.settings.input.logicFunctionId;
for (const logicFunctionId of logicFunctionIds) {
const flatLogicFunction = findFlatEntityByIdInFlatEntityMaps({
flatEntityId: logicFunctionId,
flatEntityMaps: flatLogicFunctionMaps,
@@ -105,6 +103,11 @@ export class CodeStepBuildService {
: undefined;
if (!isDefined(applicationUniversalIdentifier)) {
this.logger.warn(
`Skipping build for logic function '${logicFunctionId}' (workspace=${workspaceId}): ` +
`applicationId=${flatLogicFunction.applicationId ?? 'null'} did not resolve to an application. ` +
`The function will not be rebuilt and may stay out of date.`,
);
continue;
}
@@ -114,4 +117,94 @@ export class CodeStepBuildService {
});
}
}
async switchCodeStepLogicFunctionsToPrebuilt({
workspaceId,
steps,
}: {
workspaceId: string;
steps: WorkflowAction[];
}): Promise<void> {
const isPrebuiltModeEnabled =
await this.featureFlagService.isFeatureEnabled(
FeatureFlagKey.IS_LOGIC_FUNCTION_PREBUILT_MODE_ENABLED,
workspaceId,
);
if (!isPrebuiltModeEnabled) {
return;
}
const logicFunctionIds =
extractCodeStepLogicFunctionIdsFromWorkflowSteps(steps);
if (logicFunctionIds.length === 0) {
return;
}
const { flatLogicFunctionMaps, flatApplicationMaps } =
await this.workspaceManyOrAllFlatEntityMapsCacheService.getOrRecomputeManyOrAllFlatEntityMaps(
{
workspaceId,
flatMapsKeys: ['flatLogicFunctionMaps', 'flatApplicationMaps'],
},
);
for (const logicFunctionId of logicFunctionIds) {
const flatLogicFunction = findFlatEntityByIdInFlatEntityMaps({
flatEntityId: logicFunctionId,
flatEntityMaps: flatLogicFunctionMaps,
});
if (
!isDefined(flatLogicFunction) ||
isDefined(flatLogicFunction.deletedAt)
) {
throw new WorkflowTriggerException(
`CODE step references logic function '${logicFunctionId}' that is missing or deleted`,
WorkflowTriggerExceptionCode.INVALID_WORKFLOW_VERSION,
);
}
if (
!flatLogicFunction.isBuildUpToDate ||
!isDefined(flatLogicFunction.checksum)
) {
throw new WorkflowTriggerException(
`Logic function '${logicFunctionId}' has no fresh build or checksum after the build step`,
WorkflowTriggerExceptionCode.INTERNAL_ERROR,
);
}
if (
flatLogicFunction.executionMode === LogicFunctionExecutionMode.PREBUILT
) {
continue;
}
const applicationUniversalIdentifier = isDefined(
flatLogicFunction.applicationId,
)
? flatApplicationMaps.byId[flatLogicFunction.applicationId]
?.universalIdentifier
: undefined;
if (!isDefined(applicationUniversalIdentifier)) {
throw new WorkflowTriggerException(
`Logic function '${logicFunctionId}' references applicationId='${flatLogicFunction.applicationId ?? 'null'}' that did not resolve to an application`,
WorkflowTriggerExceptionCode.INTERNAL_ERROR,
);
}
await this.logicFunctionFromSourceHelperService.updateOneFromMetadata({
flatLogicFunctionToUpdate: {
...flatLogicFunction,
executionMode: LogicFunctionExecutionMode.PREBUILT,
updatedAt: new Date().toISOString(),
},
workspaceId,
applicationUniversalIdentifier,
});
}
}
}
@@ -0,0 +1,69 @@
import { WorkflowVersionStepException } from 'src/modules/workflow/common/exceptions/workflow-version-step.exception';
import {
type CodeStepForLogicFunctionIdExtraction,
extractCodeStepLogicFunctionIdsFromWorkflowSteps,
type NonCodeStepForLogicFunctionIdExtraction,
} from 'src/modules/workflow/workflow-builder/workflow-version-step/code-step/utils/extract-code-step-logic-function-ids-from-workflow-steps.util';
import { WorkflowActionType } from 'src/modules/workflow/workflow-executor/workflow-actions/types/workflow-action.type';
const buildCodeStep = (
logicFunctionId: string,
): CodeStepForLogicFunctionIdExtraction => ({
id: 'step-id',
type: WorkflowActionType.CODE,
settings: {
input: {
logicFunctionId,
},
},
});
const buildNonCodeStep = (): NonCodeStepForLogicFunctionIdExtraction => ({
id: 'send-email-id',
type: WorkflowActionType.SEND_EMAIL,
});
describe('extractCodeStepLogicFunctionIdsFromWorkflowSteps', () => {
it('should return logic function ids of CODE steps', () => {
const steps = [
buildCodeStep('logic-function-1'),
buildCodeStep('logic-function-2'),
];
expect(extractCodeStepLogicFunctionIdsFromWorkflowSteps(steps)).toEqual([
'logic-function-1',
'logic-function-2',
]);
});
it('should ignore non-CODE steps', () => {
const steps = [
buildCodeStep('logic-function-1'),
buildNonCodeStep(),
buildCodeStep('logic-function-2'),
];
expect(extractCodeStepLogicFunctionIdsFromWorkflowSteps(steps)).toEqual([
'logic-function-1',
'logic-function-2',
]);
});
it('should throw when a CODE step has an empty logic function id', () => {
const steps = [buildCodeStep(''), buildCodeStep('logic-function-1')];
expect(() =>
extractCodeStepLogicFunctionIdsFromWorkflowSteps(steps),
).toThrow(WorkflowVersionStepException);
});
it('should return an empty array when there are no CODE steps', () => {
expect(
extractCodeStepLogicFunctionIdsFromWorkflowSteps([buildNonCodeStep()]),
).toEqual([]);
});
it('should return an empty array for an empty list of steps', () => {
expect(extractCodeStepLogicFunctionIdsFromWorkflowSteps([])).toEqual([]);
});
});
@@ -0,0 +1,54 @@
import { isNonEmptyString } from '@sniptt/guards';
import {
WorkflowVersionStepException,
WorkflowVersionStepExceptionCode,
} from 'src/modules/workflow/common/exceptions/workflow-version-step.exception';
import { type WorkflowCodeActionInput } from 'src/modules/workflow/workflow-executor/workflow-actions/code/types/workflow-code-action-input.type';
import {
type WorkflowAction,
type WorkflowCodeAction,
WorkflowActionType,
} from 'src/modules/workflow/workflow-executor/workflow-actions/types/workflow-action.type';
export type CodeStepForLogicFunctionIdExtraction = Pick<
WorkflowCodeAction,
'id' | 'type'
> & {
settings: {
input: Pick<WorkflowCodeActionInput, 'logicFunctionId'>;
};
};
export type NonCodeStepForLogicFunctionIdExtraction = Pick<
Exclude<WorkflowAction, WorkflowCodeAction>,
'id' | 'type'
>;
type WorkflowStepForLogicFunctionIdExtraction =
| CodeStepForLogicFunctionIdExtraction
| NonCodeStepForLogicFunctionIdExtraction;
const isCodeStepForLogicFunction = (
step: WorkflowStepForLogicFunctionIdExtraction,
): step is CodeStepForLogicFunctionIdExtraction =>
step.type === WorkflowActionType.CODE;
export const extractCodeStepLogicFunctionIdsFromWorkflowSteps = (
steps: WorkflowStepForLogicFunctionIdExtraction[],
): string[] => [
...new Set(
steps.filter(isCodeStepForLogicFunction).map((step) => {
const logicFunctionId = step.settings.input.logicFunctionId;
if (!isNonEmptyString(logicFunctionId)) {
throw new WorkflowVersionStepException(
`CODE step '${step.id}' has no logic function id`,
WorkflowVersionStepExceptionCode.INVALID_REQUEST,
);
}
return logicFunctionId;
}),
),
];
@@ -136,6 +136,11 @@ export class WorkflowTriggerWorkspaceService {
steps: workflowVersion.steps ?? [],
});
await this.codeStepBuildService.switchCodeStepLogicFunctionsToPrebuilt({
workspaceId,
steps: workflowVersion.steps ?? [],
});
await this.performActivationSteps(
workflow,
workflowVersion,
@@ -0,0 +1,314 @@
import gql from 'graphql-tag';
import request from 'supertest';
import {
destroyWorkflowRun,
runWorkflowVersion,
waitForWorkflowCompletion,
} from 'test/integration/graphql/suites/workflow/utils/workflow-run-test.util';
import { updateLogicFunctionSource } from 'test/integration/metadata/suites/logic-function/utils/update-logic-function-source.util';
import { makeMetadataAPIRequest } from 'test/integration/metadata/suites/utils/make-metadata-api-request.util';
import { updateFeatureFlag } from 'test/integration/metadata/suites/utils/update-feature-flag.util';
import { FeatureFlagKey } from 'twenty-shared/types';
import { LogicFunctionExecutionMode } from 'src/engine/metadata-modules/logic-function/logic-function.entity';
const client = request(`http://localhost:${APP_PORT}`);
const EXTERNAL_PACKAGES_FUNCTION_CODE = `import groupBy from 'lodash.groupby';
export const main = async (params: { items: Array<{ category: string; name: string }> }): Promise<object> => {
const grouped = groupBy(params.items, 'category');
return {
grouped,
categories: Object.keys(grouped),
};
};`;
describe('Code step workflow with PREBUILT logic function (e2e)', () => {
let createdWorkflowId: string | null = null;
let createdWorkflowVersionId: string | null = null;
let codeStepId: string | null = null;
let codeStepLogicFunctionId: string | null = null;
let createdWorkflowRunId: string | null = null;
beforeAll(async () => {
await updateFeatureFlag({
featureFlag: FeatureFlagKey.IS_LOGIC_FUNCTION_PREBUILT_MODE_ENABLED,
value: true,
expectToFail: false,
});
const createWorkflowResponse = await client
.post('/graphql')
.set('Authorization', `Bearer ${APPLE_JANE_ADMIN_ACCESS_TOKEN}`)
.send({
query: `
mutation CreateWorkflow {
createWorkflow(data: {
name: "Code Step PREBUILT Test"
}) {
id
}
}
`,
});
expect(createWorkflowResponse.body.errors).toBeUndefined();
createdWorkflowId = createWorkflowResponse.body.data.createWorkflow.id;
const getWorkflowResponse = await client
.post('/graphql')
.set('Authorization', `Bearer ${APPLE_JANE_ADMIN_ACCESS_TOKEN}`)
.send({
query: `
query GetWorkflow($id: UUID!) {
workflow(filter: { id: { eq: $id } }) {
id
versions {
edges {
node {
id
}
}
}
}
}
`,
variables: { id: createdWorkflowId },
});
expect(getWorkflowResponse.body.errors).toBeUndefined();
createdWorkflowVersionId =
getWorkflowResponse.body.data.workflow.versions.edges[0].node.id;
const manualTrigger = {
name: 'Manual Trigger',
type: 'MANUAL',
settings: {
outputSchema: {
items: { isLeaf: true, type: 'array', value: undefined },
},
},
nextStepIds: [],
position: { x: 0, y: 0 },
};
const updateTriggerResponse = await client
.post('/graphql')
.set('Authorization', `Bearer ${APPLE_JANE_ADMIN_ACCESS_TOKEN}`)
.send({
query: `
mutation UpdateWorkflowVersion($id: UUID!, $data: WorkflowVersionUpdateInput!) {
updateWorkflowVersion(id: $id, data: $data) {
id
}
}
`,
variables: {
id: createdWorkflowVersionId,
data: { trigger: manualTrigger },
},
});
expect(updateTriggerResponse.body.errors).toBeUndefined();
const createCodeStepResponse = await client
.post('/graphql')
.set('Authorization', `Bearer ${APPLE_JANE_ADMIN_ACCESS_TOKEN}`)
.send({
query: `
mutation CreateWorkflowVersionStep($input: CreateWorkflowVersionStepInput!) {
createWorkflowVersionStep(input: $input) {
stepsDiff
}
}
`,
variables: {
input: {
workflowVersionId: createdWorkflowVersionId,
stepType: 'CODE',
parentStepId: 'trigger',
position: { x: 200, y: 0 },
},
},
});
expect(createCodeStepResponse.body.errors).toBeUndefined();
const getStepsResponse = await client
.post('/graphql')
.set('Authorization', `Bearer ${APPLE_JANE_ADMIN_ACCESS_TOKEN}`)
.send({
query: `
query GetWorkflowVersion($id: UUID!) {
workflowVersion(filter: { id: { eq: $id } }) {
id
steps
}
}
`,
variables: { id: createdWorkflowVersionId },
});
expect(getStepsResponse.body.errors).toBeUndefined();
const codeStep = getStepsResponse.body.data.workflowVersion.steps.find(
(step: { type: string }) => step.type === 'CODE',
);
expect(codeStep).toBeDefined();
codeStepId = codeStep.id;
const logicFunctionId = codeStep.settings.input.logicFunctionId;
expect(logicFunctionId).toBeDefined();
codeStepLogicFunctionId = logicFunctionId;
const updateStepResponse = await client
.post('/graphql')
.set('Authorization', `Bearer ${APPLE_JANE_ADMIN_ACCESS_TOKEN}`)
.send({
query: `
mutation UpdateWorkflowVersionStep($input: UpdateWorkflowVersionStepInput!) {
updateWorkflowVersionStep(input: $input) {
id
}
}
`,
variables: {
input: {
workflowVersionId: createdWorkflowVersionId,
step: {
...codeStep,
settings: {
...codeStep.settings,
input: {
...codeStep.settings.input,
logicFunctionInput: { items: '{{trigger.items}}' },
},
},
},
},
},
});
expect(updateStepResponse.body.errors).toBeUndefined();
const updateSourceResponse = await updateLogicFunctionSource({
input: {
id: logicFunctionId,
update: {
sourceHandlerCode: EXTERNAL_PACKAGES_FUNCTION_CODE,
},
},
expectToFail: false,
});
expect(updateSourceResponse.errors).toBeUndefined();
const activateResponse = await client
.post('/graphql')
.set('Authorization', `Bearer ${APPLE_JANE_ADMIN_ACCESS_TOKEN}`)
.send({
query: `
mutation ActivateWorkflowVersion($workflowVersionId: UUID!) {
activateWorkflowVersion(workflowVersionId: $workflowVersionId)
}
`,
variables: { workflowVersionId: createdWorkflowVersionId },
});
expect(activateResponse.body.errors).toBeUndefined();
expect(activateResponse.body.data.activateWorkflowVersion).toBe(true);
});
afterAll(async () => {
if (createdWorkflowRunId) {
await destroyWorkflowRun(createdWorkflowRunId);
}
if (createdWorkflowId) {
await client
.post('/graphql')
.set('Authorization', `Bearer ${APPLE_JANE_ADMIN_ACCESS_TOKEN}`)
.send({
query: `
mutation DestroyWorkflow($id: ID!) {
destroyWorkflow(id: $id) {
id
}
}
`,
variables: { id: createdWorkflowId },
});
}
await updateFeatureFlag({
featureFlag: FeatureFlagKey.IS_LOGIC_FUNCTION_PREBUILT_MODE_ENABLED,
value: false,
expectToFail: false,
});
});
it('flips the underlying logic function to PREBUILT on workflow activation', async () => {
const findLogicFunctionResponse = await makeMetadataAPIRequest({
query: gql`
query FindOneLogicFunction($input: LogicFunctionIdInput!) {
findOneLogicFunction(input: $input) {
id
executionMode
}
}
`,
variables: { input: { id: codeStepLogicFunctionId } },
});
expect(findLogicFunctionResponse.body.errors).toBeUndefined();
const logicFunction =
findLogicFunctionResponse.body.data.findOneLogicFunction;
expect(logicFunction.executionMode).toBe(
LogicFunctionExecutionMode.PREBUILT,
);
});
it('runs the code step from its prebuilt bundle and resolves bare imports', async () => {
createdWorkflowRunId = await runWorkflowVersion({
workflowVersionId: createdWorkflowVersionId!,
payload: {
items: [
{ category: 'fruit', name: 'apple' },
{ category: 'vegetable', name: 'carrot' },
{ category: 'fruit', name: 'banana' },
],
},
});
const workflowRun = await waitForWorkflowCompletion(createdWorkflowRunId);
expect(workflowRun?.status).toBe('COMPLETED');
expect(workflowRun?.state?.stepInfos?.trigger?.status).toBe('SUCCESS');
expect(workflowRun?.state?.stepInfos?.[codeStepId!]?.status).toBe(
'SUCCESS',
);
const stepResult = workflowRun?.state?.stepInfos?.[codeStepId!]?.result as
| {
grouped?: Record<string, Array<{ category: string; name: string }>>;
categories?: string[];
}
| undefined;
expect(stepResult?.grouped).toMatchObject({
fruit: [
{ category: 'fruit', name: 'apple' },
{ category: 'fruit', name: 'banana' },
],
vegetable: [{ category: 'vegetable', name: 'carrot' }],
});
expect(stepResult?.categories).toEqual(
expect.arrayContaining(['fruit', 'vegetable']),
);
});
});
@@ -4,6 +4,7 @@ import { executeLogicFunction } from 'test/integration/metadata/suites/logic-fun
import { updateLogicFunctionSource } from 'test/integration/metadata/suites/logic-function/utils/update-logic-function-source.util';
import { LogicFunctionExecutionStatus } from 'src/engine/metadata-modules/logic-function/dtos/logic-function-execution-result.dto';
import { LogicFunctionExecutionMode } from 'src/engine/metadata-modules/logic-function/logic-function.entity';
// Default template function code that matches the expected behavior
const DEFAULT_TEMPLATE_FUNCTION_CODE = `export const main = async (params: { a: string; b: number }): Promise<object> => {
@@ -58,6 +59,9 @@ describe('Logic Function Execution', () => {
const functionId = createData?.createOneLogicFunction?.id;
expect(functionId).toBeDefined();
expect(createData?.createOneLogicFunction?.executionMode).toBe(
LogicFunctionExecutionMode.LIVE,
);
createdFunctionIds.push(functionId);
await updateLogicFunctionSource({
@@ -290,4 +294,5 @@ describe('Logic Function Execution', () => {
});
expect(errorResult?.data).toBeNull();
});
});
@@ -11,6 +11,7 @@ const LOGIC_FUNCTION_GQL_FIELDS = `
name
description
runtime
executionMode
createdAt
updatedAt
`;
@@ -6,5 +6,6 @@ export enum FeatureFlagKey {
IS_EMAIL_GROUP_ENABLED = 'IS_EMAIL_GROUP_ENABLED',
IS_JUNCTION_RELATIONS_ENABLED = 'IS_JUNCTION_RELATIONS_ENABLED',
IS_REST_METADATA_API_NEW_FORMAT_DIRECT = 'IS_REST_METADATA_API_NEW_FORMAT_DIRECT',
IS_LOGIC_FUNCTION_PREBUILT_MODE_ENABLED = 'IS_LOGIC_FUNCTION_PREBUILT_MODE_ENABLED',
IS_SETTINGS_DISCOVERY_HERO_ENABLED = 'IS_SETTINGS_DISCOVERY_HERO_ENABLED',
}