Reorganize logic function files (#17766)

reorganize according to

<img width="1243" height="725" alt="Pasted Graphic"
src="https://github.com/user-attachments/assets/ba65dd10-8eec-4b13-ad49-9726edd3b79c"
/>

Not working yet
This commit is contained in:
martmull
2026-02-09 12:36:39 +01:00
committed by GitHub
parent bf4c348c8b
commit 9162685b2e
149 changed files with 1715 additions and 5272 deletions
@@ -19,7 +19,11 @@ export const handler = async (event) => {
const mainFile = await import(mainPath);
return await mainFile[handlerName](params);
const handlerFn = handlerName
.split('.')
.reduce((obj, key) => obj[key], mainFile);
return await handlerFn(params);
} finally {
await fs.rm(mainPath, { force: true });
// eslint-disable-next-line no-undef
@@ -1 +0,0 @@
export const NODE_LAYER_SUBFOLDER = 'nodejs';
@@ -0,0 +1 @@
export const LOGIC_FUNCTION_DRIVER = Symbol('LOGIC_FUNCTION_DRIVER');
@@ -1,14 +1,14 @@
import {
type LogicFunctionExecutorDriver,
type LogicFunctionDriver,
type LogicFunctionExecuteResult,
} from 'src/engine/core-modules/logic-function/logic-function-drivers/interfaces/logic-function-executor-driver.interface';
} from 'src/engine/core-modules/logic-function/logic-function-drivers/interfaces/logic-function-driver.interface';
import {
LogicFunctionException,
LogicFunctionExceptionCode,
} from 'src/engine/metadata-modules/logic-function/logic-function.exception';
export class DisabledDriver implements LogicFunctionExecutorDriver {
export class DisabledDriver implements LogicFunctionDriver {
async delete(): Promise<void> {
// No-op when disabled
}
@@ -19,22 +19,18 @@ import {
waitUntilFunctionUpdatedV2,
} from '@aws-sdk/client-lambda';
import { AssumeRoleCommand, STSClient } from '@aws-sdk/client-sts';
import { FileFolder } from 'twenty-shared/types';
import { isDefined } from 'twenty-shared/utils';
import {
type LogicFunctionExecuteParams,
type LogicFunctionExecuteResult,
type LogicFunctionExecutorDriver,
} from 'src/engine/core-modules/logic-function/logic-function-drivers/interfaces/logic-function-executor-driver.interface';
type LogicFunctionDriver,
} from 'src/engine/core-modules/logic-function/logic-function-drivers/interfaces/logic-function-driver.interface';
import { type FlatApplication } from 'src/engine/core-modules/application/types/flat-application.type';
import { type FileStorageService } from 'src/engine/core-modules/file-storage/file-storage.service';
import { NODE_LAYER_SUBFOLDER } from 'src/engine/core-modules/logic-function/logic-function-drivers/constants/lambda-layer.constant';
import { copyExecutor } from 'src/engine/core-modules/logic-function/logic-function-drivers/utils/copy-executor';
import { copyYarnEngineAndBuildDependencies } from 'src/engine/core-modules/logic-function/logic-function-drivers/utils/copy-yarn-engine-and-build-dependencies';
import { createZipFile } from 'src/engine/core-modules/logic-function/logic-function-drivers/utils/create-zip-file';
import { LambdaBuildDirectoryManager } from 'src/engine/core-modules/logic-function/logic-function-drivers/utils/lambda-build-directory-manager';
import { TemporaryDirManager } from 'src/engine/core-modules/logic-function/logic-function-drivers/utils/temporary-dir-manager';
import { LogicFunctionExecutionStatus } from 'src/engine/metadata-modules/logic-function/dtos/logic-function-execution-result.dto';
import { LogicFunctionRuntime } from 'src/engine/metadata-modules/logic-function/logic-function.entity';
import {
@@ -42,7 +38,8 @@ import {
LogicFunctionExceptionCode,
} from 'src/engine/metadata-modules/logic-function/logic-function.exception';
import { type FlatLogicFunction } from 'src/engine/metadata-modules/logic-function/types/flat-logic-function.type';
import { streamToBuffer } from 'src/utils/stream-to-buffer';
import { copyYarnEngineAndBuildDependencies } from 'src/engine/core-modules/application-layer/utils/copy-yarn-engine-and-build-dependencies';
import { type LogicFunctionResourceService } from 'src/engine/core-modules/logic-function/logic-function-resource/logic-function-resource.service';
const UPDATE_FUNCTION_DURATION_TIMEOUT_IN_SECONDS = 60;
const CREDENTIALS_DURATION_IN_SECONDS = 60 * 60; // 1h
@@ -55,22 +52,22 @@ type LambdaDriverExecutorPayload = {
};
export interface LambdaDriverOptions extends LambdaClientConfig {
fileStorageService: FileStorageService;
logicFunctionResourceService: LogicFunctionResourceService;
region: string;
lambdaRole: string;
subhostingRole?: string;
}
export class LambdaDriver implements LogicFunctionExecutorDriver {
export class LambdaDriver implements LogicFunctionDriver {
private lambdaClient: Lambda | undefined;
private credentialsExpiry: Date | null = null;
private readonly options: LambdaDriverOptions;
private readonly fileStorageService: FileStorageService;
private readonly logicFunctionResourceService: LogicFunctionResourceService;
constructor(options: LambdaDriverOptions) {
this.options = options;
this.lambdaClient = undefined;
this.fileStorageService = options.fileStorageService;
this.logicFunctionResourceService = options.logicFunctionResourceService;
}
private async getLambdaClient() {
@@ -140,33 +137,6 @@ export class LambdaDriver implements LogicFunctionExecutorDriver {
return flatApplication.yarnLockChecksum ?? 'default';
}
private async copyDependenciesInMemory({
applicationUniversalIdentifier,
workspaceId,
inMemoryLayerFolderPath,
}: {
applicationUniversalIdentifier: string;
workspaceId: string;
inMemoryLayerFolderPath: string;
}) {
await Promise.all([
this.fileStorageService.downloadFile({
workspaceId,
applicationUniversalIdentifier,
fileFolder: FileFolder.Dependencies,
resourcePath: 'package.json',
localPath: join(inMemoryLayerFolderPath, 'package.json'),
}),
this.fileStorageService.downloadFile({
workspaceId,
applicationUniversalIdentifier,
fileFolder: FileFolder.Dependencies,
resourcePath: 'yarn.lock',
localPath: join(inMemoryLayerFolderPath, 'yarn.lock'),
}),
]);
}
private async createLayerIfNotExists({
flatApplication,
applicationUniversalIdentifier,
@@ -191,19 +161,16 @@ export class LambdaDriver implements LogicFunctionExecutorDriver {
return listLayerResult.LayerVersions[0].LayerVersionArn;
}
const buildTemporaryDirectoryManager = new LambdaBuildDirectoryManager();
const temporaryDirManager = new TemporaryDirManager();
const { sourceTemporaryDir, lambdaZipPath } =
await buildTemporaryDirectoryManager.init();
await temporaryDirManager.init();
const nodeDependenciesFolder = join(
sourceTemporaryDir,
NODE_LAYER_SUBFOLDER,
);
const nodeDependenciesFolder = join(sourceTemporaryDir, 'nodejs');
await this.copyDependenciesInMemory({
await this.logicFunctionResourceService.copyDependenciesInMemory({
applicationUniversalIdentifier,
workspaceId: flatApplication.workspaceId,
inMemoryLayerFolderPath: nodeDependenciesFolder,
inMemoryFolderPath: nodeDependenciesFolder,
});
await copyYarnEngineAndBuildDependencies(nodeDependenciesFolder);
@@ -224,7 +191,7 @@ export class LambdaDriver implements LogicFunctionExecutorDriver {
const result = await (await this.getLambdaClient()).send(command);
await buildTemporaryDirectoryManager.clean();
await temporaryDirManager.clean();
if (!isDefined(result.LayerVersionArn)) {
throw new Error('new layer version arn if undefined');
@@ -306,10 +273,10 @@ export class LambdaDriver implements LogicFunctionExecutorDriver {
applicationUniversalIdentifier,
});
const buildTemporaryDirectoryManager = new LambdaBuildDirectoryManager();
const temporaryDirManager = new TemporaryDirManager();
const { sourceTemporaryDir, lambdaZipPath } =
await buildTemporaryDirectoryManager.init();
await temporaryDirManager.init();
await copyExecutor(sourceTemporaryDir);
@@ -331,7 +298,7 @@ export class LambdaDriver implements LogicFunctionExecutorDriver {
await (await this.getLambdaClient()).send(command);
await buildTemporaryDirectoryManager.clean();
await temporaryDirManager.clean();
}
private extractLogs(logString: string): string {
@@ -366,16 +333,11 @@ export class LambdaDriver implements LogicFunctionExecutorDriver {
const startTime = Date.now();
const compiledCode = (
await streamToBuffer(
await this.fileStorageService.readFile({
workspaceId: flatLogicFunction.workspaceId,
applicationUniversalIdentifier,
fileFolder: FileFolder.BuiltLogicFunction,
resourcePath: flatLogicFunction.builtHandlerPath,
}),
)
).toString('utf-8');
const compiledCode = await this.logicFunctionResourceService.getBuiltCode({
workspaceId: flatLogicFunction.workspaceId,
applicationUniversalIdentifier,
builtHandlerPath: flatLogicFunction.builtHandlerPath,
});
const executorPayload: LambdaDriverExecutorPayload = {
params: payload,
@@ -1,34 +1,30 @@
import { promises as fs } from 'fs';
import { spawn } from 'node:child_process';
import { join } from 'path';
import { dirname } from 'node:path';
import { FileFolder } from 'twenty-shared/types';
import {
type LogicFunctionExecuteParams,
type LogicFunctionExecuteResult,
type LogicFunctionExecutorDriver,
} from 'src/engine/core-modules/logic-function/logic-function-drivers/interfaces/logic-function-executor-driver.interface';
type LogicFunctionDriver,
} from 'src/engine/core-modules/logic-function/logic-function-drivers/interfaces/logic-function-driver.interface';
import { type FlatApplication } from 'src/engine/core-modules/application/types/flat-application.type';
import { type FileStorageService } from 'src/engine/core-modules/file-storage/file-storage.service';
import { LOGIC_FUNCTION_EXECUTOR_TMPDIR_FOLDER } from 'src/engine/core-modules/logic-function/logic-function-drivers/constants/logic-function-executor-tmpdir-folder';
import { copyYarnEngineAndBuildDependencies } from 'src/engine/core-modules/logic-function/logic-function-drivers/utils/copy-yarn-engine-and-build-dependencies';
import { ConsoleListener } from 'src/engine/core-modules/logic-function/logic-function-drivers/utils/intercept-console';
import { LambdaBuildDirectoryManager } from 'src/engine/core-modules/logic-function/logic-function-drivers/utils/lambda-build-directory-manager';
import { TemporaryDirManager } from 'src/engine/core-modules/logic-function/logic-function-drivers/utils/temporary-dir-manager';
import { LogicFunctionExecutionStatus } from 'src/engine/metadata-modules/logic-function/dtos/logic-function-execution-result.dto';
import { getRelativePathFromBase } from 'src/modules/workflow/workflow-builder/workflow-version-step/code-step/utils/get-code-step-handler-path.util';
import { copyYarnEngineAndBuildDependencies } from 'src/engine/core-modules/application-layer/utils/copy-yarn-engine-and-build-dependencies';
import type { LogicFunctionResourceService } from 'src/engine/core-modules/logic-function/logic-function-resource/logic-function-resource.service';
export interface LocalDriverOptions {
fileStorageService: FileStorageService;
logicFunctionResourceService: LogicFunctionResourceService;
}
export class LocalDriver implements LogicFunctionExecutorDriver {
private readonly fileStorageService: FileStorageService;
export class LocalDriver implements LogicFunctionDriver {
private readonly logicFunctionResourceService: LogicFunctionResourceService;
constructor(options: LocalDriverOptions) {
this.fileStorageService = options.fileStorageService;
this.logicFunctionResourceService = options.logicFunctionResourceService;
}
private getInMemoryLayerFolderPath = (flatApplication: FlatApplication) => {
@@ -37,33 +33,6 @@ export class LocalDriver implements LogicFunctionExecutorDriver {
return join(LOGIC_FUNCTION_EXECUTOR_TMPDIR_FOLDER, checksum);
};
private async copyDependenciesInMemory({
applicationUniversalIdentifier,
workspaceId,
inMemoryLayerFolderPath,
}: {
applicationUniversalIdentifier: string;
workspaceId: string;
inMemoryLayerFolderPath: string;
}) {
await Promise.all([
this.fileStorageService.downloadFile({
workspaceId,
applicationUniversalIdentifier,
fileFolder: FileFolder.Dependencies,
resourcePath: 'package.json',
localPath: join(inMemoryLayerFolderPath, 'package.json'),
}),
this.fileStorageService.downloadFile({
workspaceId,
applicationUniversalIdentifier,
fileFolder: FileFolder.Dependencies,
resourcePath: 'yarn.lock',
localPath: join(inMemoryLayerFolderPath, 'yarn.lock'),
}),
]);
}
private async createLayerIfNotExists({
flatApplication,
applicationUniversalIdentifier,
@@ -77,10 +46,10 @@ export class LocalDriver implements LogicFunctionExecutorDriver {
try {
await fs.access(inMemoryLayerFolderPath);
} catch {
await this.copyDependenciesInMemory({
await this.logicFunctionResourceService.copyDependenciesInMemory({
applicationUniversalIdentifier,
workspaceId: flatApplication.workspaceId,
inMemoryLayerFolderPath,
inMemoryFolderPath: inMemoryLayerFolderPath,
});
await copyYarnEngineAndBuildDependencies(inMemoryLayerFolderPath);
}
@@ -115,19 +84,21 @@ export class LocalDriver implements LogicFunctionExecutorDriver {
const startTime = Date.now();
const lambdaBuildDirectoryManager = new LambdaBuildDirectoryManager();
const temporaryDirManager = new TemporaryDirManager();
try {
const { sourceTemporaryDir } = await lambdaBuildDirectoryManager.init();
const { sourceTemporaryDir } = await temporaryDirManager.init();
const baseFolderPath = dirname(flatLogicFunction.builtHandlerPath);
const inMemoryBuiltHandlerPath = join(
sourceTemporaryDir,
flatLogicFunction.builtHandlerPath,
);
await this.fileStorageService.downloadFolder({
await this.logicFunctionResourceService.copyBuiltCodeInMemory({
workspaceId: flatLogicFunction.workspaceId,
applicationUniversalIdentifier,
fileFolder: FileFolder.BuiltLogicFunction,
resourcePath: baseFolderPath,
localPath: sourceTemporaryDir,
builtHandlerPath: flatLogicFunction.builtHandlerPath,
inMemoryDestinationPath: inMemoryBuiltHandlerPath,
});
try {
@@ -179,15 +150,9 @@ export class LocalDriver implements LogicFunctionExecutorDriver {
});
try {
const relativeBuiltPath = getRelativePathFromBase(
flatLogicFunction.builtHandlerPath,
baseFolderPath,
);
const builtBundleFilePath = join(sourceTemporaryDir, relativeBuiltPath);
const runnerPath = await this.writeBootstrapRunner({
dir: sourceTemporaryDir,
builtFileAbsPath: builtBundleFilePath,
builtFileAbsPath: inMemoryBuiltHandlerPath,
handlerName: flatLogicFunction.handlerName,
});
@@ -240,7 +205,7 @@ export class LocalDriver implements LogicFunctionExecutorDriver {
consoleListener.release();
}
} finally {
await lambdaBuildDirectoryManager.clean();
await temporaryDirManager.clean();
}
}
@@ -1,33 +1,33 @@
import { fromNodeProviderChain } from '@aws-sdk/credential-providers';
import {
LogicFunctionExecutorDriverType,
type LogicFunctionExecutorModuleOptions,
} from 'src/engine/core-modules/logic-function/logic-function-executor/interfaces/logic-function-executor.interface';
LogicFunctionDriverType,
type LogicFunctionModuleOptions,
} from 'src/engine/core-modules/logic-function/logic-function-drivers/interfaces/logic-function-driver.interface';
import { type FileStorageService } from 'src/engine/core-modules/file-storage/file-storage.service';
import { type TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service';
import type { LogicFunctionResourceService } from 'src/engine/core-modules/logic-function/logic-function-resource/logic-function-resource.service';
export const logicFunctionModuleFactory = async (
twentyConfigService: TwentyConfigService,
fileStorageService: FileStorageService,
): Promise<LogicFunctionExecutorModuleOptions> => {
logicFunctionResourceService: LogicFunctionResourceService,
): Promise<LogicFunctionModuleOptions> => {
const driverType = twentyConfigService.get('LOGIC_FUNCTION_TYPE');
const options = { fileStorageService };
const options = { logicFunctionResourceService };
switch (driverType) {
case LogicFunctionExecutorDriverType.DISABLED: {
case LogicFunctionDriverType.DISABLED: {
return {
type: LogicFunctionExecutorDriverType.DISABLED,
type: LogicFunctionDriverType.DISABLED,
};
}
case LogicFunctionExecutorDriverType.LOCAL: {
case LogicFunctionDriverType.LOCAL: {
return {
type: LogicFunctionExecutorDriverType.LOCAL,
type: LogicFunctionDriverType.LOCAL,
options,
};
}
case LogicFunctionExecutorDriverType.LAMBDA: {
case LogicFunctionDriverType.LAMBDA: {
const region = twentyConfigService.get('LOGIC_FUNCTION_LAMBDA_REGION');
const accessKeyId = twentyConfigService.get(
'LOGIC_FUNCTION_LAMBDA_ACCESS_KEY_ID',
@@ -42,7 +42,7 @@ export const logicFunctionModuleFactory = async (
);
return {
type: LogicFunctionExecutorDriverType.LAMBDA,
type: LogicFunctionDriverType.LAMBDA,
options: {
...options,
credentials: accessKeyId
@@ -0,0 +1,69 @@
import type { FactoryProvider, ModuleMetadata } from '@nestjs/common';
import { type FlatApplication } from 'src/engine/core-modules/application/types/flat-application.type';
import { type LogicFunctionExecutionStatus } from 'src/engine/metadata-modules/logic-function/dtos/logic-function-execution-result.dto';
import { type FlatLogicFunction } from 'src/engine/metadata-modules/logic-function/types/flat-logic-function.type';
import type { LocalDriverOptions } from 'src/engine/core-modules/logic-function/logic-function-drivers/drivers/local.driver';
import type { LambdaDriverOptions } from 'src/engine/core-modules/logic-function/logic-function-drivers/drivers/lambda.driver';
export type LogicFunctionExecuteError = {
errorType: string;
errorMessage: string;
stackTrace: string | string[];
};
export type LogicFunctionExecuteResult = {
data: object | null;
duration: number;
logs: string;
status: LogicFunctionExecutionStatus;
error?: LogicFunctionExecuteError;
};
export type LogicFunctionExecuteParams = {
flatLogicFunction: FlatLogicFunction;
flatApplication: FlatApplication;
applicationUniversalIdentifier: string;
payload: object;
env?: Record<string, string>;
};
export interface LogicFunctionDriver {
delete(flatLogicFunction: FlatLogicFunction): Promise<void>;
execute(
params: LogicFunctionExecuteParams,
): Promise<LogicFunctionExecuteResult>;
}
export enum LogicFunctionDriverType {
DISABLED = 'DISABLED',
LAMBDA = 'LAMBDA',
LOCAL = 'LOCAL',
}
export interface DisabledDriverFactoryOptions {
type: LogicFunctionDriverType.DISABLED;
}
export interface LocalDriverFactoryOptions {
type: LogicFunctionDriverType.LOCAL;
options: LocalDriverOptions;
}
export interface LambdaDriverFactoryOptions {
type: LogicFunctionDriverType.LAMBDA;
options: LambdaDriverOptions;
}
export type LogicFunctionModuleOptions =
| DisabledDriverFactoryOptions
| LocalDriverFactoryOptions
| LambdaDriverFactoryOptions;
export type LogicFunctionModuleAsyncOptions = {
useFactory: (
// eslint-disable-next-line @typescript-eslint/no-explicit-any
...args: any[]
) => LogicFunctionModuleOptions | Promise<LogicFunctionModuleOptions>;
} & Pick<ModuleMetadata, 'imports'> &
Pick<FactoryProvider, 'inject'>;
@@ -1,32 +0,0 @@
import { type FlatApplication } from 'src/engine/core-modules/application/types/flat-application.type';
import { type LogicFunctionExecutionStatus } from 'src/engine/metadata-modules/logic-function/dtos/logic-function-execution-result.dto';
import { type FlatLogicFunction } from 'src/engine/metadata-modules/logic-function/types/flat-logic-function.type';
export type LogicFunctionExecuteError = {
errorType: string;
errorMessage: string;
stackTrace: string | string[];
};
export type LogicFunctionExecuteResult = {
data: object | null;
duration: number;
logs: string;
status: LogicFunctionExecutionStatus;
error?: LogicFunctionExecuteError;
};
export type LogicFunctionExecuteParams = {
flatLogicFunction: FlatLogicFunction;
flatApplication: FlatApplication;
applicationUniversalIdentifier: string;
payload: object;
env?: Record<string, string>;
};
export interface LogicFunctionExecutorDriver {
delete(flatLogicFunction: FlatLogicFunction): Promise<void>;
execute(
params: LogicFunctionExecuteParams,
): Promise<LogicFunctionExecuteResult>;
}
@@ -1,45 +0,0 @@
{
"dependencies": {
"@types/bcrypt": "^5.0.2",
"@types/deep-equal": "^1.0.4",
"@types/lodash.camelcase": "^4.3.9",
"@types/lodash.compact": "^3.0.9",
"@types/lodash.groupby": "^4.6.9",
"@types/lodash.identity": "^3.0.9",
"@types/lodash.isempty": "^4.4.9",
"@types/lodash.isequal": "^4.5.8",
"@types/lodash.isobject": "^3.0.9",
"@types/lodash.kebabcase": "^4.1.9",
"@types/lodash.mapvalues": "^4.6.9",
"@types/lodash.omit": "^4.5.9",
"@types/lodash.pickby": "^4.6.9",
"@types/lodash.snakecase": "^4.1.9",
"@types/lodash.upperfirst": "^4.3.9",
"@types/uuid": "^10.0.0",
"archiver": "^7.0.1",
"axios": "^1.12.0",
"bcrypt": "^5.1.1",
"body-parser": "^1.20.4",
"deep-equal": "^2.2.3",
"jsonwebtoken": "^9.0.2",
"lodash.camelcase": "^4.3.0",
"lodash.chunk": "^4.2.0",
"lodash.compact": "^3.0.1",
"lodash.groupby": "^4.6.0",
"lodash.identity": "^3.0.0",
"lodash.isempty": "^4.4.0",
"lodash.isequal": "^4.5.0",
"lodash.isobject": "^3.0.2",
"lodash.kebabcase": "^4.1.1",
"lodash.mapvalues": "^4.6.0",
"lodash.merge": "^4.6.2",
"lodash.omit": "^4.5.0",
"lodash.pickby": "^4.6.0",
"lodash.snakecase": "^4.1.1",
"lodash.upperfirst": "^4.3.1",
"nodemailer": "^7.0.11",
"sharp": "^0.33.5",
"uuid": "^10.0.0",
"winston": "^3.14.2"
}
}
@@ -1,5 +0,0 @@
enableInlineHunks: true
nodeLinker: node-modules
yarnPath: .yarn/releases/yarn-4.9.2.cjs
@@ -1 +0,0 @@
export const LAST_LAYER_VERSION = 1;
@@ -1,32 +1,30 @@
import { type DynamicModule, Module } from '@nestjs/common';
import {
LogicFunctionExecutorDriverType,
type LogicFunctionExecutorModuleAsyncOptions,
} from 'src/engine/core-modules/logic-function/logic-function-executor/interfaces/logic-function-executor.interface';
LogicFunctionDriverType,
LogicFunctionModuleAsyncOptions,
} from 'src/engine/core-modules/logic-function/logic-function-drivers/interfaces/logic-function-driver.interface';
import { DisabledDriver } from 'src/engine/core-modules/logic-function/logic-function-drivers/drivers/disabled.driver';
import { LambdaDriver } from 'src/engine/core-modules/logic-function/logic-function-drivers/drivers/lambda.driver';
import { LocalDriver } from 'src/engine/core-modules/logic-function/logic-function-drivers/drivers/local.driver';
import { LOGIC_FUNCTION_EXECUTOR_DRIVER } from 'src/engine/core-modules/logic-function/logic-function-executor/constants/logic-function-executor.constants';
import { LOGIC_FUNCTION_DRIVER } from 'src/engine/core-modules/logic-function/logic-function-drivers/constants/logic-function-driver.constants';
@Module({})
export class LogicFunctionDriversModule {
static forRootAsync(
options: LogicFunctionExecutorModuleAsyncOptions,
): DynamicModule {
static forRootAsync(options: LogicFunctionModuleAsyncOptions): DynamicModule {
const provider = {
provide: LOGIC_FUNCTION_EXECUTOR_DRIVER,
provide: LOGIC_FUNCTION_DRIVER,
// eslint-disable-next-line @typescript-eslint/no-explicit-any
useFactory: async (...args: any[]) => {
const config = await options.useFactory(...args);
switch (config?.type) {
case LogicFunctionExecutorDriverType.DISABLED:
case LogicFunctionDriverType.DISABLED:
return new DisabledDriver();
case LogicFunctionExecutorDriverType.LOCAL:
case LogicFunctionDriverType.LOCAL:
return new LocalDriver(config.options);
case LogicFunctionExecutorDriverType.LAMBDA:
case LogicFunctionDriverType.LAMBDA:
return new LambdaDriver(config.options);
default: {
const unknownConfig = config as { type?: string };
@@ -44,7 +42,7 @@ export class LogicFunctionDriversModule {
module: LogicFunctionDriversModule,
imports: options.imports || [],
providers: [provider],
exports: [LOGIC_FUNCTION_EXECUTOR_DRIVER],
exports: [LOGIC_FUNCTION_DRIVER],
};
}
}
@@ -1,52 +0,0 @@
import { execFile } from 'child_process';
import { promises as fs, statSync } from 'fs';
import { join } from 'path';
import { promisify } from 'util';
import { getLayerDependenciesDirName } from 'src/engine/core-modules/logic-function/logic-function-drivers/utils/get-layer-dependencies-dir-name';
const execFilePromise = promisify(execFile);
export const copyYarnEngineAndBuildDependencies = async (
buildDirectory: string,
) => {
await fs.mkdir(buildDirectory, {
recursive: true,
});
await fs.cp(getLayerDependenciesDirName('engine'), buildDirectory, {
recursive: true,
});
const localYarnPath = join(buildDirectory, '.yarn/releases/yarn-4.9.2.cjs');
// Strip NODE_OPTIONS to prevent tsx loader from interfering with yarn
const { NODE_OPTIONS: _nodeOptions, ...cleanEnv } = process.env;
try {
await execFilePromise(process.execPath, [localYarnPath], {
cwd: buildDirectory,
env: cleanEnv,
});
// eslint-disable-next-line @typescript-eslint/no-explicit-any
} catch (error: any) {
const errorMessage =
[error?.stdout, error?.stderr].filter(Boolean).join('\n') ||
'Failed to install logic function executor dependencies';
throw new Error(errorMessage);
}
const objects = await fs.readdir(buildDirectory);
await Promise.all(
objects
.filter((object) => object !== 'node_modules')
.map((object) => {
const fullPath = join(buildDirectory, object);
return statSync(fullPath).isDirectory()
? fs.rm(fullPath, { recursive: true, force: true })
: fs.rm(fullPath);
}),
);
};
@@ -1,22 +0,0 @@
import fs from 'fs/promises';
import { join } from 'path';
import { getLayerDependenciesDirName } from 'src/engine/core-modules/logic-function/logic-function-drivers/utils/get-layer-dependencies-dir-name';
import { LAST_LAYER_VERSION } from 'src/engine/core-modules/logic-function/logic-function-drivers/layers/last-layer-version';
export type LayerDependencies = {
packageJson: string;
yarnLock: string;
};
export const getLastCommonLayerDependencies = async (
layerVersion = LAST_LAYER_VERSION,
): Promise<LayerDependencies> => {
const lastVersionLayerDirName = getLayerDependenciesDirName(layerVersion);
const [packageJson, yarnLock] = await Promise.all([
fs.readFile(join(lastVersionLayerDirName, 'package.json'), 'utf8'),
fs.readFile(join(lastVersionLayerDirName, 'yarn.lock'), 'utf8'),
]);
return { packageJson, yarnLock };
};
@@ -1,14 +0,0 @@
import path from 'path';
import { ASSET_PATH } from 'src/constants/assets-path';
export const getLayerDependenciesDirName = (
version: 'engine' | number,
): string => {
const baseTypescriptProjectPath = path.join(
ASSET_PATH,
`engine/core-modules/logic-function/logic-function-drivers/layers/${version}`,
);
return path.resolve(__dirname, baseTypescriptProjectPath);
};
@@ -5,12 +5,10 @@ import { v4 } from 'uuid';
import { LOGIC_FUNCTION_EXECUTOR_TMPDIR_FOLDER } from 'src/engine/core-modules/logic-function/logic-function-drivers/constants/logic-function-executor-tmpdir-folder';
export const NODE_LAYER_SUBFOLDER = 'nodejs';
const TEMPORARY_LAMBDA_FOLDER = 'lambda-build';
const LAMBDA_ZIP_FILE_NAME = 'lambda.zip';
export class LambdaBuildDirectoryManager {
export class TemporaryDirManager {
private temporaryDir = join(
LOGIC_FUNCTION_EXECUTOR_TMPDIR_FOLDER,
`${TEMPORARY_LAMBDA_FOLDER}-${v4()}`,
@@ -1,135 +0,0 @@
import { Logger } from '@nestjs/common';
import { execFile } from 'child_process';
import * as fs from 'fs/promises';
import { resolve } from 'path';
import { promisify } from 'util';
import { Command, CommandRunner, Option } from 'nest-commander';
const execFilePromise = promisify(execFile);
@Command({
name: 'logic-function-executor:add-packages',
description:
'Create a new logic function executor layer version and install packages in it',
})
export class AddPackagesCommand extends CommandRunner {
private readonly logger = new Logger(AddPackagesCommand.name);
@Option({
flags: '-p, --packages <packages>',
description: 'comma separated packages (eg: axios,uuid@9.0.1)',
required: true,
})
parsePackages(val: string): string[] {
return val.split(',');
}
async run(
_passedParams: string[],
// eslint-disable-next-line @typescript-eslint/no-explicit-any
options: Record<string, any>,
): Promise<void> {
this.logger.log('---------------------------------------');
this.logger.warn('This command should be run locally only');
this.logger.log('');
const layersFolder = this.getAbsoluteFilePath(
`src/engine/core-modules/logic-function/logic-function-drivers/layers`,
);
const currentVersion = await this.getLastLayerVersion();
const newVersion = currentVersion + 1;
const currentVersionFolder = `${layersFolder}/${currentVersion}`;
const newVersionFolder = `${layersFolder}/${newVersion}`;
await fs.cp(currentVersionFolder, newVersionFolder, { recursive: true });
// Install each package
this.logger.log('Installing packages');
await this.installPackages(options.packages, newVersionFolder);
this.logger.log('Cleaning');
await this.cleanPackageInstallation(newVersionFolder);
this.logger.log('Updating last layer version');
await this.updateLastLayerVersion(newVersion);
this.logger.log('Add changes to git');
await this.addToGit(layersFolder);
this.logger.log('');
this.logger.log(
`New packages '${options.packages.join("', '")}' installed in new layer version '${newVersion}' `,
);
this.logger.log('Please commit your changes');
this.logger.log('---------------------------------------');
}
private getAbsoluteFilePath(path: string) {
const rootPath = process.cwd();
return resolve(rootPath, path);
}
private async addToGit(folderPath: string) {
await execFilePromise('git', ['add', folderPath]);
}
private async cleanPackageInstallation(folderPath: string) {
await fs.rm(folderPath + '/node_modules', {
recursive: true,
force: true,
});
await fs.rm(folderPath + '/.yarn', {
recursive: true,
force: true,
});
}
private async installPackages(packages: string[], folderPath: string) {
if (packages?.length) {
for (const packageName of packages) {
this.logger.log(`- adding '${packageName}'...`);
try {
await execFilePromise('yarn', ['add', packageName], {
cwd: folderPath,
});
} catch (error) {
this.logger.error(
`Failed to install ${packageName}: ${(error as Error).message}`,
);
}
}
}
}
private async getLastLayerVersion() {
const filePath = this.getAbsoluteFilePath(
'src/engine/core-modules/logic-function/logic-function-drivers/layers/last-layer-version.ts',
);
const content = await fs.readFile(filePath, 'utf8');
const match = content.match(/export const LAST_LAYER_VERSION = (\d+);/);
if (!match) {
throw new Error('LAST_LAYER_VERSION not found');
}
return parseInt(match[1], 10);
}
private async updateLastLayerVersion(newVersion: number) {
const filePath = this.getAbsoluteFilePath(
'src/engine/core-modules/logic-function/logic-function-drivers/layers/last-layer-version.ts',
);
await fs.writeFile(
filePath,
`export const LAST_LAYER_VERSION = ${newVersion};\n`,
'utf8',
);
}
}
@@ -1,3 +0,0 @@
export const LOGIC_FUNCTION_EXECUTOR_DRIVER = Symbol(
'LOGIC_FUNCTION_EXECUTOR_DRIVER',
);
@@ -1,39 +0,0 @@
import { type FactoryProvider, type ModuleMetadata } from '@nestjs/common';
import { type LambdaDriverOptions } from 'src/engine/core-modules/logic-function/logic-function-drivers/drivers/lambda.driver';
import { type LocalDriverOptions } from 'src/engine/core-modules/logic-function/logic-function-drivers/drivers/local.driver';
export enum LogicFunctionExecutorDriverType {
DISABLED = 'DISABLED',
LAMBDA = 'LAMBDA',
LOCAL = 'LOCAL',
}
export interface DisabledDriverFactoryOptions {
type: LogicFunctionExecutorDriverType.DISABLED;
}
export interface LocalDriverFactoryOptions {
type: LogicFunctionExecutorDriverType.LOCAL;
options: LocalDriverOptions;
}
export interface LambdaDriverFactoryOptions {
type: LogicFunctionExecutorDriverType.LAMBDA;
options: LambdaDriverOptions;
}
export type LogicFunctionExecutorModuleOptions =
| DisabledDriverFactoryOptions
| LocalDriverFactoryOptions
| LambdaDriverFactoryOptions;
export type LogicFunctionExecutorModuleAsyncOptions = {
useFactory: (
// eslint-disable-next-line @typescript-eslint/no-explicit-any
...args: any[]
) =>
| LogicFunctionExecutorModuleOptions
| Promise<LogicFunctionExecutorModuleOptions>;
} & Pick<ModuleMetadata, 'imports'> &
Pick<FactoryProvider, 'inject'>;
@@ -1,14 +1,10 @@
import { Module } from '@nestjs/common';
import { TypeOrmModule } from '@nestjs/typeorm';
import { LogicFunctionExecutorService } from 'src/engine/core-modules/logic-function/logic-function-executor/logic-function-executor.service';
import { ThrottlerModule } from 'src/engine/core-modules/throttler/throttler.module';
import { AuditModule } from 'src/engine/core-modules/audit/audit.module';
import { TokenModule } from 'src/engine/core-modules/auth/token/token.module';
import { FileModule } from 'src/engine/core-modules/file/file.module';
import { AddPackagesCommand } from 'src/engine/core-modules/logic-function/logic-function-executor/commands/add-packages.command';
import { LogicFunctionExecutorService } from 'src/engine/core-modules/logic-function/logic-function-executor/services/logic-function-executor.service';
import { SecretEncryptionModule } from 'src/engine/core-modules/secret-encryption/secret-encryption.module';
import { ThrottlerModule } from 'src/engine/core-modules/throttler/throttler.module';
import { LogicFunctionEntity } from 'src/engine/metadata-modules/logic-function/logic-function.entity';
import { SubscriptionsModule } from 'src/engine/subscriptions/subscriptions.module';
import { WorkspaceCacheModule } from 'src/engine/workspace-cache/workspace-cache.module';
@@ -20,10 +16,8 @@ import { WorkspaceCacheModule } from 'src/engine/workspace-cache/workspace-cache
SecretEncryptionModule,
SubscriptionsModule,
WorkspaceCacheModule,
FileModule,
TypeOrmModule.forFeature([LogicFunctionEntity]),
],
providers: [LogicFunctionExecutorService, AddPackagesCommand],
providers: [LogicFunctionExecutorService],
exports: [LogicFunctionExecutorService],
})
export class LogicFunctionExecutorModule {}
@@ -1,37 +1,32 @@
import { Inject, Injectable } from '@nestjs/common';
import { InjectRepository } from '@nestjs/typeorm';
import {
DEFAULT_API_KEY_NAME,
DEFAULT_API_URL_NAME,
} from 'twenty-shared/application';
import { FileFolder } from 'twenty-shared/types';
import { isDefined } from 'twenty-shared/utils';
import { Repository } from 'typeorm';
import {
LogicFunctionExecutorDriver,
type LogicFunctionExecuteParams,
LogicFunctionDriver,
type LogicFunctionExecuteResult,
} from 'src/engine/core-modules/logic-function/logic-function-drivers/interfaces/logic-function-executor-driver.interface';
} from 'src/engine/core-modules/logic-function/logic-function-drivers/interfaces/logic-function-driver.interface';
import { type FlatApplication } from 'src/engine/core-modules/application/types/flat-application.type';
import { AuditService } from 'src/engine/core-modules/audit/services/audit.service';
import { LOGIC_FUNCTION_EXECUTED_EVENT } from 'src/engine/core-modules/audit/utils/events/workspace-event/logic-function/logic-function-executed';
import { ApplicationTokenService } from 'src/engine/core-modules/auth/token/services/application-token.service';
import { FileStorageService } from 'src/engine/core-modules/file-storage/file-storage.service';
import { buildEnvVar } from 'src/engine/core-modules/logic-function/logic-function-drivers/utils/build-env-var';
import { LOGIC_FUNCTION_EXECUTOR_DRIVER } from 'src/engine/core-modules/logic-function/logic-function-executor/constants/logic-function-executor.constants';
import { buildEnvVar } from 'src/engine/core-modules/logic-function/logic-function-executor/utils/build-env-var';
import { LOGIC_FUNCTION_DRIVER } from 'src/engine/core-modules/logic-function/logic-function-drivers/constants/logic-function-driver.constants';
import { SecretEncryptionService } from 'src/engine/core-modules/secret-encryption/secret-encryption.service';
import { ThrottlerService } from 'src/engine/core-modules/throttler/throttler.service';
import { TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service';
import { findFlatEntityByIdInFlatEntityMaps } from 'src/engine/metadata-modules/flat-entity/utils/find-flat-entity-by-id-in-flat-entity-maps.util';
import { LogicFunctionEntity } from 'src/engine/metadata-modules/logic-function/logic-function.entity';
import { type FlatLogicFunction } from 'src/engine/metadata-modules/logic-function/types/flat-logic-function.type';
import { SubscriptionChannel } from 'src/engine/subscriptions/enums/subscription-channel.enum';
import { SubscriptionService } from 'src/engine/subscriptions/subscription.service';
import { WorkspaceCacheService } from 'src/engine/workspace-cache/services/workspace-cache.service';
import { cleanServerUrl } from 'src/utils/clean-server-url';
import type { FlatApplicationVariable } from 'src/engine/core-modules/applicationVariable/types/flat-application-variable.type';
import { FlatLogicFunction } from 'src/engine/metadata-modules/logic-function/types/flat-logic-function.type';
import { FlatApplication } from 'src/engine/core-modules/application/types/flat-application.type';
const MIN_TOKEN_EXPIRATION_IN_SECONDS = 5;
@@ -51,12 +46,10 @@ export enum LogicFunctionExecutionExceptionCode {
}
@Injectable()
export class LogicFunctionExecutorService
implements LogicFunctionExecutorDriver
{
export class LogicFunctionExecutorService {
constructor(
@Inject(LOGIC_FUNCTION_EXECUTOR_DRIVER)
private driver: LogicFunctionExecutorDriver,
@Inject(LOGIC_FUNCTION_DRIVER)
private driver: LogicFunctionDriver,
private readonly throttlerService: ThrottlerService,
private readonly twentyConfigService: TwentyConfigService,
private readonly workspaceCacheService: WorkspaceCacheService,
@@ -64,185 +57,54 @@ export class LogicFunctionExecutorService
private readonly secretEncryptionService: SecretEncryptionService,
private readonly subscriptionService: SubscriptionService,
private readonly auditService: AuditService,
private readonly fileStorageService: FileStorageService,
@InjectRepository(LogicFunctionEntity)
private readonly logicFunctionRepository: Repository<LogicFunctionEntity>,
) {}
async delete(flatLogicFunction: FlatLogicFunction): Promise<void> {
return this.driver.delete(flatLogicFunction);
}
async execute(
params: LogicFunctionExecuteParams,
): Promise<LogicFunctionExecuteResult> {
return this.driver.execute(params);
}
async executeOneLogicFunction({
id,
async execute({
logicFunctionId,
workspaceId,
payload,
}: {
id: string;
logicFunctionId: string;
workspaceId: string;
payload: object;
}): Promise<LogicFunctionExecuteResult> {
await this.throttleExecution(workspaceId);
const {
flatLogicFunctionMaps,
flatApplicationMaps,
applicationVariableMaps,
} = await this.workspaceCacheService.getOrRecompute(workspaceId, [
'flatLogicFunctionMaps',
'flatApplicationMaps',
'applicationVariableMaps',
]);
const { flatApplication, flatLogicFunction, flatApplicationVariables } =
await this.getFlatEntitiesOrThrow({
workspaceId,
logicFunctionId,
});
const flatLogicFunction = findFlatEntityByIdInFlatEntityMaps({
flatEntityId: id,
flatEntityMaps: flatLogicFunctionMaps,
const envVariables = await this.getExecutionEnvVariables({
workspaceId,
flatApplication,
flatApplicationVariables,
flatLogicFunction,
});
if (
!isDefined(flatLogicFunction) ||
isDefined(flatLogicFunction.deletedAt)
) {
throw new LogicFunctionExecutionException(
`Logic function with id ${id} not found`,
LogicFunctionExecutionExceptionCode.LOGIC_FUNCTION_NOT_FOUND,
);
}
const flatApplication = isDefined(flatLogicFunction.applicationId)
? flatApplicationMaps.byId[flatLogicFunction.applicationId]
: undefined;
if (!isDefined(flatApplication)) {
throw new LogicFunctionExecutionException(
`Application not found for logic function ${id}`,
LogicFunctionExecutionExceptionCode.LOGIC_FUNCTION_NOT_FOUND,
);
}
const applicationAccessToken = isDefined(flatLogicFunction.applicationId)
? await this.applicationTokenService.generateApplicationToken({
workspaceId,
applicationId: flatLogicFunction.applicationId,
expiresInSeconds: Math.max(
flatLogicFunction.timeoutSeconds,
MIN_TOKEN_EXPIRATION_IN_SECONDS,
),
})
: undefined;
const baseUrl = cleanServerUrl(this.twentyConfigService.get('SERVER_URL'));
const flatApplicationVariables = isDefined(flatLogicFunction.applicationId)
? (applicationVariableMaps.byApplicationId[
flatLogicFunction.applicationId
] ?? [])
: [];
const envVariables = {
...(isDefined(baseUrl)
? {
[DEFAULT_API_URL_NAME]: baseUrl,
}
: {}),
...(isDefined(applicationAccessToken)
? {
[DEFAULT_API_KEY_NAME]: applicationAccessToken.token,
}
: {}),
...buildEnvVar(flatApplicationVariables, this.secretEncryptionService),
};
const applicationUniversalIdentifier = isDefined(
flatLogicFunction.applicationId,
)
? flatApplicationMaps.byId[flatLogicFunction.applicationId]
?.universalIdentifier
: undefined;
if (!isDefined(applicationUniversalIdentifier)) {
throw new LogicFunctionExecutionException(
`Application universal identifier not found for logic function ${flatLogicFunction.id}`,
LogicFunctionExecutionExceptionCode.LOGIC_FUNCTION_NOT_FOUND,
);
}
if (
!(await this.hasLayerDependencies({
flatApplication,
applicationUniversalIdentifier,
}))
) {
throw new LogicFunctionExecutionException(
'Logic function dependencies not found',
LogicFunctionExecutionExceptionCode.LOGIC_FUNCTION_NOT_FOUND,
);
}
const resultLogicFunction = await this.callWithTimeout({
callback: () =>
this.execute({
this.driver.execute({
flatLogicFunction,
flatApplication,
applicationUniversalIdentifier,
applicationUniversalIdentifier: flatApplication.universalIdentifier,
payload,
env: envVariables,
}),
timeoutMs: flatLogicFunction.timeoutSeconds * 1000,
});
if (this.twentyConfigService.get('LOGIC_FUNCTION_LOGS_ENABLED')) {
/* eslint-disable no-console */
console.log(resultLogicFunction.logs);
}
await this.subscriptionService.publish({
channel: SubscriptionChannel.LOGIC_FUNCTION_LOGS_CHANNEL,
await this.handleExecutionResult({
result: resultLogicFunction,
flatApplication,
flatLogicFunction,
workspaceId,
payload: {
logicFunctionLogs: {
logs: resultLogicFunction.logs,
id: flatLogicFunction.id,
name: flatLogicFunction.name,
universalIdentifier: flatLogicFunction.universalIdentifier,
applicationId: flatLogicFunction.applicationId,
applicationUniversalIdentifier,
},
},
});
this.auditService
.createContext({
workspaceId,
})
.insertWorkspaceEvent(LOGIC_FUNCTION_EXECUTED_EVENT, {
duration: resultLogicFunction.duration,
status: resultLogicFunction.status,
...(resultLogicFunction.error && {
errorType: resultLogicFunction.error.errorType,
}),
functionId: flatLogicFunction.id,
functionName: flatLogicFunction.name,
});
return resultLogicFunction;
}
async getAvailablePackages(logicFunctionId: string) {
const logicFunction = await this.logicFunctionRepository.findOneOrFail({
where: { id: logicFunctionId },
relations: ['application'],
});
return logicFunction.application.availablePackages ?? {};
}
private async throttleExecution(workspaceId: string) {
try {
await this.throttlerService.tokenBucketThrottleOrThrow(
@@ -274,26 +136,128 @@ export class LogicFunctionExecutorService
]);
}
private async hasLayerDependencies({
flatApplication,
applicationUniversalIdentifier,
private async getFlatEntitiesOrThrow({
workspaceId,
logicFunctionId,
}: {
flatApplication: FlatApplication;
applicationUniversalIdentifier: string;
}): Promise<boolean> {
const packageJsonExists = await this.fileStorageService.checkFileExists({
workspaceId: flatApplication.workspaceId,
applicationUniversalIdentifier,
fileFolder: FileFolder.Dependencies,
resourcePath: 'package.json',
});
const yarnLockExists = await this.fileStorageService.checkFileExists({
workspaceId: flatApplication.workspaceId,
applicationUniversalIdentifier,
fileFolder: FileFolder.Dependencies,
resourcePath: 'yarn.lock',
workspaceId: string;
logicFunctionId: string;
}) {
const {
flatLogicFunctionMaps,
flatApplicationMaps,
applicationVariableMaps,
} = await this.workspaceCacheService.getOrRecompute(workspaceId, [
'flatLogicFunctionMaps',
'flatApplicationMaps',
'applicationVariableMaps',
]);
const flatLogicFunction = findFlatEntityByIdInFlatEntityMaps({
flatEntityId: logicFunctionId,
flatEntityMaps: flatLogicFunctionMaps,
});
return packageJsonExists && yarnLockExists;
if (
!isDefined(flatLogicFunction) ||
isDefined(flatLogicFunction.deletedAt)
) {
throw new LogicFunctionExecutionException(
`Logic function with id ${logicFunctionId} not found`,
LogicFunctionExecutionExceptionCode.LOGIC_FUNCTION_NOT_FOUND,
);
}
const flatApplication = isDefined(flatLogicFunction.applicationId)
? flatApplicationMaps.byId[flatLogicFunction.applicationId]
: undefined;
if (!isDefined(flatApplication)) {
throw new LogicFunctionExecutionException(
`Application not found for logic function ${logicFunctionId}`,
LogicFunctionExecutionExceptionCode.LOGIC_FUNCTION_NOT_FOUND,
);
}
const flatApplicationVariables =
applicationVariableMaps.byApplicationId[flatApplication.id] ?? [];
return { flatApplication, flatLogicFunction, flatApplicationVariables };
}
private async getExecutionEnvVariables({
workspaceId,
flatApplication,
flatLogicFunction,
flatApplicationVariables,
}: {
workspaceId: string;
flatApplication: FlatApplication;
flatLogicFunction: FlatLogicFunction;
flatApplicationVariables: FlatApplicationVariable[];
}) {
const applicationAccessToken =
await this.applicationTokenService.generateApplicationToken({
workspaceId,
applicationId: flatApplication.id,
expiresInSeconds: Math.max(
flatLogicFunction.timeoutSeconds,
MIN_TOKEN_EXPIRATION_IN_SECONDS,
),
});
const baseUrl = cleanServerUrl(this.twentyConfigService.get('SERVER_URL'));
return {
[DEFAULT_API_URL_NAME]: baseUrl ?? '',
[DEFAULT_API_KEY_NAME]: applicationAccessToken.token,
...buildEnvVar(flatApplicationVariables, this.secretEncryptionService),
};
}
private async handleExecutionResult({
result,
flatApplication,
flatLogicFunction,
workspaceId,
}: {
result: LogicFunctionExecuteResult;
workspaceId: string;
flatLogicFunction: FlatLogicFunction;
flatApplication: FlatApplication;
}) {
if (this.twentyConfigService.get('LOGIC_FUNCTION_LOGS_ENABLED')) {
/* eslint-disable no-console */
console.log(result.logs);
}
await this.subscriptionService.publish({
channel: SubscriptionChannel.LOGIC_FUNCTION_LOGS_CHANNEL,
workspaceId,
payload: {
logicFunctionLogs: {
logs: result.logs,
id: flatLogicFunction.id,
name: flatLogicFunction.name,
universalIdentifier: flatLogicFunction.universalIdentifier,
applicationId: flatApplication.id,
applicationUniversalIdentifier: flatApplication.universalIdentifier,
},
},
});
this.auditService
.createContext({
workspaceId,
})
.insertWorkspaceEvent(LOGIC_FUNCTION_EXECUTED_EVENT, {
duration: result.duration,
status: result.status,
...(result.error && {
errorType: result.error.errorType,
}),
functionId: flatLogicFunction.id,
functionName: flatLogicFunction.name,
});
}
}
@@ -1,6 +1,6 @@
import { type FlatApplicationVariable } from 'src/engine/core-modules/applicationVariable/types/flat-application-variable.type';
import { type SecretEncryptionService } from 'src/engine/core-modules/secret-encryption/secret-encryption.service';
import { buildEnvVar } from 'src/engine/core-modules/logic-function/logic-function-drivers/utils/build-env-var';
import { buildEnvVar } from 'src/engine/core-modules/logic-function/logic-function-executor/utils/build-env-var';
describe('buildEnvVar', () => {
const mockSecretEncryptionService = {
@@ -0,0 +1,4 @@
export const SEED_LOGIC_FUNCTION_INPUT_SCHEMA = {
a: null,
b: null,
};
@@ -0,0 +1,9 @@
import { Module } from '@nestjs/common';
import { LogicFunctionResourceService } from 'src/engine/core-modules/logic-function/logic-function-resource/logic-function-resource.service';
@Module({
providers: [LogicFunctionResourceService],
exports: [LogicFunctionResourceService],
})
export class LogicFunctionResourceModule {}
@@ -8,19 +8,20 @@ import { isObject } from '@sniptt/guards';
import { build } from 'esbuild';
import { FileFolder, Sources } from 'twenty-shared/types';
import { isDefined } from 'twenty-shared/utils';
import { NODE_ESM_CJS_BANNER } from 'twenty-shared/application';
import { FileStorageExceptionCode } from 'src/engine/core-modules/file-storage/interfaces/file-storage-exception';
import { FileStorageService } from 'src/engine/core-modules/file-storage/file-storage.service';
import { LambdaBuildDirectoryManager } from 'src/engine/core-modules/logic-function/logic-function-drivers/utils/lambda-build-directory-manager';
import { TemporaryDirManager } from 'src/engine/core-modules/logic-function/logic-function-drivers/utils/temporary-dir-manager';
import {
getLogicFunctionBaseFolderPath,
getRelativePathFromBase,
} from 'src/engine/core-modules/logic-function/logic-function-source-builder/utils/get-logic-function-handler-path.util';
} from 'src/engine/core-modules/logic-function/logic-function-resource/utils/get-logic-function-handler-path.util';
import {
getLogicFunctionSeedProjectFiles,
LogicFunctionSeedProjectFile,
} from 'src/engine/core-modules/logic-function/logic-function-source-builder/utils/get-logic-function-seed-project-files.util';
} from 'src/engine/core-modules/logic-function/logic-function-resource/utils/get-logic-function-seed-project-files.util';
import {
DEFAULT_BUILT_HANDLER_PATH,
DEFAULT_SOURCE_HANDLER_PATH,
@@ -29,17 +30,18 @@ import {
LogicFunctionException,
LogicFunctionExceptionCode,
} from 'src/engine/metadata-modules/logic-function/logic-function.exception';
import { streamToBuffer } from 'src/utils/stream-to-buffer';
type SeedSourceFilesParams = {
logicFunctionId: string;
workspaceId: string;
applicationUniversalIdentifier: string;
code?: Sources;
sourceSubfolder: string;
};
type SeedSourceFilesResult = {
sourceHandlerPath: string;
builtHandlerPath: string;
handlerName: string;
checksum: string;
};
@@ -63,7 +65,7 @@ type GetSourceCodeParams = {
applicationUniversalIdentifier: string;
};
type CopySourceAndBuiltParams = {
type CopySourceParams = {
fromSourceHandlerPath: string;
fromBuiltHandlerPath: string;
toSourceHandlerPath: string;
@@ -73,42 +75,17 @@ type CopySourceAndBuiltParams = {
};
@Injectable()
export class LogicFunctionSourceBuilderService {
export class LogicFunctionResourceService {
constructor(private readonly fileStorageService: FileStorageService) {}
async seedSourceFiles({
logicFunctionId,
workspaceId,
applicationUniversalIdentifier,
code,
sourceSubfolder,
}: SeedSourceFilesParams): Promise<SeedSourceFilesResult> {
const sourceHandlerPath = `${logicFunctionId}/${DEFAULT_SOURCE_HANDLER_PATH}`;
const builtHandlerPath = `${logicFunctionId}/${DEFAULT_BUILT_HANDLER_PATH}`;
const sourceHandlerPath = `${sourceSubfolder}/${DEFAULT_SOURCE_HANDLER_PATH}`;
const builtHandlerPath = `${sourceSubfolder}/${DEFAULT_BUILT_HANDLER_PATH}`;
if (isDefined(code)) {
// Use provided code
await this.updateSourceFiles({
sourceHandlerPath,
workspaceId,
applicationUniversalIdentifier,
code,
});
const { checksum } = await this.buildFromSource({
sourceHandlerPath,
builtHandlerPath,
workspaceId,
applicationUniversalIdentifier,
});
return {
sourceHandlerPath,
builtHandlerPath,
checksum,
};
}
// Use seed project files
const seedProjectFiles = await getLogicFunctionSeedProjectFiles();
const sourceFiles = seedProjectFiles.filter(
@@ -160,6 +137,7 @@ export class LogicFunctionSourceBuilderService {
.digest('hex');
return {
handlerName: 'main',
sourceHandlerPath,
builtHandlerPath,
checksum,
@@ -172,10 +150,10 @@ export class LogicFunctionSourceBuilderService {
applicationUniversalIdentifier,
code,
}: UpdateSourceFilesParams): Promise<void> {
const lambdaBuildDirectoryManager = new LambdaBuildDirectoryManager();
const temporaryDirManager = new TemporaryDirManager();
try {
const { sourceTemporaryDir } = await lambdaBuildDirectoryManager.init();
const { sourceTemporaryDir } = await temporaryDirManager.init();
await this.writeSourcesToLocalFolder(code, sourceTemporaryDir);
@@ -189,7 +167,7 @@ export class LogicFunctionSourceBuilderService {
localPath: sourceTemporaryDir,
});
} finally {
await lambdaBuildDirectoryManager.clean();
await temporaryDirManager.clean();
}
}
@@ -199,10 +177,10 @@ export class LogicFunctionSourceBuilderService {
workspaceId,
applicationUniversalIdentifier,
}: BuildFromSourceParams): Promise<{ checksum: string }> {
const lambdaBuildDirectoryManager = new LambdaBuildDirectoryManager();
const temporaryDirManager = new TemporaryDirManager();
try {
const { sourceTemporaryDir } = await lambdaBuildDirectoryManager.init();
const { sourceTemporaryDir } = await temporaryDirManager.init();
const baseFolderPath = getLogicFunctionBaseFolderPath(sourceHandlerPath);
@@ -248,7 +226,7 @@ export class LogicFunctionSourceBuilderService {
checksum: crypto.createHash('md5').update(builtFile).digest('hex'),
};
} finally {
await lambdaBuildDirectoryManager.clean();
await temporaryDirManager.clean();
}
}
@@ -279,14 +257,14 @@ export class LogicFunctionSourceBuilderService {
}
}
async copySourceAndBuilt({
async copyResources({
fromSourceHandlerPath,
fromBuiltHandlerPath,
toSourceHandlerPath,
toBuiltHandlerPath,
workspaceId,
applicationUniversalIdentifier,
}: CopySourceAndBuiltParams): Promise<void> {
}: CopySourceParams): Promise<void> {
const fromSourceBaseFolderPath = getLogicFunctionBaseFolderPath(
fromSourceHandlerPath,
);
@@ -328,7 +306,75 @@ export class LogicFunctionSourceBuilderService {
});
}
private async writeSourcesToLocalFolder(
async copyDependenciesInMemory({
applicationUniversalIdentifier,
workspaceId,
inMemoryFolderPath,
}: {
applicationUniversalIdentifier: string;
workspaceId: string;
inMemoryFolderPath: string;
}) {
await Promise.all([
this.fileStorageService.downloadFile({
workspaceId,
applicationUniversalIdentifier,
fileFolder: FileFolder.Dependencies,
resourcePath: 'package.json',
localPath: join(inMemoryFolderPath, 'package.json'),
}),
this.fileStorageService.downloadFile({
workspaceId,
applicationUniversalIdentifier,
fileFolder: FileFolder.Dependencies,
resourcePath: 'yarn.lock',
localPath: join(inMemoryFolderPath, 'yarn.lock'),
}),
]);
}
async getBuiltCode({
builtHandlerPath,
workspaceId,
applicationUniversalIdentifier,
}: {
builtHandlerPath: string;
workspaceId: string;
applicationUniversalIdentifier: string;
}): Promise<string> {
return (
await streamToBuffer(
await this.fileStorageService.readFile({
workspaceId: workspaceId,
applicationUniversalIdentifier,
fileFolder: FileFolder.BuiltLogicFunction,
resourcePath: builtHandlerPath,
}),
)
).toString('utf-8');
}
async copyBuiltCodeInMemory({
builtHandlerPath,
workspaceId,
applicationUniversalIdentifier,
inMemoryDestinationPath,
}: {
builtHandlerPath: string;
workspaceId: string;
applicationUniversalIdentifier: string;
inMemoryDestinationPath: string;
}): Promise<void> {
await this.fileStorageService.downloadFile({
workspaceId,
applicationUniversalIdentifier,
fileFolder: FileFolder.BuiltLogicFunction,
resourcePath: builtHandlerPath,
localPath: inMemoryDestinationPath,
});
}
async writeSourcesToLocalFolder(
sources: Sources,
localPath: string,
): Promise<void> {
@@ -368,6 +414,7 @@ export class LogicFunctionSourceBuilderService {
bundle: true,
sourcemap: true,
packages: 'external',
banner: NODE_ESM_CJS_BANNER,
});
return builtBundleFilePath;
@@ -38,7 +38,7 @@ export const getLogicFunctionSeedProjectFiles = async (): Promise<
> => {
const seedProjectPath = path.join(
ASSET_PATH,
'engine/core-modules/logic-function/logic-function-source-builder/constants/seed-project',
'engine/core-modules/logic-function/logic-function-resource/constants/seed-project',
);
return getAllFiles(seedProjectPath);
@@ -1,9 +0,0 @@
import { Module } from '@nestjs/common';
import { LogicFunctionSourceBuilderService } from './logic-function-source-builder.service';
@Module({
providers: [LogicFunctionSourceBuilderService],
exports: [LogicFunctionSourceBuilderService],
})
export class LogicFunctionSourceBuilderModule {}
@@ -3,7 +3,7 @@ import { Scope } from '@nestjs/common';
import { Process } from 'src/engine/core-modules/message-queue/decorators/process.decorator';
import { Processor } from 'src/engine/core-modules/message-queue/decorators/processor.decorator';
import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';
import { LogicFunctionExecutorService } from 'src/engine/core-modules/logic-function/logic-function-executor/services/logic-function-executor.service';
import { LogicFunctionExecutorService } from 'src/engine/core-modules/logic-function/logic-function-executor/logic-function-executor.service';
export type LogicFunctionTriggerJobData = {
logicFunctionId: string;
@@ -25,8 +25,8 @@ export class LogicFunctionTriggerJob {
await Promise.all(
logicFunctionPayloads.map(
async (logicFunctionPayload) =>
await this.logicFunctionExecutorService.executeOneLogicFunction({
id: logicFunctionPayload.logicFunctionId,
await this.logicFunctionExecutorService.execute({
logicFunctionId: logicFunctionPayload.logicFunctionId,
workspaceId: logicFunctionPayload.workspaceId,
payload: logicFunctionPayload.payload ?? {},
}),
@@ -9,13 +9,13 @@ import { HTTPMethod } from 'twenty-shared/types';
import { AccessTokenService } from 'src/engine/core-modules/auth/token/services/access-token.service';
import { WorkspaceDomainsService } from 'src/engine/core-modules/domain/workspace-domains/services/workspace-domains.service';
import { LogicFunctionExecutorService } from 'src/engine/core-modules/logic-function/logic-function-executor/services/logic-function-executor.service';
import {
RouteTriggerException,
RouteTriggerExceptionCode,
} from 'src/engine/core-modules/logic-function/logic-function-trigger/triggers/route/exceptions/route-trigger.exception';
import { buildLogicFunctionEvent } from 'src/engine/core-modules/logic-function/logic-function-trigger/triggers/route/utils/build-logic-function-event.util';
import { LogicFunctionEntity } from 'src/engine/metadata-modules/logic-function/logic-function.entity';
import { LogicFunctionExecutorService } from 'src/engine/core-modules/logic-function/logic-function-executor/logic-function-executor.service';
@Injectable()
export class RouteTriggerService {
@@ -146,12 +146,11 @@ export class RouteTriggerService {
forwardedRequestHeaders: httpRouteSettings?.forwardedRequestHeaders ?? [],
});
const result =
await this.logicFunctionExecutorService.executeOneLogicFunction({
id: logicFunction.id,
workspaceId: logicFunction.workspaceId,
payload: event,
});
const result = await this.logicFunctionExecutorService.execute({
logicFunctionId: logicFunction.id,
workspaceId: logicFunction.workspaceId,
payload: event,
});
if (!isDefined(result)) {
return result;
@@ -1,31 +1,29 @@
import { type DynamicModule, Global, Module } from '@nestjs/common';
import { type LogicFunctionExecutorModuleAsyncOptions } from 'src/engine/core-modules/logic-function/logic-function-executor/interfaces/logic-function-executor.interface';
import { LogicFunctionModuleAsyncOptions } from 'src/engine/core-modules/logic-function/logic-function-drivers/interfaces/logic-function-driver.interface';
import { LogicFunctionDriversModule } from 'src/engine/core-modules/logic-function/logic-function-drivers/logic-function-drivers.module';
import { LogicFunctionExecutorModule } from 'src/engine/core-modules/logic-function/logic-function-executor/logic-function-executor.module';
import { LogicFunctionSourceBuilderModule } from 'src/engine/core-modules/logic-function/logic-function-source-builder/logic-function-source-builder.module';
import { LogicFunctionResourceModule } from 'src/engine/core-modules/logic-function/logic-function-resource/logic-function-resource.module';
import { LogicFunctionTriggerModule } from 'src/engine/core-modules/logic-function/logic-function-trigger/logic-function-trigger.module';
import { LogicFunctionExecutorModule } from 'src/engine/core-modules/logic-function/logic-function-executor/logic-function-executor.module';
@Global()
@Module({})
export class LogicFunctionModule {
static forRootAsync(
options: LogicFunctionExecutorModuleAsyncOptions,
): DynamicModule {
static forRootAsync(options: LogicFunctionModuleAsyncOptions): DynamicModule {
return {
module: LogicFunctionModule,
imports: [
LogicFunctionDriversModule.forRootAsync(options),
LogicFunctionExecutorModule,
LogicFunctionSourceBuilderModule,
LogicFunctionResourceModule,
LogicFunctionTriggerModule,
LogicFunctionExecutorModule,
],
exports: [
LogicFunctionDriversModule,
LogicFunctionExecutorModule,
LogicFunctionSourceBuilderModule,
LogicFunctionResourceModule,
LogicFunctionTriggerModule,
LogicFunctionExecutorModule,
],
};
}