Run vulnerable operation in isolated environment (#18523)

When driver = LAMBDA:
- run esbuild ts transpilation on dedicated lambda
- run yarn install on app dependencies on a dedicated lambda
This commit is contained in:
martmull
2026-03-11 15:08:47 +01:00
committed by GitHub
parent b346f4fb59
commit d9b3507866
21 changed files with 2036 additions and 154 deletions
+12
View File
@@ -32,6 +32,18 @@
"include": "engine/core-modules/logic-function/logic-function-drivers/constants/executor/index.mjs",
"outDir": "dist/assets"
},
{
"include": "engine/core-modules/logic-function/logic-function-drivers/constants/yarn-install/index.mjs",
"outDir": "dist/assets"
},
{
"include": "engine/core-modules/logic-function/logic-function-drivers/constants/builder/index.mjs",
"outDir": "dist/assets"
},
{
"include": "engine/core-modules/logic-function/logic-function-drivers/constants/common-layer-dependencies/**",
"outDir": "dist/assets"
},
{
"include": "database/clickHouse/migrations/*.sql",
"outDir": "dist"
+1
View File
@@ -28,6 +28,7 @@
"@aws-sdk/client-sesv2": "3.1001.0",
"@aws-sdk/client-sts": "3.1001.0",
"@aws-sdk/credential-providers": "3.1001.0",
"@aws-sdk/s3-request-presigner": "3.1001.0",
"@azure/msal-node": "^3.8.4",
"@babel/preset-env": "7.26.9",
"@blocknote/server-util": "^0.47.1",
@@ -5,3 +5,13 @@ enableScripts: false
nodeLinker: node-modules
yarnPath: .yarn/releases/yarn-4.9.2.cjs
supportedArchitectures:
os:
- current
- linux
- darwin
cpu:
- current
- x64
- arm64
@@ -0,0 +1,65 @@
import { randomBytes } from 'crypto';
import { promises as fs } from 'fs';
import { dirname, resolve } from 'path';
import { build } from 'esbuild';
const BANNER = {
js: "import { createRequire as __createRequire } from 'module';\nconst require = __createRequire(import.meta.url);",
};
const assertPathInsideDir = (filePath, dir) => {
const resolved = resolve(dir, filePath);
if (!resolved.startsWith(dir + '/')) {
throw new Error(`Path traversal detected: ${filePath}`);
}
return resolved;
};
export const handler = async (event) => {
const { action, sourceCode, sourceFileName, builtFileName } = event;
if (action !== 'transpile') {
throw new Error(`Unknown action: ${action}`);
}
if (!sourceCode || !sourceFileName || !builtFileName) {
throw new Error(
'Missing required fields: sourceCode, sourceFileName, builtFileName',
);
}
const randomId = randomBytes(16).toString('hex');
const workDir = `/tmp/${randomId}`;
await fs.mkdir(workDir, { recursive: true });
try {
const entryFilePath = assertPathInsideDir(sourceFileName, workDir);
const outFilePath = assertPathInsideDir(builtFileName, workDir);
await fs.mkdir(dirname(entryFilePath), { recursive: true });
await fs.writeFile(entryFilePath, sourceCode, 'utf-8');
await fs.mkdir(dirname(outFilePath), { recursive: true });
await build({
entryPoints: [entryFilePath],
outfile: outFilePath,
platform: 'node',
format: 'esm',
target: 'es2017',
bundle: true,
sourcemap: true,
packages: 'external',
banner: BANNER,
});
const builtCode = await fs.readFile(outFilePath, 'utf-8');
return { builtCode };
} finally {
await fs.rm(workDir, { recursive: true, force: true });
}
};
@@ -0,0 +1,11 @@
import path from 'path';
import { ASSET_PATH } from 'src/constants/assets-path';
export const COMMON_LAYER_DEPENDENCIES_DIRNAME = path.resolve(
__dirname,
path.join(
ASSET_PATH,
'engine/core-modules/logic-function/logic-function-drivers/constants/common-layer-dependencies',
),
);
@@ -0,0 +1,6 @@
{
"dependencies": {
"archiver": "^7.0.1",
"esbuild": "^0.25.0"
}
}
@@ -0,0 +1,133 @@
import { randomBytes } from 'crypto';
import { createWriteStream, promises as fs } from 'fs';
import { join, resolve } from 'path';
import archiver from 'archiver';
import { pipeline } from 'stream/promises';
const YARN_INSTALL_TIMEOUT_MS = 240_000;
const YARN_ENGINE_DIR = resolve('yarn-engine');
const YARN_ENGINE_PATH = join(YARN_ENGINE_DIR, '.yarn/releases/yarn-4.9.2.cjs');
const writePackageFiles = async (nodejsDir, packageJson, yarnLock) => {
await fs.mkdir(nodejsDir, { recursive: true });
await Promise.all([
fs.writeFile(join(nodejsDir, 'package.json'), packageJson, 'utf-8'),
fs.writeFile(join(nodejsDir, 'yarn.lock'), yarnLock, 'utf-8'),
]);
};
const copyYarnEngine = async (nodejsDir) => {
await fs.cp('yarn-engine', nodejsDir, { recursive: true });
};
const runYarnInstall = async (nodejsDir) => {
const { execFile } = await import('child_process');
const { promisify } = await import('util');
const execFilePromise = promisify(execFile);
const { NODE_OPTIONS: _nodeOptions, ...cleanEnv } = process.env;
// Lambda runs as a sandboxed user whose $HOME doesn't exist.
// Yarn needs a writable HOME for its global cache/config.
cleanEnv.HOME = '/tmp';
try {
await execFilePromise(
process.execPath,
[YARN_ENGINE_PATH, 'workspaces', 'focus', '--all', '--production'],
{
cwd: nodejsDir,
env: cleanEnv,
timeout: YARN_INSTALL_TIMEOUT_MS,
},
);
} catch (error) {
const details = [error?.stdout, error?.stderr].filter(Boolean).join('\n');
throw new Error(`yarn install failed: ${details || error?.message}`);
}
// Remove everything except node_modules
const entries = await fs.readdir(nodejsDir);
await Promise.all(
entries
.filter((entry) => entry !== 'node_modules')
.map(async (entry) => {
const fullPath = join(nodejsDir, entry);
const stat = await fs.stat(fullPath);
return stat.isDirectory()
? fs.rm(fullPath, { recursive: true, force: true })
: fs.rm(fullPath);
}),
);
};
const createZip = async (buildDir, zipPath) => {
const output = createWriteStream(zipPath);
const archive = archiver('zip', { zlib: { level: 9 } });
const p = pipeline(archive, output);
archive.directory(buildDir, false);
archive.finalize();
return p;
};
const uploadToPresignedUrl = async (zipPath, presignedUploadUrl) => {
const zipBuffer = await fs.readFile(zipPath);
const response = await fetch(presignedUploadUrl, {
method: 'PUT',
body: zipBuffer,
headers: { 'Content-Type': 'application/zip' },
});
if (!response.ok) {
throw new Error(
`Failed to upload zip to S3: ${response.status} ${response.statusText}`,
);
}
};
export const handler = async (event) => {
const { action, packageJson, yarnLock, presignedUploadUrl } = event;
if (action !== 'createLayer') {
throw new Error(`Unknown action: ${action}`);
}
if (!packageJson || !yarnLock) {
throw new Error('Missing required fields: packageJson, yarnLock');
}
if (!presignedUploadUrl) {
throw new Error('Missing required field: presignedUploadUrl');
}
const randomId = randomBytes(16).toString('hex');
const buildDir = `/tmp/${randomId}`;
const nodejsDir = join(buildDir, 'nodejs');
const zipPath = `/tmp/${randomId}.zip`;
try {
await writePackageFiles(nodejsDir, packageJson, yarnLock);
await copyYarnEngine(nodejsDir);
await runYarnInstall(nodejsDir);
await createZip(buildDir, zipPath);
await uploadToPresignedUrl(zipPath, presignedUploadUrl);
return { success: true };
} finally {
await fs.rm(buildDir, { recursive: true, force: true });
await fs.rm(zipPath, { force: true });
await fs.rm(join(YARN_ENGINE_DIR, '.yarn/cache'), {
recursive: true,
force: true,
});
}
};
@@ -1,6 +1,7 @@
import {
type LogicFunctionDriver,
type LogicFunctionExecuteResult,
type LogicFunctionTranspileResult,
} from 'src/engine/core-modules/logic-function/logic-function-drivers/interfaces/logic-function-driver.interface';
import {
@@ -19,4 +20,11 @@ export class DisabledDriver implements LogicFunctionDriver {
LogicFunctionExceptionCode.LOGIC_FUNCTION_DISABLED,
);
}
async transpile(): Promise<LogicFunctionTranspileResult> {
throw new LogicFunctionException(
'Logic function transpilation is disabled. Set LOGIC_FUNCTION_TYPE to LOCAL or LAMBDA to enable.',
LogicFunctionExceptionCode.LOGIC_FUNCTION_DISABLED,
);
}
}
@@ -1,5 +1,6 @@
import { createHash } from 'crypto';
import * as fs from 'fs/promises';
import { join } from 'path';
import { resolve, join } from 'path';
import {
CreateFunctionCommand,
@@ -14,10 +15,11 @@ import {
type ListLayerVersionsCommandInput,
LogType,
PublishLayerVersionCommand,
type PublishLayerVersionCommandInput,
ResourceNotFoundException,
waitUntilFunctionUpdatedV2,
waitUntilFunctionActiveV2,
} from '@aws-sdk/client-lambda';
import { PutObjectCommand, S3Client } from '@aws-sdk/client-s3';
import { getSignedUrl } from '@aws-sdk/s3-request-presigner';
import { AssumeRoleCommand, STSClient } from '@aws-sdk/client-sts';
import { isDefined } from 'twenty-shared/utils';
@@ -25,10 +27,17 @@ import {
type LogicFunctionExecuteParams,
type LogicFunctionExecuteResult,
type LogicFunctionDriver,
type LogicFunctionTranspileParams,
type LogicFunctionTranspileResult,
} from 'src/engine/core-modules/logic-function/logic-function-drivers/interfaces/logic-function-driver.interface';
import { ASSET_PATH } from 'src/constants/assets-path';
import { type FlatApplication } from 'src/engine/core-modules/application/types/flat-application.type';
import { COMMON_LAYER_DEPENDENCIES_DIRNAME } from 'src/engine/core-modules/logic-function/logic-function-drivers/constants/common-layer-dependencies-dirname';
import { copyBuilder } from 'src/engine/core-modules/logic-function/logic-function-drivers/utils/copy-builder';
import { copyCommonLayerDependencies } from 'src/engine/core-modules/logic-function/logic-function-drivers/utils/copy-common-layer-dependencies';
import { copyExecutor } from 'src/engine/core-modules/logic-function/logic-function-drivers/utils/copy-executor';
import { copyYarnInstall } from 'src/engine/core-modules/logic-function/logic-function-drivers/utils/copy-yarn-install';
import { createZipFile } from 'src/engine/core-modules/logic-function/logic-function-drivers/utils/create-zip-file';
import { TemporaryDirManager } from 'src/engine/core-modules/logic-function/logic-function-drivers/utils/temporary-dir-manager';
import { LogicFunctionExecutionStatus } from 'src/engine/metadata-modules/logic-function/dtos/logic-function-execution-result.dto';
@@ -38,12 +47,32 @@ import {
LogicFunctionExceptionCode,
} from 'src/engine/metadata-modules/logic-function/logic-function.exception';
import { type FlatLogicFunction } from 'src/engine/metadata-modules/logic-function/types/flat-logic-function.type';
import { copyYarnEngineAndBuildDependencies } from 'src/engine/core-modules/application/application-package/utils/copy-yarn-engine-and-build-dependencies';
import { type LogicFunctionResourceService } from 'src/engine/core-modules/logic-function/logic-function-resource/logic-function-resource.service';
import { callWithTimeout } from 'src/engine/core-modules/logic-function/logic-function-drivers/utils/call-with-timeout';
const UPDATE_FUNCTION_DURATION_TIMEOUT_IN_SECONDS = 60;
const CREDENTIALS_DURATION_IN_SECONDS = 60 * 60; // 1h
const YARN_INSTALL_LAMBDA_TIMEOUT_SECONDS = 300;
const YARN_INSTALL_LAMBDA_MEMORY_MB = 1024;
const COMMON_LAYER_NAME_PREFIX = 'twenty-common-layer';
const BUILDER_LAMBDA_TIMEOUT_SECONDS = 60;
const BUILDER_LAMBDA_MEMORY_MB = 512;
const YARN_INSTALL_HANDLER_PATH = resolve(
__dirname,
join(
ASSET_PATH,
'engine/core-modules/logic-function/logic-function-drivers/constants/yarn-install/index.mjs',
),
);
const BUILDER_HANDLER_PATH = resolve(
__dirname,
join(
ASSET_PATH,
'engine/core-modules/logic-function/logic-function-drivers/constants/builder/index.mjs',
),
);
type LambdaDriverExecutorPayload = {
code: string;
@@ -52,11 +81,35 @@ type LambdaDriverExecutorPayload = {
handlerName: string;
};
export type YarnInstallLambdaPayload = {
action: 'createLayer';
packageJson: string;
yarnLock: string;
presignedUploadUrl: string;
};
export type YarnInstallLambdaResult = {
success: boolean;
};
export type BuilderLambdaPayload = {
action: 'transpile';
sourceCode: string;
sourceFileName: string;
builtFileName: string;
};
export type BuilderLambdaResult = {
builtCode: string;
};
export interface LambdaDriverOptions extends LambdaClientConfig {
logicFunctionResourceService: LogicFunctionResourceService;
region: string;
lambdaRole: string;
subhostingRole?: string;
layerBucket: string;
layerBucketRegion: string;
}
export class LambdaDriver implements LogicFunctionDriver {
@@ -120,17 +173,33 @@ export class LambdaDriver implements LogicFunctionDriver {
};
}
private async waitFunctionUpdates(
flatLogicFunction: FlatLogicFunction,
private async generatePresignedUploadUrl(
s3Key: string,
expiresIn: number = 300,
): Promise<string> {
const s3Client = new S3Client({
region: this.options.layerBucketRegion,
credentials: isDefined(this.options.subhostingRole)
? await this.getAssumeRoleCredentials()
: this.options.credentials,
});
const putCommand = new PutObjectCommand({
Bucket: this.options.layerBucket,
Key: s3Key,
ContentType: 'application/zip',
});
return getSignedUrl(s3Client, putCommand, { expiresIn });
}
private async waitFunctionActive(
functionName: string,
maxWaitTime: number = UPDATE_FUNCTION_DURATION_TIMEOUT_IN_SECONDS,
) {
const waitParams = {
FunctionName: flatLogicFunction.id,
};
await waitUntilFunctionUpdatedV2(
await waitUntilFunctionActiveV2(
{ client: await this.getLambdaClient(), maxWaitTime },
waitParams,
{ FunctionName: functionName },
);
}
@@ -138,7 +207,398 @@ export class LambdaDriver implements LogicFunctionDriver {
return flatApplication.yarnLockChecksum ?? 'default';
}
private async createLayerIfNotExists({
private yarnInstallFunctionName: string | undefined;
private builderFunctionName: string | undefined;
private commonLayerName: string | undefined;
private async getCommonLayerName(): Promise<string> {
if (isDefined(this.commonLayerName)) {
return this.commonLayerName;
}
const [packageJson, yarnLock] = await Promise.all([
fs.readFile(
join(COMMON_LAYER_DEPENDENCIES_DIRNAME, 'package.json'),
'utf-8',
),
fs.readFile(
join(COMMON_LAYER_DEPENDENCIES_DIRNAME, 'yarn.lock'),
'utf-8',
),
]);
const checksum = createHash('sha256')
.update(packageJson)
.update(yarnLock)
.digest('hex')
.slice(0, 12);
this.commonLayerName = `${COMMON_LAYER_NAME_PREFIX}-${checksum}`;
return this.commonLayerName;
}
private async getYarnInstallFunctionName(): Promise<string> {
if (isDefined(this.yarnInstallFunctionName)) {
return this.yarnInstallFunctionName;
}
const handlerContent = await fs.readFile(
YARN_INSTALL_HANDLER_PATH,
'utf-8',
);
const checksum = createHash('sha256')
.update(handlerContent)
.digest('hex')
.slice(0, 12);
this.yarnInstallFunctionName = `twenty-yarn-install-${checksum}`;
return this.yarnInstallFunctionName;
}
private async getBuilderFunctionName(): Promise<string> {
if (isDefined(this.builderFunctionName)) {
return this.builderFunctionName;
}
const handlerContent = await fs.readFile(BUILDER_HANDLER_PATH, 'utf-8');
const checksum = createHash('sha256')
.update(handlerContent)
.digest('hex')
.slice(0, 12);
this.builderFunctionName = `twenty-builder-${checksum}`;
return this.builderFunctionName;
}
private async ensureCommonLayerExists(): Promise<string> {
const commonLayerName = await this.getCommonLayerName();
const existingArn = await this.getExistingLayerArn(commonLayerName);
if (isDefined(existingArn)) {
return existingArn;
}
const temporaryDirManager = new TemporaryDirManager();
const { sourceTemporaryDir, lambdaZipPath } =
await temporaryDirManager.init();
try {
await copyCommonLayerDependencies(sourceTemporaryDir);
await createZipFile(sourceTemporaryDir, lambdaZipPath);
const lambdaClient = await this.getLambdaClient();
const result = await lambdaClient.send(
new PublishLayerVersionCommand({
LayerName: commonLayerName,
Content: { ZipFile: await fs.readFile(lambdaZipPath) },
CompatibleRuntimes: [
LogicFunctionRuntime.NODE18,
LogicFunctionRuntime.NODE22,
],
}),
);
if (!result.LayerVersionArn) {
throw new Error(
'PublishLayerVersion did not return a LayerVersionArn for common layer',
);
}
return result.LayerVersionArn;
} finally {
await temporaryDirManager.clean();
}
}
private async ensureYarnInstallLambdaExists(): Promise<void> {
const yarnInstallFunctionName = await this.getYarnInstallFunctionName();
const lambdaClient = await this.getLambdaClient();
try {
await lambdaClient.send(
new GetFunctionCommand({ FunctionName: yarnInstallFunctionName }),
);
return;
} catch (error) {
if (!(error instanceof ResourceNotFoundException)) {
throw error;
}
}
const commonLayerArn = await this.ensureCommonLayerExists();
const temporaryDirManager = new TemporaryDirManager();
const { sourceTemporaryDir, lambdaZipPath } =
await temporaryDirManager.init();
try {
await copyYarnInstall(sourceTemporaryDir);
await createZipFile(sourceTemporaryDir, lambdaZipPath);
const params: CreateFunctionCommandInput = {
Code: {
ZipFile: await fs.readFile(lambdaZipPath),
},
FunctionName: yarnInstallFunctionName,
Layers: [commonLayerArn],
Handler: 'index.handler',
Role: this.options.lambdaRole,
Runtime: LogicFunctionRuntime.NODE22,
Timeout: YARN_INSTALL_LAMBDA_TIMEOUT_SECONDS,
MemorySize: YARN_INSTALL_LAMBDA_MEMORY_MB,
};
await lambdaClient.send(new CreateFunctionCommand(params));
} finally {
await temporaryDirManager.clean();
}
await this.waitFunctionActive(yarnInstallFunctionName);
}
private async invokeYarnInstallLambda(
payload: YarnInstallLambdaPayload,
): Promise<YarnInstallLambdaResult> {
const lambdaClient = await this.getLambdaClient();
const yarnInstallFunctionName = await this.getYarnInstallFunctionName();
const result = await callWithTimeout({
callback: () =>
lambdaClient.send(
new InvokeCommand({
FunctionName: yarnInstallFunctionName,
Payload: JSON.stringify(payload),
LogType: LogType.Tail,
}),
),
timeoutMs: YARN_INSTALL_LAMBDA_TIMEOUT_SECONDS * 1000,
});
if (result.FunctionError) {
const parsedResult = result.Payload
? JSON.parse(result.Payload.transformToString())
: {};
throw new LogicFunctionException(
`Yarn install Lambda failed: ${JSON.stringify(parsedResult)}`,
LogicFunctionExceptionCode.LOGIC_FUNCTION_CREATE_FAILED,
);
}
const parsedResult: YarnInstallLambdaResult = result.Payload
? JSON.parse(result.Payload.transformToString())
: {};
if (!parsedResult.success) {
throw new Error('Yarn install Lambda did not report success');
}
return parsedResult;
}
private async ensureBuilderLambdaExists(): Promise<void> {
const builderFunctionName = await this.getBuilderFunctionName();
const lambdaClient = await this.getLambdaClient();
try {
await lambdaClient.send(
new GetFunctionCommand({ FunctionName: builderFunctionName }),
);
return;
} catch (error) {
if (!(error instanceof ResourceNotFoundException)) {
throw error;
}
}
const commonLayerArn = await this.ensureCommonLayerExists();
const temporaryDirManager = new TemporaryDirManager();
const { sourceTemporaryDir, lambdaZipPath } =
await temporaryDirManager.init();
try {
await copyBuilder(sourceTemporaryDir);
await createZipFile(sourceTemporaryDir, lambdaZipPath);
const params: CreateFunctionCommandInput = {
Code: {
ZipFile: await fs.readFile(lambdaZipPath),
},
FunctionName: builderFunctionName,
Layers: [commonLayerArn],
Handler: 'index.handler',
Role: this.options.lambdaRole,
Runtime: LogicFunctionRuntime.NODE22,
Timeout: BUILDER_LAMBDA_TIMEOUT_SECONDS,
MemorySize: BUILDER_LAMBDA_MEMORY_MB,
};
await lambdaClient.send(new CreateFunctionCommand(params));
} finally {
await temporaryDirManager.clean();
}
await this.waitFunctionActive(builderFunctionName);
}
async transpile({
sourceCode,
sourceFileName,
builtFileName,
}: LogicFunctionTranspileParams): Promise<LogicFunctionTranspileResult> {
await this.ensureBuilderLambdaExists();
const lambdaClient = await this.getLambdaClient();
const builderFunctionName = await this.getBuilderFunctionName();
const payload: BuilderLambdaPayload = {
action: 'transpile',
sourceCode,
sourceFileName,
builtFileName,
};
const result = await callWithTimeout({
callback: () =>
lambdaClient.send(
new InvokeCommand({
FunctionName: builderFunctionName,
Payload: JSON.stringify(payload),
LogType: LogType.Tail,
}),
),
timeoutMs: BUILDER_LAMBDA_TIMEOUT_SECONDS * 1000,
});
if (result.FunctionError) {
const parsedResult = result.Payload
? JSON.parse(result.Payload.transformToString())
: {};
throw new LogicFunctionException(
`Builder Lambda failed: ${JSON.stringify(parsedResult)}`,
LogicFunctionExceptionCode.LOGIC_FUNCTION_CREATE_FAILED,
);
}
const parsedResult: BuilderLambdaResult = result.Payload
? JSON.parse(result.Payload.transformToString())
: {};
if (!parsedResult.builtCode) {
throw new Error('Builder Lambda did not return builtCode');
}
return { builtCode: parsedResult.builtCode };
}
private async getExistingLayerArn(
layerName: string,
): Promise<string | undefined> {
const listLayerParams: ListLayerVersionsCommandInput = {
LayerName: layerName,
MaxItems: 1,
};
const listLayerResult = await (
await this.getLambdaClient()
).send(new ListLayerVersionsCommand(listLayerParams));
return listLayerResult.LayerVersions?.[0]?.LayerVersionArn;
}
private async getDependencyContents(
flatApplication: FlatApplication,
applicationUniversalIdentifier: string,
): Promise<{ packageJson: string; yarnLock: string }> {
const temporaryDirManager = new TemporaryDirManager();
const { sourceTemporaryDir } = await temporaryDirManager.init();
try {
await this.logicFunctionResourceService.copyDependenciesInMemory({
applicationUniversalIdentifier,
workspaceId: flatApplication.workspaceId,
inMemoryFolderPath: sourceTemporaryDir,
});
const [packageJson, yarnLock] = await Promise.all([
fs.readFile(`${sourceTemporaryDir}/package.json`, 'utf-8'),
fs.readFile(`${sourceTemporaryDir}/yarn.lock`, 'utf-8'),
]);
return { packageJson, yarnLock };
} finally {
await temporaryDirManager.clean();
}
}
private async createLayerIfNotExist({
flatApplication,
applicationUniversalIdentifier,
}: {
flatApplication: FlatApplication;
applicationUniversalIdentifier: string;
}): Promise<void> {
const layerName = this.getLayerName(flatApplication);
const existingArn = await this.getExistingLayerArn(layerName);
if (isDefined(existingArn)) {
return;
}
const { packageJson, yarnLock } = await this.getDependencyContents(
flatApplication,
applicationUniversalIdentifier,
);
await this.ensureYarnInstallLambdaExists();
const s3Key = `lambda-layers/${layerName}.zip`;
const presignedUploadUrl = await this.generatePresignedUploadUrl(s3Key);
await this.invokeYarnInstallLambda({
action: 'createLayer',
packageJson,
yarnLock,
presignedUploadUrl,
});
const bucket = this.options.layerBucket;
const lambdaClient = await this.getLambdaClient();
const publishResult = await lambdaClient.send(
new PublishLayerVersionCommand({
LayerName: layerName,
Content: { S3Bucket: bucket, S3Key: s3Key },
CompatibleRuntimes: [
LogicFunctionRuntime.NODE18,
LogicFunctionRuntime.NODE22,
],
}),
);
if (!publishResult.LayerVersionArn) {
throw new Error(
`PublishLayerVersion did not return a LayerVersionArn for layer '${layerName}'`,
);
}
}
private async getLayerArn({
flatApplication,
applicationUniversalIdentifier,
}: {
@@ -147,58 +607,26 @@ export class LambdaDriver implements LogicFunctionDriver {
}): Promise<string> {
const layerName = this.getLayerName(flatApplication);
const listLayerParams: ListLayerVersionsCommandInput = {
LayerName: layerName,
MaxItems: 1,
};
const existingArn = await this.getExistingLayerArn(layerName);
const listLayerCommand = new ListLayerVersionsCommand(listLayerParams);
const listLayerResult = await (
await this.getLambdaClient()
).send(listLayerCommand);
if (isDefined(listLayerResult.LayerVersions?.[0]?.LayerVersionArn)) {
return listLayerResult.LayerVersions[0].LayerVersionArn;
if (isDefined(existingArn)) {
return existingArn;
}
const temporaryDirManager = new TemporaryDirManager();
const { sourceTemporaryDir, lambdaZipPath } =
await temporaryDirManager.init();
const nodeDependenciesFolder = join(sourceTemporaryDir, 'nodejs');
await this.logicFunctionResourceService.copyDependenciesInMemory({
await this.createLayerIfNotExist({
flatApplication,
applicationUniversalIdentifier,
workspaceId: flatApplication.workspaceId,
inMemoryFolderPath: nodeDependenciesFolder,
});
await copyYarnEngineAndBuildDependencies(nodeDependenciesFolder);
await createZipFile(sourceTemporaryDir, lambdaZipPath);
const newArn = await this.getExistingLayerArn(layerName);
const params: PublishLayerVersionCommandInput = {
LayerName: layerName,
Content: {
ZipFile: await fs.readFile(lambdaZipPath),
},
CompatibleRuntimes: [
LogicFunctionRuntime.NODE18,
LogicFunctionRuntime.NODE22,
],
};
const command = new PublishLayerVersionCommand(params);
const result = await (await this.getLambdaClient()).send(command);
await temporaryDirManager.clean();
if (!isDefined(result.LayerVersionArn)) {
throw new Error('new layer version arn if undefined');
if (!isDefined(newArn)) {
throw new Error(
`Layer '${layerName}' was not created by the yarn install Lambda`,
);
}
return result.LayerVersionArn;
return newArn;
}
private async getLambdaExecutor(flatLogicFunction: FlatLogicFunction) {
@@ -269,7 +697,7 @@ export class LambdaDriver implements LogicFunctionDriver {
return;
}
const layerArn = await this.createLayerIfNotExists({
const layerArn = await this.getLayerArn({
flatApplication,
applicationUniversalIdentifier,
});
@@ -331,7 +759,7 @@ export class LambdaDriver implements LogicFunctionDriver {
applicationUniversalIdentifier,
});
await this.waitFunctionUpdates(flatLogicFunction);
await this.waitFunctionActive(flatLogicFunction.id);
const startTime = Date.now();
@@ -1,11 +1,16 @@
import { promises as fs } from 'fs';
import { spawn } from 'node:child_process';
import { join } from 'path';
import { dirname, join } from 'path';
import { build } from 'esbuild';
import { NODE_ESM_CJS_BANNER } from 'twenty-shared/application';
import {
type LogicFunctionExecuteParams,
type LogicFunctionExecuteResult,
type LogicFunctionDriver,
type LogicFunctionTranspileParams,
type LogicFunctionTranspileResult,
} from 'src/engine/core-modules/logic-function/logic-function-drivers/interfaces/logic-function-driver.interface';
import { type FlatApplication } from 'src/engine/core-modules/application/types/flat-application.type';
@@ -34,7 +39,7 @@ export class LocalDriver implements LogicFunctionDriver {
return join(LOGIC_FUNCTION_EXECUTOR_TMPDIR_FOLDER, checksum);
};
private async createLayerIfNotExists({
private async createLayerIfNotExist({
flatApplication,
applicationUniversalIdentifier,
}: {
@@ -56,16 +61,52 @@ export class LocalDriver implements LogicFunctionDriver {
}
}
async transpile({
sourceCode,
sourceFileName,
builtFileName,
}: LogicFunctionTranspileParams): Promise<LogicFunctionTranspileResult> {
const temporaryDirManager = new TemporaryDirManager();
const { sourceTemporaryDir } = await temporaryDirManager.init();
try {
const entryFilePath = join(sourceTemporaryDir, sourceFileName);
const builtBundleFilePath = join(sourceTemporaryDir, builtFileName);
await fs.mkdir(dirname(entryFilePath), { recursive: true });
await fs.writeFile(entryFilePath, sourceCode, 'utf-8');
await fs.mkdir(dirname(builtBundleFilePath), { recursive: true });
await build({
entryPoints: [entryFilePath],
outfile: builtBundleFilePath,
platform: 'node',
format: 'esm',
target: 'es2017',
bundle: true,
sourcemap: true,
packages: 'external',
banner: NODE_ESM_CJS_BANNER,
});
const builtCode = await fs.readFile(builtBundleFilePath, 'utf-8');
return { builtCode };
} finally {
await temporaryDirManager.clean();
}
}
async delete() {}
private async build({
async build({
flatApplication,
applicationUniversalIdentifier,
}: {
flatApplication: FlatApplication;
applicationUniversalIdentifier: string;
}) {
await this.createLayerIfNotExists({
await this.createLayerIfNotExist({
flatApplication,
applicationUniversalIdentifier,
});
@@ -79,7 +120,7 @@ export class LocalDriver implements LogicFunctionDriver {
env,
timeoutMs = 900_000,
}: LogicFunctionExecuteParams): Promise<LogicFunctionExecuteResult> {
await this.build({
await this.createLayerIfNotExist({
flatApplication,
applicationUniversalIdentifier,
});
@@ -41,6 +41,17 @@ export const logicFunctionModuleFactory = async (
'LOGIC_FUNCTION_LAMBDA_SUBHOSTING_ROLE',
);
const s3BucketName = twentyConfigService.get('STORAGE_S3_NAME');
const layerBucket =
twentyConfigService.get('LOGIC_FUNCTION_LAMBDA_LAYER_BUCKET') ??
s3BucketName ??
'twenty-lambda-layer';
const layerBucketRegion =
twentyConfigService.get('LOGIC_FUNCTION_LAMBDA_LAYER_BUCKET_REGION') ??
region;
return {
type: LogicFunctionDriverType.LAMBDA,
options: {
@@ -56,6 +67,8 @@ export const logicFunctionModuleFactory = async (
region,
lambdaRole,
subhostingRole,
layerBucket,
layerBucketRegion,
},
};
}
@@ -29,11 +29,24 @@ export type LogicFunctionExecuteParams = {
timeoutMs?: number;
};
export type LogicFunctionTranspileParams = {
sourceCode: string;
sourceFileName: string;
builtFileName: string;
};
export type LogicFunctionTranspileResult = {
builtCode: string;
};
export interface LogicFunctionDriver {
delete(flatLogicFunction: FlatLogicFunction): Promise<void>;
execute(
params: LogicFunctionExecuteParams,
): Promise<LogicFunctionExecuteResult>;
transpile(
params: LogicFunctionTranspileParams,
): Promise<LogicFunctionTranspileResult>;
}
export enum LogicFunctionDriverType {
@@ -0,0 +1,17 @@
import { promises as fs } from 'fs';
import { resolve, join } from 'path';
import { ASSET_PATH } from 'src/constants/assets-path';
const BUILDER_FILE_PATH = resolve(
__dirname,
join(
ASSET_PATH,
'engine/core-modules/logic-function/logic-function-drivers/constants/builder',
),
);
export const copyBuilder = async (buildDirectory: string) => {
await fs.mkdir(buildDirectory, { recursive: true });
await fs.cp(BUILDER_FILE_PATH, buildDirectory, { recursive: true });
};
@@ -0,0 +1,22 @@
import { promises as fs } from 'fs';
import { join } from 'path';
import { COMMON_LAYER_DEPENDENCIES_DIRNAME } from 'src/engine/core-modules/logic-function/logic-function-drivers/constants/common-layer-dependencies-dirname';
import { copyYarnEngineAndBuildDependencies } from 'src/engine/core-modules/application/application-package/utils/copy-yarn-engine-and-build-dependencies';
// Builds a Lambda layer with the common dependencies (e.g. archiver)
// by copying the package.json + yarn.lock, then running yarn install locally.
// The result follows the Lambda layer structure: nodejs/node_modules/...
export const copyCommonLayerDependencies = async (
buildDirectory: string,
): Promise<void> => {
const nodejsDir = join(buildDirectory, 'nodejs');
await fs.mkdir(nodejsDir, { recursive: true });
await fs.cp(COMMON_LAYER_DEPENDENCIES_DIRNAME, nodejsDir, {
recursive: true,
});
await copyYarnEngineAndBuildDependencies(nodejsDir);
};
@@ -0,0 +1,23 @@
import { promises as fs } from 'fs';
import { resolve, join } from 'path';
import { ASSET_PATH } from 'src/constants/assets-path';
import { YARN_ENGINE_DIRNAME } from 'src/engine/core-modules/application/application-package/constants/yarn-engine-dirname';
const YARN_INSTALL_FILE_PATH = resolve(
__dirname,
join(
ASSET_PATH,
'engine/core-modules/logic-function/logic-function-drivers/constants/yarn-install',
),
);
export const copyYarnInstall = async (buildDirectory: string) => {
await fs.mkdir(buildDirectory, { recursive: true });
await fs.cp(YARN_INSTALL_FILE_PATH, buildDirectory, { recursive: true });
const yarnEngineDestination = join(buildDirectory, 'yarn-engine');
await fs.cp(YARN_ENGINE_DIRNAME, yarnEngineDestination, { recursive: true });
};
@@ -10,6 +10,8 @@ import { isDefined } from 'twenty-shared/utils';
import {
LogicFunctionDriver,
type LogicFunctionExecuteResult,
type LogicFunctionTranspileParams,
type LogicFunctionTranspileResult,
} from 'src/engine/core-modules/logic-function/logic-function-drivers/interfaces/logic-function-driver.interface';
import { FlatApplication } from 'src/engine/core-modules/application/types/flat-application.type';
@@ -101,6 +103,12 @@ export class LogicFunctionExecutorService {
return resultLogicFunction;
}
async transpile(
params: LogicFunctionTranspileParams,
): Promise<LogicFunctionTranspileResult> {
return this.driver.transpile(params);
}
private async throttleExecution(workspaceId: string) {
try {
await this.throttlerService.tokenBucketThrottleOrThrow(
@@ -1,18 +1,14 @@
import { Injectable } from '@nestjs/common';
import crypto from 'crypto';
import fs from 'fs/promises';
import { dirname, join } from 'path';
import { join } from 'path';
import { build } from 'esbuild';
import { FileFolder } from 'twenty-shared/types';
import { isDefined } from 'twenty-shared/utils';
import { NODE_ESM_CJS_BANNER } from 'twenty-shared/application';
import { FileStorageExceptionCode } from 'src/engine/core-modules/file-storage/interfaces/file-storage-exception';
import { FileStorageService } from 'src/engine/core-modules/file-storage/file-storage.service';
import { TemporaryDirManager } from 'src/engine/core-modules/logic-function/logic-function-drivers/utils/temporary-dir-manager';
import {
getLogicFunctionSeedProjectFiles,
LogicFunctionSeedProjectFile,
@@ -45,11 +41,6 @@ type UpdateSourceFilesParams = Omit<
sourceHandlerCode: string;
};
type BuildFromSourceParams = Identifier & {
sourceHandlerPath: string;
builtHandlerPath: string;
};
type GetSourceCodeParams = Identifier & {
sourceHandlerPath: string;
};
@@ -148,52 +139,27 @@ export class LogicFunctionResourceService {
});
}
async buildFromSourceFile({
sourceHandlerPath,
builtHandlerPath,
async uploadBuiltFile({
workspaceId,
applicationUniversalIdentifier,
}: BuildFromSourceParams): Promise<{ checksum: string }> {
const temporaryDirManager = new TemporaryDirManager();
try {
const { sourceTemporaryDir } = await temporaryDirManager.init();
await this.fileStorageService.downloadFile({
workspaceId,
applicationUniversalIdentifier,
fileFolder: FileFolder.Source,
resourcePath: sourceHandlerPath,
localPath: join(sourceTemporaryDir, sourceHandlerPath),
});
const builtBundleFilePath = await this.buildInMemory({
sourceTemporaryDir,
sourceHandlerPath,
builtHandlerPath,
});
const builtFile = await fs.readFile(builtBundleFilePath, 'utf-8');
await this.fileStorageService.writeFile({
workspaceId,
applicationUniversalIdentifier,
fileFolder: FileFolder.BuiltLogicFunction,
resourcePath: builtHandlerPath,
sourceFile: builtFile,
mimeType: 'application/javascript',
settings: {
isTemporaryFile: false,
toDelete: false,
},
});
return {
checksum: crypto.createHash('md5').update(builtFile).digest('hex'),
};
} finally {
await temporaryDirManager.clean();
}
builtHandlerPath,
builtCode,
}: Identifier & {
builtHandlerPath: string;
builtCode: string;
}): Promise<void> {
await this.fileStorageService.writeFile({
workspaceId,
applicationUniversalIdentifier,
fileFolder: FileFolder.BuiltLogicFunction,
resourcePath: builtHandlerPath,
sourceFile: builtCode,
mimeType: 'application/javascript',
settings: {
isTemporaryFile: false,
toDelete: false,
},
});
}
async getSourceFile({
@@ -328,33 +294,4 @@ export class LogicFunctionResourceService {
return localPath;
}
private async buildInMemory({
sourceTemporaryDir,
sourceHandlerPath,
builtHandlerPath,
}: {
sourceTemporaryDir: string;
sourceHandlerPath: string;
builtHandlerPath: string;
}): Promise<string> {
const entryFilePath = join(sourceTemporaryDir, sourceHandlerPath);
const builtBundleFilePath = join(sourceTemporaryDir, builtHandlerPath);
await fs.mkdir(dirname(builtBundleFilePath), { recursive: true });
await build({
entryPoints: [entryFilePath],
outfile: builtBundleFilePath,
platform: 'node',
format: 'esm',
target: 'es2017',
bundle: true,
sourcemap: true,
packages: 'external',
banner: NODE_ESM_CJS_BANNER,
});
return builtBundleFilePath;
}
}
@@ -563,6 +563,30 @@ export class ConfigVariables {
@IsOptional()
LOGIC_FUNCTION_LAMBDA_SECRET_ACCESS_KEY: string;
@ConfigVariablesMetadata({
group: ConfigVariablesGroup.LOGIC_FUNCTION_CONFIG,
description: 'S3 bucket for uploading Lambda layer zip files',
type: ConfigVariableType.STRING,
})
@ValidateIf(
(env) => env.LOGIC_FUNCTION_TYPE === LogicFunctionDriverType.LAMBDA,
)
@IsOptional()
LOGIC_FUNCTION_LAMBDA_LAYER_BUCKET?: string;
@ConfigVariablesMetadata({
group: ConfigVariablesGroup.LOGIC_FUNCTION_CONFIG,
description:
'AWS region of the S3 bucket for Lambda layer uploads (defaults to LOGIC_FUNCTION_LAMBDA_REGION)',
type: ConfigVariableType.STRING,
})
@ValidateIf(
(env) => env.LOGIC_FUNCTION_TYPE === LogicFunctionDriverType.LAMBDA,
)
@IsOptional()
@IsAWSRegion()
LOGIC_FUNCTION_LAMBDA_LAYER_BUCKET_REGION?: AwsRegion;
@ConfigVariablesMetadata({
group: ConfigVariablesGroup.CODE_INTERPRETER_CONFIG,
description:
@@ -1,5 +1,7 @@
import { Injectable } from '@nestjs/common';
import crypto from 'crypto';
import { v4 } from 'uuid';
import { isDefined } from 'twenty-shared/utils';
import { SEED_LOGIC_FUNCTION_INPUT_SCHEMA } from 'twenty-shared/logic-function';
@@ -302,14 +304,33 @@ export class LogicFunctionFromSourceService {
workspaceId,
});
const { checksum } =
await this.logicFunctionResourceService.buildFromSourceFile({
workspaceId,
applicationUniversalIdentifier:
ownerFlatApplication.universalIdentifier,
sourceHandlerPath: flatLogicFunction.sourceHandlerPath,
builtHandlerPath: flatLogicFunction.builtHandlerPath,
});
const sourceCode = await this.logicFunctionResourceService.getSourceFile({
workspaceId,
applicationUniversalIdentifier: ownerFlatApplication.universalIdentifier,
sourceHandlerPath: flatLogicFunction.sourceHandlerPath,
});
if (!sourceCode) {
throw new LogicFunctionException(
'Source file not found',
LogicFunctionExceptionCode.LOGIC_FUNCTION_NOT_FOUND,
);
}
const { builtCode } = await this.logicFunctionExecutorService.transpile({
sourceCode,
sourceFileName: flatLogicFunction.sourceHandlerPath,
builtFileName: flatLogicFunction.builtHandlerPath,
});
await this.logicFunctionResourceService.uploadBuiltFile({
workspaceId,
applicationUniversalIdentifier: ownerFlatApplication.universalIdentifier,
builtHandlerPath: flatLogicFunction.builtHandlerPath,
builtCode,
});
const checksum = crypto.createHash('md5').update(builtCode).digest('hex');
await this.helperService.updateOneFromMetadata({
flatLogicFunctionToUpdate: {
+59
View File
@@ -1662,6 +1662,22 @@ __metadata:
languageName: node
linkType: hard
"@aws-sdk/s3-request-presigner@npm:3.1001.0":
version: 3.1001.0
resolution: "@aws-sdk/s3-request-presigner@npm:3.1001.0"
dependencies:
"@aws-sdk/signature-v4-multi-region": "npm:^3.996.4"
"@aws-sdk/types": "npm:^3.973.4"
"@aws-sdk/util-format-url": "npm:^3.972.6"
"@smithy/middleware-endpoint": "npm:^4.4.21"
"@smithy/protocol-http": "npm:^5.3.10"
"@smithy/smithy-client": "npm:^4.12.1"
"@smithy/types": "npm:^4.13.0"
tslib: "npm:^2.6.2"
checksum: 10c0/51cbe9bcef780a391bf5e0a084490ff40bba317d6741ed14b3b5d5f6c761fc161e71de7241527e02f8b56e0b6ac4bce9045a43aeed8a79cb0418d4f78f2134dd
languageName: node
linkType: hard
"@aws-sdk/signature-v4-multi-region@npm:^3.996.4":
version: 3.996.4
resolution: "@aws-sdk/signature-v4-multi-region@npm:3.996.4"
@@ -1711,6 +1727,16 @@ __metadata:
languageName: node
linkType: hard
"@aws-sdk/types@npm:^3.973.5":
version: 3.973.5
resolution: "@aws-sdk/types@npm:3.973.5"
dependencies:
"@smithy/types": "npm:^4.13.0"
tslib: "npm:^2.6.2"
checksum: 10c0/9d21676eb696afc71e915b96f87ae8c67f53a52b3cf3b20f6c35e05654c5a36b67f917310ecd1dd6b696be2019e184dc5bd278318ee1f0051e51f80b292ed578
languageName: node
linkType: hard
"@aws-sdk/util-arn-parser@npm:^3.972.2":
version: 3.972.2
resolution: "@aws-sdk/util-arn-parser@npm:3.972.2"
@@ -1733,6 +1759,18 @@ __metadata:
languageName: node
linkType: hard
"@aws-sdk/util-format-url@npm:^3.972.6":
version: 3.972.7
resolution: "@aws-sdk/util-format-url@npm:3.972.7"
dependencies:
"@aws-sdk/types": "npm:^3.973.5"
"@smithy/querystring-builder": "npm:^4.2.11"
"@smithy/types": "npm:^4.13.0"
tslib: "npm:^2.6.2"
checksum: 10c0/eb3511476c49c1044184f78f0ee9a3d057fb4f91711700d9e2b53ad715d59e97a2782314fb25eec0d8310c53349a4471edf6fbf3be42c6db5a7a30bb682f3e63
languageName: node
linkType: hard
"@aws-sdk/util-locate-window@npm:^3.0.0":
version: 3.568.0
resolution: "@aws-sdk/util-locate-window@npm:3.568.0"
@@ -21097,6 +21135,17 @@ __metadata:
languageName: node
linkType: hard
"@smithy/querystring-builder@npm:^4.2.11":
version: 4.2.11
resolution: "@smithy/querystring-builder@npm:4.2.11"
dependencies:
"@smithy/types": "npm:^4.13.0"
"@smithy/util-uri-escape": "npm:^4.2.2"
tslib: "npm:^2.6.2"
checksum: 10c0/88790ee8c7a01730beba913fdf38d1790113c82c956d20cccd43a5b527f19e23df7eff86ddf41e3e615d0af562e31f61a7fed014c9fce8db924623258a4d14f8
languageName: node
linkType: hard
"@smithy/querystring-parser@npm:^4.2.10":
version: 4.2.10
resolution: "@smithy/querystring-parser@npm:4.2.10"
@@ -21365,6 +21414,15 @@ __metadata:
languageName: node
linkType: hard
"@smithy/util-uri-escape@npm:^4.2.2":
version: 4.2.2
resolution: "@smithy/util-uri-escape@npm:4.2.2"
dependencies:
tslib: "npm:^2.6.2"
checksum: 10c0/33b6546086c975278d16b5029e6555df551b4bd1e3a84042544d1ef956a287fe033b317954b1737b2773e82b6f27ebde542956ff79ef0e8a813dc0dbf9d34a58
languageName: node
linkType: hard
"@smithy/util-utf8@npm:^2.0.0":
version: 2.3.0
resolution: "@smithy/util-utf8@npm:2.3.0"
@@ -55935,6 +55993,7 @@ __metadata:
"@aws-sdk/client-sesv2": "npm:3.1001.0"
"@aws-sdk/client-sts": "npm:3.1001.0"
"@aws-sdk/credential-providers": "npm:3.1001.0"
"@aws-sdk/s3-request-presigner": "npm:3.1001.0"
"@azure/msal-node": "npm:^3.8.4"
"@babel/preset-env": "npm:7.26.9"
"@blocknote/server-util": "npm:^0.47.1"