Run vulnerable operation in isolated environment (#18523)
When driver = LAMBDA: - run esbuild ts transpilation on dedicated lambda - run yarn install on app dependencies on a dedicated lambda
This commit is contained in:
@@ -32,6 +32,18 @@
|
||||
"include": "engine/core-modules/logic-function/logic-function-drivers/constants/executor/index.mjs",
|
||||
"outDir": "dist/assets"
|
||||
},
|
||||
{
|
||||
"include": "engine/core-modules/logic-function/logic-function-drivers/constants/yarn-install/index.mjs",
|
||||
"outDir": "dist/assets"
|
||||
},
|
||||
{
|
||||
"include": "engine/core-modules/logic-function/logic-function-drivers/constants/builder/index.mjs",
|
||||
"outDir": "dist/assets"
|
||||
},
|
||||
{
|
||||
"include": "engine/core-modules/logic-function/logic-function-drivers/constants/common-layer-dependencies/**",
|
||||
"outDir": "dist/assets"
|
||||
},
|
||||
{
|
||||
"include": "database/clickHouse/migrations/*.sql",
|
||||
"outDir": "dist"
|
||||
|
||||
@@ -28,6 +28,7 @@
|
||||
"@aws-sdk/client-sesv2": "3.1001.0",
|
||||
"@aws-sdk/client-sts": "3.1001.0",
|
||||
"@aws-sdk/credential-providers": "3.1001.0",
|
||||
"@aws-sdk/s3-request-presigner": "3.1001.0",
|
||||
"@azure/msal-node": "^3.8.4",
|
||||
"@babel/preset-env": "7.26.9",
|
||||
"@blocknote/server-util": "^0.47.1",
|
||||
|
||||
+10
@@ -5,3 +5,13 @@ enableScripts: false
|
||||
nodeLinker: node-modules
|
||||
|
||||
yarnPath: .yarn/releases/yarn-4.9.2.cjs
|
||||
|
||||
supportedArchitectures:
|
||||
os:
|
||||
- current
|
||||
- linux
|
||||
- darwin
|
||||
cpu:
|
||||
- current
|
||||
- x64
|
||||
- arm64
|
||||
|
||||
+65
@@ -0,0 +1,65 @@
|
||||
import { randomBytes } from 'crypto';
|
||||
import { promises as fs } from 'fs';
|
||||
import { dirname, resolve } from 'path';
|
||||
|
||||
import { build } from 'esbuild';
|
||||
|
||||
const BANNER = {
|
||||
js: "import { createRequire as __createRequire } from 'module';\nconst require = __createRequire(import.meta.url);",
|
||||
};
|
||||
|
||||
const assertPathInsideDir = (filePath, dir) => {
|
||||
const resolved = resolve(dir, filePath);
|
||||
|
||||
if (!resolved.startsWith(dir + '/')) {
|
||||
throw new Error(`Path traversal detected: ${filePath}`);
|
||||
}
|
||||
|
||||
return resolved;
|
||||
};
|
||||
|
||||
export const handler = async (event) => {
|
||||
const { action, sourceCode, sourceFileName, builtFileName } = event;
|
||||
|
||||
if (action !== 'transpile') {
|
||||
throw new Error(`Unknown action: ${action}`);
|
||||
}
|
||||
|
||||
if (!sourceCode || !sourceFileName || !builtFileName) {
|
||||
throw new Error(
|
||||
'Missing required fields: sourceCode, sourceFileName, builtFileName',
|
||||
);
|
||||
}
|
||||
|
||||
const randomId = randomBytes(16).toString('hex');
|
||||
const workDir = `/tmp/${randomId}`;
|
||||
|
||||
await fs.mkdir(workDir, { recursive: true });
|
||||
|
||||
try {
|
||||
const entryFilePath = assertPathInsideDir(sourceFileName, workDir);
|
||||
const outFilePath = assertPathInsideDir(builtFileName, workDir);
|
||||
|
||||
await fs.mkdir(dirname(entryFilePath), { recursive: true });
|
||||
await fs.writeFile(entryFilePath, sourceCode, 'utf-8');
|
||||
await fs.mkdir(dirname(outFilePath), { recursive: true });
|
||||
|
||||
await build({
|
||||
entryPoints: [entryFilePath],
|
||||
outfile: outFilePath,
|
||||
platform: 'node',
|
||||
format: 'esm',
|
||||
target: 'es2017',
|
||||
bundle: true,
|
||||
sourcemap: true,
|
||||
packages: 'external',
|
||||
banner: BANNER,
|
||||
});
|
||||
|
||||
const builtCode = await fs.readFile(outFilePath, 'utf-8');
|
||||
|
||||
return { builtCode };
|
||||
} finally {
|
||||
await fs.rm(workDir, { recursive: true, force: true });
|
||||
}
|
||||
};
|
||||
+11
@@ -0,0 +1,11 @@
|
||||
import path from 'path';
|
||||
|
||||
import { ASSET_PATH } from 'src/constants/assets-path';
|
||||
|
||||
export const COMMON_LAYER_DEPENDENCIES_DIRNAME = path.resolve(
|
||||
__dirname,
|
||||
path.join(
|
||||
ASSET_PATH,
|
||||
'engine/core-modules/logic-function/logic-function-drivers/constants/common-layer-dependencies',
|
||||
),
|
||||
);
|
||||
+6
@@ -0,0 +1,6 @@
|
||||
{
|
||||
"dependencies": {
|
||||
"archiver": "^7.0.1",
|
||||
"esbuild": "^0.25.0"
|
||||
}
|
||||
}
|
||||
+1030
File diff suppressed because it is too large
Load Diff
+133
@@ -0,0 +1,133 @@
|
||||
import { randomBytes } from 'crypto';
|
||||
import { createWriteStream, promises as fs } from 'fs';
|
||||
import { join, resolve } from 'path';
|
||||
|
||||
import archiver from 'archiver';
|
||||
import { pipeline } from 'stream/promises';
|
||||
|
||||
const YARN_INSTALL_TIMEOUT_MS = 240_000;
|
||||
const YARN_ENGINE_DIR = resolve('yarn-engine');
|
||||
const YARN_ENGINE_PATH = join(YARN_ENGINE_DIR, '.yarn/releases/yarn-4.9.2.cjs');
|
||||
|
||||
const writePackageFiles = async (nodejsDir, packageJson, yarnLock) => {
|
||||
await fs.mkdir(nodejsDir, { recursive: true });
|
||||
await Promise.all([
|
||||
fs.writeFile(join(nodejsDir, 'package.json'), packageJson, 'utf-8'),
|
||||
fs.writeFile(join(nodejsDir, 'yarn.lock'), yarnLock, 'utf-8'),
|
||||
]);
|
||||
};
|
||||
|
||||
const copyYarnEngine = async (nodejsDir) => {
|
||||
await fs.cp('yarn-engine', nodejsDir, { recursive: true });
|
||||
};
|
||||
|
||||
const runYarnInstall = async (nodejsDir) => {
|
||||
const { execFile } = await import('child_process');
|
||||
const { promisify } = await import('util');
|
||||
const execFilePromise = promisify(execFile);
|
||||
|
||||
const { NODE_OPTIONS: _nodeOptions, ...cleanEnv } = process.env;
|
||||
|
||||
// Lambda runs as a sandboxed user whose $HOME doesn't exist.
|
||||
// Yarn needs a writable HOME for its global cache/config.
|
||||
cleanEnv.HOME = '/tmp';
|
||||
|
||||
try {
|
||||
await execFilePromise(
|
||||
process.execPath,
|
||||
[YARN_ENGINE_PATH, 'workspaces', 'focus', '--all', '--production'],
|
||||
{
|
||||
cwd: nodejsDir,
|
||||
env: cleanEnv,
|
||||
timeout: YARN_INSTALL_TIMEOUT_MS,
|
||||
},
|
||||
);
|
||||
} catch (error) {
|
||||
const details = [error?.stdout, error?.stderr].filter(Boolean).join('\n');
|
||||
|
||||
throw new Error(`yarn install failed: ${details || error?.message}`);
|
||||
}
|
||||
|
||||
// Remove everything except node_modules
|
||||
const entries = await fs.readdir(nodejsDir);
|
||||
|
||||
await Promise.all(
|
||||
entries
|
||||
.filter((entry) => entry !== 'node_modules')
|
||||
.map(async (entry) => {
|
||||
const fullPath = join(nodejsDir, entry);
|
||||
const stat = await fs.stat(fullPath);
|
||||
|
||||
return stat.isDirectory()
|
||||
? fs.rm(fullPath, { recursive: true, force: true })
|
||||
: fs.rm(fullPath);
|
||||
}),
|
||||
);
|
||||
};
|
||||
|
||||
const createZip = async (buildDir, zipPath) => {
|
||||
const output = createWriteStream(zipPath);
|
||||
const archive = archiver('zip', { zlib: { level: 9 } });
|
||||
|
||||
const p = pipeline(archive, output);
|
||||
|
||||
archive.directory(buildDir, false);
|
||||
archive.finalize();
|
||||
|
||||
return p;
|
||||
};
|
||||
|
||||
const uploadToPresignedUrl = async (zipPath, presignedUploadUrl) => {
|
||||
const zipBuffer = await fs.readFile(zipPath);
|
||||
|
||||
const response = await fetch(presignedUploadUrl, {
|
||||
method: 'PUT',
|
||||
body: zipBuffer,
|
||||
headers: { 'Content-Type': 'application/zip' },
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(
|
||||
`Failed to upload zip to S3: ${response.status} ${response.statusText}`,
|
||||
);
|
||||
}
|
||||
};
|
||||
|
||||
export const handler = async (event) => {
|
||||
const { action, packageJson, yarnLock, presignedUploadUrl } = event;
|
||||
|
||||
if (action !== 'createLayer') {
|
||||
throw new Error(`Unknown action: ${action}`);
|
||||
}
|
||||
|
||||
if (!packageJson || !yarnLock) {
|
||||
throw new Error('Missing required fields: packageJson, yarnLock');
|
||||
}
|
||||
|
||||
if (!presignedUploadUrl) {
|
||||
throw new Error('Missing required field: presignedUploadUrl');
|
||||
}
|
||||
|
||||
const randomId = randomBytes(16).toString('hex');
|
||||
const buildDir = `/tmp/${randomId}`;
|
||||
const nodejsDir = join(buildDir, 'nodejs');
|
||||
const zipPath = `/tmp/${randomId}.zip`;
|
||||
|
||||
try {
|
||||
await writePackageFiles(nodejsDir, packageJson, yarnLock);
|
||||
await copyYarnEngine(nodejsDir);
|
||||
await runYarnInstall(nodejsDir);
|
||||
await createZip(buildDir, zipPath);
|
||||
|
||||
await uploadToPresignedUrl(zipPath, presignedUploadUrl);
|
||||
|
||||
return { success: true };
|
||||
} finally {
|
||||
await fs.rm(buildDir, { recursive: true, force: true });
|
||||
await fs.rm(zipPath, { force: true });
|
||||
await fs.rm(join(YARN_ENGINE_DIR, '.yarn/cache'), {
|
||||
recursive: true,
|
||||
force: true,
|
||||
});
|
||||
}
|
||||
};
|
||||
+8
@@ -1,6 +1,7 @@
|
||||
import {
|
||||
type LogicFunctionDriver,
|
||||
type LogicFunctionExecuteResult,
|
||||
type LogicFunctionTranspileResult,
|
||||
} from 'src/engine/core-modules/logic-function/logic-function-drivers/interfaces/logic-function-driver.interface';
|
||||
|
||||
import {
|
||||
@@ -19,4 +20,11 @@ export class DisabledDriver implements LogicFunctionDriver {
|
||||
LogicFunctionExceptionCode.LOGIC_FUNCTION_DISABLED,
|
||||
);
|
||||
}
|
||||
|
||||
async transpile(): Promise<LogicFunctionTranspileResult> {
|
||||
throw new LogicFunctionException(
|
||||
'Logic function transpilation is disabled. Set LOGIC_FUNCTION_TYPE to LOCAL or LAMBDA to enable.',
|
||||
LogicFunctionExceptionCode.LOGIC_FUNCTION_DISABLED,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
+486
-58
@@ -1,5 +1,6 @@
|
||||
import { createHash } from 'crypto';
|
||||
import * as fs from 'fs/promises';
|
||||
import { join } from 'path';
|
||||
import { resolve, join } from 'path';
|
||||
|
||||
import {
|
||||
CreateFunctionCommand,
|
||||
@@ -14,10 +15,11 @@ import {
|
||||
type ListLayerVersionsCommandInput,
|
||||
LogType,
|
||||
PublishLayerVersionCommand,
|
||||
type PublishLayerVersionCommandInput,
|
||||
ResourceNotFoundException,
|
||||
waitUntilFunctionUpdatedV2,
|
||||
waitUntilFunctionActiveV2,
|
||||
} from '@aws-sdk/client-lambda';
|
||||
import { PutObjectCommand, S3Client } from '@aws-sdk/client-s3';
|
||||
import { getSignedUrl } from '@aws-sdk/s3-request-presigner';
|
||||
import { AssumeRoleCommand, STSClient } from '@aws-sdk/client-sts';
|
||||
import { isDefined } from 'twenty-shared/utils';
|
||||
|
||||
@@ -25,10 +27,17 @@ import {
|
||||
type LogicFunctionExecuteParams,
|
||||
type LogicFunctionExecuteResult,
|
||||
type LogicFunctionDriver,
|
||||
type LogicFunctionTranspileParams,
|
||||
type LogicFunctionTranspileResult,
|
||||
} from 'src/engine/core-modules/logic-function/logic-function-drivers/interfaces/logic-function-driver.interface';
|
||||
|
||||
import { ASSET_PATH } from 'src/constants/assets-path';
|
||||
import { type FlatApplication } from 'src/engine/core-modules/application/types/flat-application.type';
|
||||
import { COMMON_LAYER_DEPENDENCIES_DIRNAME } from 'src/engine/core-modules/logic-function/logic-function-drivers/constants/common-layer-dependencies-dirname';
|
||||
import { copyBuilder } from 'src/engine/core-modules/logic-function/logic-function-drivers/utils/copy-builder';
|
||||
import { copyCommonLayerDependencies } from 'src/engine/core-modules/logic-function/logic-function-drivers/utils/copy-common-layer-dependencies';
|
||||
import { copyExecutor } from 'src/engine/core-modules/logic-function/logic-function-drivers/utils/copy-executor';
|
||||
import { copyYarnInstall } from 'src/engine/core-modules/logic-function/logic-function-drivers/utils/copy-yarn-install';
|
||||
import { createZipFile } from 'src/engine/core-modules/logic-function/logic-function-drivers/utils/create-zip-file';
|
||||
import { TemporaryDirManager } from 'src/engine/core-modules/logic-function/logic-function-drivers/utils/temporary-dir-manager';
|
||||
import { LogicFunctionExecutionStatus } from 'src/engine/metadata-modules/logic-function/dtos/logic-function-execution-result.dto';
|
||||
@@ -38,12 +47,32 @@ import {
|
||||
LogicFunctionExceptionCode,
|
||||
} from 'src/engine/metadata-modules/logic-function/logic-function.exception';
|
||||
import { type FlatLogicFunction } from 'src/engine/metadata-modules/logic-function/types/flat-logic-function.type';
|
||||
import { copyYarnEngineAndBuildDependencies } from 'src/engine/core-modules/application/application-package/utils/copy-yarn-engine-and-build-dependencies';
|
||||
import { type LogicFunctionResourceService } from 'src/engine/core-modules/logic-function/logic-function-resource/logic-function-resource.service';
|
||||
import { callWithTimeout } from 'src/engine/core-modules/logic-function/logic-function-drivers/utils/call-with-timeout';
|
||||
|
||||
const UPDATE_FUNCTION_DURATION_TIMEOUT_IN_SECONDS = 60;
|
||||
const CREDENTIALS_DURATION_IN_SECONDS = 60 * 60; // 1h
|
||||
const YARN_INSTALL_LAMBDA_TIMEOUT_SECONDS = 300;
|
||||
const YARN_INSTALL_LAMBDA_MEMORY_MB = 1024;
|
||||
const COMMON_LAYER_NAME_PREFIX = 'twenty-common-layer';
|
||||
const BUILDER_LAMBDA_TIMEOUT_SECONDS = 60;
|
||||
const BUILDER_LAMBDA_MEMORY_MB = 512;
|
||||
|
||||
const YARN_INSTALL_HANDLER_PATH = resolve(
|
||||
__dirname,
|
||||
join(
|
||||
ASSET_PATH,
|
||||
'engine/core-modules/logic-function/logic-function-drivers/constants/yarn-install/index.mjs',
|
||||
),
|
||||
);
|
||||
|
||||
const BUILDER_HANDLER_PATH = resolve(
|
||||
__dirname,
|
||||
join(
|
||||
ASSET_PATH,
|
||||
'engine/core-modules/logic-function/logic-function-drivers/constants/builder/index.mjs',
|
||||
),
|
||||
);
|
||||
|
||||
type LambdaDriverExecutorPayload = {
|
||||
code: string;
|
||||
@@ -52,11 +81,35 @@ type LambdaDriverExecutorPayload = {
|
||||
handlerName: string;
|
||||
};
|
||||
|
||||
export type YarnInstallLambdaPayload = {
|
||||
action: 'createLayer';
|
||||
packageJson: string;
|
||||
yarnLock: string;
|
||||
presignedUploadUrl: string;
|
||||
};
|
||||
|
||||
export type YarnInstallLambdaResult = {
|
||||
success: boolean;
|
||||
};
|
||||
|
||||
export type BuilderLambdaPayload = {
|
||||
action: 'transpile';
|
||||
sourceCode: string;
|
||||
sourceFileName: string;
|
||||
builtFileName: string;
|
||||
};
|
||||
|
||||
export type BuilderLambdaResult = {
|
||||
builtCode: string;
|
||||
};
|
||||
|
||||
export interface LambdaDriverOptions extends LambdaClientConfig {
|
||||
logicFunctionResourceService: LogicFunctionResourceService;
|
||||
region: string;
|
||||
lambdaRole: string;
|
||||
subhostingRole?: string;
|
||||
layerBucket: string;
|
||||
layerBucketRegion: string;
|
||||
}
|
||||
|
||||
export class LambdaDriver implements LogicFunctionDriver {
|
||||
@@ -120,17 +173,33 @@ export class LambdaDriver implements LogicFunctionDriver {
|
||||
};
|
||||
}
|
||||
|
||||
private async waitFunctionUpdates(
|
||||
flatLogicFunction: FlatLogicFunction,
|
||||
private async generatePresignedUploadUrl(
|
||||
s3Key: string,
|
||||
expiresIn: number = 300,
|
||||
): Promise<string> {
|
||||
const s3Client = new S3Client({
|
||||
region: this.options.layerBucketRegion,
|
||||
credentials: isDefined(this.options.subhostingRole)
|
||||
? await this.getAssumeRoleCredentials()
|
||||
: this.options.credentials,
|
||||
});
|
||||
|
||||
const putCommand = new PutObjectCommand({
|
||||
Bucket: this.options.layerBucket,
|
||||
Key: s3Key,
|
||||
ContentType: 'application/zip',
|
||||
});
|
||||
|
||||
return getSignedUrl(s3Client, putCommand, { expiresIn });
|
||||
}
|
||||
|
||||
private async waitFunctionActive(
|
||||
functionName: string,
|
||||
maxWaitTime: number = UPDATE_FUNCTION_DURATION_TIMEOUT_IN_SECONDS,
|
||||
) {
|
||||
const waitParams = {
|
||||
FunctionName: flatLogicFunction.id,
|
||||
};
|
||||
|
||||
await waitUntilFunctionUpdatedV2(
|
||||
await waitUntilFunctionActiveV2(
|
||||
{ client: await this.getLambdaClient(), maxWaitTime },
|
||||
waitParams,
|
||||
{ FunctionName: functionName },
|
||||
);
|
||||
}
|
||||
|
||||
@@ -138,7 +207,398 @@ export class LambdaDriver implements LogicFunctionDriver {
|
||||
return flatApplication.yarnLockChecksum ?? 'default';
|
||||
}
|
||||
|
||||
private async createLayerIfNotExists({
|
||||
private yarnInstallFunctionName: string | undefined;
|
||||
private builderFunctionName: string | undefined;
|
||||
private commonLayerName: string | undefined;
|
||||
|
||||
private async getCommonLayerName(): Promise<string> {
|
||||
if (isDefined(this.commonLayerName)) {
|
||||
return this.commonLayerName;
|
||||
}
|
||||
|
||||
const [packageJson, yarnLock] = await Promise.all([
|
||||
fs.readFile(
|
||||
join(COMMON_LAYER_DEPENDENCIES_DIRNAME, 'package.json'),
|
||||
'utf-8',
|
||||
),
|
||||
fs.readFile(
|
||||
join(COMMON_LAYER_DEPENDENCIES_DIRNAME, 'yarn.lock'),
|
||||
'utf-8',
|
||||
),
|
||||
]);
|
||||
|
||||
const checksum = createHash('sha256')
|
||||
.update(packageJson)
|
||||
.update(yarnLock)
|
||||
.digest('hex')
|
||||
.slice(0, 12);
|
||||
|
||||
this.commonLayerName = `${COMMON_LAYER_NAME_PREFIX}-${checksum}`;
|
||||
|
||||
return this.commonLayerName;
|
||||
}
|
||||
|
||||
private async getYarnInstallFunctionName(): Promise<string> {
|
||||
if (isDefined(this.yarnInstallFunctionName)) {
|
||||
return this.yarnInstallFunctionName;
|
||||
}
|
||||
|
||||
const handlerContent = await fs.readFile(
|
||||
YARN_INSTALL_HANDLER_PATH,
|
||||
'utf-8',
|
||||
);
|
||||
const checksum = createHash('sha256')
|
||||
.update(handlerContent)
|
||||
.digest('hex')
|
||||
.slice(0, 12);
|
||||
|
||||
this.yarnInstallFunctionName = `twenty-yarn-install-${checksum}`;
|
||||
|
||||
return this.yarnInstallFunctionName;
|
||||
}
|
||||
|
||||
private async getBuilderFunctionName(): Promise<string> {
|
||||
if (isDefined(this.builderFunctionName)) {
|
||||
return this.builderFunctionName;
|
||||
}
|
||||
|
||||
const handlerContent = await fs.readFile(BUILDER_HANDLER_PATH, 'utf-8');
|
||||
const checksum = createHash('sha256')
|
||||
.update(handlerContent)
|
||||
.digest('hex')
|
||||
.slice(0, 12);
|
||||
|
||||
this.builderFunctionName = `twenty-builder-${checksum}`;
|
||||
|
||||
return this.builderFunctionName;
|
||||
}
|
||||
|
||||
private async ensureCommonLayerExists(): Promise<string> {
|
||||
const commonLayerName = await this.getCommonLayerName();
|
||||
const existingArn = await this.getExistingLayerArn(commonLayerName);
|
||||
|
||||
if (isDefined(existingArn)) {
|
||||
return existingArn;
|
||||
}
|
||||
|
||||
const temporaryDirManager = new TemporaryDirManager();
|
||||
const { sourceTemporaryDir, lambdaZipPath } =
|
||||
await temporaryDirManager.init();
|
||||
|
||||
try {
|
||||
await copyCommonLayerDependencies(sourceTemporaryDir);
|
||||
|
||||
await createZipFile(sourceTemporaryDir, lambdaZipPath);
|
||||
|
||||
const lambdaClient = await this.getLambdaClient();
|
||||
|
||||
const result = await lambdaClient.send(
|
||||
new PublishLayerVersionCommand({
|
||||
LayerName: commonLayerName,
|
||||
Content: { ZipFile: await fs.readFile(lambdaZipPath) },
|
||||
CompatibleRuntimes: [
|
||||
LogicFunctionRuntime.NODE18,
|
||||
LogicFunctionRuntime.NODE22,
|
||||
],
|
||||
}),
|
||||
);
|
||||
|
||||
if (!result.LayerVersionArn) {
|
||||
throw new Error(
|
||||
'PublishLayerVersion did not return a LayerVersionArn for common layer',
|
||||
);
|
||||
}
|
||||
|
||||
return result.LayerVersionArn;
|
||||
} finally {
|
||||
await temporaryDirManager.clean();
|
||||
}
|
||||
}
|
||||
|
||||
private async ensureYarnInstallLambdaExists(): Promise<void> {
|
||||
const yarnInstallFunctionName = await this.getYarnInstallFunctionName();
|
||||
const lambdaClient = await this.getLambdaClient();
|
||||
|
||||
try {
|
||||
await lambdaClient.send(
|
||||
new GetFunctionCommand({ FunctionName: yarnInstallFunctionName }),
|
||||
);
|
||||
|
||||
return;
|
||||
} catch (error) {
|
||||
if (!(error instanceof ResourceNotFoundException)) {
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
const commonLayerArn = await this.ensureCommonLayerExists();
|
||||
|
||||
const temporaryDirManager = new TemporaryDirManager();
|
||||
const { sourceTemporaryDir, lambdaZipPath } =
|
||||
await temporaryDirManager.init();
|
||||
|
||||
try {
|
||||
await copyYarnInstall(sourceTemporaryDir);
|
||||
|
||||
await createZipFile(sourceTemporaryDir, lambdaZipPath);
|
||||
|
||||
const params: CreateFunctionCommandInput = {
|
||||
Code: {
|
||||
ZipFile: await fs.readFile(lambdaZipPath),
|
||||
},
|
||||
FunctionName: yarnInstallFunctionName,
|
||||
Layers: [commonLayerArn],
|
||||
Handler: 'index.handler',
|
||||
Role: this.options.lambdaRole,
|
||||
Runtime: LogicFunctionRuntime.NODE22,
|
||||
Timeout: YARN_INSTALL_LAMBDA_TIMEOUT_SECONDS,
|
||||
MemorySize: YARN_INSTALL_LAMBDA_MEMORY_MB,
|
||||
};
|
||||
|
||||
await lambdaClient.send(new CreateFunctionCommand(params));
|
||||
} finally {
|
||||
await temporaryDirManager.clean();
|
||||
}
|
||||
|
||||
await this.waitFunctionActive(yarnInstallFunctionName);
|
||||
}
|
||||
|
||||
private async invokeYarnInstallLambda(
|
||||
payload: YarnInstallLambdaPayload,
|
||||
): Promise<YarnInstallLambdaResult> {
|
||||
const lambdaClient = await this.getLambdaClient();
|
||||
|
||||
const yarnInstallFunctionName = await this.getYarnInstallFunctionName();
|
||||
|
||||
const result = await callWithTimeout({
|
||||
callback: () =>
|
||||
lambdaClient.send(
|
||||
new InvokeCommand({
|
||||
FunctionName: yarnInstallFunctionName,
|
||||
Payload: JSON.stringify(payload),
|
||||
LogType: LogType.Tail,
|
||||
}),
|
||||
),
|
||||
timeoutMs: YARN_INSTALL_LAMBDA_TIMEOUT_SECONDS * 1000,
|
||||
});
|
||||
|
||||
if (result.FunctionError) {
|
||||
const parsedResult = result.Payload
|
||||
? JSON.parse(result.Payload.transformToString())
|
||||
: {};
|
||||
|
||||
throw new LogicFunctionException(
|
||||
`Yarn install Lambda failed: ${JSON.stringify(parsedResult)}`,
|
||||
LogicFunctionExceptionCode.LOGIC_FUNCTION_CREATE_FAILED,
|
||||
);
|
||||
}
|
||||
|
||||
const parsedResult: YarnInstallLambdaResult = result.Payload
|
||||
? JSON.parse(result.Payload.transformToString())
|
||||
: {};
|
||||
|
||||
if (!parsedResult.success) {
|
||||
throw new Error('Yarn install Lambda did not report success');
|
||||
}
|
||||
|
||||
return parsedResult;
|
||||
}
|
||||
|
||||
private async ensureBuilderLambdaExists(): Promise<void> {
|
||||
const builderFunctionName = await this.getBuilderFunctionName();
|
||||
const lambdaClient = await this.getLambdaClient();
|
||||
|
||||
try {
|
||||
await lambdaClient.send(
|
||||
new GetFunctionCommand({ FunctionName: builderFunctionName }),
|
||||
);
|
||||
|
||||
return;
|
||||
} catch (error) {
|
||||
if (!(error instanceof ResourceNotFoundException)) {
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
const commonLayerArn = await this.ensureCommonLayerExists();
|
||||
|
||||
const temporaryDirManager = new TemporaryDirManager();
|
||||
const { sourceTemporaryDir, lambdaZipPath } =
|
||||
await temporaryDirManager.init();
|
||||
|
||||
try {
|
||||
await copyBuilder(sourceTemporaryDir);
|
||||
|
||||
await createZipFile(sourceTemporaryDir, lambdaZipPath);
|
||||
|
||||
const params: CreateFunctionCommandInput = {
|
||||
Code: {
|
||||
ZipFile: await fs.readFile(lambdaZipPath),
|
||||
},
|
||||
FunctionName: builderFunctionName,
|
||||
Layers: [commonLayerArn],
|
||||
Handler: 'index.handler',
|
||||
Role: this.options.lambdaRole,
|
||||
Runtime: LogicFunctionRuntime.NODE22,
|
||||
Timeout: BUILDER_LAMBDA_TIMEOUT_SECONDS,
|
||||
MemorySize: BUILDER_LAMBDA_MEMORY_MB,
|
||||
};
|
||||
|
||||
await lambdaClient.send(new CreateFunctionCommand(params));
|
||||
} finally {
|
||||
await temporaryDirManager.clean();
|
||||
}
|
||||
|
||||
await this.waitFunctionActive(builderFunctionName);
|
||||
}
|
||||
|
||||
async transpile({
|
||||
sourceCode,
|
||||
sourceFileName,
|
||||
builtFileName,
|
||||
}: LogicFunctionTranspileParams): Promise<LogicFunctionTranspileResult> {
|
||||
await this.ensureBuilderLambdaExists();
|
||||
|
||||
const lambdaClient = await this.getLambdaClient();
|
||||
const builderFunctionName = await this.getBuilderFunctionName();
|
||||
|
||||
const payload: BuilderLambdaPayload = {
|
||||
action: 'transpile',
|
||||
sourceCode,
|
||||
sourceFileName,
|
||||
builtFileName,
|
||||
};
|
||||
|
||||
const result = await callWithTimeout({
|
||||
callback: () =>
|
||||
lambdaClient.send(
|
||||
new InvokeCommand({
|
||||
FunctionName: builderFunctionName,
|
||||
Payload: JSON.stringify(payload),
|
||||
LogType: LogType.Tail,
|
||||
}),
|
||||
),
|
||||
timeoutMs: BUILDER_LAMBDA_TIMEOUT_SECONDS * 1000,
|
||||
});
|
||||
|
||||
if (result.FunctionError) {
|
||||
const parsedResult = result.Payload
|
||||
? JSON.parse(result.Payload.transformToString())
|
||||
: {};
|
||||
|
||||
throw new LogicFunctionException(
|
||||
`Builder Lambda failed: ${JSON.stringify(parsedResult)}`,
|
||||
LogicFunctionExceptionCode.LOGIC_FUNCTION_CREATE_FAILED,
|
||||
);
|
||||
}
|
||||
|
||||
const parsedResult: BuilderLambdaResult = result.Payload
|
||||
? JSON.parse(result.Payload.transformToString())
|
||||
: {};
|
||||
|
||||
if (!parsedResult.builtCode) {
|
||||
throw new Error('Builder Lambda did not return builtCode');
|
||||
}
|
||||
|
||||
return { builtCode: parsedResult.builtCode };
|
||||
}
|
||||
|
||||
private async getExistingLayerArn(
|
||||
layerName: string,
|
||||
): Promise<string | undefined> {
|
||||
const listLayerParams: ListLayerVersionsCommandInput = {
|
||||
LayerName: layerName,
|
||||
MaxItems: 1,
|
||||
};
|
||||
|
||||
const listLayerResult = await (
|
||||
await this.getLambdaClient()
|
||||
).send(new ListLayerVersionsCommand(listLayerParams));
|
||||
|
||||
return listLayerResult.LayerVersions?.[0]?.LayerVersionArn;
|
||||
}
|
||||
|
||||
private async getDependencyContents(
|
||||
flatApplication: FlatApplication,
|
||||
applicationUniversalIdentifier: string,
|
||||
): Promise<{ packageJson: string; yarnLock: string }> {
|
||||
const temporaryDirManager = new TemporaryDirManager();
|
||||
const { sourceTemporaryDir } = await temporaryDirManager.init();
|
||||
|
||||
try {
|
||||
await this.logicFunctionResourceService.copyDependenciesInMemory({
|
||||
applicationUniversalIdentifier,
|
||||
workspaceId: flatApplication.workspaceId,
|
||||
inMemoryFolderPath: sourceTemporaryDir,
|
||||
});
|
||||
|
||||
const [packageJson, yarnLock] = await Promise.all([
|
||||
fs.readFile(`${sourceTemporaryDir}/package.json`, 'utf-8'),
|
||||
fs.readFile(`${sourceTemporaryDir}/yarn.lock`, 'utf-8'),
|
||||
]);
|
||||
|
||||
return { packageJson, yarnLock };
|
||||
} finally {
|
||||
await temporaryDirManager.clean();
|
||||
}
|
||||
}
|
||||
|
||||
private async createLayerIfNotExist({
|
||||
flatApplication,
|
||||
applicationUniversalIdentifier,
|
||||
}: {
|
||||
flatApplication: FlatApplication;
|
||||
applicationUniversalIdentifier: string;
|
||||
}): Promise<void> {
|
||||
const layerName = this.getLayerName(flatApplication);
|
||||
|
||||
const existingArn = await this.getExistingLayerArn(layerName);
|
||||
|
||||
if (isDefined(existingArn)) {
|
||||
return;
|
||||
}
|
||||
|
||||
const { packageJson, yarnLock } = await this.getDependencyContents(
|
||||
flatApplication,
|
||||
applicationUniversalIdentifier,
|
||||
);
|
||||
|
||||
await this.ensureYarnInstallLambdaExists();
|
||||
|
||||
const s3Key = `lambda-layers/${layerName}.zip`;
|
||||
|
||||
const presignedUploadUrl = await this.generatePresignedUploadUrl(s3Key);
|
||||
|
||||
await this.invokeYarnInstallLambda({
|
||||
action: 'createLayer',
|
||||
packageJson,
|
||||
yarnLock,
|
||||
presignedUploadUrl,
|
||||
});
|
||||
|
||||
const bucket = this.options.layerBucket;
|
||||
|
||||
const lambdaClient = await this.getLambdaClient();
|
||||
|
||||
const publishResult = await lambdaClient.send(
|
||||
new PublishLayerVersionCommand({
|
||||
LayerName: layerName,
|
||||
Content: { S3Bucket: bucket, S3Key: s3Key },
|
||||
CompatibleRuntimes: [
|
||||
LogicFunctionRuntime.NODE18,
|
||||
LogicFunctionRuntime.NODE22,
|
||||
],
|
||||
}),
|
||||
);
|
||||
|
||||
if (!publishResult.LayerVersionArn) {
|
||||
throw new Error(
|
||||
`PublishLayerVersion did not return a LayerVersionArn for layer '${layerName}'`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
private async getLayerArn({
|
||||
flatApplication,
|
||||
applicationUniversalIdentifier,
|
||||
}: {
|
||||
@@ -147,58 +607,26 @@ export class LambdaDriver implements LogicFunctionDriver {
|
||||
}): Promise<string> {
|
||||
const layerName = this.getLayerName(flatApplication);
|
||||
|
||||
const listLayerParams: ListLayerVersionsCommandInput = {
|
||||
LayerName: layerName,
|
||||
MaxItems: 1,
|
||||
};
|
||||
const existingArn = await this.getExistingLayerArn(layerName);
|
||||
|
||||
const listLayerCommand = new ListLayerVersionsCommand(listLayerParams);
|
||||
|
||||
const listLayerResult = await (
|
||||
await this.getLambdaClient()
|
||||
).send(listLayerCommand);
|
||||
|
||||
if (isDefined(listLayerResult.LayerVersions?.[0]?.LayerVersionArn)) {
|
||||
return listLayerResult.LayerVersions[0].LayerVersionArn;
|
||||
if (isDefined(existingArn)) {
|
||||
return existingArn;
|
||||
}
|
||||
|
||||
const temporaryDirManager = new TemporaryDirManager();
|
||||
const { sourceTemporaryDir, lambdaZipPath } =
|
||||
await temporaryDirManager.init();
|
||||
|
||||
const nodeDependenciesFolder = join(sourceTemporaryDir, 'nodejs');
|
||||
|
||||
await this.logicFunctionResourceService.copyDependenciesInMemory({
|
||||
await this.createLayerIfNotExist({
|
||||
flatApplication,
|
||||
applicationUniversalIdentifier,
|
||||
workspaceId: flatApplication.workspaceId,
|
||||
inMemoryFolderPath: nodeDependenciesFolder,
|
||||
});
|
||||
await copyYarnEngineAndBuildDependencies(nodeDependenciesFolder);
|
||||
|
||||
await createZipFile(sourceTemporaryDir, lambdaZipPath);
|
||||
const newArn = await this.getExistingLayerArn(layerName);
|
||||
|
||||
const params: PublishLayerVersionCommandInput = {
|
||||
LayerName: layerName,
|
||||
Content: {
|
||||
ZipFile: await fs.readFile(lambdaZipPath),
|
||||
},
|
||||
CompatibleRuntimes: [
|
||||
LogicFunctionRuntime.NODE18,
|
||||
LogicFunctionRuntime.NODE22,
|
||||
],
|
||||
};
|
||||
|
||||
const command = new PublishLayerVersionCommand(params);
|
||||
|
||||
const result = await (await this.getLambdaClient()).send(command);
|
||||
|
||||
await temporaryDirManager.clean();
|
||||
|
||||
if (!isDefined(result.LayerVersionArn)) {
|
||||
throw new Error('new layer version arn if undefined');
|
||||
if (!isDefined(newArn)) {
|
||||
throw new Error(
|
||||
`Layer '${layerName}' was not created by the yarn install Lambda`,
|
||||
);
|
||||
}
|
||||
|
||||
return result.LayerVersionArn;
|
||||
return newArn;
|
||||
}
|
||||
|
||||
private async getLambdaExecutor(flatLogicFunction: FlatLogicFunction) {
|
||||
@@ -269,7 +697,7 @@ export class LambdaDriver implements LogicFunctionDriver {
|
||||
return;
|
||||
}
|
||||
|
||||
const layerArn = await this.createLayerIfNotExists({
|
||||
const layerArn = await this.getLayerArn({
|
||||
flatApplication,
|
||||
applicationUniversalIdentifier,
|
||||
});
|
||||
@@ -331,7 +759,7 @@ export class LambdaDriver implements LogicFunctionDriver {
|
||||
applicationUniversalIdentifier,
|
||||
});
|
||||
|
||||
await this.waitFunctionUpdates(flatLogicFunction);
|
||||
await this.waitFunctionActive(flatLogicFunction.id);
|
||||
|
||||
const startTime = Date.now();
|
||||
|
||||
|
||||
+46
-5
@@ -1,11 +1,16 @@
|
||||
import { promises as fs } from 'fs';
|
||||
import { spawn } from 'node:child_process';
|
||||
import { join } from 'path';
|
||||
import { dirname, join } from 'path';
|
||||
|
||||
import { build } from 'esbuild';
|
||||
import { NODE_ESM_CJS_BANNER } from 'twenty-shared/application';
|
||||
|
||||
import {
|
||||
type LogicFunctionExecuteParams,
|
||||
type LogicFunctionExecuteResult,
|
||||
type LogicFunctionDriver,
|
||||
type LogicFunctionTranspileParams,
|
||||
type LogicFunctionTranspileResult,
|
||||
} from 'src/engine/core-modules/logic-function/logic-function-drivers/interfaces/logic-function-driver.interface';
|
||||
|
||||
import { type FlatApplication } from 'src/engine/core-modules/application/types/flat-application.type';
|
||||
@@ -34,7 +39,7 @@ export class LocalDriver implements LogicFunctionDriver {
|
||||
return join(LOGIC_FUNCTION_EXECUTOR_TMPDIR_FOLDER, checksum);
|
||||
};
|
||||
|
||||
private async createLayerIfNotExists({
|
||||
private async createLayerIfNotExist({
|
||||
flatApplication,
|
||||
applicationUniversalIdentifier,
|
||||
}: {
|
||||
@@ -56,16 +61,52 @@ export class LocalDriver implements LogicFunctionDriver {
|
||||
}
|
||||
}
|
||||
|
||||
async transpile({
|
||||
sourceCode,
|
||||
sourceFileName,
|
||||
builtFileName,
|
||||
}: LogicFunctionTranspileParams): Promise<LogicFunctionTranspileResult> {
|
||||
const temporaryDirManager = new TemporaryDirManager();
|
||||
const { sourceTemporaryDir } = await temporaryDirManager.init();
|
||||
|
||||
try {
|
||||
const entryFilePath = join(sourceTemporaryDir, sourceFileName);
|
||||
const builtBundleFilePath = join(sourceTemporaryDir, builtFileName);
|
||||
|
||||
await fs.mkdir(dirname(entryFilePath), { recursive: true });
|
||||
await fs.writeFile(entryFilePath, sourceCode, 'utf-8');
|
||||
await fs.mkdir(dirname(builtBundleFilePath), { recursive: true });
|
||||
|
||||
await build({
|
||||
entryPoints: [entryFilePath],
|
||||
outfile: builtBundleFilePath,
|
||||
platform: 'node',
|
||||
format: 'esm',
|
||||
target: 'es2017',
|
||||
bundle: true,
|
||||
sourcemap: true,
|
||||
packages: 'external',
|
||||
banner: NODE_ESM_CJS_BANNER,
|
||||
});
|
||||
|
||||
const builtCode = await fs.readFile(builtBundleFilePath, 'utf-8');
|
||||
|
||||
return { builtCode };
|
||||
} finally {
|
||||
await temporaryDirManager.clean();
|
||||
}
|
||||
}
|
||||
|
||||
async delete() {}
|
||||
|
||||
private async build({
|
||||
async build({
|
||||
flatApplication,
|
||||
applicationUniversalIdentifier,
|
||||
}: {
|
||||
flatApplication: FlatApplication;
|
||||
applicationUniversalIdentifier: string;
|
||||
}) {
|
||||
await this.createLayerIfNotExists({
|
||||
await this.createLayerIfNotExist({
|
||||
flatApplication,
|
||||
applicationUniversalIdentifier,
|
||||
});
|
||||
@@ -79,7 +120,7 @@ export class LocalDriver implements LogicFunctionDriver {
|
||||
env,
|
||||
timeoutMs = 900_000,
|
||||
}: LogicFunctionExecuteParams): Promise<LogicFunctionExecuteResult> {
|
||||
await this.build({
|
||||
await this.createLayerIfNotExist({
|
||||
flatApplication,
|
||||
applicationUniversalIdentifier,
|
||||
});
|
||||
|
||||
+13
@@ -41,6 +41,17 @@ export const logicFunctionModuleFactory = async (
|
||||
'LOGIC_FUNCTION_LAMBDA_SUBHOSTING_ROLE',
|
||||
);
|
||||
|
||||
const s3BucketName = twentyConfigService.get('STORAGE_S3_NAME');
|
||||
|
||||
const layerBucket =
|
||||
twentyConfigService.get('LOGIC_FUNCTION_LAMBDA_LAYER_BUCKET') ??
|
||||
s3BucketName ??
|
||||
'twenty-lambda-layer';
|
||||
|
||||
const layerBucketRegion =
|
||||
twentyConfigService.get('LOGIC_FUNCTION_LAMBDA_LAYER_BUCKET_REGION') ??
|
||||
region;
|
||||
|
||||
return {
|
||||
type: LogicFunctionDriverType.LAMBDA,
|
||||
options: {
|
||||
@@ -56,6 +67,8 @@ export const logicFunctionModuleFactory = async (
|
||||
region,
|
||||
lambdaRole,
|
||||
subhostingRole,
|
||||
layerBucket,
|
||||
layerBucketRegion,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
+13
@@ -29,11 +29,24 @@ export type LogicFunctionExecuteParams = {
|
||||
timeoutMs?: number;
|
||||
};
|
||||
|
||||
export type LogicFunctionTranspileParams = {
|
||||
sourceCode: string;
|
||||
sourceFileName: string;
|
||||
builtFileName: string;
|
||||
};
|
||||
|
||||
export type LogicFunctionTranspileResult = {
|
||||
builtCode: string;
|
||||
};
|
||||
|
||||
export interface LogicFunctionDriver {
|
||||
delete(flatLogicFunction: FlatLogicFunction): Promise<void>;
|
||||
execute(
|
||||
params: LogicFunctionExecuteParams,
|
||||
): Promise<LogicFunctionExecuteResult>;
|
||||
transpile(
|
||||
params: LogicFunctionTranspileParams,
|
||||
): Promise<LogicFunctionTranspileResult>;
|
||||
}
|
||||
|
||||
export enum LogicFunctionDriverType {
|
||||
|
||||
+17
@@ -0,0 +1,17 @@
|
||||
import { promises as fs } from 'fs';
|
||||
import { resolve, join } from 'path';
|
||||
|
||||
import { ASSET_PATH } from 'src/constants/assets-path';
|
||||
|
||||
const BUILDER_FILE_PATH = resolve(
|
||||
__dirname,
|
||||
join(
|
||||
ASSET_PATH,
|
||||
'engine/core-modules/logic-function/logic-function-drivers/constants/builder',
|
||||
),
|
||||
);
|
||||
|
||||
export const copyBuilder = async (buildDirectory: string) => {
|
||||
await fs.mkdir(buildDirectory, { recursive: true });
|
||||
await fs.cp(BUILDER_FILE_PATH, buildDirectory, { recursive: true });
|
||||
};
|
||||
+22
@@ -0,0 +1,22 @@
|
||||
import { promises as fs } from 'fs';
|
||||
import { join } from 'path';
|
||||
|
||||
import { COMMON_LAYER_DEPENDENCIES_DIRNAME } from 'src/engine/core-modules/logic-function/logic-function-drivers/constants/common-layer-dependencies-dirname';
|
||||
import { copyYarnEngineAndBuildDependencies } from 'src/engine/core-modules/application/application-package/utils/copy-yarn-engine-and-build-dependencies';
|
||||
|
||||
// Builds a Lambda layer with the common dependencies (e.g. archiver)
|
||||
// by copying the package.json + yarn.lock, then running yarn install locally.
|
||||
// The result follows the Lambda layer structure: nodejs/node_modules/...
|
||||
export const copyCommonLayerDependencies = async (
|
||||
buildDirectory: string,
|
||||
): Promise<void> => {
|
||||
const nodejsDir = join(buildDirectory, 'nodejs');
|
||||
|
||||
await fs.mkdir(nodejsDir, { recursive: true });
|
||||
|
||||
await fs.cp(COMMON_LAYER_DEPENDENCIES_DIRNAME, nodejsDir, {
|
||||
recursive: true,
|
||||
});
|
||||
|
||||
await copyYarnEngineAndBuildDependencies(nodejsDir);
|
||||
};
|
||||
+23
@@ -0,0 +1,23 @@
|
||||
import { promises as fs } from 'fs';
|
||||
import { resolve, join } from 'path';
|
||||
|
||||
import { ASSET_PATH } from 'src/constants/assets-path';
|
||||
import { YARN_ENGINE_DIRNAME } from 'src/engine/core-modules/application/application-package/constants/yarn-engine-dirname';
|
||||
|
||||
const YARN_INSTALL_FILE_PATH = resolve(
|
||||
__dirname,
|
||||
join(
|
||||
ASSET_PATH,
|
||||
'engine/core-modules/logic-function/logic-function-drivers/constants/yarn-install',
|
||||
),
|
||||
);
|
||||
|
||||
export const copyYarnInstall = async (buildDirectory: string) => {
|
||||
await fs.mkdir(buildDirectory, { recursive: true });
|
||||
|
||||
await fs.cp(YARN_INSTALL_FILE_PATH, buildDirectory, { recursive: true });
|
||||
|
||||
const yarnEngineDestination = join(buildDirectory, 'yarn-engine');
|
||||
|
||||
await fs.cp(YARN_ENGINE_DIRNAME, yarnEngineDestination, { recursive: true });
|
||||
};
|
||||
+8
@@ -10,6 +10,8 @@ import { isDefined } from 'twenty-shared/utils';
|
||||
import {
|
||||
LogicFunctionDriver,
|
||||
type LogicFunctionExecuteResult,
|
||||
type LogicFunctionTranspileParams,
|
||||
type LogicFunctionTranspileResult,
|
||||
} from 'src/engine/core-modules/logic-function/logic-function-drivers/interfaces/logic-function-driver.interface';
|
||||
|
||||
import { FlatApplication } from 'src/engine/core-modules/application/types/flat-application.type';
|
||||
@@ -101,6 +103,12 @@ export class LogicFunctionExecutorService {
|
||||
return resultLogicFunction;
|
||||
}
|
||||
|
||||
async transpile(
|
||||
params: LogicFunctionTranspileParams,
|
||||
): Promise<LogicFunctionTranspileResult> {
|
||||
return this.driver.transpile(params);
|
||||
}
|
||||
|
||||
private async throttleExecution(workspaceId: string) {
|
||||
try {
|
||||
await this.throttlerService.tokenBucketThrottleOrThrow(
|
||||
|
||||
+20
-83
@@ -1,18 +1,14 @@
|
||||
import { Injectable } from '@nestjs/common';
|
||||
|
||||
import crypto from 'crypto';
|
||||
import fs from 'fs/promises';
|
||||
import { dirname, join } from 'path';
|
||||
import { join } from 'path';
|
||||
|
||||
import { build } from 'esbuild';
|
||||
import { FileFolder } from 'twenty-shared/types';
|
||||
import { isDefined } from 'twenty-shared/utils';
|
||||
import { NODE_ESM_CJS_BANNER } from 'twenty-shared/application';
|
||||
|
||||
import { FileStorageExceptionCode } from 'src/engine/core-modules/file-storage/interfaces/file-storage-exception';
|
||||
|
||||
import { FileStorageService } from 'src/engine/core-modules/file-storage/file-storage.service';
|
||||
import { TemporaryDirManager } from 'src/engine/core-modules/logic-function/logic-function-drivers/utils/temporary-dir-manager';
|
||||
import {
|
||||
getLogicFunctionSeedProjectFiles,
|
||||
LogicFunctionSeedProjectFile,
|
||||
@@ -45,11 +41,6 @@ type UpdateSourceFilesParams = Omit<
|
||||
sourceHandlerCode: string;
|
||||
};
|
||||
|
||||
type BuildFromSourceParams = Identifier & {
|
||||
sourceHandlerPath: string;
|
||||
builtHandlerPath: string;
|
||||
};
|
||||
|
||||
type GetSourceCodeParams = Identifier & {
|
||||
sourceHandlerPath: string;
|
||||
};
|
||||
@@ -148,52 +139,27 @@ export class LogicFunctionResourceService {
|
||||
});
|
||||
}
|
||||
|
||||
async buildFromSourceFile({
|
||||
sourceHandlerPath,
|
||||
builtHandlerPath,
|
||||
async uploadBuiltFile({
|
||||
workspaceId,
|
||||
applicationUniversalIdentifier,
|
||||
}: BuildFromSourceParams): Promise<{ checksum: string }> {
|
||||
const temporaryDirManager = new TemporaryDirManager();
|
||||
|
||||
try {
|
||||
const { sourceTemporaryDir } = await temporaryDirManager.init();
|
||||
|
||||
await this.fileStorageService.downloadFile({
|
||||
workspaceId,
|
||||
applicationUniversalIdentifier,
|
||||
fileFolder: FileFolder.Source,
|
||||
resourcePath: sourceHandlerPath,
|
||||
localPath: join(sourceTemporaryDir, sourceHandlerPath),
|
||||
});
|
||||
|
||||
const builtBundleFilePath = await this.buildInMemory({
|
||||
sourceTemporaryDir,
|
||||
sourceHandlerPath,
|
||||
builtHandlerPath,
|
||||
});
|
||||
|
||||
const builtFile = await fs.readFile(builtBundleFilePath, 'utf-8');
|
||||
|
||||
await this.fileStorageService.writeFile({
|
||||
workspaceId,
|
||||
applicationUniversalIdentifier,
|
||||
fileFolder: FileFolder.BuiltLogicFunction,
|
||||
resourcePath: builtHandlerPath,
|
||||
sourceFile: builtFile,
|
||||
mimeType: 'application/javascript',
|
||||
settings: {
|
||||
isTemporaryFile: false,
|
||||
toDelete: false,
|
||||
},
|
||||
});
|
||||
|
||||
return {
|
||||
checksum: crypto.createHash('md5').update(builtFile).digest('hex'),
|
||||
};
|
||||
} finally {
|
||||
await temporaryDirManager.clean();
|
||||
}
|
||||
builtHandlerPath,
|
||||
builtCode,
|
||||
}: Identifier & {
|
||||
builtHandlerPath: string;
|
||||
builtCode: string;
|
||||
}): Promise<void> {
|
||||
await this.fileStorageService.writeFile({
|
||||
workspaceId,
|
||||
applicationUniversalIdentifier,
|
||||
fileFolder: FileFolder.BuiltLogicFunction,
|
||||
resourcePath: builtHandlerPath,
|
||||
sourceFile: builtCode,
|
||||
mimeType: 'application/javascript',
|
||||
settings: {
|
||||
isTemporaryFile: false,
|
||||
toDelete: false,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
async getSourceFile({
|
||||
@@ -328,33 +294,4 @@ export class LogicFunctionResourceService {
|
||||
|
||||
return localPath;
|
||||
}
|
||||
|
||||
private async buildInMemory({
|
||||
sourceTemporaryDir,
|
||||
sourceHandlerPath,
|
||||
builtHandlerPath,
|
||||
}: {
|
||||
sourceTemporaryDir: string;
|
||||
sourceHandlerPath: string;
|
||||
builtHandlerPath: string;
|
||||
}): Promise<string> {
|
||||
const entryFilePath = join(sourceTemporaryDir, sourceHandlerPath);
|
||||
const builtBundleFilePath = join(sourceTemporaryDir, builtHandlerPath);
|
||||
|
||||
await fs.mkdir(dirname(builtBundleFilePath), { recursive: true });
|
||||
|
||||
await build({
|
||||
entryPoints: [entryFilePath],
|
||||
outfile: builtBundleFilePath,
|
||||
platform: 'node',
|
||||
format: 'esm',
|
||||
target: 'es2017',
|
||||
bundle: true,
|
||||
sourcemap: true,
|
||||
packages: 'external',
|
||||
banner: NODE_ESM_CJS_BANNER,
|
||||
});
|
||||
|
||||
return builtBundleFilePath;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -563,6 +563,30 @@ export class ConfigVariables {
|
||||
@IsOptional()
|
||||
LOGIC_FUNCTION_LAMBDA_SECRET_ACCESS_KEY: string;
|
||||
|
||||
@ConfigVariablesMetadata({
|
||||
group: ConfigVariablesGroup.LOGIC_FUNCTION_CONFIG,
|
||||
description: 'S3 bucket for uploading Lambda layer zip files',
|
||||
type: ConfigVariableType.STRING,
|
||||
})
|
||||
@ValidateIf(
|
||||
(env) => env.LOGIC_FUNCTION_TYPE === LogicFunctionDriverType.LAMBDA,
|
||||
)
|
||||
@IsOptional()
|
||||
LOGIC_FUNCTION_LAMBDA_LAYER_BUCKET?: string;
|
||||
|
||||
@ConfigVariablesMetadata({
|
||||
group: ConfigVariablesGroup.LOGIC_FUNCTION_CONFIG,
|
||||
description:
|
||||
'AWS region of the S3 bucket for Lambda layer uploads (defaults to LOGIC_FUNCTION_LAMBDA_REGION)',
|
||||
type: ConfigVariableType.STRING,
|
||||
})
|
||||
@ValidateIf(
|
||||
(env) => env.LOGIC_FUNCTION_TYPE === LogicFunctionDriverType.LAMBDA,
|
||||
)
|
||||
@IsOptional()
|
||||
@IsAWSRegion()
|
||||
LOGIC_FUNCTION_LAMBDA_LAYER_BUCKET_REGION?: AwsRegion;
|
||||
|
||||
@ConfigVariablesMetadata({
|
||||
group: ConfigVariablesGroup.CODE_INTERPRETER_CONFIG,
|
||||
description:
|
||||
|
||||
+29
-8
@@ -1,5 +1,7 @@
|
||||
import { Injectable } from '@nestjs/common';
|
||||
|
||||
import crypto from 'crypto';
|
||||
|
||||
import { v4 } from 'uuid';
|
||||
import { isDefined } from 'twenty-shared/utils';
|
||||
import { SEED_LOGIC_FUNCTION_INPUT_SCHEMA } from 'twenty-shared/logic-function';
|
||||
@@ -302,14 +304,33 @@ export class LogicFunctionFromSourceService {
|
||||
workspaceId,
|
||||
});
|
||||
|
||||
const { checksum } =
|
||||
await this.logicFunctionResourceService.buildFromSourceFile({
|
||||
workspaceId,
|
||||
applicationUniversalIdentifier:
|
||||
ownerFlatApplication.universalIdentifier,
|
||||
sourceHandlerPath: flatLogicFunction.sourceHandlerPath,
|
||||
builtHandlerPath: flatLogicFunction.builtHandlerPath,
|
||||
});
|
||||
const sourceCode = await this.logicFunctionResourceService.getSourceFile({
|
||||
workspaceId,
|
||||
applicationUniversalIdentifier: ownerFlatApplication.universalIdentifier,
|
||||
sourceHandlerPath: flatLogicFunction.sourceHandlerPath,
|
||||
});
|
||||
|
||||
if (!sourceCode) {
|
||||
throw new LogicFunctionException(
|
||||
'Source file not found',
|
||||
LogicFunctionExceptionCode.LOGIC_FUNCTION_NOT_FOUND,
|
||||
);
|
||||
}
|
||||
|
||||
const { builtCode } = await this.logicFunctionExecutorService.transpile({
|
||||
sourceCode,
|
||||
sourceFileName: flatLogicFunction.sourceHandlerPath,
|
||||
builtFileName: flatLogicFunction.builtHandlerPath,
|
||||
});
|
||||
|
||||
await this.logicFunctionResourceService.uploadBuiltFile({
|
||||
workspaceId,
|
||||
applicationUniversalIdentifier: ownerFlatApplication.universalIdentifier,
|
||||
builtHandlerPath: flatLogicFunction.builtHandlerPath,
|
||||
builtCode,
|
||||
});
|
||||
|
||||
const checksum = crypto.createHash('md5').update(builtCode).digest('hex');
|
||||
|
||||
await this.helperService.updateOneFromMetadata({
|
||||
flatLogicFunctionToUpdate: {
|
||||
|
||||
@@ -1662,6 +1662,22 @@ __metadata:
|
||||
languageName: node
|
||||
linkType: hard
|
||||
|
||||
"@aws-sdk/s3-request-presigner@npm:3.1001.0":
|
||||
version: 3.1001.0
|
||||
resolution: "@aws-sdk/s3-request-presigner@npm:3.1001.0"
|
||||
dependencies:
|
||||
"@aws-sdk/signature-v4-multi-region": "npm:^3.996.4"
|
||||
"@aws-sdk/types": "npm:^3.973.4"
|
||||
"@aws-sdk/util-format-url": "npm:^3.972.6"
|
||||
"@smithy/middleware-endpoint": "npm:^4.4.21"
|
||||
"@smithy/protocol-http": "npm:^5.3.10"
|
||||
"@smithy/smithy-client": "npm:^4.12.1"
|
||||
"@smithy/types": "npm:^4.13.0"
|
||||
tslib: "npm:^2.6.2"
|
||||
checksum: 10c0/51cbe9bcef780a391bf5e0a084490ff40bba317d6741ed14b3b5d5f6c761fc161e71de7241527e02f8b56e0b6ac4bce9045a43aeed8a79cb0418d4f78f2134dd
|
||||
languageName: node
|
||||
linkType: hard
|
||||
|
||||
"@aws-sdk/signature-v4-multi-region@npm:^3.996.4":
|
||||
version: 3.996.4
|
||||
resolution: "@aws-sdk/signature-v4-multi-region@npm:3.996.4"
|
||||
@@ -1711,6 +1727,16 @@ __metadata:
|
||||
languageName: node
|
||||
linkType: hard
|
||||
|
||||
"@aws-sdk/types@npm:^3.973.5":
|
||||
version: 3.973.5
|
||||
resolution: "@aws-sdk/types@npm:3.973.5"
|
||||
dependencies:
|
||||
"@smithy/types": "npm:^4.13.0"
|
||||
tslib: "npm:^2.6.2"
|
||||
checksum: 10c0/9d21676eb696afc71e915b96f87ae8c67f53a52b3cf3b20f6c35e05654c5a36b67f917310ecd1dd6b696be2019e184dc5bd278318ee1f0051e51f80b292ed578
|
||||
languageName: node
|
||||
linkType: hard
|
||||
|
||||
"@aws-sdk/util-arn-parser@npm:^3.972.2":
|
||||
version: 3.972.2
|
||||
resolution: "@aws-sdk/util-arn-parser@npm:3.972.2"
|
||||
@@ -1733,6 +1759,18 @@ __metadata:
|
||||
languageName: node
|
||||
linkType: hard
|
||||
|
||||
"@aws-sdk/util-format-url@npm:^3.972.6":
|
||||
version: 3.972.7
|
||||
resolution: "@aws-sdk/util-format-url@npm:3.972.7"
|
||||
dependencies:
|
||||
"@aws-sdk/types": "npm:^3.973.5"
|
||||
"@smithy/querystring-builder": "npm:^4.2.11"
|
||||
"@smithy/types": "npm:^4.13.0"
|
||||
tslib: "npm:^2.6.2"
|
||||
checksum: 10c0/eb3511476c49c1044184f78f0ee9a3d057fb4f91711700d9e2b53ad715d59e97a2782314fb25eec0d8310c53349a4471edf6fbf3be42c6db5a7a30bb682f3e63
|
||||
languageName: node
|
||||
linkType: hard
|
||||
|
||||
"@aws-sdk/util-locate-window@npm:^3.0.0":
|
||||
version: 3.568.0
|
||||
resolution: "@aws-sdk/util-locate-window@npm:3.568.0"
|
||||
@@ -21097,6 +21135,17 @@ __metadata:
|
||||
languageName: node
|
||||
linkType: hard
|
||||
|
||||
"@smithy/querystring-builder@npm:^4.2.11":
|
||||
version: 4.2.11
|
||||
resolution: "@smithy/querystring-builder@npm:4.2.11"
|
||||
dependencies:
|
||||
"@smithy/types": "npm:^4.13.0"
|
||||
"@smithy/util-uri-escape": "npm:^4.2.2"
|
||||
tslib: "npm:^2.6.2"
|
||||
checksum: 10c0/88790ee8c7a01730beba913fdf38d1790113c82c956d20cccd43a5b527f19e23df7eff86ddf41e3e615d0af562e31f61a7fed014c9fce8db924623258a4d14f8
|
||||
languageName: node
|
||||
linkType: hard
|
||||
|
||||
"@smithy/querystring-parser@npm:^4.2.10":
|
||||
version: 4.2.10
|
||||
resolution: "@smithy/querystring-parser@npm:4.2.10"
|
||||
@@ -21365,6 +21414,15 @@ __metadata:
|
||||
languageName: node
|
||||
linkType: hard
|
||||
|
||||
"@smithy/util-uri-escape@npm:^4.2.2":
|
||||
version: 4.2.2
|
||||
resolution: "@smithy/util-uri-escape@npm:4.2.2"
|
||||
dependencies:
|
||||
tslib: "npm:^2.6.2"
|
||||
checksum: 10c0/33b6546086c975278d16b5029e6555df551b4bd1e3a84042544d1ef956a287fe033b317954b1737b2773e82b6f27ebde542956ff79ef0e8a813dc0dbf9d34a58
|
||||
languageName: node
|
||||
linkType: hard
|
||||
|
||||
"@smithy/util-utf8@npm:^2.0.0":
|
||||
version: 2.3.0
|
||||
resolution: "@smithy/util-utf8@npm:2.3.0"
|
||||
@@ -55935,6 +55993,7 @@ __metadata:
|
||||
"@aws-sdk/client-sesv2": "npm:3.1001.0"
|
||||
"@aws-sdk/client-sts": "npm:3.1001.0"
|
||||
"@aws-sdk/credential-providers": "npm:3.1001.0"
|
||||
"@aws-sdk/s3-request-presigner": "npm:3.1001.0"
|
||||
"@azure/msal-node": "npm:^3.8.4"
|
||||
"@babel/preset-env": "npm:7.26.9"
|
||||
"@blocknote/server-util": "npm:^0.47.1"
|
||||
|
||||
Reference in New Issue
Block a user