houseassassin/twenty
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

esBuild serverless (#14886)

Browse Source 
Use [esbuild](https://esbuild.github.io/) to Build serverless entire
folder instead of just the src/index.ts file. Application serverless
folders can contain code organized in folders

## Small example

This now works:

<img width="1330" height="686" alt="image"
src="https://github.com/user-attachments/assets/e0639517-492d-4b64-8722-18bbe94dacb0"
/>
This commit is contained in:
martmull
2025-10-06 10:47:42 +02:00
committed by GitHub
parent 1a9bc6d3dd
commit 31b73c3580
14 changed files with 403 additions and 347 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/twenty-server/src/engine/core-modules/serverless/drivers/constants/build-file-name.ts
-1
View File
@@ -1 +0,0 @@
export const BUILD_FILE_NAME = 'build.js';
packages/twenty-server/src/engine/core-modules/serverless/drivers/constants/env-file-name.ts
-1
View File
@@ -1 +0,0 @@
export const ENV_FILE_NAME = '.env';
packages/twenty-server/src/engine/core-modules/serverless/drivers/constants/index-file-name.ts
-1
View File
@@ -1 +0,0 @@
export const INDEX_FILE_NAME = 'index.ts';
packages/twenty-server/src/engine/core-modules/serverless/drivers/constants/outdir-folder.ts
-1
View File
@@ -1 +0,0 @@
export const OUTDIR_FOLDER = 'dist';
packages/twenty-server/src/engine/core-modules/serverless/drivers/interfaces/serverless-driver.interface.ts
+1 -1
View File
@@ -4,7 +4,7 @@ import { type ServerlessFunctionExecutionStatus } from 'src/engine/metadata-modu
export type ServerlessExecuteError = {
errorType: string;
errorMessage: string;
stackTrace: string;
stackTrace: string | string[];
};
export type ServerlessExecuteResult = {
packages/twenty-server/src/engine/core-modules/serverless/drivers/lambda.driver.ts
+67 -56
View File
@@ -20,7 +20,6 @@ import {
} from '@aws-sdk/client-lambda';
import { AssumeRoleCommand, STSClient } from '@aws-sdk/client-sts';
import { isDefined } from 'twenty-shared/utils';
import ts, { transpileModule } from 'typescript';
import {
type ServerlessDriver,
@@ -28,9 +27,7 @@ import {
} from 'src/engine/core-modules/serverless/drivers/interfaces/serverless-driver.interface';
import { type FileStorageService } from 'src/engine/core-modules/file-storage/file-storage.service';
import { readFileContent } from 'src/engine/core-modules/file-storage/utils/read-file-content';
import { COMMON_LAYER_NAME } from 'src/engine/core-modules/serverless/drivers/constants/common-layer-name';
import { INDEX_FILE_NAME } from 'src/engine/core-modules/serverless/drivers/constants/index-file-name';
import { copyAndBuildDependencies } from 'src/engine/core-modules/serverless/drivers/utils/copy-and-build-dependencies';
import { copyExecutor } from 'src/engine/core-modules/serverless/drivers/utils/copy-executor';
import { createZipFile } from 'src/engine/core-modules/serverless/drivers/utils/create-zip-file';
@@ -48,6 +45,8 @@ import {
ServerlessFunctionException,
ServerlessFunctionExceptionCode,
} from 'src/engine/metadata-modules/serverless-function/serverless-function.exception';
import { buildServerlessFunctionInMemory } from 'src/engine/core-modules/serverless/drivers/utils/build-serverless-function-in-memory';
import { formatBuildError } from 'src/engine/core-modules/serverless/drivers/utils/format-build-error';
const UPDATE_FUNCTION_DURATION_TIMEOUT_IN_SECONDS = 60;
const CREDENTIALS_DURATION_IN_SECONDS = 60 * 60; // 1h
@@ -318,68 +317,80 @@ export class LambdaDriver implements ServerlessDriver {
version,
});
const tsCodeStream = await this.fileStorageService.read({
folderPath: join(folderPath, 'src'),
filename: INDEX_FILE_NAME,
});
const tsCode = await readFileContent(tsCodeStream);
const compiledCode = transpileModule(tsCode, {
compilerOptions: {
module: ts.ModuleKind.ESNext,
target: ts.ScriptTarget.ES2017,
},
}).outputText;
const executorPayload = {
params: payload,
code: compiledCode,
};
const params: InvokeCommandInput = {
FunctionName: serverlessFunction.id,
Payload: JSON.stringify(executorPayload),
LogType: LogType.Tail,
};
const command = new InvokeCommand(params);
const lambdaBuildDirectoryManager = new LambdaBuildDirectoryManager();
try {
const result = await (await this.getLambdaClient()).send(command);
const { sourceTemporaryDir } = await lambdaBuildDirectoryManager.init();
const parsedResult = result.Payload
? JSON.parse(result.Payload.transformToString())
: {};
await this.fileStorageService.download({
from: { folderPath },
to: { folderPath: sourceTemporaryDir },
});
const logs = result.LogResult ? this.extractLogs(result.LogResult) : '';
let builtBundleFilePath = '';
const duration = Date.now() - startTime;
if (result.FunctionError) {
return {
data: null,
duration,
status: ServerlessFunctionExecutionStatus.ERROR,
error: parsedResult,
logs,
};
try {
builtBundleFilePath =
await buildServerlessFunctionInMemory(sourceTemporaryDir);
} catch (error) {
return formatBuildError(error, startTime);
}
return {
data: parsedResult,
logs,
duration,
status: ServerlessFunctionExecutionStatus.SUCCESS,
const compiledCode = (await fs.readFile(builtBundleFilePath)).toString(
'utf-8',
);
const executorPayload = {
params: payload,
code: compiledCode,
};
} catch (error) {
if (error instanceof ResourceNotFoundException) {
throw new ServerlessFunctionException(
`Function Version '${version}' does not exist`,
ServerlessFunctionExceptionCode.SERVERLESS_FUNCTION_NOT_FOUND,
);
const params: InvokeCommandInput = {
FunctionName: serverlessFunction.id,
Payload: JSON.stringify(executorPayload),
LogType: LogType.Tail,
};
const command = new InvokeCommand(params);
try {
const result = await (await this.getLambdaClient()).send(command);
const parsedResult = result.Payload
? JSON.parse(result.Payload.transformToString())
: {};
const logs = result.LogResult ? this.extractLogs(result.LogResult) : '';
const duration = Date.now() - startTime;
if (result.FunctionError) {
return {
data: null,
duration,
status: ServerlessFunctionExecutionStatus.ERROR,
error: parsedResult,
logs,
};
}
return {
data: parsedResult,
logs,
duration,
status: ServerlessFunctionExecutionStatus.SUCCESS,
};
} catch (error) {
if (error instanceof ResourceNotFoundException) {
throw new ServerlessFunctionException(
`Function Version '${version}' does not exist`,
ServerlessFunctionExceptionCode.SERVERLESS_FUNCTION_NOT_FOUND,
);
}
throw error;
}
throw error;
} finally {
await lambdaBuildDirectoryManager.clean();
}
}
}
packages/twenty-server/src/engine/core-modules/serverless/drivers/local.driver.ts
+92 -98
View File
@@ -1,8 +1,6 @@
import { promises as fs } from 'fs';
import { join } from 'path';
import ts, { transpileModule } from 'typescript';
import { v4 } from 'uuid';
import { isDefined } from 'twenty-shared/utils';
import {
@@ -11,15 +9,16 @@ import {
} from 'src/engine/core-modules/serverless/drivers/interfaces/serverless-driver.interface';
import { type FileStorageService } from 'src/engine/core-modules/file-storage/file-storage.service';
import { readFileContent } from 'src/engine/core-modules/file-storage/utils/read-file-content';
import { COMMON_LAYER_NAME } from 'src/engine/core-modules/serverless/drivers/constants/common-layer-name';
import { INDEX_FILE_NAME } from 'src/engine/core-modules/serverless/drivers/constants/index-file-name';
import { SERVERLESS_TMPDIR_FOLDER } from 'src/engine/core-modules/serverless/drivers/constants/serverless-tmpdir-folder';
import { copyAndBuildDependencies } from 'src/engine/core-modules/serverless/drivers/utils/copy-and-build-dependencies';
import { ConsoleListener } from 'src/engine/core-modules/serverless/drivers/utils/intercept-console';
import { getServerlessFolder } from 'src/engine/core-modules/serverless/utils/serverless-get-folder.utils';
import { ServerlessFunctionExecutionStatus } from 'src/engine/metadata-modules/serverless-function/dtos/serverless-function-execution-result.dto';
import { type ServerlessFunctionEntity } from 'src/engine/metadata-modules/serverless-function/serverless-function.entity';
import { LambdaBuildDirectoryManager } from 'src/engine/core-modules/serverless/drivers/utils/lambda-build-directory-manager';
import { buildServerlessFunctionInMemory } from 'src/engine/core-modules/serverless/drivers/utils/build-serverless-function-in-memory';
import { formatBuildError } from 'src/engine/core-modules/serverless/drivers/utils/format-build-error';
export interface LocalDriverOptions {
fileStorageService: FileStorageService;
@@ -106,111 +105,106 @@ export class LocalDriver implements ServerlessDriver {
version,
});
const tsCodeStream = await this.fileStorageService.read({
folderPath: join(folderPath, 'src'),
filename: INDEX_FILE_NAME,
});
const tsCode = await readFileContent(tsCodeStream);
const compiledCode = transpileModule(tsCode, {
compilerOptions: {
module: ts.ModuleKind.CommonJS,
target: ts.ScriptTarget.ES2017,
},
}).outputText;
const compiledCodeFolderPath = join(
SERVERLESS_TMPDIR_FOLDER,
`compiled-code-${v4()}`,
);
const compiledCodeFilePath = join(compiledCodeFolderPath, 'main.js');
await fs.mkdir(compiledCodeFolderPath, { recursive: true });
await fs.writeFile(compiledCodeFilePath, compiledCode, 'utf8');
const lambdaBuildDirectoryManager = new LambdaBuildDirectoryManager();
try {
await fs.symlink(
join(
this.getInMemoryLayerFolderPath(serverlessFunction),
'node_modules',
),
join(compiledCodeFolderPath, 'node_modules'),
'dir',
);
} catch (err) {
if (err.code !== 'EEXIST') {
throw err;
}
}
const { sourceTemporaryDir } = await lambdaBuildDirectoryManager.init();
let logs = '';
const consoleListener = new ConsoleListener();
consoleListener.intercept((type, args) => {
const formattedArgs = args.map((arg) => {
if (typeof arg === 'object' && arg !== null) {
const seen = new WeakSet();
return JSON.stringify(
arg,
(_key, value) => {
if (typeof value === 'object' && value !== null) {
if (seen.has(value)) {
return '[Circular]'; // Handle circular references
}
seen.add(value);
}
return value;
},
2,
);
}
return arg;
await this.fileStorageService.download({
from: { folderPath },
to: { folderPath: sourceTemporaryDir },
});
const formattedType = type === 'log' ? 'info' : type;
let builtBundleFilePath = '';
logs += `${new Date().toISOString()} ${formattedType.toUpperCase()} ${formattedArgs.join(' ')}\n`;
});
try {
builtBundleFilePath =
await buildServerlessFunctionInMemory(sourceTemporaryDir);
} catch (error) {
return formatBuildError(error, startTime);
}
try {
const mainFile = await import(compiledCodeFilePath);
try {
await fs.symlink(
join(
this.getInMemoryLayerFolderPath(serverlessFunction),
'node_modules',
),
join(sourceTemporaryDir, 'node_modules'),
'dir',
);
} catch (err) {
if (err.code !== 'EEXIST') {
throw err;
}
}
const result = await this.executeWithTimeout<object | null>(
() => mainFile.main(payload),
serverlessFunction.timeoutSeconds * 1_000,
);
let logs = '';
const duration = Date.now() - startTime;
const consoleListener = new ConsoleListener();
return {
data: result,
logs,
duration,
status: ServerlessFunctionExecutionStatus.SUCCESS,
};
} catch (error) {
return {
data: null,
logs,
duration: Date.now() - startTime,
error: {
errorType: 'UnhandledError',
errorMessage: error.message || 'Unknown error',
stackTrace: error.stack ? error.stack.split('\n') : [],
},
status: ServerlessFunctionExecutionStatus.ERROR,
};
consoleListener.intercept((type, args) => {
const formattedArgs = args.map((arg) => {
if (typeof arg === 'object' && arg !== null) {
const seen = new WeakSet();
return JSON.stringify(
arg,
(_key, value) => {
if (typeof value === 'object' && value !== null) {
if (seen.has(value)) {
return '[Circular]'; // Handle circular references
}
seen.add(value);
}
return value;
},
2,
);
}
return arg;
});
const formattedType = type === 'log' ? 'info' : type;
logs += `${new Date().toISOString()} ${formattedType.toUpperCase()} ${formattedArgs.join(' ')}\n`;
});
try {
const mainFile = await import(builtBundleFilePath);
const result = await this.executeWithTimeout<object | null>(
() => mainFile.main(payload),
serverlessFunction.timeoutSeconds * 1_000,
);
const duration = Date.now() - startTime;
return {
data: result,
logs,
duration,
status: ServerlessFunctionExecutionStatus.SUCCESS,
};
} catch (error) {
return {
data: null,
logs,
duration: Date.now() - startTime,
error: {
errorType: 'UnhandledError',
errorMessage: error.message || 'Unknown error',
stackTrace: error.stack ? error.stack.split('\n') : [],
},
status: ServerlessFunctionExecutionStatus.ERROR,
};
} finally {
consoleListener.release();
}
} finally {
// Restoring originalConsole
consoleListener.release();
await fs.rm(compiledCodeFolderPath, { recursive: true, force: true });
await lambdaBuildDirectoryManager.clean();
}
}
}
packages/twenty-server/src/engine/core-modules/serverless/drivers/utils/build-serverless-function-in-memory.ts
+24
View File
@@ -0,0 +1,24 @@
import { join } from 'path';
import { build } from 'esbuild';
export const buildServerlessFunctionInMemory = async (
sourceTemporaryDir: string,
) => {
const entryFilePath = join(sourceTemporaryDir, 'src', 'index.ts');
const builtBundleFilePath = join(sourceTemporaryDir, 'dist', 'main.mjs');
await build({
entryPoints: [entryFilePath],
outfile: builtBundleFilePath,
platform: 'node',
format: 'esm',
target: 'es2017',
bundle: true,
sourcemap: true,
packages: 'external',
});
return builtBundleFilePath;
};
packages/twenty-server/src/engine/core-modules/serverless/drivers/utils/format-build-error.ts
+20
View File
@@ -0,0 +1,20 @@
import { type BuildFailure } from 'esbuild';
import { type ServerlessExecuteResult } from 'src/engine/core-modules/serverless/drivers/interfaces/serverless-driver.interface';
import { ServerlessFunctionExecutionStatus } from 'src/engine/metadata-modules/serverless-function/dtos/serverless-function-execution-result.dto';
export const formatBuildError = (
error: BuildFailure,
startTime: number,
): ServerlessExecuteResult => ({
data: null,
logs: '',
duration: Date.now() - startTime,
error: {
errorType: 'BuildError',
errorMessage: error.errors.map((e) => e.text).join('\n') || 'Unknown error',
stackTrace: error.stack ? error.stack.split('\n') : [],
},
status: ServerlessFunctionExecutionStatus.ERROR,
});
packages/twenty-server/src/engine/core-modules/serverless/drivers/utils/lambda-build-directory-manager.ts
+1 -5
View File
@@ -8,7 +8,6 @@ import { SERVERLESS_TMPDIR_FOLDER } from 'src/engine/core-modules/serverless/dri
export const NODE_LAYER_SUBFOLDER = 'nodejs';
const TEMPORARY_LAMBDA_FOLDER = 'lambda-build';
const TEMPORARY_LAMBDA_SOURCE_FOLDER = 'src';
const LAMBDA_ZIP_FILE_NAME = 'lambda.zip';
export class LambdaBuildDirectoryManager {
@@ -18,10 +17,7 @@ export class LambdaBuildDirectoryManager {
);
async init() {
const sourceTemporaryDir = join(
this.temporaryDir,
TEMPORARY_LAMBDA_SOURCE_FOLDER,
);
const sourceTemporaryDir = join(this.temporaryDir);
const lambdaZipPath = join(this.temporaryDir, LAMBDA_ZIP_FILE_NAME);
await fs.mkdir(sourceTemporaryDir, { recursive: true });