2114 extensibility manage serverless execution from built code (#17317)

Summary - Serverless functions are now built when created/updated instead of at execution time - Added builtHandlerPath field to track pre-built function artifacts - Renamed base-typescript-project to seed-project with pre-built ESM output included - Renamed file folder enums for consistency (Functions → BuiltFunction, SourceCode → Source) - supports backwards compatibility with existing serverless functions Tested with all combinations - storage : local driver and S3 driver - serverless : local driver and lambda driver
2026-01-22 12:56:44 +01:00
parent 9ee2cd0fe3
commit f92c8a98a4
34 changed files with 348 additions and 193 deletions
@@ -110,10 +110,10 @@ export class ApplicationResolver {
    }: UploadApplicationFileInput,
  ): Promise<FileDTO> {
    const allowedApplicationFileFolders: FileFolder[] = [
-      FileFolder.Functions,
-      FileFolder.FrontComponents,
-      FileFolder.Assets,
-      FileFolder.SourceCode,
+      FileFolder.BuiltFunction,
+      FileFolder.BuiltFrontComponent,
+      FileFolder.Asset,
+      FileFolder.Source,
    ];

    if (!allowedApplicationFileFolders.includes(fileFolder)) {
@@ -36,16 +36,16 @@ export const fileFolderConfigs: Record<FileFolder, FileFolderConfig> = {
  [FileFolder.AgentChat]: {
    ignoreExpirationToken: false,
  },
-  [FileFolder.Functions]: {
+  [FileFolder.BuiltFunction]: {
    ignoreExpirationToken: false,
  },
-  [FileFolder.FrontComponents]: {
+  [FileFolder.BuiltFrontComponent]: {
    ignoreExpirationToken: false,
  },
-  [FileFolder.Assets]: {
+  [FileFolder.Asset]: {
    ignoreExpirationToken: true,
  },
-  [FileFolder.SourceCode]: {
+  [FileFolder.Source]: {
    ignoreExpirationToken: false,
  },
 };
@@ -1 +0,0 @@
-!base-typescript-project/**/.env
@@ -1,4 +0,0 @@
-export const BASE_TYPESCRIPT_PROJECT_INPUT_SCHEMA = {
-  a: null,
-  b: null,
-};
@@ -1,2 +0,0 @@
-# Add your environment variables here.
-# Access them in your serverless function code using process.env.VARIABLE
@@ -0,0 +1,4 @@
+export const SEED_PROJECT_INPUT_SCHEMA = {
+  a: null,
+  b: null,
+};
@@ -20,6 +20,7 @@ import {
 } from '@aws-sdk/client-lambda';
 import { AssumeRoleCommand, STSClient } from '@aws-sdk/client-sts';
 import { isDefined } from 'twenty-shared/utils';
+import { FileFolder } from 'twenty-shared/types';

 import {
  type ServerlessDriver,
@@ -27,16 +28,14 @@ import {
 } from 'src/engine/core-modules/serverless/drivers/interfaces/serverless-driver.interface';

 import { type FileStorageService } from 'src/engine/core-modules/file-storage/file-storage.service';
-import { buildServerlessFunctionInMemory } from 'src/engine/core-modules/serverless/drivers/utils/build-serverless-function-in-memory';
 import { copyAndBuildDependencies } from 'src/engine/core-modules/serverless/drivers/utils/copy-and-build-dependencies';
 import { copyExecutor } from 'src/engine/core-modules/serverless/drivers/utils/copy-executor';
 import { createZipFile } from 'src/engine/core-modules/serverless/drivers/utils/create-zip-file';
-import { formatBuildError } from 'src/engine/core-modules/serverless/drivers/utils/format-build-error';
 import {
  LambdaBuildDirectoryManager,
  NODE_LAYER_SUBFOLDER,
 } from 'src/engine/core-modules/serverless/drivers/utils/lambda-build-directory-manager';
-import { getServerlessFolderOrThrow } from 'src/engine/core-modules/serverless/utils/serverless-get-folder.utils';
+import { getServerlessFolderOrThrow } from 'src/engine/core-modules/serverless/utils/get-serverless-folder-or-throw.utils';
 import { type FlatServerlessFunctionLayer } from 'src/engine/metadata-modules/serverless-function-layer/types/flat-serverless-function-layer.type';
 import { ServerlessFunctionExecutionStatus } from 'src/engine/metadata-modules/serverless-function/dtos/serverless-function-execution-result.dto';
 import { ServerlessFunctionRuntime } from 'src/engine/metadata-modules/serverless-function/serverless-function.entity';
@@ -45,6 +44,7 @@ import {
  ServerlessFunctionExceptionCode,
 } from 'src/engine/metadata-modules/serverless-function/serverless-function.exception';
 import { type FlatServerlessFunction } from 'src/engine/metadata-modules/serverless-function/types/flat-serverless-function.type';
+import { streamToBuffer } from 'src/utils/stream-to-buffer';

 const UPDATE_FUNCTION_DURATION_TIMEOUT_IN_SECONDS = 60;
 const CREDENTIALS_DURATION_IN_SECONDS = 60 * 60; // 1h
@@ -335,93 +335,76 @@ export class LambdaDriver implements ServerlessDriver {
    env?: Record<string, string>;
  }): Promise<ServerlessExecuteResult> {
    await this.build(flatServerlessFunction, flatServerlessFunctionLayer);
+
    await this.waitFunctionUpdates(flatServerlessFunction);

    const startTime = Date.now();

-    const folderPath = getServerlessFolderOrThrow({
+    const builtHandlerFolderPath = getServerlessFolderOrThrow({
      flatServerlessFunction,
      version,
+      fileFolder: FileFolder.BuiltFunction,
    });

-    const lambdaBuildDirectoryManager = new LambdaBuildDirectoryManager();
+    const compiledCode = (
+      await streamToBuffer(
+        await this.fileStorageService.read({
+          folderPath: builtHandlerFolderPath,
+          filename: flatServerlessFunction.builtHandlerPath,
+        }),
+      )
+    ).toString('utf-8');
+
+    const executorPayload: LambdaDriverExecutorPayload = {
+      params: payload,
+      code: compiledCode,
+      env: env ?? {},
+      handlerName: flatServerlessFunction.handlerName,
+    };
+
+    const params: InvokeCommandInput = {
+      FunctionName: flatServerlessFunction.id,
+      Payload: JSON.stringify(executorPayload),
+      LogType: LogType.Tail,
+    };
+
+    const command = new InvokeCommand(params);

    try {
-      const { sourceTemporaryDir } = await lambdaBuildDirectoryManager.init();
+      const result = await (await this.getLambdaClient()).send(command);

-      await this.fileStorageService.download({
-        from: { folderPath },
-        to: { folderPath: sourceTemporaryDir },
-      });
+      const parsedResult = result.Payload
+        ? JSON.parse(result.Payload.transformToString())
+        : {};

-      let builtBundleFilePath = '';
+      const logs = result.LogResult ? this.extractLogs(result.LogResult) : '';

-      try {
-        builtBundleFilePath = await buildServerlessFunctionInMemory({
-          sourceTemporaryDir,
-          handlerPath: flatServerlessFunction.handlerPath,
-        });
-      } catch (error) {
-        return formatBuildError(error, startTime);
-      }
-
-      const compiledCode = (await fs.readFile(builtBundleFilePath)).toString(
-        'utf-8',
-      );
-
-      const executorPayload: LambdaDriverExecutorPayload = {
-        params: payload,
-        code: compiledCode,
-        env: env ?? {},
-        handlerName: flatServerlessFunction.handlerName,
-      };
-
-      const params: InvokeCommandInput = {
-        FunctionName: flatServerlessFunction.id,
-        Payload: JSON.stringify(executorPayload),
-        LogType: LogType.Tail,
-      };
-
-      const command = new InvokeCommand(params);
-
-      try {
-        const result = await (await this.getLambdaClient()).send(command);
-
-        const parsedResult = result.Payload
-          ? JSON.parse(result.Payload.transformToString())
-          : {};
-
-        const logs = result.LogResult ? this.extractLogs(result.LogResult) : '';
-
-        const duration = Date.now() - startTime;
-
-        if (result.FunctionError) {
-          return {
-            data: null,
-            duration,
-            status: ServerlessFunctionExecutionStatus.ERROR,
-            error: parsedResult,
-            logs,
-          };
-        }
+      const duration = Date.now() - startTime;

+      if (result.FunctionError) {
        return {
-          data: parsedResult,
-          logs,
+          data: null,
          duration,
-          status: ServerlessFunctionExecutionStatus.SUCCESS,
+          status: ServerlessFunctionExecutionStatus.ERROR,
+          error: parsedResult,
+          logs,
        };
-      } catch (error) {
-        if (error instanceof ResourceNotFoundException) {
-          throw new ServerlessFunctionException(
-            `Function Version '${version}' does not exist`,
-            ServerlessFunctionExceptionCode.SERVERLESS_FUNCTION_NOT_FOUND,
-          );
-        }
-        throw error;
      }
-    } finally {
-      await lambdaBuildDirectoryManager.clean();
+
+      return {
+        data: parsedResult,
+        logs,
+        duration,
+        status: ServerlessFunctionExecutionStatus.SUCCESS,
+      };
+    } catch (error) {
+      if (error instanceof ResourceNotFoundException) {
+        throw new ServerlessFunctionException(
+          `Function Version '${version}' does not exist`,
+          ServerlessFunctionExceptionCode.SERVERLESS_FUNCTION_NOT_FOUND,
+        );
+      }
+      throw error;
    }
  }
 }
@@ -2,6 +2,8 @@ import { promises as fs } from 'fs';
 import { spawn } from 'node:child_process';
 import { join } from 'path';

+import { FileFolder } from 'twenty-shared/types';
+
 import {
  type ServerlessDriver,
  type ServerlessExecuteResult,
@@ -9,12 +11,10 @@ import {

 import { type FileStorageService } from 'src/engine/core-modules/file-storage/file-storage.service';
 import { SERVERLESS_TMPDIR_FOLDER } from 'src/engine/core-modules/serverless/drivers/constants/serverless-tmpdir-folder';
-import { buildServerlessFunctionInMemory } from 'src/engine/core-modules/serverless/drivers/utils/build-serverless-function-in-memory';
 import { copyAndBuildDependencies } from 'src/engine/core-modules/serverless/drivers/utils/copy-and-build-dependencies';
-import { formatBuildError } from 'src/engine/core-modules/serverless/drivers/utils/format-build-error';
 import { ConsoleListener } from 'src/engine/core-modules/serverless/drivers/utils/intercept-console';
 import { LambdaBuildDirectoryManager } from 'src/engine/core-modules/serverless/drivers/utils/lambda-build-directory-manager';
-import { getServerlessFolderOrThrow } from 'src/engine/core-modules/serverless/utils/serverless-get-folder.utils';
+import { getServerlessFolderOrThrow } from 'src/engine/core-modules/serverless/utils/get-serverless-folder-or-throw.utils';
 import { type FlatServerlessFunctionLayer } from 'src/engine/metadata-modules/serverless-function-layer/types/flat-serverless-function-layer.type';
 import { ServerlessFunctionExecutionStatus } from 'src/engine/metadata-modules/serverless-function/dtos/serverless-function-execution-result.dto';
 import { type FlatServerlessFunction } from 'src/engine/metadata-modules/serverless-function/types/flat-serverless-function.type';
@@ -78,9 +78,10 @@ export class LocalDriver implements ServerlessDriver {

    const startTime = Date.now();

-    const folderPath = getServerlessFolderOrThrow({
+    const builtHandlerFolderPath = getServerlessFolderOrThrow({
      flatServerlessFunction,
      version,
+      fileFolder: FileFolder.BuiltFunction,
    });

    const lambdaBuildDirectoryManager = new LambdaBuildDirectoryManager();
@@ -89,21 +90,16 @@ export class LocalDriver implements ServerlessDriver {
      const { sourceTemporaryDir } = await lambdaBuildDirectoryManager.init();

      await this.fileStorageService.download({
-        from: { folderPath },
-        to: { folderPath: sourceTemporaryDir },
+        from: {
+          folderPath: builtHandlerFolderPath,
+          filename: flatServerlessFunction.builtHandlerPath,
+        },
+        to: {
+          folderPath: sourceTemporaryDir,
+          filename: flatServerlessFunction.builtHandlerPath,
+        },
      });

-      let builtBundleFilePath = '';
-
-      try {
-        builtBundleFilePath = await buildServerlessFunctionInMemory({
-          sourceTemporaryDir,
-          handlerPath: flatServerlessFunction.handlerPath,
-        });
-      } catch (error) {
-        return formatBuildError(error, startTime);
-      }
-
      try {
        await fs.symlink(
          join(
@@ -153,6 +149,11 @@ export class LocalDriver implements ServerlessDriver {
      });

      try {
+        const builtBundleFilePath = join(
+          sourceTemporaryDir,
+          flatServerlessFunction.builtHandlerPath,
+        );
+
        const runnerPath = await this.writeBootstrapRunner({
          dir: sourceTemporaryDir,
          builtFileAbsPath: builtBundleFilePath,
@@ -0,0 +1,86 @@
+import { join } from 'path';
+import fs from 'fs/promises';
+
+import { build } from 'esbuild';
+import { FileFolder } from 'twenty-shared/types';
+
+import { getServerlessFolderOrThrow } from 'src/engine/core-modules/serverless/utils/get-serverless-folder-or-throw.utils';
+import { LambdaBuildDirectoryManager } from 'src/engine/core-modules/serverless/drivers/utils/lambda-build-directory-manager';
+import { type FlatServerlessFunction } from 'src/engine/metadata-modules/serverless-function/types/flat-serverless-function.type';
+import { type FileStorageService } from 'src/engine/core-modules/file-storage/file-storage.service';
+
+const buildServerlessFunctionInMemory = async ({
+  sourceTemporaryDir,
+  handlerPath,
+  builtHandlerPath,
+}: {
+  sourceTemporaryDir: string;
+  handlerPath: string;
+  builtHandlerPath: string;
+}): Promise<string> => {
+  const entryFilePath = join(sourceTemporaryDir, handlerPath);
+  const builtBundleFilePath = join(sourceTemporaryDir, builtHandlerPath);
+
+  await build({
+    entryPoints: [entryFilePath],
+    outfile: builtBundleFilePath,
+    platform: 'node',
+    format: 'esm',
+    target: 'es2017',
+    bundle: true,
+    sourcemap: true,
+    packages: 'external',
+  });
+
+  return builtBundleFilePath;
+};
+
+export const buildAndUploadServerlessFunction = async ({
+  flatServerlessFunction,
+  version,
+  fileStorageService,
+}: {
+  flatServerlessFunction: FlatServerlessFunction;
+  version: string;
+  fileStorageService: FileStorageService;
+}): Promise<void> => {
+  const sourceFolderPath = getServerlessFolderOrThrow({
+    flatServerlessFunction,
+    version,
+    fileFolder: FileFolder.ServerlessFunction,
+  });
+
+  const builtFolderPath = getServerlessFolderOrThrow({
+    flatServerlessFunction,
+    version,
+    fileFolder: FileFolder.BuiltFunction,
+  });
+
+  const lambdaBuildDirectoryManager = new LambdaBuildDirectoryManager();
+
+  try {
+    const { sourceTemporaryDir } = await lambdaBuildDirectoryManager.init();
+
+    await fileStorageService.download({
+      from: { folderPath: sourceFolderPath },
+      to: { folderPath: sourceTemporaryDir },
+    });
+
+    const builtBundleFilePath = await buildServerlessFunctionInMemory({
+      sourceTemporaryDir,
+      handlerPath: flatServerlessFunction.handlerPath,
+      builtHandlerPath: flatServerlessFunction.builtHandlerPath,
+    });
+
+    const builtFile = await fs.readFile(builtBundleFilePath, 'utf-8');
+
+    await fileStorageService.write({
+      file: builtFile,
+      name: flatServerlessFunction.builtHandlerPath,
+      mimeType: 'application/javascript',
+      folder: builtFolderPath,
+    });
+  } finally {
+    await lambdaBuildDirectoryManager.clean();
+  }
+};
@@ -1,28 +0,0 @@
-import { join } from 'path';
-
-import { build } from 'esbuild';
-
-export const buildServerlessFunctionInMemory = async ({
-  sourceTemporaryDir,
-  handlerPath,
-}: {
-  sourceTemporaryDir: string;
-  handlerPath: string;
-}) => {
-  const entryFilePath = join(sourceTemporaryDir, handlerPath);
-
-  const builtBundleFilePath = join(sourceTemporaryDir, 'dist', 'main.mjs');
-
-  await build({
-    entryPoints: [entryFilePath],
-    outfile: builtBundleFilePath,
-    platform: 'node',
-    format: 'esm',
-    target: 'es2017',
-    bundle: true,
-    sourcemap: true,
-    packages: 'external',
-  });
-
-  return builtBundleFilePath;
-};
@@ -1,20 +0,0 @@
-import { type BuildFailure } from 'esbuild';
-
-import { type ServerlessExecuteResult } from 'src/engine/core-modules/serverless/drivers/interfaces/serverless-driver.interface';
-
-import { ServerlessFunctionExecutionStatus } from 'src/engine/metadata-modules/serverless-function/dtos/serverless-function-execution-result.dto';
-
-export const formatBuildError = (
-  error: BuildFailure,
-  startTime: number,
-): ServerlessExecuteResult => ({
-  data: null,
-  logs: '',
-  duration: Date.now() - startTime,
-  error: {
-    errorType: 'BuildError',
-    errorMessage: error.errors.map((e) => e.text).join('\n') || 'Unknown error',
-    stackTrace: error.stack ? error.stack.split('\n') : [],
-  },
-  status: ServerlessFunctionExecutionStatus.ERROR,
-});
@@ -16,7 +16,7 @@ const getAllFiles = async (
    const fullPath = path.join(dir, entry.name);

    if (entry.isDirectory()) {
-      return getAllFiles(rootDir, fullPath, files);
+      files.push(...(await getAllFiles(rootDir, fullPath, files)));
    } else {
      files.push({
        path: path.relative(rootDir, dir),
@@ -29,11 +29,11 @@ const getAllFiles = async (
  return files;
 };

-export const getBaseTypescriptProjectFiles = (async () => {
-  const baseTypescriptProjectPath = join(
+export const getSeedProjectFiles = (async () => {
+  const seedProjectPath = join(
    ASSET_PATH,
-    `engine/core-modules/serverless/drivers/constants/base-typescript-project`,
+    `engine/core-modules/serverless/drivers/constants/seed-project`,
  );

-  return await getAllFiles(baseTypescriptProjectPath);
+  return await getAllFiles(seedProjectPath);
 })();
@@ -0,0 +1,23 @@
+import type { FlatServerlessFunction } from 'src/engine/metadata-modules/serverless-function/types/flat-serverless-function.type';
+import type { FileStorageService } from 'src/engine/core-modules/file-storage/file-storage.service';
+import { getServerlessFolderOrThrow } from 'src/engine/core-modules/serverless/utils/get-serverless-folder-or-throw.utils';
+
+export const isServerlessFunctionBuilt = async ({
+  flatServerlessFunction,
+  version,
+  fileStorageService,
+}: {
+  flatServerlessFunction: FlatServerlessFunction;
+  version: string;
+  fileStorageService: FileStorageService;
+}) => {
+  const folderPath = getServerlessFolderOrThrow({
+    flatServerlessFunction,
+    version,
+  });
+
+  return await fileStorageService.checkFileExists({
+    folderPath,
+    filename: flatServerlessFunction.builtHandlerPath,
+  });
+};
@@ -12,11 +12,14 @@ import { type FlatServerlessFunction } from 'src/engine/metadata-modules/serverl
 export const getServerlessFolderOrThrow = ({
  flatServerlessFunction,
  version,
-  toDelete = false,
+  fileFolder = FileFolder.ServerlessFunction,
 }: {
  flatServerlessFunction: FlatServerlessFunction;
  version?: 'draft' | 'latest' | (string & NonNullable<unknown>);
-  toDelete?: boolean;
+  fileFolder?:
+    | FileFolder.ServerlessFunction
+    | FileFolder.ServerlessFunctionToDelete
+    | FileFolder.BuiltFunction;
 }) => {
  if (
    version === 'latest' &&
@@ -33,9 +36,7 @@ export const getServerlessFolderOrThrow = ({

  return join(
    'workspace-' + flatServerlessFunction.workspaceId,
-    toDelete
-      ? FileFolder.ServerlessFunctionToDelete
-      : FileFolder.ServerlessFunction,
+    fileFolder,
    flatServerlessFunction.id,
    computedVersion || '',
  );