1573 extensibility twenty cli handle custom layers for serverless functions of applications (#14779)

- allow specific layers for serverless functions
- add a serverlessFunctionLayer table
- sync application layer

packages/twenty-server/src/engine/core-modules/serverless/drivers/constants/executor/index.mjs

@@ -1,20 +1,21 @@
 import { promises as fs } from 'fs';
-import { v4 } from 'uuid';
-
+import { randomBytes } from 'crypto';
 
 export const handler = async (event) => {
-  const mainPath = `/tmp/${v4()}.mjs`;
+  const randomId = randomBytes(16).toString('hex');
+
+  const mainPath = `/tmp/${randomId}.mjs`;
 
   try {
     const { code, params } = event;
 
     await fs.writeFile(mainPath, code, 'utf8');
 
-    process.env = {}
+    process.env = {};
 
     const mainFile = await import(mainPath);
 
-    return  await mainFile.main(params);
+    return await mainFile.main(params);
   } finally {
     await fs.rm(mainPath, { force: true });
   }

packages/twenty-server/src/engine/core-modules/serverless/drivers/lambda.driver.ts

@@ -51,7 +51,6 @@ import {
 
 const UPDATE_FUNCTION_DURATION_TIMEOUT_IN_SECONDS = 60;
 const CREDENTIALS_DURATION_IN_SECONDS = 60 * 60; // 1h
-const LAMBDA_EXECUTOR_DESCRIPTION = 'User script executor';
 
 export interface LambdaDriverOptions extends LambdaClientConfig {
   fileStorageService: FileStorageService;
@@ -135,22 +134,31 @@ export class LambdaDriver implements ServerlessDriver {
     );
   }
 
-  private async createLayerIfNotExists(version: number): Promise<string> {
+  private getLayerName(serverlessFunction: ServerlessFunctionEntity) {
+    if (isDefined(serverlessFunction?.serverlessFunctionLayer)) {
+      return serverlessFunction?.serverlessFunctionLayer.checksum;
+    }
+
+    return COMMON_LAYER_NAME;
+  }
+
+  private async createLayerIfNotExists(
+    serverlessFunction: ServerlessFunctionEntity,
+  ): Promise<string> {
+    const layerName = this.getLayerName(serverlessFunction);
+
     const listLayerParams: ListLayerVersionsCommandInput = {
-      LayerName: COMMON_LAYER_NAME,
+      LayerName: layerName,
       MaxItems: 1,
     };
+
     const listLayerCommand = new ListLayerVersionsCommand(listLayerParams);
+
     const listLayerResult = await (
       await this.getLambdaClient()
     ).send(listLayerCommand);
 
-    if (
-      isDefined(listLayerResult.LayerVersions) &&
-      listLayerResult.LayerVersions.length > 0 &&
-      listLayerResult.LayerVersions?.[0].Description === `${version}` &&
-      isDefined(listLayerResult.LayerVersions[0].LayerVersionArn)
-    ) {
+    if (isDefined(listLayerResult.LayerVersions?.[0]?.LayerVersionArn)) {
       return listLayerResult.LayerVersions[0].LayerVersionArn;
     }
 
@@ -163,12 +171,12 @@ export class LambdaDriver implements ServerlessDriver {
       NODE_LAYER_SUBFOLDER,
     );
 
-    await copyAndBuildDependencies(nodeDependenciesFolder);
+    await copyAndBuildDependencies(nodeDependenciesFolder, serverlessFunction);
 
     await createZipFile(sourceTemporaryDir, lambdaZipPath);
 
     const params: PublishLayerVersionCommandInput = {
-      LayerName: COMMON_LAYER_NAME,
+      LayerName: layerName,
       Content: {
         ZipFile: await fs.readFile(lambdaZipPath),
       },
@@ -176,7 +184,6 @@ export class LambdaDriver implements ServerlessDriver {
         ServerlessFunctionRuntime.NODE18,
         ServerlessFunctionRuntime.NODE22,
       ],
-      Description: `${version}`,
     };
 
     const command = new PublishLayerVersionCommand(params);
@@ -220,22 +227,38 @@ export class LambdaDriver implements ServerlessDriver {
     }
   }
 
-  private async build(serverlessFunction: ServerlessFunctionEntity) {
+  private async isAlreadyBuilt(serverlessFunction: ServerlessFunctionEntity) {
     const lambdaExecutor = await this.getLambdaExecutor(serverlessFunction);
 
-    if (isDefined(lambdaExecutor)) {
-      if (
-        lambdaExecutor.Configuration?.Description ===
-        LAMBDA_EXECUTOR_DESCRIPTION
-      ) {
-        return;
-      }
-      await this.delete(serverlessFunction);
+    if (!isDefined(lambdaExecutor)) {
+      return false;
     }
 
-    const layerArn = await this.createLayerIfNotExists(
-      serverlessFunction.layerVersion ?? 0,
-    );
+    const layers = lambdaExecutor.Configuration?.Layers;
+
+    if (!isDefined(layers) || layers.length !== 1) {
+      await this.delete(serverlessFunction);
+
+      return false;
+    }
+
+    const layerName = this.getLayerName(serverlessFunction);
+
+    if (layers[0].Arn?.includes(layerName)) {
+      return true;
+    }
+
+    await this.delete(serverlessFunction);
+
+    return false;
+  }
+
+  private async build(serverlessFunction: ServerlessFunctionEntity) {
+    if (await this.isAlreadyBuilt(serverlessFunction)) {
+      return;
+    }
+
+    const layerArn = await this.createLayerIfNotExists(serverlessFunction);
 
     const lambdaBuildDirectoryManager = new LambdaBuildDirectoryManager();
 
@@ -255,7 +278,6 @@ export class LambdaDriver implements ServerlessDriver {
       Handler: 'index.handler',
       Role: this.options.lambdaRole,
       Runtime: serverlessFunction.runtime,
-      Description: LAMBDA_EXECUTOR_DESCRIPTION,
       Timeout: serverlessFunction.timeoutSeconds,
     };
 

packages/twenty-server/src/engine/core-modules/serverless/drivers/local.driver.ts

@@ -3,6 +3,7 @@ import { join } from 'path';
 
 import ts, { transpileModule } from 'typescript';
 import { v4 } from 'uuid';
+import { isDefined } from 'twenty-shared/utils';
 
 import {
   type ServerlessDriver,
@@ -31,25 +32,43 @@ export class LocalDriver implements ServerlessDriver {
     this.fileStorageService = options.fileStorageService;
   }
 
-  private getInMemoryLayerFolderPath = (version: number) => {
-    return join(SERVERLESS_TMPDIR_FOLDER, COMMON_LAYER_NAME, `${version}`);
+  private getInMemoryLayerFolderPath = (
+    serverlessFunction: ServerlessFunctionEntity,
+  ) => {
+    if (!isDefined(serverlessFunction?.serverlessFunctionLayer?.checksum)) {
+      return join(
+        SERVERLESS_TMPDIR_FOLDER,
+        COMMON_LAYER_NAME,
+        `${serverlessFunction.layerVersion}`,
+      );
+    }
+
+    return join(
+      SERVERLESS_TMPDIR_FOLDER,
+      serverlessFunction.serverlessFunctionLayer?.checksum,
+    );
   };
 
-  private async createLayerIfNotExists(version: number) {
-    const inMemoryLastVersionLayerFolderPath =
-      this.getInMemoryLayerFolderPath(version);
+  private async createLayerIfNotExists(
+    serverlessFunction: ServerlessFunctionEntity,
+  ) {
+    const inMemoryLayerFolderPath =
+      this.getInMemoryLayerFolderPath(serverlessFunction);
 
     try {
-      await fs.access(inMemoryLastVersionLayerFolderPath);
+      await fs.access(inMemoryLayerFolderPath);
     } catch {
-      await copyAndBuildDependencies(inMemoryLastVersionLayerFolderPath);
+      await copyAndBuildDependencies(
+        inMemoryLayerFolderPath,
+        serverlessFunction,
+      );
     }
   }
 
   async delete() {}
 
   private async build(serverlessFunction: ServerlessFunctionEntity) {
-    await this.createLayerIfNotExists(serverlessFunction.layerVersion ?? 0);
+    await this.createLayerIfNotExists(serverlessFunction);
   }
 
   private async executeWithTimeout<T>(
@@ -119,7 +138,7 @@ export class LocalDriver implements ServerlessDriver {
 
       await fs.symlink(
         join(
-          this.getInMemoryLayerFolderPath(serverlessFunction.layerVersion),
+          this.getInMemoryLayerFolderPath(serverlessFunction),
           'node_modules',
         ),
         join(compiledCodeFolderPath, 'node_modules'),

packages/twenty-server/src/engine/core-modules/serverless/drivers/utils/copy-and-build-dependencies.ts

@@ -3,18 +3,46 @@ import { promisify } from 'util';
 import { exec } from 'child_process';
 import { join } from 'path';
 
+import { isDefined } from 'twenty-shared/utils';
+
 import { getLayerDependenciesDirName } from 'src/engine/core-modules/serverless/drivers/utils/get-layer-dependencies-dir-name';
+import type { ServerlessFunctionEntity } from 'src/engine/metadata-modules/serverless-function/serverless-function.entity';
+import { LAST_LAYER_VERSION } from 'src/engine/core-modules/serverless/drivers/layers/last-layer-version';
 
 const execPromise = promisify(exec);
 
-export const copyAndBuildDependencies = async (buildDirectory: string) => {
+export const copyAndBuildDependencies = async (
+  buildDirectory: string,
+  serverlessFunction: ServerlessFunctionEntity,
+) => {
   await fs.mkdir(buildDirectory, {
     recursive: true,
   });
 
-  await fs.cp(getLayerDependenciesDirName('latest'), buildDirectory, {
-    recursive: true,
-  });
+  if (!isDefined(serverlessFunction.serverlessFunctionLayer)) {
+    await fs.cp(
+      getLayerDependenciesDirName(
+        serverlessFunction.layerVersion || LAST_LAYER_VERSION,
+      ),
+      buildDirectory,
+      {
+        recursive: true,
+      },
+    );
+  } else {
+    const packageJson = serverlessFunction.serverlessFunctionLayer.packageJson;
+
+    const yarnLock = serverlessFunction.serverlessFunctionLayer.yarnLock;
+
+    await fs.writeFile(
+      join(buildDirectory, 'package.json'),
+      JSON.stringify(packageJson, null, 2),
+      'utf8',
+    );
+
+    await fs.writeFile(join(buildDirectory, 'yarn.lock'), yarnLock, 'utf8');
+  }
+
   await fs.cp(getLayerDependenciesDirName('engine'), buildDirectory, {
     recursive: true,
   });

packages/twenty-server/src/engine/core-modules/serverless/drivers/utils/get-last-layer-dependencies.ts

@@ -1,15 +1,20 @@
 import fs from 'fs/promises';
 import { join } from 'path';
 
+import { isDefined } from 'twenty-shared/utils';
+
 import { getLayerDependenciesDirName } from 'src/engine/core-modules/serverless/drivers/utils/get-layer-dependencies-dir-name';
+import { type ServerlessFunctionEntity } from 'src/engine/metadata-modules/serverless-function/serverless-function.entity';
+import { LAST_LAYER_VERSION } from 'src/engine/core-modules/serverless/drivers/layers/last-layer-version';
+import { type PackageJson } from 'src/engine/core-modules/application/types/application.types';
 
 export type LayerDependencies = {
-  packageJson: { dependencies: object };
+  packageJson: PackageJson;
   yarnLock: string;
 };
 
-export const getLayerDependencies = async (
-  layerVersion: number | 'latest',
+export const getLastCommonLayerDependencies = async (
+  layerVersion = LAST_LAYER_VERSION,
 ): Promise<LayerDependencies> => {
   const lastVersionLayerDirName = getLayerDependenciesDirName(layerVersion);
   const [packageJson, yarnLock] = await Promise.all([
@@ -19,3 +24,18 @@ export const getLayerDependencies = async (
 
   return { packageJson: JSON.parse(packageJson), yarnLock };
 };
+
+export const getLayerDependencies = async (
+  serverlessFunction: ServerlessFunctionEntity,
+): Promise<LayerDependencies> => {
+  if (!isDefined(serverlessFunction.serverlessFunctionLayer)) {
+    return getLastCommonLayerDependencies(
+      serverlessFunction.layerVersion ?? undefined,
+    );
+  }
+
+  return {
+    packageJson: serverlessFunction.serverlessFunctionLayer.packageJson,
+    yarnLock: serverlessFunction.serverlessFunctionLayer.yarnLock,
+  };
+};

packages/twenty-server/src/engine/core-modules/serverless/drivers/utils/get-layer-dependencies-dir-name.ts

@@ -1,16 +1,13 @@
 import path from 'path';
 
-import { LAST_LAYER_VERSION } from 'src/engine/core-modules/serverless/drivers/layers/last-layer-version';
 import { ASSET_PATH } from 'src/constants/assets-path';
 
 export const getLayerDependenciesDirName = (
-  version: 'latest' | 'engine' | number,
+  version: 'engine' | number,
 ): string => {
-  const formattedVersion = version === 'latest' ? LAST_LAYER_VERSION : version;
-
   const baseTypescriptProjectPath = path.join(
     ASSET_PATH,
-    `engine/core-modules/serverless/drivers/layers/${formattedVersion}`,
+    `engine/core-modules/serverless/drivers/layers/${version}`,
   );
 
   return path.resolve(__dirname, baseTypescriptProjectPath);