Add serverless function in twenty-cli (#14819)

- add serverlessFunction schema in twenty-cli
- add trigger schema in twenty-cli
- update serverless function code save and get
- sync serverless function

packages/twenty-server/src/engine/core-modules/application/application-sync.service.ts

@@ -2,22 +2,26 @@ import { Injectable, Logger } from '@nestjs/common';
 
 import { isDefined } from 'twenty-shared/utils';
 
+import {
+  AgentManifest,
+  ObjectManifest,
+  ServerlessFunctionManifest,
+} from 'src/engine/core-modules/application/types/application.types';
+import { ApplicationService } from 'src/engine/core-modules/application/application.service';
+import { WorkspaceManyOrAllFlatEntityMapsCacheService } from 'src/engine/core-modules/common/services/workspace-many-or-all-flat-entity-maps-cache.service.';
+import type { FlatObjectMetadataWithFlatFieldMaps } from 'src/engine/metadata-modules/flat-object-metadata-maps/types/flat-object-metadata-with-flat-field-metadata-maps.type';
+import { ObjectMetadataServiceV2 } from 'src/engine/metadata-modules/object-metadata/object-metadata-v2.service';
+import { DataSourceService } from 'src/engine/metadata-modules/data-source/data-source.service';
+import { AgentService } from 'src/engine/metadata-modules/agent/agent.service';
 import {
   ApplicationException,
   ApplicationExceptionCode,
 } from 'src/engine/core-modules/application/application.exception';
-import { ApplicationService } from 'src/engine/core-modules/application/application.service';
 import { ApplicationInput } from 'src/engine/core-modules/application/dtos/application.input';
-import {
-  AgentManifest,
-  ObjectManifest,
-} from 'src/engine/core-modules/application/types/application.types';
-import { WorkspaceManyOrAllFlatEntityMapsCacheService } from 'src/engine/core-modules/common/services/workspace-many-or-all-flat-entity-maps-cache.service.';
-import { AgentService } from 'src/engine/metadata-modules/agent/agent.service';
-import { DataSourceService } from 'src/engine/metadata-modules/data-source/data-source.service';
-import type { FlatObjectMetadataWithFlatFieldMaps } from 'src/engine/metadata-modules/flat-object-metadata-maps/types/flat-object-metadata-with-flat-field-metadata-maps.type';
-import { ObjectMetadataServiceV2 } from 'src/engine/metadata-modules/object-metadata/object-metadata-v2.service';
 import { ServerlessFunctionLayerService } from 'src/engine/metadata-modules/serverless-function-layer/serverless-function-layer.service';
+import { FlatServerlessFunction } from 'src/engine/metadata-modules/serverless-function/types/flat-serverless-function.type';
+import { ServerlessFunctionV2Service } from 'src/engine/metadata-modules/serverless-function/services/serverless-function-v2.service';
+import { ApplicationEntity } from 'src/engine/core-modules/application/application.entity';
 
 @Injectable()
 export class ApplicationSyncService {
@@ -27,6 +31,7 @@ export class ApplicationSyncService {
     private readonly applicationService: ApplicationService,
     private readonly serverlessFunctionLayerService: ServerlessFunctionLayerService,
     private readonly objectMetadataServiceV2: ObjectMetadataServiceV2,
+    private readonly serverlessFunctionV2Service: ServerlessFunctionV2Service,
     private readonly flatEntityMapsCacheService: WorkspaceManyOrAllFlatEntityMapsCacheService,
     private readonly dataSourceService: DataSourceService,
     private readonly agentService: AgentService,
@@ -40,7 +45,7 @@ export class ApplicationSyncService {
   }: ApplicationInput & {
     workspaceId: string;
   }) {
-    const applicationId = await this.syncApplication({
+    const application = await this.syncApplication({
       workspaceId,
       manifest,
       packageJson,
@@ -50,13 +55,20 @@ export class ApplicationSyncService {
     await this.syncAgents({
       agentsToSync: manifest.agents,
       workspaceId,
-      applicationId,
+      applicationId: application.id,
     });
 
     await this.syncObjects({
       objectsToSync: manifest.objects,
       workspaceId,
-      applicationId,
+      applicationId: application.id,
+    });
+
+    await this.syncServerlessFunctions({
+      serverlessFunctionsToSync: manifest.serverlessFunctions,
+      workspaceId,
+      applicationId: application.id,
+      serverlessFunctionLayerId: application.serverlessFunctionLayerId,
     });
 
     this.logger.log('✅ Application sync from manifest completed');
@@ -69,9 +81,9 @@ export class ApplicationSyncService {
     yarnLock,
   }: ApplicationInput & {
     workspaceId: string;
-  }): Promise<string> {
+  }): Promise<ApplicationEntity> {
     const application = await this.applicationService.findByUniversalIdentifier(
-      manifest.standardId,
+      manifest.universalIdentifier,
       workspaceId,
     );
 
@@ -84,8 +96,9 @@ export class ApplicationSyncService {
           },
           workspaceId,
         );
-      const createdApplication = await this.applicationService.create({
-        universalIdentifier: manifest.standardId,
+
+      return await this.applicationService.create({
+        universalIdentifier: manifest.universalIdentifier,
         label: manifest.label,
         description: manifest.description,
         version: manifest.version,
@@ -93,8 +106,6 @@ export class ApplicationSyncService {
         serverlessFunctionLayerId: serverlessFunctionLayer.id,
         workspaceId,
       });
-
-      return createdApplication.id;
     }
 
     await this.serverlessFunctionLayerService.update(
@@ -111,7 +122,7 @@ export class ApplicationSyncService {
       version: manifest.version,
     });
 
-    return application.id;
+    return application;
   }
 
   private async syncAgents({
@@ -265,4 +276,116 @@ export class ApplicationSyncService {
       });
     }
   }
+
+  private async syncServerlessFunctions({
+    serverlessFunctionsToSync,
+    workspaceId,
+    applicationId,
+    serverlessFunctionLayerId,
+  }: {
+    serverlessFunctionsToSync: ServerlessFunctionManifest[];
+    workspaceId: string;
+    applicationId: string;
+    serverlessFunctionLayerId: string;
+  }) {
+    const { flatServerlessFunctionMaps } =
+      await this.flatEntityMapsCacheService.getOrRecomputeManyOrAllFlatEntityMaps(
+        {
+          workspaceId,
+          flatEntities: ['flatServerlessFunctionMaps'],
+        },
+      );
+
+    const applicationServerlessFunctions = Object.values(
+      flatServerlessFunctionMaps.byId,
+    ).filter(
+      (serverlessFunction) =>
+        isDefined(serverlessFunction) &&
+        serverlessFunction.applicationId === applicationId,
+    ) as FlatServerlessFunction[];
+
+    const serverlessFunctionsToSyncUniversalIdentifiers =
+      serverlessFunctionsToSync.map(
+        (serverlessFunction) => serverlessFunction.universalIdentifier,
+      );
+
+    const applicationServerlessFunctionsUniversalIdentifiers =
+      applicationServerlessFunctions.map(
+        (serverlessFunction) => serverlessFunction.universalIdentifier,
+      );
+
+    const serverlessFunctionsToDelete = applicationServerlessFunctions.filter(
+      (serverlessFunction) =>
+        isDefined(serverlessFunction.universalIdentifier) &&
+        !serverlessFunctionsToSyncUniversalIdentifiers.includes(
+          serverlessFunction.universalIdentifier,
+        ),
+    );
+
+    const serverlessFunctionsToUpdate = applicationServerlessFunctions.filter(
+      (serverlessFunction) =>
+        isDefined(serverlessFunction.universalIdentifier) &&
+        serverlessFunctionsToSyncUniversalIdentifiers.includes(
+          serverlessFunction.universalIdentifier,
+        ),
+    );
+
+    const serverlessFunctionsToCreate = serverlessFunctionsToSync.filter(
+      (serverlessFunctionToSync) =>
+        !applicationServerlessFunctionsUniversalIdentifiers.includes(
+          serverlessFunctionToSync.universalIdentifier,
+        ),
+    );
+
+    for (const serverlessFunctionToDelete of serverlessFunctionsToDelete) {
+      await this.serverlessFunctionV2Service.destroyOne({
+        destroyServerlessFunctionInput: { id: serverlessFunctionToDelete.id },
+        workspaceId,
+        isSystemBuild: true,
+      });
+    }
+
+    for (const serverlessFunctionToUpdate of serverlessFunctionsToUpdate) {
+      const serverlessFunctionToSync = serverlessFunctionsToSync.find(
+        (serverlessFunction) =>
+          serverlessFunction.universalIdentifier ===
+          serverlessFunctionToUpdate.universalIdentifier,
+      );
+
+      if (!serverlessFunctionToSync) {
+        throw new ApplicationException(
+          `Failed to find serverlessFunction to sync with universalIdentifier ${serverlessFunctionToUpdate.universalIdentifier}`,
+          ApplicationExceptionCode.SERVERLESS_FUNCTION_NOT_FOUND,
+        );
+      }
+
+      const updateServerlessFunctionInput = {
+        id: serverlessFunctionToUpdate.id,
+        name: serverlessFunctionToSync.name,
+        timeoutSeconds: serverlessFunctionToSync.timeoutSeconds,
+        code: serverlessFunctionToSync.code,
+      };
+
+      await this.serverlessFunctionV2Service.updateOne(
+        updateServerlessFunctionInput,
+        workspaceId,
+      );
+    }
+
+    for (const serverlessFunctionToCreate of serverlessFunctionsToCreate) {
+      const createServerlessFunctionInput = {
+        name: serverlessFunctionToCreate.name,
+        code: serverlessFunctionToCreate.code,
+        universalIdentifier: serverlessFunctionToCreate.universalIdentifier,
+        timeoutSeconds: serverlessFunctionToCreate.timeoutSeconds,
+        applicationId,
+        serverlessFunctionLayerId,
+      };
+
+      await this.serverlessFunctionV2Service.createOne(
+        createServerlessFunctionInput,
+        workspaceId,
+      );
+    }
+  }
 }

packages/twenty-server/src/engine/core-modules/application/application.exception.ts

@@ -4,4 +4,5 @@ export class ApplicationException extends CustomException<ApplicationExceptionCo
 
 export enum ApplicationExceptionCode {
   OBJECT_NOT_FOUND = 'OBJECT_NOT_FOUND',
+  SERVERLESS_FUNCTION_NOT_FOUND = 'SERVERLESS_FUNCTION_NOT_FOUND',
 }

packages/twenty-server/src/engine/core-modules/application/application.module.ts

@@ -12,6 +12,7 @@ import { ObjectMetadataModule } from 'src/engine/metadata-modules/object-metadat
 import { DataSourceModule } from 'src/engine/metadata-modules/data-source/data-source.module';
 import { AgentModule } from 'src/engine/metadata-modules/agent/agent.module';
 import { ServerlessFunctionLayerModule } from 'src/engine/metadata-modules/serverless-function-layer/serverless-function-layer.module';
+import { ServerlessFunctionModule } from 'src/engine/metadata-modules/serverless-function/serverless-function.module';
 
 @Module({
   imports: [
@@ -21,6 +22,7 @@ import { ServerlessFunctionLayerModule } from 'src/engine/metadata-modules/serve
     DataSourceModule,
     AgentModule,
     ServerlessFunctionLayerModule,
+    ServerlessFunctionModule,
   ],
   providers: [ApplicationResolver, ApplicationService, ApplicationSyncService],
 })

packages/twenty-server/src/engine/core-modules/application/types/application.types.ts

@@ -1,6 +1,8 @@
+import { type ServerlessFunctionCode } from 'src/engine/metadata-modules/serverless-function/types/serverless-function-code.type';
+
 export type PackageJson = {
   $schema?: string;
-  standardId: string;
+  universalIdentifier: string;
   label: string;
   description?: string;
   engines: {
@@ -17,8 +19,35 @@ export type PackageJson = {
 export type AppManifest = PackageJson & {
   agents: AgentManifest[];
   objects: ObjectManifest[];
+  serverlessFunctions: ServerlessFunctionManifest[];
 };
 
+export type ServerlessFunctionManifest = {
+  $schema?: string;
+  universalIdentifier: string;
+  name: string;
+  description?: string;
+  timeoutSeconds?: number;
+  triggers: ServerlessFunctionTriggerManifest[];
+  code: ServerlessFunctionCode;
+};
+
+export type ServerlessFunctionTriggerManifest =
+  | {
+      type: 'cron';
+      schedule: string;
+    }
+  | {
+      type: 'databaseEvent';
+      eventName: string;
+    }
+  | {
+      type: 'route';
+      path: string;
+      method: 'GET' | 'POST' | 'PUT' | 'PATCH' | 'DELETE';
+      isAuthRequired: boolean;
+    };
+
 export type ObjectManifest = {
   $schema?: string;
   standardId: string;

packages/twenty-server/src/engine/core-modules/file-storage/drivers/interfaces/storage-driver.interface.ts

@@ -1,14 +1,18 @@
 import { type Readable } from 'stream';
 
+import { type Sources } from 'src/engine/core-modules/file-storage/types/source.type';
+
 export interface StorageDriver {
   delete(params: { folderPath: string; filename?: string }): Promise<void>;
   read(params: { folderPath: string; filename: string }): Promise<Readable>;
+  readFolder(folderPath: string): Promise<Sources>;
   write(params: {
     file: Buffer | Uint8Array | string;
     name: string;
     folder: string;
     mimeType: string | undefined;
   }): Promise<void>;
+  writeFolder(sources: Sources, folderPath: string): Promise<void>;
   move(params: {
     from: { folderPath: string; filename?: string };
     to: { folderPath: string; filename?: string };
@@ -21,6 +25,7 @@ export interface StorageDriver {
     from: { folderPath: string; filename?: string };
     to: { folderPath: string; filename?: string };
   }): Promise<void>;
+
   checkFileExists(params: {
     folderPath: string;
     filename: string;

packages/twenty-server/src/engine/core-modules/file-storage/drivers/local.driver.ts

@@ -1,14 +1,18 @@
 import { createReadStream, existsSync } from 'fs';
 import * as fs from 'fs/promises';
-import { dirname, join } from 'path';
+import path, { dirname, join } from 'path';
 import { type Readable } from 'stream';
 
+import { isObject } from '@sniptt/guards';
+
 import { type StorageDriver } from 'src/engine/core-modules/file-storage/drivers/interfaces/storage-driver.interface';
 import {
   FileStorageException,
   FileStorageExceptionCode,
 } from 'src/engine/core-modules/file-storage/interfaces/file-storage-exception';
 
+import { type Sources } from 'src/engine/core-modules/file-storage/types/source.type';
+
 export interface LocalDriverOptions {
   storagePath: string;
 }
@@ -42,6 +46,21 @@ export class LocalDriver implements StorageDriver {
     await fs.writeFile(filePath, params.file);
   }
 
+  async writeFolder(sources: Sources, folderPath: string) {
+    for (const key of Object.keys(sources)) {
+      if (isObject(sources[key])) {
+        await this.writeFolder(sources[key], join(folderPath, key));
+        continue;
+      }
+      await this.write({
+        file: sources[key],
+        name: key,
+        mimeType: undefined,
+        folder: folderPath,
+      });
+    }
+  }
+
   async delete(params: {
     folderPath: string;
     filename?: string;
@@ -86,6 +105,30 @@ export class LocalDriver implements StorageDriver {
     }
   }
 
+  async readFolder(folderPath: string): Promise<Sources> {
+    const sources: Sources = {};
+
+    const rootFolderPath = join(`${this.options.storagePath}/`, folderPath);
+
+    const resources = await fs.readdir(rootFolderPath);
+
+    for (const resource of resources) {
+      const resourcePath = path.join(rootFolderPath, resource);
+
+      const stats = await fs.stat(resourcePath);
+
+      if (stats.isFile()) {
+        sources[resource] = await fs.readFile(resourcePath, 'utf8');
+      } else {
+        sources[resource] = await this.readFolder(
+          path.join(folderPath, resource),
+        );
+      }
+    }
+
+    return sources;
+  }
+
   async move(params: {
     from: { folderPath: string; filename?: string };
     to: { folderPath: string; filename?: string };

packages/twenty-server/src/engine/core-modules/file-storage/drivers/s3.driver.ts

@@ -21,6 +21,7 @@ import {
   type S3ClientConfig,
 } from '@aws-sdk/client-s3';
 import { isDefined } from 'twenty-shared/utils';
+import { isObject } from '@sniptt/guards';
 
 import { type StorageDriver } from 'src/engine/core-modules/file-storage/drivers/interfaces/storage-driver.interface';
 import {
@@ -28,6 +29,10 @@ import {
   FileStorageExceptionCode,
 } from 'src/engine/core-modules/file-storage/interfaces/file-storage-exception';
 
+import type { Sources } from 'src/engine/core-modules/file-storage/types/source.type';
+import { readFileContent } from 'src/engine/core-modules/file-storage/utils/read-file-content';
+import { readS3FolderContent } from 'src/engine/core-modules/file-storage/utils/read-s3-folder-content';
+
 export interface S3DriverOptions extends S3ClientConfig {
   bucketName: string;
   endpoint?: string;
@@ -70,6 +75,21 @@ export class S3Driver implements StorageDriver {
     await this.s3Client.send(command);
   }
 
+  async writeFolder(sources: Sources, folderPath: string) {
+    for (const key of Object.keys(sources)) {
+      if (isObject(sources[key])) {
+        await this.writeFolder(sources[key], join(folderPath, key));
+        continue;
+      }
+      await this.write({
+        file: sources[key],
+        name: key,
+        mimeType: undefined,
+        folder: folderPath,
+      });
+    }
+  }
+
   private async fetchS3FolderContents(folderPath: string) {
     const listParams = {
       Bucket: this.bucketName,
@@ -177,6 +197,46 @@ export class S3Driver implements StorageDriver {
     }
   }
 
+  async readFolder(folderPath: string): Promise<Sources> {
+    const sources: Sources = {};
+    const listedObjects = await this.fetchS3FolderContents(folderPath);
+
+    if (!listedObjects.Contents || listedObjects.Contents.length === 0) {
+      return sources;
+    }
+
+    const files = (
+      await Promise.all(
+        listedObjects.Contents.map(async (object) => {
+          if (!object.Key) {
+            return;
+          }
+
+          const folderAndFilePaths = this.extractFolderAndFilePaths(object.Key);
+
+          if (!isDefined(folderAndFilePaths)) {
+            return;
+          }
+
+          const { fromFolderPath, filename } = folderAndFilePaths;
+
+          const fileContent = await readFileContent(
+            await this.read({ folderPath: fromFolderPath, filename }),
+          );
+
+          const formattedObjectKey = object.Key.replace(
+            folderPath + '/',
+            '',
+          ).replace(folderPath, '');
+
+          return { path: formattedObjectKey, fileContent };
+        }),
+      )
+    ).filter(isDefined);
+
+    return readS3FolderContent(files);
+  }
+
   async move(params: {
     from: { folderPath: string; filename?: string };
     to: { folderPath: string; filename?: string };

packages/twenty-server/src/engine/core-modules/file-storage/file-storage.service.ts

@@ -5,6 +5,7 @@ import { type Readable } from 'stream';
 import { type StorageDriver } from 'src/engine/core-modules/file-storage/drivers/interfaces/storage-driver.interface';
 
 import { FileStorageDriverFactory } from 'src/engine/core-modules/file-storage/file-storage-driver.factory';
+import { Sources } from 'src/engine/core-modules/file-storage/types/source.type';
 
 @Injectable()
 export class FileStorageService implements StorageDriver {
@@ -23,12 +24,24 @@ export class FileStorageService implements StorageDriver {
     return driver.write(params);
   }
 
+  writeFolder(sources: Sources, folderPath: string): Promise<void> {
+    const driver = this.fileStorageDriverFactory.getCurrentDriver();
+
+    return driver.writeFolder(sources, folderPath);
+  }
+
   read(params: { folderPath: string; filename: string }): Promise<Readable> {
     const driver = this.fileStorageDriverFactory.getCurrentDriver();
 
     return driver.read(params);
   }
 
+  readFolder(folderPath: string): Promise<Sources> {
+    const driver = this.fileStorageDriverFactory.getCurrentDriver();
+
+    return driver.readFolder(folderPath);
+  }
+
   delete(params: { folderPath: string; filename?: string }): Promise<void> {
     const driver = this.fileStorageDriverFactory.getCurrentDriver();
 

packages/twenty-server/src/engine/core-modules/file-storage/types/source.type.ts

@@ -0,0 +1 @@
+export type Sources = { [key: string]: string | Sources };

packages/twenty-server/src/engine/core-modules/file-storage/utils/__tests__/read-s3-folder-content.spec.ts

@@ -0,0 +1,25 @@
+import { readS3FolderContent } from 'src/engine/core-modules/file-storage/utils/read-s3-folder-content';
+
+describe('read-s3-folder-content', () => {
+  it('should format files to sources properly', () => {
+    const files = [
+      { path: 'f1/file1.ts', fileContent: 'content1' },
+      { path: 'f1/f11/file11.ts', fileContent: 'content11' },
+      { path: 'f1/file2.ts', fileContent: 'content2' },
+      { path: 'file3.ts', fileContent: 'content3' },
+    ];
+
+    const expectedResult = {
+      'file3.ts': 'content3',
+      f1: {
+        'file1.ts': 'content1',
+        'file2.ts': 'content2',
+        f11: { 'file11.ts': 'content11' },
+      },
+    };
+
+    const result = readS3FolderContent(files);
+
+    expect(result).toEqual(expectedResult);
+  });
+});

packages/twenty-server/src/engine/core-modules/file-storage/utils/read-s3-folder-content.ts

@@ -0,0 +1,32 @@
+import { isDefined } from 'twenty-shared/utils';
+import { isObject } from '@sniptt/guards';
+
+import { type Sources } from 'src/engine/core-modules/file-storage/types/source.type';
+
+export const readS3FolderContent = (
+  files: { path: string; fileContent: string }[],
+) => {
+  const result: Sources = {};
+
+  for (const { path, fileContent } of files) {
+    const segments = path.split('/');
+    const fileName = segments.pop();
+
+    if (!isDefined(fileName)) {
+      continue;
+    }
+
+    let cursor: Sources = result;
+
+    for (const segment of segments) {
+      if (!isObject(cursor[segment])) {
+        cursor[segment] = {};
+      }
+      cursor = cursor[segment] as Sources;
+    }
+
+    cursor[fileName] = fileContent;
+  }
+
+  return result;
+};