houseassassin/twenty
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

2114 extensibility manage serverless execution from built code (#17317)

Browse Source 
Summary

- Serverless functions are now built when created/updated instead of at
execution time
  - Added builtHandlerPath field to track pre-built function artifacts
- Renamed base-typescript-project to seed-project with pre-built ESM
output included
- Renamed file folder enums for consistency (Functions → BuiltFunction,
SourceCode → Source)
  - supports backwards compatibility with existing serverless functions

Tested with all combinations
- storage : local driver and S3 driver
- serverless : local driver and lambda driver
This commit is contained in:
martmull
2026-01-22 12:56:44 +01:00
committed by GitHub
parent 9ee2cd0fe3
commit f92c8a98a4
34 changed files with 348 additions and 193 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/twenty-front/src/generated-metadata/graphql.ts
+6 -4
View File
@@ -1046,6 +1046,7 @@ export type CreateRowLevelPermissionPredicateInput = {
};
export type CreateServerlessFunctionInput = {
builtHandlerPath?: InputMaybe<Scalars['String']>;
code?: InputMaybe<Scalars['JSON']>;
description?: InputMaybe<Scalars['String']>;
handlerName?: InputMaybe<Scalars['String']>;
@@ -1584,16 +1585,16 @@ export type File = {
export enum FileFolder {
AgentChat = 'AgentChat',
Assets = 'Assets',
Asset = 'Asset',
Attachment = 'Attachment',
BuiltFrontComponent = 'BuiltFrontComponent',
BuiltFunction = 'BuiltFunction',
File = 'File',
FrontComponents = 'FrontComponents',
Functions = 'Functions',
PersonPicture = 'PersonPicture',
ProfilePicture = 'ProfilePicture',
ServerlessFunction = 'ServerlessFunction',
ServerlessFunctionToDelete = 'ServerlessFunctionToDelete',
SourceCode = 'SourceCode',
Source = 'Source',
WorkspaceLogo = 'WorkspaceLogo'
}
@@ -4298,6 +4299,7 @@ export type Sentry = {
export type ServerlessFunction = {
__typename?: 'ServerlessFunction';
applicationId?: Maybe<Scalars['UUID']>;
builtHandlerPath: Scalars['String'];
createdAt: Scalars['DateTime'];
cronTriggers?: Maybe<Array<CronTrigger>>;
databaseEventTriggers?: Maybe<Array<DatabaseEventTrigger>>;
packages/twenty-front/src/generated/graphql.ts
+6 -4
View File
@@ -1032,6 +1032,7 @@ export type CreateRowLevelPermissionPredicateInput = {
};
export type CreateServerlessFunctionInput = {
builtHandlerPath?: InputMaybe<Scalars['String']>;
code?: InputMaybe<Scalars['JSON']>;
description?: InputMaybe<Scalars['String']>;
handlerName?: InputMaybe<Scalars['String']>;
@@ -1561,16 +1562,16 @@ export type File = {
export enum FileFolder {
AgentChat = 'AgentChat',
Assets = 'Assets',
Asset = 'Asset',
Attachment = 'Attachment',
BuiltFrontComponent = 'BuiltFrontComponent',
BuiltFunction = 'BuiltFunction',
File = 'File',
FrontComponents = 'FrontComponents',
Functions = 'Functions',
PersonPicture = 'PersonPicture',
ProfilePicture = 'ProfilePicture',
ServerlessFunction = 'ServerlessFunction',
ServerlessFunctionToDelete = 'ServerlessFunctionToDelete',
SourceCode = 'SourceCode',
Source = 'Source',
WorkspaceLogo = 'WorkspaceLogo'
}
@@ -4142,6 +4143,7 @@ export type Sentry = {
export type ServerlessFunction = {
__typename?: 'ServerlessFunction';
applicationId?: Maybe<Scalars['UUID']>;
builtHandlerPath: Scalars['String'];
createdAt: Scalars['DateTime'];
cronTriggers?: Maybe<Array<CronTrigger>>;
databaseEventTriggers?: Maybe<Array<DatabaseEventTrigger>>;
packages/twenty-server/eslint.config.mjs
+1 -1
View File
@@ -36,7 +36,7 @@ export default [
'src/engine/workspace-manager/dev-seeder/data/seeds/**',
'src/utils/email-providers.ts',
'src/engine/core-modules/i18n/locales/generated/**',
'src/engine/core-modules/serverless/drivers/constants/base-typescript-project/src/index.ts',
'src/engine/core-modules/serverless/drivers/constants/seed-project/src/index.ts',
'packages/twenty-server/src/engine/core-modules/i18n/locales/**'
],
},
packages/twenty-server/nest-cli.json
+1 -1
View File
@@ -11,7 +11,7 @@
"watchAssets": true,
"assets": [
{
"include": "engine/core-modules/serverless/drivers/constants/base-typescript-project/**",
"include": "engine/core-modules/serverless/drivers/constants/seed-project/**",
"outDir": "dist/assets"
},
{
packages/twenty-server/src/database/typeorm/core/migrations/common/1769016869438-addBuiltHandlerPathToServerlessFunctions.ts
+19
View File
@@ -0,0 +1,19 @@
import { type MigrationInterface, type QueryRunner } from 'typeorm';
export class AddBuiltHandlerPathToServerlessFunctions1769016869438
implements MigrationInterface
{
name = 'AddBuiltHandlerPathToServerlessFunctions1769016869438';
public async up(queryRunner: QueryRunner): Promise<void> {
await queryRunner.query(
`ALTER TABLE "core"."serverlessFunction" ADD "builtHandlerPath" character varying NOT NULL DEFAULT 'index.mjs'`,
);
}
public async down(queryRunner: QueryRunner): Promise<void> {
await queryRunner.query(
`ALTER TABLE "core"."serverlessFunction" DROP COLUMN "builtHandlerPath"`,
);
}
}
packages/twenty-server/src/engine/core-modules/application/application.resolver.ts
+4 -4
View File
@@ -110,10 +110,10 @@ export class ApplicationResolver {
}: UploadApplicationFileInput,
): Promise<FileDTO> {
const allowedApplicationFileFolders: FileFolder[] = [
FileFolder.Functions,
FileFolder.FrontComponents,
FileFolder.Assets,
FileFolder.SourceCode,
FileFolder.BuiltFunction,
FileFolder.BuiltFrontComponent,
FileFolder.Asset,
FileFolder.Source,
];
if (!allowedApplicationFileFolders.includes(fileFolder)) {
packages/twenty-server/src/engine/core-modules/file/interfaces/file-folder.interface.ts
+4 -4
View File
@@ -36,16 +36,16 @@ export const fileFolderConfigs: Record<FileFolder, FileFolderConfig> = {
[FileFolder.AgentChat]: {
ignoreExpirationToken: false,
},
[FileFolder.Functions]: {
[FileFolder.BuiltFunction]: {
ignoreExpirationToken: false,
},
[FileFolder.FrontComponents]: {
[FileFolder.BuiltFrontComponent]: {
ignoreExpirationToken: false,
},
[FileFolder.Assets]: {
[FileFolder.Asset]: {
ignoreExpirationToken: true,
},
[FileFolder.SourceCode]: {
[FileFolder.Source]: {
ignoreExpirationToken: false,
},
};
packages/twenty-server/src/engine/core-modules/serverless/drivers/constants/.gitignore
-1
View File
@@ -1 +0,0 @@
!base-typescript-project/**/.env
packages/twenty-server/src/engine/core-modules/serverless/drivers/constants/base-typescript-project-input-schema.ts
-4
View File
@@ -1,4 +0,0 @@
export const BASE_TYPESCRIPT_PROJECT_INPUT_SCHEMA = {
a: null,
b: null,
};
packages/twenty-server/src/engine/core-modules/serverless/drivers/constants/base-typescript-project/.env
-2
View File
@@ -1,2 +0,0 @@
# Add your environment variables here.
# Access them in your serverless function code using process.env.VARIABLE
packages/twenty-server/src/engine/core-modules/serverless/drivers/constants/seed-project-input-schema.ts
+4
View File
@@ -0,0 +1,4 @@
export const SEED_PROJECT_INPUT_SCHEMA = {
a: null,
b: null,
};
packages/twenty-server/src/engine/core-modules/serverless/drivers/constants/base-typescript-project/src/index.ts → packages/twenty-server/src/engine/core-modules/serverless/drivers/constants/seed-project/src/index.ts
View File
packages/twenty-server/src/engine/core-modules/serverless/drivers/lambda.driver.ts
+55 -72
View File
@@ -20,6 +20,7 @@ import {
} from '@aws-sdk/client-lambda';
import { AssumeRoleCommand, STSClient } from '@aws-sdk/client-sts';
import { isDefined } from 'twenty-shared/utils';
import { FileFolder } from 'twenty-shared/types';
import {
type ServerlessDriver,
@@ -27,16 +28,14 @@ import {
} from 'src/engine/core-modules/serverless/drivers/interfaces/serverless-driver.interface';
import { type FileStorageService } from 'src/engine/core-modules/file-storage/file-storage.service';
import { buildServerlessFunctionInMemory } from 'src/engine/core-modules/serverless/drivers/utils/build-serverless-function-in-memory';
import { copyAndBuildDependencies } from 'src/engine/core-modules/serverless/drivers/utils/copy-and-build-dependencies';
import { copyExecutor } from 'src/engine/core-modules/serverless/drivers/utils/copy-executor';
import { createZipFile } from 'src/engine/core-modules/serverless/drivers/utils/create-zip-file';
import { formatBuildError } from 'src/engine/core-modules/serverless/drivers/utils/format-build-error';
import {
LambdaBuildDirectoryManager,
NODE_LAYER_SUBFOLDER,
} from 'src/engine/core-modules/serverless/drivers/utils/lambda-build-directory-manager';
import { getServerlessFolderOrThrow } from 'src/engine/core-modules/serverless/utils/serverless-get-folder.utils';
import { getServerlessFolderOrThrow } from 'src/engine/core-modules/serverless/utils/get-serverless-folder-or-throw.utils';
import { type FlatServerlessFunctionLayer } from 'src/engine/metadata-modules/serverless-function-layer/types/flat-serverless-function-layer.type';
import { ServerlessFunctionExecutionStatus } from 'src/engine/metadata-modules/serverless-function/dtos/serverless-function-execution-result.dto';
import { ServerlessFunctionRuntime } from 'src/engine/metadata-modules/serverless-function/serverless-function.entity';
@@ -45,6 +44,7 @@ import {
ServerlessFunctionExceptionCode,
} from 'src/engine/metadata-modules/serverless-function/serverless-function.exception';
import { type FlatServerlessFunction } from 'src/engine/metadata-modules/serverless-function/types/flat-serverless-function.type';
import { streamToBuffer } from 'src/utils/stream-to-buffer';
const UPDATE_FUNCTION_DURATION_TIMEOUT_IN_SECONDS = 60;
const CREDENTIALS_DURATION_IN_SECONDS = 60 * 60; // 1h
@@ -335,93 +335,76 @@ export class LambdaDriver implements ServerlessDriver {
env?: Record<string, string>;
}): Promise<ServerlessExecuteResult> {
await this.build(flatServerlessFunction, flatServerlessFunctionLayer);
await this.waitFunctionUpdates(flatServerlessFunction);
const startTime = Date.now();
const folderPath = getServerlessFolderOrThrow({
const builtHandlerFolderPath = getServerlessFolderOrThrow({
flatServerlessFunction,
version,
fileFolder: FileFolder.BuiltFunction,
});
const lambdaBuildDirectoryManager = new LambdaBuildDirectoryManager();
const compiledCode = (
await streamToBuffer(
await this.fileStorageService.read({
folderPath: builtHandlerFolderPath,
filename: flatServerlessFunction.builtHandlerPath,
}),
)
).toString('utf-8');
const executorPayload: LambdaDriverExecutorPayload = {
params: payload,
code: compiledCode,
env: env ?? {},
handlerName: flatServerlessFunction.handlerName,
};
const params: InvokeCommandInput = {
FunctionName: flatServerlessFunction.id,
Payload: JSON.stringify(executorPayload),
LogType: LogType.Tail,
};
const command = new InvokeCommand(params);
try {
const { sourceTemporaryDir } = await lambdaBuildDirectoryManager.init();
const result = await (await this.getLambdaClient()).send(command);
await this.fileStorageService.download({
from: { folderPath },
to: { folderPath: sourceTemporaryDir },
});
const parsedResult = result.Payload
? JSON.parse(result.Payload.transformToString())
: {};
let builtBundleFilePath = '';
const logs = result.LogResult ? this.extractLogs(result.LogResult) : '';
try {
builtBundleFilePath = await buildServerlessFunctionInMemory({
sourceTemporaryDir,
handlerPath: flatServerlessFunction.handlerPath,
});
} catch (error) {
return formatBuildError(error, startTime);
}
const compiledCode = (await fs.readFile(builtBundleFilePath)).toString(
'utf-8',
);
const executorPayload: LambdaDriverExecutorPayload = {
params: payload,
code: compiledCode,
env: env ?? {},
handlerName: flatServerlessFunction.handlerName,
};
const params: InvokeCommandInput = {
FunctionName: flatServerlessFunction.id,
Payload: JSON.stringify(executorPayload),
LogType: LogType.Tail,
};
const command = new InvokeCommand(params);
try {
const result = await (await this.getLambdaClient()).send(command);
const parsedResult = result.Payload
? JSON.parse(result.Payload.transformToString())
: {};
const logs = result.LogResult ? this.extractLogs(result.LogResult) : '';
const duration = Date.now() - startTime;
if (result.FunctionError) {
return {
data: null,
duration,
status: ServerlessFunctionExecutionStatus.ERROR,
error: parsedResult,
logs,
};
}
const duration = Date.now() - startTime;
if (result.FunctionError) {
return {
data: parsedResult,
logs,
data: null,
duration,
status: ServerlessFunctionExecutionStatus.SUCCESS,
status: ServerlessFunctionExecutionStatus.ERROR,
error: parsedResult,
logs,
};
} catch (error) {
if (error instanceof ResourceNotFoundException) {
throw new ServerlessFunctionException(
`Function Version '${version}' does not exist`,
ServerlessFunctionExceptionCode.SERVERLESS_FUNCTION_NOT_FOUND,
);
}
throw error;
}
} finally {
await lambdaBuildDirectoryManager.clean();
return {
data: parsedResult,
logs,
duration,
status: ServerlessFunctionExecutionStatus.SUCCESS,
};
} catch (error) {
if (error instanceof ResourceNotFoundException) {
throw new ServerlessFunctionException(
`Function Version '${version}' does not exist`,
ServerlessFunctionExceptionCode.SERVERLESS_FUNCTION_NOT_FOUND,
);
}
throw error;
}
}
}
packages/twenty-server/src/engine/core-modules/serverless/drivers/local.driver.ts
+18 -17
View File
@@ -2,6 +2,8 @@ import { promises as fs } from 'fs';
import { spawn } from 'node:child_process';
import { join } from 'path';
import { FileFolder } from 'twenty-shared/types';
import {
type ServerlessDriver,
type ServerlessExecuteResult,
@@ -9,12 +11,10 @@ import {
import { type FileStorageService } from 'src/engine/core-modules/file-storage/file-storage.service';
import { SERVERLESS_TMPDIR_FOLDER } from 'src/engine/core-modules/serverless/drivers/constants/serverless-tmpdir-folder';
import { buildServerlessFunctionInMemory } from 'src/engine/core-modules/serverless/drivers/utils/build-serverless-function-in-memory';
import { copyAndBuildDependencies } from 'src/engine/core-modules/serverless/drivers/utils/copy-and-build-dependencies';
import { formatBuildError } from 'src/engine/core-modules/serverless/drivers/utils/format-build-error';
import { ConsoleListener } from 'src/engine/core-modules/serverless/drivers/utils/intercept-console';
import { LambdaBuildDirectoryManager } from 'src/engine/core-modules/serverless/drivers/utils/lambda-build-directory-manager';
import { getServerlessFolderOrThrow } from 'src/engine/core-modules/serverless/utils/serverless-get-folder.utils';
import { getServerlessFolderOrThrow } from 'src/engine/core-modules/serverless/utils/get-serverless-folder-or-throw.utils';
import { type FlatServerlessFunctionLayer } from 'src/engine/metadata-modules/serverless-function-layer/types/flat-serverless-function-layer.type';
import { ServerlessFunctionExecutionStatus } from 'src/engine/metadata-modules/serverless-function/dtos/serverless-function-execution-result.dto';
import { type FlatServerlessFunction } from 'src/engine/metadata-modules/serverless-function/types/flat-serverless-function.type';
@@ -78,9 +78,10 @@ export class LocalDriver implements ServerlessDriver {
const startTime = Date.now();
const folderPath = getServerlessFolderOrThrow({
const builtHandlerFolderPath = getServerlessFolderOrThrow({
flatServerlessFunction,
version,
fileFolder: FileFolder.BuiltFunction,
});
const lambdaBuildDirectoryManager = new LambdaBuildDirectoryManager();
@@ -89,21 +90,16 @@ export class LocalDriver implements ServerlessDriver {
const { sourceTemporaryDir } = await lambdaBuildDirectoryManager.init();
await this.fileStorageService.download({
from: { folderPath },
to: { folderPath: sourceTemporaryDir },
from: {
folderPath: builtHandlerFolderPath,
filename: flatServerlessFunction.builtHandlerPath,
},
to: {
folderPath: sourceTemporaryDir,
filename: flatServerlessFunction.builtHandlerPath,
},
});
let builtBundleFilePath = '';
try {
builtBundleFilePath = await buildServerlessFunctionInMemory({
sourceTemporaryDir,
handlerPath: flatServerlessFunction.handlerPath,
});
} catch (error) {
return formatBuildError(error, startTime);
}
try {
await fs.symlink(
join(
@@ -153,6 +149,11 @@ export class LocalDriver implements ServerlessDriver {
});
try {
const builtBundleFilePath = join(
sourceTemporaryDir,
flatServerlessFunction.builtHandlerPath,
);
const runnerPath = await this.writeBootstrapRunner({
dir: sourceTemporaryDir,
builtFileAbsPath: builtBundleFilePath,
packages/twenty-server/src/engine/core-modules/serverless/drivers/utils/build-and-upload-serverless-function.ts
+86
View File
@@ -0,0 +1,86 @@
import { join } from 'path';
import fs from 'fs/promises';
import { build } from 'esbuild';
import { FileFolder } from 'twenty-shared/types';
import { getServerlessFolderOrThrow } from 'src/engine/core-modules/serverless/utils/get-serverless-folder-or-throw.utils';
import { LambdaBuildDirectoryManager } from 'src/engine/core-modules/serverless/drivers/utils/lambda-build-directory-manager';
import { type FlatServerlessFunction } from 'src/engine/metadata-modules/serverless-function/types/flat-serverless-function.type';
import { type FileStorageService } from 'src/engine/core-modules/file-storage/file-storage.service';
const buildServerlessFunctionInMemory = async ({
sourceTemporaryDir,
handlerPath,
builtHandlerPath,
}: {
sourceTemporaryDir: string;
handlerPath: string;
builtHandlerPath: string;
}): Promise<string> => {
const entryFilePath = join(sourceTemporaryDir, handlerPath);
const builtBundleFilePath = join(sourceTemporaryDir, builtHandlerPath);
await build({
entryPoints: [entryFilePath],
outfile: builtBundleFilePath,
platform: 'node',
format: 'esm',
target: 'es2017',
bundle: true,
sourcemap: true,
packages: 'external',
});
return builtBundleFilePath;
};
export const buildAndUploadServerlessFunction = async ({
flatServerlessFunction,
version,
fileStorageService,
}: {
flatServerlessFunction: FlatServerlessFunction;
version: string;
fileStorageService: FileStorageService;
}): Promise<void> => {
const sourceFolderPath = getServerlessFolderOrThrow({
flatServerlessFunction,
version,
fileFolder: FileFolder.ServerlessFunction,
});
const builtFolderPath = getServerlessFolderOrThrow({
flatServerlessFunction,
version,
fileFolder: FileFolder.BuiltFunction,
});
const lambdaBuildDirectoryManager = new LambdaBuildDirectoryManager();
try {
const { sourceTemporaryDir } = await lambdaBuildDirectoryManager.init();
await fileStorageService.download({
from: { folderPath: sourceFolderPath },
to: { folderPath: sourceTemporaryDir },
});
const builtBundleFilePath = await buildServerlessFunctionInMemory({
sourceTemporaryDir,
handlerPath: flatServerlessFunction.handlerPath,
builtHandlerPath: flatServerlessFunction.builtHandlerPath,
});
const builtFile = await fs.readFile(builtBundleFilePath, 'utf-8');
await fileStorageService.write({
file: builtFile,
name: flatServerlessFunction.builtHandlerPath,
mimeType: 'application/javascript',
folder: builtFolderPath,
});
} finally {
await lambdaBuildDirectoryManager.clean();
}
};
packages/twenty-server/src/engine/core-modules/serverless/drivers/utils/build-serverless-function-in-memory.ts
-28
View File
@@ -1,28 +0,0 @@
import { join } from 'path';
import { build } from 'esbuild';
export const buildServerlessFunctionInMemory = async ({
sourceTemporaryDir,
handlerPath,
}: {
sourceTemporaryDir: string;
handlerPath: string;
}) => {
const entryFilePath = join(sourceTemporaryDir, handlerPath);
const builtBundleFilePath = join(sourceTemporaryDir, 'dist', 'main.mjs');
await build({
entryPoints: [entryFilePath],
outfile: builtBundleFilePath,
platform: 'node',
format: 'esm',
target: 'es2017',
bundle: true,
sourcemap: true,
packages: 'external',
});
return builtBundleFilePath;
};
packages/twenty-server/src/engine/core-modules/serverless/drivers/utils/format-build-error.ts
-20
View File
@@ -1,20 +0,0 @@
import { type BuildFailure } from 'esbuild';
import { type ServerlessExecuteResult } from 'src/engine/core-modules/serverless/drivers/interfaces/serverless-driver.interface';
import { ServerlessFunctionExecutionStatus } from 'src/engine/metadata-modules/serverless-function/dtos/serverless-function-execution-result.dto';
export const formatBuildError = (
error: BuildFailure,
startTime: number,
): ServerlessExecuteResult => ({
data: null,
logs: '',
duration: Date.now() - startTime,
error: {
errorType: 'BuildError',
errorMessage: error.errors.map((e) => e.text).join('\n') || 'Unknown error',
stackTrace: error.stack ? error.stack.split('\n') : [],
},
status: ServerlessFunctionExecutionStatus.ERROR,
});
packages/twenty-server/src/engine/core-modules/serverless/drivers/utils/get-base-typescript-project-files.ts → packages/twenty-server/src/engine/core-modules/serverless/drivers/utils/get-seed-project-files.ts
+5 -5
View File
@@ -16,7 +16,7 @@ const getAllFiles = async (
const fullPath = path.join(dir, entry.name);
if (entry.isDirectory()) {
return getAllFiles(rootDir, fullPath, files);
files.push(...(await getAllFiles(rootDir, fullPath, files)));
} else {
files.push({
path: path.relative(rootDir, dir),
@@ -29,11 +29,11 @@ const getAllFiles = async (
return files;
};
export const getBaseTypescriptProjectFiles = (async () => {
const baseTypescriptProjectPath = join(
export const getSeedProjectFiles = (async () => {
const seedProjectPath = join(
ASSET_PATH,
`engine/core-modules/serverless/drivers/constants/base-typescript-project`,
`engine/core-modules/serverless/drivers/constants/seed-project`,
);
return await getAllFiles(baseTypescriptProjectPath);
return await getAllFiles(seedProjectPath);
})();
packages/twenty-server/src/engine/core-modules/serverless/drivers/utils/is-serverless-function-built.ts
+23
View File
@@ -0,0 +1,23 @@
import type { FlatServerlessFunction } from 'src/engine/metadata-modules/serverless-function/types/flat-serverless-function.type';
import type { FileStorageService } from 'src/engine/core-modules/file-storage/file-storage.service';
import { getServerlessFolderOrThrow } from 'src/engine/core-modules/serverless/utils/get-serverless-folder-or-throw.utils';
export const isServerlessFunctionBuilt = async ({
flatServerlessFunction,
version,
fileStorageService,
}: {
flatServerlessFunction: FlatServerlessFunction;
version: string;
fileStorageService: FileStorageService;
}) => {
const folderPath = getServerlessFolderOrThrow({
flatServerlessFunction,
version,
});
return await fileStorageService.checkFileExists({
folderPath,
filename: flatServerlessFunction.builtHandlerPath,
});
};
packages/twenty-server/src/engine/core-modules/serverless/utils/serverless-get-folder.utils.ts → packages/twenty-server/src/engine/core-modules/serverless/utils/get-serverless-folder-or-throw.utils.ts
+6 -5
View File
@@ -12,11 +12,14 @@ import { type FlatServerlessFunction } from 'src/engine/metadata-modules/serverl
export const getServerlessFolderOrThrow = ({
flatServerlessFunction,
version,
toDelete = false,
fileFolder = FileFolder.ServerlessFunction,
}: {
flatServerlessFunction: FlatServerlessFunction;
version?: 'draft' | 'latest' | (string & NonNullable<unknown>);
toDelete?: boolean;
fileFolder?:
| FileFolder.ServerlessFunction
| FileFolder.ServerlessFunctionToDelete
| FileFolder.BuiltFunction;
}) => {
if (
version === 'latest' &&
@@ -33,9 +36,7 @@ export const getServerlessFolderOrThrow = ({
return join(
'workspace-' + flatServerlessFunction.workspaceId,
toDelete
? FileFolder.ServerlessFunctionToDelete
: FileFolder.ServerlessFunction,
fileFolder,
flatServerlessFunction.id,
computedVersion || '',
);
packages/twenty-server/src/engine/metadata-modules/serverless-function/constants/default-tool-input-schema.constant.ts
+1 -1
View File
@@ -1,4 +1,4 @@
// Default tool input schema matching the base-typescript-project template
// Default tool input schema matching the seed-project template
// Template params: { a: string; b: number; }
export const DEFAULT_TOOL_INPUT_SCHEMA = {
type: 'object',
packages/twenty-server/src/engine/metadata-modules/serverless-function/dtos/create-serverless-function.input.ts
+5
View File
@@ -56,6 +56,11 @@ export class CreateServerlessFunctionInput {
@IsOptional()
handlerPath?: string;
@IsString()
@Field({ nullable: true })
@IsOptional()
builtHandlerPath?: string;
@Field(() => graphqlTypeJson, { nullable: true })
@IsObject()
@IsOptional()
packages/twenty-server/src/engine/metadata-modules/serverless-function/dtos/serverless-function.dto.ts
+4
View File
@@ -65,6 +65,10 @@ export class ServerlessFunctionDTO {
@Field()
handlerPath: string;
@IsString()
@Field()
builtHandlerPath: string;
@IsString()
@Field()
handlerName: string;
packages/twenty-server/src/engine/metadata-modules/serverless-function/serverless-function.entity.ts
+4
View File
@@ -27,6 +27,7 @@ export enum ServerlessFunctionRuntime {
}
export const DEFAULT_HANDLER_PATH = 'src/index.ts';
export const DEFAULT_BUILT_HANDLER_PATH = 'index.mjs';
export const DEFAULT_HANDLER_NAME = 'main';
@Entity('serverlessFunction')
@@ -45,6 +46,9 @@ export class ServerlessFunctionEntity
@Column({ nullable: false, default: DEFAULT_HANDLER_PATH })
handlerPath: string;
@Column({ nullable: false, default: DEFAULT_BUILT_HANDLER_PATH })
builtHandlerPath: string;
@Column({ nullable: false, default: DEFAULT_HANDLER_NAME })
handlerName: string;
packages/twenty-server/src/engine/metadata-modules/serverless-function/serverless-function.service.ts
+42 -5
View File
@@ -8,6 +8,7 @@ import {
} from 'twenty-shared/application';
import { isDefined } from 'twenty-shared/utils';
import { Repository } from 'typeorm';
import { FileFolder } from 'twenty-shared/types';
import { FileStorageExceptionCode } from 'src/engine/core-modules/file-storage/interfaces/file-storage-exception';
import { type ServerlessExecuteResult } from 'src/engine/core-modules/serverless/drivers/interfaces/serverless-driver.interface';
@@ -19,7 +20,7 @@ import { ApplicationTokenService } from 'src/engine/core-modules/auth/token/serv
import { FileStorageService } from 'src/engine/core-modules/file-storage/file-storage.service';
import { buildEnvVar } from 'src/engine/core-modules/serverless/drivers/utils/build-env-var';
import { ServerlessService } from 'src/engine/core-modules/serverless/serverless.service';
import { getServerlessFolderOrThrow } from 'src/engine/core-modules/serverless/utils/serverless-get-folder.utils';
import { getServerlessFolderOrThrow } from 'src/engine/core-modules/serverless/utils/get-serverless-folder-or-throw.utils';
import { ThrottlerService } from 'src/engine/core-modules/throttler/throttler.service';
import { TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service';
import { WorkspaceManyOrAllFlatEntityMapsCacheService } from 'src/engine/metadata-modules/flat-entity/services/workspace-many-or-all-flat-entity-maps-cache.service';
@@ -47,6 +48,8 @@ import {
WorkflowVersionStepExceptionCode,
} from 'src/modules/workflow/common/exceptions/workflow-version-step.exception';
import { cleanServerUrl } from 'src/utils/clean-server-url';
import { isServerlessFunctionBuilt } from 'src/engine/core-modules/serverless/drivers/utils/is-serverless-function-built';
import { buildAndUploadServerlessFunction } from 'src/engine/core-modules/serverless/drivers/utils/build-and-upload-serverless-function';
const MIN_TOKEN_EXPIRATION_IN_SECONDS = 5;
@@ -205,6 +208,21 @@ export class ServerlessFunctionService {
...buildEnvVar(flatApplicationVariables),
};
// We keep that check to build functions
if (
!(await isServerlessFunctionBuilt({
flatServerlessFunction,
version,
fileStorageService: this.fileStorageService,
}))
) {
await buildAndUploadServerlessFunction({
flatServerlessFunction,
version,
fileStorageService: this.fileStorageService,
});
}
const resultServerlessFunction = await this.callWithTimeout({
callback: () =>
this.serverlessService.execute({
@@ -299,19 +317,38 @@ export class ServerlessFunctionService {
? `${parseInt(existingFlatServerlessFunction.latestVersion, 10) + 1}`
: '1';
const draftFolderPath = getServerlessFolderOrThrow({
const draftSourceFolderPath = getServerlessFolderOrThrow({
flatServerlessFunction: existingFlatServerlessFunction,
version: 'draft',
fileFolder: FileFolder.ServerlessFunction,
});
const newFolderPath = getServerlessFolderOrThrow({
const newSourceFolderPath = getServerlessFolderOrThrow({
flatServerlessFunction: existingFlatServerlessFunction,
version: newVersion,
fileFolder: FileFolder.ServerlessFunction,
});
await this.fileStorageService.copy({
from: { folderPath: draftFolderPath },
to: { folderPath: newFolderPath },
from: { folderPath: draftSourceFolderPath },
to: { folderPath: newSourceFolderPath },
});
const draftBuiltFolderPath = getServerlessFolderOrThrow({
flatServerlessFunction: existingFlatServerlessFunction,
version: 'draft',
fileFolder: FileFolder.BuiltFunction,
});
const newBuiltFolderPath = getServerlessFolderOrThrow({
flatServerlessFunction: existingFlatServerlessFunction,
version: newVersion,
fileFolder: FileFolder.BuiltFunction,
});
await this.fileStorageService.copy({
from: { folderPath: draftBuiltFolderPath },
to: { folderPath: newBuiltFolderPath },
});
const newPublishedVersions = [
packages/twenty-server/src/engine/metadata-modules/serverless-function/utils/from-create-serverless-function-input-to-flat-serverless-function.util.ts
+4
View File
@@ -4,6 +4,7 @@ import { v4 } from 'uuid';
import { DEFAULT_TOOL_INPUT_SCHEMA } from 'src/engine/metadata-modules/serverless-function/constants/default-tool-input-schema.constant';
import { type CreateServerlessFunctionInput } from 'src/engine/metadata-modules/serverless-function/dtos/create-serverless-function.input';
import {
DEFAULT_BUILT_HANDLER_PATH,
DEFAULT_HANDLER_NAME,
DEFAULT_HANDLER_PATH,
ServerlessFunctionRuntime,
@@ -38,6 +39,9 @@ export const fromCreateServerlessFunctionInputToFlatServerlessFunction = ({
rawCreateServerlessFunctionInput.handlerPath ?? DEFAULT_HANDLER_PATH,
handlerName:
rawCreateServerlessFunctionInput.handlerName ?? DEFAULT_HANDLER_NAME,
builtHandlerPath:
rawCreateServerlessFunctionInput.builtHandlerPath ??
DEFAULT_BUILT_HANDLER_PATH,
universalIdentifier:
rawCreateServerlessFunctionInput.universalIdentifier ?? v4(),
createdAt: currentDate.toISOString(),
packages/twenty-server/src/engine/metadata-modules/serverless-function/utils/from-flat-serverless-function-to-serverless-function-dto.util.ts
+1
View File
@@ -44,6 +44,7 @@ export const fromFlatServerlessFunctionToServerlessFunctionDto = ({
timeoutSeconds: flatServerlessFunction.timeoutSeconds,
latestVersion: flatServerlessFunction.latestVersion ?? undefined,
handlerPath: flatServerlessFunction.handlerPath,
builtHandlerPath: flatServerlessFunction.builtHandlerPath,
handlerName: flatServerlessFunction.handlerName,
publishedVersions: flatServerlessFunction.publishedVersions,
toolInputSchema: flatServerlessFunction.toolInputSchema ?? undefined,
packages/twenty-server/src/engine/workspace-manager/workspace-migration/workspace-migration-runner/action-handlers/serverless-function/services/create-serverless-function-action-handler.service.ts
+17 -4
View File
@@ -7,11 +7,13 @@ import { isDefined } from 'twenty-shared/utils';
import { WorkspaceMigrationRunnerActionHandler } from 'src/engine/workspace-manager/workspace-migration/workspace-migration-runner/interfaces/workspace-migration-runner-action-handler-service.interface';
import { FileStorageService } from 'src/engine/core-modules/file-storage/file-storage.service';
import { getBaseTypescriptProjectFiles } from 'src/engine/core-modules/serverless/drivers/utils/get-base-typescript-project-files';
import { getServerlessFolderOrThrow } from 'src/engine/core-modules/serverless/utils/serverless-get-folder.utils';
import { getSeedProjectFiles } from 'src/engine/core-modules/serverless/drivers/utils/get-seed-project-files';
import { getServerlessFolderOrThrow } from 'src/engine/core-modules/serverless/utils/get-serverless-folder-or-throw.utils';
import { ServerlessFunctionEntity } from 'src/engine/metadata-modules/serverless-function/serverless-function.entity';
import { CreateServerlessFunctionAction } from 'src/engine/workspace-manager/workspace-migration/workspace-migration-builder/builders/serverless-function/types/workspace-migration-serverless-function-action.type';
import { WorkspaceMigrationActionRunnerArgs } from 'src/engine/workspace-manager/workspace-migration/workspace-migration-runner/types/workspace-migration-action-runner-args.type';
import { FlatServerlessFunction } from 'src/engine/metadata-modules/serverless-function/types/flat-serverless-function.type';
import { buildAndUploadServerlessFunction } from 'src/engine/core-modules/serverless/drivers/utils/build-and-upload-serverless-function';
@Injectable()
export class CreateServerlessFunctionActionHandlerService extends WorkspaceMigrationRunnerActionHandler(
@@ -28,6 +30,8 @@ export class CreateServerlessFunctionActionHandlerService extends WorkspaceMigra
const { action, queryRunner, workspaceId } = context;
const { flatEntity: serverlessFunction } = action;
await this.buildAndSaveServerlessFunction(serverlessFunction);
const serverlessFunctionRepository =
queryRunner.manager.getRepository<ServerlessFunctionEntity>(
ServerlessFunctionEntity,
@@ -37,7 +41,11 @@ export class CreateServerlessFunctionActionHandlerService extends WorkspaceMigra
...serverlessFunction,
workspaceId,
});
}
private async buildAndSaveServerlessFunction(
serverlessFunction: FlatServerlessFunction,
) {
const draftFileFolder = getServerlessFolderOrThrow({
flatServerlessFunction: serverlessFunction,
version: 'draft',
@@ -49,15 +57,20 @@ export class CreateServerlessFunctionActionHandlerService extends WorkspaceMigra
draftFileFolder,
);
} else {
for (const file of await getBaseTypescriptProjectFiles) {
for (const file of await getSeedProjectFiles) {
await this.fileStorageService.write({
file: file.content,
name: file.name,
mimeType: undefined,
mimeType: 'application/typescript',
folder: join(draftFileFolder, file.path),
});
}
}
await buildAndUploadServerlessFunction({
flatServerlessFunction: serverlessFunction,
version: 'draft',
fileStorageService: this.fileStorageService,
});
}
async rollbackForMetadata(
packages/twenty-server/src/engine/workspace-manager/workspace-migration/workspace-migration-runner/action-handlers/serverless-function/services/delete-serverless-function-action-handler.service.ts
+15 -4
View File
@@ -1,9 +1,11 @@
import { Injectable } from '@nestjs/common';
import { FileFolder } from 'twenty-shared/types';
import { WorkspaceMigrationRunnerActionHandler } from 'src/engine/workspace-manager/workspace-migration/workspace-migration-runner/interfaces/workspace-migration-runner-action-handler-service.interface';
import { FileStorageService } from 'src/engine/core-modules/file-storage/file-storage.service';
import { getServerlessFolderOrThrow } from 'src/engine/core-modules/serverless/utils/serverless-get-folder.utils';
import { getServerlessFolderOrThrow } from 'src/engine/core-modules/serverless/utils/get-serverless-folder-or-throw.utils';
import { findFlatEntityByUniversalIdentifierOrThrow } from 'src/engine/metadata-modules/flat-entity/utils/find-flat-entity-by-universal-identifier-or-throw.util';
import { ServerlessFunctionEntity } from 'src/engine/metadata-modules/serverless-function/serverless-function.entity';
import { DeleteServerlessFunctionAction } from 'src/engine/workspace-manager/workspace-migration/workspace-migration-builder/builders/serverless-function/types/workspace-migration-serverless-function-action.type';
@@ -44,15 +46,24 @@ export class DeleteServerlessFunctionActionHandlerService extends WorkspaceMigra
from: {
folderPath: getServerlessFolderOrThrow({
flatServerlessFunction,
fileFolder: FileFolder.ServerlessFunction,
}),
},
to: {
folderPath: getServerlessFolderOrThrow({
flatServerlessFunction,
toDelete: true,
fileFolder: FileFolder.ServerlessFunctionToDelete,
}),
},
});
// We can delete built code as it can be computed from source code if rollback occurs
await this.fileStorageService.delete({
folderPath: getServerlessFolderOrThrow({
flatServerlessFunction,
fileFolder: FileFolder.BuiltFunction,
}),
});
}
async rollbackForMetadata(
@@ -70,13 +81,13 @@ export class DeleteServerlessFunctionActionHandlerService extends WorkspaceMigra
from: {
folderPath: getServerlessFolderOrThrow({
flatServerlessFunction,
toDelete: true,
fileFolder: FileFolder.ServerlessFunctionToDelete,
}),
},
to: {
folderPath: getServerlessFolderOrThrow({
flatServerlessFunction,
toDelete: false,
fileFolder: FileFolder.ServerlessFunction,
}),
},
});
packages/twenty-server/src/engine/workspace-manager/workspace-migration/workspace-migration-runner/action-handlers/serverless-function/services/update-serverless-function-action-handler.service.ts
+8 -1
View File
@@ -7,13 +7,14 @@ import { WorkspaceMigrationRunnerActionHandler } from 'src/engine/workspace-mana
import { FileStorageService } from 'src/engine/core-modules/file-storage/file-storage.service';
import { ServerlessService } from 'src/engine/core-modules/serverless/serverless.service';
import { getServerlessFolderOrThrow } from 'src/engine/core-modules/serverless/utils/serverless-get-folder.utils';
import { getServerlessFolderOrThrow } from 'src/engine/core-modules/serverless/utils/get-serverless-folder-or-throw.utils';
import { findFlatEntityByIdInFlatEntityMapsOrThrow } from 'src/engine/metadata-modules/flat-entity/utils/find-flat-entity-by-id-in-flat-entity-maps-or-throw.util';
import { ServerlessFunctionEntity } from 'src/engine/metadata-modules/serverless-function/serverless-function.entity';
import { FlatServerlessFunction } from 'src/engine/metadata-modules/serverless-function/types/flat-serverless-function.type';
import { UpdateServerlessFunctionAction } from 'src/engine/workspace-manager/workspace-migration/workspace-migration-builder/builders/serverless-function/types/workspace-migration-serverless-function-action.type';
import { WorkspaceMigrationActionRunnerArgs } from 'src/engine/workspace-manager/workspace-migration/workspace-migration-runner/types/workspace-migration-action-runner-args.type';
import { fromFlatEntityPropertiesUpdatesToPartialFlatEntity } from 'src/engine/workspace-manager/workspace-migration/workspace-migration-runner/utils/from-flat-entity-properties-updates-to-partial-flat-entity';
import { buildAndUploadServerlessFunction } from 'src/engine/core-modules/serverless/drivers/utils/build-and-upload-serverless-function';
@Injectable()
export class UpdateServerlessFunctionActionHandlerService extends WorkspaceMigrationRunnerActionHandler(
@@ -84,5 +85,11 @@ export class UpdateServerlessFunctionActionHandlerService extends WorkspaceMigra
});
await this.fileStorageService.writeFolder(code, fileFolder);
await buildAndUploadServerlessFunction({
flatServerlessFunction,
version: 'draft',
fileStorageService: this.fileStorageService,
});
}
}
packages/twenty-server/src/modules/workflow/workflow-builder/workflow-version-step/__tests__/workflow-version-step-operations.workspace-service.spec.ts
+2
View File
@@ -252,6 +252,7 @@ describe('WorkflowVersionStepOperationsWorkspaceService', () => {
runtime: ServerlessFunctionRuntime.NODE22,
timeoutSeconds: 30,
handlerPath: 'src/index.ts',
builtHandlerPath: 'index.mjs',
handlerName: 'main',
checksum: null,
toolInputSchema: null,
@@ -331,6 +332,7 @@ describe('WorkflowVersionStepOperationsWorkspaceService', () => {
runtime: ServerlessFunctionRuntime.NODE22,
timeoutSeconds: 30,
handlerPath: 'src/index.ts',
builtHandlerPath: 'index.mjs',
handlerName: 'main',
checksum: null,
toolInputSchema: null,
packages/twenty-server/src/modules/workflow/workflow-builder/workflow-version-step/workflow-version-step-operations.workspace-service.ts
+2 -2
View File
@@ -15,7 +15,7 @@ import { Repository } from 'typeorm';
import { v4 } from 'uuid';
import { getFlatFieldsFromFlatObjectMetadata } from 'src/engine/api/graphql/workspace-schema-builder/utils/get-flat-fields-for-flat-object-metadata.util';
import { BASE_TYPESCRIPT_PROJECT_INPUT_SCHEMA } from 'src/engine/core-modules/serverless/drivers/constants/base-typescript-project-input-schema';
import { SEED_PROJECT_INPUT_SCHEMA } from 'src/engine/core-modules/serverless/drivers/constants/seed-project-input-schema';
import { type WorkflowStepPositionInput } from 'src/engine/core-modules/workflow/dtos/update-workflow-step-position-input.dto';
import { AiAgentRoleService } from 'src/engine/metadata-modules/ai/ai-agent-role/ai-agent-role.service';
import { AgentEntity } from 'src/engine/metadata-modules/ai/ai-agent/entities/agent.entity';
@@ -189,7 +189,7 @@ export class WorkflowVersionStepOperationsWorkspaceService {
input: {
serverlessFunctionId: newServerlessFunction.id,
serverlessFunctionVersion: 'draft',
serverlessFunctionInput: BASE_TYPESCRIPT_PROJECT_INPUT_SCHEMA,
serverlessFunctionInput: SEED_PROJECT_INPUT_SCHEMA,
},
},
},
packages/twenty-shared/src/application/serverlessFunctionManifestType.ts
+1
View File
@@ -25,6 +25,7 @@ export type ServerlessFunctionManifest = SyncableEntityOptions & {
timeoutSeconds?: number;
triggers: ServerlessFunctionTriggerManifest[];
handlerPath: string;
builtHandlerPath?: string; // Should be required when build mode implemented
handlerName: string;
toolInputSchema?: InputJsonSchema;
isTool?: boolean;
packages/twenty-shared/src/types/FileFolder.ts
+4 -4
View File
@@ -7,8 +7,8 @@ export enum FileFolder {
ServerlessFunctionToDelete = 'serverless-function-to-delete',
File = 'file',
AgentChat = 'agent-chat',
Functions = 'functions',
FrontComponents = 'front-components',
Assets = 'assets',
SourceCode = 'source-code',
BuiltFunction = 'built-function',
BuiltFrontComponent = 'built-front-component',
Asset = 'asset',
Source = 'source',
}