diff --git a/packages/twenty-server/src/engine/core-modules/serverless/drivers/lambda.driver.ts b/packages/twenty-server/src/engine/core-modules/serverless/drivers/lambda.driver.ts index c6ddd31eb3..cccbf0fad3 100644 --- a/packages/twenty-server/src/engine/core-modules/serverless/drivers/lambda.driver.ts +++ b/packages/twenty-server/src/engine/core-modules/serverless/drivers/lambda.driver.ts @@ -27,9 +27,12 @@ import { } from 'src/engine/core-modules/serverless/drivers/interfaces/serverless-driver.interface'; import { type FileStorageService } from 'src/engine/core-modules/file-storage/file-storage.service'; +import { buildEnvVar } from 'src/engine/core-modules/serverless/drivers/utils/build-env-var'; +import { buildServerlessFunctionInMemory } from 'src/engine/core-modules/serverless/drivers/utils/build-serverless-function-in-memory'; import { copyAndBuildDependencies } from 'src/engine/core-modules/serverless/drivers/utils/copy-and-build-dependencies'; import { copyExecutor } from 'src/engine/core-modules/serverless/drivers/utils/copy-executor'; import { createZipFile } from 'src/engine/core-modules/serverless/drivers/utils/create-zip-file'; +import { formatBuildError } from 'src/engine/core-modules/serverless/drivers/utils/format-build-error'; import { LambdaBuildDirectoryManager, NODE_LAYER_SUBFOLDER, @@ -44,9 +47,6 @@ import { ServerlessFunctionException, ServerlessFunctionExceptionCode, } from 'src/engine/metadata-modules/serverless-function/serverless-function.exception'; -import { buildServerlessFunctionInMemory } from 'src/engine/core-modules/serverless/drivers/utils/build-serverless-function-in-memory'; -import { formatBuildError } from 'src/engine/core-modules/serverless/drivers/utils/format-build-error'; -import { buildEnvVar } from 'src/engine/core-modules/serverless/drivers/utils/build-env-var'; const UPDATE_FUNCTION_DURATION_TIMEOUT_IN_SECONDS = 60; const CREDENTIALS_DURATION_IN_SECONDS = 60 * 60; // 1h @@ -280,7 +280,7 @@ export class LambdaDriver implements ServerlessDriver { Handler: 'index.handler', Role: this.options.lambdaRole, Runtime: serverlessFunction.runtime, - Timeout: serverlessFunction.timeoutSeconds, + Timeout: 900, // timeout is handled by the serverless function service }; const command = new CreateFunctionCommand(params); diff --git a/packages/twenty-server/src/engine/core-modules/serverless/drivers/local.driver.ts b/packages/twenty-server/src/engine/core-modules/serverless/drivers/local.driver.ts index bcd4bd82e1..b9d1c2eeec 100644 --- a/packages/twenty-server/src/engine/core-modules/serverless/drivers/local.driver.ts +++ b/packages/twenty-server/src/engine/core-modules/serverless/drivers/local.driver.ts @@ -1,6 +1,6 @@ import { promises as fs } from 'fs'; -import { join } from 'path'; import { spawn } from 'node:child_process'; +import { join } from 'path'; import { type ServerlessDriver, @@ -9,15 +9,15 @@ import { import { type FileStorageService } from 'src/engine/core-modules/file-storage/file-storage.service'; import { SERVERLESS_TMPDIR_FOLDER } from 'src/engine/core-modules/serverless/drivers/constants/serverless-tmpdir-folder'; +import { buildEnvVar } from 'src/engine/core-modules/serverless/drivers/utils/build-env-var'; +import { buildServerlessFunctionInMemory } from 'src/engine/core-modules/serverless/drivers/utils/build-serverless-function-in-memory'; import { copyAndBuildDependencies } from 'src/engine/core-modules/serverless/drivers/utils/copy-and-build-dependencies'; +import { formatBuildError } from 'src/engine/core-modules/serverless/drivers/utils/format-build-error'; import { ConsoleListener } from 'src/engine/core-modules/serverless/drivers/utils/intercept-console'; +import { LambdaBuildDirectoryManager } from 'src/engine/core-modules/serverless/drivers/utils/lambda-build-directory-manager'; import { getServerlessFolder } from 'src/engine/core-modules/serverless/utils/serverless-get-folder.utils'; import { ServerlessFunctionExecutionStatus } from 'src/engine/metadata-modules/serverless-function/dtos/serverless-function-execution-result.dto'; import { type ServerlessFunctionEntity } from 'src/engine/metadata-modules/serverless-function/serverless-function.entity'; -import { LambdaBuildDirectoryManager } from 'src/engine/core-modules/serverless/drivers/utils/lambda-build-directory-manager'; -import { buildServerlessFunctionInMemory } from 'src/engine/core-modules/serverless/drivers/utils/build-serverless-function-in-memory'; -import { formatBuildError } from 'src/engine/core-modules/serverless/drivers/utils/format-build-error'; -import { buildEnvVar } from 'src/engine/core-modules/serverless/drivers/utils/build-env-var'; export interface LocalDriverOptions { fileStorageService: FileStorageService; @@ -156,7 +156,7 @@ export class LocalDriver implements ServerlessDriver { runnerPath, env: buildEnvVar(serverlessFunction), payload, - timeoutMs: serverlessFunction.timeoutSeconds * 1_000, + timeoutMs: 900_000, // timeout is handled by the serverless function service }); if (stdout) @@ -217,7 +217,7 @@ export class LocalDriver implements ServerlessDriver { const code = ` // Auto-generated. Do not edit. const { pathToFileURL } = require('node:url'); - + (async () => { try { const builtUrl = pathToFileURL(${JSON.stringify(builtFileAbsPath)}); @@ -225,7 +225,7 @@ export class LocalDriver implements ServerlessDriver { if (typeof mod.${handlerName} !== 'function') { throw new Error('Export "${handlerName}" not found in serverless bundle'); } - + let payload = undefined; if (process.send) { process.on('message', async (msg) => { diff --git a/packages/twenty-server/src/engine/metadata-modules/serverless-function/serverless-function.exception.ts b/packages/twenty-server/src/engine/metadata-modules/serverless-function/serverless-function.exception.ts index 34a8d0e784..01286903c0 100644 --- a/packages/twenty-server/src/engine/metadata-modules/serverless-function/serverless-function.exception.ts +++ b/packages/twenty-server/src/engine/metadata-modules/serverless-function/serverless-function.exception.ts @@ -11,4 +11,5 @@ export enum ServerlessFunctionExceptionCode { SERVERLESS_FUNCTION_CODE_UNCHANGED = 'SERVERLESS_FUNCTION_CODE_UNCHANGED', SERVERLESS_FUNCTION_EXECUTION_LIMIT_REACHED = 'SERVERLESS_FUNCTION_EXECUTION_LIMIT_REACHED', SERVERLESS_FUNCTION_CREATE_FAILED = 'SERVERLESS_FUNCTION_CREATE_FAILED', + SERVERLESS_FUNCTION_EXECUTION_TIMEOUT = 'SERVERLESS_FUNCTION_EXECUTION_TIMEOUT', } diff --git a/packages/twenty-server/src/engine/metadata-modules/serverless-function/serverless-function.service.ts b/packages/twenty-server/src/engine/metadata-modules/serverless-function/serverless-function.service.ts index fae6eefe24..8349f44e82 100644 --- a/packages/twenty-server/src/engine/metadata-modules/serverless-function/serverless-function.service.ts +++ b/packages/twenty-server/src/engine/metadata-modules/serverless-function/serverless-function.service.ts @@ -13,11 +13,14 @@ import { type ServerlessExecuteResult } from 'src/engine/core-modules/serverless import { AuditService } from 'src/engine/core-modules/audit/services/audit.service'; import { SERVERLESS_FUNCTION_EXECUTED_EVENT } from 'src/engine/core-modules/audit/utils/events/workspace-event/serverless-function/serverless-function-executed'; import { FileStorageService } from 'src/engine/core-modules/file-storage/file-storage.service'; +import { Sources } from 'src/engine/core-modules/file-storage/types/source.type'; import { getBaseTypescriptProjectFiles } from 'src/engine/core-modules/serverless/drivers/utils/get-base-typescript-project-files'; import { ServerlessService } from 'src/engine/core-modules/serverless/serverless.service'; import { getServerlessFolder } from 'src/engine/core-modules/serverless/utils/serverless-get-folder.utils'; import { ThrottlerService } from 'src/engine/core-modules/throttler/throttler.service'; import { TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service'; +import { ServerlessFunctionLayerService } from 'src/engine/metadata-modules/serverless-function-layer/serverless-function-layer.service'; +import { CreateServerlessFunctionInput } from 'src/engine/metadata-modules/serverless-function/dtos/create-serverless-function.input'; import { type UpdateServerlessFunctionInput } from 'src/engine/metadata-modules/serverless-function/dtos/update-serverless-function.input'; import { ServerlessFunctionEntity } from 'src/engine/metadata-modules/serverless-function/serverless-function.entity'; import { @@ -28,9 +31,6 @@ import { WorkflowVersionStepException, WorkflowVersionStepExceptionCode, } from 'src/modules/workflow/common/exceptions/workflow-version-step.exception'; -import { ServerlessFunctionLayerService } from 'src/engine/metadata-modules/serverless-function-layer/serverless-function-layer.service'; -import { Sources } from 'src/engine/core-modules/file-storage/types/source.type'; -import { CreateServerlessFunctionInput } from 'src/engine/metadata-modules/serverless-function/dtos/create-serverless-function.input'; @Injectable() export class ServerlessFunctionService { @@ -102,11 +102,11 @@ export class ServerlessFunctionService { ], }); - const resultServerlessFunction = await this.serverlessService.execute( - functionToExecute, - payload, - version, - ); + const resultServerlessFunction = await this.callWithTimeout({ + callback: () => + this.serverlessService.execute(functionToExecute, payload, version), + timeoutMs: functionToExecute.timeoutSeconds * 1000, + }); if (this.twentyConfigService.get('SERVERLESS_LOGS_ENABLED')) { /* eslint-disable no-console */ @@ -470,4 +470,31 @@ export class ServerlessFunctionService { ); } } + + private async callWithTimeout({ + callback, + timeoutMs, + }: { + callback: () => Promise; + timeoutMs: number; + }): Promise { + let timeoutId: NodeJS.Timeout; + + const timeoutPromise = new Promise((_, reject) => { + timeoutId = setTimeout( + () => + reject( + new ServerlessFunctionException( + `Execution timeout: ${timeoutMs / 1000}s`, + ServerlessFunctionExceptionCode.SERVERLESS_FUNCTION_EXECUTION_TIMEOUT, + ), + ), + timeoutMs, + ); + }); + + return Promise.race([callback(), timeoutPromise]).finally(() => + clearTimeout(timeoutId), + ) as Promise; + } } diff --git a/packages/twenty-server/src/engine/metadata-modules/serverless-function/utils/serverless-function-graphql-api-exception-handler.utils.ts b/packages/twenty-server/src/engine/metadata-modules/serverless-function/utils/serverless-function-graphql-api-exception-handler.utils.ts index a94adda8b6..5448fe0964 100644 --- a/packages/twenty-server/src/engine/metadata-modules/serverless-function/utils/serverless-function-graphql-api-exception-handler.utils.ts +++ b/packages/twenty-server/src/engine/metadata-modules/serverless-function/utils/serverless-function-graphql-api-exception-handler.utils.ts @@ -4,6 +4,7 @@ import { ConflictError, ForbiddenError, NotFoundError, + TimeoutError, } from 'src/engine/core-modules/graphql/utils/graphql-errors.util'; import { ServerlessFunctionException, @@ -23,6 +24,8 @@ export const serverlessFunctionGraphQLApiExceptionHandler = (error: any) => { case ServerlessFunctionExceptionCode.SERVERLESS_FUNCTION_BUILDING: case ServerlessFunctionExceptionCode.SERVERLESS_FUNCTION_EXECUTION_LIMIT_REACHED: throw new ForbiddenError(error); + case ServerlessFunctionExceptionCode.SERVERLESS_FUNCTION_EXECUTION_TIMEOUT: + throw new TimeoutError(error); case ServerlessFunctionExceptionCode.SERVERLESS_FUNCTION_CODE_UNCHANGED: case ServerlessFunctionExceptionCode.SERVERLESS_FUNCTION_CREATE_FAILED: throw error;