Set lambda timeout in service (#15632)

Some functions keep running without a timeout being thrown. Doing it in service directly.
2025-11-05 13:43:23 +01:00
parent c294af2944
commit f21b862d52
5 changed files with 51 additions and 20 deletions
@@ -27,9 +27,12 @@ import {
 } from 'src/engine/core-modules/serverless/drivers/interfaces/serverless-driver.interface';

 import { type FileStorageService } from 'src/engine/core-modules/file-storage/file-storage.service';
+import { buildEnvVar } from 'src/engine/core-modules/serverless/drivers/utils/build-env-var';
+import { buildServerlessFunctionInMemory } from 'src/engine/core-modules/serverless/drivers/utils/build-serverless-function-in-memory';
 import { copyAndBuildDependencies } from 'src/engine/core-modules/serverless/drivers/utils/copy-and-build-dependencies';
 import { copyExecutor } from 'src/engine/core-modules/serverless/drivers/utils/copy-executor';
 import { createZipFile } from 'src/engine/core-modules/serverless/drivers/utils/create-zip-file';
+import { formatBuildError } from 'src/engine/core-modules/serverless/drivers/utils/format-build-error';
 import {
  LambdaBuildDirectoryManager,
  NODE_LAYER_SUBFOLDER,
@@ -44,9 +47,6 @@ import {
  ServerlessFunctionException,
  ServerlessFunctionExceptionCode,
 } from 'src/engine/metadata-modules/serverless-function/serverless-function.exception';
-import { buildServerlessFunctionInMemory } from 'src/engine/core-modules/serverless/drivers/utils/build-serverless-function-in-memory';
-import { formatBuildError } from 'src/engine/core-modules/serverless/drivers/utils/format-build-error';
-import { buildEnvVar } from 'src/engine/core-modules/serverless/drivers/utils/build-env-var';

 const UPDATE_FUNCTION_DURATION_TIMEOUT_IN_SECONDS = 60;
 const CREDENTIALS_DURATION_IN_SECONDS = 60 * 60; // 1h
@@ -280,7 +280,7 @@ export class LambdaDriver implements ServerlessDriver {
      Handler: 'index.handler',
      Role: this.options.lambdaRole,
      Runtime: serverlessFunction.runtime,
-      Timeout: serverlessFunction.timeoutSeconds,
+      Timeout: 900, // timeout is handled by the serverless function service
    };

    const command = new CreateFunctionCommand(params);
@@ -1,6 +1,6 @@
 import { promises as fs } from 'fs';
-import { join } from 'path';
 import { spawn } from 'node:child_process';
+import { join } from 'path';

 import {
  type ServerlessDriver,
@@ -9,15 +9,15 @@ import {

 import { type FileStorageService } from 'src/engine/core-modules/file-storage/file-storage.service';
 import { SERVERLESS_TMPDIR_FOLDER } from 'src/engine/core-modules/serverless/drivers/constants/serverless-tmpdir-folder';
+import { buildEnvVar } from 'src/engine/core-modules/serverless/drivers/utils/build-env-var';
+import { buildServerlessFunctionInMemory } from 'src/engine/core-modules/serverless/drivers/utils/build-serverless-function-in-memory';
 import { copyAndBuildDependencies } from 'src/engine/core-modules/serverless/drivers/utils/copy-and-build-dependencies';
+import { formatBuildError } from 'src/engine/core-modules/serverless/drivers/utils/format-build-error';
 import { ConsoleListener } from 'src/engine/core-modules/serverless/drivers/utils/intercept-console';
+import { LambdaBuildDirectoryManager } from 'src/engine/core-modules/serverless/drivers/utils/lambda-build-directory-manager';
 import { getServerlessFolder } from 'src/engine/core-modules/serverless/utils/serverless-get-folder.utils';
 import { ServerlessFunctionExecutionStatus } from 'src/engine/metadata-modules/serverless-function/dtos/serverless-function-execution-result.dto';
 import { type ServerlessFunctionEntity } from 'src/engine/metadata-modules/serverless-function/serverless-function.entity';
-import { LambdaBuildDirectoryManager } from 'src/engine/core-modules/serverless/drivers/utils/lambda-build-directory-manager';
-import { buildServerlessFunctionInMemory } from 'src/engine/core-modules/serverless/drivers/utils/build-serverless-function-in-memory';
-import { formatBuildError } from 'src/engine/core-modules/serverless/drivers/utils/format-build-error';
-import { buildEnvVar } from 'src/engine/core-modules/serverless/drivers/utils/build-env-var';

 export interface LocalDriverOptions {
  fileStorageService: FileStorageService;
@@ -156,7 +156,7 @@ export class LocalDriver implements ServerlessDriver {
            runnerPath,
            env: buildEnvVar(serverlessFunction),
            payload,
-            timeoutMs: serverlessFunction.timeoutSeconds * 1_000,
+            timeoutMs: 900_000, // timeout is handled by the serverless function service
          });

        if (stdout)
@@ -217,7 +217,7 @@ export class LocalDriver implements ServerlessDriver {
    const code = `
      // Auto-generated. Do not edit.
      const { pathToFileURL } = require('node:url');
-      
+
      (async () => {
        try {
          const builtUrl = pathToFileURL(${JSON.stringify(builtFileAbsPath)});
@@ -225,7 +225,7 @@ export class LocalDriver implements ServerlessDriver {
          if (typeof mod.${handlerName} !== 'function') {
            throw new Error('Export "${handlerName}" not found in serverless bundle');
          }
-      
+
          let payload = undefined;
          if (process.send) {
            process.on('message', async (msg) => {
@@ -11,4 +11,5 @@ export enum ServerlessFunctionExceptionCode {
  SERVERLESS_FUNCTION_CODE_UNCHANGED = 'SERVERLESS_FUNCTION_CODE_UNCHANGED',
  SERVERLESS_FUNCTION_EXECUTION_LIMIT_REACHED = 'SERVERLESS_FUNCTION_EXECUTION_LIMIT_REACHED',
  SERVERLESS_FUNCTION_CREATE_FAILED = 'SERVERLESS_FUNCTION_CREATE_FAILED',
+  SERVERLESS_FUNCTION_EXECUTION_TIMEOUT = 'SERVERLESS_FUNCTION_EXECUTION_TIMEOUT',
 }
@@ -13,11 +13,14 @@ import { type ServerlessExecuteResult } from 'src/engine/core-modules/serverless
 import { AuditService } from 'src/engine/core-modules/audit/services/audit.service';
 import { SERVERLESS_FUNCTION_EXECUTED_EVENT } from 'src/engine/core-modules/audit/utils/events/workspace-event/serverless-function/serverless-function-executed';
 import { FileStorageService } from 'src/engine/core-modules/file-storage/file-storage.service';
+import { Sources } from 'src/engine/core-modules/file-storage/types/source.type';
 import { getBaseTypescriptProjectFiles } from 'src/engine/core-modules/serverless/drivers/utils/get-base-typescript-project-files';
 import { ServerlessService } from 'src/engine/core-modules/serverless/serverless.service';
 import { getServerlessFolder } from 'src/engine/core-modules/serverless/utils/serverless-get-folder.utils';
 import { ThrottlerService } from 'src/engine/core-modules/throttler/throttler.service';
 import { TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service';
+import { ServerlessFunctionLayerService } from 'src/engine/metadata-modules/serverless-function-layer/serverless-function-layer.service';
+import { CreateServerlessFunctionInput } from 'src/engine/metadata-modules/serverless-function/dtos/create-serverless-function.input';
 import { type UpdateServerlessFunctionInput } from 'src/engine/metadata-modules/serverless-function/dtos/update-serverless-function.input';
 import { ServerlessFunctionEntity } from 'src/engine/metadata-modules/serverless-function/serverless-function.entity';
 import {
@@ -28,9 +31,6 @@ import {
  WorkflowVersionStepException,
  WorkflowVersionStepExceptionCode,
 } from 'src/modules/workflow/common/exceptions/workflow-version-step.exception';
-import { ServerlessFunctionLayerService } from 'src/engine/metadata-modules/serverless-function-layer/serverless-function-layer.service';
-import { Sources } from 'src/engine/core-modules/file-storage/types/source.type';
-import { CreateServerlessFunctionInput } from 'src/engine/metadata-modules/serverless-function/dtos/create-serverless-function.input';

@Injectable()
 export class ServerlessFunctionService {
@@ -102,11 +102,11 @@ export class ServerlessFunctionService {
        ],
      });

-    const resultServerlessFunction = await this.serverlessService.execute(
-      functionToExecute,
-      payload,
-      version,
-    );
+    const resultServerlessFunction = await this.callWithTimeout({
+      callback: () =>
+        this.serverlessService.execute(functionToExecute, payload, version),
+      timeoutMs: functionToExecute.timeoutSeconds * 1000,
+    });

    if (this.twentyConfigService.get('SERVERLESS_LOGS_ENABLED')) {
      /* eslint-disable no-console */
@@ -470,4 +470,31 @@ export class ServerlessFunctionService {
      );
    }
  }
+
+  private async callWithTimeout<T>({
+    callback,
+    timeoutMs,
+  }: {
+    callback: () => Promise<T>;
+    timeoutMs: number;
+  }): Promise<T> {
+    let timeoutId: NodeJS.Timeout;
+
+    const timeoutPromise = new Promise<never>((_, reject) => {
+      timeoutId = setTimeout(
+        () =>
+          reject(
+            new ServerlessFunctionException(
+              `Execution timeout: ${timeoutMs / 1000}s`,
+              ServerlessFunctionExceptionCode.SERVERLESS_FUNCTION_EXECUTION_TIMEOUT,
+            ),
+          ),
+        timeoutMs,
+      );
+    });
+
+    return Promise.race([callback(), timeoutPromise]).finally(() =>
+      clearTimeout(timeoutId),
+    ) as Promise<T>;
+  }
 }
@@ -4,6 +4,7 @@ import {
  ConflictError,
  ForbiddenError,
  NotFoundError,
+  TimeoutError,
 } from 'src/engine/core-modules/graphql/utils/graphql-errors.util';
 import {
  ServerlessFunctionException,
@@ -23,6 +24,8 @@ export const serverlessFunctionGraphQLApiExceptionHandler = (error: any) => {
      case ServerlessFunctionExceptionCode.SERVERLESS_FUNCTION_BUILDING:
      case ServerlessFunctionExceptionCode.SERVERLESS_FUNCTION_EXECUTION_LIMIT_REACHED:
        throw new ForbiddenError(error);
+      case ServerlessFunctionExceptionCode.SERVERLESS_FUNCTION_EXECUTION_TIMEOUT:
+        throw new TimeoutError(error);
      case ServerlessFunctionExceptionCode.SERVERLESS_FUNCTION_CODE_UNCHANGED:
      case ServerlessFunctionExceptionCode.SERVERLESS_FUNCTION_CREATE_FAILED:
        throw error;