Fix lambda timeout diagnostics (#21002)

Trying to fix
https://twenty-v7.sentry.io/issues/7420384466/?environment=prod&environment=prod-eu&project=4507072499810304&query=is%3Aunresolved%20%21issue.type%3A%5Bperformance_consecutive_db_queries%2Cperformance_consecutive_http%2Cperformance_file_io_main_thread%2Cperformance_db_main_thread%2Cperformance_n_plus_one_db_queries%2Cperformance_n_plus_one_api_calls%2Cperformance_p95_endpoint_regression%2Cperformance_slow_db_query%2Cperformance_render_blocking_asset_span%2Cperformance_uncompressed_assets%2Cperformance_http_overhead%2Cperformance_large_http_payload%5D%20timesSeen%3A%3E10&referrer=issue-stream&sort=date

## Summary
- Prevent invoking Lambda functions stuck in `Pending` state by checking
`Configuration.State === 'Active'` in `checkLambdaExecutorBuildStatus` —
a Pending function now goes through `ensureLambdaExecutor` which waits
for Active.
- Track execution phase (`build`/`fetch-code`/`invoke`) via a
`LambdaExecutionPhase` enum and include phase timing + function state in
all error messages for faster debugging.

Co-authored-by: Matt Van Horn <mvanhorn@users.noreply.github.com>

packages/twenty-server/src/engine/core-modules/logic-function/logic-function-drivers/drivers/lambda.driver.ts

@@ -57,6 +57,12 @@ import {
 } from 'src/engine/metadata-modules/logic-function/logic-function.exception';
 import { type FlatLogicFunction } from 'src/engine/metadata-modules/logic-function/types/flat-logic-function.type';
 
+enum LambdaExecutionPhase {
+  BUILD = 'build',
+  FETCH_CODE = 'fetch-code',
+  INVOKE = 'invoke',
+}
+
 const UPDATE_FUNCTION_DURATION_TIMEOUT_IN_SECONDS = 60;
 const CREDENTIALS_DURATION_IN_SECONDS = 60 * 60; // 1h
 const YARN_INSTALL_LAMBDA_TIMEOUT_SECONDS = 300;
@@ -982,8 +988,11 @@ export class LambdaDriver implements LogicFunctionDriver {
   }> {
     const lambdaExecutor = await this.getLambdaExecutor(flatLogicFunction);
 
+    const isActive = lambdaExecutor?.Configuration?.State === 'Active';
+
     const canSkip =
       isDefined(lambdaExecutor) &&
+      isActive &&
       !flatApplication.isSdkLayerStale &&
       this.hasExpectedLayers({
         lambdaExecutor,
@@ -1156,41 +1165,49 @@ export class LambdaDriver implements LogicFunctionDriver {
     env,
     timeoutMs = 900_000,
   }: LogicFunctionExecuteParams): Promise<LogicFunctionExecuteResult> {
-    const buildStart = Date.now();
-
-    await this.buildLambdaExecutor({
-      flatLogicFunction,
-      flatApplication,
-      applicationUniversalIdentifier,
-    });
-    const buildExecutorMs = Date.now() - buildStart;
-
-    const startTime = Date.now();
-
-    const compiledCode = await this.logicFunctionResourceService.getBuiltCode({
-      workspaceId: flatLogicFunction.workspaceId,
-      applicationUniversalIdentifier,
-      builtHandlerPath: flatLogicFunction.builtHandlerPath,
-    });
-    const getBuiltCodeMs = Date.now() - startTime;
-
-    const executorPayload: LambdaDriverExecutorPayload = {
-      params: payload,
-      code: compiledCode,
-      env: env ?? {},
-      handlerName: flatLogicFunction.handlerName,
-    };
-
-    const payloadString = JSON.stringify(executorPayload);
-    const params: InvokeCommandInput = {
-      FunctionName: flatLogicFunction.id,
-      Payload: payloadString,
-      LogType: LogType.Tail,
-    };
-
-    const command = new InvokeCommand(params);
+    let currentPhase: LambdaExecutionPhase = LambdaExecutionPhase.BUILD;
+    let buildExecutorMs = 0;
+    let getBuiltCodeMs = 0;
 
     try {
+      const buildStart = Date.now();
+
+      await this.buildLambdaExecutor({
+        flatLogicFunction,
+        flatApplication,
+        applicationUniversalIdentifier,
+      });
+      buildExecutorMs = Date.now() - buildStart;
+
+      currentPhase = LambdaExecutionPhase.FETCH_CODE;
+      const fetchStart = Date.now();
+
+      const compiledCode = await this.logicFunctionResourceService.getBuiltCode(
+        {
+          workspaceId: flatLogicFunction.workspaceId,
+          applicationUniversalIdentifier,
+          builtHandlerPath: flatLogicFunction.builtHandlerPath,
+        },
+      );
+      getBuiltCodeMs = Date.now() - fetchStart;
+
+      currentPhase = LambdaExecutionPhase.INVOKE;
+
+      const executorPayload: LambdaDriverExecutorPayload = {
+        params: payload,
+        code: compiledCode,
+        env: env ?? {},
+        handlerName: flatLogicFunction.handlerName,
+      };
+
+      const payloadString = JSON.stringify(executorPayload);
+      const params: InvokeCommandInput = {
+        FunctionName: flatLogicFunction.id,
+        Payload: payloadString,
+        LogType: LogType.Tail,
+      };
+
+      const command = new InvokeCommand(params);
       const lambdaClient = await this.getLambdaClient();
 
       const invokeStart = Date.now();
@@ -1211,7 +1228,7 @@ export class LambdaDriver implements LogicFunctionDriver {
         coldStart,
       } = this.parseLambdaLogResult(result.LogResult);
 
-      const duration = Date.now() - startTime;
+      const duration = Date.now() - fetchStart;
 
       this.logger.log(
         `[lambda-timing] fnId=${flatLogicFunction.id} totalMs=${Date.now() - buildStart} buildExecutorMs=${buildExecutorMs} getBuiltCodeMs=${getBuiltCodeMs} payloadBytes=${Buffer.byteLength(payloadString, 'utf8')} invokeSendMs=${invokeSendMs} reportDurationMs=${reportDurationMs ?? 'n/a'} billedMs=${billedDurationMs ?? 'n/a'} initDurationMs=${initDurationMs ?? 'n/a'} coldStart=${coldStart}`,
@@ -1234,8 +1251,10 @@ export class LambdaDriver implements LogicFunctionDriver {
         status: LogicFunctionExecutionStatus.SUCCESS,
       };
     } catch (error) {
+      const phaseTiming = `phase=${currentPhase} buildMs=${buildExecutorMs} fetchCodeMs=${getBuiltCodeMs}`;
+
       this.logger.error(
-        `Lambda invocation failed for function ${flatLogicFunction.id}: ${error instanceof Error ? error.message : String(error)}`,
+        `Lambda invocation failed for function ${flatLogicFunction.id} [${phaseTiming}]: ${error instanceof Error ? error.message : String(error)}`,
         error instanceof Error ? error.stack : undefined,
       );
 
@@ -1247,8 +1266,13 @@ export class LambdaDriver implements LogicFunctionDriver {
       }
 
       if (error instanceof Error && error.name === 'TimeoutError') {
+        const executor = await this.getLambdaExecutor(flatLogicFunction).catch(
+          () => undefined,
+        );
+        const functionState = executor?.Configuration?.State ?? 'unknown';
+
         throw new LogicFunctionException(
-          `Lambda invocation timed out for function '${flatLogicFunction.id}': ${error.message}`,
+          `Lambda timed out for function '${flatLogicFunction.id}' during ${currentPhase} (functionState=${functionState}, ${phaseTiming})`,
           LogicFunctionExceptionCode.LOGIC_FUNCTION_EXECUTION_TIMEOUT,
         );
       }
@@ -1258,7 +1282,7 @@ export class LambdaDriver implements LogicFunctionDriver {
       }
 
       throw new LogicFunctionException(
-        `Lambda invocation failed for function '${flatLogicFunction.id}': ${error instanceof Error ? error.message : 'Unknown error'}`,
+        `Lambda invocation failed for function '${flatLogicFunction.id}' during ${currentPhase}: ${error instanceof Error ? error.message : 'Unknown error'}`,
         LogicFunctionExceptionCode.LOGIC_FUNCTION_EXECUTION_FAILED,
       );
     }