From 6e2cf4cbc0fc5c23325e281189d76d5d7269b39d Mon Sep 17 00:00:00 2001 From: Thomas Trompette Date: Thu, 28 May 2026 11:06:33 +0200 Subject: [PATCH] Fix lambda timeout diagnostics (#21002) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Trying to fix https://twenty-v7.sentry.io/issues/7420384466/?environment=prod&environment=prod-eu&project=4507072499810304&query=is%3Aunresolved%20%21issue.type%3A%5Bperformance_consecutive_db_queries%2Cperformance_consecutive_http%2Cperformance_file_io_main_thread%2Cperformance_db_main_thread%2Cperformance_n_plus_one_db_queries%2Cperformance_n_plus_one_api_calls%2Cperformance_p95_endpoint_regression%2Cperformance_slow_db_query%2Cperformance_render_blocking_asset_span%2Cperformance_uncompressed_assets%2Cperformance_http_overhead%2Cperformance_large_http_payload%5D%20timesSeen%3A%3E10&referrer=issue-stream&sort=date ## Summary - Prevent invoking Lambda functions stuck in `Pending` state by checking `Configuration.State === 'Active'` in `checkLambdaExecutorBuildStatus` — a Pending function now goes through `ensureLambdaExecutor` which waits for Active. - Track execution phase (`build`/`fetch-code`/`invoke`) via a `LambdaExecutionPhase` enum and include phase timing + function state in all error messages for faster debugging. Co-authored-by: Matt Van Horn --- .../drivers/lambda.driver.ts | 98 ++++++++++++------- 1 file changed, 61 insertions(+), 37 deletions(-) diff --git a/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-drivers/drivers/lambda.driver.ts b/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-drivers/drivers/lambda.driver.ts index 570fb6a0d4..7e291edcb8 100644 --- a/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-drivers/drivers/lambda.driver.ts +++ b/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-drivers/drivers/lambda.driver.ts @@ -57,6 +57,12 @@ import { } from 'src/engine/metadata-modules/logic-function/logic-function.exception'; import { type FlatLogicFunction } from 'src/engine/metadata-modules/logic-function/types/flat-logic-function.type'; +enum LambdaExecutionPhase { + BUILD = 'build', + FETCH_CODE = 'fetch-code', + INVOKE = 'invoke', +} + const UPDATE_FUNCTION_DURATION_TIMEOUT_IN_SECONDS = 60; const CREDENTIALS_DURATION_IN_SECONDS = 60 * 60; // 1h const YARN_INSTALL_LAMBDA_TIMEOUT_SECONDS = 300; @@ -982,8 +988,11 @@ export class LambdaDriver implements LogicFunctionDriver { }> { const lambdaExecutor = await this.getLambdaExecutor(flatLogicFunction); + const isActive = lambdaExecutor?.Configuration?.State === 'Active'; + const canSkip = isDefined(lambdaExecutor) && + isActive && !flatApplication.isSdkLayerStale && this.hasExpectedLayers({ lambdaExecutor, @@ -1156,41 +1165,49 @@ export class LambdaDriver implements LogicFunctionDriver { env, timeoutMs = 900_000, }: LogicFunctionExecuteParams): Promise { - const buildStart = Date.now(); - - await this.buildLambdaExecutor({ - flatLogicFunction, - flatApplication, - applicationUniversalIdentifier, - }); - const buildExecutorMs = Date.now() - buildStart; - - const startTime = Date.now(); - - const compiledCode = await this.logicFunctionResourceService.getBuiltCode({ - workspaceId: flatLogicFunction.workspaceId, - applicationUniversalIdentifier, - builtHandlerPath: flatLogicFunction.builtHandlerPath, - }); - const getBuiltCodeMs = Date.now() - startTime; - - const executorPayload: LambdaDriverExecutorPayload = { - params: payload, - code: compiledCode, - env: env ?? {}, - handlerName: flatLogicFunction.handlerName, - }; - - const payloadString = JSON.stringify(executorPayload); - const params: InvokeCommandInput = { - FunctionName: flatLogicFunction.id, - Payload: payloadString, - LogType: LogType.Tail, - }; - - const command = new InvokeCommand(params); + let currentPhase: LambdaExecutionPhase = LambdaExecutionPhase.BUILD; + let buildExecutorMs = 0; + let getBuiltCodeMs = 0; try { + const buildStart = Date.now(); + + await this.buildLambdaExecutor({ + flatLogicFunction, + flatApplication, + applicationUniversalIdentifier, + }); + buildExecutorMs = Date.now() - buildStart; + + currentPhase = LambdaExecutionPhase.FETCH_CODE; + const fetchStart = Date.now(); + + const compiledCode = await this.logicFunctionResourceService.getBuiltCode( + { + workspaceId: flatLogicFunction.workspaceId, + applicationUniversalIdentifier, + builtHandlerPath: flatLogicFunction.builtHandlerPath, + }, + ); + getBuiltCodeMs = Date.now() - fetchStart; + + currentPhase = LambdaExecutionPhase.INVOKE; + + const executorPayload: LambdaDriverExecutorPayload = { + params: payload, + code: compiledCode, + env: env ?? {}, + handlerName: flatLogicFunction.handlerName, + }; + + const payloadString = JSON.stringify(executorPayload); + const params: InvokeCommandInput = { + FunctionName: flatLogicFunction.id, + Payload: payloadString, + LogType: LogType.Tail, + }; + + const command = new InvokeCommand(params); const lambdaClient = await this.getLambdaClient(); const invokeStart = Date.now(); @@ -1211,7 +1228,7 @@ export class LambdaDriver implements LogicFunctionDriver { coldStart, } = this.parseLambdaLogResult(result.LogResult); - const duration = Date.now() - startTime; + const duration = Date.now() - fetchStart; this.logger.log( `[lambda-timing] fnId=${flatLogicFunction.id} totalMs=${Date.now() - buildStart} buildExecutorMs=${buildExecutorMs} getBuiltCodeMs=${getBuiltCodeMs} payloadBytes=${Buffer.byteLength(payloadString, 'utf8')} invokeSendMs=${invokeSendMs} reportDurationMs=${reportDurationMs ?? 'n/a'} billedMs=${billedDurationMs ?? 'n/a'} initDurationMs=${initDurationMs ?? 'n/a'} coldStart=${coldStart}`, @@ -1234,8 +1251,10 @@ export class LambdaDriver implements LogicFunctionDriver { status: LogicFunctionExecutionStatus.SUCCESS, }; } catch (error) { + const phaseTiming = `phase=${currentPhase} buildMs=${buildExecutorMs} fetchCodeMs=${getBuiltCodeMs}`; + this.logger.error( - `Lambda invocation failed for function ${flatLogicFunction.id}: ${error instanceof Error ? error.message : String(error)}`, + `Lambda invocation failed for function ${flatLogicFunction.id} [${phaseTiming}]: ${error instanceof Error ? error.message : String(error)}`, error instanceof Error ? error.stack : undefined, ); @@ -1247,8 +1266,13 @@ export class LambdaDriver implements LogicFunctionDriver { } if (error instanceof Error && error.name === 'TimeoutError') { + const executor = await this.getLambdaExecutor(flatLogicFunction).catch( + () => undefined, + ); + const functionState = executor?.Configuration?.State ?? 'unknown'; + throw new LogicFunctionException( - `Lambda invocation timed out for function '${flatLogicFunction.id}': ${error.message}`, + `Lambda timed out for function '${flatLogicFunction.id}' during ${currentPhase} (functionState=${functionState}, ${phaseTiming})`, LogicFunctionExceptionCode.LOGIC_FUNCTION_EXECUTION_TIMEOUT, ); } @@ -1258,7 +1282,7 @@ export class LambdaDriver implements LogicFunctionDriver { } throw new LogicFunctionException( - `Lambda invocation failed for function '${flatLogicFunction.id}': ${error instanceof Error ? error.message : 'Unknown error'}`, + `Lambda invocation failed for function '${flatLogicFunction.id}' during ${currentPhase}: ${error instanceof Error ? error.message : 'Unknown error'}`, LogicFunctionExceptionCode.LOGIC_FUNCTION_EXECUTION_FAILED, ); }