From fb0a54c73aac632f328046ab452ded2d7c62947e Mon Sep 17 00:00:00 2001 From: Thomas Trompette Date: Mon, 6 Jul 2026 13:44:33 +0200 Subject: [PATCH] fix(server): pace lambda control-plane calls to avoid 'Rate exceeded' on release (#22569) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Problem Logic functions intermittently fail with: ``` Lambda invocation failed for function '' during build: Rate exceeded ``` `Rate exceeded` is AWS Lambda's control-plane throttling (`TooManyRequestsException`), thrown during the **build** phase — before invoke — inside `buildExecutor`. ### Why it spikes on release A build is skipped (`canSkip = true`, zero control-plane calls) unless the executor is missing/inactive **or** `flatApplication.isSdkLayerStale` is true. `isSdkLayerStale` is flipped to `true` for the **whole application at once** whenever the SDK client regenerates (app install / development / schema change). So on release, every logic function in the app goes stale simultaneously → each enters `ensureExecutor` in its own per-function lock → a burst of `Create`/`Update`/`PublishLayer`/`GetFunction` calls across many functions at once → the low, account-region-wide control-plane quota is exceeded → `Rate exceeded`. Between releases everything is warm and no control-plane calls happen — hence "spikes on release, silent otherwise". The Lambda client was created with no retry override, so it used the SDK default (`standard` mode, `maxAttempts = 3`): a few retries with backoff, but no client-side pacing. ## Change Configure the shared Lambda client with: - `retryMode: 'adaptive'` — adds a client-side token-bucket rate limiter that slows outgoing requests when it sees throttling, instead of fire-then-backoff. - `maxAttempts: 8` — rides out the burst. Applied after the options spread so it always takes effect, and covers **every** control-plane call including the `waitUntilFunctionActive/UpdatedV2` pollers (same client). ## Scope / follow-up This is the cheap, high-leverage mitigation and dampens the burst per process. It does **not** add a cross-function/cross-pod concurrency cap, so a large enough release across multiple replicas could still exceed the account quota. A follow-up could add a limiter (in-process semaphore, or a distributed token bucket via the existing Redis cache-lock) around `ensureExecutor`. ## Testing - `tsc --noEmit` on twenty-server: clean. - Not runtime-tested — AWS control-plane throttling can't be reproduced locally. Worth confirming against a real release-time CloudWatch window after deploy. Review in cubic --- .../drivers/lambda/constants/lambda-driver.constant.ts | 3 +++ .../drivers/lambda/services/lambda-aws-client.service.ts | 4 ++++ 2 files changed, 7 insertions(+) diff --git a/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-drivers/drivers/lambda/constants/lambda-driver.constant.ts b/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-drivers/drivers/lambda/constants/lambda-driver.constant.ts index e479414f00..3a95995612 100644 --- a/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-drivers/drivers/lambda/constants/lambda-driver.constant.ts +++ b/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-drivers/drivers/lambda/constants/lambda-driver.constant.ts @@ -5,6 +5,9 @@ import { ASSET_PATH } from 'src/constants/assets-path'; export const UPDATE_FUNCTION_DURATION_TIMEOUT_IN_SECONDS = 60; export const CREDENTIALS_DURATION_IN_SECONDS = 60 * 60; // 1h +export const LAMBDA_CLIENT_MAX_ATTEMPTS = 8; +export const LAMBDA_CLIENT_RETRY_MODE = 'adaptive' as const; + export const YARN_INSTALL_LAMBDA_TIMEOUT_SECONDS = 300; export const YARN_INSTALL_LAMBDA_MEMORY_MB = 1024; export const BUILDER_LAMBDA_TIMEOUT_SECONDS = 60; diff --git a/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-drivers/drivers/lambda/services/lambda-aws-client.service.ts b/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-drivers/drivers/lambda/services/lambda-aws-client.service.ts index 009b6174ad..f95f23247f 100644 --- a/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-drivers/drivers/lambda/services/lambda-aws-client.service.ts +++ b/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-drivers/drivers/lambda/services/lambda-aws-client.service.ts @@ -11,6 +11,8 @@ import { isDefined } from 'twenty-shared/utils'; import { CREDENTIALS_DURATION_IN_SECONDS, + LAMBDA_CLIENT_MAX_ATTEMPTS, + LAMBDA_CLIENT_RETRY_MODE, UPDATE_FUNCTION_DURATION_TIMEOUT_IN_SECONDS, } from 'src/engine/core-modules/logic-function/logic-function-drivers/drivers/lambda/constants/lambda-driver.constant'; import { type LambdaDriverOptions } from 'src/engine/core-modules/logic-function/logic-function-drivers/drivers/lambda/types/lambda-driver.type'; @@ -35,6 +37,8 @@ export class LambdaAwsClientService { ...(isDefined(this.options.subhostingRole) && { credentials: await this.getAssumeRoleCredentials(), }), + maxAttempts: LAMBDA_CLIENT_MAX_ATTEMPTS, + retryMode: LAMBDA_CLIENT_RETRY_MODE, }); }