From f5b06d20e40a004c6cd9a80d21f045737d914521 Mon Sep 17 00:00:00 2001 From: martmull Date: Tue, 21 Jul 2026 16:40:04 +0200 Subject: [PATCH] Add per-application API rate limiter (#23100) ## Context Application-token API requests are not rate limited today: `throttleQueryExecution` in the common query runner only throttles API-key requests, per workspace. An application installed on hundreds of workspaces (Call Recorder is on 700+) can therefore hit the API with large synchronized bursts, as seen with the recovery crons impacting production. ## What changed - New rate limiter in `CommonBaseQueryRunnerService`, applied when the auth context is an application context, using the existing `ThrottlerService.tokenBucketThrottleOrThrow` like the per-workspace API-key throttle. Both REST and GraphQL record operations go through this path. - The limiter key is `api:throttler:application:{universalIdentifier}` with no workspace component: the budget is shared by every installation of the application on the instance, which is what protects production from install-count-proportional load. `universalIdentifier` was chosen over `applicationRegistrationId` because the latter is null for unpublished/local applications. - Two new config variables in the `RATE_LIMITING` group: `APPLICATION_API_RATE_LIMITING_LIMIT` (default 500) and `APPLICATION_API_RATE_LIMITING_TTL_IN_MS` (default 60000), i.e. 500 requests/min per application across all workspaces. - Rejections raise the existing `ThrottlerException`, already mapped by both the REST and GraphQL exception handlers, and increment a new `common-api-query/application-rate-limited` metric (only for throttler rejections, so cache infrastructure failures are not reported as rate limiting). - Cron trigger dispatch `retryLimit` raised from 3 to 10 so throttled logic function executions eventually run once the budget refills. The existing per-workspace API-key throttle is unchanged (extracted to its own method). ## Notes - The limit is tunable per environment without a deploy pipeline change. ## Test - `npx nx lint:diff-with-main twenty-server` clean. - `npx nx typecheck twenty-server` clean. - Throttler spec passes (5 tests). --------- Co-authored-by: martmull --- .../common-base-query-runner.service.ts | 33 +++++++++++++++++++ .../triggers/cron/cron-trigger.cron.job.ts | 2 +- .../metrics/types/metrics-keys.type.ts | 1 + .../twenty-config/config-variables.ts | 18 ++++++++++ 4 files changed, 53 insertions(+), 1 deletion(-) diff --git a/packages/twenty-server/src/engine/api/common/common-query-runners/common-base-query-runner.service.ts b/packages/twenty-server/src/engine/api/common/common-query-runners/common-base-query-runner.service.ts index 590c474776..c902b80942 100644 --- a/packages/twenty-server/src/engine/api/common/common-query-runners/common-base-query-runner.service.ts +++ b/packages/twenty-server/src/engine/api/common/common-query-runners/common-base-query-runner.service.ts @@ -35,11 +35,13 @@ import { GraphqlQueryParser } from 'src/engine/api/graphql/graphql-query-runner/ import { WorkspacePreQueryHookPayload } from 'src/engine/api/graphql/workspace-query-runner/workspace-query-hook/types/workspace-query-hook.type'; import { WorkspaceQueryHookService } from 'src/engine/api/graphql/workspace-query-runner/workspace-query-hook/workspace-query-hook.service'; import { isApiKeyAuthContext } from 'src/engine/core-modules/auth/guards/is-api-key-auth-context.guard'; +import { isApplicationAuthContext } from 'src/engine/core-modules/auth/guards/is-application-auth-context.guard'; import { isUserAuthContext } from 'src/engine/core-modules/auth/guards/is-user-auth-context.guard'; import { WorkspaceAuthContext } from 'src/engine/core-modules/auth/types/workspace-auth-context.type'; import { FeatureFlagService } from 'src/engine/core-modules/feature-flag/services/feature-flag.service'; import { MetricsService } from 'src/engine/core-modules/metrics/metrics.service'; import { MetricsKeys } from 'src/engine/core-modules/metrics/types/metrics-keys.type'; +import { ThrottlerException } from 'src/engine/core-modules/throttler/throttler.exception'; import { ThrottlerService } from 'src/engine/core-modules/throttler/throttler.service'; import { TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service'; import { FlatEntityMaps } from 'src/engine/metadata-modules/flat-entity/types/flat-entity-maps.type'; @@ -348,6 +350,37 @@ export abstract class CommonBaseQueryRunnerService< } private async throttleQueryExecution(authContext: WorkspaceAuthContext) { + await this.throttleApiKeyQueryExecution(authContext); + await this.throttleApplicationQueryExecution(authContext); + } + + private async throttleApplicationQueryExecution( + authContext: WorkspaceAuthContext, + ) { + if (!isApplicationAuthContext(authContext)) return; + + try { + await this.throttlerService.tokenBucketThrottleOrThrow( + `api:throttler:application:${authContext.application.universalIdentifier}`, + 1, + this.twentyConfigService.get('APPLICATION_API_RATE_LIMITING_LIMIT'), + this.twentyConfigService.get('APPLICATION_API_RATE_LIMITING_TTL_IN_MS'), + ); + } catch (error) { + if (error instanceof ThrottlerException) { + await this.metricsService.incrementCounterForEvent({ + key: MetricsKeys.CommonApiApplicationQueryRateLimited, + shouldStoreInCache: false, + }); + } + + throw error; + } + } + + private async throttleApiKeyQueryExecution( + authContext: WorkspaceAuthContext, + ) { try { if (!isApiKeyAuthContext(authContext)) return; diff --git a/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-trigger/triggers/cron/cron-trigger.cron.job.ts b/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-trigger/triggers/cron/cron-trigger.cron.job.ts index 5fc29782d7..37adbad8d6 100644 --- a/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-trigger/triggers/cron/cron-trigger.cron.job.ts +++ b/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-trigger/triggers/cron/cron-trigger.cron.job.ts @@ -94,7 +94,7 @@ export class CronTriggerCronJob { payload: {}, }, ], - { retryLimit: 3 }, + { retryLimit: 10 }, ); } } catch (error) { diff --git a/packages/twenty-server/src/engine/core-modules/metrics/types/metrics-keys.type.ts b/packages/twenty-server/src/engine/core-modules/metrics/types/metrics-keys.type.ts index 8d189bc227..7150db9695 100644 --- a/packages/twenty-server/src/engine/core-modules/metrics/types/metrics-keys.type.ts +++ b/packages/twenty-server/src/engine/core-modules/metrics/types/metrics-keys.type.ts @@ -52,6 +52,7 @@ export enum MetricsKeys { JobWebhookCallCompleted = 'job/webhook-call-completed', SignUpSuccess = 'sign-up/success', CommonApiQueryRateLimited = 'common-api-query/rate-limited', + CommonApiApplicationQueryRateLimited = 'common-api-query/application-rate-limited', JobCompleted = 'job/completed', JobFailed = 'job/failed', JobStalled = 'job/stalled', diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/config-variables.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/config-variables.ts index 453d49907a..c282133799 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/config-variables.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/config-variables.ts @@ -1483,6 +1483,24 @@ export class ConfigVariables { @CastToPositiveNumber() API_RATE_LIMITING_LONG_LIMIT = 100; + @ConfigVariablesMetadata({ + group: ConfigVariablesGroup.RATE_LIMITING, + description: + 'Time-to-live for per-application API rate limiting in milliseconds', + type: ConfigVariableType.NUMBER, + }) + @CastToPositiveNumber() + APPLICATION_API_RATE_LIMITING_TTL_IN_MS = 60_000; + + @ConfigVariablesMetadata({ + group: ConfigVariablesGroup.RATE_LIMITING, + description: + 'Maximum number of API requests allowed per application across all workspaces in the rate limiting window', + type: ConfigVariableType.NUMBER, + }) + @CastToPositiveNumber() + APPLICATION_API_RATE_LIMITING_LIMIT = 500; + @CastToPositiveNumber() @ConfigVariablesMetadata({ group: ConfigVariablesGroup.RATE_LIMITING,