Add per-application API rate limiter (#23100)

## Context

Application-token API requests are not rate limited today:
`throttleQueryExecution` in the common query runner only throttles
API-key requests, per workspace. An application installed on hundreds of
workspaces (Call Recorder is on 700+) can therefore hit the API with
large synchronized bursts, as seen with the recovery crons impacting
production.

## What changed

- New rate limiter in `CommonBaseQueryRunnerService`, applied when the
auth context is an application context, using the existing
`ThrottlerService.tokenBucketThrottleOrThrow` like the per-workspace
API-key throttle. Both REST and GraphQL record operations go through
this path.
- The limiter key is `api:throttler:application:{universalIdentifier}`
with no workspace component: the budget is shared by every installation
of the application on the instance, which is what protects production
from install-count-proportional load. `universalIdentifier` was chosen
over `applicationRegistrationId` because the latter is null for
unpublished/local applications.
- Two new config variables in the `RATE_LIMITING` group:
`APPLICATION_API_RATE_LIMITING_LIMIT` (default 500) and
`APPLICATION_API_RATE_LIMITING_TTL_IN_MS` (default 60000), i.e. 500
requests/min per application across all workspaces.
- Rejections raise the existing `ThrottlerException`, already mapped by
both the REST and GraphQL exception handlers, and increment a new
`common-api-query/application-rate-limited` metric (only for throttler
rejections, so cache infrastructure failures are not reported as rate
limiting).
- Cron trigger dispatch `retryLimit` raised from 3 to 10 so throttled
logic function executions eventually run once the budget refills.

The existing per-workspace API-key throttle is unchanged (extracted to
its own method).

## Notes

- The limit is tunable per environment without a deploy pipeline change.

## Test

- `npx nx lint:diff-with-main twenty-server` clean.
- `npx nx typecheck twenty-server` clean.
- Throttler spec passes (5 tests).

---------

Co-authored-by: martmull <martin@twenty.com>

packages/twenty-server/src/engine/api/common/common-query-runners/common-base-query-runner.service.ts

@@ -35,11 +35,13 @@ import { GraphqlQueryParser } from 'src/engine/api/graphql/graphql-query-runner/
 import { WorkspacePreQueryHookPayload } from 'src/engine/api/graphql/workspace-query-runner/workspace-query-hook/types/workspace-query-hook.type';
 import { WorkspaceQueryHookService } from 'src/engine/api/graphql/workspace-query-runner/workspace-query-hook/workspace-query-hook.service';
 import { isApiKeyAuthContext } from 'src/engine/core-modules/auth/guards/is-api-key-auth-context.guard';
+import { isApplicationAuthContext } from 'src/engine/core-modules/auth/guards/is-application-auth-context.guard';
 import { isUserAuthContext } from 'src/engine/core-modules/auth/guards/is-user-auth-context.guard';
 import { WorkspaceAuthContext } from 'src/engine/core-modules/auth/types/workspace-auth-context.type';
 import { FeatureFlagService } from 'src/engine/core-modules/feature-flag/services/feature-flag.service';
 import { MetricsService } from 'src/engine/core-modules/metrics/metrics.service';
 import { MetricsKeys } from 'src/engine/core-modules/metrics/types/metrics-keys.type';
+import { ThrottlerException } from 'src/engine/core-modules/throttler/throttler.exception';
 import { ThrottlerService } from 'src/engine/core-modules/throttler/throttler.service';
 import { TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service';
 import { FlatEntityMaps } from 'src/engine/metadata-modules/flat-entity/types/flat-entity-maps.type';
@@ -348,6 +350,37 @@ export abstract class CommonBaseQueryRunnerService<
   }
 
   private async throttleQueryExecution(authContext: WorkspaceAuthContext) {
+    await this.throttleApiKeyQueryExecution(authContext);
+    await this.throttleApplicationQueryExecution(authContext);
+  }
+
+  private async throttleApplicationQueryExecution(
+    authContext: WorkspaceAuthContext,
+  ) {
+    if (!isApplicationAuthContext(authContext)) return;
+
+    try {
+      await this.throttlerService.tokenBucketThrottleOrThrow(
+        `api:throttler:application:${authContext.application.universalIdentifier}`,
+        1,
+        this.twentyConfigService.get('APPLICATION_API_RATE_LIMITING_LIMIT'),
+        this.twentyConfigService.get('APPLICATION_API_RATE_LIMITING_TTL_IN_MS'),
+      );
+    } catch (error) {
+      if (error instanceof ThrottlerException) {
+        await this.metricsService.incrementCounterForEvent({
+          key: MetricsKeys.CommonApiApplicationQueryRateLimited,
+          shouldStoreInCache: false,
+        });
+      }
+
+      throw error;
+    }
+  }
+
+  private async throttleApiKeyQueryExecution(
+    authContext: WorkspaceAuthContext,
+  ) {
     try {
       if (!isApiKeyAuthContext(authContext)) return;
 

packages/twenty-server/src/engine/core-modules/logic-function/logic-function-trigger/triggers/cron/cron-trigger.cron.job.ts

@@ -94,7 +94,7 @@ export class CronTriggerCronJob {
                 payload: {},
               },
             ],
-            { retryLimit: 3 },
+            { retryLimit: 10 },
           );
         }
       } catch (error) {

packages/twenty-server/src/engine/core-modules/metrics/types/metrics-keys.type.ts

@@ -52,6 +52,7 @@ export enum MetricsKeys {
   JobWebhookCallCompleted = 'job/webhook-call-completed',
   SignUpSuccess = 'sign-up/success',
   CommonApiQueryRateLimited = 'common-api-query/rate-limited',
+  CommonApiApplicationQueryRateLimited = 'common-api-query/application-rate-limited',
   JobCompleted = 'job/completed',
   JobFailed = 'job/failed',
   JobStalled = 'job/stalled',

packages/twenty-server/src/engine/core-modules/twenty-config/config-variables.ts

@@ -1483,6 +1483,24 @@ export class ConfigVariables {
   @CastToPositiveNumber()
   API_RATE_LIMITING_LONG_LIMIT = 100;
 
+  @ConfigVariablesMetadata({
+    group: ConfigVariablesGroup.RATE_LIMITING,
+    description:
+      'Time-to-live for per-application API rate limiting in milliseconds',
+    type: ConfigVariableType.NUMBER,
+  })
+  @CastToPositiveNumber()
+  APPLICATION_API_RATE_LIMITING_TTL_IN_MS = 60_000;
+
+  @ConfigVariablesMetadata({
+    group: ConfigVariablesGroup.RATE_LIMITING,
+    description:
+      'Maximum number of API requests allowed per application across all workspaces in the rate limiting window',
+    type: ConfigVariableType.NUMBER,
+  })
+  @CastToPositiveNumber()
+  APPLICATION_API_RATE_LIMITING_LIMIT = 500;
+
   @CastToPositiveNumber()
   @ConfigVariablesMetadata({
     group: ConfigVariablesGroup.RATE_LIMITING,