Improve workflow throttling logic (#16260)

- if >5000 workflows per hour, new ones should failed
- if >100 workflow per min, new ones should be set as not started.
Except manual trigger
- when enqueued, we check if there a not started workflows that may be
queued. If yes, we call the associated job

---------

Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com>

packages/twenty-server/src/engine/core-modules/metrics/types/metrics-keys.type.ts

@@ -20,7 +20,7 @@ export enum MetricsKeys {
   WorkflowRunCompleted = 'workflow-run/completed',
   WorkflowRunFailed = 'workflow-run/failed',
   WorkflowRunStopped = 'workflow-run/stopped',
-  WorkflowRunFailedThrottled = 'workflow-run/failed/throttled',
+  WorkflowRunThrottled = 'workflow-run/throttled',
   WorkflowRunFailedToEnqueue = 'workflow-run/failed/to-enqueue',
   AIToolExecutionFailed = 'ai-tool-execution/failed',
   AIToolExecutionSucceeded = 'ai-tool-execution/succeeded',

packages/twenty-server/src/engine/core-modules/throttler/throttler.service.ts

@@ -20,18 +20,14 @@ export class ThrottlerService {
     tokensToConsume: number,
     maxTokens: number,
     timeWindow: number,
-  ): Promise<void> {
+  ): Promise<number> {
     const now = Date.now();
-    const refillRate = maxTokens / timeWindow;
-
-    const { tokens, lastRefillAt } = (await this.cacheStorage.get<{
-      tokens: number;
-      lastRefillAt: number;
-    }>(key)) || { tokens: maxTokens, lastRefillAt: now };
-
-    const refillAmount = Math.floor((now - lastRefillAt) * refillRate);
-
-    const availableTokens = Math.min(tokens + refillAmount, maxTokens);
+    const availableTokens = await this.getAvailableTokensCount(
+      key,
+      maxTokens,
+      timeWindow,
+      now,
+    );
 
     if (availableTokens < tokensToConsume) {
       throw new ThrottlerException(
@@ -48,5 +44,49 @@ export class ThrottlerService {
       },
       timeWindow * 2,
     );
+
+    return availableTokens - tokensToConsume;
+  }
+
+  async consumeTokens(
+    key: string,
+    tokensToConsume: number,
+    maxTokens: number,
+    timeWindow: number,
+  ) {
+    const now = Date.now();
+    const availableTokens = await this.getAvailableTokensCount(
+      key,
+      maxTokens,
+      timeWindow,
+      now,
+    );
+
+    await this.cacheStorage.set(
+      key,
+      {
+        tokens: availableTokens - tokensToConsume,
+        lastRefillAt: now,
+      },
+      timeWindow * 2,
+    );
+  }
+
+  async getAvailableTokensCount(
+    key: string,
+    maxTokens: number,
+    timeWindow: number,
+    now = Date.now(),
+  ): Promise<number> {
+    const refillRate = maxTokens / timeWindow;
+
+    const { tokens, lastRefillAt } = (await this.cacheStorage.get<{
+      tokens: number;
+      lastRefillAt: number;
+    }>(key)) || { tokens: maxTokens, lastRefillAt: now };
+
+    const refillAmount = Math.floor((now - lastRefillAt) * refillRate);
+
+    return Math.min(tokens + refillAmount, maxTokens);
   }
 }

packages/twenty-server/src/engine/core-modules/twenty-config/config-variables.ts

@@ -1219,19 +1219,39 @@ export class ConfigVariables {
 
   @ConfigVariablesMetadata({
     group: ConfigVariablesGroup.RATE_LIMITING,
-    description: 'Throttle limit for workflow execution',
+    description:
+      'Throttle limit for workflow execution. Remaining will not be enqueued immediately.',
     type: ConfigVariableType.NUMBER,
   })
   @CastToPositiveNumber()
-  WORKFLOW_EXEC_THROTTLE_LIMIT = 100;
+  WORKFLOW_EXEC_SOFT_THROTTLE_LIMIT = 100;
 
   @ConfigVariablesMetadata({
     group: ConfigVariablesGroup.RATE_LIMITING,
-    description: 'Time-to-live for workflow execution throttle in milliseconds',
+    description:
+      'Time-to-live for workflow execution throttle in milliseconds. Remaining will not be enqueued immediately.',
     type: ConfigVariableType.NUMBER,
   })
   @CastToPositiveNumber()
-  WORKFLOW_EXEC_THROTTLE_TTL = 60_000;
+  WORKFLOW_EXEC_SOFT_THROTTLE_TTL = 60_000;
+
+  @ConfigVariablesMetadata({
+    group: ConfigVariablesGroup.RATE_LIMITING,
+    description:
+      'Throttle limit for workflow execution. Remaining will be marked as failed.',
+    type: ConfigVariableType.NUMBER,
+  })
+  @CastToPositiveNumber()
+  WORKFLOW_EXEC_HARD_THROTTLE_LIMIT = 5000;
+
+  @ConfigVariablesMetadata({
+    group: ConfigVariablesGroup.RATE_LIMITING,
+    description:
+      'Time-to-live for workflow execution throttle in milliseconds. Remaining will be marked as failed.',
+    type: ConfigVariableType.NUMBER,
+  })
+  @CastToPositiveNumber()
+  WORKFLOW_EXEC_HARD_THROTTLE_TTL = 3_600_000; // 1 hour;
 
   @ConfigVariablesMetadata({
     group: ConfigVariablesGroup.CAPTCHA_CONFIG,