Tool execution metrics (#21587)

<!-- This is an auto-generated description by cubic. -->
<a
href="https://cubic.dev/pr/twentyhq/twenty/pull/21587?utm_source=github"
target="_blank" rel="noopener noreferrer"
data-no-image-dialog="true"><picture><source
media="(prefers-color-scheme: dark)"
srcset="https://www.cubic.dev/buttons/review-in-cubic-dark.svg"><source
media="(prefers-color-scheme: light)"
srcset="https://www.cubic.dev/buttons/review-in-cubic-light.svg"><img
alt="Review in cubic"
src="https://www.cubic.dev/buttons/review-in-cubic-dark.svg"></picture></a>
<!-- End of auto-generated description by cubic. -->

packages/twenty-server/src/engine/api/mcp/services/__tests__/mcp-tool-executor.service.spec.ts

@@ -4,8 +4,8 @@ import {
   MCP_PROGRESS_NOTIFICATION_METHOD,
   TOOL_CALL_PROGRESS_TOKEN_PREFIX,
 } from 'src/engine/api/mcp/constants/mcp-progress-notification.const';
-import { MetricsService } from 'src/engine/core-modules/metrics/metrics.service';
 import { McpToolExecutorService } from 'src/engine/api/mcp/services/mcp-tool-executor.service';
+import { MetricsService } from 'src/engine/core-modules/metrics/metrics.service';
 
 describe('McpToolExecutorService', () => {
   let service: McpToolExecutorService;
@@ -13,6 +13,7 @@ describe('McpToolExecutorService', () => {
   beforeEach(() => {
     const metricsService = {
       incrementCounterBy: jest.fn().mockResolvedValue(undefined),
+      recordHistogram: jest.fn(),
     } as unknown as MetricsService;
 
     service = new McpToolExecutorService(metricsService);

packages/twenty-server/src/engine/api/mcp/services/mcp-tool-executor.service.ts

@@ -1,10 +1,15 @@
 import { Injectable } from '@nestjs/common';
 
+import { isNonEmptyString } from '@sniptt/guards';
 import { type ToolSet } from 'ai';
 import { isDefined } from 'twenty-shared/utils';
 
-import { MetricsKeys } from 'src/engine/core-modules/metrics/types/metrics-keys.type';
 import { MetricsService } from 'src/engine/core-modules/metrics/metrics.service';
+import { MetricsKeys } from 'src/engine/core-modules/metrics/types/metrics-keys.type';
+import { estimateToolOutputTokens } from 'src/engine/core-modules/tool-provider/utils/estimate-tool-output-tokens.util';
+import { getToolMetricName } from 'src/engine/core-modules/tool-provider/utils/get-tool-metric-name.util';
+import { isToolOutputSuccessful } from 'src/engine/core-modules/tool-provider/utils/is-tool-output-successful.util';
+import { resolveToolName } from 'src/engine/core-modules/tool-provider/utils/resolve-tool-name.util';
 
 import { JSON_RPC_ERROR_CODE } from 'src/engine/api/mcp/constants/json-rpc-error-code.const';
 import {
@@ -33,14 +38,23 @@ export class McpToolExecutorService {
     params: Record<string, unknown>,
     sseWriter?: (data: Record<string, unknown>) => void,
   ) {
-    const toolName = params.name as keyof typeof toolSet;
+    if (!isNonEmptyString(params.name)) {
+      return wrapJsonRpcResponse(id, {
+        error: {
+          code: JSON_RPC_ERROR_CODE.INVALID_PARAMS,
+          message: 'Tool name is required',
+        },
+      });
+    }
+
+    const toolName = params.name;
     const tool = toolSet[toolName];
 
     if (!isDefined(tool) || !isDefined(tool.execute)) {
       return wrapJsonRpcResponse(id, {
         error: {
           code: JSON_RPC_ERROR_CODE.INVALID_PARAMS,
-          message: `Unknown tool: ${String(params.name)}`,
+          message: `Unknown tool: ${toolName}`,
         },
       });
     }
@@ -57,15 +71,34 @@ export class McpToolExecutorService {
       });
     }
 
+    const metricToolName = getToolMetricName(
+      resolveToolName({
+        toolName,
+        input: params.arguments,
+      }),
+    );
+
     try {
       const result = await tool.execute(params.arguments, {
         toolCallId: '1',
         messages: [],
       });
 
-      void this.metricsService.incrementCounterBy({
-        key: MetricsKeys.McpToolExecutionSucceeded,
+      const succeeded = isToolOutputSuccessful(result);
+
+      this.metricsService.incrementCounterBy({
+        key: succeeded
+          ? MetricsKeys.McpToolExecutionSucceeded
+          : MetricsKeys.McpToolExecutionFailed,
         amount: 1,
+        attributes: { tool: metricToolName },
+      });
+
+      this.metricsService.recordHistogram({
+        key: MetricsKeys.McpToolOutputTokens,
+        value: estimateToolOutputTokens(result),
+        unit: 'token',
+        attributes: { tool: metricToolName },
       });
 
       return wrapJsonRpcResponse(id, {
@@ -75,9 +108,10 @@ export class McpToolExecutorService {
         },
       });
     } catch (executionError) {
-      void this.metricsService.incrementCounterBy({
+      this.metricsService.incrementCounterBy({
         key: MetricsKeys.McpToolExecutionFailed,
         amount: 1,
+        attributes: { tool: metricToolName },
       });
 
       return wrapJsonRpcResponse(id, {

packages/twenty-server/src/engine/core-modules/metrics/types/metrics-keys.type.ts

@@ -25,8 +25,17 @@ export enum MetricsKeys {
   WorkflowRunSystemError = 'workflow-run/system-error',
   AiChatToolExecutionSucceeded = 'ai-chat/tool-execution-succeeded',
   AiChatToolExecutionFailed = 'ai-chat/tool-execution-failed',
+  AiChatToolLearnedSucceeded = 'ai-chat/tool-learned-succeeded',
+  AiChatToolLearnedFailed = 'ai-chat/tool-learned-failed',
+  AiChatSkillLoadedSucceeded = 'ai-chat/skill-loaded-succeeded',
+  AiChatSkillLoadedFailed = 'ai-chat/skill-loaded-failed',
+  WorkflowAgentToolExecutionSucceeded = 'workflow-agent/tool-execution-succeeded',
+  WorkflowAgentToolExecutionFailed = 'workflow-agent/tool-execution-failed',
   McpToolExecutionSucceeded = 'mcp/tool-execution-succeeded',
   McpToolExecutionFailed = 'mcp/tool-execution-failed',
+  AiChatToolOutputTokens = 'ai-chat/tool-output-tokens',
+  WorkflowAgentToolOutputTokens = 'workflow-agent/tool-output-tokens',
+  McpToolOutputTokens = 'mcp/tool-output-tokens',
   AiChatInputTokens = 'ai-chat/input-tokens',
   AiChatOutputTokens = 'ai-chat/output-tokens',
   AiChatCacheReadTokens = 'ai-chat/cache-read-tokens',

packages/twenty-server/src/engine/core-modules/tool-provider/constants/database-crud-operation.const.ts

@@ -0,0 +1,14 @@
+export const DATABASE_CRUD_OPERATIONS = [
+  'find_many',
+  'find_one',
+  'create_one',
+  'create_many',
+  'update_one',
+  'update_many',
+  'upsert_many',
+  'delete_one',
+  'delete_many',
+  'group_by',
+] as const;
+
+export type DatabaseCrudOperation = (typeof DATABASE_CRUD_OPERATIONS)[number];

packages/twenty-server/src/engine/core-modules/tool-provider/types/database-crud-operation.type.ts

@@ -1,11 +0,0 @@
-export type DatabaseCrudOperation =
-  | 'find_many'
-  | 'find_one'
-  | 'create_one'
-  | 'create_many'
-  | 'update_one'
-  | 'update_many'
-  | 'upsert_many'
-  | 'delete_one'
-  | 'delete_many'
-  | 'group_by';

packages/twenty-server/src/engine/core-modules/tool-provider/types/tool-execution-ref.type.ts

@@ -1,4 +1,4 @@
-import { type DatabaseCrudOperation } from 'src/engine/core-modules/tool-provider/types/database-crud-operation.type';
+import { type DatabaseCrudOperation } from 'src/engine/core-modules/tool-provider/constants/database-crud-operation.const';
 
 export type ToolExecutionRef =
   | {

packages/twenty-server/src/engine/core-modules/tool-provider/utils/__tests__/resolve-tool-name.util.spec.ts

@@ -0,0 +1,59 @@
+import { EXECUTE_TOOL_TOOL_NAME } from 'src/engine/core-modules/tool-provider/tools';
+import { resolveToolName } from 'src/engine/core-modules/tool-provider/utils/resolve-tool-name.util';
+
+describe('resolveToolName', () => {
+  it('returns the inner toolName when the wrapper is execute_tool', () => {
+    const resolved = resolveToolName({
+      toolName: EXECUTE_TOOL_TOOL_NAME,
+      input: { toolName: 'find_records', arguments: { limit: 10 } },
+    });
+
+    expect(resolved).toBe('find_records');
+  });
+
+  it('returns the part toolName as-is for non-execute_tool wrappers', () => {
+    expect(
+      resolveToolName({
+        toolName: 'learn_tools',
+        input: { toolNames: ['find_records'] },
+      }),
+    ).toBe('learn_tools');
+
+    expect(
+      resolveToolName({
+        toolName: 'load_skills',
+        input: { skillNames: ['workflow-building'] },
+      }),
+    ).toBe('load_skills');
+
+    expect(
+      resolveToolName({
+        toolName: 'app_exa_web_search',
+        input: { query: 'twenty crm' },
+      }),
+    ).toBe('app_exa_web_search');
+  });
+
+  it('falls back to a sentinel when execute_tool input is malformed', () => {
+    expect(
+      resolveToolName({
+        toolName: EXECUTE_TOOL_TOOL_NAME,
+        input: undefined,
+      }),
+    ).toBe(`${EXECUTE_TOOL_TOOL_NAME}:unknown`);
+
+    expect(
+      resolveToolName({
+        toolName: EXECUTE_TOOL_TOOL_NAME,
+        input: { toolName: '' },
+      }),
+    ).toBe(`${EXECUTE_TOOL_TOOL_NAME}:unknown`);
+
+    expect(
+      resolveToolName({
+        toolName: EXECUTE_TOOL_TOOL_NAME,
+        input: { toolName: 42 },
+      }),
+    ).toBe(`${EXECUTE_TOOL_TOOL_NAME}:unknown`);
+  });
+});

packages/twenty-server/src/engine/core-modules/tool-provider/utils/estimate-tool-output-tokens.util.ts

@@ -0,0 +1,20 @@
+import { isDefined } from 'twenty-shared/utils';
+
+const CHARS_PER_TOKEN = 4;
+
+export const estimateToolOutputTokens = (output: unknown): number => {
+  if (!isDefined(output)) {
+    return 0;
+  }
+
+  let serialized: string;
+
+  try {
+    serialized =
+      typeof output === 'string' ? output : (JSON.stringify(output) ?? '');
+  } catch {
+    return 0;
+  }
+
+  return Math.ceil(serialized.length / CHARS_PER_TOKEN);
+};

packages/twenty-server/src/engine/core-modules/tool-provider/utils/get-tool-metric-name.util.ts

@@ -0,0 +1,10 @@
+import { DATABASE_CRUD_OPERATIONS } from 'src/engine/core-modules/tool-provider/constants/database-crud-operation.const';
+
+export const getToolMetricName = (toolName: string): string => {
+  const operation = DATABASE_CRUD_OPERATIONS.find(
+    (crudOperation) =>
+      toolName === crudOperation || toolName.startsWith(`${crudOperation}_`),
+  );
+
+  return operation ?? toolName;
+};

packages/twenty-server/src/engine/core-modules/tool-provider/utils/is-tool-output-successful.util.ts

@@ -0,0 +1,8 @@
+import { isObject } from '@sniptt/guards';
+
+export const isToolOutputSuccessful = (output: unknown): boolean => {
+  const isFailure =
+    isObject(output) && 'success' in output && output.success === false;
+
+  return !isFailure;
+};

packages/twenty-server/src/engine/core-modules/tool-provider/utils/resolve-tool-name.util.ts

@@ -0,0 +1,24 @@
+import { isNonEmptyString, isObject } from '@sniptt/guards';
+
+import {
+  EXECUTE_TOOL_TOOL_NAME,
+  type ExecuteToolInput,
+} from 'src/engine/core-modules/tool-provider/tools';
+
+const hasExecuteToolName = (
+  input: unknown,
+): input is Pick<ExecuteToolInput, 'toolName'> =>
+  isObject(input) && 'toolName' in input && isNonEmptyString(input.toolName);
+
+export const resolveToolName = (part: {
+  toolName: string;
+  input?: unknown;
+}): string => {
+  if (part.toolName !== EXECUTE_TOOL_TOOL_NAME) {
+    return part.toolName;
+  }
+
+  return hasExecuteToolName(part.input)
+    ? part.input.toolName
+    : `${EXECUTE_TOOL_TOOL_NAME}:unknown`;
+};

packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/ai-agent-execution.module.ts

@@ -5,6 +5,7 @@ import { ApplicationModule } from 'src/engine/core-modules/application/applicati
 import { BillingModule } from 'src/engine/core-modules/billing/billing.module';
 import { WorkspaceDomainsModule } from 'src/engine/core-modules/domain/workspace-domains/workspace-domains.module';
 import { FileUrlModule } from 'src/engine/core-modules/file/file-url/file-url.module';
+import { MetricsModule } from 'src/engine/core-modules/metrics/metrics.module';
 import { ToolProviderModule } from 'src/engine/core-modules/tool-provider/tool-provider.module';
 import { UserWorkspaceModule } from 'src/engine/core-modules/user-workspace/user-workspace.module';
 import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity';
@@ -36,6 +37,7 @@ import { AgentRunService } from './services/agent-run.service';
     BillingModule,
     FileUrlModule,
     WorkspaceDomainsModule,
+    MetricsModule,
     UserWorkspaceModule,
     UserRoleModule,
     PermissionsModule,

packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/services/__tests__/agent-async-executor.service.spec.ts

@@ -4,6 +4,7 @@ import { getRepositoryToken } from '@nestjs/typeorm';
 import { generateText } from 'ai';
 
 import { BillingUsageService } from 'src/engine/core-modules/billing/services/billing-usage.service';
+import { MetricsService } from 'src/engine/core-modules/metrics/metrics.service';
 import { ToolRegistryService } from 'src/engine/core-modules/tool-provider/services/tool-registry.service';
 import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity';
 import { AgentAsyncExecutorService } from 'src/engine/metadata-modules/ai/ai-agent-execution/services/agent-async-executor.service';
@@ -111,6 +112,12 @@ describe('AgentAsyncExecutorService — workflow agent role-scoped tool resoluti
             hasAvailableCreditsOrThrow: jest.fn().mockResolvedValue(undefined),
           },
         },
+        {
+          provide: MetricsService,
+          useValue: {
+            incrementCounterForEvent: jest.fn(),
+          },
+        },
         {
           provide: getWorkspaceScopedRepositoryToken(RoleTargetEntity),
           useValue: roleTargetRepository,

packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/services/agent-async-executor.service.ts

@@ -18,8 +18,13 @@ import { type Repository } from 'typeorm';
 import { isUserAuthContext } from 'src/engine/core-modules/auth/guards/is-user-auth-context.guard';
 import { type WorkspaceAuthContext } from 'src/engine/core-modules/auth/types/workspace-auth-context.type';
 import { BillingUsageService } from 'src/engine/core-modules/billing/services/billing-usage.service';
+import { MetricsService } from 'src/engine/core-modules/metrics/metrics.service';
+import { MetricsKeys } from 'src/engine/core-modules/metrics/types/metrics-keys.type';
 import { type ToolProviderContext } from 'src/engine/core-modules/tool-provider/interfaces/tool-provider-context.type';
 import { ToolRegistryService } from 'src/engine/core-modules/tool-provider/services/tool-registry.service';
+import { estimateToolOutputTokens } from 'src/engine/core-modules/tool-provider/utils/estimate-tool-output-tokens.util';
+import { getToolMetricName } from 'src/engine/core-modules/tool-provider/utils/get-tool-metric-name.util';
+import { isToolOutputSuccessful } from 'src/engine/core-modules/tool-provider/utils/is-tool-output-successful.util';
 import { UsageOperationType } from 'src/engine/core-modules/usage/enums/usage-operation-type.enum';
 import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity';
 import { WORKFLOW_AGENT_REGISTRY_TOOL_CATEGORIES } from 'src/engine/metadata-modules/ai/ai-agent-execution/constants/workflow-agent-registry-tool-categories.const';
@@ -80,6 +85,7 @@ export class AgentAsyncExecutorService {
     private readonly nativeToolBinder: NativeToolBinderService,
     private readonly aiBillingService: AiBillingService,
     private readonly billingUsageService: BillingUsageService,
+    private readonly metricsService: MetricsService,
     @InjectWorkspaceScopedRepository(RoleTargetEntity)
     private readonly roleTargetRepository: WorkspaceScopedRepository<RoleTargetEntity>,
     @InjectRepository(WorkspaceEntity)
@@ -236,6 +242,38 @@ export class AgentAsyncExecutorService {
           if (stepHasNoMoreAvailableCredits) {
             hasNoMoreAvailableCredits = true;
           }
+
+          for (const part of step.content) {
+            if (part.type !== 'tool-result' && part.type !== 'tool-error') {
+              continue;
+            }
+
+            const succeeded =
+              part.type === 'tool-result' &&
+              isToolOutputSuccessful(part.output);
+
+            const toolAttributes = {
+              model: registeredModel.modelId,
+              tool: getToolMetricName(part.toolName),
+            };
+
+            this.metricsService.incrementCounterBy({
+              key: succeeded
+                ? MetricsKeys.WorkflowAgentToolExecutionSucceeded
+                : MetricsKeys.WorkflowAgentToolExecutionFailed,
+              amount: 1,
+              attributes: toolAttributes,
+            });
+
+            this.metricsService.recordHistogram({
+              key: MetricsKeys.WorkflowAgentToolOutputTokens,
+              value: estimateToolOutputTokens(
+                part.type === 'tool-result' ? part.output : part.error,
+              ),
+              unit: 'token',
+              attributes: toolAttributes,
+            });
+          }
         },
         experimental_repairToolCall: async ({
           toolCall,

packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/services/chat-execution.service.ts

@@ -1,5 +1,6 @@
 import { Injectable, Logger } from '@nestjs/common';
 
+import { isNonEmptyString, isObject } from '@sniptt/guards';
 import {
   convertToModelMessages,
   type LanguageModelUsage,
@@ -17,7 +18,6 @@ import { getAppPath, isDefined } from 'twenty-shared/utils';
 
 import { MetricsService } from 'src/engine/core-modules/metrics/metrics.service';
 import { MetricsKeys } from 'src/engine/core-modules/metrics/types/metrics-keys.type';
-import { type ToolOutput } from 'src/engine/core-modules/tool/types/tool-output.type';
 import { UsageOperationType } from 'src/engine/core-modules/usage/enums/usage-operation-type.enum';
 
 import { type CodeExecutionStreamEmitter } from 'src/engine/core-modules/tool-provider/interfaces/code-execution-stream-emitter.type';
@@ -34,6 +34,10 @@ import {
   LEARN_TOOLS_TOOL_NAME,
   LOAD_SKILL_TOOL_NAME,
 } from 'src/engine/core-modules/tool-provider/tools';
+import { estimateToolOutputTokens } from 'src/engine/core-modules/tool-provider/utils/estimate-tool-output-tokens.util';
+import { getToolMetricName } from 'src/engine/core-modules/tool-provider/utils/get-tool-metric-name.util';
+import { isToolOutputSuccessful } from 'src/engine/core-modules/tool-provider/utils/is-tool-output-successful.util';
+import { resolveToolName } from 'src/engine/core-modules/tool-provider/utils/resolve-tool-name.util';
 import { type WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity';
 import { AgentActorContextService } from 'src/engine/metadata-modules/ai/ai-agent-execution/services/agent-actor-context.service';
 import { AGENT_CONFIG } from 'src/engine/metadata-modules/ai/ai-agent/constants/agent-config.const';
@@ -463,20 +467,83 @@ export class ChatExecutionService {
             `totalTokens=${step.usage.totalTokens ?? 0}`,
         );
 
-        for (const toolResult of step.toolResults) {
-          const output = toolResult.output as ToolOutput | undefined;
-
-          if (!isDefined(output?.success)) {
+        for (const part of step.content) {
+          if (part.type !== 'tool-result' && part.type !== 'tool-error') {
             continue;
           }
 
-          void this.metricsService.incrementCounterForEvent({
-            key: output.success
+          const succeeded =
+            part.type === 'tool-result' && isToolOutputSuccessful(part.output);
+
+          const outputTokens = estimateToolOutputTokens(
+            part.type === 'tool-result' ? part.output : part.error,
+          );
+
+          const executionAttributes = {
+            model: registeredModel.modelId,
+            tool: getToolMetricName(resolveToolName(part)),
+          };
+
+          this.metricsService.incrementCounterBy({
+            key: succeeded
               ? MetricsKeys.AiChatToolExecutionSucceeded
               : MetricsKeys.AiChatToolExecutionFailed,
-            attributes: { model: registeredModel.modelId },
-            shouldStoreInCache: false,
+            amount: 1,
+            attributes: executionAttributes,
           });
+
+          this.metricsService.recordHistogram({
+            key: MetricsKeys.AiChatToolOutputTokens,
+            value: outputTokens,
+            unit: 'token',
+            attributes: executionAttributes,
+          });
+
+          const { input } = part;
+
+          if (part.toolName === LEARN_TOOLS_TOOL_NAME) {
+            const learntToolNames =
+              isObject(input) && 'toolNames' in input
+                ? input.toolNames
+                : undefined;
+
+            for (const learntToolName of Array.isArray(learntToolNames)
+              ? learntToolNames.filter(isNonEmptyString)
+              : []) {
+              this.metricsService.incrementCounterBy({
+                key: succeeded
+                  ? MetricsKeys.AiChatToolLearnedSucceeded
+                  : MetricsKeys.AiChatToolLearnedFailed,
+                amount: 1,
+                attributes: {
+                  model: registeredModel.modelId,
+                  tool: getToolMetricName(learntToolName),
+                },
+              });
+            }
+          }
+
+          if (part.toolName === LOAD_SKILL_TOOL_NAME) {
+            const loadedSkillNames =
+              isObject(input) && 'skillNames' in input
+                ? input.skillNames
+                : undefined;
+
+            for (const loadedSkillName of Array.isArray(loadedSkillNames)
+              ? loadedSkillNames.filter(isNonEmptyString)
+              : []) {
+              this.metricsService.incrementCounterBy({
+                key: succeeded
+                  ? MetricsKeys.AiChatSkillLoadedSucceeded
+                  : MetricsKeys.AiChatSkillLoadedFailed,
+                amount: 1,
+                attributes: {
+                  model: registeredModel.modelId,
+                  skill: loadedSkillName,
+                },
+              });
+            }
+          }
         }
       },
       onAbort: async ({ steps }) => {