Tool execution metrics (#21587)

<a href="https://cubic.dev/pr/twentyhq/twenty/pull/21587?utm_source=github" target="_blank" rel="noopener noreferrer" data-no-image-dialog="true"><picture><source media="(prefers-color-scheme: dark)" srcset="https://www.cubic.dev/buttons/review-in-cubic-dark.svg"><source media="(prefers-color-scheme: light)" srcset="https://www.cubic.dev/buttons/review-in-cubic-light.svg"><img alt="Review in cubic" src="https://www.cubic.dev/buttons/review-in-cubic-dark.svg"></picture></a>
2026-06-18 14:28:47 +05:30
parent 8034c7725f
commit ccc77932a0
15 changed files with 310 additions and 28 deletions
@@ -5,6 +5,7 @@ import { ApplicationModule } from 'src/engine/core-modules/application/applicati
 import { BillingModule } from 'src/engine/core-modules/billing/billing.module';
 import { WorkspaceDomainsModule } from 'src/engine/core-modules/domain/workspace-domains/workspace-domains.module';
 import { FileUrlModule } from 'src/engine/core-modules/file/file-url/file-url.module';
+import { MetricsModule } from 'src/engine/core-modules/metrics/metrics.module';
 import { ToolProviderModule } from 'src/engine/core-modules/tool-provider/tool-provider.module';
 import { UserWorkspaceModule } from 'src/engine/core-modules/user-workspace/user-workspace.module';
 import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity';
@@ -36,6 +37,7 @@ import { AgentRunService } from './services/agent-run.service';
    BillingModule,
    FileUrlModule,
    WorkspaceDomainsModule,
+    MetricsModule,
    UserWorkspaceModule,
    UserRoleModule,
    PermissionsModule,
@@ -4,6 +4,7 @@ import { getRepositoryToken } from '@nestjs/typeorm';
 import { generateText } from 'ai';

 import { BillingUsageService } from 'src/engine/core-modules/billing/services/billing-usage.service';
+import { MetricsService } from 'src/engine/core-modules/metrics/metrics.service';
 import { ToolRegistryService } from 'src/engine/core-modules/tool-provider/services/tool-registry.service';
 import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity';
 import { AgentAsyncExecutorService } from 'src/engine/metadata-modules/ai/ai-agent-execution/services/agent-async-executor.service';
@@ -111,6 +112,12 @@ describe('AgentAsyncExecutorService — workflow agent role-scoped tool resoluti
            hasAvailableCreditsOrThrow: jest.fn().mockResolvedValue(undefined),
          },
        },
+        {
+          provide: MetricsService,
+          useValue: {
+            incrementCounterForEvent: jest.fn(),
+          },
+        },
        {
          provide: getWorkspaceScopedRepositoryToken(RoleTargetEntity),
          useValue: roleTargetRepository,
@@ -18,8 +18,13 @@ import { type Repository } from 'typeorm';
 import { isUserAuthContext } from 'src/engine/core-modules/auth/guards/is-user-auth-context.guard';
 import { type WorkspaceAuthContext } from 'src/engine/core-modules/auth/types/workspace-auth-context.type';
 import { BillingUsageService } from 'src/engine/core-modules/billing/services/billing-usage.service';
+import { MetricsService } from 'src/engine/core-modules/metrics/metrics.service';
+import { MetricsKeys } from 'src/engine/core-modules/metrics/types/metrics-keys.type';
 import { type ToolProviderContext } from 'src/engine/core-modules/tool-provider/interfaces/tool-provider-context.type';
 import { ToolRegistryService } from 'src/engine/core-modules/tool-provider/services/tool-registry.service';
+import { estimateToolOutputTokens } from 'src/engine/core-modules/tool-provider/utils/estimate-tool-output-tokens.util';
+import { getToolMetricName } from 'src/engine/core-modules/tool-provider/utils/get-tool-metric-name.util';
+import { isToolOutputSuccessful } from 'src/engine/core-modules/tool-provider/utils/is-tool-output-successful.util';
 import { UsageOperationType } from 'src/engine/core-modules/usage/enums/usage-operation-type.enum';
 import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity';
 import { WORKFLOW_AGENT_REGISTRY_TOOL_CATEGORIES } from 'src/engine/metadata-modules/ai/ai-agent-execution/constants/workflow-agent-registry-tool-categories.const';
@@ -80,6 +85,7 @@ export class AgentAsyncExecutorService {
    private readonly nativeToolBinder: NativeToolBinderService,
    private readonly aiBillingService: AiBillingService,
    private readonly billingUsageService: BillingUsageService,
+    private readonly metricsService: MetricsService,
    @InjectWorkspaceScopedRepository(RoleTargetEntity)
    private readonly roleTargetRepository: WorkspaceScopedRepository<RoleTargetEntity>,
    @InjectRepository(WorkspaceEntity)
@@ -236,6 +242,38 @@ export class AgentAsyncExecutorService {
          if (stepHasNoMoreAvailableCredits) {
            hasNoMoreAvailableCredits = true;
          }
+
+          for (const part of step.content) {
+            if (part.type !== 'tool-result' && part.type !== 'tool-error') {
+              continue;
+            }
+
+            const succeeded =
+              part.type === 'tool-result' &&
+              isToolOutputSuccessful(part.output);
+
+            const toolAttributes = {
+              model: registeredModel.modelId,
+              tool: getToolMetricName(part.toolName),
+            };
+
+            this.metricsService.incrementCounterBy({
+              key: succeeded
+                ? MetricsKeys.WorkflowAgentToolExecutionSucceeded
+                : MetricsKeys.WorkflowAgentToolExecutionFailed,
+              amount: 1,
+              attributes: toolAttributes,
+            });
+
+            this.metricsService.recordHistogram({
+              key: MetricsKeys.WorkflowAgentToolOutputTokens,
+              value: estimateToolOutputTokens(
+                part.type === 'tool-result' ? part.output : part.error,
+              ),
+              unit: 'token',
+              attributes: toolAttributes,
+            });
+          }
        },
        experimental_repairToolCall: async ({
          toolCall,
@@ -1,5 +1,6 @@
 import { Injectable, Logger } from '@nestjs/common';

+import { isNonEmptyString, isObject } from '@sniptt/guards';
 import {
  convertToModelMessages,
  type LanguageModelUsage,
@@ -17,7 +18,6 @@ import { getAppPath, isDefined } from 'twenty-shared/utils';

 import { MetricsService } from 'src/engine/core-modules/metrics/metrics.service';
 import { MetricsKeys } from 'src/engine/core-modules/metrics/types/metrics-keys.type';
-import { type ToolOutput } from 'src/engine/core-modules/tool/types/tool-output.type';
 import { UsageOperationType } from 'src/engine/core-modules/usage/enums/usage-operation-type.enum';

 import { type CodeExecutionStreamEmitter } from 'src/engine/core-modules/tool-provider/interfaces/code-execution-stream-emitter.type';
@@ -34,6 +34,10 @@ import {
  LEARN_TOOLS_TOOL_NAME,
  LOAD_SKILL_TOOL_NAME,
 } from 'src/engine/core-modules/tool-provider/tools';
+import { estimateToolOutputTokens } from 'src/engine/core-modules/tool-provider/utils/estimate-tool-output-tokens.util';
+import { getToolMetricName } from 'src/engine/core-modules/tool-provider/utils/get-tool-metric-name.util';
+import { isToolOutputSuccessful } from 'src/engine/core-modules/tool-provider/utils/is-tool-output-successful.util';
+import { resolveToolName } from 'src/engine/core-modules/tool-provider/utils/resolve-tool-name.util';
 import { type WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity';
 import { AgentActorContextService } from 'src/engine/metadata-modules/ai/ai-agent-execution/services/agent-actor-context.service';
 import { AGENT_CONFIG } from 'src/engine/metadata-modules/ai/ai-agent/constants/agent-config.const';
@@ -463,20 +467,83 @@ export class ChatExecutionService {
            `totalTokens=${step.usage.totalTokens ?? 0}`,
        );

-        for (const toolResult of step.toolResults) {
-          const output = toolResult.output as ToolOutput | undefined;
-
-          if (!isDefined(output?.success)) {
+        for (const part of step.content) {
+          if (part.type !== 'tool-result' && part.type !== 'tool-error') {
            continue;
          }

-          void this.metricsService.incrementCounterForEvent({
-            key: output.success
+          const succeeded =
+            part.type === 'tool-result' && isToolOutputSuccessful(part.output);
+
+          const outputTokens = estimateToolOutputTokens(
+            part.type === 'tool-result' ? part.output : part.error,
+          );
+
+          const executionAttributes = {
+            model: registeredModel.modelId,
+            tool: getToolMetricName(resolveToolName(part)),
+          };
+
+          this.metricsService.incrementCounterBy({
+            key: succeeded
              ? MetricsKeys.AiChatToolExecutionSucceeded
              : MetricsKeys.AiChatToolExecutionFailed,
-            attributes: { model: registeredModel.modelId },
-            shouldStoreInCache: false,
+            amount: 1,
+            attributes: executionAttributes,
          });
+
+          this.metricsService.recordHistogram({
+            key: MetricsKeys.AiChatToolOutputTokens,
+            value: outputTokens,
+            unit: 'token',
+            attributes: executionAttributes,
+          });
+
+          const { input } = part;
+
+          if (part.toolName === LEARN_TOOLS_TOOL_NAME) {
+            const learntToolNames =
+              isObject(input) && 'toolNames' in input
+                ? input.toolNames
+                : undefined;
+
+            for (const learntToolName of Array.isArray(learntToolNames)
+              ? learntToolNames.filter(isNonEmptyString)
+              : []) {
+              this.metricsService.incrementCounterBy({
+                key: succeeded
+                  ? MetricsKeys.AiChatToolLearnedSucceeded
+                  : MetricsKeys.AiChatToolLearnedFailed,
+                amount: 1,
+                attributes: {
+                  model: registeredModel.modelId,
+                  tool: getToolMetricName(learntToolName),
+                },
+              });
+            }
+          }
+
+          if (part.toolName === LOAD_SKILL_TOOL_NAME) {
+            const loadedSkillNames =
+              isObject(input) && 'skillNames' in input
+                ? input.skillNames
+                : undefined;
+
+            for (const loadedSkillName of Array.isArray(loadedSkillNames)
+              ? loadedSkillNames.filter(isNonEmptyString)
+              : []) {
+              this.metricsService.incrementCounterBy({
+                key: succeeded
+                  ? MetricsKeys.AiChatSkillLoadedSucceeded
+                  : MetricsKeys.AiChatSkillLoadedFailed,
+                amount: 1,
+                attributes: {
+                  model: registeredModel.modelId,
+                  skill: loadedSkillName,
+                },
+              });
+            }
+          }
        }
      },
      onAbort: async ({ steps }) => {