From ccc77932a0bc1a5f55bc67d9aeee8aa5a34154e6 Mon Sep 17 00:00:00 2001 From: Abdul Rahman <81605929+abdulrahmancodes@users.noreply.github.com> Date: Thu, 18 Jun 2026 14:28:47 +0530 Subject: [PATCH] Tool execution metrics (#21587) Review in cubic --- .../mcp-tool-executor.service.spec.ts | 3 +- .../mcp/services/mcp-tool-executor.service.ts | 46 ++++++++-- .../metrics/types/metrics-keys.type.ts | 9 ++ .../database-crud-operation.const.ts | 14 +++ .../types/database-crud-operation.type.ts | 11 --- .../types/tool-execution-ref.type.ts | 2 +- .../__tests__/resolve-tool-name.util.spec.ts | 59 +++++++++++++ .../utils/estimate-tool-output-tokens.util.ts | 20 +++++ .../utils/get-tool-metric-name.util.ts | 10 +++ .../utils/is-tool-output-successful.util.ts | 8 ++ .../utils/resolve-tool-name.util.ts | 24 ++++++ .../ai-agent-execution.module.ts | 2 + .../agent-async-executor.service.spec.ts | 7 ++ .../services/agent-async-executor.service.ts | 38 +++++++++ .../services/chat-execution.service.ts | 85 +++++++++++++++++-- 15 files changed, 310 insertions(+), 28 deletions(-) create mode 100644 packages/twenty-server/src/engine/core-modules/tool-provider/constants/database-crud-operation.const.ts delete mode 100644 packages/twenty-server/src/engine/core-modules/tool-provider/types/database-crud-operation.type.ts create mode 100644 packages/twenty-server/src/engine/core-modules/tool-provider/utils/__tests__/resolve-tool-name.util.spec.ts create mode 100644 packages/twenty-server/src/engine/core-modules/tool-provider/utils/estimate-tool-output-tokens.util.ts create mode 100644 packages/twenty-server/src/engine/core-modules/tool-provider/utils/get-tool-metric-name.util.ts create mode 100644 packages/twenty-server/src/engine/core-modules/tool-provider/utils/is-tool-output-successful.util.ts create mode 100644 packages/twenty-server/src/engine/core-modules/tool-provider/utils/resolve-tool-name.util.ts diff --git a/packages/twenty-server/src/engine/api/mcp/services/__tests__/mcp-tool-executor.service.spec.ts b/packages/twenty-server/src/engine/api/mcp/services/__tests__/mcp-tool-executor.service.spec.ts index 5b64b8bf26..d1d2400493 100644 --- a/packages/twenty-server/src/engine/api/mcp/services/__tests__/mcp-tool-executor.service.spec.ts +++ b/packages/twenty-server/src/engine/api/mcp/services/__tests__/mcp-tool-executor.service.spec.ts @@ -4,8 +4,8 @@ import { MCP_PROGRESS_NOTIFICATION_METHOD, TOOL_CALL_PROGRESS_TOKEN_PREFIX, } from 'src/engine/api/mcp/constants/mcp-progress-notification.const'; -import { MetricsService } from 'src/engine/core-modules/metrics/metrics.service'; import { McpToolExecutorService } from 'src/engine/api/mcp/services/mcp-tool-executor.service'; +import { MetricsService } from 'src/engine/core-modules/metrics/metrics.service'; describe('McpToolExecutorService', () => { let service: McpToolExecutorService; @@ -13,6 +13,7 @@ describe('McpToolExecutorService', () => { beforeEach(() => { const metricsService = { incrementCounterBy: jest.fn().mockResolvedValue(undefined), + recordHistogram: jest.fn(), } as unknown as MetricsService; service = new McpToolExecutorService(metricsService); diff --git a/packages/twenty-server/src/engine/api/mcp/services/mcp-tool-executor.service.ts b/packages/twenty-server/src/engine/api/mcp/services/mcp-tool-executor.service.ts index 836d9b4196..38b1cb416e 100644 --- a/packages/twenty-server/src/engine/api/mcp/services/mcp-tool-executor.service.ts +++ b/packages/twenty-server/src/engine/api/mcp/services/mcp-tool-executor.service.ts @@ -1,10 +1,15 @@ import { Injectable } from '@nestjs/common'; +import { isNonEmptyString } from '@sniptt/guards'; import { type ToolSet } from 'ai'; import { isDefined } from 'twenty-shared/utils'; -import { MetricsKeys } from 'src/engine/core-modules/metrics/types/metrics-keys.type'; import { MetricsService } from 'src/engine/core-modules/metrics/metrics.service'; +import { MetricsKeys } from 'src/engine/core-modules/metrics/types/metrics-keys.type'; +import { estimateToolOutputTokens } from 'src/engine/core-modules/tool-provider/utils/estimate-tool-output-tokens.util'; +import { getToolMetricName } from 'src/engine/core-modules/tool-provider/utils/get-tool-metric-name.util'; +import { isToolOutputSuccessful } from 'src/engine/core-modules/tool-provider/utils/is-tool-output-successful.util'; +import { resolveToolName } from 'src/engine/core-modules/tool-provider/utils/resolve-tool-name.util'; import { JSON_RPC_ERROR_CODE } from 'src/engine/api/mcp/constants/json-rpc-error-code.const'; import { @@ -33,14 +38,23 @@ export class McpToolExecutorService { params: Record, sseWriter?: (data: Record) => void, ) { - const toolName = params.name as keyof typeof toolSet; + if (!isNonEmptyString(params.name)) { + return wrapJsonRpcResponse(id, { + error: { + code: JSON_RPC_ERROR_CODE.INVALID_PARAMS, + message: 'Tool name is required', + }, + }); + } + + const toolName = params.name; const tool = toolSet[toolName]; if (!isDefined(tool) || !isDefined(tool.execute)) { return wrapJsonRpcResponse(id, { error: { code: JSON_RPC_ERROR_CODE.INVALID_PARAMS, - message: `Unknown tool: ${String(params.name)}`, + message: `Unknown tool: ${toolName}`, }, }); } @@ -57,15 +71,34 @@ export class McpToolExecutorService { }); } + const metricToolName = getToolMetricName( + resolveToolName({ + toolName, + input: params.arguments, + }), + ); + try { const result = await tool.execute(params.arguments, { toolCallId: '1', messages: [], }); - void this.metricsService.incrementCounterBy({ - key: MetricsKeys.McpToolExecutionSucceeded, + const succeeded = isToolOutputSuccessful(result); + + this.metricsService.incrementCounterBy({ + key: succeeded + ? MetricsKeys.McpToolExecutionSucceeded + : MetricsKeys.McpToolExecutionFailed, amount: 1, + attributes: { tool: metricToolName }, + }); + + this.metricsService.recordHistogram({ + key: MetricsKeys.McpToolOutputTokens, + value: estimateToolOutputTokens(result), + unit: 'token', + attributes: { tool: metricToolName }, }); return wrapJsonRpcResponse(id, { @@ -75,9 +108,10 @@ export class McpToolExecutorService { }, }); } catch (executionError) { - void this.metricsService.incrementCounterBy({ + this.metricsService.incrementCounterBy({ key: MetricsKeys.McpToolExecutionFailed, amount: 1, + attributes: { tool: metricToolName }, }); return wrapJsonRpcResponse(id, { diff --git a/packages/twenty-server/src/engine/core-modules/metrics/types/metrics-keys.type.ts b/packages/twenty-server/src/engine/core-modules/metrics/types/metrics-keys.type.ts index 3435d0c41d..67546b9ef9 100644 --- a/packages/twenty-server/src/engine/core-modules/metrics/types/metrics-keys.type.ts +++ b/packages/twenty-server/src/engine/core-modules/metrics/types/metrics-keys.type.ts @@ -25,8 +25,17 @@ export enum MetricsKeys { WorkflowRunSystemError = 'workflow-run/system-error', AiChatToolExecutionSucceeded = 'ai-chat/tool-execution-succeeded', AiChatToolExecutionFailed = 'ai-chat/tool-execution-failed', + AiChatToolLearnedSucceeded = 'ai-chat/tool-learned-succeeded', + AiChatToolLearnedFailed = 'ai-chat/tool-learned-failed', + AiChatSkillLoadedSucceeded = 'ai-chat/skill-loaded-succeeded', + AiChatSkillLoadedFailed = 'ai-chat/skill-loaded-failed', + WorkflowAgentToolExecutionSucceeded = 'workflow-agent/tool-execution-succeeded', + WorkflowAgentToolExecutionFailed = 'workflow-agent/tool-execution-failed', McpToolExecutionSucceeded = 'mcp/tool-execution-succeeded', McpToolExecutionFailed = 'mcp/tool-execution-failed', + AiChatToolOutputTokens = 'ai-chat/tool-output-tokens', + WorkflowAgentToolOutputTokens = 'workflow-agent/tool-output-tokens', + McpToolOutputTokens = 'mcp/tool-output-tokens', AiChatInputTokens = 'ai-chat/input-tokens', AiChatOutputTokens = 'ai-chat/output-tokens', AiChatCacheReadTokens = 'ai-chat/cache-read-tokens', diff --git a/packages/twenty-server/src/engine/core-modules/tool-provider/constants/database-crud-operation.const.ts b/packages/twenty-server/src/engine/core-modules/tool-provider/constants/database-crud-operation.const.ts new file mode 100644 index 0000000000..6e2e516727 --- /dev/null +++ b/packages/twenty-server/src/engine/core-modules/tool-provider/constants/database-crud-operation.const.ts @@ -0,0 +1,14 @@ +export const DATABASE_CRUD_OPERATIONS = [ + 'find_many', + 'find_one', + 'create_one', + 'create_many', + 'update_one', + 'update_many', + 'upsert_many', + 'delete_one', + 'delete_many', + 'group_by', +] as const; + +export type DatabaseCrudOperation = (typeof DATABASE_CRUD_OPERATIONS)[number]; diff --git a/packages/twenty-server/src/engine/core-modules/tool-provider/types/database-crud-operation.type.ts b/packages/twenty-server/src/engine/core-modules/tool-provider/types/database-crud-operation.type.ts deleted file mode 100644 index bc326023a9..0000000000 --- a/packages/twenty-server/src/engine/core-modules/tool-provider/types/database-crud-operation.type.ts +++ /dev/null @@ -1,11 +0,0 @@ -export type DatabaseCrudOperation = - | 'find_many' - | 'find_one' - | 'create_one' - | 'create_many' - | 'update_one' - | 'update_many' - | 'upsert_many' - | 'delete_one' - | 'delete_many' - | 'group_by'; diff --git a/packages/twenty-server/src/engine/core-modules/tool-provider/types/tool-execution-ref.type.ts b/packages/twenty-server/src/engine/core-modules/tool-provider/types/tool-execution-ref.type.ts index 205e840062..73c1386103 100644 --- a/packages/twenty-server/src/engine/core-modules/tool-provider/types/tool-execution-ref.type.ts +++ b/packages/twenty-server/src/engine/core-modules/tool-provider/types/tool-execution-ref.type.ts @@ -1,4 +1,4 @@ -import { type DatabaseCrudOperation } from 'src/engine/core-modules/tool-provider/types/database-crud-operation.type'; +import { type DatabaseCrudOperation } from 'src/engine/core-modules/tool-provider/constants/database-crud-operation.const'; export type ToolExecutionRef = | { diff --git a/packages/twenty-server/src/engine/core-modules/tool-provider/utils/__tests__/resolve-tool-name.util.spec.ts b/packages/twenty-server/src/engine/core-modules/tool-provider/utils/__tests__/resolve-tool-name.util.spec.ts new file mode 100644 index 0000000000..e5d2a08a5f --- /dev/null +++ b/packages/twenty-server/src/engine/core-modules/tool-provider/utils/__tests__/resolve-tool-name.util.spec.ts @@ -0,0 +1,59 @@ +import { EXECUTE_TOOL_TOOL_NAME } from 'src/engine/core-modules/tool-provider/tools'; +import { resolveToolName } from 'src/engine/core-modules/tool-provider/utils/resolve-tool-name.util'; + +describe('resolveToolName', () => { + it('returns the inner toolName when the wrapper is execute_tool', () => { + const resolved = resolveToolName({ + toolName: EXECUTE_TOOL_TOOL_NAME, + input: { toolName: 'find_records', arguments: { limit: 10 } }, + }); + + expect(resolved).toBe('find_records'); + }); + + it('returns the part toolName as-is for non-execute_tool wrappers', () => { + expect( + resolveToolName({ + toolName: 'learn_tools', + input: { toolNames: ['find_records'] }, + }), + ).toBe('learn_tools'); + + expect( + resolveToolName({ + toolName: 'load_skills', + input: { skillNames: ['workflow-building'] }, + }), + ).toBe('load_skills'); + + expect( + resolveToolName({ + toolName: 'app_exa_web_search', + input: { query: 'twenty crm' }, + }), + ).toBe('app_exa_web_search'); + }); + + it('falls back to a sentinel when execute_tool input is malformed', () => { + expect( + resolveToolName({ + toolName: EXECUTE_TOOL_TOOL_NAME, + input: undefined, + }), + ).toBe(`${EXECUTE_TOOL_TOOL_NAME}:unknown`); + + expect( + resolveToolName({ + toolName: EXECUTE_TOOL_TOOL_NAME, + input: { toolName: '' }, + }), + ).toBe(`${EXECUTE_TOOL_TOOL_NAME}:unknown`); + + expect( + resolveToolName({ + toolName: EXECUTE_TOOL_TOOL_NAME, + input: { toolName: 42 }, + }), + ).toBe(`${EXECUTE_TOOL_TOOL_NAME}:unknown`); + }); +}); diff --git a/packages/twenty-server/src/engine/core-modules/tool-provider/utils/estimate-tool-output-tokens.util.ts b/packages/twenty-server/src/engine/core-modules/tool-provider/utils/estimate-tool-output-tokens.util.ts new file mode 100644 index 0000000000..31ab9ae178 --- /dev/null +++ b/packages/twenty-server/src/engine/core-modules/tool-provider/utils/estimate-tool-output-tokens.util.ts @@ -0,0 +1,20 @@ +import { isDefined } from 'twenty-shared/utils'; + +const CHARS_PER_TOKEN = 4; + +export const estimateToolOutputTokens = (output: unknown): number => { + if (!isDefined(output)) { + return 0; + } + + let serialized: string; + + try { + serialized = + typeof output === 'string' ? output : (JSON.stringify(output) ?? ''); + } catch { + return 0; + } + + return Math.ceil(serialized.length / CHARS_PER_TOKEN); +}; diff --git a/packages/twenty-server/src/engine/core-modules/tool-provider/utils/get-tool-metric-name.util.ts b/packages/twenty-server/src/engine/core-modules/tool-provider/utils/get-tool-metric-name.util.ts new file mode 100644 index 0000000000..982b674ab3 --- /dev/null +++ b/packages/twenty-server/src/engine/core-modules/tool-provider/utils/get-tool-metric-name.util.ts @@ -0,0 +1,10 @@ +import { DATABASE_CRUD_OPERATIONS } from 'src/engine/core-modules/tool-provider/constants/database-crud-operation.const'; + +export const getToolMetricName = (toolName: string): string => { + const operation = DATABASE_CRUD_OPERATIONS.find( + (crudOperation) => + toolName === crudOperation || toolName.startsWith(`${crudOperation}_`), + ); + + return operation ?? toolName; +}; diff --git a/packages/twenty-server/src/engine/core-modules/tool-provider/utils/is-tool-output-successful.util.ts b/packages/twenty-server/src/engine/core-modules/tool-provider/utils/is-tool-output-successful.util.ts new file mode 100644 index 0000000000..ed8cf5d16b --- /dev/null +++ b/packages/twenty-server/src/engine/core-modules/tool-provider/utils/is-tool-output-successful.util.ts @@ -0,0 +1,8 @@ +import { isObject } from '@sniptt/guards'; + +export const isToolOutputSuccessful = (output: unknown): boolean => { + const isFailure = + isObject(output) && 'success' in output && output.success === false; + + return !isFailure; +}; diff --git a/packages/twenty-server/src/engine/core-modules/tool-provider/utils/resolve-tool-name.util.ts b/packages/twenty-server/src/engine/core-modules/tool-provider/utils/resolve-tool-name.util.ts new file mode 100644 index 0000000000..296e6323d2 --- /dev/null +++ b/packages/twenty-server/src/engine/core-modules/tool-provider/utils/resolve-tool-name.util.ts @@ -0,0 +1,24 @@ +import { isNonEmptyString, isObject } from '@sniptt/guards'; + +import { + EXECUTE_TOOL_TOOL_NAME, + type ExecuteToolInput, +} from 'src/engine/core-modules/tool-provider/tools'; + +const hasExecuteToolName = ( + input: unknown, +): input is Pick => + isObject(input) && 'toolName' in input && isNonEmptyString(input.toolName); + +export const resolveToolName = (part: { + toolName: string; + input?: unknown; +}): string => { + if (part.toolName !== EXECUTE_TOOL_TOOL_NAME) { + return part.toolName; + } + + return hasExecuteToolName(part.input) + ? part.input.toolName + : `${EXECUTE_TOOL_TOOL_NAME}:unknown`; +}; diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/ai-agent-execution.module.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/ai-agent-execution.module.ts index 1b9579b22d..9b9ec2db93 100644 --- a/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/ai-agent-execution.module.ts +++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/ai-agent-execution.module.ts @@ -5,6 +5,7 @@ import { ApplicationModule } from 'src/engine/core-modules/application/applicati import { BillingModule } from 'src/engine/core-modules/billing/billing.module'; import { WorkspaceDomainsModule } from 'src/engine/core-modules/domain/workspace-domains/workspace-domains.module'; import { FileUrlModule } from 'src/engine/core-modules/file/file-url/file-url.module'; +import { MetricsModule } from 'src/engine/core-modules/metrics/metrics.module'; import { ToolProviderModule } from 'src/engine/core-modules/tool-provider/tool-provider.module'; import { UserWorkspaceModule } from 'src/engine/core-modules/user-workspace/user-workspace.module'; import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity'; @@ -36,6 +37,7 @@ import { AgentRunService } from './services/agent-run.service'; BillingModule, FileUrlModule, WorkspaceDomainsModule, + MetricsModule, UserWorkspaceModule, UserRoleModule, PermissionsModule, diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/services/__tests__/agent-async-executor.service.spec.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/services/__tests__/agent-async-executor.service.spec.ts index 2aa9aadc15..6bb9dd1b1b 100644 --- a/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/services/__tests__/agent-async-executor.service.spec.ts +++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/services/__tests__/agent-async-executor.service.spec.ts @@ -4,6 +4,7 @@ import { getRepositoryToken } from '@nestjs/typeorm'; import { generateText } from 'ai'; import { BillingUsageService } from 'src/engine/core-modules/billing/services/billing-usage.service'; +import { MetricsService } from 'src/engine/core-modules/metrics/metrics.service'; import { ToolRegistryService } from 'src/engine/core-modules/tool-provider/services/tool-registry.service'; import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity'; import { AgentAsyncExecutorService } from 'src/engine/metadata-modules/ai/ai-agent-execution/services/agent-async-executor.service'; @@ -111,6 +112,12 @@ describe('AgentAsyncExecutorService — workflow agent role-scoped tool resoluti hasAvailableCreditsOrThrow: jest.fn().mockResolvedValue(undefined), }, }, + { + provide: MetricsService, + useValue: { + incrementCounterForEvent: jest.fn(), + }, + }, { provide: getWorkspaceScopedRepositoryToken(RoleTargetEntity), useValue: roleTargetRepository, diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/services/agent-async-executor.service.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/services/agent-async-executor.service.ts index d575df992c..c48a602f38 100644 --- a/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/services/agent-async-executor.service.ts +++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/services/agent-async-executor.service.ts @@ -18,8 +18,13 @@ import { type Repository } from 'typeorm'; import { isUserAuthContext } from 'src/engine/core-modules/auth/guards/is-user-auth-context.guard'; import { type WorkspaceAuthContext } from 'src/engine/core-modules/auth/types/workspace-auth-context.type'; import { BillingUsageService } from 'src/engine/core-modules/billing/services/billing-usage.service'; +import { MetricsService } from 'src/engine/core-modules/metrics/metrics.service'; +import { MetricsKeys } from 'src/engine/core-modules/metrics/types/metrics-keys.type'; import { type ToolProviderContext } from 'src/engine/core-modules/tool-provider/interfaces/tool-provider-context.type'; import { ToolRegistryService } from 'src/engine/core-modules/tool-provider/services/tool-registry.service'; +import { estimateToolOutputTokens } from 'src/engine/core-modules/tool-provider/utils/estimate-tool-output-tokens.util'; +import { getToolMetricName } from 'src/engine/core-modules/tool-provider/utils/get-tool-metric-name.util'; +import { isToolOutputSuccessful } from 'src/engine/core-modules/tool-provider/utils/is-tool-output-successful.util'; import { UsageOperationType } from 'src/engine/core-modules/usage/enums/usage-operation-type.enum'; import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity'; import { WORKFLOW_AGENT_REGISTRY_TOOL_CATEGORIES } from 'src/engine/metadata-modules/ai/ai-agent-execution/constants/workflow-agent-registry-tool-categories.const'; @@ -80,6 +85,7 @@ export class AgentAsyncExecutorService { private readonly nativeToolBinder: NativeToolBinderService, private readonly aiBillingService: AiBillingService, private readonly billingUsageService: BillingUsageService, + private readonly metricsService: MetricsService, @InjectWorkspaceScopedRepository(RoleTargetEntity) private readonly roleTargetRepository: WorkspaceScopedRepository, @InjectRepository(WorkspaceEntity) @@ -236,6 +242,38 @@ export class AgentAsyncExecutorService { if (stepHasNoMoreAvailableCredits) { hasNoMoreAvailableCredits = true; } + + for (const part of step.content) { + if (part.type !== 'tool-result' && part.type !== 'tool-error') { + continue; + } + + const succeeded = + part.type === 'tool-result' && + isToolOutputSuccessful(part.output); + + const toolAttributes = { + model: registeredModel.modelId, + tool: getToolMetricName(part.toolName), + }; + + this.metricsService.incrementCounterBy({ + key: succeeded + ? MetricsKeys.WorkflowAgentToolExecutionSucceeded + : MetricsKeys.WorkflowAgentToolExecutionFailed, + amount: 1, + attributes: toolAttributes, + }); + + this.metricsService.recordHistogram({ + key: MetricsKeys.WorkflowAgentToolOutputTokens, + value: estimateToolOutputTokens( + part.type === 'tool-result' ? part.output : part.error, + ), + unit: 'token', + attributes: toolAttributes, + }); + } }, experimental_repairToolCall: async ({ toolCall, diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/services/chat-execution.service.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/services/chat-execution.service.ts index 8d0aeec329..900f82a33e 100644 --- a/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/services/chat-execution.service.ts +++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/services/chat-execution.service.ts @@ -1,5 +1,6 @@ import { Injectable, Logger } from '@nestjs/common'; +import { isNonEmptyString, isObject } from '@sniptt/guards'; import { convertToModelMessages, type LanguageModelUsage, @@ -17,7 +18,6 @@ import { getAppPath, isDefined } from 'twenty-shared/utils'; import { MetricsService } from 'src/engine/core-modules/metrics/metrics.service'; import { MetricsKeys } from 'src/engine/core-modules/metrics/types/metrics-keys.type'; -import { type ToolOutput } from 'src/engine/core-modules/tool/types/tool-output.type'; import { UsageOperationType } from 'src/engine/core-modules/usage/enums/usage-operation-type.enum'; import { type CodeExecutionStreamEmitter } from 'src/engine/core-modules/tool-provider/interfaces/code-execution-stream-emitter.type'; @@ -34,6 +34,10 @@ import { LEARN_TOOLS_TOOL_NAME, LOAD_SKILL_TOOL_NAME, } from 'src/engine/core-modules/tool-provider/tools'; +import { estimateToolOutputTokens } from 'src/engine/core-modules/tool-provider/utils/estimate-tool-output-tokens.util'; +import { getToolMetricName } from 'src/engine/core-modules/tool-provider/utils/get-tool-metric-name.util'; +import { isToolOutputSuccessful } from 'src/engine/core-modules/tool-provider/utils/is-tool-output-successful.util'; +import { resolveToolName } from 'src/engine/core-modules/tool-provider/utils/resolve-tool-name.util'; import { type WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity'; import { AgentActorContextService } from 'src/engine/metadata-modules/ai/ai-agent-execution/services/agent-actor-context.service'; import { AGENT_CONFIG } from 'src/engine/metadata-modules/ai/ai-agent/constants/agent-config.const'; @@ -463,20 +467,83 @@ export class ChatExecutionService { `totalTokens=${step.usage.totalTokens ?? 0}`, ); - for (const toolResult of step.toolResults) { - const output = toolResult.output as ToolOutput | undefined; - - if (!isDefined(output?.success)) { + for (const part of step.content) { + if (part.type !== 'tool-result' && part.type !== 'tool-error') { continue; } - void this.metricsService.incrementCounterForEvent({ - key: output.success + const succeeded = + part.type === 'tool-result' && isToolOutputSuccessful(part.output); + + const outputTokens = estimateToolOutputTokens( + part.type === 'tool-result' ? part.output : part.error, + ); + + const executionAttributes = { + model: registeredModel.modelId, + tool: getToolMetricName(resolveToolName(part)), + }; + + this.metricsService.incrementCounterBy({ + key: succeeded ? MetricsKeys.AiChatToolExecutionSucceeded : MetricsKeys.AiChatToolExecutionFailed, - attributes: { model: registeredModel.modelId }, - shouldStoreInCache: false, + amount: 1, + attributes: executionAttributes, }); + + this.metricsService.recordHistogram({ + key: MetricsKeys.AiChatToolOutputTokens, + value: outputTokens, + unit: 'token', + attributes: executionAttributes, + }); + + const { input } = part; + + if (part.toolName === LEARN_TOOLS_TOOL_NAME) { + const learntToolNames = + isObject(input) && 'toolNames' in input + ? input.toolNames + : undefined; + + for (const learntToolName of Array.isArray(learntToolNames) + ? learntToolNames.filter(isNonEmptyString) + : []) { + this.metricsService.incrementCounterBy({ + key: succeeded + ? MetricsKeys.AiChatToolLearnedSucceeded + : MetricsKeys.AiChatToolLearnedFailed, + amount: 1, + attributes: { + model: registeredModel.modelId, + tool: getToolMetricName(learntToolName), + }, + }); + } + } + + if (part.toolName === LOAD_SKILL_TOOL_NAME) { + const loadedSkillNames = + isObject(input) && 'skillNames' in input + ? input.skillNames + : undefined; + + for (const loadedSkillName of Array.isArray(loadedSkillNames) + ? loadedSkillNames.filter(isNonEmptyString) + : []) { + this.metricsService.incrementCounterBy({ + key: succeeded + ? MetricsKeys.AiChatSkillLoadedSucceeded + : MetricsKeys.AiChatSkillLoadedFailed, + amount: 1, + attributes: { + model: registeredModel.modelId, + skill: loadedSkillName, + }, + }); + } + } } }, onAbort: async ({ steps }) => {