Tool execution metrics (#21587)

<a href="https://cubic.dev/pr/twentyhq/twenty/pull/21587?utm_source=github" target="_blank" rel="noopener noreferrer" data-no-image-dialog="true"><picture><source media="(prefers-color-scheme: dark)" srcset="https://www.cubic.dev/buttons/review-in-cubic-dark.svg"><source media="(prefers-color-scheme: light)" srcset="https://www.cubic.dev/buttons/review-in-cubic-light.svg"><img alt="Review in cubic" src="https://www.cubic.dev/buttons/review-in-cubic-dark.svg"></picture></a>
2026-06-18 14:28:47 +05:30
parent 8034c7725f
commit ccc77932a0
15 changed files with 310 additions and 28 deletions
@@ -25,8 +25,17 @@ export enum MetricsKeys {
  WorkflowRunSystemError = 'workflow-run/system-error',
  AiChatToolExecutionSucceeded = 'ai-chat/tool-execution-succeeded',
  AiChatToolExecutionFailed = 'ai-chat/tool-execution-failed',
+  AiChatToolLearnedSucceeded = 'ai-chat/tool-learned-succeeded',
+  AiChatToolLearnedFailed = 'ai-chat/tool-learned-failed',
+  AiChatSkillLoadedSucceeded = 'ai-chat/skill-loaded-succeeded',
+  AiChatSkillLoadedFailed = 'ai-chat/skill-loaded-failed',
+  WorkflowAgentToolExecutionSucceeded = 'workflow-agent/tool-execution-succeeded',
+  WorkflowAgentToolExecutionFailed = 'workflow-agent/tool-execution-failed',
  McpToolExecutionSucceeded = 'mcp/tool-execution-succeeded',
  McpToolExecutionFailed = 'mcp/tool-execution-failed',
+  AiChatToolOutputTokens = 'ai-chat/tool-output-tokens',
+  WorkflowAgentToolOutputTokens = 'workflow-agent/tool-output-tokens',
+  McpToolOutputTokens = 'mcp/tool-output-tokens',
  AiChatInputTokens = 'ai-chat/input-tokens',
  AiChatOutputTokens = 'ai-chat/output-tokens',
  AiChatCacheReadTokens = 'ai-chat/cache-read-tokens',
@@ -0,0 +1,14 @@
+export const DATABASE_CRUD_OPERATIONS = [
+  'find_many',
+  'find_one',
+  'create_one',
+  'create_many',
+  'update_one',
+  'update_many',
+  'upsert_many',
+  'delete_one',
+  'delete_many',
+  'group_by',
+] as const;
+
+export type DatabaseCrudOperation = (typeof DATABASE_CRUD_OPERATIONS)[number];
@@ -1,11 +0,0 @@
-export type DatabaseCrudOperation =
-  | 'find_many'
-  | 'find_one'
-  | 'create_one'
-  | 'create_many'
-  | 'update_one'
-  | 'update_many'
-  | 'upsert_many'
-  | 'delete_one'
-  | 'delete_many'
-  | 'group_by';
@@ -1,4 +1,4 @@
-import { type DatabaseCrudOperation } from 'src/engine/core-modules/tool-provider/types/database-crud-operation.type';
+import { type DatabaseCrudOperation } from 'src/engine/core-modules/tool-provider/constants/database-crud-operation.const';

 export type ToolExecutionRef =
  | {
@@ -0,0 +1,59 @@
+import { EXECUTE_TOOL_TOOL_NAME } from 'src/engine/core-modules/tool-provider/tools';
+import { resolveToolName } from 'src/engine/core-modules/tool-provider/utils/resolve-tool-name.util';
+
+describe('resolveToolName', () => {
+  it('returns the inner toolName when the wrapper is execute_tool', () => {
+    const resolved = resolveToolName({
+      toolName: EXECUTE_TOOL_TOOL_NAME,
+      input: { toolName: 'find_records', arguments: { limit: 10 } },
+    });
+
+    expect(resolved).toBe('find_records');
+  });
+
+  it('returns the part toolName as-is for non-execute_tool wrappers', () => {
+    expect(
+      resolveToolName({
+        toolName: 'learn_tools',
+        input: { toolNames: ['find_records'] },
+      }),
+    ).toBe('learn_tools');
+
+    expect(
+      resolveToolName({
+        toolName: 'load_skills',
+        input: { skillNames: ['workflow-building'] },
+      }),
+    ).toBe('load_skills');
+
+    expect(
+      resolveToolName({
+        toolName: 'app_exa_web_search',
+        input: { query: 'twenty crm' },
+      }),
+    ).toBe('app_exa_web_search');
+  });
+
+  it('falls back to a sentinel when execute_tool input is malformed', () => {
+    expect(
+      resolveToolName({
+        toolName: EXECUTE_TOOL_TOOL_NAME,
+        input: undefined,
+      }),
+    ).toBe(`${EXECUTE_TOOL_TOOL_NAME}:unknown`);
+
+    expect(
+      resolveToolName({
+        toolName: EXECUTE_TOOL_TOOL_NAME,
+        input: { toolName: '' },
+      }),
+    ).toBe(`${EXECUTE_TOOL_TOOL_NAME}:unknown`);
+
+    expect(
+      resolveToolName({
+        toolName: EXECUTE_TOOL_TOOL_NAME,
+        input: { toolName: 42 },
+      }),
+    ).toBe(`${EXECUTE_TOOL_TOOL_NAME}:unknown`);
+  });
+});
@@ -0,0 +1,20 @@
+import { isDefined } from 'twenty-shared/utils';
+
+const CHARS_PER_TOKEN = 4;
+
+export const estimateToolOutputTokens = (output: unknown): number => {
+  if (!isDefined(output)) {
+    return 0;
+  }
+
+  let serialized: string;
+
+  try {
+    serialized =
+      typeof output === 'string' ? output : (JSON.stringify(output) ?? '');
+  } catch {
+    return 0;
+  }
+
+  return Math.ceil(serialized.length / CHARS_PER_TOKEN);
+};
@@ -0,0 +1,10 @@
+import { DATABASE_CRUD_OPERATIONS } from 'src/engine/core-modules/tool-provider/constants/database-crud-operation.const';
+
+export const getToolMetricName = (toolName: string): string => {
+  const operation = DATABASE_CRUD_OPERATIONS.find(
+    (crudOperation) =>
+      toolName === crudOperation || toolName.startsWith(`${crudOperation}_`),
+  );
+
+  return operation ?? toolName;
+};
@@ -0,0 +1,8 @@
+import { isObject } from '@sniptt/guards';
+
+export const isToolOutputSuccessful = (output: unknown): boolean => {
+  const isFailure =
+    isObject(output) && 'success' in output && output.success === false;
+
+  return !isFailure;
+};
@@ -0,0 +1,24 @@
+import { isNonEmptyString, isObject } from '@sniptt/guards';
+
+import {
+  EXECUTE_TOOL_TOOL_NAME,
+  type ExecuteToolInput,
+} from 'src/engine/core-modules/tool-provider/tools';
+
+const hasExecuteToolName = (
+  input: unknown,
+): input is Pick<ExecuteToolInput, 'toolName'> =>
+  isObject(input) && 'toolName' in input && isNonEmptyString(input.toolName);
+
+export const resolveToolName = (part: {
+  toolName: string;
+  input?: unknown;
+}): string => {
+  if (part.toolName !== EXECUTE_TOOL_TOOL_NAME) {
+    return part.toolName;
+  }
+
+  return hasExecuteToolName(part.input)
+    ? part.input.toolName
+    : `${EXECUTE_TOOL_TOOL_NAME}:unknown`;
+};