Tool execution metrics (#21587)

<!-- This is an auto-generated description by cubic. -->
<a
href="https://cubic.dev/pr/twentyhq/twenty/pull/21587?utm_source=github"
target="_blank" rel="noopener noreferrer"
data-no-image-dialog="true"><picture><source
media="(prefers-color-scheme: dark)"
srcset="https://www.cubic.dev/buttons/review-in-cubic-dark.svg"><source
media="(prefers-color-scheme: light)"
srcset="https://www.cubic.dev/buttons/review-in-cubic-light.svg"><img
alt="Review in cubic"
src="https://www.cubic.dev/buttons/review-in-cubic-dark.svg"></picture></a>
<!-- End of auto-generated description by cubic. -->

packages/twenty-server/src/engine/core-modules/tool-provider/constants/database-crud-operation.const.ts

@@ -0,0 +1,14 @@
+export const DATABASE_CRUD_OPERATIONS = [
+  'find_many',
+  'find_one',
+  'create_one',
+  'create_many',
+  'update_one',
+  'update_many',
+  'upsert_many',
+  'delete_one',
+  'delete_many',
+  'group_by',
+] as const;
+
+export type DatabaseCrudOperation = (typeof DATABASE_CRUD_OPERATIONS)[number];

packages/twenty-server/src/engine/core-modules/tool-provider/types/database-crud-operation.type.ts

@@ -1,11 +0,0 @@
-export type DatabaseCrudOperation =
-  | 'find_many'
-  | 'find_one'
-  | 'create_one'
-  | 'create_many'
-  | 'update_one'
-  | 'update_many'
-  | 'upsert_many'
-  | 'delete_one'
-  | 'delete_many'
-  | 'group_by';

packages/twenty-server/src/engine/core-modules/tool-provider/types/tool-execution-ref.type.ts

@@ -1,4 +1,4 @@
-import { type DatabaseCrudOperation } from 'src/engine/core-modules/tool-provider/types/database-crud-operation.type';
+import { type DatabaseCrudOperation } from 'src/engine/core-modules/tool-provider/constants/database-crud-operation.const';
 
 export type ToolExecutionRef =
   | {

packages/twenty-server/src/engine/core-modules/tool-provider/utils/__tests__/resolve-tool-name.util.spec.ts

@@ -0,0 +1,59 @@
+import { EXECUTE_TOOL_TOOL_NAME } from 'src/engine/core-modules/tool-provider/tools';
+import { resolveToolName } from 'src/engine/core-modules/tool-provider/utils/resolve-tool-name.util';
+
+describe('resolveToolName', () => {
+  it('returns the inner toolName when the wrapper is execute_tool', () => {
+    const resolved = resolveToolName({
+      toolName: EXECUTE_TOOL_TOOL_NAME,
+      input: { toolName: 'find_records', arguments: { limit: 10 } },
+    });
+
+    expect(resolved).toBe('find_records');
+  });
+
+  it('returns the part toolName as-is for non-execute_tool wrappers', () => {
+    expect(
+      resolveToolName({
+        toolName: 'learn_tools',
+        input: { toolNames: ['find_records'] },
+      }),
+    ).toBe('learn_tools');
+
+    expect(
+      resolveToolName({
+        toolName: 'load_skills',
+        input: { skillNames: ['workflow-building'] },
+      }),
+    ).toBe('load_skills');
+
+    expect(
+      resolveToolName({
+        toolName: 'app_exa_web_search',
+        input: { query: 'twenty crm' },
+      }),
+    ).toBe('app_exa_web_search');
+  });
+
+  it('falls back to a sentinel when execute_tool input is malformed', () => {
+    expect(
+      resolveToolName({
+        toolName: EXECUTE_TOOL_TOOL_NAME,
+        input: undefined,
+      }),
+    ).toBe(`${EXECUTE_TOOL_TOOL_NAME}:unknown`);
+
+    expect(
+      resolveToolName({
+        toolName: EXECUTE_TOOL_TOOL_NAME,
+        input: { toolName: '' },
+      }),
+    ).toBe(`${EXECUTE_TOOL_TOOL_NAME}:unknown`);
+
+    expect(
+      resolveToolName({
+        toolName: EXECUTE_TOOL_TOOL_NAME,
+        input: { toolName: 42 },
+      }),
+    ).toBe(`${EXECUTE_TOOL_TOOL_NAME}:unknown`);
+  });
+});

packages/twenty-server/src/engine/core-modules/tool-provider/utils/estimate-tool-output-tokens.util.ts

@@ -0,0 +1,20 @@
+import { isDefined } from 'twenty-shared/utils';
+
+const CHARS_PER_TOKEN = 4;
+
+export const estimateToolOutputTokens = (output: unknown): number => {
+  if (!isDefined(output)) {
+    return 0;
+  }
+
+  let serialized: string;
+
+  try {
+    serialized =
+      typeof output === 'string' ? output : (JSON.stringify(output) ?? '');
+  } catch {
+    return 0;
+  }
+
+  return Math.ceil(serialized.length / CHARS_PER_TOKEN);
+};

packages/twenty-server/src/engine/core-modules/tool-provider/utils/get-tool-metric-name.util.ts

@@ -0,0 +1,10 @@
+import { DATABASE_CRUD_OPERATIONS } from 'src/engine/core-modules/tool-provider/constants/database-crud-operation.const';
+
+export const getToolMetricName = (toolName: string): string => {
+  const operation = DATABASE_CRUD_OPERATIONS.find(
+    (crudOperation) =>
+      toolName === crudOperation || toolName.startsWith(`${crudOperation}_`),
+  );
+
+  return operation ?? toolName;
+};

packages/twenty-server/src/engine/core-modules/tool-provider/utils/is-tool-output-successful.util.ts

@@ -0,0 +1,8 @@
+import { isObject } from '@sniptt/guards';
+
+export const isToolOutputSuccessful = (output: unknown): boolean => {
+  const isFailure =
+    isObject(output) && 'success' in output && output.success === false;
+
+  return !isFailure;
+};

packages/twenty-server/src/engine/core-modules/tool-provider/utils/resolve-tool-name.util.ts

@@ -0,0 +1,24 @@
+import { isNonEmptyString, isObject } from '@sniptt/guards';
+
+import {
+  EXECUTE_TOOL_TOOL_NAME,
+  type ExecuteToolInput,
+} from 'src/engine/core-modules/tool-provider/tools';
+
+const hasExecuteToolName = (
+  input: unknown,
+): input is Pick<ExecuteToolInput, 'toolName'> =>
+  isObject(input) && 'toolName' in input && isNonEmptyString(input.toolName);
+
+export const resolveToolName = (part: {
+  toolName: string;
+  input?: unknown;
+}): string => {
+  if (part.toolName !== EXECUTE_TOOL_TOOL_NAME) {
+    return part.toolName;
+  }
+
+  return hasExecuteToolName(part.input)
+    ? part.input.toolName
+    : `${EXECUTE_TOOL_TOOL_NAME}:unknown`;
+};