feat(ai-agent): suggest similar tool names when tool discovery misses (#21654)

## Context When the in-product AI agent guesses a tool name that doesn't exist, the discovery tools dead-end it with no way to recover. The most common failure is a singular/plural slip — e.g. the agent tries `group_by_cloud_user` when the real tool is `group_by_cloud_users` (read/bulk tools are always plural; only `find_one_*` is singular). Today both `learn_tools` and `execute_tool` reply with a flat `Could not find: <name>` and no suggestion, so the agent burns turns guessing or gives up. ## Change - Add a `findSimilarToolNames` util that ranks catalog tool names against the missed name by Levenshtein distance (reusing the existing `getEditDistance`), with a small bonus for a shared `<operation>_` prefix so the correct same-operation plural is ranked first rather than a closer-but-different operation (e.g. `find_many_person` → `find_many_people`, not `find_one_person`). - `learn_tools`: when names aren't found, include `suggestions` in the structured result and inline them in the message — `Could not find: group_by_person (did you mean: group_by_people?).` - `execute_tool` (via `ToolRegistryService.resolveAndExecute`): append `Did you mean: …?` to the not-found error, reusing the catalog it already fetched (no extra lookup). The heuristic mirrors the existing workflow variable-path suggestion util (same edit-distance threshold), so behavior is consistent with that prior art. ## Tests - Unit tests for `findSimilarToolNames`: plural recovery, prefix-aware ranking, distance threshold, 3-suggestion cap, empty catalog. - `learn_tools` tool tests: suggestions surfaced on a miss; no suggestion lookup when all names resolve. `nx typecheck twenty-server` passes; `oxlint --type-aware` and `oxfmt --check` are clean on the changed files. https://claude.ai/code/session_01GMjZkJYkqTJogJJTM6AAV8 --- _Generated by [Claude Code](https://claude.ai/code/session_01GMjZkJYkqTJogJJTM6AAV8)_  <a href="https://cubic.dev/pr/twentyhq/twenty/pull/21654?utm_source=github" target="_blank" rel="noopener noreferrer" data-no-image-dialog="true"><picture><source media="(prefers-color-scheme: dark)" srcset="https://www.cubic.dev/buttons/review-in-cubic-dark.svg"><source media="(prefers-color-scheme: light)" srcset="https://www.cubic.dev/buttons/review-in-cubic-light.svg"><img alt="Review in cubic" src="https://www.cubic.dev/buttons/review-in-cubic-dark.svg"></picture></a>
2026-06-16 15:06:57 +02:00
parent e515f174a4
commit 1e8169ca3e
5 changed files with 281 additions and 10 deletions
@@ -13,6 +13,7 @@ import { type LearnToolsAspect } from 'src/engine/core-modules/tool-provider/too
 import { type ToolContext } from 'src/engine/core-modules/tool-provider/types/tool-context.type';
 import { type ToolDescriptor } from 'src/engine/core-modules/tool-provider/types/tool-descriptor.type';
 import { type ToolIndexEntry } from 'src/engine/core-modules/tool-provider/types/tool-index-entry.type';
+import { findSimilarToolNames } from 'src/engine/core-modules/tool-provider/utils/find-similar-tool-names.util';
 import { wrapWithErrorHandler } from 'src/engine/core-modules/tool-provider/utils/tool-error.util';
 import { type ToolOutput } from 'src/engine/core-modules/tool/types/tool-output.type';
 import {
@@ -241,6 +242,31 @@ export class ToolRegistryService {
    });
  }

+  async suggestSimilarToolNames(
+    toolNames: string[],
+    context: ToolContext,
+  ): Promise<Record<string, string[]>> {
+    const fullContext = this.buildContextFromToolContext(context);
+
+    const catalog = await this.getCatalog(fullContext);
+    const candidateToolNames = catalog.map((entry) => entry.name);
+
+    const suggestionsByToolName: Record<string, string[]> = {};
+
+    for (const toolName of toolNames) {
+      const similarToolNames = findSimilarToolNames(
+        toolName,
+        candidateToolNames,
+      );
+
+      if (similarToolNames.length > 0) {
+        suggestionsByToolName[toolName] = similarToolNames;
+      }
+    }
+
+    return suggestionsByToolName;
+  }
+
  async resolveAndExecute(
    toolName: string,
    args: Record<string, unknown> | undefined,
@@ -254,10 +280,19 @@ export class ToolRegistryService {
      const entry = index.find((indexEntry) => indexEntry.name === toolName);

      if (!entry) {
+        const similarToolNames = findSimilarToolNames(
+          toolName,
+          index.map((indexEntry) => indexEntry.name),
+        );
+        const suggestionHint =
+          similarToolNames.length > 0
+            ? ` Did you mean: ${similarToolNames.join(', ')}?`
+            : '';
+
        return {
          success: false,
          message: `Tool "${toolName}" not found`,
-          error: `Tool "${toolName}" not found. Use learn_tools to discover available tools.`,
+          error: `Tool "${toolName}" not found.${suggestionHint} Use learn_tools to discover available tools.`,
        };
      }

@@ -0,0 +1,105 @@
+import { type ToolRegistryService } from 'src/engine/core-modules/tool-provider/services/tool-registry.service';
+import { createLearnToolsTool } from 'src/engine/core-modules/tool-provider/tools/learn-tools.tool';
+import { type ToolContext } from 'src/engine/core-modules/tool-provider/types/tool-context.type';
+
+describe('createLearnToolsTool', () => {
+  const context = {} as ToolContext;
+
+  it('surfaces "did you mean" suggestions when a tool name is not found', async () => {
+    const toolRegistry = {
+      getToolInfo: jest.fn().mockResolvedValue([]),
+      suggestSimilarToolNames: jest
+        .fn()
+        .mockResolvedValue({ group_by_person: ['group_by_people'] }),
+    } as unknown as ToolRegistryService;
+
+    const learnTools = createLearnToolsTool(toolRegistry, context);
+
+    const result = await learnTools.execute({
+      toolNames: ['group_by_person'],
+      aspects: ['description', 'schema'],
+    });
+
+    expect(result.notFound).toEqual(['group_by_person']);
+    expect(result.suggestions).toEqual({
+      group_by_person: ['group_by_people'],
+    });
+    expect(result.message).toContain(
+      'group_by_person (did you mean: group_by_people?)',
+    );
+  });
+
+  it('does not look up suggestions when every tool resolves', async () => {
+    const suggestSimilarToolNames = jest.fn();
+    const toolRegistry = {
+      getToolInfo: jest
+        .fn()
+        .mockResolvedValue([
+          { name: 'group_by_people', description: 'Group people' },
+        ]),
+      suggestSimilarToolNames,
+    } as unknown as ToolRegistryService;
+
+    const learnTools = createLearnToolsTool(toolRegistry, context);
+
+    const result = await learnTools.execute({
+      toolNames: ['group_by_people'],
+      aspects: ['description'],
+    });
+
+    expect(result.notFound).toEqual([]);
+    expect(result.suggestions).toBeUndefined();
+    expect(suggestSimilarToolNames).not.toHaveBeenCalled();
+    expect(result.message).toBe('Learned 1 tool: group_by_people.');
+  });
+
+  it('pluralizes the learned-tools count', async () => {
+    const toolRegistry = {
+      getToolInfo: jest
+        .fn()
+        .mockResolvedValue([
+          { name: 'find_many_people' },
+          { name: 'group_by_people' },
+        ]),
+      suggestSimilarToolNames: jest.fn(),
+    } as unknown as ToolRegistryService;
+
+    const learnTools = createLearnToolsTool(toolRegistry, context);
+
+    const result = await learnTools.execute({
+      toolNames: ['find_many_people', 'group_by_people'],
+      aspects: ['description'],
+    });
+
+    expect(result.message).toBe(
+      'Learned 2 tools: find_many_people, group_by_people.',
+    );
+  });
+
+  it('does not report excluded tools as not found or suggest alternatives', async () => {
+    const suggestSimilarToolNames = jest.fn();
+    const toolRegistry = {
+      getToolInfo: jest.fn().mockResolvedValue([]),
+      suggestSimilarToolNames,
+    } as unknown as ToolRegistryService;
+
+    const learnTools = createLearnToolsTool(
+      toolRegistry,
+      context,
+      new Set(['code_interpreter']),
+    );
+
+    const result = await learnTools.execute({
+      toolNames: ['code_interpreter'],
+      aspects: ['description'],
+    });
+
+    expect(toolRegistry.getToolInfo).toHaveBeenCalledWith([], context, [
+      'description',
+    ]);
+    expect(result.notFound).toEqual([]);
+    expect(result.suggestions).toBeUndefined();
+    expect(suggestSimilarToolNames).not.toHaveBeenCalled();
+    expect(result.message).toBe('No matching tools found.');
+  });
+});
@@ -35,6 +35,7 @@ export type LearnToolsResultEntry = {
 export type LearnToolsResult = {
  tools: LearnToolsResultEntry[];
  notFound: string[];
+  suggestions?: Record<string, string[]>;
  message: string;
 };

@@ -59,21 +60,49 @@ export const createLearnToolsTool = (
      aspects,
    );

-    const foundNames = new Set(toolInfos.map((t) => t.name));
-    const notFound = toolNames.filter((name) => !foundNames.has(name));
+    const foundNames = new Set(toolInfos.map((toolInfo) => toolInfo.name));
+    // Base notFound on allowedNames so excluded tools aren't surfaced as
+    // missing (which would also trigger misleading suggestions).
+    const notFound = allowedNames.filter((name) => !foundNames.has(name));
+
+    const suggestions: Record<string, string[]> =
+      notFound.length > 0
+        ? await toolRegistry.suggestSimilarToolNames(notFound, context)
+        : {};
+
+    const messageParts: string[] = [];
+
+    if (toolInfos.length > 0) {
+      const learnedNames = toolInfos.map((toolInfo) => toolInfo.name);
+      const toolNoun = learnedNames.length === 1 ? 'tool' : 'tools';
+
+      messageParts.push(
+        `Learned ${learnedNames.length} ${toolNoun}: ${learnedNames.join(', ')}`,
+      );
+    }

    if (notFound.length > 0) {
-      return {
-        tools: toolInfos,
-        notFound,
-        message: `Learned ${toolInfos.length} tool(s). Could not find: ${notFound.join(', ')}.`,
-      };
+      const notFoundDescription = notFound
+        .map((name) => {
+          const similarToolNames = suggestions[name];
+
+          return similarToolNames?.length
+            ? `${name} (did you mean: ${similarToolNames.join(', ')}?)`
+            : name;
+        })
+        .join('; ');
+
+      messageParts.push(`Could not find: ${notFoundDescription}`);
    }

    return {
      tools: toolInfos,
-      notFound: [],
-      message: `Learned ${toolInfos.length} tool(s): ${toolInfos.map((t) => t.name).join(', ')}.`,
+      notFound,
+      ...(Object.keys(suggestions).length > 0 && { suggestions }),
+      message:
+        messageParts.length > 0
+          ? `${messageParts.join('. ')}.`
+          : 'No matching tools found.',
    };
  },
 });
@@ -0,0 +1,59 @@
+import { findSimilarToolNames } from 'src/engine/core-modules/tool-provider/utils/find-similar-tool-names.util';
+
+describe('findSimilarToolNames', () => {
+  const catalog = [
+    'find_many_people',
+    'find_one_person',
+    'group_by_people',
+    'group_by_companies',
+    'create_one_person',
+    'update_many_people',
+  ];
+
+  it('ranks the plural variant first for a singular group_by name', () => {
+    const suggestions = findSimilarToolNames('group_by_person', catalog);
+
+    expect(suggestions).toContain('group_by_people');
+    expect(suggestions[0]).toBe('group_by_people');
+  });
+
+  it('ranks the plural variant first for a singular find_many name', () => {
+    const suggestions = findSimilarToolNames('find_many_person', catalog);
+
+    expect(suggestions[0]).toBe('find_many_people');
+  });
+
+  it('ranks the closest candidate first for a typo', () => {
+    expect(findSimilarToolNames('group_by_peple', catalog)[0]).toBe(
+      'group_by_people',
+    );
+  });
+
+  it('returns an empty array when nothing is close enough', () => {
+    expect(findSimilarToolNames('send_email', catalog)).toEqual([]);
+  });
+
+  it('never suggests an exact match', () => {
+    expect(findSimilarToolNames('group_by_people', catalog)).not.toContain(
+      'group_by_people',
+    );
+  });
+
+  it('caps the number of suggestions at three', () => {
+    const manyCandidates = [
+      'find_many_opportunities',
+      'find_many_opportunity',
+      'find_many_opportunitiez',
+      'find_many_oportunities',
+      'find_many_opportunitis',
+    ];
+
+    expect(
+      findSimilarToolNames('find_many_opportunites', manyCandidates),
+    ).toHaveLength(3);
+  });
+
+  it('handles an empty catalog', () => {
+    expect(findSimilarToolNames('group_by_person', [])).toEqual([]);
+  });
+});
@@ -0,0 +1,43 @@
+import { getEditDistance } from 'twenty-shared/workflow';
+
+const MAX_TOOL_NAME_SUGGESTIONS = 3;
+const PREFIX_MATCH_WEIGHT = 0.5;
+
+const getCommonPrefixLength = (left: string, right: string): number => {
+  const maxLength = Math.min(left.length, right.length);
+  let index = 0;
+
+  while (index < maxLength && left[index] === right[index]) {
+    index++;
+  }
+
+  return index;
+};
+
+export const findSimilarToolNames = (
+  toolName: string,
+  candidateToolNames: string[],
+): string[] =>
+  candidateToolNames
+    .map((candidate) => ({
+      candidate,
+      distance: getEditDistance(toolName, candidate),
+      prefixLength: getCommonPrefixLength(toolName, candidate),
+    }))
+    .filter(
+      ({ candidate, distance }) =>
+        distance > 0 && distance <= Math.ceil(candidate.length / 2),
+    )
+    // Reward a shared operation prefix so the same-operation candidate (the
+    // correct plural form) ranks ahead of a closer-but-different operation.
+    .sort((left, right) => {
+      const leftScore = left.distance - PREFIX_MATCH_WEIGHT * left.prefixLength;
+      const rightScore =
+        right.distance - PREFIX_MATCH_WEIGHT * right.prefixLength;
+
+      return (
+        leftScore - rightScore || left.candidate.localeCompare(right.candidate)
+      );
+    })
+    .slice(0, MAX_TOOL_NAME_SUGGESTIONS)
+    .map(({ candidate }) => candidate);