From 1e8169ca3e32cadfce23d3cc8194c9544d8976dd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Malfait?= Date: Tue, 16 Jun 2026 15:06:57 +0200 Subject: [PATCH] feat(ai-agent): suggest similar tool names when tool discovery misses (#21654) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Context When the in-product AI agent guesses a tool name that doesn't exist, the discovery tools dead-end it with no way to recover. The most common failure is a singular/plural slip — e.g. the agent tries `group_by_cloud_user` when the real tool is `group_by_cloud_users` (read/bulk tools are always plural; only `find_one_*` is singular). Today both `learn_tools` and `execute_tool` reply with a flat `Could not find: ` and no suggestion, so the agent burns turns guessing or gives up. ## Change - Add a `findSimilarToolNames` util that ranks catalog tool names against the missed name by Levenshtein distance (reusing the existing `getEditDistance`), with a small bonus for a shared `_` prefix so the correct same-operation plural is ranked first rather than a closer-but-different operation (e.g. `find_many_person` → `find_many_people`, not `find_one_person`). - `learn_tools`: when names aren't found, include `suggestions` in the structured result and inline them in the message — `Could not find: group_by_person (did you mean: group_by_people?).` - `execute_tool` (via `ToolRegistryService.resolveAndExecute`): append `Did you mean: …?` to the not-found error, reusing the catalog it already fetched (no extra lookup). The heuristic mirrors the existing workflow variable-path suggestion util (same edit-distance threshold), so behavior is consistent with that prior art. ## Tests - Unit tests for `findSimilarToolNames`: plural recovery, prefix-aware ranking, distance threshold, 3-suggestion cap, empty catalog. - `learn_tools` tool tests: suggestions surfaced on a miss; no suggestion lookup when all names resolve. `nx typecheck twenty-server` passes; `oxlint --type-aware` and `oxfmt --check` are clean on the changed files. https://claude.ai/code/session_01GMjZkJYkqTJogJJTM6AAV8 --- _Generated by [Claude Code](https://claude.ai/code/session_01GMjZkJYkqTJogJJTM6AAV8)_ Review in cubic --- .../services/tool-registry.service.ts | 37 +++++- .../tools/__tests__/learn-tools.tool.spec.ts | 105 ++++++++++++++++++ .../tool-provider/tools/learn-tools.tool.ts | 47 ++++++-- .../find-similar-tool-names.util.spec.ts | 59 ++++++++++ .../utils/find-similar-tool-names.util.ts | 43 +++++++ 5 files changed, 281 insertions(+), 10 deletions(-) create mode 100644 packages/twenty-server/src/engine/core-modules/tool-provider/tools/__tests__/learn-tools.tool.spec.ts create mode 100644 packages/twenty-server/src/engine/core-modules/tool-provider/utils/__tests__/find-similar-tool-names.util.spec.ts create mode 100644 packages/twenty-server/src/engine/core-modules/tool-provider/utils/find-similar-tool-names.util.ts diff --git a/packages/twenty-server/src/engine/core-modules/tool-provider/services/tool-registry.service.ts b/packages/twenty-server/src/engine/core-modules/tool-provider/services/tool-registry.service.ts index 099e057943..ad17ad572f 100644 --- a/packages/twenty-server/src/engine/core-modules/tool-provider/services/tool-registry.service.ts +++ b/packages/twenty-server/src/engine/core-modules/tool-provider/services/tool-registry.service.ts @@ -13,6 +13,7 @@ import { type LearnToolsAspect } from 'src/engine/core-modules/tool-provider/too import { type ToolContext } from 'src/engine/core-modules/tool-provider/types/tool-context.type'; import { type ToolDescriptor } from 'src/engine/core-modules/tool-provider/types/tool-descriptor.type'; import { type ToolIndexEntry } from 'src/engine/core-modules/tool-provider/types/tool-index-entry.type'; +import { findSimilarToolNames } from 'src/engine/core-modules/tool-provider/utils/find-similar-tool-names.util'; import { wrapWithErrorHandler } from 'src/engine/core-modules/tool-provider/utils/tool-error.util'; import { type ToolOutput } from 'src/engine/core-modules/tool/types/tool-output.type'; import { @@ -241,6 +242,31 @@ export class ToolRegistryService { }); } + async suggestSimilarToolNames( + toolNames: string[], + context: ToolContext, + ): Promise> { + const fullContext = this.buildContextFromToolContext(context); + + const catalog = await this.getCatalog(fullContext); + const candidateToolNames = catalog.map((entry) => entry.name); + + const suggestionsByToolName: Record = {}; + + for (const toolName of toolNames) { + const similarToolNames = findSimilarToolNames( + toolName, + candidateToolNames, + ); + + if (similarToolNames.length > 0) { + suggestionsByToolName[toolName] = similarToolNames; + } + } + + return suggestionsByToolName; + } + async resolveAndExecute( toolName: string, args: Record | undefined, @@ -254,10 +280,19 @@ export class ToolRegistryService { const entry = index.find((indexEntry) => indexEntry.name === toolName); if (!entry) { + const similarToolNames = findSimilarToolNames( + toolName, + index.map((indexEntry) => indexEntry.name), + ); + const suggestionHint = + similarToolNames.length > 0 + ? ` Did you mean: ${similarToolNames.join(', ')}?` + : ''; + return { success: false, message: `Tool "${toolName}" not found`, - error: `Tool "${toolName}" not found. Use learn_tools to discover available tools.`, + error: `Tool "${toolName}" not found.${suggestionHint} Use learn_tools to discover available tools.`, }; } diff --git a/packages/twenty-server/src/engine/core-modules/tool-provider/tools/__tests__/learn-tools.tool.spec.ts b/packages/twenty-server/src/engine/core-modules/tool-provider/tools/__tests__/learn-tools.tool.spec.ts new file mode 100644 index 0000000000..14655aac88 --- /dev/null +++ b/packages/twenty-server/src/engine/core-modules/tool-provider/tools/__tests__/learn-tools.tool.spec.ts @@ -0,0 +1,105 @@ +import { type ToolRegistryService } from 'src/engine/core-modules/tool-provider/services/tool-registry.service'; +import { createLearnToolsTool } from 'src/engine/core-modules/tool-provider/tools/learn-tools.tool'; +import { type ToolContext } from 'src/engine/core-modules/tool-provider/types/tool-context.type'; + +describe('createLearnToolsTool', () => { + const context = {} as ToolContext; + + it('surfaces "did you mean" suggestions when a tool name is not found', async () => { + const toolRegistry = { + getToolInfo: jest.fn().mockResolvedValue([]), + suggestSimilarToolNames: jest + .fn() + .mockResolvedValue({ group_by_person: ['group_by_people'] }), + } as unknown as ToolRegistryService; + + const learnTools = createLearnToolsTool(toolRegistry, context); + + const result = await learnTools.execute({ + toolNames: ['group_by_person'], + aspects: ['description', 'schema'], + }); + + expect(result.notFound).toEqual(['group_by_person']); + expect(result.suggestions).toEqual({ + group_by_person: ['group_by_people'], + }); + expect(result.message).toContain( + 'group_by_person (did you mean: group_by_people?)', + ); + }); + + it('does not look up suggestions when every tool resolves', async () => { + const suggestSimilarToolNames = jest.fn(); + const toolRegistry = { + getToolInfo: jest + .fn() + .mockResolvedValue([ + { name: 'group_by_people', description: 'Group people' }, + ]), + suggestSimilarToolNames, + } as unknown as ToolRegistryService; + + const learnTools = createLearnToolsTool(toolRegistry, context); + + const result = await learnTools.execute({ + toolNames: ['group_by_people'], + aspects: ['description'], + }); + + expect(result.notFound).toEqual([]); + expect(result.suggestions).toBeUndefined(); + expect(suggestSimilarToolNames).not.toHaveBeenCalled(); + expect(result.message).toBe('Learned 1 tool: group_by_people.'); + }); + + it('pluralizes the learned-tools count', async () => { + const toolRegistry = { + getToolInfo: jest + .fn() + .mockResolvedValue([ + { name: 'find_many_people' }, + { name: 'group_by_people' }, + ]), + suggestSimilarToolNames: jest.fn(), + } as unknown as ToolRegistryService; + + const learnTools = createLearnToolsTool(toolRegistry, context); + + const result = await learnTools.execute({ + toolNames: ['find_many_people', 'group_by_people'], + aspects: ['description'], + }); + + expect(result.message).toBe( + 'Learned 2 tools: find_many_people, group_by_people.', + ); + }); + + it('does not report excluded tools as not found or suggest alternatives', async () => { + const suggestSimilarToolNames = jest.fn(); + const toolRegistry = { + getToolInfo: jest.fn().mockResolvedValue([]), + suggestSimilarToolNames, + } as unknown as ToolRegistryService; + + const learnTools = createLearnToolsTool( + toolRegistry, + context, + new Set(['code_interpreter']), + ); + + const result = await learnTools.execute({ + toolNames: ['code_interpreter'], + aspects: ['description'], + }); + + expect(toolRegistry.getToolInfo).toHaveBeenCalledWith([], context, [ + 'description', + ]); + expect(result.notFound).toEqual([]); + expect(result.suggestions).toBeUndefined(); + expect(suggestSimilarToolNames).not.toHaveBeenCalled(); + expect(result.message).toBe('No matching tools found.'); + }); +}); diff --git a/packages/twenty-server/src/engine/core-modules/tool-provider/tools/learn-tools.tool.ts b/packages/twenty-server/src/engine/core-modules/tool-provider/tools/learn-tools.tool.ts index 4f314bb487..47898ce7e2 100644 --- a/packages/twenty-server/src/engine/core-modules/tool-provider/tools/learn-tools.tool.ts +++ b/packages/twenty-server/src/engine/core-modules/tool-provider/tools/learn-tools.tool.ts @@ -35,6 +35,7 @@ export type LearnToolsResultEntry = { export type LearnToolsResult = { tools: LearnToolsResultEntry[]; notFound: string[]; + suggestions?: Record; message: string; }; @@ -59,21 +60,49 @@ export const createLearnToolsTool = ( aspects, ); - const foundNames = new Set(toolInfos.map((t) => t.name)); - const notFound = toolNames.filter((name) => !foundNames.has(name)); + const foundNames = new Set(toolInfos.map((toolInfo) => toolInfo.name)); + // Base notFound on allowedNames so excluded tools aren't surfaced as + // missing (which would also trigger misleading suggestions). + const notFound = allowedNames.filter((name) => !foundNames.has(name)); + + const suggestions: Record = + notFound.length > 0 + ? await toolRegistry.suggestSimilarToolNames(notFound, context) + : {}; + + const messageParts: string[] = []; + + if (toolInfos.length > 0) { + const learnedNames = toolInfos.map((toolInfo) => toolInfo.name); + const toolNoun = learnedNames.length === 1 ? 'tool' : 'tools'; + + messageParts.push( + `Learned ${learnedNames.length} ${toolNoun}: ${learnedNames.join(', ')}`, + ); + } if (notFound.length > 0) { - return { - tools: toolInfos, - notFound, - message: `Learned ${toolInfos.length} tool(s). Could not find: ${notFound.join(', ')}.`, - }; + const notFoundDescription = notFound + .map((name) => { + const similarToolNames = suggestions[name]; + + return similarToolNames?.length + ? `${name} (did you mean: ${similarToolNames.join(', ')}?)` + : name; + }) + .join('; '); + + messageParts.push(`Could not find: ${notFoundDescription}`); } return { tools: toolInfos, - notFound: [], - message: `Learned ${toolInfos.length} tool(s): ${toolInfos.map((t) => t.name).join(', ')}.`, + notFound, + ...(Object.keys(suggestions).length > 0 && { suggestions }), + message: + messageParts.length > 0 + ? `${messageParts.join('. ')}.` + : 'No matching tools found.', }; }, }); diff --git a/packages/twenty-server/src/engine/core-modules/tool-provider/utils/__tests__/find-similar-tool-names.util.spec.ts b/packages/twenty-server/src/engine/core-modules/tool-provider/utils/__tests__/find-similar-tool-names.util.spec.ts new file mode 100644 index 0000000000..7f548bdd91 --- /dev/null +++ b/packages/twenty-server/src/engine/core-modules/tool-provider/utils/__tests__/find-similar-tool-names.util.spec.ts @@ -0,0 +1,59 @@ +import { findSimilarToolNames } from 'src/engine/core-modules/tool-provider/utils/find-similar-tool-names.util'; + +describe('findSimilarToolNames', () => { + const catalog = [ + 'find_many_people', + 'find_one_person', + 'group_by_people', + 'group_by_companies', + 'create_one_person', + 'update_many_people', + ]; + + it('ranks the plural variant first for a singular group_by name', () => { + const suggestions = findSimilarToolNames('group_by_person', catalog); + + expect(suggestions).toContain('group_by_people'); + expect(suggestions[0]).toBe('group_by_people'); + }); + + it('ranks the plural variant first for a singular find_many name', () => { + const suggestions = findSimilarToolNames('find_many_person', catalog); + + expect(suggestions[0]).toBe('find_many_people'); + }); + + it('ranks the closest candidate first for a typo', () => { + expect(findSimilarToolNames('group_by_peple', catalog)[0]).toBe( + 'group_by_people', + ); + }); + + it('returns an empty array when nothing is close enough', () => { + expect(findSimilarToolNames('send_email', catalog)).toEqual([]); + }); + + it('never suggests an exact match', () => { + expect(findSimilarToolNames('group_by_people', catalog)).not.toContain( + 'group_by_people', + ); + }); + + it('caps the number of suggestions at three', () => { + const manyCandidates = [ + 'find_many_opportunities', + 'find_many_opportunity', + 'find_many_opportunitiez', + 'find_many_oportunities', + 'find_many_opportunitis', + ]; + + expect( + findSimilarToolNames('find_many_opportunites', manyCandidates), + ).toHaveLength(3); + }); + + it('handles an empty catalog', () => { + expect(findSimilarToolNames('group_by_person', [])).toEqual([]); + }); +}); diff --git a/packages/twenty-server/src/engine/core-modules/tool-provider/utils/find-similar-tool-names.util.ts b/packages/twenty-server/src/engine/core-modules/tool-provider/utils/find-similar-tool-names.util.ts new file mode 100644 index 0000000000..52a73664bb --- /dev/null +++ b/packages/twenty-server/src/engine/core-modules/tool-provider/utils/find-similar-tool-names.util.ts @@ -0,0 +1,43 @@ +import { getEditDistance } from 'twenty-shared/workflow'; + +const MAX_TOOL_NAME_SUGGESTIONS = 3; +const PREFIX_MATCH_WEIGHT = 0.5; + +const getCommonPrefixLength = (left: string, right: string): number => { + const maxLength = Math.min(left.length, right.length); + let index = 0; + + while (index < maxLength && left[index] === right[index]) { + index++; + } + + return index; +}; + +export const findSimilarToolNames = ( + toolName: string, + candidateToolNames: string[], +): string[] => + candidateToolNames + .map((candidate) => ({ + candidate, + distance: getEditDistance(toolName, candidate), + prefixLength: getCommonPrefixLength(toolName, candidate), + })) + .filter( + ({ candidate, distance }) => + distance > 0 && distance <= Math.ceil(candidate.length / 2), + ) + // Reward a shared operation prefix so the same-operation candidate (the + // correct plural form) ranks ahead of a closer-but-different operation. + .sort((left, right) => { + const leftScore = left.distance - PREFIX_MATCH_WEIGHT * left.prefixLength; + const rightScore = + right.distance - PREFIX_MATCH_WEIGHT * right.prefixLength; + + return ( + leftScore - rightScore || left.candidate.localeCompare(right.candidate) + ); + }) + .slice(0, MAX_TOOL_NAME_SUGGESTIONS) + .map(({ candidate }) => candidate);