From 6c1c0737b08f4a41d92b55da80852907feefad42 Mon Sep 17 00:00:00 2001 From: nitin <142569587+ehconitin@users.noreply.github.com> Date: Fri, 24 Apr 2026 20:48:19 +0530 Subject: [PATCH] Clarify registry tools vs native model tool binding (#20022) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Intent This is a small foundation cleanup for the tool architecture. The main decision is: registry tools and native SDK/model tools are different things. - Registry tools have descriptors, schemas, catalog entries, and execute through `ToolExecutorService` - Native model tools are opaque AI SDK objects, bound directly into the model `ToolSet` - Surfaces still own their policy: chat, MCP, and workflow agents decide what they expose ## What changed - Removed `NATIVE_MODEL` from `ToolCategory` - Kept `ToolRegistryService` focused on registry-backed tools only - Moved native model tool binding through `NativeToolBinderService` - Reused native binding from chat instead of duplicating provider-specific web-search logic - Kept MCP local execution exclusions in a dedicated constant - Moved surface-specific constants into dedicated constant files ## What comes next - Move hardcoded chat app preloads, like Exa web search, into app/manifest metadata - Decide a clearer policy for local runtime tools like code interpreter and HTTP request - Gradually document the three tool shapes: registry tools, native model tools, and local runtime tools --------- Co-authored-by: claude[bot] <41898282+claude[bot]@users.noreply.github.com> Co-authored-by: Félix Malfait Co-authored-by: Félix Malfait --- .../mcp-excluded-tool-names.const.ts | 4 + .../api/mcp/services/mcp-protocol.service.ts | 9 +-- .../native/native-tool-binder.interface.ts | 5 -- .../native/native-tool-binder.service.ts | 26 +++--- .../services/tool-registry.service.ts | 11 --- .../tool-provider/tool-provider.module.ts | 5 +- ...ow-agent-registry-tool-categories.const.ts | 6 ++ .../services/agent-async-executor.service.ts | 69 +++++++++------- .../ai-chat-tool-names-to-preload.const.ts | 6 ++ .../services/chat-execution.service.ts | 80 +++---------------- .../services/system-prompt-builder.service.ts | 2 - .../services/ai-model-config.service.ts | 57 ++++++------- .../services/sdk-provider-factory.service.ts | 14 +--- .../types/native-model-tool-options.type.ts | 3 + .../src/ai/constants/tool-category.const.ts | 1 - 15 files changed, 115 insertions(+), 183 deletions(-) create mode 100644 packages/twenty-server/src/engine/api/mcp/constants/mcp-excluded-tool-names.const.ts create mode 100644 packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/constants/workflow-agent-registry-tool-categories.const.ts create mode 100644 packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/constants/ai-chat-tool-names-to-preload.const.ts create mode 100644 packages/twenty-server/src/engine/metadata-modules/ai/ai-models/types/native-model-tool-options.type.ts diff --git a/packages/twenty-server/src/engine/api/mcp/constants/mcp-excluded-tool-names.const.ts b/packages/twenty-server/src/engine/api/mcp/constants/mcp-excluded-tool-names.const.ts new file mode 100644 index 0000000000..1f008e0947 --- /dev/null +++ b/packages/twenty-server/src/engine/api/mcp/constants/mcp-excluded-tool-names.const.ts @@ -0,0 +1,4 @@ +export const MCP_EXCLUDED_TOOL_NAMES = new Set([ + 'code_interpreter', + 'http_request', +]); diff --git a/packages/twenty-server/src/engine/api/mcp/services/mcp-protocol.service.ts b/packages/twenty-server/src/engine/api/mcp/services/mcp-protocol.service.ts index 9facdb5914..e2f958a4a0 100644 --- a/packages/twenty-server/src/engine/api/mcp/services/mcp-protocol.service.ts +++ b/packages/twenty-server/src/engine/api/mcp/services/mcp-protocol.service.ts @@ -4,6 +4,7 @@ import { type ToolSet, zodSchema } from 'ai'; import { isDefined } from 'twenty-shared/utils'; import { JSON_RPC_ERROR_CODE } from 'src/engine/api/mcp/constants/json-rpc-error-code.const'; +import { MCP_EXCLUDED_TOOL_NAMES } from 'src/engine/api/mcp/constants/mcp-excluded-tool-names.const'; import { MCP_PROTOCOL_VERSION } from 'src/engine/api/mcp/constants/mcp-protocol-version.const'; import { MCP_SERVER_INFO } from 'src/engine/api/mcp/constants/mcp-server-info.const'; import { MCP_SERVER_INSTRUCTIONS } from 'src/engine/api/mcp/constants/mcp-server-instructions.const'; @@ -40,8 +41,6 @@ import { type FlatWorkspace } from 'src/engine/core-modules/workspace/types/flat import { SkillService } from 'src/engine/metadata-modules/skill/skill.service'; import { UserRoleService } from 'src/engine/metadata-modules/user-role/user-role.service'; -const MCP_EXCLUDED_TOOLS = new Set(['code_interpreter', 'http_request']); - @Injectable() export class McpProtocolService { constructor( @@ -127,7 +126,7 @@ export class McpProtocolService { ...createGetToolCatalogTool(this.toolRegistry, workspace.id, roleId, { userId: options?.userId, userWorkspaceId: options?.userWorkspaceId, - excludeTools: MCP_EXCLUDED_TOOLS, + excludeTools: MCP_EXCLUDED_TOOL_NAMES, }), inputSchema: zodSchema(getToolCatalogInputSchema), }, @@ -135,13 +134,13 @@ export class McpProtocolService { ...createLearnToolsTool( this.toolRegistry, toolContext, - MCP_EXCLUDED_TOOLS, + MCP_EXCLUDED_TOOL_NAMES, ), inputSchema: zodSchema(learnToolsInputSchema), }, [EXECUTE_TOOL_TOOL_NAME]: { ...createExecuteToolTool(this.toolRegistry, toolContext, { - excludeTools: MCP_EXCLUDED_TOOLS, + excludeTools: MCP_EXCLUDED_TOOL_NAMES, }), inputSchema: executeToolInputSchema, }, diff --git a/packages/twenty-server/src/engine/core-modules/tool-provider/native/native-tool-binder.interface.ts b/packages/twenty-server/src/engine/core-modules/tool-provider/native/native-tool-binder.interface.ts index 5d32118671..bcf19785ea 100644 --- a/packages/twenty-server/src/engine/core-modules/tool-provider/native/native-tool-binder.interface.ts +++ b/packages/twenty-server/src/engine/core-modules/tool-provider/native/native-tool-binder.interface.ts @@ -1,5 +1,4 @@ import { type ToolSet } from 'ai'; -import { type ToolCategory } from 'twenty-shared/ai'; import { type ToolProviderContext } from 'src/engine/core-modules/tool-provider/interfaces/tool-provider-context.type'; @@ -10,9 +9,5 @@ import { type ToolProviderContext } from 'src/engine/core-modules/tool-provider/ // executed by ToolExecutorService. They're merged directly into the ToolSet // handed to streamText. export interface NativeToolBinder { - readonly category: ToolCategory; - - isAvailable(context: ToolProviderContext): Promise; - bind(context: ToolProviderContext): Promise; } diff --git a/packages/twenty-server/src/engine/core-modules/tool-provider/native/native-tool-binder.service.ts b/packages/twenty-server/src/engine/core-modules/tool-provider/native/native-tool-binder.service.ts index f4269f1b90..bba8ef4d8a 100644 --- a/packages/twenty-server/src/engine/core-modules/tool-provider/native/native-tool-binder.service.ts +++ b/packages/twenty-server/src/engine/core-modules/tool-provider/native/native-tool-binder.service.ts @@ -1,28 +1,24 @@ import { Injectable } from '@nestjs/common'; import { type ToolSet } from 'ai'; -import { isDefined } from 'twenty-shared/utils'; import { type NativeToolBinder } from 'src/engine/core-modules/tool-provider/native/native-tool-binder.interface'; import { type ToolProviderContext } from 'src/engine/core-modules/tool-provider/interfaces/tool-provider-context.type'; -import { ToolCategory } from 'twenty-shared/ai'; import { AiModelConfigService } from 'src/engine/metadata-modules/ai/ai-models/services/ai-model-config.service'; -import { AiModelRegistryService } from 'src/engine/metadata-modules/ai/ai-models/services/ai-model-registry.service'; +import { + AiModelRegistryService, + type RegisteredAiModel, +} from 'src/engine/metadata-modules/ai/ai-models/services/ai-model-registry.service'; +import { type NativeModelToolOptions } from 'src/engine/metadata-modules/ai/ai-models/types/native-model-tool-options.type'; @Injectable() export class NativeToolBinderService implements NativeToolBinder { - readonly category = ToolCategory.NATIVE_MODEL; - constructor( private readonly aiModelConfigService: AiModelConfigService, private readonly aiModelRegistryService: AiModelRegistryService, ) {} - async isAvailable(context: ToolProviderContext): Promise { - return isDefined(context.agent); - } - async bind(context: ToolProviderContext): Promise { if (!context.agent) { return {}; @@ -31,12 +27,16 @@ export class NativeToolBinderService implements NativeToolBinder { const registeredModel = await this.aiModelRegistryService.resolveModelForAgent(context.agent); - // Enablement is driven by the agent's model configuration - // (modelConfiguration.webSearch.enabled). If the agent does not opt into - // a capability, getNativeModelTools returns an empty ToolSet. - return this.aiModelConfigService.getNativeModelTools( + return this.aiModelConfigService.getNativeModelToolsForAgent( registeredModel, context.agent, ); } + + bindForModel( + model: RegisteredAiModel, + options: NativeModelToolOptions = {}, + ): ToolSet { + return this.aiModelConfigService.getNativeModelTools(model, options); + } } diff --git a/packages/twenty-server/src/engine/core-modules/tool-provider/services/tool-registry.service.ts b/packages/twenty-server/src/engine/core-modules/tool-provider/services/tool-registry.service.ts index bf39c6bf3a..46a5f8ecf4 100644 --- a/packages/twenty-server/src/engine/core-modules/tool-provider/services/tool-registry.service.ts +++ b/packages/twenty-server/src/engine/core-modules/tool-provider/services/tool-registry.service.ts @@ -7,8 +7,6 @@ import { type ToolProviderContext } from 'src/engine/core-modules/tool-provider/ import { type ToolRetrievalOptions } from 'src/engine/core-modules/tool-provider/interfaces/tool-retrieval-options.type'; import { TOOL_PROVIDERS } from 'src/engine/core-modules/tool-provider/constants/tool-providers.token'; -import { ToolCategory } from 'twenty-shared/ai'; -import { NativeToolBinderService } from 'src/engine/core-modules/tool-provider/native/native-tool-binder.service'; import { compactToolOutput } from 'src/engine/core-modules/tool-provider/output-serialization/compact-tool-output.util'; import { ToolExecutorService } from 'src/engine/core-modules/tool-provider/services/tool-executor.service'; import { type LearnToolsAspect } from 'src/engine/core-modules/tool-provider/tools/learn-tools.tool'; @@ -30,7 +28,6 @@ export class ToolRegistryService { constructor( @Inject(TOOL_PROVIDERS) private readonly providers: ToolProvider[], - private readonly nativeToolBinder: NativeToolBinderService, private readonly toolExecutorService: ToolExecutorService, ) {} @@ -327,14 +324,6 @@ export class ToolRegistryService { serializeOutput, }); - if (categories?.includes(ToolCategory.NATIVE_MODEL)) { - if (await this.nativeToolBinder.isAvailable(context)) { - const nativeTools = await this.nativeToolBinder.bind(context); - - Object.assign(toolSet, nativeTools); - } - } - this.logger.log( `Generated ${Object.keys(toolSet).length} tools for categories: [${categories?.join(', ') ?? 'all'}]`, ); diff --git a/packages/twenty-server/src/engine/core-modules/tool-provider/tool-provider.module.ts b/packages/twenty-server/src/engine/core-modules/tool-provider/tool-provider.module.ts index 3b5d27f220..b5bfd9c60b 100644 --- a/packages/twenty-server/src/engine/core-modules/tool-provider/tool-provider.module.ts +++ b/packages/twenty-server/src/engine/core-modules/tool-provider/tool-provider.module.ts @@ -71,7 +71,8 @@ import { ToolRegistryService } from './services/tool-registry.service'; { // TOOL_PROVIDERS contains only providers implementing ToolProvider // (registry tools with descriptors). The native tool binder is a - // parallel concept and is injected directly into ToolRegistryService. + // parallel concept and is exported for surfaces that bind SDK-native + // tools directly into their model ToolSet. provide: TOOL_PROVIDERS, useFactory: ( actionProvider: ActionToolProvider, @@ -105,6 +106,6 @@ import { ToolRegistryService } from './services/tool-registry.service'; }, ToolRegistryService, ], - exports: [ToolRegistryService], + exports: [NativeToolBinderService, ToolRegistryService], }) export class ToolProviderModule {} diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/constants/workflow-agent-registry-tool-categories.const.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/constants/workflow-agent-registry-tool-categories.const.ts new file mode 100644 index 0000000000..5493b17e12 --- /dev/null +++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/constants/workflow-agent-registry-tool-categories.const.ts @@ -0,0 +1,6 @@ +import { ToolCategory } from 'twenty-shared/ai'; + +export const WORKFLOW_AGENT_REGISTRY_TOOL_CATEGORIES: ToolCategory[] = [ + ToolCategory.DATABASE_CRUD, + ToolCategory.ACTION, +]; diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/services/agent-async-executor.service.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/services/agent-async-executor.service.ts index 950cd599e2..9a95d6629f 100644 --- a/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/services/agent-async-executor.service.ts +++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/services/agent-async-executor.service.ts @@ -12,12 +12,13 @@ import { type ActorMetadata } from 'twenty-shared/types'; import { isDefined } from 'twenty-shared/utils'; import { type Repository } from 'typeorm'; -import { type ToolProviderContext } from 'src/engine/core-modules/tool-provider/interfaces/tool-provider-context.type'; - import { isUserAuthContext } from 'src/engine/core-modules/auth/guards/is-user-auth-context.guard'; import { type WorkspaceAuthContext } from 'src/engine/core-modules/auth/types/workspace-auth-context.type'; -import { ToolCategory } from 'twenty-shared/ai'; +import { type ToolProviderContext } from 'src/engine/core-modules/tool-provider/interfaces/tool-provider-context.type'; +import { NativeToolBinderService } from 'src/engine/core-modules/tool-provider/native/native-tool-binder.service'; import { ToolRegistryService } from 'src/engine/core-modules/tool-provider/services/tool-registry.service'; +import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity'; +import { WORKFLOW_AGENT_REGISTRY_TOOL_CATEGORIES } from 'src/engine/metadata-modules/ai/ai-agent-execution/constants/workflow-agent-registry-tool-categories.const'; import { type AgentExecutionResult } from 'src/engine/metadata-modules/ai/ai-agent-execution/types/agent-execution-result.type'; import { countNativeWebSearchCallsFromSteps } from 'src/engine/metadata-modules/ai/ai-billing/utils/count-native-web-search-calls-from-steps.util'; import { extractCacheCreationTokensFromSteps } from 'src/engine/metadata-modules/ai/ai-billing/utils/extract-cache-creation-tokens.util'; @@ -32,14 +33,13 @@ import { type AgentEntity } from 'src/engine/metadata-modules/ai/ai-agent/entiti import { repairToolCall } from 'src/engine/metadata-modules/ai/ai-agent/utils/repair-tool-call.util'; import { AI_TELEMETRY_CONFIG } from 'src/engine/metadata-modules/ai/ai-models/constants/ai-telemetry.const'; import { AiModelConfigService } from 'src/engine/metadata-modules/ai/ai-models/services/ai-model-config.service'; -import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity'; import { AiModelRegistryService } from 'src/engine/metadata-modules/ai/ai-models/services/ai-model-registry.service'; import { RoleTargetEntity } from 'src/engine/metadata-modules/role-target/role-target.entity'; import { type RolePermissionConfig } from 'src/engine/twenty-orm/types/role-permission-config'; -// Agent execution within workflows uses database and action tools only. -// Workflow tools are intentionally excluded to avoid circular dependencies -// and recursive workflow execution. +// Agent execution within workflows uses registry tools plus native model tools. +// Workflow registry tools are intentionally excluded to avoid circular +// dependencies and recursive workflow execution. @Injectable() export class AgentAsyncExecutorService { private readonly logger = new Logger(AgentAsyncExecutorService.name); @@ -48,6 +48,7 @@ export class AgentAsyncExecutorService { private readonly aiModelRegistryService: AiModelRegistryService, private readonly aiModelConfigService: AiModelConfigService, private readonly toolRegistry: ToolRegistryService, + private readonly nativeToolBinder: NativeToolBinderService, @InjectRepository(RoleTargetEntity) private readonly roleTargetRepository: Repository, @InjectRepository(WorkspaceEntity) @@ -139,37 +140,43 @@ export class AgentAsyncExecutorService { rolePermissionConfig, ); - // Workflow context: DATABASE_CRUD, ACTION, and NATIVE_MODEL tools only - // Workflow tools are excluded to prevent circular dependencies + // Workflow context: registry tools come from DATABASE_CRUD and ACTION. + // Native model tools are bound separately below. const roleId = this.extractRoleIds(effectiveRoleConfig)[0] ?? ''; - tools = await this.toolRegistry.getToolsByCategories( + const toolProviderContext: ToolProviderContext = { + workspaceId: agent.workspaceId, + roleId, + rolePermissionConfig: effectiveRoleConfig ?? { unionOf: [] }, + authContext, + actorContext, + agent: agent as unknown as ToolProviderContext['agent'], + userId: + isDefined(authContext) && isUserAuthContext(authContext) + ? authContext.user.id + : undefined, + userWorkspaceId: + isDefined(authContext) && isUserAuthContext(authContext) + ? authContext.userWorkspaceId + : undefined, + }; + + const registryTools = await this.toolRegistry.getToolsByCategories( + toolProviderContext, { - workspaceId: agent.workspaceId, - roleId, - rolePermissionConfig: effectiveRoleConfig ?? { unionOf: [] }, - authContext, - actorContext, - agent: agent as unknown as ToolProviderContext['agent'], - userId: - isDefined(authContext) && isUserAuthContext(authContext) - ? authContext.user.id - : undefined, - userWorkspaceId: - isDefined(authContext) && isUserAuthContext(authContext) - ? authContext.userWorkspaceId - : undefined, - }, - { - categories: [ - ToolCategory.DATABASE_CRUD, - ToolCategory.ACTION, - ToolCategory.NATIVE_MODEL, - ], + categories: WORKFLOW_AGENT_REGISTRY_TOOL_CATEGORIES, wrapWithErrorContext: false, }, ); + const nativeTools = + await this.nativeToolBinder.bind(toolProviderContext); + + tools = { + ...registryTools, + ...nativeTools, + }; + providerOptions = this.aiModelConfigService.getProviderOptions( registeredModel, agent as unknown as Parameters< diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/constants/ai-chat-tool-names-to-preload.const.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/constants/ai-chat-tool-names-to-preload.const.ts new file mode 100644 index 0000000000..5be16a0e27 --- /dev/null +++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/constants/ai-chat-tool-names-to-preload.const.ts @@ -0,0 +1,6 @@ +import { COMMON_PRELOAD_TOOLS } from 'src/engine/core-modules/tool-provider/constants/common-preload-tools.const'; + +export const AI_CHAT_TOOL_NAMES_TO_PRELOAD: string[] = [ + ...COMMON_PRELOAD_TOOLS, + 'app_exa_web_search', +]; diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/services/chat-execution.service.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/services/chat-execution.service.ts index db87cd6897..7fc0380a74 100644 --- a/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/services/chat-execution.service.ts +++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/services/chat-execution.service.ts @@ -22,7 +22,7 @@ import { type CodeExecutionStreamEmitter } from 'src/engine/core-modules/tool-pr import { CodeInterpreterService } from 'src/engine/core-modules/code-interpreter/code-interpreter.service'; import { WorkspaceDomainsService } from 'src/engine/core-modules/domain/workspace-domains/services/workspace-domains.service'; import { ExceptionHandlerService } from 'src/engine/core-modules/exception-handler/exception-handler.service'; -import { COMMON_PRELOAD_TOOLS } from 'src/engine/core-modules/tool-provider/constants/common-preload-tools.const'; +import { NativeToolBinderService } from 'src/engine/core-modules/tool-provider/native/native-tool-binder.service'; import { ToolRegistryService } from 'src/engine/core-modules/tool-provider/services/tool-registry.service'; import { createExecuteToolTool, @@ -40,6 +40,7 @@ import { repairToolCall } from 'src/engine/metadata-modules/ai/ai-agent/utils/re import { AiBillingService } from 'src/engine/metadata-modules/ai/ai-billing/services/ai-billing.service'; import { countNativeWebSearchCallsFromSteps } from 'src/engine/metadata-modules/ai/ai-billing/utils/count-native-web-search-calls-from-steps.util'; import { extractCacheCreationTokensFromSteps } from 'src/engine/metadata-modules/ai/ai-billing/utils/extract-cache-creation-tokens.util'; +import { AI_CHAT_TOOL_NAMES_TO_PRELOAD } from 'src/engine/metadata-modules/ai/ai-chat/constants/ai-chat-tool-names-to-preload.const'; import { MessagePruningService } from 'src/engine/metadata-modules/ai/ai-chat/services/message-pruning.service'; import { SystemPromptBuilderService } from 'src/engine/metadata-modules/ai/ai-chat/services/system-prompt-builder.service'; import { @@ -49,14 +50,9 @@ import { import { AI_SDK_ANTHROPIC, AI_SDK_BEDROCK, - AI_SDK_OPENAI, } from 'src/engine/metadata-modules/ai/ai-models/constants/ai-sdk-package.const'; import { AI_TELEMETRY_CONFIG } from 'src/engine/metadata-modules/ai/ai-models/constants/ai-telemetry.const'; -import { - AiModelRegistryService, - type RegisteredAiModel, -} from 'src/engine/metadata-modules/ai/ai-models/services/ai-model-registry.service'; -import { SdkProviderFactoryService } from 'src/engine/metadata-modules/ai/ai-models/services/sdk-provider-factory.service'; +import { AiModelRegistryService } from 'src/engine/metadata-modules/ai/ai-models/services/ai-model-registry.service'; import { type AiModelConfig } from 'src/engine/metadata-modules/ai/ai-models/types/ai-model-config.type'; import { SkillService } from 'src/engine/metadata-modules/skill/skill.service'; @@ -91,7 +87,7 @@ export class ChatExecutionService { private readonly codeInterpreterService: CodeInterpreterService, private readonly systemPromptBuilder: SystemPromptBuilderService, private readonly exceptionHandlerService: ExceptionHandlerService, - private readonly sdkProviderFactory: SdkProviderFactoryService, + private readonly nativeToolBinder: NativeToolBinderService, private readonly messagePruningService: MessagePruningService, ) {} @@ -139,16 +135,8 @@ export class ChatExecutionService { `Built tool catalog with ${toolCatalog.length} tools, ${skillCatalog.length} skills available`, ); - // Preload the Exa app tool (shipped as the `twenty-exa` npm package) so chat - // has structured web search ready without discovery. getToolsByName - // silently skips the entry when the workspace doesn't have the Exa app - // installed (admin hasn't registered it + flipped `isPreInstalled`). - // TODO(app-preloading): move this list into the app manifest so any - // app can declare `preloadedInChat: true` instead of hardcoding here. - const toolNamesToPreload = [...COMMON_PRELOAD_TOOLS, 'app_exa_web_search']; - const preloadedTools = await this.toolRegistry.getToolsByName( - toolNamesToPreload, + AI_CHAT_TOOL_NAMES_TO_PRELOAD, toolContext, { serializeOutput: true }, ); @@ -169,23 +157,22 @@ export class ChatExecutionService { registeredModel.modelId, ); - // Native web_search is returned when the resolved model's SDK provider - // exposes it (Anthropic, OpenAI). Coexists with app_exa_web_search when - // both are available — the model picks based on tool descriptions. - const { tools: nativeSearchTools, callableToolNames: searchToolNames } = - this.getNativeWebSearchTools(registeredModel); + const nativeModelTools = this.nativeToolBinder.bindForModel( + registeredModel, + { webSearchEnabled: true }, + ); // Tools the model can call directly: preloaded registry tools (already // serialized by the hydrator) plus SDK-native tools (opaque, never // serialized). execute_tool routes discovered tools through the registry. const directTools: ToolSet = { ...preloadedTools, - ...nativeSearchTools, + ...nativeModelTools, }; const preloadedToolNames = [ ...Object.keys(preloadedTools), - ...searchToolNames, + ...Object.keys(nativeModelTools), ]; // ToolSet is constant for the entire conversation — no mutation. @@ -453,51 +440,6 @@ export class ChatExecutionService { return context; } - private getNativeWebSearchTools(model: RegisteredAiModel): { - tools: ToolSet; - callableToolNames: string[]; - } { - const empty = { tools: {}, callableToolNames: [] }; - const providerName = model.providerName; - - if (!providerName) { - return empty; - } - - switch (model.sdkPackage) { - case AI_SDK_ANTHROPIC: { - const provider = - this.sdkProviderFactory.getRawAnthropicProvider(providerName); - - if (!provider) { - return empty; - } - - return { - tools: { web_search: provider.tools.webSearch_20250305() }, - callableToolNames: ['web_search'], - }; - } - case AI_SDK_BEDROCK: - return empty; - case AI_SDK_OPENAI: { - const provider = - this.sdkProviderFactory.getRawOpenAIProvider(providerName); - - if (!provider) { - return empty; - } - - return { - tools: { web_search: provider.tools.webSearch() }, - callableToolNames: ['web_search'], - }; - } - default: - return empty; - } - } - private async storeExtractedFiles( files: ExtractedFile[], _workspaceId: string, diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/services/system-prompt-builder.service.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/services/system-prompt-builder.service.ts index 4ff48e4463..e2802ed6a4 100644 --- a/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/services/system-prompt-builder.service.ts +++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/services/system-prompt-builder.service.ts @@ -322,8 +322,6 @@ ${tools return 'Dashboard Tools (create/manage dashboards)'; case ToolCategory.LOGIC_FUNCTION: return 'Logic Functions (custom tools)'; - case ToolCategory.NATIVE_MODEL: - return 'Native Model Capabilities (e.g. web search)'; case ToolCategory.VIEW_FIELD: return 'View Field Tools (manage view columns)'; default: diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-models/services/ai-model-config.service.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-models/services/ai-model-config.service.ts index b0b11cb4c0..f45dfd4f83 100644 --- a/packages/twenty-server/src/engine/metadata-modules/ai/ai-models/services/ai-model-config.service.ts +++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-models/services/ai-model-config.service.ts @@ -14,6 +14,7 @@ import { AiModelRegistryService, RegisteredAiModel, } from 'src/engine/metadata-modules/ai/ai-models/services/ai-model-registry.service'; +import { type NativeModelToolOptions } from 'src/engine/metadata-modules/ai/ai-models/types/native-model-tool-options.type'; import { SdkProviderFactoryService } from 'src/engine/metadata-modules/ai/ai-models/services/sdk-provider-factory.service'; import { FlatAgentWithRoleId } from 'src/engine/metadata-modules/flat-agent/types/flat-agent.type'; @@ -42,57 +43,51 @@ export class AiModelConfigService { getNativeModelTools( model: RegisteredAiModel, - agent: FlatAgentWithRoleId, + options: NativeModelToolOptions, ): ToolSet { const tools: ToolSet = {}; - if (!agent.modelConfiguration) { + if (!options.webSearchEnabled) { return tools; } switch (model.sdkPackage) { - case AI_SDK_ANTHROPIC: - if (agent.modelConfiguration.webSearch?.enabled) { - const anthropicProvider = model.providerName - ? this.sdkProviderFactory.getRawAnthropicProvider( - model.providerName, - ) - : undefined; + case AI_SDK_ANTHROPIC: { + const anthropicProvider = model.providerName + ? this.sdkProviderFactory.getRawAnthropicProvider(model.providerName) + : undefined; - if (anthropicProvider) { - tools.web_search = anthropicProvider.tools.webSearch_20250305(); - } + if (anthropicProvider) { + tools.web_search = anthropicProvider.tools.webSearch_20250305(); } - break; - case AI_SDK_BEDROCK: { - if (agent.modelConfiguration.webSearch?.enabled) { - const bedrockProvider = model.providerName - ? this.sdkProviderFactory.getRawBedrockProvider(model.providerName) - : undefined; - if (bedrockProvider) { - tools.web_search = - bedrockProvider.tools.webSearch_20250305() as ToolSet[string]; - } - } break; } - case AI_SDK_OPENAI: - if (agent.modelConfiguration.webSearch?.enabled) { - const openaiProvider = model.providerName - ? this.sdkProviderFactory.getRawOpenAIProvider(model.providerName) - : undefined; + case AI_SDK_OPENAI: { + const openaiProvider = model.providerName + ? this.sdkProviderFactory.getRawOpenAIProvider(model.providerName) + : undefined; - if (openaiProvider) { - tools.web_search = openaiProvider.tools.webSearch(); - } + if (openaiProvider) { + tools.web_search = openaiProvider.tools.webSearch(); } + break; + } } return tools; } + getNativeModelToolsForAgent( + model: RegisteredAiModel, + agent: FlatAgentWithRoleId, + ): ToolSet { + return this.getNativeModelTools(model, { + webSearchEnabled: agent.modelConfiguration?.webSearch?.enabled === true, + }); + } + private getXaiProviderOptions(agent: FlatAgentWithRoleId): ProviderOptions { if ( !agent.modelConfiguration || diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-models/services/sdk-provider-factory.service.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-models/services/sdk-provider-factory.service.ts index 92bdbde254..9a29e5b4f1 100644 --- a/packages/twenty-server/src/engine/metadata-modules/ai/ai-models/services/sdk-provider-factory.service.ts +++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-models/services/sdk-provider-factory.service.ts @@ -1,9 +1,6 @@ import { Injectable } from '@nestjs/common'; -import { - createAmazonBedrock, - type AmazonBedrockProvider, -} from '@ai-sdk/amazon-bedrock'; +import { createAmazonBedrock } from '@ai-sdk/amazon-bedrock'; import { createAnthropic, type AnthropicProvider } from '@ai-sdk/anthropic'; import { createGoogleGenerativeAI } from '@ai-sdk/google'; import { createMistral } from '@ai-sdk/mistral'; @@ -65,15 +62,6 @@ export class SdkProviderFactoryService { return instance.rawProvider as T; } - getRawBedrockProvider( - providerName: string, - ): AmazonBedrockProvider | undefined { - return this.getRawProvider( - providerName, - AI_SDK_BEDROCK, - ); - } - getRawAnthropicProvider(providerName: string): AnthropicProvider | undefined { return this.getRawProvider( providerName, diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-models/types/native-model-tool-options.type.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-models/types/native-model-tool-options.type.ts new file mode 100644 index 0000000000..039fb3ac20 --- /dev/null +++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-models/types/native-model-tool-options.type.ts @@ -0,0 +1,3 @@ +export type NativeModelToolOptions = { + webSearchEnabled?: boolean; +}; diff --git a/packages/twenty-shared/src/ai/constants/tool-category.const.ts b/packages/twenty-shared/src/ai/constants/tool-category.const.ts index 126cc1c597..80ea69a0e6 100644 --- a/packages/twenty-shared/src/ai/constants/tool-category.const.ts +++ b/packages/twenty-shared/src/ai/constants/tool-category.const.ts @@ -3,7 +3,6 @@ export enum ToolCategory { ACTION = 'ACTION', WORKFLOW = 'WORKFLOW', METADATA = 'METADATA', - NATIVE_MODEL = 'NATIVE_MODEL', VIEW = 'VIEW', VIEW_FIELD = 'VIEW_FIELD', DASHBOARD = 'DASHBOARD',