Feat: native model capabilities (#14787)

packages/twenty-server/src/engine/metadata-modules/agent/agent-execution.service.ts

@@ -19,7 +19,6 @@ import { getAllSelectableFields } from 'src/engine/api/utils/get-all-selectable-
 import { AIBillingService } from 'src/engine/core-modules/ai/services/ai-billing.service';
 import { AiModelRegistryService } from 'src/engine/core-modules/ai/services/ai-model-registry.service';
 import { DomainManagerService } from 'src/engine/core-modules/domain-manager/services/domain-manager.service';
-import { FileEntity } from 'src/engine/core-modules/file/entities/file.entity';
 import { FileService } from 'src/engine/core-modules/file/services/file.service';
 import { type Workspace } from 'src/engine/core-modules/workspace/workspace.entity';
 import { AgentHandoffToolService } from 'src/engine/metadata-modules/agent/agent-handoff-tool.service';
@@ -30,6 +29,7 @@ import { getObjectMetadataMapItemByNameSingular } from 'src/engine/metadata-modu
 import { WorkspacePermissionsCacheService } from 'src/engine/metadata-modules/workspace-permissions-cache/workspace-permissions-cache.service';
 import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager';
 
+import { AgentModelConfigService } from './agent-model-config.service';
 import { AgentToolGeneratorService } from './agent-tool-generator.service';
 import { AgentEntity } from './agent.entity';
 import { AgentException, AgentExceptionCode } from './agent.exception';
@@ -51,11 +51,10 @@ export class AgentExecutionService {
     private readonly workspacePermissionsCacheService: WorkspacePermissionsCacheService,
     private readonly aiModelRegistryService: AiModelRegistryService,
     private readonly agentToolGeneratorService: AgentToolGeneratorService,
+    private readonly agentModelConfigService: AgentModelConfigService,
     private readonly aiBillingService: AIBillingService,
     @InjectRepository(AgentEntity)
     private readonly agentRepository: Repository<AgentEntity>,
-    @InjectRepository(FileEntity)
-    private readonly fileRepository: Repository<FileEntity>,
   ) {}
 
   async prepareAIRequestConfig({
@@ -78,6 +77,7 @@ export class AgentExecutionService {
         await this.aiModelRegistryService.resolveModelForAgent(agent);
 
       let tools: ToolSet = {};
+      let providerOptions;
 
       if (agent) {
         const baseTools =
@@ -91,8 +91,18 @@ export class AgentExecutionService {
             agent.id,
             agent.workspaceId,
           );
+        const nativeModelTools =
+          this.agentModelConfigService.getNativeModelTools(
+            registeredModel,
+            agent,
+          );
 
-        tools = { ...baseTools, ...handoffTools };
+        tools = { ...baseTools, ...handoffTools, ...nativeModelTools };
+
+        providerOptions = this.agentModelConfigService.getProviderOptions(
+          registeredModel,
+          agent,
+        );
       }
 
       this.logger.log(`Generated ${Object.keys(tools).length} tools for agent`);
@@ -103,16 +113,7 @@ export class AgentExecutionService {
         model: registeredModel.model,
         messages: convertToModelMessages(messages),
         stopWhen: stepCountIs(AGENT_CONFIG.MAX_STEPS),
-        ...(registeredModel.doesSupportThinking && {
-          providerOptions: {
-            anthropic: {
-              thinking: {
-                type: 'enabled',
-                budgetTokens: AGENT_CONFIG.REASONING_BUDGET_TOKENS,
-              },
-            },
-          },
-        }),
+        providerOptions,
       };
     } catch (error) {
       this.logger.error(

packages/twenty-server/src/engine/metadata-modules/agent/agent-model-config.service.ts

@@ -0,0 +1,100 @@
+import { Injectable } from '@nestjs/common';
+
+import { anthropic } from '@ai-sdk/anthropic';
+import { openai } from '@ai-sdk/openai';
+import { ProviderOptions } from '@ai-sdk/provider-utils';
+import { ToolSet } from 'ai';
+
+import { ModelProvider } from 'src/engine/core-modules/ai/constants/ai-models.const';
+import { RegisteredAIModel } from 'src/engine/core-modules/ai/services/ai-model-registry.service';
+import { AGENT_CONFIG } from 'src/engine/metadata-modules/agent/constants/agent-config.const';
+
+import { AgentEntity } from './agent.entity';
+
+@Injectable()
+export class AgentModelConfigService {
+  constructor() {}
+
+  getProviderOptions(
+    model: RegisteredAIModel,
+    agent: AgentEntity,
+  ): ProviderOptions {
+    switch (model.provider) {
+      case ModelProvider.XAI:
+        return this.getXaiProviderOptions(agent);
+      case ModelProvider.ANTHROPIC:
+        return this.getAnthropicProviderOptions(model);
+      default:
+        return {};
+    }
+  }
+
+  getNativeModelTools(model: RegisteredAIModel, agent: AgentEntity): ToolSet {
+    const tools: ToolSet = {};
+
+    if (!agent.modelConfiguration) {
+      return tools;
+    }
+
+    switch (model.provider) {
+      case ModelProvider.ANTHROPIC:
+        if (agent.modelConfiguration.webSearch?.enabled) {
+          tools.web_search = anthropic.tools.webSearch_20250305();
+        }
+        break;
+      case ModelProvider.OPENAI:
+        if (agent.modelConfiguration.webSearch?.enabled) {
+          tools.web_search = openai.tools.webSearch();
+        }
+        break;
+    }
+
+    return tools;
+  }
+
+  private getXaiProviderOptions(agent: AgentEntity): ProviderOptions {
+    if (
+      !agent.modelConfiguration ||
+      (!agent.modelConfiguration.webSearch?.enabled &&
+        !agent.modelConfiguration.twitterSearch?.enabled)
+    ) {
+      return {};
+    }
+
+    const sources: Array<{ type: string }> = [];
+
+    if (agent.modelConfiguration.webSearch?.enabled) {
+      sources.push({ type: 'web' });
+    }
+
+    if (agent.modelConfiguration.twitterSearch?.enabled) {
+      sources.push({ type: 'x' });
+    }
+
+    return {
+      xai: {
+        searchParameters: {
+          mode: 'auto',
+          ...(sources.length > 0 && { sources }),
+        },
+      },
+    };
+  }
+
+  private getAnthropicProviderOptions(
+    model: RegisteredAIModel,
+  ): ProviderOptions {
+    if (!model.doesSupportThinking) {
+      return {};
+    }
+
+    return {
+      anthropic: {
+        thinking: {
+          type: 'enabled',
+          budgetTokens: AGENT_CONFIG.REASONING_BUDGET_TOKENS,
+        },
+      },
+    };
+  }
+}

packages/twenty-server/src/engine/metadata-modules/agent/agent.entity.ts

@@ -16,6 +16,7 @@ import { Relation } from 'src/engine/workspace-manager/workspace-sync-metadata/i
 import { ModelId } from 'src/engine/core-modules/ai/constants/ai-models.const';
 import { ApplicationEntity } from 'src/engine/core-modules/application/application.entity';
 import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity';
+import { ModelConfiguration } from 'src/engine/metadata-modules/agent/types/modelConfiguration';
 
 import { AgentChatThreadEntity } from './agent-chat-thread.entity';
 import { AgentHandoffEntity } from './agent-handoff.entity';
@@ -93,4 +94,7 @@ export class AgentEntity {
 
   @DeleteDateColumn({ type: 'timestamptz' })
   deletedAt?: Date;
+
+  @Column({ nullable: true, type: 'jsonb' })
+  modelConfiguration: ModelConfiguration;
 }

packages/twenty-server/src/engine/metadata-modules/agent/agent.module.ts

@@ -31,6 +31,7 @@ import { AgentHandoffExecutorService } from './agent-handoff-executor.service';
 import { AgentHandoffToolService } from './agent-handoff-tool.service';
 import { AgentHandoffEntity } from './agent-handoff.entity';
 import { AgentHandoffService } from './agent-handoff.service';
+import { AgentModelConfigService } from './agent-model-config.service';
 import { AgentStreamingService } from './agent-streaming.service';
 import { AgentTitleGenerationService } from './agent-title-generation.service';
 import { AgentToolGeneratorService } from './agent-tool-generator.service';
@@ -72,6 +73,7 @@ import { AgentService } from './agent.service';
     AgentChatResolver,
     AgentService,
     AgentExecutionService,
+    AgentModelConfigService,
     AgentToolGeneratorService,
     AgentHandoffToolService,
     AgentChatService,

packages/twenty-server/src/engine/metadata-modules/agent/agent.service.ts

@@ -1,7 +1,6 @@
 import { Injectable } from '@nestjs/common';
 import { InjectRepository } from '@nestjs/typeorm';
 
-import { isDefined } from 'twenty-shared/utils';
 import { In, Repository } from 'typeorm';
 
 import { AgentRoleService } from 'src/engine/metadata-modules/agent-role/agent-role.service';
@@ -133,7 +132,7 @@ export class AgentService {
       name: updatedName,
     });
 
-    if (!isDefined(input.roleId)) {
+    if (!('roleId' in input)) {
       return updatedAgent;
     }
 

packages/twenty-server/src/engine/metadata-modules/agent/dtos/agent.dto.ts

@@ -11,6 +11,7 @@ import GraphQLJSON from 'graphql-type-json';
 
 import { UUIDScalarType } from 'src/engine/api/graphql/workspace-schema-builder/graphql-types/scalars';
 import { ModelId } from 'src/engine/core-modules/ai/constants/ai-models.const';
+import { ModelConfiguration } from 'src/engine/metadata-modules/agent/types/modelConfiguration';
 
 @ObjectType('Agent')
 export class AgentDTO {
@@ -66,4 +67,7 @@ export class AgentDTO {
   @IsDateString()
   @Field()
   updatedAt: Date;
+
+  @Field(() => GraphQLJSON, { nullable: true })
+  modelConfiguration: ModelConfiguration;
 }

packages/twenty-server/src/engine/metadata-modules/agent/dtos/create-agent.input.ts

@@ -11,6 +11,7 @@ import GraphQLJSON from 'graphql-type-json';
 
 import { UUIDScalarType } from 'src/engine/api/graphql/workspace-schema-builder/graphql-types/scalars';
 import { ModelId } from 'src/engine/core-modules/ai/constants/ai-models.const';
+import { ModelConfiguration } from 'src/engine/metadata-modules/agent/types/modelConfiguration';
 
 @InputType()
 export class CreateAgentInput {
@@ -54,6 +55,11 @@ export class CreateAgentInput {
   @Field(() => GraphQLJSON, { nullable: true })
   responseFormat?: object;
 
+  @IsObject()
+  @IsOptional()
+  @Field(() => GraphQLJSON, { nullable: true })
+  modelConfiguration?: ModelConfiguration;
+
   @HideField()
   standardId?: string;
 

packages/twenty-server/src/engine/metadata-modules/agent/dtos/update-agent.input.ts

@@ -11,6 +11,7 @@ import GraphQLJSON from 'graphql-type-json';
 
 import { UUIDScalarType } from 'src/engine/api/graphql/workspace-schema-builder/graphql-types/scalars';
 import { ModelId } from 'src/engine/core-modules/ai/constants/ai-models.const';
+import { ModelConfiguration } from 'src/engine/metadata-modules/agent/types/modelConfiguration';
 
 @InputType()
 export class UpdateAgentInput {
@@ -58,4 +59,9 @@ export class UpdateAgentInput {
   @IsOptional()
   @Field(() => GraphQLJSON, { nullable: true })
   responseFormat?: object;
+
+  @IsObject()
+  @IsOptional()
+  @Field(() => GraphQLJSON, { nullable: true })
+  modelConfiguration?: ModelConfiguration;
 }

packages/twenty-server/src/engine/metadata-modules/agent/types/modelConfiguration.ts

@@ -0,0 +1,10 @@
+export type ModelConfiguration = {
+  webSearch?: {
+    enabled: boolean;
+    configuration: object;
+  };
+  twitterSearch?: {
+    enabled: boolean;
+    configuration: object;
+  };
+};

packages/twenty-server/src/engine/metadata-modules/flat-agent/utils/transform-agent-entity-to-flat-agent.util.ts

@@ -18,5 +18,6 @@ export const transformAgentEntityToFlatAgent = (
     isCustom: agentEntity.isCustom,
     universalIdentifier: agentEntity.standardId || agentEntity.id,
     applicationId: agentEntity.applicationId,
+    modelConfiguration: agentEntity.modelConfiguration,
   };
 };