Introduce agent hints to reduce context bloat (#15763)

Introducing a new pattern that should reduce token consumption by 90%
for the most common use-cases

packages/twenty-server/src/engine/metadata-modules/agent/agent-execution.service.ts

@@ -26,6 +26,7 @@ import { AGENT_CONFIG } from 'src/engine/metadata-modules/agent/constants/agent-
 import { AGENT_SYSTEM_PROMPTS } from 'src/engine/metadata-modules/agent/constants/agent-system-prompts.const';
 import { AgentActorContextService } from 'src/engine/metadata-modules/agent/services/agent-actor-context.service';
 import { type RecordIdsByObjectMetadataNameSingularType } from 'src/engine/metadata-modules/agent/types/recordIdsByObjectMetadataNameSingular.type';
+import { type ToolHints } from 'src/engine/metadata-modules/ai-router/types/tool-hints.interface';
 import { getObjectMetadataMapItemByNameSingular } from 'src/engine/metadata-modules/utils/get-object-metadata-map-item-by-name-singular.util';
 import { WorkspacePermissionsCacheService } from 'src/engine/metadata-modules/workspace-permissions-cache/workspace-permissions-cache.service';
 import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager';
@@ -43,6 +44,16 @@ export interface AgentExecutionResult {
   usage: LanguageModelUsage;
 }
 
+export interface StreamChatResponseResult {
+  stream: ReturnType<typeof streamText>;
+  timings: {
+    contextBuildTimeMs: number;
+    toolGenerationTimeMs: number;
+    aiRequestPrepTimeMs: number;
+    toolCount: number;
+  };
+}
+
 @Injectable()
 export class AgentExecutionService implements AgentExecutionContext {
   private readonly logger = new Logger(AgentExecutionService.name);
@@ -68,6 +79,7 @@ export class AgentExecutionService implements AgentExecutionContext {
     roleIds,
     excludeHandoffTools = false,
     userWorkspaceId,
+    toolHints,
   }: {
     system: string;
     agent: AgentEntity | null;
@@ -76,6 +88,7 @@ export class AgentExecutionService implements AgentExecutionContext {
     roleIds?: string[];
     excludeHandoffTools?: boolean;
     userWorkspaceId?: string;
+    toolHints?: ToolHints;
   }) {
     try {
       if (agent) {
@@ -98,6 +111,7 @@ export class AgentExecutionService implements AgentExecutionContext {
             actorContext,
             roleIds,
             userWorkspaceId,
+            toolHints,
           );
 
         let handoffTools = {};
@@ -169,6 +183,9 @@ export class AgentExecutionService implements AgentExecutionContext {
     }
   }
 
+  // Fetches and formats record data to provide context for AI agents
+  // Respects permissions and field restrictions based on user role
+  // Returns a JSON string with record data and workspace URLs
   async getContextForSystemPrompt(
     workspace: WorkspaceEntity,
     recordIdsByObjectMetadataNameSingular: RecordIdsByObjectMetadataNameSingularType,
@@ -272,12 +289,14 @@ export class AgentExecutionService implements AgentExecutionContext {
     agentId,
     messages,
     recordIdsByObjectMetadataNameSingular,
+    toolHints,
   }: {
     workspace: WorkspaceEntity;
     userWorkspaceId: string;
     agentId: string;
     messages: UIMessage<unknown, UIDataTypes, UITools>[];
     recordIdsByObjectMetadataNameSingular: RecordIdsByObjectMetadataNameSingularType;
+    toolHints?: ToolHints;
   }): Promise<{
     stream: ReturnType<typeof streamText>;
     timings: {
@@ -335,6 +354,7 @@ export class AgentExecutionService implements AgentExecutionContext {
         actorContext,
         roleIds: [roleId, ...(agent?.roleId ? [agent?.roleId] : [])],
         userWorkspaceId,
+        toolHints,
       });
 
       const aiRequestPrepTime = Date.now() - aiRequestPrepStart;

packages/twenty-server/src/engine/metadata-modules/agent/agent-streaming.service.ts

@@ -27,6 +27,12 @@ import {
 import { type RecordIdsByObjectMetadataNameSingularType } from 'src/engine/metadata-modules/agent/types/recordIdsByObjectMetadataNameSingular.type';
 import { AiRouterService } from 'src/engine/metadata-modules/ai-router/ai-router.service';
 
+export type TokenUsage = {
+  promptTokens: number;
+  completionTokens: number;
+  totalTokens: number;
+};
+
 export type StreamAgentChatOptions = {
   threadId: string;
   userWorkspaceId: string;
@@ -87,17 +93,18 @@ export class AgentStreamingService {
           });
 
           const routingStart = Date.now();
+          const includeDebugInfo = true;
           const routeResult = await this.aiRouterService.routeMessage(
             {
               messages,
               workspaceId: workspace.id,
               routerModel: workspace.routerModel,
             },
-            true,
+            includeDebugInfo,
           );
 
           const routingTime = Date.now() - routingStart;
-          const { agent, debugInfo } = routeResult;
+          const { agent, debugInfo, toolHints } = routeResult;
 
           if (!agent) {
             writer.write({
@@ -154,6 +161,7 @@ export class AgentStreamingService {
             userWorkspaceId,
             messages,
             recordIdsByObjectMetadataNameSingular,
+            toolHints,
           });
 
           const routedStatusPart = {
@@ -199,41 +207,7 @@ export class AgentStreamingService {
                   part.type.startsWith('tool-'),
                 ).length;
 
-                let tokenUsage: {
-                  promptTokens: number;
-                  completionTokens: number;
-                  totalTokens: number;
-                } | null = null;
-
-                try {
-                  const usage = await result.usage;
-
-                  const usageWithTokens = usage as {
-                    inputTokens?: number;
-                    outputTokens?: number;
-                    promptTokens?: number;
-                    completionTokens?: number;
-                    totalTokens?: number;
-                  };
-
-                  tokenUsage = {
-                    promptTokens:
-                      usageWithTokens.inputTokens ??
-                      usageWithTokens.promptTokens ??
-                      0,
-                    completionTokens:
-                      usageWithTokens.outputTokens ??
-                      usageWithTokens.completionTokens ??
-                      0,
-                    totalTokens: usageWithTokens.totalTokens ?? 0,
-                  };
-
-                  this.logger.log(
-                    `Agent execution usage: ${tokenUsage.promptTokens} prompt + ${tokenUsage.completionTokens} completion = ${tokenUsage.totalTokens} total tokens`,
-                  );
-                } catch (error) {
-                  this.logger.warn('Failed to get token usage:', error);
-                }
+                const tokenUsage = await this.extractTokenUsage(result.usage);
 
                 const agentExecutionTime = Date.now() - agentExecutionStart;
 
@@ -325,8 +299,45 @@ export class AgentStreamingService {
 
       pipeUIMessageStreamToResponse({ stream, response });
     } catch (error) {
-      this.logger.error(error.message);
+      this.logger.error(
+        'Failed to stream agent chat:',
+        error instanceof Error ? error.message : String(error),
+      );
       response.end();
     }
   }
+
+  private async extractTokenUsage(
+    usagePromise: Promise<unknown>,
+  ): Promise<TokenUsage | null> {
+    try {
+      const usage = await usagePromise;
+
+      const usageWithTokens = usage as {
+        inputTokens?: number;
+        outputTokens?: number;
+        promptTokens?: number;
+        completionTokens?: number;
+        totalTokens?: number;
+      };
+
+      const tokenUsage = {
+        promptTokens:
+          usageWithTokens.inputTokens ?? usageWithTokens.promptTokens ?? 0,
+        completionTokens:
+          usageWithTokens.outputTokens ?? usageWithTokens.completionTokens ?? 0,
+        totalTokens: usageWithTokens.totalTokens ?? 0,
+      };
+
+      this.logger.log(
+        `Agent execution usage: ${tokenUsage.promptTokens} prompt + ${tokenUsage.completionTokens} completion = ${tokenUsage.totalTokens} total tokens`,
+      );
+
+      return tokenUsage;
+    } catch (error) {
+      this.logger.warn('Failed to get token usage:', error);
+
+      return null;
+    }
+  }
 }

packages/twenty-server/src/engine/metadata-modules/agent/agent-tool-generator.service.ts

@@ -9,6 +9,7 @@ import { ToolAdapterService } from 'src/engine/core-modules/ai/services/tool-ada
 import { ToolService } from 'src/engine/core-modules/ai/services/tool.service';
 import { SearchArticlesTool } from 'src/engine/core-modules/tool/tools/search-articles-tool/search-articles-tool';
 import { AgentEntity } from 'src/engine/metadata-modules/agent/agent.entity';
+import { type ToolHints } from 'src/engine/metadata-modules/ai-router/types/tool-hints.interface';
 import { PermissionFlagType } from 'src/engine/metadata-modules/permissions/constants/permission-flag-type.constants';
 import { PermissionsService } from 'src/engine/metadata-modules/permissions/permissions.service';
 import { RoleEntity } from 'src/engine/metadata-modules/role/role.entity';
@@ -37,6 +38,7 @@ export class AgentToolGeneratorService {
     actorContext?: ActorMetadata,
     roleIds?: string[],
     userWorkspaceId?: string,
+    toolHints?: ToolHints,
   ): Promise<ToolSet> {
     let tools: ToolSet = {};
 
@@ -78,6 +80,7 @@ export class AgentToolGeneratorService {
         workspaceId,
         actorContext,
         userWorkspaceId,
+        toolHints,
       );
 
       tools = { ...tools, ...databaseTools };