From 2fcf3e3c2b3d9664dbcac24b7464c4bab55cc980 Mon Sep 17 00:00:00 2001 From: Etienne <45695613+etiennejouan@users.noreply.github.com> Date: Thu, 21 May 2026 17:55:07 +0200 Subject: [PATCH] fix(ai-chat) - fix browser context injection (#20809) Move the browsing context out of the system prompt and injecting it directly into the last user message instead. Previously (before this PR), browsing context change, update system prompt then break whole conversation history ... and caching. Now, browsing context is sent with last message only if changed. "Benchmark" this PR vs main : - same conv with 3-4 turns - 60% -> 85% cache ratio || 0.31 credits -> 0.13 --- .../src/modules/ai/hooks/useAgentChat.ts | 18 +++++++- ...tChatLastSentBrowsingContextFamilyState.ts | 8 ++++ .../constants/chat-system-prompts.const.ts | 3 ++ .../services/chat-execution.service.ts | 45 ++++++++++++++++--- .../services/system-prompt-builder.service.ts | 8 +--- 5 files changed, 69 insertions(+), 13 deletions(-) create mode 100644 packages/twenty-front/src/modules/ai/states/agentChatLastSentBrowsingContextFamilyState.ts diff --git a/packages/twenty-front/src/modules/ai/hooks/useAgentChat.ts b/packages/twenty-front/src/modules/ai/hooks/useAgentChat.ts index 32cc1bccc7..e1cbed6694 100644 --- a/packages/twenty-front/src/modules/ai/hooks/useAgentChat.ts +++ b/packages/twenty-front/src/modules/ai/hooks/useAgentChat.ts @@ -22,6 +22,7 @@ import { } from '@/ai/states/agentChatDraftsByThreadIdState'; import { agentChatErrorComponentFamilyState } from '@/ai/states/agentChatErrorComponentFamilyState'; import { agentChatInputState } from '@/ai/states/agentChatInputState'; +import { agentChatLastSentBrowsingContextFamilyState } from '@/ai/states/agentChatLastSentBrowsingContextFamilyState'; import { agentChatMessagesComponentFamilyState } from '@/ai/states/agentChatMessagesComponentFamilyState'; import { agentChatSelectedFilesState } from '@/ai/states/agentChatSelectedFilesState'; import { agentChatUploadedFilesState } from '@/ai/states/agentChatUploadedFilesState'; @@ -96,6 +97,17 @@ export const useAgentChat = ( })); const browsingContext = getBrowsingContext(); + const lastSentBrowsingContextAtom = + agentChatLastSentBrowsingContextFamilyState.atomFamily(threadId); + const lastSentBrowsingContext = store.get(lastSentBrowsingContextAtom); + const isBrowsingContextChanged = + lastSentBrowsingContext === undefined + ? browsingContext !== null + : JSON.stringify(browsingContext) !== + JSON.stringify(lastSentBrowsingContext); + const browsingContextToSend = isBrowsingContextChanged + ? browsingContext + : null; const messageId = v4(); const optimisticMessageCreatedAt = new Date().toISOString(); const rollbackOptimisticUnarchive = applyOptimisticUnarchive( @@ -151,13 +163,17 @@ export const useAgentChat = ( threadId, text: contentToSend, messageId, - browsingContext: browsingContext ?? null, + browsingContext: browsingContextToSend, modelId: modelIdForRequest ?? undefined, fileAttachments: fileAttachments.length > 0 ? fileAttachments : undefined, }, }); + if (isBrowsingContextChanged) { + store.set(lastSentBrowsingContextAtom, browsingContext); + } + if (data?.sendChatMessage?.queued) { const latestMessages = store.get(messagesAtom); diff --git a/packages/twenty-front/src/modules/ai/states/agentChatLastSentBrowsingContextFamilyState.ts b/packages/twenty-front/src/modules/ai/states/agentChatLastSentBrowsingContextFamilyState.ts new file mode 100644 index 0000000000..eb98f90f52 --- /dev/null +++ b/packages/twenty-front/src/modules/ai/states/agentChatLastSentBrowsingContextFamilyState.ts @@ -0,0 +1,8 @@ +import { createAtomFamilyState } from '@/ui/utilities/state/jotai/utils/createAtomFamilyState'; +import { type BrowsingContext } from '@/ai/types/BrowsingContext'; + +export const agentChatLastSentBrowsingContextFamilyState = + createAtomFamilyState({ + key: 'ai/agentChatLastSentBrowsingContextFamilyState', + defaultValue: undefined, + }); diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/constants/chat-system-prompts.const.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/constants/chat-system-prompts.const.ts index cee49dbf14..4fbf936626 100644 --- a/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/constants/chat-system-prompts.const.ts +++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/constants/chat-system-prompts.const.ts @@ -52,6 +52,9 @@ For simple CRUD operations (find/create/update/delete a record), you do NOT need - Validate assumptions before making changes `, + // Browsing context hint + BROWSING_CONTEXT_INSTRUCTION: `A tag may appear in the user's last message. Only use it when directly relevant to the question.`, + // Response formatting and record references RESPONSE_FORMAT: ` Format responses with markdown for clarity (headings, lists, code blocks, tables). diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/services/chat-execution.service.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/services/chat-execution.service.ts index 47cfba114d..a1cc0098a3 100644 --- a/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/services/chat-execution.service.ts +++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/services/chat-execution.service.ts @@ -123,10 +123,6 @@ export class ChatExecutionService { onCodeExecutionUpdate, }; - const contextString = browsingContext - ? this.buildContextFromBrowsingContext(workspace, browsingContext) - : undefined; - const toolCatalog = await this.toolRegistry.buildToolIndex( workspace.id, roleId, @@ -226,11 +222,22 @@ export class ChatExecutionService { } } + if (isDefined(browsingContext)) { + const contextString = this.buildContextFromBrowsingContext( + workspace, + browsingContext, + ); + + processedMessages = this.injectBrowsingContextIntoLastUserMessage( + processedMessages, + contextString, + ); + } + const systemPrompt = this.systemPromptBuilder.buildFullPrompt( toolCatalog, skillCatalog, preloadedToolNames, - contextString, storedFiles, workspace.aiAdditionalInstructions ?? undefined, userContext, @@ -417,6 +424,34 @@ export class ChatExecutionService { }; } + private injectBrowsingContextIntoLastUserMessage( + messages: UIMessage[], + contextString: string, + ): UIMessage[] { + const lastUserIndex = messages + .map((message) => message.role) + .lastIndexOf('user'); + + if (lastUserIndex === -1) { + return messages; + } + + const lastUserMessage = messages[lastUserIndex]; + const browsingContextPart = { + type: 'text' as const, + text: `\n${contextString}\n`, + }; + + return [ + ...messages.slice(0, lastUserIndex), + { + ...lastUserMessage, + parts: [...lastUserMessage.parts, browsingContextPart], + }, + ...messages.slice(lastUserIndex + 1), + ]; + } + private buildContextFromBrowsingContext( workspace: WorkspaceEntity, browsingContext: BrowsingContextType, diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/services/system-prompt-builder.service.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/services/system-prompt-builder.service.ts index e2802ed6a4..8eed08ff8c 100644 --- a/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/services/system-prompt-builder.service.ts +++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/services/system-prompt-builder.service.ts @@ -136,7 +136,6 @@ export class SystemPromptBuilderService { toolCatalog: ToolIndexEntry[], skillCatalog: FlatSkill[], preloadedTools: string[], - contextString?: string, storedFiles?: Array<{ filename: string; fileId: string; @@ -146,6 +145,7 @@ export class SystemPromptBuilderService { ): string { const parts: string[] = [ CHAT_SYSTEM_PROMPTS.BASE, + CHAT_SYSTEM_PROMPTS.BROWSING_CONTEXT_INSTRUCTION, CHAT_SYSTEM_PROMPTS.RESPONSE_FORMAT, ]; @@ -164,12 +164,6 @@ export class SystemPromptBuilderService { parts.push(this.buildUploadedFilesSection(storedFiles)); } - if (contextString) { - parts.push( - `\nCONTEXT (what the user is currently viewing):\n${contextString}`, - ); - } - return parts.join('\n'); }