From bc57b8ee4e290c472c250c51a203053c55b64420 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Malfait?= Date: Thu, 11 Dec 2025 10:19:27 +0100 Subject: [PATCH] feat(ai): add browsing context and fix tool loading (#16476) ## Summary - Add `BrowsingContext` type to automatically pass what the user is currently viewing (recordPage or listView) to the AI chat - Simplify context architecture: remove toggleable context UI, make it automatic and invisible to the user - Fix tool loading: add `unionOf` handling in `getDatabaseToolsForObject` and fix regex ordering so `find_one_*` tools are properly registered - Use plural names for find tools (`find_people` vs `find_one_person`) for better semantics - Clean up unused components and states ## Changes ### Frontend - New `BrowsingContext` type and `useGetBrowsingContext` hook to gather context from Recoil state - Simplified `useAgentChat` to use the new browsing context - Removed toggleable context UI components (`AgentChatContextRecordPreview`, `SendMessageWithRecordsContextButton`, etc.) - Removed `isAgentChatCurrentContextActiveState` ### Backend - New `BrowsingContextType` for recordPage and listView contexts - Updated `ChatExecutionService` to build context from browsing context - Fixed `tool-registry.service.ts`: - Added `unionOf` handling in permission config - Fixed regex ordering (`find_one` before `find`) so tools load correctly - Use plural names for search tools (`find_people` instead of `find_person`) ## Test plan - [x] Typecheck passes - [x] Lint passes - [ ] Test AI chat on record page - should show context in system prompt - [ ] Test AI chat on list view - should show view name and filters - [ ] Test `find_one_*` tools now load correctly - [ ] Test `find_*` tools use plural naming --- .../ai/components/AIChatErrorMessage.tsx | 3 - .../modules/ai/components/AIChatMessage.tsx | 14 +- .../src/modules/ai/components/AIChatTab.tsx | 13 +- .../AIChatErrorMessageWithRecordsContext.tsx | 14 - .../internal/AgentChatContextPreview.tsx | 23 +- .../AgentChatContextRecordPreview.tsx | 102 ---- .../components/internal/SendMessageButton.tsx | 11 +- .../SendMessageWithRecordsContextButton.tsx | 10 - .../modules/ai/contexts/AgentChatContext.ts | 3 +- .../src/modules/ai/hooks/useAgentChat.ts | 35 +- .../modules/ai/hooks/useBrowsingContext.ts | 126 +++++ .../isAgentChatCurrentContextActiveState.ts | 6 - .../src/modules/ai/types/BrowsingContext.ts | 13 + .../direct-record-tools.factory.ts | 4 +- .../constants/tool-providers.token.ts | 1 + .../interfaces/tool-provider.interface.ts | 20 + .../providers/action-tool.provider.ts | 80 +++ .../providers/database-tool.provider.ts | 148 ++++++ .../providers/metadata-tool.provider.ts | 40 ++ .../providers/workflow-tool.provider.ts | 49 ++ .../services/tool-provider.service.ts | 8 +- .../services/tool-registry.service.ts | 497 ++---------------- .../tool-provider/tool-provider.module.ts | 35 +- .../core-modules/tool/enums/tool-type.enum.ts | 2 +- .../tool/services/tool-registry.service.ts | 4 +- .../engine/core-modules/tool/tool.module.ts | 6 +- .../search-help-center-tool.schema.ts} | 12 +- .../search-help-center-tool.ts} | 16 +- .../tools/send-email-tool/send-email-tool.ts | 2 +- .../ai/ai-agent/types/browsingContext.type.ts | 13 + ...ordIdsByObjectMetadataNameSingular.type.ts | 4 - .../controllers/agent-chat.controller.ts | 7 +- .../services/agent-chat-streaming.service.ts | 8 +- .../services/chat-execution.service.ts | 199 +++---- .../standard-agents/agents/helper-agent.ts | 2 +- 35 files changed, 696 insertions(+), 834 deletions(-) delete mode 100644 packages/twenty-front/src/modules/ai/components/internal/AIChatErrorMessageWithRecordsContext.tsx delete mode 100644 packages/twenty-front/src/modules/ai/components/internal/AgentChatContextRecordPreview.tsx delete mode 100644 packages/twenty-front/src/modules/ai/components/internal/SendMessageWithRecordsContextButton.tsx create mode 100644 packages/twenty-front/src/modules/ai/hooks/useBrowsingContext.ts delete mode 100644 packages/twenty-front/src/modules/ai/states/isAgentChatCurrentContextActiveState.ts create mode 100644 packages/twenty-front/src/modules/ai/types/BrowsingContext.ts create mode 100644 packages/twenty-server/src/engine/core-modules/tool-provider/constants/tool-providers.token.ts create mode 100644 packages/twenty-server/src/engine/core-modules/tool-provider/interfaces/tool-provider.interface.ts create mode 100644 packages/twenty-server/src/engine/core-modules/tool-provider/providers/action-tool.provider.ts create mode 100644 packages/twenty-server/src/engine/core-modules/tool-provider/providers/database-tool.provider.ts create mode 100644 packages/twenty-server/src/engine/core-modules/tool-provider/providers/metadata-tool.provider.ts create mode 100644 packages/twenty-server/src/engine/core-modules/tool-provider/providers/workflow-tool.provider.ts rename packages/twenty-server/src/engine/core-modules/tool/tools/{search-articles-tool/search-articles-tool.schema.ts => search-help-center-tool/search-help-center-tool.schema.ts} (52%) rename packages/twenty-server/src/engine/core-modules/tool/tools/{search-articles-tool/search-articles-tool.ts => search-help-center-tool/search-help-center-tool.ts} (73%) create mode 100644 packages/twenty-server/src/engine/metadata-modules/ai/ai-agent/types/browsingContext.type.ts delete mode 100644 packages/twenty-server/src/engine/metadata-modules/ai/ai-agent/types/recordIdsByObjectMetadataNameSingular.type.ts diff --git a/packages/twenty-front/src/modules/ai/components/AIChatErrorMessage.tsx b/packages/twenty-front/src/modules/ai/components/AIChatErrorMessage.tsx index e10c880fb1..a47abe027a 100644 --- a/packages/twenty-front/src/modules/ai/components/AIChatErrorMessage.tsx +++ b/packages/twenty-front/src/modules/ai/components/AIChatErrorMessage.tsx @@ -1,5 +1,4 @@ import { useAgentChatContextOrThrow } from '@/ai/hooks/useAgentChatContextOrThrow'; -import { type ObjectRecord } from '@/object-record/types/ObjectRecord'; import { useTheme } from '@emotion/react'; import styled from '@emotion/styled'; import { t } from '@lingui/core/macro'; @@ -43,8 +42,6 @@ const StyledErrorMessage = styled.div` type AIChatErrorMessageProps = { error: Error; - records?: ObjectRecord[]; - isRetrying?: boolean; }; export const AIChatErrorMessage = ({ error }: AIChatErrorMessageProps) => { diff --git a/packages/twenty-front/src/modules/ai/components/AIChatMessage.tsx b/packages/twenty-front/src/modules/ai/components/AIChatMessage.tsx index f813afde04..32bac32251 100644 --- a/packages/twenty-front/src/modules/ai/components/AIChatMessage.tsx +++ b/packages/twenty-front/src/modules/ai/components/AIChatMessage.tsx @@ -8,10 +8,7 @@ import { AgentMessageRole } from '@/ai/constants/AgentMessageRole'; import { AIChatAssistantMessageRenderer } from '@/ai/components/AIChatAssistantMessageRenderer'; import { AIChatErrorMessage } from '@/ai/components/AIChatErrorMessage'; -import { AIChatErrorMessageWithRecordsContext } from '@/ai/components/internal/AIChatErrorMessageWithRecordsContext'; -import { contextStoreCurrentObjectMetadataItemIdComponentState } from '@/context-store/states/contextStoreCurrentObjectMetadataItemIdComponentState'; import { LightCopyIconButton } from '@/object-record/record-field/ui/components/LightCopyIconButton'; -import { useRecoilComponentValue } from '@/ui/utilities/state/component-state/hooks/useRecoilComponentValue'; import { type ExtendedUIMessage } from 'twenty-shared/ai'; import { isDefined } from 'twenty-shared/utils'; import { dateLocaleState } from '~/localization/states/dateLocaleState'; @@ -156,10 +153,6 @@ export const AIChatMessage = ({ const theme = useTheme(); const { localeCatalog } = useRecoilValue(dateLocaleState); - const contextStoreCurrentObjectMetadataItemId = useRecoilComponentValue( - contextStoreCurrentObjectMetadataItemIdComponentState, - ); - const showError = isDefined(error) && message.role === AgentMessageRole.ASSISTANT; @@ -201,12 +194,7 @@ export const AIChatMessage = ({ ))} )} - {showError && - (contextStoreCurrentObjectMetadataItemId ? ( - - ) : ( - - ))} + {showError && } {message.parts.length > 0 && message.metadata?.createdAt && ( diff --git a/packages/twenty-front/src/modules/ai/components/AIChatTab.tsx b/packages/twenty-front/src/modules/ai/components/AIChatTab.tsx index c45a668cdd..1368a915a0 100644 --- a/packages/twenty-front/src/modules/ai/components/AIChatTab.tsx +++ b/packages/twenty-front/src/modules/ai/components/AIChatTab.tsx @@ -14,14 +14,11 @@ import { AIChatMessage } from '@/ai/components/AIChatMessage'; import { AIChatSkeletonLoader } from '@/ai/components/internal/AIChatSkeletonLoader'; import { AgentChatContextPreview } from '@/ai/components/internal/AgentChatContextPreview'; import { SendMessageButton } from '@/ai/components/internal/SendMessageButton'; -import { SendMessageWithRecordsContextButton } from '@/ai/components/internal/SendMessageWithRecordsContextButton'; import { AI_CHAT_INPUT_ID } from '@/ai/constants/AiChatInputId'; import { AI_CHAT_SCROLL_WRAPPER_ID } from '@/ai/constants/AiChatScrollWrapperId'; import { useAIChatFileUpload } from '@/ai/hooks/useAIChatFileUpload'; import { useAgentChatContextOrThrow } from '@/ai/hooks/useAgentChatContextOrThrow'; import { agentChatInputState } from '@/ai/states/agentChatInputState'; -import { contextStoreCurrentObjectMetadataItemIdComponentState } from '@/context-store/states/contextStoreCurrentObjectMetadataItemIdComponentState'; -import { useRecoilComponentValue } from '@/ui/utilities/state/component-state/hooks/useRecoilComponentValue'; import { t } from '@lingui/core/macro'; import { useState } from 'react'; import { useRecoilState } from 'recoil'; @@ -71,10 +68,6 @@ export const AIChatTab = () => { const [agentChatInput, setAgentChatInput] = useRecoilState(agentChatInputState); - const contextStoreCurrentObjectMetadataItemId = useRecoilComponentValue( - contextStoreCurrentObjectMetadataItemIdComponentState, - ); - const { uploadFiles } = useAIChatFileUpload(); const { createChatThread } = useCreateNewAIChatThread(); const { navigateCommandMenu } = useCommandMenu(); @@ -145,11 +138,7 @@ export const AIChatTab = () => { onClick={() => createChatThread()} /> - {contextStoreCurrentObjectMetadataItemId ? ( - - ) : ( - - )} + diff --git a/packages/twenty-front/src/modules/ai/components/internal/AIChatErrorMessageWithRecordsContext.tsx b/packages/twenty-front/src/modules/ai/components/internal/AIChatErrorMessageWithRecordsContext.tsx deleted file mode 100644 index 08a6bc3aa4..0000000000 --- a/packages/twenty-front/src/modules/ai/components/internal/AIChatErrorMessageWithRecordsContext.tsx +++ /dev/null @@ -1,14 +0,0 @@ -import { AIChatErrorMessage } from '@/ai/components/AIChatErrorMessage'; -import { useFindManyRecordsSelectedInContextStore } from '@/context-store/hooks/useFindManyRecordsSelectedInContextStore'; - -export const AIChatErrorMessageWithRecordsContext = ({ - error, -}: { - error: Error; -}) => { - const { records } = useFindManyRecordsSelectedInContextStore({ - limit: 10, - }); - - return ; -}; diff --git a/packages/twenty-front/src/modules/ai/components/internal/AgentChatContextPreview.tsx b/packages/twenty-front/src/modules/ai/components/internal/AgentChatContextPreview.tsx index 409eac7101..4f02525b1b 100644 --- a/packages/twenty-front/src/modules/ai/components/internal/AgentChatContextPreview.tsx +++ b/packages/twenty-front/src/modules/ai/components/internal/AgentChatContextPreview.tsx @@ -1,8 +1,5 @@ -import { AgentChatContextRecordPreview } from '@/ai/components/internal/AgentChatContextRecordPreview'; import { agentChatSelectedFilesState } from '@/ai/states/agentChatSelectedFilesState'; import { agentChatUploadedFilesState } from '@/ai/states/agentChatUploadedFilesState'; -import { contextStoreCurrentObjectMetadataItemIdComponentState } from '@/context-store/states/contextStoreCurrentObjectMetadataItemIdComponentState'; -import { useRecoilComponentValue } from '@/ui/utilities/state/component-state/hooks/useRecoilComponentValue'; import styled from '@emotion/styled'; import { useRecoilState } from 'recoil'; import { AgentChatFilePreview } from './AgentChatFilePreview'; @@ -29,15 +26,18 @@ export const AgentChatContextPreview = () => { agentChatUploadedFilesState, ); - const handleRemoveUploadedFile = async (fileIndex: number) => { + const handleRemoveUploadedFile = (fileIndex: number) => { setAgentChatUploadedFiles( - agentChatUploadedFiles.filter((f, index) => fileIndex !== index), + agentChatUploadedFiles.filter((_, index) => fileIndex !== index), ); }; - const contextStoreCurrentObjectMetadataItemId = useRecoilComponentValue( - contextStoreCurrentObjectMetadataItemIdComponentState, - ); + const hasFiles = + agentChatSelectedFiles.length > 0 || agentChatUploadedFiles.length > 0; + + if (!hasFiles) { + return null; + } return ( @@ -62,13 +62,6 @@ export const AgentChatContextPreview = () => { isUploading={false} /> ))} - {contextStoreCurrentObjectMetadataItemId && ( - - )} ); diff --git a/packages/twenty-front/src/modules/ai/components/internal/AgentChatContextRecordPreview.tsx b/packages/twenty-front/src/modules/ai/components/internal/AgentChatContextRecordPreview.tsx deleted file mode 100644 index 73060aa31b..0000000000 --- a/packages/twenty-front/src/modules/ai/components/internal/AgentChatContextRecordPreview.tsx +++ /dev/null @@ -1,102 +0,0 @@ -import { isAgentChatCurrentContextActiveState } from '@/ai/states/isAgentChatCurrentContextActiveState'; -import { CommandMenuContextRecordChipAvatars } from '@/command-menu/components/CommandMenuContextRecordChipAvatars'; -import { getSelectedRecordsContextText } from '@/command-menu/utils/getRecordContextText'; -import { useFindManyRecordsSelectedInContextStore } from '@/context-store/hooks/useFindManyRecordsSelectedInContextStore'; -import { useObjectMetadataItemById } from '@/object-metadata/hooks/useObjectMetadataItemById'; -import { useTheme } from '@emotion/react'; -import styled from '@emotion/styled'; -import { t } from '@lingui/core/macro'; -import { useRecoilState } from 'recoil'; -import { MultipleAvatarChip } from 'twenty-ui/components'; -import { IconReload, IconX } from 'twenty-ui/display'; - -const StyledRightIconContainer = styled.div` - display: flex; - border-left: 1px solid ${({ theme }) => theme.border.color.light}; - - svg { - cursor: pointer; - } -`; - -const StyledChipWrapper = styled.div<{ isActive: boolean }>` - opacity: ${({ isActive }) => (isActive ? 1 : 0.7)}; -`; - -export const AgentChatContextRecordPreview = ({ - contextStoreCurrentObjectMetadataItemId, -}: { - contextStoreCurrentObjectMetadataItemId: string; -}) => { - const theme = useTheme(); - - const { records, totalCount } = useFindManyRecordsSelectedInContextStore({ - limit: 3, - }); - - const { objectMetadataItem } = useObjectMetadataItemById({ - objectId: contextStoreCurrentObjectMetadataItemId, - }); - - const [isAgentChatCurrentContextActive, setIsAgentChatCurrentContextActive] = - useRecoilState(isAgentChatCurrentContextActiveState); - - const Avatars = records.map((record) => ( - // @todo move this components to be less specific. (Outside of CommandMenu - - )); - - const recordSelectionContextChip = { - // @todo move this utils outside of CommandMenu - text: getSelectedRecordsContextText( - objectMetadataItem, - records, - totalCount ?? 0, - ), - Icons: Avatars, - withIconBackground: false, - }; - - const toggleIsAgentChatCurrentContextActive = () => { - setIsAgentChatCurrentContextActive(!isAgentChatCurrentContextActive); - }; - - return ( - <> - {records.length !== 0 && ( - - - {isAgentChatCurrentContextActive ? ( - - ) : ( - - )} - - } - /> - - )} - - ); -}; diff --git a/packages/twenty-front/src/modules/ai/components/internal/SendMessageButton.tsx b/packages/twenty-front/src/modules/ai/components/internal/SendMessageButton.tsx index 09bbe5b35c..f34d78f559 100644 --- a/packages/twenty-front/src/modules/ai/components/internal/SendMessageButton.tsx +++ b/packages/twenty-front/src/modules/ai/components/internal/SendMessageButton.tsx @@ -1,18 +1,13 @@ import { AI_CHAT_INPUT_ID } from '@/ai/constants/AiChatInputId'; import { useAgentChatContextOrThrow } from '@/ai/hooks/useAgentChatContextOrThrow'; import { agentChatInputState } from '@/ai/states/agentChatInputState'; -import { type ObjectRecord } from '@/object-record/types/ObjectRecord'; import { useHotkeysOnFocusedElement } from '@/ui/utilities/hotkey/hooks/useHotkeysOnFocusedElement'; import { t } from '@lingui/core/macro'; import { useRecoilValue } from 'recoil'; import { Key } from 'ts-key-enum'; import { Button } from 'twenty-ui/input'; -export const SendMessageButton = ({ - records, -}: { - records?: ObjectRecord[]; -}) => { +export const SendMessageButton = () => { const agentChatInput = useRecoilValue(agentChatInputState); const { handleSendMessage, isLoading } = useAgentChatContextOrThrow(); @@ -21,7 +16,7 @@ export const SendMessageButton = ({ callback: (event: KeyboardEvent) => { if (!event.ctrlKey && !event.metaKey) { event.preventDefault(); - handleSendMessage(records); + handleSendMessage(); } }, focusId: AI_CHAT_INPUT_ID, @@ -34,7 +29,7 @@ export const SendMessageButton = ({ return (