From 8d9617131c4b6af3db51d262a6c6245d0c791af9 Mon Sep 17 00:00:00 2001 From: Abdul Rahman <81605929+abdulrahmancodes@users.noreply.github.com> Date: Wed, 17 Sep 2025 15:00:24 +0530 Subject: [PATCH] Feat: Add reasoning summary and tool details display for AI responses (#14414) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes [#1405](https://github.com/twentyhq/core-team-issues/issues/1405) ### Implemented Features - Reasoning Summary: Collapsible display of AI thinking process with real-time streaming - Tool Execution: Detailed tool call rendering with expandable results and loading states - Error Handling: Clean error stream event display with proper formatting ### Demo https://github.com/user-attachments/assets/76ad29b7-d831-4bc8-a115-ad25f7ace6e2 Screenshot 2025-09-12 at 3 23 39 AM --------- Co-authored-by: Félix Malfait --- .../src/generated-metadata/graphql.ts | 6 +- .../twenty-front/src/generated/graphql.ts | 2 +- .../AIChatAssistantMessageRenderer.tsx | 128 ++++++ .../modules/ai/components/AIChatMessage.tsx | 96 +--- .../ai/components/ErrorStepRenderer.tsx | 61 +++ .../ai/components/ReasoningSummaryDisplay.tsx | 117 +++++ .../modules/ai/components/ShimmeringText.tsx | 41 ++ .../ai/components/ToolStepRenderer.tsx | 159 +++++++ .../graphql/queries/getAgentChatMessages.ts | 2 +- .../src/modules/ai/hooks/useAgentChat.ts | 35 +- .../ai/states/agentStreamingMessageState.ts | 10 +- .../src/modules/ai/types/streamTypes.ts | 36 ++ .../parseAgentStreamingChunk.test.ts | 211 --------- .../ai/utils/__tests__/parseStream.test.ts | 431 ++++++++++++++++++ .../modules/ai/utils/extractErrorMessage.ts | 40 ++ .../src/modules/ai/utils/getToolIcon.ts | 28 ++ .../ai/utils/parseAgentStreamingChunk.ts | 47 -- .../src/modules/ai/utils/parseStream.ts | 128 ++++++ ...57472-RemoveContentFromAgentChatMessage.ts | 25 + .../ai/constants/ai-models.const.ts | 4 + .../__tests__/tool-adapter.service.spec.ts | 16 +- .../services/__tests__/tool.service.spec.ts | 10 +- .../ai/services/ai-model-registry.service.ts | 2 + .../core-modules/ai/services/tool.service.ts | 74 +-- .../file/services/file.service.ts | 4 +- .../tool/tools/http-tool/http-tool.schema.ts | 2 +- .../tool/tools/http-tool/http-tool.ts | 10 +- .../send-email-tool/send-email-tool.schema.ts | 2 +- .../tools/send-email-tool/send-email-tool.ts | 11 +- .../tool/types/tool-output.type.ts | 4 +- .../agent/agent-chat-message.entity.ts | 4 +- .../agent/agent-chat.service.ts | 12 +- .../agent/agent-execution.service.ts | 52 ++- .../agent/agent-handoff-executor.service.ts | 12 +- .../agent/agent-streaming.service.ts | 103 ++--- .../agent/constants/agent-config.const.ts | 1 + .../constants/agent-handoff-schema.const.ts | 2 +- .../agent/dtos/agent-chat-message.dto.ts | 4 +- .../agent/utils/agent-tool-schema.utils.ts | 2 +- ...structAssistantMessageContentFromStream.ts | 60 +++ .../workflow-tool.workspace-service.ts | 15 +- .../src/utils/stream-to-buffer.ts | 19 +- .../agent-tool.service.integration-spec.ts | 10 +- .../display/icon/components/TablerIcons.ts | 1 + packages/twenty-ui/src/display/index.ts | 1 + 45 files changed, 1500 insertions(+), 540 deletions(-) create mode 100644 packages/twenty-front/src/modules/ai/components/AIChatAssistantMessageRenderer.tsx create mode 100644 packages/twenty-front/src/modules/ai/components/ErrorStepRenderer.tsx create mode 100644 packages/twenty-front/src/modules/ai/components/ReasoningSummaryDisplay.tsx create mode 100644 packages/twenty-front/src/modules/ai/components/ShimmeringText.tsx create mode 100644 packages/twenty-front/src/modules/ai/components/ToolStepRenderer.tsx create mode 100644 packages/twenty-front/src/modules/ai/types/streamTypes.ts delete mode 100644 packages/twenty-front/src/modules/ai/utils/__tests__/parseAgentStreamingChunk.test.ts create mode 100644 packages/twenty-front/src/modules/ai/utils/__tests__/parseStream.test.ts create mode 100644 packages/twenty-front/src/modules/ai/utils/extractErrorMessage.ts create mode 100644 packages/twenty-front/src/modules/ai/utils/getToolIcon.ts delete mode 100644 packages/twenty-front/src/modules/ai/utils/parseAgentStreamingChunk.ts create mode 100644 packages/twenty-front/src/modules/ai/utils/parseStream.ts create mode 100644 packages/twenty-server/src/database/typeorm/core/migrations/common/1757991657472-RemoveContentFromAgentChatMessage.ts create mode 100644 packages/twenty-server/src/engine/metadata-modules/agent/utils/constructAssistantMessageContentFromStream.ts diff --git a/packages/twenty-front/src/generated-metadata/graphql.ts b/packages/twenty-front/src/generated-metadata/graphql.ts index bd2b95b082..b89e1669c5 100644 --- a/packages/twenty-front/src/generated-metadata/graphql.ts +++ b/packages/twenty-front/src/generated-metadata/graphql.ts @@ -68,10 +68,10 @@ export type Agent = { export type AgentChatMessage = { __typename?: 'AgentChatMessage'; - content: Scalars['String']; createdAt: Scalars['DateTime']; files: Array; id: Scalars['UUID']; + rawContent?: Maybe; role: Scalars['String']; threadId: Scalars['UUID']; }; @@ -4201,7 +4201,7 @@ export type GetAgentChatMessagesQueryVariables = Exact<{ }>; -export type GetAgentChatMessagesQuery = { __typename?: 'Query', agentChatMessages: Array<{ __typename?: 'AgentChatMessage', id: string, threadId: string, role: string, content: string, createdAt: string, files: Array<{ __typename?: 'File', id: string, name: string, fullPath: string, size: number, type: string, createdAt: string }> }> }; +export type GetAgentChatMessagesQuery = { __typename?: 'Query', agentChatMessages: Array<{ __typename?: 'AgentChatMessage', id: string, threadId: string, role: string, createdAt: string, rawContent?: string | null, files: Array<{ __typename?: 'File', id: string, name: string, fullPath: string, size: number, type: string, createdAt: string }> }> }; export type GetAgentChatThreadsQueryVariables = Exact<{ agentId: Scalars['UUID']; @@ -6348,8 +6348,8 @@ export const GetAgentChatMessagesDocument = gql` id threadId role - content createdAt + rawContent files { id name diff --git a/packages/twenty-front/src/generated/graphql.ts b/packages/twenty-front/src/generated/graphql.ts index 0c00d3e50d..9afdb7570f 100644 --- a/packages/twenty-front/src/generated/graphql.ts +++ b/packages/twenty-front/src/generated/graphql.ts @@ -68,10 +68,10 @@ export type Agent = { export type AgentChatMessage = { __typename?: 'AgentChatMessage'; - content: Scalars['String']; createdAt: Scalars['DateTime']; files: Array; id: Scalars['UUID']; + rawContent?: Maybe; role: Scalars['String']; threadId: Scalars['UUID']; }; diff --git a/packages/twenty-front/src/modules/ai/components/AIChatAssistantMessageRenderer.tsx b/packages/twenty-front/src/modules/ai/components/AIChatAssistantMessageRenderer.tsx new file mode 100644 index 0000000000..aa5c0129c1 --- /dev/null +++ b/packages/twenty-front/src/modules/ai/components/AIChatAssistantMessageRenderer.tsx @@ -0,0 +1,128 @@ +import { ErrorStepRenderer } from '@/ai/components/ErrorStepRenderer'; +import { ReasoningSummaryDisplay } from '@/ai/components/ReasoningSummaryDisplay'; +import { ToolStepRenderer } from '@/ai/components/ToolStepRenderer'; +import type { ParsedStep } from '@/ai/types/streamTypes'; +import { parseStream } from '@/ai/utils/parseStream'; +import { IconDotsVertical } from 'twenty-ui/display'; + +import { LazyMarkdownRenderer } from '@/ai/components/LazyMarkdownRenderer'; +import { agentStreamingMessageState } from '@/ai/states/agentStreamingMessageState'; +import { keyframes, useTheme } from '@emotion/react'; +import styled from '@emotion/styled'; +import { useRecoilValue } from 'recoil'; + +const StyledStepsContainer = styled.div` + display: flex; + flex-direction: column; + gap: ${({ theme }) => theme.spacing(1)}; +`; + +const StyledDotsIconContainer = styled.div` + align-items: center; + border: ${({ theme }) => `1px solid ${theme.border.color.light}`}; + border-radius: ${({ theme }) => theme.border.radius.md}; + display: flex; + justify-content: center; + padding-inline: ${({ theme }) => theme.spacing(1)}; +`; + +const StyledDotsIcon = styled(IconDotsVertical)` + color: ${({ theme }) => theme.font.color.light}; + transform: rotate(90deg); +`; + +const dots = keyframes` + 0% { content: ''; } + 33% { content: '.'; } + 66% { content: '..'; } + 100% { content: '...'; } +`; + +const StyledToolCallContainer = styled.div` + &::after { + display: inline-block; + content: ''; + animation: ${dots} 750ms steps(3, end) infinite; + width: 2ch; + text-align: left; + } +`; + +const LoadingDotsIcon = () => { + const theme = useTheme(); + + return ( + + + + ); +}; + +export const AIChatAssistantMessageRenderer = ({ + streamData, +}: { + streamData: string; +}) => { + const agentStreamingMessage = useRecoilValue(agentStreamingMessageState); + const isStreaming = + Boolean(agentStreamingMessage) && streamData === agentStreamingMessage; + + if (!streamData) { + return ; + } + + const isPlainString = + !streamData.includes('\n') || + !streamData.split('\n').some((line) => { + try { + JSON.parse(line); + return true; + } catch { + return false; + } + }); + + if (isPlainString) { + return ; + } + + const steps = parseStream(streamData); + + if (!steps.length) { + return ; + } + + const renderStep = (step: ParsedStep, index: number) => { + switch (step.type) { + case 'tool': + return ; + case 'reasoning': + return ( + + ); + case 'text': + return ; + case 'error': + return ( + + ); + default: + return null; + } + }; + + return ( +
+ {steps.map(renderStep)} + {isStreaming && } +
+ ); +}; diff --git a/packages/twenty-front/src/modules/ai/components/AIChatMessage.tsx b/packages/twenty-front/src/modules/ai/components/AIChatMessage.tsx index 379a9d2430..9696a017f9 100644 --- a/packages/twenty-front/src/modules/ai/components/AIChatMessage.tsx +++ b/packages/twenty-front/src/modules/ai/components/AIChatMessage.tsx @@ -1,17 +1,16 @@ -import { keyframes, useTheme } from '@emotion/react'; +import { useTheme } from '@emotion/react'; import styled from '@emotion/styled'; import { useRecoilValue } from 'recoil'; -import { Avatar, IconDotsVertical, IconSparkles } from 'twenty-ui/display'; +import { Avatar, IconSparkles } from 'twenty-ui/display'; -import { LazyMarkdownRenderer } from '@/ai/components/LazyMarkdownRenderer'; import { AgentChatFilePreview } from '@/ai/components/internal/AgentChatFilePreview'; import { AgentChatMessageRole } from '@/ai/constants/AgentChatMessageRole'; import { LightCopyIconButton } from '@/object-record/record-field/ui/components/LightCopyIconButton'; +import { AIChatAssistantMessageRenderer } from '@/ai/components/AIChatAssistantMessageRenderer'; import { type AgentChatMessage } from '~/generated/graphql'; import { dateLocaleState } from '~/localization/states/dateLocaleState'; import { beautifyPastDateRelativeToNow } from '~/utils/date-utils'; - const StyledMessageBubble = styled.div<{ isUser?: boolean }>` display: flex; flex-direction: column; @@ -25,11 +24,10 @@ const StyledMessageBubble = styled.div<{ isUser?: boolean }>` } `; -const StyledMessageRow = styled.div<{ isShowingToolCall?: boolean }>` +const StyledMessageRow = styled.div` display: flex; flex-direction: row; - align-items: ${({ isShowingToolCall }) => - isShowingToolCall ? 'center' : 'flex-start'}; + align-items: flex-start; gap: ${({ theme }) => theme.spacing(3)}; width: 100%; `; @@ -137,88 +135,22 @@ const StyledFilesContainer = styled.div` margin-top: ${({ theme }) => theme.spacing(2)}; `; -const dots = keyframes` - 0% { content: ''; } - 33% { content: '.'; } - 66% { content: '..'; } - 100% { content: '...'; } -`; - -const StyledToolCallContainer = styled.div` - &::after { - display: inline-block; - content: ''; - animation: ${dots} 750ms steps(3, end) infinite; - width: 2ch; - text-align: left; - } -`; - -const StyledDotsIconContainer = styled.div` - align-items: center; - border: ${({ theme }) => `1px solid ${theme.border.color.light}`}; - border-radius: ${({ theme }) => theme.border.radius.md}; - display: flex; - justify-content: center; - padding-inline: ${({ theme }) => theme.spacing(1)}; -`; - -const StyledDotsIcon = styled(IconDotsVertical)` - color: ${({ theme }) => theme.font.color.light}; - transform: rotate(90deg); -`; - export const AIChatMessage = ({ message, agentStreamingMessage, }: { message: AgentChatMessage; - agentStreamingMessage: { streamingText: string; toolCall: string }; + agentStreamingMessage: string; }) => { const theme = useTheme(); const { localeCatalog } = useRecoilValue(dateLocaleState); - const markdownRender = (text: string) => { - return ; - }; - - const getAssistantMessageContent = (message: AgentChatMessage) => { - if (message.content !== '') { - return markdownRender(message.content); - } - - if (agentStreamingMessage.streamingText !== '') { - return markdownRender(agentStreamingMessage.streamingText); - } - - if (agentStreamingMessage.toolCall !== '') { - return ( - - {agentStreamingMessage.toolCall} - - ); - } - - return ( - - - - ); - }; - return ( - + {message.role === AgentChatMessageRole.ASSISTANT && ( - {message.role === AgentChatMessageRole.ASSISTANT - ? getAssistantMessageContent(message) - : message.content} + {message.role === AgentChatMessageRole.ASSISTANT ? ( + + ) : ( + message.rawContent + )} {message.files.length > 0 && ( @@ -249,7 +185,7 @@ export const AIChatMessage = ({ ))} )} - {message.content && ( + {message.rawContent && ( {beautifyPastDateRelativeToNow( @@ -257,7 +193,7 @@ export const AIChatMessage = ({ localeCatalog, )} - + )} diff --git a/packages/twenty-front/src/modules/ai/components/ErrorStepRenderer.tsx b/packages/twenty-front/src/modules/ai/components/ErrorStepRenderer.tsx new file mode 100644 index 0000000000..0ad53d105f --- /dev/null +++ b/packages/twenty-front/src/modules/ai/components/ErrorStepRenderer.tsx @@ -0,0 +1,61 @@ +import { extractErrorMessage } from '@/ai/utils/extractErrorMessage'; +import { useTheme } from '@emotion/react'; +import styled from '@emotion/styled'; +import { IconAlertCircle } from 'twenty-ui/display'; + +const StyledContainer = styled.div` + align-items: flex-start; + background-color: ${({ theme }) => theme.color.red10}; + border: 1px solid ${({ theme }) => theme.color.red20}; + border-radius: ${({ theme }) => theme.border.radius.md}; + display: flex; + gap: ${({ theme }) => theme.spacing(2)}; + margin-block: ${({ theme }) => theme.spacing(2)}; + padding: ${({ theme }) => theme.spacing(3)}; +`; + +const StyledIconContainer = styled.div` + align-items: center; + color: ${({ theme }) => theme.color.red60}; + display: flex; + flex-shrink: 0; + justify-content: center; +`; + +const StyledContent = styled.div` + flex: 1; +`; + +const StyledTitle = styled.div` + font-weight: ${({ theme }) => theme.font.weight.medium}; + color: ${({ theme }) => theme.color.red80}; + margin-bottom: ${({ theme }) => theme.spacing(1)}; +`; + +const StyledMessage = styled.div` + color: ${({ theme }) => theme.color.red70}; + line-height: ${({ theme }) => theme.text.lineHeight.lg}; +`; + +export const ErrorStepRenderer = ({ + message, + error, +}: { + message: string; + error?: unknown; +}) => { + const theme = useTheme(); + const errorMessage = error ? extractErrorMessage(error) : message; + + return ( + + + + + + Error + {errorMessage} + + + ); +}; diff --git a/packages/twenty-front/src/modules/ai/components/ReasoningSummaryDisplay.tsx b/packages/twenty-front/src/modules/ai/components/ReasoningSummaryDisplay.tsx new file mode 100644 index 0000000000..b40233ba1b --- /dev/null +++ b/packages/twenty-front/src/modules/ai/components/ReasoningSummaryDisplay.tsx @@ -0,0 +1,117 @@ +import { useTheme } from '@emotion/react'; +import styled from '@emotion/styled'; +import { useState } from 'react'; + +import { IconBrain, IconChevronDown, IconChevronUp } from 'twenty-ui/display'; +import { AnimatedExpandableContainer } from 'twenty-ui/layout'; + +import { ShimmeringText } from '@/ai/components/ShimmeringText'; +import { t } from '@lingui/core/macro'; + +const StyledContainer = styled.div` + display: flex; + flex-direction: column; + gap: ${({ theme }) => theme.spacing(2)}; + margin-bottom: ${({ theme }) => theme.spacing(2)}; +`; + +const StyledThinkingText = styled.div` + color: ${({ theme }) => theme.font.color.tertiary}; + font-size: ${({ theme }) => theme.font.size.md}; + font-weight: ${({ theme }) => theme.font.weight.medium}; +`; + +const StyledReasoningContainer = styled.div` + background: ${({ theme }) => theme.background.transparent.lighter}; + border-radius: ${({ theme }) => theme.border.radius.sm}; + padding: ${({ theme }) => theme.spacing(3)}; + border: 1px solid ${({ theme }) => theme.border.color.light}; +`; + +const StyledReasoningText = styled.div` + color: ${({ theme }) => theme.font.color.secondary}; + font-size: ${({ theme }) => theme.font.size.sm}; + line-height: ${({ theme }) => theme.text.lineHeight.lg}; + white-space: pre-wrap; +`; + +const StyledToggleButton = styled.div` + align-items: center; + background: none; + border: none; + color: ${({ theme }) => theme.font.color.tertiary}; + cursor: pointer; + display: flex; + font-size: ${({ theme }) => theme.font.size.md}; + font-weight: ${({ theme }) => theme.font.weight.medium}; + gap: ${({ theme }) => theme.spacing(1)}; + padding: ${({ theme }) => theme.spacing(1)} 0; + transition: color ${({ theme }) => theme.animation.duration.normal}s; + + &:hover { + color: ${({ theme }) => theme.font.color.secondary}; + } +`; + +const StyledIconContainer = styled.div` + display: flex; + align-items: center; + gap: ${({ theme }) => theme.spacing(1)}; +`; + +export const ReasoningSummaryDisplay = ({ + content, + isThinking = false, +}: { + content: string; + isThinking?: boolean; +}) => { + const theme = useTheme(); + const [isExpanded, setIsExpanded] = useState(false); + + const hasContent = content.trim().length > 0; + + if (!hasContent) { + return null; + } + + return ( + + {isThinking && ( + <> + + + + {t`Thinking...`} + + + + {content} + + + )} + + {hasContent && !isThinking && ( + <> + setIsExpanded(!isExpanded)}> + + + {t`Finished thinking`} + + {isExpanded ? ( + + ) : ( + + )} + + + + + {content} + + + + )} + + ); +}; diff --git a/packages/twenty-front/src/modules/ai/components/ShimmeringText.tsx b/packages/twenty-front/src/modules/ai/components/ShimmeringText.tsx new file mode 100644 index 0000000000..df97aa6050 --- /dev/null +++ b/packages/twenty-front/src/modules/ai/components/ShimmeringText.tsx @@ -0,0 +1,41 @@ +import styled from '@emotion/styled'; + +const StyledShimmeringText = styled.div` + background: ${({ theme }) => theme.font.color.light} + linear-gradient( + 90deg, + ${({ theme }) => theme.font.color.light} 0%, + ${({ theme }) => theme.font.color.primary} 50%, + ${({ theme }) => theme.font.color.light} 100% + ); + background-size: 200% 100%; + background-position: -200% top; + background-repeat: no-repeat; + background-clip: text; + -webkit-background-clip: text; + -webkit-text-fill-color: transparent; + animation: shimmer-wave 1s infinite linear; + + @keyframes shimmer-wave { + 0% { + background-position: 200% top; + } + 100% { + background-position: -200% top; + } + } +`; + +export const ShimmeringText = ({ + children, + className, +}: { + children: React.ReactNode; + className?: string; +}) => { + return ( + + {children} + + ); +}; diff --git a/packages/twenty-front/src/modules/ai/components/ToolStepRenderer.tsx b/packages/twenty-front/src/modules/ai/components/ToolStepRenderer.tsx new file mode 100644 index 0000000000..d1d868221b --- /dev/null +++ b/packages/twenty-front/src/modules/ai/components/ToolStepRenderer.tsx @@ -0,0 +1,159 @@ +import { useTheme } from '@emotion/react'; +import styled from '@emotion/styled'; +import { useState } from 'react'; + +import { IconChevronDown, IconChevronUp } from 'twenty-ui/display'; +import { AnimatedExpandableContainer } from 'twenty-ui/layout'; + +import { ShimmeringText } from '@/ai/components/ShimmeringText'; +import type { + ToolCallEvent, + ToolEvent, + ToolResultEvent, +} from '@/ai/types/streamTypes'; +import { extractErrorMessage } from '@/ai/utils/extractErrorMessage'; +import { getToolIcon } from '@/ai/utils/getToolIcon'; + +const StyledContainer = styled.div` + display: flex; + flex-direction: column; + gap: ${({ theme }) => theme.spacing(2)}; +`; + +const StyledLoadingContainer = styled.div` + align-items: center; + display: flex; + gap: ${({ theme }) => theme.spacing(2)}; +`; + +const StyledContentContainer = styled.div` + background: ${({ theme }) => theme.background.transparent.lighter}; + border-radius: ${({ theme }) => theme.border.radius.sm}; + padding: ${({ theme }) => theme.spacing(3)}; + border: 1px solid ${({ theme }) => theme.border.color.light}; +`; + +const StyledToggleButton = styled.div<{ isExpandable: boolean }>` + align-items: center; + background: none; + border: none; + cursor: ${({ isExpandable }) => (isExpandable ? 'pointer' : 'auto')}; + display: flex; + color: ${({ theme }) => theme.font.color.tertiary}; + gap: ${({ theme }) => theme.spacing(1)}; + padding: ${({ theme }) => theme.spacing(1)} 0; + transition: color ${({ theme }) => theme.animation.duration.normal}s; + + &:hover { + color: ${({ theme }) => theme.font.color.secondary}; + } +`; + +const StyledDisplayMessage = styled.span` + color: ${({ theme }) => theme.font.color.tertiary}; + font-size: ${({ theme }) => theme.font.size.md}; + font-weight: ${({ theme }) => theme.font.weight.medium}; +`; + +const StyledPre = styled.pre` + margin-top: ${({ theme }) => theme.spacing(1)}; + white-space: pre-wrap; +`; + +const StyledIconTextContainer = styled.div` + display: flex; + align-items: center; + gap: ${({ theme }) => theme.spacing(1)}; + + svg { + min-width: ${({ theme }) => theme.icon.size.sm}px; + } +`; + +export const ToolStepRenderer = ({ events }: { events: ToolEvent[] }) => { + const theme = useTheme(); + const [isExpanded, setIsExpanded] = useState(false); + + const toolCall = events[0] as ToolCallEvent | undefined; + const toolResult = events.find( + (event): event is ToolResultEvent => event.type === 'tool-result', + ); + + if (!toolCall) { + return null; + } + + const toolOutput = toolResult?.result as ToolResultEvent['result']; + const isStandardizedFormat = + toolOutput && typeof toolOutput === 'object' && 'success' in toolOutput; + + const hasResult = isStandardizedFormat + ? Boolean(toolOutput.result) + : Boolean(toolResult?.result); + const hasError = isStandardizedFormat ? Boolean(toolOutput.error) : false; + const isExpandable = hasResult || hasError; + + if (!toolResult) { + return ( + + + + + {toolCall.args.loadingMessage} + + + + + ); + } + + const displayMessage = + toolResult?.result && + typeof toolResult.result === 'object' && + 'message' in toolResult.result + ? (toolResult.result as { message: string }).message + : undefined; + + const ToolIcon = getToolIcon(toolCall.toolName); + + return ( + + setIsExpanded(!isExpanded)} + isExpandable={isExpandable} + > + + + {displayMessage} + + {isExpandable && + (isExpanded ? ( + + ) : ( + + ))} + + + {isExpandable && ( + + + {isStandardizedFormat ? ( + <> + {hasError &&
{extractErrorMessage(toolOutput.error)}
} + {hasResult && ( +
+ + {JSON.stringify(toolOutput.result, null, 2)} + +
+ )} + + ) : toolResult?.result ? ( + JSON.stringify(toolResult.result, null, 2) + ) : undefined} + + + )} + + ); +}; diff --git a/packages/twenty-front/src/modules/ai/graphql/queries/getAgentChatMessages.ts b/packages/twenty-front/src/modules/ai/graphql/queries/getAgentChatMessages.ts index 0d0e55d97b..c151098f98 100644 --- a/packages/twenty-front/src/modules/ai/graphql/queries/getAgentChatMessages.ts +++ b/packages/twenty-front/src/modules/ai/graphql/queries/getAgentChatMessages.ts @@ -6,8 +6,8 @@ export const GET_AGENT_CHAT_MESSAGES = gql` id threadId role - content createdAt + rawContent files { id name diff --git a/packages/twenty-front/src/modules/ai/hooks/useAgentChat.ts b/packages/twenty-front/src/modules/ai/hooks/useAgentChat.ts index aaba950eb5..2d225a48fd 100644 --- a/packages/twenty-front/src/modules/ai/hooks/useAgentChat.ts +++ b/packages/twenty-front/src/modules/ai/hooks/useAgentChat.ts @@ -1,4 +1,3 @@ -import { useSnackBar } from '@/ui/feedback/snack-bar-manager/hooks/useSnackBar'; import { useRecoilComponentState } from '@/ui/utilities/state/component-state/hooks/useRecoilComponentState'; import { useState } from 'react'; import { useRecoilState } from 'recoil'; @@ -31,7 +30,6 @@ import { type AgentChatMessage } from '~/generated/graphql'; import { agentChatInputState } from '../states/agentChatInputState'; import { agentChatMessagesComponentState } from '../states/agentChatMessagesComponentState'; import { agentStreamingMessageState } from '../states/agentStreamingMessageState'; -import { parseAgentStreamingChunk } from '../utils/parseAgentStreamingChunk'; type OptimisticMessage = AgentChatMessage & { isPending: boolean; @@ -39,7 +37,6 @@ type OptimisticMessage = AgentChatMessage & { export const useAgentChat = (agentId: string, records?: ObjectRecord[]) => { const apolloClient = useApolloClient(); - const { enqueueErrorSnackBar } = useSnackBar(); const { getObjectMetadataItemById } = useGetObjectMetadataItemById(); const contextStoreCurrentObjectMetadataItemId = useRecoilComponentValue( @@ -126,7 +123,7 @@ export const useAgentChat = (agentId: string, records?: ObjectRecord[]) => { id: v4(), threadId: currentThreadId as string, role: AgentChatMessageRole.USER, - content, + rawContent: content, createdAt: new Date().toISOString(), isPending: true, files: agentChatUploadedFiles, @@ -136,7 +133,7 @@ export const useAgentChat = (agentId: string, records?: ObjectRecord[]) => { id: v4(), threadId: currentThreadId as string, role: AgentChatMessageRole.ASSISTANT, - content: '', + rawContent: '', createdAt: new Date().toISOString(), isPending: true, files: [], @@ -180,27 +177,8 @@ export const useAgentChat = (agentId: string, records?: ObjectRecord[]) => { }, context: { onChunk: (chunk: string) => { - parseAgentStreamingChunk(chunk, { - onTextDelta: (message: string) => { - setAgentStreamingMessage((prev) => ({ - ...prev, - streamingText: prev.streamingText + message, - })); - scrollToBottom(); - }, - onToolCall: (message: string) => { - setAgentStreamingMessage((prev) => ({ - ...prev, - toolCall: message, - })); - scrollToBottom(); - }, - onError: (message: string) => { - enqueueErrorSnackBar({ - message, - }); - }, - }); + setAgentStreamingMessage((prev) => prev + chunk); + scrollToBottom(); }, }, }); @@ -225,10 +203,7 @@ export const useAgentChat = (agentId: string, records?: ObjectRecord[]) => { const { data } = await refetchMessages(); setAgentChatMessages(data?.agentChatMessages); - setAgentStreamingMessage({ - toolCall: '', - streamingText: '', - }); + setAgentStreamingMessage(''); scrollToBottom(); }; diff --git a/packages/twenty-front/src/modules/ai/states/agentStreamingMessageState.ts b/packages/twenty-front/src/modules/ai/states/agentStreamingMessageState.ts index c0cc47d5ce..5aaffced58 100644 --- a/packages/twenty-front/src/modules/ai/states/agentStreamingMessageState.ts +++ b/packages/twenty-front/src/modules/ai/states/agentStreamingMessageState.ts @@ -1,12 +1,6 @@ import { atom } from 'recoil'; -export const agentStreamingMessageState = atom<{ - toolCall: string; - streamingText: string; -}>({ +export const agentStreamingMessageState = atom({ key: 'agentStreamingMessageState', - default: { - toolCall: '', - streamingText: '', - }, + default: '', }); diff --git a/packages/twenty-front/src/modules/ai/types/streamTypes.ts b/packages/twenty-front/src/modules/ai/types/streamTypes.ts new file mode 100644 index 0000000000..2d580e999d --- /dev/null +++ b/packages/twenty-front/src/modules/ai/types/streamTypes.ts @@ -0,0 +1,36 @@ +export type ToolCallEvent = { + type: 'tool-call'; + toolCallId: string; + toolName: string; + args: { + loadingMessage: string; + input: unknown; + }; +}; + +export type ToolResultEvent = { + type: 'tool-result'; + toolCallId: string; + toolName: string; + result: { + success: boolean; + result?: unknown; + error?: string; + message: string; + }; + message: string; +}; + +export type ToolEvent = ToolCallEvent | ToolResultEvent; + +export type ErrorEvent = { + type: 'error'; + message: string; + error?: unknown; +}; + +export type ParsedStep = + | { type: 'tool'; events: ToolEvent[] } + | { type: 'reasoning'; content: string; isThinking: boolean } + | { type: 'text'; content: string } + | { type: 'error'; message: string; error?: unknown }; diff --git a/packages/twenty-front/src/modules/ai/utils/__tests__/parseAgentStreamingChunk.test.ts b/packages/twenty-front/src/modules/ai/utils/__tests__/parseAgentStreamingChunk.test.ts deleted file mode 100644 index 5cc92dd79b..0000000000 --- a/packages/twenty-front/src/modules/ai/utils/__tests__/parseAgentStreamingChunk.test.ts +++ /dev/null @@ -1,211 +0,0 @@ -import { parseAgentStreamingChunk } from '../parseAgentStreamingChunk'; - -describe('parseAgentStreamingChunk', () => { - let mockCallbacks: { - onTextDelta: jest.Mock; - onToolCall: jest.Mock; - onError: jest.Mock; - onParseError: jest.Mock; - }; - - beforeEach(() => { - mockCallbacks = { - onTextDelta: jest.fn(), - onToolCall: jest.fn(), - onError: jest.fn(), - onParseError: jest.fn(), - }; - }); - - afterEach(() => { - jest.clearAllMocks(); - }); - - describe('valid event types', () => { - it('should call onTextDelta for text-delta events', () => { - const chunk = JSON.stringify({ - type: 'text-delta', - message: 'Hello world', - }); - - parseAgentStreamingChunk(chunk, mockCallbacks); - - expect(mockCallbacks.onTextDelta).toHaveBeenCalledWith('Hello world'); - expect(mockCallbacks.onToolCall).not.toHaveBeenCalled(); - expect(mockCallbacks.onError).not.toHaveBeenCalled(); - expect(mockCallbacks.onParseError).not.toHaveBeenCalled(); - }); - - it('should call onToolCall for tool-call events', () => { - const chunk = JSON.stringify({ - type: 'tool-call', - message: 'Tool execution result', - }); - - parseAgentStreamingChunk(chunk, mockCallbacks); - - expect(mockCallbacks.onToolCall).toHaveBeenCalledWith( - 'Tool execution result', - ); - expect(mockCallbacks.onTextDelta).not.toHaveBeenCalled(); - expect(mockCallbacks.onError).not.toHaveBeenCalled(); - expect(mockCallbacks.onParseError).not.toHaveBeenCalled(); - }); - - it('should call onError for error events', () => { - const chunk = JSON.stringify({ - type: 'error', - message: 'Something went wrong', - }); - - parseAgentStreamingChunk(chunk, mockCallbacks); - - expect(mockCallbacks.onError).toHaveBeenCalledWith( - 'Something went wrong', - ); - expect(mockCallbacks.onTextDelta).not.toHaveBeenCalled(); - expect(mockCallbacks.onToolCall).not.toHaveBeenCalled(); - expect(mockCallbacks.onParseError).not.toHaveBeenCalled(); - }); - }); - - describe('multiple events in chunk', () => { - it('should process multiple events separated by newlines', () => { - const chunk = [ - JSON.stringify({ type: 'text-delta', message: 'First message' }), - JSON.stringify({ type: 'tool-call', message: 'Tool result' }), - JSON.stringify({ type: 'error', message: 'Error occurred' }), - ].join('\n'); - - parseAgentStreamingChunk(chunk, mockCallbacks); - - expect(mockCallbacks.onTextDelta).toHaveBeenCalledWith('First message'); - expect(mockCallbacks.onToolCall).toHaveBeenCalledWith('Tool result'); - expect(mockCallbacks.onError).toHaveBeenCalledWith('Error occurred'); - expect(mockCallbacks.onParseError).not.toHaveBeenCalled(); - }); - - it('should skip empty lines', () => { - const chunk = [ - JSON.stringify({ type: 'text-delta', message: 'First message' }), - '', - ' ', - JSON.stringify({ type: 'tool-call', message: 'Tool result' }), - ].join('\n'); - - parseAgentStreamingChunk(chunk, mockCallbacks); - - expect(mockCallbacks.onTextDelta).toHaveBeenCalledWith('First message'); - expect(mockCallbacks.onToolCall).toHaveBeenCalledWith('Tool result'); - expect(mockCallbacks.onParseError).not.toHaveBeenCalled(); - }); - }); - - describe('JSON parsing errors', () => { - it('should call onParseError for invalid JSON', () => { - const chunk = 'invalid json content'; - - parseAgentStreamingChunk(chunk, mockCallbacks); - - expect(mockCallbacks.onParseError).toHaveBeenCalledWith( - expect.any(Error), - 'invalid json content', - ); - expect(mockCallbacks.onTextDelta).not.toHaveBeenCalled(); - expect(mockCallbacks.onToolCall).not.toHaveBeenCalled(); - expect(mockCallbacks.onError).not.toHaveBeenCalled(); - }); - - it('should call onParseError for malformed JSON', () => { - const chunk = '{"type": "text-delta", "message": "unclosed quote}'; - - parseAgentStreamingChunk(chunk, mockCallbacks); - - expect(mockCallbacks.onParseError).toHaveBeenCalledWith( - expect.any(Error), - '{"type": "text-delta", "message": "unclosed quote}', - ); - }); - - it('should handle mixed valid and invalid JSON in same chunk', () => { - const chunk = [ - JSON.stringify({ type: 'text-delta', message: 'Valid message' }), - 'invalid json', - JSON.stringify({ type: 'tool-call', message: 'Another valid message' }), - ].join('\n'); - - parseAgentStreamingChunk(chunk, mockCallbacks); - - expect(mockCallbacks.onTextDelta).toHaveBeenCalledWith('Valid message'); - expect(mockCallbacks.onToolCall).toHaveBeenCalledWith( - 'Another valid message', - ); - expect(mockCallbacks.onParseError).toHaveBeenCalledWith( - expect.any(Error), - 'invalid json', - ); - }); - }); - - describe('optional callbacks', () => { - it('should not throw when callbacks are undefined', () => { - const chunk = JSON.stringify({ - type: 'text-delta', - message: 'Test message', - }); - - expect(() => { - parseAgentStreamingChunk(chunk, {}); - }).not.toThrow(); - }); - - it('should handle partial callback definitions', () => { - const chunk = JSON.stringify({ - type: 'text-delta', - message: 'Test message', - }); - - const partialCallbacks = { - onTextDelta: jest.fn(), - }; - - parseAgentStreamingChunk(chunk, partialCallbacks); - - expect(partialCallbacks.onTextDelta).toHaveBeenCalledWith('Test message'); - }); - }); - - describe('edge cases', () => { - it('should handle empty chunk', () => { - parseAgentStreamingChunk('', mockCallbacks); - - expect(mockCallbacks.onTextDelta).not.toHaveBeenCalled(); - expect(mockCallbacks.onToolCall).not.toHaveBeenCalled(); - expect(mockCallbacks.onError).not.toHaveBeenCalled(); - expect(mockCallbacks.onParseError).not.toHaveBeenCalled(); - }); - - it('should handle chunk with only whitespace', () => { - parseAgentStreamingChunk(' \n\t\n ', mockCallbacks); - - expect(mockCallbacks.onTextDelta).not.toHaveBeenCalled(); - expect(mockCallbacks.onToolCall).not.toHaveBeenCalled(); - expect(mockCallbacks.onError).not.toHaveBeenCalled(); - expect(mockCallbacks.onParseError).not.toHaveBeenCalled(); - }); - - it('should handle unknown event types gracefully', () => { - const chunk = JSON.stringify({ - type: 'unknown-type', - message: 'Unknown event', - }); - - parseAgentStreamingChunk(chunk, mockCallbacks); - - expect(mockCallbacks.onTextDelta).not.toHaveBeenCalled(); - expect(mockCallbacks.onToolCall).not.toHaveBeenCalled(); - expect(mockCallbacks.onError).not.toHaveBeenCalled(); - expect(mockCallbacks.onParseError).not.toHaveBeenCalled(); - }); - }); -}); diff --git a/packages/twenty-front/src/modules/ai/utils/__tests__/parseStream.test.ts b/packages/twenty-front/src/modules/ai/utils/__tests__/parseStream.test.ts new file mode 100644 index 0000000000..79ec1d28bc --- /dev/null +++ b/packages/twenty-front/src/modules/ai/utils/__tests__/parseStream.test.ts @@ -0,0 +1,431 @@ +import { parseStream } from '../parseStream'; + +describe('parseStream', () => { + describe('tool-call events', () => { + it('should parse tool-call event correctly', () => { + const streamText = JSON.stringify({ + type: 'tool-call', + toolCallId: 'call-123', + toolName: 'send_email', + args: { + loadingMessage: 'Sending email...', + input: { to: 'test@example.com' }, + }, + }); + + const result = parseStream(streamText); + + expect(result).toHaveLength(1); + expect(result[0]).toEqual({ + type: 'tool', + events: [ + { + type: 'tool-call', + toolCallId: 'call-123', + toolName: 'send_email', + args: { + loadingMessage: 'Sending email...', + input: { to: 'test@example.com' }, + }, + }, + ], + }); + }); + }); + + describe('tool-result events', () => { + it('should parse tool-result event and append to existing tool entry', () => { + const streamText = [ + JSON.stringify({ + type: 'tool-call', + toolCallId: 'call-123', + toolName: 'send_email', + args: { loadingMessage: 'Sending email...', input: {} }, + }), + JSON.stringify({ + type: 'tool-result', + toolCallId: 'call-123', + toolName: 'send_email', + result: { sucess: true, result: 'Email sent', message: 'Success' }, + message: 'Email sent successfully', + }), + ].join('\n'); + + const result = parseStream(streamText); + + expect(result).toHaveLength(1); + expect(result[0]).toEqual({ + type: 'tool', + events: [ + { + type: 'tool-call', + toolCallId: 'call-123', + toolName: 'send_email', + args: { loadingMessage: 'Sending email...', input: {} }, + }, + { + type: 'tool-result', + toolCallId: 'call-123', + toolName: 'send_email', + result: { sucess: true, result: 'Email sent', message: 'Success' }, + message: 'Email sent successfully', + }, + ], + }); + }); + + it('should create new tool entry for orphaned tool-result', () => { + const streamText = JSON.stringify({ + type: 'tool-result', + toolCallId: 'call-456', + toolName: 'http_request', + result: { + sucess: true, + result: 'Response received', + message: 'Success', + }, + message: 'Request completed', + }); + + const result = parseStream(streamText); + + expect(result).toHaveLength(1); + expect(result[0]).toEqual({ + type: 'tool', + events: [ + { + type: 'tool-result', + toolCallId: 'call-456', + toolName: 'http_request', + result: { + sucess: true, + result: 'Response received', + message: 'Success', + }, + message: 'Request completed', + }, + ], + }); + }); + }); + + describe('reasoning events', () => { + it('should parse reasoning events correctly', () => { + const streamText = [ + JSON.stringify({ + type: 'reasoning', + textDelta: 'Let me think about this...', + }), + JSON.stringify({ + type: 'reasoning', + textDelta: ' I need to consider the options.', + }), + JSON.stringify({ type: 'reasoning-signature' }), + ].join('\n'); + + const result = parseStream(streamText); + + expect(result).toHaveLength(1); + expect(result[0]).toEqual({ + type: 'reasoning', + content: 'Let me think about this... I need to consider the options.', + isThinking: false, + }); + }); + + it('should handle reasoning without signature as thinking', () => { + const streamText = JSON.stringify({ + type: 'reasoning', + textDelta: 'Still thinking...', + }); + + const result = parseStream(streamText); + + expect(result).toHaveLength(1); + expect(result[0]).toEqual({ + type: 'reasoning', + content: 'Still thinking...', + isThinking: true, + }); + }); + + it('should concatenate multiple reasoning deltas', () => { + const streamText = [ + JSON.stringify({ type: 'reasoning', textDelta: 'First part' }), + JSON.stringify({ type: 'reasoning', textDelta: ' second part' }), + JSON.stringify({ type: 'reasoning', textDelta: ' third part' }), + ].join('\n'); + + const result = parseStream(streamText); + + expect(result).toHaveLength(1); + expect(result[0]).toEqual({ + type: 'reasoning', + content: 'First part second part third part', + isThinking: true, + }); + }); + }); + + describe('text-delta events', () => { + it('should parse text-delta events correctly', () => { + const streamText = [ + JSON.stringify({ type: 'text-delta', textDelta: 'Hello, ' }), + JSON.stringify({ type: 'text-delta', textDelta: 'world!' }), + ].join('\n'); + + const result = parseStream(streamText); + + expect(result).toHaveLength(1); + expect(result[0]).toEqual({ + type: 'text', + content: 'Hello, world!', + }); + }); + + it('should handle empty textDelta', () => { + const streamText = JSON.stringify({ type: 'text-delta', textDelta: '' }); + + const result = parseStream(streamText); + + expect(result).toHaveLength(1); + expect(result[0]).toEqual({ + type: 'text', + content: '', + }); + }); + }); + + describe('error events', () => { + it('should parse error events correctly', () => { + const streamText = JSON.stringify({ + type: 'error', + message: 'Something went wrong', + error: { code: 'TIMEOUT', details: 'Request timed out' }, + }); + + const result = parseStream(streamText); + + expect(result).toHaveLength(1); + expect(result[0]).toEqual({ + type: 'error', + message: 'Something went wrong', + error: { code: 'TIMEOUT', details: 'Request timed out' }, + }); + }); + + it('should handle error without error details', () => { + const streamText = JSON.stringify({ + type: 'error', + message: 'Generic error', + }); + + const result = parseStream(streamText); + + expect(result).toHaveLength(1); + expect(result[0]).toEqual({ + type: 'error', + message: 'Generic error', + error: undefined, + }); + }); + + it('should use default message when none provided', () => { + const streamText = JSON.stringify({ + type: 'error', + }); + + const result = parseStream(streamText); + + expect(result).toHaveLength(1); + expect(result[0]).toEqual({ + type: 'error', + message: 'An error occurred', + error: undefined, + }); + }); + }); + + describe('step-finish events', () => { + it('should flush current text block on step-finish', () => { + const streamText = [ + JSON.stringify({ type: 'text-delta', textDelta: 'Some text' }), + JSON.stringify({ type: 'step-finish' }), + ].join('\n'); + + const result = parseStream(streamText); + + expect(result).toHaveLength(1); + expect(result[0]).toEqual({ + type: 'text', + content: 'Some text', + }); + }); + + it('should mark reasoning as not thinking on step-finish', () => { + const streamText = [ + JSON.stringify({ type: 'reasoning', textDelta: 'Thinking...' }), + JSON.stringify({ type: 'step-finish' }), + ].join('\n'); + + const result = parseStream(streamText); + + expect(result).toHaveLength(1); + expect(result[0]).toEqual({ + type: 'reasoning', + content: 'Thinking...', + isThinking: false, + }); + }); + }); + + describe('mixed events', () => { + it('should handle mixed event types correctly', () => { + const streamText = [ + JSON.stringify({ type: 'text-delta', textDelta: 'Starting...' }), + JSON.stringify({ + type: 'tool-call', + toolCallId: 'call-1', + toolName: 'send_email', + args: { loadingMessage: 'Sending...', input: {} }, + }), + JSON.stringify({ type: 'reasoning', textDelta: 'Let me think...' }), + JSON.stringify({ + type: 'tool-result', + toolCallId: 'call-1', + toolName: 'send_email', + result: { sucess: true, message: 'Done' }, + message: 'Email sent', + }), + JSON.stringify({ type: 'text-delta', textDelta: 'Finished!' }), + ].join('\n'); + + const result = parseStream(streamText); + + expect(result).toHaveLength(4); + expect(result[0]).toEqual({ type: 'text', content: 'Starting...' }); + expect(result[1]).toEqual({ + type: 'tool', + events: [ + { + type: 'tool-call', + toolCallId: 'call-1', + toolName: 'send_email', + args: { loadingMessage: 'Sending...', input: {} }, + }, + { + type: 'tool-result', + toolCallId: 'call-1', + toolName: 'send_email', + result: { sucess: true, message: 'Done' }, + message: 'Email sent', + }, + ], + }); + expect(result[2]).toEqual({ + type: 'reasoning', + content: 'Let me think...', + isThinking: true, + }); + expect(result[3]).toEqual({ + type: 'text', + content: 'Finished!', + }); + }); + }); + + describe('edge cases', () => { + it('should handle empty stream', () => { + const result = parseStream(''); + expect(result).toEqual([]); + }); + + it('should handle whitespace-only stream', () => { + const result = parseStream(' \n \t '); + expect(result).toEqual([]); + }); + + it('should skip invalid JSON lines', () => { + const streamText = [ + 'invalid json line', + JSON.stringify({ type: 'text-delta', textDelta: 'Valid content' }), + 'another invalid line', + ].join('\n'); + + const result = parseStream(streamText); + + expect(result).toHaveLength(1); + expect(result[0]).toEqual({ + type: 'text', + content: 'Valid content', + }); + }); + + it('should handle unknown event types', () => { + const streamText = JSON.stringify({ + type: 'unknown-event', + data: 'some data', + }); + + const result = parseStream(streamText); + expect(result).toEqual([]); + }); + + it('should flush remaining text block at end', () => { + const streamText = JSON.stringify({ + type: 'text-delta', + textDelta: 'Unflushed content', + }); + + const result = parseStream(streamText); + + expect(result).toHaveLength(1); + expect(result[0]).toEqual({ + type: 'text', + content: 'Unflushed content', + }); + }); + }); + + describe('text block transitions', () => { + it('should create new text block when switching from reasoning to text', () => { + const streamText = [ + JSON.stringify({ type: 'reasoning', textDelta: 'Thinking...' }), + JSON.stringify({ type: 'text-delta', textDelta: 'Speaking...' }), + ].join('\n'); + + const result = parseStream(streamText); + + expect(result).toHaveLength(2); + expect(result[0]).toEqual({ + type: 'reasoning', + content: 'Thinking...', + isThinking: true, + }); + expect(result[1]).toEqual({ + type: 'text', + content: 'Speaking...', + }); + }); + + it('should create new reasoning block when switching from text to reasoning', () => { + const streamText = [ + JSON.stringify({ type: 'text-delta', textDelta: 'Speaking...' }), + JSON.stringify({ type: 'reasoning', textDelta: 'Thinking...' }), + ].join('\n'); + + const result = parseStream(streamText); + + expect(result).toHaveLength(2); + expect(result[0]).toEqual({ + type: 'text', + content: 'Speaking...', + }); + expect(result[1]).toEqual({ + type: 'reasoning', + content: 'Thinking...', + isThinking: true, + }); + }); + }); +}); diff --git a/packages/twenty-front/src/modules/ai/utils/extractErrorMessage.ts b/packages/twenty-front/src/modules/ai/utils/extractErrorMessage.ts new file mode 100644 index 0000000000..3e2c9de151 --- /dev/null +++ b/packages/twenty-front/src/modules/ai/utils/extractErrorMessage.ts @@ -0,0 +1,40 @@ +import { isDefined } from 'twenty-shared/utils'; + +export const extractErrorMessage = (error: unknown): string => { + if (typeof error === 'string') { + return error; + } + + if (!isDefined(error) || typeof error !== 'object') { + return 'An unexpected error occurred'; + } + + if ('message' in error && typeof error.message === 'string') { + return error.message; + } + + if ( + 'error' in error && + isDefined(error.error) && + typeof error.error === 'object' && + 'message' in error.error && + typeof error.error.message === 'string' + ) { + return error.error.message; + } + + if ( + 'data' in error && + isDefined(error.data) && + typeof error.data === 'object' && + 'error' in error.data && + isDefined(error.data.error) && + typeof error.data.error === 'object' && + 'message' in error.data.error && + typeof error.data.error.message === 'string' + ) { + return error.data.error.message; + } + + return 'An unexpected error occurred'; +}; diff --git a/packages/twenty-front/src/modules/ai/utils/getToolIcon.ts b/packages/twenty-front/src/modules/ai/utils/getToolIcon.ts new file mode 100644 index 0000000000..bc48f79f49 --- /dev/null +++ b/packages/twenty-front/src/modules/ai/utils/getToolIcon.ts @@ -0,0 +1,28 @@ +import { IconDatabase, IconMail, IconTool, IconWorld } from 'twenty-ui/display'; + +const TOOL_ICON_MAPPINGS = [ + { + keywords: ['email'], + icon: IconMail, + }, + { + keywords: ['http_request'], + icon: IconWorld, + }, + { + keywords: ['create_', 'update_', 'find_', 'delete_'], + icon: IconDatabase, + }, + { + keywords: ['workflow', 'handoff'], + icon: IconTool, + }, +] as const; + +export const getToolIcon = (toolName: string) => { + const mapping = TOOL_ICON_MAPPINGS.find(({ keywords }) => + keywords.some((keyword) => toolName.includes(keyword)), + ); + + return mapping?.icon ?? IconTool; +}; diff --git a/packages/twenty-front/src/modules/ai/utils/parseAgentStreamingChunk.ts b/packages/twenty-front/src/modules/ai/utils/parseAgentStreamingChunk.ts deleted file mode 100644 index a0cb4006cc..0000000000 --- a/packages/twenty-front/src/modules/ai/utils/parseAgentStreamingChunk.ts +++ /dev/null @@ -1,47 +0,0 @@ -export type AgentStreamingEvent = { - type: 'text-delta' | 'tool-call' | 'error'; - message: string; -}; - -export type AgentStreamingParserCallbacks = { - onTextDelta?: (message: string) => void; - onToolCall?: (message: string) => void; - onError?: (message: string) => void; - onParseError?: (error: Error, rawLine: string) => void; -}; - -export const parseAgentStreamingChunk = ( - chunk: string, - callbacks: AgentStreamingParserCallbacks, -): void => { - const lines = chunk.split('\n'); - - for (const line of lines) { - if (line.trim() !== '') { - try { - const event = JSON.parse(line) as AgentStreamingEvent; - - switch (event.type) { - case 'text-delta': - callbacks.onTextDelta?.(event.message); - break; - case 'tool-call': - callbacks.onToolCall?.(event.message); - break; - case 'error': - callbacks.onError?.(event.message); - break; - } - } catch (error) { - // eslint-disable-next-line no-console - console.error('Failed to parse stream event:', error); - - const errorMessage = - error instanceof Error - ? error - : new Error(`Unknown parsing error: ${String(error)}`); - callbacks.onParseError?.(errorMessage, line); - } - } - } -}; diff --git a/packages/twenty-front/src/modules/ai/utils/parseStream.ts b/packages/twenty-front/src/modules/ai/utils/parseStream.ts new file mode 100644 index 0000000000..6f66f93270 --- /dev/null +++ b/packages/twenty-front/src/modules/ai/utils/parseStream.ts @@ -0,0 +1,128 @@ +import { isDefined } from 'twenty-shared/utils'; + +import { + type ParsedStep, + type ToolEvent, + type ToolResultEvent, +} from '@/ai/types/streamTypes'; + +type TextBlock = + | { type: 'reasoning'; content: string; isThinking: boolean } + | { type: 'text'; content: string } + | null; + +export const parseStream = (streamText: string): ParsedStep[] => { + const lines = streamText.trim().split('\n'); + + const output: ParsedStep[] = []; + let currentTextBlock: TextBlock = null; + + const flushTextBlock = () => { + if (isDefined(currentTextBlock)) { + output.push(currentTextBlock); + currentTextBlock = null; + } + }; + + for (const line of lines) { + let event; + try { + event = JSON.parse(line); + } catch { + continue; + } + + switch (event.type) { + case 'tool-call': + flushTextBlock(); + output.push({ + type: 'tool', + events: [ + { + type: 'tool-call', + toolCallId: event.toolCallId, + toolName: event.toolName, + args: event.args, + }, + ] as ToolEvent[], + }); + break; + + case 'tool-result': { + flushTextBlock(); + + const toolEntry = output.find( + (item): item is { type: 'tool'; events: ToolEvent[] } => + item.type === 'tool' && + item.events.some( + (e) => + e.type === 'tool-call' && e.toolCallId === event.toolCallId, + ), + ); + + const resultEvent: ToolResultEvent = { + type: 'tool-result', + toolCallId: event.toolCallId, + toolName: event.toolName, + result: event.result, + message: event.message, + }; + + if (isDefined(toolEntry)) { + toolEntry.events.push(resultEvent); + } else { + output.push({ + type: 'tool', + events: [resultEvent], + }); + } + break; + } + + case 'reasoning': + if (!currentTextBlock || currentTextBlock.type !== 'reasoning') { + flushTextBlock(); + currentTextBlock = { + type: 'reasoning', + content: '', + isThinking: true, + }; + } + currentTextBlock.content += event.textDelta || ''; + break; + + case 'text-delta': + if (!currentTextBlock || currentTextBlock.type !== 'text') { + flushTextBlock(); + currentTextBlock = { type: 'text', content: '' }; + } + currentTextBlock.content += event.textDelta || ''; + break; + + case 'reasoning-signature': + if (currentTextBlock?.type === 'reasoning') { + currentTextBlock.isThinking = false; + } + break; + + case 'step-finish': + if (currentTextBlock?.type === 'reasoning') { + currentTextBlock.isThinking = false; + } + flushTextBlock(); + break; + + case 'error': + flushTextBlock(); + output.push({ + type: 'error', + message: event.message || 'An error occurred', + error: event.error, + }); + break; + } + } + + flushTextBlock(); + return output; +}; diff --git a/packages/twenty-server/src/database/typeorm/core/migrations/common/1757991657472-RemoveContentFromAgentChatMessage.ts b/packages/twenty-server/src/database/typeorm/core/migrations/common/1757991657472-RemoveContentFromAgentChatMessage.ts new file mode 100644 index 0000000000..a706a054ae --- /dev/null +++ b/packages/twenty-server/src/database/typeorm/core/migrations/common/1757991657472-RemoveContentFromAgentChatMessage.ts @@ -0,0 +1,25 @@ +import { type MigrationInterface, type QueryRunner } from 'typeorm'; + +export class RemoveContentFromAgentChatMessage1757991657472 + implements MigrationInterface +{ + name = 'RemoveContentFromAgentChatMessage1757991657472'; + + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query( + `ALTER TABLE "core"."agentChatMessage" RENAME COLUMN "content" TO "rawContent"`, + ); + await queryRunner.query( + `ALTER TABLE "core"."agentChatMessage" ALTER COLUMN "rawContent" DROP NOT NULL`, + ); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query( + `ALTER TABLE "core"."agentChatMessage" ALTER COLUMN "rawContent" SET NOT NULL`, + ); + await queryRunner.query( + `ALTER TABLE "core"."agentChatMessage" RENAME COLUMN "rawContent" TO "content"`, + ); + } +} diff --git a/packages/twenty-server/src/engine/core-modules/ai/constants/ai-models.const.ts b/packages/twenty-server/src/engine/core-modules/ai/constants/ai-models.const.ts index 4d06d3401a..840d035048 100644 --- a/packages/twenty-server/src/engine/core-modules/ai/constants/ai-models.const.ts +++ b/packages/twenty-server/src/engine/core-modules/ai/constants/ai-models.const.ts @@ -25,6 +25,7 @@ export interface AIModelConfig { provider: ModelProvider; inputCostPer1kTokensInCents: number; outputCostPer1kTokensInCents: number; + doesSupportThinking?: boolean; } export const AI_MODELS: AIModelConfig[] = [ @@ -55,6 +56,7 @@ export const AI_MODELS: AIModelConfig[] = [ provider: ModelProvider.ANTHROPIC, inputCostPer1kTokensInCents: 1.5, outputCostPer1kTokensInCents: 7.5, + doesSupportThinking: true, }, { modelId: 'claude-sonnet-4-20250514', @@ -62,6 +64,7 @@ export const AI_MODELS: AIModelConfig[] = [ provider: ModelProvider.ANTHROPIC, inputCostPer1kTokensInCents: 0.3, outputCostPer1kTokensInCents: 1.5, + doesSupportThinking: true, }, { modelId: 'claude-3-5-haiku-20241022', @@ -69,6 +72,7 @@ export const AI_MODELS: AIModelConfig[] = [ provider: ModelProvider.ANTHROPIC, inputCostPer1kTokensInCents: 0.08, outputCostPer1kTokensInCents: 0.4, + doesSupportThinking: true, }, { modelId: 'grok-3', diff --git a/packages/twenty-server/src/engine/core-modules/ai/services/__tests__/tool-adapter.service.spec.ts b/packages/twenty-server/src/engine/core-modules/ai/services/__tests__/tool-adapter.service.spec.ts index df210a8ca3..a2ea3d2050 100644 --- a/packages/twenty-server/src/engine/core-modules/ai/services/__tests__/tool-adapter.service.spec.ts +++ b/packages/twenty-server/src/engine/core-modules/ai/services/__tests__/tool-adapter.service.spec.ts @@ -1,12 +1,12 @@ import { Test } from '@nestjs/testing'; import { ToolAdapterService } from 'src/engine/core-modules/ai/services/tool-adapter.service'; -import { ToolRegistryService } from 'src/engine/core-modules/tool/services/tool-registry.service'; -import { PermissionsService } from 'src/engine/metadata-modules/permissions/permissions.service'; import { ToolType } from 'src/engine/core-modules/tool/enums/tool-type.enum'; -import { type Tool } from 'src/engine/core-modules/tool/types/tool.type'; +import { ToolRegistryService } from 'src/engine/core-modules/tool/services/tool-registry.service'; import { type ToolInput } from 'src/engine/core-modules/tool/types/tool-input.type'; +import { type Tool } from 'src/engine/core-modules/tool/types/tool.type'; import { PermissionFlagType } from 'src/engine/metadata-modules/permissions/constants/permission-flag-type.constants'; +import { PermissionsService } from 'src/engine/metadata-modules/permissions/permissions.service'; const createMockToolRegistry = () => ({ getAllToolTypes: jest.fn(), @@ -27,6 +27,8 @@ describe('ToolAdapterService', () => { // Shared tools const unflaggedToolExecute = jest.fn(async (input: ToolInput) => ({ + success: true, + message: 'Tool executed successfully', result: { echoed: input }, })); const unflaggedTool: Tool = { @@ -36,6 +38,8 @@ describe('ToolAdapterService', () => { }; const flaggedToolExecute = jest.fn(async (input: ToolInput) => ({ + success: true, + message: 'Tool executed successfully', result: { sent: input }, })); const flaggedTool: Tool = { @@ -152,6 +156,10 @@ describe('ToolAdapterService', () => { // Ensure wrapper forwards only parameters.input expect(unflaggedToolExecute).toHaveBeenCalledWith(input); - expect(result).toEqual({ result: { echoed: input } }); + expect(result).toEqual({ + success: true, + message: 'Tool executed successfully', + result: { echoed: input }, + }); }); }); diff --git a/packages/twenty-server/src/engine/core-modules/ai/services/__tests__/tool.service.spec.ts b/packages/twenty-server/src/engine/core-modules/ai/services/__tests__/tool.service.spec.ts index 3847afdbe1..677d6ef900 100644 --- a/packages/twenty-server/src/engine/core-modules/ai/services/__tests__/tool.service.spec.ts +++ b/packages/twenty-server/src/engine/core-modules/ai/services/__tests__/tool.service.spec.ts @@ -1,10 +1,10 @@ import { Test } from '@nestjs/testing'; import { ToolService } from 'src/engine/core-modules/ai/services/tool.service'; -import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager'; +import { RecordInputTransformerService } from 'src/engine/core-modules/record-transformer/services/record-input-transformer.service'; import { ObjectMetadataService } from 'src/engine/metadata-modules/object-metadata/object-metadata.service'; import { WorkspacePermissionsCacheService } from 'src/engine/metadata-modules/workspace-permissions-cache/workspace-permissions-cache.service'; -import { RecordInputTransformerService } from 'src/engine/core-modules/record-transformer/services/record-input-transformer.service'; +import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager'; import { WorkspaceCacheStorageService } from 'src/engine/workspace-cache-storage/workspace-cache-storage.service'; import { getMockObjectMetadataEntity } from 'src/utils/__test__/get-object-metadata-entity.mock'; @@ -147,7 +147,7 @@ describe('ToolService', () => { ); expect(result.success).toBe(true); - expect(result.record).toEqual(record); + expect(result.result).toEqual(record); expect(ormManager.getRepositoryForWorkspace).toHaveBeenCalledWith( workspaceId, 'testObject', @@ -189,8 +189,8 @@ describe('ToolService', () => { ); expect(result.success).toBe(true); - expect(result.records).toEqual(records); - expect(result.count).toBe(2); + expect(result.result.records).toEqual(records); + expect(result.result.count).toBe(2); expect(mockRepo.find).toHaveBeenCalled(); }); }); diff --git a/packages/twenty-server/src/engine/core-modules/ai/services/ai-model-registry.service.ts b/packages/twenty-server/src/engine/core-modules/ai/services/ai-model-registry.service.ts index 7f4b098fea..c775371f3a 100644 --- a/packages/twenty-server/src/engine/core-modules/ai/services/ai-model-registry.service.ts +++ b/packages/twenty-server/src/engine/core-modules/ai/services/ai-model-registry.service.ts @@ -16,6 +16,7 @@ export interface RegisteredAIModel { modelId: string; provider: ModelProvider; model: LanguageModel; + doesSupportThinking?: boolean; } @Injectable() @@ -86,6 +87,7 @@ export class AiModelRegistryService { modelId: modelConfig.modelId, provider: ModelProvider.ANTHROPIC, model: anthropic(modelConfig.modelId), + doesSupportThinking: modelConfig.doesSupportThinking, }); }); } diff --git a/packages/twenty-server/src/engine/core-modules/ai/services/tool.service.ts b/packages/twenty-server/src/engine/core-modules/ai/services/tool.service.ts index 699b721527..bfccb6985c 100644 --- a/packages/twenty-server/src/engine/core-modules/ai/services/tool.service.ts +++ b/packages/twenty-server/src/engine/core-modules/ai/services/tool.service.ts @@ -173,15 +173,17 @@ export class ToolService { return { success: true, - records, - count: records.length, message: `Found ${records.length} ${objectName} records`, + result: { + records, + count: records.length, + }, }; } catch (error) { return { success: false, - error: error instanceof Error ? error.message : 'Unknown error', message: `Failed to find ${objectName} records`, + error: error instanceof Error ? error.message : 'Unknown error', }; } } @@ -205,8 +207,8 @@ export class ToolService { if (!id || typeof id !== 'string') { return { success: false, - error: 'Record ID is required', message: `Failed to find ${objectName}: Record ID is required`, + error: 'Record ID is required', }; } @@ -217,21 +219,21 @@ export class ToolService { if (!record) { return { success: false, - error: 'Record not found', message: `Failed to find ${objectName}: Record with ID ${id} not found`, + error: 'Record not found', }; } return { success: true, - record, message: `Found ${objectName} record`, + result: record, }; } catch (error) { return { success: false, - error: error instanceof Error ? error.message : 'Unknown error', message: `Failed to find ${objectName} record`, + error: error instanceof Error ? error.message : 'Unknown error', }; } } @@ -261,8 +263,8 @@ export class ToolService { if (!objectMetadataItemWithFieldsMaps) { return { success: false, - error: 'Object metadata not found', message: `Failed to create ${objectName}: Object metadata not found`, + error: 'Object metadata not found', }; } @@ -276,14 +278,14 @@ export class ToolService { return { success: true, - record: createdRecord, message: `Successfully created ${objectName}`, + result: createdRecord, }; } catch (error) { return { success: false, - error: error instanceof Error ? error.message : 'Unknown error', message: `Failed to create ${objectName}`, + error: error instanceof Error ? error.message : 'Unknown error', }; } } @@ -307,8 +309,8 @@ export class ToolService { if (!id || typeof id !== 'string') { return { success: false, - error: 'Record ID is required for update', message: `Failed to update ${objectName}: Record ID is required`, + error: 'Record ID is required for update', }; } @@ -319,8 +321,8 @@ export class ToolService { if (!existingRecord) { return { success: false, - error: 'Record not found', message: `Failed to update ${objectName}: Record with ID ${id} not found`, + error: 'Record not found', }; } @@ -335,8 +337,8 @@ export class ToolService { if (!objectMetadataItemWithFieldsMaps) { return { success: false, - error: 'Object metadata not found', message: `Failed to update ${objectName}: Object metadata not found`, + error: 'Object metadata not found', }; } @@ -355,21 +357,21 @@ export class ToolService { if (!updatedRecord) { return { success: false, - error: 'Failed to retrieve updated record', message: `Failed to update ${objectName}: Could not retrieve updated record`, + error: 'Failed to retrieve updated record', }; } return { success: true, - record: updatedRecord, message: `Successfully updated ${objectName}`, + result: updatedRecord, }; } catch (error) { return { success: false, - error: error instanceof Error ? error.message : 'Unknown error', message: `Failed to update ${objectName}`, + error: error instanceof Error ? error.message : 'Unknown error', }; } } @@ -393,8 +395,8 @@ export class ToolService { if (!id || typeof id !== 'string') { return { success: false, - error: 'Record ID is required for soft delete', message: `Failed to soft delete ${objectName}: Record ID is required`, + error: 'Record ID is required for soft delete', }; } @@ -405,8 +407,8 @@ export class ToolService { if (!existingRecord) { return { success: false, - error: 'Record not found', message: `Failed to soft delete ${objectName}: Record with ID ${id} not found`, + error: 'Record not found', }; } @@ -415,12 +417,13 @@ export class ToolService { return { success: true, message: `Successfully soft deleted ${objectName}`, + result: { id }, }; } catch (error) { return { success: false, - error: error instanceof Error ? error.message : 'Unknown error', message: `Failed to soft delete ${objectName}`, + error: error instanceof Error ? error.message : 'Unknown error', }; } } @@ -444,8 +447,8 @@ export class ToolService { if (!id || typeof id !== 'string') { return { success: false, - error: 'Record ID is required for destroy', message: `Failed to destroy ${objectName}: Record ID is required`, + error: 'Record ID is required for destroy', }; } @@ -456,8 +459,8 @@ export class ToolService { if (!existingRecord) { return { success: false, - error: 'Record not found', message: `Failed to destroy ${objectName}: Record with ID ${id} not found`, + error: 'Record not found', }; } @@ -466,12 +469,13 @@ export class ToolService { return { success: true, message: `Successfully destroyed ${objectName}`, + result: { id }, }; } catch (error) { return { success: false, - error: error instanceof Error ? error.message : 'Unknown error', message: `Failed to destroy ${objectName}`, + error: error instanceof Error ? error.message : 'Unknown error', }; } } @@ -495,8 +499,8 @@ export class ToolService { if (!filter || typeof filter !== 'object' || !('id' in filter)) { return { success: false, - error: 'Filter with record IDs is required for bulk soft delete', message: `Failed to soft delete many ${objectName}: Filter with record IDs is required`, + error: 'Filter with record IDs is required for bulk soft delete', }; } @@ -506,8 +510,8 @@ export class ToolService { if (!Array.isArray(recordIds) || recordIds.length === 0) { return { success: false, - error: 'At least one record ID is required for bulk soft delete', message: `Failed to soft delete many ${objectName}: At least one record ID is required`, + error: 'At least one record ID is required for bulk soft delete', }; } @@ -518,8 +522,8 @@ export class ToolService { if (existingRecords.length === 0) { return { success: false, - error: 'No records found to soft delete', message: `Failed to soft delete many ${objectName}: No records found with the provided IDs`, + error: 'No records found to soft delete', }; } @@ -527,14 +531,17 @@ export class ToolService { return { success: true, - count: existingRecords.length, message: `Successfully soft deleted ${existingRecords.length} ${objectName} records`, + result: { + count: existingRecords.length, + deletedIds: recordIds, + }, }; } catch (error) { return { success: false, - error: error instanceof Error ? error.message : 'Unknown error', message: `Failed to soft delete many ${objectName}`, + error: error instanceof Error ? error.message : 'Unknown error', }; } } @@ -558,8 +565,8 @@ export class ToolService { if (!filter || typeof filter !== 'object' || !('id' in filter)) { return { success: false, - error: 'Filter with record IDs is required for bulk destroy', message: `Failed to destroy many ${objectName}: Filter with record IDs is required`, + error: 'Filter with record IDs is required for bulk destroy', }; } @@ -569,8 +576,8 @@ export class ToolService { if (!Array.isArray(recordIds) || recordIds.length === 0) { return { success: false, - error: 'At least one record ID is required for bulk destroy', message: `Failed to destroy many ${objectName}: At least one record ID is required`, + error: 'At least one record ID is required for bulk destroy', }; } @@ -581,8 +588,8 @@ export class ToolService { if (existingRecords.length === 0) { return { success: false, - error: 'No records found to destroy', message: `Failed to destroy many ${objectName}: No records found with the provided IDs`, + error: 'No records found to destroy', }; } @@ -590,14 +597,17 @@ export class ToolService { return { success: true, - count: existingRecords.length, message: `Successfully destroyed ${existingRecords.length} ${objectName} records`, + result: { + count: existingRecords.length, + destroyedIds: recordIds, + }, }; } catch (error) { return { success: false, - error: error instanceof Error ? error.message : 'Unknown error', message: `Failed to destroy many ${objectName}`, + error: error instanceof Error ? error.message : 'Unknown error', }; } } diff --git a/packages/twenty-server/src/engine/core-modules/file/services/file.service.ts b/packages/twenty-server/src/engine/core-modules/file/services/file.service.ts index ca03b2c6a0..0352196917 100644 --- a/packages/twenty-server/src/engine/core-modules/file/services/file.service.ts +++ b/packages/twenty-server/src/engine/core-modules/file/services/file.service.ts @@ -1,7 +1,7 @@ import { Injectable } from '@nestjs/common'; import { basename, dirname, extname } from 'path'; -import { type Stream } from 'stream'; +import { type Readable } from 'stream'; import { isNonEmptyString } from '@sniptt/guards'; import { buildSignedPath } from 'twenty-shared/utils'; @@ -28,7 +28,7 @@ export class FileService { folderPath: string, filename: string, workspaceId: string, - ): Promise { + ): Promise { const workspaceFolderPath = `workspace-${workspaceId}/${folderPath}`; return await this.fileStorageService.read({ diff --git a/packages/twenty-server/src/engine/core-modules/tool/tools/http-tool/http-tool.schema.ts b/packages/twenty-server/src/engine/core-modules/tool/tools/http-tool/http-tool.schema.ts index c19675d7bd..e3bbdfece1 100644 --- a/packages/twenty-server/src/engine/core-modules/tool/tools/http-tool/http-tool.schema.ts +++ b/packages/twenty-server/src/engine/core-modules/tool/tools/http-tool/http-tool.schema.ts @@ -16,7 +16,7 @@ export const HttpRequestInputZodSchema = z.object({ }); export const HttpToolParametersZodSchema = z.object({ - toolDescription: z + loadingMessage: z .string() .describe( "A clear, human-readable status message describing the HTTP request being made. This will be shown to the user while the tool is being called, so phrase it as a present-tense status update (e.g., 'Making a GET request to ...'). Explain what endpoint you are calling and with what parameters in natural language.", diff --git a/packages/twenty-server/src/engine/core-modules/tool/tools/http-tool/http-tool.ts b/packages/twenty-server/src/engine/core-modules/tool/tools/http-tool/http-tool.ts index 8e18cd63d2..da6d4ab8d1 100644 --- a/packages/twenty-server/src/engine/core-modules/tool/tools/http-tool/http-tool.ts +++ b/packages/twenty-server/src/engine/core-modules/tool/tools/http-tool/http-tool.ts @@ -38,15 +38,23 @@ export class HttpTool implements Tool { const response = await axios(axiosConfig); - return { result: response.data }; + return { + success: true, + message: `HTTP ${method} request to ${url} completed successfully`, + result: response.data, + }; } catch (error) { if (axios.isAxiosError(error)) { return { + success: false, + message: `HTTP ${method} request to ${url} failed`, error: error.response?.data || error.message || 'HTTP request failed', }; } return { + success: false, + message: `HTTP ${method} request to ${url} failed`, error: error instanceof Error ? error.message : 'HTTP request failed', }; } diff --git a/packages/twenty-server/src/engine/core-modules/tool/tools/send-email-tool/send-email-tool.schema.ts b/packages/twenty-server/src/engine/core-modules/tool/tools/send-email-tool/send-email-tool.schema.ts index f2ff90eafa..d9ef21756e 100644 --- a/packages/twenty-server/src/engine/core-modules/tool/tools/send-email-tool/send-email-tool.schema.ts +++ b/packages/twenty-server/src/engine/core-modules/tool/tools/send-email-tool/send-email-tool.schema.ts @@ -14,7 +14,7 @@ export const SendEmailInputZodSchema = z.object({ }); export const SendEmailToolParametersZodSchema = z.object({ - toolDescription: z + loadingMessage: z .string() .describe( "A clear, human-readable status message describing the email being sent. This will be shown to the user while the tool is being called, so phrase it as a present-tense status update (e.g., 'Sending email to customer about order status'). Explain what email you are sending and to whom in natural language.", diff --git a/packages/twenty-server/src/engine/core-modules/tool/tools/send-email-tool/send-email-tool.ts b/packages/twenty-server/src/engine/core-modules/tool/tools/send-email-tool/send-email-tool.ts index 8e615faf33..ca763aa3b2 100644 --- a/packages/twenty-server/src/engine/core-modules/tool/tools/send-email-tool/send-email-tool.ts +++ b/packages/twenty-server/src/engine/core-modules/tool/tools/send-email-tool/send-email-tool.ts @@ -133,14 +133,19 @@ export class SendEmailTool implements Tool { this.logger.log(`Email sent successfully to ${email}`); return { + success: true, + message: `Email sent successfully to ${email}`, result: { - success: true, - message: `Email sent successfully to ${email}`, + recipient: email, + subject: safeSubject, + connectedAccountId, }, }; } catch (error) { if (error instanceof SendEmailToolException) { return { + success: false, + message: `Failed to send email to ${email}`, error: error.message, }; } @@ -148,6 +153,8 @@ export class SendEmailTool implements Tool { this.logger.error(`Failed to send email: ${error}`); return { + success: false, + message: `Failed to send email to ${email}`, error: error instanceof Error ? error.message : 'Failed to send email', }; } diff --git a/packages/twenty-server/src/engine/core-modules/tool/types/tool-output.type.ts b/packages/twenty-server/src/engine/core-modules/tool/types/tool-output.type.ts index 612be0aec6..e4a9c9557f 100644 --- a/packages/twenty-server/src/engine/core-modules/tool/types/tool-output.type.ts +++ b/packages/twenty-server/src/engine/core-modules/tool/types/tool-output.type.ts @@ -1,4 +1,6 @@ export type ToolOutput = { - result?: unknown; + success: boolean; + message: string; error?: string; + result?: unknown; }; diff --git a/packages/twenty-server/src/engine/metadata-modules/agent/agent-chat-message.entity.ts b/packages/twenty-server/src/engine/metadata-modules/agent/agent-chat-message.entity.ts index 9335f46056..04c433bb88 100644 --- a/packages/twenty-server/src/engine/metadata-modules/agent/agent-chat-message.entity.ts +++ b/packages/twenty-server/src/engine/metadata-modules/agent/agent-chat-message.entity.ts @@ -36,8 +36,8 @@ export class AgentChatMessageEntity { @Column({ type: 'enum', enum: AgentChatMessageRole }) role: AgentChatMessageRole; - @Column('text') - content: string; + @Column({ type: 'text', nullable: true }) + rawContent: string | null; @OneToMany(() => FileEntity, (file) => file.message) files: Relation; diff --git a/packages/twenty-server/src/engine/metadata-modules/agent/agent-chat.service.ts b/packages/twenty-server/src/engine/metadata-modules/agent/agent-chat.service.ts index 5d7cb28abf..2e1baef9f4 100644 --- a/packages/twenty-server/src/engine/metadata-modules/agent/agent-chat.service.ts +++ b/packages/twenty-server/src/engine/metadata-modules/agent/agent-chat.service.ts @@ -68,18 +68,18 @@ export class AgentChatService { async addMessage({ threadId, role, - content, + rawContent, fileIds, }: { threadId: string; role: AgentChatMessageRole; - content: string; + rawContent: string | null; fileIds?: string[]; }) { const message = this.messageRepository.create({ threadId, role, - content, + rawContent, }); const savedMessage = await this.messageRepository.save(message); @@ -92,7 +92,7 @@ export class AgentChatService { } } - this.generateTitleIfNeeded(threadId, content); + this.generateTitleIfNeeded(threadId, rawContent); return savedMessage; } @@ -121,14 +121,14 @@ export class AgentChatService { private async generateTitleIfNeeded( threadId: string, - messageContent: string, + messageContent: string | null, ) { const thread = await this.threadRepository.findOne({ where: { id: threadId }, select: ['id', 'title'], }); - if (!thread || thread.title) { + if (!thread || thread.title || !messageContent) { return; } diff --git a/packages/twenty-server/src/engine/metadata-modules/agent/agent-execution.service.ts b/packages/twenty-server/src/engine/metadata-modules/agent/agent-execution.service.ts index a070fa6575..7d17a8dcba 100644 --- a/packages/twenty-server/src/engine/metadata-modules/agent/agent-execution.service.ts +++ b/packages/twenty-server/src/engine/metadata-modules/agent/agent-execution.service.ts @@ -1,8 +1,6 @@ import { Injectable, Logger } from '@nestjs/common'; import { InjectRepository } from '@nestjs/typeorm'; -import { type Readable } from 'stream'; - import { type CoreMessage, type CoreUserMessage, @@ -31,6 +29,7 @@ import { AgentHandoffToolService } from 'src/engine/metadata-modules/agent/agent import { AGENT_CONFIG } from 'src/engine/metadata-modules/agent/constants/agent-config.const'; import { AGENT_SYSTEM_PROMPTS } from 'src/engine/metadata-modules/agent/constants/agent-system-prompts.const'; import { type RecordIdsByObjectMetadataNameSingularType } from 'src/engine/metadata-modules/agent/types/recordIdsByObjectMetadataNameSingular.type'; +import { constructAssistantMessageContentFromStream } from 'src/engine/metadata-modules/agent/utils/constructAssistantMessageContentFromStream'; import { WorkspacePermissionsCacheService } from 'src/engine/metadata-modules/workspace-permissions-cache/workspace-permissions-cache.service'; import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager'; import { streamToBuffer } from 'src/utils/stream-to-buffer'; @@ -115,6 +114,16 @@ export class AgentExecutionService { ...(messages && { messages }), ...(prompt && { prompt }), maxSteps: AGENT_CONFIG.MAX_STEPS, + ...(registeredModel.doesSupportThinking && { + providerOptions: { + anthropic: { + thinking: { + type: 'enabled', + budgetTokens: AGENT_CONFIG.REASONING_BUDGET_TOKENS, + }, + }, + }, + }), }; } catch (error) { this.logger.error( @@ -233,7 +242,7 @@ export class AgentExecutionService { filename, file.workspaceId, ); - const fileBuffer = await streamToBuffer(fileStream as Readable); + const fileBuffer = await streamToBuffer(fileStream); if (file.type.startsWith('image')) { return { @@ -241,13 +250,33 @@ export class AgentExecutionService { image: fileBuffer, mimeType: file.type, }; - } else { - return { - type: 'file', - data: fileBuffer, - mimeType: file.type, - }; } + + return { + type: 'file', + data: fileBuffer, + mimeType: file.type, + }; + } + + private mapMessagesToCoreMessages( + messages: AgentChatMessageEntity[], + ): CoreMessage[] { + return messages + .map(({ role, rawContent }): CoreMessage => { + if (role === AgentChatMessageRole.USER) { + return { + role: 'user', + content: rawContent ?? '', + }; + } + + return { + role: 'assistant', + content: constructAssistantMessageContentFromStream(rawContent ?? ''), + }; + }) + .filter((message) => message.content.length > 0); } async streamChatResponse({ @@ -271,10 +300,7 @@ export class AgentExecutionService { where: { id: agentId }, }); - const llmMessages: CoreMessage[] = messages.map(({ role, content }) => ({ - role, - content, - })); + const llmMessages: CoreMessage[] = this.mapMessagesToCoreMessages(messages); let contextString = ''; diff --git a/packages/twenty-server/src/engine/metadata-modules/agent/agent-handoff-executor.service.ts b/packages/twenty-server/src/engine/metadata-modules/agent/agent-handoff-executor.service.ts index 7d6ea2c7d8..423c8a97e6 100644 --- a/packages/twenty-server/src/engine/metadata-modules/agent/agent-handoff-executor.service.ts +++ b/packages/twenty-server/src/engine/metadata-modules/agent/agent-handoff-executor.service.ts @@ -86,7 +86,14 @@ export class AgentHandoffExecutorService { `Successfully executed handoff to agent ${toAgentId} with response length: ${textResponse.text.length}`, ); - return textResponse.text; + return { + success: true, + message: `Successfully executed handoff to agent ${targetAgent.name}`, + result: { + response: textResponse.text, + targetAgentName: targetAgent.name, + }, + }; } catch (error) { this.logger.error( `Handoff execution failed: ${error.message}`, @@ -95,8 +102,7 @@ export class AgentHandoffExecutorService { return { success: false, - newAgentId: handoffRequest.toAgentId, - newAgentName: 'Unknown', + message: `Failed to execute handoff to agent ${handoffRequest.toAgentId}`, error: error.message, }; } diff --git a/packages/twenty-server/src/engine/metadata-modules/agent/agent-streaming.service.ts b/packages/twenty-server/src/engine/metadata-modules/agent/agent-streaming.service.ts index 991b9557ba..3e4eb18317 100644 --- a/packages/twenty-server/src/engine/metadata-modules/agent/agent-streaming.service.ts +++ b/packages/twenty-server/src/engine/metadata-modules/agent/agent-streaming.service.ts @@ -4,6 +4,7 @@ import { InjectRepository } from '@nestjs/typeorm'; import { type Response } from 'express'; import { Repository } from 'typeorm'; +import { type Workspace } from 'src/engine/core-modules/workspace/workspace.entity'; import { AgentChatMessageRole } from 'src/engine/metadata-modules/agent/agent-chat-message.entity'; import { AgentChatThreadEntity } from 'src/engine/metadata-modules/agent/agent-chat-thread.entity'; import { AgentChatService } from 'src/engine/metadata-modules/agent/agent-chat.service'; @@ -12,7 +13,6 @@ import { AgentException, AgentExceptionCode, } from 'src/engine/metadata-modules/agent/agent.exception'; -import { type Workspace } from 'src/engine/core-modules/workspace/workspace.entity'; import { type RecordIdsByObjectMetadataNameSingularType } from 'src/engine/metadata-modules/agent/types/recordIdsByObjectMetadataNameSingular.type'; export type StreamAgentChatOptions = { @@ -45,6 +45,8 @@ export class AgentStreamingService { recordIdsByObjectMetadataNameSingular, res, }: StreamAgentChatOptions) { + let rawStreamString = ''; + try { const thread = await this.threadRepository.findOne({ where: { @@ -74,66 +76,23 @@ export class AgentStreamingService { recordIdsByObjectMetadataNameSingular, }); - let aiResponse = ''; - for await (const chunk of fullStream) { - switch (chunk.type) { - case 'text-delta': - aiResponse += chunk.textDelta; - this.sendStreamEvent(res, { - type: chunk.type, - message: chunk.textDelta, - }); - break; - case 'tool-call': - this.sendStreamEvent(res, { - type: chunk.type, - message: chunk.args?.toolDescription, - }); - break; - case 'error': - { - const errorMessage = - chunk.error && - typeof chunk.error === 'object' && - 'message' in chunk.error - ? chunk.error.message - : 'Something went wrong. Please try again.'; + rawStreamString += JSON.stringify(chunk) + '\n'; - this.sendStreamEvent(res, { - type: 'error', - message: errorMessage as string, - }); - res.end(); - } - this.logger.error(`Stream error: ${JSON.stringify(chunk)}`); - break; - default: - this.logger.log(`Unknown chunk type: ${chunk.type}`); - break; - } + this.sendStreamEvent( + res, + [ + 'text-delta', + 'reasoning', + 'reasoning-signature', + 'tool-call', + 'tool-result', + 'error', + ].includes(chunk.type) + ? chunk + : { type: chunk.type }, + ); } - - if (!aiResponse) { - res.end(); - - return; - } - - await this.agentChatService.addMessage({ - threadId, - role: AgentChatMessageRole.USER, - content: userMessage, - fileIds, - }); - - await this.agentChatService.addMessage({ - threadId, - role: AgentChatMessageRole.ASSISTANT, - content: aiResponse, - }); - - res.end(); } catch (error) { const errorMessage = error instanceof Error ? error.message : 'Unknown error occurred'; @@ -146,19 +105,33 @@ export class AgentStreamingService { this.setupStreamingHeaders(res); } - this.sendStreamEvent(res, { + const errorChunk = { type: 'error', message: errorMessage, - }); + }; - res.end(); + rawStreamString += JSON.stringify(errorChunk) + '\n'; + + this.sendStreamEvent(res, errorChunk); } + + await this.agentChatService.addMessage({ + threadId, + role: AgentChatMessageRole.USER, + rawContent: userMessage, + fileIds, + }); + + await this.agentChatService.addMessage({ + threadId, + role: AgentChatMessageRole.ASSISTANT, + rawContent: rawStreamString.trim() || null, + }); + + res.end(); } - private sendStreamEvent( - res: Response, - event: { type: string; message: string }, - ): void { + private sendStreamEvent(res: Response, event: object): void { res.write(JSON.stringify(event) + '\n'); } diff --git a/packages/twenty-server/src/engine/metadata-modules/agent/constants/agent-config.const.ts b/packages/twenty-server/src/engine/metadata-modules/agent/constants/agent-config.const.ts index 2bc7eaeca1..649e408137 100644 --- a/packages/twenty-server/src/engine/metadata-modules/agent/constants/agent-config.const.ts +++ b/packages/twenty-server/src/engine/metadata-modules/agent/constants/agent-config.const.ts @@ -1,3 +1,4 @@ export const AGENT_CONFIG = { MAX_STEPS: 10, + REASONING_BUDGET_TOKENS: 12000, }; diff --git a/packages/twenty-server/src/engine/metadata-modules/agent/constants/agent-handoff-schema.const.ts b/packages/twenty-server/src/engine/metadata-modules/agent/constants/agent-handoff-schema.const.ts index b24db0470d..abc46db9ff 100644 --- a/packages/twenty-server/src/engine/metadata-modules/agent/constants/agent-handoff-schema.const.ts +++ b/packages/twenty-server/src/engine/metadata-modules/agent/constants/agent-handoff-schema.const.ts @@ -1,7 +1,7 @@ import { z } from 'zod'; export const AGENT_HANDOFF_SCHEMA = z.object({ - toolDescription: z + loadingMessage: z .string() .describe( 'A brief, user-friendly message explaining what is happening while the handoff is being processed. This will be shown to the user during the handoff execution.', diff --git a/packages/twenty-server/src/engine/metadata-modules/agent/dtos/agent-chat-message.dto.ts b/packages/twenty-server/src/engine/metadata-modules/agent/dtos/agent-chat-message.dto.ts index 99f08b18ac..0d1460091d 100644 --- a/packages/twenty-server/src/engine/metadata-modules/agent/dtos/agent-chat-message.dto.ts +++ b/packages/twenty-server/src/engine/metadata-modules/agent/dtos/agent-chat-message.dto.ts @@ -14,8 +14,8 @@ export class AgentChatMessageDTO { @Field() role: 'user' | 'assistant'; - @Field() - content: string; + @Field({ nullable: true }) + rawContent: string; @Field(() => [FileDTO]) files: FileDTO[]; diff --git a/packages/twenty-server/src/engine/metadata-modules/agent/utils/agent-tool-schema.utils.ts b/packages/twenty-server/src/engine/metadata-modules/agent/utils/agent-tool-schema.utils.ts index 85812ac46d..e6f750d7e1 100644 --- a/packages/twenty-server/src/engine/metadata-modules/agent/utils/agent-tool-schema.utils.ts +++ b/packages/twenty-server/src/engine/metadata-modules/agent/utils/agent-tool-schema.utils.ts @@ -17,7 +17,7 @@ const createToolSchema = ( return jsonSchema({ type: 'object', properties: { - toolDescription: { + loadingMessage: { type: 'string', description: 'A clear, human-readable description of the action being performed. Explain what operation you are executing and with what parameters in natural language.', diff --git a/packages/twenty-server/src/engine/metadata-modules/agent/utils/constructAssistantMessageContentFromStream.ts b/packages/twenty-server/src/engine/metadata-modules/agent/utils/constructAssistantMessageContentFromStream.ts new file mode 100644 index 0000000000..3127e455f1 --- /dev/null +++ b/packages/twenty-server/src/engine/metadata-modules/agent/utils/constructAssistantMessageContentFromStream.ts @@ -0,0 +1,60 @@ +import { type TextPart } from 'ai'; + +type ReasoningPart = { + type: 'reasoning'; + text: string; + signature: string; +}; + +export const constructAssistantMessageContentFromStream = ( + rawContent: string, +) => { + const lines = rawContent.trim().split('\n'); + + const output: Array = []; + let reasoningText = ''; + let textContent = ''; + + for (const line of lines) { + let event; + + try { + event = JSON.parse(line); + } catch { + continue; + } + + switch (event.type) { + case 'reasoning': + reasoningText += event.textDelta || ''; + break; + + case 'reasoning-signature': + if (reasoningText) { + output.push({ + type: 'reasoning', + text: reasoningText, + signature: event.signature, + }); + reasoningText = ''; + } + break; + + case 'text-delta': + textContent += event.textDelta || ''; + break; + + default: + if (textContent) { + output.push({ + type: 'text', + text: textContent, + }); + textContent = ''; + } + break; + } + } + + return output; +}; diff --git a/packages/twenty-server/src/modules/workflow/workflow-tools/services/workflow-tool.workspace-service.ts b/packages/twenty-server/src/modules/workflow/workflow-tools/services/workflow-tool.workspace-service.ts index e8cdad599c..3cfa89279e 100644 --- a/packages/twenty-server/src/modules/workflow/workflow-tools/services/workflow-tool.workspace-service.ts +++ b/packages/twenty-server/src/modules/workflow/workflow-tools/services/workflow-tool.workspace-service.ts @@ -137,18 +137,21 @@ This is the most efficient way for AI to create workflows as it handles all the } return { - workflowId, - workflowVersionId, - name: parameters.name, - trigger: parameters.trigger, - steps: parameters.steps, + success: true, message: `Workflow "${parameters.name}" created successfully with ${parameters.steps.length} steps`, + result: { + workflowId, + workflowVersionId, + name: parameters.name, + trigger: parameters.trigger, + steps: parameters.steps, + }, }; } catch (error) { return { success: false, - error: error.message, message: `Failed to create workflow "${parameters.name}": ${error.message}`, + error: error.message, }; } }, diff --git a/packages/twenty-server/src/utils/stream-to-buffer.ts b/packages/twenty-server/src/utils/stream-to-buffer.ts index 77b4781a33..a6edd160b4 100644 --- a/packages/twenty-server/src/utils/stream-to-buffer.ts +++ b/packages/twenty-server/src/utils/stream-to-buffer.ts @@ -1,12 +1,19 @@ import { type Readable } from 'stream'; export const streamToBuffer = async (stream: Readable): Promise => { - // eslint-disable-next-line @typescript-eslint/no-explicit-any - const chunks: any[] = []; + const chunks: Buffer[] = []; - for await (const chunk of stream) { - chunks.push(chunk); - } + return new Promise((resolve, reject) => { + stream.on('data', (chunk: Buffer) => { + chunks.push(chunk); + }); - return Buffer.concat(chunks); + stream.on('end', () => { + resolve(Buffer.concat(chunks)); + }); + + stream.on('error', (error) => { + reject(error); + }); + }); }; diff --git a/packages/twenty-server/test/integration/metadata/suites/agent/agent-tool.service.integration-spec.ts b/packages/twenty-server/test/integration/metadata/suites/agent/agent-tool.service.integration-spec.ts index c33f683be2..885519d150 100644 --- a/packages/twenty-server/test/integration/metadata/suites/agent/agent-tool.service.integration-spec.ts +++ b/packages/twenty-server/test/integration/metadata/suites/agent/agent-tool.service.integration-spec.ts @@ -229,7 +229,7 @@ describe('AgentToolGeneratorService Integration', () => { ); expectSuccessResult(result, 'Successfully created testObject'); - expect(result.record).toEqual(testRecord); + expect(result.result).toEqual(testRecord); expect(mockRepository.save).toHaveBeenCalledWith({ name: 'Test Record', description: 'Test description', @@ -317,8 +317,8 @@ describe('AgentToolGeneratorService Integration', () => { ); expectSuccessResult(result, 'Found 3 testObject records'); - expect(result.records).toEqual(testRecords); - expect(result.count).toBe(3); + expect(result.result.records).toEqual(testRecords); + expect(result.result.count).toBe(3); expect(mockRepository.find).toHaveBeenCalledWith({ where: {}, take: 10, @@ -365,7 +365,7 @@ describe('AgentToolGeneratorService Integration', () => { ); expectSuccessResult(result, 'Found testObject record'); - expect(result.record).toEqual(testRecord); + expect(result.result).toEqual(testRecord); expect(mockRepository.findOne).toHaveBeenCalledWith({ where: { id: 'test-record-id' }, }); @@ -506,7 +506,7 @@ describe('AgentToolGeneratorService Integration', () => { ); expectSuccessResult(result, 'Successfully updated testObject'); - expect(result.record).toEqual(updatedRecord); + expect(result.result).toEqual(updatedRecord); expect(mockRepository.update).toHaveBeenCalledWith('test-record-id', { name: 'New Name', description: 'New description', diff --git a/packages/twenty-ui/src/display/icon/components/TablerIcons.ts b/packages/twenty-ui/src/display/icon/components/TablerIcons.ts index 20912e562a..41ef1401e3 100644 --- a/packages/twenty-ui/src/display/icon/components/TablerIcons.ts +++ b/packages/twenty-ui/src/display/icon/components/TablerIcons.ts @@ -29,6 +29,7 @@ export { IconBrackets, IconBracketsAngle, IconBracketsContain, + IconBrain, IconBrandDaysCounter, IconBrandGithub, IconBrandGoogle, diff --git a/packages/twenty-ui/src/display/index.ts b/packages/twenty-ui/src/display/index.ts index cc7a44d735..16273670b8 100644 --- a/packages/twenty-ui/src/display/index.ts +++ b/packages/twenty-ui/src/display/index.ts @@ -91,6 +91,7 @@ export { IconBrackets, IconBracketsAngle, IconBracketsContain, + IconBrain, IconBrandDaysCounter, IconBrandGithub, IconBrandGoogle,