diff --git a/packages/twenty-front/src/modules/ai/components/RoutingDebugDisplay.tsx b/packages/twenty-front/src/modules/ai/components/RoutingDebugDisplay.tsx new file mode 100644 index 0000000000..7b4135d426 --- /dev/null +++ b/packages/twenty-front/src/modules/ai/components/RoutingDebugDisplay.tsx @@ -0,0 +1,362 @@ +import { useTheme } from '@emotion/react'; +import styled from '@emotion/styled'; +import { useState } from 'react'; + +import { IconChevronDown, IconChevronUp } from 'twenty-ui/display'; +import { JsonTree } from 'twenty-ui/json-visualizer'; +import { AnimatedExpandableContainer } from 'twenty-ui/layout'; + +import { useLingui } from '@lingui/react/macro'; +import { type DataMessagePart } from 'twenty-shared/ai'; +import { type JsonValue } from 'type-fest'; +import { useCopyToClipboard } from '~/hooks/useCopyToClipboard'; + +const StyledContainer = styled.div` + display: flex; + flex-direction: column; + gap: ${({ theme }) => theme.spacing(2)}; + margin-top: ${({ theme }) => theme.spacing(2)}; +`; + +const StyledToggleButton = styled.div` + align-items: center; + background: none; + border: none; + cursor: pointer; + display: flex; + color: ${({ theme }) => theme.font.color.tertiary}; + gap: ${({ theme }) => theme.spacing(1)}; + padding: ${({ theme }) => theme.spacing(1)} 0; + transition: color ${({ theme }) => theme.animation.duration.normal}s; + font-size: ${({ theme }) => theme.font.size.sm}; + + &:hover { + color: ${({ theme }) => theme.font.color.secondary}; + } +`; + +const StyledContentContainer = styled.div` + background: ${({ theme }) => theme.background.transparent.lighter}; + border: 1px solid ${({ theme }) => theme.border.color.light}; + border-radius: ${({ theme }) => theme.border.radius.sm}; + min-width: 0; + padding: ${({ theme }) => theme.spacing(3)}; +`; + +const StyledJsonTreeContainer = styled.div` + overflow-x: auto; + + ul { + min-width: 0; + } +`; + +const StyledTabContainer = styled.div` + border-bottom: 1px solid ${({ theme }) => theme.border.color.light}; + display: flex; + gap: ${({ theme }) => theme.spacing(3)}; + margin-bottom: ${({ theme }) => theme.spacing(3)}; +`; + +const StyledTab = styled.div<{ isActive: boolean }>` + color: ${({ theme, isActive }) => + isActive ? theme.font.color.primary : theme.font.color.tertiary}; + font-size: ${({ theme }) => theme.font.size.sm}; + font-weight: ${({ theme, isActive }) => + isActive ? theme.font.weight.medium : theme.font.weight.regular}; + cursor: pointer; + transition: color ${({ theme }) => theme.animation.duration.normal}s; + padding-bottom: ${({ theme }) => theme.spacing(2)}; + + &:hover { + color: ${({ theme }) => theme.font.color.secondary}; + } +`; + +const StyledTimingSection = styled.div` + display: flex; + flex-direction: column; + gap: ${({ theme }) => theme.spacing(2)}; +`; + +const StyledTimingRow = styled.div` + align-items: center; + display: flex; + font-size: ${({ theme }) => theme.font.size.sm}; + justify-content: space-between; + padding: ${({ theme }) => theme.spacing(1)} 0; +`; + +const StyledTimingLabel = styled.span` + color: ${({ theme }) => theme.font.color.secondary}; +`; + +const StyledTimingValue = styled.span` + color: ${({ theme }) => theme.font.color.primary}; + font-weight: ${({ theme }) => theme.font.weight.medium}; +`; + +type TabType = 'timing' | 'details' | 'context'; + +const TimingRow = ({ + label, + value, +}: { + label: string; + value: string | number | undefined; +}) => { + if (value === undefined) return null; + + return ( + + {label} + {value} + + ); +}; + +const formatBytes = (bytes: number) => { + if (bytes === 0) return '0 B'; + const k = 1024; + const sizes = ['B', 'KB', 'MB']; + const i = Math.floor(Math.log(bytes) / Math.log(k)); + return `${Math.round((bytes / Math.pow(k, i)) * 100) / 100} ${sizes[i]}`; +}; + +const formatNumber = (num: number) => num.toLocaleString(); + +const formatTokenBreakdown = ( + total: number, + prompt?: number, + completion?: number, +) => { + const formattedTotal = formatNumber(total); + if ( + prompt !== undefined && + completion !== undefined && + prompt > 0 && + completion > 0 + ) { + return `${formattedTotal} (${formatNumber(prompt)} → ${formatNumber(completion)})`; + } + return formattedTotal; +}; + +const TimingTab = ({ + debug, +}: { + debug: NonNullable; +}) => { + const totalTime = + debug.agentExecutionStartTimeMs !== undefined + ? `${debug.agentExecutionStartTimeMs + (debug.agentExecutionTimeMs || 0)}ms` + : undefined; + + return ( + + + + + + + + + + + + + + + + + ); +}; + +const DetailsTab = ({ + debug, + copyToClipboard, +}: { + debug: NonNullable; + copyToClipboard: (value: string) => void; +}) => { + const { t } = useLingui(); + + const detailsData = { + selectedAgent: { + id: debug.selectedAgentId, + label: debug.selectedAgentLabel, + }, + routerModel: debug.routerModel, + agentModel: debug.agentModel, + availableAgents: debug.availableAgents, + }; + + return ( + + true} + emptyArrayLabel={t`Empty Array`} + emptyObjectLabel={t`Empty Object`} + emptyStringLabel={t`[empty string]`} + arrowButtonCollapsedLabel={t`Expand`} + arrowButtonExpandedLabel={t`Collapse`} + onNodeValueClick={copyToClipboard} + /> + + ); +}; + +const ContextTab = ({ + debug, + copyToClipboard, +}: { + debug: NonNullable; + copyToClipboard: (value: string) => void; +}) => { + const { t } = useLingui(); + + if (!debug.context) { + return ( + + No context was provided for this request + + ); + } + + try { + const contextData = JSON.parse(debug.context); + + return ( + + false} + emptyArrayLabel={t`Empty Array`} + emptyObjectLabel={t`Empty Object`} + emptyStringLabel={t`[empty string]`} + arrowButtonCollapsedLabel={t`Expand`} + arrowButtonExpandedLabel={t`Collapse`} + onNodeValueClick={copyToClipboard} + /> + + ); + } catch { + return {debug.context}; + } +}; + +export const RoutingDebugDisplay = ({ + debug, +}: { + debug: NonNullable; +}) => { + const theme = useTheme(); + const { copyToClipboard } = useCopyToClipboard(); + const [isExpanded, setIsExpanded] = useState(false); + const [activeTab, setActiveTab] = useState('timing'); + + return ( + + setIsExpanded(!isExpanded)}> + Debug Info + {isExpanded ? ( + + ) : ( + + )} + + + + + + setActiveTab('timing')} + > + Timing + + setActiveTab('details')} + > + Details + + {debug.context && ( + setActiveTab('context')} + > + Context + + )} + + + {activeTab === 'timing' && } + {activeTab === 'details' && ( + + )} + {activeTab === 'context' && ( + + )} + + + + ); +}; diff --git a/packages/twenty-front/src/modules/ai/components/RoutingStatusDisplay.tsx b/packages/twenty-front/src/modules/ai/components/RoutingStatusDisplay.tsx index 4991c73eed..6e4a0110df 100644 --- a/packages/twenty-front/src/modules/ai/components/RoutingStatusDisplay.tsx +++ b/packages/twenty-front/src/modules/ai/components/RoutingStatusDisplay.tsx @@ -1,3 +1,4 @@ +import { RoutingDebugDisplay } from '@/ai/components/RoutingDebugDisplay'; import { ShimmeringText } from '@/ai/components/ShimmeringText'; import styled from '@emotion/styled'; import { type DataMessagePart } from 'twenty-shared/ai'; @@ -38,27 +39,38 @@ const StyledText = styled.div` color: ${({ theme }) => theme.font.color.tertiary}; `; +const StyledWrapper = styled.div` + display: flex; + flex-direction: column; +`; + export const RoutingStatusDisplay = ({ data, }: { data: DataMessagePart['routing-status']; }) => { const isLoading = data.state === 'loading'; + const isDebugMode = process.env.IS_DEBUG_MODE === 'true'; if (data.state === 'error') { return null; } return ( - - - {isLoading ? : } - - {isLoading ? ( - {data.text} - ) : ( - {data.text} + + + + {isLoading ? : } + + {isLoading ? ( + {data.text} + ) : ( + {data.text} + )} + + {isDebugMode && data.state === 'routed' && data.debug && ( + )} - + ); }; diff --git a/packages/twenty-front/src/modules/client-config/types/ClientConfig.ts b/packages/twenty-front/src/modules/client-config/types/ClientConfig.ts index 53b602a0f2..04c88300e1 100644 --- a/packages/twenty-front/src/modules/client-config/types/ClientConfig.ts +++ b/packages/twenty-front/src/modules/client-config/types/ClientConfig.ts @@ -20,7 +20,6 @@ export type ClientConfig = { canManageFeatureFlags: boolean; captcha: Captcha; chromeExtensionId?: string; - debugMode: boolean; defaultSubdomain?: string; frontDomain: string; isAttachmentPreviewEnabled: boolean; diff --git a/packages/twenty-front/src/modules/client-config/utils/__tests__/clientConfigUtils.test.ts b/packages/twenty-front/src/modules/client-config/utils/__tests__/clientConfigUtils.test.ts index edbd3ae717..57c08d3643 100644 --- a/packages/twenty-front/src/modules/client-config/utils/__tests__/clientConfigUtils.test.ts +++ b/packages/twenty-front/src/modules/client-config/utils/__tests__/clientConfigUtils.test.ts @@ -21,7 +21,6 @@ const mockClientConfig = { isEmailVerificationRequired: false, defaultSubdomain: 'app', frontDomain: 'localhost', - debugMode: true, support: { supportDriver: 'none', supportFrontChatId: undefined, diff --git a/packages/twenty-front/src/testing/mock-data/config.ts b/packages/twenty-front/src/testing/mock-data/config.ts index e92854299a..6fba34906d 100644 --- a/packages/twenty-front/src/testing/mock-data/config.ts +++ b/packages/twenty-front/src/testing/mock-data/config.ts @@ -16,7 +16,6 @@ export const mockedClientConfig: ClientConfig = { frontDomain: 'localhost', defaultSubdomain: 'app', chromeExtensionId: 'MOCKED_EXTENSION_ID', - debugMode: false, analyticsEnabled: true, support: { supportDriver: SupportDriver.FRONT, diff --git a/packages/twenty-server/src/engine/core-modules/ai/services/ai-billing.service.ts b/packages/twenty-server/src/engine/core-modules/ai/services/ai-billing.service.ts index b377748ba9..d7205048b1 100644 --- a/packages/twenty-server/src/engine/core-modules/ai/services/ai-billing.service.ts +++ b/packages/twenty-server/src/engine/core-modules/ai/services/ai-billing.service.ts @@ -3,8 +3,8 @@ import { Injectable, Logger } from '@nestjs/common'; import { LanguageModelUsage } from 'ai'; import { type ModelId } from 'src/engine/core-modules/ai/constants/ai-models.const'; -import { DOLLAR_TO_CREDIT_MULTIPLIER } from 'src/engine/core-modules/ai/constants/dollar-to-credit-multiplier'; import { AiModelRegistryService } from 'src/engine/core-modules/ai/services/ai-model-registry.service'; +import { convertCentsToBillingCredits } from 'src/engine/core-modules/ai/utils/convert-cents-to-billing-credits.util'; import { BILLING_FEATURE_USED } from 'src/engine/core-modules/billing/constants/billing-feature-used.constant'; import { BillingMeterEventName } from 'src/engine/core-modules/billing/enums/billing-meter-event-names'; import { type BillingUsageEvent } from 'src/engine/core-modules/billing/types/billing-usage-event.type'; @@ -49,9 +49,7 @@ export class AIBillingService { workspaceId: string, ): Promise { const costInCents = await this.calculateCost(modelId, usage); - - const costInDollars = costInCents / 100; - const creditsUsed = Math.round(costInDollars * DOLLAR_TO_CREDIT_MULTIPLIER); + const creditsUsed = Math.round(convertCentsToBillingCredits(costInCents)); this.sendAiTokenUsageEvent(workspaceId, creditsUsed); } diff --git a/packages/twenty-server/src/engine/core-modules/ai/utils/convert-cents-to-billing-credits.util.ts b/packages/twenty-server/src/engine/core-modules/ai/utils/convert-cents-to-billing-credits.util.ts index 4fb6e664dc..ea72f74393 100644 --- a/packages/twenty-server/src/engine/core-modules/ai/utils/convert-cents-to-billing-credits.util.ts +++ b/packages/twenty-server/src/engine/core-modules/ai/utils/convert-cents-to-billing-credits.util.ts @@ -1,8 +1,12 @@ +import { DOLLAR_TO_CREDIT_MULTIPLIER } from 'src/engine/core-modules/ai/constants/dollar-to-credit-multiplier'; + /** * Converts cost in cents to cost in credits - * Formula: credits = cents / 100 * 1000 = cents * 10 + * Formula: credits = (cents / 100) * DOLLAR_TO_CREDIT_MULTIPLIER + * Where DOLLAR_TO_CREDIT_MULTIPLIER = 1000000 ($0.00001 = 1 credit) + * Simplified: cents * 10000 * @param cents - Cost in cents (real cost) * @returns Cost in credits (end-user cost) */ export const convertCentsToBillingCredits = (cents: number): number => - cents * 10; + (cents / 100) * DOLLAR_TO_CREDIT_MULTIPLIER; diff --git a/packages/twenty-server/src/engine/core-modules/client-config/client-config.controller.spec.ts b/packages/twenty-server/src/engine/core-modules/client-config/client-config.controller.spec.ts index 7dd4328024..1700f3f79a 100644 --- a/packages/twenty-server/src/engine/core-modules/client-config/client-config.controller.spec.ts +++ b/packages/twenty-server/src/engine/core-modules/client-config/client-config.controller.spec.ts @@ -69,7 +69,6 @@ describe('ClientConfigController', () => { isEmailVerificationRequired: false, defaultSubdomain: 'app', frontDomain: 'localhost', - debugMode: true, support: { supportDriver: SupportDriver.NONE, supportFrontChatId: undefined, diff --git a/packages/twenty-server/src/engine/core-modules/client-config/client-config.entity.ts b/packages/twenty-server/src/engine/core-modules/client-config/client-config.entity.ts index 67f8573a51..4ddd9722ab 100644 --- a/packages/twenty-server/src/engine/core-modules/client-config/client-config.entity.ts +++ b/packages/twenty-server/src/engine/core-modules/client-config/client-config.entity.ts @@ -147,9 +147,6 @@ export class ClientConfig { @Field(() => String) frontDomain: string; - @Field(() => Boolean) - debugMode: boolean; - @Field(() => Boolean) analyticsEnabled: boolean; diff --git a/packages/twenty-server/src/engine/core-modules/client-config/services/client-config.service.spec.ts b/packages/twenty-server/src/engine/core-modules/client-config/services/client-config.service.spec.ts index 4bd4a291c3..a9fa05e009 100644 --- a/packages/twenty-server/src/engine/core-modules/client-config/services/client-config.service.spec.ts +++ b/packages/twenty-server/src/engine/core-modules/client-config/services/client-config.service.spec.ts @@ -129,7 +129,6 @@ describe('ClientConfigService', () => { isEmailVerificationRequired: true, defaultSubdomain: 'app', frontDomain: 'app.twenty.com', - debugMode: true, support: { supportDriver: 'FRONT', supportFrontChatId: 'chat-123', @@ -172,7 +171,6 @@ describe('ClientConfigService', () => { const result = await service.getClientConfig(); - expect(result.debugMode).toBe(false); expect(result.canManageFeatureFlags).toBe(false); expect(result.aiModels).toEqual([]); }); diff --git a/packages/twenty-server/src/engine/core-modules/client-config/services/client-config.service.ts b/packages/twenty-server/src/engine/core-modules/client-config/services/client-config.service.ts index 4065e3f858..8bd863a210 100644 --- a/packages/twenty-server/src/engine/core-modules/client-config/services/client-config.service.ts +++ b/packages/twenty-server/src/engine/core-modules/client-config/services/client-config.service.ts @@ -108,9 +108,6 @@ export class ClientConfigService { ), defaultSubdomain: this.twentyConfigService.get('DEFAULT_SUBDOMAIN'), frontDomain: this.domainServerConfigService.getFrontUrl().hostname, - debugMode: - this.twentyConfigService.get('NODE_ENV') === - NodeEnvironment.DEVELOPMENT, support: { supportDriver: supportDriver ? supportDriver : SupportDriver.NONE, supportFrontChatId: this.twentyConfigService.get( diff --git a/packages/twenty-server/src/engine/metadata-modules/agent/agent-execution.service.ts b/packages/twenty-server/src/engine/metadata-modules/agent/agent-execution.service.ts index 39ead58caa..ca6fd0249d 100644 --- a/packages/twenty-server/src/engine/metadata-modules/agent/agent-execution.service.ts +++ b/packages/twenty-server/src/engine/metadata-modules/agent/agent-execution.service.ts @@ -278,28 +278,56 @@ export class AgentExecutionService implements AgentExecutionContext { agentId: string; messages: UIMessage[]; recordIdsByObjectMetadataNameSingular: RecordIdsByObjectMetadataNameSingularType; - }) { + }): Promise<{ + stream: ReturnType; + timings: { + contextBuildTimeMs: number; + toolGenerationTimeMs: number; + aiRequestPrepTimeMs: number; + toolCount: number; + }; + contextInfo: { + contextString: string; + contextRecordCount: number; + contextSizeBytes: number; + }; + }> { try { const agent = await this.agentService.findOneAgent(agentId, workspace.id); - let contextString = ''; + const contextBuildStart = Date.now(); + let contextPart = ''; + let contextRecordCount = 0; if (recordIdsByObjectMetadataNameSingular.length > 0) { - const contextPart = await this.getContextForSystemPrompt( + contextPart = await this.getContextForSystemPrompt( workspace, recordIdsByObjectMetadataNameSingular, userWorkspaceId, ); - contextString = `\n\nCONTEXT:\n${contextPart}`; + try { + const contextData = JSON.parse(contextPart); + + contextRecordCount = Array.isArray(contextData) + ? contextData.length + : 0; + } catch (error) { + this.logger.warn('Failed to parse context for record count:', error); + } } + const contextString = contextPart ? `\n\nCONTEXT:\n${contextPart}` : ''; + const contextBuildTime = Date.now() - contextBuildStart; + const { actorContext, roleId } = await this.agentActorContextService.buildUserAndAgentActorContext( userWorkspaceId, workspace.id, ); + const aiRequestPrepStart = Date.now(); + const aiRequestConfig = await this.prepareAIRequestConfig({ system: `${AGENT_SYSTEM_PROMPTS.AGENT_CHAT}\n\n${agent.prompt}${contextString}`, agent, @@ -309,8 +337,12 @@ export class AgentExecutionService implements AgentExecutionContext { userWorkspaceId, }); + const aiRequestPrepTime = Date.now() - aiRequestPrepStart; + const toolCount = Object.keys(aiRequestConfig.tools || {}).length; + const toolGenerationTime = aiRequestPrepTime; + this.logger.log( - `Sending request to AI model with ${messages.length} messages`, + `Sending request to AI model with ${messages.length} messages and ${toolCount} tools`, ); const model = @@ -330,7 +362,22 @@ export class AgentExecutionService implements AgentExecutionContext { this.logger.error('Failed to get usage information:', usageError); }); - return stream; + return { + stream, + timings: { + contextBuildTimeMs: contextBuildTime, + toolGenerationTimeMs: toolGenerationTime, + aiRequestPrepTimeMs: aiRequestPrepTime, + toolCount, + }, + contextInfo: { + contextString: contextPart, + contextRecordCount, + contextSizeBytes: contextPart + ? Buffer.byteLength(contextPart, 'utf8') + : 0, + }, + }; } catch (error) { this.logger.error('Error in streamChatResponse:', error); throw new AgentException( diff --git a/packages/twenty-server/src/engine/metadata-modules/agent/agent-streaming.service.ts b/packages/twenty-server/src/engine/metadata-modules/agent/agent-streaming.service.ts index de6e822a27..ea96190844 100644 --- a/packages/twenty-server/src/engine/metadata-modules/agent/agent-streaming.service.ts +++ b/packages/twenty-server/src/engine/metadata-modules/agent/agent-streaming.service.ts @@ -4,15 +4,18 @@ import { InjectRepository } from '@nestjs/typeorm'; import { createUIMessageStream, pipeUIMessageStreamToResponse, - UIDataTypes, - UIMessage, - UITools, + type UIDataTypes, + type UIMessage, + type UITools, } from 'ai'; import { type Response } from 'express'; -import { ExtendedUIMessage } from 'twenty-shared/ai'; -import { Repository } from 'typeorm'; +import { type ExtendedUIMessage } from 'twenty-shared/ai'; +import { type Repository } from 'typeorm'; -import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity'; +import { type ModelId } from 'src/engine/core-modules/ai/constants/ai-models.const'; +import { AIBillingService } from 'src/engine/core-modules/ai/services/ai-billing.service'; +import { convertCentsToBillingCredits } from 'src/engine/core-modules/ai/utils/convert-cents-to-billing-credits.util'; +import { type WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity'; import { AgentChatMessageRole } from 'src/engine/metadata-modules/agent/agent-chat-message.entity'; import { AgentChatThreadEntity } from 'src/engine/metadata-modules/agent/agent-chat-thread.entity'; import { AgentChatService } from 'src/engine/metadata-modules/agent/agent-chat.service'; @@ -43,6 +46,7 @@ export class AgentStreamingService { private readonly agentChatService: AgentChatService, private readonly agentExecutionService: AgentExecutionService, private readonly aiRouterService: AiRouterService, + private readonly aiBillingService: AIBillingService, ) {} async streamAgentChat({ @@ -71,6 +75,8 @@ export class AgentStreamingService { const stream = createUIMessageStream({ execute: async ({ writer }) => { + const startTime = Date.now(); + writer.write({ type: 'data-routing-status' as const, id: 'routing-status', @@ -80,11 +86,18 @@ export class AgentStreamingService { }, }); - const agent = await this.aiRouterService.routeMessage({ - messages, - workspaceId: workspace.id, - routerModel: workspace.routerModel, - }); + const routingStart = Date.now(); + const routeResult = await this.aiRouterService.routeMessage( + { + messages, + workspaceId: workspace.id, + routerModel: workspace.routerModel, + }, + true, + ); + + const routingTime = Date.now() - routingStart; + const { agent, debugInfo } = routeResult; if (!agent) { writer.write({ @@ -103,18 +116,39 @@ export class AgentStreamingService { this.logger.log(`Using agent ${agent.id} for message routing`); - const routedStatusPart = { - type: 'data-routing-status' as const, - id: 'routing-status', - data: { - text: `Routed to ${agent.label} agent`, - state: 'routed', - }, - }; + let routingCostInCredits: number | undefined; - writer.write(routedStatusPart); + if ( + debugInfo?.routerModel && + debugInfo?.promptTokens !== undefined && + debugInfo?.completionTokens !== undefined + ) { + try { + const routingCostInCents = + await this.aiBillingService.calculateCost( + debugInfo.routerModel as ModelId, + { + inputTokens: debugInfo.promptTokens, + outputTokens: debugInfo.completionTokens, + totalTokens: debugInfo.totalTokens || 0, + }, + ); - const result = await this.agentExecutionService.streamChatResponse({ + routingCostInCredits = Math.round( + convertCentsToBillingCredits(routingCostInCents), + ); + } catch (error) { + this.logger.warn('Failed to calculate routing cost:', error); + } + } + + const agentExecutionStart = Date.now(); + + const { + stream: result, + timings, + contextInfo, + } = await this.agentExecutionService.streamChatResponse({ workspace, agentId: agent.id, userWorkspaceId, @@ -122,6 +156,33 @@ export class AgentStreamingService { recordIdsByObjectMetadataNameSingular, }); + const routedStatusPart = { + type: 'data-routing-status' as const, + id: 'routing-status', + data: { + text: `Routed to ${agent.label} agent`, + state: 'routed', + debug: { + routingTimeMs: routingTime, + agentExecutionStartTimeMs: Date.now() - startTime, + selectedAgentId: agent.id, + selectedAgentLabel: agent.label, + availableAgents: debugInfo?.availableAgents, + routerModel: debugInfo?.routerModel, + agentModel: agent.modelId, + context: contextInfo.contextString || undefined, + contextRecordCount: contextInfo.contextRecordCount, + contextSizeBytes: contextInfo.contextSizeBytes, + routingPromptTokens: debugInfo?.promptTokens, + routingCompletionTokens: debugInfo?.completionTokens, + routingTotalTokens: debugInfo?.totalTokens, + routingCostInCredits, + }, + }, + }; + + writer.write(routedStatusPart); + writer.merge( result.toUIMessageStream({ onError: (error) => { @@ -133,6 +194,104 @@ export class AgentStreamingService { return; } + const toolCallCount = responseMessage.parts.filter((part) => + part.type.startsWith('tool-'), + ).length; + + let tokenUsage: { + promptTokens: number; + completionTokens: number; + totalTokens: number; + } | null = null; + + try { + const usage = await result.usage; + + const usageWithTokens = usage as { + inputTokens?: number; + outputTokens?: number; + promptTokens?: number; + completionTokens?: number; + totalTokens?: number; + }; + + tokenUsage = { + promptTokens: + usageWithTokens.inputTokens ?? + usageWithTokens.promptTokens ?? + 0, + completionTokens: + usageWithTokens.outputTokens ?? + usageWithTokens.completionTokens ?? + 0, + totalTokens: usageWithTokens.totalTokens ?? 0, + }; + + this.logger.log( + `Agent execution usage: ${tokenUsage.promptTokens} prompt + ${tokenUsage.completionTokens} completion = ${tokenUsage.totalTokens} total tokens`, + ); + } catch (error) { + this.logger.warn('Failed to get token usage:', error); + } + + const agentExecutionTime = Date.now() - agentExecutionStart; + + let agentCostInCredits: number | undefined; + let totalCostInCredits: number | undefined; + + if ( + agent.modelId && + tokenUsage && + tokenUsage.promptTokens > 0 && + tokenUsage.completionTokens > 0 + ) { + try { + const agentCostInCents = + await this.aiBillingService.calculateCost( + agent.modelId as ModelId, + { + inputTokens: tokenUsage.promptTokens, + outputTokens: tokenUsage.completionTokens, + totalTokens: tokenUsage.totalTokens, + }, + ); + + agentCostInCredits = Math.round( + convertCentsToBillingCredits(agentCostInCents), + ); + + totalCostInCredits = + (routingCostInCredits || 0) + agentCostInCredits; + } catch (error) { + this.logger.warn('Failed to calculate agent cost:', error); + } + } + + const updatedRoutedStatusPart = { + ...routedStatusPart, + data: { + ...routedStatusPart.data, + debug: { + ...routedStatusPart.data.debug, + agentExecutionTimeMs: agentExecutionTime, + toolCallCount, + toolCount: timings.toolCount, + agentContextBuildTimeMs: timings.contextBuildTimeMs, + toolGenerationTimeMs: timings.toolGenerationTimeMs, + aiRequestPrepTimeMs: timings.aiRequestPrepTimeMs, + ...(tokenUsage && { + agentPromptTokens: tokenUsage.promptTokens, + agentCompletionTokens: tokenUsage.completionTokens, + agentTotalTokens: tokenUsage.totalTokens, + }), + agentCostInCredits, + totalCostInCredits, + }, + }, + }; + + writer.write(updatedRoutedStatusPart); + await this.agentChatService.addMessage({ threadId, uiMessage: { @@ -153,7 +312,7 @@ export class AgentStreamingService { threadId, uiMessage: { ...responseMessage, - parts: [routedStatusPart, ...responseMessage.parts], + parts: [updatedRoutedStatusPart, ...responseMessage.parts], }, }); }, diff --git a/packages/twenty-server/src/engine/metadata-modules/ai-router/ai-router.service.ts b/packages/twenty-server/src/engine/metadata-modules/ai-router/ai-router.service.ts index a85e03d5af..bdccb6f309 100644 --- a/packages/twenty-server/src/engine/metadata-modules/ai-router/ai-router.service.ts +++ b/packages/twenty-server/src/engine/metadata-modules/ai-router/ai-router.service.ts @@ -7,10 +7,10 @@ import { type UIMessage, type UITools, } from 'ai'; -import { Repository } from 'typeorm'; +import { type Repository } from 'typeorm'; import { z } from 'zod'; -import { ModelId } from 'src/engine/core-modules/ai/constants/ai-models.const'; +import { type ModelId } from 'src/engine/core-modules/ai/constants/ai-models.const'; import { AI_TELEMETRY_CONFIG } from 'src/engine/core-modules/ai/constants/ai-telemetry.const'; import { AiModelRegistryService } from 'src/engine/core-modules/ai/services/ai-model-registry.service'; import { AgentEntity } from 'src/engine/metadata-modules/agent/agent.entity'; @@ -22,6 +22,17 @@ export interface AiRouterContext { routerModel: ModelId; } +export interface AiRouterResult { + agent: AgentEntity | null; + debugInfo?: { + availableAgents: Array<{ id: string; label: string }>; + routerModel: string; + promptTokens?: number; + completionTokens?: number; + totalTokens?: number; + }; +} + @Injectable() export class AiRouterService { private readonly logger = new Logger(AiRouterService.name); @@ -32,7 +43,10 @@ export class AiRouterService { private readonly aiModelRegistryService: AiModelRegistryService, ) {} - async routeMessage(context: AiRouterContext) { + async routeMessage( + context: AiRouterContext, + includeDebugInfo = false, + ): Promise { try { const { messages, workspaceId, routerModel } = context; @@ -41,11 +55,21 @@ export class AiRouterService { if (availableAgents.length === 0) { this.logger.warn('No agents available for routing'); - return null; + return { agent: null }; } + const debugInfo: AiRouterResult['debugInfo'] = includeDebugInfo + ? { + availableAgents: availableAgents.map((agent) => ({ + id: agent.id, + label: agent.label, + })), + routerModel: String(routerModel), + } + : undefined; + if (availableAgents.length === 1) { - return availableAgents[0]; + return { agent: availableAgents[0], debugInfo }; } const conversationHistory = messages @@ -87,16 +111,44 @@ export class AiRouterService { experimental_telemetry: AI_TELEMETRY_CONFIG, }); - return availableAgents.find( + const selectedAgent = availableAgents.find( (agent) => agent.id === result.object.agentId, ); + + if (includeDebugInfo && debugInfo) { + try { + const usage = await result.usage; + + const usageWithTokens = usage as { + inputTokens?: number; + outputTokens?: number; + promptTokens?: number; + completionTokens?: number; + totalTokens?: number; + }; + + debugInfo.promptTokens = + usageWithTokens.inputTokens ?? usageWithTokens.promptTokens ?? 0; + debugInfo.completionTokens = + usageWithTokens.outputTokens ?? + usageWithTokens.completionTokens ?? + 0; + debugInfo.totalTokens = usageWithTokens.totalTokens ?? 0; + } catch (error) { + this.logger.warn('Failed to get routing token usage:', error); + } + } + + return { agent: selectedAgent ?? null, debugInfo }; } catch (error) { this.logger.error( 'Routing to agent failed, falling back to Helper agent:', error, ); - return this.getHelperAgent(context.workspaceId); + const helperAgent = await this.getHelperAgent(context.workspaceId); + + return { agent: helperAgent }; } } diff --git a/packages/twenty-shared/src/ai/types/DataMessagePart.ts b/packages/twenty-shared/src/ai/types/DataMessagePart.ts index 861b196662..41a670335c 100644 --- a/packages/twenty-shared/src/ai/types/DataMessagePart.ts +++ b/packages/twenty-shared/src/ai/types/DataMessagePart.ts @@ -2,5 +2,36 @@ export type DataMessagePart = { 'routing-status': { text: string; state: string; + debug?: { + routingTimeMs?: number; + contextBuildTimeMs?: number; + agentExecutionStartTimeMs?: number; + agentExecutionTimeMs?: number; + toolGenerationTimeMs?: number; + agentContextBuildTimeMs?: number; + aiRequestPrepTimeMs?: number; + selectedAgentId?: string; + selectedAgentLabel?: string; + availableAgents?: Array<{ id: string; label: string }>; + routerModel?: string; + agentModel?: string; + context?: string; + contextRecordCount?: number; + contextSizeBytes?: number; + toolCallCount?: number; + toolCount?: number; + // Routing AI call tokens + routingPromptTokens?: number; + routingCompletionTokens?: number; + routingTotalTokens?: number; + // Agent AI call tokens + agentPromptTokens?: number; + agentCompletionTokens?: number; + agentTotalTokens?: number; + // Cost in Twenty credits + routingCostInCredits?: number; + agentCostInCredits?: number; + totalCostInCredits?: number; + }; }; };