feat: Add Agent Evaluation System and Refactor AI Modules (#16111)

## Summary This PR introduces a comprehensive agent evaluation system and refactors the AI module structure for better organization. ## Key Changes ### 🎯 Agent Evaluation System - Added **Agent Turn Evaluation** entities, DTOs, and database schema - New GraphQL mutations: `evaluateAgentTurn` and `runEvaluationInput` - Added `evaluationInputs` field to Agent entity for storing test inputs - New `AgentTurnGraderService` for automatic turn evaluation - Added evaluation UI with new **Evals** and **Logs** tabs in agent detail pages ### 🏗️ Entity & Module Refactoring - Renamed `AgentChatMessage` → `AgentMessage` for clarity - Consolidated chat entities: `AgentMessage`, `AgentTurn`, and `AgentChatThread` - Reorganized AI modules under `ai/` subdirectory structure - Updated imports across codebase to reflect new module paths ### 🤖 New Agents & Roles - Added **Dashboard Builder Agent** for dashboard creation and management - Added **Dashboard Manager Role** with appropriate permissions - Updated role permissions to be more granular (users vs agents vs API keys) ### 🔐 Permission System Updates - Added `HTTP_REQUEST_TOOL` permission flag - Updated Workflow Manager role permissions (restricted tool access) - Enhanced permission flag types to differentiate between user/agent/API key contexts - Added `isRelevantForAgents`, `isRelevantForApiKeys`, `isRelevantForUsers` to permission flags ### 📨 Message Role Enhancement - Added `system` role to `AgentMessageRole` enum (alongside user/assistant) - Updated message handling to support system prompts ### 🎨 UI/UX Improvements - New tabs in agent detail: **Evals** and **Logs** - Added turn detail page: `/ai/agents/:agentId/turns/:turnId` - Fixed text overflow in `SettingsListItemCardContent` - Updated role applicability labels ("Assignable to Workspace Members") ### 🛠️ Technical Improvements - Fixed Zod schema validation for UUID and Date fields (use string validators) - Updated `ToolRegistryService` to properly register HTTP tool with permission flag - Enhanced error handling in agent execution services - Updated database migrations for new entity schema ## Database Migrations - `1764210000000-add-system-role-to-agent-message.ts` - `1764220000000-add-evaluation-inputs-to-agent.ts` - `1764200000000-add-agent-turn-evaluation.ts` - `1764100000000-refactor-agent-chat-entities.ts` ## Testing - [ ] Agent evaluation flow tested - [ ] Dashboard Builder agent tested - [ ] Permission system validated - [ ] UI tabs and navigation tested - [ ] Database migrations run successfully ## Breaking Changes ⚠️ **Entity Rename**: `AgentChatMessage` renamed to `AgentMessage` - GraphQL queries need updating ## Related Issues  ## Screenshots
2025-11-27 08:25:40 +01:00
parent 35f81805b8
commit 4f20fd35c5
158 changed files with 2954 additions and 556 deletions
@@ -0,0 +1,82 @@
+import { type ToolUIPart } from 'ai';
+import { type ExtendedUIMessagePart } from 'twenty-shared/ai';
+
+import { type AgentMessagePartEntity } from 'src/engine/metadata-modules/ai/ai-agent-execution/entities/agent-message-part.entity';
+
+const isToolPart = (part: ExtendedUIMessagePart): part is ToolUIPart => {
+  return part.type.includes('tool-') && 'toolCallId' in part;
+};
+
+export const mapUIMessagePartsToDBParts = (
+  uiMessageParts: ExtendedUIMessagePart[],
+  messageId: string,
+): Partial<AgentMessagePartEntity>[] => {
+  return uiMessageParts.map((part, index) => {
+    const basePart: Partial<AgentMessagePartEntity> = {
+      messageId,
+      orderIndex: index,
+      type: part.type,
+    };
+
+    switch (part.type) {
+      case 'text':
+        return {
+          ...basePart,
+          textContent: part.text,
+        };
+      case 'reasoning':
+        return {
+          ...basePart,
+          reasoningContent: part.text,
+        };
+      case 'file':
+        return {
+          ...basePart,
+          fileMediaType: part.mediaType,
+          fileFilename: part.filename,
+          fileUrl: part.url,
+        };
+      case 'source-url':
+        return {
+          ...basePart,
+          sourceUrlSourceId: part.sourceId,
+          sourceUrlUrl: part.url,
+          sourceUrlTitle: part.title,
+          providerMetadata: part.providerMetadata ?? null,
+        };
+      case 'source-document':
+        return {
+          ...basePart,
+          sourceDocumentSourceId: part.sourceId,
+          sourceDocumentMediaType: part.mediaType,
+          sourceDocumentTitle: part.title,
+          sourceDocumentFilename: part.filename,
+          providerMetadata: part.providerMetadata ?? null,
+        };
+      case 'step-start':
+        return basePart;
+      case 'data-routing-status':
+        return {
+          ...basePart,
+          textContent: part.data.text,
+          state: part.data.state,
+        };
+      default:
+        {
+          if (isToolPart(part)) {
+            const { toolCallId, input, output, errorText, state } = part;
+
+            return {
+              ...basePart,
+              toolCallId: toolCallId,
+              toolInput: input,
+              toolOutput: output,
+              errorMessage: errorText,
+              state,
+            };
+          }
+        }
+        throw new Error(`Unsupported part type: ${part.type}`);
+    }
+  });
+};