feat: Add Agent Evaluation System and Refactor AI Modules (#16111)

## Summary

This PR introduces a comprehensive agent evaluation system and refactors
the AI module structure for better organization.

## Key Changes

### 🎯 Agent Evaluation System
- Added **Agent Turn Evaluation** entities, DTOs, and database schema
- New GraphQL mutations: `evaluateAgentTurn` and `runEvaluationInput`
- Added `evaluationInputs` field to Agent entity for storing test inputs
- New `AgentTurnGraderService` for automatic turn evaluation
- Added evaluation UI with new **Evals** and **Logs** tabs in agent
detail pages

### 🏗️ Entity & Module Refactoring
- Renamed `AgentChatMessage` → `AgentMessage` for clarity
- Consolidated chat entities: `AgentMessage`, `AgentTurn`, and
`AgentChatThread`
- Reorganized AI modules under `ai/` subdirectory structure
- Updated imports across codebase to reflect new module paths

### 🤖 New Agents & Roles
- Added **Dashboard Builder Agent** for dashboard creation and
management
- Added **Dashboard Manager Role** with appropriate permissions
- Updated role permissions to be more granular (users vs agents vs API
keys)

### 🔐 Permission System Updates
- Added `HTTP_REQUEST_TOOL` permission flag
- Updated Workflow Manager role permissions (restricted tool access)
- Enhanced permission flag types to differentiate between user/agent/API
key contexts
- Added `isRelevantForAgents`, `isRelevantForApiKeys`,
`isRelevantForUsers` to permission flags

### 📨 Message Role Enhancement
- Added `system` role to `AgentMessageRole` enum (alongside
user/assistant)
- Updated message handling to support system prompts

### 🎨 UI/UX Improvements
- New tabs in agent detail: **Evals** and **Logs**
- Added turn detail page: `/ai/agents/:agentId/turns/:turnId`
- Fixed text overflow in `SettingsListItemCardContent`
- Updated role applicability labels ("Assignable to Workspace Members")

### 🛠️ Technical Improvements
- Fixed Zod schema validation for UUID and Date fields (use string
validators)
- Updated `ToolRegistryService` to properly register HTTP tool with
permission flag
- Enhanced error handling in agent execution services
- Updated database migrations for new entity schema

## Database Migrations
- `1764210000000-add-system-role-to-agent-message.ts`
- `1764220000000-add-evaluation-inputs-to-agent.ts`
- `1764200000000-add-agent-turn-evaluation.ts`
- `1764100000000-refactor-agent-chat-entities.ts`

## Testing
- [ ] Agent evaluation flow tested
- [ ] Dashboard Builder agent tested
- [ ] Permission system validated
- [ ] UI tabs and navigation tested
- [ ] Database migrations run successfully

## Breaking Changes
⚠️ **Entity Rename**: `AgentChatMessage` renamed to `AgentMessage` -
GraphQL queries need updating

## Related Issues
<!-- Link any related issues here -->

## Screenshots
<!-- Add screenshots if applicable -->
This commit is contained in:
Félix Malfait
2025-11-27 08:25:40 +01:00
committed by GitHub
parent 35f81805b8
commit 4f20fd35c5
158 changed files with 2954 additions and 556 deletions
@@ -2,11 +2,11 @@ import { Test, type TestingModule } from '@nestjs/testing';
import { SupportDriver } from 'src/engine/core-modules/twenty-config/interfaces/support.interface';
import { ClientConfigService } from 'src/engine/core-modules/client-config/services/client-config.service';
import {
type ModelId,
ModelProvider,
} from 'src/engine/metadata-modules/ai-models/constants/ai-models.const';
import { ClientConfigService } from 'src/engine/core-modules/client-config/services/client-config.service';
} from 'src/engine/metadata-modules/ai/ai-models/constants/ai-models.const';
import { ClientConfigController } from './client-config.controller';
@@ -2,14 +2,14 @@ import { Field, ObjectType, registerEnumType } from '@nestjs/graphql';
import { SupportDriver } from 'src/engine/core-modules/twenty-config/interfaces/support.interface';
import {
ModelId,
ModelProvider,
} from 'src/engine/metadata-modules/ai-models/constants/ai-models.const';
import { BillingTrialPeriodDTO } from 'src/engine/core-modules/billing/dtos/billing-trial-period.dto';
import { CaptchaDriverType } from 'src/engine/core-modules/captcha/interfaces';
import { FeatureFlagKey } from 'src/engine/core-modules/feature-flag/enums/feature-flag-key.enum';
import { AuthProvidersDTO } from 'src/engine/core-modules/workspace/dtos/public-workspace-data-output';
import {
ModelId,
ModelProvider,
} from 'src/engine/metadata-modules/ai/ai-models/constants/ai-models.const';
registerEnumType(FeatureFlagKey, {
name: 'FeatureFlagKey',
@@ -3,12 +3,12 @@ import { Test, type TestingModule } from '@nestjs/testing';
import { NodeEnvironment } from 'src/engine/core-modules/twenty-config/interfaces/node-environment.interface';
import { SupportDriver } from 'src/engine/core-modules/twenty-config/interfaces/support.interface';
import { AiModelRegistryService } from 'src/engine/metadata-modules/ai-models/services/ai-model-registry.service';
import { CaptchaDriverType } from 'src/engine/core-modules/captcha/interfaces';
import { ClientConfigService } from 'src/engine/core-modules/client-config/services/client-config.service';
import { DomainServerConfigService } from 'src/engine/core-modules/domain/domain-server-config/services/domain-server-config.service';
import { PUBLIC_FEATURE_FLAGS } from 'src/engine/core-modules/feature-flag/constants/public-feature-flag.const';
import { TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service';
import { AiModelRegistryService } from 'src/engine/metadata-modules/ai/ai-models/services/ai-model-registry.service';
describe('ClientConfigService', () => {
let service: ClientConfigService;
@@ -12,14 +12,14 @@ import {
import { DomainServerConfigService } from 'src/engine/core-modules/domain/domain-server-config/services/domain-server-config.service';
import { PUBLIC_FEATURE_FLAGS } from 'src/engine/core-modules/feature-flag/constants/public-feature-flag.const';
import { TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service';
import { convertCentsToBillingCredits } from 'src/engine/metadata-modules/ai-billing/utils/convert-cents-to-billing-credits.util';
import { convertCentsToBillingCredits } from 'src/engine/metadata-modules/ai/ai-billing/utils/convert-cents-to-billing-credits.util';
import {
AI_MODELS,
DEFAULT_FAST_MODEL,
DEFAULT_SMART_MODEL,
ModelProvider,
} from 'src/engine/metadata-modules/ai-models/constants/ai-models.const';
import { AiModelRegistryService } from 'src/engine/metadata-modules/ai-models/services/ai-model-registry.service';
} from 'src/engine/metadata-modules/ai/ai-models/constants/ai-models.const';
import { AiModelRegistryService } from 'src/engine/metadata-modules/ai/ai-models/services/ai-model-registry.service';
@Injectable()
export class ClientConfigService {