Files
twenty/packages/twenty-server/test/integration/graphql/suites/agent/agent.integration-spec.ts
T
Félix Malfait 4f20fd35c5 feat: Add Agent Evaluation System and Refactor AI Modules (#16111)
## Summary

This PR introduces a comprehensive agent evaluation system and refactors
the AI module structure for better organization.

## Key Changes

### 🎯 Agent Evaluation System
- Added **Agent Turn Evaluation** entities, DTOs, and database schema
- New GraphQL mutations: `evaluateAgentTurn` and `runEvaluationInput`
- Added `evaluationInputs` field to Agent entity for storing test inputs
- New `AgentTurnGraderService` for automatic turn evaluation
- Added evaluation UI with new **Evals** and **Logs** tabs in agent
detail pages

### 🏗️ Entity & Module Refactoring
- Renamed `AgentChatMessage` → `AgentMessage` for clarity
- Consolidated chat entities: `AgentMessage`, `AgentTurn`, and
`AgentChatThread`
- Reorganized AI modules under `ai/` subdirectory structure
- Updated imports across codebase to reflect new module paths

### 🤖 New Agents & Roles
- Added **Dashboard Builder Agent** for dashboard creation and
management
- Added **Dashboard Manager Role** with appropriate permissions
- Updated role permissions to be more granular (users vs agents vs API
keys)

### 🔐 Permission System Updates
- Added `HTTP_REQUEST_TOOL` permission flag
- Updated Workflow Manager role permissions (restricted tool access)
- Enhanced permission flag types to differentiate between user/agent/API
key contexts
- Added `isRelevantForAgents`, `isRelevantForApiKeys`,
`isRelevantForUsers` to permission flags

### 📨 Message Role Enhancement
- Added `system` role to `AgentMessageRole` enum (alongside
user/assistant)
- Updated message handling to support system prompts

### 🎨 UI/UX Improvements
- New tabs in agent detail: **Evals** and **Logs**
- Added turn detail page: `/ai/agents/:agentId/turns/:turnId`
- Fixed text overflow in `SettingsListItemCardContent`
- Updated role applicability labels ("Assignable to Workspace Members")

### 🛠️ Technical Improvements
- Fixed Zod schema validation for UUID and Date fields (use string
validators)
- Updated `ToolRegistryService` to properly register HTTP tool with
permission flag
- Enhanced error handling in agent execution services
- Updated database migrations for new entity schema

## Database Migrations
- `1764210000000-add-system-role-to-agent-message.ts`
- `1764220000000-add-evaluation-inputs-to-agent.ts`
- `1764200000000-add-agent-turn-evaluation.ts`
- `1764100000000-refactor-agent-chat-entities.ts`

## Testing
- [ ] Agent evaluation flow tested
- [ ] Dashboard Builder agent tested
- [ ] Permission system validated
- [ ] UI tabs and navigation tested
- [ ] Database migrations run successfully

## Breaking Changes
⚠️ **Entity Rename**: `AgentChatMessage` renamed to `AgentMessage` -
GraphQL queries need updating

## Related Issues
<!-- Link any related issues here -->

## Screenshots
<!-- Add screenshots if applicable -->
2025-11-27 08:25:40 +01:00

209 lines
6.5 KiB
TypeScript

import { Test, type TestingModule } from '@nestjs/testing';
import { getRepositoryToken } from '@nestjs/typeorm';
import { AiAgentRoleService } from 'src/engine/metadata-modules/ai/ai-agent-role/ai-agent-role.service';
import {
AgentException,
AgentExceptionCode,
} from 'src/engine/metadata-modules/ai/ai-agent/agent.exception';
import { AgentResolver } from 'src/engine/metadata-modules/ai/ai-agent/agent.resolver';
import { AgentService } from 'src/engine/metadata-modules/ai/ai-agent/agent.service';
import { AgentEntity } from 'src/engine/metadata-modules/ai/ai-agent/entities/agent.entity';
import { AgentChatService } from 'src/engine/metadata-modules/ai/ai-chat/services/agent-chat.service';
import { PermissionsService } from 'src/engine/metadata-modules/permissions/permissions.service';
import { RoleTargetsEntity } from 'src/engine/metadata-modules/role/role-targets.entity';
// Mock the guards and decorators
jest.mock('../../../../../src/engine/guards/feature-flag.guard', () => ({
FeatureFlagGuard: jest.fn().mockImplementation(() => ({
canActivate: jest.fn().mockReturnValue(true),
})),
RequireFeatureFlag: () => jest.fn(),
}));
jest.mock('../../../../../src/engine/guards/workspace-auth.guard', () => ({
WorkspaceAuthGuard: jest.fn().mockImplementation(() => ({
canActivate: jest.fn().mockReturnValue(true),
})),
}));
describe('agentResolver', () => {
let agentService: AgentService;
let agentResolver: AgentResolver;
let module: TestingModule;
beforeAll(async () => {
// Create a testing module with mocked dependencies
module = await Test.createTestingModule({
providers: [
AgentResolver,
{
provide: AgentService,
useValue: {
findOneAgent: jest.fn(),
updateOneAgent: jest.fn(),
},
},
{
provide: getRepositoryToken(AgentEntity),
useValue: {
find: jest.fn(),
findOne: jest.fn(),
save: jest.fn(),
softDelete: jest.fn(),
},
},
{
provide: getRepositoryToken(RoleTargetsEntity),
useValue: {
findOne: jest.fn(),
},
},
{
provide: AgentChatService,
useValue: {
createThread: jest.fn(),
},
},
{
provide: AiAgentRoleService,
useValue: {
assignRoleToAgent: jest.fn(),
removeRoleFromAgent: jest.fn(),
},
},
{
provide: PermissionsService,
useValue: {
userHasWorkspaceSettingPermission: jest
.fn()
.mockResolvedValue(true),
},
},
],
}).compile();
// Get the mocked services from the module
agentService = module.get<AgentService>(AgentService);
agentResolver = module.get<AgentResolver>(AgentResolver);
});
afterAll(async () => {
if (module) {
await module.close();
}
});
beforeEach(() => {
jest.clearAllMocks();
});
describe('findOneAgent', () => {
const testAgentId = 'test-agent-id';
const workspaceId = 'test-workspace-id';
const mockAgent = {
id: testAgentId,
name: 'Test Agent for Find',
description: 'A test agent for find operations',
prompt: 'You are a test agent for finding.',
modelId: 'gpt-4o',
roleId: null,
createdAt: new Date(),
updatedAt: new Date(),
};
it('should find agent by ID successfully', async () => {
// Mock the findOneAgent method to return a mock agent
(agentService.findOneAgent as jest.Mock).mockResolvedValueOnce(mockAgent);
// Call the resolver directly
const result = await agentResolver.findOneAgent({ id: testAgentId }, {
id: workspaceId,
} as any);
// Verify the service was called with the correct parameters
expect(agentService.findOneAgent).toHaveBeenCalledWith(workspaceId, {
id: testAgentId,
});
// Verify the result matches our expectations
expect(result).toBeDefined();
expect(result.id).toBe(testAgentId);
expect(result.name).toBe('Test Agent for Find');
});
it('should throw an error for non-existent agent', async () => {
const nonExistentId = '00000000-0000-0000-0000-000000000000';
// Mock the findOneAgent method to throw an exception
(agentService.findOneAgent as jest.Mock).mockRejectedValueOnce(
new AgentException(
`Agent with id ${nonExistentId} not found`,
AgentExceptionCode.AGENT_NOT_FOUND,
),
);
// Call the resolver and expect it to throw
await expect(
agentResolver.findOneAgent({ id: nonExistentId }, {
id: workspaceId,
} as any),
).rejects.toThrow(AgentException);
});
});
describe('updateOneAgent', () => {
const testAgentId = 'test-agent-id';
const workspaceId = 'test-workspace-id';
const updatedAgent = {
id: testAgentId,
name: 'Updated Test Agent Admin',
description: 'Updated description',
prompt: 'Updated prompt for admin',
modelId: 'gpt-4o-mini',
roleId: null,
createdAt: new Date(),
updatedAt: new Date(),
};
it('should update an agent successfully', async () => {
// Mock the updateOneAgent method to return the updated agent
(agentService.updateOneAgent as jest.Mock).mockResolvedValueOnce(
updatedAgent,
);
// Call the resolver directly
const result = await agentResolver.updateOneAgent(
{
id: testAgentId,
name: 'Updated Test Agent Admin',
description: 'Updated description',
prompt: 'Updated prompt for admin',
modelId: 'gpt-4o-mini',
},
{ id: workspaceId } as any,
);
// Verify the service was called with the correct parameters
expect(agentService.updateOneAgent).toHaveBeenCalledWith(
{
id: testAgentId,
name: 'Updated Test Agent Admin',
description: 'Updated description',
prompt: 'Updated prompt for admin',
modelId: 'gpt-4o-mini',
},
workspaceId,
);
// Verify the result matches our expectations
expect(result).toBeDefined();
expect(result.id).toBe(testAgentId);
expect(result.name).toBe('Updated Test Agent Admin');
expect(result.description).toBe('Updated description');
expect(result.prompt).toBe('Updated prompt for admin');
expect(result.modelId).toBe('gpt-4o-mini');
});
});
});