4f20fd35c5
## Summary This PR introduces a comprehensive agent evaluation system and refactors the AI module structure for better organization. ## Key Changes ### 🎯 Agent Evaluation System - Added **Agent Turn Evaluation** entities, DTOs, and database schema - New GraphQL mutations: `evaluateAgentTurn` and `runEvaluationInput` - Added `evaluationInputs` field to Agent entity for storing test inputs - New `AgentTurnGraderService` for automatic turn evaluation - Added evaluation UI with new **Evals** and **Logs** tabs in agent detail pages ### 🏗️ Entity & Module Refactoring - Renamed `AgentChatMessage` → `AgentMessage` for clarity - Consolidated chat entities: `AgentMessage`, `AgentTurn`, and `AgentChatThread` - Reorganized AI modules under `ai/` subdirectory structure - Updated imports across codebase to reflect new module paths ### 🤖 New Agents & Roles - Added **Dashboard Builder Agent** for dashboard creation and management - Added **Dashboard Manager Role** with appropriate permissions - Updated role permissions to be more granular (users vs agents vs API keys) ### 🔐 Permission System Updates - Added `HTTP_REQUEST_TOOL` permission flag - Updated Workflow Manager role permissions (restricted tool access) - Enhanced permission flag types to differentiate between user/agent/API key contexts - Added `isRelevantForAgents`, `isRelevantForApiKeys`, `isRelevantForUsers` to permission flags ### 📨 Message Role Enhancement - Added `system` role to `AgentMessageRole` enum (alongside user/assistant) - Updated message handling to support system prompts ### 🎨 UI/UX Improvements - New tabs in agent detail: **Evals** and **Logs** - Added turn detail page: `/ai/agents/:agentId/turns/:turnId` - Fixed text overflow in `SettingsListItemCardContent` - Updated role applicability labels ("Assignable to Workspace Members") ### 🛠️ Technical Improvements - Fixed Zod schema validation for UUID and Date fields (use string validators) - Updated `ToolRegistryService` to properly register HTTP tool with permission flag - Enhanced error handling in agent execution services - Updated database migrations for new entity schema ## Database Migrations - `1764210000000-add-system-role-to-agent-message.ts` - `1764220000000-add-evaluation-inputs-to-agent.ts` - `1764200000000-add-agent-turn-evaluation.ts` - `1764100000000-refactor-agent-chat-entities.ts` ## Testing - [ ] Agent evaluation flow tested - [ ] Dashboard Builder agent tested - [ ] Permission system validated - [ ] UI tabs and navigation tested - [ ] Database migrations run successfully ## Breaking Changes ⚠️ **Entity Rename**: `AgentChatMessage` renamed to `AgentMessage` - GraphQL queries need updating ## Related Issues <!-- Link any related issues here --> ## Screenshots <!-- Add screenshots if applicable -->
209 lines
6.5 KiB
TypeScript
209 lines
6.5 KiB
TypeScript
import { Test, type TestingModule } from '@nestjs/testing';
|
|
import { getRepositoryToken } from '@nestjs/typeorm';
|
|
|
|
import { AiAgentRoleService } from 'src/engine/metadata-modules/ai/ai-agent-role/ai-agent-role.service';
|
|
import {
|
|
AgentException,
|
|
AgentExceptionCode,
|
|
} from 'src/engine/metadata-modules/ai/ai-agent/agent.exception';
|
|
import { AgentResolver } from 'src/engine/metadata-modules/ai/ai-agent/agent.resolver';
|
|
import { AgentService } from 'src/engine/metadata-modules/ai/ai-agent/agent.service';
|
|
import { AgentEntity } from 'src/engine/metadata-modules/ai/ai-agent/entities/agent.entity';
|
|
import { AgentChatService } from 'src/engine/metadata-modules/ai/ai-chat/services/agent-chat.service';
|
|
import { PermissionsService } from 'src/engine/metadata-modules/permissions/permissions.service';
|
|
import { RoleTargetsEntity } from 'src/engine/metadata-modules/role/role-targets.entity';
|
|
|
|
// Mock the guards and decorators
|
|
jest.mock('../../../../../src/engine/guards/feature-flag.guard', () => ({
|
|
FeatureFlagGuard: jest.fn().mockImplementation(() => ({
|
|
canActivate: jest.fn().mockReturnValue(true),
|
|
})),
|
|
RequireFeatureFlag: () => jest.fn(),
|
|
}));
|
|
|
|
jest.mock('../../../../../src/engine/guards/workspace-auth.guard', () => ({
|
|
WorkspaceAuthGuard: jest.fn().mockImplementation(() => ({
|
|
canActivate: jest.fn().mockReturnValue(true),
|
|
})),
|
|
}));
|
|
|
|
describe('agentResolver', () => {
|
|
let agentService: AgentService;
|
|
let agentResolver: AgentResolver;
|
|
let module: TestingModule;
|
|
|
|
beforeAll(async () => {
|
|
// Create a testing module with mocked dependencies
|
|
module = await Test.createTestingModule({
|
|
providers: [
|
|
AgentResolver,
|
|
{
|
|
provide: AgentService,
|
|
useValue: {
|
|
findOneAgent: jest.fn(),
|
|
updateOneAgent: jest.fn(),
|
|
},
|
|
},
|
|
{
|
|
provide: getRepositoryToken(AgentEntity),
|
|
useValue: {
|
|
find: jest.fn(),
|
|
findOne: jest.fn(),
|
|
save: jest.fn(),
|
|
softDelete: jest.fn(),
|
|
},
|
|
},
|
|
{
|
|
provide: getRepositoryToken(RoleTargetsEntity),
|
|
useValue: {
|
|
findOne: jest.fn(),
|
|
},
|
|
},
|
|
{
|
|
provide: AgentChatService,
|
|
useValue: {
|
|
createThread: jest.fn(),
|
|
},
|
|
},
|
|
{
|
|
provide: AiAgentRoleService,
|
|
useValue: {
|
|
assignRoleToAgent: jest.fn(),
|
|
removeRoleFromAgent: jest.fn(),
|
|
},
|
|
},
|
|
{
|
|
provide: PermissionsService,
|
|
useValue: {
|
|
userHasWorkspaceSettingPermission: jest
|
|
.fn()
|
|
.mockResolvedValue(true),
|
|
},
|
|
},
|
|
],
|
|
}).compile();
|
|
|
|
// Get the mocked services from the module
|
|
agentService = module.get<AgentService>(AgentService);
|
|
agentResolver = module.get<AgentResolver>(AgentResolver);
|
|
});
|
|
|
|
afterAll(async () => {
|
|
if (module) {
|
|
await module.close();
|
|
}
|
|
});
|
|
|
|
beforeEach(() => {
|
|
jest.clearAllMocks();
|
|
});
|
|
describe('findOneAgent', () => {
|
|
const testAgentId = 'test-agent-id';
|
|
const workspaceId = 'test-workspace-id';
|
|
const mockAgent = {
|
|
id: testAgentId,
|
|
name: 'Test Agent for Find',
|
|
description: 'A test agent for find operations',
|
|
prompt: 'You are a test agent for finding.',
|
|
modelId: 'gpt-4o',
|
|
roleId: null,
|
|
createdAt: new Date(),
|
|
updatedAt: new Date(),
|
|
};
|
|
|
|
it('should find agent by ID successfully', async () => {
|
|
// Mock the findOneAgent method to return a mock agent
|
|
(agentService.findOneAgent as jest.Mock).mockResolvedValueOnce(mockAgent);
|
|
|
|
// Call the resolver directly
|
|
const result = await agentResolver.findOneAgent({ id: testAgentId }, {
|
|
id: workspaceId,
|
|
} as any);
|
|
|
|
// Verify the service was called with the correct parameters
|
|
expect(agentService.findOneAgent).toHaveBeenCalledWith(workspaceId, {
|
|
id: testAgentId,
|
|
});
|
|
|
|
// Verify the result matches our expectations
|
|
expect(result).toBeDefined();
|
|
expect(result.id).toBe(testAgentId);
|
|
expect(result.name).toBe('Test Agent for Find');
|
|
});
|
|
|
|
it('should throw an error for non-existent agent', async () => {
|
|
const nonExistentId = '00000000-0000-0000-0000-000000000000';
|
|
|
|
// Mock the findOneAgent method to throw an exception
|
|
(agentService.findOneAgent as jest.Mock).mockRejectedValueOnce(
|
|
new AgentException(
|
|
`Agent with id ${nonExistentId} not found`,
|
|
AgentExceptionCode.AGENT_NOT_FOUND,
|
|
),
|
|
);
|
|
|
|
// Call the resolver and expect it to throw
|
|
await expect(
|
|
agentResolver.findOneAgent({ id: nonExistentId }, {
|
|
id: workspaceId,
|
|
} as any),
|
|
).rejects.toThrow(AgentException);
|
|
});
|
|
});
|
|
|
|
describe('updateOneAgent', () => {
|
|
const testAgentId = 'test-agent-id';
|
|
const workspaceId = 'test-workspace-id';
|
|
|
|
const updatedAgent = {
|
|
id: testAgentId,
|
|
name: 'Updated Test Agent Admin',
|
|
description: 'Updated description',
|
|
prompt: 'Updated prompt for admin',
|
|
modelId: 'gpt-4o-mini',
|
|
roleId: null,
|
|
createdAt: new Date(),
|
|
updatedAt: new Date(),
|
|
};
|
|
|
|
it('should update an agent successfully', async () => {
|
|
// Mock the updateOneAgent method to return the updated agent
|
|
(agentService.updateOneAgent as jest.Mock).mockResolvedValueOnce(
|
|
updatedAgent,
|
|
);
|
|
|
|
// Call the resolver directly
|
|
const result = await agentResolver.updateOneAgent(
|
|
{
|
|
id: testAgentId,
|
|
name: 'Updated Test Agent Admin',
|
|
description: 'Updated description',
|
|
prompt: 'Updated prompt for admin',
|
|
modelId: 'gpt-4o-mini',
|
|
},
|
|
{ id: workspaceId } as any,
|
|
);
|
|
|
|
// Verify the service was called with the correct parameters
|
|
expect(agentService.updateOneAgent).toHaveBeenCalledWith(
|
|
{
|
|
id: testAgentId,
|
|
name: 'Updated Test Agent Admin',
|
|
description: 'Updated description',
|
|
prompt: 'Updated prompt for admin',
|
|
modelId: 'gpt-4o-mini',
|
|
},
|
|
workspaceId,
|
|
);
|
|
|
|
// Verify the result matches our expectations
|
|
expect(result).toBeDefined();
|
|
expect(result.id).toBe(testAgentId);
|
|
expect(result.name).toBe('Updated Test Agent Admin');
|
|
expect(result.description).toBe('Updated description');
|
|
expect(result.prompt).toBe('Updated prompt for admin');
|
|
expect(result.modelId).toBe('gpt-4o-mini');
|
|
});
|
|
});
|
|
});
|