4f20fd35c5
## Summary This PR introduces a comprehensive agent evaluation system and refactors the AI module structure for better organization. ## Key Changes ### 🎯 Agent Evaluation System - Added **Agent Turn Evaluation** entities, DTOs, and database schema - New GraphQL mutations: `evaluateAgentTurn` and `runEvaluationInput` - Added `evaluationInputs` field to Agent entity for storing test inputs - New `AgentTurnGraderService` for automatic turn evaluation - Added evaluation UI with new **Evals** and **Logs** tabs in agent detail pages ### 🏗️ Entity & Module Refactoring - Renamed `AgentChatMessage` → `AgentMessage` for clarity - Consolidated chat entities: `AgentMessage`, `AgentTurn`, and `AgentChatThread` - Reorganized AI modules under `ai/` subdirectory structure - Updated imports across codebase to reflect new module paths ### 🤖 New Agents & Roles - Added **Dashboard Builder Agent** for dashboard creation and management - Added **Dashboard Manager Role** with appropriate permissions - Updated role permissions to be more granular (users vs agents vs API keys) ### 🔐 Permission System Updates - Added `HTTP_REQUEST_TOOL` permission flag - Updated Workflow Manager role permissions (restricted tool access) - Enhanced permission flag types to differentiate between user/agent/API key contexts - Added `isRelevantForAgents`, `isRelevantForApiKeys`, `isRelevantForUsers` to permission flags ### 📨 Message Role Enhancement - Added `system` role to `AgentMessageRole` enum (alongside user/assistant) - Updated message handling to support system prompts ### 🎨 UI/UX Improvements - New tabs in agent detail: **Evals** and **Logs** - Added turn detail page: `/ai/agents/:agentId/turns/:turnId` - Fixed text overflow in `SettingsListItemCardContent` - Updated role applicability labels ("Assignable to Workspace Members") ### 🛠️ Technical Improvements - Fixed Zod schema validation for UUID and Date fields (use string validators) - Updated `ToolRegistryService` to properly register HTTP tool with permission flag - Enhanced error handling in agent execution services - Updated database migrations for new entity schema ## Database Migrations - `1764210000000-add-system-role-to-agent-message.ts` - `1764220000000-add-evaluation-inputs-to-agent.ts` - `1764200000000-add-agent-turn-evaluation.ts` - `1764100000000-refactor-agent-chat-entities.ts` ## Testing - [ ] Agent evaluation flow tested - [ ] Dashboard Builder agent tested - [ ] Permission system validated - [ ] UI tabs and navigation tested - [ ] Database migrations run successfully ## Breaking Changes ⚠️ **Entity Rename**: `AgentChatMessage` renamed to `AgentMessage` - GraphQL queries need updating ## Related Issues <!-- Link any related issues here --> ## Screenshots <!-- Add screenshots if applicable -->
307 lines
9.6 KiB
TypeScript
307 lines
9.6 KiB
TypeScript
import { Injectable } from '@nestjs/common';
|
|
import { InjectRepository } from '@nestjs/typeorm';
|
|
|
|
import { msg } from '@lingui/core/macro';
|
|
import { isDefined } from 'twenty-shared/utils';
|
|
import { In, Repository } from 'typeorm';
|
|
|
|
import { ApiKeyRoleService } from 'src/engine/core-modules/api-key/api-key-role.service';
|
|
import { PermissionFlagType } from 'src/engine/metadata-modules/permissions/constants/permission-flag-type.constants';
|
|
import { TOOL_PERMISSION_FLAGS } from 'src/engine/metadata-modules/permissions/constants/tool-permission-flags';
|
|
import {
|
|
PermissionsException,
|
|
PermissionsExceptionCode,
|
|
PermissionsExceptionMessage,
|
|
} from 'src/engine/metadata-modules/permissions/permissions.exception';
|
|
import { type UserWorkspacePermissions } from 'src/engine/metadata-modules/permissions/types/user-workspace-permissions';
|
|
import { RoleEntity } from 'src/engine/metadata-modules/role/role.entity';
|
|
import { UserRoleService } from 'src/engine/metadata-modules/user-role/user-role.service';
|
|
import { WorkspacePermissionsCacheService } from 'src/engine/metadata-modules/workspace-permissions-cache/workspace-permissions-cache.service';
|
|
import { type RolePermissionConfig } from 'src/engine/twenty-orm/types/role-permission-config';
|
|
|
|
@Injectable()
|
|
export class PermissionsService {
|
|
constructor(
|
|
private readonly userRoleService: UserRoleService,
|
|
private readonly workspacePermissionsCacheService: WorkspacePermissionsCacheService,
|
|
private readonly apiKeyRoleService: ApiKeyRoleService,
|
|
@InjectRepository(RoleEntity)
|
|
private readonly roleRepository: Repository<RoleEntity>,
|
|
) {}
|
|
|
|
private isToolPermission(feature: string) {
|
|
return TOOL_PERMISSION_FLAGS.includes(feature);
|
|
}
|
|
|
|
public async getUserWorkspacePermissions({
|
|
userWorkspaceId,
|
|
workspaceId,
|
|
}: {
|
|
userWorkspaceId: string;
|
|
workspaceId: string;
|
|
}): Promise<UserWorkspacePermissions> {
|
|
const [roleOfUserWorkspace] = await this.userRoleService
|
|
.getRolesByUserWorkspaces({
|
|
userWorkspaceIds: [userWorkspaceId],
|
|
workspaceId,
|
|
})
|
|
.then((roles) => roles?.get(userWorkspaceId) ?? []);
|
|
|
|
if (!isDefined(roleOfUserWorkspace)) {
|
|
throw new PermissionsException(
|
|
PermissionsExceptionMessage.NO_ROLE_FOUND_FOR_USER_WORKSPACE,
|
|
PermissionsExceptionCode.NO_ROLE_FOUND_FOR_USER_WORKSPACE,
|
|
{
|
|
userFriendlyMessage: msg`Your role in this workspace could not be found. Please contact your workspace administrator.`,
|
|
},
|
|
);
|
|
}
|
|
|
|
const defaultSettingsPermissions =
|
|
this.getDefaultUserWorkspacePermissions().permissionFlags;
|
|
const permissionFlags = Object.keys(PermissionFlagType).reduce(
|
|
(acc, feature) => {
|
|
const hasBasePermission = this.isToolPermission(feature)
|
|
? roleOfUserWorkspace.canAccessAllTools
|
|
: roleOfUserWorkspace.canUpdateAllSettings;
|
|
|
|
return {
|
|
...acc,
|
|
[feature]:
|
|
hasBasePermission ||
|
|
roleOfUserWorkspace.permissionFlags.some(
|
|
(permissionFlag) => permissionFlag.flag === feature,
|
|
),
|
|
};
|
|
},
|
|
defaultSettingsPermissions,
|
|
);
|
|
|
|
const { data: rolesPermissions } =
|
|
await this.workspacePermissionsCacheService.getRolesPermissionsFromCache({
|
|
workspaceId,
|
|
});
|
|
|
|
const objectsPermissions = rolesPermissions[roleOfUserWorkspace.id] ?? {};
|
|
|
|
return {
|
|
permissionFlags,
|
|
objectsPermissions,
|
|
};
|
|
}
|
|
|
|
public getDefaultUserWorkspacePermissions = () =>
|
|
({
|
|
permissionFlags: {
|
|
[PermissionFlagType.API_KEYS_AND_WEBHOOKS]: false,
|
|
[PermissionFlagType.WORKSPACE]: false,
|
|
[PermissionFlagType.WORKSPACE_MEMBERS]: false,
|
|
[PermissionFlagType.ROLES]: false,
|
|
[PermissionFlagType.DATA_MODEL]: false,
|
|
[PermissionFlagType.SECURITY]: false,
|
|
[PermissionFlagType.WORKFLOWS]: false,
|
|
[PermissionFlagType.APPLICATIONS]: false,
|
|
[PermissionFlagType.LAYOUTS]: false,
|
|
[PermissionFlagType.VIEWS]: false,
|
|
[PermissionFlagType.BILLING]: false,
|
|
[PermissionFlagType.AI_SETTINGS]: false,
|
|
[PermissionFlagType.AI]: false,
|
|
[PermissionFlagType.UPLOAD_FILE]: false,
|
|
[PermissionFlagType.DOWNLOAD_FILE]: false,
|
|
[PermissionFlagType.SEND_EMAIL_TOOL]: false,
|
|
[PermissionFlagType.HTTP_REQUEST_TOOL]: false,
|
|
[PermissionFlagType.IMPORT_CSV]: false,
|
|
[PermissionFlagType.EXPORT_CSV]: false,
|
|
[PermissionFlagType.CONNECTED_ACCOUNTS]: false,
|
|
[PermissionFlagType.IMPERSONATE]: false,
|
|
[PermissionFlagType.SSO_BYPASS]: false,
|
|
[PermissionFlagType.PROFILE_INFORMATION]: false,
|
|
},
|
|
objectsPermissions: {},
|
|
}) as const satisfies UserWorkspacePermissions;
|
|
|
|
public async userHasWorkspaceSettingPermission({
|
|
userWorkspaceId,
|
|
workspaceId,
|
|
setting,
|
|
apiKeyId,
|
|
}: {
|
|
userWorkspaceId?: string;
|
|
workspaceId: string;
|
|
setting: PermissionFlagType;
|
|
apiKeyId?: string;
|
|
}): Promise<boolean> {
|
|
if (apiKeyId) {
|
|
const roleId = await this.apiKeyRoleService.getRoleIdForApiKey(
|
|
apiKeyId,
|
|
workspaceId,
|
|
);
|
|
|
|
const role = await this.roleRepository.findOne({
|
|
where: { id: roleId, workspaceId },
|
|
relations: ['permissionFlags'],
|
|
});
|
|
|
|
if (!isDefined(role)) {
|
|
throw new PermissionsException(
|
|
PermissionsExceptionMessage.API_KEY_ROLE_NOT_FOUND,
|
|
PermissionsExceptionCode.API_KEY_ROLE_NOT_FOUND,
|
|
{
|
|
userFriendlyMessage: msg`The API key does not have a valid role assigned. Please check your API key configuration.`,
|
|
},
|
|
);
|
|
}
|
|
|
|
return this.checkRolePermissions(role, setting);
|
|
}
|
|
|
|
if (userWorkspaceId) {
|
|
const [roleOfUserWorkspace] = await this.userRoleService
|
|
.getRolesByUserWorkspaces({
|
|
userWorkspaceIds: [userWorkspaceId],
|
|
workspaceId,
|
|
})
|
|
.then((roles) => roles?.get(userWorkspaceId) ?? []);
|
|
|
|
if (!isDefined(roleOfUserWorkspace)) {
|
|
throw new PermissionsException(
|
|
PermissionsExceptionMessage.NO_ROLE_FOUND_FOR_USER_WORKSPACE,
|
|
PermissionsExceptionCode.NO_ROLE_FOUND_FOR_USER_WORKSPACE,
|
|
{
|
|
userFriendlyMessage: msg`Your role in this workspace could not be found. Please contact your workspace administrator.`,
|
|
},
|
|
);
|
|
}
|
|
|
|
return this.checkRolePermissions(roleOfUserWorkspace, setting);
|
|
}
|
|
|
|
throw new PermissionsException(
|
|
PermissionsExceptionMessage.NO_AUTHENTICATION_CONTEXT,
|
|
PermissionsExceptionCode.NO_AUTHENTICATION_CONTEXT,
|
|
{
|
|
userFriendlyMessage: msg`Authentication is required to access this feature. Please sign in and try again.`,
|
|
},
|
|
);
|
|
}
|
|
|
|
public checkRolePermissions(
|
|
role: RoleEntity,
|
|
setting: PermissionFlagType,
|
|
): boolean {
|
|
const hasBasePermission = this.isToolPermission(setting)
|
|
? role.canAccessAllTools
|
|
: role.canUpdateAllSettings;
|
|
|
|
if (hasBasePermission === true) {
|
|
return true;
|
|
}
|
|
|
|
const permissionFlags = role.permissionFlags ?? [];
|
|
|
|
return permissionFlags.some(
|
|
(permissionFlag) => permissionFlag.flag === setting,
|
|
);
|
|
}
|
|
|
|
private async getRolesFromPermissionConfig(
|
|
rolePermissionConfig: RolePermissionConfig,
|
|
workspaceId: string,
|
|
relations: string[] = [],
|
|
): Promise<{ roles: RoleEntity[]; useIntersection: boolean } | null> {
|
|
if ('shouldBypassPermissionChecks' in rolePermissionConfig) {
|
|
return null;
|
|
}
|
|
|
|
let roleIds: string[] = [];
|
|
let useIntersection = false;
|
|
|
|
if ('intersectionOf' in rolePermissionConfig) {
|
|
roleIds = rolePermissionConfig.intersectionOf;
|
|
useIntersection = true;
|
|
} else if ('unionOf' in rolePermissionConfig) {
|
|
roleIds = rolePermissionConfig.unionOf;
|
|
useIntersection = false;
|
|
}
|
|
|
|
if (roleIds.length === 0) {
|
|
throw new Error('No role IDs provided');
|
|
}
|
|
|
|
const roles = await this.roleRepository.find({
|
|
where: { id: In(roleIds), workspaceId },
|
|
relations,
|
|
});
|
|
|
|
if (roles.length !== roleIds.length) {
|
|
throw new Error('Some roles not found');
|
|
}
|
|
|
|
return { roles, useIntersection };
|
|
}
|
|
|
|
public async checkRolesPermissions(
|
|
rolePermissionConfig: RolePermissionConfig,
|
|
workspaceId: string,
|
|
setting: PermissionFlagType,
|
|
): Promise<boolean> {
|
|
try {
|
|
const result = await this.getRolesFromPermissionConfig(
|
|
rolePermissionConfig,
|
|
workspaceId,
|
|
['permissionFlags'],
|
|
);
|
|
|
|
if (result === null) {
|
|
return true;
|
|
}
|
|
|
|
const { roles, useIntersection } = result;
|
|
|
|
return useIntersection
|
|
? roles.every((role) => this.checkRolePermissions(role, setting))
|
|
: roles.some((role) => this.checkRolePermissions(role, setting));
|
|
} catch {
|
|
return false;
|
|
}
|
|
}
|
|
|
|
public async hasToolPermission(
|
|
rolePermissionConfig: RolePermissionConfig,
|
|
workspaceId: string,
|
|
flag: PermissionFlagType,
|
|
): Promise<boolean> {
|
|
try {
|
|
const result = await this.getRolesFromPermissionConfig(
|
|
rolePermissionConfig,
|
|
workspaceId,
|
|
['permissionFlags'],
|
|
);
|
|
|
|
if (result === null) {
|
|
return true;
|
|
}
|
|
|
|
const { roles, useIntersection } = result;
|
|
|
|
const checkRoleHasPermission = (role: RoleEntity) => {
|
|
if (role.canAccessAllTools === true) {
|
|
return true;
|
|
}
|
|
|
|
const permissionFlags = role.permissionFlags ?? [];
|
|
|
|
return permissionFlags.some(
|
|
(permissionFlag) => permissionFlag.flag === flag,
|
|
);
|
|
};
|
|
|
|
return useIntersection
|
|
? roles.every(checkRoleHasPermission)
|
|
: roles.some(checkRoleHasPermission);
|
|
} catch {
|
|
return false;
|
|
}
|
|
}
|
|
}
|