feat: add configurable response format for AI agents (text/JSON) (#15953)

## Summary
This PR adds configurable response format support for AI agents,
allowing them to return either plain text or structured JSON data based
on a defined schema.

## Key Features

### 1. Agent Response Format Configuration
- Added `AgentResponseFormat` type supporting:
  - `text`: Returns plain text responses (default)
  - `json`: Returns structured JSON based on defined schema
- New `AgentResponseSchema` type moved to `twenty-shared/ai` for sharing
between frontend/backend

### 2. Settings UI
- New `SettingsAgentResponseFormat` component for configuring response
format
- Visual schema builder for defining JSON output structure
- Real-time validation and preview
- Integrated into agent settings tab

### 3. Workflow Integration
- AI Agent workflow action automatically uses agent's configured
response format
- Output schema dynamically generated from agent's response format
- Workflow variable picker shows structured fields for JSON responses
- Backward compatible with existing text-only agents

### 4. Backend Implementation
- Added `convertAgentSchemaToZod` utility to validate JSON responses
- Agent executor service handles both text and JSON generation
- Automatic agent creation/cloning when adding AI agent steps to
workflows
- Unique agent naming with conflict resolution

### 5. Database Migration
- Migration `1763622159656-update-agent-response-format.ts` 
- Sets default `responseFormat` to `{"type":"text"}` for existing agents
- Updated all standard agents with proper response format

## Changes by Module

### Frontend (`twenty-front`)
- 🆕 `AgentResponseFormat` type
- 🆕 `SettingsAgentResponseFormat` component
- ✏️ Updated `WorkflowEditActionAiAgent` to support response format
configuration
- 🗑️ Removed deprecated `useAiAgentOutputSchema` hook and
`AiAgentOutputSchema` type

### Backend (`twenty-server`)
- 🆕 `AgentResponseFormat` type in agent entity
- 🆕 `convertAgentSchemaToZod` utility for schema validation
- ✏️ Updated `AiAgentExecutorService` to handle both text and JSON
generation
- ✏️ Updated `WorkflowSchemaWorkspaceService` to generate output schema
from agent config
- ✏️ Enhanced `WorkflowVersionStepOperationsWorkspaceService` with agent
creation/cloning
- 🆕 Agent naming constants for conflict resolution

### Shared (`twenty-shared`)
- 🆕 `AgentResponseSchema` type
- 🆕 `ModelConfiguration` type moved to shared package
- Updated exports in `ai/index.ts`

## Code Quality
- Removed useless comments following code style guidelines
- All linter checks passed
- Type-safe implementation with proper TypeScript types

## Testing
-  Database migration tested
-  Agent creation/cloning in workflows verified
-  Response format switching (text ↔ JSON) validated
-  Backward compatibility with existing agents confirmed

## Migration Notes
- Existing agents will have `responseFormat: {type: 'text'}` set
automatically
- No breaking changes - all existing functionality preserved
- Agents can be updated to use JSON format through settings UI
This commit is contained in:
Félix Malfait
2025-11-20 18:32:44 +01:00
committed by GitHub
parent 5476879f77
commit a281f2a773
49 changed files with 1207 additions and 641 deletions
@@ -16,6 +16,7 @@ import { SyncableEntity } from 'src/engine/workspace-manager/workspace-sync/inte
import { ModelId } from 'src/engine/core-modules/ai/constants/ai-models.const';
import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity';
import { AgentResponseFormat } from 'src/engine/metadata-modules/agent/types/agent-response-format.type';
import { ModelConfiguration } from 'src/engine/metadata-modules/agent/types/modelConfiguration';
import { AgentHandoffEntity } from './agent-handoff.entity';
@@ -54,8 +55,8 @@ export class AgentEntity
@Column({ nullable: false, type: 'varchar', default: 'auto' })
modelId: ModelId;
@Column({ nullable: true, type: 'jsonb' })
responseFormat: object;
@Column({ nullable: true, type: 'jsonb', default: { type: 'text' } })
responseFormat: AgentResponseFormat;
@Column({ nullable: false, type: 'uuid' })
workspaceId: string;
@@ -120,17 +120,20 @@ export class AgentService {
async updateOneAgent(input: UpdateAgentInput, workspaceId: string) {
const agent = await this.findOneAgent(input.id, workspaceId);
let updatedName = input.name;
const updateData: Partial<AgentEntity> = {
...agent,
...Object.fromEntries(
Object.entries(input).filter(([_, value]) => value !== undefined),
),
};
if (input.label) {
updatedName = computeMetadataNameFromLabel(input.label);
if (input.label !== undefined) {
updateData.name = computeMetadataNameFromLabel(input.label);
} else if (input.name !== undefined) {
updateData.name = input.name;
}
const updatedAgent = await this.agentRepository.save({
...agent,
...input,
name: updatedName,
});
const updatedAgent = await this.agentRepository.save(updateData);
if (!('roleId' in input)) {
return updatedAgent;
@@ -22,12 +22,12 @@ export class UpdateAgentInput {
@IsString()
@IsOptional()
@Field()
@Field({ nullable: true })
name?: string;
@IsString()
@IsOptional()
@Field()
@Field({ nullable: true })
label?: string;
@IsString()
@@ -42,12 +42,12 @@ export class UpdateAgentInput {
@IsString()
@IsOptional()
@Field()
@Field({ nullable: true })
prompt?: string;
@IsString()
@IsOptional()
@Field(() => String)
@Field(() => String, { nullable: true })
modelId?: ModelId;
@IsUUID()
@@ -0,0 +1,10 @@
import { type AgentResponseSchema } from 'twenty-shared/ai';
export type AgentResponseFormatType = 'text' | 'json';
export type AgentResponseFormat =
| { type: 'text' }
| {
type: 'json';
schema: AgentResponseSchema;
};
@@ -1,10 +1 @@
export type ModelConfiguration = {
webSearch?: {
enabled: boolean;
configuration: object;
};
twitterSearch?: {
enabled: boolean;
configuration: object;
};
};
export type { ModelConfiguration } from 'twenty-shared/ai';
@@ -1,42 +0,0 @@
import { z } from 'zod';
import { type OutputSchema } from 'src/modules/workflow/workflow-builder/workflow-schema/types/output-schema.type';
export const convertOutputSchemaToZod = (
schema: OutputSchema,
): z.ZodObject<Record<string, z.ZodTypeAny>> => {
const shape: Record<string, z.ZodTypeAny> = {};
for (const [fieldName, field] of Object.entries(schema)) {
if (field.isLeaf) {
let fieldSchema: z.ZodTypeAny;
switch (field.type) {
case 'TEXT':
fieldSchema = z.string();
break;
case 'NUMBER':
fieldSchema = z.number();
break;
case 'BOOLEAN':
fieldSchema = z.boolean();
break;
case 'DATE':
fieldSchema = z.string().describe('Date-time string');
break;
default:
throw new Error(
`Unsupported field type for AI agent output: ${field.type}`,
);
}
if (field.description) {
fieldSchema = fieldSchema.describe(field.description);
}
shape[fieldName] = fieldSchema;
}
}
return z.object(shape);
};