feat: add configurable response format for AI agents (text/JSON) (#15953)
## Summary
This PR adds configurable response format support for AI agents,
allowing them to return either plain text or structured JSON data based
on a defined schema.
## Key Features
### 1. Agent Response Format Configuration
- Added `AgentResponseFormat` type supporting:
- `text`: Returns plain text responses (default)
- `json`: Returns structured JSON based on defined schema
- New `AgentResponseSchema` type moved to `twenty-shared/ai` for sharing
between frontend/backend
### 2. Settings UI
- New `SettingsAgentResponseFormat` component for configuring response
format
- Visual schema builder for defining JSON output structure
- Real-time validation and preview
- Integrated into agent settings tab
### 3. Workflow Integration
- AI Agent workflow action automatically uses agent's configured
response format
- Output schema dynamically generated from agent's response format
- Workflow variable picker shows structured fields for JSON responses
- Backward compatible with existing text-only agents
### 4. Backend Implementation
- Added `convertAgentSchemaToZod` utility to validate JSON responses
- Agent executor service handles both text and JSON generation
- Automatic agent creation/cloning when adding AI agent steps to
workflows
- Unique agent naming with conflict resolution
### 5. Database Migration
- Migration `1763622159656-update-agent-response-format.ts`
- Sets default `responseFormat` to `{"type":"text"}` for existing agents
- Updated all standard agents with proper response format
## Changes by Module
### Frontend (`twenty-front`)
- 🆕 `AgentResponseFormat` type
- 🆕 `SettingsAgentResponseFormat` component
- ✏️ Updated `WorkflowEditActionAiAgent` to support response format
configuration
- 🗑️ Removed deprecated `useAiAgentOutputSchema` hook and
`AiAgentOutputSchema` type
### Backend (`twenty-server`)
- 🆕 `AgentResponseFormat` type in agent entity
- 🆕 `convertAgentSchemaToZod` utility for schema validation
- ✏️ Updated `AiAgentExecutorService` to handle both text and JSON
generation
- ✏️ Updated `WorkflowSchemaWorkspaceService` to generate output schema
from agent config
- ✏️ Enhanced `WorkflowVersionStepOperationsWorkspaceService` with agent
creation/cloning
- 🆕 Agent naming constants for conflict resolution
### Shared (`twenty-shared`)
- 🆕 `AgentResponseSchema` type
- 🆕 `ModelConfiguration` type moved to shared package
- Updated exports in `ai/index.ts`
## Code Quality
- Removed useless comments following code style guidelines
- All linter checks passed
- Type-safe implementation with proper TypeScript types
## Testing
- ✅ Database migration tested
- ✅ Agent creation/cloning in workflows verified
- ✅ Response format switching (text ↔ JSON) validated
- ✅ Backward compatibility with existing agents confirmed
## Migration Notes
- Existing agents will have `responseFormat: {type: 'text'}` set
automatically
- No breaking changes - all existing functionality preserved
- Agents can be updated to use JSON format through settings UI
This commit is contained in:
Félix Malfait
+152
-71
@@ -1,28 +1,30 @@
|
||||
import { useAiAgentOutputSchema } from '@/ai/hooks/useAiAgentOutputSchema';
|
||||
import { useAiModelOptions } from '@/ai/hooks/useAiModelOptions';
|
||||
import { SidePanelHeader } from '@/command-menu/components/SidePanelHeader';
|
||||
import { FormTextFieldInput } from '@/object-record/record-field/ui/form-types/components/FormTextFieldInput';
|
||||
import { Select } from '@/ui/input/components/Select';
|
||||
import { useFlowOrThrow } from '@/workflow/hooks/useFlowOrThrow';
|
||||
import { type WorkflowAiAgentAction } from '@/workflow/types/Workflow';
|
||||
import { WorkflowStepBody } from '@/workflow/workflow-steps/components/WorkflowStepBody';
|
||||
import { WorkflowStepFooter } from '@/workflow/workflow-steps/components/WorkflowStepFooter';
|
||||
import { useUpdateWorkflowVersionStep } from '@/workflow/workflow-steps/hooks/useUpdateWorkflowVersionStep';
|
||||
import { AI_AGENT_ACTION } from '@/workflow/workflow-steps/workflow-actions/constants/actions/AiAgentAction';
|
||||
import { useWorkflowActionHeader } from '@/workflow/workflow-steps/workflow-actions/hooks/useWorkflowActionHeader';
|
||||
import { WorkflowVariablePicker } from '@/workflow/workflow-variables/components/WorkflowVariablePicker';
|
||||
import { type AiAgentOutputSchema } from '@/workflow/workflow-variables/types/AiAgentOutputSchema';
|
||||
import styled from '@emotion/styled';
|
||||
import { t } from '@lingui/core/macro';
|
||||
import {
|
||||
type AgentResponseSchema,
|
||||
type ModelConfiguration,
|
||||
} from 'twenty-shared/ai';
|
||||
import { isDefined } from 'twenty-shared/utils';
|
||||
import { useIcons } from 'twenty-ui/display';
|
||||
import { type SelectOption } from 'twenty-ui/input';
|
||||
import { useFindManyAgentsQuery } from '~/generated-metadata/graphql';
|
||||
import { useDebouncedCallback } from 'use-debounce';
|
||||
import {
|
||||
useFindOneAgentQuery,
|
||||
useUpdateOneAgentMutation,
|
||||
} from '~/generated-metadata/graphql';
|
||||
import { RightDrawerSkeletonLoader } from '~/loading/components/RightDrawerSkeletonLoader';
|
||||
import { WorkflowOutputSchemaBuilder } from './WorkflowOutputSchemaBuilder';
|
||||
|
||||
const StyledErrorMessage = styled.div`
|
||||
color: ${({ theme }) => theme.font.color.danger};
|
||||
font-size: ${({ theme }) => theme.font.size.sm};
|
||||
font-weight: ${({ theme }) => theme.font.weight.regular};
|
||||
margin-top: ${({ theme }) => theme.spacing(1)};
|
||||
`;
|
||||
import { SettingsAgentModelCapabilities } from '~/pages/settings/ai/components/SettingsAgentModelCapabilities';
|
||||
import { SettingsAgentResponseFormat } from '~/pages/settings/ai/components/SettingsAgentResponseFormat';
|
||||
|
||||
type WorkflowEditActionAiAgentProps = {
|
||||
action: WorkflowAiAgentAction;
|
||||
@@ -45,62 +47,136 @@ export const WorkflowEditActionAiAgent = ({
|
||||
defaultTitle: AI_AGENT_ACTION.defaultLabel,
|
||||
});
|
||||
|
||||
const { handleOutputSchemaChange, outputFields } = useAiAgentOutputSchema(
|
||||
action.settings.outputSchema as AiAgentOutputSchema,
|
||||
actionOptions.readonly === true ? undefined : actionOptions.onActionUpdate,
|
||||
action,
|
||||
actionOptions.readonly,
|
||||
);
|
||||
const agentId = action.settings.input.agentId;
|
||||
const { data: agentData, loading: agentLoading } = useFindOneAgentQuery({
|
||||
variables: { id: agentId || '' },
|
||||
skip: !agentId,
|
||||
});
|
||||
const [updateAgent] = useUpdateOneAgentMutation();
|
||||
const aiModelOptions = useAiModelOptions();
|
||||
const { updateWorkflowVersionStep } = useUpdateWorkflowVersionStep();
|
||||
const flow = useFlowOrThrow();
|
||||
|
||||
const { data: agentsData, loading: agentsLoading } = useFindManyAgentsQuery();
|
||||
const agent = agentData?.findOneAgent;
|
||||
|
||||
const agentOptions = (agentsData?.findManyAgents || []).reduce<
|
||||
SelectOption<string>[]
|
||||
>(
|
||||
(acc, agent) => {
|
||||
acc.push({
|
||||
label: agent.label,
|
||||
value: agent.id,
|
||||
Icon: agent.icon ? getIcon(agent.icon) : undefined,
|
||||
const handleAgentPromptChange = useDebouncedCallback(
|
||||
async (newPrompt: string) => {
|
||||
if (actionOptions.readonly === true || !isDefined(agent)) {
|
||||
return;
|
||||
}
|
||||
|
||||
await updateAgent({
|
||||
variables: {
|
||||
input: {
|
||||
id: agent.id,
|
||||
prompt: newPrompt,
|
||||
},
|
||||
},
|
||||
refetchQueries: ['FindOneAgent'],
|
||||
});
|
||||
return acc;
|
||||
},
|
||||
[
|
||||
{
|
||||
label: t`No Agent`,
|
||||
value: '',
|
||||
},
|
||||
],
|
||||
500,
|
||||
);
|
||||
|
||||
const noAgentsAvailable = agentOptions.length === 0;
|
||||
|
||||
const handleFieldChange = (field: 'agentId' | 'prompt', value: string) => {
|
||||
if (actionOptions.readonly === true) {
|
||||
const handleAgentModelChange = async (modelId: string) => {
|
||||
if (actionOptions.readonly === true || !isDefined(agent)) {
|
||||
return;
|
||||
}
|
||||
actionOptions.onActionUpdate?.({
|
||||
...action,
|
||||
settings: {
|
||||
...action.settings,
|
||||
|
||||
await updateAgent({
|
||||
variables: {
|
||||
input: {
|
||||
...action.settings.input,
|
||||
[field]: value,
|
||||
id: agent.id,
|
||||
modelId,
|
||||
},
|
||||
},
|
||||
refetchQueries: ['FindOneAgent'],
|
||||
});
|
||||
};
|
||||
|
||||
return agentsLoading ? (
|
||||
const handleModelConfigurationChange = async (
|
||||
configuration: ModelConfiguration,
|
||||
) => {
|
||||
if (actionOptions.readonly === true || !isDefined(agent)) {
|
||||
return;
|
||||
}
|
||||
|
||||
await updateAgent({
|
||||
variables: {
|
||||
input: {
|
||||
id: agent.id,
|
||||
modelConfiguration: configuration,
|
||||
},
|
||||
},
|
||||
refetchQueries: ['FindOneAgent'],
|
||||
});
|
||||
};
|
||||
|
||||
const updateAgentResponseFormat = async (format: {
|
||||
type: 'text' | 'json';
|
||||
schema?: AgentResponseSchema;
|
||||
}) => {
|
||||
if (actionOptions.readonly === true || !isDefined(agent)) {
|
||||
return;
|
||||
}
|
||||
|
||||
await updateAgent({
|
||||
variables: {
|
||||
input: {
|
||||
id: agent.id,
|
||||
responseFormat: format,
|
||||
},
|
||||
},
|
||||
refetchQueries: ['FindOneAgent'],
|
||||
});
|
||||
|
||||
await updateWorkflowVersionStep({
|
||||
workflowVersionId: flow.workflowVersionId,
|
||||
step: action,
|
||||
});
|
||||
};
|
||||
|
||||
const debouncedUpdateAgentResponseFormat = useDebouncedCallback(
|
||||
updateAgentResponseFormat,
|
||||
300,
|
||||
);
|
||||
|
||||
const handleAgentResponseFormatChange = async (format: {
|
||||
type: 'text' | 'json';
|
||||
schema?: AgentResponseSchema;
|
||||
}) => {
|
||||
if (format.type !== agent?.responseFormat?.type) {
|
||||
debouncedUpdateAgentResponseFormat.cancel();
|
||||
void updateAgentResponseFormat(format);
|
||||
} else {
|
||||
void debouncedUpdateAgentResponseFormat(format);
|
||||
}
|
||||
};
|
||||
|
||||
return agentLoading ? (
|
||||
<RightDrawerSkeletonLoader />
|
||||
) : (
|
||||
<>
|
||||
<SidePanelHeader
|
||||
onTitleChange={(newName: string) => {
|
||||
onTitleChange={async (newName: string) => {
|
||||
if (actionOptions.readonly === true) {
|
||||
return;
|
||||
}
|
||||
|
||||
actionOptions.onActionUpdate?.({ ...action, name: newName });
|
||||
|
||||
// Also update agent label
|
||||
if (isDefined(agent)) {
|
||||
await updateAgent({
|
||||
variables: {
|
||||
input: {
|
||||
id: agent.id,
|
||||
label: newName,
|
||||
},
|
||||
},
|
||||
refetchQueries: ['FindOneAgent'],
|
||||
});
|
||||
}
|
||||
}}
|
||||
Icon={getIcon(headerIcon)}
|
||||
iconColor={headerIconColor}
|
||||
@@ -110,38 +186,43 @@ export const WorkflowEditActionAiAgent = ({
|
||||
iconTooltip={AI_AGENT_ACTION.defaultLabel}
|
||||
/>
|
||||
<WorkflowStepBody>
|
||||
<div>
|
||||
<Select
|
||||
dropdownId="select-agent"
|
||||
label={t`Select Agent`}
|
||||
options={agentOptions}
|
||||
value={action.settings.input.agentId || ''}
|
||||
onChange={(value) => handleFieldChange('agentId', value)}
|
||||
disabled={actionOptions.readonly || noAgentsAvailable}
|
||||
/>
|
||||
|
||||
{noAgentsAvailable && (
|
||||
<StyledErrorMessage>
|
||||
{t`Please create agents in the AI settings to use in workflows.`}
|
||||
</StyledErrorMessage>
|
||||
)}
|
||||
</div>
|
||||
|
||||
<FormTextFieldInput
|
||||
multiline
|
||||
VariablePicker={WorkflowVariablePicker}
|
||||
label={t`Instructions for AI`}
|
||||
placeholder={t`Describe what you want the AI to do...`}
|
||||
defaultValue={action.settings.input.prompt}
|
||||
onChange={(value) => handleFieldChange('prompt', value)}
|
||||
defaultValue={agent?.prompt || ''}
|
||||
onChange={handleAgentPromptChange}
|
||||
readonly={actionOptions.readonly}
|
||||
/>
|
||||
|
||||
<WorkflowOutputSchemaBuilder
|
||||
fields={outputFields}
|
||||
onChange={handleOutputSchemaChange}
|
||||
readonly={actionOptions.readonly}
|
||||
/>
|
||||
{isDefined(agent) && (
|
||||
<>
|
||||
<Select
|
||||
dropdownId="select-agent-model"
|
||||
label={t`AI Model`}
|
||||
options={aiModelOptions}
|
||||
value={agent.modelId}
|
||||
onChange={handleAgentModelChange}
|
||||
disabled={actionOptions.readonly}
|
||||
/>
|
||||
|
||||
<SettingsAgentModelCapabilities
|
||||
selectedModelId={agent.modelId}
|
||||
modelConfiguration={agent.modelConfiguration || {}}
|
||||
onConfigurationChange={handleModelConfigurationChange}
|
||||
disabled={actionOptions.readonly}
|
||||
/>
|
||||
|
||||
<SettingsAgentResponseFormat
|
||||
responseFormat={
|
||||
agent.responseFormat || { type: 'text', schema: {} }
|
||||
}
|
||||
onResponseFormatChange={handleAgentResponseFormatChange}
|
||||
disabled={actionOptions.readonly}
|
||||
/>
|
||||
</>
|
||||
)}
|
||||
</WorkflowStepBody>
|
||||
{!actionOptions.readonly && <WorkflowStepFooter stepId={action.id} />}
|
||||
</>
|
||||
|
||||
+3
-3
@@ -1,11 +1,11 @@
|
||||
import { OUTPUT_FIELD_TYPE_OPTIONS } from '@/ai/constants/OutputFieldTypeOptions';
|
||||
import { Select } from '@/ui/input/components/Select';
|
||||
import { type InputSchemaPropertyType } from '@/workflow/types/InputSchema';
|
||||
import { t } from '@lingui/core/macro';
|
||||
import { type AgentResponseFieldType } from 'twenty-shared/ai';
|
||||
|
||||
type WorkflowOutputFieldTypeSelectorProps = {
|
||||
value?: InputSchemaPropertyType;
|
||||
onChange: (value: InputSchemaPropertyType) => void;
|
||||
value?: AgentResponseFieldType;
|
||||
onChange: (value: AgentResponseFieldType) => void;
|
||||
disabled?: boolean;
|
||||
dropdownId: string;
|
||||
};
|
||||
|
||||
+1
-2
@@ -3,7 +3,6 @@ import { FormTextFieldInput } from '@/object-record/record-field/ui/form-types/c
|
||||
|
||||
import { type OutputSchemaField } from '@/ai/constants/OutputFieldTypeOptions';
|
||||
import { InputLabel } from '@/ui/input/components/InputLabel';
|
||||
import { type InputSchemaPropertyType } from '@/workflow/types/InputSchema';
|
||||
import { useTheme } from '@emotion/react';
|
||||
import styled from '@emotion/styled';
|
||||
import { t } from '@lingui/core/macro';
|
||||
@@ -114,7 +113,7 @@ export const WorkflowOutputSchemaBuilder = ({
|
||||
id: v4(),
|
||||
name: '',
|
||||
description: '',
|
||||
type: 'TEXT' as InputSchemaPropertyType,
|
||||
type: 'string',
|
||||
};
|
||||
onChange([...fields, newField]);
|
||||
};
|
||||
|
||||
-17
@@ -1,17 +0,0 @@
|
||||
import { type InputSchemaPropertyType } from '@/workflow/types/InputSchema';
|
||||
|
||||
export type AiAgentLeaf = {
|
||||
isLeaf: true;
|
||||
type: InputSchemaPropertyType | undefined;
|
||||
label: string;
|
||||
value: any;
|
||||
};
|
||||
|
||||
export type AiAgentNode = {
|
||||
isLeaf: false;
|
||||
type: 'object' | 'unknown';
|
||||
label: string;
|
||||
value: AiAgentOutputSchema;
|
||||
};
|
||||
|
||||
export type AiAgentOutputSchema = Record<string, AiAgentLeaf | AiAgentNode>;
|
||||
Reference in New Issue
Block a user