From 0064ff67410982fa0edd0d0b3330519a2c113912 Mon Sep 17 00:00:00 2001 From: Charles Bochet Date: Fri, 19 Jun 2026 13:50:44 +0200 Subject: [PATCH] fix(ai): validate AI agent output field names against schema-key constraint (#21834) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Problem On a self-hosted instance, an AI Agent workflow action fails at run time with an opaque model error: ``` The model returned the following errors: tools.0.custom.input_schema.properties: Property keys should match pattern '^[a-zA-Z0-9_.-]{1,64}$' ``` This is Anthropic's validation on tool `input_schema` **property keys**. An AI Agent's structured **Output** fields are turned into a JSON schema and passed to the model as a tool; each output **variable name** becomes a property key. Anthropic rejects any key that does not match `^[a-zA-Z0-9_.-]{1,64}$` — most commonly a name containing a **space** (e.g. `meetings brief`), but also names over 64 characters or with other symbols. Until now nothing validated this: `fieldsToSchema` writes `properties[field.name]` verbatim, so a bad name only failed once the workflow executed, with an error that gives the user no idea what to fix. It doesn't reproduce on every instance — it depends purely on how the workflow's output variables happen to be named. ## Fix Introduce a single shared check, `isValidAgentResponseSchemaPropertyKey`, and enforce it in two places: - **Backend** — `validateAgentResponseFormat` now rejects invalid output field names at agent **save time** with a clear `userFriendlyMessage`, instead of letting the broken schema reach the model. This also gates agents created via the API and re-saves of existing bad data. - **Frontend** — the output schema builder shows an inline error on the Variable Name field as soon as an invalid name is entered. ## Tests - Unit test for the shared validity check (valid + invalid cases: spaces, leading space, empty, > 64 chars, symbols, unicode). - Unit test for `validateAgentResponseFormat` covering text/json formats, valid names, a space in a name, an over-length name, and reporting multiple invalid names at once. ## Notes for the reporter The immediate unblock for an affected workflow is to rename the output variable to remove the space (e.g. `meetings brief` → `meetings_brief`) and retry the run. With this change the bad name is caught up front with an explanation rather than failing mid-run. Review in cubic --- .../WorkflowOutputSchemaBuilder.tsx | 14 +++ ...alidate-agent-response-format.util.spec.ts | 89 +++++++++++++++++++ .../validate-agent-response-format.util.ts | 19 +++- packages/twenty-shared/src/ai/index.ts | 1 + ...-response-schema-property-key.util.spec.ts | 25 ++++++ ...agent-response-schema-property-key.util.ts | 5 ++ 6 files changed, 152 insertions(+), 1 deletion(-) create mode 100644 packages/twenty-server/src/engine/workspace-manager/workspace-migration/workspace-migration-builder/validators/utils/__tests__/validate-agent-response-format.util.spec.ts create mode 100644 packages/twenty-shared/src/ai/utils/__tests__/is-valid-agent-response-schema-property-key.util.spec.ts create mode 100644 packages/twenty-shared/src/ai/utils/is-valid-agent-response-schema-property-key.util.ts diff --git a/packages/twenty-front/src/modules/workflow/workflow-steps/workflow-actions/ai-agent-action/components/WorkflowOutputSchemaBuilder.tsx b/packages/twenty-front/src/modules/workflow/workflow-steps/workflow-actions/ai-agent-action/components/WorkflowOutputSchemaBuilder.tsx index b1655e028f..fffca3c9c1 100644 --- a/packages/twenty-front/src/modules/workflow/workflow-steps/workflow-actions/ai-agent-action/components/WorkflowOutputSchemaBuilder.tsx +++ b/packages/twenty-front/src/modules/workflow/workflow-steps/workflow-actions/ai-agent-action/components/WorkflowOutputSchemaBuilder.tsx @@ -7,6 +7,8 @@ import { InputLabel } from '@/ui/input/components/InputLabel'; import { styled } from '@linaria/react'; import { t } from '@lingui/core/macro'; import { useContext, useState } from 'react'; +import { isNonEmptyString } from '@sniptt/guards'; +import { isValidAgentResponseSchemaPropertyKey } from 'twenty-shared/ai'; import { IconChevronDown, IconPlus, IconVariable, IconX } from 'twenty-ui/icon'; import { AnimatedLightIconButton, LightIconButton } from 'twenty-ui/input'; import { AnimatedExpandableContainer } from 'twenty-ui/layout'; @@ -148,6 +150,17 @@ export const WorkflowOutputSchemaBuilder = ({ ); }; + const getVariableNameError = (name: string): string | undefined => { + if ( + !isNonEmptyString(name) || + isValidAgentResponseSchemaPropertyKey(name) + ) { + return undefined; + } + + return t`Use only letters, numbers, underscores, dots or hyphens (max 64 characters).`; + }; + return ( {t`Output`} @@ -205,6 +218,7 @@ export const WorkflowOutputSchemaBuilder = ({ label={t`Variable Name`} placeholder={t`e.g., summary, status, count`} defaultValue={field.name} + error={getVariableNameError(field.name)} onChange={(value) => updateField(field.id, { name: value.trim() }) } diff --git a/packages/twenty-server/src/engine/workspace-manager/workspace-migration/workspace-migration-builder/validators/utils/__tests__/validate-agent-response-format.util.spec.ts b/packages/twenty-server/src/engine/workspace-manager/workspace-migration/workspace-migration-builder/validators/utils/__tests__/validate-agent-response-format.util.spec.ts new file mode 100644 index 0000000000..e41bfc7b42 --- /dev/null +++ b/packages/twenty-server/src/engine/workspace-manager/workspace-migration/workspace-migration-builder/validators/utils/__tests__/validate-agent-response-format.util.spec.ts @@ -0,0 +1,89 @@ +import { type AgentResponseFormat } from 'src/engine/metadata-modules/ai/ai-agent/types/agent-response-format.type'; +import { AiExceptionCode } from 'src/engine/metadata-modules/ai/ai.exception'; +import { validateAgentResponseFormat } from 'src/engine/workspace-manager/workspace-migration/workspace-migration-builder/validators/utils/validate-agent-response-format.util'; + +type JsonResponseFormat = Extract; + +const buildJsonFormat = ( + properties: JsonResponseFormat['schema']['properties'], +): AgentResponseFormat => ({ + type: 'json', + schema: { + type: 'object', + properties, + required: Object.keys(properties), + additionalProperties: false, + }, +}); + +describe('validateAgentResponseFormat', () => { + it('should return no error for a text response format', () => { + const errors = validateAgentResponseFormat({ + responseFormat: { type: 'text' }, + }); + + expect(errors).toEqual([]); + }); + + it('should return no error when all property names are valid', () => { + const errors = validateAgentResponseFormat({ + responseFormat: buildJsonFormat({ + meetings_brief: { type: 'string' }, + count: { type: 'number' }, + }), + }); + + expect(errors).toEqual([]); + }); + + it('should return an error when a property name contains a space', () => { + const errors = validateAgentResponseFormat({ + responseFormat: buildJsonFormat({ + 'meetings brief': { type: 'string' }, + }), + }); + + expect(errors).toHaveLength(1); + expect(errors[0].code).toBe(AiExceptionCode.INVALID_AGENT_INPUT); + expect(errors[0].message).toContain('meetings brief'); + }); + + it('should return an error when a property name exceeds 64 characters', () => { + const errors = validateAgentResponseFormat({ + responseFormat: buildJsonFormat({ + ['a'.repeat(65)]: { type: 'string' }, + }), + }); + + expect(errors).toHaveLength(1); + expect(errors[0].code).toBe(AiExceptionCode.INVALID_AGENT_INPUT); + }); + + it('should not throw when a json schema is missing its properties', () => { + // Legacy or API-provided data can omit properties despite the type + const malformedFormat = { + type: 'json', + schema: { type: 'object' }, + } as AgentResponseFormat; + + expect(() => + validateAgentResponseFormat({ responseFormat: malformedFormat }), + ).not.toThrow(); + expect( + validateAgentResponseFormat({ responseFormat: malformedFormat }), + ).toEqual([]); + }); + + it('should report every invalid property name at once', () => { + const errors = validateAgentResponseFormat({ + responseFormat: buildJsonFormat({ + 'meetings brief': { type: 'string' }, + 'sales rep': { type: 'string' }, + }), + }); + + expect(errors).toHaveLength(1); + expect(errors[0].message).toContain('meetings brief'); + expect(errors[0].message).toContain('sales rep'); + }); +}); diff --git a/packages/twenty-server/src/engine/workspace-manager/workspace-migration/workspace-migration-builder/validators/utils/validate-agent-response-format.util.ts b/packages/twenty-server/src/engine/workspace-manager/workspace-migration/workspace-migration-builder/validators/utils/validate-agent-response-format.util.ts index f9e73a6f2b..b9235bdb4b 100644 --- a/packages/twenty-server/src/engine/workspace-manager/workspace-migration/workspace-migration-builder/validators/utils/validate-agent-response-format.util.ts +++ b/packages/twenty-server/src/engine/workspace-manager/workspace-migration/workspace-migration-builder/validators/utils/validate-agent-response-format.util.ts @@ -1,5 +1,6 @@ import { msg, t } from '@lingui/core/macro'; -import { isDefined } from 'twenty-shared/utils'; +import { isValidAgentResponseSchemaPropertyKey } from 'twenty-shared/ai'; +import { isDefined, isNonEmptyArray } from 'twenty-shared/utils'; import { AiExceptionCode } from 'src/engine/metadata-modules/ai/ai.exception'; import { @@ -32,6 +33,22 @@ export const validateAgentResponseFormat = ({ }); } + if (type === 'json' && isDefined(responseFormat.schema)) { + const invalidPropertyNames = Object.keys( + responseFormat.schema.properties ?? {}, + ).filter( + (propertyName) => !isValidAgentResponseSchemaPropertyKey(propertyName), + ); + + if (isNonEmptyArray(invalidPropertyNames)) { + errors.push({ + code: AiExceptionCode.INVALID_AGENT_INPUT, + message: t`Output field names must use only letters, numbers, underscores, dots or hyphens and be at most 64 characters: ${invalidPropertyNames.join(', ')}`, + userFriendlyMessage: msg`Output field names can only contain letters, numbers, underscores, dots or hyphens (max 64 characters).`, + }); + } + } + if ( type === 'text' && isDefined((responseFormat as unknown as AgentJsonResponseFormat).schema) diff --git a/packages/twenty-shared/src/ai/index.ts b/packages/twenty-shared/src/ai/index.ts index 8a7b6ea4d0..f61c50f153 100644 --- a/packages/twenty-shared/src/ai/index.ts +++ b/packages/twenty-shared/src/ai/index.ts @@ -45,3 +45,4 @@ export type { NavigateAppToolOutput } from './types/NavigateAppToolOutput'; export { inferAiSdkPackage } from './utils/infer-ai-sdk-package.util'; export { isAiSdkPackage } from './utils/is-ai-sdk-package.util'; export { isDataResidency } from './utils/is-data-residency.util'; +export { isValidAgentResponseSchemaPropertyKey } from './utils/is-valid-agent-response-schema-property-key.util'; diff --git a/packages/twenty-shared/src/ai/utils/__tests__/is-valid-agent-response-schema-property-key.util.spec.ts b/packages/twenty-shared/src/ai/utils/__tests__/is-valid-agent-response-schema-property-key.util.spec.ts new file mode 100644 index 0000000000..5b8c63d123 --- /dev/null +++ b/packages/twenty-shared/src/ai/utils/__tests__/is-valid-agent-response-schema-property-key.util.spec.ts @@ -0,0 +1,25 @@ +import { isValidAgentResponseSchemaPropertyKey } from '../is-valid-agent-response-schema-property-key.util'; + +describe('isValidAgentResponseSchemaPropertyKey', () => { + it.each([ + 'summary', + 'status_2', + 'meetings.brief', + 'a-b', + 'A', + 'a'.repeat(64), + ])('should accept "%s"', (propertyKey) => { + expect(isValidAgentResponseSchemaPropertyKey(propertyKey)).toBe(true); + }); + + it.each([ + ['a name with spaces', 'meetings brief'], + ['leading space', ' summary'], + ['empty string', ''], + ['over 64 characters', 'a'.repeat(65)], + ['unsupported symbol', 'meetings@brief'], + ['unicode', 'résumé'], + ])('should reject %s', (_label, propertyKey) => { + expect(isValidAgentResponseSchemaPropertyKey(propertyKey)).toBe(false); + }); +}); diff --git a/packages/twenty-shared/src/ai/utils/is-valid-agent-response-schema-property-key.util.ts b/packages/twenty-shared/src/ai/utils/is-valid-agent-response-schema-property-key.util.ts new file mode 100644 index 0000000000..517be53b82 --- /dev/null +++ b/packages/twenty-shared/src/ai/utils/is-valid-agent-response-schema-property-key.util.ts @@ -0,0 +1,5 @@ +const AGENT_RESPONSE_SCHEMA_PROPERTY_KEY_PATTERN = /^[a-zA-Z0-9_.-]{1,64}$/; + +export const isValidAgentResponseSchemaPropertyKey = ( + propertyKey: string, +): boolean => AGENT_RESPONSE_SCHEMA_PROPERTY_KEY_PATTERN.test(propertyKey);