feat: improve AI chat - system prompt, tool output, context window display (#17769)

⚠️ **AI-generated PR — not ready for review** ⚠️

cc @FelixMalfait

---

## Changes

### System prompt improvements
- Explicit skill-before-tools workflow to prevent the model from calling
tools without loading the matching skill first
- Data efficiency guidance (default small limits, use filters)
- Pluralized `load_skill` → `load_skills` for consistency with
`load_tools`

### Token usage reduction
- Output serialization layer: strips null/undefined/empty values from
tool results
- Lowered default `find_*` limit from 100 → 10, max from 1000 → 100

### System object tool generation
- System objects (calendar events, messages, etc.) now generate AI tools
- Only workflow-related and favorite-related objects are excluded

### Context window display fix
- **Bug**: UI compared cumulative tokens (sum of all turns) against
single-request context window → showed 100% after a few turns
- **Fix**: Track `conversationSize` (last step's `inputTokens`) which
represents the actual conversation history size sent to the model
- New `conversationSize` column on thread entity with migration

### Workspace AI instructions
- Support for custom workspace-level AI instructions

---------

Co-authored-by: claude[bot] <41898282+claude[bot]@users.noreply.github.com>
This commit is contained in:
Félix Malfait
2026-02-09 14:26:02 +01:00
committed by GitHub
parent 6c7c389785
commit 3216b634a3
105 changed files with 3245 additions and 1714 deletions
@@ -12,11 +12,19 @@ import { UserRoleService } from 'src/engine/metadata-modules/user-role/user-role
import { GlobalWorkspaceOrmManager } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager';
import { buildSystemAuthContext } from 'src/engine/twenty-orm/utils/build-system-auth-context.util';
export type UserContext = {
firstName: string;
lastName: string;
locale: string;
timezone: string | null;
};
export type AgentActorContext = {
actorContext: ActorMetadata;
roleId: string;
userId: string;
userWorkspaceId: string;
userContext: UserContext;
};
@Injectable()
@@ -87,11 +95,19 @@ export class AgentActorContextService {
workspaceMemberId: workspaceMember.id,
});
const userContext: UserContext = {
firstName: workspaceMember.name?.firstName ?? '',
lastName: workspaceMember.name?.lastName ?? '',
locale: userWorkspace.locale,
timezone: workspaceMember.timeZone ?? null,
};
return {
actorContext,
roleId,
userId: userWorkspace.userId,
userWorkspaceId,
userContext,
};
}
}
@@ -0,0 +1,14 @@
import { STANDARD_OBJECTS } from 'twenty-shared/metadata';
const FAVORITE_STANDARD_OBJECT_UNIVERSAL_IDENTIFIERS = [
STANDARD_OBJECTS.favorite.universalIdentifier,
STANDARD_OBJECTS.favoriteFolder.universalIdentifier,
] as const;
export const isFavoriteRelatedObject = (objectMetadata: {
universalIdentifier: string;
}): boolean => {
return FAVORITE_STANDARD_OBJECT_UNIVERSAL_IDENTIFIERS.includes(
objectMetadata.universalIdentifier as (typeof FAVORITE_STANDARD_OBJECT_UNIVERSAL_IDENTIFIERS)[number],
);
};
@@ -29,6 +29,7 @@ import { AgentChatStreamingService } from './services/agent-chat-streaming.servi
import { AgentChatService } from './services/agent-chat.service';
import { AgentTitleGenerationService } from './services/agent-title-generation.service';
import { ChatExecutionService } from './services/chat-execution.service';
import { SystemPromptBuilderService } from './services/system-prompt-builder.service';
@Module({
imports: [
@@ -63,6 +64,7 @@ import { ChatExecutionService } from './services/chat-execution.service';
AgentChatStreamingService,
AgentTitleGenerationService,
ChatExecutionService,
SystemPromptBuilderService,
],
exports: [
AgentChatService,
@@ -1,47 +1,54 @@
// System prompts for AI Chat (user-facing conversational interface)
export const CHAT_SYSTEM_PROMPTS = {
// Core chat behavior and tool strategy
BASE: `You are a helpful AI assistant integrated into Twenty CRM.
BASE: `You are a helpful AI assistant integrated into Twenty, a CRM (similar to Salesforce).
Tool usage strategy:
- Chain multiple tools to solve complex tasks
- If a tool fails, try alternative approaches
- Use results from one tool to inform the next
- Don't give up after first failure - be persistent
- Validate assumptions before making changes
## Plan → Skill → Learn → Execute
For ANY non-trivial task, follow this order:
1. **Plan**: Identify what the user needs. Determine which domain is involved (workflows, dashboards, metadata, data, documents, etc.).
2. **Load the relevant skill FIRST**: Call \`load_skills\` to get detailed instructions, correct schemas, and parameter formats BEFORE doing anything else. Skills contain critical knowledge you don't have built-in — skipping this step leads to incorrect parameters and failed tool calls.
3. **Learn the required tools**: Call \`learn_tools\` to discover tool schemas and descriptions before using them.
4. **Execute**: Call \`execute_tool\` to run the tools following the instructions from the skill.
⚠️ NEVER call a specialized tool (workflow, dashboard, metadata, etc.) without loading its matching skill first. The Available Skills section below lists all skills — look for the one that matches the user's task domain and load it.
Examples:
- User asks to create a workflow → \`load_skills(["workflow-building"])\` then learn and execute workflow tools
- User asks to build a dashboard → \`load_skills(["dashboard-building"])\` then learn and execute dashboard tools
- User asks to export data to Excel → \`load_skills(["xlsx", "code-interpreter"])\` then \`learn_tools({toolNames: ["code_interpreter"]})\` then \`execute_tool({toolName: "code_interpreter", arguments: {...}})\`
For simple CRUD operations (find/create/update/delete a record), you do NOT need a skill — but you still MUST call \`learn_tools\` first to learn the tool schema, then \`execute_tool\` to run it.
## Skills vs Tools
- **SKILLS** = documentation/instructions (loaded via \`load_skills\`). They teach you HOW to do something — correct schemas, parameters, and patterns. They do NOT give you execution ability.
- **TOOLS** = execution capabilities via \`execute_tool\`. They let you DO something. Use \`learn_tools\` to discover the correct parameters first.
- You need BOTH: skill for knowledge, \`execute_tool\` for action.
## Database vs HTTP Tools
Database vs HTTP tools:
- Use database tools (find_*, create_*, update_*, delete_*) for ALL Twenty CRM data operations
- NEVER guess or construct API URLs - always use the appropriate database tool
- NEVER guess or construct API URLs always use the appropriate database tool
- The \`http_request\` tool is ONLY for external third-party APIs (not for Twenty's own data)
- If you need to look up a record, load and use the corresponding find_one_* or find_many_* tool
- If you need to look up a record, learn and execute the corresponding find_one_* or find_many_* tool
Error recovery:
- Analyze error messages to understand what went wrong
- Adjust parameters or try different tools
- Only give up after exhausting reasonable alternatives
## Data Efficiency
Permissions:
- Only perform actions your role allows
- Explain limitations if you lack permissions
- Use small limits (5-10 records) for initial exploration. Only increase if the user explicitly needs more.
- Always apply filters to narrow results — don't fetch all records of a type.
- Fetch one type of data at a time and check if you have what you need before fetching more.
- Every record returned consumes context. Fetching too many records at once will cause failures.
Skills vs Tools:
- SKILLS = documentation/instructions (loaded via \`load_skill\`). They teach you HOW to do something.
- TOOLS = execution capabilities (loaded via \`load_tools\`). They let you DO something.
- Skills don't give you abilities - they give you knowledge. You still need the tool to act.
## Tool Strategy
Python Code Execution:
- To run Python code, you need TWO things:
1. Load the skill for instructions: \`load_skill(["code-interpreter"])\`
2. Load the tool for execution: \`load_tools(["code_interpreter"])\`
- Then call \`code_interpreter\` with your Python code
- The Python environment includes a \`twenty\` helper to call any Twenty tool directly from code
Document Processing (Excel, PDF, Word, PowerPoint):
- For document tasks, load both the skill AND the code_interpreter tool:
1. \`load_skill(["xlsx"])\` or \`load_skill(["pdf"])\` etc. - gets you detailed instructions
2. \`load_tools(["code_interpreter"])\` - enables code execution
- Then use \`code_interpreter\` to run the Python code described in the skill`,
- Chain multiple tools to solve complex tasks
- Use results from one tool to inform the next
- If a tool fails, analyze the error, adjust parameters, and try again
- Don't give up after first failure — be persistent and try alternative approaches
- Validate assumptions before making changes
`,
// Response formatting and record references
RESPONSE_FORMAT: `
@@ -1,4 +1,4 @@
import { Field, Int, ObjectType } from '@nestjs/graphql';
import { Field, Float, Int, ObjectType } from '@nestjs/graphql';
import { UUIDScalarType } from 'src/engine/api/graphql/workspace-schema-builder/graphql-types/scalars';
@@ -20,9 +20,14 @@ export class AgentChatThreadDTO {
contextWindowTokens: number | null;
@Field(() => Int)
conversationSize: number;
// Credits are converted from internal precision to display precision
// (internal / 1000) at the resolver level
@Field(() => Float)
totalInputCredits: number;
@Field(() => Int)
@Field(() => Float)
totalOutputCredits: number;
@Field()
@@ -0,0 +1,22 @@
import { Field, Int, ObjectType } from '@nestjs/graphql';
@ObjectType('AISystemPromptSection')
export class AISystemPromptSectionDTO {
@Field(() => String)
title: string;
@Field(() => String)
content: string;
@Field(() => Int)
estimatedTokenCount: number;
}
@ObjectType('AISystemPromptPreview')
export class AISystemPromptPreviewDTO {
@Field(() => [AISystemPromptSectionDTO])
sections: AISystemPromptSectionDTO[];
@Field(() => Int)
estimatedTokenCount: number;
}
@@ -42,6 +42,9 @@ export class AgentChatThreadEntity {
@Column({ type: 'int', nullable: true })
contextWindowTokens: number | null;
@Column({ type: 'int', default: 0 })
conversationSize: number;
@Column({ type: 'bigint', default: 0 })
totalInputCredits: number;
@@ -1,11 +1,22 @@
import { UseGuards } from '@nestjs/common';
import { Args, Mutation, Query, Resolver } from '@nestjs/graphql';
import {
Args,
Float,
Mutation,
Parent,
Query,
ResolveField,
Resolver,
} from '@nestjs/graphql';
import { PermissionFlagType } from 'twenty-shared/constants';
import { UUIDScalarType } from 'src/engine/api/graphql/workspace-schema-builder/graphql-types/scalars';
import { toDisplayCredits } from 'src/engine/core-modules/billing/utils/to-display-credits.util';
import { FeatureFlagKey } from 'src/engine/core-modules/feature-flag/enums/feature-flag-key.enum';
import { type WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity';
import { AuthUserWorkspaceId } from 'src/engine/decorators/auth/auth-user-workspace-id.decorator';
import { AuthWorkspace } from 'src/engine/decorators/auth/auth-workspace.decorator';
import {
FeatureFlagGuard,
RequireFeatureFlag,
@@ -14,16 +25,22 @@ import { SettingsPermissionGuard } from 'src/engine/guards/settings-permission.g
import { WorkspaceAuthGuard } from 'src/engine/guards/workspace-auth.guard';
import { AgentMessageDTO } from 'src/engine/metadata-modules/ai/ai-agent-execution/dtos/agent-message.dto';
import { AgentChatThreadDTO } from 'src/engine/metadata-modules/ai/ai-chat/dtos/agent-chat-thread.dto';
import { AISystemPromptPreviewDTO } from 'src/engine/metadata-modules/ai/ai-chat/dtos/ai-system-prompt-preview.dto';
import { type AgentChatThreadEntity } from 'src/engine/metadata-modules/ai/ai-chat/entities/agent-chat-thread.entity';
import { AgentChatService } from 'src/engine/metadata-modules/ai/ai-chat/services/agent-chat.service';
import { SystemPromptBuilderService } from 'src/engine/metadata-modules/ai/ai-chat/services/system-prompt-builder.service';
@UseGuards(
WorkspaceAuthGuard,
FeatureFlagGuard,
SettingsPermissionGuard(PermissionFlagType.AI),
)
@Resolver()
@Resolver(() => AgentChatThreadDTO)
export class AgentChatResolver {
constructor(private readonly agentChatService: AgentChatService) {}
constructor(
private readonly agentChatService: AgentChatService,
private readonly systemPromptBuilderService: SystemPromptBuilderService,
) {}
@Query(() => [AgentChatThreadDTO])
@RequireFeatureFlag(FeatureFlagKey.IS_AI_ENABLED)
@@ -57,4 +74,27 @@ export class AgentChatResolver {
async createChatThread(@AuthUserWorkspaceId() userWorkspaceId: string) {
return this.agentChatService.createThread(userWorkspaceId);
}
@Query(() => AISystemPromptPreviewDTO)
@RequireFeatureFlag(FeatureFlagKey.IS_AI_ENABLED)
async getAISystemPromptPreview(
@AuthWorkspace() workspace: WorkspaceEntity,
@AuthUserWorkspaceId() userWorkspaceId: string,
) {
return this.systemPromptBuilderService.buildPreview(
workspace.id,
userWorkspaceId,
workspace.aiAdditionalInstructions ?? undefined,
);
}
@ResolveField(() => Float)
totalInputCredits(@Parent() thread: AgentChatThreadEntity): number {
return toDisplayCredits(thread.totalInputCredits);
}
@ResolveField(() => Float)
totalOutputCredits(@Parent() thread: AgentChatThreadEntity): number {
return toDisplayCredits(thread.totalOutputCredits);
}
}
@@ -17,6 +17,7 @@ import {
} from 'src/engine/metadata-modules/ai/ai-agent/agent.exception';
import { type BrowsingContextType } from 'src/engine/metadata-modules/ai/ai-agent/types/browsingContext.type';
import { convertCentsToBillingCredits } from 'src/engine/metadata-modules/ai/ai-billing/utils/convert-cents-to-billing-credits.util';
import { toDisplayCredits } from 'src/engine/core-modules/billing/utils/to-display-credits.util';
import { AgentChatThreadEntity } from 'src/engine/metadata-modules/ai/ai-chat/entities/agent-chat-thread.entity';
import { AgentChatService } from './agent-chat.service';
@@ -84,21 +85,14 @@ export class AgentChatStreamingService {
onCodeExecutionUpdate,
});
writer.write({
type: 'data-routing-status' as const,
id: 'execution-status',
data: {
text: 'Processing your request...',
state: 'loading',
},
});
let streamUsage = {
inputTokens: 0,
outputTokens: 0,
inputCredits: 0,
outputCredits: 0,
};
let lastStepConversationSize = 0;
let totalCacheCreationTokens = 0;
writer.merge(
stream.toUIMessageStream({
@@ -109,9 +103,37 @@ export class AgentChatStreamingService {
},
sendStart: false,
messageMetadata: ({ part }) => {
if (part.type === 'finish-step') {
const stepInput = part.usage?.inputTokens ?? 0;
const stepCached = part.usage?.cachedInputTokens ?? 0;
// Anthropic excludes cached/created tokens from input_tokens,
// reporting them separately as cache_creation_input_tokens
const anthropicUsage = (
part as {
providerMetadata?: {
anthropic?: {
usage?: { cache_creation_input_tokens?: number };
};
};
}
).providerMetadata?.anthropic?.usage;
const stepCacheCreation =
anthropicUsage?.cache_creation_input_tokens ?? 0;
totalCacheCreationTokens += stepCacheCreation;
lastStepConversationSize =
stepInput + stepCached + stepCacheCreation;
}
if (part.type === 'finish') {
const inputTokens = part.totalUsage?.inputTokens ?? 0;
const inputTokens =
(part.totalUsage?.inputTokens ?? 0) +
(part.totalUsage?.cachedInputTokens ?? 0) +
totalCacheCreationTokens;
const outputTokens = part.totalUsage?.outputTokens ?? 0;
const cachedInputTokens =
part.totalUsage?.cachedInputTokens ?? 0;
const inputCostInCents =
(inputTokens / 1000) *
@@ -139,8 +161,10 @@ export class AgentChatStreamingService {
usage: {
inputTokens,
outputTokens,
inputCredits,
outputCredits,
cachedInputTokens,
inputCredits: toDisplayCredits(inputCredits),
outputCredits: toDisplayCredits(outputCredits),
conversationSize: lastStepConversationSize,
},
model: {
contextWindowTokens: modelConfig.contextWindowTokens,
@@ -155,15 +179,6 @@ export class AgentChatStreamingService {
return;
}
writer.write({
type: 'data-routing-status' as const,
id: 'execution-status',
data: {
text: 'Completed',
state: 'routed',
},
});
const validThreadId = thread.id;
if (!validThreadId) {
@@ -205,6 +220,7 @@ export class AgentChatStreamingService {
totalOutputCredits: () =>
`"totalOutputCredits" + ${streamUsage.outputCredits}`,
contextWindowTokens: modelConfig.contextWindowTokens,
conversationSize: lastStepConversationSize,
});
} catch (saveError) {
this.logger.error(
@@ -1,11 +1,13 @@
import { Injectable, Logger } from '@nestjs/common';
import { anthropic } from '@ai-sdk/anthropic';
import { groq } from '@ai-sdk/groq';
import { openai } from '@ai-sdk/openai';
import {
convertToModelMessages,
stepCountIs,
streamText,
type SystemModelMessage,
type ToolSet,
type UIDataTypes,
type UIMessage,
@@ -17,16 +19,16 @@ import { getAppPath } from 'twenty-shared/utils';
import { type CodeExecutionStreamEmitter } from 'src/engine/core-modules/tool-provider/interfaces/tool-provider.interface';
import { WorkspaceDomainsService } from 'src/engine/core-modules/domain/workspace-domains/services/workspace-domains.service';
import { COMMON_PRELOAD_TOOLS } from 'src/engine/core-modules/tool-provider/constants/common-preload-tools.const';
import { wrapToolsWithOutputSerialization } from 'src/engine/core-modules/tool-provider/output-serialization/wrap-tools-with-output-serialization.util';
import { ToolRegistryService } from 'src/engine/core-modules/tool-provider/services/tool-registry.service';
import {
type ToolIndexEntry,
ToolRegistryService,
} from 'src/engine/core-modules/tool-provider/services/tool-registry.service';
import {
createExecuteToolTool,
createLearnToolsTool,
createLoadSkillTool,
createLoadToolsTool,
type DynamicToolStore,
EXECUTE_TOOL_TOOL_NAME,
LEARN_TOOLS_TOOL_NAME,
LOAD_SKILL_TOOL_NAME,
LOAD_TOOLS_TOOL_NAME,
} from 'src/engine/core-modules/tool-provider/tools';
import { type WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity';
import { AgentActorContextService } from 'src/engine/metadata-modules/ai/ai-agent-execution/services/agent-actor-context.service';
@@ -34,7 +36,7 @@ import { AGENT_CONFIG } from 'src/engine/metadata-modules/ai/ai-agent/constants/
import { type BrowsingContextType } from 'src/engine/metadata-modules/ai/ai-agent/types/browsingContext.type';
import { repairToolCall } from 'src/engine/metadata-modules/ai/ai-agent/utils/repair-tool-call.util';
import { AIBillingService } from 'src/engine/metadata-modules/ai/ai-billing/services/ai-billing.service';
import { CHAT_SYSTEM_PROMPTS } from 'src/engine/metadata-modules/ai/ai-chat/constants/chat-system-prompts.const';
import { SystemPromptBuilderService } from 'src/engine/metadata-modules/ai/ai-chat/services/system-prompt-builder.service';
import {
extractCodeInterpreterFiles,
type ExtractedFile,
@@ -45,7 +47,6 @@ import {
} from 'src/engine/metadata-modules/ai/ai-models/constants/ai-models.const';
import { AI_TELEMETRY_CONFIG } from 'src/engine/metadata-modules/ai/ai-models/constants/ai-telemetry.const';
import { AiModelRegistryService } from 'src/engine/metadata-modules/ai/ai-models/services/ai-model-registry.service';
import { type FlatSkill } from 'src/engine/metadata-modules/flat-skill/types/flat-skill.type';
import { SkillService } from 'src/engine/metadata-modules/skill/skill.service';
export type ChatExecutionOptions = {
@@ -58,12 +59,9 @@ export type ChatExecutionOptions = {
export type ChatExecutionResult = {
stream: ReturnType<typeof streamText>;
preloadedTools: string[];
modelConfig: AIModelConfig;
};
const COMMON_PRELOAD_TOOLS = ['search_help_center'];
@Injectable()
export class ChatExecutionService {
private readonly logger = new Logger(ChatExecutionService.name);
@@ -75,6 +73,7 @@ export class ChatExecutionService {
private readonly aiBillingService: AIBillingService,
private readonly agentActorContextService: AgentActorContextService,
private readonly workspaceDomainsService: WorkspaceDomainsService,
private readonly systemPromptBuilder: SystemPromptBuilderService,
) {}
async streamChat({
@@ -84,7 +83,7 @@ export class ChatExecutionService {
browsingContext,
onCodeExecutionUpdate,
}: ChatExecutionOptions): Promise<ChatExecutionResult> {
const { actorContext, roleId, userId } =
const { actorContext, roleId, userId, userContext } =
await this.agentActorContextService.buildUserAndAgentActorContext(
userWorkspaceId,
workspace.id,
@@ -124,33 +123,35 @@ export class ChatExecutionService {
const preloadedToolNames = Object.keys(preloadedTools);
const dynamicToolStore: DynamicToolStore = {
loadedTools: new Set(preloadedToolNames),
};
// Respect the workspace's model preference (Settings > AI > Model Router)
const registeredModel =
this.aiModelRegistryService.getDefaultPerformanceModel();
await this.aiModelRegistryService.resolveModelForAgent({
modelId: workspace.smartModel,
});
const modelConfig = this.aiModelRegistryService.getEffectiveModelConfig(
registeredModel.modelId,
);
const activeTools: ToolSet = {
...preloadedTools,
// Direct tools: native provider tools + preloaded tools.
// These are callable directly AND as fallback through execute_tool.
const directTools: ToolSet = {
...wrapToolsWithOutputSerialization(preloadedTools),
...this.getNativeWebSearchTool(registeredModel.provider),
[LOAD_TOOLS_TOOL_NAME]: createLoadToolsTool(
};
// ToolSet is constant for the entire conversation — no mutation.
// learn_tools returns schemas as text; execute_tool dispatches to cached tools.
const activeTools: ToolSet = {
...directTools,
[LEARN_TOOLS_TOOL_NAME]: createLearnToolsTool(
this.toolRegistry,
toolContext,
dynamicToolStore,
async (toolNames) => {
const newTools = await this.toolRegistry.getToolsByName(
toolNames,
toolContext,
);
Object.assign(activeTools, newTools);
this.logger.log(`Dynamically loaded tools: ${toolNames.join(', ')}`);
},
),
[EXECUTE_TOOL_TOOL_NAME]: createExecuteToolTool(
this.toolRegistry,
toolContext,
directTools,
),
[LOAD_SKILL_TOOL_NAME]: createLoadSkillTool((skillNames) =>
this.skillService.findFlatSkillsByNames(skillNames, workspace.id),
@@ -173,22 +174,32 @@ export class ChatExecutionService {
);
}
const systemPrompt = this.buildSystemPrompt(
const systemPrompt = this.systemPromptBuilder.buildFullPrompt(
toolCatalog,
skillCatalog,
preloadedToolNames,
contextString,
storedFiles,
workspace.aiAdditionalInstructions ?? undefined,
userContext,
);
this.logger.log(
`Starting chat execution with model ${registeredModel.modelId}, ${Object.keys(activeTools).length} active tools`,
);
const systemMessage: SystemModelMessage = {
role: 'system',
content: systemPrompt,
providerOptions:
registeredModel.provider === ModelProvider.ANTHROPIC
? { anthropic: { cacheControl: { type: 'ephemeral' } } }
: undefined,
};
const stream = streamText({
model: registeredModel.model,
system: systemPrompt,
messages: convertToModelMessages(processedMessages),
messages: [systemMessage, ...convertToModelMessages(processedMessages)],
tools: activeTools,
stopWhen: stepCountIs(AGENT_CONFIG.MAX_STEPS),
experimental_telemetry: AI_TELEMETRY_CONFIG,
@@ -223,7 +234,6 @@ export class ChatExecutionService {
return {
stream,
preloadedTools: preloadedToolNames,
modelConfig,
};
}
@@ -284,176 +294,18 @@ export class ChatExecutionService {
return context;
}
private buildSystemPrompt(
toolCatalog: ToolIndexEntry[],
skillCatalog: FlatSkill[],
preloadedTools: string[],
contextString?: string,
storedFiles?: Array<{ filename: string; storagePath: string; url: string }>,
): string {
const parts: string[] = [
CHAT_SYSTEM_PROMPTS.BASE,
CHAT_SYSTEM_PROMPTS.RESPONSE_FORMAT,
];
parts.push(this.buildToolCatalogSection(toolCatalog, preloadedTools));
parts.push(this.buildSkillCatalogSection(skillCatalog));
if (storedFiles && storedFiles.length > 0) {
parts.push(this.buildUploadedFilesSection(storedFiles));
}
if (contextString) {
parts.push(
`\nCONTEXT (what the user is currently viewing):\n${contextString}`,
);
}
return parts.join('\n');
}
private buildUploadedFilesSection(
storedFiles: Array<{ filename: string; storagePath: string; url: string }>,
): string {
const fileList = storedFiles.map((f) => `- ${f.filename}`).join('\n');
const filesJson = JSON.stringify(
storedFiles.map((f) => ({ filename: f.filename, url: f.url })),
);
return `
## Uploaded Files
The user has uploaded the following files:
${fileList}
**IMPORTANT**: Use the \`code_interpreter\` tool to analyze these files.
When calling code_interpreter, include the files parameter with these values:
\`\`\`json
${filesJson}
\`\`\`
In your Python code, access files at \`/home/user/{filename}\`.`;
}
private buildSkillCatalogSection(skillCatalog: FlatSkill[]): string {
if (skillCatalog.length === 0) {
return '';
}
const skillsList = skillCatalog
.map(
(skill) => `- \`${skill.name}\`: ${skill.description ?? skill.label}`,
)
.join('\n');
return `
## Available Skills
Skills provide detailed expertise for specialized tasks. Load a skill before attempting complex operations.
To load a skill, call \`${LOAD_SKILL_TOOL_NAME}\` with the skill name(s).
${skillsList}`;
}
private buildToolCatalogSection(
toolCatalog: ToolIndexEntry[],
preloadedTools: string[],
): string {
const preloadedSet = new Set(preloadedTools);
const toolsByCategory = new Map<string, ToolIndexEntry[]>();
for (const tool of toolCatalog) {
const category = tool.category;
const existing = toolsByCategory.get(category) ?? [];
existing.push(tool);
toolsByCategory.set(category, existing);
}
const sections: string[] = [];
sections.push(`
## Available Tools
You have access to ${toolCatalog.length} tools plus native web search. Some are pre-loaded and ready to use immediately.
To use a tool that isn't pre-loaded, call \`${LOAD_TOOLS_TOOL_NAME}\` with the exact tool name(s) first.
### Pre-loaded Tools (ready to use now)
- \`web_search\` ✓: Search the web for real-time information (ALWAYS use this for current data, news, research)
${preloadedTools.length > 0 ? preloadedTools.map((t) => `- \`${t}\``).join('\n') : ''}
### Tool Catalog by Category`);
const categoryOrder = [
'DATABASE',
'ACTION',
'WORKFLOW',
'DASHBOARD',
'METADATA',
'VIEW',
'LOGIC_FUNCTION',
];
for (const category of categoryOrder) {
const tools = toolsByCategory.get(category);
if (!tools || tools.length === 0) {
continue;
}
const categoryLabel = this.getCategoryLabel(category);
sections.push(`
#### ${categoryLabel} (${tools.length} tools)
${tools
.map((t) => {
const status = preloadedSet.has(t.name) ? ' ✓' : '';
return `- \`${t.name}\`${status}: ${t.description}`;
})
.join('\n')}`);
}
sections.push(`
### How to Use Tools
1. **Web search** (\`web_search\`): Use for ANY request requiring current/real-time information from the internet
2. **Pre-loaded tools** (marked with ✓): Use directly
3. **Other tools**: First call \`${LOAD_TOOLS_TOOL_NAME}({toolNames: ["tool_name"]})\`, then use the tool`);
return sections.join('\n');
}
private getCategoryLabel(category: string): string {
switch (category) {
case 'DATABASE':
return 'Database Tools (CRUD operations)';
case 'ACTION':
return 'Action Tools (HTTP, Email, etc.)';
case 'WORKFLOW':
return 'Workflow Tools (create/manage workflows)';
case 'METADATA':
return 'Metadata Tools (schema management)';
case 'VIEW':
return 'View Tools (query views)';
case 'DASHBOARD':
return 'Dashboard Tools (create/manage dashboards)';
case 'LOGIC_FUNCTION':
return 'Logic Functions (custom tools)';
default:
return category;
}
}
private getNativeWebSearchTool(provider: ModelProvider): ToolSet {
switch (provider) {
case ModelProvider.ANTHROPIC:
return { web_search: anthropic.tools.webSearch_20250305() };
case ModelProvider.OPENAI:
return { web_search: openai.tools.webSearch() };
case ModelProvider.GROQ:
// Type assertion needed due to @ai-sdk/groq tool type mismatch
return {
web_search: groq.tools.browserSearch({}) as ToolSet[string],
};
default:
// Other providers don't have native web search
return {};
}
}
@@ -0,0 +1,333 @@
import { Injectable } from '@nestjs/common';
import { COMMON_PRELOAD_TOOLS } from 'src/engine/core-modules/tool-provider/constants/common-preload-tools.const';
import { ToolCategory } from 'src/engine/core-modules/tool-provider/enums/tool-category.enum';
import { ToolRegistryService } from 'src/engine/core-modules/tool-provider/services/tool-registry.service';
import { type ToolDescriptor } from 'src/engine/core-modules/tool-provider/types/tool-descriptor.type';
import {
EXECUTE_TOOL_TOOL_NAME,
LEARN_TOOLS_TOOL_NAME,
LOAD_SKILL_TOOL_NAME,
} from 'src/engine/core-modules/tool-provider/tools';
import {
AgentActorContextService,
type UserContext,
} from 'src/engine/metadata-modules/ai/ai-agent-execution/services/agent-actor-context.service';
import { CHAT_SYSTEM_PROMPTS } from 'src/engine/metadata-modules/ai/ai-chat/constants/chat-system-prompts.const';
import { type FlatSkill } from 'src/engine/metadata-modules/flat-skill/types/flat-skill.type';
import { SkillService } from 'src/engine/metadata-modules/skill/skill.service';
export type SystemPromptSection = {
title: string;
content: string;
estimatedTokenCount: number;
};
export type SystemPromptPreview = {
sections: SystemPromptSection[];
estimatedTokenCount: number;
};
// ~4 characters per token for mixed English/code content
const estimateTokenCount = (text: string): number => Math.ceil(text.length / 4);
@Injectable()
export class SystemPromptBuilderService {
constructor(
private readonly toolRegistry: ToolRegistryService,
private readonly skillService: SkillService,
private readonly agentActorContextService: AgentActorContextService,
) {}
async buildPreview(
workspaceId: string,
userWorkspaceId: string,
workspaceInstructions?: string,
): Promise<SystemPromptPreview> {
const { roleId, userId, userContext } =
await this.agentActorContextService.buildUserAndAgentActorContext(
userWorkspaceId,
workspaceId,
);
const toolCatalog = await this.toolRegistry.buildToolIndex(
workspaceId,
roleId,
{ userId, userWorkspaceId },
);
const skillCatalog = await this.skillService.findAllFlatSkills(workspaceId);
const sections: SystemPromptSection[] = [];
const baseContent = CHAT_SYSTEM_PROMPTS.BASE;
sections.push({
title: 'Base Instructions',
content: baseContent,
estimatedTokenCount: estimateTokenCount(baseContent),
});
const responseFormatContent = CHAT_SYSTEM_PROMPTS.RESPONSE_FORMAT;
sections.push({
title: 'Response Format',
content: responseFormatContent,
estimatedTokenCount: estimateTokenCount(responseFormatContent),
});
if (workspaceInstructions) {
const workspaceSection = this.buildWorkspaceInstructionsSection(
workspaceInstructions,
);
sections.push({
title: 'Workspace Instructions',
content: workspaceSection,
estimatedTokenCount: estimateTokenCount(workspaceSection),
});
}
if (userContext) {
const userSection = this.buildUserContextSection(userContext);
sections.push({
title: 'User Context',
content: userSection,
estimatedTokenCount: estimateTokenCount(userSection),
});
}
const toolSection = this.buildToolCatalogSection(
toolCatalog,
COMMON_PRELOAD_TOOLS,
);
sections.push({
title: 'Tool Catalog',
content: toolSection,
estimatedTokenCount: estimateTokenCount(toolSection),
});
const skillSection = this.buildSkillCatalogSection(skillCatalog);
if (skillSection) {
sections.push({
title: 'Skill Catalog',
content: skillSection,
estimatedTokenCount: estimateTokenCount(skillSection),
});
}
const totalTokens = sections.reduce(
(sum, section) => sum + section.estimatedTokenCount,
0,
);
return {
sections,
estimatedTokenCount: totalTokens,
};
}
buildFullPrompt(
toolCatalog: ToolDescriptor[],
skillCatalog: FlatSkill[],
preloadedTools: string[],
contextString?: string,
storedFiles?: Array<{
filename: string;
storagePath: string;
url: string;
}>,
workspaceInstructions?: string,
userContext?: UserContext,
): string {
const parts: string[] = [
CHAT_SYSTEM_PROMPTS.BASE,
CHAT_SYSTEM_PROMPTS.RESPONSE_FORMAT,
];
if (workspaceInstructions) {
parts.push(this.buildWorkspaceInstructionsSection(workspaceInstructions));
}
if (userContext) {
parts.push(this.buildUserContextSection(userContext));
}
parts.push(this.buildToolCatalogSection(toolCatalog, preloadedTools));
parts.push(this.buildSkillCatalogSection(skillCatalog));
if (storedFiles && storedFiles.length > 0) {
parts.push(this.buildUploadedFilesSection(storedFiles));
}
if (contextString) {
parts.push(
`\nCONTEXT (what the user is currently viewing):\n${contextString}`,
);
}
return parts.join('\n');
}
buildWorkspaceInstructionsSection(instructions: string): string {
return `
## Workspace Instructions
The following are custom instructions provided by the workspace administrator:
${instructions}`;
}
buildUserContextSection(userContext: UserContext): string {
const parts = [
`User: ${userContext.firstName} ${userContext.lastName}`.trim(),
`Locale: ${userContext.locale}`,
];
if (userContext.timezone) {
parts.push(`Timezone: ${userContext.timezone}`);
}
return `
## User Context
${parts.join('\n')}`;
}
buildUploadedFilesSection(
storedFiles: Array<{ filename: string; storagePath: string; url: string }>,
): string {
const fileList = storedFiles.map((f) => `- ${f.filename}`).join('\n');
const filesJson = JSON.stringify(
storedFiles.map((f) => ({ filename: f.filename, url: f.url })),
);
return `
## Uploaded Files
The user has uploaded the following files:
${fileList}
**IMPORTANT**: Use the \`code_interpreter\` tool to analyze these files.
When calling code_interpreter, include the files parameter with these values:
\`\`\`json
${filesJson}
\`\`\`
In your Python code, access files at \`/home/user/{filename}\`.`;
}
buildSkillCatalogSection(skillCatalog: FlatSkill[]): string {
if (skillCatalog.length === 0) {
return '';
}
const skillsList = skillCatalog
.map(
(skill) => `- \`${skill.name}\`: ${skill.description ?? skill.label}`,
)
.join('\n');
return `
## Available Skills
Skills provide detailed expertise for specialized tasks. Load a skill before attempting complex operations.
To load a skill, call \`${LOAD_SKILL_TOOL_NAME}\` with the skill name(s).
${skillsList}`;
}
buildToolCatalogSection(
toolCatalog: ToolDescriptor[],
preloadedTools: string[],
): string {
const preloadedSet = new Set(preloadedTools);
const toolsByCategory = new Map<string, ToolDescriptor[]>();
for (const tool of toolCatalog) {
const category = tool.category;
const existing = toolsByCategory.get(category) ?? [];
existing.push(tool);
toolsByCategory.set(category, existing);
}
const sections: string[] = [];
sections.push(`
## Available Tools
You have access to ${toolCatalog.length} tools plus native web search. Some are pre-loaded and ready to use immediately.
To use any other tool, first call \`${LEARN_TOOLS_TOOL_NAME}\` to learn its schema, then call \`${EXECUTE_TOOL_TOOL_NAME}\` to run it.
### Pre-loaded Tools (ready to use now)
- \`web_search\` ✓: Search the web for real-time information (ALWAYS use this for current data, news, research)
${preloadedTools.length > 0 ? preloadedTools.map((toolName) => `- \`${toolName}\``).join('\n') : ''}
### Tool Catalog by Category`);
const categoryOrder = [
ToolCategory.DATABASE_CRUD,
ToolCategory.ACTION,
ToolCategory.WORKFLOW,
ToolCategory.DASHBOARD,
ToolCategory.METADATA,
ToolCategory.VIEW,
ToolCategory.LOGIC_FUNCTION,
];
for (const category of categoryOrder) {
const tools = toolsByCategory.get(category);
if (!tools || tools.length === 0) {
continue;
}
const categoryLabel = this.getCategoryLabel(category);
sections.push(`
#### ${categoryLabel} (${tools.length} tools)
${tools
.map((tool) => {
const status = preloadedSet.has(tool.name) ? ' ✓' : '';
return `- \`${tool.name}\`${status}`;
})
.join('\n')}`);
}
sections.push(`
### How to Use Tools
1. **Web search** (\`web_search\`): Use for ANY request requiring current/real-time information from the internet
2. **Pre-loaded tools** (marked with ✓): Use directly
3. **Other tools**: First call \`${LEARN_TOOLS_TOOL_NAME}({toolNames: ["tool_name"]})\` to learn the schema, then call \`${EXECUTE_TOOL_TOOL_NAME}({toolName: "tool_name", arguments: {...}})\` to run it`);
return sections.join('\n');
}
private getCategoryLabel(category: string): string {
switch (category) {
case ToolCategory.DATABASE_CRUD:
return 'Database Tools (CRUD operations)';
case ToolCategory.ACTION:
return 'Action Tools (HTTP, Email, etc.)';
case ToolCategory.WORKFLOW:
return 'Workflow Tools (create/manage workflows)';
case ToolCategory.METADATA:
return 'Metadata Tools (schema management)';
case ToolCategory.VIEW:
return 'View Tools (query views)';
case ToolCategory.DASHBOARD:
return 'Dashboard Tools (create/manage dashboards)';
case ToolCategory.LOGIC_FUNCTION:
return 'Logic Functions (custom tools)';
default:
return category;
}
}
}
@@ -4,6 +4,7 @@ export enum ModelProvider {
ANTHROPIC = 'anthropic',
OPENAI_COMPATIBLE = 'open_ai_compatible',
XAI = 'xai',
GROQ = 'groq',
}
export const DEFAULT_FAST_MODEL = 'default-fast-model' as const;
@@ -32,6 +33,8 @@ export type ModelId =
| 'grok-3-mini'
| 'grok-4'
| 'grok-4-1-fast-reasoning'
// Groq models
| 'openai/gpt-oss-120b'
| string; // Allow custom model names
export type SupportedFileType =
@@ -15,6 +15,7 @@ describe('AI_MODELS', () => {
ModelProvider.OPENAI,
ModelProvider.ANTHROPIC,
ModelProvider.XAI,
ModelProvider.GROQ,
];
providers.forEach((provider) => {
@@ -51,6 +52,7 @@ describe('AI_MODELS', () => {
ModelProvider.OPENAI,
ModelProvider.ANTHROPIC,
ModelProvider.XAI,
ModelProvider.GROQ,
];
providers.forEach((provider) => {
@@ -89,7 +91,7 @@ describe('AiModelRegistryService', () => {
MOCK_CONFIG_SERVICE.get.mockReturnValue('gpt-4o');
expect(() => SERVICE.getEffectiveModelConfig(DEFAULT_SMART_MODEL)).toThrow(
'No AI models are available. Please configure at least one AI provider API key (OPENAI_API_KEY, ANTHROPIC_API_KEY, or XAI_API_KEY).',
'No AI models are available. Please configure at least one AI provider API key (OPENAI_API_KEY, ANTHROPIC_API_KEY, XAI_API_KEY, or GROQ_API_KEY).',
);
});
@@ -9,6 +9,7 @@ export {
import { type AIModelConfig } from './ai-models-types.const';
import { ANTHROPIC_MODELS } from './anthropic-models.const';
import { GROQ_MODELS } from './groq-models.const';
import { OPENAI_MODELS } from './openai-models.const';
import { XAI_MODELS } from './xai-models.const';
@@ -16,4 +17,5 @@ export const AI_MODELS: AIModelConfig[] = [
...OPENAI_MODELS,
...ANTHROPIC_MODELS,
...XAI_MODELS,
...GROQ_MODELS,
];
@@ -0,0 +1,15 @@
import { type AIModelConfig, ModelProvider } from './ai-models-types.const';
export const GROQ_MODELS: AIModelConfig[] = [
{
modelId: 'openai/gpt-oss-120b',
label: 'GPT-OSS 120B (Groq)',
description:
'Large-scale open-source model with browser search, served via Groq inference',
provider: ModelProvider.GROQ,
inputCostPer1kTokensInCents: 0.059,
outputCostPer1kTokensInCents: 0.079,
contextWindowTokens: 128000,
maxOutputTokens: 16384,
},
];
@@ -1,6 +1,7 @@
import { Injectable } from '@nestjs/common';
import { anthropic } from '@ai-sdk/anthropic';
import { groq } from '@ai-sdk/groq';
import { createOpenAI, openai } from '@ai-sdk/openai';
import { xai } from '@ai-sdk/xai';
import { type LanguageModel } from 'ai';
@@ -18,6 +19,7 @@ import {
type AIModelConfig,
} from 'src/engine/metadata-modules/ai/ai-models/constants/ai-models.const';
import { ANTHROPIC_MODELS } from 'src/engine/metadata-modules/ai/ai-models/constants/anthropic-models.const';
import { GROQ_MODELS } from 'src/engine/metadata-modules/ai/ai-models/constants/groq-models.const';
import { OPENAI_MODELS } from 'src/engine/metadata-modules/ai/ai-models/constants/openai-models.const';
import { XAI_MODELS } from 'src/engine/metadata-modules/ai/ai-models/constants/xai-models.const';
@@ -57,6 +59,12 @@ export class AiModelRegistryService {
this.registerXaiModels();
}
const groqApiKey = this.twentyConfigService.get('GROQ_API_KEY');
if (groqApiKey) {
this.registerGroqModels();
}
const openaiCompatibleBaseUrl = this.twentyConfigService.get(
'OPENAI_COMPATIBLE_BASE_URL',
);
@@ -105,6 +113,17 @@ export class AiModelRegistryService {
});
}
private registerGroqModels(): void {
GROQ_MODELS.forEach((modelConfig) => {
this.modelRegistry.set(modelConfig.modelId, {
modelId: modelConfig.modelId,
provider: ModelProvider.GROQ,
model: groq(modelConfig.modelId),
doesSupportThinking: modelConfig.doesSupportThinking,
});
});
}
private registerOpenAICompatibleModels(
baseUrl: string,
modelNamesString: string,
@@ -170,7 +189,7 @@ export class AiModelRegistryService {
if (!model) {
throw new AgentException(
'No AI models are available. Please configure at least one AI provider API key (OPENAI_API_KEY, ANTHROPIC_API_KEY, or XAI_API_KEY).',
'No AI models are available. Please configure at least one AI provider API key (OPENAI_API_KEY, ANTHROPIC_API_KEY, XAI_API_KEY, or GROQ_API_KEY).',
AgentExceptionCode.API_KEY_NOT_CONFIGURED,
);
}
@@ -192,7 +211,7 @@ export class AiModelRegistryService {
if (!model) {
throw new AgentException(
'No AI models are available. Please configure at least one AI provider API key (OPENAI_API_KEY, ANTHROPIC_API_KEY, or XAI_API_KEY).',
'No AI models are available. Please configure at least one AI provider API key (OPENAI_API_KEY, ANTHROPIC_API_KEY, XAI_API_KEY, or GROQ_API_KEY).',
AgentExceptionCode.API_KEY_NOT_CONFIGURED,
);
}
@@ -290,6 +309,9 @@ export class AiModelRegistryService {
case ModelProvider.XAI:
apiKey = this.twentyConfigService.get('XAI_API_KEY');
break;
case ModelProvider.GROQ:
apiKey = this.twentyConfigService.get('GROQ_API_KEY');
break;
case ModelProvider.OPENAI_COMPATIBLE:
apiKey = this.twentyConfigService.get('OPENAI_COMPATIBLE_API_KEY');
break;