fix(ai) - add logs + remove dashboard building (#21440)

- add logs for thread finishing without agent message
- add logs to monitor toolCall token usage
- remove dashboard building via AI (before fixing it)
- fix Anthropic compute
This commit is contained in:
Etienne
2026-06-11 14:45:25 +02:00
committed by GitHub
parent a6fcbf58e4
commit 303c415dd1
10 changed files with 151 additions and 79 deletions
@@ -2,7 +2,6 @@ import type { MessageDescriptor } from '@lingui/core';
import { msg } from '@lingui/core/macro';
import {
type IconComponent,
IconLayoutDashboard,
IconPlus,
IconSettingsAutomation,
} from 'twenty-ui-deprecated/display';
@@ -15,16 +14,6 @@ export type SuggestedPrompt = {
};
export const DEFAULT_SUGGESTED_PROMPTS: SuggestedPrompt[] = [
{
id: 'dashboard',
label: msg`Create a dashboard`,
Icon: IconLayoutDashboard,
prefillPrompts: [
msg`Create a dashboard with a chart of deal value by pipeline stage (New, Meeting, Proposal, Negotiation, Closed Won/Lost) for the current quarter, and a table of my top 10 open opportunities with amount, stage and expected close date.`,
msg`Build a dashboard that shows: (1) total pipeline value by stage for the last 3 months, (2) count of deals won vs lost per month, (3) average deal size. Use our standard pipeline stages.`,
msg`I need a dashboard for lead conversion: number of new leads by source this month, how many moved to opportunity, and conversion rate by source. Include a simple table and a bar chart.`,
],
},
{
id: 'workflow',
label: msg`Create a workflow`,
@@ -28,7 +28,6 @@ export const buildMcpServerInstructions = (
` METADATA: get/create/update/delete_object_metadata | get/create/update/delete_field_metadata`,
` Both GET tools return system items as compact summaries by default — keep that default for listing/inspecting; only set includeFullSystemObjects / includeFullSystemFields=true when you specifically need a system item's full configuration`,
` VIEW: get_views | get_view_query_parameters | create/update/delete_view | manage view fields, filters, sorts`,
` DASHBOARD: list_dashboards | get_dashboard | create_complete_dashboard | add/update/delete_dashboard_widget`,
` WEBHOOK: list/create/update/delete_webhook`,
` NAVIGATION: list/create/update/delete_navigation_menu_item`,
` LOGIC_FUNCTION: app_{function_name} — workspace-specific; use list_logic_function_tools to discover`,
@@ -36,8 +35,13 @@ export const buildMcpServerInstructions = (
`Skills vs Tools:`,
` Skills = documentation (load_skills) — teach HOW to do something, correct schemas and patterns`,
` Tools = execution (execute_tool) — let you DO something`,
` For complex tasks (workflows, dashboards, metadata), load the matching skill BEFORE calling tools.`,
` ⚠️ Never call workflow, dashboard, or metadata tools without loading their skill first.`,
` For complex tasks (workflows, metadata), load the matching skill BEFORE calling tools.`,
` ⚠️ Never call workflow or metadata tools without loading their skill first.`,
``,
`Dashboards (coming soon):`,
` Building or editing dashboards through the AI is not available yet — it is a coming soon feature.`,
` If asked to create/build/modify a dashboard, do not attempt it: say AI-assisted dashboards are coming soon,`,
` and offer alternatives (create views, run analytics with group_by_{objects}, or build workflows).`,
``,
`Route by intent:`,
` Named entity ("Acme company") → find_many_{objects} to resolve id first, then operate on id`,
@@ -4,7 +4,6 @@ import { TypeOrmModule } from '@nestjs/typeorm';
import { RecordCrudModule } from 'src/engine/core-modules/record-crud/record-crud.module';
import { TOOL_PROVIDERS } from 'src/engine/core-modules/tool-provider/constants/tool-providers.token';
import { ActionToolProvider } from 'src/engine/core-modules/tool-provider/providers/action-tool.provider';
import { DashboardToolProvider } from 'src/engine/core-modules/tool-provider/providers/dashboard-tool.provider';
import { DatabaseToolProvider } from 'src/engine/core-modules/tool-provider/providers/database-tool.provider';
import { LogicFunctionToolProvider } from 'src/engine/core-modules/tool-provider/providers/logic-function-tool.provider';
import { MetadataToolProvider } from 'src/engine/core-modules/tool-provider/providers/metadata-tool.provider';
@@ -69,7 +68,6 @@ import { ToolRegistryService } from './services/tool-registry.service';
ToolIndexResolver,
ToolExecutorService,
ActionToolProvider,
DashboardToolProvider,
DatabaseToolProvider,
MetadataToolProvider,
NavigationMenuItemToolProvider,
@@ -85,7 +83,6 @@ import { ToolRegistryService } from './services/tool-registry.service';
provide: TOOL_PROVIDERS,
useFactory: (
actionProvider: ActionToolProvider,
dashboardProvider: DashboardToolProvider,
databaseProvider: DatabaseToolProvider,
metadataProvider: MetadataToolProvider,
logicFunctionProvider: LogicFunctionToolProvider,
@@ -95,7 +92,6 @@ import { ToolRegistryService } from './services/tool-registry.service';
workflowProvider: WorkflowToolProvider,
) => [
actionProvider,
dashboardProvider,
databaseProvider,
metadataProvider,
logicFunctionProvider,
@@ -106,7 +102,6 @@ import { ToolRegistryService } from './services/tool-registry.service';
],
inject: [
ActionToolProvider,
DashboardToolProvider,
DatabaseToolProvider,
MetadataToolProvider,
LogicFunctionToolProvider,
@@ -145,7 +145,7 @@ describe('AiBillingService', () => {
expect(costInDollars).toBeCloseTo(0.00675);
});
it('should not subtract cached tokens from input for Anthropic', () => {
it('should not double-count cached and cache-creation tokens for Anthropic', () => {
mockAiModelRegistryService.getEffectiveModelConfig.mockReturnValue(
anthropicModelConfig as ReturnType<
AiModelRegistryService['getEffectiveModelConfig']
@@ -156,13 +156,15 @@ describe('AiBillingService', () => {
'claude-sonnet-4-5-20250929',
{
usage: {
inputTokens: 400,
// @ai-sdk/anthropic reports inputTokens as the FULL prompt:
// noCache(400) + cacheRead(600) + cacheCreation(200) = 1200
inputTokens: 1200,
outputTokens: 500,
totalTokens: 900,
totalTokens: 1700,
inputTokenDetails: {
noCacheTokens: 400,
cacheReadTokens: 600,
cacheWriteTokens: 0,
cacheWriteTokens: 200,
},
outputTokenDetails: { textTokens: 500, reasoningTokens: 0 },
},
@@ -170,7 +172,8 @@ describe('AiBillingService', () => {
},
);
// Anthropic: inputTokens already excludes cached
// inputTokens already includes cached + cache-creation, so the
// full-rate portion is 1200 - 600 - 200 = 400
// inputCost = (400/1M * 3.0) = 0.0012
// cachedCost = (600/1M * 0.3) = 0.00018
// cacheCreationCost = (200/1M * 3.75) = 0.00075
@@ -290,12 +293,13 @@ describe('AiBillingService', () => {
'claude-sonnet-4-5-20250929',
{
usage: {
inputTokens: 150_000,
// Full prompt size = noCache(150k) + cacheRead(100k) = 250k
inputTokens: 250_000,
outputTokens: 1000,
totalTokens: 251_000,
cachedInputTokens: 100_000,
inputTokenDetails: {
noCacheTokens: 0,
noCacheTokens: 150_000,
cacheReadTokens: 100_000,
cacheWriteTokens: 0,
},
@@ -304,8 +308,8 @@ describe('AiBillingService', () => {
},
);
// Anthropic: total input = 150k + 100k + 0 = 250k > 200k threshold
// Uses long context rates
// Total input = 250k > 200k threshold -> long context rates
// full-rate portion = 250k - 100k - 0 = 150k
// inputCost = (150_000/1M * 6.0) = 0.9
// cachedCost = (100_000/1M * 0.6) = 0.06
// outputCost = (1000/1M * 22.5) = 0.0225
@@ -29,10 +29,14 @@ const safeNumber = (value: number | undefined): number => {
return Number.isFinite(result) ? result : 0;
};
// Input token semantics differ by model family:
// Anthropic: inputTokens excludes cached and cache creation tokens
// OpenAI/xAI/Groq/Google: inputTokens includes cached tokens
// Output token semantics also differ:
// Input token semantics (all providers we use):
// `inputTokens` is the FULL prompt size and already includes cached and
// cache-creation tokens. The @ai-sdk/anthropic provider reports
// inputTokens = noCache + cacheRead + cacheCreation, and OpenAI-style
// providers include cached tokens (and never report cache-creation tokens).
// So the uncached, full-rate portion is always inputTokens minus cached
// minus cache-creation, and the full input size is just inputTokens.
// Output token semantics still differ by model family:
// Anthropic: outputTokens excludes reasoning (thinking) tokens
// OpenAI/xAI/Groq/Google: outputTokens includes reasoning tokens
export const computeCostBreakdown = (
@@ -47,17 +51,16 @@ export const computeCostBreakdown = (
const isAnthropicTokenReporting = model.modelFamily === ModelFamily.CLAUDE;
const adjustedInputTokens = isAnthropicTokenReporting
? rawInputTokens
: Math.max(0, rawInputTokens - cachedInputTokens);
const adjustedInputTokens = Math.max(
0,
rawInputTokens - cachedInputTokens - cacheCreationTokens,
);
const adjustedOutputTokens = isAnthropicTokenReporting
? rawOutputTokens
: Math.max(0, rawOutputTokens - reasoningTokens);
const totalInputTokens = isAnthropicTokenReporting
? rawInputTokens + cachedInputTokens + cacheCreationTokens
: rawInputTokens + cacheCreationTokens;
const totalInputTokens = rawInputTokens;
const costInfo =
model.longContextCost &&
@@ -7,20 +7,23 @@ export const CHAT_SYSTEM_PROMPTS = {
For ANY non-trivial task, follow this order:
1. **Plan**: Identify what the user needs. Determine which domain is involved (workflows, dashboards, metadata, data, documents, etc.).
1. **Plan**: Identify what the user needs. Determine which domain is involved (workflows, metadata, data, documents, etc.).
2. **Load the relevant skill FIRST**: Call \`load_skills\` to get detailed instructions, correct schemas, and parameter formats BEFORE doing anything else. Skills contain critical knowledge you don't have built-in — skipping this step leads to incorrect parameters and failed tool calls.
3. **Learn the required tools**: Call \`learn_tools\` to discover tool schemas and descriptions before using them. Pass every tool you need in a single \`learn_tools\` call (\`toolNames\` is an array) — do not make one call per tool.
4. **Execute**: Call \`execute_tool\` to run the tools following the instructions from the skill.
⚠️ NEVER call a specialized tool (workflow, dashboard, metadata, etc.) without loading its matching skill first. The Available Skills section below lists all skills — look for the one that matches the user's task domain and load it.
⚠️ NEVER call a specialized tool (workflow, metadata, etc.) without loading its matching skill first. The Available Skills section below lists all skills — look for the one that matches the user's task domain and load it.
Examples:
- User asks to create a workflow → \`load_skills(["workflow-building"])\` then learn and execute workflow tools
- User asks to build a dashboard → \`load_skills(["dashboard-building"])\` then learn and execute dashboard tools
- User asks to export data to Excel → \`load_skills(["xlsx", "code-interpreter"])\` then \`learn_tools({toolNames: ["code_interpreter"]})\` then \`execute_tool({toolName: "code_interpreter", arguments: {...}})\`
For simple CRUD operations (find/create/update/delete a record), you do NOT need a skill — but you still MUST call \`learn_tools\` first to learn the tool schema, then \`execute_tool\` to run it.
## Dashboards (coming soon)
Building or editing dashboards through the AI is not available yet — it is a coming soon feature. If the user asks you to create, build, or modify a dashboard, do NOT attempt it: let them know that AI-assisted dashboards are coming soon, and offer the alternatives you can help with today (e.g. creating views, running analytics with \`group_by_*\`, or building workflows).
## Skills vs Tools
- **SKILLS** = documentation/instructions (loaded via \`load_skills\`). They teach you HOW to do something — correct schemas, parameters, and patterns. They do NOT give you execution ability.
@@ -7,7 +7,9 @@ import type {
ExtendedUIMessage,
ExtendedUIMessagePart,
} from 'twenty-shared/ai';
import { isNonEmptyString } from '@sniptt/guards';
import { Repository } from 'typeorm';
import { isDefined } from 'twenty-shared/utils';
import { InjectWorkspaceScopedRepository } from 'src/engine/twenty-orm/workspace-scoped-repository/inject-workspace-scoped-repository.decorator';
import { WorkspaceScopedRepository } from 'src/engine/twenty-orm/workspace-scoped-repository/workspace-scoped-repository';
@@ -274,10 +276,13 @@ export class StreamAgentChatJob {
},
});
},
onFinish: async ({ responseMessage }) => {
onFinish: async ({ responseMessage, isAborted }) => {
try {
await this.handleStreamFinish({
responseMessage,
isAborted,
streamError,
outOfCredits: checkHasNoMoreAvailableCredits(),
threadId: data.threadId,
workspaceId: data.workspaceId,
userWorkspaceId: data.userWorkspaceId,
@@ -353,7 +358,6 @@ export class StreamAgentChatJob {
type: string;
usage?: {
inputTokens?: number;
inputTokenDetails?: { cacheReadTokens?: number };
};
totalUsage?: {
inputTokens?: number;
@@ -378,13 +382,12 @@ export class StreamAgentChatJob {
}) {
if (part.type === 'finish-step') {
const stepInput = part.usage?.inputTokens ?? 0;
const stepCached = part.usage?.inputTokenDetails?.cacheReadTokens ?? 0;
const stepCacheCreation = extractCacheCreationTokens(
part.providerMetadata,
);
onUpdateCacheCreationTokens(totalCacheCreationTokens + stepCacheCreation);
onUpdateConversationSize(stepInput + stepCached + stepCacheCreation);
onUpdateConversationSize(stepInput);
}
if (part.type === 'finish') {
@@ -432,6 +435,9 @@ export class StreamAgentChatJob {
private async handleStreamFinish({
responseMessage,
isAborted,
streamError,
outOfCredits,
threadId,
workspaceId,
userWorkspaceId,
@@ -442,6 +448,9 @@ export class StreamAgentChatJob {
userMessagePromise,
}: {
responseMessage: Omit<ExtendedUIMessage, 'id'>;
isAborted: boolean;
streamError: unknown;
outOfCredits: boolean;
threadId: string;
workspaceId: string;
userWorkspaceId: string;
@@ -457,6 +466,23 @@ export class StreamAgentChatJob {
modelConfig: AiModelConfig;
userMessagePromise: Promise<{ turnId: string | null }>;
}): Promise<void> {
const hasText = responseMessage.parts.some(
(part) => part.type === 'text' && isNonEmptyString(part.text),
);
if (isAborted || !hasText) {
this.logAssistantTurnWithoutText({
responseMessage,
isAborted,
streamError,
outOfCredits,
hasText,
threadId,
workspaceId,
streamUsage,
});
}
if (responseMessage.parts.length === 0) {
return;
}
@@ -506,4 +532,57 @@ export class StreamAgentChatJob {
workspaceId,
});
}
private logAssistantTurnWithoutText({
responseMessage,
isAborted,
streamError,
outOfCredits,
hasText,
threadId,
workspaceId,
streamUsage,
}: {
responseMessage: Omit<ExtendedUIMessage, 'id'>;
isAborted: boolean;
streamError: unknown;
outOfCredits: boolean;
hasText: boolean;
threadId: string;
workspaceId: string;
streamUsage: {
inputTokens: number;
outputTokens: number;
};
}): void {
const reason = isAborted
? 'user-cancelled'
: streamError
? 'stream-error'
: outOfCredits
? 'credits-exhausted'
: 'empty-completion';
const errorDetail =
streamError instanceof Error
? `${streamError.name}: ${streamError.message}`
: isDefined(streamError)
? String(streamError)
: 'none';
this.logger.warn(
`[AI_CHAT_NO_TEXT] Assistant turn ended without a text reply — ` +
`reason=${reason}, threadId=${threadId}, workspaceId=${workspaceId}, ` +
`isAborted=${isAborted}, outOfCredits=${outOfCredits}, hasText=${hasText}, ` +
`streamError=${errorDetail}, ` +
`inputTokens=${streamUsage.inputTokens},` +
`responseMessage.parts=${JSON.stringify(responseMessage.parts)}`,
);
if (streamError instanceof Error && isDefined(streamError.stack)) {
this.logger.warn(
`[AI_CHAT_NO_TEXT] streamError stack — threadId=${threadId}: ${streamError.stack}`,
);
}
}
}
@@ -288,6 +288,7 @@ export class ChatExecutionService {
const streamStartedAt = performance.now();
let stepStartedAt = streamStartedAt;
let ttftRecorded = false;
let stepIndex = 0;
const emitTurnUsageEvent = async (steps: StepResult<ToolSet>[]) => {
const usage = steps.reduce<LanguageModelUsage>(
@@ -447,6 +448,18 @@ export class ChatExecutionService {
hasNoMoreAvailableCredits = true;
}
this.logger.log(
`[AI_CHAT_TOKENS] step #${++stepIndex}` +
`toolCallIds=[${step.toolCalls.map((toolCall) => toolCall.toolCallId).join(', ')}]: ` +
`outputTokens=${step.usage.outputTokens ?? 0}, ` +
`reasoningTokens=${step.usage.outputTokenDetails?.reasoningTokens ?? 0}, ` +
`inputTokens(fullContext)=${step.usage.inputTokens ?? 0}, ` +
`cacheReadTokens=${step.usage.inputTokenDetails?.cacheReadTokens ?? 0}, ` +
`cacheWriteTokens=${step.usage.inputTokenDetails?.cacheWriteTokens ?? 0}, ` +
`cacheCreationTokens=${extractCacheCreationTokens(step.providerMetadata)}, ` +
`totalTokens=${step.usage.totalTokens ?? 0}`,
);
for (const toolResult of step.toolResults) {
const output = toolResult.output as ToolOutput | undefined;
@@ -276,38 +276,6 @@ Also create additional views for the standard objects (People, Companies, Opport
Navigate to each view after creating it. Wait 3 seconds.
Loop STEP 8 for all the custom objects
STEP 9: Create a multi-tab dashboard that tells the full story of the business.
Use create_complete_dashboard to create the first tab, then add_dashboard_tab + add_dashboard_widget for subsequent tabs.
**Structure: 3 tabs**
Tab 1 — "Overview": high-level KPIs and charts across the whole workspace
- Row 0: 34 AGGREGATE_CHART widgets (KPIs) — one per key metric (e.g. total revenue from Opportunities, count of active People, count of open deals). columnSpan 34, rowSpan 3.
- Row 3: 12 BAR_CHART or LINE_CHART widgets showing trends over time (group by a DATE_TIME field with MONTH granularity). columnSpan 6, rowSpan 7.
- Row 3: 1 PIE_CHART showing distribution by a SELECT field (e.g. status, type). columnSpan 6, rowSpan 7.
- Row 10: 1 STANDALONE_RICH_TEXT widget summarising the dashboard story. columnSpan 12, rowSpan 3.
Tab 2 — "[Domain object] pipeline" (e.g. "Deals", "Applications", "Repairs"): focus on Opportunities enriched with domain data
- Before adding the RECORD_TABLE widget, run this 3-step sequence:
1. create_view (type TABLE, name e.g. "Active Deals") → get the new viewId
2. create_many_view_fields on the new viewId — add 46 key fields (name, the new stage/status SELECT, a CURRENCY/NUMERIC field, a DATE field, linked Person or Company). Use positions 0, 1, 2… and isVisible: true.
3. create_many_view_filters + create_view_sort — e.g. filter out CLOSED/LOST records (SELECT IS_NOT "CLOSED"), sort by value DESC
- Row 0: 1 RECORD_TABLE widget. Set objectMetadataId to Opportunity, configuration.viewId to the dedicated view. columnSpan 12, rowSpan 8.
- Row 8: 1 BAR_CHART grouped by the stage SELECT field. columnSpan 6, rowSpan 7.
- Row 8: 1 PIE_CHART or AGGREGATE_CHART on the CURRENCY field. columnSpan 6, rowSpan 7.
Tab 3 — "[Domain people role] list" (e.g. "Clients", "Candidates", "Contacts"): focus on People enriched with domain data
- Before adding the RECORD_TABLE widget, run this 3-step sequence:
1. create_view (type TABLE, name e.g. "All Clients") → get the new viewId
2. create_many_view_fields — add 45 key fields (name, email, the new SELECT/status field, a DATE field, linked Company)
3. create_view_sort — sort by createdAt DESC or by name ASC
- Row 0: 1 RECORD_TABLE widget with the dedicated view. columnSpan 12, rowSpan 8.
- Row 8: 23 AGGREGATE_CHART KPIs (count, totals). columnSpan 4, rowSpan 3.
- Row 11: 1 BAR_CHART or LINE_CHART. columnSpan 12, rowSpan 7.
After creating the dashboard, navigate to the dashboard page.
`,
isCustom: false,
},
@@ -437,6 +405,10 @@ After creating a tab, use its returned tabId as pageLayoutTabId when calling add
- When modifying a chart, confirm whether the user wants to change settings or change chart type
- Use RECORD_TABLE widgets to give users direct access to filtered record lists without leaving the dashboard`,
isCustom: false,
// Dashboard tools are temporarily disabled in AI chat / MCP because the
// generated dashboards are not reliable yet. Keeping the skill defined
// (inactive) so it can be re-enabled once the tooling is trustworthy.
isActive: false,
},
}),
@@ -14,6 +14,7 @@ export type CreateStandardSkillContext = {
description: string | null;
content: string;
isCustom: boolean;
isActive?: boolean;
};
export type CreateStandardSkillArgs = StandardBuilderArgs<'skill'> & {
@@ -21,7 +22,16 @@ export type CreateStandardSkillArgs = StandardBuilderArgs<'skill'> & {
};
export const createStandardSkillFlatMetadata = ({
context: { skillName, name, label, icon, description, content, isCustom },
context: {
skillName,
name,
label,
icon,
description,
content,
isCustom,
isActive = true,
},
workspaceId,
twentyStandardApplicationId,
now,
@@ -37,7 +47,7 @@ export const createStandardSkillFlatMetadata = ({
description,
content,
isCustom,
isActive: true,
isActive,
workspaceId,
applicationId: twentyStandardApplicationId,
applicationUniversalIdentifier: