fix(ai) - add logs + remove dashboard building (#21440)

- add logs for thread finishing without agent message - add logs to monitor toolCall token usage - remove dashboard building via AI (before fixing it) - fix Anthropic compute
2026-06-11 14:45:25 +02:00
parent a6fcbf58e4
commit 303c415dd1
10 changed files with 151 additions and 79 deletions
@@ -145,7 +145,7 @@ describe('AiBillingService', () => {
      expect(costInDollars).toBeCloseTo(0.00675);
    });

-    it('should not subtract cached tokens from input for Anthropic', () => {
+    it('should not double-count cached and cache-creation tokens for Anthropic', () => {
      mockAiModelRegistryService.getEffectiveModelConfig.mockReturnValue(
        anthropicModelConfig as ReturnType<
          AiModelRegistryService['getEffectiveModelConfig']
@@ -156,13 +156,15 @@ describe('AiBillingService', () => {
        'claude-sonnet-4-5-20250929',
        {
          usage: {
-            inputTokens: 400,
+            // @ai-sdk/anthropic reports inputTokens as the FULL prompt:
+            // noCache(400) + cacheRead(600) + cacheCreation(200) = 1200
+            inputTokens: 1200,
            outputTokens: 500,
-            totalTokens: 900,
+            totalTokens: 1700,
            inputTokenDetails: {
              noCacheTokens: 400,
              cacheReadTokens: 600,
-              cacheWriteTokens: 0,
+              cacheWriteTokens: 200,
            },
            outputTokenDetails: { textTokens: 500, reasoningTokens: 0 },
          },
@@ -170,7 +172,8 @@ describe('AiBillingService', () => {
        },
      );

-      // Anthropic: inputTokens already excludes cached
+      // inputTokens already includes cached + cache-creation, so the
+      // full-rate portion is 1200 - 600 - 200 = 400
      // inputCost = (400/1M * 3.0) = 0.0012
      // cachedCost = (600/1M * 0.3) = 0.00018
      // cacheCreationCost = (200/1M * 3.75) = 0.00075
@@ -290,12 +293,13 @@ describe('AiBillingService', () => {
        'claude-sonnet-4-5-20250929',
        {
          usage: {
-            inputTokens: 150_000,
+            // Full prompt size = noCache(150k) + cacheRead(100k) = 250k
+            inputTokens: 250_000,
            outputTokens: 1000,
            totalTokens: 251_000,
            cachedInputTokens: 100_000,
            inputTokenDetails: {
-              noCacheTokens: 0,
+              noCacheTokens: 150_000,
              cacheReadTokens: 100_000,
              cacheWriteTokens: 0,
            },
@@ -304,8 +308,8 @@ describe('AiBillingService', () => {
        },
      );

-      // Anthropic: total input = 150k + 100k + 0 = 250k > 200k threshold
-      // Uses long context rates
+      // Total input = 250k > 200k threshold -> long context rates
+      // full-rate portion = 250k - 100k - 0 = 150k
      // inputCost = (150_000/1M * 6.0) = 0.9
      // cachedCost = (100_000/1M * 0.6) = 0.06
      // outputCost = (1000/1M * 22.5) = 0.0225
@@ -29,10 +29,14 @@ const safeNumber = (value: number | undefined): number => {
  return Number.isFinite(result) ? result : 0;
 };

-// Input token semantics differ by model family:
-//   Anthropic: inputTokens excludes cached and cache creation tokens
-//   OpenAI/xAI/Groq/Google: inputTokens includes cached tokens
-// Output token semantics also differ:
+// Input token semantics (all providers we use):
+//   `inputTokens` is the FULL prompt size and already includes cached and
+//   cache-creation tokens. The @ai-sdk/anthropic provider reports
+//   inputTokens = noCache + cacheRead + cacheCreation, and OpenAI-style
+//   providers include cached tokens (and never report cache-creation tokens).
+//   So the uncached, full-rate portion is always inputTokens minus cached
+//   minus cache-creation, and the full input size is just inputTokens.
+// Output token semantics still differ by model family:
 //   Anthropic: outputTokens excludes reasoning (thinking) tokens
 //   OpenAI/xAI/Groq/Google: outputTokens includes reasoning tokens
 export const computeCostBreakdown = (
@@ -47,17 +51,16 @@ export const computeCostBreakdown = (

  const isAnthropicTokenReporting = model.modelFamily === ModelFamily.CLAUDE;

-  const adjustedInputTokens = isAnthropicTokenReporting
-    ? rawInputTokens
-    : Math.max(0, rawInputTokens - cachedInputTokens);
+  const adjustedInputTokens = Math.max(
+    0,
+    rawInputTokens - cachedInputTokens - cacheCreationTokens,
+  );

  const adjustedOutputTokens = isAnthropicTokenReporting
    ? rawOutputTokens
    : Math.max(0, rawOutputTokens - reasoningTokens);

-  const totalInputTokens = isAnthropicTokenReporting
-    ? rawInputTokens + cachedInputTokens + cacheCreationTokens
-    : rawInputTokens + cacheCreationTokens;
+  const totalInputTokens = rawInputTokens;

  const costInfo =
    model.longContextCost &&
@@ -7,20 +7,23 @@ export const CHAT_SYSTEM_PROMPTS = {

 For ANY non-trivial task, follow this order:

-1. **Plan**: Identify what the user needs. Determine which domain is involved (workflows, dashboards, metadata, data, documents, etc.).
+1. **Plan**: Identify what the user needs. Determine which domain is involved (workflows, metadata, data, documents, etc.).
 2. **Load the relevant skill FIRST**: Call \`load_skills\` to get detailed instructions, correct schemas, and parameter formats BEFORE doing anything else. Skills contain critical knowledge you don't have built-in — skipping this step leads to incorrect parameters and failed tool calls.
 3. **Learn the required tools**: Call \`learn_tools\` to discover tool schemas and descriptions before using them. Pass every tool you need in a single \`learn_tools\` call (\`toolNames\` is an array) — do not make one call per tool.
 4. **Execute**: Call \`execute_tool\` to run the tools following the instructions from the skill.

-⚠️ NEVER call a specialized tool (workflow, dashboard, metadata, etc.) without loading its matching skill first. The Available Skills section below lists all skills — look for the one that matches the user's task domain and load it.
+⚠️ NEVER call a specialized tool (workflow, metadata, etc.) without loading its matching skill first. The Available Skills section below lists all skills — look for the one that matches the user's task domain and load it.

 Examples:
 - User asks to create a workflow → \`load_skills(["workflow-building"])\` then learn and execute workflow tools
- User asks to build a dashboard → \`load_skills(["dashboard-building"])\` then learn and execute dashboard tools
 - User asks to export data to Excel → \`load_skills(["xlsx", "code-interpreter"])\` then \`learn_tools({toolNames: ["code_interpreter"]})\` then \`execute_tool({toolName: "code_interpreter", arguments: {...}})\`

 For simple CRUD operations (find/create/update/delete a record), you do NOT need a skill — but you still MUST call \`learn_tools\` first to learn the tool schema, then \`execute_tool\` to run it.

+## Dashboards (coming soon)
+
+Building or editing dashboards through the AI is not available yet — it is a coming soon feature. If the user asks you to create, build, or modify a dashboard, do NOT attempt it: let them know that AI-assisted dashboards are coming soon, and offer the alternatives you can help with today (e.g. creating views, running analytics with \`group_by_*\`, or building workflows).
+
 ## Skills vs Tools

 - **SKILLS** = documentation/instructions (loaded via \`load_skills\`). They teach you HOW to do something — correct schemas, parameters, and patterns. They do NOT give you execution ability.
@@ -7,7 +7,9 @@ import type {
  ExtendedUIMessage,
  ExtendedUIMessagePart,
 } from 'twenty-shared/ai';
+import { isNonEmptyString } from '@sniptt/guards';
 import { Repository } from 'typeorm';
+import { isDefined } from 'twenty-shared/utils';

 import { InjectWorkspaceScopedRepository } from 'src/engine/twenty-orm/workspace-scoped-repository/inject-workspace-scoped-repository.decorator';
 import { WorkspaceScopedRepository } from 'src/engine/twenty-orm/workspace-scoped-repository/workspace-scoped-repository';
@@ -274,10 +276,13 @@ export class StreamAgentChatJob {
                  },
                });
              },
-              onFinish: async ({ responseMessage }) => {
+              onFinish: async ({ responseMessage, isAborted }) => {
                try {
                  await this.handleStreamFinish({
                    responseMessage,
+                    isAborted,
+                    streamError,
+                    outOfCredits: checkHasNoMoreAvailableCredits(),
                    threadId: data.threadId,
                    workspaceId: data.workspaceId,
                    userWorkspaceId: data.userWorkspaceId,
@@ -353,7 +358,6 @@ export class StreamAgentChatJob {
      type: string;
      usage?: {
        inputTokens?: number;
-        inputTokenDetails?: { cacheReadTokens?: number };
      };
      totalUsage?: {
        inputTokens?: number;
@@ -378,13 +382,12 @@ export class StreamAgentChatJob {
  }) {
    if (part.type === 'finish-step') {
      const stepInput = part.usage?.inputTokens ?? 0;
-      const stepCached = part.usage?.inputTokenDetails?.cacheReadTokens ?? 0;
      const stepCacheCreation = extractCacheCreationTokens(
        part.providerMetadata,
      );

      onUpdateCacheCreationTokens(totalCacheCreationTokens + stepCacheCreation);
-      onUpdateConversationSize(stepInput + stepCached + stepCacheCreation);
+      onUpdateConversationSize(stepInput);
    }

    if (part.type === 'finish') {
@@ -432,6 +435,9 @@ export class StreamAgentChatJob {

  private async handleStreamFinish({
    responseMessage,
+    isAborted,
+    streamError,
+    outOfCredits,
    threadId,
    workspaceId,
    userWorkspaceId,
@@ -442,6 +448,9 @@ export class StreamAgentChatJob {
    userMessagePromise,
  }: {
    responseMessage: Omit<ExtendedUIMessage, 'id'>;
+    isAborted: boolean;
+    streamError: unknown;
+    outOfCredits: boolean;
    threadId: string;
    workspaceId: string;
    userWorkspaceId: string;
@@ -457,6 +466,23 @@ export class StreamAgentChatJob {
    modelConfig: AiModelConfig;
    userMessagePromise: Promise<{ turnId: string | null }>;
  }): Promise<void> {
+    const hasText = responseMessage.parts.some(
+      (part) => part.type === 'text' && isNonEmptyString(part.text),
+    );
+
+    if (isAborted || !hasText) {
+      this.logAssistantTurnWithoutText({
+        responseMessage,
+        isAborted,
+        streamError,
+        outOfCredits,
+        hasText,
+        threadId,
+        workspaceId,
+        streamUsage,
+      });
+    }
+
    if (responseMessage.parts.length === 0) {
      return;
    }
@@ -506,4 +532,57 @@ export class StreamAgentChatJob {
      workspaceId,
    });
  }
+
+  private logAssistantTurnWithoutText({
+    responseMessage,
+    isAborted,
+    streamError,
+    outOfCredits,
+    hasText,
+    threadId,
+    workspaceId,
+    streamUsage,
+  }: {
+    responseMessage: Omit<ExtendedUIMessage, 'id'>;
+    isAborted: boolean;
+    streamError: unknown;
+    outOfCredits: boolean;
+    hasText: boolean;
+    threadId: string;
+    workspaceId: string;
+    streamUsage: {
+      inputTokens: number;
+      outputTokens: number;
+    };
+  }): void {
+    const reason = isAborted
+      ? 'user-cancelled'
+      : streamError
+        ? 'stream-error'
+        : outOfCredits
+          ? 'credits-exhausted'
+          : 'empty-completion';
+
+    const errorDetail =
+      streamError instanceof Error
+        ? `${streamError.name}: ${streamError.message}`
+        : isDefined(streamError)
+          ? String(streamError)
+          : 'none';
+
+    this.logger.warn(
+      `[AI_CHAT_NO_TEXT] Assistant turn ended without a text reply — ` +
+        `reason=${reason}, threadId=${threadId}, workspaceId=${workspaceId}, ` +
+        `isAborted=${isAborted}, outOfCredits=${outOfCredits}, hasText=${hasText}, ` +
+        `streamError=${errorDetail}, ` +
+        `inputTokens=${streamUsage.inputTokens},` +
+        `responseMessage.parts=${JSON.stringify(responseMessage.parts)}`,
+    );
+
+    if (streamError instanceof Error && isDefined(streamError.stack)) {
+      this.logger.warn(
+        `[AI_CHAT_NO_TEXT] streamError stack — threadId=${threadId}: ${streamError.stack}`,
+      );
+    }
+  }
 }
@@ -288,6 +288,7 @@ export class ChatExecutionService {
    const streamStartedAt = performance.now();
    let stepStartedAt = streamStartedAt;
    let ttftRecorded = false;
+    let stepIndex = 0;

    const emitTurnUsageEvent = async (steps: StepResult<ToolSet>[]) => {
      const usage = steps.reduce<LanguageModelUsage>(
@@ -447,6 +448,18 @@ export class ChatExecutionService {
          hasNoMoreAvailableCredits = true;
        }

+        this.logger.log(
+          `[AI_CHAT_TOKENS] step #${++stepIndex} — ` +
+            `toolCallIds=[${step.toolCalls.map((toolCall) => toolCall.toolCallId).join(', ')}]: ` +
+            `outputTokens=${step.usage.outputTokens ?? 0}, ` +
+            `reasoningTokens=${step.usage.outputTokenDetails?.reasoningTokens ?? 0}, ` +
+            `inputTokens(fullContext)=${step.usage.inputTokens ?? 0}, ` +
+            `cacheReadTokens=${step.usage.inputTokenDetails?.cacheReadTokens ?? 0}, ` +
+            `cacheWriteTokens=${step.usage.inputTokenDetails?.cacheWriteTokens ?? 0}, ` +
+            `cacheCreationTokens=${extractCacheCreationTokens(step.providerMetadata)}, ` +
+            `totalTokens=${step.usage.totalTokens ?? 0}`,
+        );
+
        for (const toolResult of step.toolResults) {
          const output = toolResult.output as ToolOutput | undefined;