feat(ai) - add observability (#20850)

**AI Chat - Tool Executions (counters, tagged with model)**
ai-chat/tool-execution-succeeded: number of tool calls invoked by the AI
that completed without error
ai-chat/tool-execution-failed: number of tool calls invoked by the AI
that threw an error
**AI Chat - Token Usage (counters, tagged with model)**
ai-chat/input-tokens: total input tokens sent to the model across all
turns
ai-chat/output-tokens: total output tokens generated by the model
ai-chat/cache-read-tokens: input tokens served from the model's prompt
cache (cheaper)
ai-chat/cache-write-tokens: input tokens written into the prompt cache
for future reuse
**AI Chat - Latency (histograms in ms, tagged with model)**
ai-chat/turn-latency-ms: total duration of a full chat turn (from stream
start to stream end)
ai-chat/step-latency-ms: duration of a single reasoning/tool-call step
within a turn
ai-chat/ttft-ms: time-to-first-token, i.e. how long until the model
starts streaming output
**MCP - Tool Executions (counters)**
mcp/tool-execution-succeeded: number of MCP tool calls that completed
successfully
mcp/tool-execution-failed: number of MCP tool calls that threw an error

packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/ai-chat.module.ts

@@ -12,6 +12,7 @@ import { UserWorkspaceEntity } from 'src/engine/core-modules/user-workspace/user
 import { UserWorkspaceModule } from 'src/engine/core-modules/user-workspace/user-workspace.module';
 import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity';
 import { AiAgentExecutionModule } from 'src/engine/metadata-modules/ai/ai-agent-execution/ai-agent-execution.module';
+import { MetricsModule } from 'src/engine/core-modules/metrics/metrics.module';
 import { AiBillingModule } from 'src/engine/metadata-modules/ai/ai-billing/ai-billing.module';
 import { AiGraphqlApiExceptionInterceptor } from 'src/engine/metadata-modules/ai/interceptors/ai-graphql-api-exception.interceptor';
 import { PermissionsModule } from 'src/engine/metadata-modules/permissions/permissions.module';
@@ -56,6 +57,7 @@ import { SystemPromptBuilderService } from './services/system-prompt-builder.ser
     TokenModule,
     UserWorkspaceModule,
     AiBillingModule,
+    MetricsModule,
     ToolProviderModule,
     DashboardToolsModule,
     WorkflowToolsModule,

packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/services/chat-execution.service.ts

@@ -15,6 +15,9 @@ import {
 import { AppPath } from 'twenty-shared/types';
 import { getAppPath, isDefined } from 'twenty-shared/utils';
 
+import { MetricsService } from 'src/engine/core-modules/metrics/metrics.service';
+import { MetricsKeys } from 'src/engine/core-modules/metrics/types/metrics-keys.type';
+import { type ToolOutput } from 'src/engine/core-modules/tool/types/tool-output.type';
 import { UsageOperationType } from 'src/engine/core-modules/usage/enums/usage-operation-type.enum';
 
 import { type CodeExecutionStreamEmitter } from 'src/engine/core-modules/tool-provider/interfaces/code-execution-stream-emitter.type';
@@ -95,6 +98,7 @@ export class ChatExecutionService {
     private readonly exceptionHandlerService: ExceptionHandlerService,
     private readonly nativeToolBinder: NativeToolBinderService,
     private readonly messagePruningService: MessagePruningService,
+    private readonly metricsService: MetricsService,
   ) {}
 
   async streamChat({
@@ -275,6 +279,9 @@ export class ChatExecutionService {
     const modelMessages = pruningResult.messages;
 
     let hasNoMoreAvailableCredits = false;
+    const streamStartedAt = performance.now();
+    let stepStartedAt = streamStartedAt;
+    let ttftRecorded = false;
 
     const emitTurnUsageEvent = async (steps: StepResult<ToolSet>[]) => {
       const usage = steps.reduce<LanguageModelUsage>(
@@ -347,6 +354,35 @@ export class ChatExecutionService {
         workspace.id,
         userWorkspaceId,
       );
+
+      const modelAttr = { model: registeredModel.modelId };
+
+      this.metricsService.incrementCounterBy({
+        key: MetricsKeys.AiChatInputTokens,
+        amount: usage.inputTokens ?? 0,
+        attributes: modelAttr,
+      });
+      this.metricsService.incrementCounterBy({
+        key: MetricsKeys.AiChatOutputTokens,
+        amount: usage.outputTokens ?? 0,
+        attributes: modelAttr,
+      });
+      this.metricsService.incrementCounterBy({
+        key: MetricsKeys.AiChatCacheReadTokens,
+        amount: usage.inputTokenDetails?.cacheReadTokens ?? 0,
+        attributes: modelAttr,
+      });
+      this.metricsService.incrementCounterBy({
+        key: MetricsKeys.AiChatCacheWriteTokens,
+        amount: cacheCreationTokens,
+        attributes: modelAttr,
+      });
+      this.metricsService.recordHistogram({
+        key: MetricsKeys.AiChatTurnLatencyMs,
+        value: performance.now() - streamStartedAt,
+        unit: 'ms',
+        attributes: modelAttr,
+      });
     };
 
     const stream = streamText({
@@ -360,10 +396,35 @@ export class ChatExecutionService {
       providerOptions: getCallLevelCacheProviderOptions(
         registeredModel.sdkPackage,
       ),
-      prepareStep: ({ messages }) => ({
-        messages: injectCacheBreakpoint(messages, registeredModel.sdkPackage),
-      }),
+      prepareStep: ({ messages }) => {
+        stepStartedAt = performance.now();
+
+        return {
+          messages: injectCacheBreakpoint(messages, registeredModel.sdkPackage),
+        };
+      },
+      onChunk: ({ chunk }) => {
+        if (
+          !ttftRecorded &&
+          (chunk.type === 'text-delta' || chunk.type === 'tool-call')
+        ) {
+          ttftRecorded = true;
+          this.metricsService.recordHistogram({
+            key: MetricsKeys.AiChatTtftMs,
+            value: performance.now() - streamStartedAt,
+            unit: 'ms',
+            attributes: { model: registeredModel.modelId },
+          });
+        }
+      },
       onStepFinish: async (step) => {
+        this.metricsService.recordHistogram({
+          key: MetricsKeys.AiChatStepLatencyMs,
+          value: performance.now() - stepStartedAt,
+          unit: 'ms',
+          attributes: { model: registeredModel.modelId },
+        });
+
         const { hasNoMoreAvailableCredits: stepHasNoMoreAvailableCredits } =
           await this.aiBillingService.decrementAndCheckAvailableCredits(
             registeredModel.modelId,
@@ -379,6 +440,22 @@ export class ChatExecutionService {
         if (stepHasNoMoreAvailableCredits) {
           hasNoMoreAvailableCredits = true;
         }
+
+        for (const toolResult of step.toolResults) {
+          const output = toolResult.output as ToolOutput | undefined;
+
+          if (!isDefined(output?.success)) {
+            continue;
+          }
+
+          void this.metricsService.incrementCounterForEvent({
+            key: output.success
+              ? MetricsKeys.AiChatToolExecutionSucceeded
+              : MetricsKeys.AiChatToolExecutionFailed,
+            attributes: { model: registeredModel.modelId },
+            shouldStoreInCache: false,
+          });
+        }
       },
       onAbort: async ({ steps }) => {
         await emitTurnUsageEvent(steps);