feat(ai) - add observability (#20850)

**AI Chat - Tool Executions (counters, tagged with model)**
ai-chat/tool-execution-succeeded: number of tool calls invoked by the AI
that completed without error
ai-chat/tool-execution-failed: number of tool calls invoked by the AI
that threw an error
**AI Chat - Token Usage (counters, tagged with model)**
ai-chat/input-tokens: total input tokens sent to the model across all
turns
ai-chat/output-tokens: total output tokens generated by the model
ai-chat/cache-read-tokens: input tokens served from the model's prompt
cache (cheaper)
ai-chat/cache-write-tokens: input tokens written into the prompt cache
for future reuse
**AI Chat - Latency (histograms in ms, tagged with model)**
ai-chat/turn-latency-ms: total duration of a full chat turn (from stream
start to stream end)
ai-chat/step-latency-ms: duration of a single reasoning/tool-call step
within a turn
ai-chat/ttft-ms: time-to-first-token, i.e. how long until the model
starts streaming output
**MCP - Tool Executions (counters)**
mcp/tool-execution-succeeded: number of MCP tool calls that completed
successfully
mcp/tool-execution-failed: number of MCP tool calls that threw an error
This commit is contained in:
Etienne
2026-05-22 17:32:51 +02:00
committed by GitHub
parent de044f4b45
commit eda41b4eba
23 changed files with 169 additions and 33 deletions
@@ -385,7 +385,7 @@ export abstract class CommonBaseQueryRunnerService<
longConfig.timeWindow,
);
} catch (error) {
await this.metricsService.incrementCounter({
await this.metricsService.incrementCounterForEvent({
key: MetricsKeys.CommonApiQueryRateLimited,
shouldStoreInCache: false,
});
@@ -5,6 +5,7 @@ import { McpAuthGuard } from 'src/engine/api/mcp/guards/mcp-auth.guard';
import { McpProtocolService } from 'src/engine/api/mcp/services/mcp-protocol.service';
import { McpToolExecutorService } from 'src/engine/api/mcp/services/mcp-tool-executor.service';
import { ApiKeyModule } from 'src/engine/core-modules/api-key/api-key.module';
import { MetricsModule } from 'src/engine/core-modules/metrics/metrics.module';
import { TokenModule } from 'src/engine/core-modules/auth/token/token.module';
import { TwentyConfigModule } from 'src/engine/core-modules/twenty-config/twenty-config.module';
import { ToolProviderModule } from 'src/engine/core-modules/tool-provider/tool-provider.module';
@@ -17,6 +18,7 @@ import { WorkspaceCacheStorageModule } from 'src/engine/workspace-cache-storage/
@Module({
imports: [
ApiKeyModule,
MetricsModule,
TokenModule,
WorkspaceCacheStorageModule,
UserRoleModule,
@@ -4,13 +4,18 @@ import {
MCP_PROGRESS_NOTIFICATION_METHOD,
TOOL_CALL_PROGRESS_TOKEN_PREFIX,
} from 'src/engine/api/mcp/constants/mcp-progress-notification.const';
import { MetricsService } from 'src/engine/core-modules/metrics/metrics.service';
import { McpToolExecutorService } from 'src/engine/api/mcp/services/mcp-tool-executor.service';
describe('McpToolExecutorService', () => {
let service: McpToolExecutorService;
beforeEach(() => {
service = new McpToolExecutorService();
const metricsService = {
incrementCounterBy: jest.fn().mockResolvedValue(undefined),
} as unknown as MetricsService;
service = new McpToolExecutorService(metricsService);
});
describe('handleToolsListing', () => {
@@ -3,6 +3,9 @@ import { Injectable } from '@nestjs/common';
import { type ToolSet } from 'ai';
import { isDefined } from 'twenty-shared/utils';
import { MetricsKeys } from 'src/engine/core-modules/metrics/types/metrics-keys.type';
import { MetricsService } from 'src/engine/core-modules/metrics/metrics.service';
import { JSON_RPC_ERROR_CODE } from 'src/engine/api/mcp/constants/json-rpc-error-code.const';
import {
MCP_PROGRESS_NOTIFICATION_METHOD,
@@ -22,6 +25,8 @@ const unwrapJsonSchema = (schema: unknown) =>
@Injectable()
export class McpToolExecutorService {
constructor(private readonly metricsService: MetricsService) {}
async handleToolCall(
id: string | number,
toolSet: ToolSet,
@@ -58,6 +63,11 @@ export class McpToolExecutorService {
messages: [],
});
void this.metricsService.incrementCounterBy({
key: MetricsKeys.McpToolExecutionSucceeded,
amount: 1,
});
return wrapJsonRpcResponse(id, {
result: {
content: [{ type: 'text', text: JSON.stringify(result) }],
@@ -65,6 +75,11 @@ export class McpToolExecutorService {
},
});
} catch (executionError) {
void this.metricsService.incrementCounterBy({
key: MetricsKeys.McpToolExecutionFailed,
amount: 1,
});
return wrapJsonRpcResponse(id, {
result: {
content: [