feat(ai) - add observability (#20850)
**AI Chat - Tool Executions (counters, tagged with model)** ai-chat/tool-execution-succeeded: number of tool calls invoked by the AI that completed without error ai-chat/tool-execution-failed: number of tool calls invoked by the AI that threw an error **AI Chat - Token Usage (counters, tagged with model)** ai-chat/input-tokens: total input tokens sent to the model across all turns ai-chat/output-tokens: total output tokens generated by the model ai-chat/cache-read-tokens: input tokens served from the model's prompt cache (cheaper) ai-chat/cache-write-tokens: input tokens written into the prompt cache for future reuse **AI Chat - Latency (histograms in ms, tagged with model)** ai-chat/turn-latency-ms: total duration of a full chat turn (from stream start to stream end) ai-chat/step-latency-ms: duration of a single reasoning/tool-call step within a turn ai-chat/ttft-ms: time-to-first-token, i.e. how long until the model starts streaming output **MCP - Tool Executions (counters)** mcp/tool-execution-succeeded: number of MCP tool calls that completed successfully mcp/tool-execution-failed: number of MCP tool calls that threw an error
This commit is contained in:
+1
-1
@@ -385,7 +385,7 @@ export abstract class CommonBaseQueryRunnerService<
|
||||
longConfig.timeWindow,
|
||||
);
|
||||
} catch (error) {
|
||||
await this.metricsService.incrementCounter({
|
||||
await this.metricsService.incrementCounterForEvent({
|
||||
key: MetricsKeys.CommonApiQueryRateLimited,
|
||||
shouldStoreInCache: false,
|
||||
});
|
||||
|
||||
@@ -5,6 +5,7 @@ import { McpAuthGuard } from 'src/engine/api/mcp/guards/mcp-auth.guard';
|
||||
import { McpProtocolService } from 'src/engine/api/mcp/services/mcp-protocol.service';
|
||||
import { McpToolExecutorService } from 'src/engine/api/mcp/services/mcp-tool-executor.service';
|
||||
import { ApiKeyModule } from 'src/engine/core-modules/api-key/api-key.module';
|
||||
import { MetricsModule } from 'src/engine/core-modules/metrics/metrics.module';
|
||||
import { TokenModule } from 'src/engine/core-modules/auth/token/token.module';
|
||||
import { TwentyConfigModule } from 'src/engine/core-modules/twenty-config/twenty-config.module';
|
||||
import { ToolProviderModule } from 'src/engine/core-modules/tool-provider/tool-provider.module';
|
||||
@@ -17,6 +18,7 @@ import { WorkspaceCacheStorageModule } from 'src/engine/workspace-cache-storage/
|
||||
@Module({
|
||||
imports: [
|
||||
ApiKeyModule,
|
||||
MetricsModule,
|
||||
TokenModule,
|
||||
WorkspaceCacheStorageModule,
|
||||
UserRoleModule,
|
||||
|
||||
+6
-1
@@ -4,13 +4,18 @@ import {
|
||||
MCP_PROGRESS_NOTIFICATION_METHOD,
|
||||
TOOL_CALL_PROGRESS_TOKEN_PREFIX,
|
||||
} from 'src/engine/api/mcp/constants/mcp-progress-notification.const';
|
||||
import { MetricsService } from 'src/engine/core-modules/metrics/metrics.service';
|
||||
import { McpToolExecutorService } from 'src/engine/api/mcp/services/mcp-tool-executor.service';
|
||||
|
||||
describe('McpToolExecutorService', () => {
|
||||
let service: McpToolExecutorService;
|
||||
|
||||
beforeEach(() => {
|
||||
service = new McpToolExecutorService();
|
||||
const metricsService = {
|
||||
incrementCounterBy: jest.fn().mockResolvedValue(undefined),
|
||||
} as unknown as MetricsService;
|
||||
|
||||
service = new McpToolExecutorService(metricsService);
|
||||
});
|
||||
|
||||
describe('handleToolsListing', () => {
|
||||
|
||||
@@ -3,6 +3,9 @@ import { Injectable } from '@nestjs/common';
|
||||
import { type ToolSet } from 'ai';
|
||||
import { isDefined } from 'twenty-shared/utils';
|
||||
|
||||
import { MetricsKeys } from 'src/engine/core-modules/metrics/types/metrics-keys.type';
|
||||
import { MetricsService } from 'src/engine/core-modules/metrics/metrics.service';
|
||||
|
||||
import { JSON_RPC_ERROR_CODE } from 'src/engine/api/mcp/constants/json-rpc-error-code.const';
|
||||
import {
|
||||
MCP_PROGRESS_NOTIFICATION_METHOD,
|
||||
@@ -22,6 +25,8 @@ const unwrapJsonSchema = (schema: unknown) =>
|
||||
|
||||
@Injectable()
|
||||
export class McpToolExecutorService {
|
||||
constructor(private readonly metricsService: MetricsService) {}
|
||||
|
||||
async handleToolCall(
|
||||
id: string | number,
|
||||
toolSet: ToolSet,
|
||||
@@ -58,6 +63,11 @@ export class McpToolExecutorService {
|
||||
messages: [],
|
||||
});
|
||||
|
||||
void this.metricsService.incrementCounterBy({
|
||||
key: MetricsKeys.McpToolExecutionSucceeded,
|
||||
amount: 1,
|
||||
});
|
||||
|
||||
return wrapJsonRpcResponse(id, {
|
||||
result: {
|
||||
content: [{ type: 'text', text: JSON.stringify(result) }],
|
||||
@@ -65,6 +75,11 @@ export class McpToolExecutorService {
|
||||
},
|
||||
});
|
||||
} catch (executionError) {
|
||||
void this.metricsService.incrementCounterBy({
|
||||
key: MetricsKeys.McpToolExecutionFailed,
|
||||
amount: 1,
|
||||
});
|
||||
|
||||
return wrapJsonRpcResponse(id, {
|
||||
result: {
|
||||
content: [
|
||||
|
||||
Reference in New Issue
Block a user