From dc8c62a7a9b15c183f51314c5b4b0d1b37c527c8 Mon Sep 17 00:00:00 2001 From: Paul Rastoin <45004772+prastoin@users.noreply.github.com> Date: Wed, 5 Aug 2026 18:40:32 +0200 Subject: [PATCH] Add OpenTelemetry metrics to workspace migration build and run (#23797) ## Context ## AI generated graph goal example image image The workspace migration builder and runner already log phase timings via `logger.perfTime`, but those logs are only useful for local debugging: they're gated behind the `performance` log level and never leave the process. This PR routes the same timings into the existing `MetricsService` (OpenTelemetry) so migration performance can be tracked over time and regressions detected. ## What's recorded All metrics are histograms recorded via `MetricsService.recordHistogram`, which is a synchronous in-memory aggregation - export happens on the background OTel readers (Prometheus scrape or periodic OTLP push), so nothing is added to the migration hot path. When `METER_DRIVER` is unset, records are no-ops. | Metric | Where | Attributes | |---|---|---| | `workspace-migration/build-duration-ms` | `WorkspaceMigrationValidateBuildAndRunService` | `status`: `success`, `fail` (validation errors), `error` (builder threw) | | `workspace-migration/build-entity-duration-ms` | `WorkspaceEntityMigrationBuilderService.validateAndBuild` | `metadataName`, `status` | | `workspace-migration/build-entity-phase-duration-ms` | entity builder | `metadataName`, `phase`: `matrix-computation`, `deletion-validation`, `creation-validation`, `update-validation` | | `workspace-migration/run-duration-ms` | `WorkspaceMigrationRunnerService.run` | `status`: `success`, `fail` - recorded on every exit path, including pre-transaction failures (DDL locked, cache retrieval, application not found) | | `workspace-migration/run-phase-duration-ms` | runner | `phase`: `initial-cache-retrieval`, `action-execution`, `commit`, `cache-invalidation`; `status` - on failure the elapsed transaction time is recorded as `action-execution` with `status: 'fail'` (the action/commit split is unknowable mid-failure) | | `workspace-migration/action-duration-ms` | base action handler wrapper | `actionType`, `metadataName`, `step`, `status` - failed steps are recorded via try/finally | | `workspace-migration/action-count` | validate-build-and-run service | - | Phase notes: phases are disjoint (commit is subtracted from transaction time) but not exhaustive - connection setup, the application-map fetch, rollback, and after-commit side effects sit outside them, so the stack approximates rather than equals the run total. The `cache-invalidation` phase is recorded only at the run's post-commit call site, not inside the public `invalidateCache`, so standalone callers (upgrade backfill commands, `FlatCacheInvalidateCommand`) don't pollute the series. ## Implementation notes - Durations are measured with local `performance.now()` instead of reusing `perfTime`/`perfTimeEnd`, so metrics are recorded regardless of the `performance` log level and are immune to key collisions in the logger's shared timer map under concurrent migrations. Existing perf logs are untouched. - `WorkspaceMigrationRunnerService.run` is now a thin wrapper around the previous body (`executeRun`) so the total run duration and its success/fail status are recorded in one place for every throw path. - No `workspaceId` in attributes to keep cardinality bounded. - Two new bucket-boundary constants follow the existing `AI_LATENCY_MS_BUCKET_BOUNDARIES` pattern: durations 5ms-120s, action counts 1-5000. - `MetricsModule` imported into `WorkspaceMigrationModule`, `WorkspaceMigrationRunnerModule`, `WorkspaceMigrationBuilderModule`, and the action-handlers module. ## Test - `nx typecheck twenty-server` passes - oxlint + oxfmt clean on changed files - `workspace.service.spec.ts` passes --- ...action-count-bucket-boundaries.constant.ts | 3 + ...-duration-ms-bucket-boundaries.constant.ts | 3 + .../metrics/types/metrics-keys.type.ts | 7 ++ ...igration-validate-build-and-run-service.ts | 27 +++++ ...kspace-entity-migration-builder.service.ts | 82 +++++++++++++++ .../workspace-migration-builder.module.ts | 7 +- ...migration-runner-action-handlers.module.ts | 2 + ...runner-action-handler-service.interface.ts | 35 ++++++- .../workspace-migration-runner.service.ts | 99 ++++++++++++++++++- .../workspace-migration-runner.module.ts | 2 + .../workspace-migration.module.ts | 2 + 11 files changed, 265 insertions(+), 4 deletions(-) create mode 100644 packages/twenty-server/src/engine/core-modules/metrics/constants/workspace-migration-action-count-bucket-boundaries.constant.ts create mode 100644 packages/twenty-server/src/engine/core-modules/metrics/constants/workspace-migration-duration-ms-bucket-boundaries.constant.ts diff --git a/packages/twenty-server/src/engine/core-modules/metrics/constants/workspace-migration-action-count-bucket-boundaries.constant.ts b/packages/twenty-server/src/engine/core-modules/metrics/constants/workspace-migration-action-count-bucket-boundaries.constant.ts new file mode 100644 index 0000000000..56396ba2db --- /dev/null +++ b/packages/twenty-server/src/engine/core-modules/metrics/constants/workspace-migration-action-count-bucket-boundaries.constant.ts @@ -0,0 +1,3 @@ +export const WORKSPACE_MIGRATION_ACTION_COUNT_BUCKET_BOUNDARIES = [ + 1, 5, 10, 25, 50, 100, 250, 500, 1000, 2500, 5000, +] as const; diff --git a/packages/twenty-server/src/engine/core-modules/metrics/constants/workspace-migration-duration-ms-bucket-boundaries.constant.ts b/packages/twenty-server/src/engine/core-modules/metrics/constants/workspace-migration-duration-ms-bucket-boundaries.constant.ts new file mode 100644 index 0000000000..80c701c515 --- /dev/null +++ b/packages/twenty-server/src/engine/core-modules/metrics/constants/workspace-migration-duration-ms-bucket-boundaries.constant.ts @@ -0,0 +1,3 @@ +export const WORKSPACE_MIGRATION_DURATION_MS_BUCKET_BOUNDARIES = [ + 5, 10, 25, 50, 100, 250, 500, 1000, 2500, 5000, 10000, 30000, 60000, 120000, +] as const; diff --git a/packages/twenty-server/src/engine/core-modules/metrics/types/metrics-keys.type.ts b/packages/twenty-server/src/engine/core-modules/metrics/types/metrics-keys.type.ts index 736b484e0c..8aa0d5b1ca 100644 --- a/packages/twenty-server/src/engine/core-modules/metrics/types/metrics-keys.type.ts +++ b/packages/twenty-server/src/engine/core-modules/metrics/types/metrics-keys.type.ts @@ -83,4 +83,11 @@ export enum MetricsKeys { ConnectedAccountWebhookSubscriptionRenewalFailed = 'connected-account-webhook-subscription/renewal-failed', ConnectedAccountWebhookSubscriptionDeleted = 'connected-account-webhook-subscription/deleted', ConnectedAccountWebhookSubscriptionDeletionFailed = 'connected-account-webhook-subscription/deletion-failed', + WorkspaceMigrationBuildDurationMs = 'workspace-migration/build-duration-ms', + WorkspaceMigrationBuildEntityDurationMs = 'workspace-migration/build-entity-duration-ms', + WorkspaceMigrationBuildEntityPhaseDurationMs = 'workspace-migration/build-entity-phase-duration-ms', + WorkspaceMigrationRunDurationMs = 'workspace-migration/run-duration-ms', + WorkspaceMigrationRunPhaseDurationMs = 'workspace-migration/run-phase-duration-ms', + WorkspaceMigrationActionDurationMs = 'workspace-migration/action-duration-ms', + WorkspaceMigrationActionCount = 'workspace-migration/action-count', } diff --git a/packages/twenty-server/src/engine/workspace-manager/workspace-migration/services/workspace-migration-validate-build-and-run-service.ts b/packages/twenty-server/src/engine/workspace-manager/workspace-migration/services/workspace-migration-validate-build-and-run-service.ts index a9ad88fc56..8d38daee74 100644 --- a/packages/twenty-server/src/engine/workspace-manager/workspace-migration/services/workspace-migration-validate-build-and-run-service.ts +++ b/packages/twenty-server/src/engine/workspace-manager/workspace-migration/services/workspace-migration-validate-build-and-run-service.ts @@ -6,6 +6,10 @@ import { } from 'twenty-shared/metadata'; import { LoggerService } from 'src/engine/core-modules/logger/logger.service'; +import { WORKSPACE_MIGRATION_ACTION_COUNT_BUCKET_BOUNDARIES } from 'src/engine/core-modules/metrics/constants/workspace-migration-action-count-bucket-boundaries.constant'; +import { WORKSPACE_MIGRATION_DURATION_MS_BUCKET_BOUNDARIES } from 'src/engine/core-modules/metrics/constants/workspace-migration-duration-ms-bucket-boundaries.constant'; +import { MetricsService } from 'src/engine/core-modules/metrics/metrics.service'; +import { MetricsKeys } from 'src/engine/core-modules/metrics/types/metrics-keys.type'; import { TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service'; import { AllFlatEntityOperationRecordByMetadataName } from 'src/engine/metadata-modules/flat-entity/types/all-flat-entity-operation-record-by-metadata-name.type'; import { AllFlatEntityOperationByMetadataName } from 'src/engine/metadata-modules/flat-entity/types/flat-entity-to-create-delete-update.type'; @@ -71,6 +75,7 @@ export class WorkspaceMigrationValidateBuildAndRunService { private readonly workspaceMigrationFlatEntityMapsService: WorkspaceMigrationFlatEntityMapsService, private readonly metadataEventEmitter: MetadataEventEmitter, private readonly metadataSideEffectEngineService: MetadataSideEffectEngineService, + private readonly metricsService: MetricsService, private readonly logger: LoggerService, twentyConfigService: TwentyConfigService, ) { @@ -98,6 +103,14 @@ export class WorkspaceMigrationValidateBuildAndRunService { await this.workspaceMigrationBuildOrchestratorService .buildWorkspaceMigration(buildArgs) .catch((error) => { + this.metricsService.recordHistogram({ + key: MetricsKeys.WorkspaceMigrationBuildDurationMs, + value: performance.now() - buildStart, + unit: 'ms', + attributes: { status: 'error' }, + bucketBoundaries: WORKSPACE_MIGRATION_DURATION_MS_BUCKET_BOUNDARIES, + }); + this.logger.error( error, WorkspaceMigrationValidateBuildAndRunService.name, @@ -110,6 +123,14 @@ export class WorkspaceMigrationValidateBuildAndRunService { }); const buildMs = performance.now() - buildStart; + this.metricsService.recordHistogram({ + key: MetricsKeys.WorkspaceMigrationBuildDurationMs, + value: buildMs, + unit: 'ms', + attributes: { status: validateAndBuildResult.status }, + bucketBoundaries: WORKSPACE_MIGRATION_DURATION_MS_BUCKET_BOUNDARIES, + }); + this.logger.perf( `[install-perf] buildWorkspaceMigration took ${buildMs.toFixed(1)}ms (status=${validateAndBuildResult.status})`, WorkspaceMigrationValidateBuildAndRunService.name, @@ -154,6 +175,12 @@ export class WorkspaceMigrationValidateBuildAndRunService { WorkspaceMigrationValidateBuildAndRunService.name, ); + this.metricsService.recordHistogram({ + key: MetricsKeys.WorkspaceMigrationActionCount, + value: workspaceMigration.actions.length, + bucketBoundaries: WORKSPACE_MIGRATION_ACTION_COUNT_BUCKET_BOUNDARIES, + }); + const runStart = performance.now(); const { hasSchemaMetadataChanged, metadataEvents } = await this.workspaceMigrationRunnerService.run({ diff --git a/packages/twenty-server/src/engine/workspace-manager/workspace-migration/workspace-migration-builder/services/workspace-entity-migration-builder.service.ts b/packages/twenty-server/src/engine/workspace-manager/workspace-migration/workspace-migration-builder/services/workspace-entity-migration-builder.service.ts index 59c98990d6..f4384fe6a1 100644 --- a/packages/twenty-server/src/engine/workspace-manager/workspace-migration/workspace-migration-builder/services/workspace-entity-migration-builder.service.ts +++ b/packages/twenty-server/src/engine/workspace-manager/workspace-migration/workspace-migration-builder/services/workspace-entity-migration-builder.service.ts @@ -6,6 +6,9 @@ import { isDefined } from 'twenty-shared/utils'; import { validate as uuidValidate, version as uuidVersion } from 'uuid'; import { LoggerService } from 'src/engine/core-modules/logger/logger.service'; +import { WORKSPACE_MIGRATION_DURATION_MS_BUCKET_BOUNDARIES } from 'src/engine/core-modules/metrics/constants/workspace-migration-duration-ms-bucket-boundaries.constant'; +import { MetricsService } from 'src/engine/core-modules/metrics/metrics.service'; +import { MetricsKeys } from 'src/engine/core-modules/metrics/types/metrics-keys.type'; import { FlatEntityMapsException, FlatEntityMapsExceptionCode, @@ -56,6 +59,10 @@ export abstract class WorkspaceEntityMigrationBuilderService< > { @Inject(LoggerService) protected readonly logger: LoggerService; + + @Inject(MetricsService) + protected readonly metricsService: MetricsService; + private metadataName: T; constructor(metadataName: T) { @@ -80,6 +87,9 @@ export abstract class WorkspaceEntityMigrationBuilderService< 'matrix computation', ); + const validateAndBuildStart = performance.now(); + const matrixComputationStart = performance.now(); + const fromFlatEntities = Object.values( fromFlatEntityMaps.byUniversalIdentifier, ).filter(isDefined); @@ -107,6 +117,11 @@ export abstract class WorkspaceEntityMigrationBuilderService< 'entity processing', ); + this.recordBuildEntityPhaseMetric({ + phase: 'matrix-computation', + startedAt: matrixComputationStart, + }); + const flatEntityMapsKey = getMetadataFlatEntityMapsKey(this.metadataName); const actionsResult = getMetadataEmptyWorkspaceMigrationActionRecord( this.metadataName, @@ -119,6 +134,8 @@ export abstract class WorkspaceEntityMigrationBuilderService< 'deletion validation', ); + const deletionValidationStart = performance.now(); + const remainingFlatEntityMapsToDelete = structuredClone( deletedFlatEntityMaps, ); @@ -195,6 +212,13 @@ export abstract class WorkspaceEntityMigrationBuilderService< 'creation validation', ); + this.recordBuildEntityPhaseMetric({ + phase: 'deletion-validation', + startedAt: deletionValidationStart, + }); + + const creationValidationStart = performance.now(); + const remainingFlatEntityMapsToCreate = structuredClone( createdFlatEntityMaps, ); @@ -275,6 +299,13 @@ export abstract class WorkspaceEntityMigrationBuilderService< 'update validation', ); + this.recordBuildEntityPhaseMetric({ + phase: 'creation-validation', + startedAt: creationValidationStart, + }); + + const updateValidationStart = performance.now(); + for (const flatEntityToUpdateUniversalIdentifier in updatedFlatEntityMaps.byUniversalIdentifier) { const flatEntityUpdate = updatedFlatEntityMaps.byUniversalIdentifier[ @@ -362,13 +393,28 @@ export abstract class WorkspaceEntityMigrationBuilderService< 'entity processing', ); + this.recordBuildEntityPhaseMetric({ + phase: 'update-validation', + startedAt: updateValidationStart, + }); + if (allValidationResult.length > 0) { + this.recordBuildEntityDurationMetric({ + status: 'fail', + startedAt: validateAndBuildStart, + }); + return { status: 'fail', errors: allValidationResult, }; } + this.recordBuildEntityDurationMetric({ + status: 'success', + startedAt: validateAndBuildStart, + }); + this.logger.perfTimeEnd( `EntityBuilder ${this.metadataName}`, 'validateAndBuild', @@ -380,6 +426,42 @@ export abstract class WorkspaceEntityMigrationBuilderService< }; } + private recordBuildEntityDurationMetric({ + status, + startedAt, + }: { + status: 'success' | 'fail'; + startedAt: number; + }): void { + this.metricsService.recordHistogram({ + key: MetricsKeys.WorkspaceMigrationBuildEntityDurationMs, + value: performance.now() - startedAt, + unit: 'ms', + attributes: { metadataName: this.metadataName, status }, + bucketBoundaries: WORKSPACE_MIGRATION_DURATION_MS_BUCKET_BOUNDARIES, + }); + } + + private recordBuildEntityPhaseMetric({ + phase, + startedAt, + }: { + phase: + | 'matrix-computation' + | 'deletion-validation' + | 'creation-validation' + | 'update-validation'; + startedAt: number; + }): void { + this.metricsService.recordHistogram({ + key: MetricsKeys.WorkspaceMigrationBuildEntityPhaseDurationMs, + value: performance.now() - startedAt, + unit: 'ms', + attributes: { metadataName: this.metadataName, phase }, + bucketBoundaries: WORKSPACE_MIGRATION_DURATION_MS_BUCKET_BOUNDARIES, + }); + } + private validateUniversalIdentifier({ flatEntityToValidate: { universalIdentifier }, }: UniversalFlatEntityValidationArgs): FlatEntityValidationError[] { diff --git a/packages/twenty-server/src/engine/workspace-manager/workspace-migration/workspace-migration-builder/workspace-migration-builder.module.ts b/packages/twenty-server/src/engine/workspace-manager/workspace-migration/workspace-migration-builder/workspace-migration-builder.module.ts index fb7406e2b6..176d67f7c2 100644 --- a/packages/twenty-server/src/engine/workspace-manager/workspace-migration/workspace-migration-builder/workspace-migration-builder.module.ts +++ b/packages/twenty-server/src/engine/workspace-manager/workspace-migration/workspace-migration-builder/workspace-migration-builder.module.ts @@ -1,6 +1,7 @@ import { Module } from '@nestjs/common'; import { FeatureFlagModule } from 'src/engine/core-modules/feature-flag/feature-flag.module'; +import { MetricsModule } from 'src/engine/core-modules/metrics/metrics.module'; import { FlatFieldMetadataTypeValidatorService } from 'src/engine/metadata-modules/flat-field-metadata/services/flat-field-metadata-type-validator.service'; import { WorkspaceMigrationAgentActionsBuilderService } from 'src/engine/workspace-manager/workspace-migration/workspace-migration-builder/builders/agent/workspace-migration-agent-actions-builder.service'; import { WorkspaceMigrationCommandMenuItemActionsBuilderService } from 'src/engine/workspace-manager/workspace-migration/workspace-migration-builder/builders/command-menu-item/workspace-migration-command-menu-item-actions-builder.service'; @@ -36,7 +37,11 @@ import { WorkspaceMigrationSearchFieldMetadataActionsBuilderService } from 'src/ import { WorkspaceMigrationBuilderValidatorsModule } from 'src/engine/workspace-manager/workspace-migration/workspace-migration-builder/validators/workspace-migration-builder-validators.module'; @Module({ - imports: [FeatureFlagModule, WorkspaceMigrationBuilderValidatorsModule], + imports: [ + FeatureFlagModule, + WorkspaceMigrationBuilderValidatorsModule, + MetricsModule, + ], providers: [ FlatFieldMetadataTypeValidatorService, WorkspaceMigrationObjectActionsBuilderService, diff --git a/packages/twenty-server/src/engine/workspace-manager/workspace-migration/workspace-migration-runner/action-handlers/workspace-schema-migration-runner-action-handlers.module.ts b/packages/twenty-server/src/engine/workspace-manager/workspace-migration/workspace-migration-runner/action-handlers/workspace-schema-migration-runner-action-handlers.module.ts index 8fa05ac9eb..dc8ed7c32f 100644 --- a/packages/twenty-server/src/engine/workspace-manager/workspace-migration/workspace-migration-runner/action-handlers/workspace-schema-migration-runner-action-handlers.module.ts +++ b/packages/twenty-server/src/engine/workspace-manager/workspace-migration/workspace-migration-runner/action-handlers/workspace-schema-migration-runner-action-handlers.module.ts @@ -2,6 +2,7 @@ import { Module } from '@nestjs/common'; import { TypeOrmModule } from '@nestjs/typeorm'; import { ApplicationEntity } from 'src/engine/core-modules/application/application.entity'; +import { MetricsModule } from 'src/engine/core-modules/metrics/metrics.module'; import { SecretEncryptionModule } from 'src/engine/core-modules/secret-encryption/secret-encryption.module'; import { WorkspaceSchemaManagerModule } from 'src/engine/twenty-orm/workspace-schema-manager/workspace-schema-manager.module'; import { CreateAgentActionHandlerService } from 'src/engine/workspace-manager/workspace-migration/workspace-migration-runner/action-handlers/agent/services/create-agent-action-handler.service'; @@ -103,6 +104,7 @@ import { UpdateSearchFieldMetadataActionHandlerService } from 'src/engine/worksp TypeOrmModule.forFeature([ApplicationEntity]), WorkspaceSchemaManagerModule, SecretEncryptionModule, + MetricsModule, ], providers: [ CreateFieldActionHandlerService, diff --git a/packages/twenty-server/src/engine/workspace-manager/workspace-migration/workspace-migration-runner/interfaces/workspace-migration-runner-action-handler-service.interface.ts b/packages/twenty-server/src/engine/workspace-manager/workspace-migration/workspace-migration-runner/interfaces/workspace-migration-runner-action-handler-service.interface.ts index ccc0635e70..d4807fb4c5 100644 --- a/packages/twenty-server/src/engine/workspace-manager/workspace-migration/workspace-migration-runner/interfaces/workspace-migration-runner-action-handler-service.interface.ts +++ b/packages/twenty-server/src/engine/workspace-manager/workspace-migration/workspace-migration-runner/interfaces/workspace-migration-runner-action-handler-service.interface.ts @@ -4,6 +4,9 @@ import { AllMetadataName } from 'twenty-shared/metadata'; import { QueryRunner } from 'typeorm'; import { LoggerService } from 'src/engine/core-modules/logger/logger.service'; +import { WORKSPACE_MIGRATION_DURATION_MS_BUCKET_BOUNDARIES } from 'src/engine/core-modules/metrics/constants/workspace-migration-duration-ms-bucket-boundaries.constant'; +import { MetricsService } from 'src/engine/core-modules/metrics/metrics.service'; +import { MetricsKeys } from 'src/engine/core-modules/metrics/types/metrics-keys.type'; import { ALL_METADATA_ENTITY_BY_METADATA_NAME } from 'src/engine/metadata-modules/flat-entity/constant/all-metadata-entity-by-metadata-name.constant'; import { type AllFlatEntityMaps } from 'src/engine/metadata-modules/flat-entity/types/all-flat-entity-maps.type'; import { FlatEntityMaps } from 'src/engine/metadata-modules/flat-entity/types/flat-entity-maps.type'; @@ -75,6 +78,9 @@ export abstract class BaseWorkspaceMigrationRunnerActionHandlerService< @Inject(LoggerService) protected readonly logger: LoggerService; + @Inject(MetricsService) + protected readonly metricsService: MetricsService; + public abstract transpileUniversalActionToFlatAction( context: WorkspaceMigrationActionRunnerArgs, ): Promise; @@ -320,18 +326,43 @@ export abstract class BaseWorkspaceMigrationRunnerActionHandlerService< label, method, }: { - label: string; + label: 'executeForMetadata' | 'executeForWorkspaceSchema'; method: () => Promise; }): Promise { + const startedAt = performance.now(); + + const recordActionDuration = (status: 'success' | 'fail') => + this.metricsService.recordHistogram({ + key: MetricsKeys.WorkspaceMigrationActionDurationMs, + value: performance.now() - startedAt, + unit: 'ms', + attributes: { + actionType: this.actionType, + metadataName: this.metadataName, + step: label, + status, + }, + bucketBoundaries: WORKSPACE_MIGRATION_DURATION_MS_BUCKET_BOUNDARIES, + }); + this.logger.perfTime( 'BaseWorkspaceMigrationRunnerActionHandlerService', `${this.actionType}_${this.metadataName} ${label}`, ); - await method(); + + try { + await method(); + } catch (error) { + recordActionDuration('fail'); + throw error; + } + this.logger.perfTimeEnd( 'BaseWorkspaceMigrationRunnerActionHandlerService', `${this.actionType}_${this.metadataName} ${label}`, ); + + recordActionDuration('success'); } } diff --git a/packages/twenty-server/src/engine/workspace-manager/workspace-migration/workspace-migration-runner/services/workspace-migration-runner.service.ts b/packages/twenty-server/src/engine/workspace-manager/workspace-migration/workspace-migration-runner/services/workspace-migration-runner.service.ts index 7434cbb771..2c55e328e4 100644 --- a/packages/twenty-server/src/engine/workspace-manager/workspace-migration/workspace-migration-runner/services/workspace-migration-runner.service.ts +++ b/packages/twenty-server/src/engine/workspace-manager/workspace-migration/workspace-migration-runner/services/workspace-migration-runner.service.ts @@ -6,6 +6,9 @@ import { isDefined } from 'twenty-shared/utils'; import { DataSource } from 'typeorm'; import { LoggerService } from 'src/engine/core-modules/logger/logger.service'; +import { WORKSPACE_MIGRATION_DURATION_MS_BUCKET_BOUNDARIES } from 'src/engine/core-modules/metrics/constants/workspace-migration-duration-ms-bucket-boundaries.constant'; +import { MetricsService } from 'src/engine/core-modules/metrics/metrics.service'; +import { MetricsKeys } from 'src/engine/core-modules/metrics/types/metrics-keys.type'; import { TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service'; import { WorkspaceManyOrAllFlatEntityMapsCacheService } from 'src/engine/metadata-modules/flat-entity/services/workspace-many-or-all-flat-entity-maps-cache.service'; import { AllFlatEntityMaps } from 'src/engine/metadata-modules/flat-entity/types/all-flat-entity-maps.type'; @@ -36,6 +39,7 @@ export class WorkspaceMigrationRunnerService { private readonly workspaceMigrationRunnerActionHandlerRegistry: WorkspaceMigrationRunnerActionHandlerRegistryService, private readonly workspaceMetadataVersionService: WorkspaceMetadataVersionService, private readonly workspaceCacheService: WorkspaceCacheService, + private readonly metricsService: MetricsService, private readonly logger: LoggerService, private readonly twentyConfigService: TwentyConfigService, ) {} @@ -149,6 +153,28 @@ export class WorkspaceMigrationRunnerService { ); } + private recordRunPhaseMetric({ + phase, + status, + value, + }: { + phase: + | 'initial-cache-retrieval' + | 'action-execution' + | 'commit' + | 'cache-invalidation'; + status: 'success' | 'fail'; + value: number; + }): void { + this.metricsService.recordHistogram({ + key: MetricsKeys.WorkspaceMigrationRunPhaseDurationMs, + value, + unit: 'ms', + attributes: { phase, status }, + bucketBoundaries: WORKSPACE_MIGRATION_DURATION_MS_BUCKET_BOUNDARIES, + }); + } + private async logBlockingDbActivity(): Promise { try { // Metadata only (no query text) to avoid logging literals from other sessions. @@ -178,7 +204,42 @@ export class WorkspaceMigrationRunnerService { } } - run = async ({ + run = async (args: { + workspaceMigration: WorkspaceMigration; + workspaceId: string; + }): Promise<{ + allFlatEntityMaps: AllFlatEntityMaps; + metadataEvents: MetadataEvent[]; + hasSchemaMetadataChanged: boolean; + }> => { + const runStart = performance.now(); + + try { + const result = await this.executeRun(args); + + this.metricsService.recordHistogram({ + key: MetricsKeys.WorkspaceMigrationRunDurationMs, + value: performance.now() - runStart, + unit: 'ms', + attributes: { status: 'success' }, + bucketBoundaries: WORKSPACE_MIGRATION_DURATION_MS_BUCKET_BOUNDARIES, + }); + + return result; + } catch (error) { + this.metricsService.recordHistogram({ + key: MetricsKeys.WorkspaceMigrationRunDurationMs, + value: performance.now() - runStart, + unit: 'ms', + attributes: { status: 'fail' }, + bucketBoundaries: WORKSPACE_MIGRATION_DURATION_MS_BUCKET_BOUNDARIES, + }); + + throw error; + } + }; + + private executeRun = async ({ workspaceMigration: { actions, applicationUniversalIdentifier }, workspaceId, }: { @@ -246,6 +307,12 @@ export class WorkspaceMigrationRunnerService { const initialCacheRetrievalMs = performance.now() - initialCacheRetrievalStart; + this.recordRunPhaseMetric({ + phase: 'initial-cache-retrieval', + status: 'success', + value: initialCacheRetrievalMs, + }); + this.logger.perf( `[install-perf] Runner initial cache retrieval (getOrRecomputeManyOrAllFlatEntityMaps) took ${initialCacheRetrievalMs.toFixed(1)}ms for ${allFlatEntityMapsKeys.length} flat-maps keys`, 'Runner', @@ -354,6 +421,18 @@ export class WorkspaceMigrationRunnerService { const commitMs = performance.now() - commitStart; const transactionMs = performance.now() - transactionStart; + this.recordRunPhaseMetric({ + phase: 'action-execution', + status: 'success', + value: transactionMs - commitMs, + }); + + this.recordRunPhaseMetric({ + phase: 'commit', + status: 'success', + value: commitMs, + }); + this.logger.perf( `[install-perf] Runner transaction summary: ${actionCount} actions, total transaction ${transactionMs.toFixed(1)}ms (commit ${commitMs.toFixed(1)}ms), slowest action ${slowestActionLabel} ${slowestActionMs.toFixed(1)}ms`, 'Runner', @@ -361,6 +440,12 @@ export class WorkspaceMigrationRunnerService { this.logger.perfTimeEnd('Runner', 'Transaction execution'); } catch (error) { + this.recordRunPhaseMetric({ + phase: 'action-execution', + status: 'fail', + value: performance.now() - transactionStart, + }); + this.logger.error( `[install-perf] migration failed after ${actionCount} action(s): ${ error instanceof Error ? error.message : String(error) @@ -433,7 +518,19 @@ export class WorkspaceMigrationRunnerService { allFlatEntityMapsKeys, workspaceId, }); + + this.recordRunPhaseMetric({ + phase: 'cache-invalidation', + status: 'success', + value: performance.now() - postCommitInvalidateStart, + }); } catch (cacheError) { + this.recordRunPhaseMetric({ + phase: 'cache-invalidation', + status: 'fail', + value: performance.now() - postCommitInvalidateStart, + }); + this.logger.error( `Cache invalidation failed after committed transaction: ${cacheError}`, 'Runner', diff --git a/packages/twenty-server/src/engine/workspace-manager/workspace-migration/workspace-migration-runner/workspace-migration-runner.module.ts b/packages/twenty-server/src/engine/workspace-manager/workspace-migration/workspace-migration-runner/workspace-migration-runner.module.ts index 2812a3c067..8986b11bc0 100644 --- a/packages/twenty-server/src/engine/workspace-manager/workspace-migration/workspace-migration-runner/workspace-migration-runner.module.ts +++ b/packages/twenty-server/src/engine/workspace-manager/workspace-migration/workspace-migration-runner/workspace-migration-runner.module.ts @@ -5,6 +5,7 @@ import { TypeOrmModule } from '@nestjs/typeorm'; import { WorkspaceIteratorModule } from 'src/database/commands/command-runners/workspace-iterator.module'; import { TypeORMModule } from 'src/database/typeorm/typeorm.module'; import { FeatureFlagModule } from 'src/engine/core-modules/feature-flag/feature-flag.module'; +import { MetricsModule } from 'src/engine/core-modules/metrics/metrics.module'; import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity'; import { WorkspaceManyOrAllFlatEntityMapsCacheModule } from 'src/engine/metadata-modules/flat-entity/services/workspace-many-or-all-flat-entity-maps-cache.module'; import { WorkspaceMetadataVersionModule } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.module'; @@ -27,6 +28,7 @@ import { WorkspaceMigrationRunnerService } from 'src/engine/workspace-manager/wo WorkspaceCacheModule, TypeOrmModule.forFeature([WorkspaceEntity]), WorkspaceIteratorModule, + MetricsModule, ], providers: [ WorkspaceMigrationRunnerService, diff --git a/packages/twenty-server/src/engine/workspace-manager/workspace-migration/workspace-migration.module.ts b/packages/twenty-server/src/engine/workspace-manager/workspace-migration/workspace-migration.module.ts index feb4325fbe..aeb6e1ff40 100644 --- a/packages/twenty-server/src/engine/workspace-manager/workspace-migration/workspace-migration.module.ts +++ b/packages/twenty-server/src/engine/workspace-manager/workspace-migration/workspace-migration.module.ts @@ -1,5 +1,6 @@ import { Module } from '@nestjs/common'; +import { MetricsModule } from 'src/engine/core-modules/metrics/metrics.module'; import { MetadataSideEffectModule } from 'src/engine/metadata-modules/metadata-side-effect/metadata-side-effect.module'; import { WorkspaceCacheModule } from 'src/engine/workspace-cache/workspace-cache.module'; import { WorkspaceMigrationBuildOrchestratorService } from 'src/engine/workspace-manager/workspace-migration/services/workspace-migration-build-orchestrator.service'; @@ -14,6 +15,7 @@ import { WorkspaceMigrationRunnerModule } from 'src/engine/workspace-manager/wor WorkspaceMigrationRunnerModule, WorkspaceCacheModule, MetadataSideEffectModule, + MetricsModule, ], providers: [ WorkspaceMigrationValidateBuildAndRunService,