Add OpenTelemetry metrics to workspace migration build and run (#23797)
## Context ## AI generated graph goal example <img width="2078" height="914" alt="image" src="https://github.com/user-attachments/assets/14c760df-5ac8-4526-a5e0-40b57344be90" /> <img width="2158" height="1850" alt="image" src="https://github.com/user-attachments/assets/2324779e-9ab8-47fc-a014-78269a78823e" /> The workspace migration builder and runner already log phase timings via `logger.perfTime`, but those logs are only useful for local debugging: they're gated behind the `performance` log level and never leave the process. This PR routes the same timings into the existing `MetricsService` (OpenTelemetry) so migration performance can be tracked over time and regressions detected. ## What's recorded All metrics are histograms recorded via `MetricsService.recordHistogram`, which is a synchronous in-memory aggregation - export happens on the background OTel readers (Prometheus scrape or periodic OTLP push), so nothing is added to the migration hot path. When `METER_DRIVER` is unset, records are no-ops. | Metric | Where | Attributes | |---|---|---| | `workspace-migration/build-duration-ms` | `WorkspaceMigrationValidateBuildAndRunService` | `status`: `success`, `fail` (validation errors), `error` (builder threw) | | `workspace-migration/build-entity-duration-ms` | `WorkspaceEntityMigrationBuilderService.validateAndBuild` | `metadataName`, `status` | | `workspace-migration/build-entity-phase-duration-ms` | entity builder | `metadataName`, `phase`: `matrix-computation`, `deletion-validation`, `creation-validation`, `update-validation` | | `workspace-migration/run-duration-ms` | `WorkspaceMigrationRunnerService.run` | `status`: `success`, `fail` - recorded on every exit path, including pre-transaction failures (DDL locked, cache retrieval, application not found) | | `workspace-migration/run-phase-duration-ms` | runner | `phase`: `initial-cache-retrieval`, `action-execution`, `commit`, `cache-invalidation`; `status` - on failure the elapsed transaction time is recorded as `action-execution` with `status: 'fail'` (the action/commit split is unknowable mid-failure) | | `workspace-migration/action-duration-ms` | base action handler wrapper | `actionType`, `metadataName`, `step`, `status` - failed steps are recorded via try/finally | | `workspace-migration/action-count` | validate-build-and-run service | - | Phase notes: phases are disjoint (commit is subtracted from transaction time) but not exhaustive - connection setup, the application-map fetch, rollback, and after-commit side effects sit outside them, so the stack approximates rather than equals the run total. The `cache-invalidation` phase is recorded only at the run's post-commit call site, not inside the public `invalidateCache`, so standalone callers (upgrade backfill commands, `FlatCacheInvalidateCommand`) don't pollute the series. ## Implementation notes - Durations are measured with local `performance.now()` instead of reusing `perfTime`/`perfTimeEnd`, so metrics are recorded regardless of the `performance` log level and are immune to key collisions in the logger's shared timer map under concurrent migrations. Existing perf logs are untouched. - `WorkspaceMigrationRunnerService.run` is now a thin wrapper around the previous body (`executeRun`) so the total run duration and its success/fail status are recorded in one place for every throw path. - No `workspaceId` in attributes to keep cardinality bounded. - Two new bucket-boundary constants follow the existing `AI_LATENCY_MS_BUCKET_BOUNDARIES` pattern: durations 5ms-120s, action counts 1-5000. - `MetricsModule` imported into `WorkspaceMigrationModule`, `WorkspaceMigrationRunnerModule`, `WorkspaceMigrationBuilderModule`, and the action-handlers module. ## Test - `nx typecheck twenty-server` passes - oxlint + oxfmt clean on changed files - `workspace.service.spec.ts` passes
This commit is contained in:
+3
@@ -0,0 +1,3 @@
|
||||
export const WORKSPACE_MIGRATION_ACTION_COUNT_BUCKET_BOUNDARIES = [
|
||||
1, 5, 10, 25, 50, 100, 250, 500, 1000, 2500, 5000,
|
||||
] as const;
|
||||
+3
@@ -0,0 +1,3 @@
|
||||
export const WORKSPACE_MIGRATION_DURATION_MS_BUCKET_BOUNDARIES = [
|
||||
5, 10, 25, 50, 100, 250, 500, 1000, 2500, 5000, 10000, 30000, 60000, 120000,
|
||||
] as const;
|
||||
@@ -83,4 +83,11 @@ export enum MetricsKeys {
|
||||
ConnectedAccountWebhookSubscriptionRenewalFailed = 'connected-account-webhook-subscription/renewal-failed',
|
||||
ConnectedAccountWebhookSubscriptionDeleted = 'connected-account-webhook-subscription/deleted',
|
||||
ConnectedAccountWebhookSubscriptionDeletionFailed = 'connected-account-webhook-subscription/deletion-failed',
|
||||
WorkspaceMigrationBuildDurationMs = 'workspace-migration/build-duration-ms',
|
||||
WorkspaceMigrationBuildEntityDurationMs = 'workspace-migration/build-entity-duration-ms',
|
||||
WorkspaceMigrationBuildEntityPhaseDurationMs = 'workspace-migration/build-entity-phase-duration-ms',
|
||||
WorkspaceMigrationRunDurationMs = 'workspace-migration/run-duration-ms',
|
||||
WorkspaceMigrationRunPhaseDurationMs = 'workspace-migration/run-phase-duration-ms',
|
||||
WorkspaceMigrationActionDurationMs = 'workspace-migration/action-duration-ms',
|
||||
WorkspaceMigrationActionCount = 'workspace-migration/action-count',
|
||||
}
|
||||
|
||||
+27
@@ -6,6 +6,10 @@ import {
|
||||
} from 'twenty-shared/metadata';
|
||||
|
||||
import { LoggerService } from 'src/engine/core-modules/logger/logger.service';
|
||||
import { WORKSPACE_MIGRATION_ACTION_COUNT_BUCKET_BOUNDARIES } from 'src/engine/core-modules/metrics/constants/workspace-migration-action-count-bucket-boundaries.constant';
|
||||
import { WORKSPACE_MIGRATION_DURATION_MS_BUCKET_BOUNDARIES } from 'src/engine/core-modules/metrics/constants/workspace-migration-duration-ms-bucket-boundaries.constant';
|
||||
import { MetricsService } from 'src/engine/core-modules/metrics/metrics.service';
|
||||
import { MetricsKeys } from 'src/engine/core-modules/metrics/types/metrics-keys.type';
|
||||
import { TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service';
|
||||
import { AllFlatEntityOperationRecordByMetadataName } from 'src/engine/metadata-modules/flat-entity/types/all-flat-entity-operation-record-by-metadata-name.type';
|
||||
import { AllFlatEntityOperationByMetadataName } from 'src/engine/metadata-modules/flat-entity/types/flat-entity-to-create-delete-update.type';
|
||||
@@ -71,6 +75,7 @@ export class WorkspaceMigrationValidateBuildAndRunService {
|
||||
private readonly workspaceMigrationFlatEntityMapsService: WorkspaceMigrationFlatEntityMapsService,
|
||||
private readonly metadataEventEmitter: MetadataEventEmitter,
|
||||
private readonly metadataSideEffectEngineService: MetadataSideEffectEngineService,
|
||||
private readonly metricsService: MetricsService,
|
||||
private readonly logger: LoggerService,
|
||||
twentyConfigService: TwentyConfigService,
|
||||
) {
|
||||
@@ -98,6 +103,14 @@ export class WorkspaceMigrationValidateBuildAndRunService {
|
||||
await this.workspaceMigrationBuildOrchestratorService
|
||||
.buildWorkspaceMigration(buildArgs)
|
||||
.catch((error) => {
|
||||
this.metricsService.recordHistogram({
|
||||
key: MetricsKeys.WorkspaceMigrationBuildDurationMs,
|
||||
value: performance.now() - buildStart,
|
||||
unit: 'ms',
|
||||
attributes: { status: 'error' },
|
||||
bucketBoundaries: WORKSPACE_MIGRATION_DURATION_MS_BUCKET_BOUNDARIES,
|
||||
});
|
||||
|
||||
this.logger.error(
|
||||
error,
|
||||
WorkspaceMigrationValidateBuildAndRunService.name,
|
||||
@@ -110,6 +123,14 @@ export class WorkspaceMigrationValidateBuildAndRunService {
|
||||
});
|
||||
const buildMs = performance.now() - buildStart;
|
||||
|
||||
this.metricsService.recordHistogram({
|
||||
key: MetricsKeys.WorkspaceMigrationBuildDurationMs,
|
||||
value: buildMs,
|
||||
unit: 'ms',
|
||||
attributes: { status: validateAndBuildResult.status },
|
||||
bucketBoundaries: WORKSPACE_MIGRATION_DURATION_MS_BUCKET_BOUNDARIES,
|
||||
});
|
||||
|
||||
this.logger.perf(
|
||||
`[install-perf] buildWorkspaceMigration took ${buildMs.toFixed(1)}ms (status=${validateAndBuildResult.status})`,
|
||||
WorkspaceMigrationValidateBuildAndRunService.name,
|
||||
@@ -154,6 +175,12 @@ export class WorkspaceMigrationValidateBuildAndRunService {
|
||||
WorkspaceMigrationValidateBuildAndRunService.name,
|
||||
);
|
||||
|
||||
this.metricsService.recordHistogram({
|
||||
key: MetricsKeys.WorkspaceMigrationActionCount,
|
||||
value: workspaceMigration.actions.length,
|
||||
bucketBoundaries: WORKSPACE_MIGRATION_ACTION_COUNT_BUCKET_BOUNDARIES,
|
||||
});
|
||||
|
||||
const runStart = performance.now();
|
||||
const { hasSchemaMetadataChanged, metadataEvents } =
|
||||
await this.workspaceMigrationRunnerService.run({
|
||||
|
||||
+82
@@ -6,6 +6,9 @@ import { isDefined } from 'twenty-shared/utils';
|
||||
import { validate as uuidValidate, version as uuidVersion } from 'uuid';
|
||||
|
||||
import { LoggerService } from 'src/engine/core-modules/logger/logger.service';
|
||||
import { WORKSPACE_MIGRATION_DURATION_MS_BUCKET_BOUNDARIES } from 'src/engine/core-modules/metrics/constants/workspace-migration-duration-ms-bucket-boundaries.constant';
|
||||
import { MetricsService } from 'src/engine/core-modules/metrics/metrics.service';
|
||||
import { MetricsKeys } from 'src/engine/core-modules/metrics/types/metrics-keys.type';
|
||||
import {
|
||||
FlatEntityMapsException,
|
||||
FlatEntityMapsExceptionCode,
|
||||
@@ -56,6 +59,10 @@ export abstract class WorkspaceEntityMigrationBuilderService<
|
||||
> {
|
||||
@Inject(LoggerService)
|
||||
protected readonly logger: LoggerService;
|
||||
|
||||
@Inject(MetricsService)
|
||||
protected readonly metricsService: MetricsService;
|
||||
|
||||
private metadataName: T;
|
||||
|
||||
constructor(metadataName: T) {
|
||||
@@ -80,6 +87,9 @@ export abstract class WorkspaceEntityMigrationBuilderService<
|
||||
'matrix computation',
|
||||
);
|
||||
|
||||
const validateAndBuildStart = performance.now();
|
||||
const matrixComputationStart = performance.now();
|
||||
|
||||
const fromFlatEntities = Object.values(
|
||||
fromFlatEntityMaps.byUniversalIdentifier,
|
||||
).filter(isDefined);
|
||||
@@ -107,6 +117,11 @@ export abstract class WorkspaceEntityMigrationBuilderService<
|
||||
'entity processing',
|
||||
);
|
||||
|
||||
this.recordBuildEntityPhaseMetric({
|
||||
phase: 'matrix-computation',
|
||||
startedAt: matrixComputationStart,
|
||||
});
|
||||
|
||||
const flatEntityMapsKey = getMetadataFlatEntityMapsKey(this.metadataName);
|
||||
const actionsResult = getMetadataEmptyWorkspaceMigrationActionRecord(
|
||||
this.metadataName,
|
||||
@@ -119,6 +134,8 @@ export abstract class WorkspaceEntityMigrationBuilderService<
|
||||
'deletion validation',
|
||||
);
|
||||
|
||||
const deletionValidationStart = performance.now();
|
||||
|
||||
const remainingFlatEntityMapsToDelete = structuredClone(
|
||||
deletedFlatEntityMaps,
|
||||
);
|
||||
@@ -195,6 +212,13 @@ export abstract class WorkspaceEntityMigrationBuilderService<
|
||||
'creation validation',
|
||||
);
|
||||
|
||||
this.recordBuildEntityPhaseMetric({
|
||||
phase: 'deletion-validation',
|
||||
startedAt: deletionValidationStart,
|
||||
});
|
||||
|
||||
const creationValidationStart = performance.now();
|
||||
|
||||
const remainingFlatEntityMapsToCreate = structuredClone(
|
||||
createdFlatEntityMaps,
|
||||
);
|
||||
@@ -275,6 +299,13 @@ export abstract class WorkspaceEntityMigrationBuilderService<
|
||||
'update validation',
|
||||
);
|
||||
|
||||
this.recordBuildEntityPhaseMetric({
|
||||
phase: 'creation-validation',
|
||||
startedAt: creationValidationStart,
|
||||
});
|
||||
|
||||
const updateValidationStart = performance.now();
|
||||
|
||||
for (const flatEntityToUpdateUniversalIdentifier in updatedFlatEntityMaps.byUniversalIdentifier) {
|
||||
const flatEntityUpdate =
|
||||
updatedFlatEntityMaps.byUniversalIdentifier[
|
||||
@@ -362,13 +393,28 @@ export abstract class WorkspaceEntityMigrationBuilderService<
|
||||
'entity processing',
|
||||
);
|
||||
|
||||
this.recordBuildEntityPhaseMetric({
|
||||
phase: 'update-validation',
|
||||
startedAt: updateValidationStart,
|
||||
});
|
||||
|
||||
if (allValidationResult.length > 0) {
|
||||
this.recordBuildEntityDurationMetric({
|
||||
status: 'fail',
|
||||
startedAt: validateAndBuildStart,
|
||||
});
|
||||
|
||||
return {
|
||||
status: 'fail',
|
||||
errors: allValidationResult,
|
||||
};
|
||||
}
|
||||
|
||||
this.recordBuildEntityDurationMetric({
|
||||
status: 'success',
|
||||
startedAt: validateAndBuildStart,
|
||||
});
|
||||
|
||||
this.logger.perfTimeEnd(
|
||||
`EntityBuilder ${this.metadataName}`,
|
||||
'validateAndBuild',
|
||||
@@ -380,6 +426,42 @@ export abstract class WorkspaceEntityMigrationBuilderService<
|
||||
};
|
||||
}
|
||||
|
||||
private recordBuildEntityDurationMetric({
|
||||
status,
|
||||
startedAt,
|
||||
}: {
|
||||
status: 'success' | 'fail';
|
||||
startedAt: number;
|
||||
}): void {
|
||||
this.metricsService.recordHistogram({
|
||||
key: MetricsKeys.WorkspaceMigrationBuildEntityDurationMs,
|
||||
value: performance.now() - startedAt,
|
||||
unit: 'ms',
|
||||
attributes: { metadataName: this.metadataName, status },
|
||||
bucketBoundaries: WORKSPACE_MIGRATION_DURATION_MS_BUCKET_BOUNDARIES,
|
||||
});
|
||||
}
|
||||
|
||||
private recordBuildEntityPhaseMetric({
|
||||
phase,
|
||||
startedAt,
|
||||
}: {
|
||||
phase:
|
||||
| 'matrix-computation'
|
||||
| 'deletion-validation'
|
||||
| 'creation-validation'
|
||||
| 'update-validation';
|
||||
startedAt: number;
|
||||
}): void {
|
||||
this.metricsService.recordHistogram({
|
||||
key: MetricsKeys.WorkspaceMigrationBuildEntityPhaseDurationMs,
|
||||
value: performance.now() - startedAt,
|
||||
unit: 'ms',
|
||||
attributes: { metadataName: this.metadataName, phase },
|
||||
bucketBoundaries: WORKSPACE_MIGRATION_DURATION_MS_BUCKET_BOUNDARIES,
|
||||
});
|
||||
}
|
||||
|
||||
private validateUniversalIdentifier({
|
||||
flatEntityToValidate: { universalIdentifier },
|
||||
}: UniversalFlatEntityValidationArgs<T>): FlatEntityValidationError[] {
|
||||
|
||||
+6
-1
@@ -1,6 +1,7 @@
|
||||
import { Module } from '@nestjs/common';
|
||||
|
||||
import { FeatureFlagModule } from 'src/engine/core-modules/feature-flag/feature-flag.module';
|
||||
import { MetricsModule } from 'src/engine/core-modules/metrics/metrics.module';
|
||||
import { FlatFieldMetadataTypeValidatorService } from 'src/engine/metadata-modules/flat-field-metadata/services/flat-field-metadata-type-validator.service';
|
||||
import { WorkspaceMigrationAgentActionsBuilderService } from 'src/engine/workspace-manager/workspace-migration/workspace-migration-builder/builders/agent/workspace-migration-agent-actions-builder.service';
|
||||
import { WorkspaceMigrationCommandMenuItemActionsBuilderService } from 'src/engine/workspace-manager/workspace-migration/workspace-migration-builder/builders/command-menu-item/workspace-migration-command-menu-item-actions-builder.service';
|
||||
@@ -36,7 +37,11 @@ import { WorkspaceMigrationSearchFieldMetadataActionsBuilderService } from 'src/
|
||||
import { WorkspaceMigrationBuilderValidatorsModule } from 'src/engine/workspace-manager/workspace-migration/workspace-migration-builder/validators/workspace-migration-builder-validators.module';
|
||||
|
||||
@Module({
|
||||
imports: [FeatureFlagModule, WorkspaceMigrationBuilderValidatorsModule],
|
||||
imports: [
|
||||
FeatureFlagModule,
|
||||
WorkspaceMigrationBuilderValidatorsModule,
|
||||
MetricsModule,
|
||||
],
|
||||
providers: [
|
||||
FlatFieldMetadataTypeValidatorService,
|
||||
WorkspaceMigrationObjectActionsBuilderService,
|
||||
|
||||
+2
@@ -2,6 +2,7 @@ import { Module } from '@nestjs/common';
|
||||
import { TypeOrmModule } from '@nestjs/typeorm';
|
||||
|
||||
import { ApplicationEntity } from 'src/engine/core-modules/application/application.entity';
|
||||
import { MetricsModule } from 'src/engine/core-modules/metrics/metrics.module';
|
||||
import { SecretEncryptionModule } from 'src/engine/core-modules/secret-encryption/secret-encryption.module';
|
||||
import { WorkspaceSchemaManagerModule } from 'src/engine/twenty-orm/workspace-schema-manager/workspace-schema-manager.module';
|
||||
import { CreateAgentActionHandlerService } from 'src/engine/workspace-manager/workspace-migration/workspace-migration-runner/action-handlers/agent/services/create-agent-action-handler.service';
|
||||
@@ -103,6 +104,7 @@ import { UpdateSearchFieldMetadataActionHandlerService } from 'src/engine/worksp
|
||||
TypeOrmModule.forFeature([ApplicationEntity]),
|
||||
WorkspaceSchemaManagerModule,
|
||||
SecretEncryptionModule,
|
||||
MetricsModule,
|
||||
],
|
||||
providers: [
|
||||
CreateFieldActionHandlerService,
|
||||
|
||||
+32
-1
@@ -4,6 +4,9 @@ import { AllMetadataName } from 'twenty-shared/metadata';
|
||||
import { QueryRunner } from 'typeorm';
|
||||
|
||||
import { LoggerService } from 'src/engine/core-modules/logger/logger.service';
|
||||
import { WORKSPACE_MIGRATION_DURATION_MS_BUCKET_BOUNDARIES } from 'src/engine/core-modules/metrics/constants/workspace-migration-duration-ms-bucket-boundaries.constant';
|
||||
import { MetricsService } from 'src/engine/core-modules/metrics/metrics.service';
|
||||
import { MetricsKeys } from 'src/engine/core-modules/metrics/types/metrics-keys.type';
|
||||
import { ALL_METADATA_ENTITY_BY_METADATA_NAME } from 'src/engine/metadata-modules/flat-entity/constant/all-metadata-entity-by-metadata-name.constant';
|
||||
import { type AllFlatEntityMaps } from 'src/engine/metadata-modules/flat-entity/types/all-flat-entity-maps.type';
|
||||
import { FlatEntityMaps } from 'src/engine/metadata-modules/flat-entity/types/flat-entity-maps.type';
|
||||
@@ -75,6 +78,9 @@ export abstract class BaseWorkspaceMigrationRunnerActionHandlerService<
|
||||
@Inject(LoggerService)
|
||||
protected readonly logger: LoggerService;
|
||||
|
||||
@Inject(MetricsService)
|
||||
protected readonly metricsService: MetricsService;
|
||||
|
||||
public abstract transpileUniversalActionToFlatAction(
|
||||
context: WorkspaceMigrationActionRunnerArgs<TUniversalAction>,
|
||||
): Promise<TFlatAction>;
|
||||
@@ -320,18 +326,43 @@ export abstract class BaseWorkspaceMigrationRunnerActionHandlerService<
|
||||
label,
|
||||
method,
|
||||
}: {
|
||||
label: string;
|
||||
label: 'executeForMetadata' | 'executeForWorkspaceSchema';
|
||||
method: () => Promise<void>;
|
||||
}): Promise<void> {
|
||||
const startedAt = performance.now();
|
||||
|
||||
const recordActionDuration = (status: 'success' | 'fail') =>
|
||||
this.metricsService.recordHistogram({
|
||||
key: MetricsKeys.WorkspaceMigrationActionDurationMs,
|
||||
value: performance.now() - startedAt,
|
||||
unit: 'ms',
|
||||
attributes: {
|
||||
actionType: this.actionType,
|
||||
metadataName: this.metadataName,
|
||||
step: label,
|
||||
status,
|
||||
},
|
||||
bucketBoundaries: WORKSPACE_MIGRATION_DURATION_MS_BUCKET_BOUNDARIES,
|
||||
});
|
||||
|
||||
this.logger.perfTime(
|
||||
'BaseWorkspaceMigrationRunnerActionHandlerService',
|
||||
`${this.actionType}_${this.metadataName} ${label}`,
|
||||
);
|
||||
|
||||
try {
|
||||
await method();
|
||||
} catch (error) {
|
||||
recordActionDuration('fail');
|
||||
throw error;
|
||||
}
|
||||
|
||||
this.logger.perfTimeEnd(
|
||||
'BaseWorkspaceMigrationRunnerActionHandlerService',
|
||||
`${this.actionType}_${this.metadataName} ${label}`,
|
||||
);
|
||||
|
||||
recordActionDuration('success');
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+98
-1
@@ -6,6 +6,9 @@ import { isDefined } from 'twenty-shared/utils';
|
||||
import { DataSource } from 'typeorm';
|
||||
|
||||
import { LoggerService } from 'src/engine/core-modules/logger/logger.service';
|
||||
import { WORKSPACE_MIGRATION_DURATION_MS_BUCKET_BOUNDARIES } from 'src/engine/core-modules/metrics/constants/workspace-migration-duration-ms-bucket-boundaries.constant';
|
||||
import { MetricsService } from 'src/engine/core-modules/metrics/metrics.service';
|
||||
import { MetricsKeys } from 'src/engine/core-modules/metrics/types/metrics-keys.type';
|
||||
import { TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service';
|
||||
import { WorkspaceManyOrAllFlatEntityMapsCacheService } from 'src/engine/metadata-modules/flat-entity/services/workspace-many-or-all-flat-entity-maps-cache.service';
|
||||
import { AllFlatEntityMaps } from 'src/engine/metadata-modules/flat-entity/types/all-flat-entity-maps.type';
|
||||
@@ -36,6 +39,7 @@ export class WorkspaceMigrationRunnerService {
|
||||
private readonly workspaceMigrationRunnerActionHandlerRegistry: WorkspaceMigrationRunnerActionHandlerRegistryService,
|
||||
private readonly workspaceMetadataVersionService: WorkspaceMetadataVersionService,
|
||||
private readonly workspaceCacheService: WorkspaceCacheService,
|
||||
private readonly metricsService: MetricsService,
|
||||
private readonly logger: LoggerService,
|
||||
private readonly twentyConfigService: TwentyConfigService,
|
||||
) {}
|
||||
@@ -149,6 +153,28 @@ export class WorkspaceMigrationRunnerService {
|
||||
);
|
||||
}
|
||||
|
||||
private recordRunPhaseMetric({
|
||||
phase,
|
||||
status,
|
||||
value,
|
||||
}: {
|
||||
phase:
|
||||
| 'initial-cache-retrieval'
|
||||
| 'action-execution'
|
||||
| 'commit'
|
||||
| 'cache-invalidation';
|
||||
status: 'success' | 'fail';
|
||||
value: number;
|
||||
}): void {
|
||||
this.metricsService.recordHistogram({
|
||||
key: MetricsKeys.WorkspaceMigrationRunPhaseDurationMs,
|
||||
value,
|
||||
unit: 'ms',
|
||||
attributes: { phase, status },
|
||||
bucketBoundaries: WORKSPACE_MIGRATION_DURATION_MS_BUCKET_BOUNDARIES,
|
||||
});
|
||||
}
|
||||
|
||||
private async logBlockingDbActivity(): Promise<void> {
|
||||
try {
|
||||
// Metadata only (no query text) to avoid logging literals from other sessions.
|
||||
@@ -178,7 +204,42 @@ export class WorkspaceMigrationRunnerService {
|
||||
}
|
||||
}
|
||||
|
||||
run = async ({
|
||||
run = async (args: {
|
||||
workspaceMigration: WorkspaceMigration;
|
||||
workspaceId: string;
|
||||
}): Promise<{
|
||||
allFlatEntityMaps: AllFlatEntityMaps;
|
||||
metadataEvents: MetadataEvent[];
|
||||
hasSchemaMetadataChanged: boolean;
|
||||
}> => {
|
||||
const runStart = performance.now();
|
||||
|
||||
try {
|
||||
const result = await this.executeRun(args);
|
||||
|
||||
this.metricsService.recordHistogram({
|
||||
key: MetricsKeys.WorkspaceMigrationRunDurationMs,
|
||||
value: performance.now() - runStart,
|
||||
unit: 'ms',
|
||||
attributes: { status: 'success' },
|
||||
bucketBoundaries: WORKSPACE_MIGRATION_DURATION_MS_BUCKET_BOUNDARIES,
|
||||
});
|
||||
|
||||
return result;
|
||||
} catch (error) {
|
||||
this.metricsService.recordHistogram({
|
||||
key: MetricsKeys.WorkspaceMigrationRunDurationMs,
|
||||
value: performance.now() - runStart,
|
||||
unit: 'ms',
|
||||
attributes: { status: 'fail' },
|
||||
bucketBoundaries: WORKSPACE_MIGRATION_DURATION_MS_BUCKET_BOUNDARIES,
|
||||
});
|
||||
|
||||
throw error;
|
||||
}
|
||||
};
|
||||
|
||||
private executeRun = async ({
|
||||
workspaceMigration: { actions, applicationUniversalIdentifier },
|
||||
workspaceId,
|
||||
}: {
|
||||
@@ -246,6 +307,12 @@ export class WorkspaceMigrationRunnerService {
|
||||
const initialCacheRetrievalMs =
|
||||
performance.now() - initialCacheRetrievalStart;
|
||||
|
||||
this.recordRunPhaseMetric({
|
||||
phase: 'initial-cache-retrieval',
|
||||
status: 'success',
|
||||
value: initialCacheRetrievalMs,
|
||||
});
|
||||
|
||||
this.logger.perf(
|
||||
`[install-perf] Runner initial cache retrieval (getOrRecomputeManyOrAllFlatEntityMaps) took ${initialCacheRetrievalMs.toFixed(1)}ms for ${allFlatEntityMapsKeys.length} flat-maps keys`,
|
||||
'Runner',
|
||||
@@ -354,6 +421,18 @@ export class WorkspaceMigrationRunnerService {
|
||||
const commitMs = performance.now() - commitStart;
|
||||
const transactionMs = performance.now() - transactionStart;
|
||||
|
||||
this.recordRunPhaseMetric({
|
||||
phase: 'action-execution',
|
||||
status: 'success',
|
||||
value: transactionMs - commitMs,
|
||||
});
|
||||
|
||||
this.recordRunPhaseMetric({
|
||||
phase: 'commit',
|
||||
status: 'success',
|
||||
value: commitMs,
|
||||
});
|
||||
|
||||
this.logger.perf(
|
||||
`[install-perf] Runner transaction summary: ${actionCount} actions, total transaction ${transactionMs.toFixed(1)}ms (commit ${commitMs.toFixed(1)}ms), slowest action ${slowestActionLabel} ${slowestActionMs.toFixed(1)}ms`,
|
||||
'Runner',
|
||||
@@ -361,6 +440,12 @@ export class WorkspaceMigrationRunnerService {
|
||||
|
||||
this.logger.perfTimeEnd('Runner', 'Transaction execution');
|
||||
} catch (error) {
|
||||
this.recordRunPhaseMetric({
|
||||
phase: 'action-execution',
|
||||
status: 'fail',
|
||||
value: performance.now() - transactionStart,
|
||||
});
|
||||
|
||||
this.logger.error(
|
||||
`[install-perf] migration failed after ${actionCount} action(s): ${
|
||||
error instanceof Error ? error.message : String(error)
|
||||
@@ -433,7 +518,19 @@ export class WorkspaceMigrationRunnerService {
|
||||
allFlatEntityMapsKeys,
|
||||
workspaceId,
|
||||
});
|
||||
|
||||
this.recordRunPhaseMetric({
|
||||
phase: 'cache-invalidation',
|
||||
status: 'success',
|
||||
value: performance.now() - postCommitInvalidateStart,
|
||||
});
|
||||
} catch (cacheError) {
|
||||
this.recordRunPhaseMetric({
|
||||
phase: 'cache-invalidation',
|
||||
status: 'fail',
|
||||
value: performance.now() - postCommitInvalidateStart,
|
||||
});
|
||||
|
||||
this.logger.error(
|
||||
`Cache invalidation failed after committed transaction: ${cacheError}`,
|
||||
'Runner',
|
||||
|
||||
+2
@@ -5,6 +5,7 @@ import { TypeOrmModule } from '@nestjs/typeorm';
|
||||
import { WorkspaceIteratorModule } from 'src/database/commands/command-runners/workspace-iterator.module';
|
||||
import { TypeORMModule } from 'src/database/typeorm/typeorm.module';
|
||||
import { FeatureFlagModule } from 'src/engine/core-modules/feature-flag/feature-flag.module';
|
||||
import { MetricsModule } from 'src/engine/core-modules/metrics/metrics.module';
|
||||
import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity';
|
||||
import { WorkspaceManyOrAllFlatEntityMapsCacheModule } from 'src/engine/metadata-modules/flat-entity/services/workspace-many-or-all-flat-entity-maps-cache.module';
|
||||
import { WorkspaceMetadataVersionModule } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.module';
|
||||
@@ -27,6 +28,7 @@ import { WorkspaceMigrationRunnerService } from 'src/engine/workspace-manager/wo
|
||||
WorkspaceCacheModule,
|
||||
TypeOrmModule.forFeature([WorkspaceEntity]),
|
||||
WorkspaceIteratorModule,
|
||||
MetricsModule,
|
||||
],
|
||||
providers: [
|
||||
WorkspaceMigrationRunnerService,
|
||||
|
||||
+2
@@ -1,5 +1,6 @@
|
||||
import { Module } from '@nestjs/common';
|
||||
|
||||
import { MetricsModule } from 'src/engine/core-modules/metrics/metrics.module';
|
||||
import { MetadataSideEffectModule } from 'src/engine/metadata-modules/metadata-side-effect/metadata-side-effect.module';
|
||||
import { WorkspaceCacheModule } from 'src/engine/workspace-cache/workspace-cache.module';
|
||||
import { WorkspaceMigrationBuildOrchestratorService } from 'src/engine/workspace-manager/workspace-migration/services/workspace-migration-build-orchestrator.service';
|
||||
@@ -14,6 +15,7 @@ import { WorkspaceMigrationRunnerModule } from 'src/engine/workspace-manager/wor
|
||||
WorkspaceMigrationRunnerModule,
|
||||
WorkspaceCacheModule,
|
||||
MetadataSideEffectModule,
|
||||
MetricsModule,
|
||||
],
|
||||
providers: [
|
||||
WorkspaceMigrationValidateBuildAndRunService,
|
||||
|
||||
Reference in New Issue
Block a user