From fda2295beb8416e916ecd5bc9e6ef2483f70577e Mon Sep 17 00:00:00 2001 From: Charles Bochet Date: Tue, 5 May 2026 09:56:13 +0200 Subject: [PATCH] feat: expose upgrade status as Prometheus gauge metrics (#20262) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary - Adds `UpgradeGaugeService` that exposes three observable Prometheus gauges based on the recently merged upgrade status service: - `twenty_upgrade_instance_health` — 1 (up-to-date), 0 (behind), -1 (failed) - `twenty_upgrade_workspaces_behind_total` — count of workspaces with pending upgrade commands - `twenty_upgrade_workspaces_failed_total` — count of workspaces with a failed upgrade command - Follows the existing gauge pattern (`WorkspaceGaugeService`, `BillingGaugeService`, `DatabaseGaugeService`) ### Caching & QPS design Prometheus scrapes every **15s** via `ServiceMonitor`. Each gauge uses the `MetricsService.createObservableGauge({ cacheValue: true })` pattern which caches the value in Redis for **60 seconds**. Under that, `UpgradeStatusService.getInstanceAndAllWorkspacesStatus()` uses `UpgradeStatusCacheService` with a **1-hour TTL** in Redis. Result: at most 1 DB query per hour regardless of scrape frequency. --------- Co-authored-by: Cursor --- .../upgrade/upgrade-gauge.service.ts | 112 ++++++++++++++++++ .../core-modules/upgrade/upgrade.module.ts | 4 + 2 files changed, 116 insertions(+) create mode 100644 packages/twenty-server/src/engine/core-modules/upgrade/upgrade-gauge.service.ts diff --git a/packages/twenty-server/src/engine/core-modules/upgrade/upgrade-gauge.service.ts b/packages/twenty-server/src/engine/core-modules/upgrade/upgrade-gauge.service.ts new file mode 100644 index 0000000000..78f8da74f0 --- /dev/null +++ b/packages/twenty-server/src/engine/core-modules/upgrade/upgrade-gauge.service.ts @@ -0,0 +1,112 @@ +import { Injectable, Logger, OnModuleInit } from '@nestjs/common'; + +import { UpgradeHealthEnum } from 'twenty-shared/types'; + +import { MetricsService } from 'src/engine/core-modules/metrics/metrics.service'; +import { + type InstanceAndAllWorkspacesUpgradeStatus, + UpgradeStatusService, +} from 'src/engine/core-modules/upgrade/services/upgrade-status.service'; + +const HEALTH_TO_GAUGE_VALUE: Record = { + [UpgradeHealthEnum.UP_TO_DATE]: 1, + [UpgradeHealthEnum.BEHIND]: 0, + [UpgradeHealthEnum.FAILED]: -1, +}; + +const HEALTH_UNKNOWN = -2; +const UPGRADE_STATUS_TTL_MS = 60_000; + +@Injectable() +export class UpgradeGaugeService implements OnModuleInit { + private readonly logger = new Logger(UpgradeGaugeService.name); + + private cachedUpgradeStatus: InstanceAndAllWorkspacesUpgradeStatus | null = + null; + private cachedUpgradeStatusExpiresAt = 0; + private inflightUpgradeStatusPromise: Promise | null = + null; + + constructor( + private readonly metricsService: MetricsService, + private readonly upgradeStatusService: UpgradeStatusService, + ) {} + + onModuleInit() { + this.metricsService.createObservableGauge({ + metricName: 'twenty_upgrade_instance_health', + options: { + description: + 'Instance upgrade health (1 = up-to-date, 0 = behind, -1 = failed, -2 = unknown)', + }, + callback: async () => { + const upgradeStatus = await this.getCachedUpgradeStatus(); + + if (!upgradeStatus) { + return HEALTH_UNKNOWN; + } + + return ( + HEALTH_TO_GAUGE_VALUE[upgradeStatus.instanceUpgradeStatus.health] ?? + HEALTH_UNKNOWN + ); + }, + cacheValue: true, + }); + + this.metricsService.createObservableGauge({ + metricName: 'twenty_upgrade_workspaces_behind_total', + options: { + description: 'Number of workspaces behind on upgrade commands', + }, + callback: async () => { + const upgradeStatus = await this.getCachedUpgradeStatus(); + + return upgradeStatus?.workspacesBehind.length ?? 0; + }, + cacheValue: true, + }); + + this.metricsService.createObservableGauge({ + metricName: 'twenty_upgrade_workspaces_failed_total', + options: { + description: 'Number of workspaces with a failed upgrade command', + }, + callback: async () => { + const upgradeStatus = await this.getCachedUpgradeStatus(); + + return upgradeStatus?.workspacesFailed.length ?? 0; + }, + cacheValue: true, + }); + } + + private async getCachedUpgradeStatus(): Promise { + if ( + this.cachedUpgradeStatus && + Date.now() < this.cachedUpgradeStatusExpiresAt + ) { + return this.cachedUpgradeStatus; + } + + if (this.inflightUpgradeStatusPromise) { + return this.inflightUpgradeStatusPromise.catch(() => null); + } + + this.inflightUpgradeStatusPromise = + this.upgradeStatusService.getInstanceAndAllWorkspacesStatus(); + + try { + this.cachedUpgradeStatus = await this.inflightUpgradeStatusPromise; + this.cachedUpgradeStatusExpiresAt = Date.now() + UPGRADE_STATUS_TTL_MS; + + return this.cachedUpgradeStatus; + } catch (error) { + this.logger.error('Failed to fetch upgrade status for gauges', error); + + return null; + } finally { + this.inflightUpgradeStatusPromise = null; + } + } +} diff --git a/packages/twenty-server/src/engine/core-modules/upgrade/upgrade.module.ts b/packages/twenty-server/src/engine/core-modules/upgrade/upgrade.module.ts index b6415c16d7..a9bad4f5bc 100644 --- a/packages/twenty-server/src/engine/core-modules/upgrade/upgrade.module.ts +++ b/packages/twenty-server/src/engine/core-modules/upgrade/upgrade.module.ts @@ -6,6 +6,7 @@ import { WorkspaceIteratorModule } from 'src/database/commands/command-runners/w import { InstanceCommandProviderModule } from 'src/database/commands/upgrade-version-command/instance-command-provider.module'; import { WorkspaceCommandProviderModule } from 'src/database/commands/upgrade-version-command/workspace-command-provider.module'; import { CoreEntityCacheModule } from 'src/engine/core-entity-cache/core-entity-cache.module'; +import { MetricsModule } from 'src/engine/core-modules/metrics/metrics.module'; import { InstanceCommandRunnerService } from 'src/engine/core-modules/upgrade/services/instance-command-runner.service'; import { UpgradeCommandRegistryService } from 'src/engine/core-modules/upgrade/services/upgrade-command-registry.service'; import { UpgradeMigrationService } from 'src/engine/core-modules/upgrade/services/upgrade-migration.service'; @@ -14,6 +15,7 @@ import { UpgradeSequenceRunnerService } from 'src/engine/core-modules/upgrade/se import { UpgradeStatusCacheService } from 'src/engine/core-modules/upgrade/services/upgrade-status-cache.service'; import { UpgradeStatusService } from 'src/engine/core-modules/upgrade/services/upgrade-status.service'; import { WorkspaceCommandRunnerService } from 'src/engine/core-modules/upgrade/services/workspace-command-runner.service'; +import { UpgradeGaugeService } from 'src/engine/core-modules/upgrade/upgrade-gauge.service'; import { UpgradeMigrationEntity } from 'src/engine/core-modules/upgrade/upgrade-migration.entity'; import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity'; import { WorkspaceVersionModule } from 'src/engine/workspace-manager/workspace-version/workspace-version.module'; @@ -23,6 +25,7 @@ import { WorkspaceVersionModule } from 'src/engine/workspace-manager/workspace-v CoreEntityCacheModule, DiscoveryModule, InstanceCommandProviderModule, + MetricsModule, WorkspaceCommandProviderModule, WorkspaceIteratorModule, WorkspaceVersionModule, @@ -37,6 +40,7 @@ import { WorkspaceVersionModule } from 'src/engine/workspace-manager/workspace-v UpgradeSequenceRunnerService, UpgradeStatusService, UpgradeStatusCacheService, + UpgradeGaugeService, ], exports: [ UpgradeMigrationService,