feat: expose upgrade status as Prometheus gauge metrics (#20262)

## Summary - Adds `UpgradeGaugeService` that exposes three observable Prometheus gauges based on the recently merged upgrade status service: - `twenty_upgrade_instance_health` — 1 (up-to-date), 0 (behind), -1 (failed) - `twenty_upgrade_workspaces_behind_total` — count of workspaces with pending upgrade commands - `twenty_upgrade_workspaces_failed_total` — count of workspaces with a failed upgrade command - Follows the existing gauge pattern (`WorkspaceGaugeService`, `BillingGaugeService`, `DatabaseGaugeService`) ### Caching & QPS design Prometheus scrapes every **15s** via `ServiceMonitor`. Each gauge uses the `MetricsService.createObservableGauge({ cacheValue: true })` pattern which caches the value in Redis for **60 seconds**. Under that, `UpgradeStatusService.getInstanceAndAllWorkspacesStatus()` uses `UpgradeStatusCacheService` with a **1-hour TTL** in Redis. Result: at most 1 DB query per hour regardless of scrape frequency. --------- Co-authored-by: Cursor <cursoragent@cursor.com>
2026-05-05 09:56:13 +02:00
parent df63dbff05
commit fda2295beb
2 changed files with 116 additions and 0 deletions
@@ -0,0 +1,112 @@
+import { Injectable, Logger, OnModuleInit } from '@nestjs/common';
+
+import { UpgradeHealthEnum } from 'twenty-shared/types';
+
+import { MetricsService } from 'src/engine/core-modules/metrics/metrics.service';
+import {
+  type InstanceAndAllWorkspacesUpgradeStatus,
+  UpgradeStatusService,
+} from 'src/engine/core-modules/upgrade/services/upgrade-status.service';
+
+const HEALTH_TO_GAUGE_VALUE: Record<UpgradeHealthEnum, number> = {
+  [UpgradeHealthEnum.UP_TO_DATE]: 1,
+  [UpgradeHealthEnum.BEHIND]: 0,
+  [UpgradeHealthEnum.FAILED]: -1,
+};
+
+const HEALTH_UNKNOWN = -2;
+const UPGRADE_STATUS_TTL_MS = 60_000;
+
+@Injectable()
+export class UpgradeGaugeService implements OnModuleInit {
+  private readonly logger = new Logger(UpgradeGaugeService.name);
+
+  private cachedUpgradeStatus: InstanceAndAllWorkspacesUpgradeStatus | null =
+    null;
+  private cachedUpgradeStatusExpiresAt = 0;
+  private inflightUpgradeStatusPromise: Promise<InstanceAndAllWorkspacesUpgradeStatus> | null =
+    null;
+
+  constructor(
+    private readonly metricsService: MetricsService,
+    private readonly upgradeStatusService: UpgradeStatusService,
+  ) {}
+
+  onModuleInit() {
+    this.metricsService.createObservableGauge({
+      metricName: 'twenty_upgrade_instance_health',
+      options: {
+        description:
+          'Instance upgrade health (1 = up-to-date, 0 = behind, -1 = failed, -2 = unknown)',
+      },
+      callback: async () => {
+        const upgradeStatus = await this.getCachedUpgradeStatus();
+
+        if (!upgradeStatus) {
+          return HEALTH_UNKNOWN;
+        }
+
+        return (
+          HEALTH_TO_GAUGE_VALUE[upgradeStatus.instanceUpgradeStatus.health] ??
+          HEALTH_UNKNOWN
+        );
+      },
+      cacheValue: true,
+    });
+
+    this.metricsService.createObservableGauge({
+      metricName: 'twenty_upgrade_workspaces_behind_total',
+      options: {
+        description: 'Number of workspaces behind on upgrade commands',
+      },
+      callback: async () => {
+        const upgradeStatus = await this.getCachedUpgradeStatus();
+
+        return upgradeStatus?.workspacesBehind.length ?? 0;
+      },
+      cacheValue: true,
+    });
+
+    this.metricsService.createObservableGauge({
+      metricName: 'twenty_upgrade_workspaces_failed_total',
+      options: {
+        description: 'Number of workspaces with a failed upgrade command',
+      },
+      callback: async () => {
+        const upgradeStatus = await this.getCachedUpgradeStatus();
+
+        return upgradeStatus?.workspacesFailed.length ?? 0;
+      },
+      cacheValue: true,
+    });
+  }
+
+  private async getCachedUpgradeStatus(): Promise<InstanceAndAllWorkspacesUpgradeStatus | null> {
+    if (
+      this.cachedUpgradeStatus &&
+      Date.now() < this.cachedUpgradeStatusExpiresAt
+    ) {
+      return this.cachedUpgradeStatus;
+    }
+
+    if (this.inflightUpgradeStatusPromise) {
+      return this.inflightUpgradeStatusPromise.catch(() => null);
+    }
+
+    this.inflightUpgradeStatusPromise =
+      this.upgradeStatusService.getInstanceAndAllWorkspacesStatus();
+
+    try {
+      this.cachedUpgradeStatus = await this.inflightUpgradeStatusPromise;
+      this.cachedUpgradeStatusExpiresAt = Date.now() + UPGRADE_STATUS_TTL_MS;
+
+      return this.cachedUpgradeStatus;
+    } catch (error) {
+      this.logger.error('Failed to fetch upgrade status for gauges', error);
+
+      return null;
+    } finally {
+      this.inflightUpgradeStatusPromise = null;
+    }
+  }
+}
@@ -6,6 +6,7 @@ import { WorkspaceIteratorModule } from 'src/database/commands/command-runners/w
 import { InstanceCommandProviderModule } from 'src/database/commands/upgrade-version-command/instance-command-provider.module';
 import { WorkspaceCommandProviderModule } from 'src/database/commands/upgrade-version-command/workspace-command-provider.module';
 import { CoreEntityCacheModule } from 'src/engine/core-entity-cache/core-entity-cache.module';
+import { MetricsModule } from 'src/engine/core-modules/metrics/metrics.module';
 import { InstanceCommandRunnerService } from 'src/engine/core-modules/upgrade/services/instance-command-runner.service';
 import { UpgradeCommandRegistryService } from 'src/engine/core-modules/upgrade/services/upgrade-command-registry.service';
 import { UpgradeMigrationService } from 'src/engine/core-modules/upgrade/services/upgrade-migration.service';
@@ -14,6 +15,7 @@ import { UpgradeSequenceRunnerService } from 'src/engine/core-modules/upgrade/se
 import { UpgradeStatusCacheService } from 'src/engine/core-modules/upgrade/services/upgrade-status-cache.service';
 import { UpgradeStatusService } from 'src/engine/core-modules/upgrade/services/upgrade-status.service';
 import { WorkspaceCommandRunnerService } from 'src/engine/core-modules/upgrade/services/workspace-command-runner.service';
+import { UpgradeGaugeService } from 'src/engine/core-modules/upgrade/upgrade-gauge.service';
 import { UpgradeMigrationEntity } from 'src/engine/core-modules/upgrade/upgrade-migration.entity';
 import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity';
 import { WorkspaceVersionModule } from 'src/engine/workspace-manager/workspace-version/workspace-version.module';
@@ -23,6 +25,7 @@ import { WorkspaceVersionModule } from 'src/engine/workspace-manager/workspace-v
    CoreEntityCacheModule,
    DiscoveryModule,
    InstanceCommandProviderModule,
+    MetricsModule,
    WorkspaceCommandProviderModule,
    WorkspaceIteratorModule,
    WorkspaceVersionModule,
@@ -37,6 +40,7 @@ import { WorkspaceVersionModule } from 'src/engine/workspace-manager/workspace-v
    UpgradeSequenceRunnerService,
    UpgradeStatusService,
    UpgradeStatusCacheService,
+    UpgradeGaugeService,
  ],
  exports: [
    UpgradeMigrationService,