Optimize upgrade status gauges with count-only queries (#23574)

## Context Upgrade health metrics and the admin upgrade-status query currently share `getInstanceAndAllWorkspacesStatus`. On a cache hit, that method reads the cached behind/failed workspace IDs, then hydrates every workspace name with an individual `CoreEntityCacheService.get` call. This is useful for the admin response, but the gauges only need the number of workspaces in each state. As the number of behind or failed workspaces grows, every gauge refresh therefore creates a fan-out of entity-cache lookups. Those lookups can include Redis validation and response deserialization. Production profiling of slow upgrade-status requests showed `loadWorkspaceNamesById` and `CoreEntityCacheService.get` on the hot path, so this PR removes that unnecessary repeated work. ## What changed - Added a count-only upgrade-status method for metric collection. - Updated upgrade gauges to use cached ID counts without loading workspace names. - Replaced the admin path's per-workspace cache lookups with one repository query selecting only `id` and `displayName`. - Removed the upgrade module's now-unused core-entity-cache dependency. ## Why this improves performance ### Metrics path Before: - Read the cached upgrade-status IDs. - Run one entity-cache lookup per behind/failed workspace. - Discard the hydrated names and only use the array lengths. After: - Read the same cached upgrade-status IDs. - Derive counts directly from those IDs. - Perform no workspace-name lookup. **This changes metric collection from a fixed set of status-cache calls plus `N` entity-cache calls to only the fixed status-cache calls. The amount of ID data still scales with the number of affected workspaces, but the Redis/client round-trip fan-out does not.** ### Admin path The admin response still needs workspace names. It now loads them with one primary-key `IN` query instead of `N` independent entity-cache calls. This reduces round trips and repeated cache validation while preserving the response shape. ## Safety and behavior preservation - Upgrade-status cache keys, TTLs and invalidation behavior are unchanged. - A missing cache marker still triggers the existing full status refresh. - Metrics names and values are unchanged. - The admin GraphQL response is unchanged. - Cached workspace IDs missing from the database still produce a `null` name, matching the previous behavior. - The batched query runs only for callers that request the detailed admin payload, not for metric collection. ## Expected impact - Remove recurring per-workspace cache fan-out from every API process collecting upgrade gauges. - Reduce Redis client work, response deserialization and event-loop pressure during metric collection. - Reduce latency for detailed admin upgrade-status requests. This targets one profiled source of tail latency. It is not expected to eliminate all API p99 outliers, which also have independent causes. ## Validation - 36 focused upgrade-status and gauge tests pass. - `yarn nx typecheck twenty-server` passes. - Oxlint passes with zero warnings and errors. - Oxfmt and `git diff --check` pass.
2026-07-30 16:38:48 +02:00
parent ad271ee639
commit 3689e89440
5 changed files with 199 additions and 79 deletions
@@ -0,0 +1,41 @@
+import { type MetricsService } from 'src/engine/core-modules/metrics/metrics.service';
+import { type UpgradeStatusService } from 'src/engine/core-modules/upgrade/services/upgrade-status.service';
+import { UpgradeGaugeService } from 'src/engine/core-modules/upgrade/upgrade-gauge.service';
+import { UpgradeHealthEnum } from 'twenty-shared/types';
+
+describe('UpgradeGaugeService', () => {
+  it('collects workspace counts without hydrating workspace names', async () => {
+    const createObservableGauge = jest.fn();
+    const createInfoGauge = jest.fn();
+    const getInstanceAndWorkspaceCountsStatus = jest.fn().mockResolvedValue({
+      instanceUpgradeStatus: {
+        inferredVersion: 'v1.2.3',
+        health: UpgradeHealthEnum.UP_TO_DATE,
+        latestCommand: null,
+      },
+      behindWorkspaceCount: 2,
+      failedWorkspaceCount: 1,
+      upToDateWorkspaceCount: 5,
+      computedAt: new Date(),
+    });
+    const service = new UpgradeGaugeService(
+      {
+        createObservableGauge,
+        createInfoGauge,
+      } as unknown as MetricsService,
+      {
+        getInstanceAndWorkspaceCountsStatus,
+      } as unknown as UpgradeStatusService,
+    );
+
+    service.onModuleInit();
+
+    const behindGaugeCallback = createObservableGauge.mock.calls.find(
+      ([options]) =>
+        options.metricName === 'twenty_upgrade_workspaces_behind_total',
+    )?.[0].callback;
+
+    await expect(behindGaugeCallback()).resolves.toBe(2);
+    expect(getInstanceAndWorkspaceCountsStatus).toHaveBeenCalledTimes(1);
+  });
+});
@@ -3,9 +3,7 @@ import { getRepositoryToken } from '@nestjs/typeorm';

 import { UpgradeHealthEnum } from 'twenty-shared/types';
 import { isDefined } from 'twenty-shared/utils';
-import { WorkspaceActivationStatus } from 'twenty-shared/workspace';

-import { CoreEntityCacheService } from 'src/engine/core-entity-cache/services/core-entity-cache.service';
 import { TWENTY_CROSS_UPGRADE_SUPPORTED_VERSIONS } from 'src/engine/core-modules/upgrade/constants/twenty-cross-upgrade-supported-version.constant';
 import { UpgradeMigrationService } from 'src/engine/core-modules/upgrade/services/upgrade-migration.service';
 import { UpgradeSequenceReaderService } from 'src/engine/core-modules/upgrade/services/upgrade-sequence-reader.service';
@@ -52,34 +50,12 @@ type WorkspaceRecord = {
  displayName: string | null;
 };

-const buildWorkspaceCacheGetMock = (
-  workspaces: WorkspaceRecord[],
-): jest.Mock => {
-  const byId = new Map(
-    workspaces.map((workspace) => [workspace.id, workspace]),
-  );
-
-  return jest.fn(async (_cacheKey: string, workspaceId: string) => {
-    const workspace = byId.get(workspaceId);
-
-    if (!workspace) {
-      return null;
-    }
-
-    return {
-      activationStatus: WorkspaceActivationStatus.ACTIVE,
-      ...workspace,
-    };
-  });
-};
-
 describe('UpgradeStatusService', () => {
  let service: UpgradeStatusService;
  let sequence: { kind: string; name: string }[];
  let getLastAttemptedInstanceCommand: jest.Mock;
  let getWorkspaceLastAttemptedCommandName: jest.Mock;
  let workspaceFind: jest.Mock;
-  let coreEntityCacheGet: jest.Mock;
  let cacheGetComputedAt: jest.Mock;
  let cacheGetBehindWorkspaceIds: jest.Mock;
  let cacheGetFailedWorkspaceIds: jest.Mock;
@@ -89,9 +65,6 @@ describe('UpgradeStatusService', () => {

  const mockActiveWorkspaces = (workspaces: WorkspaceRecord[]) => {
    workspaceFind.mockResolvedValue(workspaces);
-    coreEntityCacheGet.mockImplementation(
-      buildWorkspaceCacheGetMock(workspaces),
-    );
  };

  beforeEach(async () => {
@@ -99,7 +72,6 @@ describe('UpgradeStatusService', () => {
    getLastAttemptedInstanceCommand = jest.fn();
    getWorkspaceLastAttemptedCommandName = jest.fn();
    workspaceFind = jest.fn().mockResolvedValue([]);
-    coreEntityCacheGet = jest.fn().mockResolvedValue(null);
    cacheGetComputedAt = jest.fn();
    cacheGetBehindWorkspaceIds = jest.fn().mockResolvedValue([]);
    cacheGetFailedWorkspaceIds = jest.fn().mockResolvedValue([]);
@@ -133,10 +105,6 @@ describe('UpgradeStatusService', () => {
          provide: getRepositoryToken(WorkspaceEntity),
          useValue: { find: workspaceFind },
        },
-        {
-          provide: CoreEntityCacheService,
-          useValue: { get: coreEntityCacheGet },
-        },
        {
          provide: UpgradeStatusCacheService,
          useValue: {
@@ -551,12 +519,10 @@ describe('UpgradeStatusService', () => {
        errorMessage: null,
        createdAt: new Date('2025-06-01T00:00:00Z'),
      });
-      coreEntityCacheGet.mockImplementation(
-        buildWorkspaceCacheGetMock([
+      workspaceFind.mockResolvedValue([
        { id: 'ws-2', displayName: 'Banana' },
        { id: 'ws-3', displayName: 'Cherry' },
-        ]),
-      );
+      ]);

      const result = await service.getInstanceAndAllWorkspacesStatus();

@@ -564,6 +530,7 @@ describe('UpgradeStatusService', () => {
      expect(result.workspacesFailed).toEqual([{ id: 'ws-3', name: 'Cherry' }]);
      expect(result.upToDateWorkspaceCount).toBe(5);
      expect(result.computedAt).toEqual(computedAt);
+      expect(workspaceFind).toHaveBeenCalledTimes(1);
      expect(getWorkspaceLastAttemptedCommandName).not.toHaveBeenCalled();
      expect(cacheWrite).not.toHaveBeenCalled();
    });
@@ -580,11 +547,10 @@ describe('UpgradeStatusService', () => {
      expect(result.workspacesBehind).toEqual([{ id: 'ws-1', name: 'Apple' }]);
    });

-    it('should use null name when a cached id is missing from the cache', async () => {
+    it('should use null name when a cached id is missing from the database', async () => {
      cacheGetComputedAt.mockResolvedValue(new Date());
      cacheGetBehindWorkspaceIds.mockResolvedValue(['ws-orphan']);
      getLastAttemptedInstanceCommand.mockResolvedValue(null);
-      coreEntityCacheGet.mockResolvedValue(null);

      const result = await service.getInstanceAndAllWorkspacesStatus();

@@ -599,7 +565,66 @@ describe('UpgradeStatusService', () => {

      await service.getInstanceAndAllWorkspacesStatus();

-      expect(coreEntityCacheGet).not.toHaveBeenCalled();
+      expect(workspaceFind).not.toHaveBeenCalled();
+    });
+  });
+
+  describe('getInstanceAndWorkspaceCountsStatus', () => {
+    it('should derive counts from cached ids without loading workspace names', async () => {
+      const computedAt = new Date('2025-06-02T10:00:00Z');
+
+      cacheGetComputedAt.mockResolvedValue(computedAt);
+      cacheGetBehindWorkspaceIds.mockResolvedValue(['ws-1', 'ws-2']);
+      cacheGetFailedWorkspaceIds.mockResolvedValue(['ws-3']);
+      cacheGetUpToDateWorkspaceCount.mockResolvedValue(5);
+      getLastAttemptedInstanceCommand.mockResolvedValue(null);
+
+      const result = await service.getInstanceAndWorkspaceCountsStatus();
+
+      expect(result).toEqual({
+        instanceUpgradeStatus: {
+          inferredVersion: null,
+          health: UpgradeHealthEnum.BEHIND,
+          latestCommand: null,
+        },
+        behindWorkspaceCount: 2,
+        failedWorkspaceCount: 1,
+        upToDateWorkspaceCount: 5,
+        computedAt,
+      });
+      expect(workspaceFind).not.toHaveBeenCalled();
+      expect(cacheWrite).not.toHaveBeenCalled();
+    });
+
+    it('should derive counts from a refresh when the cache marker is missing', async () => {
+      cacheGetComputedAt.mockResolvedValue(null);
+      getLastAttemptedInstanceCommand.mockResolvedValue(null);
+      mockActiveWorkspaces([
+        { id: 'ws-1', displayName: 'Apple' },
+        { id: 'ws-2', displayName: 'Banana' },
+      ]);
+      getWorkspaceLastAttemptedCommandName.mockResolvedValue(
+        new Map([
+          [
+            'ws-2',
+            {
+              workspaceId: 'ws-2',
+              name: NEWEST_WORKSPACE_COMMAND,
+              status: 'failed',
+              executedByVersion: NEWEST_VERSION,
+              errorMessage: 'boom',
+              createdAt: new Date('2025-06-01T00:00:00Z'),
+            },
+          ],
+        ]),
+      );
+
+      const result = await service.getInstanceAndWorkspaceCountsStatus();
+
+      expect(result.behindWorkspaceCount).toBe(1);
+      expect(result.failedWorkspaceCount).toBe(1);
+      expect(result.upToDateWorkspaceCount).toBe(0);
+      expect(cacheWrite).toHaveBeenCalledTimes(1);
    });
  });

@@ -5,7 +5,6 @@ import { isDefined } from 'twenty-shared/utils';
 import { PROVISIONED_WORKSPACE_ACTIVATION_STATUSES } from 'twenty-shared/workspace';

 import { InjectRepository } from '@nestjs/typeorm';
-import { CoreEntityCacheService } from 'src/engine/core-entity-cache/services/core-entity-cache.service';
 import { TWENTY_CROSS_UPGRADE_SUPPORTED_VERSIONS } from 'src/engine/core-modules/upgrade/constants/twenty-cross-upgrade-supported-version.constant';
 import { UpgradeMigrationService } from 'src/engine/core-modules/upgrade/services/upgrade-migration.service';
 import { UpgradeSequenceReaderService } from 'src/engine/core-modules/upgrade/services/upgrade-sequence-reader.service';
@@ -54,6 +53,22 @@ export type InstanceAndAllWorkspacesUpgradeStatus = {
  computedAt: Date;
 };

+export type InstanceAndWorkspaceCountsUpgradeStatus = {
+  instanceUpgradeStatus: InstanceUpgradeStatus;
+  behindWorkspaceCount: number;
+  failedWorkspaceCount: number;
+  upToDateWorkspaceCount: number;
+  computedAt: Date;
+};
+
+type CachedInstanceAndWorkspaceUpgradeStatus = {
+  instanceUpgradeStatus: InstanceUpgradeStatus;
+  behindWorkspaceIds: string[];
+  failedWorkspaceIds: string[];
+  upToDateWorkspaceCount: number;
+  computedAt: Date;
+};
+
 const deriveHealth = (
  cursor: UpgradeCursor,
  lastExpectedCommandName: string | null,
@@ -82,7 +97,6 @@ export class UpgradeStatusService {
    @InjectRepository(WorkspaceEntity)
    private readonly workspaceRepository: Repository<WorkspaceEntity>,
    private readonly upgradeStatusCacheService: UpgradeStatusCacheService,
-    private readonly coreEntityCacheService: CoreEntityCacheService,
  ) {}

  async getInstanceStatus(): Promise<InstanceUpgradeStatus> {
@@ -163,42 +177,55 @@ export class UpgradeStatusService {
    });
  }

-  async getInstanceAndAllWorkspacesStatus(): Promise<InstanceAndAllWorkspacesUpgradeStatus> {
-    const computedAt = await this.upgradeStatusCacheService.getComputedAt();
+  async getInstanceAndWorkspaceCountsStatus(): Promise<InstanceAndWorkspaceCountsUpgradeStatus> {
+    const cachedStatus = await this.getCachedInstanceAndWorkspaceStatus();

-    if (!isDefined(computedAt)) {
+    if (!isDefined(cachedStatus)) {
+      const refreshedStatus =
+        await this.refreshInstanceAndAllWorkspacesStatus();
+
+      return {
+        instanceUpgradeStatus: refreshedStatus.instanceUpgradeStatus,
+        behindWorkspaceCount: refreshedStatus.workspacesBehind.length,
+        failedWorkspaceCount: refreshedStatus.workspacesFailed.length,
+        upToDateWorkspaceCount: refreshedStatus.upToDateWorkspaceCount,
+        computedAt: refreshedStatus.computedAt,
+      };
+    }
+
+    return {
+      instanceUpgradeStatus: cachedStatus.instanceUpgradeStatus,
+      behindWorkspaceCount: cachedStatus.behindWorkspaceIds.length,
+      failedWorkspaceCount: cachedStatus.failedWorkspaceIds.length,
+      upToDateWorkspaceCount: cachedStatus.upToDateWorkspaceCount,
+      computedAt: cachedStatus.computedAt,
+    };
+  }
+
+  async getInstanceAndAllWorkspacesStatus(): Promise<InstanceAndAllWorkspacesUpgradeStatus> {
+    const cachedStatus = await this.getCachedInstanceAndWorkspaceStatus();
+
+    if (!isDefined(cachedStatus)) {
      return this.refreshInstanceAndAllWorkspacesStatus();
    }

-    const [
-      instanceUpgradeStatus,
-      behindWorkspaceIds,
-      failedWorkspaceIds,
-      upToDateWorkspaceCount,
-    ] = await Promise.all([
-      this.getInstanceStatus(),
-      this.upgradeStatusCacheService.getBehindWorkspaceIds(),
-      this.upgradeStatusCacheService.getFailedWorkspaceIds(),
-      this.upgradeStatusCacheService.getUpToDateWorkspaceCount(),
-    ]);
-
    const workspaceNamesById = await this.loadWorkspaceNamesById([
-      ...behindWorkspaceIds,
-      ...failedWorkspaceIds,
+      ...cachedStatus.behindWorkspaceIds,
+      ...cachedStatus.failedWorkspaceIds,
    ]);

    return {
-      instanceUpgradeStatus,
+      instanceUpgradeStatus: cachedStatus.instanceUpgradeStatus,
      workspacesBehind: this.toWorkspaceRefs(
-        behindWorkspaceIds,
+        cachedStatus.behindWorkspaceIds,
        workspaceNamesById,
      ),
      workspacesFailed: this.toWorkspaceRefs(
-        failedWorkspaceIds,
+        cachedStatus.failedWorkspaceIds,
        workspaceNamesById,
      ),
-      upToDateWorkspaceCount,
-      computedAt,
+      upToDateWorkspaceCount: cachedStatus.upToDateWorkspaceCount,
+      computedAt: cachedStatus.computedAt,
    };
  }

@@ -251,6 +278,34 @@ export class UpgradeStatusService {
    await this.upgradeStatusCacheService.invalidate();
  }

+  private async getCachedInstanceAndWorkspaceStatus(): Promise<CachedInstanceAndWorkspaceUpgradeStatus | null> {
+    const computedAt = await this.upgradeStatusCacheService.getComputedAt();
+
+    if (!isDefined(computedAt)) {
+      return null;
+    }
+
+    const [
+      instanceUpgradeStatus,
+      behindWorkspaceIds,
+      failedWorkspaceIds,
+      upToDateWorkspaceCount,
+    ] = await Promise.all([
+      this.getInstanceStatus(),
+      this.upgradeStatusCacheService.getBehindWorkspaceIds(),
+      this.upgradeStatusCacheService.getFailedWorkspaceIds(),
+      this.upgradeStatusCacheService.getUpToDateWorkspaceCount(),
+    ]);
+
+    return {
+      instanceUpgradeStatus,
+      behindWorkspaceIds,
+      failedWorkspaceIds,
+      upToDateWorkspaceCount,
+      computedAt,
+    };
+  }
+
  private resolveInstanceCompletedVersion(
    cursor: UpgradeCursor | null,
  ): string | null {
@@ -332,11 +387,12 @@ export class UpgradeStatusService {
      return namesById;
    }

-    const workspaces = await Promise.all(
-      workspaceIds.map((workspaceId) =>
-        this.coreEntityCacheService.get('workspaceEntity', workspaceId),
-      ),
-    );
+    const workspaces = await this.workspaceRepository.find({
+      select: ['id', 'displayName'],
+      where: {
+        id: In(workspaceIds),
+      },
+    });

    for (const workspace of workspaces) {
      if (isDefined(workspace)) {
@@ -4,7 +4,7 @@ import { UpgradeHealthEnum } from 'twenty-shared/types';

 import { MetricsService } from 'src/engine/core-modules/metrics/metrics.service';
 import {
-  type InstanceAndAllWorkspacesUpgradeStatus,
+  type InstanceAndWorkspaceCountsUpgradeStatus,
  UpgradeStatusService,
 } from 'src/engine/core-modules/upgrade/services/upgrade-status.service';

@@ -21,10 +21,10 @@ const UPGRADE_STATUS_TTL_MS = 60_000;
 export class UpgradeGaugeService implements OnModuleInit {
  private readonly logger = new Logger(UpgradeGaugeService.name);

-  private cachedUpgradeStatus: InstanceAndAllWorkspacesUpgradeStatus | null =
+  private cachedUpgradeStatus: InstanceAndWorkspaceCountsUpgradeStatus | null =
    null;
  private cachedUpgradeStatusExpiresAt = 0;
-  private inflightUpgradeStatusPromise: Promise<InstanceAndAllWorkspacesUpgradeStatus> | null =
+  private inflightUpgradeStatusPromise: Promise<InstanceAndWorkspaceCountsUpgradeStatus> | null =
    null;

  constructor(
@@ -62,7 +62,7 @@ export class UpgradeGaugeService implements OnModuleInit {
      callback: async () => {
        const upgradeStatus = await this.getCachedUpgradeStatus();

-        return upgradeStatus?.workspacesBehind.length ?? 0;
+        return upgradeStatus?.behindWorkspaceCount ?? 0;
      },
      cacheValue: true,
    });
@@ -75,7 +75,7 @@ export class UpgradeGaugeService implements OnModuleInit {
      callback: async () => {
        const upgradeStatus = await this.getCachedUpgradeStatus();

-        return upgradeStatus?.workspacesFailed.length ?? 0;
+        return upgradeStatus?.failedWorkspaceCount ?? 0;
      },
      cacheValue: true,
    });
@@ -110,7 +110,7 @@ export class UpgradeGaugeService implements OnModuleInit {
    });
  }

-  private async getCachedUpgradeStatus(): Promise<InstanceAndAllWorkspacesUpgradeStatus | null> {
+  private async getCachedUpgradeStatus(): Promise<InstanceAndWorkspaceCountsUpgradeStatus | null> {
    if (
      this.cachedUpgradeStatus &&
      Date.now() < this.cachedUpgradeStatusExpiresAt
@@ -123,7 +123,7 @@ export class UpgradeGaugeService implements OnModuleInit {
    }

    this.inflightUpgradeStatusPromise =
-      this.upgradeStatusService.getInstanceAndAllWorkspacesStatus();
+      this.upgradeStatusService.getInstanceAndWorkspaceCountsStatus();

    try {
      this.cachedUpgradeStatus = await this.inflightUpgradeStatusPromise;
@@ -2,7 +2,6 @@ import { Module } from '@nestjs/common';
 import { DiscoveryModule } from '@nestjs/core';
 import { TypeOrmModule } from '@nestjs/typeorm';

-import { CoreEntityCacheModule } from 'src/engine/core-entity-cache/core-entity-cache.module';
 import { UpgradeCommandRegistryService } from 'src/engine/core-modules/upgrade/services/upgrade-command-registry.service';
 import { UpgradeMigrationService } from 'src/engine/core-modules/upgrade/services/upgrade-migration.service';
 import { UpgradeSequenceReaderService } from 'src/engine/core-modules/upgrade/services/upgrade-sequence-reader.service';
@@ -13,7 +12,6 @@ import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.ent

@Module({
  imports: [
-    CoreEntityCacheModule,
    DiscoveryModule,
    TypeOrmModule.forFeature([UpgradeMigrationEntity, WorkspaceEntity]),
  ],