Add server upgrade status on admin panel (#20107)

## Summary

Adds an admin upgrade-status panel that surfaces per-instance and
per-workspace migration health, backed by a Redis-cached aggregate to
keep the page snappy on large fleets.

<img width="827" height="880" alt="Screenshot 2026-04-28 at 10 21 03"
src="https://github.com/user-attachments/assets/8f88baa9-7268-4eff-bf6a-906a7f06ca91"
/>
<img width="804" height="892" alt="Screenshot 2026-04-28 at 10 21 11"
src="https://github.com/user-attachments/assets/1e6decf8-766a-4d0e-96b1-03a9962bba3c"
/>


## Computed metrics

**Instance** (`InstanceUpgradeStatus`)
- `inferredVersion` — version derived from the latest non-initial
instance command name
- `health` — `upToDate` | `behind` | `failed`, derived from the latest
attempt vs. the last expected instance step in the upgrade sequence
- `latestCommand` — `{ name, status, executedByVersion, errorMessage,
createdAt }` from the most recent attempt

**Per-workspace** (`WorkspaceUpgradeStatus`)
- `workspaceId`, `displayName`
- `inferredVersion`, `health`, `latestCommand` (same shape as instance),
computed against the latest expected step in the sequence

**Aggregate** (`AllWorkspacesUpgradeStatus`, only across `ACTIVE` /
`SUSPENDED` workspaces)
- `instanceUpgradeStatus`
- `totalCount`, `upToDateCount`, `behindCount`, `failedCount`
- `workspacesBehindIds[]`, `workspacesFailedIds[]`
- `computedAt`

## Fetching strategy

All reads go through `UpgradeStatusCacheService` (cache namespace:
`EngineHealth`).

- **Aggregate read** (`getAllWorkspacesStatus` →
`getAllWorkspacesUpgradeStatus` query):
reads summary + behind-ids + failed-ids in parallel; if any of the three
keys is missing, full recompute (`recomputeAllWorkspaces`) is triggered,
which also primes per-workspace entries.
- **Per-workspace read** (`getWorkspacesStatus(ids)` →
`getUpgradeStatus(ids)` query):
`mget` on workspace keys; misses are recomputed individually
(`recomputeWorkspace`), and aggregates are reconciled in place (count +
id list deltas) without a full recompute.
- **Recompute on demand**: `refreshUpgradeStatus` mutation calls
`recomputeAllWorkspaces` to bypass cache and rewrite all keys.
- **Auto-invalidation**: `InstanceCommandRunnerService` (fast + slow
paths) and `WorkspaceCommandRunnerService` invalidate after every run
via `safeInvalidateUpgradeStatusCache()`
(`flushByPattern('upgrade-status:*')`). Failures in cache invalidation
are swallowed and logged so they never break the migration runner.
- **TTL**: `60 * 60 * 1000` ms (1 hour) on every key — protects against
stale data even if a runner crashes before invalidating.

## Introduced cache keys

All under the `EngineHealth` cache-storage namespace:

| Key | Type | Purpose |
| --- | --- | --- |
| `upgrade-status:all-workspaces:summary` |
`CachedAllWorkspacesStatusSummary` | Counts + instance status +
`computedAt` |
| `upgrade-status:all-workspaces:behind-ids` | `string[]` | Workspace
ids in `behind` state |
| `upgrade-status:all-workspaces:failed-ids` | `string[]` | Workspace
ids in `failed` state |
| `upgrade-status:workspace:<workspaceId>` |
`CachedWorkspaceUpgradeStatus` | Per-workspace status (one key per
workspace) |

Full invalidation uses the pattern `upgrade-status:*`.

## Index added on `upgradeMigration` (already added on prod)

Migration
`2-2-instance-command-fast-1777308014234-addUpgradeMigrationWorkspaceIdIndex.ts`:

```sql
CREATE INDEX "IDX_upgradeMigration_workspaceId_name_attempt"
  ON "core"."upgradeMigration" ("workspaceId", "name", "attempt")
  WHERE "workspaceId" IS NOT NULL;
This commit is contained in:
Marie
2026-05-04 17:06:07 +02:00
committed by GitHub
parent 1cd983a330
commit 4852ac401a
49 changed files with 2100 additions and 213 deletions
@@ -0,0 +1,21 @@
import { QueryRunner } from 'typeorm';
import { RegisteredInstanceCommand } from 'src/engine/core-modules/upgrade/decorators/registered-instance-command.decorator';
import { FastInstanceCommand } from 'src/engine/core-modules/upgrade/interfaces/fast-instance-command.interface';
@RegisteredInstanceCommand('2.3.0', 1777308014234)
export class AddUpgradeMigrationWorkspaceIdIndexFastInstanceCommand
implements FastInstanceCommand
{
public async up(queryRunner: QueryRunner): Promise<void> {
await queryRunner.query(
'CREATE INDEX IF NOT EXISTS "IDX_UPGRADE_MIGRATION_WORKSPACE_ID_NAME_ATTEMPT" ON "core"."upgradeMigration" ("workspaceId", "name", "attempt") WHERE "workspaceId" IS NOT NULL',
);
}
public async down(queryRunner: QueryRunner): Promise<void> {
await queryRunner.query(
'DROP INDEX IF EXISTS "core"."IDX_UPGRADE_MIGRATION_WORKSPACE_ID_NAME_ATTEMPT"',
);
}
}
@@ -3,21 +3,22 @@
import { AddViewFieldGroupIdIndexOnViewFieldFastInstanceCommand } from 'src/database/commands/upgrade-version-command/1-21/1-21-instance-command-fast-1775129420309-add-view-field-group-id-index-on-view-field';
import { MigrateMessagingCalendarToCoreFastInstanceCommand } from 'src/database/commands/upgrade-version-command/1-21/1-21-instance-command-fast-1775165049548-migrate-messaging-calendar-to-core';
import { AddEmailThreadWidgetTypeFastInstanceCommand } from 'src/database/commands/upgrade-version-command/1-21/1-21-instance-command-fast-1775200000000-add-email-thread-widget-type';
import { AddStandalonePageFastInstanceCommand } from 'src/database/commands/upgrade-version-command/1-23/1-23-instance-command-fast-1775752781995-add-standalone-page';
import { AddPermissionFlagRoleIdIndexFastInstanceCommand } from 'src/database/commands/upgrade-version-command/1-22/1-22-instance-command-fast-1775749486425-add-permission-flag-role-id-index';
import { AddTableWidgetViewTypeFastInstanceCommand } from 'src/database/commands/upgrade-version-command/1-23/1-23-instance-command-fast-1775752190522-add-table-widget-view-type';
import { AddWorkspaceIdToIndirectEntitiesFastInstanceCommand } from 'src/database/commands/upgrade-version-command/1-22/1-22-instance-command-fast-1775758621017-add-workspace-id-to-indirect-entities';
import { AddWorkspaceIdIndexesAndFksFastInstanceCommand } from 'src/database/commands/upgrade-version-command/1-22/1-22-instance-command-fast-1775761294897-add-workspace-id-indexes-and-fks-to-indirect-entities';
import { DropObjectMetadataDataSourceFkFastInstanceCommand } from 'src/database/commands/upgrade-version-command/1-22/1-22-instance-command-fast-1775804361516-drop-object-metadata-data-source-fk';
import { AddCreditBalanceToBillingCustomerFastInstanceCommand } from 'src/database/commands/upgrade-version-command/1-22/1-22-instance-command-fast-1776078919203-add-credit-balance-to-billing-customer';
import { BackfillWorkspaceIdOnIndirectEntitiesSlowInstanceCommand } from 'src/database/commands/upgrade-version-command/1-22/1-22-instance-command-slow-1775758621018-backfill-workspace-id-on-indirect-entities';
import { DropWorkspaceVersionColumnFastInstanceCommand } from 'src/database/commands/upgrade-version-command/1-23/1-23-instance-command-fast-1785000000000-drop-workspace-version-column';
import { AddConditionalAvailabilityExpressionToPageLayoutWidgetFastInstanceCommand } from 'src/database/commands/upgrade-version-command/1-23/1-23-instance-command-fast-1775654781000-add-conditional-availability-expression-to-page-layout-widget';
import { AddTableWidgetViewTypeFastInstanceCommand } from 'src/database/commands/upgrade-version-command/1-23/1-23-instance-command-fast-1775752190522-add-table-widget-view-type';
import { AddStandalonePageFastInstanceCommand } from 'src/database/commands/upgrade-version-command/1-23/1-23-instance-command-fast-1775752781995-add-standalone-page';
import { AddGlobalObjectContextToCommandMenuItemAvailabilityTypeFastInstanceCommand } from 'src/database/commands/upgrade-version-command/1-23/1-23-instance-command-fast-1776090711153-add-global-object-context-to-command-menu-item-availability-type';
import { AddPageLayoutIdToCommandMenuItemFastInstanceCommand } from 'src/database/commands/upgrade-version-command/1-23/1-23-instance-command-fast-1776168404836-add-page-layout-id-to-command-menu-item';
import { AddConditionalAvailabilityExpressionToPageLayoutWidgetFastInstanceCommand } from 'src/database/commands/upgrade-version-command/1-23/1-23-instance-command-fast-1775654781000-add-conditional-availability-expression-to-page-layout-widget';
import { DropWorkspaceVersionColumnFastInstanceCommand } from 'src/database/commands/upgrade-version-command/1-23/1-23-instance-command-fast-1785000000000-drop-workspace-version-column';
import { AddIsPreInstalledToApplicationRegistrationFastInstanceCommand } from 'src/database/commands/upgrade-version-command/2-1/2-1-instance-command-fast-1776886452831-add-is-pre-installed-to-application-registration';
import { AddProviderExecutedToAgentMessagePartFastInstanceCommand } from 'src/database/commands/upgrade-version-command/2-1/2-1-instance-command-fast-1777012800000-add-provider-executed-to-agent-message-part';
import { BackfillPageLayoutWidgetPositionSlowInstanceCommand } from 'src/database/commands/upgrade-version-command/2-1/2-1-instance-command-slow-1795000002000-backfill-page-layout-widget-position';
import { AddUpgradeMigrationWorkspaceIdIndexFastInstanceCommand } from 'src/database/commands/upgrade-version-command/2-3/2-3-instance-command-fast-1777308014234-add-upgrade-migration-workspace-id-index';
import { AddCacheTokensToAgentChatThreadFastInstanceCommand } from 'src/database/commands/upgrade-version-command/2-2/2-2-instance-command-fast-1777455269302-add-cache-tokens-to-agent-chat-thread';
import { AddLogoToApplicationFastInstanceCommand } from 'src/database/commands/upgrade-version-command/2-2/2-2-instance-command-fast-1777539664664-add-logo-to-application';
import { AddDeletedAtToAgentChatThreadFastInstanceCommand } from 'src/database/commands/upgrade-version-command/2-3/2-3-instance-command-fast-1777682000000-add-deleted-at-to-agent-chat-thread';
@@ -38,6 +39,7 @@ export const INSTANCE_COMMANDS = [
AddGlobalObjectContextToCommandMenuItemAvailabilityTypeFastInstanceCommand,
AddPageLayoutIdToCommandMenuItemFastInstanceCommand,
AddConditionalAvailabilityExpressionToPageLayoutWidgetFastInstanceCommand,
AddUpgradeMigrationWorkspaceIdIndexFastInstanceCommand,
AddIsPreInstalledToApplicationRegistrationFastInstanceCommand,
AddProviderExecutedToAgentMessagePartFastInstanceCommand,
BackfillPageLayoutWidgetPositionSlowInstanceCommand,