Add server upgrade status on admin panel (#20107)

## Summary

Adds an admin upgrade-status panel that surfaces per-instance and
per-workspace migration health, backed by a Redis-cached aggregate to
keep the page snappy on large fleets.

<img width="827" height="880" alt="Screenshot 2026-04-28 at 10 21 03"
src="https://github.com/user-attachments/assets/8f88baa9-7268-4eff-bf6a-906a7f06ca91"
/>
<img width="804" height="892" alt="Screenshot 2026-04-28 at 10 21 11"
src="https://github.com/user-attachments/assets/1e6decf8-766a-4d0e-96b1-03a9962bba3c"
/>


## Computed metrics

**Instance** (`InstanceUpgradeStatus`)
- `inferredVersion` — version derived from the latest non-initial
instance command name
- `health` — `upToDate` | `behind` | `failed`, derived from the latest
attempt vs. the last expected instance step in the upgrade sequence
- `latestCommand` — `{ name, status, executedByVersion, errorMessage,
createdAt }` from the most recent attempt

**Per-workspace** (`WorkspaceUpgradeStatus`)
- `workspaceId`, `displayName`
- `inferredVersion`, `health`, `latestCommand` (same shape as instance),
computed against the latest expected step in the sequence

**Aggregate** (`AllWorkspacesUpgradeStatus`, only across `ACTIVE` /
`SUSPENDED` workspaces)
- `instanceUpgradeStatus`
- `totalCount`, `upToDateCount`, `behindCount`, `failedCount`
- `workspacesBehindIds[]`, `workspacesFailedIds[]`
- `computedAt`

## Fetching strategy

All reads go through `UpgradeStatusCacheService` (cache namespace:
`EngineHealth`).

- **Aggregate read** (`getAllWorkspacesStatus` →
`getAllWorkspacesUpgradeStatus` query):
reads summary + behind-ids + failed-ids in parallel; if any of the three
keys is missing, full recompute (`recomputeAllWorkspaces`) is triggered,
which also primes per-workspace entries.
- **Per-workspace read** (`getWorkspacesStatus(ids)` →
`getUpgradeStatus(ids)` query):
`mget` on workspace keys; misses are recomputed individually
(`recomputeWorkspace`), and aggregates are reconciled in place (count +
id list deltas) without a full recompute.
- **Recompute on demand**: `refreshUpgradeStatus` mutation calls
`recomputeAllWorkspaces` to bypass cache and rewrite all keys.
- **Auto-invalidation**: `InstanceCommandRunnerService` (fast + slow
paths) and `WorkspaceCommandRunnerService` invalidate after every run
via `safeInvalidateUpgradeStatusCache()`
(`flushByPattern('upgrade-status:*')`). Failures in cache invalidation
are swallowed and logged so they never break the migration runner.
- **TTL**: `60 * 60 * 1000` ms (1 hour) on every key — protects against
stale data even if a runner crashes before invalidating.

## Introduced cache keys

All under the `EngineHealth` cache-storage namespace:

| Key | Type | Purpose |
| --- | --- | --- |
| `upgrade-status:all-workspaces:summary` |
`CachedAllWorkspacesStatusSummary` | Counts + instance status +
`computedAt` |
| `upgrade-status:all-workspaces:behind-ids` | `string[]` | Workspace
ids in `behind` state |
| `upgrade-status:all-workspaces:failed-ids` | `string[]` | Workspace
ids in `failed` state |
| `upgrade-status:workspace:<workspaceId>` |
`CachedWorkspaceUpgradeStatus` | Per-workspace status (one key per
workspace) |

Full invalidation uses the pattern `upgrade-status:*`.

## Index added on `upgradeMigration` (already added on prod)

Migration
`2-2-instance-command-fast-1777308014234-addUpgradeMigrationWorkspaceIdIndex.ts`:

```sql
CREATE INDEX "IDX_upgradeMigration_workspaceId_name_attempt"
  ON "core"."upgradeMigration" ("workspaceId", "name", "attempt")
  WHERE "workspaceId" IS NOT NULL;
This commit is contained in:
Marie
2026-05-04 17:06:07 +02:00
committed by GitHub
parent 1cd983a330
commit 4852ac401a
49 changed files with 2100 additions and 213 deletions
@@ -0,0 +1,23 @@
import { t } from '@lingui/core/macro';
import { type ThemeColor } from 'twenty-ui/theme';
import { UpgradeHealth } from '~/generated-admin/graphql';
type UpgradeHealthStatusBadge = {
color: ThemeColor;
label: string;
};
export const getUpgradeHealthStatusBadge = (
health: UpgradeHealth | undefined,
): UpgradeHealthStatusBadge => {
switch (health) {
case UpgradeHealth.UP_TO_DATE:
return { color: 'green', label: t`Up to date` };
case UpgradeHealth.BEHIND:
return { color: 'orange', label: t`Behind` };
case UpgradeHealth.FAILED:
return { color: 'red', label: t`Failed` };
default:
return { color: 'gray', label: t`Unknown` };
}
};
@@ -0,0 +1,16 @@
import { UpgradeHealth } from '~/generated-admin/graphql';
export const getWorkspacesUpgradeHealth = (
behindCount: number,
failedCount: number,
): UpgradeHealth => {
if (failedCount > 0) {
return UpgradeHealth.FAILED;
}
if (behindCount > 0) {
return UpgradeHealth.BEHIND;
}
return UpgradeHealth.UP_TO_DATE;
};
@@ -0,0 +1,30 @@
import { plural } from '@lingui/core/macro';
export const getWorkspacesUpgradeHealthText = (
behindCount: number,
failedCount: number,
upToDateLabel: string,
): string => {
if (failedCount > 0 && behindCount > 0) {
return plural(failedCount + behindCount, {
one: '# workspace failed or behind',
other: '# workspaces failed or behind',
});
}
if (failedCount > 0) {
return plural(failedCount, {
one: '# workspace failed',
other: '# workspaces failed',
});
}
if (behindCount > 0) {
return plural(behindCount, {
one: '# workspace behind',
other: '# workspaces behind',
});
}
return upToDateLabel;
};