Add server upgrade status on admin panel (#20107)
## Summary Adds an admin upgrade-status panel that surfaces per-instance and per-workspace migration health, backed by a Redis-cached aggregate to keep the page snappy on large fleets. <img width="827" height="880" alt="Screenshot 2026-04-28 at 10 21 03" src="https://github.com/user-attachments/assets/8f88baa9-7268-4eff-bf6a-906a7f06ca91" /> <img width="804" height="892" alt="Screenshot 2026-04-28 at 10 21 11" src="https://github.com/user-attachments/assets/1e6decf8-766a-4d0e-96b1-03a9962bba3c" /> ## Computed metrics **Instance** (`InstanceUpgradeStatus`) - `inferredVersion` — version derived from the latest non-initial instance command name - `health` — `upToDate` | `behind` | `failed`, derived from the latest attempt vs. the last expected instance step in the upgrade sequence - `latestCommand` — `{ name, status, executedByVersion, errorMessage, createdAt }` from the most recent attempt **Per-workspace** (`WorkspaceUpgradeStatus`) - `workspaceId`, `displayName` - `inferredVersion`, `health`, `latestCommand` (same shape as instance), computed against the latest expected step in the sequence **Aggregate** (`AllWorkspacesUpgradeStatus`, only across `ACTIVE` / `SUSPENDED` workspaces) - `instanceUpgradeStatus` - `totalCount`, `upToDateCount`, `behindCount`, `failedCount` - `workspacesBehindIds[]`, `workspacesFailedIds[]` - `computedAt` ## Fetching strategy All reads go through `UpgradeStatusCacheService` (cache namespace: `EngineHealth`). - **Aggregate read** (`getAllWorkspacesStatus` → `getAllWorkspacesUpgradeStatus` query): reads summary + behind-ids + failed-ids in parallel; if any of the three keys is missing, full recompute (`recomputeAllWorkspaces`) is triggered, which also primes per-workspace entries. - **Per-workspace read** (`getWorkspacesStatus(ids)` → `getUpgradeStatus(ids)` query): `mget` on workspace keys; misses are recomputed individually (`recomputeWorkspace`), and aggregates are reconciled in place (count + id list deltas) without a full recompute. - **Recompute on demand**: `refreshUpgradeStatus` mutation calls `recomputeAllWorkspaces` to bypass cache and rewrite all keys. - **Auto-invalidation**: `InstanceCommandRunnerService` (fast + slow paths) and `WorkspaceCommandRunnerService` invalidate after every run via `safeInvalidateUpgradeStatusCache()` (`flushByPattern('upgrade-status:*')`). Failures in cache invalidation are swallowed and logged so they never break the migration runner. - **TTL**: `60 * 60 * 1000` ms (1 hour) on every key — protects against stale data even if a runner crashes before invalidating. ## Introduced cache keys All under the `EngineHealth` cache-storage namespace: | Key | Type | Purpose | | --- | --- | --- | | `upgrade-status:all-workspaces:summary` | `CachedAllWorkspacesStatusSummary` | Counts + instance status + `computedAt` | | `upgrade-status:all-workspaces:behind-ids` | `string[]` | Workspace ids in `behind` state | | `upgrade-status:all-workspaces:failed-ids` | `string[]` | Workspace ids in `failed` state | | `upgrade-status:workspace:<workspaceId>` | `CachedWorkspaceUpgradeStatus` | Per-workspace status (one key per workspace) | Full invalidation uses the pattern `upgrade-status:*`. ## Index added on `upgradeMigration` (already added on prod) Migration `2-2-instance-command-fast-1777308014234-addUpgradeMigrationWorkspaceIdIndex.ts`: ```sql CREATE INDEX "IDX_upgradeMigration_workspaceId_name_attempt" ON "core"."upgradeMigration" ("workspaceId", "name", "attempt") WHERE "workspaceId" IS NOT NULL;
This commit is contained in:
@@ -189,6 +189,7 @@ export { isMetadataGqlOperationSignature } from './typeguard/isMetadataGqlOperat
|
||||
export { isPlainObject } from './typeguard/isPlainObject';
|
||||
export { isRecordGqlOperationSignature } from './typeguard/isRecordGqlOperationSignature';
|
||||
export { throwIfNotDefined } from './typeguard/throwIfNotDefined';
|
||||
export { formatUpgradeCommandName } from './upgrade/formatUpgradeCommandName';
|
||||
export { absoluteUrlSchema } from './url/absoluteUrlSchema';
|
||||
export { buildSignedPath } from './url/buildSignedPath';
|
||||
export { ensureAbsoluteUrl } from './url/ensureAbsoluteUrl';
|
||||
|
||||
@@ -0,0 +1,32 @@
|
||||
import { formatUpgradeCommandName } from '../formatUpgradeCommandName';
|
||||
|
||||
describe('formatUpgradeCommandName', () => {
|
||||
it('should append (instance fast) for fast instance commands', () => {
|
||||
expect(
|
||||
formatUpgradeCommandName(
|
||||
'1.23.0_DropWorkspaceVersionColumnFastInstanceCommand_1785000000000',
|
||||
),
|
||||
).toBe('DropWorkspaceVersionColumn 1785000000000 (1.23.0) (instance fast)');
|
||||
});
|
||||
|
||||
it('should append (instance slow) for slow instance commands', () => {
|
||||
expect(
|
||||
formatUpgradeCommandName(
|
||||
'1.23.0_BackfillWorkspaceIdSlowInstanceCommand_1785000000001',
|
||||
),
|
||||
).toBe('BackfillWorkspaceId 1785000000001 (1.23.0) (instance slow)');
|
||||
});
|
||||
|
||||
it('should append (workspace) for workspace commands', () => {
|
||||
expect(
|
||||
formatUpgradeCommandName(
|
||||
'1.22.0_BackfillStandardSkillsCommand_1780000002000',
|
||||
),
|
||||
).toBe('BackfillStandardSkills 1780000002000 (1.22.0) (workspace)');
|
||||
});
|
||||
|
||||
it('should return raw name when fewer than three segments', () => {
|
||||
expect(formatUpgradeCommandName('short')).toBe('short');
|
||||
expect(formatUpgradeCommandName('a_b')).toBe('a_b');
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,41 @@
|
||||
const TRAILING_TIMESTAMP_PATTERN = /^\d+$/;
|
||||
|
||||
const getCommandKindLabel = (className: string): string => {
|
||||
if (className.endsWith('FastInstanceCommand')) {
|
||||
return '(instance fast)';
|
||||
}
|
||||
|
||||
if (className.endsWith('SlowInstanceCommand')) {
|
||||
return '(instance slow)';
|
||||
}
|
||||
|
||||
return '(workspace)';
|
||||
};
|
||||
|
||||
const stripCommandSuffix = (className: string): string =>
|
||||
className
|
||||
.replace(/FastInstanceCommand$/, '')
|
||||
.replace(/SlowInstanceCommand$/, '')
|
||||
.replace(/Command$/, '');
|
||||
|
||||
export const formatUpgradeCommandName = (commandName: string): string => {
|
||||
const commandNameParts = commandName.split('_');
|
||||
|
||||
if (commandNameParts.length < 3) {
|
||||
return commandName;
|
||||
}
|
||||
|
||||
const version = commandNameParts[0];
|
||||
const lastPart = commandNameParts[commandNameParts.length - 1];
|
||||
const hasTrailingTimestamp = TRAILING_TIMESTAMP_PATTERN.test(lastPart);
|
||||
|
||||
const className = commandNameParts.slice(1, -1).join('_');
|
||||
const friendlyCommandName = stripCommandSuffix(className);
|
||||
const kindLabel = getCommandKindLabel(className);
|
||||
|
||||
if (hasTrailingTimestamp) {
|
||||
return `${friendlyCommandName} ${lastPart} (${version}) ${kindLabel}`;
|
||||
}
|
||||
|
||||
return `${friendlyCommandName} (${version}) ${kindLabel}`;
|
||||
};
|
||||
Reference in New Issue
Block a user