Add server upgrade status on admin panel (#20107)

## Summary

Adds an admin upgrade-status panel that surfaces per-instance and
per-workspace migration health, backed by a Redis-cached aggregate to
keep the page snappy on large fleets.

<img width="827" height="880" alt="Screenshot 2026-04-28 at 10 21 03"
src="https://github.com/user-attachments/assets/8f88baa9-7268-4eff-bf6a-906a7f06ca91"
/>
<img width="804" height="892" alt="Screenshot 2026-04-28 at 10 21 11"
src="https://github.com/user-attachments/assets/1e6decf8-766a-4d0e-96b1-03a9962bba3c"
/>


## Computed metrics

**Instance** (`InstanceUpgradeStatus`)
- `inferredVersion` — version derived from the latest non-initial
instance command name
- `health` — `upToDate` | `behind` | `failed`, derived from the latest
attempt vs. the last expected instance step in the upgrade sequence
- `latestCommand` — `{ name, status, executedByVersion, errorMessage,
createdAt }` from the most recent attempt

**Per-workspace** (`WorkspaceUpgradeStatus`)
- `workspaceId`, `displayName`
- `inferredVersion`, `health`, `latestCommand` (same shape as instance),
computed against the latest expected step in the sequence

**Aggregate** (`AllWorkspacesUpgradeStatus`, only across `ACTIVE` /
`SUSPENDED` workspaces)
- `instanceUpgradeStatus`
- `totalCount`, `upToDateCount`, `behindCount`, `failedCount`
- `workspacesBehindIds[]`, `workspacesFailedIds[]`
- `computedAt`

## Fetching strategy

All reads go through `UpgradeStatusCacheService` (cache namespace:
`EngineHealth`).

- **Aggregate read** (`getAllWorkspacesStatus` →
`getAllWorkspacesUpgradeStatus` query):
reads summary + behind-ids + failed-ids in parallel; if any of the three
keys is missing, full recompute (`recomputeAllWorkspaces`) is triggered,
which also primes per-workspace entries.
- **Per-workspace read** (`getWorkspacesStatus(ids)` →
`getUpgradeStatus(ids)` query):
`mget` on workspace keys; misses are recomputed individually
(`recomputeWorkspace`), and aggregates are reconciled in place (count +
id list deltas) without a full recompute.
- **Recompute on demand**: `refreshUpgradeStatus` mutation calls
`recomputeAllWorkspaces` to bypass cache and rewrite all keys.
- **Auto-invalidation**: `InstanceCommandRunnerService` (fast + slow
paths) and `WorkspaceCommandRunnerService` invalidate after every run
via `safeInvalidateUpgradeStatusCache()`
(`flushByPattern('upgrade-status:*')`). Failures in cache invalidation
are swallowed and logged so they never break the migration runner.
- **TTL**: `60 * 60 * 1000` ms (1 hour) on every key — protects against
stale data even if a runner crashes before invalidating.

## Introduced cache keys

All under the `EngineHealth` cache-storage namespace:

| Key | Type | Purpose |
| --- | --- | --- |
| `upgrade-status:all-workspaces:summary` |
`CachedAllWorkspacesStatusSummary` | Counts + instance status +
`computedAt` |
| `upgrade-status:all-workspaces:behind-ids` | `string[]` | Workspace
ids in `behind` state |
| `upgrade-status:all-workspaces:failed-ids` | `string[]` | Workspace
ids in `failed` state |
| `upgrade-status:workspace:<workspaceId>` |
`CachedWorkspaceUpgradeStatus` | Per-workspace status (one key per
workspace) |

Full invalidation uses the pattern `upgrade-status:*`.

## Index added on `upgradeMigration` (already added on prod)

Migration
`2-2-instance-command-fast-1777308014234-addUpgradeMigrationWorkspaceIdIndex.ts`:

```sql
CREATE INDEX "IDX_upgradeMigration_workspaceId_name_attempt"
  ON "core"."upgradeMigration" ("workspaceId", "name", "attempt")
  WHERE "workspaceId" IS NOT NULL;
This commit is contained in:
Marie
2026-05-04 17:06:07 +02:00
committed by GitHub
parent 1cd983a330
commit 4852ac401a
49 changed files with 2100 additions and 213 deletions
@@ -5,13 +5,18 @@ import { TypeOrmModule } from '@nestjs/typeorm';
import { AdminPanelHealthService } from 'src/engine/core-modules/admin-panel/admin-panel-health.service';
import { AdminPanelQueueService } from 'src/engine/core-modules/admin-panel/admin-panel-queue.service';
import { AdminPanelResolver } from 'src/engine/core-modules/admin-panel/admin-panel.resolver';
import { AppHealthIndicator } from 'src/engine/core-modules/admin-panel/indicators/app.health';
import { ConnectedAccountHealth } from 'src/engine/core-modules/admin-panel/indicators/connected-account.health';
import { DatabaseHealthIndicator } from 'src/engine/core-modules/admin-panel/indicators/database.health';
import { RedisHealthIndicator } from 'src/engine/core-modules/admin-panel/indicators/redis.health';
import { WorkerHealthIndicator } from 'src/engine/core-modules/admin-panel/indicators/worker.health';
import { MaintenanceModeService } from 'src/engine/core-modules/admin-panel/maintenance-mode.service';
import { AdminPanelBillingService } from 'src/engine/core-modules/admin-panel/services/admin-panel-billing.service';
import { AdminPanelChatService } from 'src/engine/core-modules/admin-panel/services/admin-panel-chat.service';
import { AdminPanelConfigService } from 'src/engine/core-modules/admin-panel/services/admin-panel-config.service';
import { AdminPanelStatisticsService } from 'src/engine/core-modules/admin-panel/services/admin-panel-statistics.service';
import { AdminPanelUserLookupService } from 'src/engine/core-modules/admin-panel/services/admin-panel-user-lookup.service';
import { AdminPanelVersionService } from 'src/engine/core-modules/admin-panel/services/admin-panel-version.service';
import { MaintenanceModeService } from 'src/engine/core-modules/admin-panel/maintenance-mode.service';
import { ApplicationRegistrationModule } from 'src/engine/core-modules/application/application-registration/application-registration.module';
import { AuditModule } from 'src/engine/core-modules/audit/audit.module';
import { AuthModule } from 'src/engine/core-modules/auth/auth.module';
@@ -19,28 +24,24 @@ import { BillingModule } from 'src/engine/core-modules/billing/billing.module';
import { BillingCustomerEntity } from 'src/engine/core-modules/billing/entities/billing-customer.entity';
import { BillingPriceEntity } from 'src/engine/core-modules/billing/entities/billing-price.entity';
import { WorkspaceDomainsModule } from 'src/engine/core-modules/domain/workspace-domains/workspace-domains.module';
import { FeatureFlagEntity } from 'src/engine/core-modules/feature-flag/feature-flag.entity';
import { FeatureFlagModule } from 'src/engine/core-modules/feature-flag/feature-flag.module';
import { FileModule } from 'src/engine/core-modules/file/file.module';
import { AppHealthIndicator } from 'src/engine/core-modules/admin-panel/indicators/app.health';
import { ConnectedAccountHealth } from 'src/engine/core-modules/admin-panel/indicators/connected-account.health';
import { DatabaseHealthIndicator } from 'src/engine/core-modules/admin-panel/indicators/database.health';
import { RedisHealthIndicator } from 'src/engine/core-modules/admin-panel/indicators/redis.health';
import { WorkerHealthIndicator } from 'src/engine/core-modules/admin-panel/indicators/worker.health';
import { ImpersonationModule } from 'src/engine/core-modules/impersonation/impersonation.module';
import { KeyValuePairModule } from 'src/engine/core-modules/key-value-pair/key-value-pair.module';
import { MetricsModule } from 'src/engine/core-modules/metrics/metrics.module';
import { RedisClientModule } from 'src/engine/core-modules/redis-client/redis-client.module';
import { SecureHttpClientModule } from 'src/engine/core-modules/secure-http-client/secure-http-client.module';
import { TelemetryModule } from 'src/engine/core-modules/telemetry/telemetry.module';
import { UpgradeModule } from 'src/engine/core-modules/upgrade/upgrade.module';
import { UsageModule } from 'src/engine/core-modules/usage/usage.module';
import { FeatureFlagEntity } from 'src/engine/core-modules/feature-flag/feature-flag.entity';
import { UserModule } from 'src/engine/core-modules/user/user.module';
import { UserEntity } from 'src/engine/core-modules/user/user.entity';
import { UserVarsModule } from 'src/engine/core-modules/user/user-vars/user-vars.module';
import { UserWorkspaceEntity } from 'src/engine/core-modules/user-workspace/user-workspace.entity';
import { UserVarsModule } from 'src/engine/core-modules/user/user-vars/user-vars.module';
import { UserEntity } from 'src/engine/core-modules/user/user.entity';
import { UserModule } from 'src/engine/core-modules/user/user.module';
import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity';
import { AgentChatThreadEntity } from 'src/engine/metadata-modules/ai/ai-chat/entities/agent-chat-thread.entity';
import { AgentMessageEntity } from 'src/engine/metadata-modules/ai/ai-agent-execution/entities/agent-message.entity';
import { KeyValuePairModule } from 'src/engine/core-modules/key-value-pair/key-value-pair.module';
import { AgentChatThreadEntity } from 'src/engine/metadata-modules/ai/ai-chat/entities/agent-chat-thread.entity';
import { PermissionsModule } from 'src/engine/metadata-modules/permissions/permissions.module';
@Module({
@@ -72,6 +73,7 @@ import { PermissionsModule } from 'src/engine/metadata-modules/permissions/permi
UsageModule,
KeyValuePairModule,
UserVarsModule,
UpgradeModule,
UserModule,
],
providers: [
@@ -3,32 +3,23 @@ import { Args, Int, Mutation, Query } from '@nestjs/graphql';
import { InjectRepository } from '@nestjs/typeorm';
import GraphQLJSON from 'graphql-type-json';
import { In, type Repository } from 'typeorm';
import { isDefined } from 'twenty-shared/utils';
import { PermissionFlagType } from 'twenty-shared/constants';
import { isDefined } from 'twenty-shared/utils';
import { In, type Repository } from 'typeorm';
import { UUIDScalarType } from 'src/engine/api/graphql/workspace-schema-builder/graphql-types/scalars';
import { InstanceAndAllWorkspacesUpgradeStatusDTO } from 'src/engine/core-modules/upgrade/dtos/instance-and-all-workspaces-upgrade-status.dto';
import { WorkspaceUpgradeStatusDTO } from 'src/engine/core-modules/upgrade/dtos/workspace-upgrade-status.dto';
import { UpgradeStatusService } from 'src/engine/core-modules/upgrade/services/upgrade-status.service';
import { AdminResolver } from 'src/engine/api/graphql/graphql-config/decorators/admin-resolver.decorator';
import { AdminPanelHealthService } from 'src/engine/core-modules/admin-panel/admin-panel-health.service';
import { AdminPanelQueueService } from 'src/engine/core-modules/admin-panel/admin-panel-queue.service';
import { AdminPanelBillingService } from 'src/engine/core-modules/admin-panel/services/admin-panel-billing.service';
import { AdminPanelChatService } from 'src/engine/core-modules/admin-panel/services/admin-panel-chat.service';
import { AdminPanelConfigService } from 'src/engine/core-modules/admin-panel/services/admin-panel-config.service';
import { AdminPanelStatisticsService } from 'src/engine/core-modules/admin-panel/services/admin-panel-statistics.service';
import { AdminPanelUserLookupService } from 'src/engine/core-modules/admin-panel/services/admin-panel-user-lookup.service';
import { AdminPanelVersionService } from 'src/engine/core-modules/admin-panel/services/admin-panel-version.service';
import { MaintenanceModeService } from 'src/engine/core-modules/admin-panel/maintenance-mode.service';
import { AdminChatThreadMessagesDTO } from 'src/engine/core-modules/admin-panel/dtos/admin-chat-thread-messages.dto';
import { AdminPanelRecentUserDTO } from 'src/engine/core-modules/admin-panel/dtos/admin-panel-recent-user.dto';
import { AdminPanelTopWorkspaceDTO } from 'src/engine/core-modules/admin-panel/dtos/admin-panel-top-workspace.dto';
import { AdminPanelWorkspaceBillingDTO } from 'src/engine/core-modules/admin-panel/dtos/admin-panel-workspace-billing.dto';
import { AdminWorkspaceChatThreadDTO } from 'src/engine/core-modules/admin-panel/dtos/admin-workspace-chat-thread.dto';
import { AdminChatThreadMessagesDTO } from 'src/engine/core-modules/admin-panel/dtos/admin-chat-thread-messages.dto';
import { ApplicationRegistrationEntity } from 'src/engine/core-modules/application/application-registration/application-registration.entity';
import { ApplicationRegistrationService } from 'src/engine/core-modules/application/application-registration/application-registration.service';
import { AdminAiModelsDTO } from 'src/engine/core-modules/client-config/client-config.entity';
import { UsageBreakdownItemDTO } from 'src/engine/core-modules/usage/dtos/usage-breakdown-item.dto';
import { UsageAnalyticsService } from 'src/engine/core-modules/usage/services/usage-analytics.service';
import { AiModelRole } from 'src/engine/metadata-modules/ai/ai-models/types/ai-model-role.enum';
import { ConfigVariableDTO } from 'src/engine/core-modules/admin-panel/dtos/config-variable.dto';
import { ConfigVariablesDTO } from 'src/engine/core-modules/admin-panel/dtos/config-variables.dto';
import { DeleteJobsResponseDTO } from 'src/engine/core-modules/admin-panel/dtos/delete-jobs-response.dto';
@@ -39,33 +30,45 @@ import { UpdateWorkspaceFeatureFlagInput } from 'src/engine/core-modules/admin-p
import { UserLookup } from 'src/engine/core-modules/admin-panel/dtos/user-lookup.dto';
import { UserLookupInput } from 'src/engine/core-modules/admin-panel/dtos/user-lookup.input';
import { VersionInfoDTO } from 'src/engine/core-modules/admin-panel/dtos/version-info.dto';
import { HealthIndicatorId } from 'src/engine/core-modules/admin-panel/enums/health-indicator-id.enum';
import { JobStateEnum } from 'src/engine/core-modules/admin-panel/enums/job-state.enum';
import { QueueMetricsTimeRange } from 'src/engine/core-modules/admin-panel/enums/queue-metrics-time-range.enum';
import { MaintenanceModeService } from 'src/engine/core-modules/admin-panel/maintenance-mode.service';
import { AdminPanelBillingService } from 'src/engine/core-modules/admin-panel/services/admin-panel-billing.service';
import { AdminPanelChatService } from 'src/engine/core-modules/admin-panel/services/admin-panel-chat.service';
import { AdminPanelConfigService } from 'src/engine/core-modules/admin-panel/services/admin-panel-config.service';
import { AdminPanelStatisticsService } from 'src/engine/core-modules/admin-panel/services/admin-panel-statistics.service';
import { AdminPanelUserLookupService } from 'src/engine/core-modules/admin-panel/services/admin-panel-user-lookup.service';
import { AdminPanelVersionService } from 'src/engine/core-modules/admin-panel/services/admin-panel-version.service';
import { ApplicationRegistrationEntity } from 'src/engine/core-modules/application/application-registration/application-registration.entity';
import { ApplicationRegistrationService } from 'src/engine/core-modules/application/application-registration/application-registration.service';
import { AuthGraphqlApiExceptionFilter } from 'src/engine/core-modules/auth/filters/auth-graphql-api-exception.filter';
import { AdminAiModelsDTO } from 'src/engine/core-modules/client-config/client-config.entity';
import { FeatureFlagException } from 'src/engine/core-modules/feature-flag/feature-flag.exception';
import { FeatureFlagService } from 'src/engine/core-modules/feature-flag/services/feature-flag.service';
import { PreventNestToAutoLogGraphqlErrorsFilter } from 'src/engine/core-modules/graphql/filters/prevent-nest-to-auto-log-graphql-errors.filter';
import { ResolverValidationPipe } from 'src/engine/core-modules/graphql/pipes/resolver-validation.pipe';
import { UserInputError } from 'src/engine/core-modules/graphql/utils/graphql-errors.util';
import { HealthIndicatorId } from 'src/engine/core-modules/admin-panel/enums/health-indicator-id.enum';
import { type MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';
import { type ConfigVariables } from 'src/engine/core-modules/twenty-config/config-variables';
import { ConfigVariableGraphqlApiExceptionFilter } from 'src/engine/core-modules/twenty-config/filters/config-variable-graphql-api-exception.filter';
import { TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service';
import { AiModelRegistryService } from 'src/engine/metadata-modules/ai/ai-models/services/ai-model-registry.service';
import { ModelsDevCatalogService } from 'src/engine/metadata-modules/ai/ai-models/services/models-dev-catalog.service';
import { MODEL_FAMILY_LABELS } from 'src/engine/metadata-modules/ai/ai-models/constants/model-family-labels.const';
import { type AiProviderConfig } from 'src/engine/metadata-modules/ai/ai-models/types/ai-provider-config.type';
import { type AiProviderModelConfig } from 'src/engine/metadata-modules/ai/ai-models/types/ai-provider-model-config.type';
import { extractConfigVariableName } from 'src/engine/metadata-modules/ai/ai-models/utils/extract-config-variable-name.util';
import { DefaultAiCatalogService } from 'src/engine/metadata-modules/ai/ai-models/services/default-ai-catalog.service';
import { UsageBreakdownItemDTO } from 'src/engine/core-modules/usage/dtos/usage-breakdown-item.dto';
import { UsageAnalyticsService } from 'src/engine/core-modules/usage/services/usage-analytics.service';
import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity';
import { AdminResolver } from 'src/engine/api/graphql/graphql-config/decorators/admin-resolver.decorator';
import { AdminPanelGuard } from 'src/engine/guards/admin-panel-guard';
import { ServerLevelImpersonateGuard } from 'src/engine/guards/server-level-impersonate.guard';
import { SettingsPermissionGuard } from 'src/engine/guards/settings-permission.guard';
import { UserAuthGuard } from 'src/engine/guards/user-auth.guard';
import { WorkspaceAuthGuard } from 'src/engine/guards/workspace-auth.guard';
import { MODEL_FAMILY_LABELS } from 'src/engine/metadata-modules/ai/ai-models/constants/model-family-labels.const';
import { AiModelRegistryService } from 'src/engine/metadata-modules/ai/ai-models/services/ai-model-registry.service';
import { DefaultAiCatalogService } from 'src/engine/metadata-modules/ai/ai-models/services/default-ai-catalog.service';
import { ModelsDevCatalogService } from 'src/engine/metadata-modules/ai/ai-models/services/models-dev-catalog.service';
import { AiModelRole } from 'src/engine/metadata-modules/ai/ai-models/types/ai-model-role.enum';
import { type AiProviderConfig } from 'src/engine/metadata-modules/ai/ai-models/types/ai-provider-config.type';
import { type AiProviderModelConfig } from 'src/engine/metadata-modules/ai/ai-models/types/ai-provider-model-config.type';
import { extractConfigVariableName } from 'src/engine/metadata-modules/ai/ai-models/utils/extract-config-variable-name.util';
import { AdminPanelHealthServiceDataDTO } from './dtos/admin-panel-health-service-data.dto';
import { MaintenanceModeDTO } from './dtos/maintenance-mode.dto';
@@ -73,7 +76,6 @@ import { ModelsDevModelSuggestionDTO } from './dtos/models-dev-model-suggestion.
import { ModelsDevProviderSuggestionDTO } from './dtos/models-dev-provider-suggestion.dto';
import { QueueMetricsDataDTO } from './dtos/queue-metrics-data.dto';
import { SetMaintenanceModeInput } from './dtos/set-maintenance-mode.input';
import { AuthWorkspace } from 'src/engine/decorators/auth/auth-workspace.decorator';
@UsePipes(ResolverValidationPipe)
@AdminResolver()
@@ -105,6 +107,7 @@ export class AdminPanelResolver {
private readonly modelsDevCatalogService: ModelsDevCatalogService,
private readonly usageAnalyticsService: UsageAnalyticsService,
private readonly maintenanceModeService: MaintenanceModeService,
private readonly upgradeStatusService: UpgradeStatusService,
@InjectRepository(WorkspaceEntity)
private readonly workspaceRepository: Repository<WorkspaceEntity>,
) {}
@@ -701,4 +704,29 @@ export class AdminPanelResolver {
): Promise<ApplicationRegistrationEntity> {
return this.applicationRegistrationService.findOneByIdGlobal(id);
}
@UseGuards(AdminPanelGuard)
@Query(() => InstanceAndAllWorkspacesUpgradeStatusDTO)
async getInstanceAndAllWorkspacesUpgradeStatus(): Promise<InstanceAndAllWorkspacesUpgradeStatusDTO> {
return this.upgradeStatusService.getInstanceAndAllWorkspacesStatus();
}
@UseGuards(AdminPanelGuard)
@Mutation(() => InstanceAndAllWorkspacesUpgradeStatusDTO)
async refreshUpgradeStatus(): Promise<InstanceAndAllWorkspacesUpgradeStatusDTO> {
return this.upgradeStatusService.refreshInstanceAndAllWorkspacesStatus();
}
@UseGuards(AdminPanelGuard)
@Query(() => [WorkspaceUpgradeStatusDTO])
async getUpgradeStatus(
@Args('workspaceIds', { type: () => [UUIDScalarType] })
workspaceIds: string[],
): Promise<WorkspaceUpgradeStatusDTO[]> {
if (workspaceIds.length === 0) {
return [];
}
return this.upgradeStatusService.getWorkspaceStatuses(workspaceIds);
}
}
@@ -62,8 +62,8 @@ class WorkspaceInfoDTO {
@ObjectType('UserLookup')
export class UserLookup {
@Field(() => UserInfoDTO)
user: UserInfoDTO;
@Field(() => UserInfoDTO, { nullable: true })
user?: UserInfoDTO | null;
@Field(() => [WorkspaceInfoDTO])
workspaces: WorkspaceInfoDTO[];
@@ -14,9 +14,9 @@ import {
import { WorkspaceDomainsService } from 'src/engine/core-modules/domain/workspace-domains/services/workspace-domains.service';
import { FeatureFlagEntity } from 'src/engine/core-modules/feature-flag/feature-flag.entity';
import { FileUrlService } from 'src/engine/core-modules/file/file-url/file-url.service';
import { UserEntity } from 'src/engine/core-modules/user/user.entity';
import { UserWorkspaceEntity } from 'src/engine/core-modules/user-workspace/user-workspace.entity';
import { UserService } from 'src/engine/core-modules/user/services/user.service';
import { UserEntity } from 'src/engine/core-modules/user/user.entity';
import { userValidator } from 'src/engine/core-modules/user/user.validate';
import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity';
@@ -208,16 +208,16 @@ export class AdminPanelUserLookupService {
const firstUser = workspaceUsers.find((wu) => isDefined(wu.user))?.user;
return {
user: {
id: firstUser?.id ?? '',
email: firstUser?.email ?? '',
firstName: firstUser?.firstName,
lastName: firstUser?.lastName,
avatarUrl: firstUser
? (avatarUrlsByUserId.get(firstUser.id) ?? null)
: null,
createdAt: firstUser?.createdAt ?? new Date(),
},
user: isDefined(firstUser)
? {
id: firstUser.id,
email: firstUser.email,
firstName: firstUser.firstName,
lastName: firstUser.lastName,
avatarUrl: avatarUrlsByUserId.get(firstUser.id) ?? null,
createdAt: firstUser.createdAt,
}
: null,
workspaces: [workspaceInfo],
};
}