Files
twenty/packages/twenty-server/src/database/commands/run-instance-commands.command.ts
T
Charles Bochet 577b22df46 fix(upgrade): invalidate upgrade-status cache on command end (#21497)
## Problem

The "Twenty / Upgrade Status" Grafana dashboard shows stale workspace
counts (e.g. `N behind / 0 up-to-date` while the instance reads
`UP_TO_DATE`) that disagree with `command:prod upgrade:status`. The CLI
is correct; the dashboard lags, sometimes for the full hour.

## Root cause

The dashboard is fed by the `twenty_upgrade_workspaces_*` gauges, which
read their workspace counts from a Redis snapshot
(`UpgradeStatusCacheService`). That snapshot is only invalidated
**per-command, inside the runners' `finally` blocks**. Two gaps:

1. An instance command that is already applied returns **before** its
invalidation runs (`isAlreadyCompleted` early-return in
`InstanceCommandRunnerService`). So a plain **redeploy** — which changes
the deployed upgrade sequence, and thus the "behind" answer, without
executing any command — never refreshes the snapshot. This is most
visible on an instance-only release.
2. The snapshot then stays frozen until its 60-minute TTL, while the CLI
reads live and disagrees.

"Behind" is derived from the deployed sequence, not just the ledger, so
the correct answer changes on events (deploys) that run no command —
which is exactly why per-command invalidation isn't enough on its own.

## Fix

Invalidate the upgrade-status cache **once, unconditionally, at the end
of both upgrade entrypoints** — `run-instance-commands` (the
deploy/migrate step) and `upgrade` — in a `finally`. Every run,
including a no-op redeploy where all commands are already applied, now
clears the snapshot, so the next gauge scrape recomputes against the
current sequence. Best-effort (failures are logged, never block the
command). The existing per-command invalidation is kept for mid-run
progress.

This keeps the read path untouched.

## Reproduction + verification (live, local)

Served twenty-server (`NODE_PORT=4000`, `METER_DRIVER=prometheus`)
against the seeded DB, whose latest version `2.12.0` is instance-only.

1. Froze the gauge at `behind 4 / up_to_date 0` while the DB was brought
up-to-date (snapshot not invalidated) — reproduced the dashboard/CLI
divergence.
2. Ran the **patched** `run-instance-commands --force`. Every step
logged `already executed, skipping` — and the `finally` still deleted
the Redis snapshot.
3. On the next recompute the gauge self-healed to `instance_health 1,
behind 0, up_to_date 4`, matching the live CLI.

With the old code the snapshot stayed frozen at `behind 4` until the
TTL.
2026-06-12 16:01:59 +00:00

186 lines
6.0 KiB
TypeScript

import { Logger } from '@nestjs/common';
import { InjectDataSource } from '@nestjs/typeorm';
import chalk from 'chalk';
import { Command, CommandRunner, Option } from 'nest-commander';
import { DataSource } from 'typeorm';
import { TWENTY_PREVIOUS_VERSIONS } from 'src/engine/core-modules/upgrade/constants/twenty-previous-versions.constant';
import { InstanceCommandRunnerService } from 'src/engine/core-modules/upgrade/services/instance-command-runner.service';
import { UpgradeCommandRegistryService } from 'src/engine/core-modules/upgrade/services/upgrade-command-registry.service';
import { UpgradeMigrationService } from 'src/engine/core-modules/upgrade/services/upgrade-migration.service';
import { UpgradeSequenceReaderService } from 'src/engine/core-modules/upgrade/services/upgrade-sequence-reader.service';
import { UpgradeStatusService } from 'src/engine/core-modules/upgrade/services/upgrade-status.service';
import { WorkspaceVersionService } from 'src/engine/workspace-manager/workspace-version/services/workspace-version.service';
type RunInstanceCommandsOptions = {
force?: boolean;
includeSlow?: boolean;
};
// TODO should be replaced by a specific call to the upgrade
@Command({
name: 'run-instance-commands',
description:
'Run legacy TypeORM migrations and all registered instance commands',
})
export class RunInstanceCommandsCommand extends CommandRunner {
private readonly logger = new Logger(RunInstanceCommandsCommand.name);
constructor(
@InjectDataSource()
private readonly dataSource: DataSource,
private readonly workspaceVersionService: WorkspaceVersionService,
private readonly upgradeCommandRegistryService: UpgradeCommandRegistryService,
private readonly upgradeSequenceReaderService: UpgradeSequenceReaderService,
private readonly instanceUpgradeService: InstanceCommandRunnerService,
private readonly upgradeMigrationService: UpgradeMigrationService,
private readonly upgradeStatusService: UpgradeStatusService,
) {
super();
}
@Option({
flags: '-f, --force',
description: 'Skip workspace version safety check',
required: false,
})
parseForce(): boolean {
return true;
}
@Option({
flags: '--include-slow',
description: 'Also run slow instance commands (data migration + DDL)',
required: false,
})
parseIncludeSlow(): boolean {
return true;
}
async run(
_passedParams: string[],
options: RunInstanceCommandsOptions,
): Promise<void> {
try {
await this.checkWorkspaceVersionSafety(options);
await this.runLegacyPendingTypeOrmMigrations();
const activeOrSuspendedWorkspaceIds =
await this.workspaceVersionService.getActiveOrSuspendedWorkspaceIds();
const sequence = this.upgradeSequenceReaderService.getUpgradeSequence();
for (const step of sequence) {
if (step.kind === 'fast-instance') {
const result =
await this.instanceUpgradeService.runFastInstanceCommand({
command: step.command,
name: step.name,
});
if (result.status === 'failed') {
throw result.error;
}
}
if (step.kind === 'slow-instance' && options.includeSlow) {
const result =
await this.instanceUpgradeService.runSlowInstanceCommand({
command: step.command,
name: step.name,
skipDataMigration: activeOrSuspendedWorkspaceIds.length === 0,
});
if (result.status === 'failed') {
throw result.error;
}
}
}
this.logger.log(chalk.green('Instance commands completed'));
} catch (error) {
this.logger.error(
chalk.red(`Instance commands failed: ${error.message}`),
);
throw error;
} finally {
await this.safeInvalidateUpgradeStatusCache();
}
}
private async safeInvalidateUpgradeStatusCache(): Promise<void> {
try {
await this.upgradeStatusService.invalidateInstanceAndAllWorkspacesStatus();
} catch (error) {
this.logger.warn(
`Failed to invalidate upgrade-status cache: ${
error instanceof Error ? error.message : String(error)
}`,
);
}
}
private async checkWorkspaceVersionSafety(
options: RunInstanceCommandsOptions,
): Promise<void> {
if (options.force) {
this.logger.warn(
chalk.yellow('Skipping workspace version check (--force flag used)'),
);
return;
}
const activeOrSuspendedWorkspaceIds =
await this.workspaceVersionService.getActiveOrSuspendedWorkspaceIds();
if (activeOrSuspendedWorkspaceIds.length === 0) {
return;
}
const previousVersion =
TWENTY_PREVIOUS_VERSIONS[TWENTY_PREVIOUS_VERSIONS.length - 1];
const lastWorkspaceCommand =
this.upgradeCommandRegistryService.getLastWorkspaceCommandForVersion(
previousVersion,
);
if (!lastWorkspaceCommand) {
return;
}
const allAtPreviousVersion =
await this.upgradeMigrationService.areAllWorkspacesAtCommand({
commandName: lastWorkspaceCommand.name,
workspaceIds: activeOrSuspendedWorkspaceIds,
});
if (!allAtPreviousVersion) {
throw new Error(
'Unable to run instance commands. Some workspace(s) have not completed ' +
`the last workspace command for ${previousVersion} ("${lastWorkspaceCommand.name}").\n` +
'Please ensure all workspaces are upgraded to at least the previous version before running migrations.\n' +
'Use --force to bypass this check (not recommended).',
);
}
}
private async runLegacyPendingTypeOrmMigrations(): Promise<void> {
this.logger.log('Running legacy TypeORM migrations...');
const migrations = await this.dataSource.runMigrations({
transaction: 'each',
});
if (migrations.length === 0) {
this.logger.log('No pending legacy migrations');
} else {
this.logger.log(
`Executed ${migrations.length} legacy migration(s): ${migrations.map((migration) => migration.name).join(', ')}`,
);
}
}
}