577b22df46
## Problem The "Twenty / Upgrade Status" Grafana dashboard shows stale workspace counts (e.g. `N behind / 0 up-to-date` while the instance reads `UP_TO_DATE`) that disagree with `command:prod upgrade:status`. The CLI is correct; the dashboard lags, sometimes for the full hour. ## Root cause The dashboard is fed by the `twenty_upgrade_workspaces_*` gauges, which read their workspace counts from a Redis snapshot (`UpgradeStatusCacheService`). That snapshot is only invalidated **per-command, inside the runners' `finally` blocks**. Two gaps: 1. An instance command that is already applied returns **before** its invalidation runs (`isAlreadyCompleted` early-return in `InstanceCommandRunnerService`). So a plain **redeploy** — which changes the deployed upgrade sequence, and thus the "behind" answer, without executing any command — never refreshes the snapshot. This is most visible on an instance-only release. 2. The snapshot then stays frozen until its 60-minute TTL, while the CLI reads live and disagrees. "Behind" is derived from the deployed sequence, not just the ledger, so the correct answer changes on events (deploys) that run no command — which is exactly why per-command invalidation isn't enough on its own. ## Fix Invalidate the upgrade-status cache **once, unconditionally, at the end of both upgrade entrypoints** — `run-instance-commands` (the deploy/migrate step) and `upgrade` — in a `finally`. Every run, including a no-op redeploy where all commands are already applied, now clears the snapshot, so the next gauge scrape recomputes against the current sequence. Best-effort (failures are logged, never block the command). The existing per-command invalidation is kept for mid-run progress. This keeps the read path untouched. ## Reproduction + verification (live, local) Served twenty-server (`NODE_PORT=4000`, `METER_DRIVER=prometheus`) against the seeded DB, whose latest version `2.12.0` is instance-only. 1. Froze the gauge at `behind 4 / up_to_date 0` while the DB was brought up-to-date (snapshot not invalidated) — reproduced the dashboard/CLI divergence. 2. Ran the **patched** `run-instance-commands --force`. Every step logged `already executed, skipping` — and the `finally` still deleted the Redis snapshot. 3. On the next recompute the gauge self-healed to `instance_health 1, behind 0, up_to_date 4`, matching the live CLI. With the old code the snapshot stayed frozen at `behind 4` until the TTL.
186 lines
6.0 KiB
TypeScript
186 lines
6.0 KiB
TypeScript
import { Logger } from '@nestjs/common';
|
|
import { InjectDataSource } from '@nestjs/typeorm';
|
|
|
|
import chalk from 'chalk';
|
|
import { Command, CommandRunner, Option } from 'nest-commander';
|
|
import { DataSource } from 'typeorm';
|
|
|
|
import { TWENTY_PREVIOUS_VERSIONS } from 'src/engine/core-modules/upgrade/constants/twenty-previous-versions.constant';
|
|
import { InstanceCommandRunnerService } from 'src/engine/core-modules/upgrade/services/instance-command-runner.service';
|
|
import { UpgradeCommandRegistryService } from 'src/engine/core-modules/upgrade/services/upgrade-command-registry.service';
|
|
import { UpgradeMigrationService } from 'src/engine/core-modules/upgrade/services/upgrade-migration.service';
|
|
import { UpgradeSequenceReaderService } from 'src/engine/core-modules/upgrade/services/upgrade-sequence-reader.service';
|
|
import { UpgradeStatusService } from 'src/engine/core-modules/upgrade/services/upgrade-status.service';
|
|
import { WorkspaceVersionService } from 'src/engine/workspace-manager/workspace-version/services/workspace-version.service';
|
|
|
|
type RunInstanceCommandsOptions = {
|
|
force?: boolean;
|
|
includeSlow?: boolean;
|
|
};
|
|
|
|
// TODO should be replaced by a specific call to the upgrade
|
|
@Command({
|
|
name: 'run-instance-commands',
|
|
description:
|
|
'Run legacy TypeORM migrations and all registered instance commands',
|
|
})
|
|
export class RunInstanceCommandsCommand extends CommandRunner {
|
|
private readonly logger = new Logger(RunInstanceCommandsCommand.name);
|
|
|
|
constructor(
|
|
@InjectDataSource()
|
|
private readonly dataSource: DataSource,
|
|
private readonly workspaceVersionService: WorkspaceVersionService,
|
|
private readonly upgradeCommandRegistryService: UpgradeCommandRegistryService,
|
|
private readonly upgradeSequenceReaderService: UpgradeSequenceReaderService,
|
|
private readonly instanceUpgradeService: InstanceCommandRunnerService,
|
|
private readonly upgradeMigrationService: UpgradeMigrationService,
|
|
private readonly upgradeStatusService: UpgradeStatusService,
|
|
) {
|
|
super();
|
|
}
|
|
|
|
@Option({
|
|
flags: '-f, --force',
|
|
description: 'Skip workspace version safety check',
|
|
required: false,
|
|
})
|
|
parseForce(): boolean {
|
|
return true;
|
|
}
|
|
|
|
@Option({
|
|
flags: '--include-slow',
|
|
description: 'Also run slow instance commands (data migration + DDL)',
|
|
required: false,
|
|
})
|
|
parseIncludeSlow(): boolean {
|
|
return true;
|
|
}
|
|
|
|
async run(
|
|
_passedParams: string[],
|
|
options: RunInstanceCommandsOptions,
|
|
): Promise<void> {
|
|
try {
|
|
await this.checkWorkspaceVersionSafety(options);
|
|
await this.runLegacyPendingTypeOrmMigrations();
|
|
|
|
const activeOrSuspendedWorkspaceIds =
|
|
await this.workspaceVersionService.getActiveOrSuspendedWorkspaceIds();
|
|
|
|
const sequence = this.upgradeSequenceReaderService.getUpgradeSequence();
|
|
|
|
for (const step of sequence) {
|
|
if (step.kind === 'fast-instance') {
|
|
const result =
|
|
await this.instanceUpgradeService.runFastInstanceCommand({
|
|
command: step.command,
|
|
name: step.name,
|
|
});
|
|
|
|
if (result.status === 'failed') {
|
|
throw result.error;
|
|
}
|
|
}
|
|
|
|
if (step.kind === 'slow-instance' && options.includeSlow) {
|
|
const result =
|
|
await this.instanceUpgradeService.runSlowInstanceCommand({
|
|
command: step.command,
|
|
name: step.name,
|
|
skipDataMigration: activeOrSuspendedWorkspaceIds.length === 0,
|
|
});
|
|
|
|
if (result.status === 'failed') {
|
|
throw result.error;
|
|
}
|
|
}
|
|
}
|
|
|
|
this.logger.log(chalk.green('Instance commands completed'));
|
|
} catch (error) {
|
|
this.logger.error(
|
|
chalk.red(`Instance commands failed: ${error.message}`),
|
|
);
|
|
throw error;
|
|
} finally {
|
|
await this.safeInvalidateUpgradeStatusCache();
|
|
}
|
|
}
|
|
|
|
private async safeInvalidateUpgradeStatusCache(): Promise<void> {
|
|
try {
|
|
await this.upgradeStatusService.invalidateInstanceAndAllWorkspacesStatus();
|
|
} catch (error) {
|
|
this.logger.warn(
|
|
`Failed to invalidate upgrade-status cache: ${
|
|
error instanceof Error ? error.message : String(error)
|
|
}`,
|
|
);
|
|
}
|
|
}
|
|
|
|
private async checkWorkspaceVersionSafety(
|
|
options: RunInstanceCommandsOptions,
|
|
): Promise<void> {
|
|
if (options.force) {
|
|
this.logger.warn(
|
|
chalk.yellow('Skipping workspace version check (--force flag used)'),
|
|
);
|
|
|
|
return;
|
|
}
|
|
|
|
const activeOrSuspendedWorkspaceIds =
|
|
await this.workspaceVersionService.getActiveOrSuspendedWorkspaceIds();
|
|
|
|
if (activeOrSuspendedWorkspaceIds.length === 0) {
|
|
return;
|
|
}
|
|
|
|
const previousVersion =
|
|
TWENTY_PREVIOUS_VERSIONS[TWENTY_PREVIOUS_VERSIONS.length - 1];
|
|
|
|
const lastWorkspaceCommand =
|
|
this.upgradeCommandRegistryService.getLastWorkspaceCommandForVersion(
|
|
previousVersion,
|
|
);
|
|
|
|
if (!lastWorkspaceCommand) {
|
|
return;
|
|
}
|
|
|
|
const allAtPreviousVersion =
|
|
await this.upgradeMigrationService.areAllWorkspacesAtCommand({
|
|
commandName: lastWorkspaceCommand.name,
|
|
workspaceIds: activeOrSuspendedWorkspaceIds,
|
|
});
|
|
|
|
if (!allAtPreviousVersion) {
|
|
throw new Error(
|
|
'Unable to run instance commands. Some workspace(s) have not completed ' +
|
|
`the last workspace command for ${previousVersion} ("${lastWorkspaceCommand.name}").\n` +
|
|
'Please ensure all workspaces are upgraded to at least the previous version before running migrations.\n' +
|
|
'Use --force to bypass this check (not recommended).',
|
|
);
|
|
}
|
|
}
|
|
|
|
private async runLegacyPendingTypeOrmMigrations(): Promise<void> {
|
|
this.logger.log('Running legacy TypeORM migrations...');
|
|
|
|
const migrations = await this.dataSource.runMigrations({
|
|
transaction: 'each',
|
|
});
|
|
|
|
if (migrations.length === 0) {
|
|
this.logger.log('No pending legacy migrations');
|
|
} else {
|
|
this.logger.log(
|
|
`Executed ${migrations.length} legacy migration(s): ${migrations.map((migration) => migration.name).join(', ')}`,
|
|
);
|
|
}
|
|
}
|
|
}
|