Files
twenty/packages/twenty-server/src/database/commands/upgrade-version-command/upgrade.command.ts
T
Charles Bochet 577b22df46 fix(upgrade): invalidate upgrade-status cache on command end (#21497)
## Problem

The "Twenty / Upgrade Status" Grafana dashboard shows stale workspace
counts (e.g. `N behind / 0 up-to-date` while the instance reads
`UP_TO_DATE`) that disagree with `command:prod upgrade:status`. The CLI
is correct; the dashboard lags, sometimes for the full hour.

## Root cause

The dashboard is fed by the `twenty_upgrade_workspaces_*` gauges, which
read their workspace counts from a Redis snapshot
(`UpgradeStatusCacheService`). That snapshot is only invalidated
**per-command, inside the runners' `finally` blocks**. Two gaps:

1. An instance command that is already applied returns **before** its
invalidation runs (`isAlreadyCompleted` early-return in
`InstanceCommandRunnerService`). So a plain **redeploy** — which changes
the deployed upgrade sequence, and thus the "behind" answer, without
executing any command — never refreshes the snapshot. This is most
visible on an instance-only release.
2. The snapshot then stays frozen until its 60-minute TTL, while the CLI
reads live and disagrees.

"Behind" is derived from the deployed sequence, not just the ledger, so
the correct answer changes on events (deploys) that run no command —
which is exactly why per-command invalidation isn't enough on its own.

## Fix

Invalidate the upgrade-status cache **once, unconditionally, at the end
of both upgrade entrypoints** — `run-instance-commands` (the
deploy/migrate step) and `upgrade` — in a `finally`. Every run,
including a no-op redeploy where all commands are already applied, now
clears the snapshot, so the next gauge scrape recomputes against the
current sequence. Best-effort (failures are logged, never block the
command). The existing per-command invalidation is kept for mid-run
progress.

This keeps the read path untouched.

## Reproduction + verification (live, local)

Served twenty-server (`NODE_PORT=4000`, `METER_DRIVER=prometheus`)
against the seeded DB, whose latest version `2.12.0` is instance-only.

1. Froze the gauge at `behind 4 / up_to_date 0` while the DB was brought
up-to-date (snapshot not invalidated) — reproduced the dashboard/CLI
divergence.
2. Ran the **patched** `run-instance-commands --force`. Every step
logged `already executed, skipping` — and the `finally` still deleted
the Redis snapshot.
3. On the next recompute the gauge self-healed to `instance_health 1,
behind 0, up_to_date 4`, matching the live CLI.

With the old code the snapshot stayed frozen at `behind 4` until the
TTL.
2026-06-12 16:01:59 +00:00

215 lines
5.9 KiB
TypeScript

import { Command, CommandRunner, Option } from 'nest-commander';
import { isDefined } from 'twenty-shared/utils';
import { CommandLogger } from 'src/database/commands/logger';
import { UpgradeSequenceReaderService } from 'src/engine/core-modules/upgrade/services/upgrade-sequence-reader.service';
import { UpgradeSequenceRunnerService } from 'src/engine/core-modules/upgrade/services/upgrade-sequence-runner.service';
import { UpgradeStatusService } from 'src/engine/core-modules/upgrade/services/upgrade-status.service';
import { formatUpgradeLog } from 'src/engine/core-modules/upgrade/utils/format-upgrade-log.util';
type RawUpgradeCommandOptions = {
workspaceId?: Set<string>;
startFromWorkspaceId?: string;
workspaceCountLimit?: number;
dryRun?: boolean;
verbose?: boolean;
};
export type ParsedUpgradeCommandOptions = {
workspaceIds?: string[];
startFromWorkspaceId?: string;
workspaceCountLimit?: number;
dryRun?: boolean;
verbose?: boolean;
};
@Command({
name: 'upgrade',
description: 'Upgrade workspaces to the latest version',
})
export class UpgradeCommand extends CommandRunner {
protected logger: CommandLogger;
constructor(
protected readonly upgradeSequenceReaderService: UpgradeSequenceReaderService,
protected readonly upgradeSequenceRunnerService: UpgradeSequenceRunnerService,
protected readonly upgradeStatusService: UpgradeStatusService,
) {
super();
this.logger = new CommandLogger({
verbose: false,
constructorName: this.constructor.name,
});
}
@Option({
flags: '-d, --dry-run',
description: 'Simulate the command without making actual changes',
required: false,
})
parseDryRun(): boolean {
return true;
}
@Option({
flags: '-v, --verbose',
description: 'Verbose output',
required: false,
})
parseVerbose(): boolean {
return true;
}
@Option({
flags: '-w, --workspace-id [workspace_id]',
description:
'workspace id. Command runs on all active/suspended workspaces if not provided.',
required: false,
})
parseWorkspaceId(val: string, previous?: Set<string>): Set<string> {
const accumulator = previous ?? new Set<string>();
accumulator.add(val);
return accumulator;
}
@Option({
flags: '--start-from-workspace-id [workspace_id]',
description:
'Start from a specific workspace id. Workspaces are processed in ascending order of id.',
required: false,
})
parseStartFromWorkspaceId(val: string): string {
return val;
}
@Option({
flags: '--workspace-count-limit [count]',
description:
'Limit the number of workspaces to process. Workspaces are processed in ascending order of id.',
required: false,
})
parseWorkspaceCountLimit(val: string): number {
const limit = parseInt(val);
if (isNaN(limit)) {
throw new Error('Workspace count limit must be a number');
}
if (limit <= 0) {
throw new Error('Workspace count limit must be greater than 0');
}
return limit;
}
override async run(
_passedParams: string[],
options: RawUpgradeCommandOptions,
): Promise<void> {
if (options.verbose) {
this.logger = new CommandLogger({
verbose: true,
constructorName: this.constructor.name,
});
}
if (
isDefined(options.workspaceId) &&
isDefined(options.startFromWorkspaceId)
) {
throw new Error(
'Cannot use --start-from-workspace-id together with -w/--workspace-id',
);
}
try {
const sequence = this.upgradeSequenceReaderService.getUpgradeSequence();
this.logger.log(
formatUpgradeLog({
humanMessage: `Initialized upgrade sequence: ${sequence.length} step(s)`,
event: 'sequence.initialized',
logFields: {
stepCount: sequence.length,
dryRun: options.dryRun ?? false,
},
}),
);
for (const [index, step] of sequence.entries()) {
this.logger.verbose(
formatUpgradeLog({
humanMessage: ` [${index}] ${step.kind}${step.name} (${step.version})`,
event: 'sequence.step',
logFields: {
index,
kind: step.kind,
name: step.name,
version: step.version,
},
}),
);
}
const { totalSuccesses, totalFailures } =
await this.upgradeSequenceRunnerService.run({
sequence,
options: {
...options,
workspaceIds: isDefined(options.workspaceId)
? Array.from(options.workspaceId)
: undefined,
},
});
this.logger.log(
formatUpgradeLog({
humanMessage: `Upgrade summary: ${totalSuccesses} workspace(s) succeeded, ${totalFailures} workspace(s) failed`,
event: 'summary',
logFields: {
totalSuccesses,
totalFailures,
dryRun: options.dryRun ?? false,
},
}),
);
if (totalFailures > 0) {
throw new Error(
`Upgrade completed with ${totalFailures} workspace failure(s)`,
);
}
} catch (error) {
const errorMessage =
error instanceof Error ? error.message : String(error);
this.logger.error(
formatUpgradeLog({
humanMessage: `Upgrade failed: ${errorMessage}`,
event: 'aborted',
}),
);
throw error;
} finally {
await this.safeInvalidateUpgradeStatusCache();
}
}
private async safeInvalidateUpgradeStatusCache(): Promise<void> {
try {
await this.upgradeStatusService.invalidateInstanceAndAllWorkspacesStatus();
} catch (error) {
this.logger.error(
formatUpgradeLog({
humanMessage: `Failed to invalidate upgrade-status cache: ${
error instanceof Error ? error.message : String(error)
}`,
event: 'cache.invalidate.failed',
}),
);
}
}
}