577b22df46
## Problem The "Twenty / Upgrade Status" Grafana dashboard shows stale workspace counts (e.g. `N behind / 0 up-to-date` while the instance reads `UP_TO_DATE`) that disagree with `command:prod upgrade:status`. The CLI is correct; the dashboard lags, sometimes for the full hour. ## Root cause The dashboard is fed by the `twenty_upgrade_workspaces_*` gauges, which read their workspace counts from a Redis snapshot (`UpgradeStatusCacheService`). That snapshot is only invalidated **per-command, inside the runners' `finally` blocks**. Two gaps: 1. An instance command that is already applied returns **before** its invalidation runs (`isAlreadyCompleted` early-return in `InstanceCommandRunnerService`). So a plain **redeploy** — which changes the deployed upgrade sequence, and thus the "behind" answer, without executing any command — never refreshes the snapshot. This is most visible on an instance-only release. 2. The snapshot then stays frozen until its 60-minute TTL, while the CLI reads live and disagrees. "Behind" is derived from the deployed sequence, not just the ledger, so the correct answer changes on events (deploys) that run no command — which is exactly why per-command invalidation isn't enough on its own. ## Fix Invalidate the upgrade-status cache **once, unconditionally, at the end of both upgrade entrypoints** — `run-instance-commands` (the deploy/migrate step) and `upgrade` — in a `finally`. Every run, including a no-op redeploy where all commands are already applied, now clears the snapshot, so the next gauge scrape recomputes against the current sequence. Best-effort (failures are logged, never block the command). The existing per-command invalidation is kept for mid-run progress. This keeps the read path untouched. ## Reproduction + verification (live, local) Served twenty-server (`NODE_PORT=4000`, `METER_DRIVER=prometheus`) against the seeded DB, whose latest version `2.12.0` is instance-only. 1. Froze the gauge at `behind 4 / up_to_date 0` while the DB was brought up-to-date (snapshot not invalidated) — reproduced the dashboard/CLI divergence. 2. Ran the **patched** `run-instance-commands --force`. Every step logged `already executed, skipping` — and the `finally` still deleted the Redis snapshot. 3. On the next recompute the gauge self-healed to `instance_health 1, behind 0, up_to_date 4`, matching the live CLI. With the old code the snapshot stayed frozen at `behind 4` until the TTL.
215 lines
5.9 KiB
TypeScript
215 lines
5.9 KiB
TypeScript
import { Command, CommandRunner, Option } from 'nest-commander';
|
|
import { isDefined } from 'twenty-shared/utils';
|
|
|
|
import { CommandLogger } from 'src/database/commands/logger';
|
|
import { UpgradeSequenceReaderService } from 'src/engine/core-modules/upgrade/services/upgrade-sequence-reader.service';
|
|
import { UpgradeSequenceRunnerService } from 'src/engine/core-modules/upgrade/services/upgrade-sequence-runner.service';
|
|
import { UpgradeStatusService } from 'src/engine/core-modules/upgrade/services/upgrade-status.service';
|
|
import { formatUpgradeLog } from 'src/engine/core-modules/upgrade/utils/format-upgrade-log.util';
|
|
|
|
type RawUpgradeCommandOptions = {
|
|
workspaceId?: Set<string>;
|
|
startFromWorkspaceId?: string;
|
|
workspaceCountLimit?: number;
|
|
dryRun?: boolean;
|
|
verbose?: boolean;
|
|
};
|
|
|
|
export type ParsedUpgradeCommandOptions = {
|
|
workspaceIds?: string[];
|
|
startFromWorkspaceId?: string;
|
|
workspaceCountLimit?: number;
|
|
dryRun?: boolean;
|
|
verbose?: boolean;
|
|
};
|
|
|
|
@Command({
|
|
name: 'upgrade',
|
|
description: 'Upgrade workspaces to the latest version',
|
|
})
|
|
export class UpgradeCommand extends CommandRunner {
|
|
protected logger: CommandLogger;
|
|
|
|
constructor(
|
|
protected readonly upgradeSequenceReaderService: UpgradeSequenceReaderService,
|
|
protected readonly upgradeSequenceRunnerService: UpgradeSequenceRunnerService,
|
|
protected readonly upgradeStatusService: UpgradeStatusService,
|
|
) {
|
|
super();
|
|
this.logger = new CommandLogger({
|
|
verbose: false,
|
|
constructorName: this.constructor.name,
|
|
});
|
|
}
|
|
|
|
@Option({
|
|
flags: '-d, --dry-run',
|
|
description: 'Simulate the command without making actual changes',
|
|
required: false,
|
|
})
|
|
parseDryRun(): boolean {
|
|
return true;
|
|
}
|
|
|
|
@Option({
|
|
flags: '-v, --verbose',
|
|
description: 'Verbose output',
|
|
required: false,
|
|
})
|
|
parseVerbose(): boolean {
|
|
return true;
|
|
}
|
|
|
|
@Option({
|
|
flags: '-w, --workspace-id [workspace_id]',
|
|
description:
|
|
'workspace id. Command runs on all active/suspended workspaces if not provided.',
|
|
required: false,
|
|
})
|
|
parseWorkspaceId(val: string, previous?: Set<string>): Set<string> {
|
|
const accumulator = previous ?? new Set<string>();
|
|
|
|
accumulator.add(val);
|
|
|
|
return accumulator;
|
|
}
|
|
|
|
@Option({
|
|
flags: '--start-from-workspace-id [workspace_id]',
|
|
description:
|
|
'Start from a specific workspace id. Workspaces are processed in ascending order of id.',
|
|
required: false,
|
|
})
|
|
parseStartFromWorkspaceId(val: string): string {
|
|
return val;
|
|
}
|
|
|
|
@Option({
|
|
flags: '--workspace-count-limit [count]',
|
|
description:
|
|
'Limit the number of workspaces to process. Workspaces are processed in ascending order of id.',
|
|
required: false,
|
|
})
|
|
parseWorkspaceCountLimit(val: string): number {
|
|
const limit = parseInt(val);
|
|
|
|
if (isNaN(limit)) {
|
|
throw new Error('Workspace count limit must be a number');
|
|
}
|
|
|
|
if (limit <= 0) {
|
|
throw new Error('Workspace count limit must be greater than 0');
|
|
}
|
|
|
|
return limit;
|
|
}
|
|
|
|
override async run(
|
|
_passedParams: string[],
|
|
options: RawUpgradeCommandOptions,
|
|
): Promise<void> {
|
|
if (options.verbose) {
|
|
this.logger = new CommandLogger({
|
|
verbose: true,
|
|
constructorName: this.constructor.name,
|
|
});
|
|
}
|
|
|
|
if (
|
|
isDefined(options.workspaceId) &&
|
|
isDefined(options.startFromWorkspaceId)
|
|
) {
|
|
throw new Error(
|
|
'Cannot use --start-from-workspace-id together with -w/--workspace-id',
|
|
);
|
|
}
|
|
|
|
try {
|
|
const sequence = this.upgradeSequenceReaderService.getUpgradeSequence();
|
|
|
|
this.logger.log(
|
|
formatUpgradeLog({
|
|
humanMessage: `Initialized upgrade sequence: ${sequence.length} step(s)`,
|
|
event: 'sequence.initialized',
|
|
logFields: {
|
|
stepCount: sequence.length,
|
|
dryRun: options.dryRun ?? false,
|
|
},
|
|
}),
|
|
);
|
|
|
|
for (const [index, step] of sequence.entries()) {
|
|
this.logger.verbose(
|
|
formatUpgradeLog({
|
|
humanMessage: ` [${index}] ${step.kind} — ${step.name} (${step.version})`,
|
|
event: 'sequence.step',
|
|
logFields: {
|
|
index,
|
|
kind: step.kind,
|
|
name: step.name,
|
|
version: step.version,
|
|
},
|
|
}),
|
|
);
|
|
}
|
|
|
|
const { totalSuccesses, totalFailures } =
|
|
await this.upgradeSequenceRunnerService.run({
|
|
sequence,
|
|
options: {
|
|
...options,
|
|
workspaceIds: isDefined(options.workspaceId)
|
|
? Array.from(options.workspaceId)
|
|
: undefined,
|
|
},
|
|
});
|
|
|
|
this.logger.log(
|
|
formatUpgradeLog({
|
|
humanMessage: `Upgrade summary: ${totalSuccesses} workspace(s) succeeded, ${totalFailures} workspace(s) failed`,
|
|
event: 'summary',
|
|
logFields: {
|
|
totalSuccesses,
|
|
totalFailures,
|
|
dryRun: options.dryRun ?? false,
|
|
},
|
|
}),
|
|
);
|
|
|
|
if (totalFailures > 0) {
|
|
throw new Error(
|
|
`Upgrade completed with ${totalFailures} workspace failure(s)`,
|
|
);
|
|
}
|
|
} catch (error) {
|
|
const errorMessage =
|
|
error instanceof Error ? error.message : String(error);
|
|
|
|
this.logger.error(
|
|
formatUpgradeLog({
|
|
humanMessage: `Upgrade failed: ${errorMessage}`,
|
|
event: 'aborted',
|
|
}),
|
|
);
|
|
throw error;
|
|
} finally {
|
|
await this.safeInvalidateUpgradeStatusCache();
|
|
}
|
|
}
|
|
|
|
private async safeInvalidateUpgradeStatusCache(): Promise<void> {
|
|
try {
|
|
await this.upgradeStatusService.invalidateInstanceAndAllWorkspacesStatus();
|
|
} catch (error) {
|
|
this.logger.error(
|
|
formatUpgradeLog({
|
|
humanMessage: `Failed to invalidate upgrade-status cache: ${
|
|
error instanceof Error ? error.message : String(error)
|
|
}`,
|
|
event: 'cache.invalidate.failed',
|
|
}),
|
|
);
|
|
}
|
|
}
|
|
}
|