[Upgrade] Stop command gracefully on SIGINT/SIGTERM (#23481)
Ctrl+C on `upgrade` used to kill the process wherever it happened to be, potentially in the middle of a workspace command. It now stops at the next iteration boundary instead. ## Behavior - **First SIGINT/SIGTERM** — the runner finishes what it started, then stops instead of starting new work. Exits with `130` (SIGINT) or `143` (SIGTERM), following the 128+signal convention, so orchestrators can tell an interruption apart from a failure. - **Second signal** — immediate exit, leaving the command in progress unfinished. - **SIGKILL** — untrappable, same outcome as a second signal. Nothing is rolled back on stop: the run resumes from the last command recorded in `upgradeMigration`. ## Opt-in per command Registering a `SIGINT` listener removes Node's default kill-on-signal behavior, so a command that installs a handler without honoring the flag would ignore the first Ctrl+C entirely. Handlers are therefore opt-in via `CommandShutdownService.listenToShutdownSignals()`, called by the two commands that stop at a boundary: - `UpgradeCommand` - `WorkspaceCommandRunner`, the base for standalone workspace commands Everything else keeps today's behavior and dies on the first signal, `run-instance-commands` included: instance commands are transactional and cursor-guarded, so a hard kill rolls back and a rerun skips what completed. `install-application`, `rebuild-application-default-deps` and `install-pre-installed-apps` iterate over workspaces without going through `WorkspaceCommandRunner`, so they are not armed either; they are one call away if we want them. The server and worker processes share these services and never arm anything, so their shutdown semantics are unchanged. ## Where the flag is checked `CommandShutdownService` exposes a single boolean, `isShutdownRequested()`, read only by the iteration runners: - `UpgradeSequenceRunnerService.runInner` — before each sequence step - `WorkspaceIteratorService.iterate` — before each workspace There is deliberately no `AbortSignal`: in-flight work is never cancelled, it is allowed to finish. Individual commands know nothing about shutdown, so a workspace that has started runs its whole pending segment before the run stops. Each workspace ends up either fully done with the segment or untouched, never scattered at some cursor inside it. That keeps resume state coarse and the change out of the command layer, at the cost of a longer stop latency, which the second Ctrl+C covers. `WorkspaceIteratorReport` gained an `interrupted` flag. The sequence runner needs it: stopping partway through the workspace list and then advancing the cursor would run an instance step against workspaces that are not aligned yet, so it returns instead. ## Deployment note Under Kubernetes, `terminationGracePeriodSeconds` must exceed the time for one workspace to finish its segment, otherwise the SIGTERM path degrades into a SIGKILL. Documented in `docs/UPGRADE_COMMANDS.md`. ## Testing - New unit test for `CommandShutdownService` (7 cases); 293 tests pass across `database/commands` and `core-modules/upgrade` - `tsgo -p tsconfig.json` clean - oxlint and oxfmt clean on all touched files
This commit is contained in:
@@ -134,6 +134,14 @@ Within a given version of Twenty, the upgrade pipeline runs commands in this ord
|
||||
|
||||
Workspace commands are executed sequentially across all active/suspended workspaces.
|
||||
|
||||
## Interrupting a run (Ctrl+C, SIGTERM)
|
||||
|
||||
Ctrl+C during an `upgrade` stops it gracefully: the workspace being processed finishes its commands, then the run stops instead of starting the next one. Ctrl+C again forces an immediate exit, leaving the command in progress unfinished.
|
||||
|
||||
Rerun the command to resume. Nothing is rolled back, and the run picks up from the last command recorded in `upgradeMigration`.
|
||||
|
||||
Expect the first Ctrl+C to look like it did nothing while a long step is running: it takes effect once the step ends.
|
||||
|
||||
## Shipping a command for a future version (deferred drops)
|
||||
|
||||
You can write a command for a version listed in `TWENTY_NEXT_VERSIONS` — typically the second half of a zero-downtime migration, e.g. dropping a column one release after its replacement ships. Pass the target version to the generator:
|
||||
|
||||
+98
@@ -0,0 +1,98 @@
|
||||
import { CommandShutdownService } from 'src/database/commands/command-runners/command-shutdown.service';
|
||||
|
||||
describe('CommandShutdownService', () => {
|
||||
let service: CommandShutdownService;
|
||||
let exitSpy: jest.SpyInstance;
|
||||
let signalListeners: Map<string, () => void>;
|
||||
const initialExitCode = process.exitCode;
|
||||
|
||||
beforeEach(() => {
|
||||
service = new CommandShutdownService();
|
||||
signalListeners = new Map();
|
||||
|
||||
exitSpy = jest.spyOn(process, 'exit').mockImplementation((() => {
|
||||
// oxlint-disable-next-line typescript/no-explicit-any
|
||||
}) as any);
|
||||
|
||||
jest
|
||||
.spyOn(process, 'on')
|
||||
.mockImplementation((event: string | symbol, listener) => {
|
||||
signalListeners.set(String(event), listener as () => void);
|
||||
|
||||
return process;
|
||||
});
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
jest.restoreAllMocks();
|
||||
process.exitCode = initialExitCode;
|
||||
});
|
||||
|
||||
it('should not install listeners until it is asked to listen', () => {
|
||||
expect(process.on).not.toHaveBeenCalled();
|
||||
|
||||
service.listenToShutdownSignals();
|
||||
|
||||
expect(process.on).toHaveBeenCalledWith('SIGINT', expect.any(Function));
|
||||
expect(process.on).toHaveBeenCalledWith('SIGTERM', expect.any(Function));
|
||||
});
|
||||
|
||||
it('should install listeners only once', () => {
|
||||
service.listenToShutdownSignals();
|
||||
service.listenToShutdownSignals();
|
||||
|
||||
const registrationsBySignal = (process.on as jest.Mock).mock.calls.reduce<
|
||||
Record<string, number>
|
||||
>((accumulator, [event]) => {
|
||||
accumulator[String(event)] = (accumulator[String(event)] ?? 0) + 1;
|
||||
|
||||
return accumulator;
|
||||
}, {});
|
||||
|
||||
expect(registrationsBySignal).toEqual({ SIGINT: 1, SIGTERM: 1 });
|
||||
});
|
||||
|
||||
it('should not report a shutdown before any signal is received', () => {
|
||||
service.listenToShutdownSignals();
|
||||
|
||||
expect(service.isShutdownRequested()).toBe(false);
|
||||
});
|
||||
|
||||
it('should request a graceful shutdown on the first SIGINT', () => {
|
||||
service.listenToShutdownSignals();
|
||||
|
||||
signalListeners.get('SIGINT')?.();
|
||||
|
||||
expect(service.isShutdownRequested()).toBe(true);
|
||||
expect(process.exitCode).toBe(130);
|
||||
expect(exitSpy).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('should force an immediate exit on the second SIGINT', () => {
|
||||
service.listenToShutdownSignals();
|
||||
|
||||
signalListeners.get('SIGINT')?.();
|
||||
signalListeners.get('SIGINT')?.();
|
||||
|
||||
expect(exitSpy).toHaveBeenCalledWith(130);
|
||||
});
|
||||
|
||||
it('should use the SIGTERM exit code on SIGTERM', () => {
|
||||
service.listenToShutdownSignals();
|
||||
|
||||
signalListeners.get('SIGTERM')?.();
|
||||
|
||||
expect(service.isShutdownRequested()).toBe(true);
|
||||
expect(process.exitCode).toBe(143);
|
||||
expect(exitSpy).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('should force an immediate exit when a second signal of another kind arrives', () => {
|
||||
service.listenToShutdownSignals();
|
||||
|
||||
signalListeners.get('SIGTERM')?.();
|
||||
signalListeners.get('SIGINT')?.();
|
||||
|
||||
expect(exitSpy).toHaveBeenCalledWith(130);
|
||||
});
|
||||
});
|
||||
+9
@@ -0,0 +1,9 @@
|
||||
import { Module } from '@nestjs/common';
|
||||
|
||||
import { CommandShutdownService } from 'src/database/commands/command-runners/command-shutdown.service';
|
||||
|
||||
@Module({
|
||||
providers: [CommandShutdownService],
|
||||
exports: [CommandShutdownService],
|
||||
})
|
||||
export class CommandShutdownModule {}
|
||||
+55
@@ -0,0 +1,55 @@
|
||||
import { Injectable, Logger } from '@nestjs/common';
|
||||
|
||||
import { isDefined } from 'twenty-shared/utils';
|
||||
|
||||
const SHUTDOWN_SIGNALS = ['SIGINT', 'SIGTERM'] as const;
|
||||
|
||||
type ShutdownSignal = (typeof SHUTDOWN_SIGNALS)[number];
|
||||
|
||||
const EXIT_CODE_BY_SIGNAL: Record<ShutdownSignal, number> = {
|
||||
SIGINT: 130,
|
||||
SIGTERM: 143,
|
||||
};
|
||||
|
||||
@Injectable()
|
||||
export class CommandShutdownService {
|
||||
private readonly logger = new Logger(CommandShutdownService.name);
|
||||
|
||||
private receivedSignal: ShutdownSignal | undefined;
|
||||
private isListening = false;
|
||||
|
||||
listenToShutdownSignals(): void {
|
||||
if (this.isListening) {
|
||||
return;
|
||||
}
|
||||
|
||||
this.isListening = true;
|
||||
|
||||
for (const shutdownSignal of SHUTDOWN_SIGNALS) {
|
||||
process.on(shutdownSignal, () => this.handleSignal(shutdownSignal));
|
||||
}
|
||||
}
|
||||
|
||||
isShutdownRequested(): boolean {
|
||||
return isDefined(this.receivedSignal);
|
||||
}
|
||||
|
||||
private handleSignal(shutdownSignal: ShutdownSignal): void {
|
||||
if (this.isShutdownRequested()) {
|
||||
this.logger.warn(
|
||||
`Received ${shutdownSignal} again, exiting immediately. ` +
|
||||
'The step in progress is left unfinished, rerun the command to resume from the last recorded step.',
|
||||
);
|
||||
|
||||
process.exit(EXIT_CODE_BY_SIGNAL[shutdownSignal]);
|
||||
}
|
||||
|
||||
this.receivedSignal = shutdownSignal;
|
||||
process.exitCode = EXIT_CODE_BY_SIGNAL[shutdownSignal];
|
||||
|
||||
this.logger.warn(
|
||||
`Received ${shutdownSignal}, finishing the step in progress then stopping. ` +
|
||||
`Send ${shutdownSignal} again to exit immediately.`,
|
||||
);
|
||||
}
|
||||
}
|
||||
+2
-1
@@ -1,11 +1,12 @@
|
||||
import { Module } from '@nestjs/common';
|
||||
import { TypeOrmModule } from '@nestjs/typeorm';
|
||||
|
||||
import { CommandShutdownModule } from 'src/database/commands/command-runners/command-shutdown.module';
|
||||
import { WorkspaceIteratorService } from 'src/database/commands/command-runners/workspace-iterator.service';
|
||||
import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity';
|
||||
|
||||
@Module({
|
||||
imports: [TypeOrmModule.forFeature([WorkspaceEntity])],
|
||||
imports: [CommandShutdownModule, TypeOrmModule.forFeature([WorkspaceEntity])],
|
||||
providers: [WorkspaceIteratorService],
|
||||
exports: [WorkspaceIteratorService],
|
||||
})
|
||||
|
||||
+19
@@ -10,6 +10,7 @@ import {
|
||||
import { isDefined } from 'twenty-shared/utils';
|
||||
import { MoreThanOrEqual, Repository } from 'typeorm';
|
||||
|
||||
import { CommandShutdownService } from 'src/database/commands/command-runners/command-shutdown.service';
|
||||
import { activationStatusIn } from 'src/database/commands/command-runners/utils/activation-status-in.util';
|
||||
import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity';
|
||||
import { GlobalWorkspaceDataSource } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-datasource';
|
||||
@@ -41,6 +42,7 @@ export type WorkspaceIteratorReport = {
|
||||
success: {
|
||||
workspaceId: string;
|
||||
}[];
|
||||
interrupted: boolean;
|
||||
};
|
||||
|
||||
const DEFAULT_ACTIVATION_STATUSES = PROVISIONED_WORKSPACE_ACTIVATION_STATUSES;
|
||||
@@ -53,14 +55,20 @@ export class WorkspaceIteratorService {
|
||||
@InjectRepository(WorkspaceEntity)
|
||||
private readonly workspaceRepository: Repository<WorkspaceEntity>,
|
||||
private readonly globalWorkspaceOrmManager: GlobalWorkspaceOrmManager,
|
||||
private readonly commandShutdownService: CommandShutdownService,
|
||||
) {}
|
||||
|
||||
listenToShutdownSignals(): void {
|
||||
this.commandShutdownService.listenToShutdownSignals();
|
||||
}
|
||||
|
||||
async iterate(args: WorkspaceIteratorArgs): Promise<WorkspaceIteratorReport> {
|
||||
const { callback, ...options } = args;
|
||||
|
||||
const report: WorkspaceIteratorReport = {
|
||||
fail: [],
|
||||
success: [],
|
||||
interrupted: false,
|
||||
};
|
||||
|
||||
const workspaceIdsToProcess =
|
||||
@@ -73,6 +81,17 @@ export class WorkspaceIteratorService {
|
||||
}
|
||||
|
||||
for (const [index, workspaceId] of workspaceIdsToProcess.entries()) {
|
||||
if (this.commandShutdownService.isShutdownRequested()) {
|
||||
this.logger.warn(
|
||||
`Shutdown requested, stopping before workspace ${workspaceId}. ` +
|
||||
`${workspaceIdsToProcess.length - index} workspace(s) left untouched.`,
|
||||
);
|
||||
|
||||
report.interrupted = true;
|
||||
|
||||
break;
|
||||
}
|
||||
|
||||
this.logger.log(
|
||||
`Running on workspace ${workspaceId} ${index + 1}/${workspaceIdsToProcess.length}`,
|
||||
);
|
||||
|
||||
+13
-1
@@ -108,8 +108,10 @@ export abstract class WorkspaceCommandRunner<
|
||||
});
|
||||
}
|
||||
|
||||
this.workspaceIteratorService.listenToShutdownSignals();
|
||||
|
||||
try {
|
||||
await this.workspaceIteratorService.iterate({
|
||||
const report = await this.workspaceIteratorService.iterate({
|
||||
workspaceIds:
|
||||
options.workspaceId && options.workspaceId.size > 0
|
||||
? Array.from(options.workspaceId)
|
||||
@@ -129,6 +131,16 @@ export abstract class WorkspaceCommandRunner<
|
||||
},
|
||||
});
|
||||
|
||||
if (report.interrupted) {
|
||||
this.logger.warn(
|
||||
chalk.yellow(
|
||||
'Command interrupted before processing every workspace. Rerun it to process the remaining ones.',
|
||||
),
|
||||
);
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
this.logger.log(chalk.blue('Command completed!'));
|
||||
} catch (error) {
|
||||
this.logger.error(chalk.red(`Command failed`));
|
||||
|
||||
+2
-1
@@ -1,11 +1,12 @@
|
||||
import { Module } from '@nestjs/common';
|
||||
|
||||
import { CommandShutdownModule } from 'src/database/commands/command-runners/command-shutdown.module';
|
||||
import { WorkspaceIteratorModule } from 'src/database/commands/command-runners/workspace-iterator.module';
|
||||
import { UpgradeCommand } from 'src/database/commands/upgrade-version-command/upgrade.command';
|
||||
import { UpgradeModule } from 'src/engine/core-modules/upgrade/upgrade.module';
|
||||
|
||||
@Module({
|
||||
imports: [UpgradeModule, WorkspaceIteratorModule],
|
||||
imports: [CommandShutdownModule, UpgradeModule, WorkspaceIteratorModule],
|
||||
providers: [UpgradeCommand],
|
||||
})
|
||||
export class UpgradeVersionCommandModule {}
|
||||
|
||||
+4
@@ -1,6 +1,7 @@
|
||||
import { Command, CommandRunner, Option } from 'nest-commander';
|
||||
import { isDefined } from 'twenty-shared/utils';
|
||||
|
||||
import { CommandShutdownService } from 'src/database/commands/command-runners/command-shutdown.service';
|
||||
import { CommandLogger } from 'src/database/commands/logger';
|
||||
import { UpgradeSequenceReaderService } from 'src/engine/core-modules/upgrade/services/upgrade-sequence-reader.service';
|
||||
import { UpgradeSequenceRunnerService } from 'src/engine/core-modules/upgrade/services/upgrade-sequence-runner.service';
|
||||
@@ -34,6 +35,7 @@ export class UpgradeCommand extends CommandRunner {
|
||||
protected readonly upgradeSequenceReaderService: UpgradeSequenceReaderService,
|
||||
protected readonly upgradeSequenceRunnerService: UpgradeSequenceRunnerService,
|
||||
protected readonly upgradeStatusService: UpgradeStatusService,
|
||||
protected readonly commandShutdownService: CommandShutdownService,
|
||||
) {
|
||||
super();
|
||||
this.logger = new CommandLogger({
|
||||
@@ -124,6 +126,8 @@ export class UpgradeCommand extends CommandRunner {
|
||||
);
|
||||
}
|
||||
|
||||
this.commandShutdownService.listenToShutdownSignals();
|
||||
|
||||
try {
|
||||
const sequence = this.upgradeSequenceReaderService.getUpgradeSequence();
|
||||
|
||||
|
||||
+1
-1
@@ -180,7 +180,7 @@ export class ApplicationUpgradeService {
|
||||
// provisioned workspace, which would upgrade workspaces that were
|
||||
// filtered out.
|
||||
if (!isNonEmptyArray(applications)) {
|
||||
return { success: [], fail: [] };
|
||||
return { success: [], fail: [], interrupted: false };
|
||||
}
|
||||
|
||||
return this.workspaceIteratorService.iterate({
|
||||
|
||||
+139
@@ -0,0 +1,139 @@
|
||||
import { type WorkspaceIteratorService } from 'src/database/commands/command-runners/workspace-iterator.service';
|
||||
import { type CommandShutdownService } from 'src/database/commands/command-runners/command-shutdown.service';
|
||||
import { type InstanceCommandRunnerService } from 'src/engine/core-modules/upgrade/services/instance-command-runner.service';
|
||||
import { type UpgradeMigrationService } from 'src/engine/core-modules/upgrade/services/upgrade-migration.service';
|
||||
import {
|
||||
type UpgradeStep,
|
||||
type UpgradeSequenceReaderService,
|
||||
} from 'src/engine/core-modules/upgrade/services/upgrade-sequence-reader.service';
|
||||
import { UpgradeSequenceRunnerService } from 'src/engine/core-modules/upgrade/services/upgrade-sequence-runner.service';
|
||||
import { type WorkspaceCommandRunnerService } from 'src/engine/core-modules/upgrade/services/workspace-command-runner.service';
|
||||
import { type UpgradeAwareEntityMetadataAdapter } from 'src/engine/twenty-orm/upgrade-aware/upgrade-aware-entity-metadata.adapter';
|
||||
import { type WorkspaceVersionService } from 'src/engine/workspace-manager/workspace-version/services/workspace-version.service';
|
||||
|
||||
const WORKSPACE_ID = 'workspace-1';
|
||||
|
||||
const makeStep = (kind: UpgradeStep['kind'], name: string) =>
|
||||
({
|
||||
kind,
|
||||
name,
|
||||
command: {},
|
||||
version: '1.0.0',
|
||||
timestamp: 0,
|
||||
}) as unknown as UpgradeStep;
|
||||
|
||||
describe('UpgradeSequenceRunnerService shutdown handling', () => {
|
||||
let service: UpgradeSequenceRunnerService;
|
||||
let isShutdownRequested: jest.Mock<boolean, []>;
|
||||
let runFastInstanceCommand: jest.Mock;
|
||||
let iterate: jest.Mock;
|
||||
|
||||
beforeEach(() => {
|
||||
isShutdownRequested = jest.fn().mockReturnValue(false);
|
||||
runFastInstanceCommand = jest.fn().mockResolvedValue({ status: 'success' });
|
||||
iterate = jest.fn().mockResolvedValue({
|
||||
success: [{ workspaceId: WORKSPACE_ID }],
|
||||
fail: [],
|
||||
interrupted: false,
|
||||
});
|
||||
|
||||
const upgradeMigrationService = {
|
||||
getLastAttemptedCommandNameOrThrow: jest
|
||||
.fn()
|
||||
.mockResolvedValue({ name: 'step-1', status: 'failed' }),
|
||||
getWorkspaceLastAttemptedCommandNameOrThrow: jest
|
||||
.fn()
|
||||
.mockResolvedValue(new Map()),
|
||||
} as unknown as UpgradeMigrationService;
|
||||
|
||||
const upgradeSequenceReaderService = {
|
||||
locateStepInSequenceOrThrow: jest.fn().mockReturnValue(0),
|
||||
getWorkspaceSegmentBounds: jest
|
||||
.fn()
|
||||
.mockReturnValue({ startCursor: 0, endCursor: 0 }),
|
||||
collectWorkspaceCommandsStartingFrom: jest
|
||||
.fn()
|
||||
.mockImplementation(({ fromWorkspaceCommand }) => [
|
||||
fromWorkspaceCommand,
|
||||
]),
|
||||
getPendingWorkspaceCommands: jest.fn().mockReturnValue([]),
|
||||
} as unknown as UpgradeSequenceReaderService;
|
||||
|
||||
service = new UpgradeSequenceRunnerService(
|
||||
upgradeMigrationService,
|
||||
{ runFastInstanceCommand } as unknown as InstanceCommandRunnerService,
|
||||
{
|
||||
runWorkspaceCommands: jest.fn(),
|
||||
} as unknown as WorkspaceCommandRunnerService,
|
||||
upgradeSequenceReaderService,
|
||||
{
|
||||
refresh: jest.fn().mockResolvedValue(undefined),
|
||||
} as unknown as UpgradeAwareEntityMetadataAdapter,
|
||||
{ iterate } as unknown as WorkspaceIteratorService,
|
||||
{
|
||||
getProvisionedWorkspaceIds: jest.fn().mockResolvedValue([WORKSPACE_ID]),
|
||||
} as unknown as WorkspaceVersionService,
|
||||
{ isShutdownRequested } as unknown as CommandShutdownService,
|
||||
);
|
||||
|
||||
jest.spyOn(service['logger'], 'log').mockImplementation();
|
||||
jest.spyOn(service['logger'], 'warn').mockImplementation();
|
||||
jest.spyOn(service['logger'], 'error').mockImplementation();
|
||||
});
|
||||
|
||||
it('should run every step when no shutdown is requested', async () => {
|
||||
const sequence = [
|
||||
makeStep('fast-instance', 'step-1'),
|
||||
makeStep('fast-instance', 'step-2'),
|
||||
];
|
||||
|
||||
await service.run({ sequence, options: {} });
|
||||
|
||||
expect(runFastInstanceCommand).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it('should stop before the next step when a shutdown is requested', async () => {
|
||||
const sequence = [
|
||||
makeStep('fast-instance', 'step-1'),
|
||||
makeStep('fast-instance', 'step-2'),
|
||||
];
|
||||
|
||||
isShutdownRequested.mockReturnValueOnce(false).mockReturnValue(true);
|
||||
|
||||
await service.run({ sequence, options: {} });
|
||||
|
||||
expect(runFastInstanceCommand).toHaveBeenCalledTimes(1);
|
||||
expect(runFastInstanceCommand).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ name: 'step-1' }),
|
||||
);
|
||||
});
|
||||
|
||||
it('should not advance to the next instance step when a workspace segment is interrupted', async () => {
|
||||
const sequence = [
|
||||
makeStep('workspace', 'workspace-step-1'),
|
||||
makeStep('fast-instance', 'instance-step-1'),
|
||||
];
|
||||
|
||||
iterate.mockResolvedValue({
|
||||
success: [{ workspaceId: WORKSPACE_ID }],
|
||||
fail: [],
|
||||
interrupted: true,
|
||||
});
|
||||
|
||||
const report = await service.run({ sequence, options: {} });
|
||||
|
||||
expect(runFastInstanceCommand).not.toHaveBeenCalled();
|
||||
expect(report).toEqual({ totalSuccesses: 1, totalFailures: 0 });
|
||||
});
|
||||
|
||||
it('should advance to the next instance step when a workspace segment completes', async () => {
|
||||
const sequence = [
|
||||
makeStep('workspace', 'workspace-step-1'),
|
||||
makeStep('fast-instance', 'instance-step-1'),
|
||||
];
|
||||
|
||||
await service.run({ sequence, options: {} });
|
||||
|
||||
expect(runFastInstanceCommand).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
});
|
||||
+36
@@ -1,5 +1,6 @@
|
||||
import { Injectable, Logger } from '@nestjs/common';
|
||||
|
||||
import { CommandShutdownService } from 'src/database/commands/command-runners/command-shutdown.service';
|
||||
import {
|
||||
type WorkspaceIteratorReport,
|
||||
WorkspaceIteratorService,
|
||||
@@ -39,6 +40,7 @@ export class UpgradeSequenceRunnerService {
|
||||
private readonly upgradeAwareEntityMetadataAdapter: UpgradeAwareEntityMetadataAdapter,
|
||||
private readonly workspaceIteratorService: WorkspaceIteratorService,
|
||||
private readonly workspaceVersionService: WorkspaceVersionService,
|
||||
private readonly commandShutdownService: CommandShutdownService,
|
||||
) {}
|
||||
|
||||
async run({
|
||||
@@ -95,6 +97,23 @@ export class UpgradeSequenceRunnerService {
|
||||
while (cursor < sequence.length) {
|
||||
const step = sequence[cursor];
|
||||
|
||||
if (this.commandShutdownService.isShutdownRequested()) {
|
||||
this.logger.warn(
|
||||
formatUpgradeLog({
|
||||
humanMessage:
|
||||
`Stopping before step "${step.name}": shutdown requested. ` +
|
||||
'Rerun the upgrade to resume from this step.',
|
||||
event: 'sequence.stopped',
|
||||
logFields: {
|
||||
before: step.name,
|
||||
reason: 'shutdown-requested',
|
||||
},
|
||||
}),
|
||||
);
|
||||
|
||||
break;
|
||||
}
|
||||
|
||||
if (step.kind === 'fast-instance' || step.kind === 'slow-instance') {
|
||||
if (
|
||||
(isDefined(options.workspaceIds) &&
|
||||
@@ -173,6 +192,23 @@ export class UpgradeSequenceRunnerService {
|
||||
return { totalSuccesses, totalFailures };
|
||||
}
|
||||
|
||||
if (report.interrupted) {
|
||||
this.logger.warn(
|
||||
formatUpgradeLog({
|
||||
humanMessage:
|
||||
'Stopped during workspace steps: shutdown requested. ' +
|
||||
'Rerun the upgrade to process the remaining workspaces.',
|
||||
event: 'sequence.stopped',
|
||||
logFields: {
|
||||
reason: 'shutdown-requested',
|
||||
processedWorkspaces: report.success.length,
|
||||
},
|
||||
}),
|
||||
);
|
||||
|
||||
return { totalSuccesses, totalFailures };
|
||||
}
|
||||
|
||||
cursor += workspaceCommandsSegment.length;
|
||||
|
||||
workspaceCursors = await this.fetchWorkspaceCursors(
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import { Module } from '@nestjs/common';
|
||||
|
||||
import { CommandShutdownModule } from 'src/database/commands/command-runners/command-shutdown.module';
|
||||
import { WorkspaceIteratorModule } from 'src/database/commands/command-runners/workspace-iterator.module';
|
||||
import { InstanceCommandProviderModule } from 'src/database/commands/upgrade-version-command/instance-command-provider.module';
|
||||
import { WorkspaceCommandProviderModule } from 'src/database/commands/upgrade-version-command/workspace-command-provider.module';
|
||||
@@ -14,6 +15,7 @@ import { WorkspaceVersionModule } from 'src/engine/workspace-manager/workspace-v
|
||||
|
||||
@Module({
|
||||
imports: [
|
||||
CommandShutdownModule,
|
||||
InstanceCommandProviderModule,
|
||||
MetricsModule,
|
||||
UpgradeStatusModule,
|
||||
|
||||
+7
-1
@@ -5,6 +5,7 @@ import { config } from 'dotenv';
|
||||
import { WorkspaceActivationStatus } from 'twenty-shared/workspace';
|
||||
import { DataSource, type Repository } from 'typeorm';
|
||||
|
||||
import { CommandShutdownService } from 'src/database/commands/command-runners/command-shutdown.service';
|
||||
import { WorkspaceIteratorService } from 'src/database/commands/command-runners/workspace-iterator.service';
|
||||
import { ApplicationEntity } from 'src/engine/core-modules/application/application.entity';
|
||||
import { TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service';
|
||||
@@ -228,7 +229,11 @@ export const createUpgradeSequenceRunnerIntegrationTestModule = async () => {
|
||||
iterate: jest.fn().mockImplementation(async (args: any) => {
|
||||
const { callback, workspaceIds } = args;
|
||||
const ids = workspaceIds ?? [WS_1];
|
||||
const report = { fail: [] as any[], success: [] as any[] };
|
||||
const report = {
|
||||
fail: [] as any[],
|
||||
success: [] as any[],
|
||||
interrupted: false,
|
||||
};
|
||||
|
||||
for (const [index, workspaceId] of ids.entries()) {
|
||||
try {
|
||||
@@ -256,6 +261,7 @@ export const createUpgradeSequenceRunnerIntegrationTestModule = async () => {
|
||||
getHiddenColumnPropertyNames: jest.fn().mockReturnValue(new Set()),
|
||||
},
|
||||
},
|
||||
CommandShutdownService,
|
||||
UpgradeSequenceRunnerService,
|
||||
],
|
||||
}).compile();
|
||||
|
||||
Reference in New Issue
Block a user