[Upgrade] Stop command gracefully on SIGINT/SIGTERM (#23481)

Ctrl+C on `upgrade` used to kill the process wherever it happened to be, potentially in the middle of a workspace command. It now stops at the next iteration boundary instead. ## Behavior - **First SIGINT/SIGTERM** — the runner finishes what it started, then stops instead of starting new work. Exits with `130` (SIGINT) or `143` (SIGTERM), following the 128+signal convention, so orchestrators can tell an interruption apart from a failure. - **Second signal** — immediate exit, leaving the command in progress unfinished. - **SIGKILL** — untrappable, same outcome as a second signal. Nothing is rolled back on stop: the run resumes from the last command recorded in `upgradeMigration`. ## Opt-in per command Registering a `SIGINT` listener removes Node's default kill-on-signal behavior, so a command that installs a handler without honoring the flag would ignore the first Ctrl+C entirely. Handlers are therefore opt-in via `CommandShutdownService.listenToShutdownSignals()`, called by the two commands that stop at a boundary: - `UpgradeCommand` - `WorkspaceCommandRunner`, the base for standalone workspace commands Everything else keeps today's behavior and dies on the first signal, `run-instance-commands` included: instance commands are transactional and cursor-guarded, so a hard kill rolls back and a rerun skips what completed. `install-application`, `rebuild-application-default-deps` and `install-pre-installed-apps` iterate over workspaces without going through `WorkspaceCommandRunner`, so they are not armed either; they are one call away if we want them. The server and worker processes share these services and never arm anything, so their shutdown semantics are unchanged. ## Where the flag is checked `CommandShutdownService` exposes a single boolean, `isShutdownRequested()`, read only by the iteration runners: - `UpgradeSequenceRunnerService.runInner` — before each sequence step - `WorkspaceIteratorService.iterate` — before each workspace There is deliberately no `AbortSignal`: in-flight work is never cancelled, it is allowed to finish. Individual commands know nothing about shutdown, so a workspace that has started runs its whole pending segment before the run stops. Each workspace ends up either fully done with the segment or untouched, never scattered at some cursor inside it. That keeps resume state coarse and the change out of the command layer, at the cost of a longer stop latency, which the second Ctrl+C covers. `WorkspaceIteratorReport` gained an `interrupted` flag. The sequence runner needs it: stopping partway through the workspace list and then advancing the cursor would run an instance step against workspaces that are not aligned yet, so it returns instead. ## Deployment note Under Kubernetes, `terminationGracePeriodSeconds` must exceed the time for one workspace to finish its segment, otherwise the SIGTERM path degrades into a SIGKILL. Documented in `docs/UPGRADE_COMMANDS.md`. ## Testing - New unit test for `CommandShutdownService` (7 cases); 293 tests pass across `database/commands` and `core-modules/upgrade` - `tsgo -p tsconfig.json` clean - oxlint and oxfmt clean on all touched files
2026-07-29 16:49:47 +02:00
parent 742f76e318
commit 133b3375b6
14 changed files with 395 additions and 5 deletions
@@ -0,0 +1,98 @@
+import { CommandShutdownService } from 'src/database/commands/command-runners/command-shutdown.service';
+
+describe('CommandShutdownService', () => {
+  let service: CommandShutdownService;
+  let exitSpy: jest.SpyInstance;
+  let signalListeners: Map<string, () => void>;
+  const initialExitCode = process.exitCode;
+
+  beforeEach(() => {
+    service = new CommandShutdownService();
+    signalListeners = new Map();
+
+    exitSpy = jest.spyOn(process, 'exit').mockImplementation((() => {
+      // oxlint-disable-next-line typescript/no-explicit-any
+    }) as any);
+
+    jest
+      .spyOn(process, 'on')
+      .mockImplementation((event: string | symbol, listener) => {
+        signalListeners.set(String(event), listener as () => void);
+
+        return process;
+      });
+  });
+
+  afterEach(() => {
+    jest.restoreAllMocks();
+    process.exitCode = initialExitCode;
+  });
+
+  it('should not install listeners until it is asked to listen', () => {
+    expect(process.on).not.toHaveBeenCalled();
+
+    service.listenToShutdownSignals();
+
+    expect(process.on).toHaveBeenCalledWith('SIGINT', expect.any(Function));
+    expect(process.on).toHaveBeenCalledWith('SIGTERM', expect.any(Function));
+  });
+
+  it('should install listeners only once', () => {
+    service.listenToShutdownSignals();
+    service.listenToShutdownSignals();
+
+    const registrationsBySignal = (process.on as jest.Mock).mock.calls.reduce<
+      Record<string, number>
+    >((accumulator, [event]) => {
+      accumulator[String(event)] = (accumulator[String(event)] ?? 0) + 1;
+
+      return accumulator;
+    }, {});
+
+    expect(registrationsBySignal).toEqual({ SIGINT: 1, SIGTERM: 1 });
+  });
+
+  it('should not report a shutdown before any signal is received', () => {
+    service.listenToShutdownSignals();
+
+    expect(service.isShutdownRequested()).toBe(false);
+  });
+
+  it('should request a graceful shutdown on the first SIGINT', () => {
+    service.listenToShutdownSignals();
+
+    signalListeners.get('SIGINT')?.();
+
+    expect(service.isShutdownRequested()).toBe(true);
+    expect(process.exitCode).toBe(130);
+    expect(exitSpy).not.toHaveBeenCalled();
+  });
+
+  it('should force an immediate exit on the second SIGINT', () => {
+    service.listenToShutdownSignals();
+
+    signalListeners.get('SIGINT')?.();
+    signalListeners.get('SIGINT')?.();
+
+    expect(exitSpy).toHaveBeenCalledWith(130);
+  });
+
+  it('should use the SIGTERM exit code on SIGTERM', () => {
+    service.listenToShutdownSignals();
+
+    signalListeners.get('SIGTERM')?.();
+
+    expect(service.isShutdownRequested()).toBe(true);
+    expect(process.exitCode).toBe(143);
+    expect(exitSpy).not.toHaveBeenCalled();
+  });
+
+  it('should force an immediate exit when a second signal of another kind arrives', () => {
+    service.listenToShutdownSignals();
+
+    signalListeners.get('SIGTERM')?.();
+    signalListeners.get('SIGINT')?.();
+
+    expect(exitSpy).toHaveBeenCalledWith(130);
+  });
+});
@@ -0,0 +1,9 @@
+import { Module } from '@nestjs/common';
+
+import { CommandShutdownService } from 'src/database/commands/command-runners/command-shutdown.service';
+
+@Module({
+  providers: [CommandShutdownService],
+  exports: [CommandShutdownService],
+})
+export class CommandShutdownModule {}
@@ -0,0 +1,55 @@
+import { Injectable, Logger } from '@nestjs/common';
+
+import { isDefined } from 'twenty-shared/utils';
+
+const SHUTDOWN_SIGNALS = ['SIGINT', 'SIGTERM'] as const;
+
+type ShutdownSignal = (typeof SHUTDOWN_SIGNALS)[number];
+
+const EXIT_CODE_BY_SIGNAL: Record<ShutdownSignal, number> = {
+  SIGINT: 130,
+  SIGTERM: 143,
+};
+
+@Injectable()
+export class CommandShutdownService {
+  private readonly logger = new Logger(CommandShutdownService.name);
+
+  private receivedSignal: ShutdownSignal | undefined;
+  private isListening = false;
+
+  listenToShutdownSignals(): void {
+    if (this.isListening) {
+      return;
+    }
+
+    this.isListening = true;
+
+    for (const shutdownSignal of SHUTDOWN_SIGNALS) {
+      process.on(shutdownSignal, () => this.handleSignal(shutdownSignal));
+    }
+  }
+
+  isShutdownRequested(): boolean {
+    return isDefined(this.receivedSignal);
+  }
+
+  private handleSignal(shutdownSignal: ShutdownSignal): void {
+    if (this.isShutdownRequested()) {
+      this.logger.warn(
+        `Received ${shutdownSignal} again, exiting immediately. ` +
+          'The step in progress is left unfinished, rerun the command to resume from the last recorded step.',
+      );
+
+      process.exit(EXIT_CODE_BY_SIGNAL[shutdownSignal]);
+    }
+
+    this.receivedSignal = shutdownSignal;
+    process.exitCode = EXIT_CODE_BY_SIGNAL[shutdownSignal];
+
+    this.logger.warn(
+      `Received ${shutdownSignal}, finishing the step in progress then stopping. ` +
+        `Send ${shutdownSignal} again to exit immediately.`,
+    );
+  }
+}
@@ -1,11 +1,12 @@
 import { Module } from '@nestjs/common';
 import { TypeOrmModule } from '@nestjs/typeorm';

+import { CommandShutdownModule } from 'src/database/commands/command-runners/command-shutdown.module';
 import { WorkspaceIteratorService } from 'src/database/commands/command-runners/workspace-iterator.service';
 import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity';

@Module({
-  imports: [TypeOrmModule.forFeature([WorkspaceEntity])],
+  imports: [CommandShutdownModule, TypeOrmModule.forFeature([WorkspaceEntity])],
  providers: [WorkspaceIteratorService],
  exports: [WorkspaceIteratorService],
 })
@@ -10,6 +10,7 @@ import {
 import { isDefined } from 'twenty-shared/utils';
 import { MoreThanOrEqual, Repository } from 'typeorm';

+import { CommandShutdownService } from 'src/database/commands/command-runners/command-shutdown.service';
 import { activationStatusIn } from 'src/database/commands/command-runners/utils/activation-status-in.util';
 import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity';
 import { GlobalWorkspaceDataSource } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-datasource';
@@ -41,6 +42,7 @@ export type WorkspaceIteratorReport = {
  success: {
    workspaceId: string;
  }[];
+  interrupted: boolean;
 };

 const DEFAULT_ACTIVATION_STATUSES = PROVISIONED_WORKSPACE_ACTIVATION_STATUSES;
@@ -53,14 +55,20 @@ export class WorkspaceIteratorService {
    @InjectRepository(WorkspaceEntity)
    private readonly workspaceRepository: Repository<WorkspaceEntity>,
    private readonly globalWorkspaceOrmManager: GlobalWorkspaceOrmManager,
+    private readonly commandShutdownService: CommandShutdownService,
  ) {}

+  listenToShutdownSignals(): void {
+    this.commandShutdownService.listenToShutdownSignals();
+  }
+
  async iterate(args: WorkspaceIteratorArgs): Promise<WorkspaceIteratorReport> {
    const { callback, ...options } = args;

    const report: WorkspaceIteratorReport = {
      fail: [],
      success: [],
+      interrupted: false,
    };

    const workspaceIdsToProcess =
@@ -73,6 +81,17 @@ export class WorkspaceIteratorService {
    }

    for (const [index, workspaceId] of workspaceIdsToProcess.entries()) {
+      if (this.commandShutdownService.isShutdownRequested()) {
+        this.logger.warn(
+          `Shutdown requested, stopping before workspace ${workspaceId}. ` +
+            `${workspaceIdsToProcess.length - index} workspace(s) left untouched.`,
+        );
+
+        report.interrupted = true;
+
+        break;
+      }
+
      this.logger.log(
        `Running on workspace ${workspaceId} ${index + 1}/${workspaceIdsToProcess.length}`,
      );
@@ -108,8 +108,10 @@ export abstract class WorkspaceCommandRunner<
      });
    }

+    this.workspaceIteratorService.listenToShutdownSignals();
+
    try {
-      await this.workspaceIteratorService.iterate({
+      const report = await this.workspaceIteratorService.iterate({
        workspaceIds:
          options.workspaceId && options.workspaceId.size > 0
            ? Array.from(options.workspaceId)
@@ -129,6 +131,16 @@ export abstract class WorkspaceCommandRunner<
        },
      });

+      if (report.interrupted) {
+        this.logger.warn(
+          chalk.yellow(
+            'Command interrupted before processing every workspace. Rerun it to process the remaining ones.',
+          ),
+        );
+
+        return;
+      }
+
      this.logger.log(chalk.blue('Command completed!'));
    } catch (error) {
      this.logger.error(chalk.red(`Command failed`));
@@ -1,11 +1,12 @@
 import { Module } from '@nestjs/common';

+import { CommandShutdownModule } from 'src/database/commands/command-runners/command-shutdown.module';
 import { WorkspaceIteratorModule } from 'src/database/commands/command-runners/workspace-iterator.module';
 import { UpgradeCommand } from 'src/database/commands/upgrade-version-command/upgrade.command';
 import { UpgradeModule } from 'src/engine/core-modules/upgrade/upgrade.module';

@Module({
-  imports: [UpgradeModule, WorkspaceIteratorModule],
+  imports: [CommandShutdownModule, UpgradeModule, WorkspaceIteratorModule],
  providers: [UpgradeCommand],
 })
 export class UpgradeVersionCommandModule {}
@@ -1,6 +1,7 @@
 import { Command, CommandRunner, Option } from 'nest-commander';
 import { isDefined } from 'twenty-shared/utils';

+import { CommandShutdownService } from 'src/database/commands/command-runners/command-shutdown.service';
 import { CommandLogger } from 'src/database/commands/logger';
 import { UpgradeSequenceReaderService } from 'src/engine/core-modules/upgrade/services/upgrade-sequence-reader.service';
 import { UpgradeSequenceRunnerService } from 'src/engine/core-modules/upgrade/services/upgrade-sequence-runner.service';
@@ -34,6 +35,7 @@ export class UpgradeCommand extends CommandRunner {
    protected readonly upgradeSequenceReaderService: UpgradeSequenceReaderService,
    protected readonly upgradeSequenceRunnerService: UpgradeSequenceRunnerService,
    protected readonly upgradeStatusService: UpgradeStatusService,
+    protected readonly commandShutdownService: CommandShutdownService,
  ) {
    super();
    this.logger = new CommandLogger({
@@ -124,6 +126,8 @@ export class UpgradeCommand extends CommandRunner {
      );
    }

+    this.commandShutdownService.listenToShutdownSignals();
+
    try {
      const sequence = this.upgradeSequenceReaderService.getUpgradeSequence();