Add workspace and server level stop commands for applications (#23183)

## Context

When an installed application misbehaves (e.g. a logic function loop
DDoSing the server or the database), we currently have no targeted way
to shut it down in production: the only kill switch is
`LOGIC_FUNCTION_TYPE=DISABLED`, which disables logic functions for the
whole instance. This PR adds an emergency stop mechanism at two levels:

- **Workspace level**: stop one installed application in its workspace.
- **Server level**: stop every application installed from an
`applicationRegistration`, across all workspaces.

## How it works

**New nullable `stoppedAt` columns** on `core.application` and
`core.applicationRegistration` (fast instance command `2.24.0`, with
`up`/`down` and `@WasIntroducedInUpgrade` decorators on the entities).

**Enforcement in a single choke point**:
`LogicFunctionExecutorService.execute()` is the funnel behind every
execution path (public route triggers, server route triggers, cron
triggers, database event triggers, workflow actions, agent tool calls,
manual GraphQL execution, install hooks). A new
`assertApplicationNotStopped` guard runs right after the flat entities
are resolved and throws `LOGIC_FUNCTION_DISABLED` (already mapped to a
403 on route triggers and handled by the GraphQL exception handler)
when:
- `flatApplication.stoppedAt` is set (workspace-level stop, read from
the cached flat application maps: zero extra runtime cost), or
- the linked registration is stopped (one indexed PK lookup, same
pattern as the existing per-execution server-variable query).

**Propagation**: the workspace-level stop invalidates and recomputes
`flatApplicationMaps` for the workspace, so all server instances pick
the flag up within the local cache TTL (100ms). The registration-level
flag is read live, so it is effective immediately.

## Ops commands

```bash
# Workspace level
yarn command:prod application:stop -a <application-id>
yarn command:prod application:start -a <application-id>

# Server level (all applications of the registration, all workspaces)
yarn command:prod application-registration:stop -r <application-registration-id>
yarn command:prod application-registration:start -r <application-registration-id>
```

Each command logs what was stopped/started and, for registrations, how
many installed applications are affected.

## Notes

- Stopped executions fail fast at the guard, so queued trigger jobs
(cron/db-event) burn a negligible amount of work while stopped.
- The two flags are independent: lifting a registration-level stop does
not clear workspace-level stops that were set individually, and vice
versa.
- Unit tests added for `ApplicationStopService`.


---
_Generated by [Claude
Code](https://claude.ai/code/session_01CjEnKUACn89aSgK1wEMH2d)_

<!-- This is an auto-generated description by cubic. -->
<a
href="https://cubic.dev/pr/twentyhq/twenty/pull/23183?utm_source=github"
target="_blank" rel="noopener noreferrer"
data-no-image-dialog="true"><picture><source
media="(prefers-color-scheme: dark)"
srcset="https://www.cubic.dev/buttons/review-in-cubic-dark.svg"><source
media="(prefers-color-scheme: light)"
srcset="https://www.cubic.dev/buttons/review-in-cubic-light.svg"><img
alt="Review in cubic"
src="https://www.cubic.dev/buttons/review-in-cubic-dark.svg"></picture></a>
<!-- End of auto-generated description by cubic. -->
This commit is contained in:
martmull
2026-07-22 20:30:56 +02:00
committed by GitHub
parent a65da48591
commit a3f4acadb1
21 changed files with 799 additions and 8 deletions
@@ -12,6 +12,10 @@ import { ListOrphanedWorkspaceEntitiesCommand } from 'src/database/commands/list
import { ConfirmationQuestion } from 'src/database/commands/questions/confirmation.question';
import { RebuildApplicationDefaultDepsCommand } from 'src/database/commands/rebuild-application-default-deps.command';
import { RunInstanceCommandsCommand } from 'src/database/commands/run-instance-commands.command';
import { StartApplicationRegistrationCommand } from 'src/database/commands/start-application-registration.command';
import { StartApplicationCommand } from 'src/database/commands/start-application.command';
import { StopApplicationRegistrationCommand } from 'src/database/commands/stop-application-registration.command';
import { StopApplicationCommand } from 'src/database/commands/stop-application.command';
import { UpgradeVersionCommandModule } from 'src/database/commands/upgrade-version-command/upgrade-version-command.module';
import { WorkspaceExportModule } from 'src/database/commands/workspace-export/workspace-export.module';
import { TypeORMModule } from 'src/database/typeorm/typeorm.module';
@@ -110,6 +114,10 @@ import { AutomatedTriggerModule } from 'src/modules/workflow/workflow-trigger/au
UpgradeStatusCommand,
RebuildApplicationDefaultDepsCommand,
InstallPreInstalledAppsCommand,
StopApplicationCommand,
StartApplicationCommand,
StopApplicationRegistrationCommand,
StartApplicationRegistrationCommand,
provideWorkspaceScopedRepository(RoleEntity),
],
})
@@ -0,0 +1,46 @@
import { Logger } from '@nestjs/common';
import { Command, CommandRunner, Option } from 'nest-commander';
import { ApplicationStopService } from 'src/engine/core-modules/application/application-stop.service';
@Command({
name: 'application-registration:start',
description:
'Lift the server-level kill switch set by application-registration:stop and resume logic function executions of all applications installed from the registration.',
})
export class StartApplicationRegistrationCommand extends CommandRunner {
private readonly logger = new Logger(
StartApplicationRegistrationCommand.name,
);
constructor(private readonly applicationStopService: ApplicationStopService) {
super();
}
@Option({
flags:
'-r, --application-registration-universal-identifier <universal_identifier>',
description:
'universal identifier of the application registration to start',
required: true,
})
parseApplicationRegistrationUniversalIdentifier(value: string): string {
return value;
}
override async run(
_passedParams: string[],
options: { applicationRegistrationUniversalIdentifier: string },
): Promise<void> {
const { applicationRegistration, installedApplicationCount } =
await this.applicationStopService.startApplicationRegistration({
applicationRegistrationUniversalIdentifier:
options.applicationRegistrationUniversalIdentifier,
});
this.logger.log(
`Started application registration "${applicationRegistration.name}" (universalIdentifier ${applicationRegistration.universalIdentifier}): the server-level stop is lifted for its ${installedApplicationCount} installed application(s). Workspace-level stops set with application:stop remain in effect.`,
);
}
}
@@ -0,0 +1,67 @@
import { Logger } from '@nestjs/common';
import { Command, CommandRunner, Option } from 'nest-commander';
import { isDefined } from 'twenty-shared/utils';
import { ApplicationStopService } from 'src/engine/core-modules/application/application-stop.service';
type StartApplicationCommandOptions = {
workspaceId: string;
applicationUniversalIdentifier: string;
};
@Command({
name: 'application:start',
description:
'Lift the workspace-level kill switch set by application:stop and resume logic function executions of the application.',
})
export class StartApplicationCommand extends CommandRunner {
private readonly logger = new Logger(StartApplicationCommand.name);
constructor(private readonly applicationStopService: ApplicationStopService) {
super();
}
@Option({
flags: '-w, --workspace-id <workspace_id>',
description: 'id of the workspace the application is installed in',
required: true,
})
parseWorkspaceId(value: string): string {
return value;
}
@Option({
flags: '-a, --application-universal-identifier <universal_identifier>',
description: 'universal identifier of the application to start',
required: true,
})
parseApplicationUniversalIdentifier(value: string): string {
return value;
}
override async run(
_passedParams: string[],
options: StartApplicationCommandOptions,
): Promise<void> {
const application = await this.applicationStopService.startApplication({
workspaceId: options.workspaceId,
applicationUniversalIdentifier: options.applicationUniversalIdentifier,
});
this.logger.log(
`Started application "${application.name}" (universalIdentifier ${application.universalIdentifier}) in workspace ${application.workspaceId}: the workspace-level stop is lifted.`,
);
if (
isDefined(application.applicationRegistrationId) &&
(await this.applicationStopService.isApplicationRegistrationStopped(
application.applicationRegistrationId,
))
) {
this.logger.warn(
`Application registration ${application.applicationRegistrationId} is still stopped server-wide: executions of this application remain blocked until application-registration:start is run.`,
);
}
}
}
@@ -0,0 +1,43 @@
import { Logger } from '@nestjs/common';
import { Command, CommandRunner, Option } from 'nest-commander';
import { ApplicationStopService } from 'src/engine/core-modules/application/application-stop.service';
@Command({
name: 'application-registration:stop',
description:
'Server-level kill switch: block all logic function executions of every application installed from a registration, across all workspaces. Reverse with application-registration:start.',
})
export class StopApplicationRegistrationCommand extends CommandRunner {
private readonly logger = new Logger(StopApplicationRegistrationCommand.name);
constructor(private readonly applicationStopService: ApplicationStopService) {
super();
}
@Option({
flags:
'-r, --application-registration-universal-identifier <universal_identifier>',
description: 'universal identifier of the application registration to stop',
required: true,
})
parseApplicationRegistrationUniversalIdentifier(value: string): string {
return value;
}
override async run(
_passedParams: string[],
options: { applicationRegistrationUniversalIdentifier: string },
): Promise<void> {
const { applicationRegistration, installedApplicationCount } =
await this.applicationStopService.stopApplicationRegistration({
applicationRegistrationUniversalIdentifier:
options.applicationRegistrationUniversalIdentifier,
});
this.logger.log(
`Stopped application registration "${applicationRegistration.name}" (universalIdentifier ${applicationRegistration.universalIdentifier}). All logic function executions of its ${installedApplicationCount} installed application(s) are now blocked, across all workspaces.`,
);
}
}
@@ -0,0 +1,55 @@
import { Logger } from '@nestjs/common';
import { Command, CommandRunner, Option } from 'nest-commander';
import { ApplicationStopService } from 'src/engine/core-modules/application/application-stop.service';
type StopApplicationCommandOptions = {
workspaceId: string;
applicationUniversalIdentifier: string;
};
@Command({
name: 'application:stop',
description:
'Workspace-level kill switch: block all logic function executions of one installed application. Reverse with application:start.',
})
export class StopApplicationCommand extends CommandRunner {
private readonly logger = new Logger(StopApplicationCommand.name);
constructor(private readonly applicationStopService: ApplicationStopService) {
super();
}
@Option({
flags: '-w, --workspace-id <workspace_id>',
description: 'id of the workspace the application is installed in',
required: true,
})
parseWorkspaceId(value: string): string {
return value;
}
@Option({
flags: '-a, --application-universal-identifier <universal_identifier>',
description: 'universal identifier of the application to stop',
required: true,
})
parseApplicationUniversalIdentifier(value: string): string {
return value;
}
override async run(
_passedParams: string[],
options: StopApplicationCommandOptions,
): Promise<void> {
const application = await this.applicationStopService.stopApplication({
workspaceId: options.workspaceId,
applicationUniversalIdentifier: options.applicationUniversalIdentifier,
});
this.logger.log(
`Stopped application "${application.name}" (universalIdentifier ${application.universalIdentifier}) in workspace ${application.workspaceId}. All its logic function executions are now blocked.`,
);
}
}
@@ -0,0 +1,27 @@
import { QueryRunner } from 'typeorm';
import { RegisteredInstanceCommand } from 'src/engine/core-modules/upgrade/decorators/registered-instance-command.decorator';
import { FastInstanceCommand } from 'src/engine/core-modules/upgrade/interfaces/fast-instance-command.interface';
@RegisteredInstanceCommand('2.24.0', 1784734278506)
export class AddStoppedAtToApplicationAndApplicationRegistrationFastInstanceCommand
implements FastInstanceCommand
{
public async up(queryRunner: QueryRunner): Promise<void> {
await queryRunner.query(
'ALTER TABLE "core"."application" ADD COLUMN IF NOT EXISTS "stoppedAt" TIMESTAMP WITH TIME ZONE',
);
await queryRunner.query(
'ALTER TABLE "core"."applicationRegistration" ADD COLUMN IF NOT EXISTS "stoppedAt" TIMESTAMP WITH TIME ZONE',
);
}
public async down(queryRunner: QueryRunner): Promise<void> {
await queryRunner.query(
'ALTER TABLE "core"."applicationRegistration" DROP COLUMN IF EXISTS "stoppedAt"',
);
await queryRunner.query(
'ALTER TABLE "core"."application" DROP COLUMN IF EXISTS "stoppedAt"',
);
}
}
@@ -53,6 +53,7 @@ import { AllowServerScopedFileFastInstanceCommand } from 'src/database/commands/
import { AddCalendarEndFieldMetadataIdToViewFastInstanceCommand } from 'src/database/commands/upgrade-version-command/2-22/2-22-instance-command-fast-1783956795000-add-calendar-end-field-metadata-id-to-view';
import { AddSdkClientCoreChecksumToApplicationFastInstanceCommand } from 'src/database/commands/upgrade-version-command/2-23/2-23-instance-command-fast-1784625638000-add-sdk-client-core-checksum-to-application';
import { AddAutoUpgradeToApplicationFastInstanceCommand } from 'src/database/commands/upgrade-version-command/2-23/2-23-instance-command-fast-1784297307235-add-auto-upgrade-to-application';
import { AddStoppedAtToApplicationAndApplicationRegistrationFastInstanceCommand } from 'src/database/commands/upgrade-version-command/2-24/2-24-instance-command-fast-1784734278506-add-stopped-at-to-application-and-application-registration';
import { AddSubFieldNameToViewSortEarlyFastInstanceCommand } from 'src/database/commands/upgrade-version-command/2-3/2-3-instance-command-fast-1747234200000-add-sub-field-name-to-view-sort';
import { AddRelationTargetFieldMetadataIdToViewFilterEarlyFastInstanceCommand } from 'src/database/commands/upgrade-version-command/2-3/2-3-instance-command-fast-1747234300000-add-relation-target-field-metadata-id-to-view-filter';
import { AddUpgradeMigrationWorkspaceIdIndexFastInstanceCommand } from 'src/database/commands/upgrade-version-command/2-3/2-3-instance-command-fast-1777308014234-add-upgrade-migration-workspace-id-index';
@@ -244,4 +245,5 @@ export const INSTANCE_COMMANDS = [
AddSdkClientCoreChecksumToApplicationFastInstanceCommand,
AddStatusesToBillingSubscriptionIndexSlowInstanceCommand,
AddOnConnectLogicFunctionToConnectionProviderFastInstanceCommand,
AddStoppedAtToApplicationAndApplicationRegistrationFastInstanceCommand,
];
@@ -0,0 +1,243 @@
import { Test, type TestingModule } from '@nestjs/testing';
import { getRepositoryToken } from '@nestjs/typeorm';
import { ApplicationRegistrationEntity } from 'src/engine/core-modules/application/application-registration/application-registration.entity';
import { ApplicationRegistrationException } from 'src/engine/core-modules/application/application-registration/application-registration.exception';
import { ApplicationStopService } from 'src/engine/core-modules/application/application-stop.service';
import { ApplicationEntity } from 'src/engine/core-modules/application/application.entity';
import { ApplicationException } from 'src/engine/core-modules/application/application.exception';
import { WorkspaceCacheService } from 'src/engine/workspace-cache/services/workspace-cache.service';
const APPLICATION_ID = 'application-1';
const APPLICATION_UNIVERSAL_IDENTIFIER = 'application-universal-identifier-1';
const APPLICATION_REGISTRATION_ID = 'application-registration-1';
const APPLICATION_REGISTRATION_UNIVERSAL_IDENTIFIER =
'application-registration-universal-identifier-1';
const WORKSPACE_ID = 'workspace-1';
describe('ApplicationStopService', () => {
let applicationStopService: ApplicationStopService;
const applicationRepository = {
findOne: jest.fn(),
update: jest.fn(),
count: jest.fn(),
};
const applicationRegistrationRepository = {
findOne: jest.fn(),
update: jest.fn(),
count: jest.fn(),
};
const workspaceCacheService = {
invalidateAndRecompute: jest.fn(),
};
beforeEach(async () => {
jest.clearAllMocks();
const module: TestingModule = await Test.createTestingModule({
providers: [
ApplicationStopService,
{
provide: getRepositoryToken(ApplicationEntity),
useValue: applicationRepository,
},
{
provide: getRepositoryToken(ApplicationRegistrationEntity),
useValue: applicationRegistrationRepository,
},
{
provide: WorkspaceCacheService,
useValue: workspaceCacheService,
},
],
}).compile();
applicationStopService = module.get<ApplicationStopService>(
ApplicationStopService,
);
});
describe('stopApplication', () => {
it('should set stoppedAt and invalidate the workspace application cache', async () => {
applicationRepository.findOne.mockResolvedValue({
id: APPLICATION_ID,
universalIdentifier: APPLICATION_UNIVERSAL_IDENTIFIER,
workspaceId: WORKSPACE_ID,
stoppedAt: null,
});
const application = await applicationStopService.stopApplication({
workspaceId: WORKSPACE_ID,
applicationUniversalIdentifier: APPLICATION_UNIVERSAL_IDENTIFIER,
});
expect(applicationRepository.update).toHaveBeenCalledWith(
APPLICATION_ID,
{ stoppedAt: expect.any(Date) },
);
expect(workspaceCacheService.invalidateAndRecompute).toHaveBeenCalledWith(
WORKSPACE_ID,
['flatApplicationMaps'],
);
expect(application.stoppedAt).toBeInstanceOf(Date);
});
it('should throw when the application does not exist', async () => {
applicationRepository.findOne.mockResolvedValue(null);
await expect(
applicationStopService.stopApplication({
workspaceId: WORKSPACE_ID,
applicationUniversalIdentifier: APPLICATION_UNIVERSAL_IDENTIFIER,
}),
).rejects.toThrow(ApplicationException);
expect(applicationRepository.update).not.toHaveBeenCalled();
expect(
workspaceCacheService.invalidateAndRecompute,
).not.toHaveBeenCalled();
});
});
describe('startApplication', () => {
it('should clear stoppedAt and invalidate the workspace application cache', async () => {
applicationRepository.findOne.mockResolvedValue({
id: APPLICATION_ID,
universalIdentifier: APPLICATION_UNIVERSAL_IDENTIFIER,
workspaceId: WORKSPACE_ID,
stoppedAt: new Date(),
});
const application = await applicationStopService.startApplication({
workspaceId: WORKSPACE_ID,
applicationUniversalIdentifier: APPLICATION_UNIVERSAL_IDENTIFIER,
});
expect(applicationRepository.update).toHaveBeenCalledWith(
APPLICATION_ID,
{ stoppedAt: null },
);
expect(workspaceCacheService.invalidateAndRecompute).toHaveBeenCalledWith(
WORKSPACE_ID,
['flatApplicationMaps'],
);
expect(application.stoppedAt).toBeNull();
});
it('should throw when the application does not exist', async () => {
applicationRepository.findOne.mockResolvedValue(null);
await expect(
applicationStopService.startApplication({
workspaceId: WORKSPACE_ID,
applicationUniversalIdentifier: APPLICATION_UNIVERSAL_IDENTIFIER,
}),
).rejects.toThrow(ApplicationException);
expect(applicationRepository.update).not.toHaveBeenCalled();
expect(
workspaceCacheService.invalidateAndRecompute,
).not.toHaveBeenCalled();
});
});
describe('stopApplicationRegistration', () => {
it('should set stoppedAt on the registration and report the installed application count', async () => {
applicationRegistrationRepository.findOne.mockResolvedValue({
id: APPLICATION_REGISTRATION_ID,
universalIdentifier: APPLICATION_REGISTRATION_UNIVERSAL_IDENTIFIER,
name: 'My App',
stoppedAt: null,
});
applicationRepository.count.mockResolvedValue(3);
const { applicationRegistration, installedApplicationCount } =
await applicationStopService.stopApplicationRegistration({
applicationRegistrationUniversalIdentifier:
APPLICATION_REGISTRATION_UNIVERSAL_IDENTIFIER,
});
expect(applicationRegistrationRepository.update).toHaveBeenCalledWith(
APPLICATION_REGISTRATION_ID,
{ stoppedAt: expect.any(Date) },
);
expect(applicationRegistration.stoppedAt).toBeInstanceOf(Date);
expect(installedApplicationCount).toBe(3);
});
it('should throw when the registration does not exist', async () => {
applicationRegistrationRepository.findOne.mockResolvedValue(null);
await expect(
applicationStopService.stopApplicationRegistration({
applicationRegistrationUniversalIdentifier:
APPLICATION_REGISTRATION_UNIVERSAL_IDENTIFIER,
}),
).rejects.toThrow(ApplicationRegistrationException);
expect(applicationRegistrationRepository.update).not.toHaveBeenCalled();
});
});
describe('startApplicationRegistration', () => {
it('should clear stoppedAt on the registration', async () => {
applicationRegistrationRepository.findOne.mockResolvedValue({
id: APPLICATION_REGISTRATION_ID,
universalIdentifier: APPLICATION_REGISTRATION_UNIVERSAL_IDENTIFIER,
name: 'My App',
stoppedAt: new Date(),
});
applicationRepository.count.mockResolvedValue(1);
const { applicationRegistration } =
await applicationStopService.startApplicationRegistration({
applicationRegistrationUniversalIdentifier:
APPLICATION_REGISTRATION_UNIVERSAL_IDENTIFIER,
});
expect(applicationRegistrationRepository.update).toHaveBeenCalledWith(
APPLICATION_REGISTRATION_ID,
{ stoppedAt: null },
);
expect(applicationRegistration.stoppedAt).toBeNull();
});
it('should throw when the registration does not exist', async () => {
applicationRegistrationRepository.findOne.mockResolvedValue(null);
await expect(
applicationStopService.startApplicationRegistration({
applicationRegistrationUniversalIdentifier:
APPLICATION_REGISTRATION_UNIVERSAL_IDENTIFIER,
}),
).rejects.toThrow(ApplicationRegistrationException);
expect(applicationRegistrationRepository.update).not.toHaveBeenCalled();
});
});
describe('isApplicationRegistrationStopped', () => {
it('should return true when a stopped registration matches', async () => {
applicationRegistrationRepository.count.mockResolvedValue(1);
await expect(
applicationStopService.isApplicationRegistrationStopped(
APPLICATION_REGISTRATION_ID,
),
).resolves.toBe(true);
});
it('should return false when the registration is not stopped', async () => {
applicationRegistrationRepository.count.mockResolvedValue(0);
await expect(
applicationStopService.isApplicationRegistrationStopped(
APPLICATION_REGISTRATION_ID,
),
).resolves.toBe(false);
});
});
});
@@ -162,6 +162,7 @@ export class ApplicationSyncService {
settingsCustomTabFrontComponentId: null,
canBeUninstalled: true,
autoUpgrade: false,
stoppedAt: null,
isSdkLayerStale: false,
sdkClientCoreChecksum: null,
applicationRegistrationId: null,
@@ -138,6 +138,16 @@ export class ApplicationRegistrationEntity {
@Column({ type: 'boolean', default: false })
isPreInstalled: boolean;
// Emergency kill switch: while set, every logic function execution of every
// application installed from this registration is blocked, across all
// workspaces (see `application-registration:stop` command).
@Column({ nullable: true, type: 'timestamptz' })
@WasIntroducedInUpgrade({
upgradeCommandName:
'2.24.0_AddStoppedAtToApplicationAndApplicationRegistrationFastInstanceCommand_1784734278506',
})
stoppedAt: Date | null;
@Column({ type: 'jsonb', nullable: true })
manifest: Manifest | null;
@@ -0,0 +1,175 @@
import { Injectable } from '@nestjs/common';
import { InjectRepository } from '@nestjs/typeorm';
import { isDefined } from 'twenty-shared/utils';
import { IsNull, Not, Repository } from 'typeorm';
import { ApplicationRegistrationEntity } from 'src/engine/core-modules/application/application-registration/application-registration.entity';
import {
ApplicationRegistrationException,
ApplicationRegistrationExceptionCode,
} from 'src/engine/core-modules/application/application-registration/application-registration.exception';
import { ApplicationEntity } from 'src/engine/core-modules/application/application.entity';
import {
ApplicationException,
ApplicationExceptionCode,
} from 'src/engine/core-modules/application/application.exception';
import { WorkspaceCacheService } from 'src/engine/workspace-cache/services/workspace-cache.service';
// Emergency kill switch for applications DDoSing the server or the database.
// Workspace level: stops one installed application. Server level: stops every
// application installed from an application registration, in all workspaces.
@Injectable()
export class ApplicationStopService {
constructor(
@InjectRepository(ApplicationEntity)
private readonly applicationRepository: Repository<ApplicationEntity>,
@InjectRepository(ApplicationRegistrationEntity)
private readonly applicationRegistrationRepository: Repository<ApplicationRegistrationEntity>,
private readonly workspaceCacheService: WorkspaceCacheService,
) {}
async stopApplication({
workspaceId,
applicationUniversalIdentifier,
}: {
workspaceId: string;
applicationUniversalIdentifier: string;
}): Promise<ApplicationEntity> {
return this.setApplicationStoppedAt({
workspaceId,
applicationUniversalIdentifier,
stoppedAt: new Date(),
});
}
async startApplication({
workspaceId,
applicationUniversalIdentifier,
}: {
workspaceId: string;
applicationUniversalIdentifier: string;
}): Promise<ApplicationEntity> {
return this.setApplicationStoppedAt({
workspaceId,
applicationUniversalIdentifier,
stoppedAt: null,
});
}
async stopApplicationRegistration({
applicationRegistrationUniversalIdentifier,
}: {
applicationRegistrationUniversalIdentifier: string;
}): Promise<{
applicationRegistration: ApplicationRegistrationEntity;
installedApplicationCount: number;
}> {
return this.setApplicationRegistrationStoppedAt({
applicationRegistrationUniversalIdentifier,
stoppedAt: new Date(),
});
}
async startApplicationRegistration({
applicationRegistrationUniversalIdentifier,
}: {
applicationRegistrationUniversalIdentifier: string;
}): Promise<{
applicationRegistration: ApplicationRegistrationEntity;
installedApplicationCount: number;
}> {
return this.setApplicationRegistrationStoppedAt({
applicationRegistrationUniversalIdentifier,
stoppedAt: null,
});
}
async isApplicationRegistrationStopped(
applicationRegistrationId: string,
): Promise<boolean> {
const stoppedApplicationRegistrationCount =
await this.applicationRegistrationRepository.count({
where: {
id: applicationRegistrationId,
stoppedAt: Not(IsNull()),
},
});
return stoppedApplicationRegistrationCount > 0;
}
private async setApplicationStoppedAt({
workspaceId,
applicationUniversalIdentifier,
stoppedAt,
}: {
workspaceId: string;
applicationUniversalIdentifier: string;
stoppedAt: Date | null;
}): Promise<ApplicationEntity> {
const application = await this.applicationRepository.findOne({
where: {
workspaceId,
universalIdentifier: applicationUniversalIdentifier,
},
});
if (!isDefined(application)) {
throw new ApplicationException(
`Application ${applicationUniversalIdentifier} not found in workspace ${workspaceId}`,
ApplicationExceptionCode.APPLICATION_NOT_FOUND,
);
}
await this.applicationRepository.update(application.id, { stoppedAt });
// The executor reads applications from the workspace cache, so the flag
// only takes effect once the cached flat application maps are rebuilt.
await this.workspaceCacheService.invalidateAndRecompute(
application.workspaceId,
['flatApplicationMaps'],
);
return { ...application, stoppedAt };
}
private async setApplicationRegistrationStoppedAt({
applicationRegistrationUniversalIdentifier,
stoppedAt,
}: {
applicationRegistrationUniversalIdentifier: string;
stoppedAt: Date | null;
}): Promise<{
applicationRegistration: ApplicationRegistrationEntity;
installedApplicationCount: number;
}> {
const applicationRegistration =
await this.applicationRegistrationRepository.findOne({
where: {
universalIdentifier: applicationRegistrationUniversalIdentifier,
},
});
if (!isDefined(applicationRegistration)) {
throw new ApplicationRegistrationException(
`Application registration ${applicationRegistrationUniversalIdentifier} not found`,
ApplicationRegistrationExceptionCode.APPLICATION_REGISTRATION_NOT_FOUND,
);
}
await this.applicationRegistrationRepository.update(
applicationRegistration.id,
{ stoppedAt },
);
const installedApplicationCount = await this.applicationRepository.count({
where: { applicationRegistrationId: applicationRegistration.id },
});
return {
applicationRegistration: { ...applicationRegistration, stoppedAt },
installedApplicationCount,
};
}
}
@@ -131,6 +131,15 @@ export class ApplicationEntity extends WorkspaceRelatedEntity {
})
autoUpgrade: boolean;
// Emergency kill switch: while set, every logic function execution of this
// application is blocked in this workspace (see `application:stop` command).
@Column({ nullable: true, type: 'timestamptz' })
@WasIntroducedInUpgrade({
upgradeCommandName:
'2.24.0_AddStoppedAtToApplicationAndApplicationRegistrationFastInstanceCommand_1784734278506',
})
stoppedAt: Date | null;
@Column({ nullable: false, type: 'boolean', default: false })
isSdkLayerStale: boolean;
@@ -3,6 +3,7 @@ import { TypeOrmModule } from '@nestjs/typeorm';
import { ApplicationRegistrationEntity } from 'src/engine/core-modules/application/application-registration/application-registration.entity';
import { ApplicationGaugeService } from 'src/engine/core-modules/application/application-gauge.service';
import { ApplicationStopService } from 'src/engine/core-modules/application/application-stop.service';
import { ApplicationEntity } from 'src/engine/core-modules/application/application.entity';
import { ApplicationResolver } from 'src/engine/core-modules/application/application.resolver';
import { ApplicationService } from 'src/engine/core-modules/application/application.service';
@@ -40,10 +41,15 @@ import { WorkspaceCacheModule } from 'src/engine/workspace-cache/workspace-cache
FeatureFlagModule,
MetricsModule,
],
exports: [ApplicationService, WorkspaceFlatApplicationMapCacheService],
exports: [
ApplicationService,
ApplicationStopService,
WorkspaceFlatApplicationMapCacheService,
],
providers: [
ApplicationResolver,
ApplicationService,
ApplicationStopService,
ApplicationGaugeService,
WorkspaceFlatApplicationMapCacheService,
provideWorkspaceScopedRepository(AgentEntity),
@@ -21,6 +21,7 @@ import {
import { buildApplicationLogEnvelopes } from 'src/engine/core-modules/event-logs/producers/application-log/build-application-log-envelopes';
import { parseApplicationLogLines } from 'src/engine/core-modules/event-logs/producers/application-log/parse-application-log-lines';
import { ApplicationRegistrationVariableEntity } from 'src/engine/core-modules/application/application-registration-variable/application-registration-variable.entity';
import { ApplicationStopService } from 'src/engine/core-modules/application/application-stop.service';
import { ApplicationService } from 'src/engine/core-modules/application/application.service';
import type { FlatApplicationVariable } from 'src/engine/metadata-modules/flat-application-variable/types/flat-application-variable.type';
import { FlatApplication } from 'src/engine/core-modules/application/types/flat-application.type';
@@ -92,6 +93,7 @@ export class LogicFunctionExecutorService {
private readonly featureFlagService: FeatureFlagService,
private readonly workspaceDomainsService: WorkspaceDomainsService,
private readonly applicationService: ApplicationService,
private readonly applicationStopService: ApplicationStopService,
@InjectRepository(WorkspaceEntity)
private readonly workspaceRepository: Repository<WorkspaceEntity>,
@InjectRepository(ApplicationRegistrationVariableEntity)
@@ -113,14 +115,18 @@ export class LogicFunctionExecutorService {
userWorkspaceId?: string;
executionMode?: LogicFunctionExecutionMode;
}): Promise<LogicFunctionExecuteResult> {
await this.throttleExecution(workspaceId);
const { flatApplication, flatLogicFunction, flatApplicationVariables } =
await this.getFlatEntitiesOrThrow({
workspaceId,
logicFunctionId,
});
// Checked before the shared workspace throttle so a flood from a stopped
// application cannot exhaust the token bucket of the other applications.
await this.assertApplicationNotStopped(flatApplication);
await this.throttleExecution(workspaceId);
const envVariables = await this.getExecutionEnvVariables({
workspaceId,
flatApplication,
@@ -233,6 +239,32 @@ export class LogicFunctionExecutorService {
return driver.transpile(params);
}
// Emergency kill switch checked on every execution: an application can be
// stopped for one workspace (application.stoppedAt) or server-wide for all
// workspaces (applicationRegistration.stoppedAt) when it degrades production.
private async assertApplicationNotStopped(
flatApplication: FlatApplication,
): Promise<void> {
if (isDefined(flatApplication.stoppedAt)) {
throw new LogicFunctionException(
`Application ${flatApplication.id} is stopped in workspace ${flatApplication.workspaceId}`,
LogicFunctionExceptionCode.LOGIC_FUNCTION_DISABLED,
);
}
if (
isDefined(flatApplication.applicationRegistrationId) &&
(await this.applicationStopService.isApplicationRegistrationStopped(
flatApplication.applicationRegistrationId,
))
) {
throw new LogicFunctionException(
`Application registration ${flatApplication.applicationRegistrationId} is stopped server-wide`,
LogicFunctionExceptionCode.LOGIC_FUNCTION_DISABLED,
);
}
}
private async throttleExecution(workspaceId: string) {
try {
await this.throttlerService.tokenBucketThrottleOrThrow(
@@ -1,9 +1,13 @@
import { Scope } from '@nestjs/common';
import { Logger, Scope } from '@nestjs/common';
import { Process } from 'src/engine/core-modules/message-queue/decorators/process.decorator';
import { Processor } from 'src/engine/core-modules/message-queue/decorators/processor.decorator';
import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';
import { LogicFunctionExecutorService } from 'src/engine/core-modules/logic-function/logic-function-executor/logic-function-executor.service';
import {
LogicFunctionException,
LogicFunctionExceptionCode,
} from 'src/engine/metadata-modules/logic-function/logic-function.exception';
export type LogicFunctionTriggerJobData = {
logicFunctionId: string;
@@ -18,6 +22,8 @@ export type LogicFunctionTriggerJobData = {
scope: Scope.REQUEST,
})
export class LogicFunctionTriggerJob {
private readonly logger = new Logger(LogicFunctionTriggerJob.name);
constructor(
private readonly logicFunctionExecutorService: LogicFunctionExecutorService,
) {}
@@ -25,16 +31,32 @@ export class LogicFunctionTriggerJob {
@Process(LogicFunctionTriggerJob.name)
async handle(logicFunctionPayloads: LogicFunctionTriggerJobData[]) {
await Promise.all(
logicFunctionPayloads.map(
async (logicFunctionPayload) =>
logicFunctionPayloads.map(async (logicFunctionPayload) => {
try {
await this.logicFunctionExecutorService.execute({
logicFunctionId: logicFunctionPayload.logicFunctionId,
workspaceId: logicFunctionPayload.workspaceId,
payload: logicFunctionPayload.payload ?? {},
userId: logicFunctionPayload.userId,
userWorkspaceId: logicFunctionPayload.userWorkspaceId,
}),
),
});
} catch (error) {
// A stopped application must not fail the job: failing would make
// the queue retry an execution that is intentionally blocked.
if (
error instanceof LogicFunctionException &&
error.code === LogicFunctionExceptionCode.LOGIC_FUNCTION_DISABLED
) {
this.logger.warn(
`Skipping execution of logic function ${logicFunctionPayload.logicFunctionId} in workspace ${logicFunctionPayload.workspaceId}: ${error.message}`,
);
return;
}
throw error;
}
}),
);
}
}
@@ -12,6 +12,10 @@ import { type MessageQueueService } from 'src/engine/core-modules/message-queue/
import { ServerRouteTriggerExceptionCode } from 'src/engine/core-modules/server-route-trigger/exceptions/server-route-trigger.exception';
import { ServerRouteTriggerService } from 'src/engine/core-modules/server-route-trigger/server-route-trigger.service';
import { type LogicFunctionEntity } from 'src/engine/metadata-modules/logic-function/logic-function.entity';
import {
LogicFunctionException,
LogicFunctionExceptionCode,
} from 'src/engine/metadata-modules/logic-function/logic-function.exception';
import { LogicFunctionExecutionStatus } from 'src/engine/metadata-modules/logic-function/dtos/logic-function-execution-result.dto';
const RESOLVER_UID = 'resolver-uid';
@@ -302,6 +306,20 @@ describe('ServerRouteTriggerService', () => {
});
});
it('maps a LogicFunctionException(LOGIC_FUNCTION_DISABLED) to the server-route disabled code', async () => {
logicFunctionExecutorService.execute.mockReset();
logicFunctionExecutorService.execute.mockRejectedValue(
new LogicFunctionException(
'application is stopped',
LogicFunctionExceptionCode.LOGIC_FUNCTION_DISABLED,
),
);
await expect(handle()).rejects.toMatchObject({
code: ServerRouteTriggerExceptionCode.LOGIC_FUNCTION_DISABLED,
});
});
it('does not leak the raw resolver executor error message to the caller', async () => {
logicFunctionExecutorService.execute.mockReset();
logicFunctionExecutorService.execute.mockRejectedValue(
@@ -30,6 +30,15 @@ export class ServerRouteTriggerRestApiExceptionFilter implements ExceptionFilter
response,
404,
);
case ServerRouteTriggerExceptionCode.LOGIC_FUNCTION_DISABLED:
return this.httpExceptionHandlerService.handleError(
exception as CustomException,
response,
403,
undefined,
undefined,
{ shouldBeCapturedBySentry: false },
);
case ServerRouteTriggerExceptionCode.RATE_LIMIT_EXCEEDED:
return this.httpExceptionHandlerService.handleError(
exception as CustomException,
@@ -6,6 +6,7 @@ import { CustomException } from 'src/utils/custom-exception';
export enum ServerRouteTriggerExceptionCode {
LOGIC_FUNCTION_NOT_FOUND = 'LOGIC_FUNCTION_NOT_FOUND',
LOGIC_FUNCTION_DISABLED = 'LOGIC_FUNCTION_DISABLED',
RATE_LIMIT_EXCEEDED = 'RATE_LIMIT_EXCEEDED',
SERVER_ROUTE_USER_UNCAUGHT_ERROR = 'SERVER_ROUTE_USER_UNCAUGHT_ERROR',
SERVER_ROUTE_PLATFORM_ERROR = 'SERVER_ROUTE_PLATFORM_ERROR',
@@ -19,6 +20,8 @@ const getServerRouteTriggerExceptionUserFriendlyMessage = (
switch (code) {
case ServerRouteTriggerExceptionCode.LOGIC_FUNCTION_NOT_FOUND:
return msg`Server logic function not found.`;
case ServerRouteTriggerExceptionCode.LOGIC_FUNCTION_DISABLED:
return msg`This action is currently unavailable.`;
case ServerRouteTriggerExceptionCode.RATE_LIMIT_EXCEEDED:
return msg`Rate limit exceeded.`;
case ServerRouteTriggerExceptionCode.SERVER_ROUTE_USER_UNCAUGHT_ERROR:
@@ -25,6 +25,10 @@ import {
ServerRouteTriggerExceptionCode,
} from 'src/engine/core-modules/server-route-trigger/exceptions/server-route-trigger.exception';
import { LogicFunctionEntity } from 'src/engine/metadata-modules/logic-function/logic-function.entity';
import {
LogicFunctionException,
LogicFunctionExceptionCode,
} from 'src/engine/metadata-modules/logic-function/logic-function.exception';
type ResolverResult = {
workspaceId: string;
@@ -274,6 +278,8 @@ export class ServerRouteTriggerService {
return 'Rate limit exceeded';
case ServerRouteTriggerExceptionCode.LOGIC_FUNCTION_NOT_FOUND:
return 'Logic function not found';
case ServerRouteTriggerExceptionCode.LOGIC_FUNCTION_DISABLED:
return 'Logic function execution is disabled';
default:
return 'An unexpected error occurred while handling the server route';
}
@@ -282,6 +288,13 @@ export class ServerRouteTriggerService {
private mapExecutorErrorToServerRouteCode(
error: unknown,
): ServerRouteTriggerExceptionCode {
if (
error instanceof LogicFunctionException &&
error.code === LogicFunctionExceptionCode.LOGIC_FUNCTION_DISABLED
) {
return ServerRouteTriggerExceptionCode.LOGIC_FUNCTION_DISABLED;
}
if (!(error instanceof LogicFunctionExecutionException)) {
return ServerRouteTriggerExceptionCode.SERVER_ROUTE_PLATFORM_ERROR;
}
@@ -42,6 +42,7 @@ const MOCK_FLAT_APPLICATION: FlatApplication = {
settingsCustomTabFrontComponentId: null,
canBeUninstalled: false,
autoUpgrade: false,
stoppedAt: null,
applicationRegistrationId: null,
primaryPublicDomainId: null,
createdAt: new Date(),
@@ -37,6 +37,7 @@ const MOCK_FLAT_APPLICATION: FlatApplication = {
settingsCustomTabFrontComponentId: null,
canBeUninstalled: false,
autoUpgrade: false,
stoppedAt: null,
applicationRegistrationId: null,
primaryPublicDomainId: null,
createdAt: new Date(),