houseassassin/twenty
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Clean soft deleted suspended workspace command and ignore-grace-period flag (#17144)

Browse Source 
# Introduction

In the https://github.com/twentyhq/core-team-issues/issues/1989 's
context we will apply migration through an upgrade command post entity
backfill to fit the applied constraint. But suspended soft deleted
workspace are not included in the upgrade workspace batches

In order to be able to pass the constraint in production, discussed with
@FelixMalfait, we will manually clear all the currently suspended and
soft deleted workspace ignore their grace period

## Force mode
When running the command in force it will ignore the limit per execution
( which really serve the cron job ) and the grace period

## Test
Tested on a production extract
This commit is contained in:
Paul Rastoin
2026-01-14 15:15:15 +01:00
committed by GitHub
parent 2c24180b44
commit 9e4247c934
11 changed files with 204 additions and 95 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/twenty-server/src/engine/core-modules/workspace/services/__tests__/workspace.service.spec.ts
+7 -9
View File
@@ -26,6 +26,7 @@ import { WorkspaceInvitationService } from 'src/engine/core-modules/workspace-in
import { WorkspaceService } from 'src/engine/core-modules/workspace/services/workspace.service';
import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity';
import { WorkspaceManyOrAllFlatEntityMapsCacheService } from 'src/engine/metadata-modules/flat-entity/services/workspace-many-or-all-flat-entity-maps-cache.service';
import { ObjectMetadataService } from 'src/engine/metadata-modules/object-metadata/object-metadata.service';
import { PermissionsService } from 'src/engine/metadata-modules/permissions/permissions.service';
import { WorkspaceCacheStorageService } from 'src/engine/workspace-cache-storage/workspace-cache-storage.service';
import { WorkspaceDataSourceService } from 'src/engine/workspace-datasource/workspace-datasource.service';
@@ -60,6 +61,12 @@ describe('WorkspaceService', () => {
findOneBy: jest.fn(),
},
},
{
provide: ObjectMetadataService,
useValue: {
deleteWorkspaceAllObjectMetadata: jest.fn(),
},
},
{
provide: getRepositoryToken(UserWorkspaceEntity),
useValue: {
@@ -280,16 +287,10 @@ describe('WorkspaceService', () => {
.spyOn(workspaceRepository, 'findOne')
.mockResolvedValue(mockWorkspace);
jest.spyOn(userWorkspaceRepository, 'find').mockResolvedValue([]);
jest
.spyOn(service, 'deleteMetadataSchemaCacheAndUserWorkspace')
.mockResolvedValue({} as WorkspaceEntity);
await service.deleteWorkspace(mockWorkspace.id, false);
expect(workspaceRepository.delete).toHaveBeenCalledWith(mockWorkspace.id);
expect(
service.deleteMetadataSchemaCacheAndUserWorkspace,
).toHaveBeenCalled();
expect(workspaceRepository.softDelete).not.toHaveBeenCalled();
expect(workspaceCacheStorageService.flush).toHaveBeenCalledWith(
mockWorkspace.id,
@@ -331,9 +332,6 @@ describe('WorkspaceService', () => {
.spyOn(workspaceRepository, 'findOne')
.mockResolvedValue(mockWorkspace);
jest.spyOn(userWorkspaceRepository, 'find').mockResolvedValue([]);
jest
.spyOn(service, 'deleteMetadataSchemaCacheAndUserWorkspace')
.mockResolvedValue({} as WorkspaceEntity);
await service.deleteWorkspace(mockWorkspace.id, false);
packages/twenty-server/src/engine/core-modules/workspace/services/workspace.service.ts
+15 -23
View File
@@ -37,6 +37,7 @@ import {
WorkspaceNotFoundDefaultError,
} from 'src/engine/core-modules/workspace/workspace.exception';
import { WorkspaceManyOrAllFlatEntityMapsCacheService } from 'src/engine/metadata-modules/flat-entity/services/workspace-many-or-all-flat-entity-maps-cache.service';
import { ObjectMetadataService } from 'src/engine/metadata-modules/object-metadata/object-metadata.service';
import {
PermissionsException,
PermissionsExceptionCode,
@@ -44,8 +45,8 @@ import {
} from 'src/engine/metadata-modules/permissions/permissions.exception';
import { PermissionsService } from 'src/engine/metadata-modules/permissions/permissions.service';
import { WorkspaceCacheStorageService } from 'src/engine/workspace-cache-storage/workspace-cache-storage.service';
import { WorkspaceDataSourceService } from 'src/engine/workspace-datasource/workspace-datasource.service';
import { getWorkspaceSchemaName } from 'src/engine/workspace-datasource/utils/get-workspace-schema-name.util';
import { WorkspaceDataSourceService } from 'src/engine/workspace-datasource/workspace-datasource.service';
import { prefillCompanies } from 'src/engine/workspace-manager/standard-objects-prefill-data/prefill-companies';
import { prefillDashboards } from 'src/engine/workspace-manager/standard-objects-prefill-data/prefill-dashboards';
import { prefillOpportunities } from 'src/engine/workspace-manager/standard-objects-prefill-data/prefill-opportunities';
@@ -102,6 +103,7 @@ export class WorkspaceService extends TypeOrmQueryService<WorkspaceEntity> {
private readonly workspaceCacheStorageService: WorkspaceCacheStorageService,
private readonly subdomainManagerService: SubdomainManagerService,
private readonly workspaceDataSourceService: WorkspaceDataSourceService,
private readonly objectMetadataService: ObjectMetadataService,
private readonly customDomainManagerService: CustomDomainManagerService,
@InjectMessageQueue(MessageQueue.deleteCascadeQueue)
private readonly messageQueueService: MessageQueueService,
@@ -282,20 +284,6 @@ export class WorkspaceService extends TypeOrmQueryService<WorkspaceEntity> {
});
}
/**
* @deprecated Should be removed once AddWorkspaceForeignKeysMigrationCommand has been run successfully in production
* As we will be able to rely on foreignKey delete cascading
*/
async deleteMetadataSchemaCacheAndUserWorkspace(workspace: WorkspaceEntity) {
await this.userWorkspaceService.deleteUserWorkspace({
userWorkspaceId: workspace.id,
});
await this.workspaceManagerService.delete(workspace.id);
return workspace;
}
async deleteWorkspace(id: string, softDelete = false) {
const workspace = await this.workspaceRepository.findOne({
where: { id },
@@ -320,13 +308,6 @@ export class WorkspaceService extends TypeOrmQueryService<WorkspaceEntity> {
}
this.logger.log(`workspace ${id} user workspaces deleted`);
await this.workspaceCacheStorageService.flush(
workspace.id,
workspace.metadataVersion,
);
await this.flatEntityMapsCacheService.flushFlatEntityMaps({
workspaceId: workspace.id,
});
this.logger.log(`workspace ${id} cache flushed`);
if (this.billingService.isBillingEnabled()) {
@@ -341,10 +322,21 @@ export class WorkspaceService extends TypeOrmQueryService<WorkspaceEntity> {
return workspace;
}
await this.deleteMetadataSchemaCacheAndUserWorkspace(workspace);
// Note: not relying on workspace id FK cascade deletion here to avoid query read timeout later on workspace deletion
await this.objectMetadataService.deleteWorkspaceAllObjectMetadata({
workspaceId: workspace.id,
});
await this.workspaceDataSourceService.deleteWorkspaceDBSchema(workspace.id);
await this.workspaceCacheStorageService.flush(
workspace.id,
workspace.metadataVersion,
);
await this.flatEntityMapsCacheService.flushFlatEntityMaps({
workspaceId: workspace.id,
});
await this.messageQueueService.add<FileWorkspaceFolderDeletionJobData>(
FileWorkspaceFolderDeletionJob.name,
{ workspaceId: id },
packages/twenty-server/src/engine/core-modules/workspace/workspace.module.ts
+2
View File
@@ -31,6 +31,7 @@ import { WorkspaceResolver } from 'src/engine/core-modules/workspace/workspace.r
import { AiAgentModule } from 'src/engine/metadata-modules/ai/ai-agent/ai-agent.module';
import { DataSourceModule } from 'src/engine/metadata-modules/data-source/data-source.module';
import { WorkspaceManyOrAllFlatEntityMapsCacheModule } from 'src/engine/metadata-modules/flat-entity/services/workspace-many-or-all-flat-entity-maps-cache.module';
import { ObjectMetadataModule } from 'src/engine/metadata-modules/object-metadata/object-metadata.module';
import { PermissionsModule } from 'src/engine/metadata-modules/permissions/permissions.module';
import { RoleModule } from 'src/engine/metadata-modules/role/role.module';
import { ViewModule } from 'src/engine/metadata-modules/view/view.module';
@@ -55,6 +56,7 @@ import { WorkspaceManagerModule } from 'src/engine/workspace-manager/workspace-m
UserWorkspaceEntity,
PublicDomainEntity,
]),
ObjectMetadataModule,
UserWorkspaceModule,
WorkspaceManagerModule,
FeatureFlagModule,
packages/twenty-server/src/engine/metadata-modules/workspace-metadata-version/services/workspace-metadata-version.service.ts
+1
View File
@@ -24,6 +24,7 @@ export class WorkspaceMetadataVersionService {
async incrementMetadataVersion(workspaceId: string): Promise<void> {
const workspace = await this.workspaceRepository.findOne({
where: { id: workspaceId },
withDeleted: true,
});
const metadataVersion = workspace?.metadataVersion;
packages/twenty-server/src/engine/workspace-manager/workspace-cleaner/commands/clean-soft-deleted-suspended-workspaces.command.ts
+106
View File
@@ -0,0 +1,106 @@
import { InjectRepository } from '@nestjs/typeorm';
import { Command, Option } from 'nest-commander';
import { isDefined } from 'twenty-shared/utils';
import { WorkspaceActivationStatus } from 'twenty-shared/workspace';
import { In, IsNull, Not, Repository } from 'typeorm';
import {
type MigrationCommandOptions,
MigrationCommandRunner,
} from 'src/database/commands/command-runners/migration.command-runner';
import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity';
import { CleanerWorkspaceService } from 'src/engine/workspace-manager/workspace-cleaner/services/cleaner.workspace-service';
type HardDeleteSoftDeletedSuspendedWorkspacesCommandOptions =
MigrationCommandOptions & {
ignoreGracePeriod?: boolean;
};
@Command({
name: 'workspace:hard-delete-soft-deleted',
description: 'Hard delete soft-deleted suspended workspaces',
})
export class HardDeleteSoftDeletedSuspendedWorkspacesCommand extends MigrationCommandRunner {
private workspaceIds: string[] = [];
constructor(
private readonly cleanerWorkspaceService: CleanerWorkspaceService,
@InjectRepository(WorkspaceEntity)
protected readonly workspaceRepository: Repository<WorkspaceEntity>,
) {
super();
}
@Option({
flags: '-w, --workspace-id [workspace_id]',
description:
'workspace id. Command runs on all suspended workspaces if not provided',
required: false,
})
parseWorkspaceId(val: string): string[] {
this.workspaceIds.push(val);
return this.workspaceIds;
}
@Option({
flags: '--ignore-grace-period',
description:
'Ignore the grace period and hard delete soft-deleted workspaces immediately',
required: false,
})
parseIgnoreGracePeriod(): boolean {
return true;
}
async fetchSuspendedSoftDeletedWorkspaces(): Promise<WorkspaceEntity[]> {
return await this.workspaceRepository.find({
where: {
activationStatus: In([WorkspaceActivationStatus.SUSPENDED]),
deletedAt: Not(IsNull()),
...(this.workspaceIds.length > 0 ? { id: In(this.workspaceIds) } : {}),
},
withDeleted: true,
});
}
override async runMigrationCommand(
_passedParams: string[],
options: HardDeleteSoftDeletedSuspendedWorkspacesCommandOptions,
): Promise<void> {
const { dryRun, ignoreGracePeriod } = options;
const softDeletedSuspendedWorkspaces =
await this.fetchSuspendedSoftDeletedWorkspaces();
let deletedWorkspaceCounter = 0;
this.logger.log(
`${dryRun ? 'DRY RUN - ' : ''}${ignoreGracePeriod ? 'IGNORING GRACE PERIOD - ' : ''}Iterating over ${softDeletedSuspendedWorkspaces.length} soft deleted suspended workspaces`,
);
for (const workspace of softDeletedSuspendedWorkspaces) {
try {
const result =
await this.cleanerWorkspaceService.hardDeleteSoftDeletedWorkspace({
workspace,
dryRun,
ignoreGracePeriod,
});
if (isDefined(result)) {
deletedWorkspaceCounter++;
}
} catch (error) {
this.logger.error(
`Failed to destroy soft deleted workspace ${workspace.id}`,
error,
);
}
}
this.logger.log(
`Destroyed ${deletedWorkspaceCounter}/${softDeletedSuspendedWorkspaces.length}`,
);
}
}
packages/twenty-server/src/engine/workspace-manager/workspace-cleaner/commands/clean-suspended-workspaces.command.ts
+3 -3
View File
@@ -65,9 +65,9 @@ export class CleanSuspendedWorkspacesCommand extends MigrationCommandRunner {
`${dryRun ? 'DRY RUN - ' : ''}Cleaning ${suspendedWorkspaceIds.length} suspended workspaces`,
);
await this.cleanerWorkspaceService.batchWarnOrCleanSuspendedWorkspaces(
suspendedWorkspaceIds,
await this.cleanerWorkspaceService.batchWarnOrCleanSuspendedWorkspaces({
workspaceIds: suspendedWorkspaceIds,
dryRun,
);
});
}
}
packages/twenty-server/src/engine/workspace-manager/workspace-cleaner/crons/clean-suspended-workspaces.job.ts
+3 -3
View File
@@ -33,8 +33,8 @@ export class CleanSuspendedWorkspacesJob {
withDeleted: true,
});
await this.cleanerWorkspaceService.batchWarnOrCleanSuspendedWorkspaces(
suspendedWorkspaceIds.map((workspace) => workspace.id),
);
await this.cleanerWorkspaceService.batchWarnOrCleanSuspendedWorkspaces({
workspaceIds: suspendedWorkspaceIds.map((workspace) => workspace.id),
});
}
}
packages/twenty-server/src/engine/workspace-manager/workspace-cleaner/services/cleaner.workspace-service.ts
+59 -26
View File
@@ -24,7 +24,6 @@ import { UserService } from 'src/engine/core-modules/user/services/user.service'
import { UserVarsService } from 'src/engine/core-modules/user/user-vars/services/user-vars.service';
import { WorkspaceService } from 'src/engine/core-modules/workspace/services/workspace.service';
import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity';
import { GlobalWorkspaceOrmManager } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager';
import { USER_WORKSPACE_DELETION_WARNING_SENT_KEY } from 'src/engine/workspace-manager/workspace-cleaner/constants/user-workspace-deletion-warning-sent-key.constant';
import {
WorkspaceCleanerException,
@@ -49,7 +48,6 @@ export class CleanerWorkspaceService {
private readonly workspaceRepository: Repository<WorkspaceEntity>,
@InjectRepository(BillingSubscriptionEntity)
private readonly billingSubscriptionRepository: Repository<BillingSubscriptionEntity>,
private readonly globalWorkspaceOrmManager: GlobalWorkspaceOrmManager,
@InjectRepository(UserWorkspaceEntity)
private readonly userWorkspaceRepository: Repository<UserWorkspaceEntity>,
private readonly i18nService: I18nService,
@@ -331,10 +329,56 @@ export class CleanerWorkspaceService {
}
}
async batchWarnOrCleanSuspendedWorkspaces(
workspaceIds: string[],
async hardDeleteSoftDeletedWorkspace({
workspace,
ignoreGracePeriod = false,
dryRun = false,
): Promise<void> {
}: {
ignoreGracePeriod?: boolean;
dryRun?: boolean;
workspace: WorkspaceEntity;
}): Promise<WorkspaceEntity | undefined> {
if (!isDefined(workspace.deletedAt)) {
return;
}
const daysSinceSoftDeleted = workspace.deletedAt
? differenceInDays(new Date(), workspace.deletedAt)
: 0;
const hasPassedGracePeriod =
daysSinceSoftDeleted >
this.inactiveDaysBeforeDelete - this.inactiveDaysBeforeSoftDelete;
const canHardDelete = ignoreGracePeriod || hasPassedGracePeriod;
if (!canHardDelete) {
return;
}
this.logger.log(
`${dryRun ? 'DRY RUN - ' : ''}Destroying workspace ${workspace.id} ${workspace.displayName}`,
);
if (dryRun) {
return;
}
await this.workspaceService.deleteWorkspace(workspace.id);
this.metricsService.incrementCounter({
key: MetricsKeys.CronJobDeletedWorkspace,
shouldStoreInCache: false,
});
return workspace;
}
async batchWarnOrCleanSuspendedWorkspaces({
workspaceIds,
dryRun = false,
}: {
workspaceIds: string[];
dryRun?: boolean;
}): Promise<void> {
this.logger.log(
`${dryRun ? 'DRY RUN - ' : ''}batchWarnOrCleanSuspendedWorkspaces running...`,
);
@@ -356,29 +400,18 @@ export class CleanerWorkspaceService {
try {
const isSoftDeletedWorkspace = isDefined(workspace.deletedAt);
const isWithinDeletionLimit =
deletedWorkspacesCount <
this.maxNumberOfWorkspacesDeletedPerExecution;
if (isSoftDeletedWorkspace) {
const daysSinceSoftDeleted = workspace.deletedAt
? differenceInDays(new Date(), workspace.deletedAt)
: 0;
if (isSoftDeletedWorkspace && isWithinDeletionLimit) {
const result = await this.hardDeleteSoftDeletedWorkspace({
workspace,
dryRun,
ignoreGracePeriod: false,
});
if (
daysSinceSoftDeleted >
this.inactiveDaysBeforeDelete -
this.inactiveDaysBeforeSoftDelete &&
deletedWorkspacesCount <
this.maxNumberOfWorkspacesDeletedPerExecution
) {
this.logger.log(
`${dryRun ? 'DRY RUN - ' : ''}Destroying workspace ${workspace.id} ${workspace.displayName}`,
);
if (!dryRun) {
await this.workspaceService.deleteWorkspace(workspace.id);
this.metricsService.incrementCounter({
key: MetricsKeys.CronJobDeletedWorkspace,
shouldStoreInCache: false,
});
}
if (isDefined(result)) {
deletedWorkspacesCount++;
}
continue;
packages/twenty-server/src/engine/workspace-manager/workspace-cleaner/workspace-cleaner.module.ts
+2
View File
@@ -13,6 +13,7 @@ import { WorkspaceModule } from 'src/engine/core-modules/workspace/workspace.mod
import { DataSourceModule } from 'src/engine/metadata-modules/data-source/data-source.module';
import { CleanOnboardingWorkspacesCommand } from 'src/engine/workspace-manager/workspace-cleaner/commands/clean-onboarding-workspaces.command';
import { CleanOnboardingWorkspacesCronCommand } from 'src/engine/workspace-manager/workspace-cleaner/commands/clean-onboarding-workspaces.cron.command';
import { HardDeleteSoftDeletedSuspendedWorkspacesCommand } from 'src/engine/workspace-manager/workspace-cleaner/commands/clean-soft-deleted-suspended-workspaces.command';
import { CleanSuspendedWorkspacesCommand } from 'src/engine/workspace-manager/workspace-cleaner/commands/clean-suspended-workspaces.command';
import { CleanSuspendedWorkspacesCronCommand } from 'src/engine/workspace-manager/workspace-cleaner/commands/clean-suspended-workspaces.cron.command';
import { DeleteWorkspacesCommand } from 'src/engine/workspace-manager/workspace-cleaner/commands/delete-workspaces.command';
@@ -39,6 +40,7 @@ import { CleanerWorkspaceService } from 'src/engine/workspace-manager/workspace-
DestroyWorkspaceCommand,
CleanSuspendedWorkspacesCronCommand,
CleanSuspendedWorkspacesCommand,
HardDeleteSoftDeletedSuspendedWorkspacesCommand,
CleanOnboardingWorkspacesCommand,
CleanOnboardingWorkspacesCronCommand,
CleanerWorkspaceService,
packages/twenty-server/src/engine/workspace-manager/workspace-manager.service.ts
-31
View File
@@ -8,12 +8,9 @@ import { FlatApplication } from 'src/engine/core-modules/application/types/flat-
import { UserWorkspaceEntity } from 'src/engine/core-modules/user-workspace/user-workspace.entity';
import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity';
import { DataSourceService } from 'src/engine/metadata-modules/data-source/data-source.service';
import { ObjectMetadataService } from 'src/engine/metadata-modules/object-metadata/object-metadata.service';
import { RoleTargetEntity } from 'src/engine/metadata-modules/role-target/role-target.entity';
import { ADMIN_ROLE } from 'src/engine/metadata-modules/role/constants/admin-role';
import { RoleEntity } from 'src/engine/metadata-modules/role/role.entity';
import { RoleService } from 'src/engine/metadata-modules/role/role.service';
import { ServerlessFunctionEntity } from 'src/engine/metadata-modules/serverless-function/serverless-function.entity';
import { UserRoleService } from 'src/engine/metadata-modules/user-role/user-role.service';
import { WorkspaceDataSourceService } from 'src/engine/workspace-datasource/workspace-datasource.service';
import { TwentyStandardApplicationService } from 'src/engine/workspace-manager/twenty-standard-application/services/twenty-standard-application.service';
@@ -24,7 +21,6 @@ export class WorkspaceManagerService {
constructor(
private readonly workspaceDataSourceService: WorkspaceDataSourceService,
private readonly objectMetadataService: ObjectMetadataService,
private readonly dataSourceService: DataSourceService,
@InjectRepository(UserWorkspaceEntity)
private readonly userWorkspaceRepository: Repository<UserWorkspaceEntity>,
@@ -35,10 +31,6 @@ export class WorkspaceManagerService {
private readonly workspaceRepository: Repository<WorkspaceEntity>,
@InjectRepository(RoleEntity)
private readonly roleRepository: Repository<RoleEntity>,
@InjectRepository(RoleTargetEntity)
private readonly roleTargetRepository: Repository<RoleTargetEntity>,
@InjectRepository(ServerlessFunctionEntity)
private readonly serverlessFunctionRepository: Repository<ServerlessFunctionEntity>,
private readonly applicationService: ApplicationService,
) {}
@@ -99,29 +91,6 @@ export class WorkspaceManagerService {
});
}
/**
* @deprecated Should be removed once AddWorkspaceForeignKeysMigrationCommand has been run successfully in production
* As we will be able to rely on foreignKey delete cascading
*/
public async delete(workspaceId: string): Promise<void> {
await this.roleTargetRepository.delete({
workspaceId,
});
await this.roleRepository.delete({
workspaceId,
});
await this.serverlessFunctionRepository.delete({
workspaceId,
});
await this.objectMetadataService.deleteWorkspaceAllObjectMetadata({
workspaceId,
});
await this.dataSourceService.delete(workspaceId);
}
private async setupDefaultRoles({
userId,
workspaceId,
packages/twenty-server/src/engine/workspace-manager/workspace-migration/workspace-migration-runner/services/workspace-migration-runner.service.ts
+6
View File
@@ -199,6 +199,12 @@ export class WorkspaceMigrationRunnerService {
);
if (invalidationFailures.length > 0) {
invalidationFailures.forEach((err) =>
this.logger.error(
`Failed to invalidate a legacy cache ${err.reason}`,
'Runner',
),
);
throw new Error(
`Failed to invalidate ${invalidationFailures.length} cache operations`,
);