Improve cleaning job (#17208)
# Introduction Refactored the workspace deletion to dynamically iterate over all known v2 syncable entities repos and delete all of them from child to parent Exception for field metadata that we chunk delete in order to avoid locking the core schema too long, it does not have an impact on perfs at all ( neither plus or less ) Chunking by constraint within a transaction is not necessary both does not cost more ## From 30s for a workspace complete deletion ```ts [Nest] 93244 - 01/16/2026, 10:24:52 PM LOG [WorkspaceService] workspace WS_ID cache flushed [Runner] Total execution: 26.290s // ( deleteAllObjectMetadatas v2 ) [Nest] 93244 - 01/16/2026, 10:25:22 PM LOG [WorkspaceService] workspace WS_ID hard deleted ``` ## To 3s ! ```ts [Nest] 65112 - 01/18/2026, 4:37:38 PM LOG [DatabaseConfigDriver] [INIT] Config variables loaded: 0 values found in DB, 69 falling to env vars/defaults [Nest] 65112 - 01/18/2026, 4:37:38 PM LOG [CleanSuspendedWorkspacesCommand] IGNORING GRACE PERIOD - Cleaning 1 suspended workspaces [Nest] 65112 - 01/18/2026, 4:37:38 PM LOG [CleanerWorkspaceService] batchWarnOrCleanSuspendedWorkspaces running... [Nest] 65112 - 01/18/2026, 4:37:38 PM LOG [CleanerWorkspaceService] Processing workspace - 1/1 [Nest] 65112 - 01/18/2026, 4:37:38 PM LOG [CleanerWorkspaceService] Destroying workspace Twenty Eng [Nest] 65112 - 01/18/2026, 4:37:38 PM LOG [WorkspaceService] workspace user workspaces deleted [Nest] 65112 - 01/18/2026, 4:37:38 PM LOG [WorkspaceService] workspace cache flushed [Nest] 65112 - 01/18/2026, 4:37:38 PM LOG [WorkspaceService] workspace: deleted 80 viewFilter record(s) [Nest] 65112 - 01/18/2026, 4:37:38 PM LOG [WorkspaceService] workspace: deleted 21 pageLayoutWidget record(s) [Nest] 65112 - 01/18/2026, 4:37:38 PM LOG [WorkspaceService] workspace: deleted 1515 viewField record(s) [Nest] 65112 - 01/18/2026, 4:37:38 PM LOG [WorkspaceService] workspace: deleted 91 index record(s) [Nest] 65112 - 01/18/2026, 4:37:38 PM LOG [WorkspaceService] workspace: deleted 66 roleTarget record(s) [Nest] 65112 - 01/18/2026, 4:37:38 PM LOG [WorkspaceService] workspace: deleted 174 viewGroup record(s) [Nest] 65112 - 01/18/2026, 4:37:38 PM LOG [WorkspaceService] workspace: deleted 1 agent record(s) [Nest] 65112 - 01/18/2026, 4:37:38 PM LOG [WorkspaceService] workspace: deleted 7 pageLayout record(s) [Nest] 65112 - 01/18/2026, 4:37:38 PM LOG [WorkspaceService] workspace: deleted 111 view record(s) [Nest] 65112 - 01/18/2026, 4:37:38 PM LOG [WorkspaceService] workspace: fieldMetadata chunk 1/15 - deleted 50 record(s) [Nest] 65112 - 01/18/2026, 4:37:38 PM LOG [WorkspaceService] workspace: fieldMetadata chunk 2/15 - deleted 50 record(s) [Nest] 65112 - 01/18/2026, 4:37:38 PM LOG [WorkspaceService] workspace: fieldMetadata chunk 3/15 - deleted 50 record(s) [Nest] 65112 - 01/18/2026, 4:37:38 PM LOG [WorkspaceService] workspace: fieldMetadata chunk 4/15 - deleted 50 record(s) [Nest] 65112 - 01/18/2026, 4:37:38 PM LOG [WorkspaceService] workspace: fieldMetadata chunk 5/15 - deleted 50 record(s) [Nest] 65112 - 01/18/2026, 4:37:38 PM LOG [WorkspaceService] workspace: fieldMetadata chunk 6/15 - deleted 50 record(s) [Nest] 65112 - 01/18/2026, 4:37:38 PM LOG [WorkspaceService] workspace: fieldMetadata chunk 7/15 - deleted 50 record(s) [Nest] 65112 - 01/18/2026, 4:37:38 PM LOG [WorkspaceService] workspace: fieldMetadata chunk 8/15 - deleted 50 record(s) [Nest] 65112 - 01/18/2026, 4:37:38 PM LOG [WorkspaceService] workspace: fieldMetadata chunk 9/15 - deleted 50 record(s) [Nest] 65112 - 01/18/2026, 4:37:38 PM LOG [WorkspaceService] workspace: fieldMetadata chunk 10/15 - deleted 50 record(s) [Nest] 65112 - 01/18/2026, 4:37:38 PM LOG [WorkspaceService] workspace: fieldMetadata chunk 11/15 - deleted 50 record(s) [Nest] 65112 - 01/18/2026, 4:37:38 PM LOG [WorkspaceService] workspace: fieldMetadata chunk 12/15 - deleted 51 record(s) [Nest] 65112 - 01/18/2026, 4:37:38 PM LOG [WorkspaceService] workspace: fieldMetadata chunk 13/15 - deleted 50 record(s) [Nest] 65112 - 01/18/2026, 4:37:38 PM LOG [WorkspaceService] workspace: fieldMetadata chunk 14/15 - deleted 50 record(s) [Nest] 65112 - 01/18/2026, 4:37:38 PM LOG [WorkspaceService] workspace: fieldMetadata chunk 15/15 - deleted 36 record(s) [Nest] 65112 - 01/18/2026, 4:37:38 PM LOG [WorkspaceService] workspace: deleted 737 fieldMetadata record(s) [Nest] 65112 - 01/18/2026, 4:37:38 PM LOG [WorkspaceService] workspace: deleted 6 role record(s) [Nest] 65112 - 01/18/2026, 4:37:38 PM LOG [WorkspaceService] workspace: deleted 78 serverlessFunction record(s) [Nest] 65112 - 01/18/2026, 4:37:39 PM LOG [WorkspaceService] workspace: deleted 43 objectMetadata record(s) [Nest] 65112 - 01/18/2026, 4:37:41 PM LOG [WorkspaceService] workspace hard deleted [Nest] 65112 - 01/18/2026, 4:37:41 PM LOG [CleanerWorkspaceService] Destroyed 1 workspaces on 5 limit durings this execution [Nest] 65112 - 01/18/2026, 4:37:41 PM LOG [CleanerWorkspaceService] batchWarnOrCleanSuspendedWorkspaces done! [Nest] 65112 - 01/18/2026, 4:37:41 PM LOG [CleanSuspendedWorkspacesCommand] Command completed! ``` ## Update Discussed with @charlesBochet ended debugging and analyzing sql query operations He discovered that we were not indexing foreignKey effectively We've ended up fixing all the FK indeces coverage leading to ## Cleaning Removed the ```sh npx nx run twenty-server:command workspace:clean-soft-deleted-suspended-workspaces --ignore-grace-period ``` In favor of ```sh npx nx run twenty-server:command workspace:clean --only-operation destroy --ignore-destroy-grace-period ``` ## Conclusion Not that crazy but still worth it and could demultiply in production
This commit is contained in:
Paul Rastoin
+7
-3
@@ -25,6 +25,7 @@ import { UserEntity } from 'src/engine/core-modules/user/user.entity';
|
||||
import { WorkspaceInvitationService } from 'src/engine/core-modules/workspace-invitation/services/workspace-invitation.service';
|
||||
import { WorkspaceService } from 'src/engine/core-modules/workspace/services/workspace.service';
|
||||
import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity';
|
||||
import { createEmptyAllFlatEntityMaps } from 'src/engine/metadata-modules/flat-entity/constant/create-empty-all-flat-entity-maps.constant';
|
||||
import { WorkspaceManyOrAllFlatEntityMapsCacheService } from 'src/engine/metadata-modules/flat-entity/services/workspace-many-or-all-flat-entity-maps-cache.service';
|
||||
import { ObjectMetadataService } from 'src/engine/metadata-modules/object-metadata/object-metadata.service';
|
||||
import { PermissionsService } from 'src/engine/metadata-modules/permissions/permissions.service';
|
||||
@@ -134,6 +135,9 @@ describe('WorkspaceService', () => {
|
||||
provide: WorkspaceManyOrAllFlatEntityMapsCacheService,
|
||||
useValue: {
|
||||
flushFlatEntityMaps: jest.fn(),
|
||||
getOrRecomputeManyOrAllFlatEntityMaps: jest
|
||||
.fn()
|
||||
.mockResolvedValue(createEmptyAllFlatEntityMaps()),
|
||||
},
|
||||
},
|
||||
{
|
||||
@@ -157,7 +161,9 @@ describe('WorkspaceService', () => {
|
||||
commitTransaction: jest.fn(),
|
||||
rollbackTransaction: jest.fn(),
|
||||
release: jest.fn(),
|
||||
manager: {},
|
||||
manager: {
|
||||
delete: jest.fn().mockResolvedValue({ affected: 0 }),
|
||||
},
|
||||
}),
|
||||
},
|
||||
},
|
||||
@@ -290,7 +296,6 @@ describe('WorkspaceService', () => {
|
||||
|
||||
await service.deleteWorkspace(mockWorkspace.id, false);
|
||||
|
||||
expect(workspaceRepository.delete).toHaveBeenCalledWith(mockWorkspace.id);
|
||||
expect(workspaceRepository.softDelete).not.toHaveBeenCalled();
|
||||
expect(workspaceCacheStorageService.flush).toHaveBeenCalledWith(
|
||||
mockWorkspace.id,
|
||||
@@ -338,7 +343,6 @@ describe('WorkspaceService', () => {
|
||||
expect(dnsManagerService.deleteHostnameSilently).toHaveBeenCalledWith(
|
||||
customDomain,
|
||||
);
|
||||
expect(workspaceRepository.delete).toHaveBeenCalledWith(mockWorkspace.id);
|
||||
});
|
||||
|
||||
it('should not delete the custom domain when soft deleting a workspace with a custom domain', async () => {
|
||||
|
||||
+129
-7
@@ -8,7 +8,7 @@ import { TypeOrmQueryService } from '@ptc-org/nestjs-query-typeorm';
|
||||
import { PermissionFlagType } from 'twenty-shared/constants';
|
||||
import { assertIsDefinedOrThrow, isDefined } from 'twenty-shared/utils';
|
||||
import { WorkspaceActivationStatus } from 'twenty-shared/workspace';
|
||||
import { DataSource, Repository } from 'typeorm';
|
||||
import { DataSource, QueryRunner, Repository } from 'typeorm';
|
||||
|
||||
import { ApiKeyEntity } from 'src/engine/core-modules/api-key/api-key.entity';
|
||||
import { BillingSubscriptionService } from 'src/engine/core-modules/billing/services/billing-subscription.service';
|
||||
@@ -36,8 +36,10 @@ import {
|
||||
WorkspaceExceptionCode,
|
||||
WorkspaceNotFoundDefaultError,
|
||||
} from 'src/engine/core-modules/workspace/workspace.exception';
|
||||
import { FieldMetadataEntity } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity';
|
||||
import { ALL_METADATA_ENTITY_BY_METADATA_NAME } from 'src/engine/metadata-modules/flat-entity/constant/all-metadata-entity-by-metadata-name.constant';
|
||||
import { ALL_METADATA_NAMES_SORTED_ATOMICALLY } from 'src/engine/metadata-modules/flat-entity/constant/all-metadata-names-sorted-atomically.constant';
|
||||
import { WorkspaceManyOrAllFlatEntityMapsCacheService } from 'src/engine/metadata-modules/flat-entity/services/workspace-many-or-all-flat-entity-maps-cache.service';
|
||||
import { ObjectMetadataService } from 'src/engine/metadata-modules/object-metadata/object-metadata.service';
|
||||
import {
|
||||
PermissionsException,
|
||||
PermissionsExceptionCode,
|
||||
@@ -103,7 +105,6 @@ export class WorkspaceService extends TypeOrmQueryService<WorkspaceEntity> {
|
||||
private readonly workspaceCacheStorageService: WorkspaceCacheStorageService,
|
||||
private readonly subdomainManagerService: SubdomainManagerService,
|
||||
private readonly workspaceDataSourceService: WorkspaceDataSourceService,
|
||||
private readonly objectMetadataService: ObjectMetadataService,
|
||||
private readonly customDomainManagerService: CustomDomainManagerService,
|
||||
@InjectMessageQueue(MessageQueue.deleteCascadeQueue)
|
||||
private readonly messageQueueService: MessageQueueService,
|
||||
@@ -322,10 +323,7 @@ export class WorkspaceService extends TypeOrmQueryService<WorkspaceEntity> {
|
||||
return workspace;
|
||||
}
|
||||
|
||||
// Note: not relying on workspace id FK cascade deletion here to avoid query read timeout later on workspace deletion
|
||||
await this.objectMetadataService.deleteWorkspaceAllObjectMetadata({
|
||||
workspaceId: workspace.id,
|
||||
});
|
||||
await this.deleteWorkspaceSyncableMetadataEntities(workspace);
|
||||
|
||||
await this.workspaceDataSourceService.deleteWorkspaceDBSchema(workspace.id);
|
||||
|
||||
@@ -356,6 +354,130 @@ export class WorkspaceService extends TypeOrmQueryService<WorkspaceEntity> {
|
||||
return workspace;
|
||||
}
|
||||
|
||||
private async deleteWorkspaceSyncableMetadataEntities(
|
||||
workspace: WorkspaceEntity,
|
||||
): Promise<void> {
|
||||
const queryRunner = this.coreDataSource.createQueryRunner();
|
||||
|
||||
await queryRunner.connect();
|
||||
|
||||
try {
|
||||
await queryRunner.startTransaction();
|
||||
|
||||
for (const metadataName of ALL_METADATA_NAMES_SORTED_ATOMICALLY) {
|
||||
if (metadataName === 'fieldMetadata') {
|
||||
const deletedCount = await this.deleteFieldMetadataInChunks(
|
||||
queryRunner,
|
||||
workspace.id,
|
||||
);
|
||||
|
||||
if (deletedCount > 0) {
|
||||
this.logger.log(
|
||||
`workspace ${workspace.id}: deleted ${deletedCount} ${metadataName} record(s)`,
|
||||
);
|
||||
}
|
||||
|
||||
continue;
|
||||
}
|
||||
|
||||
const entity = ALL_METADATA_ENTITY_BY_METADATA_NAME[metadataName];
|
||||
|
||||
const result = await queryRunner.manager.delete(entity, {
|
||||
workspaceId: workspace.id,
|
||||
});
|
||||
|
||||
if (result.affected && result.affected > 0) {
|
||||
this.logger.log(
|
||||
`workspace ${workspace.id}: deleted ${result.affected} ${metadataName} record(s)`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
await queryRunner.commitTransaction();
|
||||
} catch (error) {
|
||||
await queryRunner.rollbackTransaction();
|
||||
|
||||
throw error;
|
||||
} finally {
|
||||
await queryRunner.release();
|
||||
}
|
||||
}
|
||||
|
||||
// FieldMetadataEntity has a self-referencing FK (relationTargetFieldMetadataId)
|
||||
// Related fields must be deleted together to avoid constraint violations
|
||||
private async deleteFieldMetadataInChunks(
|
||||
queryRunner: QueryRunner,
|
||||
workspaceId: string,
|
||||
): Promise<number> {
|
||||
const CHUNK_SIZE = 50;
|
||||
let totalDeleted = 0;
|
||||
|
||||
const { flatFieldMetadataMaps } =
|
||||
await this.flatEntityMapsCacheService.getOrRecomputeManyOrAllFlatEntityMaps(
|
||||
{
|
||||
workspaceId,
|
||||
flatMapsKeys: ['flatFieldMetadataMaps'],
|
||||
},
|
||||
);
|
||||
|
||||
const fields = Object.values(flatFieldMetadataMaps.byId).filter(isDefined);
|
||||
|
||||
if (fields.length === 0) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
const processedIds = new Set<string>();
|
||||
const fieldsMap = new Map(fields.map((field) => [field.id, field]));
|
||||
const chunks: string[][] = [];
|
||||
let currentChunk: string[] = [];
|
||||
|
||||
for (const field of fields) {
|
||||
if (processedIds.has(field.id)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
currentChunk.push(field.id);
|
||||
processedIds.add(field.id);
|
||||
|
||||
if (field.relationTargetFieldMetadataId) {
|
||||
const relatedField = fieldsMap.get(field.relationTargetFieldMetadataId);
|
||||
|
||||
if (relatedField && !processedIds.has(relatedField.id)) {
|
||||
currentChunk.push(relatedField.id);
|
||||
processedIds.add(relatedField.id);
|
||||
}
|
||||
}
|
||||
|
||||
if (currentChunk.length >= CHUNK_SIZE) {
|
||||
chunks.push([...currentChunk]);
|
||||
currentChunk = [];
|
||||
}
|
||||
}
|
||||
|
||||
if (currentChunk.length > 0) {
|
||||
chunks.push(currentChunk);
|
||||
}
|
||||
|
||||
for (const [index, chunk] of chunks.entries()) {
|
||||
const result = await queryRunner.manager
|
||||
.createQueryBuilder()
|
||||
.delete()
|
||||
.from(FieldMetadataEntity)
|
||||
.whereInIds(chunk)
|
||||
.execute();
|
||||
|
||||
const deletedInChunk = result.affected || 0;
|
||||
|
||||
totalDeleted += deletedInChunk;
|
||||
|
||||
this.logger.log(
|
||||
`workspace ${workspaceId}: fieldMetadata chunk ${index + 1}/${chunks.length} - deleted ${deletedInChunk} record(s)`,
|
||||
);
|
||||
}
|
||||
|
||||
return totalDeleted;
|
||||
}
|
||||
|
||||
async handleRemoveWorkspaceMember(
|
||||
workspaceId: string,
|
||||
userId: string,
|
||||
|
||||
Reference in New Issue
Block a user