From fae6d0e262e11cbaa8220853a3c7cfe3a2e86a33 Mon Sep 17 00:00:00 2001 From: Paul Rastoin <45004772+prastoin@users.noreply.github.com> Date: Sun, 18 Jan 2026 17:22:26 +0100 Subject: [PATCH] Improve cleaning job (#17208) # Introduction Refactored the workspace deletion to dynamically iterate over all known v2 syncable entities repos and delete all of them from child to parent Exception for field metadata that we chunk delete in order to avoid locking the core schema too long, it does not have an impact on perfs at all ( neither plus or less ) Chunking by constraint within a transaction is not necessary both does not cost more ## From 30s for a workspace complete deletion ```ts [Nest] 93244 - 01/16/2026, 10:24:52 PM LOG [WorkspaceService] workspace WS_ID cache flushed [Runner] Total execution: 26.290s // ( deleteAllObjectMetadatas v2 ) [Nest] 93244 - 01/16/2026, 10:25:22 PM LOG [WorkspaceService] workspace WS_ID hard deleted ``` ## To 3s ! ```ts [Nest] 65112 - 01/18/2026, 4:37:38 PM LOG [DatabaseConfigDriver] [INIT] Config variables loaded: 0 values found in DB, 69 falling to env vars/defaults [Nest] 65112 - 01/18/2026, 4:37:38 PM LOG [CleanSuspendedWorkspacesCommand] IGNORING GRACE PERIOD - Cleaning 1 suspended workspaces [Nest] 65112 - 01/18/2026, 4:37:38 PM LOG [CleanerWorkspaceService] batchWarnOrCleanSuspendedWorkspaces running... [Nest] 65112 - 01/18/2026, 4:37:38 PM LOG [CleanerWorkspaceService] Processing workspace - 1/1 [Nest] 65112 - 01/18/2026, 4:37:38 PM LOG [CleanerWorkspaceService] Destroying workspace Twenty Eng [Nest] 65112 - 01/18/2026, 4:37:38 PM LOG [WorkspaceService] workspace user workspaces deleted [Nest] 65112 - 01/18/2026, 4:37:38 PM LOG [WorkspaceService] workspace cache flushed [Nest] 65112 - 01/18/2026, 4:37:38 PM LOG [WorkspaceService] workspace: deleted 80 viewFilter record(s) [Nest] 65112 - 01/18/2026, 4:37:38 PM LOG [WorkspaceService] workspace: deleted 21 pageLayoutWidget record(s) [Nest] 65112 - 01/18/2026, 4:37:38 PM LOG [WorkspaceService] workspace: deleted 1515 viewField record(s) [Nest] 65112 - 01/18/2026, 4:37:38 PM LOG [WorkspaceService] workspace: deleted 91 index record(s) [Nest] 65112 - 01/18/2026, 4:37:38 PM LOG [WorkspaceService] workspace: deleted 66 roleTarget record(s) [Nest] 65112 - 01/18/2026, 4:37:38 PM LOG [WorkspaceService] workspace: deleted 174 viewGroup record(s) [Nest] 65112 - 01/18/2026, 4:37:38 PM LOG [WorkspaceService] workspace: deleted 1 agent record(s) [Nest] 65112 - 01/18/2026, 4:37:38 PM LOG [WorkspaceService] workspace: deleted 7 pageLayout record(s) [Nest] 65112 - 01/18/2026, 4:37:38 PM LOG [WorkspaceService] workspace: deleted 111 view record(s) [Nest] 65112 - 01/18/2026, 4:37:38 PM LOG [WorkspaceService] workspace: fieldMetadata chunk 1/15 - deleted 50 record(s) [Nest] 65112 - 01/18/2026, 4:37:38 PM LOG [WorkspaceService] workspace: fieldMetadata chunk 2/15 - deleted 50 record(s) [Nest] 65112 - 01/18/2026, 4:37:38 PM LOG [WorkspaceService] workspace: fieldMetadata chunk 3/15 - deleted 50 record(s) [Nest] 65112 - 01/18/2026, 4:37:38 PM LOG [WorkspaceService] workspace: fieldMetadata chunk 4/15 - deleted 50 record(s) [Nest] 65112 - 01/18/2026, 4:37:38 PM LOG [WorkspaceService] workspace: fieldMetadata chunk 5/15 - deleted 50 record(s) [Nest] 65112 - 01/18/2026, 4:37:38 PM LOG [WorkspaceService] workspace: fieldMetadata chunk 6/15 - deleted 50 record(s) [Nest] 65112 - 01/18/2026, 4:37:38 PM LOG [WorkspaceService] workspace: fieldMetadata chunk 7/15 - deleted 50 record(s) [Nest] 65112 - 01/18/2026, 4:37:38 PM LOG [WorkspaceService] workspace: fieldMetadata chunk 8/15 - deleted 50 record(s) [Nest] 65112 - 01/18/2026, 4:37:38 PM LOG [WorkspaceService] workspace: fieldMetadata chunk 9/15 - deleted 50 record(s) [Nest] 65112 - 01/18/2026, 4:37:38 PM LOG [WorkspaceService] workspace: fieldMetadata chunk 10/15 - deleted 50 record(s) [Nest] 65112 - 01/18/2026, 4:37:38 PM LOG [WorkspaceService] workspace: fieldMetadata chunk 11/15 - deleted 50 record(s) [Nest] 65112 - 01/18/2026, 4:37:38 PM LOG [WorkspaceService] workspace: fieldMetadata chunk 12/15 - deleted 51 record(s) [Nest] 65112 - 01/18/2026, 4:37:38 PM LOG [WorkspaceService] workspace: fieldMetadata chunk 13/15 - deleted 50 record(s) [Nest] 65112 - 01/18/2026, 4:37:38 PM LOG [WorkspaceService] workspace: fieldMetadata chunk 14/15 - deleted 50 record(s) [Nest] 65112 - 01/18/2026, 4:37:38 PM LOG [WorkspaceService] workspace: fieldMetadata chunk 15/15 - deleted 36 record(s) [Nest] 65112 - 01/18/2026, 4:37:38 PM LOG [WorkspaceService] workspace: deleted 737 fieldMetadata record(s) [Nest] 65112 - 01/18/2026, 4:37:38 PM LOG [WorkspaceService] workspace: deleted 6 role record(s) [Nest] 65112 - 01/18/2026, 4:37:38 PM LOG [WorkspaceService] workspace: deleted 78 serverlessFunction record(s) [Nest] 65112 - 01/18/2026, 4:37:39 PM LOG [WorkspaceService] workspace: deleted 43 objectMetadata record(s) [Nest] 65112 - 01/18/2026, 4:37:41 PM LOG [WorkspaceService] workspace hard deleted [Nest] 65112 - 01/18/2026, 4:37:41 PM LOG [CleanerWorkspaceService] Destroyed 1 workspaces on 5 limit durings this execution [Nest] 65112 - 01/18/2026, 4:37:41 PM LOG [CleanerWorkspaceService] batchWarnOrCleanSuspendedWorkspaces done! [Nest] 65112 - 01/18/2026, 4:37:41 PM LOG [CleanSuspendedWorkspacesCommand] Command completed! ``` ## Update Discussed with @charlesBochet ended debugging and analyzing sql query operations He discovered that we were not indexing foreignKey effectively We've ended up fixing all the FK indeces coverage leading to ## Cleaning Removed the ```sh npx nx run twenty-server:command workspace:clean-soft-deleted-suspended-workspaces --ignore-grace-period ``` In favor of ```sh npx nx run twenty-server:command workspace:clean --only-operation destroy --ignore-destroy-grace-period ``` ## Conclusion Not that crazy but still worth it and could demultiply in production --- ...50308557-foreignKeyIndexStandardization.ts | 155 ++++++++++++++++++ .../__tests__/workspace.service.spec.ts | 10 +- .../workspace/services/workspace.service.ts | 136 ++++++++++++++- .../entities/command-menu-item.entity.ts | 3 + .../entities/cron-trigger.entity.ts | 1 + .../entities/database-event-trigger.entity.ts | 3 + ...tadata-entity-by-metadata-name.constant.ts | 52 ++++++ ...tadata-names-sorted-atomically.constant.ts | 4 + .../all-flat-entity-types-by-metadata-name.ts | 70 +++----- .../flat-entity/types/metadata-entity.type.ts | 7 +- ...ata-names-children-first.util.spec.ts.snap | 29 ++++ ...metadata-names-children-first.util.spec.ts | 9 + ...metadata-many-to-one-related-names.util.ts | 16 ++ ...metadata-one-to-many-related-names.util.ts | 16 ++ ...et-metadata-related-metadata-names.util.ts | 14 +- ...sort-metadata-names-children-first.util.ts | 29 ++++ .../object-metadata/object-metadata.entity.ts | 2 + .../entities/page-layout-widget.entity.ts | 1 + .../role-target/role-target.entity.ts | 1 + .../route-trigger/route-trigger.entity.ts | 2 + ...level-permission-predicate-group.entity.ts | 3 + .../row-level-permission-predicate.entity.ts | 3 + .../serverless-function.entity.ts | 1 + .../view-field/entities/view-field.entity.ts | 5 +- .../entities/view-filter-group.entity.ts | 5 +- .../entities/view-filter.entity.ts | 4 +- .../view-group/entities/view-group.entity.ts | 4 +- .../view-sort/entities/view-sort.entity.ts | 5 +- .../view/entities/view.entity.ts | 6 + ...ft-deleted-suspended-workspaces.command.ts | 106 ------------ .../clean-suspended-workspaces.command.ts | 52 +++++- .../commands/delete-workspaces.command.ts | 91 ---------- .../services/cleaner.workspace-service.ts | 105 +++++++----- .../workspace-cleaner.module.ts | 4 - 34 files changed, 616 insertions(+), 338 deletions(-) create mode 100644 packages/twenty-server/src/database/typeorm/core/migrations/common/1768750308557-foreignKeyIndexStandardization.ts create mode 100644 packages/twenty-server/src/engine/metadata-modules/flat-entity/constant/all-metadata-entity-by-metadata-name.constant.ts create mode 100644 packages/twenty-server/src/engine/metadata-modules/flat-entity/constant/all-metadata-names-sorted-atomically.constant.ts create mode 100644 packages/twenty-server/src/engine/metadata-modules/flat-entity/utils/__tests__/__snapshots__/sort-metadata-names-children-first.util.spec.ts.snap create mode 100644 packages/twenty-server/src/engine/metadata-modules/flat-entity/utils/__tests__/sort-metadata-names-children-first.util.spec.ts create mode 100644 packages/twenty-server/src/engine/metadata-modules/flat-entity/utils/get-metadata-many-to-one-related-names.util.ts create mode 100644 packages/twenty-server/src/engine/metadata-modules/flat-entity/utils/get-metadata-one-to-many-related-names.util.ts create mode 100644 packages/twenty-server/src/engine/metadata-modules/flat-entity/utils/sort-metadata-names-children-first.util.ts delete mode 100644 packages/twenty-server/src/engine/workspace-manager/workspace-cleaner/commands/clean-soft-deleted-suspended-workspaces.command.ts delete mode 100644 packages/twenty-server/src/engine/workspace-manager/workspace-cleaner/commands/delete-workspaces.command.ts diff --git a/packages/twenty-server/src/database/typeorm/core/migrations/common/1768750308557-foreignKeyIndexStandardization.ts b/packages/twenty-server/src/database/typeorm/core/migrations/common/1768750308557-foreignKeyIndexStandardization.ts new file mode 100644 index 0000000000..c267df188a --- /dev/null +++ b/packages/twenty-server/src/database/typeorm/core/migrations/common/1768750308557-foreignKeyIndexStandardization.ts @@ -0,0 +1,155 @@ +import { type MigrationInterface, type QueryRunner } from 'typeorm'; + +export class ForeignKeyIndexStandardization1768750308557 + implements MigrationInterface +{ + name = 'ForeignKeyIndexStandardization1768750308557'; + + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query(`DROP INDEX "core"."IDX_VIEW_FILTER_VIEW_ID"`); + await queryRunner.query( + `DROP INDEX "core"."IDX_VIEW_FILTER_GROUP_VIEW_ID"`, + ); + await queryRunner.query(`DROP INDEX "core"."IDX_VIEW_GROUP_VIEW_ID"`); + await queryRunner.query(`DROP INDEX "core"."IDX_VIEW_SORT_VIEW_ID"`); + await queryRunner.query(`DROP INDEX "core"."IDX_VIEW_FIELD_VIEW_ID"`); + await queryRunner.query( + `CREATE INDEX "IDX_ROLE_TARGET_ROLE_ID" ON "core"."roleTarget" ("roleId") `, + ); + await queryRunner.query( + `CREATE INDEX "IDX_RLPP_WORKSPACE_MEMBER_FIELD_METADATA_ID" ON "core"."rowLevelPermissionPredicate" ("workspaceMemberFieldMetadataId") `, + ); + await queryRunner.query( + `CREATE INDEX "IDX_RLPPG_PARENT_GROUP_ID" ON "core"."rowLevelPermissionPredicateGroup" ("parentRowLevelPermissionPredicateGroupId") `, + ); + await queryRunner.query( + `CREATE INDEX "IDX_VIEW_FILTER_VIEW_ID" ON "core"."viewFilter" ("viewId") `, + ); + await queryRunner.query( + `CREATE INDEX "IDX_VIEW_FILTER_GROUP_PARENT_ID" ON "core"."viewFilterGroup" ("parentViewFilterGroupId") `, + ); + await queryRunner.query( + `CREATE INDEX "IDX_VIEW_FILTER_GROUP_VIEW_ID" ON "core"."viewFilterGroup" ("viewId") `, + ); + await queryRunner.query( + `CREATE INDEX "IDX_VIEW_GROUP_VIEW_ID" ON "core"."viewGroup" ("viewId") `, + ); + await queryRunner.query( + `CREATE INDEX "IDX_VIEW_SORT_FIELD_METADATA_ID" ON "core"."viewSort" ("fieldMetadataId") `, + ); + await queryRunner.query( + `CREATE INDEX "IDX_VIEW_SORT_VIEW_ID" ON "core"."viewSort" ("viewId") `, + ); + await queryRunner.query( + `CREATE INDEX "IDX_VIEW_CREATED_BY_USER_WORKSPACE" ON "core"."view" ("createdByUserWorkspaceId") `, + ); + await queryRunner.query( + `CREATE INDEX "IDX_VIEW_MAIN_GROUP_BY_FIELD_METADATA" ON "core"."view" ("mainGroupByFieldMetadataId") `, + ); + await queryRunner.query( + `CREATE INDEX "IDX_VIEW_KANBAN_FIELD_METADATA" ON "core"."view" ("kanbanAggregateOperationFieldMetadataId") `, + ); + await queryRunner.query( + `CREATE INDEX "IDX_VIEW_CALENDAR_FIELD_METADATA" ON "core"."view" ("calendarFieldMetadataId") `, + ); + await queryRunner.query( + `CREATE INDEX "IDX_VIEW_FIELD_FIELD_METADATA_ID" ON "core"."viewField" ("fieldMetadataId") `, + ); + await queryRunner.query( + `CREATE INDEX "IDX_VIEW_FIELD_VIEW_ID" ON "core"."viewField" ("viewId") `, + ); + await queryRunner.query( + `CREATE INDEX "IDX_OBJECT_METADATA_DATA_SOURCE_ID" ON "core"."objectMetadata" ("dataSourceId") `, + ); + await queryRunner.query( + `CREATE INDEX "IDX_CRON_TRIGGER_SERVERLESS_FUNCTION_ID" ON "core"."cronTrigger" ("serverlessFunctionId") `, + ); + await queryRunner.query( + `CREATE INDEX "IDX_DATABASE_EVENT_TRIGGER_SERVERLESS_FUNCTION_ID" ON "core"."databaseEventTrigger" ("serverlessFunctionId") `, + ); + await queryRunner.query( + `CREATE INDEX "IDX_ROUTE_TRIGGER_SERVERLESS_FUNCTION_ID" ON "core"."routeTrigger" ("serverlessFunctionId") `, + ); + await queryRunner.query( + `CREATE INDEX "IDX_SERVERLESS_FUNCTION_LAYER_ID" ON "core"."serverlessFunction" ("serverlessFunctionLayerId") `, + ); + await queryRunner.query( + `CREATE INDEX "IDX_PAGE_LAYOUT_WIDGET_OBJECT_METADATA_ID" ON "core"."pageLayoutWidget" ("objectMetadataId") `, + ); + await queryRunner.query( + `CREATE INDEX "IDX_COMMAND_MENU_ITEM_AVAILABILITY_OBJECT_METADATA_ID" ON "core"."commandMenuItem" ("availabilityObjectMetadataId") `, + ); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query( + `DROP INDEX "core"."IDX_COMMAND_MENU_ITEM_AVAILABILITY_OBJECT_METADATA_ID"`, + ); + await queryRunner.query( + `DROP INDEX "core"."IDX_PAGE_LAYOUT_WIDGET_OBJECT_METADATA_ID"`, + ); + await queryRunner.query( + `DROP INDEX "core"."IDX_SERVERLESS_FUNCTION_LAYER_ID"`, + ); + await queryRunner.query( + `DROP INDEX "core"."IDX_ROUTE_TRIGGER_SERVERLESS_FUNCTION_ID"`, + ); + await queryRunner.query( + `DROP INDEX "core"."IDX_DATABASE_EVENT_TRIGGER_SERVERLESS_FUNCTION_ID"`, + ); + await queryRunner.query( + `DROP INDEX "core"."IDX_CRON_TRIGGER_SERVERLESS_FUNCTION_ID"`, + ); + await queryRunner.query( + `DROP INDEX "core"."IDX_OBJECT_METADATA_DATA_SOURCE_ID"`, + ); + await queryRunner.query(`DROP INDEX "core"."IDX_VIEW_FIELD_VIEW_ID"`); + await queryRunner.query( + `DROP INDEX "core"."IDX_VIEW_FIELD_FIELD_METADATA_ID"`, + ); + await queryRunner.query( + `DROP INDEX "core"."IDX_VIEW_CALENDAR_FIELD_METADATA"`, + ); + await queryRunner.query( + `DROP INDEX "core"."IDX_VIEW_KANBAN_FIELD_METADATA"`, + ); + await queryRunner.query( + `DROP INDEX "core"."IDX_VIEW_MAIN_GROUP_BY_FIELD_METADATA"`, + ); + await queryRunner.query( + `DROP INDEX "core"."IDX_VIEW_CREATED_BY_USER_WORKSPACE"`, + ); + await queryRunner.query(`DROP INDEX "core"."IDX_VIEW_SORT_VIEW_ID"`); + await queryRunner.query( + `DROP INDEX "core"."IDX_VIEW_SORT_FIELD_METADATA_ID"`, + ); + await queryRunner.query(`DROP INDEX "core"."IDX_VIEW_GROUP_VIEW_ID"`); + await queryRunner.query( + `DROP INDEX "core"."IDX_VIEW_FILTER_GROUP_VIEW_ID"`, + ); + await queryRunner.query( + `DROP INDEX "core"."IDX_VIEW_FILTER_GROUP_PARENT_ID"`, + ); + await queryRunner.query(`DROP INDEX "core"."IDX_VIEW_FILTER_VIEW_ID"`); + await queryRunner.query(`DROP INDEX "core"."IDX_RLPPG_PARENT_GROUP_ID"`); + await queryRunner.query( + `DROP INDEX "core"."IDX_RLPP_WORKSPACE_MEMBER_FIELD_METADATA_ID"`, + ); + await queryRunner.query(`DROP INDEX "core"."IDX_ROLE_TARGET_ROLE_ID"`); + await queryRunner.query( + `CREATE INDEX "IDX_VIEW_FIELD_VIEW_ID" ON "core"."viewField" ("viewId") WHERE ("deletedAt" IS NULL)`, + ); + await queryRunner.query( + `CREATE INDEX "IDX_VIEW_SORT_VIEW_ID" ON "core"."viewSort" ("viewId") WHERE ("deletedAt" IS NULL)`, + ); + await queryRunner.query( + `CREATE INDEX "IDX_VIEW_GROUP_VIEW_ID" ON "core"."viewGroup" ("viewId") WHERE ("deletedAt" IS NULL)`, + ); + await queryRunner.query( + `CREATE INDEX "IDX_VIEW_FILTER_GROUP_VIEW_ID" ON "core"."viewFilterGroup" ("viewId") WHERE ("deletedAt" IS NULL)`, + ); + await queryRunner.query( + `CREATE INDEX "IDX_VIEW_FILTER_VIEW_ID" ON "core"."viewFilter" ("viewId") WHERE ("deletedAt" IS NULL)`, + ); + } +} diff --git a/packages/twenty-server/src/engine/core-modules/workspace/services/__tests__/workspace.service.spec.ts b/packages/twenty-server/src/engine/core-modules/workspace/services/__tests__/workspace.service.spec.ts index 3d491ddfde..30eff2426e 100644 --- a/packages/twenty-server/src/engine/core-modules/workspace/services/__tests__/workspace.service.spec.ts +++ b/packages/twenty-server/src/engine/core-modules/workspace/services/__tests__/workspace.service.spec.ts @@ -25,6 +25,7 @@ import { UserEntity } from 'src/engine/core-modules/user/user.entity'; import { WorkspaceInvitationService } from 'src/engine/core-modules/workspace-invitation/services/workspace-invitation.service'; import { WorkspaceService } from 'src/engine/core-modules/workspace/services/workspace.service'; import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity'; +import { createEmptyAllFlatEntityMaps } from 'src/engine/metadata-modules/flat-entity/constant/create-empty-all-flat-entity-maps.constant'; import { WorkspaceManyOrAllFlatEntityMapsCacheService } from 'src/engine/metadata-modules/flat-entity/services/workspace-many-or-all-flat-entity-maps-cache.service'; import { ObjectMetadataService } from 'src/engine/metadata-modules/object-metadata/object-metadata.service'; import { PermissionsService } from 'src/engine/metadata-modules/permissions/permissions.service'; @@ -134,6 +135,9 @@ describe('WorkspaceService', () => { provide: WorkspaceManyOrAllFlatEntityMapsCacheService, useValue: { flushFlatEntityMaps: jest.fn(), + getOrRecomputeManyOrAllFlatEntityMaps: jest + .fn() + .mockResolvedValue(createEmptyAllFlatEntityMaps()), }, }, { @@ -157,7 +161,9 @@ describe('WorkspaceService', () => { commitTransaction: jest.fn(), rollbackTransaction: jest.fn(), release: jest.fn(), - manager: {}, + manager: { + delete: jest.fn().mockResolvedValue({ affected: 0 }), + }, }), }, }, @@ -290,7 +296,6 @@ describe('WorkspaceService', () => { await service.deleteWorkspace(mockWorkspace.id, false); - expect(workspaceRepository.delete).toHaveBeenCalledWith(mockWorkspace.id); expect(workspaceRepository.softDelete).not.toHaveBeenCalled(); expect(workspaceCacheStorageService.flush).toHaveBeenCalledWith( mockWorkspace.id, @@ -338,7 +343,6 @@ describe('WorkspaceService', () => { expect(dnsManagerService.deleteHostnameSilently).toHaveBeenCalledWith( customDomain, ); - expect(workspaceRepository.delete).toHaveBeenCalledWith(mockWorkspace.id); }); it('should not delete the custom domain when soft deleting a workspace with a custom domain', async () => { diff --git a/packages/twenty-server/src/engine/core-modules/workspace/services/workspace.service.ts b/packages/twenty-server/src/engine/core-modules/workspace/services/workspace.service.ts index b12596ef3e..787304182c 100644 --- a/packages/twenty-server/src/engine/core-modules/workspace/services/workspace.service.ts +++ b/packages/twenty-server/src/engine/core-modules/workspace/services/workspace.service.ts @@ -8,7 +8,7 @@ import { TypeOrmQueryService } from '@ptc-org/nestjs-query-typeorm'; import { PermissionFlagType } from 'twenty-shared/constants'; import { assertIsDefinedOrThrow, isDefined } from 'twenty-shared/utils'; import { WorkspaceActivationStatus } from 'twenty-shared/workspace'; -import { DataSource, Repository } from 'typeorm'; +import { DataSource, QueryRunner, Repository } from 'typeorm'; import { ApiKeyEntity } from 'src/engine/core-modules/api-key/api-key.entity'; import { BillingSubscriptionService } from 'src/engine/core-modules/billing/services/billing-subscription.service'; @@ -36,8 +36,10 @@ import { WorkspaceExceptionCode, WorkspaceNotFoundDefaultError, } from 'src/engine/core-modules/workspace/workspace.exception'; +import { FieldMetadataEntity } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity'; +import { ALL_METADATA_ENTITY_BY_METADATA_NAME } from 'src/engine/metadata-modules/flat-entity/constant/all-metadata-entity-by-metadata-name.constant'; +import { ALL_METADATA_NAMES_SORTED_ATOMICALLY } from 'src/engine/metadata-modules/flat-entity/constant/all-metadata-names-sorted-atomically.constant'; import { WorkspaceManyOrAllFlatEntityMapsCacheService } from 'src/engine/metadata-modules/flat-entity/services/workspace-many-or-all-flat-entity-maps-cache.service'; -import { ObjectMetadataService } from 'src/engine/metadata-modules/object-metadata/object-metadata.service'; import { PermissionsException, PermissionsExceptionCode, @@ -103,7 +105,6 @@ export class WorkspaceService extends TypeOrmQueryService { private readonly workspaceCacheStorageService: WorkspaceCacheStorageService, private readonly subdomainManagerService: SubdomainManagerService, private readonly workspaceDataSourceService: WorkspaceDataSourceService, - private readonly objectMetadataService: ObjectMetadataService, private readonly customDomainManagerService: CustomDomainManagerService, @InjectMessageQueue(MessageQueue.deleteCascadeQueue) private readonly messageQueueService: MessageQueueService, @@ -322,10 +323,7 @@ export class WorkspaceService extends TypeOrmQueryService { return workspace; } - // Note: not relying on workspace id FK cascade deletion here to avoid query read timeout later on workspace deletion - await this.objectMetadataService.deleteWorkspaceAllObjectMetadata({ - workspaceId: workspace.id, - }); + await this.deleteWorkspaceSyncableMetadataEntities(workspace); await this.workspaceDataSourceService.deleteWorkspaceDBSchema(workspace.id); @@ -356,6 +354,130 @@ export class WorkspaceService extends TypeOrmQueryService { return workspace; } + private async deleteWorkspaceSyncableMetadataEntities( + workspace: WorkspaceEntity, + ): Promise { + const queryRunner = this.coreDataSource.createQueryRunner(); + + await queryRunner.connect(); + + try { + await queryRunner.startTransaction(); + + for (const metadataName of ALL_METADATA_NAMES_SORTED_ATOMICALLY) { + if (metadataName === 'fieldMetadata') { + const deletedCount = await this.deleteFieldMetadataInChunks( + queryRunner, + workspace.id, + ); + + if (deletedCount > 0) { + this.logger.log( + `workspace ${workspace.id}: deleted ${deletedCount} ${metadataName} record(s)`, + ); + } + + continue; + } + + const entity = ALL_METADATA_ENTITY_BY_METADATA_NAME[metadataName]; + + const result = await queryRunner.manager.delete(entity, { + workspaceId: workspace.id, + }); + + if (result.affected && result.affected > 0) { + this.logger.log( + `workspace ${workspace.id}: deleted ${result.affected} ${metadataName} record(s)`, + ); + } + } + + await queryRunner.commitTransaction(); + } catch (error) { + await queryRunner.rollbackTransaction(); + + throw error; + } finally { + await queryRunner.release(); + } + } + + // FieldMetadataEntity has a self-referencing FK (relationTargetFieldMetadataId) + // Related fields must be deleted together to avoid constraint violations + private async deleteFieldMetadataInChunks( + queryRunner: QueryRunner, + workspaceId: string, + ): Promise { + const CHUNK_SIZE = 50; + let totalDeleted = 0; + + const { flatFieldMetadataMaps } = + await this.flatEntityMapsCacheService.getOrRecomputeManyOrAllFlatEntityMaps( + { + workspaceId, + flatMapsKeys: ['flatFieldMetadataMaps'], + }, + ); + + const fields = Object.values(flatFieldMetadataMaps.byId).filter(isDefined); + + if (fields.length === 0) { + return 0; + } + + const processedIds = new Set(); + const fieldsMap = new Map(fields.map((field) => [field.id, field])); + const chunks: string[][] = []; + let currentChunk: string[] = []; + + for (const field of fields) { + if (processedIds.has(field.id)) { + continue; + } + + currentChunk.push(field.id); + processedIds.add(field.id); + + if (field.relationTargetFieldMetadataId) { + const relatedField = fieldsMap.get(field.relationTargetFieldMetadataId); + + if (relatedField && !processedIds.has(relatedField.id)) { + currentChunk.push(relatedField.id); + processedIds.add(relatedField.id); + } + } + + if (currentChunk.length >= CHUNK_SIZE) { + chunks.push([...currentChunk]); + currentChunk = []; + } + } + + if (currentChunk.length > 0) { + chunks.push(currentChunk); + } + + for (const [index, chunk] of chunks.entries()) { + const result = await queryRunner.manager + .createQueryBuilder() + .delete() + .from(FieldMetadataEntity) + .whereInIds(chunk) + .execute(); + + const deletedInChunk = result.affected || 0; + + totalDeleted += deletedInChunk; + + this.logger.log( + `workspace ${workspaceId}: fieldMetadata chunk ${index + 1}/${chunks.length} - deleted ${deletedInChunk} record(s)`, + ); + } + + return totalDeleted; + } + async handleRemoveWorkspaceMember( workspaceId: string, userId: string, diff --git a/packages/twenty-server/src/engine/metadata-modules/command-menu-item/entities/command-menu-item.entity.ts b/packages/twenty-server/src/engine/metadata-modules/command-menu-item/entities/command-menu-item.entity.ts index e2443913d8..20b7d57f1f 100644 --- a/packages/twenty-server/src/engine/metadata-modules/command-menu-item/entities/command-menu-item.entity.ts +++ b/packages/twenty-server/src/engine/metadata-modules/command-menu-item/entities/command-menu-item.entity.ts @@ -24,6 +24,9 @@ export enum CommandMenuItemAvailabilityType { 'workflowVersionId', 'workspaceId', ]) +@Index('IDX_COMMAND_MENU_ITEM_AVAILABILITY_OBJECT_METADATA_ID', [ + 'availabilityObjectMetadataId', +]) export class CommandMenuItemEntity extends SyncableEntityRequired implements Required diff --git a/packages/twenty-server/src/engine/metadata-modules/cron-trigger/entities/cron-trigger.entity.ts b/packages/twenty-server/src/engine/metadata-modules/cron-trigger/entities/cron-trigger.entity.ts index 16fe7b3fb4..282ebe5158 100644 --- a/packages/twenty-server/src/engine/metadata-modules/cron-trigger/entities/cron-trigger.entity.ts +++ b/packages/twenty-server/src/engine/metadata-modules/cron-trigger/entities/cron-trigger.entity.ts @@ -19,6 +19,7 @@ export type CronTriggerSettings = { @Entity({ name: 'cronTrigger', schema: 'core' }) @Index('IDX_CRON_TRIGGER_WORKSPACE_ID', ['workspaceId']) +@Index('IDX_CRON_TRIGGER_SERVERLESS_FUNCTION_ID', ['serverlessFunctionId']) export class CronTriggerEntity extends SyncableEntity implements Required diff --git a/packages/twenty-server/src/engine/metadata-modules/database-event-trigger/entities/database-event-trigger.entity.ts b/packages/twenty-server/src/engine/metadata-modules/database-event-trigger/entities/database-event-trigger.entity.ts index 1dc24b847c..7cc8b97bcf 100644 --- a/packages/twenty-server/src/engine/metadata-modules/database-event-trigger/entities/database-event-trigger.entity.ts +++ b/packages/twenty-server/src/engine/metadata-modules/database-event-trigger/entities/database-event-trigger.entity.ts @@ -19,6 +19,9 @@ export type DatabaseEventTriggerSettings = { @Entity('databaseEventTrigger') @Index('IDX_DATABASE_EVENT_TRIGGER_WORKSPACE_ID', ['workspaceId']) +@Index('IDX_DATABASE_EVENT_TRIGGER_SERVERLESS_FUNCTION_ID', [ + 'serverlessFunctionId', +]) export class DatabaseEventTriggerEntity extends SyncableEntity implements Required diff --git a/packages/twenty-server/src/engine/metadata-modules/flat-entity/constant/all-metadata-entity-by-metadata-name.constant.ts b/packages/twenty-server/src/engine/metadata-modules/flat-entity/constant/all-metadata-entity-by-metadata-name.constant.ts new file mode 100644 index 0000000000..12462cc1c4 --- /dev/null +++ b/packages/twenty-server/src/engine/metadata-modules/flat-entity/constant/all-metadata-entity-by-metadata-name.constant.ts @@ -0,0 +1,52 @@ +import { type AllMetadataName } from 'twenty-shared/metadata'; +import { type EntityTarget, type ObjectLiteral } from 'typeorm'; + +import { AgentEntity } from 'src/engine/metadata-modules/ai/ai-agent/entities/agent.entity'; +import { CommandMenuItemEntity } from 'src/engine/metadata-modules/command-menu-item/entities/command-menu-item.entity'; +import { CronTriggerEntity } from 'src/engine/metadata-modules/cron-trigger/entities/cron-trigger.entity'; +import { DatabaseEventTriggerEntity } from 'src/engine/metadata-modules/database-event-trigger/entities/database-event-trigger.entity'; +import { FieldMetadataEntity } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity'; +import { FrontComponentEntity } from 'src/engine/metadata-modules/front-component/entities/front-component.entity'; +import { IndexMetadataEntity } from 'src/engine/metadata-modules/index-metadata/index-metadata.entity'; +import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity'; +import { PageLayoutTabEntity } from 'src/engine/metadata-modules/page-layout-tab/entities/page-layout-tab.entity'; +import { PageLayoutWidgetEntity } from 'src/engine/metadata-modules/page-layout-widget/entities/page-layout-widget.entity'; +import { PageLayoutEntity } from 'src/engine/metadata-modules/page-layout/entities/page-layout.entity'; +import { RoleTargetEntity } from 'src/engine/metadata-modules/role-target/role-target.entity'; +import { RoleEntity } from 'src/engine/metadata-modules/role/role.entity'; +import { RouteTriggerEntity } from 'src/engine/metadata-modules/route-trigger/route-trigger.entity'; +import { RowLevelPermissionPredicateGroupEntity } from 'src/engine/metadata-modules/row-level-permission-predicate/entities/row-level-permission-predicate-group.entity'; +import { RowLevelPermissionPredicateEntity } from 'src/engine/metadata-modules/row-level-permission-predicate/entities/row-level-permission-predicate.entity'; +import { ServerlessFunctionEntity } from 'src/engine/metadata-modules/serverless-function/serverless-function.entity'; +import { SkillEntity } from 'src/engine/metadata-modules/skill/entities/skill.entity'; +import { ViewFieldEntity } from 'src/engine/metadata-modules/view-field/entities/view-field.entity'; +import { ViewFilterGroupEntity } from 'src/engine/metadata-modules/view-filter-group/entities/view-filter-group.entity'; +import { ViewFilterEntity } from 'src/engine/metadata-modules/view-filter/entities/view-filter.entity'; +import { ViewGroupEntity } from 'src/engine/metadata-modules/view-group/entities/view-group.entity'; +import { ViewEntity } from 'src/engine/metadata-modules/view/entities/view.entity'; + +export const ALL_METADATA_ENTITY_BY_METADATA_NAME = { + viewField: ViewFieldEntity, + viewFilter: ViewFilterEntity, + viewGroup: ViewGroupEntity, + viewFilterGroup: ViewFilterGroupEntity, + roleTarget: RoleTargetEntity, + rowLevelPermissionPredicate: RowLevelPermissionPredicateEntity, + pageLayoutWidget: PageLayoutWidgetEntity, + rowLevelPermissionPredicateGroup: RowLevelPermissionPredicateGroupEntity, + view: ViewEntity, + index: IndexMetadataEntity, + pageLayoutTab: PageLayoutTabEntity, + routeTrigger: RouteTriggerEntity, + cronTrigger: CronTriggerEntity, + databaseEventTrigger: DatabaseEventTriggerEntity, + frontComponent: FrontComponentEntity, + fieldMetadata: FieldMetadataEntity, + pageLayout: PageLayoutEntity, + skill: SkillEntity, + serverlessFunction: ServerlessFunctionEntity, + objectMetadata: ObjectMetadataEntity, + role: RoleEntity, + agent: AgentEntity, + commandMenuItem: CommandMenuItemEntity, +} as const satisfies Record>; diff --git a/packages/twenty-server/src/engine/metadata-modules/flat-entity/constant/all-metadata-names-sorted-atomically.constant.ts b/packages/twenty-server/src/engine/metadata-modules/flat-entity/constant/all-metadata-names-sorted-atomically.constant.ts new file mode 100644 index 0000000000..a62783ec0e --- /dev/null +++ b/packages/twenty-server/src/engine/metadata-modules/flat-entity/constant/all-metadata-names-sorted-atomically.constant.ts @@ -0,0 +1,4 @@ +import { sortMetadataNamesChildrenFirst } from 'src/engine/metadata-modules/flat-entity/utils/sort-metadata-names-children-first.util'; + +export const ALL_METADATA_NAMES_SORTED_ATOMICALLY = + sortMetadataNamesChildrenFirst(); diff --git a/packages/twenty-server/src/engine/metadata-modules/flat-entity/types/all-flat-entity-types-by-metadata-name.ts b/packages/twenty-server/src/engine/metadata-modules/flat-entity/types/all-flat-entity-types-by-metadata-name.ts index a1d3b5ca25..bcb734ec25 100644 --- a/packages/twenty-server/src/engine/metadata-modules/flat-entity/types/all-flat-entity-types-by-metadata-name.ts +++ b/packages/twenty-server/src/engine/metadata-modules/flat-entity/types/all-flat-entity-types-by-metadata-name.ts @@ -1,12 +1,8 @@ -import { type AgentEntity } from 'src/engine/metadata-modules/ai/ai-agent/entities/agent.entity'; -import { type CommandMenuItemEntity } from 'src/engine/metadata-modules/command-menu-item/entities/command-menu-item.entity'; -import { type CronTriggerEntity } from 'src/engine/metadata-modules/cron-trigger/entities/cron-trigger.entity'; import { type FlatCronTrigger } from 'src/engine/metadata-modules/cron-trigger/types/flat-cron-trigger.type'; -import { type DatabaseEventTriggerEntity } from 'src/engine/metadata-modules/database-event-trigger/entities/database-event-trigger.entity'; import { type FlatDatabaseEventTrigger } from 'src/engine/metadata-modules/database-event-trigger/types/flat-database-event-trigger.type'; -import { type FieldMetadataEntity } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity'; import { type FlatAgent } from 'src/engine/metadata-modules/flat-agent/types/flat-agent.type'; import { type FlatCommandMenuItem } from 'src/engine/metadata-modules/flat-command-menu-item/types/flat-command-menu-item.type'; +import { type MetadataEntity } from 'src/engine/metadata-modules/flat-entity/types/metadata-entity.type'; import { type FlatFieldMetadata } from 'src/engine/metadata-modules/flat-field-metadata/types/flat-field-metadata.type'; import { type FlatFrontComponent } from 'src/engine/metadata-modules/flat-front-component/types/flat-front-component.type'; import { type FlatIndexMetadata } from 'src/engine/metadata-modules/flat-index-metadata/types/flat-index-metadata.type'; @@ -22,28 +18,10 @@ import { type FlatViewFilterGroup } from 'src/engine/metadata-modules/flat-view- import { type FlatViewFilter } from 'src/engine/metadata-modules/flat-view-filter/types/flat-view-filter.type'; import { type FlatViewGroup } from 'src/engine/metadata-modules/flat-view-group/types/flat-view-group.type'; import { type FlatView } from 'src/engine/metadata-modules/flat-view/types/flat-view.type'; -import { type FrontComponentEntity } from 'src/engine/metadata-modules/front-component/entities/front-component.entity'; -import { type IndexMetadataEntity } from 'src/engine/metadata-modules/index-metadata/index-metadata.entity'; -import { type ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity'; -import { type PageLayoutTabEntity } from 'src/engine/metadata-modules/page-layout-tab/entities/page-layout-tab.entity'; -import { type PageLayoutWidgetEntity } from 'src/engine/metadata-modules/page-layout-widget/entities/page-layout-widget.entity'; -import { type PageLayoutEntity } from 'src/engine/metadata-modules/page-layout/entities/page-layout.entity'; -import { type RoleTargetEntity } from 'src/engine/metadata-modules/role-target/role-target.entity'; -import { type RoleEntity } from 'src/engine/metadata-modules/role/role.entity'; -import { type RouteTriggerEntity } from 'src/engine/metadata-modules/route-trigger/route-trigger.entity'; import { type FlatRouteTrigger } from 'src/engine/metadata-modules/route-trigger/types/flat-route-trigger.type'; -import { type RowLevelPermissionPredicateGroupEntity } from 'src/engine/metadata-modules/row-level-permission-predicate/entities/row-level-permission-predicate-group.entity'; -import { type RowLevelPermissionPredicateEntity } from 'src/engine/metadata-modules/row-level-permission-predicate/entities/row-level-permission-predicate.entity'; import { type FlatRowLevelPermissionPredicateGroup } from 'src/engine/metadata-modules/row-level-permission-predicate/types/flat-row-level-permission-predicate-group.type'; import { type FlatRowLevelPermissionPredicate } from 'src/engine/metadata-modules/row-level-permission-predicate/types/flat-row-level-permission-predicate.type'; -import { type ServerlessFunctionEntity } from 'src/engine/metadata-modules/serverless-function/serverless-function.entity'; import { type FlatServerlessFunction } from 'src/engine/metadata-modules/serverless-function/types/flat-serverless-function.type'; -import { type SkillEntity } from 'src/engine/metadata-modules/skill/entities/skill.entity'; -import { type ViewFieldEntity } from 'src/engine/metadata-modules/view-field/entities/view-field.entity'; -import { type ViewFilterGroupEntity } from 'src/engine/metadata-modules/view-filter-group/entities/view-filter-group.entity'; -import { type ViewFilterEntity } from 'src/engine/metadata-modules/view-filter/entities/view-filter.entity'; -import { type ViewGroupEntity } from 'src/engine/metadata-modules/view-group/entities/view-group.entity'; -import { type ViewEntity } from 'src/engine/metadata-modules/view/entities/view.entity'; import { type CreateAgentAction, type DeleteAgentAction, @@ -168,7 +146,7 @@ export type AllFlatEntityTypesByMetadataName = { delete: DeleteFieldAction; }; flatEntity: FlatFieldMetadata; - entity: FieldMetadataEntity; + entity: MetadataEntity<'fieldMetadata'>; }; objectMetadata: { actions: { @@ -177,7 +155,7 @@ export type AllFlatEntityTypesByMetadataName = { delete: DeleteObjectAction; }; flatEntity: FlatObjectMetadata; - entity: ObjectMetadataEntity; + entity: MetadataEntity<'objectMetadata'>; }; view: { actions: { @@ -186,7 +164,7 @@ export type AllFlatEntityTypesByMetadataName = { delete: DeleteViewAction; }; flatEntity: FlatView; - entity: ViewEntity; + entity: MetadataEntity<'view'>; }; viewField: { actions: { @@ -195,7 +173,7 @@ export type AllFlatEntityTypesByMetadataName = { delete: DeleteViewFieldAction; }; flatEntity: FlatViewField; - entity: ViewFieldEntity; + entity: MetadataEntity<'viewField'>; }; viewGroup: { actions: { @@ -204,7 +182,7 @@ export type AllFlatEntityTypesByMetadataName = { delete: DeleteViewGroupAction; }; flatEntity: FlatViewGroup; - entity: ViewGroupEntity; + entity: MetadataEntity<'viewGroup'>; }; rowLevelPermissionPredicate: { actions: { @@ -213,7 +191,7 @@ export type AllFlatEntityTypesByMetadataName = { delete: DeleteRowLevelPermissionPredicateAction; }; flatEntity: FlatRowLevelPermissionPredicate; - entity: RowLevelPermissionPredicateEntity; + entity: MetadataEntity<'rowLevelPermissionPredicate'>; }; rowLevelPermissionPredicateGroup: { actions: { @@ -222,7 +200,7 @@ export type AllFlatEntityTypesByMetadataName = { delete: DeleteRowLevelPermissionPredicateGroupAction; }; flatEntity: FlatRowLevelPermissionPredicateGroup; - entity: RowLevelPermissionPredicateGroupEntity; + entity: MetadataEntity<'rowLevelPermissionPredicateGroup'>; }; viewFilterGroup: { actions: { @@ -231,7 +209,7 @@ export type AllFlatEntityTypesByMetadataName = { delete: DeleteViewFilterGroupAction; }; flatEntity: FlatViewFilterGroup; - entity: ViewFilterGroupEntity; + entity: MetadataEntity<'viewFilterGroup'>; }; index: { actions: { @@ -240,7 +218,7 @@ export type AllFlatEntityTypesByMetadataName = { delete: DeleteIndexAction; }; flatEntity: FlatIndexMetadata; - entity: IndexMetadataEntity; + entity: MetadataEntity<'index'>; }; serverlessFunction: { actions: { @@ -249,7 +227,7 @@ export type AllFlatEntityTypesByMetadataName = { delete: DeleteServerlessFunctionAction; }; flatEntity: FlatServerlessFunction; - entity: ServerlessFunctionEntity; + entity: MetadataEntity<'serverlessFunction'>; }; cronTrigger: { actions: { @@ -258,7 +236,7 @@ export type AllFlatEntityTypesByMetadataName = { delete: DeleteCronTriggerAction; }; flatEntity: FlatCronTrigger; - entity: CronTriggerEntity; + entity: MetadataEntity<'cronTrigger'>; }; databaseEventTrigger: { actions: { @@ -267,7 +245,7 @@ export type AllFlatEntityTypesByMetadataName = { delete: DeleteDatabaseEventTriggerAction; }; flatEntity: FlatDatabaseEventTrigger; - entity: DatabaseEventTriggerEntity; + entity: MetadataEntity<'databaseEventTrigger'>; }; routeTrigger: { actions: { @@ -276,7 +254,7 @@ export type AllFlatEntityTypesByMetadataName = { delete: DeleteRouteTriggerAction; }; flatEntity: FlatRouteTrigger; - entity: RouteTriggerEntity; + entity: MetadataEntity<'routeTrigger'>; }; viewFilter: { actions: { @@ -285,7 +263,7 @@ export type AllFlatEntityTypesByMetadataName = { delete: DeleteViewFilterAction; }; flatEntity: FlatViewFilter; - entity: ViewFilterEntity; + entity: MetadataEntity<'viewFilter'>; }; role: { actions: { @@ -294,7 +272,7 @@ export type AllFlatEntityTypesByMetadataName = { delete: DeleteRoleAction; }; flatEntity: FlatRole; - entity: RoleEntity; + entity: MetadataEntity<'role'>; }; roleTarget: { actions: { @@ -303,7 +281,7 @@ export type AllFlatEntityTypesByMetadataName = { delete: DeleteRoleTargetAction; }; flatEntity: FlatRoleTarget; - entity: RoleTargetEntity; + entity: MetadataEntity<'roleTarget'>; }; agent: { actions: { @@ -312,7 +290,7 @@ export type AllFlatEntityTypesByMetadataName = { delete: DeleteAgentAction; }; flatEntity: FlatAgent; - entity: AgentEntity; + entity: MetadataEntity<'agent'>; }; skill: { actions: { @@ -321,7 +299,7 @@ export type AllFlatEntityTypesByMetadataName = { delete: DeleteSkillAction; }; flatEntity: FlatSkill; - entity: SkillEntity; + entity: MetadataEntity<'skill'>; }; commandMenuItem: { actions: { @@ -330,7 +308,7 @@ export type AllFlatEntityTypesByMetadataName = { delete: DeleteCommandMenuItemAction; }; flatEntity: FlatCommandMenuItem; - entity: CommandMenuItemEntity; + entity: MetadataEntity<'commandMenuItem'>; }; pageLayout: { actions: { @@ -339,7 +317,7 @@ export type AllFlatEntityTypesByMetadataName = { delete: DeletePageLayoutAction; }; flatEntity: FlatPageLayout; - entity: PageLayoutEntity; + entity: MetadataEntity<'pageLayout'>; }; pageLayoutWidget: { actions: { @@ -348,7 +326,7 @@ export type AllFlatEntityTypesByMetadataName = { delete: DeletePageLayoutWidgetAction; }; flatEntity: FlatPageLayoutWidget; - entity: PageLayoutWidgetEntity; + entity: MetadataEntity<'pageLayoutWidget'>; }; pageLayoutTab: { actions: { @@ -357,7 +335,7 @@ export type AllFlatEntityTypesByMetadataName = { delete: DeletePageLayoutTabAction; }; flatEntity: FlatPageLayoutTab; - entity: PageLayoutTabEntity; + entity: MetadataEntity<'pageLayoutTab'>; }; frontComponent: { actions: { @@ -366,6 +344,6 @@ export type AllFlatEntityTypesByMetadataName = { delete: DeleteFrontComponentAction; }; flatEntity: FlatFrontComponent; - entity: FrontComponentEntity; + entity: MetadataEntity<'frontComponent'>; }; }; diff --git a/packages/twenty-server/src/engine/metadata-modules/flat-entity/types/metadata-entity.type.ts b/packages/twenty-server/src/engine/metadata-modules/flat-entity/types/metadata-entity.type.ts index a6521d7738..889d95d2c0 100644 --- a/packages/twenty-server/src/engine/metadata-modules/flat-entity/types/metadata-entity.type.ts +++ b/packages/twenty-server/src/engine/metadata-modules/flat-entity/types/metadata-entity.type.ts @@ -1,6 +1,7 @@ import { type AllMetadataName } from 'twenty-shared/metadata'; -import { type AllFlatEntityTypesByMetadataName } from 'src/engine/metadata-modules/flat-entity/types/all-flat-entity-types-by-metadata-name'; +import { type ALL_METADATA_ENTITY_BY_METADATA_NAME } from 'src/engine/metadata-modules/flat-entity/constant/all-metadata-entity-by-metadata-name.constant'; -export type MetadataEntity = - AllFlatEntityTypesByMetadataName[T]['entity']; +export type MetadataEntity = InstanceType< + (typeof ALL_METADATA_ENTITY_BY_METADATA_NAME)[K] +>; diff --git a/packages/twenty-server/src/engine/metadata-modules/flat-entity/utils/__tests__/__snapshots__/sort-metadata-names-children-first.util.spec.ts.snap b/packages/twenty-server/src/engine/metadata-modules/flat-entity/utils/__tests__/__snapshots__/sort-metadata-names-children-first.util.spec.ts.snap new file mode 100644 index 0000000000..82689758b5 --- /dev/null +++ b/packages/twenty-server/src/engine/metadata-modules/flat-entity/utils/__tests__/__snapshots__/sort-metadata-names-children-first.util.spec.ts.snap @@ -0,0 +1,29 @@ +// Jest Snapshot v1, https://jestjs.io/docs/snapshot-testing + +exports[`sortMetadataNamesChildrenFirst should return metadata names sorted with children first (most manyToOne relations first) 1`] = ` +[ + "rowLevelPermissionPredicate", + "viewFilter", + "pageLayoutWidget", + "viewField", + "commandMenuItem", + "cronTrigger", + "databaseEventTrigger", + "index", + "roleTarget", + "routeTrigger", + "rowLevelPermissionPredicateGroup", + "viewGroup", + "agent", + "frontComponent", + "pageLayout", + "pageLayoutTab", + "skill", + "view", + "viewFilterGroup", + "fieldMetadata", + "role", + "serverlessFunction", + "objectMetadata", +] +`; diff --git a/packages/twenty-server/src/engine/metadata-modules/flat-entity/utils/__tests__/sort-metadata-names-children-first.util.spec.ts b/packages/twenty-server/src/engine/metadata-modules/flat-entity/utils/__tests__/sort-metadata-names-children-first.util.spec.ts new file mode 100644 index 0000000000..34690921de --- /dev/null +++ b/packages/twenty-server/src/engine/metadata-modules/flat-entity/utils/__tests__/sort-metadata-names-children-first.util.spec.ts @@ -0,0 +1,9 @@ +import { sortMetadataNamesChildrenFirst } from 'src/engine/metadata-modules/flat-entity/utils/sort-metadata-names-children-first.util'; + +describe('sortMetadataNamesChildrenFirst', () => { + it('should return metadata names sorted with children first (most manyToOne relations first)', () => { + const result = sortMetadataNamesChildrenFirst(); + + expect(result).toMatchSnapshot(); + }); +}); diff --git a/packages/twenty-server/src/engine/metadata-modules/flat-entity/utils/get-metadata-many-to-one-related-names.util.ts b/packages/twenty-server/src/engine/metadata-modules/flat-entity/utils/get-metadata-many-to-one-related-names.util.ts new file mode 100644 index 0000000000..0e5b5aade3 --- /dev/null +++ b/packages/twenty-server/src/engine/metadata-modules/flat-entity/utils/get-metadata-many-to-one-related-names.util.ts @@ -0,0 +1,16 @@ +import { type AllMetadataName } from 'twenty-shared/metadata'; + +import { ALL_METADATA_RELATIONS } from 'src/engine/metadata-modules/flat-entity/constant/all-metadata-relations.constant'; +import { type MetadataManyToOneRelatedMetadataNames } from 'src/engine/metadata-modules/flat-entity/types/metadata-many-to-one-related-metadata-names.type'; + +export const getMetadataManyToOneRelatedNames = ( + metadataName: T, +): MetadataManyToOneRelatedMetadataNames[] => { + const relations = ALL_METADATA_RELATIONS[metadataName]; + + return Object.values(relations.manyToOne) + .filter((relation) => relation !== null) + .map( + (relation) => relation.metadataName, + ) as MetadataManyToOneRelatedMetadataNames[]; +}; diff --git a/packages/twenty-server/src/engine/metadata-modules/flat-entity/utils/get-metadata-one-to-many-related-names.util.ts b/packages/twenty-server/src/engine/metadata-modules/flat-entity/utils/get-metadata-one-to-many-related-names.util.ts new file mode 100644 index 0000000000..0369e7aa80 --- /dev/null +++ b/packages/twenty-server/src/engine/metadata-modules/flat-entity/utils/get-metadata-one-to-many-related-names.util.ts @@ -0,0 +1,16 @@ +import { type AllMetadataName } from 'twenty-shared/metadata'; + +import { ALL_METADATA_RELATIONS } from 'src/engine/metadata-modules/flat-entity/constant/all-metadata-relations.constant'; +import { type MetadataOneToManyRelatedMetadataNames } from 'src/engine/metadata-modules/flat-entity/types/metadata-one-to-many-related-metadata-names.type'; + +export const getMetadataOneToManyRelatedNames = ( + metadataName: T, +): MetadataOneToManyRelatedMetadataNames[] => { + const relations = ALL_METADATA_RELATIONS[metadataName]; + + return Object.values(relations.oneToMany) + .filter((relation) => relation !== null) + .map( + (relation) => relation.metadataName, + ) as MetadataOneToManyRelatedMetadataNames[]; +}; diff --git a/packages/twenty-server/src/engine/metadata-modules/flat-entity/utils/get-metadata-related-metadata-names.util.ts b/packages/twenty-server/src/engine/metadata-modules/flat-entity/utils/get-metadata-related-metadata-names.util.ts index 3f49220442..f1348610c9 100644 --- a/packages/twenty-server/src/engine/metadata-modules/flat-entity/utils/get-metadata-related-metadata-names.util.ts +++ b/packages/twenty-server/src/engine/metadata-modules/flat-entity/utils/get-metadata-related-metadata-names.util.ts @@ -1,21 +1,15 @@ import { type AllMetadataName } from 'twenty-shared/metadata'; -import { ALL_METADATA_RELATIONS } from 'src/engine/metadata-modules/flat-entity/constant/all-metadata-relations.constant'; import { type MetadataManyToOneRelatedMetadataNames } from 'src/engine/metadata-modules/flat-entity/types/metadata-many-to-one-related-metadata-names.type'; import { type MetadataOneToManyRelatedMetadataNames } from 'src/engine/metadata-modules/flat-entity/types/metadata-one-to-many-related-metadata-names.type'; +import { getMetadataManyToOneRelatedNames } from 'src/engine/metadata-modules/flat-entity/utils/get-metadata-many-to-one-related-names.util'; +import { getMetadataOneToManyRelatedNames } from 'src/engine/metadata-modules/flat-entity/utils/get-metadata-one-to-many-related-names.util'; export const getMetadataRelatedMetadataNames = ( metadataName: T, ) => { - const relationProperties = ALL_METADATA_RELATIONS[metadataName]; - - const manyToOneMetadataNames = Object.values(relationProperties.manyToOne) - .filter((relation) => relation !== null) - .map((relation) => relation.metadataName); - - const oneToManyMetadataNames = Object.values(relationProperties.oneToMany) - .filter((relation) => relation !== null) - .map((relation) => relation.metadataName); + const manyToOneMetadataNames = getMetadataManyToOneRelatedNames(metadataName); + const oneToManyMetadataNames = getMetadataOneToManyRelatedNames(metadataName); return [ ...new Set([...manyToOneMetadataNames, ...oneToManyMetadataNames]), diff --git a/packages/twenty-server/src/engine/metadata-modules/flat-entity/utils/sort-metadata-names-children-first.util.ts b/packages/twenty-server/src/engine/metadata-modules/flat-entity/utils/sort-metadata-names-children-first.util.ts new file mode 100644 index 0000000000..58ff917d03 --- /dev/null +++ b/packages/twenty-server/src/engine/metadata-modules/flat-entity/utils/sort-metadata-names-children-first.util.ts @@ -0,0 +1,29 @@ +import { + ALL_METADATA_NAME, + type AllMetadataName, +} from 'twenty-shared/metadata'; + +import { getMetadataManyToOneRelatedNames } from 'src/engine/metadata-modules/flat-entity/utils/get-metadata-many-to-one-related-names.util'; +import { getMetadataOneToManyRelatedNames } from 'src/engine/metadata-modules/flat-entity/utils/get-metadata-one-to-many-related-names.util'; + +const computeDependencyScore = (metadataName: AllMetadataName): number => { + const manyToOneCount = getMetadataManyToOneRelatedNames(metadataName).length; + const oneToManyCount = getMetadataOneToManyRelatedNames(metadataName).length; + + return manyToOneCount - oneToManyCount; +}; + +export const sortMetadataNamesChildrenFirst = (): AllMetadataName[] => { + const metadataNames = Object.keys(ALL_METADATA_NAME) as AllMetadataName[]; + + return metadataNames.sort((metadataNameA, metadataNameB) => { + const scoreA = computeDependencyScore(metadataNameA); + const scoreB = computeDependencyScore(metadataNameB); + + if (scoreA !== scoreB) { + return scoreB - scoreA; + } + + return metadataNameA.localeCompare(metadataNameB); + }); +}; diff --git a/packages/twenty-server/src/engine/metadata-modules/object-metadata/object-metadata.entity.ts b/packages/twenty-server/src/engine/metadata-modules/object-metadata/object-metadata.entity.ts index a65b1229cf..d48021ae34 100644 --- a/packages/twenty-server/src/engine/metadata-modules/object-metadata/object-metadata.entity.ts +++ b/packages/twenty-server/src/engine/metadata-modules/object-metadata/object-metadata.entity.ts @@ -2,6 +2,7 @@ import { Column, CreateDateColumn, Entity, + Index, ManyToOne, OneToMany, PrimaryGeneratedColumn, @@ -29,6 +30,7 @@ import { SyncableEntityRequired } from 'src/engine/workspace-manager/types/synca 'namePlural', 'workspaceId', ]) +@Index('IDX_OBJECT_METADATA_DATA_SOURCE_ID', ['dataSourceId']) export class ObjectMetadataEntity extends SyncableEntityRequired implements Required diff --git a/packages/twenty-server/src/engine/metadata-modules/page-layout-widget/entities/page-layout-widget.entity.ts b/packages/twenty-server/src/engine/metadata-modules/page-layout-widget/entities/page-layout-widget.entity.ts index e5c79e12c5..ad065cf889 100644 --- a/packages/twenty-server/src/engine/metadata-modules/page-layout-widget/entities/page-layout-widget.entity.ts +++ b/packages/twenty-server/src/engine/metadata-modules/page-layout-widget/entities/page-layout-widget.entity.ts @@ -28,6 +28,7 @@ import { SyncableEntity } from 'src/engine/workspace-manager/types/syncable-enti ['workspaceId', 'pageLayoutTabId'], { where: '"deletedAt" IS NULL' }, ) +@Index('IDX_PAGE_LAYOUT_WIDGET_OBJECT_METADATA_ID', ['objectMetadataId']) export class PageLayoutWidgetEntity< TWidgetConfigurationType extends WidgetConfigurationType = WidgetConfigurationType, diff --git a/packages/twenty-server/src/engine/metadata-modules/role-target/role-target.entity.ts b/packages/twenty-server/src/engine/metadata-modules/role-target/role-target.entity.ts index d5025f7288..95eb8cd6de 100644 --- a/packages/twenty-server/src/engine/metadata-modules/role-target/role-target.entity.ts +++ b/packages/twenty-server/src/engine/metadata-modules/role-target/role-target.entity.ts @@ -26,6 +26,7 @@ import { SyncableEntity } from 'src/engine/workspace-manager/types/syncable-enti @Index('IDX_ROLE_TARGET_WORKSPACE_ID', ['userWorkspaceId', 'workspaceId']) @Index('IDX_ROLE_TARGET_AGENT_ID', ['agentId']) @Index('IDX_ROLE_TARGET_API_KEY_ID', ['apiKeyId']) +@Index('IDX_ROLE_TARGET_ROLE_ID', ['roleId']) @Check( 'CHK_role_target_single_entity', '("agentId" IS NOT NULL AND "userWorkspaceId" IS NULL AND "apiKeyId" IS NULL) OR ("agentId" IS NULL AND "userWorkspaceId" IS NOT NULL AND "apiKeyId" IS NULL) OR ("agentId" IS NULL AND "userWorkspaceId" IS NULL AND "apiKeyId" IS NOT NULL)', diff --git a/packages/twenty-server/src/engine/metadata-modules/route-trigger/route-trigger.entity.ts b/packages/twenty-server/src/engine/metadata-modules/route-trigger/route-trigger.entity.ts index 4682ecd906..9586970c91 100644 --- a/packages/twenty-server/src/engine/metadata-modules/route-trigger/route-trigger.entity.ts +++ b/packages/twenty-server/src/engine/metadata-modules/route-trigger/route-trigger.entity.ts @@ -3,6 +3,7 @@ import { Column, CreateDateColumn, Entity, + Index, JoinColumn, ManyToOne, PrimaryGeneratedColumn, @@ -20,6 +21,7 @@ import { SyncableEntity } from 'src/engine/workspace-manager/types/syncable-enti 'httpMethod', 'workspaceId', ]) +@Index('IDX_ROUTE_TRIGGER_SERVERLESS_FUNCTION_ID', ['serverlessFunctionId']) export class RouteTriggerEntity extends SyncableEntity implements Required diff --git a/packages/twenty-server/src/engine/metadata-modules/row-level-permission-predicate/entities/row-level-permission-predicate-group.entity.ts b/packages/twenty-server/src/engine/metadata-modules/row-level-permission-predicate/entities/row-level-permission-predicate-group.entity.ts index c1e812b619..4c041657c5 100644 --- a/packages/twenty-server/src/engine/metadata-modules/row-level-permission-predicate/entities/row-level-permission-predicate-group.entity.ts +++ b/packages/twenty-server/src/engine/metadata-modules/row-level-permission-predicate/entities/row-level-permission-predicate-group.entity.ts @@ -29,6 +29,9 @@ import { SyncableEntity } from 'src/engine/workspace-manager/types/syncable-enti 'roleId', 'objectMetadataId', ]) +@Index('IDX_RLPPG_PARENT_GROUP_ID', [ + 'parentRowLevelPermissionPredicateGroupId', +]) export class RowLevelPermissionPredicateGroupEntity extends SyncableEntity implements diff --git a/packages/twenty-server/src/engine/metadata-modules/row-level-permission-predicate/entities/row-level-permission-predicate.entity.ts b/packages/twenty-server/src/engine/metadata-modules/row-level-permission-predicate/entities/row-level-permission-predicate.entity.ts index 56413fc080..07f03b0041 100644 --- a/packages/twenty-server/src/engine/metadata-modules/row-level-permission-predicate/entities/row-level-permission-predicate.entity.ts +++ b/packages/twenty-server/src/engine/metadata-modules/row-level-permission-predicate/entities/row-level-permission-predicate.entity.ts @@ -32,6 +32,9 @@ import { SyncableEntity } from 'src/engine/workspace-manager/types/syncable-enti ]) @Index('IDX_RLPP_FIELD_METADATA_ID', ['fieldMetadataId']) @Index('IDX_RLPP_GROUP_ID', ['rowLevelPermissionPredicateGroupId']) +@Index('IDX_RLPP_WORKSPACE_MEMBER_FIELD_METADATA_ID', [ + 'workspaceMemberFieldMetadataId', +]) export class RowLevelPermissionPredicateEntity extends SyncableEntity implements diff --git a/packages/twenty-server/src/engine/metadata-modules/serverless-function/serverless-function.entity.ts b/packages/twenty-server/src/engine/metadata-modules/serverless-function/serverless-function.entity.ts index 3442d2dc6b..33e4cf858d 100644 --- a/packages/twenty-server/src/engine/metadata-modules/serverless-function/serverless-function.entity.ts +++ b/packages/twenty-server/src/engine/metadata-modules/serverless-function/serverless-function.entity.ts @@ -31,6 +31,7 @@ export const DEFAULT_HANDLER_NAME = 'main'; @Entity('serverlessFunction') @Index('IDX_SERVERLESS_FUNCTION_ID_DELETED_AT', ['id', 'deletedAt']) +@Index('IDX_SERVERLESS_FUNCTION_LAYER_ID', ['serverlessFunctionLayerId']) export class ServerlessFunctionEntity extends SyncableEntity implements Required diff --git a/packages/twenty-server/src/engine/metadata-modules/view-field/entities/view-field.entity.ts b/packages/twenty-server/src/engine/metadata-modules/view-field/entities/view-field.entity.ts index ac2feaec88..199c7aae27 100644 --- a/packages/twenty-server/src/engine/metadata-modules/view-field/entities/view-field.entity.ts +++ b/packages/twenty-server/src/engine/metadata-modules/view-field/entities/view-field.entity.ts @@ -18,9 +18,8 @@ import { SyncableEntityRequired } from 'src/engine/workspace-manager/types/synca @Entity({ name: 'viewField', schema: 'core' }) @Index('IDX_VIEW_FIELD_WORKSPACE_ID_VIEW_ID', ['workspaceId', 'viewId']) -@Index('IDX_VIEW_FIELD_VIEW_ID', ['viewId'], { - where: '"deletedAt" IS NULL', -}) +@Index('IDX_VIEW_FIELD_VIEW_ID', ['viewId']) +@Index('IDX_VIEW_FIELD_FIELD_METADATA_ID', ['fieldMetadataId']) @Index( 'IDX_VIEW_FIELD_FIELD_METADATA_ID_VIEW_ID_UNIQUE', ['fieldMetadataId', 'viewId'], diff --git a/packages/twenty-server/src/engine/metadata-modules/view-filter-group/entities/view-filter-group.entity.ts b/packages/twenty-server/src/engine/metadata-modules/view-filter-group/entities/view-filter-group.entity.ts index 78b6269964..c39f71166b 100644 --- a/packages/twenty-server/src/engine/metadata-modules/view-filter-group/entities/view-filter-group.entity.ts +++ b/packages/twenty-server/src/engine/metadata-modules/view-filter-group/entities/view-filter-group.entity.ts @@ -19,9 +19,8 @@ import { SyncableEntity } from 'src/engine/workspace-manager/types/syncable-enti @Entity({ name: 'viewFilterGroup', schema: 'core' }) @Index('IDX_VIEW_FILTER_GROUP_WORKSPACE_ID_VIEW_ID', ['workspaceId', 'viewId']) -@Index('IDX_VIEW_FILTER_GROUP_VIEW_ID', ['viewId'], { - where: '"deletedAt" IS NULL', -}) +@Index('IDX_VIEW_FILTER_GROUP_VIEW_ID', ['viewId']) +@Index('IDX_VIEW_FILTER_GROUP_PARENT_ID', ['parentViewFilterGroupId']) export class ViewFilterGroupEntity extends SyncableEntity { @PrimaryGeneratedColumn('uuid') id: string; diff --git a/packages/twenty-server/src/engine/metadata-modules/view-filter/entities/view-filter.entity.ts b/packages/twenty-server/src/engine/metadata-modules/view-filter/entities/view-filter.entity.ts index 05412d1703..387ccedcc7 100644 --- a/packages/twenty-server/src/engine/metadata-modules/view-filter/entities/view-filter.entity.ts +++ b/packages/twenty-server/src/engine/metadata-modules/view-filter/entities/view-filter.entity.ts @@ -20,9 +20,7 @@ import { SyncableEntityRequired } from 'src/engine/workspace-manager/types/synca @Entity({ name: 'viewFilter', schema: 'core' }) @Index('IDX_VIEW_FILTER_WORKSPACE_ID_VIEW_ID', ['workspaceId', 'viewId']) -@Index('IDX_VIEW_FILTER_VIEW_ID', ['viewId'], { - where: '"deletedAt" IS NULL', -}) +@Index('IDX_VIEW_FILTER_VIEW_ID', ['viewId']) @Index('IDX_VIEW_FILTER_FIELD_METADATA_ID', ['fieldMetadataId']) export class ViewFilterEntity extends SyncableEntityRequired diff --git a/packages/twenty-server/src/engine/metadata-modules/view-group/entities/view-group.entity.ts b/packages/twenty-server/src/engine/metadata-modules/view-group/entities/view-group.entity.ts index f944f5ded7..824b73b3e9 100644 --- a/packages/twenty-server/src/engine/metadata-modules/view-group/entities/view-group.entity.ts +++ b/packages/twenty-server/src/engine/metadata-modules/view-group/entities/view-group.entity.ts @@ -16,9 +16,7 @@ import { SyncableEntity } from 'src/engine/workspace-manager/types/syncable-enti @Entity({ name: 'viewGroup', schema: 'core' }) @Index('IDX_VIEW_GROUP_WORKSPACE_ID_VIEW_ID', ['workspaceId', 'viewId']) -@Index('IDX_VIEW_GROUP_VIEW_ID', ['viewId'], { - where: '"deletedAt" IS NULL', -}) +@Index('IDX_VIEW_GROUP_VIEW_ID', ['viewId']) export class ViewGroupEntity extends SyncableEntity implements Required diff --git a/packages/twenty-server/src/engine/metadata-modules/view-sort/entities/view-sort.entity.ts b/packages/twenty-server/src/engine/metadata-modules/view-sort/entities/view-sort.entity.ts index eb88d4c0c7..d6b4bba8ff 100644 --- a/packages/twenty-server/src/engine/metadata-modules/view-sort/entities/view-sort.entity.ts +++ b/packages/twenty-server/src/engine/metadata-modules/view-sort/entities/view-sort.entity.ts @@ -18,9 +18,8 @@ import { SyncableEntity } from 'src/engine/workspace-manager/types/syncable-enti @Entity({ name: 'viewSort', schema: 'core' }) @Index('IDX_VIEW_SORT_WORKSPACE_ID_VIEW_ID', ['workspaceId', 'viewId']) -@Index('IDX_VIEW_SORT_VIEW_ID', ['viewId'], { - where: '"deletedAt" IS NULL', -}) +@Index('IDX_VIEW_SORT_VIEW_ID', ['viewId']) +@Index('IDX_VIEW_SORT_FIELD_METADATA_ID', ['fieldMetadataId']) @Index( 'IDX_VIEW_SORT_FIELD_METADATA_ID_VIEW_ID_UNIQUE', ['fieldMetadataId', 'viewId'], diff --git a/packages/twenty-server/src/engine/metadata-modules/view/entities/view.entity.ts b/packages/twenty-server/src/engine/metadata-modules/view/entities/view.entity.ts index 17f048cc64..8dbee99bb6 100644 --- a/packages/twenty-server/src/engine/metadata-modules/view/entities/view.entity.ts +++ b/packages/twenty-server/src/engine/metadata-modules/view/entities/view.entity.ts @@ -36,6 +36,12 @@ import { SyncableEntityRequired } from 'src/engine/workspace-manager/types/synca 'objectMetadataId', ]) @Index('IDX_VIEW_VISIBILITY', ['visibility']) +@Index('IDX_VIEW_CALENDAR_FIELD_METADATA', ['calendarFieldMetadataId']) +@Index('IDX_VIEW_KANBAN_FIELD_METADATA', [ + 'kanbanAggregateOperationFieldMetadataId', +]) +@Index('IDX_VIEW_MAIN_GROUP_BY_FIELD_METADATA', ['mainGroupByFieldMetadataId']) +@Index('IDX_VIEW_CREATED_BY_USER_WORKSPACE', ['createdByUserWorkspaceId']) @Check( 'CHK_VIEW_CALENDAR_INTEGRITY', `("type" != 'CALENDAR' OR ("calendarLayout" IS NOT NULL AND "calendarFieldMetadataId" IS NOT NULL))`, diff --git a/packages/twenty-server/src/engine/workspace-manager/workspace-cleaner/commands/clean-soft-deleted-suspended-workspaces.command.ts b/packages/twenty-server/src/engine/workspace-manager/workspace-cleaner/commands/clean-soft-deleted-suspended-workspaces.command.ts deleted file mode 100644 index 9c4c6744e4..0000000000 --- a/packages/twenty-server/src/engine/workspace-manager/workspace-cleaner/commands/clean-soft-deleted-suspended-workspaces.command.ts +++ /dev/null @@ -1,106 +0,0 @@ -import { InjectRepository } from '@nestjs/typeorm'; - -import { Command, Option } from 'nest-commander'; -import { isDefined } from 'twenty-shared/utils'; -import { WorkspaceActivationStatus } from 'twenty-shared/workspace'; -import { In, IsNull, Not, Repository } from 'typeorm'; - -import { - type MigrationCommandOptions, - MigrationCommandRunner, -} from 'src/database/commands/command-runners/migration.command-runner'; -import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity'; -import { CleanerWorkspaceService } from 'src/engine/workspace-manager/workspace-cleaner/services/cleaner.workspace-service'; - -type HardDeleteSoftDeletedSuspendedWorkspacesCommandOptions = - MigrationCommandOptions & { - ignoreGracePeriod?: boolean; - }; - -@Command({ - name: 'workspace:hard-delete-soft-deleted', - description: 'Hard delete soft-deleted suspended workspaces', -}) -export class HardDeleteSoftDeletedSuspendedWorkspacesCommand extends MigrationCommandRunner { - private workspaceIds: string[] = []; - - constructor( - private readonly cleanerWorkspaceService: CleanerWorkspaceService, - @InjectRepository(WorkspaceEntity) - protected readonly workspaceRepository: Repository, - ) { - super(); - } - - @Option({ - flags: '-w, --workspace-id [workspace_id]', - description: - 'workspace id. Command runs on all suspended workspaces if not provided', - required: false, - }) - parseWorkspaceId(val: string): string[] { - this.workspaceIds.push(val); - - return this.workspaceIds; - } - - @Option({ - flags: '--ignore-grace-period', - description: - 'Ignore the grace period and hard delete soft-deleted workspaces immediately', - required: false, - }) - parseIgnoreGracePeriod(): boolean { - return true; - } - - async fetchSuspendedSoftDeletedWorkspaces(): Promise { - return await this.workspaceRepository.find({ - where: { - activationStatus: In([WorkspaceActivationStatus.SUSPENDED]), - deletedAt: Not(IsNull()), - ...(this.workspaceIds.length > 0 ? { id: In(this.workspaceIds) } : {}), - }, - withDeleted: true, - }); - } - - override async runMigrationCommand( - _passedParams: string[], - options: HardDeleteSoftDeletedSuspendedWorkspacesCommandOptions, - ): Promise { - const { dryRun, ignoreGracePeriod } = options; - - const softDeletedSuspendedWorkspaces = - await this.fetchSuspendedSoftDeletedWorkspaces(); - let deletedWorkspaceCounter = 0; - - this.logger.log( - `${dryRun ? 'DRY RUN - ' : ''}${ignoreGracePeriod ? 'IGNORING GRACE PERIOD - ' : ''}Iterating over ${softDeletedSuspendedWorkspaces.length} soft deleted suspended workspaces`, - ); - - for (const workspace of softDeletedSuspendedWorkspaces) { - try { - const result = - await this.cleanerWorkspaceService.hardDeleteSoftDeletedWorkspace({ - workspace, - dryRun, - ignoreGracePeriod, - }); - - if (isDefined(result)) { - deletedWorkspaceCounter++; - } - } catch (error) { - this.logger.error( - `Failed to destroy soft deleted workspace ${workspace.id}`, - error, - ); - } - } - - this.logger.log( - `Destroyed ${deletedWorkspaceCounter}/${softDeletedSuspendedWorkspaces.length}`, - ); - } -} diff --git a/packages/twenty-server/src/engine/workspace-manager/workspace-cleaner/commands/clean-suspended-workspaces.command.ts b/packages/twenty-server/src/engine/workspace-manager/workspace-cleaner/commands/clean-suspended-workspaces.command.ts index 2dab102a03..e887a2afb8 100644 --- a/packages/twenty-server/src/engine/workspace-manager/workspace-cleaner/commands/clean-suspended-workspaces.command.ts +++ b/packages/twenty-server/src/engine/workspace-manager/workspace-cleaner/commands/clean-suspended-workspaces.command.ts @@ -9,7 +9,17 @@ import { MigrationCommandRunner, } from 'src/database/commands/command-runners/migration.command-runner'; import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity'; -import { CleanerWorkspaceService } from 'src/engine/workspace-manager/workspace-cleaner/services/cleaner.workspace-service'; +import { + CLEAN_SUSPENDED_WORKSPACES_OPERATIONS, + CleanerWorkspaceService, + CleanSuspendedWorkspacesOperation, + type CleanSuspendedWorkspacesOptions, +} from 'src/engine/workspace-manager/workspace-cleaner/services/cleaner.workspace-service'; + +type CleanSuspendedWorkspacesCommandOptions = MigrationCommandOptions & + Pick & { + onlyOperation?: CleanSuspendedWorkspacesOperation; + }; @Command({ name: 'workspace:clean', @@ -38,11 +48,32 @@ export class CleanSuspendedWorkspacesCommand extends MigrationCommandRunner { return this.workspaceIds; } + @Option({ + flags: '--ignore-destroy-grace-period', + description: + 'Ignore the grace period and hard delete soft-deleted workspaces immediately', + required: false, + }) + parseIgnoreDestroyGracePeriod(): boolean { + return true; + } + + @Option({ + flags: '--only-operation ', + description: + 'Run only a specific operation: warn, destroy (hard delete), or soft-delete', + required: false, + choices: [...CLEAN_SUSPENDED_WORKSPACES_OPERATIONS], + }) + parseOnlyOperation(val: string): CleanSuspendedWorkspacesOperation { + return val as CleanSuspendedWorkspacesOperation; + } + async fetchSuspendedWorkspaceIds(): Promise { const suspendedWorkspaces = await this.workspaceRepository.find({ - select: ['id'], where: { activationStatus: In([WorkspaceActivationStatus.SUSPENDED]), + ...(this.workspaceIds.length > 0 ? { id: In(this.workspaceIds) } : {}), }, withDeleted: true, }); @@ -52,22 +83,25 @@ export class CleanSuspendedWorkspacesCommand extends MigrationCommandRunner { override async runMigrationCommand( _passedParams: string[], - options: MigrationCommandOptions, + options: CleanSuspendedWorkspacesCommandOptions, ): Promise { - const { dryRun } = options; + const { dryRun, ignoreDestroyGracePeriod, onlyOperation } = options; - const suspendedWorkspaceIds = - this.workspaceIds.length > 0 - ? this.workspaceIds - : await this.fetchSuspendedWorkspaceIds(); + const suspendedWorkspaceIds = await this.fetchSuspendedWorkspaceIds(); + + const operationLabel = onlyOperation + ? `ONLY ${onlyOperation.toUpperCase()} - ` + : ''; this.logger.log( - `${dryRun ? 'DRY RUN - ' : ''}Cleaning ${suspendedWorkspaceIds.length} suspended workspaces`, + `${dryRun ? 'DRY RUN - ' : ''}${ignoreDestroyGracePeriod ? 'IGNORING GRACE PERIOD - ' : ''}${operationLabel}Cleaning ${suspendedWorkspaceIds.length} suspended workspaces`, ); await this.cleanerWorkspaceService.batchWarnOrCleanSuspendedWorkspaces({ workspaceIds: suspendedWorkspaceIds, dryRun, + ignoreDestroyGracePeriod, + onlyOperation, }); } } diff --git a/packages/twenty-server/src/engine/workspace-manager/workspace-cleaner/commands/delete-workspaces.command.ts b/packages/twenty-server/src/engine/workspace-manager/workspace-cleaner/commands/delete-workspaces.command.ts deleted file mode 100644 index 39bf8681ad..0000000000 --- a/packages/twenty-server/src/engine/workspace-manager/workspace-cleaner/commands/delete-workspaces.command.ts +++ /dev/null @@ -1,91 +0,0 @@ -import { Logger } from '@nestjs/common'; -import { InjectRepository } from '@nestjs/typeorm'; - -import { Command, CommandRunner, Option } from 'nest-commander'; -import { In, Repository } from 'typeorm'; - -import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity'; -import { DataSourceService } from 'src/engine/metadata-modules/data-source/data-source.service'; -import { getDryRunLogHeader } from 'src/utils/get-dry-run-log-header'; - -type DeleteWorkspacesCommandOptions = { - dryRun?: boolean; - workspaceIds: string[]; -}; - -@Command({ - name: 'workspace:delete', - description: 'Delete workspace', -}) -export class DeleteWorkspacesCommand extends CommandRunner { - private readonly logger = new Logger(DeleteWorkspacesCommand.name); - - constructor( - @InjectRepository(WorkspaceEntity) - private readonly workspaceRepository: Repository, - private readonly dataSourceService: DataSourceService, - ) { - super(); - } - - @Option({ - flags: '-d, --dry-run [dry run]', - description: 'Dry run: Log delete actions without executing them.', - required: false, - }) - dryRun(value: string): boolean { - return Boolean(value); - } - - @Option({ - flags: '-w, --workspace-ids [workspace_ids]', - description: 'comma separated workspace ids', - required: true, - }) - parseWorkspaceIds(value: string): string[] { - return value.split(','); - } - - async run( - _passedParam: string[], - options: DeleteWorkspacesCommandOptions, - ): Promise { - const workspaces = await this.workspaceRepository.find({ - where: { id: In(options.workspaceIds) }, - }); - - const dataSources = - await this.dataSourceService.getManyDataSourceMetadata(); - - const workspaceIdsWithSchema = dataSources.map( - (dataSource) => dataSource.workspaceId, - ); - - const workspacesToDelete = workspaces.filter((WorkspaceEntity) => - workspaceIdsWithSchema.includes(WorkspaceEntity.id), - ); - - if (workspacesToDelete.length) { - this.logger.log( - `Running Deleting workspaces on ${workspacesToDelete.length} workspaces`, - ); - } - - for (const workspace of workspacesToDelete) { - this.logger.log( - `${getDryRunLogHeader(options.dryRun)}Deleting workspace ${ - workspace.id - } name: '${workspace.displayName}'`, - ); - // const workspaceServiceInstance = - // await this.loadServiceWithWorkspaceContext.load( - // this.workspaceService, - // workspace.id, - // ); - - // if (!options.dryRun) { - // await workspaceServiceInstance.softDeleteWorkspace(workspace.id); - // } - } - } -} diff --git a/packages/twenty-server/src/engine/workspace-manager/workspace-cleaner/services/cleaner.workspace-service.ts b/packages/twenty-server/src/engine/workspace-manager/workspace-cleaner/services/cleaner.workspace-service.ts index 87764a404c..5ae551fb86 100644 --- a/packages/twenty-server/src/engine/workspace-manager/workspace-cleaner/services/cleaner.workspace-service.ts +++ b/packages/twenty-server/src/engine/workspace-manager/workspace-cleaner/services/cleaner.workspace-service.ts @@ -25,12 +25,23 @@ import { UserVarsService } from 'src/engine/core-modules/user/user-vars/services import { WorkspaceService } from 'src/engine/core-modules/workspace/services/workspace.service'; import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity'; import { USER_WORKSPACE_DELETION_WARNING_SENT_KEY } from 'src/engine/workspace-manager/workspace-cleaner/constants/user-workspace-deletion-warning-sent-key.constant'; -import { - WorkspaceCleanerException, - WorkspaceCleanerExceptionCode, -} from 'src/engine/workspace-manager/workspace-cleaner/exceptions/workspace-cleaner.exception'; import { WorkspaceMemberWorkspaceEntity } from 'src/modules/workspace-member/standard-objects/workspace-member.workspace-entity'; +export const CLEAN_SUSPENDED_WORKSPACES_OPERATIONS = [ + 'warn', + 'destroy', + 'soft-delete', +] as const; +export type CleanSuspendedWorkspacesOperation = + (typeof CLEAN_SUSPENDED_WORKSPACES_OPERATIONS)[number]; + +export type CleanSuspendedWorkspacesOptions = { + workspaceIds: string[]; + dryRun?: boolean; + ignoreDestroyGracePeriod?: boolean; + onlyOperation?: CleanSuspendedWorkspacesOperation; +}; + @Injectable() export class CleanerWorkspaceService { private readonly logger = new Logger(CleanerWorkspaceService.name); @@ -68,39 +79,30 @@ export class CleanerWorkspaceService { ); } - async computeDaysSinceSubscriptionUnpaidOrThrow( + async computeDaysSinceSubscriptionUnpaid( workspace: WorkspaceEntity, - ): Promise { - try { - const lastSubscription = - await this.billingSubscriptionRepository.findOneOrFail({ - where: { - workspaceId: workspace.id, - }, - order: { updatedAt: 'DESC' }, - }); + ): Promise { + const lastSubscription = + await this.billingSubscriptionRepository.findOneOrFail({ + where: { + workspaceId: workspace.id, + }, + order: { updatedAt: 'DESC' }, + }); - if ( - lastSubscription.status !== SubscriptionStatus.Unpaid && - lastSubscription.status !== SubscriptionStatus.Canceled - ) { - throw new Error( - 'No cancelled or unpaid billing subscription found for workspace', - ); - } - - const daysSinceSubscriptionUnpaid = differenceInDays( - new Date(), - lastSubscription.currentPeriodStart, - ); - - return daysSinceSubscriptionUnpaid; - } catch { - throw new WorkspaceCleanerException( - `No cancelled or unpaid billing subscription found for workspace ${workspace.id} ${workspace.displayName}`, - WorkspaceCleanerExceptionCode.BILLING_SUBSCRIPTION_NOT_FOUND, - ); + if ( + lastSubscription.status !== SubscriptionStatus.Unpaid && + lastSubscription.status !== SubscriptionStatus.Canceled + ) { + return null; } + + const daysSinceSubscriptionUnpaid = differenceInDays( + new Date(), + lastSubscription.currentPeriodStart, + ); + + return daysSinceSubscriptionUnpaid; } async checkIfAtLeastOneWorkspaceMemberWarned( @@ -329,7 +331,7 @@ export class CleanerWorkspaceService { } } - async hardDeleteSoftDeletedWorkspace({ + async destroySoftDeletedWorkspace({ workspace, ignoreGracePeriod = false, dryRun = false, @@ -375,10 +377,9 @@ export class CleanerWorkspaceService { async batchWarnOrCleanSuspendedWorkspaces({ workspaceIds, dryRun = false, - }: { - workspaceIds: string[]; - dryRun?: boolean; - }): Promise { + ignoreDestroyGracePeriod = false, + onlyOperation, + }: CleanSuspendedWorkspacesOptions): Promise { this.logger.log( `${dryRun ? 'DRY RUN - ' : ''}batchWarnOrCleanSuspendedWorkspaces running...`, ); @@ -404,23 +405,37 @@ export class CleanerWorkspaceService { deletedWorkspacesCount < this.maxNumberOfWorkspacesDeletedPerExecution; - if (isSoftDeletedWorkspace && isWithinDeletionLimit) { - const result = await this.hardDeleteSoftDeletedWorkspace({ + if ( + (!isDefined(onlyOperation) || onlyOperation === 'destroy') && + isSoftDeletedWorkspace && + isWithinDeletionLimit + ) { + const result = await this.destroySoftDeletedWorkspace({ workspace, dryRun, - ignoreGracePeriod: false, + ignoreGracePeriod: ignoreDestroyGracePeriod, }); if (isDefined(result)) { deletedWorkspacesCount++; + this.logger.log( + `Destroyed ${deletedWorkspacesCount} workspaces on ${this.maxNumberOfWorkspacesDeletedPerExecution} limit durings this execution`, + ); } continue; } const workspaceInactivity = - await this.computeDaysSinceSubscriptionUnpaidOrThrow(workspace); + await this.computeDaysSinceSubscriptionUnpaid(workspace); - if (workspaceInactivity > this.inactiveDaysBeforeSoftDelete) { + if (workspaceInactivity === null) { + continue; + } + + if ( + (!isDefined(onlyOperation) || onlyOperation === 'soft-delete') && + workspaceInactivity > this.inactiveDaysBeforeSoftDelete + ) { await this.informWorkspaceMembersAndSoftDeleteWorkspace( workspace, workspaceInactivity, @@ -429,7 +444,9 @@ export class CleanerWorkspaceService { continue; } + if ( + (!isDefined(onlyOperation) || onlyOperation === 'warn') && workspaceInactivity > this.inactiveDaysBeforeWarn && workspaceInactivity <= this.inactiveDaysBeforeSoftDelete ) { diff --git a/packages/twenty-server/src/engine/workspace-manager/workspace-cleaner/workspace-cleaner.module.ts b/packages/twenty-server/src/engine/workspace-manager/workspace-cleaner/workspace-cleaner.module.ts index 826ad675f0..2d087e9fc0 100644 --- a/packages/twenty-server/src/engine/workspace-manager/workspace-cleaner/workspace-cleaner.module.ts +++ b/packages/twenty-server/src/engine/workspace-manager/workspace-cleaner/workspace-cleaner.module.ts @@ -13,10 +13,8 @@ import { WorkspaceModule } from 'src/engine/core-modules/workspace/workspace.mod import { DataSourceModule } from 'src/engine/metadata-modules/data-source/data-source.module'; import { CleanOnboardingWorkspacesCommand } from 'src/engine/workspace-manager/workspace-cleaner/commands/clean-onboarding-workspaces.command'; import { CleanOnboardingWorkspacesCronCommand } from 'src/engine/workspace-manager/workspace-cleaner/commands/clean-onboarding-workspaces.cron.command'; -import { HardDeleteSoftDeletedSuspendedWorkspacesCommand } from 'src/engine/workspace-manager/workspace-cleaner/commands/clean-soft-deleted-suspended-workspaces.command'; import { CleanSuspendedWorkspacesCommand } from 'src/engine/workspace-manager/workspace-cleaner/commands/clean-suspended-workspaces.command'; import { CleanSuspendedWorkspacesCronCommand } from 'src/engine/workspace-manager/workspace-cleaner/commands/clean-suspended-workspaces.cron.command'; -import { DeleteWorkspacesCommand } from 'src/engine/workspace-manager/workspace-cleaner/commands/delete-workspaces.command'; import { DestroyWorkspaceCommand } from 'src/engine/workspace-manager/workspace-cleaner/commands/destroy-workspace.command'; import { CleanerWorkspaceService } from 'src/engine/workspace-manager/workspace-cleaner/services/cleaner.workspace-service'; @@ -36,11 +34,9 @@ import { CleanerWorkspaceService } from 'src/engine/workspace-manager/workspace- MetricsModule, ], providers: [ - DeleteWorkspacesCommand, DestroyWorkspaceCommand, CleanSuspendedWorkspacesCronCommand, CleanSuspendedWorkspacesCommand, - HardDeleteSoftDeletedSuspendedWorkspacesCommand, CleanOnboardingWorkspacesCommand, CleanOnboardingWorkspacesCronCommand, CleanerWorkspaceService,