Automatically clean up soft-deleted records after X days. (#14862)
Closes #14726 ### Added - `trashRetentionDays` field to workspace entity (default: 14 days) - Automated trash cleanup using BullMQ jobs - Daily cron (00:10 UTC) that enqueues cleanup jobs for all active workspaces - Per-workspace limit: 100k records deleted per day - Calendar-based retention: records deleted on day X are cleaned up X+14 days later (at midnight UTC boundaries) ### Architecture - **Cron (WorkspaceTrashCleanupCronJob):** Runs daily, enqueues jobs in parallel for all workspaces - **Job (WorkspaceTrashCleanupJob):** Processes individual workspace cleanup - **Service (WorkspaceTrashCleanupService):** Discovers tables with `deletedAt`, deletes old records with quota enforcement - **Command:** `npx nx run twenty-server:command cron:workspace:cleanup-trash` to register the cron ### Testing - Unit tests for service with 100% coverage of public API - Tested quota enforcement, error handling, and edge cases --------- Co-authored-by: Félix Malfait <felix.malfait@gmail.com>
This commit is contained in:
@@ -7,6 +7,7 @@ import { CheckCustomDomainValidRecordsCronCommand } from 'src/engine/core-module
|
||||
import { CronTriggerCronCommand } from 'src/engine/metadata-modules/cron-trigger/crons/commands/cron-trigger.cron.command';
|
||||
import { CleanOnboardingWorkspacesCronCommand } from 'src/engine/workspace-manager/workspace-cleaner/commands/clean-onboarding-workspaces.cron.command';
|
||||
import { CleanSuspendedWorkspacesCronCommand } from 'src/engine/workspace-manager/workspace-cleaner/commands/clean-suspended-workspaces.cron.command';
|
||||
import { TrashCleanupCronCommand } from 'src/engine/trash-cleanup/commands/trash-cleanup.cron.command';
|
||||
import { CalendarEventListFetchCronCommand } from 'src/modules/calendar/calendar-event-import-manager/crons/commands/calendar-event-list-fetch.cron.command';
|
||||
import { CalendarEventsImportCronCommand } from 'src/modules/calendar/calendar-event-import-manager/crons/commands/calendar-import.cron.command';
|
||||
import { CalendarOngoingStaleCronCommand } from 'src/modules/calendar/calendar-event-import-manager/crons/commands/calendar-ongoing-stale.cron.command';
|
||||
@@ -42,6 +43,7 @@ export class CronRegisterAllCommand extends CommandRunner {
|
||||
private readonly cronTriggerCronCommand: CronTriggerCronCommand,
|
||||
private readonly cleanSuspendedWorkspacesCronCommand: CleanSuspendedWorkspacesCronCommand,
|
||||
private readonly cleanOnboardingWorkspacesCronCommand: CleanOnboardingWorkspacesCronCommand,
|
||||
private readonly trashCleanupCronCommand: TrashCleanupCronCommand,
|
||||
) {
|
||||
super();
|
||||
}
|
||||
@@ -110,6 +112,10 @@ export class CronRegisterAllCommand extends CommandRunner {
|
||||
name: 'CleanOnboardingWorkspaces',
|
||||
command: this.cleanOnboardingWorkspacesCronCommand,
|
||||
},
|
||||
{
|
||||
name: 'TrashCleanup',
|
||||
command: this.trashCleanupCronCommand,
|
||||
},
|
||||
];
|
||||
|
||||
let successCount = 0;
|
||||
|
||||
@@ -20,6 +20,7 @@ import { ObjectMetadataModule } from 'src/engine/metadata-modules/object-metadat
|
||||
import { WorkspaceCacheStorageModule } from 'src/engine/workspace-cache-storage/workspace-cache-storage.module';
|
||||
import { DevSeederModule } from 'src/engine/workspace-manager/dev-seeder/dev-seeder.module';
|
||||
import { WorkspaceCleanerModule } from 'src/engine/workspace-manager/workspace-cleaner/workspace-cleaner.module';
|
||||
import { TrashCleanupModule } from 'src/engine/trash-cleanup/trash-cleanup.module';
|
||||
import { WorkspaceManagerModule } from 'src/engine/workspace-manager/workspace-manager.module';
|
||||
import { CalendarEventImportManagerModule } from 'src/modules/calendar/calendar-event-import-manager/calendar-event-import-manager.module';
|
||||
import { MessagingImportManagerModule } from 'src/modules/messaging/message-import-manager/messaging-import-manager.module';
|
||||
@@ -50,6 +51,7 @@ import { AutomatedTriggerModule } from 'src/modules/workflow/workflow-trigger/au
|
||||
CronTriggerModule,
|
||||
DatabaseEventTriggerModule,
|
||||
WorkspaceCleanerModule,
|
||||
TrashCleanupModule,
|
||||
PublicDomainModule,
|
||||
],
|
||||
providers: [
|
||||
|
||||
+19
@@ -0,0 +1,19 @@
|
||||
import { type MigrationInterface, type QueryRunner } from 'typeorm';
|
||||
|
||||
export class AddWorkspaceTrashRetention1760356369619
|
||||
implements MigrationInterface
|
||||
{
|
||||
name = 'AddWorkspaceTrashRetention1760356369619';
|
||||
|
||||
public async up(queryRunner: QueryRunner): Promise<void> {
|
||||
await queryRunner.query(
|
||||
`ALTER TABLE "core"."workspace" ADD "trashRetentionDays" integer NOT NULL DEFAULT '14'`,
|
||||
);
|
||||
}
|
||||
|
||||
public async down(queryRunner: QueryRunner): Promise<void> {
|
||||
await queryRunner.query(
|
||||
`ALTER TABLE "core"."workspace" DROP COLUMN "trashRetentionDays"`,
|
||||
);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user