Automatically clean up soft-deleted records after X days. (#14862)

Closes #14726

### Added
- `trashRetentionDays` field to workspace entity (default: 14 days)  
- Automated trash cleanup using BullMQ jobs  
- Daily cron (00:10 UTC) that enqueues cleanup jobs for all active
workspaces
- Per-workspace limit: 100k records deleted per day  
- Calendar-based retention: records deleted on day X are cleaned up X+14
days later (at midnight UTC boundaries)

### Architecture
- **Cron (WorkspaceTrashCleanupCronJob):** Runs daily, enqueues jobs in
parallel for all workspaces
- **Job (WorkspaceTrashCleanupJob):** Processes individual workspace
cleanup
- **Service (WorkspaceTrashCleanupService):** Discovers tables with
`deletedAt`, deletes old records with quota enforcement
- **Command:** `npx nx run twenty-server:command
cron:workspace:cleanup-trash` to register the cron

### Testing
- Unit tests for service with 100% coverage of public API  
- Tested quota enforcement, error handling, and edge cases

---------

Co-authored-by: Félix Malfait <felix.malfait@gmail.com>
This commit is contained in:
Abdullah.
2025-10-14 21:49:40 +05:00
committed by GitHub
parent bca79f91ac
commit d750df7fff
26 changed files with 714 additions and 28 deletions
@@ -7,6 +7,7 @@ import { CheckCustomDomainValidRecordsCronCommand } from 'src/engine/core-module
import { CronTriggerCronCommand } from 'src/engine/metadata-modules/cron-trigger/crons/commands/cron-trigger.cron.command';
import { CleanOnboardingWorkspacesCronCommand } from 'src/engine/workspace-manager/workspace-cleaner/commands/clean-onboarding-workspaces.cron.command';
import { CleanSuspendedWorkspacesCronCommand } from 'src/engine/workspace-manager/workspace-cleaner/commands/clean-suspended-workspaces.cron.command';
import { TrashCleanupCronCommand } from 'src/engine/trash-cleanup/commands/trash-cleanup.cron.command';
import { CalendarEventListFetchCronCommand } from 'src/modules/calendar/calendar-event-import-manager/crons/commands/calendar-event-list-fetch.cron.command';
import { CalendarEventsImportCronCommand } from 'src/modules/calendar/calendar-event-import-manager/crons/commands/calendar-import.cron.command';
import { CalendarOngoingStaleCronCommand } from 'src/modules/calendar/calendar-event-import-manager/crons/commands/calendar-ongoing-stale.cron.command';
@@ -42,6 +43,7 @@ export class CronRegisterAllCommand extends CommandRunner {
private readonly cronTriggerCronCommand: CronTriggerCronCommand,
private readonly cleanSuspendedWorkspacesCronCommand: CleanSuspendedWorkspacesCronCommand,
private readonly cleanOnboardingWorkspacesCronCommand: CleanOnboardingWorkspacesCronCommand,
private readonly trashCleanupCronCommand: TrashCleanupCronCommand,
) {
super();
}
@@ -110,6 +112,10 @@ export class CronRegisterAllCommand extends CommandRunner {
name: 'CleanOnboardingWorkspaces',
command: this.cleanOnboardingWorkspacesCronCommand,
},
{
name: 'TrashCleanup',
command: this.trashCleanupCronCommand,
},
];
let successCount = 0;
@@ -20,6 +20,7 @@ import { ObjectMetadataModule } from 'src/engine/metadata-modules/object-metadat
import { WorkspaceCacheStorageModule } from 'src/engine/workspace-cache-storage/workspace-cache-storage.module';
import { DevSeederModule } from 'src/engine/workspace-manager/dev-seeder/dev-seeder.module';
import { WorkspaceCleanerModule } from 'src/engine/workspace-manager/workspace-cleaner/workspace-cleaner.module';
import { TrashCleanupModule } from 'src/engine/trash-cleanup/trash-cleanup.module';
import { WorkspaceManagerModule } from 'src/engine/workspace-manager/workspace-manager.module';
import { CalendarEventImportManagerModule } from 'src/modules/calendar/calendar-event-import-manager/calendar-event-import-manager.module';
import { MessagingImportManagerModule } from 'src/modules/messaging/message-import-manager/messaging-import-manager.module';
@@ -50,6 +51,7 @@ import { AutomatedTriggerModule } from 'src/modules/workflow/workflow-trigger/au
CronTriggerModule,
DatabaseEventTriggerModule,
WorkspaceCleanerModule,
TrashCleanupModule,
PublicDomainModule,
],
providers: [
@@ -0,0 +1,19 @@
import { type MigrationInterface, type QueryRunner } from 'typeorm';
export class AddWorkspaceTrashRetention1760356369619
implements MigrationInterface
{
name = 'AddWorkspaceTrashRetention1760356369619';
public async up(queryRunner: QueryRunner): Promise<void> {
await queryRunner.query(
`ALTER TABLE "core"."workspace" ADD "trashRetentionDays" integer NOT NULL DEFAULT '14'`,
);
}
public async down(queryRunner: QueryRunner): Promise<void> {
await queryRunner.query(
`ALTER TABLE "core"."workspace" DROP COLUMN "trashRetentionDays"`,
);
}
}