Automatically clean up soft-deleted records after X days. (#14862)

Closes #14726

### Added
- `trashRetentionDays` field to workspace entity (default: 14 days)  
- Automated trash cleanup using BullMQ jobs  
- Daily cron (00:10 UTC) that enqueues cleanup jobs for all active
workspaces
- Per-workspace limit: 100k records deleted per day  
- Calendar-based retention: records deleted on day X are cleaned up X+14
days later (at midnight UTC boundaries)

### Architecture
- **Cron (WorkspaceTrashCleanupCronJob):** Runs daily, enqueues jobs in
parallel for all workspaces
- **Job (WorkspaceTrashCleanupJob):** Processes individual workspace
cleanup
- **Service (WorkspaceTrashCleanupService):** Discovers tables with
`deletedAt`, deletes old records with quota enforcement
- **Command:** `npx nx run twenty-server:command
cron:workspace:cleanup-trash` to register the cron

### Testing
- Unit tests for service with 100% coverage of public API  
- Tested quota enforcement, error handling, and edge cases

---------

Co-authored-by: Félix Malfait <felix.malfait@gmail.com>
This commit is contained in:
Abdullah.
2025-10-14 21:49:40 +05:00
committed by GitHub
parent bca79f91ac
commit d750df7fff
26 changed files with 714 additions and 28 deletions
File diff suppressed because one or more lines are too long
@@ -4077,6 +4077,7 @@ export type UpdateWorkspaceInput = {
isTwoFactorAuthenticationEnforced?: InputMaybe<Scalars['Boolean']>;
logo?: InputMaybe<Scalars['String']>;
subdomain?: InputMaybe<Scalars['String']>;
trashRetentionDays?: InputMaybe<Scalars['Float']>;
};
export type UpsertFieldPermissionsInput = {
@@ -4362,6 +4363,7 @@ export type Workspace = {
logo?: Maybe<Scalars['String']>;
metadataVersion: Scalars['Float'];
subdomain: Scalars['String'];
trashRetentionDays: Scalars['Float'];
updatedAt: Scalars['DateTime'];
version?: Maybe<Scalars['String']>;
viewFields?: Maybe<Array<CoreViewField>>;
@@ -57,6 +57,7 @@ const mockWorkspace = {
customUrl: 'test.com',
},
isTwoFactorAuthenticationEnforced: false,
trashRetentionDays: 14,
};
const createMockOptions = (): Options<any> => ({
@@ -24,6 +24,7 @@ export type CurrentWorkspace = Pick<
| 'workspaceUrls'
| 'metadataVersion'
| 'isTwoFactorAuthenticationEnforced'
| 'trashRetentionDays'
> & {
defaultRole?: Omit<Role, 'workspaceMembers' | 'agents' | 'apiKeys'> | null;
defaultAgent?: { id: string } | null;
@@ -50,6 +50,7 @@ const Wrapper = getJestMetadataAndApolloMocksAndActionMenuWrapper({
},
],
isTwoFactorAuthenticationEnforced: false,
trashRetentionDays: 14,
});
},
});
@@ -10,14 +10,16 @@ type SettingsCounterProps = {
minValue?: number;
maxValue?: number;
disabled?: boolean;
showButtons?: boolean;
};
const StyledCounterContainer = styled.div`
const StyledCounterContainer = styled.div<{ showButtons: boolean }>`
align-items: center;
display: flex;
gap: ${({ theme }) => theme.spacing(1)};
margin-left: auto;
width: ${({ theme }) => theme.spacing(30)};
width: ${({ theme, showButtons }) =>
showButtons ? theme.spacing(30) : theme.spacing(16)};
`;
const StyledTextInput = styled(SettingsTextInput)`
@@ -34,11 +36,12 @@ export const SettingsCounter = ({
value,
onChange,
minValue = 0,
maxValue = 100,
maxValue,
disabled = false,
showButtons = true,
}: SettingsCounterProps) => {
const handleIncrementCounter = () => {
if (value < maxValue) {
if (maxValue === undefined || value < maxValue) {
onChange(value + 1);
}
};
@@ -60,7 +63,7 @@ export const SettingsCounter = ({
return;
}
if (castedNumber > maxValue) {
if (maxValue !== undefined && castedNumber > maxValue) {
onChange(maxValue);
return;
}
@@ -68,14 +71,16 @@ export const SettingsCounter = ({
};
return (
<StyledCounterContainer>
<IconButton
size="small"
Icon={IconMinus}
variant="secondary"
onClick={handleDecrementCounter}
disabled={disabled}
/>
<StyledCounterContainer showButtons={showButtons}>
{showButtons && (
<IconButton
size="small"
Icon={IconMinus}
variant="secondary"
onClick={handleDecrementCounter}
disabled={disabled}
/>
)}
<StyledTextInput
instanceId="settings-counter-input"
name="counter"
@@ -84,13 +89,15 @@ export const SettingsCounter = ({
onChange={handleTextInputChange}
disabled={disabled}
/>
<IconButton
size="small"
Icon={IconPlus}
variant="secondary"
onClick={handleIncrementCounter}
disabled={disabled}
/>
{showButtons && (
<IconButton
size="small"
Icon={IconPlus}
variant="secondary"
onClick={handleIncrementCounter}
disabled={disabled}
/>
)}
</StyledCounterContainer>
);
};
@@ -17,6 +17,7 @@ type SettingsOptionCardContentCounterProps = {
onChange: (value: number) => void;
minValue?: number;
maxValue?: number;
showButtons?: boolean;
};
export const SettingsOptionCardContentCounter = ({
@@ -28,6 +29,7 @@ export const SettingsOptionCardContentCounter = ({
onChange,
minValue,
maxValue,
showButtons = true,
}: SettingsOptionCardContentCounterProps) => {
return (
<StyledSettingsOptionCardContent disabled={disabled}>
@@ -50,6 +52,7 @@ export const SettingsOptionCardContentCounter = ({
minValue={minValue}
maxValue={maxValue}
disabled={disabled}
showButtons={showButtons}
/>
</StyledSettingsOptionCardContent>
);
@@ -25,6 +25,7 @@ const SettingsOptionCardContentCounterWrapper = (
disabled={args.disabled}
minValue={args.minValue}
maxValue={args.maxValue}
showButtons={args.showButtons}
/>
</StyledContainer>
);
@@ -50,6 +51,7 @@ export const Default: Story = {
value: 5,
minValue: 1,
maxValue: 10,
showButtons: true,
},
argTypes: {
Icon: { control: false },
@@ -64,6 +66,7 @@ export const WithoutIcon: Story = {
value: 20,
minValue: 10,
maxValue: 50,
showButtons: true,
},
};
@@ -76,5 +79,17 @@ export const Disabled: Story = {
disabled: true,
minValue: 1,
maxValue: 10,
showButtons: true,
},
};
export const WithoutButtons: Story = {
args: {
Icon: IconUsers,
title: 'Trash Retention',
description: 'Adjust the number of days before deletion',
value: 14,
minValue: 0,
showButtons: false,
},
};
@@ -79,6 +79,7 @@ export const USER_QUERY_FRAGMENT = gql`
id
}
isTwoFactorAuthenticationEnforced
trashRetentionDays
}
availableWorkspaces {
...AvailableWorkspacesFragment
@@ -1,19 +1,24 @@
import styled from '@emotion/styled';
import { Trans, useLingui } from '@lingui/react/macro';
import { useDebouncedCallback } from 'use-debounce';
import { currentWorkspaceState } from '@/auth/states/currentWorkspaceState';
import { isMultiWorkspaceEnabledState } from '@/client-config/states/isMultiWorkspaceEnabledState';
import { SettingsOptionCardContentCounter } from '@/settings/components/SettingsOptions/SettingsOptionCardContentCounter';
import { SettingsPageContainer } from '@/settings/components/SettingsPageContainer';
import { SettingsSSOIdentitiesProvidersListCard } from '@/settings/security/components/SSO/SettingsSSOIdentitiesProvidersListCard';
import { SettingsSecurityAuthProvidersOptionsList } from '@/settings/security/components/SettingsSecurityAuthProvidersOptionsList';
import { ToggleImpersonate } from '@/settings/workspace/components/ToggleImpersonate';
import { useSnackBar } from '@/ui/feedback/snack-bar-manager/hooks/useSnackBar';
import { SubMenuTopBarContainer } from '@/ui/layout/page/components/SubMenuTopBarContainer';
import { useRecoilValue } from 'recoil';
import { ApolloError } from '@apollo/client';
import { useRecoilState, useRecoilValue } from 'recoil';
import { SettingsPath } from 'twenty-shared/types';
import { getSettingsPath } from 'twenty-shared/utils';
import { Tag } from 'twenty-ui/components';
import { H2Title, IconLock } from 'twenty-ui/display';
import { Section } from 'twenty-ui/layout';
import { H2Title, IconLock, IconTrash } from 'twenty-ui/display';
import { Card, Section } from 'twenty-ui/layout';
import { useUpdateWorkspaceMutation } from '~/generated-metadata/graphql';
const StyledContainer = styled.div`
width: 100%;
@@ -32,8 +37,50 @@ const StyledSection = styled(Section)`
export const SettingsSecurity = () => {
const { t } = useLingui();
const { enqueueErrorSnackBar } = useSnackBar();
const isMultiWorkspaceEnabled = useRecoilValue(isMultiWorkspaceEnabledState);
const [currentWorkspace, setCurrentWorkspace] = useRecoilState(
currentWorkspaceState,
);
const [updateWorkspace] = useUpdateWorkspaceMutation();
const saveWorkspace = useDebouncedCallback(async (value: number) => {
try {
if (!currentWorkspace?.id) {
throw new Error('User is not logged in');
}
await updateWorkspace({
variables: {
input: {
trashRetentionDays: value,
},
},
});
} catch (err) {
enqueueErrorSnackBar({
apolloError: err instanceof ApolloError ? err : undefined,
});
}
}, 500);
const handleTrashRetentionDaysChange = (value: number) => {
if (!currentWorkspace) {
return;
}
if (value === currentWorkspace.trashRetentionDays) {
return;
}
setCurrentWorkspace({
...currentWorkspace,
trashRetentionDays: value,
});
saveWorkspace(value);
};
return (
<SubMenuTopBarContainer
@@ -82,6 +129,23 @@ export const SettingsSecurity = () => {
<ToggleImpersonate />
</Section>
)}
<Section>
<H2Title
title={t`Other`}
description={t`Other security settings`}
/>
<Card rounded>
<SettingsOptionCardContentCounter
Icon={IconTrash}
title={t`Erasure of soft-deleted records`}
description={t`Permanent deletion. Enter the number of days.`}
value={currentWorkspace?.trashRetentionDays ?? 14}
onChange={handleTrashRetentionDaysChange}
minValue={0}
showButtons={false}
/>
</Card>
</Section>
</StyledMainContent>
</SettingsPageContainer>
</SubMenuTopBarContainer>
@@ -84,6 +84,7 @@ export const mockCurrentWorkspace: Workspace = {
createdAt: '2023-04-26T10:23:42.33625+00:00',
updatedAt: '2023-04-26T10:23:42.33625+00:00',
metadataVersion: 1,
trashRetentionDays: 14,
currentBillingSubscription: {
__typename: 'BillingSubscription',
id: '7efbc3f7-6e5e-4128-957e-8d86808cdf6a',
@@ -151,6 +152,7 @@ export const mockCurrentWorkspace: Workspace = {
databaseSchema: '',
databaseUrl: '',
isTwoFactorAuthenticationEnforced: false,
__typename: 'Workspace',
};
export const mockedWorkspaceMemberData: WorkspaceMember = {
+1 -1
View File
@@ -78,4 +78,4 @@ FRONTEND_URL=http://localhost:3001
# CLOUDFLARE_WEBHOOK_SECRET=
# IS_CONFIG_VARIABLES_IN_DB_ENABLED=false
# ANALYTICS_ENABLED=
# CLICKHOUSE_URL=http://default:clickhousePassword@localhost:8123/twenty
# CLICKHOUSE_URL=http://default:clickhousePassword@localhost:8123/twenty
@@ -7,6 +7,7 @@ import { CheckCustomDomainValidRecordsCronCommand } from 'src/engine/core-module
import { CronTriggerCronCommand } from 'src/engine/metadata-modules/cron-trigger/crons/commands/cron-trigger.cron.command';
import { CleanOnboardingWorkspacesCronCommand } from 'src/engine/workspace-manager/workspace-cleaner/commands/clean-onboarding-workspaces.cron.command';
import { CleanSuspendedWorkspacesCronCommand } from 'src/engine/workspace-manager/workspace-cleaner/commands/clean-suspended-workspaces.cron.command';
import { TrashCleanupCronCommand } from 'src/engine/trash-cleanup/commands/trash-cleanup.cron.command';
import { CalendarEventListFetchCronCommand } from 'src/modules/calendar/calendar-event-import-manager/crons/commands/calendar-event-list-fetch.cron.command';
import { CalendarEventsImportCronCommand } from 'src/modules/calendar/calendar-event-import-manager/crons/commands/calendar-import.cron.command';
import { CalendarOngoingStaleCronCommand } from 'src/modules/calendar/calendar-event-import-manager/crons/commands/calendar-ongoing-stale.cron.command';
@@ -42,6 +43,7 @@ export class CronRegisterAllCommand extends CommandRunner {
private readonly cronTriggerCronCommand: CronTriggerCronCommand,
private readonly cleanSuspendedWorkspacesCronCommand: CleanSuspendedWorkspacesCronCommand,
private readonly cleanOnboardingWorkspacesCronCommand: CleanOnboardingWorkspacesCronCommand,
private readonly trashCleanupCronCommand: TrashCleanupCronCommand,
) {
super();
}
@@ -110,6 +112,10 @@ export class CronRegisterAllCommand extends CommandRunner {
name: 'CleanOnboardingWorkspaces',
command: this.cleanOnboardingWorkspacesCronCommand,
},
{
name: 'TrashCleanup',
command: this.trashCleanupCronCommand,
},
];
let successCount = 0;
@@ -20,6 +20,7 @@ import { ObjectMetadataModule } from 'src/engine/metadata-modules/object-metadat
import { WorkspaceCacheStorageModule } from 'src/engine/workspace-cache-storage/workspace-cache-storage.module';
import { DevSeederModule } from 'src/engine/workspace-manager/dev-seeder/dev-seeder.module';
import { WorkspaceCleanerModule } from 'src/engine/workspace-manager/workspace-cleaner/workspace-cleaner.module';
import { TrashCleanupModule } from 'src/engine/trash-cleanup/trash-cleanup.module';
import { WorkspaceManagerModule } from 'src/engine/workspace-manager/workspace-manager.module';
import { CalendarEventImportManagerModule } from 'src/modules/calendar/calendar-event-import-manager/calendar-event-import-manager.module';
import { MessagingImportManagerModule } from 'src/modules/messaging/message-import-manager/messaging-import-manager.module';
@@ -50,6 +51,7 @@ import { AutomatedTriggerModule } from 'src/modules/workflow/workflow-trigger/au
CronTriggerModule,
DatabaseEventTriggerModule,
WorkspaceCleanerModule,
TrashCleanupModule,
PublicDomainModule,
],
providers: [
@@ -0,0 +1,19 @@
import { type MigrationInterface, type QueryRunner } from 'typeorm';
export class AddWorkspaceTrashRetention1760356369619
implements MigrationInterface
{
name = 'AddWorkspaceTrashRetention1760356369619';
public async up(queryRunner: QueryRunner): Promise<void> {
await queryRunner.query(
`ALTER TABLE "core"."workspace" ADD "trashRetentionDays" integer NOT NULL DEFAULT '14'`,
);
}
public async down(queryRunner: QueryRunner): Promise<void> {
await queryRunner.query(
`ALTER TABLE "core"."workspace" DROP COLUMN "trashRetentionDays"`,
);
}
}
@@ -56,6 +56,7 @@ import { WorkspaceInvitationModule } from 'src/engine/core-modules/workspace-inv
import { WorkspaceModule } from 'src/engine/core-modules/workspace/workspace.module';
import { RoleModule } from 'src/engine/metadata-modules/role/role.module';
import { SubscriptionsModule } from 'src/engine/subscriptions/subscriptions.module';
import { TrashCleanupModule } from 'src/engine/trash-cleanup/trash-cleanup.module';
import { WorkspaceEventEmitterModule } from 'src/engine/workspace-event-emitter/workspace-event-emitter.module';
import { ChannelSyncModule } from 'src/modules/connected-account/channel-sync/channel-sync.module';
@@ -134,6 +135,7 @@ import { FileModule } from './file/file.module';
WebhookModule,
PageLayoutModule,
ImpersonationModule,
TrashCleanupModule,
],
exports: [
AuditModule,
@@ -2,11 +2,13 @@ import { Field, InputType } from '@nestjs/graphql';
import {
IsBoolean,
IsInt,
IsNotIn,
IsOptional,
IsString,
IsUUID,
Matches,
Min,
} from 'class-validator';
import { UUIDScalarType } from 'src/engine/api/graphql/workspace-schema-builder/graphql-types/scalars';
@@ -197,4 +199,10 @@ export class UpdateWorkspaceInput {
@IsBoolean()
@IsOptional()
isTwoFactorAuthenticationEnforced?: boolean;
@Field({ nullable: true })
@IsInt()
@Min(0)
@IsOptional()
trashRetentionDays?: number;
}
@@ -476,7 +476,8 @@ export class WorkspaceService extends TypeOrmQueryService<Workspace> {
'displayName' in payload ||
'subdomain' in payload ||
'customDomain' in payload ||
'logo' in payload
'logo' in payload ||
'trashRetentionDays' in payload
) {
if (!userWorkspaceId) {
throw new Error('Missing userWorkspaceId in authContext');
@@ -92,6 +92,10 @@ export class Workspace {
@Column({ default: true })
isPublicInviteLinkEnabled: boolean;
@Field()
@Column({ type: 'integer', default: 14 })
trashRetentionDays: number;
// Relations
@OneToMany(() => AppToken, (appToken) => appToken.workspace, {
cascade: true,
@@ -0,0 +1,32 @@
import { Command, CommandRunner } from 'nest-commander';
import { InjectMessageQueue } from 'src/engine/core-modules/message-queue/decorators/message-queue.decorator';
import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';
import { MessageQueueService } from 'src/engine/core-modules/message-queue/services/message-queue.service';
import { TRASH_CLEANUP_CRON_PATTERN } from 'src/engine/trash-cleanup/constants/trash-cleanup.constants';
import { TrashCleanupCronJob } from 'src/engine/trash-cleanup/crons/trash-cleanup.cron.job';
@Command({
name: 'cron:trash-cleanup',
description: 'Starts a cron job to clean up soft-deleted records',
})
export class TrashCleanupCronCommand extends CommandRunner {
constructor(
@InjectMessageQueue(MessageQueue.cronQueue)
private readonly messageQueueService: MessageQueueService,
) {
super();
}
async run(): Promise<void> {
await this.messageQueueService.addCron<undefined>({
jobName: TrashCleanupCronJob.name,
data: undefined,
options: {
repeat: {
pattern: TRASH_CLEANUP_CRON_PATTERN,
},
},
});
}
}
@@ -0,0 +1,5 @@
// Daily at 00:10 UTC
export const TRASH_CLEANUP_CRON_PATTERN = '10 0 * * *';
export const TRASH_CLEANUP_MAX_RECORDS_PER_WORKSPACE = 1_000_000;
export const TRASH_CLEANUP_BATCH_SIZE = 1_000;
@@ -0,0 +1,92 @@
import { Injectable, Logger } from '@nestjs/common';
import { InjectRepository } from '@nestjs/typeorm';
import { WorkspaceActivationStatus } from 'twenty-shared/workspace';
import { Repository } from 'typeorm';
import { SentryCronMonitor } from 'src/engine/core-modules/cron/sentry-cron-monitor.decorator';
import { ExceptionHandlerService } from 'src/engine/core-modules/exception-handler/exception-handler.service';
import { InjectMessageQueue } from 'src/engine/core-modules/message-queue/decorators/message-queue.decorator';
import { Process } from 'src/engine/core-modules/message-queue/decorators/process.decorator';
import { Processor } from 'src/engine/core-modules/message-queue/decorators/processor.decorator';
import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';
import { MessageQueueService } from 'src/engine/core-modules/message-queue/services/message-queue.service';
import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity';
import { TRASH_CLEANUP_CRON_PATTERN } from 'src/engine/trash-cleanup/constants/trash-cleanup.constants';
import {
TrashCleanupJob,
type TrashCleanupJobData,
} from 'src/engine/trash-cleanup/jobs/trash-cleanup.job';
@Injectable()
@Processor(MessageQueue.cronQueue)
export class TrashCleanupCronJob {
private readonly logger = new Logger(TrashCleanupCronJob.name);
constructor(
@InjectRepository(Workspace)
private readonly workspaceRepository: Repository<Workspace>,
@InjectMessageQueue(MessageQueue.workspaceQueue)
private readonly messageQueueService: MessageQueueService,
private readonly exceptionHandlerService: ExceptionHandlerService,
) {}
@Process(TrashCleanupCronJob.name)
@SentryCronMonitor(TrashCleanupCronJob.name, TRASH_CLEANUP_CRON_PATTERN)
async handle(): Promise<void> {
const workspaces = await this.getActiveWorkspaces();
if (workspaces.length === 0) {
this.logger.log('No active workspaces found for trash cleanup');
return;
}
this.logger.log(
`Enqueuing trash cleanup jobs for ${workspaces.length} workspace(s)`,
);
for (const workspace of workspaces) {
try {
await this.messageQueueService.add<TrashCleanupJobData>(
TrashCleanupJob.name,
{
workspaceId: workspace.id,
trashRetentionDays: workspace.trashRetentionDays,
},
);
} catch (error) {
this.exceptionHandlerService.captureExceptions([error], {
workspace: {
id: workspace.id,
},
});
}
}
this.logger.log(
`Successfully enqueued ${workspaces.length} trash cleanup job(s)`,
);
}
private async getActiveWorkspaces(): Promise<
Array<{ id: string; trashRetentionDays: number }>
> {
const workspaces = await this.workspaceRepository.find({
where: {
activationStatus: WorkspaceActivationStatus.ACTIVE,
},
select: ['id', 'trashRetentionDays'],
order: { id: 'ASC' },
});
if (workspaces.length === 0) {
return [];
}
return workspaces.map((workspace) => ({
id: workspace.id,
trashRetentionDays: workspace.trashRetentionDays,
}));
}
}
@@ -0,0 +1,37 @@
import { Injectable, Logger } from '@nestjs/common';
import { Process } from 'src/engine/core-modules/message-queue/decorators/process.decorator';
import { Processor } from 'src/engine/core-modules/message-queue/decorators/processor.decorator';
import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';
import { TrashCleanupService } from 'src/engine/trash-cleanup/services/trash-cleanup.service';
export type TrashCleanupJobData = {
workspaceId: string;
trashRetentionDays: number;
};
@Injectable()
@Processor(MessageQueue.workspaceQueue)
export class TrashCleanupJob {
private readonly logger = new Logger(TrashCleanupJob.name);
constructor(private readonly trashCleanupService: TrashCleanupService) {}
@Process(TrashCleanupJob.name)
async handle(data: TrashCleanupJobData): Promise<void> {
const { workspaceId, trashRetentionDays } = data;
try {
await this.trashCleanupService.cleanupWorkspaceTrash({
workspaceId,
trashRetentionDays,
});
} catch (error) {
this.logger.error(
`Trash cleanup failed for workspace ${workspaceId}`,
error instanceof Error ? error.stack : String(error),
);
throw error;
}
}
}
@@ -0,0 +1,208 @@
import { Test, type TestingModule } from '@nestjs/testing';
import { WorkspaceManyOrAllFlatEntityMapsCacheService } from 'src/engine/metadata-modules/flat-entity/services/workspace-many-or-all-flat-entity-maps-cache.service';
import { TrashCleanupService } from 'src/engine/trash-cleanup/services/trash-cleanup.service';
import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager';
describe('TrashCleanupService', () => {
let service: TrashCleanupService;
let mockFlatEntityMapsCacheService: any;
let mockTwentyORMGlobalManager: any;
beforeEach(async () => {
mockFlatEntityMapsCacheService = {
getOrRecomputeManyOrAllFlatEntityMaps: jest.fn(),
};
mockTwentyORMGlobalManager = {
getRepositoryForWorkspace: jest.fn(),
};
const module: TestingModule = await Test.createTestingModule({
providers: [
TrashCleanupService,
{
provide: WorkspaceManyOrAllFlatEntityMapsCacheService,
useValue: mockFlatEntityMapsCacheService,
},
{
provide: TwentyORMGlobalManager,
useValue: mockTwentyORMGlobalManager,
},
],
}).compile();
service = module.get<TrashCleanupService>(TrashCleanupService);
// Suppress logger output in tests
jest.spyOn(service['logger'], 'log').mockImplementation();
jest.spyOn(service['logger'], 'error').mockImplementation();
jest.clearAllMocks();
});
it('should be defined', () => {
expect(service).toBeDefined();
});
describe('cleanupWorkspaceTrash', () => {
const createRepositoryMock = (name: string, initialCount: number) => {
let remaining = initialCount;
let counter = 0;
return {
find: jest.fn().mockImplementation(({ take }) => {
const amount = Math.min(take ?? remaining, remaining);
const records = Array.from({ length: amount }, () => ({
id: `${name}-${counter++}`,
}));
remaining -= amount;
return Promise.resolve(records);
}),
delete: jest.fn().mockResolvedValue(undefined),
};
};
const setObjectMetadataCache = (
entries: Array<{ id: string; nameSingular: string }>,
) => {
const byId = entries.reduce<Record<string, any>>(
(acc, { id, nameSingular }) => {
acc[id] = {
id,
nameSingular,
};
return acc;
},
{},
);
mockFlatEntityMapsCacheService.getOrRecomputeManyOrAllFlatEntityMaps.mockResolvedValue(
{
flatObjectMetadataMaps: {
byId,
idByUniversalIdentifier: {},
},
},
);
};
it('should return deleted count when cleanup succeeds', async () => {
setObjectMetadataCache([
{ id: 'obj-company', nameSingular: 'company' },
{ id: 'obj-person', nameSingular: 'person' },
]);
const companyRepository = createRepositoryMock('company', 2);
const personRepository = createRepositoryMock('person', 1);
mockTwentyORMGlobalManager.getRepositoryForWorkspace
.mockResolvedValueOnce(companyRepository)
.mockResolvedValueOnce(personRepository);
const result = await service.cleanupWorkspaceTrash({
workspaceId: 'workspace-id',
trashRetentionDays: 14,
});
expect(result).toEqual(3);
expect(companyRepository.find).toHaveBeenCalled();
expect(personRepository.find).toHaveBeenCalled();
expect(companyRepository.delete).toHaveBeenCalledTimes(1);
expect(personRepository.delete).toHaveBeenCalledTimes(1);
const findArgs = companyRepository.find.mock.calls[0][0];
expect(findArgs.withDeleted).toBe(true);
expect(findArgs.order).toEqual({ deletedAt: 'ASC' });
});
it('should return zero when no objects are found', async () => {
setObjectMetadataCache([]);
const result = await service.cleanupWorkspaceTrash({
workspaceId: 'workspace-id',
trashRetentionDays: 14,
});
expect(result).toEqual(0);
expect(
mockTwentyORMGlobalManager.getRepositoryForWorkspace,
).not.toHaveBeenCalled();
});
it('should respect max records limit across objects', async () => {
(service as any).maxRecordsPerWorkspace = 3;
(service as any).batchSize = 3;
setObjectMetadataCache([
{ id: 'obj-company', nameSingular: 'company' },
{ id: 'obj-person', nameSingular: 'person' },
]);
const companyRepository = createRepositoryMock('company', 2);
const personRepository = createRepositoryMock('person', 5);
mockTwentyORMGlobalManager.getRepositoryForWorkspace
.mockResolvedValueOnce(companyRepository)
.mockResolvedValueOnce(personRepository);
const result = await service.cleanupWorkspaceTrash({
workspaceId: 'workspace-id',
trashRetentionDays: 14,
});
expect(result).toEqual(3);
expect(companyRepository.delete).toHaveBeenCalledTimes(1);
expect(personRepository.delete).toHaveBeenCalledTimes(1);
const personDeleteArgs = personRepository.delete.mock.calls[0][0];
const deletedIds =
personDeleteArgs.id._value ?? personDeleteArgs.id.value;
expect(deletedIds).toHaveLength(1);
expect(personRepository.find).toHaveBeenCalledTimes(1);
});
it('should ignore objects without soft deleted records', async () => {
setObjectMetadataCache([{ id: 'obj-company', nameSingular: 'company' }]);
const companyRepository = createRepositoryMock('company', 0);
mockTwentyORMGlobalManager.getRepositoryForWorkspace.mockResolvedValueOnce(
companyRepository,
);
const result = await service.cleanupWorkspaceTrash({
workspaceId: 'workspace-id',
trashRetentionDays: 14,
});
expect(result).toEqual(0);
expect(companyRepository.delete).not.toHaveBeenCalled();
});
it('should delete records across multiple batches', async () => {
setObjectMetadataCache([{ id: 'obj-company', nameSingular: 'company' }]);
const companyRepository = createRepositoryMock('company', 5);
mockTwentyORMGlobalManager.getRepositoryForWorkspace.mockResolvedValueOnce(
companyRepository,
);
(service as any).batchSize = 2;
(service as any).maxRecordsPerWorkspace = 10;
const result = await service.cleanupWorkspaceTrash({
workspaceId: 'workspace-id',
trashRetentionDays: 14,
});
expect(result).toEqual(5);
expect(companyRepository.find).toHaveBeenCalledTimes(4);
expect(companyRepository.delete).toHaveBeenCalledTimes(3);
});
});
});
@@ -0,0 +1,146 @@
import { Injectable, Logger } from '@nestjs/common';
import { isDefined } from 'twenty-shared/utils';
import { In, LessThan } from 'typeorm';
import { WorkspaceManyOrAllFlatEntityMapsCacheService } from 'src/engine/metadata-modules/flat-entity/services/workspace-many-or-all-flat-entity-maps-cache.service';
import {
TRASH_CLEANUP_BATCH_SIZE,
TRASH_CLEANUP_MAX_RECORDS_PER_WORKSPACE,
} from 'src/engine/trash-cleanup/constants/trash-cleanup.constants';
import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager';
export type TrashCleanupInput = {
workspaceId: string;
trashRetentionDays: number;
};
@Injectable()
export class TrashCleanupService {
private readonly logger = new Logger(TrashCleanupService.name);
private readonly maxRecordsPerWorkspace =
TRASH_CLEANUP_MAX_RECORDS_PER_WORKSPACE;
private readonly batchSize = TRASH_CLEANUP_BATCH_SIZE;
constructor(
private readonly flatEntityMapsCacheService: WorkspaceManyOrAllFlatEntityMapsCacheService,
private readonly twentyORMGlobalManager: TwentyORMGlobalManager,
) {}
async cleanupWorkspaceTrash(input: TrashCleanupInput): Promise<number> {
const { workspaceId, trashRetentionDays } = input;
const { flatObjectMetadataMaps } =
await this.flatEntityMapsCacheService.getOrRecomputeManyOrAllFlatEntityMaps(
{
workspaceId,
flatMapsKeys: ['flatObjectMetadataMaps'],
},
);
const objectNames = Object.values(flatObjectMetadataMaps.byId ?? {})
.map((metadata) => metadata?.nameSingular)
.filter(isDefined);
if (objectNames.length === 0) {
this.logger.log(`No objects found in workspace ${workspaceId}`);
return 0;
}
const cutoffDate = this.calculateCutoffDate(trashRetentionDays);
let deletedCount = 0;
for (const objectName of objectNames) {
if (deletedCount >= this.maxRecordsPerWorkspace) {
this.logger.log(
`Reached deletion limit (${this.maxRecordsPerWorkspace}) for workspace ${workspaceId}`,
);
break;
}
const remainingQuota = this.maxRecordsPerWorkspace - deletedCount;
const deletedForObject = await this.deleteSoftDeletedRecords({
workspaceId,
objectName,
cutoffDate,
remainingQuota,
});
if (deletedForObject > 0) {
this.logger.log(
`Deleted ${deletedForObject} record(s) from ${objectName} in workspace ${workspaceId}`,
);
}
deletedCount += deletedForObject;
}
this.logger.log(
`Deleted ${deletedCount} record(s) from workspace ${workspaceId}`,
);
return deletedCount;
}
private async deleteSoftDeletedRecords({
workspaceId,
objectName,
cutoffDate,
remainingQuota,
}: {
workspaceId: string;
objectName: string;
cutoffDate: Date;
remainingQuota: number;
}): Promise<number> {
if (remainingQuota <= 0) {
return 0;
}
const repository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace(
workspaceId,
objectName,
{ shouldBypassPermissionChecks: true },
);
let deleted = 0;
while (deleted < remainingQuota) {
const take = Math.min(this.batchSize, remainingQuota - deleted);
const recordsToDelete = await repository.find({
withDeleted: true,
select: ['id'],
where: {
deletedAt: LessThan(cutoffDate),
},
order: { deletedAt: 'ASC' },
take,
loadEagerRelations: false,
});
if (recordsToDelete.length === 0) {
break;
}
await repository.delete({
id: In(recordsToDelete.map((record) => record.id)),
});
deleted += recordsToDelete.length;
}
return deleted;
}
private calculateCutoffDate(trashRetentionDays: number): Date {
const cutoffDate = new Date();
cutoffDate.setUTCHours(0, 0, 0, 0);
cutoffDate.setDate(cutoffDate.getDate() - trashRetentionDays + 1);
return cutoffDate;
}
}
@@ -0,0 +1,24 @@
import { Module } from '@nestjs/common';
import { TypeOrmModule } from '@nestjs/typeorm';
import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity';
import { WorkspaceManyOrAllFlatEntityMapsCacheModule } from 'src/engine/metadata-modules/flat-entity/services/workspace-many-or-all-flat-entity-maps-cache.module';
import { TrashCleanupCronCommand } from 'src/engine/trash-cleanup/commands/trash-cleanup.cron.command';
import { TrashCleanupCronJob } from 'src/engine/trash-cleanup/crons/trash-cleanup.cron.job';
import { TrashCleanupJob } from 'src/engine/trash-cleanup/jobs/trash-cleanup.job';
import { TrashCleanupService } from 'src/engine/trash-cleanup/services/trash-cleanup.service';
@Module({
imports: [
TypeOrmModule.forFeature([Workspace]),
WorkspaceManyOrAllFlatEntityMapsCacheModule,
],
providers: [
TrashCleanupService,
TrashCleanupJob,
TrashCleanupCronJob,
TrashCleanupCronCommand,
],
exports: [TrashCleanupCronCommand],
})
export class TrashCleanupModule {}