Automatically clean up soft-deleted records after X days. (#14862)
Closes #14726 ### Added - `trashRetentionDays` field to workspace entity (default: 14 days) - Automated trash cleanup using BullMQ jobs - Daily cron (00:10 UTC) that enqueues cleanup jobs for all active workspaces - Per-workspace limit: 100k records deleted per day - Calendar-based retention: records deleted on day X are cleaned up X+14 days later (at midnight UTC boundaries) ### Architecture - **Cron (WorkspaceTrashCleanupCronJob):** Runs daily, enqueues jobs in parallel for all workspaces - **Job (WorkspaceTrashCleanupJob):** Processes individual workspace cleanup - **Service (WorkspaceTrashCleanupService):** Discovers tables with `deletedAt`, deletes old records with quota enforcement - **Command:** `npx nx run twenty-server:command cron:workspace:cleanup-trash` to register the cron ### Testing - Unit tests for service with 100% coverage of public API - Tested quota enforcement, error handling, and edge cases --------- Co-authored-by: Félix Malfait <felix.malfait@gmail.com>
This commit is contained in:
File diff suppressed because one or more lines are too long
@@ -4077,6 +4077,7 @@ export type UpdateWorkspaceInput = {
|
||||
isTwoFactorAuthenticationEnforced?: InputMaybe<Scalars['Boolean']>;
|
||||
logo?: InputMaybe<Scalars['String']>;
|
||||
subdomain?: InputMaybe<Scalars['String']>;
|
||||
trashRetentionDays?: InputMaybe<Scalars['Float']>;
|
||||
};
|
||||
|
||||
export type UpsertFieldPermissionsInput = {
|
||||
@@ -4362,6 +4363,7 @@ export type Workspace = {
|
||||
logo?: Maybe<Scalars['String']>;
|
||||
metadataVersion: Scalars['Float'];
|
||||
subdomain: Scalars['String'];
|
||||
trashRetentionDays: Scalars['Float'];
|
||||
updatedAt: Scalars['DateTime'];
|
||||
version?: Maybe<Scalars['String']>;
|
||||
viewFields?: Maybe<Array<CoreViewField>>;
|
||||
|
||||
@@ -57,6 +57,7 @@ const mockWorkspace = {
|
||||
customUrl: 'test.com',
|
||||
},
|
||||
isTwoFactorAuthenticationEnforced: false,
|
||||
trashRetentionDays: 14,
|
||||
};
|
||||
|
||||
const createMockOptions = (): Options<any> => ({
|
||||
|
||||
@@ -24,6 +24,7 @@ export type CurrentWorkspace = Pick<
|
||||
| 'workspaceUrls'
|
||||
| 'metadataVersion'
|
||||
| 'isTwoFactorAuthenticationEnforced'
|
||||
| 'trashRetentionDays'
|
||||
> & {
|
||||
defaultRole?: Omit<Role, 'workspaceMembers' | 'agents' | 'apiKeys'> | null;
|
||||
defaultAgent?: { id: string } | null;
|
||||
|
||||
+1
@@ -50,6 +50,7 @@ const Wrapper = getJestMetadataAndApolloMocksAndActionMenuWrapper({
|
||||
},
|
||||
],
|
||||
isTwoFactorAuthenticationEnforced: false,
|
||||
trashRetentionDays: 14,
|
||||
});
|
||||
},
|
||||
});
|
||||
|
||||
@@ -10,14 +10,16 @@ type SettingsCounterProps = {
|
||||
minValue?: number;
|
||||
maxValue?: number;
|
||||
disabled?: boolean;
|
||||
showButtons?: boolean;
|
||||
};
|
||||
|
||||
const StyledCounterContainer = styled.div`
|
||||
const StyledCounterContainer = styled.div<{ showButtons: boolean }>`
|
||||
align-items: center;
|
||||
display: flex;
|
||||
gap: ${({ theme }) => theme.spacing(1)};
|
||||
margin-left: auto;
|
||||
width: ${({ theme }) => theme.spacing(30)};
|
||||
width: ${({ theme, showButtons }) =>
|
||||
showButtons ? theme.spacing(30) : theme.spacing(16)};
|
||||
`;
|
||||
|
||||
const StyledTextInput = styled(SettingsTextInput)`
|
||||
@@ -34,11 +36,12 @@ export const SettingsCounter = ({
|
||||
value,
|
||||
onChange,
|
||||
minValue = 0,
|
||||
maxValue = 100,
|
||||
maxValue,
|
||||
disabled = false,
|
||||
showButtons = true,
|
||||
}: SettingsCounterProps) => {
|
||||
const handleIncrementCounter = () => {
|
||||
if (value < maxValue) {
|
||||
if (maxValue === undefined || value < maxValue) {
|
||||
onChange(value + 1);
|
||||
}
|
||||
};
|
||||
@@ -60,7 +63,7 @@ export const SettingsCounter = ({
|
||||
return;
|
||||
}
|
||||
|
||||
if (castedNumber > maxValue) {
|
||||
if (maxValue !== undefined && castedNumber > maxValue) {
|
||||
onChange(maxValue);
|
||||
return;
|
||||
}
|
||||
@@ -68,14 +71,16 @@ export const SettingsCounter = ({
|
||||
};
|
||||
|
||||
return (
|
||||
<StyledCounterContainer>
|
||||
<IconButton
|
||||
size="small"
|
||||
Icon={IconMinus}
|
||||
variant="secondary"
|
||||
onClick={handleDecrementCounter}
|
||||
disabled={disabled}
|
||||
/>
|
||||
<StyledCounterContainer showButtons={showButtons}>
|
||||
{showButtons && (
|
||||
<IconButton
|
||||
size="small"
|
||||
Icon={IconMinus}
|
||||
variant="secondary"
|
||||
onClick={handleDecrementCounter}
|
||||
disabled={disabled}
|
||||
/>
|
||||
)}
|
||||
<StyledTextInput
|
||||
instanceId="settings-counter-input"
|
||||
name="counter"
|
||||
@@ -84,13 +89,15 @@ export const SettingsCounter = ({
|
||||
onChange={handleTextInputChange}
|
||||
disabled={disabled}
|
||||
/>
|
||||
<IconButton
|
||||
size="small"
|
||||
Icon={IconPlus}
|
||||
variant="secondary"
|
||||
onClick={handleIncrementCounter}
|
||||
disabled={disabled}
|
||||
/>
|
||||
{showButtons && (
|
||||
<IconButton
|
||||
size="small"
|
||||
Icon={IconPlus}
|
||||
variant="secondary"
|
||||
onClick={handleIncrementCounter}
|
||||
disabled={disabled}
|
||||
/>
|
||||
)}
|
||||
</StyledCounterContainer>
|
||||
);
|
||||
};
|
||||
|
||||
+3
@@ -17,6 +17,7 @@ type SettingsOptionCardContentCounterProps = {
|
||||
onChange: (value: number) => void;
|
||||
minValue?: number;
|
||||
maxValue?: number;
|
||||
showButtons?: boolean;
|
||||
};
|
||||
|
||||
export const SettingsOptionCardContentCounter = ({
|
||||
@@ -28,6 +29,7 @@ export const SettingsOptionCardContentCounter = ({
|
||||
onChange,
|
||||
minValue,
|
||||
maxValue,
|
||||
showButtons = true,
|
||||
}: SettingsOptionCardContentCounterProps) => {
|
||||
return (
|
||||
<StyledSettingsOptionCardContent disabled={disabled}>
|
||||
@@ -50,6 +52,7 @@ export const SettingsOptionCardContentCounter = ({
|
||||
minValue={minValue}
|
||||
maxValue={maxValue}
|
||||
disabled={disabled}
|
||||
showButtons={showButtons}
|
||||
/>
|
||||
</StyledSettingsOptionCardContent>
|
||||
);
|
||||
|
||||
+15
@@ -25,6 +25,7 @@ const SettingsOptionCardContentCounterWrapper = (
|
||||
disabled={args.disabled}
|
||||
minValue={args.minValue}
|
||||
maxValue={args.maxValue}
|
||||
showButtons={args.showButtons}
|
||||
/>
|
||||
</StyledContainer>
|
||||
);
|
||||
@@ -50,6 +51,7 @@ export const Default: Story = {
|
||||
value: 5,
|
||||
minValue: 1,
|
||||
maxValue: 10,
|
||||
showButtons: true,
|
||||
},
|
||||
argTypes: {
|
||||
Icon: { control: false },
|
||||
@@ -64,6 +66,7 @@ export const WithoutIcon: Story = {
|
||||
value: 20,
|
||||
minValue: 10,
|
||||
maxValue: 50,
|
||||
showButtons: true,
|
||||
},
|
||||
};
|
||||
|
||||
@@ -76,5 +79,17 @@ export const Disabled: Story = {
|
||||
disabled: true,
|
||||
minValue: 1,
|
||||
maxValue: 10,
|
||||
showButtons: true,
|
||||
},
|
||||
};
|
||||
|
||||
export const WithoutButtons: Story = {
|
||||
args: {
|
||||
Icon: IconUsers,
|
||||
title: 'Trash Retention',
|
||||
description: 'Adjust the number of days before deletion',
|
||||
value: 14,
|
||||
minValue: 0,
|
||||
showButtons: false,
|
||||
},
|
||||
};
|
||||
|
||||
@@ -79,6 +79,7 @@ export const USER_QUERY_FRAGMENT = gql`
|
||||
id
|
||||
}
|
||||
isTwoFactorAuthenticationEnforced
|
||||
trashRetentionDays
|
||||
}
|
||||
availableWorkspaces {
|
||||
...AvailableWorkspacesFragment
|
||||
|
||||
@@ -1,19 +1,24 @@
|
||||
import styled from '@emotion/styled';
|
||||
import { Trans, useLingui } from '@lingui/react/macro';
|
||||
import { useDebouncedCallback } from 'use-debounce';
|
||||
|
||||
import { currentWorkspaceState } from '@/auth/states/currentWorkspaceState';
|
||||
import { isMultiWorkspaceEnabledState } from '@/client-config/states/isMultiWorkspaceEnabledState';
|
||||
import { SettingsOptionCardContentCounter } from '@/settings/components/SettingsOptions/SettingsOptionCardContentCounter';
|
||||
import { SettingsPageContainer } from '@/settings/components/SettingsPageContainer';
|
||||
import { SettingsSSOIdentitiesProvidersListCard } from '@/settings/security/components/SSO/SettingsSSOIdentitiesProvidersListCard';
|
||||
import { SettingsSecurityAuthProvidersOptionsList } from '@/settings/security/components/SettingsSecurityAuthProvidersOptionsList';
|
||||
|
||||
import { ToggleImpersonate } from '@/settings/workspace/components/ToggleImpersonate';
|
||||
import { useSnackBar } from '@/ui/feedback/snack-bar-manager/hooks/useSnackBar';
|
||||
import { SubMenuTopBarContainer } from '@/ui/layout/page/components/SubMenuTopBarContainer';
|
||||
import { useRecoilValue } from 'recoil';
|
||||
import { ApolloError } from '@apollo/client';
|
||||
import { useRecoilState, useRecoilValue } from 'recoil';
|
||||
import { SettingsPath } from 'twenty-shared/types';
|
||||
import { getSettingsPath } from 'twenty-shared/utils';
|
||||
import { Tag } from 'twenty-ui/components';
|
||||
import { H2Title, IconLock } from 'twenty-ui/display';
|
||||
import { Section } from 'twenty-ui/layout';
|
||||
import { H2Title, IconLock, IconTrash } from 'twenty-ui/display';
|
||||
import { Card, Section } from 'twenty-ui/layout';
|
||||
import { useUpdateWorkspaceMutation } from '~/generated-metadata/graphql';
|
||||
|
||||
const StyledContainer = styled.div`
|
||||
width: 100%;
|
||||
@@ -32,8 +37,50 @@ const StyledSection = styled(Section)`
|
||||
|
||||
export const SettingsSecurity = () => {
|
||||
const { t } = useLingui();
|
||||
const { enqueueErrorSnackBar } = useSnackBar();
|
||||
|
||||
const isMultiWorkspaceEnabled = useRecoilValue(isMultiWorkspaceEnabledState);
|
||||
const [currentWorkspace, setCurrentWorkspace] = useRecoilState(
|
||||
currentWorkspaceState,
|
||||
);
|
||||
const [updateWorkspace] = useUpdateWorkspaceMutation();
|
||||
|
||||
const saveWorkspace = useDebouncedCallback(async (value: number) => {
|
||||
try {
|
||||
if (!currentWorkspace?.id) {
|
||||
throw new Error('User is not logged in');
|
||||
}
|
||||
|
||||
await updateWorkspace({
|
||||
variables: {
|
||||
input: {
|
||||
trashRetentionDays: value,
|
||||
},
|
||||
},
|
||||
});
|
||||
} catch (err) {
|
||||
enqueueErrorSnackBar({
|
||||
apolloError: err instanceof ApolloError ? err : undefined,
|
||||
});
|
||||
}
|
||||
}, 500);
|
||||
|
||||
const handleTrashRetentionDaysChange = (value: number) => {
|
||||
if (!currentWorkspace) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (value === currentWorkspace.trashRetentionDays) {
|
||||
return;
|
||||
}
|
||||
|
||||
setCurrentWorkspace({
|
||||
...currentWorkspace,
|
||||
trashRetentionDays: value,
|
||||
});
|
||||
|
||||
saveWorkspace(value);
|
||||
};
|
||||
|
||||
return (
|
||||
<SubMenuTopBarContainer
|
||||
@@ -82,6 +129,23 @@ export const SettingsSecurity = () => {
|
||||
<ToggleImpersonate />
|
||||
</Section>
|
||||
)}
|
||||
<Section>
|
||||
<H2Title
|
||||
title={t`Other`}
|
||||
description={t`Other security settings`}
|
||||
/>
|
||||
<Card rounded>
|
||||
<SettingsOptionCardContentCounter
|
||||
Icon={IconTrash}
|
||||
title={t`Erasure of soft-deleted records`}
|
||||
description={t`Permanent deletion. Enter the number of days.`}
|
||||
value={currentWorkspace?.trashRetentionDays ?? 14}
|
||||
onChange={handleTrashRetentionDaysChange}
|
||||
minValue={0}
|
||||
showButtons={false}
|
||||
/>
|
||||
</Card>
|
||||
</Section>
|
||||
</StyledMainContent>
|
||||
</SettingsPageContainer>
|
||||
</SubMenuTopBarContainer>
|
||||
|
||||
@@ -84,6 +84,7 @@ export const mockCurrentWorkspace: Workspace = {
|
||||
createdAt: '2023-04-26T10:23:42.33625+00:00',
|
||||
updatedAt: '2023-04-26T10:23:42.33625+00:00',
|
||||
metadataVersion: 1,
|
||||
trashRetentionDays: 14,
|
||||
currentBillingSubscription: {
|
||||
__typename: 'BillingSubscription',
|
||||
id: '7efbc3f7-6e5e-4128-957e-8d86808cdf6a',
|
||||
@@ -151,6 +152,7 @@ export const mockCurrentWorkspace: Workspace = {
|
||||
databaseSchema: '',
|
||||
databaseUrl: '',
|
||||
isTwoFactorAuthenticationEnforced: false,
|
||||
__typename: 'Workspace',
|
||||
};
|
||||
|
||||
export const mockedWorkspaceMemberData: WorkspaceMember = {
|
||||
|
||||
@@ -78,4 +78,4 @@ FRONTEND_URL=http://localhost:3001
|
||||
# CLOUDFLARE_WEBHOOK_SECRET=
|
||||
# IS_CONFIG_VARIABLES_IN_DB_ENABLED=false
|
||||
# ANALYTICS_ENABLED=
|
||||
# CLICKHOUSE_URL=http://default:clickhousePassword@localhost:8123/twenty
|
||||
# CLICKHOUSE_URL=http://default:clickhousePassword@localhost:8123/twenty
|
||||
@@ -7,6 +7,7 @@ import { CheckCustomDomainValidRecordsCronCommand } from 'src/engine/core-module
|
||||
import { CronTriggerCronCommand } from 'src/engine/metadata-modules/cron-trigger/crons/commands/cron-trigger.cron.command';
|
||||
import { CleanOnboardingWorkspacesCronCommand } from 'src/engine/workspace-manager/workspace-cleaner/commands/clean-onboarding-workspaces.cron.command';
|
||||
import { CleanSuspendedWorkspacesCronCommand } from 'src/engine/workspace-manager/workspace-cleaner/commands/clean-suspended-workspaces.cron.command';
|
||||
import { TrashCleanupCronCommand } from 'src/engine/trash-cleanup/commands/trash-cleanup.cron.command';
|
||||
import { CalendarEventListFetchCronCommand } from 'src/modules/calendar/calendar-event-import-manager/crons/commands/calendar-event-list-fetch.cron.command';
|
||||
import { CalendarEventsImportCronCommand } from 'src/modules/calendar/calendar-event-import-manager/crons/commands/calendar-import.cron.command';
|
||||
import { CalendarOngoingStaleCronCommand } from 'src/modules/calendar/calendar-event-import-manager/crons/commands/calendar-ongoing-stale.cron.command';
|
||||
@@ -42,6 +43,7 @@ export class CronRegisterAllCommand extends CommandRunner {
|
||||
private readonly cronTriggerCronCommand: CronTriggerCronCommand,
|
||||
private readonly cleanSuspendedWorkspacesCronCommand: CleanSuspendedWorkspacesCronCommand,
|
||||
private readonly cleanOnboardingWorkspacesCronCommand: CleanOnboardingWorkspacesCronCommand,
|
||||
private readonly trashCleanupCronCommand: TrashCleanupCronCommand,
|
||||
) {
|
||||
super();
|
||||
}
|
||||
@@ -110,6 +112,10 @@ export class CronRegisterAllCommand extends CommandRunner {
|
||||
name: 'CleanOnboardingWorkspaces',
|
||||
command: this.cleanOnboardingWorkspacesCronCommand,
|
||||
},
|
||||
{
|
||||
name: 'TrashCleanup',
|
||||
command: this.trashCleanupCronCommand,
|
||||
},
|
||||
];
|
||||
|
||||
let successCount = 0;
|
||||
|
||||
@@ -20,6 +20,7 @@ import { ObjectMetadataModule } from 'src/engine/metadata-modules/object-metadat
|
||||
import { WorkspaceCacheStorageModule } from 'src/engine/workspace-cache-storage/workspace-cache-storage.module';
|
||||
import { DevSeederModule } from 'src/engine/workspace-manager/dev-seeder/dev-seeder.module';
|
||||
import { WorkspaceCleanerModule } from 'src/engine/workspace-manager/workspace-cleaner/workspace-cleaner.module';
|
||||
import { TrashCleanupModule } from 'src/engine/trash-cleanup/trash-cleanup.module';
|
||||
import { WorkspaceManagerModule } from 'src/engine/workspace-manager/workspace-manager.module';
|
||||
import { CalendarEventImportManagerModule } from 'src/modules/calendar/calendar-event-import-manager/calendar-event-import-manager.module';
|
||||
import { MessagingImportManagerModule } from 'src/modules/messaging/message-import-manager/messaging-import-manager.module';
|
||||
@@ -50,6 +51,7 @@ import { AutomatedTriggerModule } from 'src/modules/workflow/workflow-trigger/au
|
||||
CronTriggerModule,
|
||||
DatabaseEventTriggerModule,
|
||||
WorkspaceCleanerModule,
|
||||
TrashCleanupModule,
|
||||
PublicDomainModule,
|
||||
],
|
||||
providers: [
|
||||
|
||||
+19
@@ -0,0 +1,19 @@
|
||||
import { type MigrationInterface, type QueryRunner } from 'typeorm';
|
||||
|
||||
export class AddWorkspaceTrashRetention1760356369619
|
||||
implements MigrationInterface
|
||||
{
|
||||
name = 'AddWorkspaceTrashRetention1760356369619';
|
||||
|
||||
public async up(queryRunner: QueryRunner): Promise<void> {
|
||||
await queryRunner.query(
|
||||
`ALTER TABLE "core"."workspace" ADD "trashRetentionDays" integer NOT NULL DEFAULT '14'`,
|
||||
);
|
||||
}
|
||||
|
||||
public async down(queryRunner: QueryRunner): Promise<void> {
|
||||
await queryRunner.query(
|
||||
`ALTER TABLE "core"."workspace" DROP COLUMN "trashRetentionDays"`,
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -56,6 +56,7 @@ import { WorkspaceInvitationModule } from 'src/engine/core-modules/workspace-inv
|
||||
import { WorkspaceModule } from 'src/engine/core-modules/workspace/workspace.module';
|
||||
import { RoleModule } from 'src/engine/metadata-modules/role/role.module';
|
||||
import { SubscriptionsModule } from 'src/engine/subscriptions/subscriptions.module';
|
||||
import { TrashCleanupModule } from 'src/engine/trash-cleanup/trash-cleanup.module';
|
||||
import { WorkspaceEventEmitterModule } from 'src/engine/workspace-event-emitter/workspace-event-emitter.module';
|
||||
import { ChannelSyncModule } from 'src/modules/connected-account/channel-sync/channel-sync.module';
|
||||
|
||||
@@ -134,6 +135,7 @@ import { FileModule } from './file/file.module';
|
||||
WebhookModule,
|
||||
PageLayoutModule,
|
||||
ImpersonationModule,
|
||||
TrashCleanupModule,
|
||||
],
|
||||
exports: [
|
||||
AuditModule,
|
||||
|
||||
+8
@@ -2,11 +2,13 @@ import { Field, InputType } from '@nestjs/graphql';
|
||||
|
||||
import {
|
||||
IsBoolean,
|
||||
IsInt,
|
||||
IsNotIn,
|
||||
IsOptional,
|
||||
IsString,
|
||||
IsUUID,
|
||||
Matches,
|
||||
Min,
|
||||
} from 'class-validator';
|
||||
|
||||
import { UUIDScalarType } from 'src/engine/api/graphql/workspace-schema-builder/graphql-types/scalars';
|
||||
@@ -197,4 +199,10 @@ export class UpdateWorkspaceInput {
|
||||
@IsBoolean()
|
||||
@IsOptional()
|
||||
isTwoFactorAuthenticationEnforced?: boolean;
|
||||
|
||||
@Field({ nullable: true })
|
||||
@IsInt()
|
||||
@Min(0)
|
||||
@IsOptional()
|
||||
trashRetentionDays?: number;
|
||||
}
|
||||
|
||||
+2
-1
@@ -476,7 +476,8 @@ export class WorkspaceService extends TypeOrmQueryService<Workspace> {
|
||||
'displayName' in payload ||
|
||||
'subdomain' in payload ||
|
||||
'customDomain' in payload ||
|
||||
'logo' in payload
|
||||
'logo' in payload ||
|
||||
'trashRetentionDays' in payload
|
||||
) {
|
||||
if (!userWorkspaceId) {
|
||||
throw new Error('Missing userWorkspaceId in authContext');
|
||||
|
||||
@@ -92,6 +92,10 @@ export class Workspace {
|
||||
@Column({ default: true })
|
||||
isPublicInviteLinkEnabled: boolean;
|
||||
|
||||
@Field()
|
||||
@Column({ type: 'integer', default: 14 })
|
||||
trashRetentionDays: number;
|
||||
|
||||
// Relations
|
||||
@OneToMany(() => AppToken, (appToken) => appToken.workspace, {
|
||||
cascade: true,
|
||||
|
||||
+32
@@ -0,0 +1,32 @@
|
||||
import { Command, CommandRunner } from 'nest-commander';
|
||||
|
||||
import { InjectMessageQueue } from 'src/engine/core-modules/message-queue/decorators/message-queue.decorator';
|
||||
import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';
|
||||
import { MessageQueueService } from 'src/engine/core-modules/message-queue/services/message-queue.service';
|
||||
import { TRASH_CLEANUP_CRON_PATTERN } from 'src/engine/trash-cleanup/constants/trash-cleanup.constants';
|
||||
import { TrashCleanupCronJob } from 'src/engine/trash-cleanup/crons/trash-cleanup.cron.job';
|
||||
|
||||
@Command({
|
||||
name: 'cron:trash-cleanup',
|
||||
description: 'Starts a cron job to clean up soft-deleted records',
|
||||
})
|
||||
export class TrashCleanupCronCommand extends CommandRunner {
|
||||
constructor(
|
||||
@InjectMessageQueue(MessageQueue.cronQueue)
|
||||
private readonly messageQueueService: MessageQueueService,
|
||||
) {
|
||||
super();
|
||||
}
|
||||
|
||||
async run(): Promise<void> {
|
||||
await this.messageQueueService.addCron<undefined>({
|
||||
jobName: TrashCleanupCronJob.name,
|
||||
data: undefined,
|
||||
options: {
|
||||
repeat: {
|
||||
pattern: TRASH_CLEANUP_CRON_PATTERN,
|
||||
},
|
||||
},
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
// Daily at 00:10 UTC
|
||||
export const TRASH_CLEANUP_CRON_PATTERN = '10 0 * * *';
|
||||
|
||||
export const TRASH_CLEANUP_MAX_RECORDS_PER_WORKSPACE = 1_000_000;
|
||||
export const TRASH_CLEANUP_BATCH_SIZE = 1_000;
|
||||
@@ -0,0 +1,92 @@
|
||||
import { Injectable, Logger } from '@nestjs/common';
|
||||
import { InjectRepository } from '@nestjs/typeorm';
|
||||
|
||||
import { WorkspaceActivationStatus } from 'twenty-shared/workspace';
|
||||
import { Repository } from 'typeorm';
|
||||
|
||||
import { SentryCronMonitor } from 'src/engine/core-modules/cron/sentry-cron-monitor.decorator';
|
||||
import { ExceptionHandlerService } from 'src/engine/core-modules/exception-handler/exception-handler.service';
|
||||
import { InjectMessageQueue } from 'src/engine/core-modules/message-queue/decorators/message-queue.decorator';
|
||||
import { Process } from 'src/engine/core-modules/message-queue/decorators/process.decorator';
|
||||
import { Processor } from 'src/engine/core-modules/message-queue/decorators/processor.decorator';
|
||||
import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';
|
||||
import { MessageQueueService } from 'src/engine/core-modules/message-queue/services/message-queue.service';
|
||||
import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity';
|
||||
import { TRASH_CLEANUP_CRON_PATTERN } from 'src/engine/trash-cleanup/constants/trash-cleanup.constants';
|
||||
import {
|
||||
TrashCleanupJob,
|
||||
type TrashCleanupJobData,
|
||||
} from 'src/engine/trash-cleanup/jobs/trash-cleanup.job';
|
||||
|
||||
@Injectable()
|
||||
@Processor(MessageQueue.cronQueue)
|
||||
export class TrashCleanupCronJob {
|
||||
private readonly logger = new Logger(TrashCleanupCronJob.name);
|
||||
|
||||
constructor(
|
||||
@InjectRepository(Workspace)
|
||||
private readonly workspaceRepository: Repository<Workspace>,
|
||||
@InjectMessageQueue(MessageQueue.workspaceQueue)
|
||||
private readonly messageQueueService: MessageQueueService,
|
||||
private readonly exceptionHandlerService: ExceptionHandlerService,
|
||||
) {}
|
||||
|
||||
@Process(TrashCleanupCronJob.name)
|
||||
@SentryCronMonitor(TrashCleanupCronJob.name, TRASH_CLEANUP_CRON_PATTERN)
|
||||
async handle(): Promise<void> {
|
||||
const workspaces = await this.getActiveWorkspaces();
|
||||
|
||||
if (workspaces.length === 0) {
|
||||
this.logger.log('No active workspaces found for trash cleanup');
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
this.logger.log(
|
||||
`Enqueuing trash cleanup jobs for ${workspaces.length} workspace(s)`,
|
||||
);
|
||||
|
||||
for (const workspace of workspaces) {
|
||||
try {
|
||||
await this.messageQueueService.add<TrashCleanupJobData>(
|
||||
TrashCleanupJob.name,
|
||||
{
|
||||
workspaceId: workspace.id,
|
||||
trashRetentionDays: workspace.trashRetentionDays,
|
||||
},
|
||||
);
|
||||
} catch (error) {
|
||||
this.exceptionHandlerService.captureExceptions([error], {
|
||||
workspace: {
|
||||
id: workspace.id,
|
||||
},
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
this.logger.log(
|
||||
`Successfully enqueued ${workspaces.length} trash cleanup job(s)`,
|
||||
);
|
||||
}
|
||||
|
||||
private async getActiveWorkspaces(): Promise<
|
||||
Array<{ id: string; trashRetentionDays: number }>
|
||||
> {
|
||||
const workspaces = await this.workspaceRepository.find({
|
||||
where: {
|
||||
activationStatus: WorkspaceActivationStatus.ACTIVE,
|
||||
},
|
||||
select: ['id', 'trashRetentionDays'],
|
||||
order: { id: 'ASC' },
|
||||
});
|
||||
|
||||
if (workspaces.length === 0) {
|
||||
return [];
|
||||
}
|
||||
|
||||
return workspaces.map((workspace) => ({
|
||||
id: workspace.id,
|
||||
trashRetentionDays: workspace.trashRetentionDays,
|
||||
}));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,37 @@
|
||||
import { Injectable, Logger } from '@nestjs/common';
|
||||
|
||||
import { Process } from 'src/engine/core-modules/message-queue/decorators/process.decorator';
|
||||
import { Processor } from 'src/engine/core-modules/message-queue/decorators/processor.decorator';
|
||||
import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';
|
||||
import { TrashCleanupService } from 'src/engine/trash-cleanup/services/trash-cleanup.service';
|
||||
|
||||
export type TrashCleanupJobData = {
|
||||
workspaceId: string;
|
||||
trashRetentionDays: number;
|
||||
};
|
||||
|
||||
@Injectable()
|
||||
@Processor(MessageQueue.workspaceQueue)
|
||||
export class TrashCleanupJob {
|
||||
private readonly logger = new Logger(TrashCleanupJob.name);
|
||||
|
||||
constructor(private readonly trashCleanupService: TrashCleanupService) {}
|
||||
|
||||
@Process(TrashCleanupJob.name)
|
||||
async handle(data: TrashCleanupJobData): Promise<void> {
|
||||
const { workspaceId, trashRetentionDays } = data;
|
||||
|
||||
try {
|
||||
await this.trashCleanupService.cleanupWorkspaceTrash({
|
||||
workspaceId,
|
||||
trashRetentionDays,
|
||||
});
|
||||
} catch (error) {
|
||||
this.logger.error(
|
||||
`Trash cleanup failed for workspace ${workspaceId}`,
|
||||
error instanceof Error ? error.stack : String(error),
|
||||
);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
+208
@@ -0,0 +1,208 @@
|
||||
import { Test, type TestingModule } from '@nestjs/testing';
|
||||
|
||||
import { WorkspaceManyOrAllFlatEntityMapsCacheService } from 'src/engine/metadata-modules/flat-entity/services/workspace-many-or-all-flat-entity-maps-cache.service';
|
||||
import { TrashCleanupService } from 'src/engine/trash-cleanup/services/trash-cleanup.service';
|
||||
import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager';
|
||||
|
||||
describe('TrashCleanupService', () => {
|
||||
let service: TrashCleanupService;
|
||||
let mockFlatEntityMapsCacheService: any;
|
||||
let mockTwentyORMGlobalManager: any;
|
||||
|
||||
beforeEach(async () => {
|
||||
mockFlatEntityMapsCacheService = {
|
||||
getOrRecomputeManyOrAllFlatEntityMaps: jest.fn(),
|
||||
};
|
||||
|
||||
mockTwentyORMGlobalManager = {
|
||||
getRepositoryForWorkspace: jest.fn(),
|
||||
};
|
||||
|
||||
const module: TestingModule = await Test.createTestingModule({
|
||||
providers: [
|
||||
TrashCleanupService,
|
||||
{
|
||||
provide: WorkspaceManyOrAllFlatEntityMapsCacheService,
|
||||
useValue: mockFlatEntityMapsCacheService,
|
||||
},
|
||||
{
|
||||
provide: TwentyORMGlobalManager,
|
||||
useValue: mockTwentyORMGlobalManager,
|
||||
},
|
||||
],
|
||||
}).compile();
|
||||
|
||||
service = module.get<TrashCleanupService>(TrashCleanupService);
|
||||
|
||||
// Suppress logger output in tests
|
||||
jest.spyOn(service['logger'], 'log').mockImplementation();
|
||||
jest.spyOn(service['logger'], 'error').mockImplementation();
|
||||
|
||||
jest.clearAllMocks();
|
||||
});
|
||||
|
||||
it('should be defined', () => {
|
||||
expect(service).toBeDefined();
|
||||
});
|
||||
|
||||
describe('cleanupWorkspaceTrash', () => {
|
||||
const createRepositoryMock = (name: string, initialCount: number) => {
|
||||
let remaining = initialCount;
|
||||
let counter = 0;
|
||||
|
||||
return {
|
||||
find: jest.fn().mockImplementation(({ take }) => {
|
||||
const amount = Math.min(take ?? remaining, remaining);
|
||||
const records = Array.from({ length: amount }, () => ({
|
||||
id: `${name}-${counter++}`,
|
||||
}));
|
||||
|
||||
remaining -= amount;
|
||||
|
||||
return Promise.resolve(records);
|
||||
}),
|
||||
delete: jest.fn().mockResolvedValue(undefined),
|
||||
};
|
||||
};
|
||||
|
||||
const setObjectMetadataCache = (
|
||||
entries: Array<{ id: string; nameSingular: string }>,
|
||||
) => {
|
||||
const byId = entries.reduce<Record<string, any>>(
|
||||
(acc, { id, nameSingular }) => {
|
||||
acc[id] = {
|
||||
id,
|
||||
nameSingular,
|
||||
};
|
||||
|
||||
return acc;
|
||||
},
|
||||
{},
|
||||
);
|
||||
|
||||
mockFlatEntityMapsCacheService.getOrRecomputeManyOrAllFlatEntityMaps.mockResolvedValue(
|
||||
{
|
||||
flatObjectMetadataMaps: {
|
||||
byId,
|
||||
idByUniversalIdentifier: {},
|
||||
},
|
||||
},
|
||||
);
|
||||
};
|
||||
|
||||
it('should return deleted count when cleanup succeeds', async () => {
|
||||
setObjectMetadataCache([
|
||||
{ id: 'obj-company', nameSingular: 'company' },
|
||||
{ id: 'obj-person', nameSingular: 'person' },
|
||||
]);
|
||||
|
||||
const companyRepository = createRepositoryMock('company', 2);
|
||||
const personRepository = createRepositoryMock('person', 1);
|
||||
|
||||
mockTwentyORMGlobalManager.getRepositoryForWorkspace
|
||||
.mockResolvedValueOnce(companyRepository)
|
||||
.mockResolvedValueOnce(personRepository);
|
||||
|
||||
const result = await service.cleanupWorkspaceTrash({
|
||||
workspaceId: 'workspace-id',
|
||||
trashRetentionDays: 14,
|
||||
});
|
||||
|
||||
expect(result).toEqual(3);
|
||||
expect(companyRepository.find).toHaveBeenCalled();
|
||||
expect(personRepository.find).toHaveBeenCalled();
|
||||
expect(companyRepository.delete).toHaveBeenCalledTimes(1);
|
||||
expect(personRepository.delete).toHaveBeenCalledTimes(1);
|
||||
|
||||
const findArgs = companyRepository.find.mock.calls[0][0];
|
||||
|
||||
expect(findArgs.withDeleted).toBe(true);
|
||||
expect(findArgs.order).toEqual({ deletedAt: 'ASC' });
|
||||
});
|
||||
|
||||
it('should return zero when no objects are found', async () => {
|
||||
setObjectMetadataCache([]);
|
||||
|
||||
const result = await service.cleanupWorkspaceTrash({
|
||||
workspaceId: 'workspace-id',
|
||||
trashRetentionDays: 14,
|
||||
});
|
||||
|
||||
expect(result).toEqual(0);
|
||||
expect(
|
||||
mockTwentyORMGlobalManager.getRepositoryForWorkspace,
|
||||
).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('should respect max records limit across objects', async () => {
|
||||
(service as any).maxRecordsPerWorkspace = 3;
|
||||
(service as any).batchSize = 3;
|
||||
setObjectMetadataCache([
|
||||
{ id: 'obj-company', nameSingular: 'company' },
|
||||
{ id: 'obj-person', nameSingular: 'person' },
|
||||
]);
|
||||
|
||||
const companyRepository = createRepositoryMock('company', 2);
|
||||
const personRepository = createRepositoryMock('person', 5);
|
||||
|
||||
mockTwentyORMGlobalManager.getRepositoryForWorkspace
|
||||
.mockResolvedValueOnce(companyRepository)
|
||||
.mockResolvedValueOnce(personRepository);
|
||||
|
||||
const result = await service.cleanupWorkspaceTrash({
|
||||
workspaceId: 'workspace-id',
|
||||
trashRetentionDays: 14,
|
||||
});
|
||||
|
||||
expect(result).toEqual(3);
|
||||
expect(companyRepository.delete).toHaveBeenCalledTimes(1);
|
||||
expect(personRepository.delete).toHaveBeenCalledTimes(1);
|
||||
const personDeleteArgs = personRepository.delete.mock.calls[0][0];
|
||||
const deletedIds =
|
||||
personDeleteArgs.id._value ?? personDeleteArgs.id.value;
|
||||
|
||||
expect(deletedIds).toHaveLength(1);
|
||||
expect(personRepository.find).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('should ignore objects without soft deleted records', async () => {
|
||||
setObjectMetadataCache([{ id: 'obj-company', nameSingular: 'company' }]);
|
||||
|
||||
const companyRepository = createRepositoryMock('company', 0);
|
||||
|
||||
mockTwentyORMGlobalManager.getRepositoryForWorkspace.mockResolvedValueOnce(
|
||||
companyRepository,
|
||||
);
|
||||
|
||||
const result = await service.cleanupWorkspaceTrash({
|
||||
workspaceId: 'workspace-id',
|
||||
trashRetentionDays: 14,
|
||||
});
|
||||
|
||||
expect(result).toEqual(0);
|
||||
expect(companyRepository.delete).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('should delete records across multiple batches', async () => {
|
||||
setObjectMetadataCache([{ id: 'obj-company', nameSingular: 'company' }]);
|
||||
|
||||
const companyRepository = createRepositoryMock('company', 5);
|
||||
|
||||
mockTwentyORMGlobalManager.getRepositoryForWorkspace.mockResolvedValueOnce(
|
||||
companyRepository,
|
||||
);
|
||||
|
||||
(service as any).batchSize = 2;
|
||||
(service as any).maxRecordsPerWorkspace = 10;
|
||||
|
||||
const result = await service.cleanupWorkspaceTrash({
|
||||
workspaceId: 'workspace-id',
|
||||
trashRetentionDays: 14,
|
||||
});
|
||||
|
||||
expect(result).toEqual(5);
|
||||
expect(companyRepository.find).toHaveBeenCalledTimes(4);
|
||||
expect(companyRepository.delete).toHaveBeenCalledTimes(3);
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,146 @@
|
||||
import { Injectable, Logger } from '@nestjs/common';
|
||||
|
||||
import { isDefined } from 'twenty-shared/utils';
|
||||
import { In, LessThan } from 'typeorm';
|
||||
|
||||
import { WorkspaceManyOrAllFlatEntityMapsCacheService } from 'src/engine/metadata-modules/flat-entity/services/workspace-many-or-all-flat-entity-maps-cache.service';
|
||||
import {
|
||||
TRASH_CLEANUP_BATCH_SIZE,
|
||||
TRASH_CLEANUP_MAX_RECORDS_PER_WORKSPACE,
|
||||
} from 'src/engine/trash-cleanup/constants/trash-cleanup.constants';
|
||||
import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager';
|
||||
|
||||
export type TrashCleanupInput = {
|
||||
workspaceId: string;
|
||||
trashRetentionDays: number;
|
||||
};
|
||||
|
||||
@Injectable()
|
||||
export class TrashCleanupService {
|
||||
private readonly logger = new Logger(TrashCleanupService.name);
|
||||
private readonly maxRecordsPerWorkspace =
|
||||
TRASH_CLEANUP_MAX_RECORDS_PER_WORKSPACE;
|
||||
private readonly batchSize = TRASH_CLEANUP_BATCH_SIZE;
|
||||
|
||||
constructor(
|
||||
private readonly flatEntityMapsCacheService: WorkspaceManyOrAllFlatEntityMapsCacheService,
|
||||
private readonly twentyORMGlobalManager: TwentyORMGlobalManager,
|
||||
) {}
|
||||
|
||||
async cleanupWorkspaceTrash(input: TrashCleanupInput): Promise<number> {
|
||||
const { workspaceId, trashRetentionDays } = input;
|
||||
|
||||
const { flatObjectMetadataMaps } =
|
||||
await this.flatEntityMapsCacheService.getOrRecomputeManyOrAllFlatEntityMaps(
|
||||
{
|
||||
workspaceId,
|
||||
flatMapsKeys: ['flatObjectMetadataMaps'],
|
||||
},
|
||||
);
|
||||
|
||||
const objectNames = Object.values(flatObjectMetadataMaps.byId ?? {})
|
||||
.map((metadata) => metadata?.nameSingular)
|
||||
.filter(isDefined);
|
||||
|
||||
if (objectNames.length === 0) {
|
||||
this.logger.log(`No objects found in workspace ${workspaceId}`);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
const cutoffDate = this.calculateCutoffDate(trashRetentionDays);
|
||||
let deletedCount = 0;
|
||||
|
||||
for (const objectName of objectNames) {
|
||||
if (deletedCount >= this.maxRecordsPerWorkspace) {
|
||||
this.logger.log(
|
||||
`Reached deletion limit (${this.maxRecordsPerWorkspace}) for workspace ${workspaceId}`,
|
||||
);
|
||||
break;
|
||||
}
|
||||
|
||||
const remainingQuota = this.maxRecordsPerWorkspace - deletedCount;
|
||||
const deletedForObject = await this.deleteSoftDeletedRecords({
|
||||
workspaceId,
|
||||
objectName,
|
||||
cutoffDate,
|
||||
remainingQuota,
|
||||
});
|
||||
|
||||
if (deletedForObject > 0) {
|
||||
this.logger.log(
|
||||
`Deleted ${deletedForObject} record(s) from ${objectName} in workspace ${workspaceId}`,
|
||||
);
|
||||
}
|
||||
|
||||
deletedCount += deletedForObject;
|
||||
}
|
||||
|
||||
this.logger.log(
|
||||
`Deleted ${deletedCount} record(s) from workspace ${workspaceId}`,
|
||||
);
|
||||
|
||||
return deletedCount;
|
||||
}
|
||||
|
||||
private async deleteSoftDeletedRecords({
|
||||
workspaceId,
|
||||
objectName,
|
||||
cutoffDate,
|
||||
remainingQuota,
|
||||
}: {
|
||||
workspaceId: string;
|
||||
objectName: string;
|
||||
cutoffDate: Date;
|
||||
remainingQuota: number;
|
||||
}): Promise<number> {
|
||||
if (remainingQuota <= 0) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
const repository =
|
||||
await this.twentyORMGlobalManager.getRepositoryForWorkspace(
|
||||
workspaceId,
|
||||
objectName,
|
||||
{ shouldBypassPermissionChecks: true },
|
||||
);
|
||||
|
||||
let deleted = 0;
|
||||
|
||||
while (deleted < remainingQuota) {
|
||||
const take = Math.min(this.batchSize, remainingQuota - deleted);
|
||||
|
||||
const recordsToDelete = await repository.find({
|
||||
withDeleted: true,
|
||||
select: ['id'],
|
||||
where: {
|
||||
deletedAt: LessThan(cutoffDate),
|
||||
},
|
||||
order: { deletedAt: 'ASC' },
|
||||
take,
|
||||
loadEagerRelations: false,
|
||||
});
|
||||
|
||||
if (recordsToDelete.length === 0) {
|
||||
break;
|
||||
}
|
||||
|
||||
await repository.delete({
|
||||
id: In(recordsToDelete.map((record) => record.id)),
|
||||
});
|
||||
|
||||
deleted += recordsToDelete.length;
|
||||
}
|
||||
|
||||
return deleted;
|
||||
}
|
||||
|
||||
private calculateCutoffDate(trashRetentionDays: number): Date {
|
||||
const cutoffDate = new Date();
|
||||
|
||||
cutoffDate.setUTCHours(0, 0, 0, 0);
|
||||
cutoffDate.setDate(cutoffDate.getDate() - trashRetentionDays + 1);
|
||||
|
||||
return cutoffDate;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
import { Module } from '@nestjs/common';
|
||||
import { TypeOrmModule } from '@nestjs/typeorm';
|
||||
|
||||
import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity';
|
||||
import { WorkspaceManyOrAllFlatEntityMapsCacheModule } from 'src/engine/metadata-modules/flat-entity/services/workspace-many-or-all-flat-entity-maps-cache.module';
|
||||
import { TrashCleanupCronCommand } from 'src/engine/trash-cleanup/commands/trash-cleanup.cron.command';
|
||||
import { TrashCleanupCronJob } from 'src/engine/trash-cleanup/crons/trash-cleanup.cron.job';
|
||||
import { TrashCleanupJob } from 'src/engine/trash-cleanup/jobs/trash-cleanup.job';
|
||||
import { TrashCleanupService } from 'src/engine/trash-cleanup/services/trash-cleanup.service';
|
||||
|
||||
@Module({
|
||||
imports: [
|
||||
TypeOrmModule.forFeature([Workspace]),
|
||||
WorkspaceManyOrAllFlatEntityMapsCacheModule,
|
||||
],
|
||||
providers: [
|
||||
TrashCleanupService,
|
||||
TrashCleanupJob,
|
||||
TrashCleanupCronJob,
|
||||
TrashCleanupCronCommand,
|
||||
],
|
||||
exports: [TrashCleanupCronCommand],
|
||||
})
|
||||
export class TrashCleanupModule {}
|
||||
Reference in New Issue
Block a user