Message folders optimization (#16479)
- Batch Gmail API calls using `googleapis-batcher` for folder processing - Add concurrency limit for Microsoft Graph folder processing - Skip IMAP folder sync when no new messages (checks UIDVALIDITY/MODSEQ) - Refactored `syncMessageFolders` to return folder state directly, avoiding extra DB round-trips - Refactored `processPendingFolderActions` to reuse state instead of querying DB again - Add unique index on message folders entity --------- Co-authored-by: Charles Bochet <charles@twenty.com>
This commit is contained in:
neo773
+543
@@ -0,0 +1,543 @@
|
||||
import { Test, type TestingModule } from '@nestjs/testing';
|
||||
|
||||
import { ConnectedAccountProvider } from 'twenty-shared/types';
|
||||
|
||||
import { type DiscoveredMessageFolder } from 'src/modules/messaging/message-folder-manager/interfaces/message-folder-driver.interface';
|
||||
|
||||
import { GlobalWorkspaceOrmManager } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager';
|
||||
import {
|
||||
MessageChannelContactAutoCreationPolicy,
|
||||
MessageChannelType,
|
||||
MessageChannelVisibility,
|
||||
MessageFolderImportPolicy,
|
||||
} from 'src/modules/messaging/common/standard-objects/message-channel.workspace-entity';
|
||||
import {
|
||||
MessageFolderPendingSyncAction,
|
||||
type MessageFolderWorkspaceEntity,
|
||||
} from 'src/modules/messaging/common/standard-objects/message-folder.workspace-entity';
|
||||
import { GmailGetAllFoldersService } from 'src/modules/messaging/message-folder-manager/drivers/gmail/services/gmail-get-all-folders.service';
|
||||
import { ImapGetAllFoldersService } from 'src/modules/messaging/message-folder-manager/drivers/imap/services/imap-get-all-folders.service';
|
||||
import { MicrosoftGetAllFoldersService } from 'src/modules/messaging/message-folder-manager/drivers/microsoft/services/microsoft-get-all-folders.service';
|
||||
import { SyncMessageFoldersService } from 'src/modules/messaging/message-folder-manager/services/sync-message-folders.service';
|
||||
|
||||
type SyncedMessageFolder = Pick<
|
||||
MessageFolderWorkspaceEntity,
|
||||
| 'id'
|
||||
| 'name'
|
||||
| 'isSynced'
|
||||
| 'isSentFolder'
|
||||
| 'externalId'
|
||||
| 'syncCursor'
|
||||
| 'parentFolderId'
|
||||
| 'pendingSyncAction'
|
||||
>;
|
||||
|
||||
const createMockMessageChannel = (
|
||||
overrides: {
|
||||
provider?: ConnectedAccountProvider;
|
||||
messageFolders?: SyncedMessageFolder[];
|
||||
} = {},
|
||||
) => ({
|
||||
id: 'channel-123',
|
||||
handle: 'test@gmail.com',
|
||||
type: MessageChannelType.EMAIL,
|
||||
messageFolderImportPolicy: MessageFolderImportPolicy.ALL_FOLDERS,
|
||||
connectedAccount: {
|
||||
id: 'account-456',
|
||||
handle: 'test@gmail.com',
|
||||
provider: overrides.provider ?? ConnectedAccountProvider.GOOGLE,
|
||||
accessToken: 'mock-access-token',
|
||||
refreshToken: 'mock-refresh-token',
|
||||
connectionParameters: {},
|
||||
},
|
||||
messageFolders: overrides.messageFolders ?? [],
|
||||
visibility: MessageChannelVisibility.SHARE_EVERYTHING,
|
||||
isContactAutoCreationEnabled: false,
|
||||
contactAutoCreationPolicy: MessageChannelContactAutoCreationPolicy.NONE,
|
||||
excludeNonProfessionalEmails: false,
|
||||
excludeGroupEmails: false,
|
||||
});
|
||||
|
||||
const createMockDiscoveredFolder = (
|
||||
overrides: Partial<DiscoveredMessageFolder> = {},
|
||||
): DiscoveredMessageFolder => ({
|
||||
externalId: `external-${Math.random().toString(36).substring(7)}`,
|
||||
name: 'Test Folder',
|
||||
isSynced: false,
|
||||
isSentFolder: false,
|
||||
parentFolderId: null,
|
||||
...overrides,
|
||||
});
|
||||
|
||||
const createMockExistingFolder = (
|
||||
overrides: Partial<SyncedMessageFolder> = {},
|
||||
): SyncedMessageFolder => ({
|
||||
id: `folder-${Math.random().toString(36).substring(7)}`,
|
||||
externalId: `external-${Math.random().toString(36).substring(7)}`,
|
||||
name: 'Existing Folder',
|
||||
isSynced: true,
|
||||
isSentFolder: false,
|
||||
syncCursor: null,
|
||||
parentFolderId: null,
|
||||
pendingSyncAction: MessageFolderPendingSyncAction.NONE,
|
||||
...overrides,
|
||||
});
|
||||
|
||||
describe('SyncMessageFoldersService', () => {
|
||||
let service: SyncMessageFoldersService;
|
||||
let gmailGetAllFoldersService: jest.Mocked<GmailGetAllFoldersService>;
|
||||
|
||||
let mockRepository: {
|
||||
delete: jest.Mock;
|
||||
update: jest.Mock;
|
||||
updateMany: jest.Mock;
|
||||
save: jest.Mock;
|
||||
};
|
||||
let mockTransactionManager: object;
|
||||
|
||||
beforeEach(async () => {
|
||||
mockRepository = {
|
||||
delete: jest.fn(),
|
||||
update: jest.fn(),
|
||||
updateMany: jest.fn(),
|
||||
save: jest.fn().mockImplementation((folders) =>
|
||||
folders.map((folder: Partial<MessageFolderWorkspaceEntity>) => ({
|
||||
...folder,
|
||||
id: `new-folder-${Math.random().toString(36).substring(7)}`,
|
||||
isSynced: false,
|
||||
syncCursor: null,
|
||||
})),
|
||||
),
|
||||
};
|
||||
|
||||
mockTransactionManager = {};
|
||||
|
||||
const mockDataSource = {
|
||||
transaction: jest
|
||||
.fn()
|
||||
.mockImplementation((callback) => callback(mockTransactionManager)),
|
||||
};
|
||||
|
||||
const module: TestingModule = await Test.createTestingModule({
|
||||
providers: [
|
||||
SyncMessageFoldersService,
|
||||
{
|
||||
provide: GlobalWorkspaceOrmManager,
|
||||
useValue: {
|
||||
executeInWorkspaceContext: jest
|
||||
.fn()
|
||||
.mockImplementation((_, callback) => callback()),
|
||||
getRepository: jest.fn().mockResolvedValue(mockRepository),
|
||||
getDataSourceForWorkspace: jest
|
||||
.fn()
|
||||
.mockResolvedValue(mockDataSource),
|
||||
getGlobalWorkspaceDataSource: jest
|
||||
.fn()
|
||||
.mockResolvedValue(mockDataSource),
|
||||
},
|
||||
},
|
||||
{
|
||||
provide: GmailGetAllFoldersService,
|
||||
useValue: {
|
||||
getAllMessageFolders: jest.fn(),
|
||||
},
|
||||
},
|
||||
{
|
||||
provide: MicrosoftGetAllFoldersService,
|
||||
useValue: {
|
||||
getAllMessageFolders: jest.fn(),
|
||||
},
|
||||
},
|
||||
{
|
||||
provide: ImapGetAllFoldersService,
|
||||
useValue: {
|
||||
getAllMessageFolders: jest.fn(),
|
||||
},
|
||||
},
|
||||
],
|
||||
}).compile();
|
||||
|
||||
service = module.get<SyncMessageFoldersService>(SyncMessageFoldersService);
|
||||
gmailGetAllFoldersService = module.get(GmailGetAllFoldersService);
|
||||
});
|
||||
|
||||
describe('syncMessageFolders', () => {
|
||||
const workspaceId = 'workspace-789';
|
||||
|
||||
describe('folder creation scenarios', () => {
|
||||
it('should create new folders when none exist locally', async () => {
|
||||
const discoveredFolders = [
|
||||
createMockDiscoveredFolder({
|
||||
externalId: 'inbox-ext',
|
||||
name: 'INBOX',
|
||||
}),
|
||||
createMockDiscoveredFolder({
|
||||
externalId: 'sent-ext',
|
||||
name: 'Sent',
|
||||
isSentFolder: true,
|
||||
}),
|
||||
];
|
||||
const messageChannel = createMockMessageChannel({
|
||||
messageFolders: [],
|
||||
});
|
||||
|
||||
gmailGetAllFoldersService.getAllMessageFolders.mockResolvedValue(
|
||||
discoveredFolders,
|
||||
);
|
||||
|
||||
const result = await service.syncMessageFolders({
|
||||
messageChannel,
|
||||
workspaceId,
|
||||
});
|
||||
|
||||
expect(mockRepository.save).toHaveBeenCalledWith(
|
||||
expect.arrayContaining([
|
||||
expect.objectContaining({
|
||||
name: 'INBOX',
|
||||
externalId: 'inbox-ext',
|
||||
messageChannelId: 'channel-123',
|
||||
isSentFolder: false,
|
||||
}),
|
||||
expect.objectContaining({
|
||||
name: 'Sent',
|
||||
externalId: 'sent-ext',
|
||||
messageChannelId: 'channel-123',
|
||||
isSentFolder: true,
|
||||
}),
|
||||
]),
|
||||
{},
|
||||
mockTransactionManager,
|
||||
);
|
||||
expect(result).toHaveLength(2);
|
||||
});
|
||||
|
||||
it('should handle nested folder creation with parent references', async () => {
|
||||
const discoveredFolders = [
|
||||
createMockDiscoveredFolder({
|
||||
externalId: 'parent-ext',
|
||||
name: 'Work',
|
||||
parentFolderId: null,
|
||||
}),
|
||||
createMockDiscoveredFolder({
|
||||
externalId: 'child-ext',
|
||||
name: 'Projects',
|
||||
parentFolderId: 'parent-folder-id',
|
||||
}),
|
||||
];
|
||||
const messageChannel = createMockMessageChannel({
|
||||
messageFolders: [],
|
||||
});
|
||||
|
||||
gmailGetAllFoldersService.getAllMessageFolders.mockResolvedValue(
|
||||
discoveredFolders,
|
||||
);
|
||||
|
||||
await service.syncMessageFolders({
|
||||
messageChannel,
|
||||
workspaceId,
|
||||
});
|
||||
|
||||
expect(mockRepository.save).toHaveBeenCalledWith(
|
||||
expect.arrayContaining([
|
||||
expect.objectContaining({
|
||||
name: 'Projects',
|
||||
parentFolderId: 'parent-folder-id',
|
||||
}),
|
||||
]),
|
||||
{},
|
||||
mockTransactionManager,
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('folder update scenarios', () => {
|
||||
it('should update folder when name changes', async () => {
|
||||
const existingFolder = createMockExistingFolder({
|
||||
id: 'folder-1',
|
||||
externalId: 'inbox-ext',
|
||||
name: 'INBOX',
|
||||
});
|
||||
const discoveredFolders = [
|
||||
createMockDiscoveredFolder({
|
||||
externalId: 'inbox-ext',
|
||||
name: 'Primary Inbox',
|
||||
}),
|
||||
];
|
||||
const messageChannel = createMockMessageChannel({
|
||||
messageFolders: [existingFolder],
|
||||
});
|
||||
|
||||
gmailGetAllFoldersService.getAllMessageFolders.mockResolvedValue(
|
||||
discoveredFolders,
|
||||
);
|
||||
|
||||
const result = await service.syncMessageFolders({
|
||||
messageChannel,
|
||||
workspaceId,
|
||||
});
|
||||
|
||||
expect(mockRepository.updateMany).toHaveBeenCalledWith(
|
||||
expect.arrayContaining([
|
||||
expect.objectContaining({
|
||||
criteria: 'folder-1',
|
||||
partialEntity: expect.objectContaining({ name: 'Primary Inbox' }),
|
||||
}),
|
||||
]),
|
||||
mockTransactionManager,
|
||||
);
|
||||
expect(result).toContainEqual(
|
||||
expect.objectContaining({
|
||||
id: 'folder-1',
|
||||
name: 'Primary Inbox',
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it('should update folder when parent folder changes', async () => {
|
||||
const existingFolder = createMockExistingFolder({
|
||||
id: 'folder-1',
|
||||
externalId: 'child-ext',
|
||||
name: 'Projects',
|
||||
parentFolderId: 'old-parent-id',
|
||||
});
|
||||
const discoveredFolders = [
|
||||
createMockDiscoveredFolder({
|
||||
externalId: 'child-ext',
|
||||
name: 'Projects',
|
||||
parentFolderId: 'new-parent-id',
|
||||
}),
|
||||
];
|
||||
const messageChannel = createMockMessageChannel({
|
||||
messageFolders: [existingFolder],
|
||||
});
|
||||
|
||||
gmailGetAllFoldersService.getAllMessageFolders.mockResolvedValue(
|
||||
discoveredFolders,
|
||||
);
|
||||
|
||||
await service.syncMessageFolders({
|
||||
messageChannel,
|
||||
workspaceId,
|
||||
});
|
||||
|
||||
expect(mockRepository.updateMany).toHaveBeenCalledWith(
|
||||
expect.arrayContaining([
|
||||
expect.objectContaining({
|
||||
criteria: 'folder-1',
|
||||
partialEntity: expect.objectContaining({
|
||||
parentFolderId: 'new-parent-id',
|
||||
}),
|
||||
}),
|
||||
]),
|
||||
mockTransactionManager,
|
||||
);
|
||||
});
|
||||
|
||||
it('should not update folder when nothing has changed', async () => {
|
||||
const existingFolder = createMockExistingFolder({
|
||||
id: 'folder-1',
|
||||
externalId: 'inbox-ext',
|
||||
name: 'INBOX',
|
||||
isSentFolder: false,
|
||||
parentFolderId: null,
|
||||
});
|
||||
const discoveredFolders = [
|
||||
createMockDiscoveredFolder({
|
||||
externalId: 'inbox-ext',
|
||||
name: 'INBOX',
|
||||
isSentFolder: false,
|
||||
parentFolderId: null,
|
||||
}),
|
||||
];
|
||||
const messageChannel = createMockMessageChannel({
|
||||
messageFolders: [existingFolder],
|
||||
});
|
||||
|
||||
gmailGetAllFoldersService.getAllMessageFolders.mockResolvedValue(
|
||||
discoveredFolders,
|
||||
);
|
||||
|
||||
await service.syncMessageFolders({
|
||||
messageChannel,
|
||||
workspaceId,
|
||||
});
|
||||
|
||||
expect(mockRepository.updateMany).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
describe('folder deletion scenarios', () => {
|
||||
it('should delete folders that no longer exist remotely', async () => {
|
||||
const existingFolders = [
|
||||
createMockExistingFolder({
|
||||
id: 'folder-1',
|
||||
externalId: 'inbox-ext',
|
||||
name: 'INBOX',
|
||||
}),
|
||||
createMockExistingFolder({
|
||||
id: 'folder-2',
|
||||
externalId: 'deleted-ext',
|
||||
name: 'Old Folder',
|
||||
}),
|
||||
];
|
||||
const discoveredFolders = [
|
||||
createMockDiscoveredFolder({
|
||||
externalId: 'inbox-ext',
|
||||
name: 'INBOX',
|
||||
}),
|
||||
];
|
||||
const messageChannel = createMockMessageChannel({
|
||||
messageFolders: existingFolders,
|
||||
});
|
||||
|
||||
gmailGetAllFoldersService.getAllMessageFolders.mockResolvedValue(
|
||||
discoveredFolders,
|
||||
);
|
||||
|
||||
const result = await service.syncMessageFolders({
|
||||
messageChannel: messageChannel,
|
||||
workspaceId,
|
||||
});
|
||||
|
||||
expect(mockRepository.updateMany).toHaveBeenCalledWith(
|
||||
expect.arrayContaining([
|
||||
expect.objectContaining({
|
||||
criteria: 'folder-2',
|
||||
partialEntity: expect.objectContaining({
|
||||
pendingSyncAction: 'FOLDER_DELETION',
|
||||
}),
|
||||
}),
|
||||
]),
|
||||
mockTransactionManager,
|
||||
);
|
||||
expect(result).toContainEqual(
|
||||
expect.objectContaining({
|
||||
id: 'folder-2',
|
||||
pendingSyncAction: MessageFolderPendingSyncAction.FOLDER_DELETION,
|
||||
}),
|
||||
);
|
||||
expect(result).toHaveLength(2);
|
||||
});
|
||||
});
|
||||
|
||||
describe('complex sync scenarios', () => {
|
||||
it('should handle simultaneous create, update, and delete operations', async () => {
|
||||
const existingFolders = [
|
||||
createMockExistingFolder({
|
||||
id: 'folder-to-update',
|
||||
externalId: 'update-ext',
|
||||
name: 'Old Name',
|
||||
}),
|
||||
createMockExistingFolder({
|
||||
id: 'folder-to-delete',
|
||||
externalId: 'delete-ext',
|
||||
name: 'To Delete',
|
||||
}),
|
||||
createMockExistingFolder({
|
||||
id: 'folder-unchanged',
|
||||
externalId: 'unchanged-ext',
|
||||
name: 'Unchanged',
|
||||
}),
|
||||
];
|
||||
const discoveredFolders = [
|
||||
createMockDiscoveredFolder({
|
||||
externalId: 'update-ext',
|
||||
name: 'New Name',
|
||||
}),
|
||||
createMockDiscoveredFolder({
|
||||
externalId: 'unchanged-ext',
|
||||
name: 'Unchanged',
|
||||
}),
|
||||
createMockDiscoveredFolder({
|
||||
externalId: 'new-ext',
|
||||
name: 'New Folder',
|
||||
}),
|
||||
];
|
||||
const messageChannel = createMockMessageChannel({
|
||||
messageFolders: existingFolders,
|
||||
});
|
||||
|
||||
gmailGetAllFoldersService.getAllMessageFolders.mockResolvedValue(
|
||||
discoveredFolders,
|
||||
);
|
||||
|
||||
const result = await service.syncMessageFolders({
|
||||
messageChannel,
|
||||
workspaceId,
|
||||
});
|
||||
|
||||
expect(mockRepository.updateMany).toHaveBeenCalledWith(
|
||||
expect.arrayContaining([
|
||||
expect.objectContaining({
|
||||
criteria: 'folder-to-delete',
|
||||
partialEntity: expect.objectContaining({
|
||||
pendingSyncAction: 'FOLDER_DELETION',
|
||||
}),
|
||||
}),
|
||||
]),
|
||||
mockTransactionManager,
|
||||
);
|
||||
expect(mockRepository.updateMany).toHaveBeenCalledWith(
|
||||
expect.arrayContaining([
|
||||
expect.objectContaining({
|
||||
criteria: 'folder-to-update',
|
||||
partialEntity: expect.objectContaining({ name: 'New Name' }),
|
||||
}),
|
||||
]),
|
||||
mockTransactionManager,
|
||||
);
|
||||
expect(mockRepository.save).toHaveBeenCalledWith(
|
||||
expect.arrayContaining([
|
||||
expect.objectContaining({ externalId: 'new-ext' }),
|
||||
]),
|
||||
{},
|
||||
mockTransactionManager,
|
||||
);
|
||||
expect(result).toHaveLength(4);
|
||||
expect(result).toContainEqual(
|
||||
expect.objectContaining({
|
||||
id: 'folder-to-delete',
|
||||
pendingSyncAction: MessageFolderPendingSyncAction.FOLDER_DELETION,
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it('should preserve syncCursor and isSynced for unchanged folders', async () => {
|
||||
const existingFolder = createMockExistingFolder({
|
||||
id: 'folder-1',
|
||||
externalId: 'inbox-ext',
|
||||
name: 'INBOX',
|
||||
isSynced: true,
|
||||
syncCursor: 'cursor-abc123',
|
||||
});
|
||||
const discoveredFolders = [
|
||||
createMockDiscoveredFolder({
|
||||
externalId: 'inbox-ext',
|
||||
name: 'INBOX',
|
||||
}),
|
||||
];
|
||||
const messageChannel = createMockMessageChannel({
|
||||
messageFolders: [existingFolder],
|
||||
});
|
||||
|
||||
gmailGetAllFoldersService.getAllMessageFolders.mockResolvedValue(
|
||||
discoveredFolders,
|
||||
);
|
||||
|
||||
const result = await service.syncMessageFolders({
|
||||
messageChannel,
|
||||
workspaceId,
|
||||
});
|
||||
|
||||
expect(result).toContainEqual(
|
||||
expect.objectContaining({
|
||||
id: 'folder-1',
|
||||
isSynced: true,
|
||||
syncCursor: 'cursor-abc123',
|
||||
pendingSyncAction: MessageFolderPendingSyncAction.NONE,
|
||||
}),
|
||||
);
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
+123
-207
@@ -1,54 +1,28 @@
|
||||
import { Injectable } from '@nestjs/common';
|
||||
|
||||
import { isNonEmptyString } from '@sniptt/guards';
|
||||
import deepEqual from 'deep-equal';
|
||||
import { ConnectedAccountProvider } from 'twenty-shared/types';
|
||||
import { isDefined } from 'twenty-shared/utils';
|
||||
import { v4 } from 'uuid';
|
||||
|
||||
import { MessageFolder } from 'src/modules/messaging/message-folder-manager/interfaces/message-folder-driver.interface';
|
||||
import {
|
||||
DiscoveredMessageFolder,
|
||||
MessageFolder,
|
||||
} from 'src/modules/messaging/message-folder-manager/interfaces/message-folder-driver.interface';
|
||||
|
||||
import { type WorkspaceEntityManager } from 'src/engine/twenty-orm/entity-manager/workspace-entity-manager';
|
||||
import { WorkspaceEntityManager } from 'src/engine/twenty-orm/entity-manager/workspace-entity-manager';
|
||||
import { GlobalWorkspaceOrmManager } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager';
|
||||
import { WorkspaceRepository } from 'src/engine/twenty-orm/repository/workspace.repository';
|
||||
import { buildSystemAuthContext } from 'src/engine/twenty-orm/utils/build-system-auth-context.util';
|
||||
import { ConnectedAccountWorkspaceEntity } from 'src/modules/connected-account/standard-objects/connected-account.workspace-entity';
|
||||
import { type MessageChannelWorkspaceEntity } from 'src/modules/messaging/common/standard-objects/message-channel.workspace-entity';
|
||||
import {
|
||||
MessageFolderPendingSyncAction,
|
||||
type MessageFolderWorkspaceEntity,
|
||||
MessageFolderWorkspaceEntity,
|
||||
} from 'src/modules/messaging/common/standard-objects/message-folder.workspace-entity';
|
||||
import { GmailGetAllFoldersService } from 'src/modules/messaging/message-folder-manager/drivers/gmail/services/gmail-get-all-folders.service';
|
||||
import { ImapGetAllFoldersService } from 'src/modules/messaging/message-folder-manager/drivers/imap/services/imap-get-all-folders.service';
|
||||
import { MicrosoftGetAllFoldersService } from 'src/modules/messaging/message-folder-manager/drivers/microsoft/services/microsoft-get-all-folders.service';
|
||||
import { MessageFolderName } from 'src/modules/messaging/message-import-manager/drivers/microsoft/types/folders';
|
||||
|
||||
type SyncMessageFoldersInput = {
|
||||
workspaceId: string;
|
||||
messageChannel: Pick<
|
||||
MessageChannelWorkspaceEntity,
|
||||
'messageFolderImportPolicy' | 'connectedAccount' | 'id'
|
||||
>;
|
||||
manager: WorkspaceEntityManager;
|
||||
};
|
||||
|
||||
type MessageFolderToInsert = Pick<
|
||||
MessageFolderWorkspaceEntity,
|
||||
| 'id'
|
||||
| 'messageChannelId'
|
||||
| 'name'
|
||||
| 'syncCursor'
|
||||
| 'isSynced'
|
||||
| 'isSentFolder'
|
||||
| 'externalId'
|
||||
| 'parentFolderId'
|
||||
>;
|
||||
|
||||
type MessageFolderToUpdate = Partial<
|
||||
Pick<
|
||||
MessageFolderWorkspaceEntity,
|
||||
'name' | 'externalId' | 'isSentFolder' | 'parentFolderId'
|
||||
>
|
||||
>;
|
||||
import { computeFolderIdsToDelete } from 'src/modules/messaging/message-folder-manager/utils/compute-folder-ids-to-delete.util';
|
||||
import { computeFoldersToCreate } from 'src/modules/messaging/message-folder-manager/utils/compute-folders-to-create.util';
|
||||
import { computeFoldersToUpdate } from 'src/modules/messaging/message-folder-manager/utils/compute-folders-to-update.util';
|
||||
import { computeUpdatedFolders } from 'src/modules/messaging/message-folder-manager/utils/compute-updated-folders.util';
|
||||
|
||||
@Injectable()
|
||||
export class SyncMessageFoldersService {
|
||||
@@ -59,160 +33,71 @@ export class SyncMessageFoldersService {
|
||||
private readonly imapGetAllFoldersService: ImapGetAllFoldersService,
|
||||
) {}
|
||||
|
||||
async syncMessageFolders(input: SyncMessageFoldersInput): Promise<void> {
|
||||
const { workspaceId, messageChannel, manager } = input;
|
||||
|
||||
const authContext = buildSystemAuthContext(workspaceId);
|
||||
|
||||
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
|
||||
authContext,
|
||||
async () => {
|
||||
const folders = await this.discoverAllFolders(
|
||||
messageChannel.connectedAccount,
|
||||
messageChannel,
|
||||
);
|
||||
|
||||
await this.upsertDiscoveredFolders({
|
||||
workspaceId,
|
||||
messageChannelId: messageChannel.id,
|
||||
folders,
|
||||
manager,
|
||||
});
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
private async upsertDiscoveredFolders({
|
||||
async syncMessageFolders({
|
||||
messageChannel,
|
||||
workspaceId,
|
||||
messageChannelId,
|
||||
folders,
|
||||
manager,
|
||||
}: {
|
||||
messageChannel: Pick<
|
||||
MessageChannelWorkspaceEntity,
|
||||
'id' | 'messageFolderImportPolicy'
|
||||
> & {
|
||||
connectedAccount: Pick<
|
||||
ConnectedAccountWorkspaceEntity,
|
||||
| 'provider'
|
||||
| 'accessToken'
|
||||
| 'refreshToken'
|
||||
| 'id'
|
||||
| 'handle'
|
||||
| 'connectionParameters'
|
||||
>;
|
||||
messageFolders: MessageFolder[];
|
||||
};
|
||||
workspaceId: string;
|
||||
messageChannelId: string;
|
||||
folders: MessageFolder[];
|
||||
manager: WorkspaceEntityManager;
|
||||
}): Promise<void> {
|
||||
const messageFolderRepository =
|
||||
await this.globalWorkspaceOrmManager.getRepository<MessageFolderWorkspaceEntity>(
|
||||
workspaceId,
|
||||
'messageFolder',
|
||||
);
|
||||
|
||||
const existingFolderMap = await this.buildExistingFolderMap({
|
||||
messageChannelId,
|
||||
messageFolderRepository,
|
||||
});
|
||||
|
||||
const inserts: MessageFolderToInsert[] = [];
|
||||
const updates: [string, MessageFolderToUpdate][] = [];
|
||||
const foldersToMarkForDeletion: string[] = [];
|
||||
|
||||
const discoveredExternalIds = new Set(
|
||||
folders
|
||||
.filter((folder) => folder.externalId)
|
||||
.map((folder) => folder.externalId!),
|
||||
}): Promise<MessageFolder[]> {
|
||||
const discoveredFolders = await this.discoverAllFolders(
|
||||
messageChannel.connectedAccount,
|
||||
messageChannel,
|
||||
);
|
||||
|
||||
for (const existingFolder of existingFolderMap.values()) {
|
||||
if (
|
||||
existingFolder.externalId &&
|
||||
!discoveredExternalIds.has(existingFolder.externalId)
|
||||
) {
|
||||
foldersToMarkForDeletion.push(existingFolder.id);
|
||||
}
|
||||
}
|
||||
const { messageFolders: existingFolders, id: messageChannelId } =
|
||||
messageChannel;
|
||||
|
||||
for (const folder of folders) {
|
||||
const existingFolder = this.findExistingFolderInMap(
|
||||
existingFolderMap,
|
||||
folder,
|
||||
);
|
||||
|
||||
if (existingFolder) {
|
||||
const folderSyncData = {
|
||||
name: folder.name,
|
||||
externalId: folder.externalId,
|
||||
isSentFolder: folder.isSentFolder,
|
||||
parentFolderId: isNonEmptyString(folder.parentFolderId)
|
||||
? folder.parentFolderId
|
||||
: null,
|
||||
};
|
||||
|
||||
const existingFolderData = {
|
||||
name: existingFolder.name,
|
||||
externalId: existingFolder.externalId,
|
||||
isSentFolder: existingFolder.isSentFolder,
|
||||
parentFolderId: isNonEmptyString(existingFolder.parentFolderId)
|
||||
? existingFolder.parentFolderId
|
||||
: null,
|
||||
};
|
||||
|
||||
if (!deepEqual(folderSyncData, existingFolderData)) {
|
||||
updates.push([existingFolder.id, folderSyncData]);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
inserts.push({
|
||||
id: v4(),
|
||||
messageChannelId,
|
||||
name: folder.name,
|
||||
syncCursor: '',
|
||||
isSynced: folder.isSynced,
|
||||
isSentFolder: folder.isSentFolder,
|
||||
externalId: folder.externalId,
|
||||
parentFolderId: folder.parentFolderId,
|
||||
});
|
||||
}
|
||||
|
||||
if (inserts.length > 0) {
|
||||
await messageFolderRepository.insert(inserts, manager);
|
||||
}
|
||||
|
||||
if (updates.length > 0) {
|
||||
await messageFolderRepository.updateMany(
|
||||
updates.map(([id, data]) => ({
|
||||
criteria: id,
|
||||
partialEntity: data,
|
||||
})),
|
||||
manager,
|
||||
);
|
||||
}
|
||||
|
||||
if (foldersToMarkForDeletion.length > 0) {
|
||||
await messageFolderRepository.updateMany(
|
||||
foldersToMarkForDeletion.map((id) => ({
|
||||
criteria: id,
|
||||
partialEntity: {
|
||||
pendingSyncAction: MessageFolderPendingSyncAction.FOLDER_DELETION,
|
||||
},
|
||||
})),
|
||||
manager,
|
||||
);
|
||||
}
|
||||
return this.syncFolderChanges(
|
||||
discoveredFolders,
|
||||
existingFolders,
|
||||
messageChannelId,
|
||||
workspaceId,
|
||||
);
|
||||
}
|
||||
|
||||
async discoverAllFolders(
|
||||
connectedAccount: MessageChannelWorkspaceEntity['connectedAccount'],
|
||||
connectedAccount: Pick<
|
||||
ConnectedAccountWorkspaceEntity,
|
||||
| 'accessToken'
|
||||
| 'refreshToken'
|
||||
| 'id'
|
||||
| 'handle'
|
||||
| 'provider'
|
||||
| 'connectionParameters'
|
||||
>,
|
||||
messageChannel: Pick<
|
||||
MessageChannelWorkspaceEntity,
|
||||
'messageFolderImportPolicy'
|
||||
>,
|
||||
): Promise<MessageFolder[]> {
|
||||
): Promise<DiscoveredMessageFolder[]> {
|
||||
switch (connectedAccount.provider) {
|
||||
case ConnectedAccountProvider.GOOGLE:
|
||||
return await this.gmailGetAllFoldersService.getAllMessageFolders(
|
||||
return this.gmailGetAllFoldersService.getAllMessageFolders(
|
||||
connectedAccount,
|
||||
messageChannel,
|
||||
);
|
||||
case ConnectedAccountProvider.MICROSOFT:
|
||||
return await this.microsoftGetAllFoldersService.getAllMessageFolders(
|
||||
return this.microsoftGetAllFoldersService.getAllMessageFolders(
|
||||
connectedAccount,
|
||||
messageChannel,
|
||||
);
|
||||
case ConnectedAccountProvider.IMAP_SMTP_CALDAV:
|
||||
return await this.imapGetAllFoldersService.getAllMessageFolders(
|
||||
return this.imapGetAllFoldersService.getAllMessageFolders(
|
||||
connectedAccount,
|
||||
messageChannel,
|
||||
);
|
||||
@@ -223,55 +108,86 @@ export class SyncMessageFoldersService {
|
||||
}
|
||||
}
|
||||
|
||||
private async buildExistingFolderMap({
|
||||
messageChannelId,
|
||||
messageFolderRepository,
|
||||
}: {
|
||||
messageChannelId: string;
|
||||
messageFolderRepository: WorkspaceRepository<MessageFolderWorkspaceEntity>;
|
||||
}): Promise<Map<string, MessageFolderWorkspaceEntity>> {
|
||||
const existingFolders = await messageFolderRepository.find({
|
||||
where: { messageChannelId },
|
||||
private async syncFolderChanges(
|
||||
discoveredFolders: DiscoveredMessageFolder[],
|
||||
existingFolders: MessageFolder[],
|
||||
messageChannelId: string,
|
||||
workspaceId: string,
|
||||
): Promise<MessageFolder[]> {
|
||||
const foldersToCreate = computeFoldersToCreate({
|
||||
discoveredFolders,
|
||||
existingFolders,
|
||||
messageChannelId,
|
||||
});
|
||||
|
||||
const existingFolderMap = new Map<string, MessageFolderWorkspaceEntity>();
|
||||
const foldersToUpdate = computeFoldersToUpdate({
|
||||
discoveredFolders,
|
||||
existingFolders,
|
||||
});
|
||||
|
||||
for (const existingFolder of existingFolders) {
|
||||
if (isDefined(existingFolder.externalId)) {
|
||||
existingFolderMap.set(existingFolder.externalId, existingFolder);
|
||||
}
|
||||
existingFolderMap.set(existingFolder.name ?? '', existingFolder);
|
||||
}
|
||||
const folderIdsToDelete = computeFolderIdsToDelete({
|
||||
discoveredFolders,
|
||||
existingFolders,
|
||||
});
|
||||
|
||||
return existingFolderMap;
|
||||
}
|
||||
const authContext = buildSystemAuthContext(workspaceId);
|
||||
|
||||
private findExistingFolderInMap(
|
||||
existingFolderMap: Map<string, MessageFolderWorkspaceEntity>,
|
||||
folder: MessageFolder,
|
||||
): MessageFolderWorkspaceEntity | undefined {
|
||||
if (isDefined(folder.externalId)) {
|
||||
const existingFolder = existingFolderMap.get(folder.externalId);
|
||||
return this.globalWorkspaceOrmManager.executeInWorkspaceContext(
|
||||
authContext,
|
||||
async () => {
|
||||
const messageFolderRepository =
|
||||
await this.globalWorkspaceOrmManager.getRepository<MessageFolderWorkspaceEntity>(
|
||||
workspaceId,
|
||||
'messageFolder',
|
||||
);
|
||||
|
||||
if (existingFolder) {
|
||||
return existingFolder;
|
||||
}
|
||||
}
|
||||
const workspaceDataSource =
|
||||
await this.globalWorkspaceOrmManager.getGlobalWorkspaceDataSource();
|
||||
|
||||
const legacyFolderName = this.getLegacyFolderName(folder);
|
||||
return workspaceDataSource.transaction(
|
||||
async (transactionManager: WorkspaceEntityManager) => {
|
||||
if (folderIdsToDelete.length > 0) {
|
||||
await messageFolderRepository.updateMany(
|
||||
folderIdsToDelete.map((id) => ({
|
||||
criteria: id,
|
||||
partialEntity: {
|
||||
pendingSyncAction:
|
||||
MessageFolderPendingSyncAction.FOLDER_DELETION,
|
||||
},
|
||||
})),
|
||||
transactionManager,
|
||||
);
|
||||
}
|
||||
|
||||
return existingFolderMap.get(legacyFolderName);
|
||||
}
|
||||
if (foldersToUpdate.size > 0) {
|
||||
await messageFolderRepository.updateMany(
|
||||
Array.from(foldersToUpdate.entries()).map(([id, data]) => ({
|
||||
criteria: id,
|
||||
partialEntity: data,
|
||||
})),
|
||||
transactionManager,
|
||||
);
|
||||
}
|
||||
|
||||
private getLegacyFolderName(folder: MessageFolder): string {
|
||||
if (folder.isSynced && !folder.isSentFolder) {
|
||||
return MessageFolderName.INBOX;
|
||||
}
|
||||
const createdFolders =
|
||||
foldersToCreate.length > 0
|
||||
? await messageFolderRepository.save(
|
||||
foldersToCreate,
|
||||
{},
|
||||
transactionManager,
|
||||
)
|
||||
: [];
|
||||
|
||||
if (folder.isSynced && folder.isSentFolder) {
|
||||
return MessageFolderName.SENT_ITEMS;
|
||||
}
|
||||
const updatedExistingFolders = computeUpdatedFolders({
|
||||
existingFolders,
|
||||
foldersToUpdate,
|
||||
folderIdsToDelete,
|
||||
});
|
||||
|
||||
return folder.name ?? '';
|
||||
return [...updatedExistingFolders, ...createdFolders];
|
||||
},
|
||||
);
|
||||
},
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user