feat: message folders control (#14144)
Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com>
This commit is contained in:
+17
-30
@@ -3,12 +3,10 @@ import { Injectable } from '@nestjs/common';
|
||||
import { ConnectedAccountProvider } from 'twenty-shared/types';
|
||||
import { v4 } from 'uuid';
|
||||
|
||||
import { CreateMessageFolderService } from 'src/engine/core-modules/auth/services/create-message-folder.service';
|
||||
import { type EmailAccountConnectionParameters } from 'src/engine/core-modules/imap-smtp-caldav-connection/dtos/imap-smtp-caldav-connection.dto';
|
||||
import { InjectMessageQueue } from 'src/engine/core-modules/message-queue/decorators/message-queue.decorator';
|
||||
import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';
|
||||
import { MessageQueueService } from 'src/engine/core-modules/message-queue/services/message-queue.service';
|
||||
import { type WorkspaceEntityManager } from 'src/engine/twenty-orm/entity-manager/workspace-entity-manager';
|
||||
import { type WorkspaceRepository } from 'src/engine/twenty-orm/repository/workspace.repository';
|
||||
import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager';
|
||||
import {
|
||||
@@ -40,7 +38,6 @@ export class ImapSmtpCalDavAPIService {
|
||||
private readonly messageQueueService: MessageQueueService,
|
||||
@InjectMessageQueue(MessageQueue.calendarQueue)
|
||||
private readonly calendarQueueService: MessageQueueService,
|
||||
private readonly createMessageFolderService: CreateMessageFolderService,
|
||||
) {}
|
||||
|
||||
async setupCompleteAccount(input: {
|
||||
@@ -89,28 +86,25 @@ export class ImapSmtpCalDavAPIService {
|
||||
let createdMessageChannel: MessageChannelWorkspaceEntity | null = null;
|
||||
let createdCalendarChannel: CalendarChannelWorkspaceEntity | null = null;
|
||||
|
||||
await workspaceDataSource.transaction(
|
||||
async (manager: WorkspaceEntityManager) => {
|
||||
await this.upsertConnectedAccount(
|
||||
input,
|
||||
accountId,
|
||||
connectedAccountRepository,
|
||||
);
|
||||
await workspaceDataSource.transaction(async () => {
|
||||
await this.upsertConnectedAccount(
|
||||
input,
|
||||
accountId,
|
||||
connectedAccountRepository,
|
||||
);
|
||||
|
||||
createdMessageChannel = await this.setupMessageChannels(
|
||||
input,
|
||||
accountId,
|
||||
messageChannelRepository,
|
||||
manager,
|
||||
);
|
||||
createdMessageChannel = await this.setupMessageChannels(
|
||||
input,
|
||||
accountId,
|
||||
messageChannelRepository,
|
||||
);
|
||||
|
||||
createdCalendarChannel = await this.setupCalendarChannels(
|
||||
input,
|
||||
accountId,
|
||||
calendarChannelRepository,
|
||||
);
|
||||
},
|
||||
);
|
||||
createdCalendarChannel = await this.setupCalendarChannels(
|
||||
input,
|
||||
accountId,
|
||||
calendarChannelRepository,
|
||||
);
|
||||
});
|
||||
|
||||
await this.enqueueSyncJobs(
|
||||
input,
|
||||
@@ -149,7 +143,6 @@ export class ImapSmtpCalDavAPIService {
|
||||
},
|
||||
accountId: string,
|
||||
messageChannelRepository: WorkspaceRepository<MessageChannelWorkspaceEntity>,
|
||||
manager: WorkspaceEntityManager,
|
||||
): Promise<MessageChannelWorkspaceEntity | null> {
|
||||
const existingChannels = await messageChannelRepository.find({
|
||||
where: { connectedAccountId: accountId },
|
||||
@@ -182,12 +175,6 @@ export class ImapSmtpCalDavAPIService {
|
||||
{},
|
||||
);
|
||||
|
||||
await this.createMessageFolderService.createMessageFolders({
|
||||
workspaceId: input.workspaceId,
|
||||
messageChannelId: newMessageChannel.id,
|
||||
manager,
|
||||
});
|
||||
|
||||
return shouldEnableSync ? newMessageChannel : null;
|
||||
}
|
||||
|
||||
|
||||
+34
-2
@@ -1,14 +1,15 @@
|
||||
import { msg } from '@lingui/core/macro';
|
||||
import { Relation } from 'typeorm';
|
||||
import { FieldMetadataType } from 'twenty-shared/types';
|
||||
import { Relation } from 'typeorm';
|
||||
|
||||
import { RelationType } from 'src/engine/metadata-modules/field-metadata/interfaces/relation-type.interface';
|
||||
import { RelationOnDeleteAction } from 'src/engine/metadata-modules/field-metadata/interfaces/relation-on-delete-action.interface';
|
||||
import { RelationType } from 'src/engine/metadata-modules/field-metadata/interfaces/relation-type.interface';
|
||||
|
||||
import { BaseWorkspaceEntity } from 'src/engine/twenty-orm/base.workspace-entity';
|
||||
import { WorkspaceEntity } from 'src/engine/twenty-orm/decorators/workspace-entity.decorator';
|
||||
import { WorkspaceField } from 'src/engine/twenty-orm/decorators/workspace-field.decorator';
|
||||
import { WorkspaceIsNotAuditLogged } from 'src/engine/twenty-orm/decorators/workspace-is-not-audit-logged.decorator';
|
||||
import { WorkspaceIsNullable } from 'src/engine/twenty-orm/decorators/workspace-is-nullable.decorator';
|
||||
import { WorkspaceIsSystem } from 'src/engine/twenty-orm/decorators/workspace-is-system.decorator';
|
||||
import { WorkspaceJoinColumn } from 'src/engine/twenty-orm/decorators/workspace-join-column.decorator';
|
||||
import { WorkspaceRelation } from 'src/engine/twenty-orm/decorators/workspace-relation.decorator';
|
||||
@@ -58,6 +59,37 @@ export class MessageFolderWorkspaceEntity extends BaseWorkspaceEntity {
|
||||
})
|
||||
syncCursor: string;
|
||||
|
||||
@WorkspaceField({
|
||||
standardId: MESSAGE_FOLDER_STANDARD_FIELD_IDS.isSentFolder,
|
||||
type: FieldMetadataType.BOOLEAN,
|
||||
label: msg`Is Sent Folder`,
|
||||
description: msg`Is Sent Folder`,
|
||||
icon: 'IconCheck',
|
||||
defaultValue: false,
|
||||
})
|
||||
isSentFolder: boolean;
|
||||
|
||||
@WorkspaceField({
|
||||
standardId: MESSAGE_FOLDER_STANDARD_FIELD_IDS.isSynced,
|
||||
type: FieldMetadataType.BOOLEAN,
|
||||
label: msg`Is Synced`,
|
||||
description: msg`Is Synced`,
|
||||
icon: 'IconCheck',
|
||||
defaultValue: false,
|
||||
})
|
||||
isSynced: boolean;
|
||||
|
||||
@WorkspaceField({
|
||||
standardId: MESSAGE_FOLDER_STANDARD_FIELD_IDS.externalId,
|
||||
type: FieldMetadataType.TEXT,
|
||||
label: msg`External ID`,
|
||||
description: msg`External ID`,
|
||||
icon: 'IconHash',
|
||||
defaultValue: null,
|
||||
})
|
||||
@WorkspaceIsNullable()
|
||||
externalId: string | null;
|
||||
|
||||
@WorkspaceJoinColumn('messageChannel')
|
||||
messageChannelId: string;
|
||||
}
|
||||
|
||||
+110
@@ -0,0 +1,110 @@
|
||||
import { Injectable, Logger } from '@nestjs/common';
|
||||
|
||||
import { gmail_v1 } from 'googleapis';
|
||||
import { isDefined } from 'twenty-shared/utils';
|
||||
|
||||
import {
|
||||
MessageFolder,
|
||||
MessageFolderDriver,
|
||||
} from 'src/modules/messaging/message-folder-manager/interfaces/message-folder-driver.interface';
|
||||
|
||||
import { type ConnectedAccountWorkspaceEntity } from 'src/modules/connected-account/standard-objects/connected-account.workspace-entity';
|
||||
import { MESSAGING_GMAIL_EXCLUDED_CATEGORIES } from 'src/modules/messaging/message-import-manager/drivers/gmail/constants/messaging-gmail-excluded-categories';
|
||||
import { GmailClientProvider } from 'src/modules/messaging/message-import-manager/drivers/gmail/providers/gmail-client.provider';
|
||||
import { GmailHandleErrorService } from 'src/modules/messaging/message-import-manager/drivers/gmail/services/gmail-handle-error.service';
|
||||
import { computeGmailCategoryLabelId } from 'src/modules/messaging/message-import-manager/drivers/gmail/utils/compute-gmail-category-label-id.util';
|
||||
|
||||
@Injectable()
|
||||
export class GmailGetAllFoldersService implements MessageFolderDriver {
|
||||
private readonly logger = new Logger(GmailGetAllFoldersService.name);
|
||||
|
||||
constructor(
|
||||
private readonly gmailClientProvider: GmailClientProvider,
|
||||
private readonly gmailHandleErrorService: GmailHandleErrorService,
|
||||
) {}
|
||||
|
||||
private isExcludedCategoryFolder(labelId: string): boolean {
|
||||
const excludedCategoryIds = MESSAGING_GMAIL_EXCLUDED_CATEGORIES.map(
|
||||
(category) => computeGmailCategoryLabelId(category),
|
||||
);
|
||||
|
||||
return excludedCategoryIds.includes(labelId);
|
||||
}
|
||||
|
||||
private isIncludedFolder(label: gmail_v1.Schema$Label): boolean {
|
||||
if (!isDefined(label.id)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const isTargetSystemFolder =
|
||||
label.type === 'system' && (label.id === 'INBOX' || label.id === 'SENT');
|
||||
const isUserFolder = label.type === 'user';
|
||||
|
||||
return isTargetSystemFolder || isUserFolder;
|
||||
}
|
||||
|
||||
async getAllMessageFolders(
|
||||
connectedAccount: Pick<
|
||||
ConnectedAccountWorkspaceEntity,
|
||||
'provider' | 'refreshToken' | 'id' | 'handle'
|
||||
>,
|
||||
): Promise<MessageFolder[]> {
|
||||
try {
|
||||
const gmailClient =
|
||||
await this.gmailClientProvider.getGmailClient(connectedAccount);
|
||||
|
||||
const response = await gmailClient.users.labels
|
||||
.list({ userId: 'me' })
|
||||
.catch((error) => {
|
||||
this.logger.error(
|
||||
`Connected account ${connectedAccount.id}: Error fetching labels: ${error.message}`,
|
||||
);
|
||||
|
||||
this.gmailHandleErrorService.handleGmailMessageListFetchError(error);
|
||||
|
||||
return { data: { labels: [] } };
|
||||
});
|
||||
|
||||
const labels = response.data.labels || [];
|
||||
|
||||
const folders: MessageFolder[] = [];
|
||||
|
||||
for (const label of labels) {
|
||||
if (!label.name || !label.id) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (this.isExcludedCategoryFolder(label.id)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!this.isIncludedFolder(label)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const isSentFolder = label.id === 'SENT';
|
||||
const isSyncedByDefault = label.id === 'INBOX' || label.id === 'SENT';
|
||||
|
||||
folders.push({
|
||||
externalId: label.id,
|
||||
name: label.name,
|
||||
isSynced: isSyncedByDefault,
|
||||
isSentFolder,
|
||||
});
|
||||
}
|
||||
|
||||
this.logger.log(
|
||||
`Found ${folders.length} folders for Gmail account ${connectedAccount.handle}`,
|
||||
);
|
||||
|
||||
return folders;
|
||||
} catch (error) {
|
||||
this.logger.error(
|
||||
`Failed to get Gmail folders for account ${connectedAccount.handle}:`,
|
||||
error,
|
||||
);
|
||||
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
+171
@@ -0,0 +1,171 @@
|
||||
import { Injectable, Logger } from '@nestjs/common';
|
||||
|
||||
import { ImapFlow, type ListResponse } from 'imapflow';
|
||||
import { isDefined } from 'twenty-shared/utils';
|
||||
|
||||
import {
|
||||
MessageFolder,
|
||||
MessageFolderDriver,
|
||||
} from 'src/modules/messaging/message-folder-manager/interfaces/message-folder-driver.interface';
|
||||
|
||||
import { type ConnectedAccountWorkspaceEntity } from 'src/modules/connected-account/standard-objects/connected-account.workspace-entity';
|
||||
import { ImapClientProvider } from 'src/modules/messaging/message-import-manager/drivers/imap/providers/imap-client.provider';
|
||||
import { ImapFindSentFolderService } from 'src/modules/messaging/message-import-manager/drivers/imap/services/imap-find-sent-folder.service';
|
||||
import { MessageFolderName } from 'src/modules/messaging/message-import-manager/drivers/imap/types/folders';
|
||||
import { StandardFolder } from 'src/modules/messaging/message-import-manager/drivers/types/standard-folder';
|
||||
import { getStandardFolderByRegex } from 'src/modules/messaging/message-import-manager/drivers/utils/get-standard-folder-by-regex';
|
||||
|
||||
@Injectable()
|
||||
export class ImapGetAllFoldersService implements MessageFolderDriver {
|
||||
private readonly logger = new Logger(ImapGetAllFoldersService.name);
|
||||
|
||||
constructor(
|
||||
private readonly imapClientProvider: ImapClientProvider,
|
||||
private readonly imapFindSentFolderService: ImapFindSentFolderService,
|
||||
) {}
|
||||
|
||||
public async getAllMessageFolders(
|
||||
connectedAccount: Pick<
|
||||
ConnectedAccountWorkspaceEntity,
|
||||
'id' | 'provider' | 'connectionParameters' | 'handle'
|
||||
>,
|
||||
): Promise<MessageFolder[]> {
|
||||
try {
|
||||
const client = await this.imapClientProvider.getClient(connectedAccount);
|
||||
|
||||
const mailboxList = await client.list();
|
||||
|
||||
const folders = await this.filterAndMapFolders(client, mailboxList);
|
||||
|
||||
await this.imapClientProvider.closeClient(client);
|
||||
|
||||
return folders;
|
||||
} catch (error) {
|
||||
this.logger.error(
|
||||
`Failed to get IMAP folders for account ${connectedAccount.handle}:`,
|
||||
error,
|
||||
);
|
||||
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
private async filterAndMapFolders(
|
||||
client: ImapFlow,
|
||||
mailboxList: ListResponse[],
|
||||
): Promise<MessageFolder[]> {
|
||||
const folders: MessageFolder[] = [];
|
||||
const sentFolderPath =
|
||||
await this.imapFindSentFolderService.findSentFolder(client);
|
||||
|
||||
if (isDefined(sentFolderPath)) {
|
||||
const sentMailbox = mailboxList.find((m) => m.path === sentFolderPath);
|
||||
const uidValidity = sentMailbox
|
||||
? await this.getUidValidity(client, sentMailbox)
|
||||
: null;
|
||||
|
||||
folders.push({
|
||||
externalId: uidValidity
|
||||
? `${sentFolderPath}:${uidValidity.toString()}`
|
||||
: sentFolderPath,
|
||||
name: sentFolderPath,
|
||||
isSynced: true,
|
||||
isSentFolder: true,
|
||||
});
|
||||
}
|
||||
|
||||
const validMailboxes = mailboxList.filter((mailbox) =>
|
||||
this.isValidMailbox(mailbox, folders),
|
||||
);
|
||||
|
||||
for (const mailbox of validMailboxes) {
|
||||
const isInbox = await this.isInboxFolder(mailbox);
|
||||
const uidValidity = await this.getUidValidity(client, mailbox);
|
||||
|
||||
folders.push({
|
||||
externalId: uidValidity
|
||||
? `${mailbox.path}:${uidValidity}`
|
||||
: mailbox.path,
|
||||
name: mailbox.path,
|
||||
isSynced: isInbox,
|
||||
isSentFolder: false,
|
||||
});
|
||||
}
|
||||
|
||||
return folders;
|
||||
}
|
||||
|
||||
private isValidMailbox(
|
||||
mailbox: ListResponse,
|
||||
existingFolders: MessageFolder[],
|
||||
): boolean {
|
||||
if (this.shouldExcludeFolder(mailbox)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const isDuplicate = existingFolders.some(
|
||||
(folder) => folder.name === mailbox.path,
|
||||
);
|
||||
|
||||
return !isDuplicate;
|
||||
}
|
||||
|
||||
private async isInboxFolder(mailbox: ListResponse): Promise<boolean> {
|
||||
if (
|
||||
mailbox.path.toLowerCase() === MessageFolderName.INBOX ||
|
||||
mailbox.specialUse === '\\Inbox'
|
||||
) {
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
private shouldExcludeFolder(mailbox: ListResponse): boolean {
|
||||
if (mailbox.flags?.has('\\Noselect')) {
|
||||
return true;
|
||||
}
|
||||
|
||||
if (
|
||||
mailbox.specialUse === '\\Drafts' ||
|
||||
mailbox.specialUse === '\\Trash' ||
|
||||
mailbox.specialUse === '\\Junk'
|
||||
) {
|
||||
return true;
|
||||
}
|
||||
const standardFolder = getStandardFolderByRegex(mailbox.path);
|
||||
|
||||
if (!standardFolder) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return (
|
||||
standardFolder !== StandardFolder.SENT &&
|
||||
standardFolder !== StandardFolder.INBOX
|
||||
);
|
||||
}
|
||||
|
||||
private async getUidValidity(
|
||||
client: ImapFlow,
|
||||
mailbox: ListResponse,
|
||||
): Promise<bigint | null> {
|
||||
if (mailbox.status?.uidValidity) {
|
||||
return mailbox.status.uidValidity;
|
||||
}
|
||||
|
||||
try {
|
||||
const status = await client.status(mailbox.path, {
|
||||
uidValidity: true,
|
||||
});
|
||||
|
||||
return status.uidValidity ?? null;
|
||||
} catch (error) {
|
||||
this.logger.warn(
|
||||
`Failed to get uidValidity for folder ${mailbox.path}:`,
|
||||
error,
|
||||
);
|
||||
|
||||
return null;
|
||||
}
|
||||
}
|
||||
}
|
||||
+107
@@ -0,0 +1,107 @@
|
||||
import { Injectable, Logger } from '@nestjs/common';
|
||||
|
||||
import {
|
||||
MessageFolder,
|
||||
MessageFolderDriver,
|
||||
} from 'src/modules/messaging/message-folder-manager/interfaces/message-folder-driver.interface';
|
||||
|
||||
import { type ConnectedAccountWorkspaceEntity } from 'src/modules/connected-account/standard-objects/connected-account.workspace-entity';
|
||||
import { MicrosoftClientProvider } from 'src/modules/messaging/message-import-manager/drivers/microsoft/providers/microsoft-client.provider';
|
||||
import { MicrosoftHandleErrorService } from 'src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-handle-error.service';
|
||||
import { StandardFolder } from 'src/modules/messaging/message-import-manager/drivers/types/standard-folder';
|
||||
import { getStandardFolderByRegex } from 'src/modules/messaging/message-import-manager/drivers/utils/get-standard-folder-by-regex';
|
||||
|
||||
type MicrosoftGraphFolder = {
|
||||
id: string;
|
||||
displayName: string;
|
||||
};
|
||||
|
||||
@Injectable()
|
||||
export class MicrosoftGetAllFoldersService implements MessageFolderDriver {
|
||||
private readonly logger = new Logger(MicrosoftGetAllFoldersService.name);
|
||||
|
||||
constructor(
|
||||
private readonly microsoftClientProvider: MicrosoftClientProvider,
|
||||
private readonly microsoftHandleErrorService: MicrosoftHandleErrorService,
|
||||
) {}
|
||||
|
||||
async getAllMessageFolders(
|
||||
connectedAccount: Pick<
|
||||
ConnectedAccountWorkspaceEntity,
|
||||
'refreshToken' | 'id' | 'handle'
|
||||
>,
|
||||
): Promise<MessageFolder[]> {
|
||||
try {
|
||||
const microsoftClient =
|
||||
await this.microsoftClientProvider.getMicrosoftClient(connectedAccount);
|
||||
|
||||
const response = await microsoftClient
|
||||
.api('/me/mailFolders')
|
||||
.get()
|
||||
.catch((error) => {
|
||||
this.logger.error(
|
||||
`Connected account ${connectedAccount.id}: Error fetching folders: ${error.message}`,
|
||||
);
|
||||
this.microsoftHandleErrorService.handleMicrosoftGetMessageListError(
|
||||
error,
|
||||
);
|
||||
|
||||
return { value: [] };
|
||||
});
|
||||
|
||||
const folders = (response.value as MicrosoftGraphFolder[]) || [];
|
||||
const folderInfos: MessageFolder[] = [];
|
||||
|
||||
for (const folder of folders) {
|
||||
if (!folder.displayName) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const standardFolder = getStandardFolderByRegex(folder.displayName);
|
||||
|
||||
if (this.shouldExcludeFolder(standardFolder)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const isInbox = this.isInboxFolder(standardFolder);
|
||||
const isSentFolder = this.isSentFolder(standardFolder);
|
||||
|
||||
folderInfos.push({
|
||||
externalId: folder.id,
|
||||
name: folder.displayName,
|
||||
isSynced: isInbox || isSentFolder,
|
||||
isSentFolder,
|
||||
});
|
||||
}
|
||||
|
||||
this.logger.log(
|
||||
`Found ${folderInfos.length} folders for Microsoft account ${connectedAccount.handle}`,
|
||||
);
|
||||
|
||||
return folderInfos;
|
||||
} catch (error) {
|
||||
this.logger.error(
|
||||
`Failed to get Microsoft folders for account ${connectedAccount.handle}:`,
|
||||
error,
|
||||
);
|
||||
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
private isInboxFolder(standardFolder: StandardFolder | null): boolean {
|
||||
return standardFolder === StandardFolder.INBOX;
|
||||
}
|
||||
|
||||
private isSentFolder(standardFolder: StandardFolder | null): boolean {
|
||||
return standardFolder === StandardFolder.SENT;
|
||||
}
|
||||
|
||||
private shouldExcludeFolder(standardFolder: StandardFolder | null): boolean {
|
||||
return (
|
||||
standardFolder !== null &&
|
||||
standardFolder !== StandardFolder.SENT &&
|
||||
standardFolder !== StandardFolder.INBOX
|
||||
);
|
||||
}
|
||||
}
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
import { type ConnectedAccountWorkspaceEntity } from 'src/modules/connected-account/standard-objects/connected-account.workspace-entity';
|
||||
import { type MessageFolderWorkspaceEntity } from 'src/modules/messaging/common/standard-objects/message-folder.workspace-entity';
|
||||
|
||||
export type MessageFolder = Pick<
|
||||
MessageFolderWorkspaceEntity,
|
||||
'name' | 'isSynced' | 'isSentFolder' | 'externalId'
|
||||
>;
|
||||
|
||||
export interface MessageFolderDriver {
|
||||
getAllMessageFolders(
|
||||
connectedAccount: Pick<
|
||||
ConnectedAccountWorkspaceEntity,
|
||||
'provider' | 'refreshToken' | 'id' | 'handle' | 'connectionParameters'
|
||||
>,
|
||||
): Promise<MessageFolder[]>;
|
||||
}
|
||||
+34
@@ -0,0 +1,34 @@
|
||||
import { Module } from '@nestjs/common';
|
||||
import { TypeOrmModule } from '@nestjs/typeorm';
|
||||
|
||||
import { FeatureFlagModule } from 'src/engine/core-modules/feature-flag/feature-flag.module';
|
||||
import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity';
|
||||
import { DataSourceModule } from 'src/engine/metadata-modules/data-source/data-source.module';
|
||||
import { WorkspaceDataSourceModule } from 'src/engine/workspace-datasource/workspace-datasource.module';
|
||||
import { SyncMessageFoldersService } from 'src/modules/messaging/message-folder-manager/services/sync-message-folders.service';
|
||||
import { GmailGetAllFoldersService } from 'src/modules/messaging/message-folder-manager/drivers/gmail/gmail-get-all-folders.service';
|
||||
import { ImapGetAllFoldersService } from 'src/modules/messaging/message-folder-manager/drivers/imap/imap-get-all-folders.service';
|
||||
import { MicrosoftGetAllFoldersService } from 'src/modules/messaging/message-folder-manager/drivers/microsoft/microsoft-get-all-folders.service';
|
||||
import { MessagingGmailDriverModule } from 'src/modules/messaging/message-import-manager/drivers/gmail/messaging-gmail-driver.module';
|
||||
import { MessagingIMAPDriverModule } from 'src/modules/messaging/message-import-manager/drivers/imap/messaging-imap-driver.module';
|
||||
import { MessagingMicrosoftDriverModule } from 'src/modules/messaging/message-import-manager/drivers/microsoft/messaging-microsoft-driver.module';
|
||||
|
||||
@Module({
|
||||
imports: [
|
||||
FeatureFlagModule,
|
||||
WorkspaceDataSourceModule,
|
||||
DataSourceModule,
|
||||
TypeOrmModule.forFeature([Workspace]),
|
||||
MessagingGmailDriverModule,
|
||||
MessagingMicrosoftDriverModule,
|
||||
MessagingIMAPDriverModule,
|
||||
],
|
||||
providers: [
|
||||
SyncMessageFoldersService,
|
||||
GmailGetAllFoldersService,
|
||||
ImapGetAllFoldersService,
|
||||
MicrosoftGetAllFoldersService,
|
||||
],
|
||||
exports: [SyncMessageFoldersService],
|
||||
})
|
||||
export class MessagingFolderSyncManagerModule {}
|
||||
+181
@@ -0,0 +1,181 @@
|
||||
import { Injectable } from '@nestjs/common';
|
||||
|
||||
import { ConnectedAccountProvider } from 'twenty-shared/types';
|
||||
import { isDefined } from 'twenty-shared/utils';
|
||||
import { v4 } from 'uuid';
|
||||
|
||||
import { MessageFolder } from 'src/modules/messaging/message-folder-manager/interfaces/message-folder-driver.interface';
|
||||
|
||||
import { FeatureFlagService } from 'src/engine/core-modules/feature-flag/services/feature-flag.service';
|
||||
import { type WorkspaceEntityManager } from 'src/engine/twenty-orm/entity-manager/workspace-entity-manager';
|
||||
import { WorkspaceRepository } from 'src/engine/twenty-orm/repository/workspace.repository';
|
||||
import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager';
|
||||
import { type MessageChannelWorkspaceEntity } from 'src/modules/messaging/common/standard-objects/message-channel.workspace-entity';
|
||||
import { type MessageFolderWorkspaceEntity } from 'src/modules/messaging/common/standard-objects/message-folder.workspace-entity';
|
||||
import { GmailGetAllFoldersService } from 'src/modules/messaging/message-folder-manager/drivers/gmail/gmail-get-all-folders.service';
|
||||
import { ImapGetAllFoldersService } from 'src/modules/messaging/message-folder-manager/drivers/imap/imap-get-all-folders.service';
|
||||
import { MicrosoftGetAllFoldersService } from 'src/modules/messaging/message-folder-manager/drivers/microsoft/microsoft-get-all-folders.service';
|
||||
import { MessageFolderName } from 'src/modules/messaging/message-import-manager/drivers/microsoft/types/folders';
|
||||
|
||||
type SyncMessageFoldersInput = {
|
||||
workspaceId: string;
|
||||
messageChannelId: string;
|
||||
connectedAccount: MessageChannelWorkspaceEntity['connectedAccount'];
|
||||
manager: WorkspaceEntityManager;
|
||||
};
|
||||
|
||||
@Injectable()
|
||||
export class SyncMessageFoldersService {
|
||||
constructor(
|
||||
private readonly twentyORMGlobalManager: TwentyORMGlobalManager,
|
||||
private readonly featureFlagService: FeatureFlagService,
|
||||
private readonly gmailGetAllFoldersService: GmailGetAllFoldersService,
|
||||
private readonly microsoftGetAllFoldersService: MicrosoftGetAllFoldersService,
|
||||
private readonly imapGetAllFoldersService: ImapGetAllFoldersService,
|
||||
) {}
|
||||
|
||||
async syncMessageFolders(input: SyncMessageFoldersInput): Promise<void> {
|
||||
const { workspaceId, messageChannelId, connectedAccount, manager } = input;
|
||||
|
||||
const folders = await this.discoverAllFolders(connectedAccount);
|
||||
|
||||
await this.upsertDiscoveredFolders({
|
||||
workspaceId,
|
||||
messageChannelId,
|
||||
folders,
|
||||
manager,
|
||||
});
|
||||
}
|
||||
|
||||
private async upsertDiscoveredFolders({
|
||||
workspaceId,
|
||||
messageChannelId,
|
||||
folders,
|
||||
manager,
|
||||
}: {
|
||||
workspaceId: string;
|
||||
messageChannelId: string;
|
||||
folders: MessageFolder[];
|
||||
manager: WorkspaceEntityManager;
|
||||
}): Promise<void> {
|
||||
const messageFolderRepository =
|
||||
await this.twentyORMGlobalManager.getRepositoryForWorkspace<MessageFolderWorkspaceEntity>(
|
||||
workspaceId,
|
||||
'messageFolder',
|
||||
);
|
||||
|
||||
const existingFolderMap = await this.buildExistingFolderMap({
|
||||
messageChannelId,
|
||||
messageFolderRepository,
|
||||
});
|
||||
|
||||
for (const folder of folders) {
|
||||
const existingFolder = this.findExistingFolderInMap(
|
||||
existingFolderMap,
|
||||
folder,
|
||||
);
|
||||
|
||||
if (existingFolder) {
|
||||
await messageFolderRepository.update(
|
||||
existingFolder.id,
|
||||
{
|
||||
name: folder.name,
|
||||
isSynced: folder.isSynced,
|
||||
isSentFolder: folder.isSentFolder,
|
||||
externalId: folder.externalId,
|
||||
},
|
||||
manager,
|
||||
);
|
||||
} else {
|
||||
await messageFolderRepository.save(
|
||||
{
|
||||
id: v4(),
|
||||
messageChannelId,
|
||||
name: folder.name,
|
||||
syncCursor: '',
|
||||
isSynced: folder.isSynced,
|
||||
isSentFolder: folder.isSentFolder,
|
||||
externalId: folder.externalId,
|
||||
},
|
||||
{},
|
||||
manager,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async discoverAllFolders(
|
||||
connectedAccount: MessageChannelWorkspaceEntity['connectedAccount'],
|
||||
): Promise<MessageFolder[]> {
|
||||
switch (connectedAccount.provider) {
|
||||
case ConnectedAccountProvider.GOOGLE:
|
||||
return await this.gmailGetAllFoldersService.getAllMessageFolders(
|
||||
connectedAccount,
|
||||
);
|
||||
case ConnectedAccountProvider.MICROSOFT:
|
||||
return await this.microsoftGetAllFoldersService.getAllMessageFolders(
|
||||
connectedAccount,
|
||||
);
|
||||
case ConnectedAccountProvider.IMAP_SMTP_CALDAV:
|
||||
return await this.imapGetAllFoldersService.getAllMessageFolders(
|
||||
connectedAccount,
|
||||
);
|
||||
default:
|
||||
throw new Error(
|
||||
`Provider ${connectedAccount.provider} is not supported`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
private async buildExistingFolderMap({
|
||||
messageChannelId,
|
||||
messageFolderRepository,
|
||||
}: {
|
||||
messageChannelId: string;
|
||||
messageFolderRepository: WorkspaceRepository<MessageFolderWorkspaceEntity>;
|
||||
}): Promise<Map<string, MessageFolderWorkspaceEntity>> {
|
||||
const existingFolders = await messageFolderRepository.find({
|
||||
where: { messageChannelId },
|
||||
});
|
||||
|
||||
const existingFolderMap = new Map<string, MessageFolderWorkspaceEntity>();
|
||||
|
||||
for (const existingFolder of existingFolders) {
|
||||
if (isDefined(existingFolder.externalId)) {
|
||||
existingFolderMap.set(existingFolder.externalId, existingFolder);
|
||||
}
|
||||
existingFolderMap.set(existingFolder.name, existingFolder);
|
||||
}
|
||||
|
||||
return existingFolderMap;
|
||||
}
|
||||
|
||||
private findExistingFolderInMap(
|
||||
existingFolderMap: Map<string, MessageFolderWorkspaceEntity>,
|
||||
folder: MessageFolder,
|
||||
): MessageFolderWorkspaceEntity | undefined {
|
||||
if (isDefined(folder.externalId)) {
|
||||
const existingFolder = existingFolderMap.get(folder.externalId);
|
||||
|
||||
if (existingFolder) {
|
||||
return existingFolder;
|
||||
}
|
||||
}
|
||||
|
||||
const legacyFolderName = this.getLegacyFolderName(folder);
|
||||
|
||||
return existingFolderMap.get(legacyFolderName);
|
||||
}
|
||||
|
||||
private getLegacyFolderName(folder: MessageFolder): string {
|
||||
if (folder.isSynced && !folder.isSentFolder) {
|
||||
return MessageFolderName.INBOX;
|
||||
}
|
||||
|
||||
if (folder.isSynced && folder.isSentFolder) {
|
||||
return MessageFolderName.SENT_ITEMS;
|
||||
}
|
||||
|
||||
return folder.name;
|
||||
}
|
||||
}
|
||||
+2
-1
@@ -12,13 +12,13 @@ import { EmailAliasManagerModule } from 'src/modules/connected-account/email-ali
|
||||
import { OAuth2ClientManagerModule } from 'src/modules/connected-account/oauth2-client-manager/oauth2-client-manager.module';
|
||||
import { MessagingCommonModule } from 'src/modules/messaging/common/messaging-common.module';
|
||||
import { GmailClientProvider } from 'src/modules/messaging/message-import-manager/drivers/gmail/providers/gmail-client.provider';
|
||||
import { OAuth2ClientProvider } from 'src/modules/messaging/message-import-manager/drivers/gmail/providers/oauth2-client.provider';
|
||||
import { GmailFetchByBatchService } from 'src/modules/messaging/message-import-manager/drivers/gmail/services/gmail-fetch-by-batch.service';
|
||||
import { GmailGetHistoryService } from 'src/modules/messaging/message-import-manager/drivers/gmail/services/gmail-get-history.service';
|
||||
import { GmailGetMessageListService } from 'src/modules/messaging/message-import-manager/drivers/gmail/services/gmail-get-message-list.service';
|
||||
import { GmailGetMessagesService } from 'src/modules/messaging/message-import-manager/drivers/gmail/services/gmail-get-messages.service';
|
||||
import { GmailHandleErrorService } from 'src/modules/messaging/message-import-manager/drivers/gmail/services/gmail-handle-error.service';
|
||||
import { MessageParticipantManagerModule } from 'src/modules/messaging/message-participant-manager/message-participant-manager.module';
|
||||
import { OAuth2ClientProvider } from 'src/modules/messaging/message-import-manager/drivers/gmail/providers/oauth2-client.provider';
|
||||
|
||||
@Module({
|
||||
imports: [
|
||||
@@ -49,6 +49,7 @@ import { OAuth2ClientProvider } from 'src/modules/messaging/message-import-manag
|
||||
GmailGetMessageListService,
|
||||
GmailClientProvider,
|
||||
OAuth2ClientProvider,
|
||||
GmailHandleErrorService,
|
||||
],
|
||||
})
|
||||
export class MessagingGmailDriverModule {}
|
||||
|
||||
+29
-6
@@ -36,7 +36,10 @@ export class GmailGetMessageListService {
|
||||
ConnectedAccountWorkspaceEntity,
|
||||
'provider' | 'refreshToken' | 'id' | 'handle'
|
||||
>,
|
||||
messageFolders: Pick<MessageFolderWorkspaceEntity, 'name'>[],
|
||||
messageFolders: Pick<
|
||||
MessageFolderWorkspaceEntity,
|
||||
'name' | 'externalId' | 'isSynced'
|
||||
>[],
|
||||
): Promise<GetMessageListsResponse> {
|
||||
const gmailClient =
|
||||
await this.gmailClientProvider.getGmailClient(connectedAccount);
|
||||
@@ -45,7 +48,7 @@ export class GmailGetMessageListService {
|
||||
let hasMoreMessages = true;
|
||||
|
||||
const messageExternalIds: string[] = [];
|
||||
const excludedCategories = this.comptuteExcludedCategories(messageFolders);
|
||||
const excludedCategories = this.computeExcludedCategories(messageFolders);
|
||||
|
||||
while (hasMoreMessages) {
|
||||
const messageList = await gmailClient.users.messages
|
||||
@@ -54,6 +57,7 @@ export class GmailGetMessageListService {
|
||||
maxResults: MESSAGING_GMAIL_USERS_MESSAGES_LIST_MAX_RESULT,
|
||||
pageToken,
|
||||
q: computeGmailCategoryExcludeSearchFilter(excludedCategories),
|
||||
labelIds: this.getCustomLabelIds(messageFolders),
|
||||
})
|
||||
.catch((error) => {
|
||||
this.logger.error(
|
||||
@@ -181,8 +185,8 @@ export class GmailGetMessageListService {
|
||||
];
|
||||
}
|
||||
|
||||
private comptuteExcludedCategories(
|
||||
messageFolders: Pick<MessageFolderWorkspaceEntity, 'name'>[],
|
||||
private computeExcludedCategories(
|
||||
messageFolders: Pick<MessageFolderWorkspaceEntity, 'name' | 'externalId'>[],
|
||||
) {
|
||||
const includedDefaultCategories = messageFolders
|
||||
.map((messageFolder) =>
|
||||
@@ -199,11 +203,11 @@ export class GmailGetMessageListService {
|
||||
private async getEmailIdsFromExcludedCategories(
|
||||
gmailClient: gmailV1.Gmail,
|
||||
lastSyncHistoryId: string,
|
||||
messageFolders: Pick<MessageFolderWorkspaceEntity, 'name'>[],
|
||||
messageFolders: Pick<MessageFolderWorkspaceEntity, 'name' | 'externalId'>[],
|
||||
): Promise<string[]> {
|
||||
const emailIds: string[] = [];
|
||||
|
||||
const excludedCategories = this.comptuteExcludedCategories(messageFolders);
|
||||
const excludedCategories = this.computeExcludedCategories(messageFolders);
|
||||
|
||||
for (const category of excludedCategories) {
|
||||
const { history } = await this.gmailGetHistoryService.getHistory(
|
||||
@@ -225,4 +229,23 @@ export class GmailGetMessageListService {
|
||||
|
||||
return emailIds;
|
||||
}
|
||||
|
||||
private getCustomLabelIds(
|
||||
messageFolders: Pick<
|
||||
MessageFolderWorkspaceEntity,
|
||||
'name' | 'externalId' | 'isSynced'
|
||||
>[],
|
||||
): string[] | undefined {
|
||||
const customLabelIds = messageFolders
|
||||
.filter(
|
||||
(folder) =>
|
||||
folder.externalId &&
|
||||
folder.isSynced &&
|
||||
!mapGmailDefaultFolderToCategoryOrUndefined(folder.name),
|
||||
)
|
||||
.map((folder) => folder.externalId)
|
||||
.filter((id): id is string => !!id);
|
||||
|
||||
return customLabelIds.length > 0 ? customLabelIds : undefined;
|
||||
}
|
||||
}
|
||||
|
||||
+1
@@ -46,6 +46,7 @@ import { MessageParticipantManagerModule } from 'src/modules/messaging/message-p
|
||||
ImapGetMessagesService,
|
||||
ImapGetMessageListService,
|
||||
ImapClientProvider,
|
||||
ImapFindSentFolderService,
|
||||
],
|
||||
})
|
||||
export class MessagingIMAPDriverModule {}
|
||||
|
||||
+2
-37
@@ -4,10 +4,8 @@ import { type ImapFlow } from 'imapflow';
|
||||
|
||||
import { type MessageFolderWorkspaceEntity } from 'src/modules/messaging/common/standard-objects/message-folder.workspace-entity';
|
||||
import { ImapClientProvider } from 'src/modules/messaging/message-import-manager/drivers/imap/providers/imap-client.provider';
|
||||
import { ImapFindSentFolderService } from 'src/modules/messaging/message-import-manager/drivers/imap/services/imap-find-sent-folder.service';
|
||||
import { ImapHandleErrorService } from 'src/modules/messaging/message-import-manager/drivers/imap/services/imap-handle-error.service';
|
||||
import { ImapIncrementalSyncService } from 'src/modules/messaging/message-import-manager/drivers/imap/services/imap-incremental-sync.service';
|
||||
import { MessageFolderName } from 'src/modules/messaging/message-import-manager/drivers/imap/types/folders';
|
||||
import { createSyncCursor } from 'src/modules/messaging/message-import-manager/drivers/imap/utils/create-sync-cursor.util';
|
||||
import { extractMailboxState } from 'src/modules/messaging/message-import-manager/drivers/imap/utils/extract-mailbox-state.util';
|
||||
import {
|
||||
@@ -26,7 +24,6 @@ export class ImapGetMessageListService {
|
||||
|
||||
constructor(
|
||||
private readonly imapClientProvider: ImapClientProvider,
|
||||
private readonly imapFindSentFolderService: ImapFindSentFolderService,
|
||||
private readonly imapIncrementalSyncService: ImapIncrementalSyncService,
|
||||
private readonly imapHandleErrorService: ImapHandleErrorService,
|
||||
) {}
|
||||
@@ -43,19 +40,11 @@ export class ImapGetMessageListService {
|
||||
|
||||
for (const folder of messageFolders) {
|
||||
this.logger.log(`Processing folder: ${folder.name}`);
|
||||
const folderName = await this.getFolderName(client, folder.name);
|
||||
|
||||
if (!folderName) {
|
||||
this.logger.warn(
|
||||
`No IMAP folder found for message folder: ${folder.name}`,
|
||||
);
|
||||
continue;
|
||||
}
|
||||
|
||||
try {
|
||||
const response = await this.getMessageList(
|
||||
client,
|
||||
folderName,
|
||||
folder.name,
|
||||
folder,
|
||||
);
|
||||
|
||||
@@ -65,7 +54,7 @@ export class ImapGetMessageListService {
|
||||
});
|
||||
} catch (error) {
|
||||
this.logger.warn(
|
||||
`Error fetching from folder ${folder.name} (${folderName}): ${error.message}. Continuing with other folders.`,
|
||||
`Error fetching from folder ${folder.name}: ${error.message}. Continuing with other folders.`,
|
||||
);
|
||||
|
||||
result.push({
|
||||
@@ -130,30 +119,6 @@ export class ImapGetMessageListService {
|
||||
};
|
||||
}
|
||||
|
||||
private async getFolderName(
|
||||
client: ImapFlow,
|
||||
folderName: string,
|
||||
): Promise<string | null> {
|
||||
if (folderName === MessageFolderName.INBOX) {
|
||||
return 'INBOX';
|
||||
}
|
||||
|
||||
if (folderName === MessageFolderName.SENT_ITEMS) {
|
||||
const sentFolder =
|
||||
await this.imapFindSentFolderService.findSentFolder(client);
|
||||
|
||||
if (!sentFolder) {
|
||||
this.logger.warn('SENT folder not found, skipping');
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
return sentFolder;
|
||||
}
|
||||
|
||||
return folderName;
|
||||
}
|
||||
|
||||
private async getMessagesFromFolder(
|
||||
client: ImapFlow,
|
||||
folder: string,
|
||||
|
||||
+450
@@ -0,0 +1,450 @@
|
||||
import { StandardFolder } from 'src/modules/messaging/message-import-manager/drivers/types/standard-folder';
|
||||
import { getStandardFolderByRegex } from 'src/modules/messaging/message-import-manager/drivers/utils/get-standard-folder-by-regex';
|
||||
|
||||
function testFolderMatches(
|
||||
variants: string[],
|
||||
expectedStandardFolder: StandardFolder,
|
||||
) {
|
||||
variants.forEach((variant) => {
|
||||
const result = getStandardFolderByRegex(variant);
|
||||
|
||||
expect(result).toBe(expectedStandardFolder);
|
||||
});
|
||||
}
|
||||
|
||||
describe('getStandardFolderByRegex', () => {
|
||||
describe('INBOX folder detection', () => {
|
||||
it('matches English variants', () => {
|
||||
const englishVariants = [
|
||||
'Inbox',
|
||||
'Mail',
|
||||
'Messages',
|
||||
'Message',
|
||||
'Received',
|
||||
];
|
||||
|
||||
testFolderMatches(englishVariants, StandardFolder.INBOX);
|
||||
});
|
||||
|
||||
it('matches French variants', () => {
|
||||
const frenchVariants = [
|
||||
'Boîte de réception',
|
||||
'Courrier entrant',
|
||||
'Messages reçus',
|
||||
'Réception',
|
||||
];
|
||||
|
||||
testFolderMatches(frenchVariants, StandardFolder.INBOX);
|
||||
});
|
||||
|
||||
it('matches German variants', () => {
|
||||
const germanVariants = [
|
||||
'Posteingang',
|
||||
'Eingang',
|
||||
'Eingangsmails',
|
||||
'Empfangen',
|
||||
];
|
||||
|
||||
testFolderMatches(germanVariants, StandardFolder.INBOX);
|
||||
});
|
||||
|
||||
it('matches Spanish variants', () => {
|
||||
const spanishVariants = [
|
||||
'Bandeja de entrada',
|
||||
'Entrada',
|
||||
'Correo entrante',
|
||||
'Recibidos',
|
||||
];
|
||||
|
||||
testFolderMatches(spanishVariants, StandardFolder.INBOX);
|
||||
});
|
||||
|
||||
it('matches Portuguese variants', () => {
|
||||
const portugueseVariants = [
|
||||
'Caixa de entrada',
|
||||
'Entrada',
|
||||
'Correio de entrada',
|
||||
'Recebidos',
|
||||
];
|
||||
|
||||
testFolderMatches(portugueseVariants, StandardFolder.INBOX);
|
||||
});
|
||||
|
||||
it('matches Italian variants', () => {
|
||||
const italianVariants = [
|
||||
'Posta in arrivo',
|
||||
'Arrivo',
|
||||
'Casella postale',
|
||||
'Ricevuti',
|
||||
];
|
||||
|
||||
testFolderMatches(italianVariants, StandardFolder.INBOX);
|
||||
});
|
||||
|
||||
it('matches Korean variants', () => {
|
||||
const koreanVariants = ['받은편지함', '수신함', '받은메일'];
|
||||
|
||||
testFolderMatches(koreanVariants, StandardFolder.INBOX);
|
||||
});
|
||||
|
||||
it('matches Japanese variants', () => {
|
||||
const japaneseVariants = ['受信トレイ', '受信箱', '受信メール'];
|
||||
|
||||
testFolderMatches(japaneseVariants, StandardFolder.INBOX);
|
||||
});
|
||||
|
||||
it('matches Polish variants', () => {
|
||||
const polishVariants = [
|
||||
'Odebrane',
|
||||
'Skrzynka odbiorcza',
|
||||
'Wiadomości przychodzące',
|
||||
];
|
||||
|
||||
testFolderMatches(polishVariants, StandardFolder.INBOX);
|
||||
});
|
||||
|
||||
it('matches Russian variants', () => {
|
||||
const russianVariants = [
|
||||
'Входящие',
|
||||
'Папка входящих',
|
||||
'Полученные сообщения',
|
||||
'Полученные письма',
|
||||
];
|
||||
|
||||
testFolderMatches(russianVariants, StandardFolder.INBOX);
|
||||
});
|
||||
|
||||
it('matches Gmail special folder', () => {
|
||||
const gmailVariants = ['[Gmail]/Inbox', '[Gmail]\\Inbox'];
|
||||
|
||||
testFolderMatches(gmailVariants, StandardFolder.INBOX);
|
||||
});
|
||||
});
|
||||
|
||||
describe('DRAFTS folder detection', () => {
|
||||
it('matches English variants', () => {
|
||||
const englishVariants = [
|
||||
'Drafts',
|
||||
'Draft',
|
||||
'Draft Items',
|
||||
'Draft Mail',
|
||||
'Draft Messages',
|
||||
];
|
||||
|
||||
testFolderMatches(englishVariants, StandardFolder.DRAFTS);
|
||||
});
|
||||
|
||||
it('matches French variants', () => {
|
||||
const frenchVariants = ['Brouillons', 'Éléments brouillons'];
|
||||
|
||||
testFolderMatches(frenchVariants, StandardFolder.DRAFTS);
|
||||
});
|
||||
|
||||
it('matches German variants', () => {
|
||||
const germanVariants = ['Entwürfe', 'Entwurf'];
|
||||
|
||||
testFolderMatches(germanVariants, StandardFolder.DRAFTS);
|
||||
});
|
||||
|
||||
it('matches Spanish variants', () => {
|
||||
const spanishVariants = ['Borradores', 'Elementos borrador'];
|
||||
|
||||
testFolderMatches(spanishVariants, StandardFolder.DRAFTS);
|
||||
});
|
||||
|
||||
it('matches Portuguese variants', () => {
|
||||
const portugueseVariants = ['Rascunhos', 'Itens rascunho'];
|
||||
|
||||
testFolderMatches(portugueseVariants, StandardFolder.DRAFTS);
|
||||
});
|
||||
|
||||
it('matches Italian variants', () => {
|
||||
const italianVariants = ['Bozze', 'Bozze salvate'];
|
||||
|
||||
testFolderMatches(italianVariants, StandardFolder.DRAFTS);
|
||||
});
|
||||
|
||||
it('matches Korean variants', () => {
|
||||
const koreanVariants = ['임시보관함', '초안'];
|
||||
|
||||
testFolderMatches(koreanVariants, StandardFolder.DRAFTS);
|
||||
});
|
||||
|
||||
it('matches Japanese variants', () => {
|
||||
const japaneseVariants = ['下書き', '草稿'];
|
||||
|
||||
testFolderMatches(japaneseVariants, StandardFolder.DRAFTS);
|
||||
});
|
||||
|
||||
it('matches Polish variants', () => {
|
||||
const polishVariants = ['Wersje robocze', 'Szkice'];
|
||||
|
||||
testFolderMatches(polishVariants, StandardFolder.DRAFTS);
|
||||
});
|
||||
|
||||
it('matches Russian variants', () => {
|
||||
const russianVariants = [
|
||||
'Черновики',
|
||||
'Черновые сообщения',
|
||||
'Неотправленные',
|
||||
];
|
||||
|
||||
testFolderMatches(russianVariants, StandardFolder.DRAFTS);
|
||||
});
|
||||
|
||||
it('matches Gmail special folder', () => {
|
||||
const gmailVariants = ['[Gmail]/Drafts', '[Gmail]\\Drafts'];
|
||||
|
||||
testFolderMatches(gmailVariants, StandardFolder.DRAFTS);
|
||||
});
|
||||
});
|
||||
|
||||
describe('SENT folder detection', () => {
|
||||
it('matches English variants', () => {
|
||||
const englishVariants = [
|
||||
'Sent',
|
||||
'Sent Items',
|
||||
'Sent Mail',
|
||||
'Sent Messages',
|
||||
'sent-elements',
|
||||
];
|
||||
|
||||
testFolderMatches(englishVariants, StandardFolder.SENT);
|
||||
});
|
||||
|
||||
it('matches French variants', () => {
|
||||
const frenchVariants = ['Envoyés', 'Éléments envoyés', 'Objets envoyés'];
|
||||
|
||||
testFolderMatches(frenchVariants, StandardFolder.SENT);
|
||||
});
|
||||
|
||||
it('matches German variants', () => {
|
||||
const germanVariants = ['Gesendet', 'Gesendete Elemente'];
|
||||
|
||||
testFolderMatches(germanVariants, StandardFolder.SENT);
|
||||
});
|
||||
|
||||
it('matches Spanish variants', () => {
|
||||
const spanishVariants = ['Enviados', 'Elementos enviados'];
|
||||
|
||||
testFolderMatches(spanishVariants, StandardFolder.SENT);
|
||||
});
|
||||
|
||||
it('matches Portuguese variants', () => {
|
||||
const portugueseVariants = ['Itens enviados'];
|
||||
|
||||
testFolderMatches(portugueseVariants, StandardFolder.SENT);
|
||||
});
|
||||
|
||||
it('matches Italian variants', () => {
|
||||
const italianVariants = ['Posta inviata', 'Inviati'];
|
||||
|
||||
testFolderMatches(italianVariants, StandardFolder.SENT);
|
||||
});
|
||||
|
||||
it('matches Korean variant', () => {
|
||||
const koreanVariants = ['보낸편지함'];
|
||||
|
||||
testFolderMatches(koreanVariants, StandardFolder.SENT);
|
||||
});
|
||||
|
||||
it('matches Japanese variants', () => {
|
||||
const japaneseVariants = ['送信済みメール', '送信済み'];
|
||||
|
||||
testFolderMatches(japaneseVariants, StandardFolder.SENT);
|
||||
});
|
||||
|
||||
it('matches Polish variants', () => {
|
||||
const polishVariants = ['Wysłane', 'Elementy wysłane'];
|
||||
|
||||
testFolderMatches(polishVariants, StandardFolder.SENT);
|
||||
});
|
||||
|
||||
it('matches Russian variants', () => {
|
||||
const russianVariants = [
|
||||
'Отправленные',
|
||||
'Отправленные письма',
|
||||
'Отправленные сообщения',
|
||||
'Исходящие',
|
||||
];
|
||||
|
||||
testFolderMatches(russianVariants, StandardFolder.SENT);
|
||||
});
|
||||
|
||||
it('matches Gmail special folder', () => {
|
||||
const gmailVariants = ['[Gmail]/Sent Mail', '[Gmail]\\Sent Mail'];
|
||||
|
||||
testFolderMatches(gmailVariants, StandardFolder.SENT);
|
||||
});
|
||||
|
||||
it('does not match unrelated folders', () => {
|
||||
const unrelatedFolders = [
|
||||
'Inbox',
|
||||
'Drafts',
|
||||
'Trash',
|
||||
'Archive',
|
||||
'Junk',
|
||||
'Important',
|
||||
'RandomFolder',
|
||||
];
|
||||
|
||||
unrelatedFolders.forEach((folder) => {
|
||||
const result = getStandardFolderByRegex(folder);
|
||||
|
||||
expect(result).not.toBe(StandardFolder.SENT);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('TRASH folder detection', () => {
|
||||
it('matches English variants', () => {
|
||||
const englishVariants = [
|
||||
'Trash',
|
||||
'Deleted Items',
|
||||
'Deleted Messages',
|
||||
'Bin',
|
||||
'Recycle Bin',
|
||||
];
|
||||
|
||||
testFolderMatches(englishVariants, StandardFolder.TRASH);
|
||||
});
|
||||
|
||||
it('matches French variants', () => {
|
||||
const frenchVariants = ['Corbeille', 'Supprimés', 'Éléments supprimés'];
|
||||
|
||||
testFolderMatches(frenchVariants, StandardFolder.TRASH);
|
||||
});
|
||||
|
||||
it('matches German variants', () => {
|
||||
const germanVariants = ['Gelöscht', 'Gelöschte Elemente', 'Papierkorb'];
|
||||
|
||||
testFolderMatches(germanVariants, StandardFolder.TRASH);
|
||||
});
|
||||
|
||||
it('matches Spanish variants', () => {
|
||||
const spanishVariants = [
|
||||
'Papelera',
|
||||
'Eliminados',
|
||||
'Elementos eliminados',
|
||||
];
|
||||
|
||||
testFolderMatches(spanishVariants, StandardFolder.TRASH);
|
||||
});
|
||||
|
||||
it('matches Portuguese variants', () => {
|
||||
const portugueseVariants = ['Lixeira', 'Itens excluídos'];
|
||||
|
||||
testFolderMatches(portugueseVariants, StandardFolder.TRASH);
|
||||
});
|
||||
|
||||
it('matches Italian variants', () => {
|
||||
const italianVariants = ['Cestino', 'Posta eliminata', 'Eliminati'];
|
||||
|
||||
testFolderMatches(italianVariants, StandardFolder.TRASH);
|
||||
});
|
||||
|
||||
it('matches Korean variants', () => {
|
||||
const koreanVariants = ['휴지통', '삭제된편지함'];
|
||||
|
||||
testFolderMatches(koreanVariants, StandardFolder.TRASH);
|
||||
});
|
||||
|
||||
it('matches Japanese variants', () => {
|
||||
const japaneseVariants = ['ごみ箱', '削除済み', '削除済みメール'];
|
||||
|
||||
testFolderMatches(japaneseVariants, StandardFolder.TRASH);
|
||||
});
|
||||
|
||||
it('matches Polish variants', () => {
|
||||
const polishVariants = ['Kosz', 'Usunięte', 'Elementy usunięte'];
|
||||
|
||||
testFolderMatches(polishVariants, StandardFolder.TRASH);
|
||||
});
|
||||
|
||||
it('matches Russian variants', () => {
|
||||
const russianVariants = ['Удалённые', 'Корзина', 'Удалённые сообщения'];
|
||||
|
||||
testFolderMatches(russianVariants, StandardFolder.TRASH);
|
||||
});
|
||||
|
||||
it('matches Gmail special folder', () => {
|
||||
const gmailVariants = ['[Gmail]/Trash', '[Gmail]\\Trash'];
|
||||
|
||||
testFolderMatches(gmailVariants, StandardFolder.TRASH);
|
||||
});
|
||||
});
|
||||
|
||||
describe('JUNK/SPAM folder detection', () => {
|
||||
it('matches English variants', () => {
|
||||
const englishVariants = [
|
||||
'Spam',
|
||||
'Junk Mail',
|
||||
'Junk Messages',
|
||||
'Bulk Mail',
|
||||
'Bulk Messages',
|
||||
];
|
||||
|
||||
testFolderMatches(englishVariants, StandardFolder.JUNK);
|
||||
});
|
||||
|
||||
it('matches French variants', () => {
|
||||
const frenchVariants = ['Indésirables', 'Courrier indésirable', 'Spam'];
|
||||
|
||||
testFolderMatches(frenchVariants, StandardFolder.JUNK);
|
||||
});
|
||||
|
||||
it('matches German variants', () => {
|
||||
const germanVariants = ['Spam', 'Junk Mail', 'Unerwünscht'];
|
||||
|
||||
testFolderMatches(germanVariants, StandardFolder.JUNK);
|
||||
});
|
||||
|
||||
it('matches Spanish variants', () => {
|
||||
const spanishVariants = ['Spam', 'Correo basura', 'No deseado'];
|
||||
|
||||
testFolderMatches(spanishVariants, StandardFolder.JUNK);
|
||||
});
|
||||
|
||||
it('matches Portuguese variants', () => {
|
||||
const portugueseVariants = ['Spam', 'Lixo eletrônico', 'Indesejados'];
|
||||
|
||||
testFolderMatches(portugueseVariants, StandardFolder.JUNK);
|
||||
});
|
||||
|
||||
it('matches Italian variants', () => {
|
||||
const italianVariants = ['Spam', 'Posta indesiderata', 'Indesiderata'];
|
||||
|
||||
testFolderMatches(italianVariants, StandardFolder.JUNK);
|
||||
});
|
||||
|
||||
it('matches Korean variants', () => {
|
||||
const koreanVariants = ['스팸', '정크메일'];
|
||||
|
||||
testFolderMatches(koreanVariants, StandardFolder.JUNK);
|
||||
});
|
||||
|
||||
it('matches Japanese variants', () => {
|
||||
const japaneseVariants = ['スパム', '迷惑メール'];
|
||||
|
||||
testFolderMatches(japaneseVariants, StandardFolder.JUNK);
|
||||
});
|
||||
|
||||
it('matches Polish variants', () => {
|
||||
const polishVariants = ['Spam', 'Niechciane', 'Śmieci'];
|
||||
|
||||
testFolderMatches(polishVariants, StandardFolder.JUNK);
|
||||
});
|
||||
|
||||
it('matches Russian variants', () => {
|
||||
const russianVariants = ['Спам', 'Нежелательные', 'Мусор'];
|
||||
|
||||
testFolderMatches(russianVariants, StandardFolder.JUNK);
|
||||
});
|
||||
|
||||
it('matches Gmail special folder', () => {
|
||||
const gmailVariants = ['[Gmail]/Spam', '[Gmail]\\Spam'];
|
||||
|
||||
testFolderMatches(gmailVariants, StandardFolder.JUNK);
|
||||
});
|
||||
});
|
||||
});
|
||||
+7
-40
@@ -1,50 +1,17 @@
|
||||
import { type ListResponse } from 'imapflow';
|
||||
|
||||
import { StandardFolder } from 'src/modules/messaging/message-import-manager/drivers/types/standard-folder';
|
||||
import { getStandardFolderByRegex } from 'src/modules/messaging/message-import-manager/drivers/utils/get-standard-folder-by-regex';
|
||||
|
||||
export function getImapSentFolderCandidatesByRegex(
|
||||
list: ListResponse[],
|
||||
): string[] {
|
||||
const sentFolderPattern = new RegExp(
|
||||
[
|
||||
// EN
|
||||
'sent([\\s_-]?(items|mail|messages|elements))?',
|
||||
// FR
|
||||
'envoy[éê]s?',
|
||||
'[ée]l[ée]ments[\\s_-]?envoy[éê]s',
|
||||
// DE
|
||||
'gesendet',
|
||||
'gesendete[\\s_-]?elemente',
|
||||
// ES
|
||||
'enviados?',
|
||||
'elementos[\\s_-]?enviados',
|
||||
// PT
|
||||
'itens[\\s_-]?enviados',
|
||||
// IT
|
||||
'posta[\\s_-]?inviata',
|
||||
'inviati',
|
||||
// KO
|
||||
'보낸편지함',
|
||||
// JA
|
||||
'送信済みメール',
|
||||
'送信済み',
|
||||
// PL
|
||||
'elementy[\\s_-]?wysłane',
|
||||
'wysłane',
|
||||
// RU
|
||||
'отправленные',
|
||||
'отправленные[\\s_-]?(сообщения|письма)?',
|
||||
'исходящие',
|
||||
// GMAIL
|
||||
'\\[gmail\\][\\/]+sent[\\s_-]?mail',
|
||||
]
|
||||
.map((s) => `(${s})`)
|
||||
.join('|'),
|
||||
'i',
|
||||
);
|
||||
|
||||
const regexCandidateFolders = [];
|
||||
const regexCandidateFolders: string[] = [];
|
||||
|
||||
for (const folder of list) {
|
||||
if (sentFolderPattern.test(folder.path)) {
|
||||
const standardFolder = getStandardFolderByRegex(folder.path);
|
||||
|
||||
if (standardFolder === StandardFolder.SENT) {
|
||||
regexCandidateFolders.push(folder.path);
|
||||
}
|
||||
}
|
||||
|
||||
+2
@@ -27,6 +27,7 @@ import { MicrosoftGetMessageListService } from './services/microsoft-get-message
|
||||
MicrosoftClientProvider,
|
||||
MicrosoftGetMessageListService,
|
||||
MicrosoftGetMessagesService,
|
||||
|
||||
MicrosoftFetchByBatchService,
|
||||
MicrosoftHandleErrorService,
|
||||
MicrosoftOAuth2ClientManagerService,
|
||||
@@ -35,6 +36,7 @@ import { MicrosoftGetMessageListService } from './services/microsoft-get-message
|
||||
MicrosoftGetMessageListService,
|
||||
MicrosoftClientProvider,
|
||||
MicrosoftGetMessagesService,
|
||||
MicrosoftHandleErrorService,
|
||||
],
|
||||
})
|
||||
export class MessagingMicrosoftDriverModule {}
|
||||
|
||||
+12
@@ -66,6 +66,9 @@ xdescribe('Microsoft dev tests : get message list service', () => {
|
||||
id: 'inbox-folder-id',
|
||||
name: MessageFolderName.INBOX,
|
||||
syncCursor: 'inbox-sync-cursor',
|
||||
isSynced: false,
|
||||
isSentFolder: false,
|
||||
externalId: null,
|
||||
},
|
||||
],
|
||||
});
|
||||
@@ -91,6 +94,9 @@ xdescribe('Microsoft dev tests : get message list service', () => {
|
||||
id: 'inbox-folder-id',
|
||||
name: MessageFolderName.INBOX,
|
||||
syncCursor: 'inbox-sync-cursor',
|
||||
isSynced: false,
|
||||
isSentFolder: false,
|
||||
externalId: null,
|
||||
},
|
||||
],
|
||||
}),
|
||||
@@ -107,6 +113,9 @@ xdescribe('Microsoft dev tests : get message list service', () => {
|
||||
id: 'inbox-folder-id',
|
||||
name: MessageFolderName.INBOX,
|
||||
syncCursor: syncCursor,
|
||||
isSynced: false,
|
||||
isSentFolder: false,
|
||||
externalId: null,
|
||||
},
|
||||
],
|
||||
});
|
||||
@@ -127,6 +136,9 @@ xdescribe('Microsoft dev tests : get message list service', () => {
|
||||
id: 'inbox-folder-id',
|
||||
name: MessageFolderName.INBOX,
|
||||
syncCursor: 'invalid-syncCursor',
|
||||
isSynced: false,
|
||||
isSentFolder: false,
|
||||
externalId: null,
|
||||
},
|
||||
],
|
||||
}),
|
||||
|
||||
+10
-46
@@ -6,9 +6,7 @@ import {
|
||||
type PageIteratorCallback,
|
||||
} from '@microsoft/microsoft-graph-client';
|
||||
import { isNonEmptyString } from '@sniptt/guards';
|
||||
import { v4 } from 'uuid';
|
||||
|
||||
import { TwentyORMManager } from 'src/engine/twenty-orm/twenty-orm.manager';
|
||||
import { type ConnectedAccountWorkspaceEntity } from 'src/modules/connected-account/standard-objects/connected-account.workspace-entity';
|
||||
import { type MessageFolderWorkspaceEntity } from 'src/modules/messaging/common/standard-objects/message-folder.workspace-entity';
|
||||
import {
|
||||
@@ -17,7 +15,6 @@ import {
|
||||
} from 'src/modules/messaging/message-import-manager/drivers/exceptions/message-import-driver.exception';
|
||||
import { MicrosoftClientProvider } from 'src/modules/messaging/message-import-manager/drivers/microsoft/providers/microsoft-client.provider';
|
||||
import { MicrosoftHandleErrorService } from 'src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-handle-error.service';
|
||||
import { MessageFolderName } from 'src/modules/messaging/message-import-manager/drivers/microsoft/types/folders';
|
||||
import { isAccessTokenRefreshingError } from 'src/modules/messaging/message-import-manager/drivers/microsoft/utils/is-access-token-refreshing-error.utils';
|
||||
import { type GetMessageListsArgs } from 'src/modules/messaging/message-import-manager/types/get-message-lists-args.type';
|
||||
import {
|
||||
@@ -34,7 +31,6 @@ export class MicrosoftGetMessageListService {
|
||||
constructor(
|
||||
private readonly microsoftClientProvider: MicrosoftClientProvider,
|
||||
private readonly microsoftHandleErrorService: MicrosoftHandleErrorService,
|
||||
private readonly twentyORMManager: TwentyORMManager,
|
||||
) {}
|
||||
|
||||
public async getMessageLists({
|
||||
@@ -45,46 +41,10 @@ export class MicrosoftGetMessageListService {
|
||||
const result: GetMessageListsResponse = [];
|
||||
|
||||
if (messageFolders.length === 0) {
|
||||
// permanent solution:
|
||||
// throw new MessageImportDriverException(
|
||||
// `Message channel ${messageChannel.id} has no message folders`,
|
||||
// MessageImportDriverExceptionCode.NOT_FOUND,
|
||||
// );
|
||||
|
||||
// temporary solution: TODO: remove this once we have a permanent solution
|
||||
// if no folders exist, most probably a first time sync for microsoft
|
||||
// so we create the folders INBOX and SENTITEMS
|
||||
// and fill the INBOX with the previous sync cursor
|
||||
// and for sentitms, we do the full message list fetch
|
||||
// console.warn(
|
||||
// `Message channel ${messageChannel.id} has no message folders, most probably a first time`,
|
||||
// );
|
||||
|
||||
const messageFolderRepository =
|
||||
await this.twentyORMManager.getRepository<MessageFolderWorkspaceEntity>(
|
||||
'messageFolder',
|
||||
);
|
||||
|
||||
const newFolder = await messageFolderRepository.save({
|
||||
id: v4(),
|
||||
messageChannelId: messageChannel.id,
|
||||
name: MessageFolderName.INBOX,
|
||||
syncCursor: messageChannel.syncCursor,
|
||||
});
|
||||
|
||||
const response = await this.getMessageList(connectedAccount, {
|
||||
name: MessageFolderName.INBOX,
|
||||
syncCursor: messageChannel.syncCursor,
|
||||
});
|
||||
|
||||
result.push({
|
||||
...response,
|
||||
folderId: newFolder.id,
|
||||
});
|
||||
|
||||
// we are ok with not synchronizing the legacy connected microsoft accounts.
|
||||
// so we return an empty array.
|
||||
return result;
|
||||
throw new MessageImportDriverException(
|
||||
`Message channel ${messageChannel.id} has no message folders`,
|
||||
MessageImportDriverExceptionCode.NOT_FOUND,
|
||||
);
|
||||
}
|
||||
|
||||
for (const folder of messageFolders) {
|
||||
@@ -104,7 +64,10 @@ export class MicrosoftGetMessageListService {
|
||||
ConnectedAccountWorkspaceEntity,
|
||||
'provider' | 'refreshToken' | 'id'
|
||||
>,
|
||||
messageFolder: Pick<MessageFolderWorkspaceEntity, 'name' | 'syncCursor'>,
|
||||
messageFolder: Pick<
|
||||
MessageFolderWorkspaceEntity,
|
||||
'name' | 'syncCursor' | 'externalId'
|
||||
>,
|
||||
): Promise<GetOneMessageListResponse> {
|
||||
const messageExternalIds: string[] = [];
|
||||
const messageExternalIdsToDelete: string[] = [];
|
||||
@@ -112,9 +75,10 @@ export class MicrosoftGetMessageListService {
|
||||
const microsoftClient =
|
||||
await this.microsoftClientProvider.getMicrosoftClient(connectedAccount);
|
||||
|
||||
const folderId = messageFolder.externalId || messageFolder.name;
|
||||
const apiUrl = isNonEmptyString(messageFolder.syncCursor)
|
||||
? messageFolder.syncCursor
|
||||
: `/me/mailfolders/${messageFolder.name}/messages/delta?$select=id`;
|
||||
: `/me/mailfolders/${folderId}/messages/delta?$select=id`;
|
||||
|
||||
const response: PageCollection = await microsoftClient
|
||||
.api(apiUrl)
|
||||
|
||||
+7
@@ -0,0 +1,7 @@
|
||||
export enum StandardFolder {
|
||||
INBOX = 'inbox',
|
||||
DRAFTS = 'drafts',
|
||||
SENT = 'sent',
|
||||
TRASH = 'trash',
|
||||
JUNK = 'junk',
|
||||
}
|
||||
+225
@@ -0,0 +1,225 @@
|
||||
import { StandardFolder } from 'src/modules/messaging/message-import-manager/drivers/types/standard-folder';
|
||||
|
||||
const FOLDER_REGEX_PATTERNS: Record<StandardFolder, string[]> = {
|
||||
[StandardFolder.INBOX]: [
|
||||
// EN
|
||||
'inbox',
|
||||
'^mail$',
|
||||
'^messages?$',
|
||||
'received',
|
||||
// FR
|
||||
'boîte[\\s_-]?de[\\s_-]?réception',
|
||||
'courrier[\\s_-]?entrant',
|
||||
'messages[\\s_-]?reçus',
|
||||
'réception',
|
||||
// DE
|
||||
'posteingang',
|
||||
'eingang',
|
||||
'eingangsmails?',
|
||||
'empfangen',
|
||||
// ES
|
||||
'bandeja[\\s_-]?de[\\s_-]?entrada',
|
||||
'entrada',
|
||||
'correo[\\s_-]?entrante',
|
||||
'recibidos?',
|
||||
// PT
|
||||
'caixa[\\s_-]?de[\\s_-]?entrada',
|
||||
'entrada',
|
||||
'correio[\\s_-]?de[\\s_-]?entrada',
|
||||
'recebidos?',
|
||||
// IT
|
||||
'posta[\\s_-]?in[\\s_-]?arrivo',
|
||||
'arrivo',
|
||||
'casella[\\s_-]?postale',
|
||||
'ricevuti',
|
||||
// KO
|
||||
'받은편지함',
|
||||
'수신함',
|
||||
'받은메일',
|
||||
// JA
|
||||
'受信トレイ',
|
||||
'受信箱',
|
||||
'受信メール',
|
||||
// PL
|
||||
'odebrane',
|
||||
'skrzynka[\\s_-]?odbiorcza',
|
||||
'wiadomości[\\s_-]?przychodzące',
|
||||
// RU
|
||||
'входящие',
|
||||
'папка[\\s_-]?входящих',
|
||||
'полученные[\\s_-]?(сообщения|письма)?',
|
||||
// GMAIL
|
||||
'\\[gmail\\][\\/]+inbox',
|
||||
],
|
||||
[StandardFolder.DRAFTS]: [
|
||||
// EN
|
||||
'drafts?',
|
||||
'draft[\\s_-]?(items|mail|messages|elements)?',
|
||||
// FR
|
||||
'brouillons?',
|
||||
'[ée]l[ée]ments[\\s_-]?brouillons?',
|
||||
// DE
|
||||
'entwürfe',
|
||||
'entwurf',
|
||||
// ES
|
||||
'borradores?',
|
||||
'elementos[\\s_-]?borrador',
|
||||
// PT
|
||||
'rascunhos?',
|
||||
'itens[\\s_-]?rascunho',
|
||||
// IT
|
||||
'bozze',
|
||||
'bozze[\\s_-]?salvate',
|
||||
// KO
|
||||
'임시보관함',
|
||||
'초안',
|
||||
// JA
|
||||
'下書き',
|
||||
'草稿',
|
||||
// PL
|
||||
'wersje[\\s_-]?robocze',
|
||||
'szkice',
|
||||
// RU
|
||||
'черновики',
|
||||
'черновые[\\s_-]?(сообщения|письма)?',
|
||||
'неотправленные',
|
||||
// GMAIL
|
||||
'\\[gmail\\][\\/]+drafts',
|
||||
],
|
||||
[StandardFolder.SENT]: [
|
||||
// EN
|
||||
'sent([\\s_-]?(items|mail|messages|elements))?',
|
||||
// FR
|
||||
'envoy[éê]s?',
|
||||
'[ée]l[ée]ments[\\s_-]?envoy[éê]s',
|
||||
// DE
|
||||
'gesendet',
|
||||
'gesendete[\\s_-]?elemente',
|
||||
// ES
|
||||
'enviados?',
|
||||
'elementos[\\s_-]?enviados',
|
||||
// PT
|
||||
'itens[\\s_-]?enviados',
|
||||
// IT
|
||||
'posta[\\s_-]?inviata',
|
||||
'inviati',
|
||||
// KO
|
||||
'보낸편지함',
|
||||
// JA
|
||||
'送信済みメール',
|
||||
'送信済み',
|
||||
// PL
|
||||
'elementy[\\s_-]?wysłane',
|
||||
'wysłane',
|
||||
// RU
|
||||
'отправленные',
|
||||
'отправленные[\\s_-]?(сообщения|письма)?',
|
||||
'исходящие',
|
||||
// GMAIL
|
||||
'\\[gmail\\][\\/]+sent[\\s_-]?mail',
|
||||
],
|
||||
[StandardFolder.TRASH]: [
|
||||
// EN
|
||||
'trash',
|
||||
'deleted[\\s_-]?(items|messages|mail)?',
|
||||
'bin',
|
||||
'recycle[\\s_-]?bin',
|
||||
// FR
|
||||
'corbeille',
|
||||
'supprim[ée]s',
|
||||
'[ée]l[ée]ments[\\s_-]?supprim[ée]s',
|
||||
// DE
|
||||
'gelöscht',
|
||||
'gelöschte[\\s_-]?elemente',
|
||||
'papierkorb',
|
||||
// ES
|
||||
'papelera',
|
||||
'eliminados?',
|
||||
'elementos[\\s_-]?eliminados',
|
||||
// PT
|
||||
'lixeira',
|
||||
'itens[\\s_-]?excluídos',
|
||||
// IT
|
||||
'cestino',
|
||||
'posta[\\s_-]?eliminata',
|
||||
'eliminati',
|
||||
// KO
|
||||
'휴지통',
|
||||
'삭제된편지함',
|
||||
// JA
|
||||
'ごみ箱',
|
||||
'削除済み',
|
||||
'削除済みメール',
|
||||
// PL
|
||||
'kosz',
|
||||
'usunięte',
|
||||
'elementy[\\s_-]?usunięte',
|
||||
// RU
|
||||
'удалённые',
|
||||
'корзина',
|
||||
'удалённые[\\s_-]?(сообщения|письма)?',
|
||||
// GMAIL
|
||||
'\\[gmail\\][\\/]+trash',
|
||||
],
|
||||
[StandardFolder.JUNK]: [
|
||||
// EN
|
||||
'spam',
|
||||
'junk[\\s_-]?(mail|messages|email)?',
|
||||
'bulk[\\s_-]?(mail|messages)?',
|
||||
// FR
|
||||
'indésirables',
|
||||
'courrier[\\s_-]?indésirable',
|
||||
'spam',
|
||||
// DE
|
||||
'spam',
|
||||
'junk[\\s_-]?mail',
|
||||
'unerwünscht',
|
||||
// ES
|
||||
'spam',
|
||||
'correo[\\s_-]?basura',
|
||||
'no[\\s_-]?deseado',
|
||||
// PT
|
||||
'spam',
|
||||
'lixo[\\s_-]?eletrônico',
|
||||
'indesejados',
|
||||
// IT
|
||||
'spam',
|
||||
'posta[\\s_-]?indesiderata',
|
||||
'indesiderata',
|
||||
// KO
|
||||
'스팸',
|
||||
'정크메일',
|
||||
// JA
|
||||
'スパム',
|
||||
'迷惑メール',
|
||||
// PL
|
||||
'spam',
|
||||
'niechciane',
|
||||
'śmieci',
|
||||
// RU
|
||||
'спам',
|
||||
'нежелательные',
|
||||
'мусор',
|
||||
// GMAIL
|
||||
'\\[gmail\\][\\/]+spam',
|
||||
],
|
||||
};
|
||||
|
||||
const CACHED_REGEX_PATTERNS = Object.fromEntries(
|
||||
Object.entries(FOLDER_REGEX_PATTERNS).map(([standardFolder, patterns]) => [
|
||||
standardFolder,
|
||||
new RegExp(patterns.map((s) => `(${s})`).join('|'), 'i'),
|
||||
]),
|
||||
);
|
||||
|
||||
export function getStandardFolderByRegex(
|
||||
folderName: string,
|
||||
): StandardFolder | null {
|
||||
for (const [standardFolder, regex] of Object.entries(CACHED_REGEX_PATTERNS)) {
|
||||
if (regex.test(folderName)) {
|
||||
return standardFolder as StandardFolder;
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
+2
@@ -39,6 +39,7 @@ import { MessagingMessagesImportService } from 'src/modules/messaging/message-im
|
||||
import { MessagingSaveMessagesAndEnqueueContactCreationService } from 'src/modules/messaging/message-import-manager/services/messaging-save-messages-and-enqueue-contact-creation.service';
|
||||
import { MessagingSendMessageService } from 'src/modules/messaging/message-import-manager/services/messaging-send-message.service';
|
||||
import { MessageParticipantManagerModule } from 'src/modules/messaging/message-participant-manager/message-participant-manager.module';
|
||||
import { MessagingFolderSyncManagerModule } from 'src/modules/messaging/message-folder-manager/messaging-folder-sync-manager.module';
|
||||
import { MessagingMonitoringModule } from 'src/modules/messaging/monitoring/messaging-monitoring.module';
|
||||
@Module({
|
||||
imports: [
|
||||
@@ -57,6 +58,7 @@ import { MessagingMonitoringModule } from 'src/modules/messaging/monitoring/mess
|
||||
EmailAliasManagerModule,
|
||||
FeatureFlagModule,
|
||||
MessageParticipantManagerModule,
|
||||
MessagingFolderSyncManagerModule,
|
||||
MessagingMonitoringModule,
|
||||
MessagingMessageCleanerModule,
|
||||
WorkspaceEventEmitterModule,
|
||||
|
||||
+10
-3
@@ -3,6 +3,7 @@ import { Injectable } from '@nestjs/common';
|
||||
import { ConnectedAccountProvider } from 'twenty-shared/types';
|
||||
|
||||
import { type MessageChannelWorkspaceEntity } from 'src/modules/messaging/common/standard-objects/message-channel.workspace-entity';
|
||||
import { MessageFolderWorkspaceEntity } from 'src/modules/messaging/common/standard-objects/message-folder.workspace-entity';
|
||||
import { GmailGetMessageListService } from 'src/modules/messaging/message-import-manager/drivers/gmail/services/gmail-get-message-list.service';
|
||||
import { ImapGetMessageListService } from 'src/modules/messaging/message-import-manager/drivers/imap/services/imap-get-message-list.service';
|
||||
import { MicrosoftGetMessageListService } from 'src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-get-message-list.service';
|
||||
@@ -12,6 +13,11 @@ import {
|
||||
} from 'src/modules/messaging/message-import-manager/exceptions/message-import.exception';
|
||||
import { type GetMessageListsResponse } from 'src/modules/messaging/message-import-manager/types/get-message-lists-response.type';
|
||||
|
||||
type MessageFolder = Pick<
|
||||
MessageFolderWorkspaceEntity,
|
||||
'name' | 'isSynced' | 'isSentFolder' | 'externalId' | 'syncCursor' | 'id'
|
||||
>;
|
||||
|
||||
@Injectable()
|
||||
export class MessagingGetMessageListService {
|
||||
constructor(
|
||||
@@ -22,25 +28,26 @@ export class MessagingGetMessageListService {
|
||||
|
||||
public async getMessageLists(
|
||||
messageChannel: MessageChannelWorkspaceEntity,
|
||||
messageFoldersToSync: MessageFolder[],
|
||||
): Promise<GetMessageListsResponse> {
|
||||
switch (messageChannel.connectedAccount.provider) {
|
||||
case ConnectedAccountProvider.GOOGLE:
|
||||
return await this.gmailGetMessageListService.getMessageLists({
|
||||
messageChannel,
|
||||
connectedAccount: messageChannel.connectedAccount,
|
||||
messageFolders: messageChannel.messageFolders,
|
||||
messageFolders: messageFoldersToSync,
|
||||
});
|
||||
case ConnectedAccountProvider.MICROSOFT:
|
||||
return this.microsoftGetMessageListService.getMessageLists({
|
||||
messageChannel,
|
||||
connectedAccount: messageChannel.connectedAccount,
|
||||
messageFolders: messageChannel.messageFolders,
|
||||
messageFolders: messageFoldersToSync,
|
||||
});
|
||||
case ConnectedAccountProvider.IMAP_SMTP_CALDAV: {
|
||||
return await this.imapGetMessageListService.getMessageLists({
|
||||
messageChannel,
|
||||
connectedAccount: messageChannel.connectedAccount,
|
||||
messageFolders: messageChannel.messageFolders,
|
||||
messageFolders: messageFoldersToSync,
|
||||
});
|
||||
}
|
||||
default:
|
||||
|
||||
+82
-40
@@ -9,6 +9,7 @@ import { MessageChannelSyncStatusService } from 'src/modules/messaging/common/se
|
||||
import { type MessageChannelWorkspaceEntity } from 'src/modules/messaging/common/standard-objects/message-channel.workspace-entity';
|
||||
import { type MessageFolderWorkspaceEntity } from 'src/modules/messaging/common/standard-objects/message-folder.workspace-entity';
|
||||
import { MessagingMessageCleanerService } from 'src/modules/messaging/message-cleaner/services/messaging-message-cleaner.service';
|
||||
import { SyncMessageFoldersService } from 'src/modules/messaging/message-folder-manager/services/sync-message-folders.service';
|
||||
import { MessagingAccountAuthenticationService } from 'src/modules/messaging/message-import-manager/services/messaging-account-authentication.service';
|
||||
import { MessagingCursorService } from 'src/modules/messaging/message-import-manager/services/messaging-cursor.service';
|
||||
import { MessagingGetMessageListService } from 'src/modules/messaging/message-import-manager/services/messaging-get-message-list.service';
|
||||
@@ -75,6 +76,18 @@ describe('MessagingMessageListFetchService', () => {
|
||||
delete: jest.fn().mockResolvedValue(undefined),
|
||||
};
|
||||
|
||||
const mockMessageFolderRepository = {
|
||||
find: jest.fn().mockResolvedValue([
|
||||
{
|
||||
id: 'inbox-folder-id',
|
||||
name: 'inbox',
|
||||
syncCursor: 'inbox-sync-cursor',
|
||||
messageChannelId: 'microsoft-message-channel-id',
|
||||
isSynced: true,
|
||||
},
|
||||
]),
|
||||
};
|
||||
|
||||
const module: TestingModule = await Test.createTestingModule({
|
||||
providers: [
|
||||
MessagingMessageListFetchService,
|
||||
@@ -88,41 +101,40 @@ describe('MessagingMessageListFetchService', () => {
|
||||
{
|
||||
provide: MessagingGetMessageListService,
|
||||
useValue: {
|
||||
getMessageLists: jest
|
||||
.fn()
|
||||
.mockImplementation(({ connectedAccount }) => {
|
||||
if (
|
||||
connectedAccount.provider === ConnectedAccountProvider.GOOGLE
|
||||
) {
|
||||
return [
|
||||
{
|
||||
messageExternalIds: [
|
||||
'external-id-existing-message-1',
|
||||
'external-id-google-message-1',
|
||||
'external-id-google-message-2',
|
||||
],
|
||||
nextSyncCursor: 'new-google-history-id',
|
||||
folderId: undefined,
|
||||
messageExternalIdsToDelete: [],
|
||||
previousSyncCursor: 'google-sync-cursor',
|
||||
},
|
||||
];
|
||||
} else {
|
||||
return [
|
||||
{
|
||||
messageExternalIds: [
|
||||
'external-id-existing-message-1',
|
||||
'external-id-new-message-1',
|
||||
'external-id-new-message-2',
|
||||
],
|
||||
nextSyncCursor: 'new-sync-cursor',
|
||||
folderId: 'inbox-folder-id',
|
||||
messageExternalIdsToDelete: [],
|
||||
previousSyncCursor: 'inbox-sync-cursor',
|
||||
},
|
||||
];
|
||||
}
|
||||
}),
|
||||
getMessageLists: jest.fn().mockImplementation((messageChannel) => {
|
||||
if (
|
||||
messageChannel.connectedAccount.provider ===
|
||||
ConnectedAccountProvider.GOOGLE
|
||||
) {
|
||||
return [
|
||||
{
|
||||
messageExternalIds: [
|
||||
'external-id-existing-message-1',
|
||||
'external-id-google-message-1',
|
||||
'external-id-google-message-2',
|
||||
],
|
||||
nextSyncCursor: 'new-google-history-id',
|
||||
folderId: undefined,
|
||||
messageExternalIdsToDelete: [],
|
||||
previousSyncCursor: 'google-sync-cursor',
|
||||
},
|
||||
];
|
||||
} else {
|
||||
return [
|
||||
{
|
||||
messageExternalIds: [
|
||||
'external-id-existing-message-1',
|
||||
'external-id-new-message-1',
|
||||
'external-id-new-message-2',
|
||||
],
|
||||
nextSyncCursor: 'new-sync-cursor',
|
||||
folderId: 'inbox-folder-id',
|
||||
messageExternalIdsToDelete: [],
|
||||
previousSyncCursor: 'inbox-sync-cursor',
|
||||
},
|
||||
];
|
||||
}
|
||||
}),
|
||||
},
|
||||
},
|
||||
{
|
||||
@@ -175,11 +187,17 @@ describe('MessagingMessageListFetchService', () => {
|
||||
{
|
||||
provide: TwentyORMManager,
|
||||
useValue: {
|
||||
getRepository: jest
|
||||
.fn()
|
||||
.mockResolvedValue(
|
||||
mockMessageChannelMessageAssociationRepository,
|
||||
),
|
||||
getDatasource: jest.fn().mockResolvedValue({
|
||||
manager: {},
|
||||
}),
|
||||
getRepository: jest.fn().mockImplementation((name) => {
|
||||
if (name === 'messageChannelMessageAssociation') {
|
||||
return mockMessageChannelMessageAssociationRepository;
|
||||
}
|
||||
if (name === 'messageFolder') {
|
||||
return mockMessageFolderRepository;
|
||||
}
|
||||
}),
|
||||
},
|
||||
},
|
||||
{
|
||||
@@ -207,6 +225,12 @@ describe('MessagingMessageListFetchService', () => {
|
||||
cleanWorkspaceThreads: jest.fn().mockResolvedValue(undefined),
|
||||
},
|
||||
},
|
||||
{
|
||||
provide: SyncMessageFoldersService,
|
||||
useValue: {
|
||||
syncMessageFolders: jest.fn().mockResolvedValue(undefined),
|
||||
},
|
||||
},
|
||||
],
|
||||
}).compile();
|
||||
|
||||
@@ -258,6 +282,15 @@ describe('MessagingMessageListFetchService', () => {
|
||||
refreshToken: 'new-microsoft-refresh-token',
|
||||
},
|
||||
},
|
||||
[
|
||||
{
|
||||
id: 'inbox-folder-id',
|
||||
name: 'inbox',
|
||||
syncCursor: 'inbox-sync-cursor',
|
||||
messageChannelId: 'microsoft-message-channel-id',
|
||||
isSynced: true,
|
||||
},
|
||||
],
|
||||
);
|
||||
|
||||
expect(twentyORMManager.getRepository).toHaveBeenCalledWith(
|
||||
@@ -308,6 +341,15 @@ describe('MessagingMessageListFetchService', () => {
|
||||
refreshToken: 'new-google-refresh-token',
|
||||
},
|
||||
},
|
||||
[
|
||||
{
|
||||
id: 'inbox-folder-id',
|
||||
name: 'inbox',
|
||||
syncCursor: 'inbox-sync-cursor',
|
||||
messageChannelId: 'microsoft-message-channel-id',
|
||||
isSynced: true,
|
||||
},
|
||||
],
|
||||
);
|
||||
|
||||
expect(twentyORMManager.getRepository).toHaveBeenCalledWith(
|
||||
|
||||
+26
-1
@@ -13,9 +13,11 @@ import { MessageChannelSyncStatusService } from 'src/modules/messaging/common/se
|
||||
import { type MessageChannelMessageAssociationWorkspaceEntity } from 'src/modules/messaging/common/standard-objects/message-channel-message-association.workspace-entity';
|
||||
import {
|
||||
MessageChannelSyncStage,
|
||||
type MessageChannelWorkspaceEntity,
|
||||
MessageChannelWorkspaceEntity,
|
||||
} from 'src/modules/messaging/common/standard-objects/message-channel.workspace-entity';
|
||||
import { MessageFolderWorkspaceEntity } from 'src/modules/messaging/common/standard-objects/message-folder.workspace-entity';
|
||||
import { MessagingMessageCleanerService } from 'src/modules/messaging/message-cleaner/services/messaging-message-cleaner.service';
|
||||
import { SyncMessageFoldersService } from 'src/modules/messaging/message-folder-manager/services/sync-message-folders.service';
|
||||
import { MessagingAccountAuthenticationService } from 'src/modules/messaging/message-import-manager/services/messaging-account-authentication.service';
|
||||
import { MessagingCursorService } from 'src/modules/messaging/message-import-manager/services/messaging-cursor.service';
|
||||
import { MessagingGetMessageListService } from 'src/modules/messaging/message-import-manager/services/messaging-get-message-list.service';
|
||||
@@ -41,6 +43,7 @@ export class MessagingMessageListFetchService {
|
||||
private readonly messagingCursorService: MessagingCursorService,
|
||||
private readonly messagingMessagesImportService: MessagingMessagesImportService,
|
||||
private readonly messagingAccountAuthenticationService: MessagingAccountAuthenticationService,
|
||||
private readonly syncMessageFoldersService: SyncMessageFoldersService,
|
||||
) {}
|
||||
|
||||
public async processMessageListFetch(
|
||||
@@ -74,9 +77,31 @@ export class MessagingMessageListFetchService {
|
||||
},
|
||||
};
|
||||
|
||||
const datasource = await this.twentyORMManager.getDatasource();
|
||||
|
||||
await this.syncMessageFoldersService.syncMessageFolders({
|
||||
workspaceId,
|
||||
messageChannelId: messageChannelWithFreshTokens.id,
|
||||
connectedAccount: messageChannelWithFreshTokens.connectedAccount,
|
||||
manager: datasource.manager,
|
||||
});
|
||||
|
||||
const messageFolderRepository =
|
||||
await this.twentyORMManager.getRepository<MessageFolderWorkspaceEntity>(
|
||||
'messageFolder',
|
||||
);
|
||||
|
||||
const messageFoldersToSync = await messageFolderRepository.find({
|
||||
where: {
|
||||
messageChannelId: messageChannel.id,
|
||||
isSynced: true,
|
||||
},
|
||||
});
|
||||
|
||||
const messageLists =
|
||||
await this.messagingGetMessageListService.getMessageLists(
|
||||
messageChannelWithFreshTokens,
|
||||
messageFoldersToSync,
|
||||
);
|
||||
|
||||
await this.cacheStorage.del(
|
||||
|
||||
+1
-1
@@ -10,6 +10,6 @@ export type GetMessageListsArgs = {
|
||||
>;
|
||||
messageFolders: Pick<
|
||||
MessageFolderWorkspaceEntity,
|
||||
'name' | 'syncCursor' | 'id'
|
||||
'name' | 'syncCursor' | 'id' | 'isSynced' | 'isSentFolder' | 'externalId'
|
||||
>[];
|
||||
};
|
||||
|
||||
Reference in New Issue
Block a user