diff --git a/packages/twenty-front/src/generated-metadata/graphql.ts b/packages/twenty-front/src/generated-metadata/graphql.ts index 2b63721f22..3cf53dc876 100644 --- a/packages/twenty-front/src/generated-metadata/graphql.ts +++ b/packages/twenty-front/src/generated-metadata/graphql.ts @@ -972,6 +972,7 @@ export enum FeatureFlagKey { IS_CORE_VIEW_SYNCING_ENABLED = 'IS_CORE_VIEW_SYNCING_ENABLED', IS_IMAP_SMTP_CALDAV_ENABLED = 'IS_IMAP_SMTP_CALDAV_ENABLED', IS_JSON_FILTER_ENABLED = 'IS_JSON_FILTER_ENABLED', + IS_MESSAGE_FOLDER_CONTROL_ENABLED = 'IS_MESSAGE_FOLDER_CONTROL_ENABLED', IS_MORPH_RELATION_ENABLED = 'IS_MORPH_RELATION_ENABLED', IS_PAGE_LAYOUT_ENABLED = 'IS_PAGE_LAYOUT_ENABLED', IS_POSTGRESQL_INTEGRATION_ENABLED = 'IS_POSTGRESQL_INTEGRATION_ENABLED', diff --git a/packages/twenty-front/src/generated/graphql.ts b/packages/twenty-front/src/generated/graphql.ts index 4a6daafd53..39d7feae44 100644 --- a/packages/twenty-front/src/generated/graphql.ts +++ b/packages/twenty-front/src/generated/graphql.ts @@ -936,6 +936,7 @@ export enum FeatureFlagKey { IS_CORE_VIEW_SYNCING_ENABLED = 'IS_CORE_VIEW_SYNCING_ENABLED', IS_IMAP_SMTP_CALDAV_ENABLED = 'IS_IMAP_SMTP_CALDAV_ENABLED', IS_JSON_FILTER_ENABLED = 'IS_JSON_FILTER_ENABLED', + IS_MESSAGE_FOLDER_CONTROL_ENABLED = 'IS_MESSAGE_FOLDER_CONTROL_ENABLED', IS_MORPH_RELATION_ENABLED = 'IS_MORPH_RELATION_ENABLED', IS_PAGE_LAYOUT_ENABLED = 'IS_PAGE_LAYOUT_ENABLED', IS_POSTGRESQL_INTEGRATION_ENABLED = 'IS_POSTGRESQL_INTEGRATION_ENABLED', diff --git a/packages/twenty-front/src/modules/accounts/types/MessageChannel.ts b/packages/twenty-front/src/modules/accounts/types/MessageChannel.ts index 8ce6ea9b24..4b8321115f 100644 --- a/packages/twenty-front/src/modules/accounts/types/MessageChannel.ts +++ b/packages/twenty-front/src/modules/accounts/types/MessageChannel.ts @@ -1,4 +1,5 @@ import { type ImapSmtpCaldavAccount } from '@/accounts/types/ImapSmtpCaldavAccount'; +import { type MessageFolder } from '@/accounts/types/MessageFolder'; import { type ConnectedAccountProvider } from 'twenty-shared/types'; import { type MessageChannelVisibility } from '~/generated/graphql'; @@ -35,6 +36,7 @@ export type MessageChannel = { excludeNonProfessionalEmails: boolean; excludeGroupEmails: boolean; isSyncEnabled: boolean; + messageFolders: MessageFolder[]; visibility: MessageChannelVisibility; syncStatus: MessageChannelSyncStatus; syncStage: MessageChannelSyncStage; diff --git a/packages/twenty-front/src/modules/accounts/types/MessageFolder.ts b/packages/twenty-front/src/modules/accounts/types/MessageFolder.ts new file mode 100644 index 0000000000..471ba6a006 --- /dev/null +++ b/packages/twenty-front/src/modules/accounts/types/MessageFolder.ts @@ -0,0 +1,9 @@ +export type MessageFolder = { + id: string; + name: string; + syncCursor: string; + isSentFolder: boolean; + isSynced: boolean; + messageChannelId: string; + __typename: 'MessageFolder'; +}; diff --git a/packages/twenty-front/src/modules/apollo/optimistic-effect/utils/triggerAttachRelationOptimisticEffect.ts b/packages/twenty-front/src/modules/apollo/optimistic-effect/utils/triggerAttachRelationOptimisticEffect.ts index c67ccf8c00..91a30e5a96 100644 --- a/packages/twenty-front/src/modules/apollo/optimistic-effect/utils/triggerAttachRelationOptimisticEffect.ts +++ b/packages/twenty-front/src/modules/apollo/optimistic-effect/utils/triggerAttachRelationOptimisticEffect.ts @@ -47,6 +47,15 @@ export const triggerAttachRelationOptimisticEffect = ({ } if (fieldValueIsObjectRecordConnectionWithRefs) { + const recordAlreadyExists = targetRecordFieldValue.edges.some( + (edge: RecordGqlRefEdge) => + edge.node.__ref === sourceRecordReference.__ref, + ); + + if (recordAlreadyExists) { + return targetRecordFieldValue; + } + const nextEdges: RecordGqlRefEdge[] = [ ...targetRecordFieldValue.edges, { diff --git a/packages/twenty-front/src/modules/object-metadata/types/CoreObjectNameSingular.ts b/packages/twenty-front/src/modules/object-metadata/types/CoreObjectNameSingular.ts index cf8e14ad95..164c995322 100644 --- a/packages/twenty-front/src/modules/object-metadata/types/CoreObjectNameSingular.ts +++ b/packages/twenty-front/src/modules/object-metadata/types/CoreObjectNameSingular.ts @@ -15,6 +15,7 @@ export enum CoreObjectNameSingular { Message = 'message', MessageChannel = 'messageChannel', MessageParticipant = 'messageParticipant', + MessageFolder = 'messageFolder', MessageThread = 'messageThread', Note = 'note', NoteTarget = 'noteTarget', diff --git a/packages/twenty-front/src/modules/settings/accounts/components/SettingsAccountsMessageChannelDetails.tsx b/packages/twenty-front/src/modules/settings/accounts/components/SettingsAccountsMessageChannelDetails.tsx index 175ca7ad39..be802b0332 100644 --- a/packages/twenty-front/src/modules/settings/accounts/components/SettingsAccountsMessageChannelDetails.tsx +++ b/packages/twenty-front/src/modules/settings/accounts/components/SettingsAccountsMessageChannelDetails.tsx @@ -7,12 +7,17 @@ import { import { CoreObjectNameSingular } from '@/object-metadata/types/CoreObjectNameSingular'; import { useUpdateOneRecord } from '@/object-record/hooks/useUpdateOneRecord'; import { SettingsAccountsMessageAutoCreationCard } from '@/settings/accounts/components/SettingsAccountsMessageAutoCreationCard'; +import { SettingsAccountsMessageFoldersCard } from '@/settings/accounts/components/message-folders/SettingsAccountsMessageFoldersCard'; import { SettingsAccountsMessageVisibilityCard } from '@/settings/accounts/components/SettingsAccountsMessageVisibilityCard'; import { SettingsOptionCardContentToggle } from '@/settings/components/SettingsOptions/SettingsOptionCardContentToggle'; -import { type MessageChannelVisibility } from '~/generated-metadata/graphql'; +import { useIsFeatureEnabled } from '@/workspace/hooks/useIsFeatureEnabled'; import { t } from '@lingui/core/macro'; -import { Card, Section } from 'twenty-ui/layout'; import { H2Title, IconBriefcase, IconUsers } from 'twenty-ui/display'; +import { Card, Section } from 'twenty-ui/layout'; +import { + FeatureFlagKey, + type MessageChannelVisibility, +} from '~/generated-metadata/graphql'; type SettingsAccountsMessageChannelDetailsProps = { messageChannel: Pick< @@ -23,6 +28,7 @@ type SettingsAccountsMessageChannelDetailsProps = { | 'excludeNonProfessionalEmails' | 'excludeGroupEmails' | 'isSyncEnabled' + | 'messageFolders' >; }; @@ -39,6 +45,10 @@ export const SettingsAccountsMessageChannelDetails = ({ objectNameSingular: CoreObjectNameSingular.MessageChannel, }); + const isFolderControlEnabled = useIsFeatureEnabled( + FeatureFlagKey.IS_MESSAGE_FOLDER_CONTROL_ENABLED, + ); + const handleVisibilityChange = (value: MessageChannelVisibility) => { updateOneRecord({ idToUpdate: messageChannel.id, @@ -79,6 +89,18 @@ export const SettingsAccountsMessageChannelDetails = ({ return ( + {isFolderControlEnabled && messageChannel.messageFolders && ( +
+ + +
+ )}
{ ); const currentWorkspaceMember = useRecoilValue(currentWorkspaceMemberState); + const messageChannelObjectMetadataItem = useObjectMetadataItem({ + objectNameSingular: CoreObjectNameSingular.MessageChannel, + }); + const { records: accounts } = useFindManyRecords({ objectNameSingular: CoreObjectNameSingular.ConnectedAccount, filter: { @@ -48,15 +54,16 @@ export const SettingsAccountsMessageChannelsContainer = () => { eq: true, }, }, + recordGqlFields: generateDepthOneRecordGqlFields( + messageChannelObjectMetadataItem, + ), skip: !accounts.length, }); - const tabs = [ - ...messageChannels.map((messageChannel) => ({ - id: messageChannel.id, - title: messageChannel.handle, - })), - ]; + const tabs = messageChannels.map((messageChannel) => ({ + id: messageChannel.id, + title: messageChannel.handle, + })); if (!messageChannels.length) { return ; diff --git a/packages/twenty-front/src/modules/settings/accounts/components/__stories__/SettingsAccountsMessageChannelDetails.stories.tsx b/packages/twenty-front/src/modules/settings/accounts/components/__stories__/SettingsAccountsMessageChannelDetails.stories.tsx index 1c8323b25b..05d62de37a 100644 --- a/packages/twenty-front/src/modules/settings/accounts/components/__stories__/SettingsAccountsMessageChannelDetails.stories.tsx +++ b/packages/twenty-front/src/modules/settings/accounts/components/__stories__/SettingsAccountsMessageChannelDetails.stories.tsx @@ -2,11 +2,11 @@ import { type Meta, type StoryObj } from '@storybook/react'; import { MessageChannelContactAutoCreationPolicy } from '@/accounts/types/MessageChannel'; import { SettingsAccountsMessageChannelDetails } from '@/settings/accounts/components/SettingsAccountsMessageChannelDetails'; +import { ComponentDecorator } from 'twenty-ui/testing'; import { MessageChannelVisibility } from '~/generated/graphql'; import { I18nFrontDecorator } from '~/testing/decorators/I18nFrontDecorator'; import { ObjectMetadataItemsDecorator } from '~/testing/decorators/ObjectMetadataItemsDecorator'; import { SnackBarDecorator } from '~/testing/decorators/SnackBarDecorator'; -import { ComponentDecorator } from 'twenty-ui/testing'; const meta: Meta = { title: @@ -26,6 +26,7 @@ const meta: Meta = { excludeGroupEmails: false, isSyncEnabled: true, visibility: MessageChannelVisibility.SHARE_EVERYTHING, + messageFolders: [], }, }, argTypes: { diff --git a/packages/twenty-front/src/modules/settings/accounts/components/message-folders/SettingsAccountsMessageFoldersCard.tsx b/packages/twenty-front/src/modules/settings/accounts/components/message-folders/SettingsAccountsMessageFoldersCard.tsx new file mode 100644 index 0000000000..c3c49ec336 --- /dev/null +++ b/packages/twenty-front/src/modules/settings/accounts/components/message-folders/SettingsAccountsMessageFoldersCard.tsx @@ -0,0 +1,38 @@ +import { type MessageFolder } from '@/accounts/types/MessageFolder'; +import { SettingsMessageFoldersEmptyStateCard } from '@/settings/accounts/components/message-folders/SettingsMessageFoldersEmptyStateCard'; +import { SettingsMessageFoldersTableHeader } from '@/settings/accounts/components/message-folders/SettingsMessageFoldersTableHeader'; +import { SettingsMessageFoldersTableRow } from '@/settings/accounts/components/message-folders/SettingsMessageFoldersTableRow'; +import { Table } from '@/ui/layout/table/components/Table'; +import styled from '@emotion/styled'; +import { Section } from 'twenty-ui/layout'; + +type SettingsAccountsMessageFoldersCardProps = { + messageChannelId: string; + messageFolders: MessageFolder[]; +}; + +const StyledTableRows = styled.div` + padding-bottom: ${({ theme }) => theme.spacing(2)}; + padding-top: ${({ theme }) => theme.spacing(2)}; +`; + +export const SettingsAccountsMessageFoldersCard = ({ + messageFolders, +}: SettingsAccountsMessageFoldersCardProps) => { + if (!messageFolders || messageFolders.length === 0) { + return ; + } + + return ( +
+ + + + {messageFolders.map((folder) => ( + + ))} + +
+
+ ); +}; diff --git a/packages/twenty-front/src/modules/settings/accounts/components/message-folders/SettingsMessageFoldersEmptyStateCard.tsx b/packages/twenty-front/src/modules/settings/accounts/components/message-folders/SettingsMessageFoldersEmptyStateCard.tsx new file mode 100644 index 0000000000..277b787b3b --- /dev/null +++ b/packages/twenty-front/src/modules/settings/accounts/components/message-folders/SettingsMessageFoldersEmptyStateCard.tsx @@ -0,0 +1,28 @@ +import { useTheme } from '@emotion/react'; +import styled from '@emotion/styled'; +import { t } from '@lingui/core/macro'; +import { IconFolder } from 'twenty-ui/display'; +import { Section } from 'twenty-ui/layout'; + +const StyledEmptyState = styled.div` + align-items: center; + color: ${({ theme }) => theme.font.color.tertiary}; + display: flex; + flex-direction: column; + gap: ${({ theme }) => theme.spacing(2)}; + padding: ${({ theme }) => theme.spacing(8)}; + text-align: center; +`; + +export const SettingsMessageFoldersEmptyStateCard = () => { + const theme = useTheme(); + + return ( +
+ + +
{t`No folders found for this account`}
+
+
+ ); +}; diff --git a/packages/twenty-front/src/modules/settings/accounts/components/message-folders/SettingsMessageFoldersTableHeader.tsx b/packages/twenty-front/src/modules/settings/accounts/components/message-folders/SettingsMessageFoldersTableHeader.tsx new file mode 100644 index 0000000000..d72beeb1f7 --- /dev/null +++ b/packages/twenty-front/src/modules/settings/accounts/components/message-folders/SettingsMessageFoldersTableHeader.tsx @@ -0,0 +1,19 @@ +import { Table } from '@/ui/layout/table/components/Table'; +import { TableHeader } from '@/ui/layout/table/components/TableHeader'; +import { TableRow } from '@/ui/layout/table/components/TableRow'; +import { Trans } from '@lingui/react/macro'; + +export const SettingsMessageFoldersTableHeader = () => { + return ( + + + + Folder + + + Sync + + +
+ ); +}; diff --git a/packages/twenty-front/src/modules/settings/accounts/components/message-folders/SettingsMessageFoldersTableRow.tsx b/packages/twenty-front/src/modules/settings/accounts/components/message-folders/SettingsMessageFoldersTableRow.tsx new file mode 100644 index 0000000000..4ef0ec6141 --- /dev/null +++ b/packages/twenty-front/src/modules/settings/accounts/components/message-folders/SettingsMessageFoldersTableRow.tsx @@ -0,0 +1,75 @@ +import { type MessageFolder } from '@/accounts/types/MessageFolder'; +import { CoreObjectNameSingular } from '@/object-metadata/types/CoreObjectNameSingular'; +import { useUpdateOneRecord } from '@/object-record/hooks/useUpdateOneRecord'; +import { TableCell } from '@/ui/layout/table/components/TableCell'; +import { TableRow } from '@/ui/layout/table/components/TableRow'; +import { useTheme } from '@emotion/react'; +import styled from '@emotion/styled'; +import { IconFolder, IconSend } from 'twenty-ui/display'; +import { Toggle } from 'twenty-ui/input'; + +const StyledFolderNameCell = styled.div` + align-items: center; + display: flex; + gap: ${({ theme }) => theme.spacing(2)}; +`; + +const StyledTableRow = styled(TableRow)` + &:hover { + background: ${({ theme }) => theme.background.transparent.light}; + } +`; + +type SettingsMessageFoldersTableRowProps = { + folder: MessageFolder; +}; + +export const SettingsMessageFoldersTableRow = ({ + folder, +}: SettingsMessageFoldersTableRowProps) => { + const theme = useTheme(); + const { updateOneRecord } = useUpdateOneRecord({ + objectNameSingular: CoreObjectNameSingular.MessageFolder, + }); + + const handleSyncToggle = (value: boolean) => { + updateOneRecord({ + idToUpdate: folder.id, + updateOneRecordInput: { + isSynced: value, + }, + }); + }; + + const formatName = (name: string) => { + return name + .split(' ') + .map((word) => word.charAt(0).toUpperCase() + word.slice(1).toLowerCase()) + .join(' '); + }; + + return ( + + + + {folder.isSentFolder ? ( + + ) : ( + + )} + {formatName(folder.name)} + + + + + + + ); +}; diff --git a/packages/twenty-server/src/database/commands/cron-register-all.command.ts b/packages/twenty-server/src/database/commands/cron-register-all.command.ts index 70264dc1af..8dd5554c8c 100644 --- a/packages/twenty-server/src/database/commands/cron-register-all.command.ts +++ b/packages/twenty-server/src/database/commands/cron-register-all.command.ts @@ -4,6 +4,7 @@ import { Command, CommandRunner } from 'nest-commander'; import { CheckCustomDomainValidRecordsCronCommand } from 'src/engine/core-modules/domain-manager/crons/commands/check-custom-domain-valid-records.cron.command'; import { CleanupOrphanedFilesCronCommand } from 'src/engine/core-modules/file/crons/commands/cleanup-orphaned-files.cron.command'; +import { CronTriggerCronCommand } from 'src/engine/metadata-modules/trigger/crons/commands/cron-trigger.cron.command'; import { CalendarEventListFetchCronCommand } from 'src/modules/calendar/calendar-event-import-manager/crons/commands/calendar-event-list-fetch.cron.command'; import { CalendarEventsImportCronCommand } from 'src/modules/calendar/calendar-event-import-manager/crons/commands/calendar-import.cron.command'; import { CalendarOngoingStaleCronCommand } from 'src/modules/calendar/calendar-event-import-manager/crons/commands/calendar-ongoing-stale.cron.command'; @@ -14,7 +15,6 @@ import { WorkflowCleanWorkflowRunsCommand } from 'src/modules/workflow/workflow- import { WorkflowHandleStaledRunsCronCommand } from 'src/modules/workflow/workflow-runner/workflow-run-queue/cron/command/workflow-handle-staled-runs.cron.command'; import { WorkflowRunEnqueueCronCommand } from 'src/modules/workflow/workflow-runner/workflow-run-queue/cron/command/workflow-run-enqueue.cron.command'; import { WorkflowCronTriggerCronCommand } from 'src/modules/workflow/workflow-trigger/automated-trigger/crons/commands/workflow-cron-trigger.cron.command'; -import { CronTriggerCronCommand } from 'src/engine/metadata-modules/trigger/crons/commands/cron-trigger.cron.command'; @Command({ name: 'cron:register:all', @@ -27,6 +27,7 @@ export class CronRegisterAllCommand extends CommandRunner { private readonly messagingMessagesImportCronCommand: MessagingMessagesImportCronCommand, private readonly messagingMessageListFetchCronCommand: MessagingMessageListFetchCronCommand, private readonly messagingOngoingStaleCronCommand: MessagingOngoingStaleCronCommand, + private readonly calendarEventListFetchCronCommand: CalendarEventListFetchCronCommand, private readonly calendarEventsImportCronCommand: CalendarEventsImportCronCommand, private readonly calendarOngoingStaleCronCommand: CalendarOngoingStaleCronCommand, diff --git a/packages/twenty-server/src/database/commands/database-command.module.ts b/packages/twenty-server/src/database/commands/database-command.module.ts index a4b45502f6..baaeb1311f 100644 --- a/packages/twenty-server/src/database/commands/database-command.module.ts +++ b/packages/twenty-server/src/database/commands/database-command.module.ts @@ -14,6 +14,7 @@ import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity'; import { DataSourceModule } from 'src/engine/metadata-modules/data-source/data-source.module'; import { FieldMetadataModule } from 'src/engine/metadata-modules/field-metadata/field-metadata.module'; import { ObjectMetadataModule } from 'src/engine/metadata-modules/object-metadata/object-metadata.module'; +import { TriggerModule } from 'src/engine/metadata-modules/trigger/trigger.module'; import { WorkspaceCacheStorageModule } from 'src/engine/workspace-cache-storage/workspace-cache-storage.module'; import { DevSeederModule } from 'src/engine/workspace-manager/dev-seeder/dev-seeder.module'; import { WorkspaceManagerModule } from 'src/engine/workspace-manager/workspace-manager.module'; @@ -21,7 +22,6 @@ import { CalendarEventImportManagerModule } from 'src/modules/calendar/calendar- import { MessagingImportManagerModule } from 'src/modules/messaging/message-import-manager/messaging-import-manager.module'; import { WorkflowRunQueueModule } from 'src/modules/workflow/workflow-runner/workflow-run-queue/workflow-run-queue.module'; import { AutomatedTriggerModule } from 'src/modules/workflow/workflow-trigger/automated-trigger/automated-trigger.module'; -import { TriggerModule } from 'src/engine/metadata-modules/trigger/trigger.module'; import { DataSeedWorkspaceCommand } from './data-seed-dev-workspace.command'; diff --git a/packages/twenty-server/src/database/commands/upgrade-version-command/1-5/1-5-upgrade-version-command.module.ts b/packages/twenty-server/src/database/commands/upgrade-version-command/1-5/1-5-upgrade-version-command.module.ts index dd3d332d68..dc521dac37 100644 --- a/packages/twenty-server/src/database/commands/upgrade-version-command/1-5/1-5-upgrade-version-command.module.ts +++ b/packages/twenty-server/src/database/commands/upgrade-version-command/1-5/1-5-upgrade-version-command.module.ts @@ -1,6 +1,7 @@ import { Module } from '@nestjs/common'; import { TypeOrmModule } from '@nestjs/typeorm'; +import { AddPositionsToWorkflowVersionsAndWorkflowRuns } from 'src/database/commands/upgrade-version-command/1-5/1-5-add-positions-to-workflow-versions-and-workflow-runs.command'; import { RemoveFavoriteViewRelation } from 'src/database/commands/upgrade-version-command/1-5/1-5-remove-favorite-view-relation.command'; import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity'; import { FieldMetadataEntity } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity'; @@ -8,7 +9,6 @@ import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadat import { WorkspaceMetadataVersionModule } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.module'; import { WorkspaceSchemaManagerModule } from 'src/engine/twenty-orm/workspace-schema-manager/workspace-schema-manager.module'; import { WorkspaceDataSourceModule } from 'src/engine/workspace-datasource/workspace-datasource.module'; -import { AddPositionsToWorkflowVersionsAndWorkflowRuns } from 'src/database/commands/upgrade-version-command/1-5/1-5-add-positions-to-workflow-versions-and-workflow-runs.command'; @Module({ imports: [ diff --git a/packages/twenty-server/src/database/commands/upgrade-version-command/upgrade.command.ts b/packages/twenty-server/src/database/commands/upgrade-version-command/upgrade.command.ts index b74b03c043..eb216e31ff 100644 --- a/packages/twenty-server/src/database/commands/upgrade-version-command/upgrade.command.ts +++ b/packages/twenty-server/src/database/commands/upgrade-version-command/upgrade.command.ts @@ -31,13 +31,13 @@ import { RemoveWorkflowRunsWithoutState } from 'src/database/commands/upgrade-ve import { AddNextStepIdsToWorkflowRunsTrigger } from 'src/database/commands/upgrade-version-command/1-3/1-3-add-next-step-ids-to-workflow-runs-trigger.command'; import { AssignRolesToExistingApiKeysCommand } from 'src/database/commands/upgrade-version-command/1-3/1-3-assign-roles-to-existing-api-keys.command'; import { UpdateTimestampColumnTypeInWorkspaceSchemaCommand } from 'src/database/commands/upgrade-version-command/1-3/1-3-update-timestamp-column-type-in-workspace-schema.command'; +import { AddPositionsToWorkflowVersionsAndWorkflowRuns } from 'src/database/commands/upgrade-version-command/1-5/1-5-add-positions-to-workflow-versions-and-workflow-runs.command'; import { RemoveFavoriteViewRelation } from 'src/database/commands/upgrade-version-command/1-5/1-5-remove-favorite-view-relation.command'; import { TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service'; import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity'; import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager'; import { SyncWorkspaceMetadataCommand } from 'src/engine/workspace-manager/workspace-sync-metadata/commands/sync-workspace-metadata.command'; import { compareVersionMajorAndMinor } from 'src/utils/version/compare-version-minor-and-major'; -import { AddPositionsToWorkflowVersionsAndWorkflowRuns } from 'src/database/commands/upgrade-version-command/1-5/1-5-add-positions-to-workflow-versions-and-workflow-runs.command'; const execPromise = promisify(exec); diff --git a/packages/twenty-server/src/engine/core-modules/auth/auth.module.ts b/packages/twenty-server/src/engine/core-modules/auth/auth.module.ts index f1af4a33b3..e30905af97 100644 --- a/packages/twenty-server/src/engine/core-modules/auth/auth.module.ts +++ b/packages/twenty-server/src/engine/core-modules/auth/auth.module.ts @@ -17,7 +17,6 @@ import { AuthSsoService } from 'src/engine/core-modules/auth/services/auth-sso.s import { CreateCalendarChannelService } from 'src/engine/core-modules/auth/services/create-calendar-channel.service'; import { CreateConnectedAccountService } from 'src/engine/core-modules/auth/services/create-connected-account.service'; import { CreateMessageChannelService } from 'src/engine/core-modules/auth/services/create-message-channel.service'; -import { CreateMessageFolderService } from 'src/engine/core-modules/auth/services/create-message-folder.service'; import { GoogleAPIScopesService } from 'src/engine/core-modules/auth/services/google-apis-scopes'; import { GoogleAPIsService } from 'src/engine/core-modules/auth/services/google-apis.service'; import { MicrosoftAPIsService } from 'src/engine/core-modules/auth/services/microsoft-apis.service'; @@ -59,6 +58,7 @@ import { UserRoleModule } from 'src/engine/metadata-modules/user-role/user-role. import { WorkspaceDataSourceModule } from 'src/engine/workspace-datasource/workspace-datasource.module'; import { WorkspaceManagerModule } from 'src/engine/workspace-manager/workspace-manager.module'; import { ConnectedAccountModule } from 'src/modules/connected-account/connected-account.module'; +import { MessagingFolderSyncManagerModule } from 'src/modules/messaging/message-folder-manager/messaging-folder-sync-manager.module'; import { TwoFactorAuthenticationMethod } from '../two-factor-authentication/entities/two-factor-authentication-method.entity'; import { TwoFactorAuthenticationModule } from '../two-factor-authentication/two-factor-authentication.module'; @@ -96,6 +96,7 @@ import { JwtAuthStrategy } from './strategies/jwt.auth.strategy'; OnboardingModule, WorkspaceDataSourceModule, ConnectedAccountModule, + MessagingFolderSyncManagerModule, WorkspaceSSOModule, FeatureFlagModule, WorkspaceInvitationModule, @@ -136,17 +137,11 @@ import { JwtAuthStrategy } from './strategies/jwt.auth.strategy'; ResetMessageFolderService, CreateMessageChannelService, CreateCalendarChannelService, - CreateMessageFolderService, CreateConnectedAccountService, UpdateConnectedAccountOnReconnectService, TransientTokenService, AuthSsoService, ], - exports: [ - AccessTokenService, - LoginTokenService, - RefreshTokenService, - CreateMessageFolderService, - ], + exports: [AccessTokenService, LoginTokenService, RefreshTokenService], }) export class AuthModule {} diff --git a/packages/twenty-server/src/engine/core-modules/auth/services/create-message-folder.service.ts b/packages/twenty-server/src/engine/core-modules/auth/services/create-message-folder.service.ts deleted file mode 100644 index 6d19c8c9ed..0000000000 --- a/packages/twenty-server/src/engine/core-modules/auth/services/create-message-folder.service.ts +++ /dev/null @@ -1,53 +0,0 @@ -import { Injectable } from '@nestjs/common'; - -import { v4 } from 'uuid'; - -import { type WorkspaceEntityManager } from 'src/engine/twenty-orm/entity-manager/workspace-entity-manager'; -import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager'; -import { type MessageFolderWorkspaceEntity } from 'src/modules/messaging/common/standard-objects/message-folder.workspace-entity'; -import { MessageFolderName } from 'src/modules/messaging/message-import-manager/drivers/microsoft/types/folders'; - -export type CreateMessageFoldersInput = { - workspaceId: string; - messageChannelId: string; - manager: WorkspaceEntityManager; -}; - -@Injectable() -export class CreateMessageFolderService { - constructor( - private readonly twentyORMGlobalManager: TwentyORMGlobalManager, - ) {} - - async createMessageFolders(input: CreateMessageFoldersInput): Promise { - const { workspaceId, messageChannelId, manager } = input; - - const messageFolderRepository = - await this.twentyORMGlobalManager.getRepositoryForWorkspace( - workspaceId, - 'messageFolder', - ); - - await messageFolderRepository.save( - { - id: v4(), - messageChannelId, - name: MessageFolderName.INBOX, - syncCursor: '', - }, - {}, - manager, - ); - - await messageFolderRepository.save( - { - id: v4(), - messageChannelId, - name: MessageFolderName.SENT_ITEMS, - syncCursor: '', - }, - {}, - manager, - ); - } -} diff --git a/packages/twenty-server/src/engine/core-modules/auth/services/microsoft-apis.service.spec.ts b/packages/twenty-server/src/engine/core-modules/auth/services/microsoft-apis.service.spec.ts index 01fe176058..7cddccdade 100644 --- a/packages/twenty-server/src/engine/core-modules/auth/services/microsoft-apis.service.spec.ts +++ b/packages/twenty-server/src/engine/core-modules/auth/services/microsoft-apis.service.spec.ts @@ -6,7 +6,6 @@ import { ConnectedAccountProvider } from 'twenty-shared/types'; import { CreateCalendarChannelService } from 'src/engine/core-modules/auth/services/create-calendar-channel.service'; import { CreateConnectedAccountService } from 'src/engine/core-modules/auth/services/create-connected-account.service'; import { CreateMessageChannelService } from 'src/engine/core-modules/auth/services/create-message-channel.service'; -import { CreateMessageFolderService } from 'src/engine/core-modules/auth/services/create-message-folder.service'; import { MicrosoftAPIsService } from 'src/engine/core-modules/auth/services/microsoft-apis.service'; import { ResetCalendarChannelService } from 'src/engine/core-modules/auth/services/reset-calendar-channel.service'; import { ResetMessageChannelService } from 'src/engine/core-modules/auth/services/reset-message-channel.service'; @@ -141,12 +140,6 @@ describe('MicrosoftAPIsService', () => { .mockResolvedValue('message-channel-id'), }, }, - { - provide: CreateMessageFolderService, - useValue: { - createMessageFolders: jest.fn(), - }, - }, { provide: CreateCalendarChannelService, useValue: { diff --git a/packages/twenty-server/src/engine/core-modules/auth/services/microsoft-apis.service.ts b/packages/twenty-server/src/engine/core-modules/auth/services/microsoft-apis.service.ts index d7aa01afed..6d25b5887a 100644 --- a/packages/twenty-server/src/engine/core-modules/auth/services/microsoft-apis.service.ts +++ b/packages/twenty-server/src/engine/core-modules/auth/services/microsoft-apis.service.ts @@ -6,7 +6,6 @@ import { v4 } from 'uuid'; import { CreateCalendarChannelService } from 'src/engine/core-modules/auth/services/create-calendar-channel.service'; import { CreateConnectedAccountService } from 'src/engine/core-modules/auth/services/create-connected-account.service'; import { CreateMessageChannelService } from 'src/engine/core-modules/auth/services/create-message-channel.service'; -import { CreateMessageFolderService } from 'src/engine/core-modules/auth/services/create-message-folder.service'; import { ResetCalendarChannelService } from 'src/engine/core-modules/auth/services/reset-calendar-channel.service'; import { ResetMessageChannelService } from 'src/engine/core-modules/auth/services/reset-message-channel.service'; import { ResetMessageFolderService } from 'src/engine/core-modules/auth/services/reset-message-folder.service'; @@ -52,7 +51,6 @@ export class MicrosoftAPIsService { private readonly resetCalendarChannelService: ResetCalendarChannelService, private readonly createMessageChannelService: CreateMessageChannelService, private readonly createCalendarChannelService: CreateCalendarChannelService, - private readonly createMessageFolderService: CreateMessageFolderService, private readonly createConnectedAccountService: CreateConnectedAccountService, private readonly updateConnectedAccountOnReconnectService: UpdateConnectedAccountOnReconnectService, private readonly twentyConfigService: TwentyConfigService, @@ -122,18 +120,11 @@ export class MicrosoftAPIsService { manager, }); - const newMessageChannelId = - await this.createMessageChannelService.createMessageChannel({ - workspaceId, - connectedAccountId: newOrExistingConnectedAccountId, - handle, - messageVisibility, - manager, - }); - - await this.createMessageFolderService.createMessageFolders({ + await this.createMessageChannelService.createMessageChannel({ workspaceId, - messageChannelId: newMessageChannelId, + connectedAccountId: newOrExistingConnectedAccountId, + handle, + messageVisibility, manager, }); diff --git a/packages/twenty-server/src/engine/core-modules/feature-flag/constants/public-feature-flag.const.ts b/packages/twenty-server/src/engine/core-modules/feature-flag/constants/public-feature-flag.const.ts index 3d69ca5c9b..f8a7720f53 100644 --- a/packages/twenty-server/src/engine/core-modules/feature-flag/constants/public-feature-flag.const.ts +++ b/packages/twenty-server/src/engine/core-modules/feature-flag/constants/public-feature-flag.const.ts @@ -30,6 +30,15 @@ export const PUBLIC_FEATURE_FLAGS: PublicFeatureFlag[] = [ imagePath: 'https://twenty.com/images/lab/is-workflow-branch-enabled.png', }, }, + { + key: FeatureFlagKey.IS_MESSAGE_FOLDER_CONTROL_ENABLED, + metadata: { + label: 'Message Folder Control', + description: 'Control which folders are synced', + imagePath: + 'https://twenty.com/images/lab/is-message-folder-control-enabled.png', + }, + }, ...(process.env.CLOUDFLARE_API_KEY ? [ // { diff --git a/packages/twenty-server/src/engine/core-modules/feature-flag/enums/feature-flag-key.enum.ts b/packages/twenty-server/src/engine/core-modules/feature-flag/enums/feature-flag-key.enum.ts index 9ccfd7b3aa..6ee344972c 100644 --- a/packages/twenty-server/src/engine/core-modules/feature-flag/enums/feature-flag-key.enum.ts +++ b/packages/twenty-server/src/engine/core-modules/feature-flag/enums/feature-flag-key.enum.ts @@ -14,4 +14,5 @@ export enum FeatureFlagKey { IS_WORKSPACE_MIGRATION_V2_ENABLED = 'IS_WORKSPACE_MIGRATION_V2_ENABLED', IS_API_KEY_ROLES_ENABLED = 'IS_API_KEY_ROLES_ENABLED', IS_PAGE_LAYOUT_ENABLED = 'IS_PAGE_LAYOUT_ENABLED', + IS_MESSAGE_FOLDER_CONTROL_ENABLED = 'IS_MESSAGE_FOLDER_CONTROL_ENABLED', } diff --git a/packages/twenty-server/src/engine/twenty-orm/entity-manager/workspace-entity-manager.spec.ts b/packages/twenty-server/src/engine/twenty-orm/entity-manager/workspace-entity-manager.spec.ts index e83dcd074e..6c6ebfa3e1 100644 --- a/packages/twenty-server/src/engine/twenty-orm/entity-manager/workspace-entity-manager.spec.ts +++ b/packages/twenty-server/src/engine/twenty-orm/entity-manager/workspace-entity-manager.spec.ts @@ -137,6 +137,7 @@ describe('WorkspaceEntityManager', () => { IS_WORKSPACE_MIGRATION_V2_ENABLED: false, IS_API_KEY_ROLES_ENABLED: false, IS_PAGE_LAYOUT_ENABLED: false, + IS_MESSAGE_FOLDER_CONTROL_ENABLED: false, }, eventEmitterService: { emitMutationEvent: jest.fn(), diff --git a/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/constants/standard-field-ids.ts b/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/constants/standard-field-ids.ts index 8d18e3ef8f..356b1830ef 100644 --- a/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/constants/standard-field-ids.ts +++ b/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/constants/standard-field-ids.ts @@ -253,6 +253,9 @@ export const MESSAGE_FOLDER_STANDARD_FIELD_IDS = { name: '20202020-7cf8-40bc-a681-b80b771449b7', messageChannel: '20202020-b658-408f-bd46-3bd2d15d7e52', syncCursor: '20202020-98cd-49ed-8dfc-cb5796400e64', + isSentFolder: '20202020-2af5-4a25-b2de-3c9386da941b', + isSynced: '20202020-764f-4e09-8f95-cd46b6bfe3c4', + externalId: '20202020-f3a8-4d2b-9c7e-1b5f9a8e4c6d', } as const; export const MESSAGE_PARTICIPANT_STANDARD_FIELD_IDS = { diff --git a/packages/twenty-server/src/modules/connected-account/services/imap-smtp-caldav-apis.service.ts b/packages/twenty-server/src/modules/connected-account/services/imap-smtp-caldav-apis.service.ts index 5fcb2c78cc..c855283ea4 100644 --- a/packages/twenty-server/src/modules/connected-account/services/imap-smtp-caldav-apis.service.ts +++ b/packages/twenty-server/src/modules/connected-account/services/imap-smtp-caldav-apis.service.ts @@ -3,12 +3,10 @@ import { Injectable } from '@nestjs/common'; import { ConnectedAccountProvider } from 'twenty-shared/types'; import { v4 } from 'uuid'; -import { CreateMessageFolderService } from 'src/engine/core-modules/auth/services/create-message-folder.service'; import { type EmailAccountConnectionParameters } from 'src/engine/core-modules/imap-smtp-caldav-connection/dtos/imap-smtp-caldav-connection.dto'; import { InjectMessageQueue } from 'src/engine/core-modules/message-queue/decorators/message-queue.decorator'; import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants'; import { MessageQueueService } from 'src/engine/core-modules/message-queue/services/message-queue.service'; -import { type WorkspaceEntityManager } from 'src/engine/twenty-orm/entity-manager/workspace-entity-manager'; import { type WorkspaceRepository } from 'src/engine/twenty-orm/repository/workspace.repository'; import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager'; import { @@ -40,7 +38,6 @@ export class ImapSmtpCalDavAPIService { private readonly messageQueueService: MessageQueueService, @InjectMessageQueue(MessageQueue.calendarQueue) private readonly calendarQueueService: MessageQueueService, - private readonly createMessageFolderService: CreateMessageFolderService, ) {} async setupCompleteAccount(input: { @@ -89,28 +86,25 @@ export class ImapSmtpCalDavAPIService { let createdMessageChannel: MessageChannelWorkspaceEntity | null = null; let createdCalendarChannel: CalendarChannelWorkspaceEntity | null = null; - await workspaceDataSource.transaction( - async (manager: WorkspaceEntityManager) => { - await this.upsertConnectedAccount( - input, - accountId, - connectedAccountRepository, - ); + await workspaceDataSource.transaction(async () => { + await this.upsertConnectedAccount( + input, + accountId, + connectedAccountRepository, + ); - createdMessageChannel = await this.setupMessageChannels( - input, - accountId, - messageChannelRepository, - manager, - ); + createdMessageChannel = await this.setupMessageChannels( + input, + accountId, + messageChannelRepository, + ); - createdCalendarChannel = await this.setupCalendarChannels( - input, - accountId, - calendarChannelRepository, - ); - }, - ); + createdCalendarChannel = await this.setupCalendarChannels( + input, + accountId, + calendarChannelRepository, + ); + }); await this.enqueueSyncJobs( input, @@ -149,7 +143,6 @@ export class ImapSmtpCalDavAPIService { }, accountId: string, messageChannelRepository: WorkspaceRepository, - manager: WorkspaceEntityManager, ): Promise { const existingChannels = await messageChannelRepository.find({ where: { connectedAccountId: accountId }, @@ -182,12 +175,6 @@ export class ImapSmtpCalDavAPIService { {}, ); - await this.createMessageFolderService.createMessageFolders({ - workspaceId: input.workspaceId, - messageChannelId: newMessageChannel.id, - manager, - }); - return shouldEnableSync ? newMessageChannel : null; } diff --git a/packages/twenty-server/src/modules/messaging/common/standard-objects/message-folder.workspace-entity.ts b/packages/twenty-server/src/modules/messaging/common/standard-objects/message-folder.workspace-entity.ts index 5cb4c73ba1..0505319f6b 100644 --- a/packages/twenty-server/src/modules/messaging/common/standard-objects/message-folder.workspace-entity.ts +++ b/packages/twenty-server/src/modules/messaging/common/standard-objects/message-folder.workspace-entity.ts @@ -1,14 +1,15 @@ import { msg } from '@lingui/core/macro'; -import { Relation } from 'typeorm'; import { FieldMetadataType } from 'twenty-shared/types'; +import { Relation } from 'typeorm'; -import { RelationType } from 'src/engine/metadata-modules/field-metadata/interfaces/relation-type.interface'; import { RelationOnDeleteAction } from 'src/engine/metadata-modules/field-metadata/interfaces/relation-on-delete-action.interface'; +import { RelationType } from 'src/engine/metadata-modules/field-metadata/interfaces/relation-type.interface'; import { BaseWorkspaceEntity } from 'src/engine/twenty-orm/base.workspace-entity'; import { WorkspaceEntity } from 'src/engine/twenty-orm/decorators/workspace-entity.decorator'; import { WorkspaceField } from 'src/engine/twenty-orm/decorators/workspace-field.decorator'; import { WorkspaceIsNotAuditLogged } from 'src/engine/twenty-orm/decorators/workspace-is-not-audit-logged.decorator'; +import { WorkspaceIsNullable } from 'src/engine/twenty-orm/decorators/workspace-is-nullable.decorator'; import { WorkspaceIsSystem } from 'src/engine/twenty-orm/decorators/workspace-is-system.decorator'; import { WorkspaceJoinColumn } from 'src/engine/twenty-orm/decorators/workspace-join-column.decorator'; import { WorkspaceRelation } from 'src/engine/twenty-orm/decorators/workspace-relation.decorator'; @@ -58,6 +59,37 @@ export class MessageFolderWorkspaceEntity extends BaseWorkspaceEntity { }) syncCursor: string; + @WorkspaceField({ + standardId: MESSAGE_FOLDER_STANDARD_FIELD_IDS.isSentFolder, + type: FieldMetadataType.BOOLEAN, + label: msg`Is Sent Folder`, + description: msg`Is Sent Folder`, + icon: 'IconCheck', + defaultValue: false, + }) + isSentFolder: boolean; + + @WorkspaceField({ + standardId: MESSAGE_FOLDER_STANDARD_FIELD_IDS.isSynced, + type: FieldMetadataType.BOOLEAN, + label: msg`Is Synced`, + description: msg`Is Synced`, + icon: 'IconCheck', + defaultValue: false, + }) + isSynced: boolean; + + @WorkspaceField({ + standardId: MESSAGE_FOLDER_STANDARD_FIELD_IDS.externalId, + type: FieldMetadataType.TEXT, + label: msg`External ID`, + description: msg`External ID`, + icon: 'IconHash', + defaultValue: null, + }) + @WorkspaceIsNullable() + externalId: string | null; + @WorkspaceJoinColumn('messageChannel') messageChannelId: string; } diff --git a/packages/twenty-server/src/modules/messaging/message-folder-manager/drivers/gmail/gmail-get-all-folders.service.ts b/packages/twenty-server/src/modules/messaging/message-folder-manager/drivers/gmail/gmail-get-all-folders.service.ts new file mode 100644 index 0000000000..6a6376c47e --- /dev/null +++ b/packages/twenty-server/src/modules/messaging/message-folder-manager/drivers/gmail/gmail-get-all-folders.service.ts @@ -0,0 +1,110 @@ +import { Injectable, Logger } from '@nestjs/common'; + +import { gmail_v1 } from 'googleapis'; +import { isDefined } from 'twenty-shared/utils'; + +import { + MessageFolder, + MessageFolderDriver, +} from 'src/modules/messaging/message-folder-manager/interfaces/message-folder-driver.interface'; + +import { type ConnectedAccountWorkspaceEntity } from 'src/modules/connected-account/standard-objects/connected-account.workspace-entity'; +import { MESSAGING_GMAIL_EXCLUDED_CATEGORIES } from 'src/modules/messaging/message-import-manager/drivers/gmail/constants/messaging-gmail-excluded-categories'; +import { GmailClientProvider } from 'src/modules/messaging/message-import-manager/drivers/gmail/providers/gmail-client.provider'; +import { GmailHandleErrorService } from 'src/modules/messaging/message-import-manager/drivers/gmail/services/gmail-handle-error.service'; +import { computeGmailCategoryLabelId } from 'src/modules/messaging/message-import-manager/drivers/gmail/utils/compute-gmail-category-label-id.util'; + +@Injectable() +export class GmailGetAllFoldersService implements MessageFolderDriver { + private readonly logger = new Logger(GmailGetAllFoldersService.name); + + constructor( + private readonly gmailClientProvider: GmailClientProvider, + private readonly gmailHandleErrorService: GmailHandleErrorService, + ) {} + + private isExcludedCategoryFolder(labelId: string): boolean { + const excludedCategoryIds = MESSAGING_GMAIL_EXCLUDED_CATEGORIES.map( + (category) => computeGmailCategoryLabelId(category), + ); + + return excludedCategoryIds.includes(labelId); + } + + private isIncludedFolder(label: gmail_v1.Schema$Label): boolean { + if (!isDefined(label.id)) { + return false; + } + + const isTargetSystemFolder = + label.type === 'system' && (label.id === 'INBOX' || label.id === 'SENT'); + const isUserFolder = label.type === 'user'; + + return isTargetSystemFolder || isUserFolder; + } + + async getAllMessageFolders( + connectedAccount: Pick< + ConnectedAccountWorkspaceEntity, + 'provider' | 'refreshToken' | 'id' | 'handle' + >, + ): Promise { + try { + const gmailClient = + await this.gmailClientProvider.getGmailClient(connectedAccount); + + const response = await gmailClient.users.labels + .list({ userId: 'me' }) + .catch((error) => { + this.logger.error( + `Connected account ${connectedAccount.id}: Error fetching labels: ${error.message}`, + ); + + this.gmailHandleErrorService.handleGmailMessageListFetchError(error); + + return { data: { labels: [] } }; + }); + + const labels = response.data.labels || []; + + const folders: MessageFolder[] = []; + + for (const label of labels) { + if (!label.name || !label.id) { + continue; + } + + if (this.isExcludedCategoryFolder(label.id)) { + continue; + } + + if (!this.isIncludedFolder(label)) { + continue; + } + + const isSentFolder = label.id === 'SENT'; + const isSyncedByDefault = label.id === 'INBOX' || label.id === 'SENT'; + + folders.push({ + externalId: label.id, + name: label.name, + isSynced: isSyncedByDefault, + isSentFolder, + }); + } + + this.logger.log( + `Found ${folders.length} folders for Gmail account ${connectedAccount.handle}`, + ); + + return folders; + } catch (error) { + this.logger.error( + `Failed to get Gmail folders for account ${connectedAccount.handle}:`, + error, + ); + + throw error; + } + } +} diff --git a/packages/twenty-server/src/modules/messaging/message-folder-manager/drivers/imap/imap-get-all-folders.service.ts b/packages/twenty-server/src/modules/messaging/message-folder-manager/drivers/imap/imap-get-all-folders.service.ts new file mode 100644 index 0000000000..bb50d8dc4a --- /dev/null +++ b/packages/twenty-server/src/modules/messaging/message-folder-manager/drivers/imap/imap-get-all-folders.service.ts @@ -0,0 +1,171 @@ +import { Injectable, Logger } from '@nestjs/common'; + +import { ImapFlow, type ListResponse } from 'imapflow'; +import { isDefined } from 'twenty-shared/utils'; + +import { + MessageFolder, + MessageFolderDriver, +} from 'src/modules/messaging/message-folder-manager/interfaces/message-folder-driver.interface'; + +import { type ConnectedAccountWorkspaceEntity } from 'src/modules/connected-account/standard-objects/connected-account.workspace-entity'; +import { ImapClientProvider } from 'src/modules/messaging/message-import-manager/drivers/imap/providers/imap-client.provider'; +import { ImapFindSentFolderService } from 'src/modules/messaging/message-import-manager/drivers/imap/services/imap-find-sent-folder.service'; +import { MessageFolderName } from 'src/modules/messaging/message-import-manager/drivers/imap/types/folders'; +import { StandardFolder } from 'src/modules/messaging/message-import-manager/drivers/types/standard-folder'; +import { getStandardFolderByRegex } from 'src/modules/messaging/message-import-manager/drivers/utils/get-standard-folder-by-regex'; + +@Injectable() +export class ImapGetAllFoldersService implements MessageFolderDriver { + private readonly logger = new Logger(ImapGetAllFoldersService.name); + + constructor( + private readonly imapClientProvider: ImapClientProvider, + private readonly imapFindSentFolderService: ImapFindSentFolderService, + ) {} + + public async getAllMessageFolders( + connectedAccount: Pick< + ConnectedAccountWorkspaceEntity, + 'id' | 'provider' | 'connectionParameters' | 'handle' + >, + ): Promise { + try { + const client = await this.imapClientProvider.getClient(connectedAccount); + + const mailboxList = await client.list(); + + const folders = await this.filterAndMapFolders(client, mailboxList); + + await this.imapClientProvider.closeClient(client); + + return folders; + } catch (error) { + this.logger.error( + `Failed to get IMAP folders for account ${connectedAccount.handle}:`, + error, + ); + + throw error; + } + } + + private async filterAndMapFolders( + client: ImapFlow, + mailboxList: ListResponse[], + ): Promise { + const folders: MessageFolder[] = []; + const sentFolderPath = + await this.imapFindSentFolderService.findSentFolder(client); + + if (isDefined(sentFolderPath)) { + const sentMailbox = mailboxList.find((m) => m.path === sentFolderPath); + const uidValidity = sentMailbox + ? await this.getUidValidity(client, sentMailbox) + : null; + + folders.push({ + externalId: uidValidity + ? `${sentFolderPath}:${uidValidity.toString()}` + : sentFolderPath, + name: sentFolderPath, + isSynced: true, + isSentFolder: true, + }); + } + + const validMailboxes = mailboxList.filter((mailbox) => + this.isValidMailbox(mailbox, folders), + ); + + for (const mailbox of validMailboxes) { + const isInbox = await this.isInboxFolder(mailbox); + const uidValidity = await this.getUidValidity(client, mailbox); + + folders.push({ + externalId: uidValidity + ? `${mailbox.path}:${uidValidity}` + : mailbox.path, + name: mailbox.path, + isSynced: isInbox, + isSentFolder: false, + }); + } + + return folders; + } + + private isValidMailbox( + mailbox: ListResponse, + existingFolders: MessageFolder[], + ): boolean { + if (this.shouldExcludeFolder(mailbox)) { + return false; + } + + const isDuplicate = existingFolders.some( + (folder) => folder.name === mailbox.path, + ); + + return !isDuplicate; + } + + private async isInboxFolder(mailbox: ListResponse): Promise { + if ( + mailbox.path.toLowerCase() === MessageFolderName.INBOX || + mailbox.specialUse === '\\Inbox' + ) { + return true; + } + + return false; + } + + private shouldExcludeFolder(mailbox: ListResponse): boolean { + if (mailbox.flags?.has('\\Noselect')) { + return true; + } + + if ( + mailbox.specialUse === '\\Drafts' || + mailbox.specialUse === '\\Trash' || + mailbox.specialUse === '\\Junk' + ) { + return true; + } + const standardFolder = getStandardFolderByRegex(mailbox.path); + + if (!standardFolder) { + return false; + } + + return ( + standardFolder !== StandardFolder.SENT && + standardFolder !== StandardFolder.INBOX + ); + } + + private async getUidValidity( + client: ImapFlow, + mailbox: ListResponse, + ): Promise { + if (mailbox.status?.uidValidity) { + return mailbox.status.uidValidity; + } + + try { + const status = await client.status(mailbox.path, { + uidValidity: true, + }); + + return status.uidValidity ?? null; + } catch (error) { + this.logger.warn( + `Failed to get uidValidity for folder ${mailbox.path}:`, + error, + ); + + return null; + } + } +} diff --git a/packages/twenty-server/src/modules/messaging/message-folder-manager/drivers/microsoft/microsoft-get-all-folders.service.ts b/packages/twenty-server/src/modules/messaging/message-folder-manager/drivers/microsoft/microsoft-get-all-folders.service.ts new file mode 100644 index 0000000000..cbc0062a48 --- /dev/null +++ b/packages/twenty-server/src/modules/messaging/message-folder-manager/drivers/microsoft/microsoft-get-all-folders.service.ts @@ -0,0 +1,107 @@ +import { Injectable, Logger } from '@nestjs/common'; + +import { + MessageFolder, + MessageFolderDriver, +} from 'src/modules/messaging/message-folder-manager/interfaces/message-folder-driver.interface'; + +import { type ConnectedAccountWorkspaceEntity } from 'src/modules/connected-account/standard-objects/connected-account.workspace-entity'; +import { MicrosoftClientProvider } from 'src/modules/messaging/message-import-manager/drivers/microsoft/providers/microsoft-client.provider'; +import { MicrosoftHandleErrorService } from 'src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-handle-error.service'; +import { StandardFolder } from 'src/modules/messaging/message-import-manager/drivers/types/standard-folder'; +import { getStandardFolderByRegex } from 'src/modules/messaging/message-import-manager/drivers/utils/get-standard-folder-by-regex'; + +type MicrosoftGraphFolder = { + id: string; + displayName: string; +}; + +@Injectable() +export class MicrosoftGetAllFoldersService implements MessageFolderDriver { + private readonly logger = new Logger(MicrosoftGetAllFoldersService.name); + + constructor( + private readonly microsoftClientProvider: MicrosoftClientProvider, + private readonly microsoftHandleErrorService: MicrosoftHandleErrorService, + ) {} + + async getAllMessageFolders( + connectedAccount: Pick< + ConnectedAccountWorkspaceEntity, + 'refreshToken' | 'id' | 'handle' + >, + ): Promise { + try { + const microsoftClient = + await this.microsoftClientProvider.getMicrosoftClient(connectedAccount); + + const response = await microsoftClient + .api('/me/mailFolders') + .get() + .catch((error) => { + this.logger.error( + `Connected account ${connectedAccount.id}: Error fetching folders: ${error.message}`, + ); + this.microsoftHandleErrorService.handleMicrosoftGetMessageListError( + error, + ); + + return { value: [] }; + }); + + const folders = (response.value as MicrosoftGraphFolder[]) || []; + const folderInfos: MessageFolder[] = []; + + for (const folder of folders) { + if (!folder.displayName) { + continue; + } + + const standardFolder = getStandardFolderByRegex(folder.displayName); + + if (this.shouldExcludeFolder(standardFolder)) { + continue; + } + + const isInbox = this.isInboxFolder(standardFolder); + const isSentFolder = this.isSentFolder(standardFolder); + + folderInfos.push({ + externalId: folder.id, + name: folder.displayName, + isSynced: isInbox || isSentFolder, + isSentFolder, + }); + } + + this.logger.log( + `Found ${folderInfos.length} folders for Microsoft account ${connectedAccount.handle}`, + ); + + return folderInfos; + } catch (error) { + this.logger.error( + `Failed to get Microsoft folders for account ${connectedAccount.handle}:`, + error, + ); + + throw error; + } + } + + private isInboxFolder(standardFolder: StandardFolder | null): boolean { + return standardFolder === StandardFolder.INBOX; + } + + private isSentFolder(standardFolder: StandardFolder | null): boolean { + return standardFolder === StandardFolder.SENT; + } + + private shouldExcludeFolder(standardFolder: StandardFolder | null): boolean { + return ( + standardFolder !== null && + standardFolder !== StandardFolder.SENT && + standardFolder !== StandardFolder.INBOX + ); + } +} diff --git a/packages/twenty-server/src/modules/messaging/message-folder-manager/interfaces/message-folder-driver.interface.ts b/packages/twenty-server/src/modules/messaging/message-folder-manager/interfaces/message-folder-driver.interface.ts new file mode 100644 index 0000000000..8b22b31885 --- /dev/null +++ b/packages/twenty-server/src/modules/messaging/message-folder-manager/interfaces/message-folder-driver.interface.ts @@ -0,0 +1,16 @@ +import { type ConnectedAccountWorkspaceEntity } from 'src/modules/connected-account/standard-objects/connected-account.workspace-entity'; +import { type MessageFolderWorkspaceEntity } from 'src/modules/messaging/common/standard-objects/message-folder.workspace-entity'; + +export type MessageFolder = Pick< + MessageFolderWorkspaceEntity, + 'name' | 'isSynced' | 'isSentFolder' | 'externalId' +>; + +export interface MessageFolderDriver { + getAllMessageFolders( + connectedAccount: Pick< + ConnectedAccountWorkspaceEntity, + 'provider' | 'refreshToken' | 'id' | 'handle' | 'connectionParameters' + >, + ): Promise; +} diff --git a/packages/twenty-server/src/modules/messaging/message-folder-manager/messaging-folder-sync-manager.module.ts b/packages/twenty-server/src/modules/messaging/message-folder-manager/messaging-folder-sync-manager.module.ts new file mode 100644 index 0000000000..f31bea3776 --- /dev/null +++ b/packages/twenty-server/src/modules/messaging/message-folder-manager/messaging-folder-sync-manager.module.ts @@ -0,0 +1,34 @@ +import { Module } from '@nestjs/common'; +import { TypeOrmModule } from '@nestjs/typeorm'; + +import { FeatureFlagModule } from 'src/engine/core-modules/feature-flag/feature-flag.module'; +import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity'; +import { DataSourceModule } from 'src/engine/metadata-modules/data-source/data-source.module'; +import { WorkspaceDataSourceModule } from 'src/engine/workspace-datasource/workspace-datasource.module'; +import { SyncMessageFoldersService } from 'src/modules/messaging/message-folder-manager/services/sync-message-folders.service'; +import { GmailGetAllFoldersService } from 'src/modules/messaging/message-folder-manager/drivers/gmail/gmail-get-all-folders.service'; +import { ImapGetAllFoldersService } from 'src/modules/messaging/message-folder-manager/drivers/imap/imap-get-all-folders.service'; +import { MicrosoftGetAllFoldersService } from 'src/modules/messaging/message-folder-manager/drivers/microsoft/microsoft-get-all-folders.service'; +import { MessagingGmailDriverModule } from 'src/modules/messaging/message-import-manager/drivers/gmail/messaging-gmail-driver.module'; +import { MessagingIMAPDriverModule } from 'src/modules/messaging/message-import-manager/drivers/imap/messaging-imap-driver.module'; +import { MessagingMicrosoftDriverModule } from 'src/modules/messaging/message-import-manager/drivers/microsoft/messaging-microsoft-driver.module'; + +@Module({ + imports: [ + FeatureFlagModule, + WorkspaceDataSourceModule, + DataSourceModule, + TypeOrmModule.forFeature([Workspace]), + MessagingGmailDriverModule, + MessagingMicrosoftDriverModule, + MessagingIMAPDriverModule, + ], + providers: [ + SyncMessageFoldersService, + GmailGetAllFoldersService, + ImapGetAllFoldersService, + MicrosoftGetAllFoldersService, + ], + exports: [SyncMessageFoldersService], +}) +export class MessagingFolderSyncManagerModule {} diff --git a/packages/twenty-server/src/modules/messaging/message-folder-manager/services/sync-message-folders.service.ts b/packages/twenty-server/src/modules/messaging/message-folder-manager/services/sync-message-folders.service.ts new file mode 100644 index 0000000000..f02fb279b1 --- /dev/null +++ b/packages/twenty-server/src/modules/messaging/message-folder-manager/services/sync-message-folders.service.ts @@ -0,0 +1,181 @@ +import { Injectable } from '@nestjs/common'; + +import { ConnectedAccountProvider } from 'twenty-shared/types'; +import { isDefined } from 'twenty-shared/utils'; +import { v4 } from 'uuid'; + +import { MessageFolder } from 'src/modules/messaging/message-folder-manager/interfaces/message-folder-driver.interface'; + +import { FeatureFlagService } from 'src/engine/core-modules/feature-flag/services/feature-flag.service'; +import { type WorkspaceEntityManager } from 'src/engine/twenty-orm/entity-manager/workspace-entity-manager'; +import { WorkspaceRepository } from 'src/engine/twenty-orm/repository/workspace.repository'; +import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager'; +import { type MessageChannelWorkspaceEntity } from 'src/modules/messaging/common/standard-objects/message-channel.workspace-entity'; +import { type MessageFolderWorkspaceEntity } from 'src/modules/messaging/common/standard-objects/message-folder.workspace-entity'; +import { GmailGetAllFoldersService } from 'src/modules/messaging/message-folder-manager/drivers/gmail/gmail-get-all-folders.service'; +import { ImapGetAllFoldersService } from 'src/modules/messaging/message-folder-manager/drivers/imap/imap-get-all-folders.service'; +import { MicrosoftGetAllFoldersService } from 'src/modules/messaging/message-folder-manager/drivers/microsoft/microsoft-get-all-folders.service'; +import { MessageFolderName } from 'src/modules/messaging/message-import-manager/drivers/microsoft/types/folders'; + +type SyncMessageFoldersInput = { + workspaceId: string; + messageChannelId: string; + connectedAccount: MessageChannelWorkspaceEntity['connectedAccount']; + manager: WorkspaceEntityManager; +}; + +@Injectable() +export class SyncMessageFoldersService { + constructor( + private readonly twentyORMGlobalManager: TwentyORMGlobalManager, + private readonly featureFlagService: FeatureFlagService, + private readonly gmailGetAllFoldersService: GmailGetAllFoldersService, + private readonly microsoftGetAllFoldersService: MicrosoftGetAllFoldersService, + private readonly imapGetAllFoldersService: ImapGetAllFoldersService, + ) {} + + async syncMessageFolders(input: SyncMessageFoldersInput): Promise { + const { workspaceId, messageChannelId, connectedAccount, manager } = input; + + const folders = await this.discoverAllFolders(connectedAccount); + + await this.upsertDiscoveredFolders({ + workspaceId, + messageChannelId, + folders, + manager, + }); + } + + private async upsertDiscoveredFolders({ + workspaceId, + messageChannelId, + folders, + manager, + }: { + workspaceId: string; + messageChannelId: string; + folders: MessageFolder[]; + manager: WorkspaceEntityManager; + }): Promise { + const messageFolderRepository = + await this.twentyORMGlobalManager.getRepositoryForWorkspace( + workspaceId, + 'messageFolder', + ); + + const existingFolderMap = await this.buildExistingFolderMap({ + messageChannelId, + messageFolderRepository, + }); + + for (const folder of folders) { + const existingFolder = this.findExistingFolderInMap( + existingFolderMap, + folder, + ); + + if (existingFolder) { + await messageFolderRepository.update( + existingFolder.id, + { + name: folder.name, + isSynced: folder.isSynced, + isSentFolder: folder.isSentFolder, + externalId: folder.externalId, + }, + manager, + ); + } else { + await messageFolderRepository.save( + { + id: v4(), + messageChannelId, + name: folder.name, + syncCursor: '', + isSynced: folder.isSynced, + isSentFolder: folder.isSentFolder, + externalId: folder.externalId, + }, + {}, + manager, + ); + } + } + } + + async discoverAllFolders( + connectedAccount: MessageChannelWorkspaceEntity['connectedAccount'], + ): Promise { + switch (connectedAccount.provider) { + case ConnectedAccountProvider.GOOGLE: + return await this.gmailGetAllFoldersService.getAllMessageFolders( + connectedAccount, + ); + case ConnectedAccountProvider.MICROSOFT: + return await this.microsoftGetAllFoldersService.getAllMessageFolders( + connectedAccount, + ); + case ConnectedAccountProvider.IMAP_SMTP_CALDAV: + return await this.imapGetAllFoldersService.getAllMessageFolders( + connectedAccount, + ); + default: + throw new Error( + `Provider ${connectedAccount.provider} is not supported`, + ); + } + } + + private async buildExistingFolderMap({ + messageChannelId, + messageFolderRepository, + }: { + messageChannelId: string; + messageFolderRepository: WorkspaceRepository; + }): Promise> { + const existingFolders = await messageFolderRepository.find({ + where: { messageChannelId }, + }); + + const existingFolderMap = new Map(); + + for (const existingFolder of existingFolders) { + if (isDefined(existingFolder.externalId)) { + existingFolderMap.set(existingFolder.externalId, existingFolder); + } + existingFolderMap.set(existingFolder.name, existingFolder); + } + + return existingFolderMap; + } + + private findExistingFolderInMap( + existingFolderMap: Map, + folder: MessageFolder, + ): MessageFolderWorkspaceEntity | undefined { + if (isDefined(folder.externalId)) { + const existingFolder = existingFolderMap.get(folder.externalId); + + if (existingFolder) { + return existingFolder; + } + } + + const legacyFolderName = this.getLegacyFolderName(folder); + + return existingFolderMap.get(legacyFolderName); + } + + private getLegacyFolderName(folder: MessageFolder): string { + if (folder.isSynced && !folder.isSentFolder) { + return MessageFolderName.INBOX; + } + + if (folder.isSynced && folder.isSentFolder) { + return MessageFolderName.SENT_ITEMS; + } + + return folder.name; + } +} diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/messaging-gmail-driver.module.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/messaging-gmail-driver.module.ts index 077f3862d3..7b4fe879ac 100644 --- a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/messaging-gmail-driver.module.ts +++ b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/messaging-gmail-driver.module.ts @@ -12,13 +12,13 @@ import { EmailAliasManagerModule } from 'src/modules/connected-account/email-ali import { OAuth2ClientManagerModule } from 'src/modules/connected-account/oauth2-client-manager/oauth2-client-manager.module'; import { MessagingCommonModule } from 'src/modules/messaging/common/messaging-common.module'; import { GmailClientProvider } from 'src/modules/messaging/message-import-manager/drivers/gmail/providers/gmail-client.provider'; +import { OAuth2ClientProvider } from 'src/modules/messaging/message-import-manager/drivers/gmail/providers/oauth2-client.provider'; import { GmailFetchByBatchService } from 'src/modules/messaging/message-import-manager/drivers/gmail/services/gmail-fetch-by-batch.service'; import { GmailGetHistoryService } from 'src/modules/messaging/message-import-manager/drivers/gmail/services/gmail-get-history.service'; import { GmailGetMessageListService } from 'src/modules/messaging/message-import-manager/drivers/gmail/services/gmail-get-message-list.service'; import { GmailGetMessagesService } from 'src/modules/messaging/message-import-manager/drivers/gmail/services/gmail-get-messages.service'; import { GmailHandleErrorService } from 'src/modules/messaging/message-import-manager/drivers/gmail/services/gmail-handle-error.service'; import { MessageParticipantManagerModule } from 'src/modules/messaging/message-participant-manager/message-participant-manager.module'; -import { OAuth2ClientProvider } from 'src/modules/messaging/message-import-manager/drivers/gmail/providers/oauth2-client.provider'; @Module({ imports: [ @@ -49,6 +49,7 @@ import { OAuth2ClientProvider } from 'src/modules/messaging/message-import-manag GmailGetMessageListService, GmailClientProvider, OAuth2ClientProvider, + GmailHandleErrorService, ], }) export class MessagingGmailDriverModule {} diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/services/gmail-get-message-list.service.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/services/gmail-get-message-list.service.ts index 8d8031e0db..e9bdfb0670 100644 --- a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/services/gmail-get-message-list.service.ts +++ b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/services/gmail-get-message-list.service.ts @@ -36,7 +36,10 @@ export class GmailGetMessageListService { ConnectedAccountWorkspaceEntity, 'provider' | 'refreshToken' | 'id' | 'handle' >, - messageFolders: Pick[], + messageFolders: Pick< + MessageFolderWorkspaceEntity, + 'name' | 'externalId' | 'isSynced' + >[], ): Promise { const gmailClient = await this.gmailClientProvider.getGmailClient(connectedAccount); @@ -45,7 +48,7 @@ export class GmailGetMessageListService { let hasMoreMessages = true; const messageExternalIds: string[] = []; - const excludedCategories = this.comptuteExcludedCategories(messageFolders); + const excludedCategories = this.computeExcludedCategories(messageFolders); while (hasMoreMessages) { const messageList = await gmailClient.users.messages @@ -54,6 +57,7 @@ export class GmailGetMessageListService { maxResults: MESSAGING_GMAIL_USERS_MESSAGES_LIST_MAX_RESULT, pageToken, q: computeGmailCategoryExcludeSearchFilter(excludedCategories), + labelIds: this.getCustomLabelIds(messageFolders), }) .catch((error) => { this.logger.error( @@ -181,8 +185,8 @@ export class GmailGetMessageListService { ]; } - private comptuteExcludedCategories( - messageFolders: Pick[], + private computeExcludedCategories( + messageFolders: Pick[], ) { const includedDefaultCategories = messageFolders .map((messageFolder) => @@ -199,11 +203,11 @@ export class GmailGetMessageListService { private async getEmailIdsFromExcludedCategories( gmailClient: gmailV1.Gmail, lastSyncHistoryId: string, - messageFolders: Pick[], + messageFolders: Pick[], ): Promise { const emailIds: string[] = []; - const excludedCategories = this.comptuteExcludedCategories(messageFolders); + const excludedCategories = this.computeExcludedCategories(messageFolders); for (const category of excludedCategories) { const { history } = await this.gmailGetHistoryService.getHistory( @@ -225,4 +229,23 @@ export class GmailGetMessageListService { return emailIds; } + + private getCustomLabelIds( + messageFolders: Pick< + MessageFolderWorkspaceEntity, + 'name' | 'externalId' | 'isSynced' + >[], + ): string[] | undefined { + const customLabelIds = messageFolders + .filter( + (folder) => + folder.externalId && + folder.isSynced && + !mapGmailDefaultFolderToCategoryOrUndefined(folder.name), + ) + .map((folder) => folder.externalId) + .filter((id): id is string => !!id); + + return customLabelIds.length > 0 ? customLabelIds : undefined; + } } diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/imap/messaging-imap-driver.module.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/imap/messaging-imap-driver.module.ts index ecd07e533c..f6974af96a 100644 --- a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/imap/messaging-imap-driver.module.ts +++ b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/imap/messaging-imap-driver.module.ts @@ -46,6 +46,7 @@ import { MessageParticipantManagerModule } from 'src/modules/messaging/message-p ImapGetMessagesService, ImapGetMessageListService, ImapClientProvider, + ImapFindSentFolderService, ], }) export class MessagingIMAPDriverModule {} diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/imap/services/imap-get-message-list.service.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/imap/services/imap-get-message-list.service.ts index f997d27e1a..288834d288 100644 --- a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/imap/services/imap-get-message-list.service.ts +++ b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/imap/services/imap-get-message-list.service.ts @@ -4,10 +4,8 @@ import { type ImapFlow } from 'imapflow'; import { type MessageFolderWorkspaceEntity } from 'src/modules/messaging/common/standard-objects/message-folder.workspace-entity'; import { ImapClientProvider } from 'src/modules/messaging/message-import-manager/drivers/imap/providers/imap-client.provider'; -import { ImapFindSentFolderService } from 'src/modules/messaging/message-import-manager/drivers/imap/services/imap-find-sent-folder.service'; import { ImapHandleErrorService } from 'src/modules/messaging/message-import-manager/drivers/imap/services/imap-handle-error.service'; import { ImapIncrementalSyncService } from 'src/modules/messaging/message-import-manager/drivers/imap/services/imap-incremental-sync.service'; -import { MessageFolderName } from 'src/modules/messaging/message-import-manager/drivers/imap/types/folders'; import { createSyncCursor } from 'src/modules/messaging/message-import-manager/drivers/imap/utils/create-sync-cursor.util'; import { extractMailboxState } from 'src/modules/messaging/message-import-manager/drivers/imap/utils/extract-mailbox-state.util'; import { @@ -26,7 +24,6 @@ export class ImapGetMessageListService { constructor( private readonly imapClientProvider: ImapClientProvider, - private readonly imapFindSentFolderService: ImapFindSentFolderService, private readonly imapIncrementalSyncService: ImapIncrementalSyncService, private readonly imapHandleErrorService: ImapHandleErrorService, ) {} @@ -43,19 +40,11 @@ export class ImapGetMessageListService { for (const folder of messageFolders) { this.logger.log(`Processing folder: ${folder.name}`); - const folderName = await this.getFolderName(client, folder.name); - - if (!folderName) { - this.logger.warn( - `No IMAP folder found for message folder: ${folder.name}`, - ); - continue; - } try { const response = await this.getMessageList( client, - folderName, + folder.name, folder, ); @@ -65,7 +54,7 @@ export class ImapGetMessageListService { }); } catch (error) { this.logger.warn( - `Error fetching from folder ${folder.name} (${folderName}): ${error.message}. Continuing with other folders.`, + `Error fetching from folder ${folder.name}: ${error.message}. Continuing with other folders.`, ); result.push({ @@ -130,30 +119,6 @@ export class ImapGetMessageListService { }; } - private async getFolderName( - client: ImapFlow, - folderName: string, - ): Promise { - if (folderName === MessageFolderName.INBOX) { - return 'INBOX'; - } - - if (folderName === MessageFolderName.SENT_ITEMS) { - const sentFolder = - await this.imapFindSentFolderService.findSentFolder(client); - - if (!sentFolder) { - this.logger.warn('SENT folder not found, skipping'); - - return null; - } - - return sentFolder; - } - - return folderName; - } - private async getMessagesFromFolder( client: ImapFlow, folder: string, diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/imap/utils/__tests__/get-standard-folder-by-regex.util.spec.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/imap/utils/__tests__/get-standard-folder-by-regex.util.spec.ts new file mode 100644 index 0000000000..c4806ebe9c --- /dev/null +++ b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/imap/utils/__tests__/get-standard-folder-by-regex.util.spec.ts @@ -0,0 +1,450 @@ +import { StandardFolder } from 'src/modules/messaging/message-import-manager/drivers/types/standard-folder'; +import { getStandardFolderByRegex } from 'src/modules/messaging/message-import-manager/drivers/utils/get-standard-folder-by-regex'; + +function testFolderMatches( + variants: string[], + expectedStandardFolder: StandardFolder, +) { + variants.forEach((variant) => { + const result = getStandardFolderByRegex(variant); + + expect(result).toBe(expectedStandardFolder); + }); +} + +describe('getStandardFolderByRegex', () => { + describe('INBOX folder detection', () => { + it('matches English variants', () => { + const englishVariants = [ + 'Inbox', + 'Mail', + 'Messages', + 'Message', + 'Received', + ]; + + testFolderMatches(englishVariants, StandardFolder.INBOX); + }); + + it('matches French variants', () => { + const frenchVariants = [ + 'Boîte de réception', + 'Courrier entrant', + 'Messages reçus', + 'Réception', + ]; + + testFolderMatches(frenchVariants, StandardFolder.INBOX); + }); + + it('matches German variants', () => { + const germanVariants = [ + 'Posteingang', + 'Eingang', + 'Eingangsmails', + 'Empfangen', + ]; + + testFolderMatches(germanVariants, StandardFolder.INBOX); + }); + + it('matches Spanish variants', () => { + const spanishVariants = [ + 'Bandeja de entrada', + 'Entrada', + 'Correo entrante', + 'Recibidos', + ]; + + testFolderMatches(spanishVariants, StandardFolder.INBOX); + }); + + it('matches Portuguese variants', () => { + const portugueseVariants = [ + 'Caixa de entrada', + 'Entrada', + 'Correio de entrada', + 'Recebidos', + ]; + + testFolderMatches(portugueseVariants, StandardFolder.INBOX); + }); + + it('matches Italian variants', () => { + const italianVariants = [ + 'Posta in arrivo', + 'Arrivo', + 'Casella postale', + 'Ricevuti', + ]; + + testFolderMatches(italianVariants, StandardFolder.INBOX); + }); + + it('matches Korean variants', () => { + const koreanVariants = ['받은편지함', '수신함', '받은메일']; + + testFolderMatches(koreanVariants, StandardFolder.INBOX); + }); + + it('matches Japanese variants', () => { + const japaneseVariants = ['受信トレイ', '受信箱', '受信メール']; + + testFolderMatches(japaneseVariants, StandardFolder.INBOX); + }); + + it('matches Polish variants', () => { + const polishVariants = [ + 'Odebrane', + 'Skrzynka odbiorcza', + 'Wiadomości przychodzące', + ]; + + testFolderMatches(polishVariants, StandardFolder.INBOX); + }); + + it('matches Russian variants', () => { + const russianVariants = [ + 'Входящие', + 'Папка входящих', + 'Полученные сообщения', + 'Полученные письма', + ]; + + testFolderMatches(russianVariants, StandardFolder.INBOX); + }); + + it('matches Gmail special folder', () => { + const gmailVariants = ['[Gmail]/Inbox', '[Gmail]\\Inbox']; + + testFolderMatches(gmailVariants, StandardFolder.INBOX); + }); + }); + + describe('DRAFTS folder detection', () => { + it('matches English variants', () => { + const englishVariants = [ + 'Drafts', + 'Draft', + 'Draft Items', + 'Draft Mail', + 'Draft Messages', + ]; + + testFolderMatches(englishVariants, StandardFolder.DRAFTS); + }); + + it('matches French variants', () => { + const frenchVariants = ['Brouillons', 'Éléments brouillons']; + + testFolderMatches(frenchVariants, StandardFolder.DRAFTS); + }); + + it('matches German variants', () => { + const germanVariants = ['Entwürfe', 'Entwurf']; + + testFolderMatches(germanVariants, StandardFolder.DRAFTS); + }); + + it('matches Spanish variants', () => { + const spanishVariants = ['Borradores', 'Elementos borrador']; + + testFolderMatches(spanishVariants, StandardFolder.DRAFTS); + }); + + it('matches Portuguese variants', () => { + const portugueseVariants = ['Rascunhos', 'Itens rascunho']; + + testFolderMatches(portugueseVariants, StandardFolder.DRAFTS); + }); + + it('matches Italian variants', () => { + const italianVariants = ['Bozze', 'Bozze salvate']; + + testFolderMatches(italianVariants, StandardFolder.DRAFTS); + }); + + it('matches Korean variants', () => { + const koreanVariants = ['임시보관함', '초안']; + + testFolderMatches(koreanVariants, StandardFolder.DRAFTS); + }); + + it('matches Japanese variants', () => { + const japaneseVariants = ['下書き', '草稿']; + + testFolderMatches(japaneseVariants, StandardFolder.DRAFTS); + }); + + it('matches Polish variants', () => { + const polishVariants = ['Wersje robocze', 'Szkice']; + + testFolderMatches(polishVariants, StandardFolder.DRAFTS); + }); + + it('matches Russian variants', () => { + const russianVariants = [ + 'Черновики', + 'Черновые сообщения', + 'Неотправленные', + ]; + + testFolderMatches(russianVariants, StandardFolder.DRAFTS); + }); + + it('matches Gmail special folder', () => { + const gmailVariants = ['[Gmail]/Drafts', '[Gmail]\\Drafts']; + + testFolderMatches(gmailVariants, StandardFolder.DRAFTS); + }); + }); + + describe('SENT folder detection', () => { + it('matches English variants', () => { + const englishVariants = [ + 'Sent', + 'Sent Items', + 'Sent Mail', + 'Sent Messages', + 'sent-elements', + ]; + + testFolderMatches(englishVariants, StandardFolder.SENT); + }); + + it('matches French variants', () => { + const frenchVariants = ['Envoyés', 'Éléments envoyés', 'Objets envoyés']; + + testFolderMatches(frenchVariants, StandardFolder.SENT); + }); + + it('matches German variants', () => { + const germanVariants = ['Gesendet', 'Gesendete Elemente']; + + testFolderMatches(germanVariants, StandardFolder.SENT); + }); + + it('matches Spanish variants', () => { + const spanishVariants = ['Enviados', 'Elementos enviados']; + + testFolderMatches(spanishVariants, StandardFolder.SENT); + }); + + it('matches Portuguese variants', () => { + const portugueseVariants = ['Itens enviados']; + + testFolderMatches(portugueseVariants, StandardFolder.SENT); + }); + + it('matches Italian variants', () => { + const italianVariants = ['Posta inviata', 'Inviati']; + + testFolderMatches(italianVariants, StandardFolder.SENT); + }); + + it('matches Korean variant', () => { + const koreanVariants = ['보낸편지함']; + + testFolderMatches(koreanVariants, StandardFolder.SENT); + }); + + it('matches Japanese variants', () => { + const japaneseVariants = ['送信済みメール', '送信済み']; + + testFolderMatches(japaneseVariants, StandardFolder.SENT); + }); + + it('matches Polish variants', () => { + const polishVariants = ['Wysłane', 'Elementy wysłane']; + + testFolderMatches(polishVariants, StandardFolder.SENT); + }); + + it('matches Russian variants', () => { + const russianVariants = [ + 'Отправленные', + 'Отправленные письма', + 'Отправленные сообщения', + 'Исходящие', + ]; + + testFolderMatches(russianVariants, StandardFolder.SENT); + }); + + it('matches Gmail special folder', () => { + const gmailVariants = ['[Gmail]/Sent Mail', '[Gmail]\\Sent Mail']; + + testFolderMatches(gmailVariants, StandardFolder.SENT); + }); + + it('does not match unrelated folders', () => { + const unrelatedFolders = [ + 'Inbox', + 'Drafts', + 'Trash', + 'Archive', + 'Junk', + 'Important', + 'RandomFolder', + ]; + + unrelatedFolders.forEach((folder) => { + const result = getStandardFolderByRegex(folder); + + expect(result).not.toBe(StandardFolder.SENT); + }); + }); + }); + + describe('TRASH folder detection', () => { + it('matches English variants', () => { + const englishVariants = [ + 'Trash', + 'Deleted Items', + 'Deleted Messages', + 'Bin', + 'Recycle Bin', + ]; + + testFolderMatches(englishVariants, StandardFolder.TRASH); + }); + + it('matches French variants', () => { + const frenchVariants = ['Corbeille', 'Supprimés', 'Éléments supprimés']; + + testFolderMatches(frenchVariants, StandardFolder.TRASH); + }); + + it('matches German variants', () => { + const germanVariants = ['Gelöscht', 'Gelöschte Elemente', 'Papierkorb']; + + testFolderMatches(germanVariants, StandardFolder.TRASH); + }); + + it('matches Spanish variants', () => { + const spanishVariants = [ + 'Papelera', + 'Eliminados', + 'Elementos eliminados', + ]; + + testFolderMatches(spanishVariants, StandardFolder.TRASH); + }); + + it('matches Portuguese variants', () => { + const portugueseVariants = ['Lixeira', 'Itens excluídos']; + + testFolderMatches(portugueseVariants, StandardFolder.TRASH); + }); + + it('matches Italian variants', () => { + const italianVariants = ['Cestino', 'Posta eliminata', 'Eliminati']; + + testFolderMatches(italianVariants, StandardFolder.TRASH); + }); + + it('matches Korean variants', () => { + const koreanVariants = ['휴지통', '삭제된편지함']; + + testFolderMatches(koreanVariants, StandardFolder.TRASH); + }); + + it('matches Japanese variants', () => { + const japaneseVariants = ['ごみ箱', '削除済み', '削除済みメール']; + + testFolderMatches(japaneseVariants, StandardFolder.TRASH); + }); + + it('matches Polish variants', () => { + const polishVariants = ['Kosz', 'Usunięte', 'Elementy usunięte']; + + testFolderMatches(polishVariants, StandardFolder.TRASH); + }); + + it('matches Russian variants', () => { + const russianVariants = ['Удалённые', 'Корзина', 'Удалённые сообщения']; + + testFolderMatches(russianVariants, StandardFolder.TRASH); + }); + + it('matches Gmail special folder', () => { + const gmailVariants = ['[Gmail]/Trash', '[Gmail]\\Trash']; + + testFolderMatches(gmailVariants, StandardFolder.TRASH); + }); + }); + + describe('JUNK/SPAM folder detection', () => { + it('matches English variants', () => { + const englishVariants = [ + 'Spam', + 'Junk Mail', + 'Junk Messages', + 'Bulk Mail', + 'Bulk Messages', + ]; + + testFolderMatches(englishVariants, StandardFolder.JUNK); + }); + + it('matches French variants', () => { + const frenchVariants = ['Indésirables', 'Courrier indésirable', 'Spam']; + + testFolderMatches(frenchVariants, StandardFolder.JUNK); + }); + + it('matches German variants', () => { + const germanVariants = ['Spam', 'Junk Mail', 'Unerwünscht']; + + testFolderMatches(germanVariants, StandardFolder.JUNK); + }); + + it('matches Spanish variants', () => { + const spanishVariants = ['Spam', 'Correo basura', 'No deseado']; + + testFolderMatches(spanishVariants, StandardFolder.JUNK); + }); + + it('matches Portuguese variants', () => { + const portugueseVariants = ['Spam', 'Lixo eletrônico', 'Indesejados']; + + testFolderMatches(portugueseVariants, StandardFolder.JUNK); + }); + + it('matches Italian variants', () => { + const italianVariants = ['Spam', 'Posta indesiderata', 'Indesiderata']; + + testFolderMatches(italianVariants, StandardFolder.JUNK); + }); + + it('matches Korean variants', () => { + const koreanVariants = ['스팸', '정크메일']; + + testFolderMatches(koreanVariants, StandardFolder.JUNK); + }); + + it('matches Japanese variants', () => { + const japaneseVariants = ['スパム', '迷惑メール']; + + testFolderMatches(japaneseVariants, StandardFolder.JUNK); + }); + + it('matches Polish variants', () => { + const polishVariants = ['Spam', 'Niechciane', 'Śmieci']; + + testFolderMatches(polishVariants, StandardFolder.JUNK); + }); + + it('matches Russian variants', () => { + const russianVariants = ['Спам', 'Нежелательные', 'Мусор']; + + testFolderMatches(russianVariants, StandardFolder.JUNK); + }); + + it('matches Gmail special folder', () => { + const gmailVariants = ['[Gmail]/Spam', '[Gmail]\\Spam']; + + testFolderMatches(gmailVariants, StandardFolder.JUNK); + }); + }); +}); diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/imap/utils/get-sent-folder-candidates-by-regex.util.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/imap/utils/get-sent-folder-candidates-by-regex.util.ts index 7f0d1466fd..3bbd05f63b 100644 --- a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/imap/utils/get-sent-folder-candidates-by-regex.util.ts +++ b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/imap/utils/get-sent-folder-candidates-by-regex.util.ts @@ -1,50 +1,17 @@ import { type ListResponse } from 'imapflow'; +import { StandardFolder } from 'src/modules/messaging/message-import-manager/drivers/types/standard-folder'; +import { getStandardFolderByRegex } from 'src/modules/messaging/message-import-manager/drivers/utils/get-standard-folder-by-regex'; + export function getImapSentFolderCandidatesByRegex( list: ListResponse[], ): string[] { - const sentFolderPattern = new RegExp( - [ - // EN - 'sent([\\s_-]?(items|mail|messages|elements))?', - // FR - 'envoy[éê]s?', - '[ée]l[ée]ments[\\s_-]?envoy[éê]s', - // DE - 'gesendet', - 'gesendete[\\s_-]?elemente', - // ES - 'enviados?', - 'elementos[\\s_-]?enviados', - // PT - 'itens[\\s_-]?enviados', - // IT - 'posta[\\s_-]?inviata', - 'inviati', - // KO - '보낸편지함', - // JA - '送信済みメール', - '送信済み', - // PL - 'elementy[\\s_-]?wysłane', - 'wysłane', - // RU - 'отправленные', - 'отправленные[\\s_-]?(сообщения|письма)?', - 'исходящие', - // GMAIL - '\\[gmail\\][\\/]+sent[\\s_-]?mail', - ] - .map((s) => `(${s})`) - .join('|'), - 'i', - ); - - const regexCandidateFolders = []; + const regexCandidateFolders: string[] = []; for (const folder of list) { - if (sentFolderPattern.test(folder.path)) { + const standardFolder = getStandardFolderByRegex(folder.path); + + if (standardFolder === StandardFolder.SENT) { regexCandidateFolders.push(folder.path); } } diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/messaging-microsoft-driver.module.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/messaging-microsoft-driver.module.ts index 8835d9a1f3..24ac407b7b 100644 --- a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/messaging-microsoft-driver.module.ts +++ b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/messaging-microsoft-driver.module.ts @@ -27,6 +27,7 @@ import { MicrosoftGetMessageListService } from './services/microsoft-get-message MicrosoftClientProvider, MicrosoftGetMessageListService, MicrosoftGetMessagesService, + MicrosoftFetchByBatchService, MicrosoftHandleErrorService, MicrosoftOAuth2ClientManagerService, @@ -35,6 +36,7 @@ import { MicrosoftGetMessageListService } from './services/microsoft-get-message MicrosoftGetMessageListService, MicrosoftClientProvider, MicrosoftGetMessagesService, + MicrosoftHandleErrorService, ], }) export class MessagingMicrosoftDriverModule {} diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-get-message-list.service.dev.spec.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-get-message-list.service.dev.spec.ts index ee7d703c8a..90e70e72ce 100644 --- a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-get-message-list.service.dev.spec.ts +++ b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-get-message-list.service.dev.spec.ts @@ -66,6 +66,9 @@ xdescribe('Microsoft dev tests : get message list service', () => { id: 'inbox-folder-id', name: MessageFolderName.INBOX, syncCursor: 'inbox-sync-cursor', + isSynced: false, + isSentFolder: false, + externalId: null, }, ], }); @@ -91,6 +94,9 @@ xdescribe('Microsoft dev tests : get message list service', () => { id: 'inbox-folder-id', name: MessageFolderName.INBOX, syncCursor: 'inbox-sync-cursor', + isSynced: false, + isSentFolder: false, + externalId: null, }, ], }), @@ -107,6 +113,9 @@ xdescribe('Microsoft dev tests : get message list service', () => { id: 'inbox-folder-id', name: MessageFolderName.INBOX, syncCursor: syncCursor, + isSynced: false, + isSentFolder: false, + externalId: null, }, ], }); @@ -127,6 +136,9 @@ xdescribe('Microsoft dev tests : get message list service', () => { id: 'inbox-folder-id', name: MessageFolderName.INBOX, syncCursor: 'invalid-syncCursor', + isSynced: false, + isSentFolder: false, + externalId: null, }, ], }), diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-get-message-list.service.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-get-message-list.service.ts index e4ac4a0f22..6f06a9151b 100644 --- a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-get-message-list.service.ts +++ b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-get-message-list.service.ts @@ -6,9 +6,7 @@ import { type PageIteratorCallback, } from '@microsoft/microsoft-graph-client'; import { isNonEmptyString } from '@sniptt/guards'; -import { v4 } from 'uuid'; -import { TwentyORMManager } from 'src/engine/twenty-orm/twenty-orm.manager'; import { type ConnectedAccountWorkspaceEntity } from 'src/modules/connected-account/standard-objects/connected-account.workspace-entity'; import { type MessageFolderWorkspaceEntity } from 'src/modules/messaging/common/standard-objects/message-folder.workspace-entity'; import { @@ -17,7 +15,6 @@ import { } from 'src/modules/messaging/message-import-manager/drivers/exceptions/message-import-driver.exception'; import { MicrosoftClientProvider } from 'src/modules/messaging/message-import-manager/drivers/microsoft/providers/microsoft-client.provider'; import { MicrosoftHandleErrorService } from 'src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-handle-error.service'; -import { MessageFolderName } from 'src/modules/messaging/message-import-manager/drivers/microsoft/types/folders'; import { isAccessTokenRefreshingError } from 'src/modules/messaging/message-import-manager/drivers/microsoft/utils/is-access-token-refreshing-error.utils'; import { type GetMessageListsArgs } from 'src/modules/messaging/message-import-manager/types/get-message-lists-args.type'; import { @@ -34,7 +31,6 @@ export class MicrosoftGetMessageListService { constructor( private readonly microsoftClientProvider: MicrosoftClientProvider, private readonly microsoftHandleErrorService: MicrosoftHandleErrorService, - private readonly twentyORMManager: TwentyORMManager, ) {} public async getMessageLists({ @@ -45,46 +41,10 @@ export class MicrosoftGetMessageListService { const result: GetMessageListsResponse = []; if (messageFolders.length === 0) { - // permanent solution: - // throw new MessageImportDriverException( - // `Message channel ${messageChannel.id} has no message folders`, - // MessageImportDriverExceptionCode.NOT_FOUND, - // ); - - // temporary solution: TODO: remove this once we have a permanent solution - // if no folders exist, most probably a first time sync for microsoft - // so we create the folders INBOX and SENTITEMS - // and fill the INBOX with the previous sync cursor - // and for sentitms, we do the full message list fetch - // console.warn( - // `Message channel ${messageChannel.id} has no message folders, most probably a first time`, - // ); - - const messageFolderRepository = - await this.twentyORMManager.getRepository( - 'messageFolder', - ); - - const newFolder = await messageFolderRepository.save({ - id: v4(), - messageChannelId: messageChannel.id, - name: MessageFolderName.INBOX, - syncCursor: messageChannel.syncCursor, - }); - - const response = await this.getMessageList(connectedAccount, { - name: MessageFolderName.INBOX, - syncCursor: messageChannel.syncCursor, - }); - - result.push({ - ...response, - folderId: newFolder.id, - }); - - // we are ok with not synchronizing the legacy connected microsoft accounts. - // so we return an empty array. - return result; + throw new MessageImportDriverException( + `Message channel ${messageChannel.id} has no message folders`, + MessageImportDriverExceptionCode.NOT_FOUND, + ); } for (const folder of messageFolders) { @@ -104,7 +64,10 @@ export class MicrosoftGetMessageListService { ConnectedAccountWorkspaceEntity, 'provider' | 'refreshToken' | 'id' >, - messageFolder: Pick, + messageFolder: Pick< + MessageFolderWorkspaceEntity, + 'name' | 'syncCursor' | 'externalId' + >, ): Promise { const messageExternalIds: string[] = []; const messageExternalIdsToDelete: string[] = []; @@ -112,9 +75,10 @@ export class MicrosoftGetMessageListService { const microsoftClient = await this.microsoftClientProvider.getMicrosoftClient(connectedAccount); + const folderId = messageFolder.externalId || messageFolder.name; const apiUrl = isNonEmptyString(messageFolder.syncCursor) ? messageFolder.syncCursor - : `/me/mailfolders/${messageFolder.name}/messages/delta?$select=id`; + : `/me/mailfolders/${folderId}/messages/delta?$select=id`; const response: PageCollection = await microsoftClient .api(apiUrl) diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/types/standard-folder.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/types/standard-folder.ts new file mode 100644 index 0000000000..71982d1f31 --- /dev/null +++ b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/types/standard-folder.ts @@ -0,0 +1,7 @@ +export enum StandardFolder { + INBOX = 'inbox', + DRAFTS = 'drafts', + SENT = 'sent', + TRASH = 'trash', + JUNK = 'junk', +} diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/utils/get-standard-folder-by-regex.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/utils/get-standard-folder-by-regex.ts new file mode 100644 index 0000000000..ef7fb61fc0 --- /dev/null +++ b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/utils/get-standard-folder-by-regex.ts @@ -0,0 +1,225 @@ +import { StandardFolder } from 'src/modules/messaging/message-import-manager/drivers/types/standard-folder'; + +const FOLDER_REGEX_PATTERNS: Record = { + [StandardFolder.INBOX]: [ + // EN + 'inbox', + '^mail$', + '^messages?$', + 'received', + // FR + 'boîte[\\s_-]?de[\\s_-]?réception', + 'courrier[\\s_-]?entrant', + 'messages[\\s_-]?reçus', + 'réception', + // DE + 'posteingang', + 'eingang', + 'eingangsmails?', + 'empfangen', + // ES + 'bandeja[\\s_-]?de[\\s_-]?entrada', + 'entrada', + 'correo[\\s_-]?entrante', + 'recibidos?', + // PT + 'caixa[\\s_-]?de[\\s_-]?entrada', + 'entrada', + 'correio[\\s_-]?de[\\s_-]?entrada', + 'recebidos?', + // IT + 'posta[\\s_-]?in[\\s_-]?arrivo', + 'arrivo', + 'casella[\\s_-]?postale', + 'ricevuti', + // KO + '받은편지함', + '수신함', + '받은메일', + // JA + '受信トレイ', + '受信箱', + '受信メール', + // PL + 'odebrane', + 'skrzynka[\\s_-]?odbiorcza', + 'wiadomości[\\s_-]?przychodzące', + // RU + 'входящие', + 'папка[\\s_-]?входящих', + 'полученные[\\s_-]?(сообщения|письма)?', + // GMAIL + '\\[gmail\\][\\/]+inbox', + ], + [StandardFolder.DRAFTS]: [ + // EN + 'drafts?', + 'draft[\\s_-]?(items|mail|messages|elements)?', + // FR + 'brouillons?', + '[ée]l[ée]ments[\\s_-]?brouillons?', + // DE + 'entwürfe', + 'entwurf', + // ES + 'borradores?', + 'elementos[\\s_-]?borrador', + // PT + 'rascunhos?', + 'itens[\\s_-]?rascunho', + // IT + 'bozze', + 'bozze[\\s_-]?salvate', + // KO + '임시보관함', + '초안', + // JA + '下書き', + '草稿', + // PL + 'wersje[\\s_-]?robocze', + 'szkice', + // RU + 'черновики', + 'черновые[\\s_-]?(сообщения|письма)?', + 'неотправленные', + // GMAIL + '\\[gmail\\][\\/]+drafts', + ], + [StandardFolder.SENT]: [ + // EN + 'sent([\\s_-]?(items|mail|messages|elements))?', + // FR + 'envoy[éê]s?', + '[ée]l[ée]ments[\\s_-]?envoy[éê]s', + // DE + 'gesendet', + 'gesendete[\\s_-]?elemente', + // ES + 'enviados?', + 'elementos[\\s_-]?enviados', + // PT + 'itens[\\s_-]?enviados', + // IT + 'posta[\\s_-]?inviata', + 'inviati', + // KO + '보낸편지함', + // JA + '送信済みメール', + '送信済み', + // PL + 'elementy[\\s_-]?wysłane', + 'wysłane', + // RU + 'отправленные', + 'отправленные[\\s_-]?(сообщения|письма)?', + 'исходящие', + // GMAIL + '\\[gmail\\][\\/]+sent[\\s_-]?mail', + ], + [StandardFolder.TRASH]: [ + // EN + 'trash', + 'deleted[\\s_-]?(items|messages|mail)?', + 'bin', + 'recycle[\\s_-]?bin', + // FR + 'corbeille', + 'supprim[ée]s', + '[ée]l[ée]ments[\\s_-]?supprim[ée]s', + // DE + 'gelöscht', + 'gelöschte[\\s_-]?elemente', + 'papierkorb', + // ES + 'papelera', + 'eliminados?', + 'elementos[\\s_-]?eliminados', + // PT + 'lixeira', + 'itens[\\s_-]?excluídos', + // IT + 'cestino', + 'posta[\\s_-]?eliminata', + 'eliminati', + // KO + '휴지통', + '삭제된편지함', + // JA + 'ごみ箱', + '削除済み', + '削除済みメール', + // PL + 'kosz', + 'usunięte', + 'elementy[\\s_-]?usunięte', + // RU + 'удалённые', + 'корзина', + 'удалённые[\\s_-]?(сообщения|письма)?', + // GMAIL + '\\[gmail\\][\\/]+trash', + ], + [StandardFolder.JUNK]: [ + // EN + 'spam', + 'junk[\\s_-]?(mail|messages|email)?', + 'bulk[\\s_-]?(mail|messages)?', + // FR + 'indésirables', + 'courrier[\\s_-]?indésirable', + 'spam', + // DE + 'spam', + 'junk[\\s_-]?mail', + 'unerwünscht', + // ES + 'spam', + 'correo[\\s_-]?basura', + 'no[\\s_-]?deseado', + // PT + 'spam', + 'lixo[\\s_-]?eletrônico', + 'indesejados', + // IT + 'spam', + 'posta[\\s_-]?indesiderata', + 'indesiderata', + // KO + '스팸', + '정크메일', + // JA + 'スパム', + '迷惑メール', + // PL + 'spam', + 'niechciane', + 'śmieci', + // RU + 'спам', + 'нежелательные', + 'мусор', + // GMAIL + '\\[gmail\\][\\/]+spam', + ], +}; + +const CACHED_REGEX_PATTERNS = Object.fromEntries( + Object.entries(FOLDER_REGEX_PATTERNS).map(([standardFolder, patterns]) => [ + standardFolder, + new RegExp(patterns.map((s) => `(${s})`).join('|'), 'i'), + ]), +); + +export function getStandardFolderByRegex( + folderName: string, +): StandardFolder | null { + for (const [standardFolder, regex] of Object.entries(CACHED_REGEX_PATTERNS)) { + if (regex.test(folderName)) { + return standardFolder as StandardFolder; + } + } + + return null; +} diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/messaging-import-manager.module.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/messaging-import-manager.module.ts index bf468ae355..b216a1e1b6 100644 --- a/packages/twenty-server/src/modules/messaging/message-import-manager/messaging-import-manager.module.ts +++ b/packages/twenty-server/src/modules/messaging/message-import-manager/messaging-import-manager.module.ts @@ -39,6 +39,7 @@ import { MessagingMessagesImportService } from 'src/modules/messaging/message-im import { MessagingSaveMessagesAndEnqueueContactCreationService } from 'src/modules/messaging/message-import-manager/services/messaging-save-messages-and-enqueue-contact-creation.service'; import { MessagingSendMessageService } from 'src/modules/messaging/message-import-manager/services/messaging-send-message.service'; import { MessageParticipantManagerModule } from 'src/modules/messaging/message-participant-manager/message-participant-manager.module'; +import { MessagingFolderSyncManagerModule } from 'src/modules/messaging/message-folder-manager/messaging-folder-sync-manager.module'; import { MessagingMonitoringModule } from 'src/modules/messaging/monitoring/messaging-monitoring.module'; @Module({ imports: [ @@ -57,6 +58,7 @@ import { MessagingMonitoringModule } from 'src/modules/messaging/monitoring/mess EmailAliasManagerModule, FeatureFlagModule, MessageParticipantManagerModule, + MessagingFolderSyncManagerModule, MessagingMonitoringModule, MessagingMessageCleanerModule, WorkspaceEventEmitterModule, diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-get-message-list.service.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-get-message-list.service.ts index 08b3b86249..3b67bbf7f0 100644 --- a/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-get-message-list.service.ts +++ b/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-get-message-list.service.ts @@ -3,6 +3,7 @@ import { Injectable } from '@nestjs/common'; import { ConnectedAccountProvider } from 'twenty-shared/types'; import { type MessageChannelWorkspaceEntity } from 'src/modules/messaging/common/standard-objects/message-channel.workspace-entity'; +import { MessageFolderWorkspaceEntity } from 'src/modules/messaging/common/standard-objects/message-folder.workspace-entity'; import { GmailGetMessageListService } from 'src/modules/messaging/message-import-manager/drivers/gmail/services/gmail-get-message-list.service'; import { ImapGetMessageListService } from 'src/modules/messaging/message-import-manager/drivers/imap/services/imap-get-message-list.service'; import { MicrosoftGetMessageListService } from 'src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-get-message-list.service'; @@ -12,6 +13,11 @@ import { } from 'src/modules/messaging/message-import-manager/exceptions/message-import.exception'; import { type GetMessageListsResponse } from 'src/modules/messaging/message-import-manager/types/get-message-lists-response.type'; +type MessageFolder = Pick< + MessageFolderWorkspaceEntity, + 'name' | 'isSynced' | 'isSentFolder' | 'externalId' | 'syncCursor' | 'id' +>; + @Injectable() export class MessagingGetMessageListService { constructor( @@ -22,25 +28,26 @@ export class MessagingGetMessageListService { public async getMessageLists( messageChannel: MessageChannelWorkspaceEntity, + messageFoldersToSync: MessageFolder[], ): Promise { switch (messageChannel.connectedAccount.provider) { case ConnectedAccountProvider.GOOGLE: return await this.gmailGetMessageListService.getMessageLists({ messageChannel, connectedAccount: messageChannel.connectedAccount, - messageFolders: messageChannel.messageFolders, + messageFolders: messageFoldersToSync, }); case ConnectedAccountProvider.MICROSOFT: return this.microsoftGetMessageListService.getMessageLists({ messageChannel, connectedAccount: messageChannel.connectedAccount, - messageFolders: messageChannel.messageFolders, + messageFolders: messageFoldersToSync, }); case ConnectedAccountProvider.IMAP_SMTP_CALDAV: { return await this.imapGetMessageListService.getMessageLists({ messageChannel, connectedAccount: messageChannel.connectedAccount, - messageFolders: messageChannel.messageFolders, + messageFolders: messageFoldersToSync, }); } default: diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-message-list-fetch.service.spec.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-message-list-fetch.service.spec.ts index 1f86761d3b..1899591efd 100644 --- a/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-message-list-fetch.service.spec.ts +++ b/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-message-list-fetch.service.spec.ts @@ -9,6 +9,7 @@ import { MessageChannelSyncStatusService } from 'src/modules/messaging/common/se import { type MessageChannelWorkspaceEntity } from 'src/modules/messaging/common/standard-objects/message-channel.workspace-entity'; import { type MessageFolderWorkspaceEntity } from 'src/modules/messaging/common/standard-objects/message-folder.workspace-entity'; import { MessagingMessageCleanerService } from 'src/modules/messaging/message-cleaner/services/messaging-message-cleaner.service'; +import { SyncMessageFoldersService } from 'src/modules/messaging/message-folder-manager/services/sync-message-folders.service'; import { MessagingAccountAuthenticationService } from 'src/modules/messaging/message-import-manager/services/messaging-account-authentication.service'; import { MessagingCursorService } from 'src/modules/messaging/message-import-manager/services/messaging-cursor.service'; import { MessagingGetMessageListService } from 'src/modules/messaging/message-import-manager/services/messaging-get-message-list.service'; @@ -75,6 +76,18 @@ describe('MessagingMessageListFetchService', () => { delete: jest.fn().mockResolvedValue(undefined), }; + const mockMessageFolderRepository = { + find: jest.fn().mockResolvedValue([ + { + id: 'inbox-folder-id', + name: 'inbox', + syncCursor: 'inbox-sync-cursor', + messageChannelId: 'microsoft-message-channel-id', + isSynced: true, + }, + ]), + }; + const module: TestingModule = await Test.createTestingModule({ providers: [ MessagingMessageListFetchService, @@ -88,41 +101,40 @@ describe('MessagingMessageListFetchService', () => { { provide: MessagingGetMessageListService, useValue: { - getMessageLists: jest - .fn() - .mockImplementation(({ connectedAccount }) => { - if ( - connectedAccount.provider === ConnectedAccountProvider.GOOGLE - ) { - return [ - { - messageExternalIds: [ - 'external-id-existing-message-1', - 'external-id-google-message-1', - 'external-id-google-message-2', - ], - nextSyncCursor: 'new-google-history-id', - folderId: undefined, - messageExternalIdsToDelete: [], - previousSyncCursor: 'google-sync-cursor', - }, - ]; - } else { - return [ - { - messageExternalIds: [ - 'external-id-existing-message-1', - 'external-id-new-message-1', - 'external-id-new-message-2', - ], - nextSyncCursor: 'new-sync-cursor', - folderId: 'inbox-folder-id', - messageExternalIdsToDelete: [], - previousSyncCursor: 'inbox-sync-cursor', - }, - ]; - } - }), + getMessageLists: jest.fn().mockImplementation((messageChannel) => { + if ( + messageChannel.connectedAccount.provider === + ConnectedAccountProvider.GOOGLE + ) { + return [ + { + messageExternalIds: [ + 'external-id-existing-message-1', + 'external-id-google-message-1', + 'external-id-google-message-2', + ], + nextSyncCursor: 'new-google-history-id', + folderId: undefined, + messageExternalIdsToDelete: [], + previousSyncCursor: 'google-sync-cursor', + }, + ]; + } else { + return [ + { + messageExternalIds: [ + 'external-id-existing-message-1', + 'external-id-new-message-1', + 'external-id-new-message-2', + ], + nextSyncCursor: 'new-sync-cursor', + folderId: 'inbox-folder-id', + messageExternalIdsToDelete: [], + previousSyncCursor: 'inbox-sync-cursor', + }, + ]; + } + }), }, }, { @@ -175,11 +187,17 @@ describe('MessagingMessageListFetchService', () => { { provide: TwentyORMManager, useValue: { - getRepository: jest - .fn() - .mockResolvedValue( - mockMessageChannelMessageAssociationRepository, - ), + getDatasource: jest.fn().mockResolvedValue({ + manager: {}, + }), + getRepository: jest.fn().mockImplementation((name) => { + if (name === 'messageChannelMessageAssociation') { + return mockMessageChannelMessageAssociationRepository; + } + if (name === 'messageFolder') { + return mockMessageFolderRepository; + } + }), }, }, { @@ -207,6 +225,12 @@ describe('MessagingMessageListFetchService', () => { cleanWorkspaceThreads: jest.fn().mockResolvedValue(undefined), }, }, + { + provide: SyncMessageFoldersService, + useValue: { + syncMessageFolders: jest.fn().mockResolvedValue(undefined), + }, + }, ], }).compile(); @@ -258,6 +282,15 @@ describe('MessagingMessageListFetchService', () => { refreshToken: 'new-microsoft-refresh-token', }, }, + [ + { + id: 'inbox-folder-id', + name: 'inbox', + syncCursor: 'inbox-sync-cursor', + messageChannelId: 'microsoft-message-channel-id', + isSynced: true, + }, + ], ); expect(twentyORMManager.getRepository).toHaveBeenCalledWith( @@ -308,6 +341,15 @@ describe('MessagingMessageListFetchService', () => { refreshToken: 'new-google-refresh-token', }, }, + [ + { + id: 'inbox-folder-id', + name: 'inbox', + syncCursor: 'inbox-sync-cursor', + messageChannelId: 'microsoft-message-channel-id', + isSynced: true, + }, + ], ); expect(twentyORMManager.getRepository).toHaveBeenCalledWith( diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-message-list-fetch.service.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-message-list-fetch.service.ts index 031959aed5..0c99028cfe 100644 --- a/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-message-list-fetch.service.ts +++ b/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-message-list-fetch.service.ts @@ -13,9 +13,11 @@ import { MessageChannelSyncStatusService } from 'src/modules/messaging/common/se import { type MessageChannelMessageAssociationWorkspaceEntity } from 'src/modules/messaging/common/standard-objects/message-channel-message-association.workspace-entity'; import { MessageChannelSyncStage, - type MessageChannelWorkspaceEntity, + MessageChannelWorkspaceEntity, } from 'src/modules/messaging/common/standard-objects/message-channel.workspace-entity'; +import { MessageFolderWorkspaceEntity } from 'src/modules/messaging/common/standard-objects/message-folder.workspace-entity'; import { MessagingMessageCleanerService } from 'src/modules/messaging/message-cleaner/services/messaging-message-cleaner.service'; +import { SyncMessageFoldersService } from 'src/modules/messaging/message-folder-manager/services/sync-message-folders.service'; import { MessagingAccountAuthenticationService } from 'src/modules/messaging/message-import-manager/services/messaging-account-authentication.service'; import { MessagingCursorService } from 'src/modules/messaging/message-import-manager/services/messaging-cursor.service'; import { MessagingGetMessageListService } from 'src/modules/messaging/message-import-manager/services/messaging-get-message-list.service'; @@ -41,6 +43,7 @@ export class MessagingMessageListFetchService { private readonly messagingCursorService: MessagingCursorService, private readonly messagingMessagesImportService: MessagingMessagesImportService, private readonly messagingAccountAuthenticationService: MessagingAccountAuthenticationService, + private readonly syncMessageFoldersService: SyncMessageFoldersService, ) {} public async processMessageListFetch( @@ -74,9 +77,31 @@ export class MessagingMessageListFetchService { }, }; + const datasource = await this.twentyORMManager.getDatasource(); + + await this.syncMessageFoldersService.syncMessageFolders({ + workspaceId, + messageChannelId: messageChannelWithFreshTokens.id, + connectedAccount: messageChannelWithFreshTokens.connectedAccount, + manager: datasource.manager, + }); + + const messageFolderRepository = + await this.twentyORMManager.getRepository( + 'messageFolder', + ); + + const messageFoldersToSync = await messageFolderRepository.find({ + where: { + messageChannelId: messageChannel.id, + isSynced: true, + }, + }); + const messageLists = await this.messagingGetMessageListService.getMessageLists( messageChannelWithFreshTokens, + messageFoldersToSync, ); await this.cacheStorage.del( diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/types/get-message-lists-args.type.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/types/get-message-lists-args.type.ts index 8f4a65cd79..aa7d77ea23 100644 --- a/packages/twenty-server/src/modules/messaging/message-import-manager/types/get-message-lists-args.type.ts +++ b/packages/twenty-server/src/modules/messaging/message-import-manager/types/get-message-lists-args.type.ts @@ -10,6 +10,6 @@ export type GetMessageListsArgs = { >; messageFolders: Pick< MessageFolderWorkspaceEntity, - 'name' | 'syncCursor' | 'id' + 'name' | 'syncCursor' | 'id' | 'isSynced' | 'isSentFolder' | 'externalId' >[]; };