feat: migrate ConnectedAccount infrastructure entities to metadata schema (#18784)

## Summary - Migrates 4 entities (`connectedAccount`, `messageChannel`, `calendarChannel`, `messageFolder`) from per-workspace schemas to the shared `core` metadata schema - Introduces a `IS_CONNECTED_ACCOUNT_MIGRATED` feature flag to control the migration: when enabled, reads come from core metadata and all writes are dual-written to both workspace and core - Extracts 12 enums from workspace entity files to `twenty-shared` for reuse across frontend and backend - Creates new TypeORM entities, metadata services, GraphQL resolvers/DTOs, and exception interceptors per entity - Each entity owns its own data access module (`ConnectedAccountDataAccessModule`, `MessageChannelDataAccessModule`, `CalendarChannelDataAccessModule`, `MessageFolderDataAccessModule`) — no umbrella infrastructure module - Adds a 1.20 upgrade command that backfills data from workspace schemas to core (preserving UUIDs) and enables the feature flag - Replaces direct repository access with data access service calls across ~50 files in messaging, calendar, and connected-account modules - Adds `lastSignedInAt` and `oidcTokenClaims` fields to the new `ConnectedAccountEntity` - Drops unused `lastSyncHistoryId` field from the migrated connected account entity ## Test plan - [x] Lint passes (`npx nx lint:diff-with-main twenty-server`) - [x] Typecheck passes (`npx nx typecheck twenty-server`) - [x] All unit tests pass (477 suites, 4267 tests, 0 failures) - [ ] Manual test: verify messaging sync works with feature flag disabled (existing behavior) - [ ] Manual test: run upgrade command on a workspace, verify data backfilled to core tables - [ ] Manual test: verify messaging/calendar sync works with feature flag enabled (dual-write path) - [ ] Manual test: verify GraphQL metadata resolvers return correct data when flag enabled
2026-03-20 00:34:58 +01:00
parent cd594ce8bd
commit cee4cf6452
149 changed files with 7338 additions and 1699 deletions
@@ -1,9 +1,11 @@
 import { Test, type TestingModule } from '@nestjs/testing';

 import { ConnectedAccountProvider } from 'twenty-shared/types';
+import { In } from 'typeorm';

 import { type DiscoveredMessageFolder } from 'src/modules/messaging/message-folder-manager/interfaces/message-folder-driver.interface';

+import { MessageFolderDataAccessService } from 'src/engine/metadata-modules/message-folder/data-access/services/message-folder-data-access.service';
 import { GlobalWorkspaceOrmManager } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager';
 import {
  MessageChannelContactAutoCreationPolicy,
@@ -83,39 +85,73 @@ const createMockExistingFolder = (
  ...overrides,
 });

+const getInValuesFromWhere = (externalIdClause: unknown): string[] => {
+  if (
+    externalIdClause &&
+    typeof externalIdClause === 'object' &&
+    '_value' in externalIdClause
+  ) {
+    return (externalIdClause as { _value: string[] })._value;
+  }
+
+  if (
+    externalIdClause &&
+    typeof externalIdClause === 'object' &&
+    'value' in externalIdClause
+  ) {
+    const value = (externalIdClause as { value: unknown }).value;
+
+    return Array.isArray(value) ? value : [];
+  }
+
+  return [];
+};
+
 describe('SyncMessageFoldersService', () => {
  let service: SyncMessageFoldersService;
  let gmailGetAllFoldersService: jest.Mocked<GmailGetAllFoldersService>;

-  let mockRepository: {
+  let mockMessageFolderDataAccessService: {
    delete: jest.Mock;
    update: jest.Mock;
-    updateMany: jest.Mock;
    save: jest.Mock;
+    find: jest.Mock;
  };
-  let mockTransactionManager: object;
+
+  let createdFolderRecords: Array<
+    Partial<MessageFolderWorkspaceEntity> & {
+      id: string;
+      externalId: string;
+    }
+  >;

  beforeEach(async () => {
-    mockRepository = {
+    createdFolderRecords = [];
+
+    mockMessageFolderDataAccessService = {
      delete: jest.fn(),
-      update: jest.fn(),
-      updateMany: jest.fn(),
-      save: jest.fn().mockImplementation((folders) =>
-        folders.map((folder: Partial<MessageFolderWorkspaceEntity>) => ({
+      update: jest.fn().mockResolvedValue(undefined),
+      save: jest.fn().mockImplementation(async (_workspaceId, folder) => {
+        createdFolderRecords.push({
          ...folder,
-          id: `new-folder-${Math.random().toString(36).substring(7)}`,
+          id: `new-folder-${createdFolderRecords.length}-${Math.random().toString(36).substring(7)}`,
          isSynced: false,
          syncCursor: null,
-        })),
-      ),
-    };
+          pendingSyncAction: MessageFolderPendingSyncAction.NONE,
+          externalId: folder.externalId as string,
+        });
+      }),
+      find: jest.fn().mockImplementation(async (_workspaceId, where) => {
+        if (!where?.externalId) {
+          return [];
+        }

-    mockTransactionManager = {};
+        const externalIds = getInValuesFromWhere(where.externalId);

-    const mockDataSource = {
-      transaction: jest
-        .fn()
-        .mockImplementation((callback) => callback(mockTransactionManager)),
+        return createdFolderRecords.filter((folder) =>
+          externalIds.includes(folder.externalId as string),
+        );
+      }),
    };

    const module: TestingModule = await Test.createTestingModule({
@@ -129,15 +165,15 @@ describe('SyncMessageFoldersService', () => {
              .mockImplementation((callback: () => any, _authContext?: any) =>
                callback(),
              ),
-            getRepository: jest.fn().mockResolvedValue(mockRepository),
-            getDataSourceForWorkspace: jest
-              .fn()
-              .mockResolvedValue(mockDataSource),
-            getGlobalWorkspaceDataSource: jest
-              .fn()
-              .mockResolvedValue(mockDataSource),
+            getRepository: jest.fn(),
+            getDataSourceForWorkspace: jest.fn(),
+            getGlobalWorkspaceDataSource: jest.fn(),
          },
        },
+        {
+          provide: MessageFolderDataAccessService,
+          useValue: mockMessageFolderDataAccessService,
+        },
        {
          provide: GmailGetAllFoldersService,
          useValue: {
@@ -192,23 +228,23 @@ describe('SyncMessageFoldersService', () => {
          workspaceId,
        });

-        expect(mockRepository.save).toHaveBeenCalledWith(
-          expect.arrayContaining([
-            expect.objectContaining({
-              name: 'INBOX',
-              externalId: 'inbox-ext',
-              messageChannelId: 'channel-123',
-              isSentFolder: false,
-            }),
-            expect.objectContaining({
-              name: 'Sent',
-              externalId: 'sent-ext',
-              messageChannelId: 'channel-123',
-              isSentFolder: true,
-            }),
-          ]),
-          {},
-          mockTransactionManager,
+        expect(mockMessageFolderDataAccessService.save).toHaveBeenCalledWith(
+          workspaceId,
+          expect.objectContaining({
+            name: 'INBOX',
+            externalId: 'inbox-ext',
+            messageChannelId: 'channel-123',
+            isSentFolder: false,
+          }),
+        );
+        expect(mockMessageFolderDataAccessService.save).toHaveBeenCalledWith(
+          workspaceId,
+          expect.objectContaining({
+            name: 'Sent',
+            externalId: 'sent-ext',
+            messageChannelId: 'channel-123',
+            isSentFolder: true,
+          }),
        );
        expect(result).toHaveLength(2);
      });
@@ -239,15 +275,12 @@ describe('SyncMessageFoldersService', () => {
          workspaceId,
        });

-        expect(mockRepository.save).toHaveBeenCalledWith(
-          expect.arrayContaining([
-            expect.objectContaining({
-              name: 'Projects',
-              parentFolderId: 'parent-folder-id',
-            }),
-          ]),
-          {},
-          mockTransactionManager,
+        expect(mockMessageFolderDataAccessService.save).toHaveBeenCalledWith(
+          workspaceId,
+          expect.objectContaining({
+            name: 'Projects',
+            parentFolderId: 'parent-folder-id',
+          }),
        );
      });
    });
@@ -278,14 +311,10 @@ describe('SyncMessageFoldersService', () => {
          workspaceId,
        });

-        expect(mockRepository.updateMany).toHaveBeenCalledWith(
-          expect.arrayContaining([
-            expect.objectContaining({
-              criteria: 'folder-1',
-              partialEntity: expect.objectContaining({ name: 'Primary Inbox' }),
-            }),
-          ]),
-          mockTransactionManager,
+        expect(mockMessageFolderDataAccessService.update).toHaveBeenCalledWith(
+          workspaceId,
+          { id: 'folder-1' },
+          expect.objectContaining({ name: 'Primary Inbox' }),
        );
        expect(result).toContainEqual(
          expect.objectContaining({
@@ -322,16 +351,12 @@ describe('SyncMessageFoldersService', () => {
          workspaceId,
        });

-        expect(mockRepository.updateMany).toHaveBeenCalledWith(
-          expect.arrayContaining([
-            expect.objectContaining({
-              criteria: 'folder-1',
-              partialEntity: expect.objectContaining({
-                parentFolderId: 'new-parent-id',
-              }),
-            }),
-          ]),
-          mockTransactionManager,
+        expect(mockMessageFolderDataAccessService.update).toHaveBeenCalledWith(
+          workspaceId,
+          { id: 'folder-1' },
+          expect.objectContaining({
+            parentFolderId: 'new-parent-id',
+          }),
        );
      });

@@ -364,7 +389,9 @@ describe('SyncMessageFoldersService', () => {
          workspaceId,
        });

-        expect(mockRepository.updateMany).not.toHaveBeenCalled();
+        expect(
+          mockMessageFolderDataAccessService.update,
+        ).not.toHaveBeenCalled();
      });
    });

@@ -401,16 +428,12 @@ describe('SyncMessageFoldersService', () => {
          workspaceId,
        });

-        expect(mockRepository.updateMany).toHaveBeenCalledWith(
-          expect.arrayContaining([
-            expect.objectContaining({
-              criteria: 'folder-2',
-              partialEntity: expect.objectContaining({
-                pendingSyncAction: 'FOLDER_DELETION',
-              }),
-            }),
-          ]),
-          mockTransactionManager,
+        expect(mockMessageFolderDataAccessService.update).toHaveBeenCalledWith(
+          workspaceId,
+          { id: In(['folder-2']) },
+          expect.objectContaining({
+            pendingSyncAction: 'FOLDER_DELETION',
+          }),
        );
        expect(result).toContainEqual(
          expect.objectContaining({
@@ -468,32 +491,21 @@ describe('SyncMessageFoldersService', () => {
          workspaceId,
        });

-        expect(mockRepository.updateMany).toHaveBeenCalledWith(
-          expect.arrayContaining([
-            expect.objectContaining({
-              criteria: 'folder-to-delete',
-              partialEntity: expect.objectContaining({
-                pendingSyncAction: 'FOLDER_DELETION',
-              }),
-            }),
-          ]),
-          mockTransactionManager,
+        expect(mockMessageFolderDataAccessService.update).toHaveBeenCalledWith(
+          workspaceId,
+          { id: In(['folder-to-delete']) },
+          expect.objectContaining({
+            pendingSyncAction: 'FOLDER_DELETION',
+          }),
        );
-        expect(mockRepository.updateMany).toHaveBeenCalledWith(
-          expect.arrayContaining([
-            expect.objectContaining({
-              criteria: 'folder-to-update',
-              partialEntity: expect.objectContaining({ name: 'New Name' }),
-            }),
-          ]),
-          mockTransactionManager,
+        expect(mockMessageFolderDataAccessService.update).toHaveBeenCalledWith(
+          workspaceId,
+          { id: 'folder-to-update' },
+          expect.objectContaining({ name: 'New Name' }),
        );
-        expect(mockRepository.save).toHaveBeenCalledWith(
-          expect.arrayContaining([
-            expect.objectContaining({ externalId: 'new-ext' }),
-          ]),
-          {},
-          mockTransactionManager,
+        expect(mockMessageFolderDataAccessService.save).toHaveBeenCalledWith(
+          workspaceId,
+          expect.objectContaining({ externalId: 'new-ext' }),
        );
        expect(result).toHaveLength(4);
        expect(result).toContainEqual(
@@ -1,21 +1,20 @@
 import { Injectable } from '@nestjs/common';

 import { ConnectedAccountProvider } from 'twenty-shared/types';
+import { isDefined } from 'twenty-shared/utils';
+import { In } from 'typeorm';

 import {
  DiscoveredMessageFolder,
  MessageFolder,
 } from 'src/modules/messaging/message-folder-manager/interfaces/message-folder-driver.interface';

-import { WorkspaceEntityManager } from 'src/engine/twenty-orm/entity-manager/workspace-entity-manager';
+import { MessageFolderDataAccessService } from 'src/engine/metadata-modules/message-folder/data-access/services/message-folder-data-access.service';
 import { GlobalWorkspaceOrmManager } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager';
 import { buildSystemAuthContext } from 'src/engine/twenty-orm/utils/build-system-auth-context.util';
 import { ConnectedAccountWorkspaceEntity } from 'src/modules/connected-account/standard-objects/connected-account.workspace-entity';
 import { type MessageChannelWorkspaceEntity } from 'src/modules/messaging/common/standard-objects/message-channel.workspace-entity';
-import {
-  MessageFolderPendingSyncAction,
-  MessageFolderWorkspaceEntity,
-} from 'src/modules/messaging/common/standard-objects/message-folder.workspace-entity';
+import { MessageFolderPendingSyncAction } from 'src/modules/messaging/common/standard-objects/message-folder.workspace-entity';
 import { GmailGetAllFoldersService } from 'src/modules/messaging/message-folder-manager/drivers/gmail/services/gmail-get-all-folders.service';
 import { ImapGetAllFoldersService } from 'src/modules/messaging/message-folder-manager/drivers/imap/services/imap-get-all-folders.service';
 import { MicrosoftGetAllFoldersService } from 'src/modules/messaging/message-folder-manager/drivers/microsoft/services/microsoft-get-all-folders.service';
@@ -28,6 +27,7 @@ import { computeUpdatedFolders } from 'src/modules/messaging/message-folder-mana
 export class SyncMessageFoldersService {
  constructor(
    private readonly globalWorkspaceOrmManager: GlobalWorkspaceOrmManager,
+    private readonly messageFolderDataAccessService: MessageFolderDataAccessService,
    private readonly gmailGetAllFoldersService: GmailGetAllFoldersService,
    private readonly microsoftGetAllFoldersService: MicrosoftGetAllFoldersService,
    private readonly imapGetAllFoldersService: ImapGetAllFoldersService,
@@ -132,60 +132,63 @@ export class SyncMessageFoldersService {

    const authContext = buildSystemAuthContext(workspaceId);

+    // TODO: Restore transaction wrapper once migration is complete — folder
+    // sync operations (create/update/delete) are no longer atomic since
+    // the data access layer routes writes across workspace and core schemas.
+    // Acceptable during transition as sync is idempotent and self-corrects.
    return this.globalWorkspaceOrmManager.executeInWorkspaceContext(
      async () => {
-        const messageFolderRepository =
-          await this.globalWorkspaceOrmManager.getRepository<MessageFolderWorkspaceEntity>(
+        if (folderIdsToDelete.length > 0) {
+          await this.messageFolderDataAccessService.update(
            workspaceId,
-            'messageFolder',
+            { id: In(folderIdsToDelete) },
+            {
+              pendingSyncAction: MessageFolderPendingSyncAction.FOLDER_DELETION,
+            },
          );
+        }

-        const workspaceDataSource =
-          await this.globalWorkspaceOrmManager.getGlobalWorkspaceDataSource();
+        if (foldersToUpdate.size > 0) {
+          for (const [id, data] of foldersToUpdate.entries()) {
+            await this.messageFolderDataAccessService.update(
+              workspaceId,
+              { id },
+              data,
+            );
+          }
+        }

-        return workspaceDataSource.transaction(
-          async (transactionManager: WorkspaceEntityManager) => {
-            if (folderIdsToDelete.length > 0) {
-              await messageFolderRepository.updateMany(
-                folderIdsToDelete.map((id) => ({
-                  criteria: id,
-                  partialEntity: {
-                    pendingSyncAction:
-                      MessageFolderPendingSyncAction.FOLDER_DELETION,
-                  },
-                })),
-                transactionManager,
-              );
-            }
+        if (foldersToCreate.length > 0) {
+          for (const folderToCreate of foldersToCreate) {
+            await this.messageFolderDataAccessService.save(
+              workspaceId,
+              folderToCreate,
+            );
+          }
+        }

-            if (foldersToUpdate.size > 0) {
-              await messageFolderRepository.updateMany(
-                Array.from(foldersToUpdate.entries()).map(([id, data]) => ({
-                  criteria: id,
-                  partialEntity: data,
-                })),
-                transactionManager,
-              );
-            }
+        const createdFolders =
+          foldersToCreate.length > 0
+            ? await this.messageFolderDataAccessService.find(workspaceId, {
+                messageChannelId,
+                externalId: In(
+                  foldersToCreate
+                    .map((folder) => folder.externalId)
+                    .filter(isDefined),
+                ),
+              })
+            : [];

-            const createdFolders =
-              foldersToCreate.length > 0
-                ? await messageFolderRepository.save(
-                    foldersToCreate,
-                    {},
-                    transactionManager,
-                  )
-                : [];
+        const updatedExistingFolders = computeUpdatedFolders({
+          existingFolders,
+          foldersToUpdate,
+          folderIdsToDelete,
+        });

-            const updatedExistingFolders = computeUpdatedFolders({
-              existingFolders,
-              foldersToUpdate,
-              folderIdsToDelete,
-            });
-
-            return [...updatedExistingFolders, ...createdFolders];
-          },
-        );
+        return [
+          ...updatedExistingFolders,
+          ...(createdFolders as MessageFolder[]),
+        ];
      },
      authContext,
    );