feat: migrate ConnectedAccount infrastructure entities to metadata schema (#18784)

## Summary

- Migrates 4 entities (`connectedAccount`, `messageChannel`,
`calendarChannel`, `messageFolder`) from per-workspace schemas to the
shared `core` metadata schema
- Introduces a `IS_CONNECTED_ACCOUNT_MIGRATED` feature flag to control
the migration: when enabled, reads come from core metadata and all
writes are dual-written to both workspace and core
- Extracts 12 enums from workspace entity files to `twenty-shared` for
reuse across frontend and backend
- Creates new TypeORM entities, metadata services, GraphQL
resolvers/DTOs, and exception interceptors per entity
- Each entity owns its own data access module
(`ConnectedAccountDataAccessModule`, `MessageChannelDataAccessModule`,
`CalendarChannelDataAccessModule`, `MessageFolderDataAccessModule`) — no
umbrella infrastructure module
- Adds a 1.20 upgrade command that backfills data from workspace schemas
to core (preserving UUIDs) and enables the feature flag
- Replaces direct repository access with data access service calls
across ~50 files in messaging, calendar, and connected-account modules
- Adds `lastSignedInAt` and `oidcTokenClaims` fields to the new
`ConnectedAccountEntity`
- Drops unused `lastSyncHistoryId` field from the migrated connected
account entity

## Test plan

- [x] Lint passes (`npx nx lint:diff-with-main twenty-server`)
- [x] Typecheck passes (`npx nx typecheck twenty-server`)
- [x] All unit tests pass (477 suites, 4267 tests, 0 failures)
- [ ] Manual test: verify messaging sync works with feature flag
disabled (existing behavior)
- [ ] Manual test: run upgrade command on a workspace, verify data
backfilled to core tables
- [ ] Manual test: verify messaging/calendar sync works with feature
flag enabled (dual-write path)
- [ ] Manual test: verify GraphQL metadata resolvers return correct data
when flag enabled
This commit is contained in:
Charles Bochet
2026-03-20 00:34:58 +01:00
committed by GitHub
parent cd594ce8bd
commit cee4cf6452
149 changed files with 7338 additions and 1699 deletions
@@ -0,0 +1,382 @@
import { InjectRepository } from '@nestjs/typeorm';
import { isNonEmptyString } from '@sniptt/guards';
import { Command } from 'nest-commander';
import { FeatureFlagKey } from 'twenty-shared/types';
import { Repository } from 'typeorm';
import { ActiveOrSuspendedWorkspacesMigrationCommandRunner } from 'src/database/commands/command-runners/active-or-suspended-workspaces-migration.command-runner';
import { type RunOnWorkspaceArgs } from 'src/database/commands/command-runners/workspaces-migration.command-runner';
import { FeatureFlagService } from 'src/engine/core-modules/feature-flag/services/feature-flag.service';
import { UserWorkspaceEntity } from 'src/engine/core-modules/user-workspace/user-workspace.entity';
import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity';
import { CalendarChannelEntity } from 'src/engine/metadata-modules/calendar-channel/entities/calendar-channel.entity';
import { ConnectedAccountEntity } from 'src/engine/metadata-modules/connected-account/entities/connected-account.entity';
import { DataSourceService } from 'src/engine/metadata-modules/data-source/data-source.service';
import { MessageChannelEntity } from 'src/engine/metadata-modules/message-channel/entities/message-channel.entity';
import { MessageFolderEntity } from 'src/engine/metadata-modules/message-folder/entities/message-folder.entity';
import { GlobalWorkspaceOrmManager } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager';
import { type CalendarChannelWorkspaceEntity } from 'src/modules/calendar/common/standard-objects/calendar-channel.workspace-entity';
import { type ConnectedAccountWorkspaceEntity } from 'src/modules/connected-account/standard-objects/connected-account.workspace-entity';
import { type MessageChannelWorkspaceEntity } from 'src/modules/messaging/common/standard-objects/message-channel.workspace-entity';
import { type MessageFolderWorkspaceEntity } from 'src/modules/messaging/common/standard-objects/message-folder.workspace-entity';
import { type WorkspaceMemberWorkspaceEntity } from 'src/modules/workspace-member/standard-objects/workspace-member.workspace-entity';
@Command({
name: 'upgrade:1-20:migrate-messaging-infrastructure-to-metadata',
description:
'Backfill connectedAccount, messageChannel, calendarChannel, and messageFolder to core metadata schema',
})
export class MigrateMessagingInfrastructureToMetadataCommand extends ActiveOrSuspendedWorkspacesMigrationCommandRunner {
constructor(
@InjectRepository(WorkspaceEntity)
protected readonly workspaceRepository: Repository<WorkspaceEntity>,
protected readonly twentyORMGlobalManager: GlobalWorkspaceOrmManager,
protected readonly dataSourceService: DataSourceService,
@InjectRepository(ConnectedAccountEntity)
private readonly connectedAccountRepository: Repository<ConnectedAccountEntity>,
@InjectRepository(MessageChannelEntity)
private readonly messageChannelRepository: Repository<MessageChannelEntity>,
@InjectRepository(CalendarChannelEntity)
private readonly calendarChannelRepository: Repository<CalendarChannelEntity>,
@InjectRepository(MessageFolderEntity)
private readonly messageFolderRepository: Repository<MessageFolderEntity>,
@InjectRepository(UserWorkspaceEntity)
private readonly userWorkspaceRepository: Repository<UserWorkspaceEntity>,
private readonly featureFlagService: FeatureFlagService,
) {
super(workspaceRepository, twentyORMGlobalManager, dataSourceService);
}
override async runOnWorkspace({
workspaceId,
options,
}: RunOnWorkspaceArgs): Promise<void> {
const isDryRun = options.dryRun ?? false;
const isAlreadyMigrated = await this.featureFlagService.isFeatureEnabled(
FeatureFlagKey.IS_CONNECTED_ACCOUNT_MIGRATED,
workspaceId,
);
if (isAlreadyMigrated) {
this.logger.log(
`IS_CONNECTED_ACCOUNT_MIGRATED already enabled for workspace ${workspaceId}, skipping`,
);
return;
}
const connectedAccountWorkspaceRepository =
await this.twentyORMGlobalManager.getRepository<ConnectedAccountWorkspaceEntity>(
workspaceId,
'connectedAccount',
);
const messageChannelWorkspaceRepository =
await this.twentyORMGlobalManager.getRepository<MessageChannelWorkspaceEntity>(
workspaceId,
'messageChannel',
);
const calendarChannelWorkspaceRepository =
await this.twentyORMGlobalManager.getRepository<CalendarChannelWorkspaceEntity>(
workspaceId,
'calendarChannel',
);
const messageFolderWorkspaceRepository =
await this.twentyORMGlobalManager.getRepository<MessageFolderWorkspaceEntity>(
workspaceId,
'messageFolder',
);
const connectedAccounts = await connectedAccountWorkspaceRepository.find();
const messageChannels = await messageChannelWorkspaceRepository.find();
const calendarChannels = await calendarChannelWorkspaceRepository.find();
const messageFolders = await messageFolderWorkspaceRepository.find();
const workspaceMemberIdToUserWorkspaceIdMap =
await this.buildWorkspaceMemberIdToUserWorkspaceIdMap(workspaceId);
const connectedAccountsWithMissingHandle = connectedAccounts.filter(
(account) => !account.handle,
);
const connectedAccountsWithUnresolvedOwner = connectedAccounts.filter(
(account) =>
!workspaceMemberIdToUserWorkspaceIdMap.has(account.accountOwnerId),
);
const messageChannelsWithMissingHandle = messageChannels.filter(
(channel) => !channel.handle,
);
const calendarChannelsWithMissingHandle = calendarChannels.filter(
(channel) => !channel.handle,
);
if (isDryRun) {
this.logger.log(
`[DRY RUN] Workspace ${workspaceId}: ` +
`${connectedAccounts.length} connected accounts, ` +
`${messageChannels.length} message channels, ` +
`${calendarChannels.length} calendar channels, ` +
`${messageFolders.length} message folders`,
);
if (connectedAccountsWithMissingHandle.length > 0) {
this.logger.warn(
`[DRY RUN] ${connectedAccountsWithMissingHandle.length} connected accounts have empty handle`,
);
}
if (connectedAccountsWithUnresolvedOwner.length > 0) {
this.logger.warn(
`[DRY RUN] ${connectedAccountsWithUnresolvedOwner.length} connected accounts have unresolvable accountOwnerId (no matching userWorkspace)`,
);
}
if (messageChannelsWithMissingHandle.length > 0) {
this.logger.warn(
`[DRY RUN] ${messageChannelsWithMissingHandle.length} message channels have empty handle`,
);
}
if (calendarChannelsWithMissingHandle.length > 0) {
this.logger.warn(
`[DRY RUN] ${calendarChannelsWithMissingHandle.length} calendar channels have empty handle`,
);
}
return;
}
let migratedConnectedAccountIds = new Set(
connectedAccounts.map((account) => account.id),
);
let migratedMessageChannelIds = new Set(
messageChannels.map((channel) => channel.id),
);
if (connectedAccounts.length > 0) {
const coreConnectedAccounts = connectedAccounts
.filter((workspaceEntity) => {
const userWorkspaceId = workspaceMemberIdToUserWorkspaceIdMap.get(
workspaceEntity.accountOwnerId,
);
if (!userWorkspaceId) {
this.logger.warn(
`Skipping connected account ${workspaceEntity.id}: no userWorkspace found for workspaceMember ${workspaceEntity.accountOwnerId}`,
);
return false;
}
return true;
})
.map((workspaceEntity) => {
const handleAliases = isNonEmptyString(workspaceEntity.handleAliases)
? workspaceEntity.handleAliases
.split(',')
.map((alias) => alias.trim())
: null;
return {
id: workspaceEntity.id,
handle: workspaceEntity.handle ?? '',
provider: workspaceEntity.provider,
accessToken: workspaceEntity.accessToken,
refreshToken: workspaceEntity.refreshToken,
lastCredentialsRefreshedAt:
workspaceEntity.lastCredentialsRefreshedAt,
authFailedAt: workspaceEntity.authFailedAt,
handleAliases,
scopes: workspaceEntity.scopes,
connectionParameters:
workspaceEntity.connectionParameters as Record<
string,
unknown
> | null,
userWorkspaceId: workspaceMemberIdToUserWorkspaceIdMap.get(
workspaceEntity.accountOwnerId,
)!,
workspaceId,
createdAt: workspaceEntity.createdAt,
updatedAt: workspaceEntity.updatedAt,
};
});
if (coreConnectedAccounts.length > 0) {
await this.connectedAccountRepository.save(
coreConnectedAccounts as unknown as ConnectedAccountEntity[],
);
this.logger.log(
`Migrated ${coreConnectedAccounts.length} connected accounts for workspace ${workspaceId}`,
);
}
migratedConnectedAccountIds = new Set(
coreConnectedAccounts.map((account) => account.id),
);
}
if (messageChannels.length > 0) {
const coreMessageChannels = messageChannels
.filter((workspaceEntity) =>
migratedConnectedAccountIds.has(workspaceEntity.connectedAccountId),
)
.map((workspaceEntity) => ({
id: workspaceEntity.id,
visibility: workspaceEntity.visibility,
handle: workspaceEntity.handle ?? '',
type: workspaceEntity.type,
isContactAutoCreationEnabled:
workspaceEntity.isContactAutoCreationEnabled,
contactAutoCreationPolicy: workspaceEntity.contactAutoCreationPolicy,
messageFolderImportPolicy: workspaceEntity.messageFolderImportPolicy,
excludeNonProfessionalEmails:
workspaceEntity.excludeNonProfessionalEmails,
excludeGroupEmails: workspaceEntity.excludeGroupEmails,
pendingGroupEmailsAction: workspaceEntity.pendingGroupEmailsAction,
isSyncEnabled: workspaceEntity.isSyncEnabled,
syncCursor: workspaceEntity.syncCursor,
syncedAt: workspaceEntity.syncedAt
? new Date(workspaceEntity.syncedAt)
: null,
syncStatus: workspaceEntity.syncStatus ?? 'NOT_SYNCED',
syncStage: workspaceEntity.syncStage,
syncStageStartedAt: workspaceEntity.syncStageStartedAt
? new Date(workspaceEntity.syncStageStartedAt)
: null,
throttleFailureCount: workspaceEntity.throttleFailureCount,
throttleRetryAfter: workspaceEntity.throttleRetryAfter
? new Date(workspaceEntity.throttleRetryAfter)
: null,
connectedAccountId: workspaceEntity.connectedAccountId,
workspaceId,
createdAt: workspaceEntity.createdAt,
updatedAt: workspaceEntity.updatedAt,
}));
if (coreMessageChannels.length > 0) {
await this.messageChannelRepository.save(
coreMessageChannels as unknown as MessageChannelEntity[],
);
this.logger.log(
`Migrated ${coreMessageChannels.length} message channels for workspace ${workspaceId}`,
);
}
migratedMessageChannelIds = new Set(
coreMessageChannels.map((channel) => channel.id),
);
}
if (calendarChannels.length > 0) {
const coreCalendarChannels = calendarChannels
.filter((workspaceEntity) =>
migratedConnectedAccountIds.has(workspaceEntity.connectedAccountId),
)
.map((workspaceEntity) => ({
id: workspaceEntity.id,
handle: workspaceEntity.handle ?? '',
syncStatus: workspaceEntity.syncStatus ?? 'NOT_SYNCED',
syncStage: workspaceEntity.syncStage,
visibility: workspaceEntity.visibility,
isContactAutoCreationEnabled:
workspaceEntity.isContactAutoCreationEnabled,
contactAutoCreationPolicy: workspaceEntity.contactAutoCreationPolicy,
isSyncEnabled: workspaceEntity.isSyncEnabled,
syncCursor: workspaceEntity.syncCursor,
syncedAt: workspaceEntity.syncedAt
? new Date(workspaceEntity.syncedAt)
: null,
syncStageStartedAt: workspaceEntity.syncStageStartedAt
? new Date(workspaceEntity.syncStageStartedAt)
: null,
throttleFailureCount: workspaceEntity.throttleFailureCount,
connectedAccountId: workspaceEntity.connectedAccountId,
workspaceId,
createdAt: workspaceEntity.createdAt,
updatedAt: workspaceEntity.updatedAt,
}));
await this.calendarChannelRepository.save(
coreCalendarChannels as unknown as CalendarChannelEntity[],
);
this.logger.log(
`Migrated ${coreCalendarChannels.length} calendar channels for workspace ${workspaceId}`,
);
}
if (messageFolders.length > 0) {
const coreMessageFolders = messageFolders
.filter((workspaceEntity) =>
migratedMessageChannelIds.has(workspaceEntity.messageChannelId),
)
.map((workspaceEntity) => ({
id: workspaceEntity.id,
name: workspaceEntity.name,
syncCursor: workspaceEntity.syncCursor,
isSentFolder: workspaceEntity.isSentFolder,
isSynced: workspaceEntity.isSynced,
parentFolderId: isNonEmptyString(workspaceEntity.parentFolderId)
? workspaceEntity.parentFolderId
: null,
externalId: workspaceEntity.externalId,
pendingSyncAction: workspaceEntity.pendingSyncAction,
messageChannelId: workspaceEntity.messageChannelId,
workspaceId,
createdAt: workspaceEntity.createdAt,
updatedAt: workspaceEntity.updatedAt,
}));
await this.messageFolderRepository.save(
coreMessageFolders as unknown as MessageFolderEntity[],
);
this.logger.log(
`Migrated ${coreMessageFolders.length} message folders for workspace ${workspaceId}`,
);
}
await this.featureFlagService.enableFeatureFlags(
[FeatureFlagKey.IS_CONNECTED_ACCOUNT_MIGRATED],
workspaceId,
);
this.logger.log(
`Enabled IS_CONNECTED_ACCOUNT_MIGRATED for workspace ${workspaceId}`,
);
}
private async buildWorkspaceMemberIdToUserWorkspaceIdMap(
workspaceId: string,
): Promise<Map<string, string>> {
const workspaceMemberRepository =
await this.twentyORMGlobalManager.getRepository<WorkspaceMemberWorkspaceEntity>(
workspaceId,
'workspaceMember',
);
const workspaceMembers = await workspaceMemberRepository.find();
const userWorkspaces = await this.userWorkspaceRepository.find({
where: { workspaceId },
select: ['id', 'userId'],
});
const userWorkspaceIdByUserId = new Map(
userWorkspaces.map((userWorkspace) => [
userWorkspace.userId,
userWorkspace.id,
]),
);
return new Map(
workspaceMembers
.filter((member) => userWorkspaceIdByUserId.has(member.userId))
.map((member) => [
member.id,
userWorkspaceIdByUserId.get(member.userId)!,
]),
);
}
}
@@ -8,13 +8,19 @@ import { IdentifyObjectPermissionMetadataCommand } from 'src/database/commands/u
import { IdentifyPermissionFlagMetadataCommand } from 'src/database/commands/upgrade-version-command/1-20/1-20-identify-permission-flag-metadata.command';
import { MakeObjectPermissionUniversalIdentifierAndApplicationIdNotNullableMigrationCommand } from 'src/database/commands/upgrade-version-command/1-20/1-20-make-object-permission-universal-identifier-and-application-id-not-nullable-migration.command';
import { MakePermissionFlagUniversalIdentifierAndApplicationIdNotNullableMigrationCommand } from 'src/database/commands/upgrade-version-command/1-20/1-20-make-permission-flag-universal-identifier-and-application-id-not-nullable-migration.command';
import { MigrateMessagingInfrastructureToMetadataCommand } from 'src/database/commands/upgrade-version-command/1-20/1-20-migrate-messaging-infrastructure-to-metadata.command';
import { MigrateRichTextToTextCommand } from 'src/database/commands/upgrade-version-command/1-20/1-20-migrate-rich-text-to-text.command';
import { SeedCliApplicationRegistrationCommand } from 'src/database/commands/upgrade-version-command/1-20/1-20-seed-cli-application-registration.command';
import { ApplicationRegistrationModule } from 'src/engine/core-modules/application/application-registration/application-registration.module';
import { ApplicationModule } from 'src/engine/core-modules/application/application.module';
import { FeatureFlagModule } from 'src/engine/core-modules/feature-flag/feature-flag.module';
import { UserWorkspaceEntity } from 'src/engine/core-modules/user-workspace/user-workspace.entity';
import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity';
import { CalendarChannelEntity } from 'src/engine/metadata-modules/calendar-channel/entities/calendar-channel.entity';
import { ConnectedAccountEntity } from 'src/engine/metadata-modules/connected-account/entities/connected-account.entity';
import { DataSourceModule } from 'src/engine/metadata-modules/data-source/data-source.module';
import { MessageChannelEntity } from 'src/engine/metadata-modules/message-channel/entities/message-channel.entity';
import { MessageFolderEntity } from 'src/engine/metadata-modules/message-folder/entities/message-folder.entity';
import { WorkspaceMetadataVersionModule } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.module';
import { WorkspaceCacheStorageModule } from 'src/engine/workspace-cache-storage/workspace-cache-storage.module';
import { WorkspaceCacheModule } from 'src/engine/workspace-cache/workspace-cache.module';
@@ -23,7 +29,14 @@ import { WorkspaceMigrationModule } from 'src/engine/workspace-manager/workspace
@Module({
imports: [
TypeOrmModule.forFeature([WorkspaceEntity]),
TypeOrmModule.forFeature([
WorkspaceEntity,
ConnectedAccountEntity,
MessageChannelEntity,
CalendarChannelEntity,
MessageFolderEntity,
UserWorkspaceEntity,
]),
DataSourceModule,
WorkspaceCacheModule,
WorkspaceCacheStorageModule,
@@ -44,6 +57,7 @@ import { WorkspaceMigrationModule } from 'src/engine/workspace-manager/workspace
BackfillPageLayoutsCommand,
SeedCliApplicationRegistrationCommand,
MigrateRichTextToTextCommand,
MigrateMessagingInfrastructureToMetadataCommand,
],
exports: [
IdentifyPermissionFlagMetadataCommand,
@@ -55,6 +69,7 @@ import { WorkspaceMigrationModule } from 'src/engine/workspace-manager/workspace
BackfillPageLayoutsCommand,
SeedCliApplicationRegistrationCommand,
MigrateRichTextToTextCommand,
MigrateMessagingInfrastructureToMetadataCommand,
],
})
export class V1_20_UpgradeVersionCommandModule {}
@@ -40,6 +40,7 @@ import { IdentifyObjectPermissionMetadataCommand } from 'src/database/commands/u
import { IdentifyPermissionFlagMetadataCommand } from 'src/database/commands/upgrade-version-command/1-20/1-20-identify-permission-flag-metadata.command';
import { MakeObjectPermissionUniversalIdentifierAndApplicationIdNotNullableMigrationCommand } from 'src/database/commands/upgrade-version-command/1-20/1-20-make-object-permission-universal-identifier-and-application-id-not-nullable-migration.command';
import { MakePermissionFlagUniversalIdentifierAndApplicationIdNotNullableMigrationCommand } from 'src/database/commands/upgrade-version-command/1-20/1-20-make-permission-flag-universal-identifier-and-application-id-not-nullable-migration.command';
import { MigrateMessagingInfrastructureToMetadataCommand } from 'src/database/commands/upgrade-version-command/1-20/1-20-migrate-messaging-infrastructure-to-metadata.command';
import { MigrateRichTextToTextCommand } from 'src/database/commands/upgrade-version-command/1-20/1-20-migrate-rich-text-to-text.command';
import { SeedCliApplicationRegistrationCommand } from 'src/database/commands/upgrade-version-command/1-20/1-20-seed-cli-application-registration.command';
import { TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service';
@@ -101,6 +102,7 @@ export class UpgradeCommand extends UpgradeCommandRunner {
protected readonly backfillPageLayoutsCommand: BackfillPageLayoutsCommand,
protected readonly seedCliApplicationRegistrationCommand: SeedCliApplicationRegistrationCommand,
protected readonly migrateRichTextToTextCommand: MigrateRichTextToTextCommand,
protected readonly migrateMessagingInfrastructureToMetadataCommand: MigrateMessagingInfrastructureToMetadataCommand,
) {
super(
workspaceRepository,
@@ -159,6 +161,7 @@ export class UpgradeCommand extends UpgradeCommandRunner {
this.backfillCommandMenuItemsCommand,
this.backfillPageLayoutsCommand,
this.seedCliApplicationRegistrationCommand,
this.migrateMessagingInfrastructureToMetadataCommand,
];
this.allCommands = {