From 0dc1cd9df13c2e99a86befc2e8cddab2a83f13f7 Mon Sep 17 00:00:00 2001 From: Paul Rastoin <45004772+prastoin@users.noreply.github.com> Date: Thu, 27 Feb 2025 18:44:41 +0100 Subject: [PATCH] [REG/FEAT] Sync metadata after upgrade (#10557) # Introduction Historically we've been programmatically running sync metadata just after all upgrade command's migration. Adding back this behavior as default to the new dynamic modules Duplicated already existing synchronize metadata logic as a quick fix as we're about to iterate over commands next sprint --- .../create-upgrade-all-command.factory.ts | 80 ++++++++++++++++++- .../migration-command.module.ts | 10 ++- .../services/sync-workspace-logger.module.ts | 9 +++ .../services/sync-workspace-logger.service.ts | 4 +- ...workspace-sync-metadata-commands.module.ts | 10 +-- 5 files changed, 100 insertions(+), 13 deletions(-) create mode 100644 packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/commands/services/sync-workspace-logger.module.ts diff --git a/packages/twenty-server/src/database/commands/migration-command/create-upgrade-all-command.factory.ts b/packages/twenty-server/src/database/commands/migration-command/create-upgrade-all-command.factory.ts index 4d95da353a..5eb173585b 100644 --- a/packages/twenty-server/src/database/commands/migration-command/create-upgrade-all-command.factory.ts +++ b/packages/twenty-server/src/database/commands/migration-command/create-upgrade-all-command.factory.ts @@ -1,11 +1,19 @@ import { Inject } from '@nestjs/common'; +import { InjectRepository } from '@nestjs/typeorm'; import { Command } from 'nest-commander'; +import { Repository } from 'typeorm'; import { MigrationCommandInterface } from 'src/database/commands/migration-command/interfaces/migration-command.interface'; +import { MaintainedWorkspacesMigrationCommandRunner } from 'src/database/commands/migration-command/maintained-workspaces-migration-command.runner'; import { MIGRATION_COMMAND_INJECTION_TOKEN } from 'src/database/commands/migration-command/migration-command.constants'; import { MigrationCommandRunner } from 'src/database/commands/migration-command/migration-command.runner'; +import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity'; +import { DataSourceService } from 'src/engine/metadata-modules/data-source/data-source.service'; +import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager'; +import { SyncWorkspaceLoggerService } from 'src/engine/workspace-manager/workspace-sync-metadata/commands/services/sync-workspace-logger.service'; +import { WorkspaceSyncMetadataService } from 'src/engine/workspace-manager/workspace-sync-metadata/workspace-sync-metadata.service'; export function createUpgradeAllCommand( version: string, @@ -14,17 +22,81 @@ export function createUpgradeAllCommand( name: `upgrade-${version}`, description: `Upgrade to version ${version}`, }) - class UpgradeCommand extends MigrationCommandRunner { + class UpgradeCommand extends MaintainedWorkspacesMigrationCommandRunner { constructor( + @InjectRepository(Workspace, 'core') + protected readonly workspaceRepository: Repository, + protected readonly twentyORMGlobalManager: TwentyORMGlobalManager, @Inject(MIGRATION_COMMAND_INJECTION_TOKEN) private readonly subCommands: MigrationCommandInterface[], + private readonly dataSourceService: DataSourceService, + private readonly workspaceSyncMetadataService: WorkspaceSyncMetadataService, + private readonly syncWorkspaceLoggerService: SyncWorkspaceLoggerService, ) { - super(); + super(workspaceRepository, twentyORMGlobalManager); } - async runMigrationCommand( + // TODO Remove and avoid duplicated synchronize logic with SyncWorkspaceMetadataCommand after command refactoring + private async synchronizeWorkspaceMetadata({ + workspaceIds, + options, + }: { + workspaceIds: string[]; + options: Record; + }) { + this.logger.log(`Attempting to sync ${workspaceIds.length} workspaces.`); + const errorsDuringSync: string[] = []; + + for (const [index, workspaceId] of workspaceIds.entries()) { + try { + this.logger.log( + `Running workspace sync for workspace: ${workspaceId} (${index + 1} out of ${workspaceIds.length})`, + ); + const dataSourceMetadata = + await this.dataSourceService.getLastDataSourceMetadataFromWorkspaceIdOrFail( + workspaceId, + ); + + const { storage, workspaceMigrations } = + await this.workspaceSyncMetadataService.synchronize( + { + workspaceId, + dataSourceId: dataSourceMetadata.id, + }, + { applyChanges: !options.dryRun }, + ); + + if (options.dryRun) { + await this.syncWorkspaceLoggerService.saveLogs( + workspaceId, + storage, + workspaceMigrations, + ); + } + } catch (error) { + const errorMessage = `Failed to synchronize workspace ${workspaceId}: ${error.message}`; + + this.logger.error(errorMessage); + errorsDuringSync.push(errorMessage); + + continue; + } + } + this.logger.log( + `Finished synchronizing all active workspaces (${ + workspaceIds.length + } workspaces). ${ + errorsDuringSync.length > 0 + ? 'Errors during sync:\n' + errorsDuringSync.join('.\n') + : '' + }`, + ); + } + + async runMigrationCommandOnMaintainedWorkspaces( passedParams: string[], options: Record, + workspaceIds: string[], ): Promise { this.logger.log(`Running upgrade command for version ${version}`); @@ -32,6 +104,8 @@ export function createUpgradeAllCommand( await command.runMigrationCommand(passedParams, options); } + await this.synchronizeWorkspaceMetadata({ options, workspaceIds }); + this.logger.log(`Upgrade ${version} command completed!`); } } diff --git a/packages/twenty-server/src/database/commands/migration-command/migration-command.module.ts b/packages/twenty-server/src/database/commands/migration-command/migration-command.module.ts index 404245d5f1..d4958249a9 100644 --- a/packages/twenty-server/src/database/commands/migration-command/migration-command.module.ts +++ b/packages/twenty-server/src/database/commands/migration-command/migration-command.module.ts @@ -5,6 +5,9 @@ import { MigrationCommandInterface } from 'src/database/commands/migration-comma import { createUpgradeAllCommand } from 'src/database/commands/migration-command/create-upgrade-all-command.factory'; import { getMigrationCommandsForVersion } from 'src/database/commands/migration-command/decorators/migration-command.decorator'; import { MIGRATION_COMMAND_INJECTION_TOKEN } from 'src/database/commands/migration-command/migration-command.constants'; +import { DataSourceModule } from 'src/engine/metadata-modules/data-source/data-source.module'; +import { SyncWorkspaceLoggerModule } from 'src/engine/workspace-manager/workspace-sync-metadata/commands/services/sync-workspace-logger.module'; +import { WorkspaceSyncMetadataModule } from 'src/engine/workspace-manager/workspace-sync-metadata/workspace-sync-metadata.module'; @Module({}) export class MigrationCommandModule { @@ -17,7 +20,12 @@ export class MigrationCommandModule { return { module: MigrationCommandModule, - imports: moduleMetadata.imports, + imports: [ + SyncWorkspaceLoggerModule, + ...(moduleMetadata.imports ?? []), + WorkspaceSyncMetadataModule, + DataSourceModule, + ], providers: [ ...(moduleMetadata.providers ?? []), ...commandClasses, diff --git a/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/commands/services/sync-workspace-logger.module.ts b/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/commands/services/sync-workspace-logger.module.ts new file mode 100644 index 0000000000..2ac0ec229e --- /dev/null +++ b/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/commands/services/sync-workspace-logger.module.ts @@ -0,0 +1,9 @@ +import { Module } from '@nestjs/common'; + +import { SyncWorkspaceLoggerService } from 'src/engine/workspace-manager/workspace-sync-metadata/commands/services/sync-workspace-logger.service'; + +@Module({ + providers: [SyncWorkspaceLoggerService], + exports: [SyncWorkspaceLoggerService], +}) +export class SyncWorkspaceLoggerModule {} diff --git a/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/commands/services/sync-workspace-logger.service.ts b/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/commands/services/sync-workspace-logger.service.ts index 25f92347d9..f587a3294c 100644 --- a/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/commands/services/sync-workspace-logger.service.ts +++ b/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/commands/services/sync-workspace-logger.service.ts @@ -1,8 +1,8 @@ import { Injectable } from '@nestjs/common'; -import { WorkspaceSyncStorage } from 'src/engine/workspace-manager/workspace-sync-metadata/storage/workspace-sync.storage'; -import { WorkspaceMigrationEntity } from 'src/engine/metadata-modules/workspace-migration/workspace-migration.entity'; import { CommandLogger } from 'src/command/command-logger'; +import { WorkspaceMigrationEntity } from 'src/engine/metadata-modules/workspace-migration/workspace-migration.entity'; +import { WorkspaceSyncStorage } from 'src/engine/workspace-manager/workspace-sync-metadata/storage/workspace-sync.storage'; @Injectable() export class SyncWorkspaceLoggerService { diff --git a/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/commands/workspace-sync-metadata-commands.module.ts b/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/commands/workspace-sync-metadata-commands.module.ts index 6ae43c4041..5a753043c2 100644 --- a/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/commands/workspace-sync-metadata-commands.module.ts +++ b/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/commands/workspace-sync-metadata-commands.module.ts @@ -8,11 +8,10 @@ import { WorkspaceDataSourceModule } from 'src/engine/workspace-datasource/works import { WorkspaceHealthModule } from 'src/engine/workspace-manager/workspace-health/workspace-health.module'; import { ConvertRecordPositionsToIntegers } from 'src/engine/workspace-manager/workspace-sync-metadata/commands/convert-record-positions-to-integers.command'; import { WorkspaceSyncMetadataModule } from 'src/engine/workspace-manager/workspace-sync-metadata/workspace-sync-metadata.module'; +import { SyncWorkspaceLoggerModule } from 'src/engine/workspace-manager/workspace-sync-metadata/commands/services/sync-workspace-logger.module'; import { SyncWorkspaceMetadataCommand } from './sync-workspace-metadata.command'; -import { SyncWorkspaceLoggerService } from './services/sync-workspace-logger.service'; - @Module({ imports: [ WorkspaceSyncMetadataModule, @@ -21,12 +20,9 @@ import { SyncWorkspaceLoggerService } from './services/sync-workspace-logger.ser DataSourceModule, WorkspaceDataSourceModule, TypeOrmModule.forFeature([Workspace], 'core'), + SyncWorkspaceLoggerModule, ], - providers: [ - SyncWorkspaceMetadataCommand, - ConvertRecordPositionsToIntegers, - SyncWorkspaceLoggerService, - ], + providers: [SyncWorkspaceMetadataCommand, ConvertRecordPositionsToIntegers], exports: [SyncWorkspaceMetadataCommand], }) export class WorkspaceSyncMetadataCommandsModule {}