diff --git a/.github/workflows/ci-server.yaml b/.github/workflows/ci-server.yaml index dec140049f..a18755905e 100644 --- a/.github/workflows/ci-server.yaml +++ b/.github/workflows/ci-server.yaml @@ -144,15 +144,15 @@ jobs: exit 1 - name: Server / Check for Pending Migrations run: | - CORE_MIGRATION_OUTPUT=$(npx nx run twenty-server:typeorm migration:generate core-migration-check -d src/database/typeorm/core/core.datasource.ts || true) + CORE_MIGRATION_OUTPUT=$(npx nx database:migrate:generate twenty-server -- --name core-migration-check || true) - CORE_MIGRATION_FILE=$(ls packages/twenty-server/*core-migration-check.ts 2>/dev/null || echo "") + CORE_MIGRATION_FILE=$(ls packages/twenty-server/src/database/typeorm/core/migrations/common/*core-migration-check.ts 2>/dev/null || echo "") if [ -n "$CORE_MIGRATION_FILE" ]; then - echo "::error::Unexpected migration files were generated. Please create a proper migration manually." + echo "::error::Unexpected migration files were generated. Please run 'npx nx database:migrate:generate twenty-server -- --name ' and commit the result." echo "$CORE_MIGRATION_OUTPUT" - rm -f packages/twenty-server/*core-migration-check.ts + rm -f packages/twenty-server/src/database/typeorm/core/migrations/common/*core-migration-check.ts exit 1 fi diff --git a/packages/twenty-server/package.json b/packages/twenty-server/package.json index 2fc470989d..f1c1d27724 100644 --- a/packages/twenty-server/package.json +++ b/packages/twenty-server/package.json @@ -10,7 +10,7 @@ "command:prod": "node dist/command/command", "worker:prod": "node dist/queue-worker/queue-worker", "database:init:prod": "node dist/database/scripts/setup-db.js && yarn database:migrate:prod --force", - "database:migrate:prod": "node dist/command/command run-typeorm-migration", + "database:migrate:prod": "node dist/command/command run-core-migration", "clickhouse:migrate:prod": "node dist/database/clickHouse/migrations/run-migrations.js", "typeorm": "../../node_modules/typeorm/.bin/typeorm" }, diff --git a/packages/twenty-server/project.json b/packages/twenty-server/project.json index ef27c877ec..6a4e9184f6 100644 --- a/packages/twenty-server/project.json +++ b/packages/twenty-server/project.json @@ -204,29 +204,18 @@ }, "database:migrate": { "executor": "nx:run-commands", - "dependsOn": ["^build"], + "dependsOn": ["build"], "options": { "cwd": "packages/twenty-server", - "commands": [ - "nx typeorm -- migration:run -d src/database/typeorm/core/core.datasource" - ] + "command": "node dist/command/command.js run-core-migration --force" } }, "database:migrate:generate": { "executor": "nx:run-commands", + "dependsOn": ["build"], "options": { "cwd": "packages/twenty-server", - "command": "npx nx typeorm -- migration:generate src/database/typeorm/core/migrations/common/{args.migrationName} -d src/database/typeorm/core/core.datasource.ts" - } - }, - "database:migrate:revert": { - "executor": "nx:run-commands", - "dependsOn": ["^build"], - "options": { - "cwd": "packages/twenty-server", - "commands": [ - "nx typeorm -- migration:revert -d src/database/typeorm/core/core.datasource" - ] + "command": "node dist/command/command.js generate:versioned-migration" } }, "generate:integration-test": { diff --git a/packages/twenty-server/src/database/commands/command-runners/__tests__/upgrade.command-runner.spec.ts b/packages/twenty-server/src/database/commands/command-runners/__tests__/upgrade.command-runner.spec.ts index 62942bfafa..437746ebb5 100644 --- a/packages/twenty-server/src/database/commands/command-runners/__tests__/upgrade.command-runner.spec.ts +++ b/packages/twenty-server/src/database/commands/command-runners/__tests__/upgrade.command-runner.spec.ts @@ -5,14 +5,21 @@ import { eachTestingContextFilter, type EachTestingContext, } from 'twenty-shared/testing'; -import { type Repository } from 'typeorm'; +import { + type MigrationInterface, + type QueryRunner, + type Repository, +} from 'typeorm'; import { + UpgradeCommandOptions, UpgradeCommandRunner, type AllCommands, } from 'src/database/commands/command-runners/upgrade.command-runner'; import { WorkspaceIteratorService } from 'src/database/commands/command-runners/workspace-iterator.service'; -import { CoreMigrationRunnerService } from 'src/database/commands/core-migration-runner/services/core-migration-runner.service'; +import { CoreMigrationRunnerService } from 'src/database/commands/core-migration/services/core-migration-runner.service'; +import { RegisteredCoreMigrationService } from 'src/database/commands/core-migration/services/registered-core-migration-registry.service'; +import { RegisteredCoreMigration } from 'src/database/typeorm/core/decorators/registered-core-migration.decorator'; import { UPGRADE_COMMAND_SUPPORTED_VERSIONS } from 'src/engine/constants/upgrade-command-supported-versions.constant'; import { CoreEngineVersionService } from 'src/engine/core-engine-version/services/core-engine-version.service'; import { type ConfigVariables } from 'src/engine/core-modules/twenty-config/config-variables'; @@ -59,12 +66,35 @@ type BuildUpgradeCommandModuleArgs = { workspaces: WorkspaceEntity[]; appVersion: string | null; commandRunner: CommandRunnerValues; + migrations?: MigrationInterface[]; }; const buildUpgradeCommandModule = async ({ workspaces, appVersion, commandRunner, + migrations, }: BuildUpgradeCommandModuleArgs) => { + const registryProvider = migrations + ? { + provide: RegisteredCoreMigrationService, + useFactory: () => { + const fakeDataSource = { + migrations, + } as unknown as import('typeorm').DataSource; + const registry = new RegisteredCoreMigrationService(fakeDataSource); + + registry.onModuleInit(); + + return registry; + }, + } + : { + provide: RegisteredCoreMigrationService, + useValue: { + getInstanceCommandsForVersion: jest.fn().mockReturnValue([]), + }, + }; + const module: TestingModule = await Test.createTestingModule({ providers: [ { @@ -74,6 +104,7 @@ const buildUpgradeCommandModule = async ({ coreEngineVersionService: CoreEngineVersionService, workspaceVersionService: WorkspaceVersionService, coreMigrationRunnerService: CoreMigrationRunnerService, + versionedMigrationRegistryService: RegisteredCoreMigrationService, workspaceIteratorService: WorkspaceIteratorService, ) => { return new commandRunner( @@ -81,6 +112,7 @@ const buildUpgradeCommandModule = async ({ coreEngineVersionService, workspaceVersionService, coreMigrationRunnerService, + versionedMigrationRegistryService, workspaceIteratorService, ); }, @@ -89,6 +121,7 @@ const buildUpgradeCommandModule = async ({ CoreEngineVersionService, WorkspaceVersionService, CoreMigrationRunnerService, + RegisteredCoreMigrationService, WorkspaceIteratorService, ], }, @@ -124,8 +157,13 @@ const buildUpgradeCommandModule = async ({ WorkspaceVersionService, { provide: CoreMigrationRunnerService, - useValue: { run: jest.fn().mockResolvedValue(undefined) }, + useValue: { + runSingleMigration: jest + .fn() + .mockResolvedValue({ status: 'success' }), + }, }, + registryProvider, { provide: WorkspaceIteratorService, useValue: { @@ -173,6 +211,7 @@ describe('UpgradeCommandRunner', () => { workspaces?: WorkspaceEntity[]; appVersion?: string | null; commandRunner?: CommandRunnerValues; + migrations?: MigrationInterface[]; }; const buildModuleAndSetupSpies = async ({ numberOfWorkspace = 1, @@ -180,6 +219,7 @@ describe('UpgradeCommandRunner', () => { workspaces, commandRunner = BasicUpgradeCommandRunner, appVersion = CURRENT_VERSION, + migrations, }: BuildModuleAndSetupSpiesArgs) => { const generatedWorkspaces = Array.from( { length: numberOfWorkspace }, @@ -193,6 +233,7 @@ describe('UpgradeCommandRunner', () => { commandRunner, appVersion, workspaces: [...generatedWorkspaces, ...(workspaces ?? [])], + migrations, }); upgradeCommandRunner = module.get(commandRunner); @@ -204,9 +245,11 @@ describe('UpgradeCommandRunner', () => { workspaceRepository = module.get>( getRepositoryToken(WorkspaceEntity), ); + + return module; }; - it('should ignore and list as succesfull upgrade on workspace with higher version', async () => { + it('should ignore and list as successful upgrade on workspace with higher version', async () => { const higherVersionWorkspace = generateMockWorkspace({ id: 'higher_version_workspace', version: '42.42.42', @@ -216,11 +259,9 @@ describe('UpgradeCommandRunner', () => { numberOfWorkspace: 0, workspaces: [higherVersionWorkspace], }); - // @ts-expect-error legacy noImplicitAny - const passedParams = []; - const options = {}; + const passedParams: string[] = []; + const options: UpgradeCommandOptions = {}; - // @ts-expect-error legacy noImplicitAny await upgradeCommandRunner.run(passedParams, options); [workspaceRepository.update].forEach((fn) => @@ -234,11 +275,9 @@ describe('UpgradeCommandRunner', () => { await buildModuleAndSetupSpies({ numberOfWorkspace, }); - // @ts-expect-error legacy noImplicitAny - const passedParams = []; - const options = {}; + const passedParams: string[] = []; + const options: UpgradeCommandOptions = {}; - // @ts-expect-error legacy noImplicitAny await upgradeCommandRunner.run(passedParams, options); expect(workspaceRepository.update).toHaveBeenNthCalledWith( @@ -294,11 +333,9 @@ describe('UpgradeCommandRunner', () => { async ({ context: { input } }) => { await buildModuleAndSetupSpies(input); - // @ts-expect-error legacy noImplicitAny - const passedParams = []; - const options = {}; + const passedParams: string[] = []; + const options: UpgradeCommandOptions = {}; - // @ts-expect-error legacy noImplicitAny await upgradeCommandRunner.run(passedParams, options); expect(workspaceRepository.update).toHaveBeenCalledWith( @@ -309,6 +346,57 @@ describe('UpgradeCommandRunner', () => { ); }); + it('should only run instance commands for the current version', async () => { + @RegisteredCoreMigration(CURRENT_VERSION) + class AddIndexToUsers1770000000000 implements MigrationInterface { + async up(_queryRunner: QueryRunner) {} + async down(_queryRunner: QueryRunner) {} + } + + @RegisteredCoreMigration(CURRENT_VERSION) + class AddColumnToAccounts1771000000000 implements MigrationInterface { + async up(_queryRunner: QueryRunner) {} + async down(_queryRunner: QueryRunner) {} + } + + @RegisteredCoreMigration(PREVIOUS_VERSION) + class DropLegacyTable1769000000000 implements MigrationInterface { + async up(_queryRunner: QueryRunner) {} + async down(_queryRunner: QueryRunner) {} + } + + class UndecoratedMigration1768000000000 implements MigrationInterface { + async up(_queryRunner: QueryRunner) {} + async down(_queryRunner: QueryRunner) {} + } + + const module = await buildModuleAndSetupSpies({ + migrations: [ + new UndecoratedMigration1768000000000(), + new DropLegacyTable1769000000000(), + new AddIndexToUsers1770000000000(), + new AddColumnToAccounts1771000000000(), + ], + }); + + const migrationRunnerService = module.get(CoreMigrationRunnerService); + + const passedParams: string[] = []; + const options: UpgradeCommandOptions = {}; + + await upgradeCommandRunner.run(passedParams, options); + + expect(migrationRunnerService.runSingleMigration).toHaveBeenCalledTimes(2); + expect(migrationRunnerService.runSingleMigration).toHaveBeenNthCalledWith( + 1, + 'AddIndexToUsers1770000000000', + ); + expect(migrationRunnerService.runSingleMigration).toHaveBeenNthCalledWith( + 2, + 'AddColumnToAccounts1771000000000', + ); + }); + describe('Workspace upgrade should fail', () => { const failingTestUseCases: EachTestingContext<{ input: Omit; diff --git a/packages/twenty-server/src/database/commands/command-runners/upgrade.command-runner.ts b/packages/twenty-server/src/database/commands/command-runners/upgrade.command-runner.ts index 89120392eb..93401aad50 100644 --- a/packages/twenty-server/src/database/commands/command-runners/upgrade.command-runner.ts +++ b/packages/twenty-server/src/database/commands/command-runners/upgrade.command-runner.ts @@ -4,17 +4,19 @@ import chalk from 'chalk'; import { CommandRunner, Option } from 'nest-commander'; import { SemVer } from 'semver'; import { assertUnreachable, isDefined } from 'twenty-shared/utils'; -import { Repository } from 'typeorm'; +import { MigrationInterface, Repository } from 'typeorm'; -import { - type RunOnWorkspaceArgs, - WorkspaceCommandRunner, -} from 'src/database/commands/command-runners/workspace.command-runner'; +import { ActiveOrSuspendedWorkspaceCommandRunner } from 'src/database/commands/command-runners/active-or-suspended-workspace.command-runner'; import { type WorkspaceIteratorContext, WorkspaceIteratorService, } from 'src/database/commands/command-runners/workspace-iterator.service'; -import { CoreMigrationRunnerService } from 'src/database/commands/core-migration-runner/services/core-migration-runner.service'; +import { + type RunOnWorkspaceArgs, + WorkspaceCommandRunner, +} from 'src/database/commands/command-runners/workspace.command-runner'; +import { CoreMigrationRunnerService } from 'src/database/commands/core-migration/services/core-migration-runner.service'; +import { RegisteredCoreMigrationService } from 'src/database/commands/core-migration/services/registered-core-migration-registry.service'; import { CommandLogger } from 'src/database/commands/logger'; import { type UpgradeCommandVersion } from 'src/engine/constants/upgrade-command-supported-versions.constant'; import { CoreEngineVersionService } from 'src/engine/core-engine-version/services/core-engine-version.service'; @@ -25,7 +27,10 @@ import { compareVersionMajorAndMinor, } from 'src/utils/version/compare-version-minor-and-major'; -export type VersionCommands = WorkspaceCommandRunner[]; +export type VersionCommands = ( + | WorkspaceCommandRunner + | ActiveOrSuspendedWorkspaceCommandRunner +)[]; export type AllCommands = Record; export type UpgradeCommandOptions = { @@ -39,7 +44,9 @@ export type UpgradeCommandOptions = { type VersionContext = { fromWorkspaceVersion: SemVer; currentAppVersion: SemVer; - commands: VersionCommands; + currentVersionMajorMinor: UpgradeCommandVersion; + instanceCommands: MigrationInterface[]; + workspaceCommands: VersionCommands; }; export abstract class UpgradeCommandRunner extends CommandRunner { @@ -53,6 +60,7 @@ export abstract class UpgradeCommandRunner extends CommandRunner { protected readonly coreEngineVersionService: CoreEngineVersionService, protected readonly workspaceVersionService: WorkspaceVersionService, protected readonly coreMigrationRunnerService: CoreMigrationRunnerService, + protected readonly versionedMigrationRegistryService: RegisteredCoreMigrationService, protected readonly workspaceIteratorService: WorkspaceIteratorService, ) { super(); @@ -138,6 +146,18 @@ export abstract class UpgradeCommandRunner extends CommandRunner { try { const versionContext = this.resolveVersionContext(); + this.logger.log( + chalk.blue( + [ + 'Initialized upgrade context with:', + `- currentVersion (migrating to): ${versionContext.currentAppVersion}`, + `- fromWorkspaceVersion: ${versionContext.fromWorkspaceVersion}`, + `- ${versionContext.instanceCommands.length} instance commands (from registry)`, + `- ${versionContext.workspaceCommands.length} workspace commands`, + ].join('\n '), + ), + ); + const hasWorkspaces = await this.workspaceVersionService.hasActiveOrSuspendedWorkspaces(); @@ -166,7 +186,43 @@ Please roll back to that version and run the upgrade command again.`, ); } - await this.coreMigrationRunnerService.run(); + for (const instanceCommand of versionContext.instanceCommands) { + const migrationName = instanceCommand.constructor.name; + const result = + await this.coreMigrationRunnerService.runSingleMigration( + migrationName, + ); + + if (result.status === 'fail') { + if (result.code === 'already-executed') { + this.logger.warn( + `Core migration ${migrationName} already executed, skipping`, + ); + + continue; + } + + this.logger.error( + `Core migration ${migrationName} failed with code: ${result.code}`, + ); + + if (isDefined(result.error)) { + this.logger.error( + result.error instanceof Error + ? (result.error.stack ?? result.error.message) + : String(result.error), + ); + } + + throw new Error( + `Core migration ${migrationName} failed: ${result.code}`, + ); + } + + this.logger.log( + `Core migration ${migrationName} executed successfully`, + ); + } const iteratorReport = await this.workspaceIteratorService.iterate({ workspaceIds: @@ -205,9 +261,9 @@ Please roll back to that version and run the upgrade command again.`, const currentAppVersion = this.coreEngineVersionService.getCurrentVersion(); const currentVersionMajorMinor = `${currentAppVersion.major}.${currentAppVersion.minor}.0` as UpgradeCommandVersion; - const commands = this.allCommands[currentVersionMajorMinor]; + const workspaceCommands = this.allCommands[currentVersionMajorMinor]; - if (!isDefined(commands)) { + if (!isDefined(workspaceCommands)) { throw new Error( `No command found for version ${currentAppVersion}. Please check the commands record.`, ); @@ -216,18 +272,18 @@ Please roll back to that version and run the upgrade command again.`, const fromWorkspaceVersion = this.coreEngineVersionService.getPreviousVersion(); - this.logger.log( - chalk.blue( - [ - 'Initialized upgrade context with:', - `- currentVersion (migrating to): ${currentAppVersion}`, - `- fromWorkspaceVersion: ${fromWorkspaceVersion}`, - `- ${commands.length} commands`, - ].join('\n '), - ), - ); + const instanceCommands = + this.versionedMigrationRegistryService.getInstanceCommandsForVersion( + currentVersionMajorMinor, + ); - return { fromWorkspaceVersion, currentAppVersion, commands }; + return { + fromWorkspaceVersion, + currentAppVersion, + currentVersionMajorMinor, + workspaceCommands, + instanceCommands, + }; } private async runOnWorkspace( @@ -236,7 +292,7 @@ Please roll back to that version and run the upgrade command again.`, versionContext: VersionContext, ): Promise { const { workspaceId, index, total } = iteratorContext; - const { fromWorkspaceVersion, currentAppVersion, commands } = + const { fromWorkspaceVersion, currentAppVersion, workspaceCommands } = versionContext; this.logger.log( @@ -258,8 +314,8 @@ Please roll back to that version and run the upgrade command again.`, ); } case 'equal': { - for (const command of commands) { - await command.runOnWorkspace({ + for (const workspaceCommand of workspaceCommands) { + await workspaceCommand.runOnWorkspace({ options: options as RunOnWorkspaceArgs['options'], workspaceId, dataSource: iteratorContext.dataSource, diff --git a/packages/twenty-server/src/database/commands/core-migration-runner/core-migration-runner.module.ts b/packages/twenty-server/src/database/commands/core-migration-runner/core-migration-runner.module.ts deleted file mode 100644 index 927a69092b..0000000000 --- a/packages/twenty-server/src/database/commands/core-migration-runner/core-migration-runner.module.ts +++ /dev/null @@ -1,9 +0,0 @@ -import { Module } from '@nestjs/common'; - -import { CoreMigrationRunnerService } from 'src/database/commands/core-migration-runner/services/core-migration-runner.service'; - -@Module({ - providers: [CoreMigrationRunnerService], - exports: [CoreMigrationRunnerService], -}) -export class CoreMigrationRunnerModule {} diff --git a/packages/twenty-server/src/database/commands/core-migration-runner/services/core-migration-runner.service.ts b/packages/twenty-server/src/database/commands/core-migration-runner/services/core-migration-runner.service.ts deleted file mode 100644 index 498ebfdf29..0000000000 --- a/packages/twenty-server/src/database/commands/core-migration-runner/services/core-migration-runner.service.ts +++ /dev/null @@ -1,37 +0,0 @@ -import { Injectable, Logger } from '@nestjs/common'; -import { InjectDataSource } from '@nestjs/typeorm'; - -import { DataSource } from 'typeorm'; - -@Injectable() -export class CoreMigrationRunnerService { - private readonly logger = new Logger(CoreMigrationRunnerService.name); - - constructor( - @InjectDataSource() - private readonly dataSource: DataSource, - ) {} - - async run(): Promise { - this.logger.log('Running core datasource migrations...'); - - try { - const migrations = await this.dataSource.runMigrations({ - transaction: 'each', - }); - - if (migrations.length === 0) { - this.logger.log('No pending migrations'); - } else { - this.logger.log( - `Executed ${migrations.length} migration(s): ${migrations.map((migration) => migration.name).join(', ')}`, - ); - } - - this.logger.log('Database migrations completed successfully'); - } catch (error) { - this.logger.error('Error running database migrations:', error); - throw error; - } - } -} diff --git a/packages/twenty-server/src/database/commands/core-migration/__tests__/__snapshots__/core-migration-generator.service.spec.ts.snap b/packages/twenty-server/src/database/commands/core-migration/__tests__/__snapshots__/core-migration-generator.service.spec.ts.snap new file mode 100644 index 0000000000..a4335f558b --- /dev/null +++ b/packages/twenty-server/src/database/commands/core-migration/__tests__/__snapshots__/core-migration-generator.service.spec.ts.snap @@ -0,0 +1,171 @@ +// Jest Snapshot v1, https://jestjs.io/docs/snapshot-testing + +exports[`CoreMigrationGeneratorService should encode version correctly in file and class names 1`] = ` +{ + "className": "TestV11901775000000000", + "fileName": "1775000000000-1-19-0-test.ts", + "fileTemplate": "import { MigrationInterface, QueryRunner } from 'typeorm'; + +import { RegisteredCoreMigration } from 'src/database/typeorm/core/decorators/registered-core-migration.decorator'; + +@RegisteredCoreMigration('1.19.0') +export class TestV11901775000000000 implements MigrationInterface { + name = 'TestV11901775000000000'; + + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query('SELECT 1'); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query('SELECT 1'); + } +} +", +} +`; + +exports[`CoreMigrationGeneratorService should escape backslashes in SQL queries 1`] = ` +{ + "className": "UpdatePathV12101775000000000", + "fileName": "1775000000000-1-21-0-update-path.ts", + "fileTemplate": "import { MigrationInterface, QueryRunner } from 'typeorm'; + +import { RegisteredCoreMigration } from 'src/database/typeorm/core/decorators/registered-core-migration.decorator'; + +@RegisteredCoreMigration('1.21.0') +export class UpdatePathV12101775000000000 implements MigrationInterface { + name = 'UpdatePathV12101775000000000'; + + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query('UPDATE "core"."config" SET "value" = E\\'path\\\\\\\\to\\\\\\\\file\\''); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query('UPDATE "core"."config" SET "value" = NULL'); + } +} +", +} +`; + +exports[`CoreMigrationGeneratorService should escape single quotes in SQL queries 1`] = ` +{ + "className": "UpdateConfigV12101775000000000", + "fileName": "1775000000000-1-21-0-update-config.ts", + "fileTemplate": "import { MigrationInterface, QueryRunner } from 'typeorm'; + +import { RegisteredCoreMigration } from 'src/database/typeorm/core/decorators/registered-core-migration.decorator'; + +@RegisteredCoreMigration('1.21.0') +export class UpdateConfigV12101775000000000 implements MigrationInterface { + name = 'UpdateConfigV12101775000000000'; + + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query('UPDATE "core"."config" SET "value" = \\'it\\'\\'s done\\''); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query('UPDATE "core"."config" SET "value" = \\'original\\''); + } +} +", +} +`; + +exports[`CoreMigrationGeneratorService should generate a migration with a single up/down query 1`] = ` +{ + "className": "AddFooColumnV12101775000000000", + "fileName": "1775000000000-1-21-0-add-foo-column.ts", + "fileTemplate": "import { MigrationInterface, QueryRunner } from 'typeorm'; + +import { RegisteredCoreMigration } from 'src/database/typeorm/core/decorators/registered-core-migration.decorator'; + +@RegisteredCoreMigration('1.21.0') +export class AddFooColumnV12101775000000000 implements MigrationInterface { + name = 'AddFooColumnV12101775000000000'; + + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query('ALTER TABLE "core"."user" ADD "foo" varchar'); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query('ALTER TABLE "core"."user" DROP COLUMN "foo"'); + } +} +", +} +`; + +exports[`CoreMigrationGeneratorService should generate a migration with multiple queries 1`] = ` +{ + "className": "CreateTaskTableV12101775000000000", + "fileName": "1775000000000-1-21-0-create-task-table.ts", + "fileTemplate": "import { MigrationInterface, QueryRunner } from 'typeorm'; + +import { RegisteredCoreMigration } from 'src/database/typeorm/core/decorators/registered-core-migration.decorator'; + +@RegisteredCoreMigration('1.21.0') +export class CreateTaskTableV12101775000000000 implements MigrationInterface { + name = 'CreateTaskTableV12101775000000000'; + + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query('CREATE TABLE "core"."task" ("id" uuid NOT NULL DEFAULT uuid_generate_v4(), "name" varchar NOT NULL)'); + await queryRunner.query('ALTER TABLE "core"."task" ADD CONSTRAINT "PK_task" PRIMARY KEY ("id")'); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query('DROP TABLE "core"."task"'); + await queryRunner.query('ALTER TABLE "core"."task" DROP CONSTRAINT "PK_task"'); + } +} +", +} +`; + +exports[`CoreMigrationGeneratorService should generate a migration with query parameters 1`] = ` +{ + "className": "SeedSettingV12101775000000000", + "fileName": "1775000000000-1-21-0-seed-setting.ts", + "fileTemplate": "import { MigrationInterface, QueryRunner } from 'typeorm'; + +import { RegisteredCoreMigration } from 'src/database/typeorm/core/decorators/registered-core-migration.decorator'; + +@RegisteredCoreMigration('1.21.0') +export class SeedSettingV12101775000000000 implements MigrationInterface { + name = 'SeedSettingV12101775000000000'; + + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query('INSERT INTO "core"."setting" ("key", "value") VALUES ($1, $2)', ["theme","dark"]); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query('DELETE FROM "core"."setting" WHERE "key" = $1', ["theme"]); + } +} +", +} +`; + +exports[`CoreMigrationGeneratorService should use default migration name in class and file names 1`] = ` +{ + "className": "AutoGeneratedV12101775000000000", + "fileName": "1775000000000-1-21-0-auto-generated.ts", + "fileTemplate": "import { MigrationInterface, QueryRunner } from 'typeorm'; + +import { RegisteredCoreMigration } from 'src/database/typeorm/core/decorators/registered-core-migration.decorator'; + +@RegisteredCoreMigration('1.21.0') +export class AutoGeneratedV12101775000000000 implements MigrationInterface { + name = 'AutoGeneratedV12101775000000000'; + + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query('ALTER TABLE "core"."user" ADD "bar" integer'); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query('ALTER TABLE "core"."user" DROP COLUMN "bar"'); + } +} +", +} +`; diff --git a/packages/twenty-server/src/database/commands/core-migration/__tests__/core-migration-generator.service.spec.ts b/packages/twenty-server/src/database/commands/core-migration/__tests__/core-migration-generator.service.spec.ts new file mode 100644 index 0000000000..b05429e5c6 --- /dev/null +++ b/packages/twenty-server/src/database/commands/core-migration/__tests__/core-migration-generator.service.spec.ts @@ -0,0 +1,180 @@ +import { Test } from '@nestjs/testing'; +import { getDataSourceToken } from '@nestjs/typeorm'; + +import { CoreMigrationGeneratorService } from 'src/database/commands/core-migration/services/core-migration-generator.service'; + +const FIXED_TIMESTAMP = 1775000000000; + +const buildMockDataSource = ( + upQueries: { query: string; parameters?: unknown[] }[], + downQueries: { query: string; parameters?: unknown[] }[], +) => ({ + driver: { + createSchemaBuilder: () => ({ + log: jest.fn().mockResolvedValue({ upQueries, downQueries }), + }), + }, +}); + +describe('CoreMigrationGeneratorService', () => { + const buildService = async ( + upQueries: { query: string; parameters?: unknown[] }[] = [], + downQueries: { query: string; parameters?: unknown[] }[] = [], + ) => { + const module = await Test.createTestingModule({ + providers: [ + CoreMigrationGeneratorService, + { + provide: getDataSourceToken(), + useValue: buildMockDataSource(upQueries, downQueries), + }, + ], + }).compile(); + + return module.get(CoreMigrationGeneratorService); + }; + + it('should return null when no schema changes are detected', async () => { + const service = await buildService(); + + const result = await service.generate({ + migrationName: 'no-changes', + version: '1.21.0', + timestamp: FIXED_TIMESTAMP, + }); + + expect(result).toBeNull(); + }); + + it('should generate a migration with a single up/down query', async () => { + const service = await buildService( + [{ query: 'ALTER TABLE "core"."user" ADD "foo" varchar' }], + [{ query: 'ALTER TABLE "core"."user" DROP COLUMN "foo"' }], + ); + + const result = await service.generate({ + migrationName: 'add-foo-column', + version: '1.21.0', + timestamp: FIXED_TIMESTAMP, + }); + + expect(result).toMatchSnapshot(); + }); + + it('should generate a migration with multiple queries', async () => { + const service = await buildService( + [ + { + query: + 'CREATE TABLE "core"."task" ("id" uuid NOT NULL DEFAULT uuid_generate_v4(), "name" varchar NOT NULL)', + }, + { + query: + 'ALTER TABLE "core"."task" ADD CONSTRAINT "PK_task" PRIMARY KEY ("id")', + }, + ], + [ + { query: 'ALTER TABLE "core"."task" DROP CONSTRAINT "PK_task"' }, + { query: 'DROP TABLE "core"."task"' }, + ], + ); + + const result = await service.generate({ + migrationName: 'create-task-table', + version: '1.21.0', + timestamp: FIXED_TIMESTAMP, + }); + + expect(result).toMatchSnapshot(); + }); + + it('should generate a migration with query parameters', async () => { + const service = await buildService( + [ + { + query: + 'INSERT INTO "core"."setting" ("key", "value") VALUES ($1, $2)', + parameters: ['theme', 'dark'], + }, + ], + [ + { + query: 'DELETE FROM "core"."setting" WHERE "key" = $1', + parameters: ['theme'], + }, + ], + ); + + const result = await service.generate({ + migrationName: 'seed-setting', + version: '1.21.0', + timestamp: FIXED_TIMESTAMP, + }); + + expect(result).toMatchSnapshot(); + }); + + it('should escape single quotes in SQL queries', async () => { + const service = await buildService( + [{ query: 'UPDATE "core"."config" SET "value" = \'it\'\'s done\'' }], + [{ query: 'UPDATE "core"."config" SET "value" = \'original\'' }], + ); + + const result = await service.generate({ + migrationName: 'update-config', + version: '1.21.0', + timestamp: FIXED_TIMESTAMP, + }); + + expect(result).toMatchSnapshot(); + }); + + it('should escape backslashes in SQL queries', async () => { + const service = await buildService( + [ + { + query: 'UPDATE "core"."config" SET "value" = E\'path\\\\to\\\\file\'', + }, + ], + [{ query: 'UPDATE "core"."config" SET "value" = NULL' }], + ); + + const result = await service.generate({ + migrationName: 'update-path', + version: '1.21.0', + timestamp: FIXED_TIMESTAMP, + }); + + expect(result).toMatchSnapshot(); + }); + + it('should use default migration name in class and file names', async () => { + const service = await buildService( + [{ query: 'ALTER TABLE "core"."user" ADD "bar" integer' }], + [{ query: 'ALTER TABLE "core"."user" DROP COLUMN "bar"' }], + ); + + const result = await service.generate({ + migrationName: 'auto-generated', + version: '1.21.0', + timestamp: FIXED_TIMESTAMP, + }); + + expect(result).toMatchSnapshot(); + }); + + it('should encode version correctly in file and class names', async () => { + const service = await buildService( + [{ query: 'SELECT 1' }], + [{ query: 'SELECT 1' }], + ); + + const result = await service.generate({ + migrationName: 'test', + version: '1.19.0', + timestamp: FIXED_TIMESTAMP, + }); + + expect(result).toMatchSnapshot(); + }); +}); diff --git a/packages/twenty-server/src/database/commands/core-migration/__tests__/versioned-migration-registry.service.spec.ts b/packages/twenty-server/src/database/commands/core-migration/__tests__/versioned-migration-registry.service.spec.ts new file mode 100644 index 0000000000..ce02df1e90 --- /dev/null +++ b/packages/twenty-server/src/database/commands/core-migration/__tests__/versioned-migration-registry.service.spec.ts @@ -0,0 +1,138 @@ +import 'reflect-metadata'; + +import { Test } from '@nestjs/testing'; +import { getDataSourceToken } from '@nestjs/typeorm'; + +import { type MigrationInterface } from 'typeorm'; + +import { RegisteredCoreMigrationService } from 'src/database/commands/core-migration/services/registered-core-migration-registry.service'; +import { RegisteredCoreMigration } from 'src/database/typeorm/core/decorators/registered-core-migration.decorator'; + +@RegisteredCoreMigration('1.21.0') +class MigrationA1770000000000 implements MigrationInterface { + name = 'MigrationA1770000000000'; + + async up(): Promise {} + async down(): Promise {} +} + +@RegisteredCoreMigration('1.21.0') +class MigrationB1771000000000 implements MigrationInterface { + name = 'MigrationB1771000000000'; + + async up(): Promise {} + async down(): Promise {} +} + +@RegisteredCoreMigration('1.21.0') +class MigrationC1772000000000 implements MigrationInterface { + name = 'MigrationC1772000000000'; + + async up(): Promise {} + async down(): Promise {} +} + +@RegisteredCoreMigration('1.20.0') +class MigrationD1769000000000 implements MigrationInterface { + name = 'MigrationD1769000000000'; + + async up(): Promise {} + async down(): Promise {} +} + +class UndecoratedMigration1768000000000 implements MigrationInterface { + name = 'UndecoratedMigration1768000000000'; + + async up(): Promise {} + async down(): Promise {} +} + +const buildRegistryService = async ( + migrations: MigrationInterface[], +): Promise => { + const module = await Test.createTestingModule({ + providers: [ + RegisteredCoreMigrationService, + { + provide: getDataSourceToken(), + useValue: { migrations }, + }, + ], + }).compile(); + + const service = module.get(RegisteredCoreMigrationService); + + service.onModuleInit(); + + return service; +}; + +describe('VersionedMigrationRegistryService', () => { + it('should group migrations by version', async () => { + const service = await buildRegistryService([ + new MigrationD1769000000000(), + new MigrationA1770000000000(), + new MigrationB1771000000000(), + new MigrationC1772000000000(), + ]); + + const v120 = service.getInstanceCommandsForVersion('1.20.0'); + const v121 = service.getInstanceCommandsForVersion('1.21.0'); + + expect(v120.map((m) => m.constructor.name)).toStrictEqual([ + 'MigrationD1769000000000', + ]); + + expect(v121.map((m) => m.constructor.name)).toStrictEqual([ + 'MigrationA1770000000000', + 'MigrationB1771000000000', + 'MigrationC1772000000000', + ]); + }); + + it('should preserve input order within a version bucket', async () => { + const service = await buildRegistryService([ + new MigrationA1770000000000(), + new MigrationC1772000000000(), + new MigrationB1771000000000(), + ]); + + const names = service + .getInstanceCommandsForVersion('1.21.0') + .map((m) => m.constructor.name); + + expect(names).toStrictEqual([ + 'MigrationA1770000000000', + 'MigrationC1772000000000', + 'MigrationB1771000000000', + ]); + }); + + it('should skip undecorated migrations', async () => { + const service = await buildRegistryService([ + new UndecoratedMigration1768000000000(), + new MigrationA1770000000000(), + ]); + + const v121 = service.getInstanceCommandsForVersion('1.21.0'); + + expect(v121).toHaveLength(1); + expect(v121[0].constructor.name).toBe('MigrationA1770000000000'); + }); + + it('should return empty array for version with no migrations', async () => { + const service = await buildRegistryService([]); + + expect(service.getInstanceCommandsForVersion('1.19.0')).toStrictEqual([]); + expect(service.getInstanceCommandsForVersion('1.20.0')).toStrictEqual([]); + expect(service.getInstanceCommandsForVersion('1.21.0')).toStrictEqual([]); + }); + + it('should return empty array for unsupported version', async () => { + const service = await buildRegistryService([]); + + expect( + service.getInstanceCommandsForVersion('99.0.0' as unknown as '1.21.0'), + ).toStrictEqual([]); + }); +}); diff --git a/packages/twenty-server/src/database/commands/core-migration/core-migration.module.ts b/packages/twenty-server/src/database/commands/core-migration/core-migration.module.ts new file mode 100644 index 0000000000..17b083fff1 --- /dev/null +++ b/packages/twenty-server/src/database/commands/core-migration/core-migration.module.ts @@ -0,0 +1,19 @@ +import { Module } from '@nestjs/common'; + +import { CoreMigrationGeneratorService } from 'src/database/commands/core-migration/services/core-migration-generator.service'; +import { CoreMigrationRunnerService } from 'src/database/commands/core-migration/services/core-migration-runner.service'; +import { RegisteredCoreMigrationService } from 'src/database/commands/core-migration/services/registered-core-migration-registry.service'; + +@Module({ + providers: [ + CoreMigrationGeneratorService, + CoreMigrationRunnerService, + RegisteredCoreMigrationService, + ], + exports: [ + CoreMigrationGeneratorService, + CoreMigrationRunnerService, + RegisteredCoreMigrationService, + ], +}) +export class CoreMigrationModule {} diff --git a/packages/twenty-server/src/database/commands/core-migration/services/core-migration-generator.service.ts b/packages/twenty-server/src/database/commands/core-migration/services/core-migration-generator.service.ts new file mode 100644 index 0000000000..a64ae3209f --- /dev/null +++ b/packages/twenty-server/src/database/commands/core-migration/services/core-migration-generator.service.ts @@ -0,0 +1,111 @@ +import { Injectable } from '@nestjs/common'; +import { InjectDataSource } from '@nestjs/typeorm'; + +import { pascalCase } from 'twenty-shared/utils'; +import { DataSource } from 'typeorm'; + +import { type UpgradeCommandVersion } from 'src/engine/constants/upgrade-command-supported-versions.constant'; + +type GenerateMigrationArgs = { + migrationName: string; + version: UpgradeCommandVersion; + timestamp: number; +}; + +export type GeneratedMigrationResult = { + fileName: string; + fileTemplate: string; + className: string; +}; + +@Injectable() +export class CoreMigrationGeneratorService { + constructor( + @InjectDataSource() + private readonly dataSource: DataSource, + ) {} + + async generate({ + migrationName, + version, + timestamp, + }: GenerateMigrationArgs): Promise { + const sqlInMemory = await this.dataSource.driver + .createSchemaBuilder() + .log(); + + if (sqlInMemory.upQueries.length === 0) { + return null; + } + + const className = this.buildClassName(migrationName, version, timestamp); + + const upStatements = sqlInMemory.upQueries.map( + ({ query, parameters }) => + ` await queryRunner.query('${this.escapeForSingleQuotedString(query)}'${this.formatQueryParams(parameters)});`, + ); + + const downStatements = sqlInMemory.downQueries + .reverse() + .map( + ({ query, parameters }) => + ` await queryRunner.query('${this.escapeForSingleQuotedString(query)}'${this.formatQueryParams(parameters)});`, + ); + + const fileTemplate = this.buildMigrationFileContent( + className, + version, + upStatements, + downStatements, + ); + + const fileName = `${timestamp}-${version.replace(/\./g, '-')}-${migrationName}.ts`; + + return { fileName, fileTemplate, className }; + } + + private buildClassName( + name: string, + version: string, + timestamp: number, + ): string { + return `${pascalCase(name)}V${version.replace(/\./g, '')}${timestamp}`; + } + + private formatQueryParams(parameters: unknown[] | undefined): string { + if (!parameters || !parameters.length) { + return ''; + } + + return `, ${JSON.stringify(parameters)}`; + } + + private escapeForSingleQuotedString(query: string): string { + return query.replace(/\\/g, '\\\\').replace(/'/g, "\\'"); + } + + private buildMigrationFileContent( + className: string, + version: string, + upStatements: string[], + downStatements: string[], + ): string { + return `import { MigrationInterface, QueryRunner } from 'typeorm'; + +import { RegisteredCoreMigration } from 'src/database/typeorm/core/decorators/registered-core-migration.decorator'; + +@RegisteredCoreMigration('${version}') +export class ${className} implements MigrationInterface { + name = '${className}'; + + public async up(queryRunner: QueryRunner): Promise { +${upStatements.join('\n')} + } + + public async down(queryRunner: QueryRunner): Promise { +${downStatements.join('\n')} + } +} +`; + } +} diff --git a/packages/twenty-server/src/database/commands/core-migration/services/core-migration-runner.service.ts b/packages/twenty-server/src/database/commands/core-migration/services/core-migration-runner.service.ts new file mode 100644 index 0000000000..5d05a19160 --- /dev/null +++ b/packages/twenty-server/src/database/commands/core-migration/services/core-migration-runner.service.ts @@ -0,0 +1,147 @@ +import { Injectable, Logger } from '@nestjs/common'; +import { InjectDataSource } from '@nestjs/typeorm'; + +import { DataSource, MigrationExecutor, type QueryRunner } from 'typeorm'; + +export type RunSingleMigrationError = + | 'already-executed' + | 'migration-execution-failed' + | 'migration-instance-not-defined' + | 'migration-not-registered'; + +export type RunSingleMigrationResult = + | { status: 'success' } + | { code: RunSingleMigrationError; error?: unknown; status: 'fail' }; + +@Injectable() +export class CoreMigrationRunnerService { + private readonly logger = new Logger(CoreMigrationRunnerService.name); + + constructor( + @InjectDataSource() + private readonly dataSource: DataSource, + ) {} + + async runAllPendingMigrations(): Promise { + this.logger.log('Running core datasource migrations...'); + + try { + const migrations = await this.dataSource.runMigrations({ + transaction: 'each', + }); + + if (migrations.length === 0) { + this.logger.log('No pending migrations'); + } else { + this.logger.log( + `Executed ${migrations.length} migration(s): ${migrations.map((migration) => migration.name).join(', ')}`, + ); + } + + this.logger.log('Database migrations completed successfully'); + } catch (error) { + this.logger.error('Error running database migrations:', error); + throw error; + } + } + + async runSingleMigration( + migrationName: string, + ): Promise { + this.logger.log(`Running core datasource migration ${migrationName}...`); + + const queryRunner = this.dataSource.createQueryRunner(); + const migrationExecutor = new MigrationExecutor( + this.dataSource, + queryRunner, + ); + + try { + await queryRunner.connect(); + + const pendingMigrations = await migrationExecutor.getPendingMigrations(); + const pendingMigration = pendingMigrations.find( + (migration) => migration.name === migrationName, + ); + + if (!pendingMigration) { + const registeredMigration = ( + await migrationExecutor.getAllMigrations() + ).find((migration) => migration.name === migrationName); + + if (!registeredMigration) { + return { + code: 'migration-not-registered', + status: 'fail', + }; + } + + return { + code: 'already-executed', + status: 'fail', + }; + } + + if (!pendingMigration.instance) { + return { + code: 'migration-instance-not-defined', + status: 'fail', + }; + } + + await this.createMetadataTableIfNecessary(queryRunner); + + const shouldRunInTransaction = + pendingMigration.instance.transaction ?? true; + + await queryRunner.beforeMigration(); + + try { + if (shouldRunInTransaction) { + await queryRunner.startTransaction(); + } + + await pendingMigration.instance.up(queryRunner); + await migrationExecutor.insertMigration(pendingMigration); + + if (shouldRunInTransaction) { + await queryRunner.commitTransaction(); + } + } catch (error) { + if (shouldRunInTransaction && queryRunner.isTransactionActive) { + await queryRunner.rollbackTransaction(); + } + + throw error; + } finally { + await queryRunner.afterMigration(); + } + + return { status: 'success' }; + } catch (error) { + this.logger.error('Error running database migration:', error); + + return { + code: 'migration-execution-failed', + error, + status: 'fail', + }; + } finally { + await queryRunner.release(); + } + } + + private async createMetadataTableIfNecessary( + queryRunner: QueryRunner, + ): Promise { + const schemaBuilder = this.dataSource.driver.createSchemaBuilder() as { + createMetadataTableIfNecessary?: ( + queryRunnerToUse: QueryRunner, + ) => Promise; + }; + + if (schemaBuilder.createMetadataTableIfNecessary) { + await schemaBuilder.createMetadataTableIfNecessary(queryRunner); + } + } +} diff --git a/packages/twenty-server/src/database/commands/core-migration/services/registered-core-migration-registry.service.ts b/packages/twenty-server/src/database/commands/core-migration/services/registered-core-migration-registry.service.ts new file mode 100644 index 0000000000..2dba9ae6a6 --- /dev/null +++ b/packages/twenty-server/src/database/commands/core-migration/services/registered-core-migration-registry.service.ts @@ -0,0 +1,64 @@ +import { Injectable, Logger, type OnModuleInit } from '@nestjs/common'; +import { InjectDataSource } from '@nestjs/typeorm'; + +import { DataSource, type MigrationInterface } from 'typeorm'; + +import { getRegisteredCoreMigrationVersion } from 'src/database/typeorm/core/decorators/registered-core-migration.decorator'; +import { + UPGRADE_COMMAND_SUPPORTED_VERSIONS, + type UpgradeCommandVersion, +} from 'src/engine/constants/upgrade-command-supported-versions.constant'; + +@Injectable() +export class RegisteredCoreMigrationService implements OnModuleInit { + private readonly logger = new Logger(RegisteredCoreMigrationService.name); + + private readonly migrationsByVersion = new Map< + UpgradeCommandVersion, + MigrationInterface[] + >(); + + constructor( + @InjectDataSource() + private readonly dataSource: DataSource, + ) {} + + onModuleInit(): void { + for (const version of UPGRADE_COMMAND_SUPPORTED_VERSIONS) { + this.migrationsByVersion.set(version, []); + } + + // dataSource.migrations is already sorted by timestamp (TypeORM sorts + // ascending by the 13-digit suffix of the class name) + for (const migration of this.dataSource.migrations) { + const constructor = migration.constructor; + const version = getRegisteredCoreMigrationVersion(constructor); + + if (version === undefined) { + continue; + } + + const bucket = this.migrationsByVersion.get(version); + + if (!bucket) { + continue; + } + + bucket.push(migration); + } + + for (const [version, migrations] of this.migrationsByVersion) { + if (migrations.length > 0) { + this.logger.log( + `Registered ${migrations.length} versioned migration(s) for ${version}: ${migrations.map((migration) => migration.constructor.name).join(', ')}`, + ); + } + } + } + + getInstanceCommandsForVersion( + version: UpgradeCommandVersion, + ): MigrationInterface[] { + return this.migrationsByVersion.get(version) ?? []; + } +} diff --git a/packages/twenty-server/src/database/commands/database-command.module.ts b/packages/twenty-server/src/database/commands/database-command.module.ts index ba7d429da6..6a44c66f61 100644 --- a/packages/twenty-server/src/database/commands/database-command.module.ts +++ b/packages/twenty-server/src/database/commands/database-command.module.ts @@ -3,14 +3,15 @@ import { TypeOrmModule } from '@nestjs/typeorm'; import { CronRegisterAllCommand } from 'src/database/commands/cron-register-all.command'; import { DataSeedWorkspaceCommand } from 'src/database/commands/data-seed-dev-workspace.command'; +import { GenerateVersionedMigrationCommand } from 'src/database/commands/generate-versioned-migration.command'; import { ListOrphanedWorkspaceEntitiesCommand } from 'src/database/commands/list-and-delete-orphaned-workspace-entities.command'; import { ConfirmationQuestion } from 'src/database/commands/questions/confirmation.question'; -import { RunTypeormMigrationCommand } from 'src/database/commands/run-typeorm-migration.command'; +import { RunCoreMigrationCommand } from 'src/database/commands/run-core-migration.command'; import { UpgradeVersionCommandModule } from 'src/database/commands/upgrade-version-command/upgrade-version-command.module'; import { WorkspaceExportModule } from 'src/database/commands/workspace-export/workspace-export.module'; import { TypeORMModule } from 'src/database/typeorm/typeorm.module'; import { CoreEngineVersionModule } from 'src/engine/core-engine-version/core-engine-version.module'; -import { CoreMigrationRunnerModule } from 'src/database/commands/core-migration-runner/core-migration-runner.module'; +import { CoreMigrationModule } from 'src/database/commands/core-migration/core-migration.module'; import { ApiKeyModule } from 'src/engine/core-modules/api-key/api-key.module'; import { GenerateApiKeyCommand } from 'src/engine/core-modules/api-key/commands/generate-api-key.command'; import { MarketplaceModule } from 'src/engine/core-modules/application/application-marketplace/marketplace.module'; @@ -73,17 +74,18 @@ import { AutomatedTriggerModule } from 'src/modules/workflow/workflow-trigger/au ApplicationUpgradeModule, StaleRegistrationCleanupModule, CoreEngineVersionModule, - CoreMigrationRunnerModule, + CoreMigrationModule, WorkspaceVersionModule, ], providers: [ DataSeedWorkspaceCommand, ConfirmationQuestion, CronRegisterAllCommand, + GenerateVersionedMigrationCommand, ListOrphanedWorkspaceEntitiesCommand, EnterpriseKeyValidationCronCommand, GenerateApiKeyCommand, - RunTypeormMigrationCommand, + RunCoreMigrationCommand, ], }) export class DatabaseCommandModule {} diff --git a/packages/twenty-server/src/database/commands/generate-versioned-migration.command.ts b/packages/twenty-server/src/database/commands/generate-versioned-migration.command.ts new file mode 100644 index 0000000000..753c70ed6c --- /dev/null +++ b/packages/twenty-server/src/database/commands/generate-versioned-migration.command.ts @@ -0,0 +1,81 @@ +import * as fs from 'fs'; +import * as path from 'path'; + +import { Logger } from '@nestjs/common'; + +import { Command, CommandRunner, Option } from 'nest-commander'; + +import { CoreMigrationGeneratorService } from 'src/database/commands/core-migration/services/core-migration-generator.service'; +import { UPGRADE_COMMAND_SUPPORTED_VERSIONS } from 'src/engine/constants/upgrade-command-supported-versions.constant'; + +const MIGRATIONS_DIR = path.resolve( + process.cwd(), + 'src/database/typeorm/core/migrations/common', +); + +type GenerateVersionedMigrationCommandOptions = { + name: string; +}; + +@Command({ + name: 'generate:versioned-migration', + description: + 'Generate a TypeORM migration with @RegisteredCoreMigration decorator for the latest supported version', +}) +export class GenerateVersionedMigrationCommand extends CommandRunner { + private readonly logger = new Logger(GenerateVersionedMigrationCommand.name); + + constructor( + private readonly coreMigrationGeneratorService: CoreMigrationGeneratorService, + ) { + super(); + } + + @Option({ + flags: '-n, --name ', + description: 'Migration name (kebab-case)', + defaultValue: 'auto-generated', + }) + parseName(value: string): string { + return value; + } + + async run( + _passedParams: string[], + options: GenerateVersionedMigrationCommandOptions, + ): Promise { + const migrationName = options.name; + + const version = UPGRADE_COMMAND_SUPPORTED_VERSIONS.slice(-1)[0]; + + if (!version) { + throw new Error('No supported versions found'); + } + + this.logger.log(`Generating versioned migration for version ${version}...`); + + const timestamp = Date.now(); + + const result = await this.coreMigrationGeneratorService.generate({ + migrationName, + version, + timestamp, + }); + + if (!result) { + this.logger.warn( + 'No changes in database schema were found - cannot generate a migration.', + ); + + return; + } + + const filePath = path.join(MIGRATIONS_DIR, result.fileName); + + fs.writeFileSync(filePath, result.fileTemplate); + + this.logger.log(`Migration generated successfully: ${filePath}`); + this.logger.log(` Class: ${result.className}`); + this.logger.log(` Version: ${version}`); + } +} diff --git a/packages/twenty-server/src/database/commands/run-typeorm-migration.command.ts b/packages/twenty-server/src/database/commands/run-core-migration.command.ts similarity index 85% rename from packages/twenty-server/src/database/commands/run-typeorm-migration.command.ts rename to packages/twenty-server/src/database/commands/run-core-migration.command.ts index 1dc4320830..feee605bb4 100644 --- a/packages/twenty-server/src/database/commands/run-typeorm-migration.command.ts +++ b/packages/twenty-server/src/database/commands/run-core-migration.command.ts @@ -3,21 +3,21 @@ import { Logger } from '@nestjs/common'; import chalk from 'chalk'; import { Command, CommandRunner, Option } from 'nest-commander'; -import { CoreMigrationRunnerService } from 'src/database/commands/core-migration-runner/services/core-migration-runner.service'; +import { CoreMigrationRunnerService } from 'src/database/commands/core-migration/services/core-migration-runner.service'; import { CoreEngineVersionService } from 'src/engine/core-engine-version/services/core-engine-version.service'; import { WorkspaceVersionService } from 'src/engine/workspace-manager/workspace-version/services/workspace-version.service'; -type RunTypeormMigrationCommandOptions = { +type RunCoreMigrationCommandOptions = { force?: boolean; }; @Command({ - name: 'run-typeorm-migration', + name: 'run-core-migration', description: 'Run TypeORM core migrations with workspace version safety check', }) -export class RunTypeormMigrationCommand extends CommandRunner { - private readonly logger = new Logger(RunTypeormMigrationCommand.name); +export class RunCoreMigrationCommand extends CommandRunner { + private readonly logger = new Logger(RunCoreMigrationCommand.name); constructor( private readonly coreEngineVersionService: CoreEngineVersionService, @@ -38,7 +38,7 @@ export class RunTypeormMigrationCommand extends CommandRunner { async run( _passedParams: string[], - options: RunTypeormMigrationCommandOptions, + options: RunCoreMigrationCommandOptions, ): Promise { if (options.force) { this.logger.warn( @@ -70,6 +70,6 @@ export class RunTypeormMigrationCommand extends CommandRunner { } } - await this.coreMigrationRunnerService.run(); + await this.coreMigrationRunnerService.runAllPendingMigrations(); } } diff --git a/packages/twenty-server/src/database/commands/upgrade-version-command/upgrade-version-command.module.ts b/packages/twenty-server/src/database/commands/upgrade-version-command/upgrade-version-command.module.ts index c4d1628676..bae34fffa1 100644 --- a/packages/twenty-server/src/database/commands/upgrade-version-command/upgrade-version-command.module.ts +++ b/packages/twenty-server/src/database/commands/upgrade-version-command/upgrade-version-command.module.ts @@ -2,7 +2,7 @@ import { Module } from '@nestjs/common'; import { TypeOrmModule } from '@nestjs/typeorm'; import { WorkspaceIteratorModule } from 'src/database/commands/command-runners/workspace-iterator.module'; -import { CoreMigrationRunnerModule } from 'src/database/commands/core-migration-runner/core-migration-runner.module'; +import { CoreMigrationModule } from 'src/database/commands/core-migration/core-migration.module'; import { V1_20_UpgradeVersionCommandModule } from 'src/database/commands/upgrade-version-command/1-20/1-20-upgrade-version-command.module'; import { V1_21_UpgradeVersionCommandModule } from 'src/database/commands/upgrade-version-command/1-21/1-21-upgrade-version-command.module'; import { UpgradeCommand } from 'src/database/commands/upgrade-version-command/upgrade.command'; @@ -18,7 +18,7 @@ import { WorkspaceVersionModule } from 'src/engine/workspace-manager/workspace-v V1_21_UpgradeVersionCommandModule, DataSourceModule, CoreEngineVersionModule, - CoreMigrationRunnerModule, + CoreMigrationModule, WorkspaceVersionModule, WorkspaceIteratorModule, ], diff --git a/packages/twenty-server/src/database/commands/upgrade-version-command/upgrade.command.ts b/packages/twenty-server/src/database/commands/upgrade-version-command/upgrade.command.ts index 24dc758219..cbe8a9ac0f 100644 --- a/packages/twenty-server/src/database/commands/upgrade-version-command/upgrade.command.ts +++ b/packages/twenty-server/src/database/commands/upgrade-version-command/upgrade.command.ts @@ -9,7 +9,8 @@ import { type VersionCommands, } from 'src/database/commands/command-runners/upgrade.command-runner'; import { WorkspaceIteratorService } from 'src/database/commands/command-runners/workspace-iterator.service'; -import { CoreMigrationRunnerService } from 'src/database/commands/core-migration-runner/services/core-migration-runner.service'; +import { CoreMigrationRunnerService } from 'src/database/commands/core-migration/services/core-migration-runner.service'; +import { RegisteredCoreMigrationService } from 'src/database/commands/core-migration/services/registered-core-migration-registry.service'; import { BackfillCommandMenuItemsCommand } from 'src/database/commands/upgrade-version-command/1-20/1-20-backfill-command-menu-items.command'; import { BackfillNavigationMenuItemTypeCommand } from 'src/database/commands/upgrade-version-command/1-20/1-20-backfill-navigation-menu-item-type.command'; import { BackfillSelectFieldOptionIdsCommand } from 'src/database/commands/upgrade-version-command/1-20/1-20-backfill-select-field-option-ids.command'; @@ -50,6 +51,7 @@ export class UpgradeCommand extends UpgradeCommandRunner { protected readonly coreEngineVersionService: CoreEngineVersionService, protected readonly workspaceVersionService: WorkspaceVersionService, protected readonly coreMigrationRunnerService: CoreMigrationRunnerService, + protected readonly versionedMigrationRegistryService: RegisteredCoreMigrationService, protected readonly workspaceIteratorService: WorkspaceIteratorService, // 1.20 Commands @@ -84,6 +86,7 @@ export class UpgradeCommand extends UpgradeCommandRunner { coreEngineVersionService, workspaceVersionService, coreMigrationRunnerService, + versionedMigrationRegistryService, workspaceIteratorService, ); diff --git a/packages/twenty-server/src/database/typeorm/core/decorators/registered-core-migration.decorator.ts b/packages/twenty-server/src/database/typeorm/core/decorators/registered-core-migration.decorator.ts new file mode 100644 index 0000000000..9a70d02299 --- /dev/null +++ b/packages/twenty-server/src/database/typeorm/core/decorators/registered-core-migration.decorator.ts @@ -0,0 +1,18 @@ +import 'reflect-metadata'; + +import { type UpgradeCommandVersion } from 'src/engine/constants/upgrade-command-supported-versions.constant'; + +const REGISTERED_CORE_MIGRATION_KEY = 'REGISTERED_CORE_MIGRATION_VERSION'; + +// When dropping a version from UPGRADE_COMMAND_SUPPORTED_VERSIONS, also +// remove the @RegisteredCoreMigration decorator from its associated migration files. +export const RegisteredCoreMigration = + (version: UpgradeCommandVersion): ClassDecorator => + (target) => { + Reflect.defineMetadata(REGISTERED_CORE_MIGRATION_KEY, version, target); + }; + +export const getRegisteredCoreMigrationVersion = ( + target: Function, +): UpgradeCommandVersion | undefined => + Reflect.getMetadata(REGISTERED_CORE_MIGRATION_KEY, target); diff --git a/packages/twenty-server/src/database/typeorm/core/migrations/common/1774700000000-add-global-key-value-pair-unique-index.ts b/packages/twenty-server/src/database/typeorm/core/migrations/common/1774700000000-add-global-key-value-pair-unique-index.ts index a187f96a34..3a90f013be 100644 --- a/packages/twenty-server/src/database/typeorm/core/migrations/common/1774700000000-add-global-key-value-pair-unique-index.ts +++ b/packages/twenty-server/src/database/typeorm/core/migrations/common/1774700000000-add-global-key-value-pair-unique-index.ts @@ -1,7 +1,9 @@ import { type MigrationInterface, type QueryRunner } from 'typeorm'; +import { RegisteredCoreMigration } from 'src/database/typeorm/core/decorators/registered-core-migration.decorator'; import { addGlobalKeyValuePairUniqueIndexQueries } from 'src/database/typeorm/core/migrations/utils/1774700000000-add-global-key-value-pair-unique-index.util'; +@RegisteredCoreMigration('1.21.0') export class AddGlobalKeyValuePairUniqueIndex1774700000000 implements MigrationInterface { diff --git a/packages/twenty-server/src/database/typeorm/core/migrations/common/1774966727625-addIsActiveToOverridableEntities.ts b/packages/twenty-server/src/database/typeorm/core/migrations/common/1774966727625-addIsActiveToOverridableEntities.ts index ba75edd24d..772f38cd88 100644 --- a/packages/twenty-server/src/database/typeorm/core/migrations/common/1774966727625-addIsActiveToOverridableEntities.ts +++ b/packages/twenty-server/src/database/typeorm/core/migrations/common/1774966727625-addIsActiveToOverridableEntities.ts @@ -1,5 +1,8 @@ import { MigrationInterface, QueryRunner } from 'typeorm'; +import { RegisteredCoreMigration } from 'src/database/typeorm/core/decorators/registered-core-migration.decorator'; + +@RegisteredCoreMigration('1.21.0') export class AddIsActiveToOverridableEntities1774966727625 implements MigrationInterface { diff --git a/packages/twenty-server/src/database/typeorm/core/migrations/common/1775001600000-add-status-to-agent-message.ts b/packages/twenty-server/src/database/typeorm/core/migrations/common/1775001600000-add-status-to-agent-message.ts index 2b81a7562b..84510b1714 100644 --- a/packages/twenty-server/src/database/typeorm/core/migrations/common/1775001600000-add-status-to-agent-message.ts +++ b/packages/twenty-server/src/database/typeorm/core/migrations/common/1775001600000-add-status-to-agent-message.ts @@ -1,5 +1,8 @@ import { MigrationInterface, QueryRunner } from 'typeorm'; +import { RegisteredCoreMigration } from 'src/database/typeorm/core/decorators/registered-core-migration.decorator'; + +@RegisteredCoreMigration('1.21.0') export class AddStatusToAgentMessage1775001600000 implements MigrationInterface { diff --git a/packages/twenty-server/src/database/typeorm/core/migrations/common/1775129420309-add-view-field-group-id-index-on-view-field.ts b/packages/twenty-server/src/database/typeorm/core/migrations/common/1775129420309-add-view-field-group-id-index-on-view-field.ts index de55c0f4c1..6c1bf809d5 100644 --- a/packages/twenty-server/src/database/typeorm/core/migrations/common/1775129420309-add-view-field-group-id-index-on-view-field.ts +++ b/packages/twenty-server/src/database/typeorm/core/migrations/common/1775129420309-add-view-field-group-id-index-on-view-field.ts @@ -1,5 +1,8 @@ import { type MigrationInterface, type QueryRunner } from 'typeorm'; +import { RegisteredCoreMigration } from 'src/database/typeorm/core/decorators/registered-core-migration.decorator'; + +@RegisteredCoreMigration('1.21.0') export class AddViewFieldGroupIdIndexOnViewField1775129420309 implements MigrationInterface {