Refactor typeorm migration lifecycle and generation (#19275)
# Introduction Typeorm migration are now associated to a given twenty-version from the `UPGRADE_COMMAND_SUPPORTED_VERSIONS` that the current twenty core engine handles This way when we upgrade we retrieve the migrations that need to be run, this will be useful for the cross-version incremental upgrade so we preserve sequentiality ## What's new To generate ```sh npx nx database:migrate:generate twenty-server -- --name add-index-to-users ``` To apply all ```sh npx nx database:migrate twenty-server ``` ## Next Introduce slow and fast typeorm migration in order to get rid of the save point pattern in our code base Create a clean and dedicated `InstanceUpgradeService` abstraction
This commit is contained in:
Paul Rastoin
+104
-16
@@ -5,14 +5,21 @@ import {
|
||||
eachTestingContextFilter,
|
||||
type EachTestingContext,
|
||||
} from 'twenty-shared/testing';
|
||||
import { type Repository } from 'typeorm';
|
||||
import {
|
||||
type MigrationInterface,
|
||||
type QueryRunner,
|
||||
type Repository,
|
||||
} from 'typeorm';
|
||||
|
||||
import {
|
||||
UpgradeCommandOptions,
|
||||
UpgradeCommandRunner,
|
||||
type AllCommands,
|
||||
} from 'src/database/commands/command-runners/upgrade.command-runner';
|
||||
import { WorkspaceIteratorService } from 'src/database/commands/command-runners/workspace-iterator.service';
|
||||
import { CoreMigrationRunnerService } from 'src/database/commands/core-migration-runner/services/core-migration-runner.service';
|
||||
import { CoreMigrationRunnerService } from 'src/database/commands/core-migration/services/core-migration-runner.service';
|
||||
import { RegisteredCoreMigrationService } from 'src/database/commands/core-migration/services/registered-core-migration-registry.service';
|
||||
import { RegisteredCoreMigration } from 'src/database/typeorm/core/decorators/registered-core-migration.decorator';
|
||||
import { UPGRADE_COMMAND_SUPPORTED_VERSIONS } from 'src/engine/constants/upgrade-command-supported-versions.constant';
|
||||
import { CoreEngineVersionService } from 'src/engine/core-engine-version/services/core-engine-version.service';
|
||||
import { type ConfigVariables } from 'src/engine/core-modules/twenty-config/config-variables';
|
||||
@@ -59,12 +66,35 @@ type BuildUpgradeCommandModuleArgs = {
|
||||
workspaces: WorkspaceEntity[];
|
||||
appVersion: string | null;
|
||||
commandRunner: CommandRunnerValues;
|
||||
migrations?: MigrationInterface[];
|
||||
};
|
||||
const buildUpgradeCommandModule = async ({
|
||||
workspaces,
|
||||
appVersion,
|
||||
commandRunner,
|
||||
migrations,
|
||||
}: BuildUpgradeCommandModuleArgs) => {
|
||||
const registryProvider = migrations
|
||||
? {
|
||||
provide: RegisteredCoreMigrationService,
|
||||
useFactory: () => {
|
||||
const fakeDataSource = {
|
||||
migrations,
|
||||
} as unknown as import('typeorm').DataSource;
|
||||
const registry = new RegisteredCoreMigrationService(fakeDataSource);
|
||||
|
||||
registry.onModuleInit();
|
||||
|
||||
return registry;
|
||||
},
|
||||
}
|
||||
: {
|
||||
provide: RegisteredCoreMigrationService,
|
||||
useValue: {
|
||||
getInstanceCommandsForVersion: jest.fn().mockReturnValue([]),
|
||||
},
|
||||
};
|
||||
|
||||
const module: TestingModule = await Test.createTestingModule({
|
||||
providers: [
|
||||
{
|
||||
@@ -74,6 +104,7 @@ const buildUpgradeCommandModule = async ({
|
||||
coreEngineVersionService: CoreEngineVersionService,
|
||||
workspaceVersionService: WorkspaceVersionService,
|
||||
coreMigrationRunnerService: CoreMigrationRunnerService,
|
||||
versionedMigrationRegistryService: RegisteredCoreMigrationService,
|
||||
workspaceIteratorService: WorkspaceIteratorService,
|
||||
) => {
|
||||
return new commandRunner(
|
||||
@@ -81,6 +112,7 @@ const buildUpgradeCommandModule = async ({
|
||||
coreEngineVersionService,
|
||||
workspaceVersionService,
|
||||
coreMigrationRunnerService,
|
||||
versionedMigrationRegistryService,
|
||||
workspaceIteratorService,
|
||||
);
|
||||
},
|
||||
@@ -89,6 +121,7 @@ const buildUpgradeCommandModule = async ({
|
||||
CoreEngineVersionService,
|
||||
WorkspaceVersionService,
|
||||
CoreMigrationRunnerService,
|
||||
RegisteredCoreMigrationService,
|
||||
WorkspaceIteratorService,
|
||||
],
|
||||
},
|
||||
@@ -124,8 +157,13 @@ const buildUpgradeCommandModule = async ({
|
||||
WorkspaceVersionService,
|
||||
{
|
||||
provide: CoreMigrationRunnerService,
|
||||
useValue: { run: jest.fn().mockResolvedValue(undefined) },
|
||||
useValue: {
|
||||
runSingleMigration: jest
|
||||
.fn()
|
||||
.mockResolvedValue({ status: 'success' }),
|
||||
},
|
||||
},
|
||||
registryProvider,
|
||||
{
|
||||
provide: WorkspaceIteratorService,
|
||||
useValue: {
|
||||
@@ -173,6 +211,7 @@ describe('UpgradeCommandRunner', () => {
|
||||
workspaces?: WorkspaceEntity[];
|
||||
appVersion?: string | null;
|
||||
commandRunner?: CommandRunnerValues;
|
||||
migrations?: MigrationInterface[];
|
||||
};
|
||||
const buildModuleAndSetupSpies = async ({
|
||||
numberOfWorkspace = 1,
|
||||
@@ -180,6 +219,7 @@ describe('UpgradeCommandRunner', () => {
|
||||
workspaces,
|
||||
commandRunner = BasicUpgradeCommandRunner,
|
||||
appVersion = CURRENT_VERSION,
|
||||
migrations,
|
||||
}: BuildModuleAndSetupSpiesArgs) => {
|
||||
const generatedWorkspaces = Array.from(
|
||||
{ length: numberOfWorkspace },
|
||||
@@ -193,6 +233,7 @@ describe('UpgradeCommandRunner', () => {
|
||||
commandRunner,
|
||||
appVersion,
|
||||
workspaces: [...generatedWorkspaces, ...(workspaces ?? [])],
|
||||
migrations,
|
||||
});
|
||||
|
||||
upgradeCommandRunner = module.get(commandRunner);
|
||||
@@ -204,9 +245,11 @@ describe('UpgradeCommandRunner', () => {
|
||||
workspaceRepository = module.get<Repository<WorkspaceEntity>>(
|
||||
getRepositoryToken(WorkspaceEntity),
|
||||
);
|
||||
|
||||
return module;
|
||||
};
|
||||
|
||||
it('should ignore and list as succesfull upgrade on workspace with higher version', async () => {
|
||||
it('should ignore and list as successful upgrade on workspace with higher version', async () => {
|
||||
const higherVersionWorkspace = generateMockWorkspace({
|
||||
id: 'higher_version_workspace',
|
||||
version: '42.42.42',
|
||||
@@ -216,11 +259,9 @@ describe('UpgradeCommandRunner', () => {
|
||||
numberOfWorkspace: 0,
|
||||
workspaces: [higherVersionWorkspace],
|
||||
});
|
||||
// @ts-expect-error legacy noImplicitAny
|
||||
const passedParams = [];
|
||||
const options = {};
|
||||
const passedParams: string[] = [];
|
||||
const options: UpgradeCommandOptions = {};
|
||||
|
||||
// @ts-expect-error legacy noImplicitAny
|
||||
await upgradeCommandRunner.run(passedParams, options);
|
||||
|
||||
[workspaceRepository.update].forEach((fn) =>
|
||||
@@ -234,11 +275,9 @@ describe('UpgradeCommandRunner', () => {
|
||||
await buildModuleAndSetupSpies({
|
||||
numberOfWorkspace,
|
||||
});
|
||||
// @ts-expect-error legacy noImplicitAny
|
||||
const passedParams = [];
|
||||
const options = {};
|
||||
const passedParams: string[] = [];
|
||||
const options: UpgradeCommandOptions = {};
|
||||
|
||||
// @ts-expect-error legacy noImplicitAny
|
||||
await upgradeCommandRunner.run(passedParams, options);
|
||||
|
||||
expect(workspaceRepository.update).toHaveBeenNthCalledWith(
|
||||
@@ -294,11 +333,9 @@ describe('UpgradeCommandRunner', () => {
|
||||
async ({ context: { input } }) => {
|
||||
await buildModuleAndSetupSpies(input);
|
||||
|
||||
// @ts-expect-error legacy noImplicitAny
|
||||
const passedParams = [];
|
||||
const options = {};
|
||||
const passedParams: string[] = [];
|
||||
const options: UpgradeCommandOptions = {};
|
||||
|
||||
// @ts-expect-error legacy noImplicitAny
|
||||
await upgradeCommandRunner.run(passedParams, options);
|
||||
|
||||
expect(workspaceRepository.update).toHaveBeenCalledWith(
|
||||
@@ -309,6 +346,57 @@ describe('UpgradeCommandRunner', () => {
|
||||
);
|
||||
});
|
||||
|
||||
it('should only run instance commands for the current version', async () => {
|
||||
@RegisteredCoreMigration(CURRENT_VERSION)
|
||||
class AddIndexToUsers1770000000000 implements MigrationInterface {
|
||||
async up(_queryRunner: QueryRunner) {}
|
||||
async down(_queryRunner: QueryRunner) {}
|
||||
}
|
||||
|
||||
@RegisteredCoreMigration(CURRENT_VERSION)
|
||||
class AddColumnToAccounts1771000000000 implements MigrationInterface {
|
||||
async up(_queryRunner: QueryRunner) {}
|
||||
async down(_queryRunner: QueryRunner) {}
|
||||
}
|
||||
|
||||
@RegisteredCoreMigration(PREVIOUS_VERSION)
|
||||
class DropLegacyTable1769000000000 implements MigrationInterface {
|
||||
async up(_queryRunner: QueryRunner) {}
|
||||
async down(_queryRunner: QueryRunner) {}
|
||||
}
|
||||
|
||||
class UndecoratedMigration1768000000000 implements MigrationInterface {
|
||||
async up(_queryRunner: QueryRunner) {}
|
||||
async down(_queryRunner: QueryRunner) {}
|
||||
}
|
||||
|
||||
const module = await buildModuleAndSetupSpies({
|
||||
migrations: [
|
||||
new UndecoratedMigration1768000000000(),
|
||||
new DropLegacyTable1769000000000(),
|
||||
new AddIndexToUsers1770000000000(),
|
||||
new AddColumnToAccounts1771000000000(),
|
||||
],
|
||||
});
|
||||
|
||||
const migrationRunnerService = module.get(CoreMigrationRunnerService);
|
||||
|
||||
const passedParams: string[] = [];
|
||||
const options: UpgradeCommandOptions = {};
|
||||
|
||||
await upgradeCommandRunner.run(passedParams, options);
|
||||
|
||||
expect(migrationRunnerService.runSingleMigration).toHaveBeenCalledTimes(2);
|
||||
expect(migrationRunnerService.runSingleMigration).toHaveBeenNthCalledWith(
|
||||
1,
|
||||
'AddIndexToUsers1770000000000',
|
||||
);
|
||||
expect(migrationRunnerService.runSingleMigration).toHaveBeenNthCalledWith(
|
||||
2,
|
||||
'AddColumnToAccounts1771000000000',
|
||||
);
|
||||
});
|
||||
|
||||
describe('Workspace upgrade should fail', () => {
|
||||
const failingTestUseCases: EachTestingContext<{
|
||||
input: Omit<BuildModuleAndSetupSpiesArgs, 'numberOfWorkspace'>;
|
||||
|
||||
+81
-25
@@ -4,17 +4,19 @@ import chalk from 'chalk';
|
||||
import { CommandRunner, Option } from 'nest-commander';
|
||||
import { SemVer } from 'semver';
|
||||
import { assertUnreachable, isDefined } from 'twenty-shared/utils';
|
||||
import { Repository } from 'typeorm';
|
||||
import { MigrationInterface, Repository } from 'typeorm';
|
||||
|
||||
import {
|
||||
type RunOnWorkspaceArgs,
|
||||
WorkspaceCommandRunner,
|
||||
} from 'src/database/commands/command-runners/workspace.command-runner';
|
||||
import { ActiveOrSuspendedWorkspaceCommandRunner } from 'src/database/commands/command-runners/active-or-suspended-workspace.command-runner';
|
||||
import {
|
||||
type WorkspaceIteratorContext,
|
||||
WorkspaceIteratorService,
|
||||
} from 'src/database/commands/command-runners/workspace-iterator.service';
|
||||
import { CoreMigrationRunnerService } from 'src/database/commands/core-migration-runner/services/core-migration-runner.service';
|
||||
import {
|
||||
type RunOnWorkspaceArgs,
|
||||
WorkspaceCommandRunner,
|
||||
} from 'src/database/commands/command-runners/workspace.command-runner';
|
||||
import { CoreMigrationRunnerService } from 'src/database/commands/core-migration/services/core-migration-runner.service';
|
||||
import { RegisteredCoreMigrationService } from 'src/database/commands/core-migration/services/registered-core-migration-registry.service';
|
||||
import { CommandLogger } from 'src/database/commands/logger';
|
||||
import { type UpgradeCommandVersion } from 'src/engine/constants/upgrade-command-supported-versions.constant';
|
||||
import { CoreEngineVersionService } from 'src/engine/core-engine-version/services/core-engine-version.service';
|
||||
@@ -25,7 +27,10 @@ import {
|
||||
compareVersionMajorAndMinor,
|
||||
} from 'src/utils/version/compare-version-minor-and-major';
|
||||
|
||||
export type VersionCommands = WorkspaceCommandRunner[];
|
||||
export type VersionCommands = (
|
||||
| WorkspaceCommandRunner
|
||||
| ActiveOrSuspendedWorkspaceCommandRunner
|
||||
)[];
|
||||
export type AllCommands = Record<UpgradeCommandVersion, VersionCommands>;
|
||||
|
||||
export type UpgradeCommandOptions = {
|
||||
@@ -39,7 +44,9 @@ export type UpgradeCommandOptions = {
|
||||
type VersionContext = {
|
||||
fromWorkspaceVersion: SemVer;
|
||||
currentAppVersion: SemVer;
|
||||
commands: VersionCommands;
|
||||
currentVersionMajorMinor: UpgradeCommandVersion;
|
||||
instanceCommands: MigrationInterface[];
|
||||
workspaceCommands: VersionCommands;
|
||||
};
|
||||
|
||||
export abstract class UpgradeCommandRunner extends CommandRunner {
|
||||
@@ -53,6 +60,7 @@ export abstract class UpgradeCommandRunner extends CommandRunner {
|
||||
protected readonly coreEngineVersionService: CoreEngineVersionService,
|
||||
protected readonly workspaceVersionService: WorkspaceVersionService,
|
||||
protected readonly coreMigrationRunnerService: CoreMigrationRunnerService,
|
||||
protected readonly versionedMigrationRegistryService: RegisteredCoreMigrationService,
|
||||
protected readonly workspaceIteratorService: WorkspaceIteratorService,
|
||||
) {
|
||||
super();
|
||||
@@ -138,6 +146,18 @@ export abstract class UpgradeCommandRunner extends CommandRunner {
|
||||
try {
|
||||
const versionContext = this.resolveVersionContext();
|
||||
|
||||
this.logger.log(
|
||||
chalk.blue(
|
||||
[
|
||||
'Initialized upgrade context with:',
|
||||
`- currentVersion (migrating to): ${versionContext.currentAppVersion}`,
|
||||
`- fromWorkspaceVersion: ${versionContext.fromWorkspaceVersion}`,
|
||||
`- ${versionContext.instanceCommands.length} instance commands (from registry)`,
|
||||
`- ${versionContext.workspaceCommands.length} workspace commands`,
|
||||
].join('\n '),
|
||||
),
|
||||
);
|
||||
|
||||
const hasWorkspaces =
|
||||
await this.workspaceVersionService.hasActiveOrSuspendedWorkspaces();
|
||||
|
||||
@@ -166,7 +186,43 @@ Please roll back to that version and run the upgrade command again.`,
|
||||
);
|
||||
}
|
||||
|
||||
await this.coreMigrationRunnerService.run();
|
||||
for (const instanceCommand of versionContext.instanceCommands) {
|
||||
const migrationName = instanceCommand.constructor.name;
|
||||
const result =
|
||||
await this.coreMigrationRunnerService.runSingleMigration(
|
||||
migrationName,
|
||||
);
|
||||
|
||||
if (result.status === 'fail') {
|
||||
if (result.code === 'already-executed') {
|
||||
this.logger.warn(
|
||||
`Core migration ${migrationName} already executed, skipping`,
|
||||
);
|
||||
|
||||
continue;
|
||||
}
|
||||
|
||||
this.logger.error(
|
||||
`Core migration ${migrationName} failed with code: ${result.code}`,
|
||||
);
|
||||
|
||||
if (isDefined(result.error)) {
|
||||
this.logger.error(
|
||||
result.error instanceof Error
|
||||
? (result.error.stack ?? result.error.message)
|
||||
: String(result.error),
|
||||
);
|
||||
}
|
||||
|
||||
throw new Error(
|
||||
`Core migration ${migrationName} failed: ${result.code}`,
|
||||
);
|
||||
}
|
||||
|
||||
this.logger.log(
|
||||
`Core migration ${migrationName} executed successfully`,
|
||||
);
|
||||
}
|
||||
|
||||
const iteratorReport = await this.workspaceIteratorService.iterate({
|
||||
workspaceIds:
|
||||
@@ -205,9 +261,9 @@ Please roll back to that version and run the upgrade command again.`,
|
||||
const currentAppVersion = this.coreEngineVersionService.getCurrentVersion();
|
||||
const currentVersionMajorMinor =
|
||||
`${currentAppVersion.major}.${currentAppVersion.minor}.0` as UpgradeCommandVersion;
|
||||
const commands = this.allCommands[currentVersionMajorMinor];
|
||||
const workspaceCommands = this.allCommands[currentVersionMajorMinor];
|
||||
|
||||
if (!isDefined(commands)) {
|
||||
if (!isDefined(workspaceCommands)) {
|
||||
throw new Error(
|
||||
`No command found for version ${currentAppVersion}. Please check the commands record.`,
|
||||
);
|
||||
@@ -216,18 +272,18 @@ Please roll back to that version and run the upgrade command again.`,
|
||||
const fromWorkspaceVersion =
|
||||
this.coreEngineVersionService.getPreviousVersion();
|
||||
|
||||
this.logger.log(
|
||||
chalk.blue(
|
||||
[
|
||||
'Initialized upgrade context with:',
|
||||
`- currentVersion (migrating to): ${currentAppVersion}`,
|
||||
`- fromWorkspaceVersion: ${fromWorkspaceVersion}`,
|
||||
`- ${commands.length} commands`,
|
||||
].join('\n '),
|
||||
),
|
||||
);
|
||||
const instanceCommands =
|
||||
this.versionedMigrationRegistryService.getInstanceCommandsForVersion(
|
||||
currentVersionMajorMinor,
|
||||
);
|
||||
|
||||
return { fromWorkspaceVersion, currentAppVersion, commands };
|
||||
return {
|
||||
fromWorkspaceVersion,
|
||||
currentAppVersion,
|
||||
currentVersionMajorMinor,
|
||||
workspaceCommands,
|
||||
instanceCommands,
|
||||
};
|
||||
}
|
||||
|
||||
private async runOnWorkspace(
|
||||
@@ -236,7 +292,7 @@ Please roll back to that version and run the upgrade command again.`,
|
||||
versionContext: VersionContext,
|
||||
): Promise<void> {
|
||||
const { workspaceId, index, total } = iteratorContext;
|
||||
const { fromWorkspaceVersion, currentAppVersion, commands } =
|
||||
const { fromWorkspaceVersion, currentAppVersion, workspaceCommands } =
|
||||
versionContext;
|
||||
|
||||
this.logger.log(
|
||||
@@ -258,8 +314,8 @@ Please roll back to that version and run the upgrade command again.`,
|
||||
);
|
||||
}
|
||||
case 'equal': {
|
||||
for (const command of commands) {
|
||||
await command.runOnWorkspace({
|
||||
for (const workspaceCommand of workspaceCommands) {
|
||||
await workspaceCommand.runOnWorkspace({
|
||||
options: options as RunOnWorkspaceArgs['options'],
|
||||
workspaceId,
|
||||
dataSource: iteratorContext.dataSource,
|
||||
|
||||
Reference in New Issue
Block a user