Guard yarn database:migrate:prod (#19008)
## Motivations A lot of self hosters hands up using the `yarn database:migrated:prod` either manually or through AI assisted debug while they try to upgrade an instance while their workspace is still blocked in a previous one Leading to their whole database permanent corruption ## What happened Replaced the direct call the the typeorm cli to a command calling it programmatically, adding a layer of security in case a workspace seems to be blocked in a previous version than the one just before the one being installed ( e.g 1.0 when you try to upgrade from 1.1 to 1.2 ) For our cloud we still need a way to bypass this security explaining the -f flag ## Remark Centralized this logic and refactored creating new services `WorkspaceVersionService` and `CoreEngineVersionService` that will become useful for the upcoming upgrade refactor Related to https://github.com/twentyhq/twenty-infra/pull/529
This commit is contained in:
Paul Rastoin
-21
@@ -1,21 +0,0 @@
|
||||
// Jest Snapshot v1, https://jestjs.io/docs/snapshot-testing
|
||||
|
||||
exports[`UpgradeCommandRunner Workspace upgrade should fail when APP_VERSION is not defined 1`] = `[Error: Cannot run upgrade command when APP_VERSION is not defined, please double check your env variables]`;
|
||||
|
||||
exports[`UpgradeCommandRunner Workspace upgrade should fail when all commands contains invalid semver keys 1`] = `[Error: No previous version found for version 2.0.0. Please review the "allCommands" record. Available versions are: invalid, 2.0.0]`;
|
||||
|
||||
exports[`UpgradeCommandRunner Workspace upgrade should fail when current version commands are not found 1`] = `[Error: No command found for version 42.0.0. Please check the commands record.]`;
|
||||
|
||||
exports[`UpgradeCommandRunner Workspace upgrade should fail when previous version is not found 1`] = `[Error: No previous version found for version 1.0.0. Please review the "allCommands" record. Available versions are: 1.0.0, 2.0.0]`;
|
||||
|
||||
exports[`UpgradeCommandRunner Workspace upgrade should fail when workspace version is not defined 1`] = `
|
||||
[Error: Unable to run the upgrade command. Aborting the upgrade process.
|
||||
Please ensure that all workspaces are on at least the previous minor version (1.0.0).
|
||||
If any workspaces are not on the previous minor version, roll back to that version and run the upgrade command again.]
|
||||
`;
|
||||
|
||||
exports[`UpgradeCommandRunner Workspace upgrade should fail when workspace version is not equal to fromVersion 1`] = `
|
||||
[Error: Unable to run the upgrade command. Aborting the upgrade process.
|
||||
Please ensure that all workspaces are on at least the previous minor version (1.0.0).
|
||||
If any workspaces are not on the previous minor version, roll back to that version and run the upgrade command again.]
|
||||
`;
|
||||
+62
-43
@@ -7,35 +7,41 @@ import {
|
||||
} from 'twenty-shared/testing';
|
||||
import { type Repository } from 'typeorm';
|
||||
|
||||
import { UpgradeCommandRunner } from 'src/database/commands/command-runners/upgrade.command-runner';
|
||||
import {
|
||||
UpgradeCommandRunner,
|
||||
type AllCommands,
|
||||
} from 'src/database/commands/command-runners/upgrade.command-runner';
|
||||
import { CoreMigrationRunnerService } from 'src/database/commands/core-migration-runner/services/core-migration-runner.service';
|
||||
import { UPGRADE_COMMAND_SUPPORTED_VERSIONS } from 'src/engine/constants/upgrade-command-supported-versions.constant';
|
||||
import { CoreEngineVersionService } from 'src/engine/core-engine-version/services/core-engine-version.service';
|
||||
import { type ConfigVariables } from 'src/engine/core-modules/twenty-config/config-variables';
|
||||
import { TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service';
|
||||
import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity';
|
||||
import { DataSourceService } from 'src/engine/metadata-modules/data-source/data-source.service';
|
||||
import { GlobalWorkspaceOrmManager } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager';
|
||||
import { WorkspaceVersionService } from 'src/engine/workspace-manager/workspace-version/services/workspace-version.service';
|
||||
|
||||
const CURRENT_VERSION =
|
||||
UPGRADE_COMMAND_SUPPORTED_VERSIONS[
|
||||
UPGRADE_COMMAND_SUPPORTED_VERSIONS.length - 1
|
||||
];
|
||||
const PREVIOUS_VERSION =
|
||||
UPGRADE_COMMAND_SUPPORTED_VERSIONS[
|
||||
UPGRADE_COMMAND_SUPPORTED_VERSIONS.length - 2
|
||||
];
|
||||
|
||||
class BasicUpgradeCommandRunner extends UpgradeCommandRunner {
|
||||
allCommands = {
|
||||
'1.0.0': [],
|
||||
'2.0.0': [],
|
||||
};
|
||||
allCommands = Object.fromEntries(
|
||||
UPGRADE_COMMAND_SUPPORTED_VERSIONS.map((version) => [version, []]),
|
||||
) as unknown as AllCommands;
|
||||
}
|
||||
|
||||
class InvalidUpgradeCommandRunner extends UpgradeCommandRunner {
|
||||
allCommands = {
|
||||
invalid: [],
|
||||
'2.0.0': [],
|
||||
};
|
||||
}
|
||||
|
||||
type CommandRunnerValues =
|
||||
| typeof BasicUpgradeCommandRunner
|
||||
| typeof InvalidUpgradeCommandRunner;
|
||||
type CommandRunnerValues = typeof BasicUpgradeCommandRunner;
|
||||
|
||||
const generateMockWorkspace = (overrides?: Partial<WorkspaceEntity>) =>
|
||||
({
|
||||
id: 'workspace-id',
|
||||
version: '1.0.0',
|
||||
version: PREVIOUS_VERSION,
|
||||
createdAt: new Date(),
|
||||
updatedAt: new Date(),
|
||||
allowImpersonation: false,
|
||||
@@ -73,12 +79,18 @@ const buildUpgradeCommandModule = async ({
|
||||
twentyConfigService: TwentyConfigService,
|
||||
globalWorkspaceOrmManager: GlobalWorkspaceOrmManager,
|
||||
dataSourceService: DataSourceService,
|
||||
coreEngineVersionService: CoreEngineVersionService,
|
||||
workspaceVersionService: WorkspaceVersionService,
|
||||
coreMigrationRunnerService: CoreMigrationRunnerService,
|
||||
) => {
|
||||
return new commandRunner(
|
||||
workspaceRepository,
|
||||
twentyConfigService,
|
||||
globalWorkspaceOrmManager,
|
||||
dataSourceService,
|
||||
coreEngineVersionService,
|
||||
workspaceVersionService,
|
||||
coreMigrationRunnerService,
|
||||
);
|
||||
},
|
||||
inject: [
|
||||
@@ -86,6 +98,9 @@ const buildUpgradeCommandModule = async ({
|
||||
TwentyConfigService,
|
||||
GlobalWorkspaceOrmManager,
|
||||
DataSourceService,
|
||||
CoreEngineVersionService,
|
||||
WorkspaceVersionService,
|
||||
CoreMigrationRunnerService,
|
||||
],
|
||||
},
|
||||
{
|
||||
@@ -98,6 +113,7 @@ const buildUpgradeCommandModule = async ({
|
||||
),
|
||||
update: jest.fn(),
|
||||
find: jest.fn().mockResolvedValue(workspaces),
|
||||
exists: jest.fn().mockResolvedValue(workspaces.length > 0),
|
||||
},
|
||||
},
|
||||
{
|
||||
@@ -130,6 +146,12 @@ const buildUpgradeCommandModule = async ({
|
||||
provide: DataSourceService,
|
||||
useValue: mockDataSourceService,
|
||||
},
|
||||
CoreEngineVersionService,
|
||||
WorkspaceVersionService,
|
||||
{
|
||||
provide: CoreMigrationRunnerService,
|
||||
useValue: { run: jest.fn().mockResolvedValue(undefined) },
|
||||
},
|
||||
],
|
||||
}).compile();
|
||||
|
||||
@@ -139,7 +161,7 @@ const buildUpgradeCommandModule = async ({
|
||||
describe('UpgradeCommandRunner', () => {
|
||||
let upgradeCommandRunner: BasicUpgradeCommandRunner;
|
||||
let workspaceRepository: Repository<WorkspaceEntity>;
|
||||
let runCoreMigrationsSpy: jest.SpyInstance;
|
||||
let coreMigrationRunnerService: CoreMigrationRunnerService;
|
||||
|
||||
type BuildModuleAndSetupSpiesArgs = {
|
||||
numberOfWorkspace?: number;
|
||||
@@ -153,7 +175,7 @@ describe('UpgradeCommandRunner', () => {
|
||||
workspaceOverride,
|
||||
workspaces,
|
||||
commandRunner = BasicUpgradeCommandRunner,
|
||||
appVersion = '2.0.0',
|
||||
appVersion = CURRENT_VERSION,
|
||||
}: BuildModuleAndSetupSpiesArgs) => {
|
||||
const generatedWorkspaces = Array.from(
|
||||
{ length: numberOfWorkspace },
|
||||
@@ -176,9 +198,8 @@ describe('UpgradeCommandRunner', () => {
|
||||
jest.spyOn(upgradeCommandRunner['logger'], 'warn').mockImplementation();
|
||||
|
||||
jest.spyOn(upgradeCommandRunner, 'runOnWorkspace');
|
||||
runCoreMigrationsSpy = jest
|
||||
.spyOn(upgradeCommandRunner, 'runCoreMigrations')
|
||||
.mockImplementation(() => Promise.resolve());
|
||||
|
||||
coreMigrationRunnerService = module.get(CoreMigrationRunnerService);
|
||||
|
||||
workspaceRepository = module.get<Repository<WorkspaceEntity>>(
|
||||
getRepositoryToken(WorkspaceEntity),
|
||||
@@ -190,12 +211,10 @@ describe('UpgradeCommandRunner', () => {
|
||||
id: 'higher_version_workspace',
|
||||
version: '42.42.42',
|
||||
});
|
||||
const appVersion = '2.0.0';
|
||||
|
||||
await buildModuleAndSetupSpies({
|
||||
numberOfWorkspace: 0,
|
||||
workspaces: [higherVersionWorkspace],
|
||||
appVersion,
|
||||
});
|
||||
// @ts-expect-error legacy noImplicitAny
|
||||
const passedParams = [];
|
||||
@@ -221,11 +240,9 @@ describe('UpgradeCommandRunner', () => {
|
||||
|
||||
it('should run upgrade over several workspaces', async () => {
|
||||
const numberOfWorkspace = 42;
|
||||
const appVersion = '2.0.0';
|
||||
|
||||
await buildModuleAndSetupSpies({
|
||||
numberOfWorkspace,
|
||||
appVersion,
|
||||
});
|
||||
// @ts-expect-error legacy noImplicitAny
|
||||
const passedParams = [];
|
||||
@@ -240,7 +257,7 @@ describe('UpgradeCommandRunner', () => {
|
||||
expect(workspaceRepository.update).toHaveBeenNthCalledWith(
|
||||
numberOfWorkspace,
|
||||
{ id: expect.any(String) },
|
||||
{ version: appVersion },
|
||||
{ version: CURRENT_VERSION },
|
||||
);
|
||||
expect(upgradeCommandRunner.migrationReport.success.length).toBe(42);
|
||||
expect(upgradeCommandRunner.migrationReport.fail.length).toBe(0);
|
||||
@@ -254,9 +271,9 @@ describe('UpgradeCommandRunner', () => {
|
||||
title: 'even if workspace version and app version differ in patch',
|
||||
context: {
|
||||
input: {
|
||||
appVersion: 'v2.0.0',
|
||||
appVersion: `v${CURRENT_VERSION}`,
|
||||
workspaceOverride: {
|
||||
version: 'v1.0.12',
|
||||
version: `v${PREVIOUS_VERSION.replace('.0', '.12')}`,
|
||||
},
|
||||
},
|
||||
},
|
||||
@@ -266,9 +283,9 @@ describe('UpgradeCommandRunner', () => {
|
||||
'even if workspace version and app version differ in patch and semantic',
|
||||
context: {
|
||||
input: {
|
||||
appVersion: 'v2.0.0',
|
||||
appVersion: `v${CURRENT_VERSION}`,
|
||||
workspaceOverride: {
|
||||
version: '1.0.12',
|
||||
version: PREVIOUS_VERSION.replace('.0', '.12'),
|
||||
},
|
||||
},
|
||||
},
|
||||
@@ -277,9 +294,9 @@ describe('UpgradeCommandRunner', () => {
|
||||
title: 'even if app version contains a patch value',
|
||||
context: {
|
||||
input: {
|
||||
appVersion: '2.0.24',
|
||||
appVersion: CURRENT_VERSION.replace('.0', '.24'),
|
||||
workspaceOverride: {
|
||||
version: '1.0.12',
|
||||
version: PREVIOUS_VERSION.replace('.0', '.12'),
|
||||
},
|
||||
},
|
||||
},
|
||||
@@ -303,7 +320,7 @@ describe('UpgradeCommandRunner', () => {
|
||||
|
||||
expect(failReport.length).toBe(0);
|
||||
expect(successReport.length).toBe(1);
|
||||
expect(runCoreMigrationsSpy).toHaveBeenCalledTimes(1);
|
||||
expect(coreMigrationRunnerService.run).toHaveBeenCalledTimes(1);
|
||||
const { workspaceId } = successReport[0];
|
||||
|
||||
expect(workspaceId).toBe('workspace_0');
|
||||
@@ -316,19 +333,20 @@ describe('UpgradeCommandRunner', () => {
|
||||
input: Omit<BuildModuleAndSetupSpiesArgs, 'numberOfWorkspace'>;
|
||||
output?: {
|
||||
failReportWorkspaceId: string;
|
||||
expectedErrorMessage: string;
|
||||
};
|
||||
}>[] = [
|
||||
{
|
||||
title: 'when workspace version is not equal to fromVersion',
|
||||
context: {
|
||||
input: {
|
||||
appVersion: '2.0.0',
|
||||
workspaceOverride: {
|
||||
version: '0.1.0',
|
||||
},
|
||||
},
|
||||
output: {
|
||||
failReportWorkspaceId: 'workspace_0',
|
||||
expectedErrorMessage: `Unable to run the upgrade command. Aborting the upgrade process.\nPlease ensure that all workspaces are on at least the previous minor version (${PREVIOUS_VERSION}).\nIf any workspaces are not on the previous minor version, roll back to that version and run the upgrade command again.`,
|
||||
},
|
||||
},
|
||||
},
|
||||
@@ -342,6 +360,7 @@ describe('UpgradeCommandRunner', () => {
|
||||
},
|
||||
output: {
|
||||
failReportWorkspaceId: 'workspace_0',
|
||||
expectedErrorMessage: `Unable to run the upgrade command. Aborting the upgrade process.\nPlease ensure that all workspaces are on at least the previous minor version (${PREVIOUS_VERSION}).\nIf any workspaces are not on the previous minor version, roll back to that version and run the upgrade command again.`,
|
||||
},
|
||||
},
|
||||
},
|
||||
@@ -353,6 +372,8 @@ describe('UpgradeCommandRunner', () => {
|
||||
},
|
||||
output: {
|
||||
failReportWorkspaceId: 'global',
|
||||
expectedErrorMessage:
|
||||
'APP_VERSION is not defined, please double check your env variables',
|
||||
},
|
||||
},
|
||||
},
|
||||
@@ -364,6 +385,8 @@ describe('UpgradeCommandRunner', () => {
|
||||
},
|
||||
output: {
|
||||
failReportWorkspaceId: 'global',
|
||||
expectedErrorMessage:
|
||||
'No command found for version 42.0.0. Please check the commands record.',
|
||||
},
|
||||
},
|
||||
},
|
||||
@@ -371,15 +394,11 @@ describe('UpgradeCommandRunner', () => {
|
||||
title: 'when previous version is not found',
|
||||
context: {
|
||||
input: {
|
||||
appVersion: '1.0.0',
|
||||
appVersion: UPGRADE_COMMAND_SUPPORTED_VERSIONS[0],
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
title: 'when all commands contains invalid semver keys',
|
||||
context: {
|
||||
input: {
|
||||
commandRunner: InvalidUpgradeCommandRunner,
|
||||
output: {
|
||||
failReportWorkspaceId: 'global',
|
||||
expectedErrorMessage: `No previous version found for version ${UPGRADE_COMMAND_SUPPORTED_VERSIONS[0]}. Available versions: ${UPGRADE_COMMAND_SUPPORTED_VERSIONS.join(', ')}`,
|
||||
},
|
||||
},
|
||||
},
|
||||
@@ -403,7 +422,7 @@ describe('UpgradeCommandRunner', () => {
|
||||
const { workspaceId, error } = failReport[0];
|
||||
|
||||
expect(workspaceId).toBe(output?.failReportWorkspaceId ?? 'global');
|
||||
expect(error).toMatchSnapshot();
|
||||
expect(error).toEqual(new Error(output?.expectedErrorMessage ?? ''));
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
+21
-129
@@ -1,13 +1,9 @@
|
||||
import { InjectRepository } from '@nestjs/typeorm';
|
||||
|
||||
import { exec } from 'child_process';
|
||||
import { promisify } from 'util';
|
||||
|
||||
import chalk from 'chalk';
|
||||
import { SemVer } from 'semver';
|
||||
import { isDefined } from 'twenty-shared/utils';
|
||||
import { WorkspaceActivationStatus } from 'twenty-shared/workspace';
|
||||
import { In, Repository } from 'typeorm';
|
||||
import { Repository } from 'typeorm';
|
||||
|
||||
import {
|
||||
ActiveOrSuspendedWorkspacesMigrationCommandOptions,
|
||||
@@ -17,22 +13,24 @@ import {
|
||||
RunOnWorkspaceArgs,
|
||||
WorkspacesMigrationCommandRunner,
|
||||
} from 'src/database/commands/command-runners/workspaces-migration.command-runner';
|
||||
import { CoreMigrationRunnerService } from 'src/database/commands/core-migration-runner/services/core-migration-runner.service';
|
||||
import { type UpgradeCommandVersion } from 'src/engine/constants/upgrade-command-supported-versions.constant';
|
||||
import { CoreEngineVersionService } from 'src/engine/core-engine-version/services/core-engine-version.service';
|
||||
import { TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service';
|
||||
import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity';
|
||||
import { type DataSourceService } from 'src/engine/metadata-modules/data-source/data-source.service';
|
||||
import { GlobalWorkspaceOrmManager } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager';
|
||||
import { WorkspaceVersionService } from 'src/engine/workspace-manager/workspace-version/services/workspace-version.service';
|
||||
import {
|
||||
type CompareVersionMajorAndMinorReturnType,
|
||||
compareVersionMajorAndMinor,
|
||||
} from 'src/utils/version/compare-version-minor-and-major';
|
||||
import { getPreviousVersion } from 'src/utils/version/get-previous-version';
|
||||
|
||||
export type VersionCommands = (
|
||||
| WorkspacesMigrationCommandRunner
|
||||
| ActiveOrSuspendedWorkspacesMigrationCommandRunner
|
||||
)[];
|
||||
export type AllCommands = Record<string, VersionCommands>;
|
||||
const execPromise = promisify(exec);
|
||||
export type AllCommands = Record<UpgradeCommandVersion, VersionCommands>;
|
||||
|
||||
export abstract class UpgradeCommandRunner extends ActiveOrSuspendedWorkspacesMigrationCommandRunner {
|
||||
private fromWorkspaceVersion: SemVer;
|
||||
@@ -47,95 +45,13 @@ export abstract class UpgradeCommandRunner extends ActiveOrSuspendedWorkspacesMi
|
||||
protected readonly twentyConfigService: TwentyConfigService,
|
||||
protected readonly globalWorkspaceOrmManager: GlobalWorkspaceOrmManager,
|
||||
protected readonly dataSourceService: DataSourceService,
|
||||
protected readonly coreEngineVersionService: CoreEngineVersionService,
|
||||
protected readonly workspaceVersionService: WorkspaceVersionService,
|
||||
protected readonly coreMigrationRunnerService: CoreMigrationRunnerService,
|
||||
) {
|
||||
super(workspaceRepository, globalWorkspaceOrmManager, dataSourceService);
|
||||
}
|
||||
|
||||
private async loadActiveOrSuspendedWorkspace() {
|
||||
return await this.workspaceRepository.find({
|
||||
select: ['id', 'version', 'displayName'],
|
||||
where: {
|
||||
activationStatus: In([
|
||||
WorkspaceActivationStatus.ACTIVE,
|
||||
WorkspaceActivationStatus.SUSPENDED,
|
||||
]),
|
||||
},
|
||||
order: {
|
||||
id: 'ASC',
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
private async shouldSkipUpgradeIfFreshInstallation(): Promise<boolean> {
|
||||
const activeWorkspaceOrSuspendedWorkspaceCount =
|
||||
await this.loadActiveOrSuspendedWorkspace();
|
||||
|
||||
return activeWorkspaceOrSuspendedWorkspaceCount.length === 0;
|
||||
}
|
||||
|
||||
async runCoreMigrations(): Promise<void> {
|
||||
this.logger.log('Running global database migrations');
|
||||
|
||||
try {
|
||||
this.logger.log('Running core datasource migrations...');
|
||||
const coreResult = await execPromise(
|
||||
'npx -y typeorm migration:run -d dist/database/typeorm/core/core.datasource',
|
||||
);
|
||||
|
||||
this.logger.log(coreResult.stdout);
|
||||
|
||||
this.logger.log('Database migrations completed successfully');
|
||||
} catch (error) {
|
||||
this.logger.error('Error running database migrations:', error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
private async workspacesThatAreBelowFromWorkspaceVersion(
|
||||
fromWorkspaceVersion: SemVer,
|
||||
): Promise<Pick<WorkspaceEntity, 'id' | 'displayName' | 'version'>[]> {
|
||||
try {
|
||||
const allActiveOrSuspendedWorkspaces =
|
||||
await this.loadActiveOrSuspendedWorkspace();
|
||||
|
||||
if (allActiveOrSuspendedWorkspaces.length === 0) {
|
||||
this.logger.log(
|
||||
'No workspaces found. Running migrations for fresh installation.',
|
||||
);
|
||||
|
||||
return [];
|
||||
}
|
||||
|
||||
const workspacesThatAreBelowFromWorkspaceVersion =
|
||||
allActiveOrSuspendedWorkspaces.filter((workspace) => {
|
||||
if (!isDefined(workspace.version)) {
|
||||
return true;
|
||||
}
|
||||
|
||||
try {
|
||||
const versionCompareResult = compareVersionMajorAndMinor(
|
||||
workspace.version,
|
||||
fromWorkspaceVersion.version,
|
||||
);
|
||||
|
||||
return versionCompareResult === 'lower';
|
||||
} catch (error) {
|
||||
this.logger.error(
|
||||
`Error checking workspace ${workspace.id} version: ${error.message}`,
|
||||
);
|
||||
|
||||
return true;
|
||||
}
|
||||
});
|
||||
|
||||
return workspacesThatAreBelowFromWorkspaceVersion;
|
||||
} catch (error) {
|
||||
this.logger.error('Error checking workspaces below version:', error);
|
||||
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
private setUpgradeContextVersionsAndCommandsForCurrentAppVersion() {
|
||||
const upgradeContextIsAlreadyDefined = [
|
||||
this.currentAppVersion,
|
||||
@@ -147,8 +63,9 @@ export abstract class UpgradeCommandRunner extends ActiveOrSuspendedWorkspacesMi
|
||||
return;
|
||||
}
|
||||
|
||||
const currentAppVersion = this.retrieveCurrentAppVersion();
|
||||
const currentVersionMajorMinor = `${currentAppVersion.major}.${currentAppVersion.minor}.0`;
|
||||
const currentAppVersion = this.coreEngineVersionService.getCurrentVersion();
|
||||
const currentVersionMajorMinor =
|
||||
`${currentAppVersion.major}.${currentAppVersion.minor}.0` as UpgradeCommandVersion;
|
||||
const currentCommands = this.allCommands[currentVersionMajorMinor];
|
||||
|
||||
if (!isDefined(currentCommands)) {
|
||||
@@ -157,17 +74,8 @@ export abstract class UpgradeCommandRunner extends ActiveOrSuspendedWorkspacesMi
|
||||
);
|
||||
}
|
||||
|
||||
const allCommandsVersions = Object.keys(this.allCommands);
|
||||
const previousVersion = getPreviousVersion({
|
||||
currentVersion: currentVersionMajorMinor,
|
||||
versions: allCommandsVersions,
|
||||
});
|
||||
const previousVersion = this.coreEngineVersionService.getPreviousVersion();
|
||||
|
||||
if (!isDefined(previousVersion)) {
|
||||
throw new Error(
|
||||
`No previous version found for version ${currentAppVersion}. Please review the "allCommands" record. Available versions are: ${allCommandsVersions.join(', ')}`,
|
||||
);
|
||||
}
|
||||
this.commands = currentCommands;
|
||||
this.fromWorkspaceVersion = previousVersion;
|
||||
this.currentAppVersion = currentAppVersion;
|
||||
@@ -189,10 +97,12 @@ export abstract class UpgradeCommandRunner extends ActiveOrSuspendedWorkspacesMi
|
||||
try {
|
||||
this.setUpgradeContextVersionsAndCommandsForCurrentAppVersion();
|
||||
|
||||
const shouldSkipUpgradeIfFreshInstallation =
|
||||
await this.shouldSkipUpgradeIfFreshInstallation();
|
||||
// On fresh installs there are no workspaces yet, so skip the
|
||||
// per-workspace upgrade loop (core migrations already ran above).
|
||||
const hasWorkspaces =
|
||||
await this.workspaceVersionService.hasActiveOrSuspendedWorkspaces();
|
||||
|
||||
if (shouldSkipUpgradeIfFreshInstallation) {
|
||||
if (!hasWorkspaces) {
|
||||
this.logger.log(
|
||||
chalk.blue('Fresh installation detected, skipping migration'),
|
||||
);
|
||||
@@ -201,8 +111,8 @@ export abstract class UpgradeCommandRunner extends ActiveOrSuspendedWorkspacesMi
|
||||
}
|
||||
|
||||
const workspacesThatAreBelowFromWorkspaceVersion =
|
||||
await this.workspacesThatAreBelowFromWorkspaceVersion(
|
||||
this.fromWorkspaceVersion,
|
||||
await this.workspaceVersionService.getWorkspacesBelowVersion(
|
||||
this.fromWorkspaceVersion.version,
|
||||
);
|
||||
|
||||
if (workspacesThatAreBelowFromWorkspaceVersion.length > 0) {
|
||||
@@ -234,7 +144,7 @@ If any workspaces are not on the previous minor version, roll back to that versi
|
||||
return;
|
||||
}
|
||||
|
||||
await this.runCoreMigrations();
|
||||
await this.coreMigrationRunnerService.run();
|
||||
await super.runMigrationCommand(passedParams, options);
|
||||
}
|
||||
|
||||
@@ -295,24 +205,6 @@ If any workspaces are not on the previous minor version, roll back to that versi
|
||||
}
|
||||
}
|
||||
|
||||
private retrieveCurrentAppVersion() {
|
||||
const appVersion = this.twentyConfigService.get('APP_VERSION');
|
||||
|
||||
if (!isDefined(appVersion)) {
|
||||
throw new Error(
|
||||
'Cannot run upgrade command when APP_VERSION is not defined, please double check your env variables',
|
||||
);
|
||||
}
|
||||
|
||||
try {
|
||||
return new SemVer(appVersion);
|
||||
} catch {
|
||||
throw new Error(
|
||||
`Should never occur, APP_VERSION is invalid ${appVersion}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
private async retrieveWorkspaceVersionAndCompareToWorkspaceFromVersion(
|
||||
workspaceId: string,
|
||||
): Promise<CompareVersionMajorAndMinorReturnType> {
|
||||
|
||||
+9
@@ -0,0 +1,9 @@
|
||||
import { Module } from '@nestjs/common';
|
||||
|
||||
import { CoreMigrationRunnerService } from 'src/database/commands/core-migration-runner/services/core-migration-runner.service';
|
||||
|
||||
@Module({
|
||||
providers: [CoreMigrationRunnerService],
|
||||
exports: [CoreMigrationRunnerService],
|
||||
})
|
||||
export class CoreMigrationRunnerModule {}
|
||||
+37
@@ -0,0 +1,37 @@
|
||||
import { Injectable, Logger } from '@nestjs/common';
|
||||
import { InjectDataSource } from '@nestjs/typeorm';
|
||||
|
||||
import { DataSource } from 'typeorm';
|
||||
|
||||
@Injectable()
|
||||
export class CoreMigrationRunnerService {
|
||||
private readonly logger = new Logger(CoreMigrationRunnerService.name);
|
||||
|
||||
constructor(
|
||||
@InjectDataSource()
|
||||
private readonly dataSource: DataSource,
|
||||
) {}
|
||||
|
||||
async run(): Promise<void> {
|
||||
this.logger.log('Running core datasource migrations...');
|
||||
|
||||
try {
|
||||
const migrations = await this.dataSource.runMigrations({
|
||||
transaction: 'each',
|
||||
});
|
||||
|
||||
if (migrations.length === 0) {
|
||||
this.logger.log('No pending migrations');
|
||||
} else {
|
||||
this.logger.log(
|
||||
`Executed ${migrations.length} migration(s): ${migrations.map((migration) => migration.name).join(', ')}`,
|
||||
);
|
||||
}
|
||||
|
||||
this.logger.log('Database migrations completed successfully');
|
||||
} catch (error) {
|
||||
this.logger.error('Error running database migrations:', error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -5,9 +5,12 @@ import { CronRegisterAllCommand } from 'src/database/commands/cron-register-all.
|
||||
import { DataSeedWorkspaceCommand } from 'src/database/commands/data-seed-dev-workspace.command';
|
||||
import { ListOrphanedWorkspaceEntitiesCommand } from 'src/database/commands/list-and-delete-orphaned-workspace-entities.command';
|
||||
import { ConfirmationQuestion } from 'src/database/commands/questions/confirmation.question';
|
||||
import { WorkspaceExportModule } from 'src/database/commands/workspace-export/workspace-export.module';
|
||||
import { RunTypeormMigrationCommand } from 'src/database/commands/run-typeorm-migration.command';
|
||||
import { UpgradeVersionCommandModule } from 'src/database/commands/upgrade-version-command/upgrade-version-command.module';
|
||||
import { WorkspaceExportModule } from 'src/database/commands/workspace-export/workspace-export.module';
|
||||
import { TypeORMModule } from 'src/database/typeorm/typeorm.module';
|
||||
import { CoreEngineVersionModule } from 'src/engine/core-engine-version/core-engine-version.module';
|
||||
import { CoreMigrationRunnerModule } from 'src/database/commands/core-migration-runner/core-migration-runner.module';
|
||||
import { ApiKeyModule } from 'src/engine/core-modules/api-key/api-key.module';
|
||||
import { GenerateApiKeyCommand } from 'src/engine/core-modules/api-key/commands/generate-api-key.command';
|
||||
import { MarketplaceModule } from 'src/engine/core-modules/application/application-marketplace/marketplace.module';
|
||||
@@ -31,6 +34,7 @@ import { DevSeederModule } from 'src/engine/workspace-manager/dev-seeder/dev-see
|
||||
import { WorkspaceCleanerModule } from 'src/engine/workspace-manager/workspace-cleaner/workspace-cleaner.module';
|
||||
import { WorkspaceManagerModule } from 'src/engine/workspace-manager/workspace-manager.module';
|
||||
import { WorkspaceMigrationModule } from 'src/engine/workspace-manager/workspace-migration/workspace-migration.module';
|
||||
import { WorkspaceVersionModule } from 'src/engine/workspace-manager/workspace-version/workspace-version.module';
|
||||
import { CalendarEventImportManagerModule } from 'src/modules/calendar/calendar-event-import-manager/calendar-event-import-manager.module';
|
||||
import { MessagingImportManagerModule } from 'src/modules/messaging/message-import-manager/messaging-import-manager.module';
|
||||
import { WorkflowRunQueueModule } from 'src/modules/workflow/workflow-runner/workflow-run-queue/workflow-run-queue.module';
|
||||
@@ -68,6 +72,9 @@ import { AutomatedTriggerModule } from 'src/modules/workflow/workflow-trigger/au
|
||||
MarketplaceModule,
|
||||
ApplicationUpgradeModule,
|
||||
StaleRegistrationCleanupModule,
|
||||
CoreEngineVersionModule,
|
||||
CoreMigrationRunnerModule,
|
||||
WorkspaceVersionModule,
|
||||
],
|
||||
providers: [
|
||||
DataSeedWorkspaceCommand,
|
||||
@@ -76,6 +83,7 @@ import { AutomatedTriggerModule } from 'src/modules/workflow/workflow-trigger/au
|
||||
ListOrphanedWorkspaceEntitiesCommand,
|
||||
EnterpriseKeyValidationCronCommand,
|
||||
GenerateApiKeyCommand,
|
||||
RunTypeormMigrationCommand,
|
||||
],
|
||||
})
|
||||
export class DatabaseCommandModule {}
|
||||
|
||||
@@ -0,0 +1,75 @@
|
||||
import { Logger } from '@nestjs/common';
|
||||
|
||||
import chalk from 'chalk';
|
||||
import { Command, CommandRunner, Option } from 'nest-commander';
|
||||
|
||||
import { CoreMigrationRunnerService } from 'src/database/commands/core-migration-runner/services/core-migration-runner.service';
|
||||
import { CoreEngineVersionService } from 'src/engine/core-engine-version/services/core-engine-version.service';
|
||||
import { WorkspaceVersionService } from 'src/engine/workspace-manager/workspace-version/services/workspace-version.service';
|
||||
|
||||
type RunTypeormMigrationCommandOptions = {
|
||||
force?: boolean;
|
||||
};
|
||||
|
||||
@Command({
|
||||
name: 'run-typeorm-migration',
|
||||
description:
|
||||
'Run TypeORM core migrations with workspace version safety check',
|
||||
})
|
||||
export class RunTypeormMigrationCommand extends CommandRunner {
|
||||
private readonly logger = new Logger(RunTypeormMigrationCommand.name);
|
||||
|
||||
constructor(
|
||||
private readonly coreEngineVersionService: CoreEngineVersionService,
|
||||
private readonly workspaceVersionService: WorkspaceVersionService,
|
||||
private readonly coreMigrationRunnerService: CoreMigrationRunnerService,
|
||||
) {
|
||||
super();
|
||||
}
|
||||
|
||||
@Option({
|
||||
flags: '-f, --force',
|
||||
description: 'Skip workspace version safety check',
|
||||
required: false,
|
||||
})
|
||||
parseForce(): boolean {
|
||||
return true;
|
||||
}
|
||||
|
||||
async run(
|
||||
_passedParams: string[],
|
||||
options: RunTypeormMigrationCommandOptions,
|
||||
): Promise<void> {
|
||||
if (options.force) {
|
||||
this.logger.warn(
|
||||
chalk.yellow('Skipping workspace version check (--force flag used)'),
|
||||
);
|
||||
} else {
|
||||
const previousVersion =
|
||||
this.coreEngineVersionService.getPreviousVersion();
|
||||
|
||||
const workspacesBelow =
|
||||
await this.workspaceVersionService.getWorkspacesBelowVersion(
|
||||
previousVersion.version,
|
||||
);
|
||||
|
||||
if (workspacesBelow.length > 0) {
|
||||
for (const workspace of workspacesBelow) {
|
||||
this.logger.error(
|
||||
chalk.red(
|
||||
`Workspace ${workspace.id} (${workspace.displayName}) is at version ${workspace.version ?? 'undefined'}, which is below the minimum required version.`,
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
throw new Error(
|
||||
'Unable to run TypeORM migrations. Some workspace(s) are below the minimum required version.\n' +
|
||||
'Please ensure all workspaces are on at least the previous minor version before running migrations.\n' +
|
||||
'Use --force to bypass this check (not recommended).',
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
await this.coreMigrationRunnerService.run();
|
||||
}
|
||||
}
|
||||
+6
@@ -6,8 +6,11 @@ import { V1_18_UpgradeVersionCommandModule } from 'src/database/commands/upgrade
|
||||
import { V1_19_UpgradeVersionCommandModule } from 'src/database/commands/upgrade-version-command/1-19/1-19-upgrade-version-command.module';
|
||||
import { V1_20_UpgradeVersionCommandModule } from 'src/database/commands/upgrade-version-command/1-20/1-20-upgrade-version-command.module';
|
||||
import { UpgradeCommand } from 'src/database/commands/upgrade-version-command/upgrade.command';
|
||||
import { CoreEngineVersionModule } from 'src/engine/core-engine-version/core-engine-version.module';
|
||||
import { CoreMigrationRunnerModule } from 'src/database/commands/core-migration-runner/core-migration-runner.module';
|
||||
import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity';
|
||||
import { DataSourceModule } from 'src/engine/metadata-modules/data-source/data-source.module';
|
||||
import { WorkspaceVersionModule } from 'src/engine/workspace-manager/workspace-version/workspace-version.module';
|
||||
|
||||
@Module({
|
||||
imports: [
|
||||
@@ -17,6 +20,9 @@ import { DataSourceModule } from 'src/engine/metadata-modules/data-source/data-s
|
||||
V1_19_UpgradeVersionCommandModule,
|
||||
V1_20_UpgradeVersionCommandModule,
|
||||
DataSourceModule,
|
||||
CoreEngineVersionModule,
|
||||
CoreMigrationRunnerModule,
|
||||
WorkspaceVersionModule,
|
||||
],
|
||||
providers: [UpgradeCommand],
|
||||
})
|
||||
|
||||
+9
@@ -9,6 +9,7 @@ import {
|
||||
UpgradeCommandRunner,
|
||||
type VersionCommands,
|
||||
} from 'src/database/commands/command-runners/upgrade.command-runner';
|
||||
import { CoreMigrationRunnerService } from 'src/database/commands/core-migration-runner/services/core-migration-runner.service';
|
||||
import { BackfillApplicationPackageFilesCommand } from 'src/database/commands/upgrade-version-command/1-17/1-17-backfill-application-package-files.command';
|
||||
import { DeleteFileRecordsAndUpdateTableCommand } from 'src/database/commands/upgrade-version-command/1-17/1-17-delete-all-files-and-update-table.command';
|
||||
import { FixMorphRelationFieldNamesCommand } from 'src/database/commands/upgrade-version-command/1-17/1-17-fix-morph-relation-field-names.command';
|
||||
@@ -49,10 +50,12 @@ import { MigrateMessagingInfrastructureToMetadataCommand } from 'src/database/co
|
||||
import { MigrateRichTextToTextCommand } from 'src/database/commands/upgrade-version-command/1-20/1-20-migrate-rich-text-to-text.command';
|
||||
import { SeedCliApplicationRegistrationCommand } from 'src/database/commands/upgrade-version-command/1-20/1-20-seed-cli-application-registration.command';
|
||||
import { UpdateStandardIndexViewNamesCommand } from 'src/database/commands/upgrade-version-command/1-20/1-20-update-standard-index-view-names.command';
|
||||
import { CoreEngineVersionService } from 'src/engine/core-engine-version/services/core-engine-version.service';
|
||||
import { TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service';
|
||||
import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity';
|
||||
import { DataSourceService } from 'src/engine/metadata-modules/data-source/data-source.service';
|
||||
import { GlobalWorkspaceOrmManager } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager';
|
||||
import { WorkspaceVersionService } from 'src/engine/workspace-manager/workspace-version/services/workspace-version.service';
|
||||
|
||||
@Command({
|
||||
name: 'upgrade',
|
||||
@@ -67,6 +70,9 @@ export class UpgradeCommand extends UpgradeCommandRunner {
|
||||
protected readonly twentyConfigService: TwentyConfigService,
|
||||
protected readonly globalWorkspaceOrmManager: GlobalWorkspaceOrmManager,
|
||||
protected readonly dataSourceService: DataSourceService,
|
||||
protected readonly coreEngineVersionService: CoreEngineVersionService,
|
||||
protected readonly workspaceVersionService: WorkspaceVersionService,
|
||||
protected readonly coreMigrationRunnerService: CoreMigrationRunnerService,
|
||||
|
||||
// 1.17 Commands
|
||||
protected readonly backfillApplicationPackageFilesCommand: BackfillApplicationPackageFilesCommand,
|
||||
@@ -121,6 +127,9 @@ export class UpgradeCommand extends UpgradeCommandRunner {
|
||||
twentyConfigService,
|
||||
globalWorkspaceOrmManager,
|
||||
dataSourceService,
|
||||
coreEngineVersionService,
|
||||
workspaceVersionService,
|
||||
coreMigrationRunnerService,
|
||||
);
|
||||
|
||||
// Note: Required empty commands array to allow retrieving previous version
|
||||
|
||||
@@ -0,0 +1,34 @@
|
||||
import { rawDataSource } from 'src/database/typeorm/raw/raw.datasource';
|
||||
|
||||
export const camelToSnakeCase = (str: string) =>
|
||||
str.replace(/[A-Z]/g, (letter) => `_${letter.toLowerCase()}`);
|
||||
|
||||
export const performQuery = async <T = unknown>(
|
||||
query: string,
|
||||
consoleDescription: string,
|
||||
withLog = true,
|
||||
ignoreAlreadyExistsError = false,
|
||||
) => {
|
||||
try {
|
||||
const result = await rawDataSource.query<T>(query);
|
||||
|
||||
if (withLog) {
|
||||
// oxlint-disable-next-line no-console
|
||||
console.log(`Performed '${consoleDescription}' successfully`);
|
||||
}
|
||||
|
||||
return result;
|
||||
} catch (err) {
|
||||
let message = '';
|
||||
|
||||
if (ignoreAlreadyExistsError && `${err}`.includes('already exists')) {
|
||||
message = `Performed '${consoleDescription}' successfully`;
|
||||
} else {
|
||||
message = `Failed to perform '${consoleDescription}': ${err}`;
|
||||
}
|
||||
if (withLog) {
|
||||
// oxlint-disable-next-line no-console
|
||||
console.error(message);
|
||||
}
|
||||
}
|
||||
};
|
||||
@@ -0,0 +1,98 @@
|
||||
import { rawDataSource } from 'src/database/typeorm/raw/raw.datasource';
|
||||
|
||||
import { camelToSnakeCase, performQuery } from './setup-db-utils';
|
||||
|
||||
rawDataSource
|
||||
.initialize()
|
||||
.then(async () => {
|
||||
await performQuery(
|
||||
'CREATE SCHEMA IF NOT EXISTS "public"',
|
||||
'create schema "public"',
|
||||
);
|
||||
await performQuery(
|
||||
'CREATE SCHEMA IF NOT EXISTS "core"',
|
||||
'create schema "core"',
|
||||
);
|
||||
|
||||
await performQuery(
|
||||
'CREATE EXTENSION IF NOT EXISTS "uuid-ossp"',
|
||||
'create extension "uuid-ossp"',
|
||||
);
|
||||
|
||||
await performQuery(
|
||||
'CREATE EXTENSION IF NOT EXISTS "unaccent"',
|
||||
'create extension "unaccent"',
|
||||
);
|
||||
|
||||
await performQuery(
|
||||
`CREATE OR REPLACE FUNCTION public.unaccent_immutable(input text)
|
||||
RETURNS text
|
||||
LANGUAGE sql
|
||||
IMMUTABLE
|
||||
AS $$
|
||||
SELECT public.unaccent('public.unaccent'::regdictionary, input)
|
||||
$$;`,
|
||||
'create immutable unaccent wrapper function',
|
||||
);
|
||||
|
||||
// We paused the work on FDW
|
||||
if (process.env.IS_FDW_ENABLED !== 'true') {
|
||||
return;
|
||||
}
|
||||
|
||||
await performQuery(
|
||||
'CREATE EXTENSION IF NOT EXISTS "postgres_fdw"',
|
||||
'create extension "postgres_fdw"',
|
||||
);
|
||||
|
||||
await performQuery(
|
||||
'CREATE EXTENSION IF NOT EXISTS "wrappers"',
|
||||
'create extension "wrappers"',
|
||||
);
|
||||
|
||||
await performQuery(
|
||||
'CREATE EXTENSION IF NOT EXISTS "mysql_fdw"',
|
||||
'create extension "mysql_fdw"',
|
||||
);
|
||||
|
||||
const supabaseWrappers = [
|
||||
'airtable',
|
||||
'bigQuery',
|
||||
'clickHouse',
|
||||
'firebase',
|
||||
'logflare',
|
||||
's3',
|
||||
'stripe',
|
||||
]; // See https://supabase.github.io/wrappers/
|
||||
|
||||
for (const wrapper of supabaseWrappers) {
|
||||
if (await checkForeignDataWrapperExists(`${wrapper.toLowerCase()}_fdw`)) {
|
||||
continue;
|
||||
}
|
||||
await performQuery(
|
||||
`
|
||||
CREATE FOREIGN DATA WRAPPER "${wrapper.toLowerCase()}_fdw"
|
||||
HANDLER "${camelToSnakeCase(wrapper)}_fdw_handler"
|
||||
VALIDATOR "${camelToSnakeCase(wrapper)}_fdw_validator";
|
||||
`,
|
||||
`create ${wrapper} "wrappers"`,
|
||||
true,
|
||||
true,
|
||||
);
|
||||
}
|
||||
})
|
||||
.catch((err) => {
|
||||
// oxlint-disable-next-line no-console
|
||||
console.error('Error during Data Source initialization:', err);
|
||||
});
|
||||
|
||||
async function checkForeignDataWrapperExists(
|
||||
wrapperName: string,
|
||||
): Promise<boolean> {
|
||||
const result = await rawDataSource.query(
|
||||
`SELECT 1 FROM pg_foreign_data_wrapper WHERE fdwname = $1`,
|
||||
[wrapperName],
|
||||
);
|
||||
|
||||
return result.length > 0;
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
import { rawDataSource } from 'src/database/typeorm/raw/raw.datasource';
|
||||
|
||||
import { performQuery } from './setup-db-utils';
|
||||
|
||||
async function dropSchemasSequentially() {
|
||||
try {
|
||||
await rawDataSource.initialize();
|
||||
|
||||
// Fetch all schemas excluding the ones we want to keep
|
||||
const schemas =
|
||||
(await performQuery<{ schema_name: string }[]>(
|
||||
`
|
||||
SELECT n.nspname AS "schema_name"
|
||||
FROM pg_catalog.pg_namespace n
|
||||
WHERE n.nspname !~ '^pg_'
|
||||
AND n.nspname <> 'information_schema'
|
||||
AND n.nspname NOT IN ('metric_helpers', 'user_management', 'public')
|
||||
`,
|
||||
'Fetching schemas...',
|
||||
)) ?? [];
|
||||
|
||||
const batchSize = 10;
|
||||
|
||||
for (let i = 0; i < schemas.length; i += batchSize) {
|
||||
const batch = schemas.slice(i, i + batchSize);
|
||||
|
||||
await Promise.all(
|
||||
batch.map((schema) =>
|
||||
performQuery(
|
||||
`DROP SCHEMA IF EXISTS "${schema.schema_name}" CASCADE;`,
|
||||
`Dropping schema ${schema.schema_name}...`,
|
||||
),
|
||||
),
|
||||
);
|
||||
}
|
||||
// oxlint-disable-next-line no-console
|
||||
console.log('All schemas dropped successfully.');
|
||||
} catch (err) {
|
||||
// oxlint-disable-next-line no-console
|
||||
console.error('Error during schema dropping:', err);
|
||||
}
|
||||
}
|
||||
|
||||
dropSchemasSequentially();
|
||||
Reference in New Issue
Block a user