Guard yarn database:migrate:prod (#19008)

## Motivations
A lot of self hosters hands up using the `yarn database:migrated:prod`
either manually or through AI assisted debug while they try to upgrade
an instance while their workspace is still blocked in a previous one
Leading to their whole database permanent corruption

## What happened
Replaced the direct call the the typeorm cli to a command calling it
programmatically, adding a layer of security in case a workspace seems
to be blocked in a previous version than the one just before the one
being installed ( e.g 1.0 when you try to upgrade from 1.1 to 1.2 )

For our cloud we still need a way to bypass this security explaining the
-f flag

## Remark
Centralized this logic and refactored creating new services
`WorkspaceVersionService` and `CoreEngineVersionService` that will
become useful for the upcoming upgrade refactor

Related to https://github.com/twentyhq/twenty-infra/pull/529
This commit is contained in:
Paul Rastoin
2026-03-27 15:39:18 +01:00
committed by GitHub
parent c96c034908
commit 281bb6d783
25 changed files with 401 additions and 216 deletions
@@ -143,7 +143,6 @@ jobs:
set_env_var "CLICKHOUSE_PASSWORD" "clickhousePassword"
npx nx run twenty-server:database:init:prod
npx nx run twenty-server:database:migrate:prod
- name: Seed current branch database with test data
run: |
@@ -296,7 +295,6 @@ jobs:
set_env_var "CLICKHOUSE_PASSWORD" "clickhousePassword"
npx nx run twenty-server:database:init:prod
npx nx run twenty-server:database:migrate:prod
- name: Seed main branch database with test data
run: |
-1
View File
@@ -122,7 +122,6 @@ jobs:
PGPASSWORD=postgres psql -h localhost -p 5432 -U postgres -d postgres -c 'CREATE DATABASE "default";'
PGPASSWORD=postgres psql -h localhost -p 5432 -U postgres -d postgres -c 'CREATE DATABASE "test";'
npx nx run twenty-server:database:init:prod
npx nx run twenty-server:database:migrate:prod
- name: Worker / Run
run: |
timeout 30s npx nx run twenty-server:worker || exit_code=$?
@@ -32,11 +32,11 @@ has_schema=$(PGPASSWORD=twenty psql -h localhost -U twenty -d default -tAc \
if [ "$has_schema" = "f" ]; then
echo "Database appears to be empty, running initial setup..."
NODE_OPTIONS="--max-old-space-size=1500" node ./dist/scripts/setup-db.js
NODE_OPTIONS="--max-old-space-size=1500" node ./dist/database/scripts/setup-db.js
fi
# Always run migrations (idempotent — skips already-applied ones)
yarn database:migrate:prod
yarn database:migrate:prod --force
yarn command:prod cache:flush
yarn command:prod upgrade
-6
View File
@@ -38,12 +38,6 @@ RUN npx nx run twenty-server:lingui:extract && \
RUN npx nx run twenty-server:build
# Bundle setup-db script into a standalone JS file so the final image
# doesn't need tsx or the TypeScript source tree at runtime.
RUN npx esbuild packages/twenty-server/scripts/setup-db.ts \
--bundle --platform=node --outfile=packages/twenty-server/dist/scripts/setup-db.js \
--external:typeorm --external:dotenv --external:pg
# Clean server build output (type declarations and compiled tests are not needed at runtime;
# source maps are kept because twenty-infra extracts them from the image for Sentry uploads)
RUN find /app/packages/twenty-server/dist -name '*.d.ts' -delete \
+1 -2
View File
@@ -13,8 +13,7 @@ setup_and_migrate_db() {
has_schema=$(psql -tAc "SELECT EXISTS (SELECT 1 FROM information_schema.schemata WHERE schema_name = 'core')" ${PG_DATABASE_URL})
if [ "$has_schema" = "f" ]; then
echo "Database appears to be empty, running migrations."
NODE_OPTIONS="--max-old-space-size=1500" node ./dist/scripts/setup-db.js
yarn database:migrate:prod
yarn database:init:prod
fi
yarn command:prod cache:flush
+2 -2
View File
@@ -9,8 +9,8 @@
"start:prod": "node dist/main",
"command:prod": "node dist/command/command",
"worker:prod": "node dist/queue-worker/queue-worker",
"database:init:prod": "npx ts-node ./scripts/setup-db.ts && yarn database:migrate:prod",
"database:migrate:prod": "npx -y typeorm migration:run -d dist/database/typeorm/core/core.datasource",
"database:init:prod": "node dist/database/scripts/setup-db.js && yarn database:migrate:prod --force",
"database:migrate:prod": "node dist/command/command run-typeorm-migration",
"clickhouse:migrate:prod": "node dist/database/clickHouse/migrations/run-migrations.js",
"typeorm": "../../node_modules/typeorm/.bin/typeorm"
},
+4 -4
View File
@@ -246,8 +246,8 @@
"no-seed": {
"cwd": "packages/twenty-server",
"commands": [
"nx ts-node-no-deps-transpile-only -- ./scripts/truncate-db.ts",
"nx ts-node-no-deps-transpile-only -- ./scripts/setup-db.ts",
"node dist/database/scripts/truncate-db.js",
"node dist/database/scripts/setup-db.js",
"nx database:migrate",
"nx command-no-deps -- cache:flush"
],
@@ -256,8 +256,8 @@
"seed": {
"cwd": "packages/twenty-server",
"commands": [
"nx ts-node-no-deps-transpile-only -- ./scripts/truncate-db.ts",
"nx ts-node-no-deps-transpile-only -- ./scripts/setup-db.ts",
"node dist/database/scripts/truncate-db.js",
"node dist/database/scripts/setup-db.js",
"nx database:migrate",
"nx command-no-deps -- cache:flush",
"nx command-no-deps -- workspace:seed:dev"
@@ -1,21 +0,0 @@
// Jest Snapshot v1, https://jestjs.io/docs/snapshot-testing
exports[`UpgradeCommandRunner Workspace upgrade should fail when APP_VERSION is not defined 1`] = `[Error: Cannot run upgrade command when APP_VERSION is not defined, please double check your env variables]`;
exports[`UpgradeCommandRunner Workspace upgrade should fail when all commands contains invalid semver keys 1`] = `[Error: No previous version found for version 2.0.0. Please review the "allCommands" record. Available versions are: invalid, 2.0.0]`;
exports[`UpgradeCommandRunner Workspace upgrade should fail when current version commands are not found 1`] = `[Error: No command found for version 42.0.0. Please check the commands record.]`;
exports[`UpgradeCommandRunner Workspace upgrade should fail when previous version is not found 1`] = `[Error: No previous version found for version 1.0.0. Please review the "allCommands" record. Available versions are: 1.0.0, 2.0.0]`;
exports[`UpgradeCommandRunner Workspace upgrade should fail when workspace version is not defined 1`] = `
[Error: Unable to run the upgrade command. Aborting the upgrade process.
Please ensure that all workspaces are on at least the previous minor version (1.0.0).
If any workspaces are not on the previous minor version, roll back to that version and run the upgrade command again.]
`;
exports[`UpgradeCommandRunner Workspace upgrade should fail when workspace version is not equal to fromVersion 1`] = `
[Error: Unable to run the upgrade command. Aborting the upgrade process.
Please ensure that all workspaces are on at least the previous minor version (1.0.0).
If any workspaces are not on the previous minor version, roll back to that version and run the upgrade command again.]
`;
@@ -7,35 +7,41 @@ import {
} from 'twenty-shared/testing';
import { type Repository } from 'typeorm';
import { UpgradeCommandRunner } from 'src/database/commands/command-runners/upgrade.command-runner';
import {
UpgradeCommandRunner,
type AllCommands,
} from 'src/database/commands/command-runners/upgrade.command-runner';
import { CoreMigrationRunnerService } from 'src/database/commands/core-migration-runner/services/core-migration-runner.service';
import { UPGRADE_COMMAND_SUPPORTED_VERSIONS } from 'src/engine/constants/upgrade-command-supported-versions.constant';
import { CoreEngineVersionService } from 'src/engine/core-engine-version/services/core-engine-version.service';
import { type ConfigVariables } from 'src/engine/core-modules/twenty-config/config-variables';
import { TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service';
import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity';
import { DataSourceService } from 'src/engine/metadata-modules/data-source/data-source.service';
import { GlobalWorkspaceOrmManager } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager';
import { WorkspaceVersionService } from 'src/engine/workspace-manager/workspace-version/services/workspace-version.service';
const CURRENT_VERSION =
UPGRADE_COMMAND_SUPPORTED_VERSIONS[
UPGRADE_COMMAND_SUPPORTED_VERSIONS.length - 1
];
const PREVIOUS_VERSION =
UPGRADE_COMMAND_SUPPORTED_VERSIONS[
UPGRADE_COMMAND_SUPPORTED_VERSIONS.length - 2
];
class BasicUpgradeCommandRunner extends UpgradeCommandRunner {
allCommands = {
'1.0.0': [],
'2.0.0': [],
};
allCommands = Object.fromEntries(
UPGRADE_COMMAND_SUPPORTED_VERSIONS.map((version) => [version, []]),
) as unknown as AllCommands;
}
class InvalidUpgradeCommandRunner extends UpgradeCommandRunner {
allCommands = {
invalid: [],
'2.0.0': [],
};
}
type CommandRunnerValues =
| typeof BasicUpgradeCommandRunner
| typeof InvalidUpgradeCommandRunner;
type CommandRunnerValues = typeof BasicUpgradeCommandRunner;
const generateMockWorkspace = (overrides?: Partial<WorkspaceEntity>) =>
({
id: 'workspace-id',
version: '1.0.0',
version: PREVIOUS_VERSION,
createdAt: new Date(),
updatedAt: new Date(),
allowImpersonation: false,
@@ -73,12 +79,18 @@ const buildUpgradeCommandModule = async ({
twentyConfigService: TwentyConfigService,
globalWorkspaceOrmManager: GlobalWorkspaceOrmManager,
dataSourceService: DataSourceService,
coreEngineVersionService: CoreEngineVersionService,
workspaceVersionService: WorkspaceVersionService,
coreMigrationRunnerService: CoreMigrationRunnerService,
) => {
return new commandRunner(
workspaceRepository,
twentyConfigService,
globalWorkspaceOrmManager,
dataSourceService,
coreEngineVersionService,
workspaceVersionService,
coreMigrationRunnerService,
);
},
inject: [
@@ -86,6 +98,9 @@ const buildUpgradeCommandModule = async ({
TwentyConfigService,
GlobalWorkspaceOrmManager,
DataSourceService,
CoreEngineVersionService,
WorkspaceVersionService,
CoreMigrationRunnerService,
],
},
{
@@ -98,6 +113,7 @@ const buildUpgradeCommandModule = async ({
),
update: jest.fn(),
find: jest.fn().mockResolvedValue(workspaces),
exists: jest.fn().mockResolvedValue(workspaces.length > 0),
},
},
{
@@ -130,6 +146,12 @@ const buildUpgradeCommandModule = async ({
provide: DataSourceService,
useValue: mockDataSourceService,
},
CoreEngineVersionService,
WorkspaceVersionService,
{
provide: CoreMigrationRunnerService,
useValue: { run: jest.fn().mockResolvedValue(undefined) },
},
],
}).compile();
@@ -139,7 +161,7 @@ const buildUpgradeCommandModule = async ({
describe('UpgradeCommandRunner', () => {
let upgradeCommandRunner: BasicUpgradeCommandRunner;
let workspaceRepository: Repository<WorkspaceEntity>;
let runCoreMigrationsSpy: jest.SpyInstance;
let coreMigrationRunnerService: CoreMigrationRunnerService;
type BuildModuleAndSetupSpiesArgs = {
numberOfWorkspace?: number;
@@ -153,7 +175,7 @@ describe('UpgradeCommandRunner', () => {
workspaceOverride,
workspaces,
commandRunner = BasicUpgradeCommandRunner,
appVersion = '2.0.0',
appVersion = CURRENT_VERSION,
}: BuildModuleAndSetupSpiesArgs) => {
const generatedWorkspaces = Array.from(
{ length: numberOfWorkspace },
@@ -176,9 +198,8 @@ describe('UpgradeCommandRunner', () => {
jest.spyOn(upgradeCommandRunner['logger'], 'warn').mockImplementation();
jest.spyOn(upgradeCommandRunner, 'runOnWorkspace');
runCoreMigrationsSpy = jest
.spyOn(upgradeCommandRunner, 'runCoreMigrations')
.mockImplementation(() => Promise.resolve());
coreMigrationRunnerService = module.get(CoreMigrationRunnerService);
workspaceRepository = module.get<Repository<WorkspaceEntity>>(
getRepositoryToken(WorkspaceEntity),
@@ -190,12 +211,10 @@ describe('UpgradeCommandRunner', () => {
id: 'higher_version_workspace',
version: '42.42.42',
});
const appVersion = '2.0.0';
await buildModuleAndSetupSpies({
numberOfWorkspace: 0,
workspaces: [higherVersionWorkspace],
appVersion,
});
// @ts-expect-error legacy noImplicitAny
const passedParams = [];
@@ -221,11 +240,9 @@ describe('UpgradeCommandRunner', () => {
it('should run upgrade over several workspaces', async () => {
const numberOfWorkspace = 42;
const appVersion = '2.0.0';
await buildModuleAndSetupSpies({
numberOfWorkspace,
appVersion,
});
// @ts-expect-error legacy noImplicitAny
const passedParams = [];
@@ -240,7 +257,7 @@ describe('UpgradeCommandRunner', () => {
expect(workspaceRepository.update).toHaveBeenNthCalledWith(
numberOfWorkspace,
{ id: expect.any(String) },
{ version: appVersion },
{ version: CURRENT_VERSION },
);
expect(upgradeCommandRunner.migrationReport.success.length).toBe(42);
expect(upgradeCommandRunner.migrationReport.fail.length).toBe(0);
@@ -254,9 +271,9 @@ describe('UpgradeCommandRunner', () => {
title: 'even if workspace version and app version differ in patch',
context: {
input: {
appVersion: 'v2.0.0',
appVersion: `v${CURRENT_VERSION}`,
workspaceOverride: {
version: 'v1.0.12',
version: `v${PREVIOUS_VERSION.replace('.0', '.12')}`,
},
},
},
@@ -266,9 +283,9 @@ describe('UpgradeCommandRunner', () => {
'even if workspace version and app version differ in patch and semantic',
context: {
input: {
appVersion: 'v2.0.0',
appVersion: `v${CURRENT_VERSION}`,
workspaceOverride: {
version: '1.0.12',
version: PREVIOUS_VERSION.replace('.0', '.12'),
},
},
},
@@ -277,9 +294,9 @@ describe('UpgradeCommandRunner', () => {
title: 'even if app version contains a patch value',
context: {
input: {
appVersion: '2.0.24',
appVersion: CURRENT_VERSION.replace('.0', '.24'),
workspaceOverride: {
version: '1.0.12',
version: PREVIOUS_VERSION.replace('.0', '.12'),
},
},
},
@@ -303,7 +320,7 @@ describe('UpgradeCommandRunner', () => {
expect(failReport.length).toBe(0);
expect(successReport.length).toBe(1);
expect(runCoreMigrationsSpy).toHaveBeenCalledTimes(1);
expect(coreMigrationRunnerService.run).toHaveBeenCalledTimes(1);
const { workspaceId } = successReport[0];
expect(workspaceId).toBe('workspace_0');
@@ -316,19 +333,20 @@ describe('UpgradeCommandRunner', () => {
input: Omit<BuildModuleAndSetupSpiesArgs, 'numberOfWorkspace'>;
output?: {
failReportWorkspaceId: string;
expectedErrorMessage: string;
};
}>[] = [
{
title: 'when workspace version is not equal to fromVersion',
context: {
input: {
appVersion: '2.0.0',
workspaceOverride: {
version: '0.1.0',
},
},
output: {
failReportWorkspaceId: 'workspace_0',
expectedErrorMessage: `Unable to run the upgrade command. Aborting the upgrade process.\nPlease ensure that all workspaces are on at least the previous minor version (${PREVIOUS_VERSION}).\nIf any workspaces are not on the previous minor version, roll back to that version and run the upgrade command again.`,
},
},
},
@@ -342,6 +360,7 @@ describe('UpgradeCommandRunner', () => {
},
output: {
failReportWorkspaceId: 'workspace_0',
expectedErrorMessage: `Unable to run the upgrade command. Aborting the upgrade process.\nPlease ensure that all workspaces are on at least the previous minor version (${PREVIOUS_VERSION}).\nIf any workspaces are not on the previous minor version, roll back to that version and run the upgrade command again.`,
},
},
},
@@ -353,6 +372,8 @@ describe('UpgradeCommandRunner', () => {
},
output: {
failReportWorkspaceId: 'global',
expectedErrorMessage:
'APP_VERSION is not defined, please double check your env variables',
},
},
},
@@ -364,6 +385,8 @@ describe('UpgradeCommandRunner', () => {
},
output: {
failReportWorkspaceId: 'global',
expectedErrorMessage:
'No command found for version 42.0.0. Please check the commands record.',
},
},
},
@@ -371,15 +394,11 @@ describe('UpgradeCommandRunner', () => {
title: 'when previous version is not found',
context: {
input: {
appVersion: '1.0.0',
appVersion: UPGRADE_COMMAND_SUPPORTED_VERSIONS[0],
},
},
},
{
title: 'when all commands contains invalid semver keys',
context: {
input: {
commandRunner: InvalidUpgradeCommandRunner,
output: {
failReportWorkspaceId: 'global',
expectedErrorMessage: `No previous version found for version ${UPGRADE_COMMAND_SUPPORTED_VERSIONS[0]}. Available versions: ${UPGRADE_COMMAND_SUPPORTED_VERSIONS.join(', ')}`,
},
},
},
@@ -403,7 +422,7 @@ describe('UpgradeCommandRunner', () => {
const { workspaceId, error } = failReport[0];
expect(workspaceId).toBe(output?.failReportWorkspaceId ?? 'global');
expect(error).toMatchSnapshot();
expect(error).toEqual(new Error(output?.expectedErrorMessage ?? ''));
},
);
});
@@ -1,13 +1,9 @@
import { InjectRepository } from '@nestjs/typeorm';
import { exec } from 'child_process';
import { promisify } from 'util';
import chalk from 'chalk';
import { SemVer } from 'semver';
import { isDefined } from 'twenty-shared/utils';
import { WorkspaceActivationStatus } from 'twenty-shared/workspace';
import { In, Repository } from 'typeorm';
import { Repository } from 'typeorm';
import {
ActiveOrSuspendedWorkspacesMigrationCommandOptions,
@@ -17,22 +13,24 @@ import {
RunOnWorkspaceArgs,
WorkspacesMigrationCommandRunner,
} from 'src/database/commands/command-runners/workspaces-migration.command-runner';
import { CoreMigrationRunnerService } from 'src/database/commands/core-migration-runner/services/core-migration-runner.service';
import { type UpgradeCommandVersion } from 'src/engine/constants/upgrade-command-supported-versions.constant';
import { CoreEngineVersionService } from 'src/engine/core-engine-version/services/core-engine-version.service';
import { TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service';
import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity';
import { type DataSourceService } from 'src/engine/metadata-modules/data-source/data-source.service';
import { GlobalWorkspaceOrmManager } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager';
import { WorkspaceVersionService } from 'src/engine/workspace-manager/workspace-version/services/workspace-version.service';
import {
type CompareVersionMajorAndMinorReturnType,
compareVersionMajorAndMinor,
} from 'src/utils/version/compare-version-minor-and-major';
import { getPreviousVersion } from 'src/utils/version/get-previous-version';
export type VersionCommands = (
| WorkspacesMigrationCommandRunner
| ActiveOrSuspendedWorkspacesMigrationCommandRunner
)[];
export type AllCommands = Record<string, VersionCommands>;
const execPromise = promisify(exec);
export type AllCommands = Record<UpgradeCommandVersion, VersionCommands>;
export abstract class UpgradeCommandRunner extends ActiveOrSuspendedWorkspacesMigrationCommandRunner {
private fromWorkspaceVersion: SemVer;
@@ -47,95 +45,13 @@ export abstract class UpgradeCommandRunner extends ActiveOrSuspendedWorkspacesMi
protected readonly twentyConfigService: TwentyConfigService,
protected readonly globalWorkspaceOrmManager: GlobalWorkspaceOrmManager,
protected readonly dataSourceService: DataSourceService,
protected readonly coreEngineVersionService: CoreEngineVersionService,
protected readonly workspaceVersionService: WorkspaceVersionService,
protected readonly coreMigrationRunnerService: CoreMigrationRunnerService,
) {
super(workspaceRepository, globalWorkspaceOrmManager, dataSourceService);
}
private async loadActiveOrSuspendedWorkspace() {
return await this.workspaceRepository.find({
select: ['id', 'version', 'displayName'],
where: {
activationStatus: In([
WorkspaceActivationStatus.ACTIVE,
WorkspaceActivationStatus.SUSPENDED,
]),
},
order: {
id: 'ASC',
},
});
}
private async shouldSkipUpgradeIfFreshInstallation(): Promise<boolean> {
const activeWorkspaceOrSuspendedWorkspaceCount =
await this.loadActiveOrSuspendedWorkspace();
return activeWorkspaceOrSuspendedWorkspaceCount.length === 0;
}
async runCoreMigrations(): Promise<void> {
this.logger.log('Running global database migrations');
try {
this.logger.log('Running core datasource migrations...');
const coreResult = await execPromise(
'npx -y typeorm migration:run -d dist/database/typeorm/core/core.datasource',
);
this.logger.log(coreResult.stdout);
this.logger.log('Database migrations completed successfully');
} catch (error) {
this.logger.error('Error running database migrations:', error);
throw error;
}
}
private async workspacesThatAreBelowFromWorkspaceVersion(
fromWorkspaceVersion: SemVer,
): Promise<Pick<WorkspaceEntity, 'id' | 'displayName' | 'version'>[]> {
try {
const allActiveOrSuspendedWorkspaces =
await this.loadActiveOrSuspendedWorkspace();
if (allActiveOrSuspendedWorkspaces.length === 0) {
this.logger.log(
'No workspaces found. Running migrations for fresh installation.',
);
return [];
}
const workspacesThatAreBelowFromWorkspaceVersion =
allActiveOrSuspendedWorkspaces.filter((workspace) => {
if (!isDefined(workspace.version)) {
return true;
}
try {
const versionCompareResult = compareVersionMajorAndMinor(
workspace.version,
fromWorkspaceVersion.version,
);
return versionCompareResult === 'lower';
} catch (error) {
this.logger.error(
`Error checking workspace ${workspace.id} version: ${error.message}`,
);
return true;
}
});
return workspacesThatAreBelowFromWorkspaceVersion;
} catch (error) {
this.logger.error('Error checking workspaces below version:', error);
throw error;
}
}
private setUpgradeContextVersionsAndCommandsForCurrentAppVersion() {
const upgradeContextIsAlreadyDefined = [
this.currentAppVersion,
@@ -147,8 +63,9 @@ export abstract class UpgradeCommandRunner extends ActiveOrSuspendedWorkspacesMi
return;
}
const currentAppVersion = this.retrieveCurrentAppVersion();
const currentVersionMajorMinor = `${currentAppVersion.major}.${currentAppVersion.minor}.0`;
const currentAppVersion = this.coreEngineVersionService.getCurrentVersion();
const currentVersionMajorMinor =
`${currentAppVersion.major}.${currentAppVersion.minor}.0` as UpgradeCommandVersion;
const currentCommands = this.allCommands[currentVersionMajorMinor];
if (!isDefined(currentCommands)) {
@@ -157,17 +74,8 @@ export abstract class UpgradeCommandRunner extends ActiveOrSuspendedWorkspacesMi
);
}
const allCommandsVersions = Object.keys(this.allCommands);
const previousVersion = getPreviousVersion({
currentVersion: currentVersionMajorMinor,
versions: allCommandsVersions,
});
const previousVersion = this.coreEngineVersionService.getPreviousVersion();
if (!isDefined(previousVersion)) {
throw new Error(
`No previous version found for version ${currentAppVersion}. Please review the "allCommands" record. Available versions are: ${allCommandsVersions.join(', ')}`,
);
}
this.commands = currentCommands;
this.fromWorkspaceVersion = previousVersion;
this.currentAppVersion = currentAppVersion;
@@ -189,10 +97,12 @@ export abstract class UpgradeCommandRunner extends ActiveOrSuspendedWorkspacesMi
try {
this.setUpgradeContextVersionsAndCommandsForCurrentAppVersion();
const shouldSkipUpgradeIfFreshInstallation =
await this.shouldSkipUpgradeIfFreshInstallation();
// On fresh installs there are no workspaces yet, so skip the
// per-workspace upgrade loop (core migrations already ran above).
const hasWorkspaces =
await this.workspaceVersionService.hasActiveOrSuspendedWorkspaces();
if (shouldSkipUpgradeIfFreshInstallation) {
if (!hasWorkspaces) {
this.logger.log(
chalk.blue('Fresh installation detected, skipping migration'),
);
@@ -201,8 +111,8 @@ export abstract class UpgradeCommandRunner extends ActiveOrSuspendedWorkspacesMi
}
const workspacesThatAreBelowFromWorkspaceVersion =
await this.workspacesThatAreBelowFromWorkspaceVersion(
this.fromWorkspaceVersion,
await this.workspaceVersionService.getWorkspacesBelowVersion(
this.fromWorkspaceVersion.version,
);
if (workspacesThatAreBelowFromWorkspaceVersion.length > 0) {
@@ -234,7 +144,7 @@ If any workspaces are not on the previous minor version, roll back to that versi
return;
}
await this.runCoreMigrations();
await this.coreMigrationRunnerService.run();
await super.runMigrationCommand(passedParams, options);
}
@@ -295,24 +205,6 @@ If any workspaces are not on the previous minor version, roll back to that versi
}
}
private retrieveCurrentAppVersion() {
const appVersion = this.twentyConfigService.get('APP_VERSION');
if (!isDefined(appVersion)) {
throw new Error(
'Cannot run upgrade command when APP_VERSION is not defined, please double check your env variables',
);
}
try {
return new SemVer(appVersion);
} catch {
throw new Error(
`Should never occur, APP_VERSION is invalid ${appVersion}`,
);
}
}
private async retrieveWorkspaceVersionAndCompareToWorkspaceFromVersion(
workspaceId: string,
): Promise<CompareVersionMajorAndMinorReturnType> {
@@ -0,0 +1,9 @@
import { Module } from '@nestjs/common';
import { CoreMigrationRunnerService } from 'src/database/commands/core-migration-runner/services/core-migration-runner.service';
@Module({
providers: [CoreMigrationRunnerService],
exports: [CoreMigrationRunnerService],
})
export class CoreMigrationRunnerModule {}
@@ -0,0 +1,37 @@
import { Injectable, Logger } from '@nestjs/common';
import { InjectDataSource } from '@nestjs/typeorm';
import { DataSource } from 'typeorm';
@Injectable()
export class CoreMigrationRunnerService {
private readonly logger = new Logger(CoreMigrationRunnerService.name);
constructor(
@InjectDataSource()
private readonly dataSource: DataSource,
) {}
async run(): Promise<void> {
this.logger.log('Running core datasource migrations...');
try {
const migrations = await this.dataSource.runMigrations({
transaction: 'each',
});
if (migrations.length === 0) {
this.logger.log('No pending migrations');
} else {
this.logger.log(
`Executed ${migrations.length} migration(s): ${migrations.map((migration) => migration.name).join(', ')}`,
);
}
this.logger.log('Database migrations completed successfully');
} catch (error) {
this.logger.error('Error running database migrations:', error);
throw error;
}
}
}
@@ -5,9 +5,12 @@ import { CronRegisterAllCommand } from 'src/database/commands/cron-register-all.
import { DataSeedWorkspaceCommand } from 'src/database/commands/data-seed-dev-workspace.command';
import { ListOrphanedWorkspaceEntitiesCommand } from 'src/database/commands/list-and-delete-orphaned-workspace-entities.command';
import { ConfirmationQuestion } from 'src/database/commands/questions/confirmation.question';
import { WorkspaceExportModule } from 'src/database/commands/workspace-export/workspace-export.module';
import { RunTypeormMigrationCommand } from 'src/database/commands/run-typeorm-migration.command';
import { UpgradeVersionCommandModule } from 'src/database/commands/upgrade-version-command/upgrade-version-command.module';
import { WorkspaceExportModule } from 'src/database/commands/workspace-export/workspace-export.module';
import { TypeORMModule } from 'src/database/typeorm/typeorm.module';
import { CoreEngineVersionModule } from 'src/engine/core-engine-version/core-engine-version.module';
import { CoreMigrationRunnerModule } from 'src/database/commands/core-migration-runner/core-migration-runner.module';
import { ApiKeyModule } from 'src/engine/core-modules/api-key/api-key.module';
import { GenerateApiKeyCommand } from 'src/engine/core-modules/api-key/commands/generate-api-key.command';
import { MarketplaceModule } from 'src/engine/core-modules/application/application-marketplace/marketplace.module';
@@ -31,6 +34,7 @@ import { DevSeederModule } from 'src/engine/workspace-manager/dev-seeder/dev-see
import { WorkspaceCleanerModule } from 'src/engine/workspace-manager/workspace-cleaner/workspace-cleaner.module';
import { WorkspaceManagerModule } from 'src/engine/workspace-manager/workspace-manager.module';
import { WorkspaceMigrationModule } from 'src/engine/workspace-manager/workspace-migration/workspace-migration.module';
import { WorkspaceVersionModule } from 'src/engine/workspace-manager/workspace-version/workspace-version.module';
import { CalendarEventImportManagerModule } from 'src/modules/calendar/calendar-event-import-manager/calendar-event-import-manager.module';
import { MessagingImportManagerModule } from 'src/modules/messaging/message-import-manager/messaging-import-manager.module';
import { WorkflowRunQueueModule } from 'src/modules/workflow/workflow-runner/workflow-run-queue/workflow-run-queue.module';
@@ -68,6 +72,9 @@ import { AutomatedTriggerModule } from 'src/modules/workflow/workflow-trigger/au
MarketplaceModule,
ApplicationUpgradeModule,
StaleRegistrationCleanupModule,
CoreEngineVersionModule,
CoreMigrationRunnerModule,
WorkspaceVersionModule,
],
providers: [
DataSeedWorkspaceCommand,
@@ -76,6 +83,7 @@ import { AutomatedTriggerModule } from 'src/modules/workflow/workflow-trigger/au
ListOrphanedWorkspaceEntitiesCommand,
EnterpriseKeyValidationCronCommand,
GenerateApiKeyCommand,
RunTypeormMigrationCommand,
],
})
export class DatabaseCommandModule {}
@@ -0,0 +1,75 @@
import { Logger } from '@nestjs/common';
import chalk from 'chalk';
import { Command, CommandRunner, Option } from 'nest-commander';
import { CoreMigrationRunnerService } from 'src/database/commands/core-migration-runner/services/core-migration-runner.service';
import { CoreEngineVersionService } from 'src/engine/core-engine-version/services/core-engine-version.service';
import { WorkspaceVersionService } from 'src/engine/workspace-manager/workspace-version/services/workspace-version.service';
type RunTypeormMigrationCommandOptions = {
force?: boolean;
};
@Command({
name: 'run-typeorm-migration',
description:
'Run TypeORM core migrations with workspace version safety check',
})
export class RunTypeormMigrationCommand extends CommandRunner {
private readonly logger = new Logger(RunTypeormMigrationCommand.name);
constructor(
private readonly coreEngineVersionService: CoreEngineVersionService,
private readonly workspaceVersionService: WorkspaceVersionService,
private readonly coreMigrationRunnerService: CoreMigrationRunnerService,
) {
super();
}
@Option({
flags: '-f, --force',
description: 'Skip workspace version safety check',
required: false,
})
parseForce(): boolean {
return true;
}
async run(
_passedParams: string[],
options: RunTypeormMigrationCommandOptions,
): Promise<void> {
if (options.force) {
this.logger.warn(
chalk.yellow('Skipping workspace version check (--force flag used)'),
);
} else {
const previousVersion =
this.coreEngineVersionService.getPreviousVersion();
const workspacesBelow =
await this.workspaceVersionService.getWorkspacesBelowVersion(
previousVersion.version,
);
if (workspacesBelow.length > 0) {
for (const workspace of workspacesBelow) {
this.logger.error(
chalk.red(
`Workspace ${workspace.id} (${workspace.displayName}) is at version ${workspace.version ?? 'undefined'}, which is below the minimum required version.`,
),
);
}
throw new Error(
'Unable to run TypeORM migrations. Some workspace(s) are below the minimum required version.\n' +
'Please ensure all workspaces are on at least the previous minor version before running migrations.\n' +
'Use --force to bypass this check (not recommended).',
);
}
}
await this.coreMigrationRunnerService.run();
}
}
@@ -6,8 +6,11 @@ import { V1_18_UpgradeVersionCommandModule } from 'src/database/commands/upgrade
import { V1_19_UpgradeVersionCommandModule } from 'src/database/commands/upgrade-version-command/1-19/1-19-upgrade-version-command.module';
import { V1_20_UpgradeVersionCommandModule } from 'src/database/commands/upgrade-version-command/1-20/1-20-upgrade-version-command.module';
import { UpgradeCommand } from 'src/database/commands/upgrade-version-command/upgrade.command';
import { CoreEngineVersionModule } from 'src/engine/core-engine-version/core-engine-version.module';
import { CoreMigrationRunnerModule } from 'src/database/commands/core-migration-runner/core-migration-runner.module';
import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity';
import { DataSourceModule } from 'src/engine/metadata-modules/data-source/data-source.module';
import { WorkspaceVersionModule } from 'src/engine/workspace-manager/workspace-version/workspace-version.module';
@Module({
imports: [
@@ -17,6 +20,9 @@ import { DataSourceModule } from 'src/engine/metadata-modules/data-source/data-s
V1_19_UpgradeVersionCommandModule,
V1_20_UpgradeVersionCommandModule,
DataSourceModule,
CoreEngineVersionModule,
CoreMigrationRunnerModule,
WorkspaceVersionModule,
],
providers: [UpgradeCommand],
})
@@ -9,6 +9,7 @@ import {
UpgradeCommandRunner,
type VersionCommands,
} from 'src/database/commands/command-runners/upgrade.command-runner';
import { CoreMigrationRunnerService } from 'src/database/commands/core-migration-runner/services/core-migration-runner.service';
import { BackfillApplicationPackageFilesCommand } from 'src/database/commands/upgrade-version-command/1-17/1-17-backfill-application-package-files.command';
import { DeleteFileRecordsAndUpdateTableCommand } from 'src/database/commands/upgrade-version-command/1-17/1-17-delete-all-files-and-update-table.command';
import { FixMorphRelationFieldNamesCommand } from 'src/database/commands/upgrade-version-command/1-17/1-17-fix-morph-relation-field-names.command';
@@ -49,10 +50,12 @@ import { MigrateMessagingInfrastructureToMetadataCommand } from 'src/database/co
import { MigrateRichTextToTextCommand } from 'src/database/commands/upgrade-version-command/1-20/1-20-migrate-rich-text-to-text.command';
import { SeedCliApplicationRegistrationCommand } from 'src/database/commands/upgrade-version-command/1-20/1-20-seed-cli-application-registration.command';
import { UpdateStandardIndexViewNamesCommand } from 'src/database/commands/upgrade-version-command/1-20/1-20-update-standard-index-view-names.command';
import { CoreEngineVersionService } from 'src/engine/core-engine-version/services/core-engine-version.service';
import { TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service';
import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity';
import { DataSourceService } from 'src/engine/metadata-modules/data-source/data-source.service';
import { GlobalWorkspaceOrmManager } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager';
import { WorkspaceVersionService } from 'src/engine/workspace-manager/workspace-version/services/workspace-version.service';
@Command({
name: 'upgrade',
@@ -67,6 +70,9 @@ export class UpgradeCommand extends UpgradeCommandRunner {
protected readonly twentyConfigService: TwentyConfigService,
protected readonly globalWorkspaceOrmManager: GlobalWorkspaceOrmManager,
protected readonly dataSourceService: DataSourceService,
protected readonly coreEngineVersionService: CoreEngineVersionService,
protected readonly workspaceVersionService: WorkspaceVersionService,
protected readonly coreMigrationRunnerService: CoreMigrationRunnerService,
// 1.17 Commands
protected readonly backfillApplicationPackageFilesCommand: BackfillApplicationPackageFilesCommand,
@@ -121,6 +127,9 @@ export class UpgradeCommand extends UpgradeCommandRunner {
twentyConfigService,
globalWorkspaceOrmManager,
dataSourceService,
coreEngineVersionService,
workspaceVersionService,
coreMigrationRunnerService,
);
// Note: Required empty commands array to allow retrieving previous version
@@ -1,6 +1,6 @@
import { rawDataSource } from 'src/database/typeorm/raw/raw.datasource';
import { camelToSnakeCase, performQuery } from './utils';
import { camelToSnakeCase, performQuery } from './setup-db-utils';
rawDataSource
.initialize()
@@ -1,6 +1,6 @@
import { rawDataSource } from 'src/database/typeorm/raw/raw.datasource';
import { performQuery } from './utils';
import { performQuery } from './setup-db-utils';
async function dropSchemasSequentially() {
try {
@@ -23,8 +23,8 @@ import { useDirectExecution } from 'src/engine/api/graphql/direct-execution/hook
import { WorkspaceSchemaFactory } from 'src/engine/api/graphql/workspace-schema.factory';
import { CoreEngineModule } from 'src/engine/core-modules/core-engine.module';
import { ExceptionHandlerService } from 'src/engine/core-modules/exception-handler/exception-handler.service';
import { FeatureFlagService } from 'src/engine/core-modules/feature-flag/services/feature-flag.service';
import { useSentryTracing } from 'src/engine/core-modules/exception-handler/hooks/use-sentry-tracing';
import { FeatureFlagService } from 'src/engine/core-modules/feature-flag/services/feature-flag.service';
import { useDisableIntrospectionAndSuggestionsForUnauthenticatedUsers } from 'src/engine/core-modules/graphql/hooks/use-disable-introspection-and-suggestions-for-unauthenticated-users.hook';
import { useGraphQLErrorHandlerHook } from 'src/engine/core-modules/graphql/hooks/use-graphql-error-handler.hook';
import { useGraphQLQueryTiming } from 'src/engine/core-modules/graphql/hooks/use-graphql-query-timing.hook';
@@ -0,0 +1,10 @@
export const UPGRADE_COMMAND_SUPPORTED_VERSIONS = [
'1.16.0',
'1.17.0',
'1.18.0',
'1.19.0',
'1.20.0',
] as const;
export type UpgradeCommandVersion =
(typeof UPGRADE_COMMAND_SUPPORTED_VERSIONS)[number];
@@ -0,0 +1,9 @@
import { Module } from '@nestjs/common';
import { CoreEngineVersionService } from 'src/engine/core-engine-version/services/core-engine-version.service';
@Module({
providers: [CoreEngineVersionService],
exports: [CoreEngineVersionService],
})
export class CoreEngineVersionModule {}
@@ -0,0 +1,47 @@
import { Injectable } from '@nestjs/common';
import { SemVer } from 'semver';
import { isDefined } from 'twenty-shared/utils';
import { UPGRADE_COMMAND_SUPPORTED_VERSIONS } from 'src/engine/constants/upgrade-command-supported-versions.constant';
import { TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service';
import { getPreviousVersion } from 'src/utils/version/get-previous-version';
@Injectable()
export class CoreEngineVersionService {
constructor(private readonly twentyConfigService: TwentyConfigService) {}
getCurrentVersion(): SemVer {
const appVersion = this.twentyConfigService.get('APP_VERSION');
if (!isDefined(appVersion)) {
throw new Error(
'APP_VERSION is not defined, please double check your env variables',
);
}
try {
return new SemVer(appVersion);
} catch {
throw new Error(`APP_VERSION is not a valid semver: "${appVersion}"`);
}
}
getPreviousVersion(): SemVer {
const currentAppVersion = this.getCurrentVersion();
const currentVersionMajorMinor = `${currentAppVersion.major}.${currentAppVersion.minor}.0`;
const previousVersion = getPreviousVersion({
currentVersion: currentVersionMajorMinor,
versions: [...UPGRADE_COMMAND_SUPPORTED_VERSIONS],
});
if (!isDefined(previousVersion)) {
throw new Error(
`No previous version found for version ${currentAppVersion}. Available versions: ${UPGRADE_COMMAND_SUPPORTED_VERSIONS.join(', ')}`,
);
}
return previousVersion;
}
}
@@ -0,0 +1,83 @@
import { Injectable, Logger } from '@nestjs/common';
import { InjectRepository } from '@nestjs/typeorm';
import { isDefined } from 'twenty-shared/utils';
import { WorkspaceActivationStatus } from 'twenty-shared/workspace';
import { In, Repository } from 'typeorm';
import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity';
import { compareVersionMajorAndMinor } from 'src/utils/version/compare-version-minor-and-major';
@Injectable()
export class WorkspaceVersionService {
private readonly logger = new Logger(WorkspaceVersionService.name);
constructor(
@InjectRepository(WorkspaceEntity)
private readonly workspaceRepository: Repository<WorkspaceEntity>,
) {}
async hasActiveOrSuspendedWorkspaces(): Promise<boolean> {
return this.workspaceRepository.exists({
where: {
activationStatus: In([
WorkspaceActivationStatus.ACTIVE,
WorkspaceActivationStatus.SUSPENDED,
]),
},
});
}
async getWorkspacesBelowVersion(
version: string,
): Promise<Pick<WorkspaceEntity, 'id' | 'displayName' | 'version'>[]> {
const allActiveOrSuspendedWorkspaces =
await this.loadActiveOrSuspendedWorkspaces();
if (allActiveOrSuspendedWorkspaces.length === 0) {
this.logger.log(
'No workspaces found. Running migrations for fresh installation.',
);
return [];
}
return allActiveOrSuspendedWorkspaces.filter((workspace) => {
if (!isDefined(workspace.version)) {
return true;
}
try {
const versionCompareResult = compareVersionMajorAndMinor(
workspace.version,
version,
);
return versionCompareResult === 'lower';
} catch (error) {
this.logger.error(
`Error checking workspace ${workspace.id} version: ${error.message}`,
);
return true;
}
});
}
private async loadActiveOrSuspendedWorkspaces(): Promise<
Pick<WorkspaceEntity, 'id' | 'version' | 'displayName'>[]
> {
return this.workspaceRepository.find({
select: ['id', 'version', 'displayName'],
where: {
activationStatus: In([
WorkspaceActivationStatus.ACTIVE,
WorkspaceActivationStatus.SUSPENDED,
]),
},
order: {
id: 'ASC',
},
});
}
}
@@ -0,0 +1,12 @@
import { Module } from '@nestjs/common';
import { TypeOrmModule } from '@nestjs/typeorm';
import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity';
import { WorkspaceVersionService } from 'src/engine/workspace-manager/workspace-version/services/workspace-version.service';
@Module({
imports: [TypeOrmModule.forFeature([WorkspaceEntity])],
providers: [WorkspaceVersionService],
exports: [WorkspaceVersionService],
})
export class WorkspaceVersionModule {}