workspace:export command (#18695)
This PR adds a `workspace:export` server command for ongoing debug tooling/administrative work Demo Video shows 1. Exporting a sample workspace YC 2. Restoring Local DB Docker volume snapshot to Dropped YC Workspace 3. Importing exported SQL 4. Booting and navigating the imported Workspace https://github.com/user-attachments/assets/0e1ac6cb-8ce1-440b-8b56-f81dcb27a9c8
This commit is contained in:
@@ -5,6 +5,7 @@ import { CronRegisterAllCommand } from 'src/database/commands/cron-register-all.
|
||||
import { DataSeedWorkspaceCommand } from 'src/database/commands/data-seed-dev-workspace.command';
|
||||
import { ListOrphanedWorkspaceEntitiesCommand } from 'src/database/commands/list-and-delete-orphaned-workspace-entities.command';
|
||||
import { ConfirmationQuestion } from 'src/database/commands/questions/confirmation.question';
|
||||
import { WorkspaceExportModule } from 'src/database/commands/workspace-export/workspace-export.module';
|
||||
import { UpgradeVersionCommandModule } from 'src/database/commands/upgrade-version-command/upgrade-version-command.module';
|
||||
import { TypeORMModule } from 'src/database/typeorm/typeorm.module';
|
||||
import { ApiKeyModule } from 'src/engine/core-modules/api-key/api-key.module';
|
||||
@@ -39,6 +40,7 @@ import { AutomatedTriggerModule } from 'src/modules/workflow/workflow-trigger/au
|
||||
imports: [
|
||||
UpgradeVersionCommandModule,
|
||||
TypeOrmModule.forFeature([WorkspaceEntity]),
|
||||
WorkspaceExportModule,
|
||||
// Cron command dependencies
|
||||
MessagingImportManagerModule,
|
||||
CalendarEventImportManagerModule,
|
||||
|
||||
+84
@@ -0,0 +1,84 @@
|
||||
import { formatSqlValue } from 'src/database/commands/workspace-export/utils/format-sql-value.util';
|
||||
|
||||
describe('formatSqlValue', () => {
|
||||
it('should return NULL for null and undefined', () => {
|
||||
expect(formatSqlValue(null)).toBe('NULL');
|
||||
expect(formatSqlValue(undefined)).toBe('NULL');
|
||||
});
|
||||
|
||||
it('should return unquoted TRUE/FALSE for booleans', () => {
|
||||
expect(formatSqlValue(true)).toBe('TRUE');
|
||||
expect(formatSqlValue(false)).toBe('FALSE');
|
||||
});
|
||||
|
||||
it('should return unquoted numbers', () => {
|
||||
expect(formatSqlValue(42)).toBe('42');
|
||||
expect(formatSqlValue(3.14)).toBe('3.14');
|
||||
expect(formatSqlValue(-1)).toBe('-1');
|
||||
expect(formatSqlValue(0)).toBe('0');
|
||||
});
|
||||
|
||||
it('should return NULL for NaN and Infinity', () => {
|
||||
expect(formatSqlValue(NaN)).toBe('NULL');
|
||||
expect(formatSqlValue(Infinity)).toBe('NULL');
|
||||
expect(formatSqlValue(-Infinity)).toBe('NULL');
|
||||
});
|
||||
|
||||
it('should return unquoted bigint', () => {
|
||||
expect(formatSqlValue(BigInt(9007199254740991))).toBe('9007199254740991');
|
||||
});
|
||||
|
||||
it('should escape single quotes in strings', () => {
|
||||
expect(formatSqlValue("it's")).toBe("'it''s'");
|
||||
});
|
||||
|
||||
it('should handle backslashes with E-string prefix', () => {
|
||||
expect(formatSqlValue('path\\to\\file')).toBe("E'path\\\\to\\\\file'");
|
||||
});
|
||||
|
||||
it('should format dates as escaped ISO strings', () => {
|
||||
const date = new Date('2024-01-15T10:30:00.000Z');
|
||||
|
||||
expect(formatSqlValue(date)).toBe("'2024-01-15T10:30:00.000Z'");
|
||||
});
|
||||
|
||||
it('should JSON-serialize objects when isJsonColumn is true', () => {
|
||||
const value = { key: 'value' };
|
||||
|
||||
expect(formatSqlValue(value, true)).toBe('\'{"key":"value"}\'');
|
||||
});
|
||||
|
||||
it('should JSON-serialize plain objects even when isJsonColumn is false', () => {
|
||||
const value = { key: 'value' };
|
||||
|
||||
expect(formatSqlValue(value, false)).toBe('\'{"key":"value"}\'');
|
||||
});
|
||||
|
||||
it('should return empty PostgreSQL array literal for empty arrays', () => {
|
||||
expect(formatSqlValue([])).toBe("'{}'");
|
||||
});
|
||||
|
||||
it('should format string arrays as PostgreSQL array literals', () => {
|
||||
expect(formatSqlValue(['a', 'b', 'c'])).toBe('\'{"a","b","c"}\'');
|
||||
});
|
||||
|
||||
it('should escape single quotes in array elements', () => {
|
||||
expect(formatSqlValue(["O'Reilly"])).toBe("'{\"O''Reilly\"}'");
|
||||
});
|
||||
|
||||
it('should format arrays with null elements as PostgreSQL array literals', () => {
|
||||
expect(formatSqlValue([null, 'foo', 'bar'])).toBe('\'{NULL,"foo","bar"}\'');
|
||||
});
|
||||
|
||||
it('should JSON-serialize arrays of objects', () => {
|
||||
const value = [{ id: 1 }, { id: 2 }];
|
||||
|
||||
expect(formatSqlValue(value)).toBe('\'[{"id":1},{"id":2}]\'');
|
||||
});
|
||||
|
||||
it('should throw on strings containing null bytes', () => {
|
||||
expect(() => formatSqlValue('hello\0world')).toThrow(
|
||||
'Null bytes are not allowed',
|
||||
);
|
||||
});
|
||||
});
|
||||
+11
@@ -0,0 +1,11 @@
|
||||
import { escapeIdentifier } from 'src/engine/workspace-manager/workspace-migration/utils/remove-sql-injection.util';
|
||||
|
||||
export const buildInsertPrefix = (
|
||||
schemaName: string,
|
||||
tableName: string,
|
||||
columnNames: string[],
|
||||
): string => {
|
||||
const escapedColumnNames = columnNames.map(escapeIdentifier).join(', ');
|
||||
|
||||
return `INSERT INTO ${escapeIdentifier(schemaName)}.${escapeIdentifier(tableName)} (${escapedColumnNames}) VALUES `;
|
||||
};
|
||||
+45
@@ -0,0 +1,45 @@
|
||||
import { type FieldMetadataEntity } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity';
|
||||
import { type FlatFieldMetadata } from 'src/engine/metadata-modules/flat-field-metadata/types/flat-field-metadata.type';
|
||||
import { type FlatObjectMetadata } from 'src/engine/metadata-modules/flat-object-metadata/types/flat-object-metadata.type';
|
||||
import { type ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity';
|
||||
import { generateColumnDefinitions } from 'src/engine/workspace-manager/workspace-migration/workspace-migration-runner/utils/generate-column-definitions.util';
|
||||
|
||||
const JSON_COLUMN_TYPES = new Set(['json', 'jsonb']);
|
||||
|
||||
type WorkspaceTableColumnSets = {
|
||||
jsonColumns: Set<string>;
|
||||
generatedColumns: Set<string>;
|
||||
};
|
||||
|
||||
export const buildWorkspaceTableColumnSets = (
|
||||
workspaceId: string,
|
||||
objectMetadata: ObjectMetadataEntity,
|
||||
fieldMetadatas: FieldMetadataEntity[],
|
||||
): WorkspaceTableColumnSets => {
|
||||
const jsonColumns = new Set<string>();
|
||||
const generatedColumns = new Set<string>();
|
||||
|
||||
const flatObjectMetadata = objectMetadata as unknown as FlatObjectMetadata;
|
||||
|
||||
for (const fieldMetadata of fieldMetadatas) {
|
||||
const flatFieldMetadata = fieldMetadata as unknown as FlatFieldMetadata;
|
||||
|
||||
const columnDefinitions = generateColumnDefinitions({
|
||||
flatFieldMetadata,
|
||||
flatObjectMetadata,
|
||||
workspaceId,
|
||||
});
|
||||
|
||||
for (const columnDefinition of columnDefinitions) {
|
||||
if (JSON_COLUMN_TYPES.has(columnDefinition.type)) {
|
||||
jsonColumns.add(columnDefinition.name);
|
||||
}
|
||||
|
||||
if (columnDefinition.type === 'tsvector') {
|
||||
generatedColumns.add(columnDefinition.name);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return { jsonColumns, generatedColumns };
|
||||
};
|
||||
+55
@@ -0,0 +1,55 @@
|
||||
import { isDefined } from 'twenty-shared/utils';
|
||||
|
||||
import { escapeLiteral } from 'src/engine/workspace-manager/workspace-migration/utils/remove-sql-injection.util';
|
||||
|
||||
export const formatSqlValue = (
|
||||
value: unknown,
|
||||
isJsonColumn = false,
|
||||
): string => {
|
||||
if (!isDefined(value)) return 'NULL';
|
||||
|
||||
if (isJsonColumn) {
|
||||
return escapeLiteral(JSON.stringify(value));
|
||||
}
|
||||
|
||||
if (typeof value === 'boolean') return value ? 'TRUE' : 'FALSE';
|
||||
|
||||
if (typeof value === 'number') {
|
||||
if (!Number.isFinite(value)) return 'NULL';
|
||||
|
||||
return String(value);
|
||||
}
|
||||
|
||||
if (typeof value === 'bigint') return String(value);
|
||||
|
||||
if (value instanceof Date) return escapeLiteral(value.toISOString());
|
||||
|
||||
if (Array.isArray(value)) {
|
||||
if (value.length === 0) return "'{}'";
|
||||
|
||||
if (isDefined(value[0]) && typeof value[0] === 'object') {
|
||||
return escapeLiteral(JSON.stringify(value));
|
||||
}
|
||||
|
||||
const formattedElements = value.map((element) => {
|
||||
if (!isDefined(element)) return 'NULL';
|
||||
|
||||
const stringElement = String(element);
|
||||
const escapedElement = stringElement
|
||||
.replace(/\\/g, '\\\\')
|
||||
.replace(/"/g, '\\"');
|
||||
|
||||
return `"${escapedElement}"`;
|
||||
});
|
||||
|
||||
const arrayLiteral = `{${formattedElements.join(',')}}`;
|
||||
|
||||
return `'${arrayLiteral.replace(/'/g, "''")}'`;
|
||||
}
|
||||
|
||||
if (typeof value === 'object') {
|
||||
return escapeLiteral(JSON.stringify(value));
|
||||
}
|
||||
|
||||
return escapeLiteral(String(value));
|
||||
};
|
||||
+4
@@ -0,0 +1,4 @@
|
||||
export const generateInsertStatement = (
|
||||
insertPrefix: string,
|
||||
formattedValues: string[],
|
||||
): string => `${insertPrefix}(${formattedValues.join(', ')});\n`;
|
||||
+75
@@ -0,0 +1,75 @@
|
||||
import { type FieldMetadataEntity } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity';
|
||||
import { type FlatFieldMetadata } from 'src/engine/metadata-modules/flat-field-metadata/types/flat-field-metadata.type';
|
||||
import { type FlatObjectMetadata } from 'src/engine/metadata-modules/flat-object-metadata/types/flat-object-metadata.type';
|
||||
import { type ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity';
|
||||
import { buildSqlColumnDefinition } from 'src/engine/twenty-orm/workspace-schema-manager/utils/build-sql-column-definition.util';
|
||||
import { computeTableName } from 'src/engine/utils/compute-table-name.util';
|
||||
import {
|
||||
escapeIdentifier,
|
||||
escapeLiteral,
|
||||
} from 'src/engine/workspace-manager/workspace-migration/utils/remove-sql-injection.util';
|
||||
import { generateColumnDefinitions } from 'src/engine/workspace-manager/workspace-migration/workspace-migration-runner/utils/generate-column-definitions.util';
|
||||
import {
|
||||
type CreateEnumOperationSpec,
|
||||
EnumOperation,
|
||||
collectEnumOperationsForObject,
|
||||
} from 'src/engine/workspace-manager/workspace-migration/workspace-migration-runner/utils/workspace-schema-enum-operations.util';
|
||||
|
||||
export const generateWorkspaceSchemaDdl = (
|
||||
workspaceId: string,
|
||||
schemaName: string,
|
||||
objectMetadatas: ObjectMetadataEntity[],
|
||||
fieldsByObjectId: Map<string, FieldMetadataEntity[]>,
|
||||
): string[] => {
|
||||
const statements: string[] = [];
|
||||
|
||||
for (const objectMetadata of objectMetadatas) {
|
||||
if (!objectMetadata.isActive) continue;
|
||||
|
||||
const tableName = computeTableName(
|
||||
objectMetadata.nameSingular,
|
||||
objectMetadata.isCustom,
|
||||
);
|
||||
const fieldMetadatas = fieldsByObjectId.get(objectMetadata.id) ?? [];
|
||||
|
||||
const flatFieldMetadatas = fieldMetadatas as unknown as FlatFieldMetadata[];
|
||||
const flatObjectMetadata = objectMetadata as unknown as FlatObjectMetadata;
|
||||
|
||||
const enumOperations = collectEnumOperationsForObject({
|
||||
tableName,
|
||||
operation: EnumOperation.CREATE,
|
||||
flatFieldMetadatas,
|
||||
});
|
||||
|
||||
for (const enumOperation of enumOperations) {
|
||||
const createOp = enumOperation as CreateEnumOperationSpec;
|
||||
const escapedValues = createOp.values.map(escapeLiteral).join(', ');
|
||||
|
||||
statements.push(
|
||||
`CREATE TYPE ${escapeIdentifier(schemaName)}.${escapeIdentifier(createOp.enumName)} AS ENUM (${escapedValues});`,
|
||||
);
|
||||
}
|
||||
|
||||
const columnDefinitions = flatFieldMetadatas.flatMap((flatFieldMetadata) =>
|
||||
generateColumnDefinitions({
|
||||
flatFieldMetadata,
|
||||
flatObjectMetadata,
|
||||
workspaceId,
|
||||
}),
|
||||
);
|
||||
|
||||
if (columnDefinitions.length === 0) continue;
|
||||
|
||||
const columnsSql = columnDefinitions
|
||||
.map(
|
||||
(columnDefinition) => ` ${buildSqlColumnDefinition(columnDefinition)}`,
|
||||
)
|
||||
.join(',\n');
|
||||
|
||||
statements.push(
|
||||
`CREATE TABLE ${escapeIdentifier(schemaName)}.${escapeIdentifier(tableName)} (\n${columnsSql}\n);`,
|
||||
);
|
||||
}
|
||||
|
||||
return statements;
|
||||
};
|
||||
+11
@@ -0,0 +1,11 @@
|
||||
import { DataSource } from 'typeorm';
|
||||
|
||||
export const getCoreEntityMetadatasWithWorkspaceId = (
|
||||
dataSource: DataSource,
|
||||
) => {
|
||||
return dataSource.entityMetadatas.filter((entityMetadata) =>
|
||||
entityMetadata.columns.some(
|
||||
(column) => column.propertyName === 'workspaceId',
|
||||
),
|
||||
);
|
||||
};
|
||||
+70
@@ -0,0 +1,70 @@
|
||||
import { Logger } from '@nestjs/common';
|
||||
|
||||
import { Command, CommandRunner, Option } from 'nest-commander';
|
||||
|
||||
import { WorkspaceExportService } from 'src/database/commands/workspace-export/workspace-export.service';
|
||||
|
||||
type WorkspaceExportCommandOptions = {
|
||||
workspaceId: string;
|
||||
outputPath: string;
|
||||
tables?: string;
|
||||
};
|
||||
|
||||
@Command({
|
||||
name: 'workspace:export',
|
||||
description: 'Export a workspace as SQL INSERT statements',
|
||||
})
|
||||
export class WorkspaceExportCommand extends CommandRunner {
|
||||
private readonly logger = new Logger(WorkspaceExportCommand.name);
|
||||
|
||||
constructor(private readonly workspaceExportService: WorkspaceExportService) {
|
||||
super();
|
||||
}
|
||||
|
||||
@Option({
|
||||
flags: '--workspace-id <workspaceId>',
|
||||
description: 'Workspace UUID to export',
|
||||
required: true,
|
||||
})
|
||||
parseWorkspaceId(value: string): string {
|
||||
return value;
|
||||
}
|
||||
|
||||
@Option({
|
||||
flags: '--output-path <outputPath>',
|
||||
description: 'Directory to write the .sql file',
|
||||
defaultValue: '/tmp/exports',
|
||||
})
|
||||
parseOutputPath(value: string): string {
|
||||
return value;
|
||||
}
|
||||
|
||||
@Option({
|
||||
flags: '--tables <tables>',
|
||||
description:
|
||||
'Comma-separated workspace table names to export (uses nameSingular from ObjectMetadata)',
|
||||
})
|
||||
parseTables(value: string): string {
|
||||
return value;
|
||||
}
|
||||
|
||||
async run(
|
||||
_passedParams: string[],
|
||||
options: WorkspaceExportCommandOptions,
|
||||
): Promise<void> {
|
||||
const tableFilter = options.tables?.split(',').map((table) => table.trim());
|
||||
|
||||
try {
|
||||
const filePath = await this.workspaceExportService.exportWorkspace({
|
||||
workspaceId: options.workspaceId,
|
||||
outputPath: options.outputPath,
|
||||
tableFilter,
|
||||
});
|
||||
|
||||
this.logger.log(`Export complete: ${filePath}`);
|
||||
} catch (error) {
|
||||
this.logger.error('Export failed', error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
+15
@@ -0,0 +1,15 @@
|
||||
import { Module } from '@nestjs/common';
|
||||
import { TypeOrmModule } from '@nestjs/typeorm';
|
||||
|
||||
import { WorkspaceExportCommand } from 'src/database/commands/workspace-export/workspace-export.command';
|
||||
import { WorkspaceExportService } from 'src/database/commands/workspace-export/workspace-export.service';
|
||||
import { FieldMetadataEntity } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity';
|
||||
import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity';
|
||||
|
||||
@Module({
|
||||
imports: [
|
||||
TypeOrmModule.forFeature([ObjectMetadataEntity, FieldMetadataEntity]),
|
||||
],
|
||||
providers: [WorkspaceExportCommand, WorkspaceExportService],
|
||||
})
|
||||
export class WorkspaceExportModule {}
|
||||
+325
@@ -0,0 +1,325 @@
|
||||
import { Injectable, Logger } from '@nestjs/common';
|
||||
import { InjectDataSource, InjectRepository } from '@nestjs/typeorm';
|
||||
import { once } from 'events';
|
||||
import { type WriteStream, createWriteStream, mkdirSync } from 'fs';
|
||||
import { finished } from 'stream/promises';
|
||||
|
||||
import {
|
||||
DataSource,
|
||||
type EntityMetadata,
|
||||
type QueryRunner,
|
||||
Repository,
|
||||
} from 'typeorm';
|
||||
|
||||
import { FieldMetadataEntity } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity';
|
||||
import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity';
|
||||
import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity';
|
||||
import { getWorkspaceSchemaName } from 'src/engine/workspace-datasource/utils/get-workspace-schema-name.util';
|
||||
import { computeTableName } from 'src/engine/utils/compute-table-name.util';
|
||||
import { escapeIdentifier } from 'src/engine/workspace-manager/workspace-migration/utils/remove-sql-injection.util';
|
||||
import { getCoreEntityMetadatasWithWorkspaceId } from 'src/database/commands/workspace-export/utils/get-core-entity-metadatas-with-workspace-id.util';
|
||||
import { generateWorkspaceSchemaDdl } from 'src/database/commands/workspace-export/utils/generate-workspace-schema-ddl.util';
|
||||
import { buildInsertPrefix } from 'src/database/commands/workspace-export/utils/build-insert-prefix.util';
|
||||
import { buildWorkspaceTableColumnSets } from 'src/database/commands/workspace-export/utils/build-workspace-table-column-sets.util';
|
||||
import { formatSqlValue } from 'src/database/commands/workspace-export/utils/format-sql-value.util';
|
||||
import { generateInsertStatement } from 'src/database/commands/workspace-export/utils/generate-insert-statement.util';
|
||||
|
||||
const BATCH_SIZE = 5000;
|
||||
|
||||
type WorkspaceExportParams = {
|
||||
workspaceId: string;
|
||||
outputPath: string;
|
||||
tableFilter?: string[];
|
||||
};
|
||||
|
||||
type RowFilter = {
|
||||
filterColumn: string;
|
||||
filterValue: string;
|
||||
};
|
||||
|
||||
type WriteRowsOptions = {
|
||||
schemaName: string;
|
||||
tableName: string;
|
||||
displayName: string;
|
||||
queryRunner: QueryRunner;
|
||||
stream: WriteStream;
|
||||
rowFilter?: RowFilter;
|
||||
jsonColumns?: Set<string>;
|
||||
excludedColumns?: Set<string>;
|
||||
};
|
||||
|
||||
@Injectable()
|
||||
export class WorkspaceExportService {
|
||||
private readonly logger = new Logger(WorkspaceExportService.name);
|
||||
|
||||
constructor(
|
||||
@InjectDataSource()
|
||||
private readonly dataSource: DataSource,
|
||||
@InjectRepository(ObjectMetadataEntity)
|
||||
private readonly objectMetadataRepository: Repository<ObjectMetadataEntity>,
|
||||
@InjectRepository(FieldMetadataEntity)
|
||||
private readonly fieldMetadataRepository: Repository<FieldMetadataEntity>,
|
||||
) {}
|
||||
|
||||
async exportWorkspace({
|
||||
workspaceId,
|
||||
outputPath,
|
||||
tableFilter,
|
||||
}: WorkspaceExportParams): Promise<string> {
|
||||
const workspace = await this.dataSource
|
||||
.getRepository(WorkspaceEntity)
|
||||
.findOne({ where: { id: workspaceId } });
|
||||
|
||||
if (!workspace) {
|
||||
throw new Error(`Workspace ${workspaceId} not found`);
|
||||
}
|
||||
|
||||
const schemaName = getWorkspaceSchemaName(workspaceId);
|
||||
|
||||
this.logger.log(`Exporting workspace ${workspaceId} (${schemaName})`);
|
||||
|
||||
const objectMetadatas = await this.objectMetadataRepository.find({
|
||||
where: { workspaceId },
|
||||
});
|
||||
|
||||
const fieldMetadatas = await this.fieldMetadataRepository.find({
|
||||
where: { workspaceId },
|
||||
});
|
||||
|
||||
const fieldsByObjectId = new Map<string, FieldMetadataEntity[]>();
|
||||
|
||||
for (const fieldMetadata of fieldMetadatas) {
|
||||
const objectFields =
|
||||
fieldsByObjectId.get(fieldMetadata.objectMetadataId) ?? [];
|
||||
|
||||
objectFields.push(fieldMetadata);
|
||||
fieldsByObjectId.set(fieldMetadata.objectMetadataId, objectFields);
|
||||
}
|
||||
|
||||
mkdirSync(outputPath, { recursive: true });
|
||||
|
||||
const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
|
||||
const filePath = `${outputPath}/${workspaceId}-${timestamp}.sql`;
|
||||
const stream = createWriteStream(filePath);
|
||||
|
||||
const queryRunner = this.dataSource.createQueryRunner();
|
||||
|
||||
try {
|
||||
stream.write("SET session_replication_role = 'replica';\n\n");
|
||||
|
||||
await this.writeCoreEntityRows(workspaceId, queryRunner, stream);
|
||||
|
||||
stream.write(
|
||||
`\nCREATE SCHEMA IF NOT EXISTS ${escapeIdentifier(schemaName)};\n\n`,
|
||||
);
|
||||
|
||||
this.writeWorkspaceSchemaDdl(
|
||||
workspaceId,
|
||||
schemaName,
|
||||
objectMetadatas,
|
||||
fieldsByObjectId,
|
||||
stream,
|
||||
);
|
||||
|
||||
await this.writeWorkspaceDataRows(
|
||||
workspaceId,
|
||||
schemaName,
|
||||
objectMetadatas,
|
||||
fieldsByObjectId,
|
||||
tableFilter,
|
||||
queryRunner,
|
||||
stream,
|
||||
);
|
||||
|
||||
stream.write("\nSET session_replication_role = 'origin';\n");
|
||||
} finally {
|
||||
await queryRunner.release();
|
||||
stream.end();
|
||||
await finished(stream);
|
||||
}
|
||||
|
||||
return filePath;
|
||||
}
|
||||
|
||||
private async writeCoreEntityRows(
|
||||
workspaceId: string,
|
||||
queryRunner: QueryRunner,
|
||||
stream: WriteStream,
|
||||
): Promise<void> {
|
||||
const workspaceEntityMetadata = this.dataSource.entityMetadatas.find(
|
||||
(entityMetadata) => entityMetadata.tableName === 'workspace',
|
||||
);
|
||||
|
||||
if (workspaceEntityMetadata) {
|
||||
await this.writeRows({
|
||||
schemaName: workspaceEntityMetadata.schema || 'core',
|
||||
tableName: workspaceEntityMetadata.tableName,
|
||||
displayName: workspaceEntityMetadata.tableName,
|
||||
queryRunner,
|
||||
stream,
|
||||
rowFilter: { filterColumn: 'id', filterValue: workspaceId },
|
||||
jsonColumns: this.buildJsonColumnSet(workspaceEntityMetadata),
|
||||
});
|
||||
}
|
||||
|
||||
const coreEntityMetadatas = getCoreEntityMetadatasWithWorkspaceId(
|
||||
this.dataSource,
|
||||
);
|
||||
|
||||
for (const entityMetadata of coreEntityMetadatas) {
|
||||
try {
|
||||
await this.writeRows({
|
||||
schemaName: entityMetadata.schema || 'core',
|
||||
tableName: entityMetadata.tableName,
|
||||
displayName: entityMetadata.tableName,
|
||||
queryRunner,
|
||||
stream,
|
||||
rowFilter: {
|
||||
filterColumn: 'workspaceId',
|
||||
filterValue: workspaceId,
|
||||
},
|
||||
jsonColumns: this.buildJsonColumnSet(entityMetadata),
|
||||
});
|
||||
} catch (error) {
|
||||
this.logger.warn(`${entityMetadata.tableName}: skipped`, error);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private buildJsonColumnSet(entityMetadata: EntityMetadata): Set<string> {
|
||||
return new Set(
|
||||
entityMetadata.columns
|
||||
.filter((column) => column.type === 'jsonb' || column.type === 'json')
|
||||
.map((column) => column.databaseName),
|
||||
);
|
||||
}
|
||||
|
||||
private async writeRows({
|
||||
schemaName,
|
||||
tableName,
|
||||
displayName,
|
||||
queryRunner,
|
||||
stream,
|
||||
rowFilter,
|
||||
jsonColumns,
|
||||
excludedColumns,
|
||||
}: WriteRowsOptions): Promise<void> {
|
||||
const whereClause = rowFilter
|
||||
? ` WHERE "${rowFilter.filterColumn}" = $1`
|
||||
: '';
|
||||
const queryParameters = rowFilter ? [rowFilter.filterValue] : [];
|
||||
|
||||
const [{ count: totalCount }] = await queryRunner.query(
|
||||
`SELECT COUNT(*)::int as count FROM "${schemaName}"."${tableName}"${whereClause}`,
|
||||
queryParameters,
|
||||
);
|
||||
|
||||
if (totalCount === 0) return;
|
||||
|
||||
this.logger.log(` ${displayName}: ${totalCount} rows`);
|
||||
|
||||
let insertPrefix: string | undefined;
|
||||
|
||||
for (let offset = 0; offset < totalCount; offset += BATCH_SIZE) {
|
||||
const rows: Record<string, unknown>[] = await queryRunner.query(
|
||||
`SELECT * FROM "${schemaName}"."${tableName}"${whereClause} ORDER BY "id" LIMIT ${BATCH_SIZE} OFFSET ${offset}`,
|
||||
queryParameters,
|
||||
);
|
||||
|
||||
const batchStatements: string[] = [];
|
||||
|
||||
for (const row of rows) {
|
||||
const columnNames = Object.keys(row).filter(
|
||||
(columnName) => !excludedColumns?.has(columnName),
|
||||
);
|
||||
|
||||
if (!insertPrefix) {
|
||||
insertPrefix = buildInsertPrefix(schemaName, tableName, columnNames);
|
||||
}
|
||||
|
||||
const formattedValues = columnNames.map((columnName) =>
|
||||
formatSqlValue(row[columnName], jsonColumns?.has(columnName)),
|
||||
);
|
||||
|
||||
batchStatements.push(
|
||||
generateInsertStatement(insertPrefix, formattedValues),
|
||||
);
|
||||
}
|
||||
|
||||
if (!stream.write(batchStatements.join(''))) {
|
||||
await once(stream, 'drain');
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private writeWorkspaceSchemaDdl(
|
||||
workspaceId: string,
|
||||
schemaName: string,
|
||||
objectMetadatas: ObjectMetadataEntity[],
|
||||
fieldsByObjectId: Map<string, FieldMetadataEntity[]>,
|
||||
stream: WriteStream,
|
||||
): void {
|
||||
this.logger.log('Generating workspace schema DDL from metadata...');
|
||||
|
||||
const ddlStatements = generateWorkspaceSchemaDdl(
|
||||
workspaceId,
|
||||
schemaName,
|
||||
objectMetadatas,
|
||||
fieldsByObjectId,
|
||||
);
|
||||
|
||||
this.logger.log(` ${ddlStatements.length} DDL statements`);
|
||||
|
||||
for (const statement of ddlStatements) {
|
||||
stream.write(statement + '\n');
|
||||
}
|
||||
|
||||
stream.write('\n');
|
||||
}
|
||||
|
||||
private async writeWorkspaceDataRows(
|
||||
workspaceId: string,
|
||||
schemaName: string,
|
||||
objectMetadatas: ObjectMetadataEntity[],
|
||||
fieldsByObjectId: Map<string, FieldMetadataEntity[]>,
|
||||
tableFilter: string[] | undefined,
|
||||
queryRunner: QueryRunner,
|
||||
stream: WriteStream,
|
||||
): Promise<void> {
|
||||
for (const objectMetadata of objectMetadatas) {
|
||||
if (!objectMetadata.isActive) continue;
|
||||
|
||||
const tableName = computeTableName(
|
||||
objectMetadata.nameSingular,
|
||||
objectMetadata.isCustom,
|
||||
);
|
||||
|
||||
if (tableFilter && !tableFilter.includes(objectMetadata.nameSingular)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const objectFieldMetadatas =
|
||||
fieldsByObjectId.get(objectMetadata.id) ?? [];
|
||||
|
||||
const { jsonColumns, generatedColumns } = buildWorkspaceTableColumnSets(
|
||||
workspaceId,
|
||||
objectMetadata,
|
||||
objectFieldMetadatas,
|
||||
);
|
||||
|
||||
try {
|
||||
await this.writeRows({
|
||||
schemaName,
|
||||
tableName,
|
||||
displayName: objectMetadata.nameSingular,
|
||||
queryRunner,
|
||||
stream,
|
||||
jsonColumns,
|
||||
excludedColumns: generatedColumns,
|
||||
});
|
||||
} catch (error) {
|
||||
this.logger.warn(`${objectMetadata.nameSingular}: skipped`, error);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user