workspace:export command (#18695)

This PR adds a `workspace:export` server command for ongoing debug
tooling/administrative work

Demo Video shows
1. Exporting a sample workspace YC
2. Restoring Local DB Docker volume snapshot to Dropped YC Workspace
3. Importing exported SQL
4. Booting and navigating the imported Workspace



https://github.com/user-attachments/assets/0e1ac6cb-8ce1-440b-8b56-f81dcb27a9c8
This commit is contained in:
neo773
2026-03-18 16:10:12 +05:30
committed by GitHub
parent dedcf4e9b9
commit 87efaf2ff8
11 changed files with 697 additions and 0 deletions
@@ -5,6 +5,7 @@ import { CronRegisterAllCommand } from 'src/database/commands/cron-register-all.
import { DataSeedWorkspaceCommand } from 'src/database/commands/data-seed-dev-workspace.command';
import { ListOrphanedWorkspaceEntitiesCommand } from 'src/database/commands/list-and-delete-orphaned-workspace-entities.command';
import { ConfirmationQuestion } from 'src/database/commands/questions/confirmation.question';
import { WorkspaceExportModule } from 'src/database/commands/workspace-export/workspace-export.module';
import { UpgradeVersionCommandModule } from 'src/database/commands/upgrade-version-command/upgrade-version-command.module';
import { TypeORMModule } from 'src/database/typeorm/typeorm.module';
import { ApiKeyModule } from 'src/engine/core-modules/api-key/api-key.module';
@@ -39,6 +40,7 @@ import { AutomatedTriggerModule } from 'src/modules/workflow/workflow-trigger/au
imports: [
UpgradeVersionCommandModule,
TypeOrmModule.forFeature([WorkspaceEntity]),
WorkspaceExportModule,
// Cron command dependencies
MessagingImportManagerModule,
CalendarEventImportManagerModule,
@@ -0,0 +1,84 @@
import { formatSqlValue } from 'src/database/commands/workspace-export/utils/format-sql-value.util';
describe('formatSqlValue', () => {
it('should return NULL for null and undefined', () => {
expect(formatSqlValue(null)).toBe('NULL');
expect(formatSqlValue(undefined)).toBe('NULL');
});
it('should return unquoted TRUE/FALSE for booleans', () => {
expect(formatSqlValue(true)).toBe('TRUE');
expect(formatSqlValue(false)).toBe('FALSE');
});
it('should return unquoted numbers', () => {
expect(formatSqlValue(42)).toBe('42');
expect(formatSqlValue(3.14)).toBe('3.14');
expect(formatSqlValue(-1)).toBe('-1');
expect(formatSqlValue(0)).toBe('0');
});
it('should return NULL for NaN and Infinity', () => {
expect(formatSqlValue(NaN)).toBe('NULL');
expect(formatSqlValue(Infinity)).toBe('NULL');
expect(formatSqlValue(-Infinity)).toBe('NULL');
});
it('should return unquoted bigint', () => {
expect(formatSqlValue(BigInt(9007199254740991))).toBe('9007199254740991');
});
it('should escape single quotes in strings', () => {
expect(formatSqlValue("it's")).toBe("'it''s'");
});
it('should handle backslashes with E-string prefix', () => {
expect(formatSqlValue('path\\to\\file')).toBe("E'path\\\\to\\\\file'");
});
it('should format dates as escaped ISO strings', () => {
const date = new Date('2024-01-15T10:30:00.000Z');
expect(formatSqlValue(date)).toBe("'2024-01-15T10:30:00.000Z'");
});
it('should JSON-serialize objects when isJsonColumn is true', () => {
const value = { key: 'value' };
expect(formatSqlValue(value, true)).toBe('\'{"key":"value"}\'');
});
it('should JSON-serialize plain objects even when isJsonColumn is false', () => {
const value = { key: 'value' };
expect(formatSqlValue(value, false)).toBe('\'{"key":"value"}\'');
});
it('should return empty PostgreSQL array literal for empty arrays', () => {
expect(formatSqlValue([])).toBe("'{}'");
});
it('should format string arrays as PostgreSQL array literals', () => {
expect(formatSqlValue(['a', 'b', 'c'])).toBe('\'{"a","b","c"}\'');
});
it('should escape single quotes in array elements', () => {
expect(formatSqlValue(["O'Reilly"])).toBe("'{\"O''Reilly\"}'");
});
it('should format arrays with null elements as PostgreSQL array literals', () => {
expect(formatSqlValue([null, 'foo', 'bar'])).toBe('\'{NULL,"foo","bar"}\'');
});
it('should JSON-serialize arrays of objects', () => {
const value = [{ id: 1 }, { id: 2 }];
expect(formatSqlValue(value)).toBe('\'[{"id":1},{"id":2}]\'');
});
it('should throw on strings containing null bytes', () => {
expect(() => formatSqlValue('hello\0world')).toThrow(
'Null bytes are not allowed',
);
});
});
@@ -0,0 +1,11 @@
import { escapeIdentifier } from 'src/engine/workspace-manager/workspace-migration/utils/remove-sql-injection.util';
export const buildInsertPrefix = (
schemaName: string,
tableName: string,
columnNames: string[],
): string => {
const escapedColumnNames = columnNames.map(escapeIdentifier).join(', ');
return `INSERT INTO ${escapeIdentifier(schemaName)}.${escapeIdentifier(tableName)} (${escapedColumnNames}) VALUES `;
};
@@ -0,0 +1,45 @@
import { type FieldMetadataEntity } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity';
import { type FlatFieldMetadata } from 'src/engine/metadata-modules/flat-field-metadata/types/flat-field-metadata.type';
import { type FlatObjectMetadata } from 'src/engine/metadata-modules/flat-object-metadata/types/flat-object-metadata.type';
import { type ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity';
import { generateColumnDefinitions } from 'src/engine/workspace-manager/workspace-migration/workspace-migration-runner/utils/generate-column-definitions.util';
const JSON_COLUMN_TYPES = new Set(['json', 'jsonb']);
type WorkspaceTableColumnSets = {
jsonColumns: Set<string>;
generatedColumns: Set<string>;
};
export const buildWorkspaceTableColumnSets = (
workspaceId: string,
objectMetadata: ObjectMetadataEntity,
fieldMetadatas: FieldMetadataEntity[],
): WorkspaceTableColumnSets => {
const jsonColumns = new Set<string>();
const generatedColumns = new Set<string>();
const flatObjectMetadata = objectMetadata as unknown as FlatObjectMetadata;
for (const fieldMetadata of fieldMetadatas) {
const flatFieldMetadata = fieldMetadata as unknown as FlatFieldMetadata;
const columnDefinitions = generateColumnDefinitions({
flatFieldMetadata,
flatObjectMetadata,
workspaceId,
});
for (const columnDefinition of columnDefinitions) {
if (JSON_COLUMN_TYPES.has(columnDefinition.type)) {
jsonColumns.add(columnDefinition.name);
}
if (columnDefinition.type === 'tsvector') {
generatedColumns.add(columnDefinition.name);
}
}
}
return { jsonColumns, generatedColumns };
};
@@ -0,0 +1,55 @@
import { isDefined } from 'twenty-shared/utils';
import { escapeLiteral } from 'src/engine/workspace-manager/workspace-migration/utils/remove-sql-injection.util';
export const formatSqlValue = (
value: unknown,
isJsonColumn = false,
): string => {
if (!isDefined(value)) return 'NULL';
if (isJsonColumn) {
return escapeLiteral(JSON.stringify(value));
}
if (typeof value === 'boolean') return value ? 'TRUE' : 'FALSE';
if (typeof value === 'number') {
if (!Number.isFinite(value)) return 'NULL';
return String(value);
}
if (typeof value === 'bigint') return String(value);
if (value instanceof Date) return escapeLiteral(value.toISOString());
if (Array.isArray(value)) {
if (value.length === 0) return "'{}'";
if (isDefined(value[0]) && typeof value[0] === 'object') {
return escapeLiteral(JSON.stringify(value));
}
const formattedElements = value.map((element) => {
if (!isDefined(element)) return 'NULL';
const stringElement = String(element);
const escapedElement = stringElement
.replace(/\\/g, '\\\\')
.replace(/"/g, '\\"');
return `"${escapedElement}"`;
});
const arrayLiteral = `{${formattedElements.join(',')}}`;
return `'${arrayLiteral.replace(/'/g, "''")}'`;
}
if (typeof value === 'object') {
return escapeLiteral(JSON.stringify(value));
}
return escapeLiteral(String(value));
};
@@ -0,0 +1,4 @@
export const generateInsertStatement = (
insertPrefix: string,
formattedValues: string[],
): string => `${insertPrefix}(${formattedValues.join(', ')});\n`;
@@ -0,0 +1,75 @@
import { type FieldMetadataEntity } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity';
import { type FlatFieldMetadata } from 'src/engine/metadata-modules/flat-field-metadata/types/flat-field-metadata.type';
import { type FlatObjectMetadata } from 'src/engine/metadata-modules/flat-object-metadata/types/flat-object-metadata.type';
import { type ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity';
import { buildSqlColumnDefinition } from 'src/engine/twenty-orm/workspace-schema-manager/utils/build-sql-column-definition.util';
import { computeTableName } from 'src/engine/utils/compute-table-name.util';
import {
escapeIdentifier,
escapeLiteral,
} from 'src/engine/workspace-manager/workspace-migration/utils/remove-sql-injection.util';
import { generateColumnDefinitions } from 'src/engine/workspace-manager/workspace-migration/workspace-migration-runner/utils/generate-column-definitions.util';
import {
type CreateEnumOperationSpec,
EnumOperation,
collectEnumOperationsForObject,
} from 'src/engine/workspace-manager/workspace-migration/workspace-migration-runner/utils/workspace-schema-enum-operations.util';
export const generateWorkspaceSchemaDdl = (
workspaceId: string,
schemaName: string,
objectMetadatas: ObjectMetadataEntity[],
fieldsByObjectId: Map<string, FieldMetadataEntity[]>,
): string[] => {
const statements: string[] = [];
for (const objectMetadata of objectMetadatas) {
if (!objectMetadata.isActive) continue;
const tableName = computeTableName(
objectMetadata.nameSingular,
objectMetadata.isCustom,
);
const fieldMetadatas = fieldsByObjectId.get(objectMetadata.id) ?? [];
const flatFieldMetadatas = fieldMetadatas as unknown as FlatFieldMetadata[];
const flatObjectMetadata = objectMetadata as unknown as FlatObjectMetadata;
const enumOperations = collectEnumOperationsForObject({
tableName,
operation: EnumOperation.CREATE,
flatFieldMetadatas,
});
for (const enumOperation of enumOperations) {
const createOp = enumOperation as CreateEnumOperationSpec;
const escapedValues = createOp.values.map(escapeLiteral).join(', ');
statements.push(
`CREATE TYPE ${escapeIdentifier(schemaName)}.${escapeIdentifier(createOp.enumName)} AS ENUM (${escapedValues});`,
);
}
const columnDefinitions = flatFieldMetadatas.flatMap((flatFieldMetadata) =>
generateColumnDefinitions({
flatFieldMetadata,
flatObjectMetadata,
workspaceId,
}),
);
if (columnDefinitions.length === 0) continue;
const columnsSql = columnDefinitions
.map(
(columnDefinition) => ` ${buildSqlColumnDefinition(columnDefinition)}`,
)
.join(',\n');
statements.push(
`CREATE TABLE ${escapeIdentifier(schemaName)}.${escapeIdentifier(tableName)} (\n${columnsSql}\n);`,
);
}
return statements;
};
@@ -0,0 +1,11 @@
import { DataSource } from 'typeorm';
export const getCoreEntityMetadatasWithWorkspaceId = (
dataSource: DataSource,
) => {
return dataSource.entityMetadatas.filter((entityMetadata) =>
entityMetadata.columns.some(
(column) => column.propertyName === 'workspaceId',
),
);
};
@@ -0,0 +1,70 @@
import { Logger } from '@nestjs/common';
import { Command, CommandRunner, Option } from 'nest-commander';
import { WorkspaceExportService } from 'src/database/commands/workspace-export/workspace-export.service';
type WorkspaceExportCommandOptions = {
workspaceId: string;
outputPath: string;
tables?: string;
};
@Command({
name: 'workspace:export',
description: 'Export a workspace as SQL INSERT statements',
})
export class WorkspaceExportCommand extends CommandRunner {
private readonly logger = new Logger(WorkspaceExportCommand.name);
constructor(private readonly workspaceExportService: WorkspaceExportService) {
super();
}
@Option({
flags: '--workspace-id <workspaceId>',
description: 'Workspace UUID to export',
required: true,
})
parseWorkspaceId(value: string): string {
return value;
}
@Option({
flags: '--output-path <outputPath>',
description: 'Directory to write the .sql file',
defaultValue: '/tmp/exports',
})
parseOutputPath(value: string): string {
return value;
}
@Option({
flags: '--tables <tables>',
description:
'Comma-separated workspace table names to export (uses nameSingular from ObjectMetadata)',
})
parseTables(value: string): string {
return value;
}
async run(
_passedParams: string[],
options: WorkspaceExportCommandOptions,
): Promise<void> {
const tableFilter = options.tables?.split(',').map((table) => table.trim());
try {
const filePath = await this.workspaceExportService.exportWorkspace({
workspaceId: options.workspaceId,
outputPath: options.outputPath,
tableFilter,
});
this.logger.log(`Export complete: ${filePath}`);
} catch (error) {
this.logger.error('Export failed', error);
throw error;
}
}
}
@@ -0,0 +1,15 @@
import { Module } from '@nestjs/common';
import { TypeOrmModule } from '@nestjs/typeorm';
import { WorkspaceExportCommand } from 'src/database/commands/workspace-export/workspace-export.command';
import { WorkspaceExportService } from 'src/database/commands/workspace-export/workspace-export.service';
import { FieldMetadataEntity } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity';
import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity';
@Module({
imports: [
TypeOrmModule.forFeature([ObjectMetadataEntity, FieldMetadataEntity]),
],
providers: [WorkspaceExportCommand, WorkspaceExportService],
})
export class WorkspaceExportModule {}
@@ -0,0 +1,325 @@
import { Injectable, Logger } from '@nestjs/common';
import { InjectDataSource, InjectRepository } from '@nestjs/typeorm';
import { once } from 'events';
import { type WriteStream, createWriteStream, mkdirSync } from 'fs';
import { finished } from 'stream/promises';
import {
DataSource,
type EntityMetadata,
type QueryRunner,
Repository,
} from 'typeorm';
import { FieldMetadataEntity } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity';
import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity';
import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity';
import { getWorkspaceSchemaName } from 'src/engine/workspace-datasource/utils/get-workspace-schema-name.util';
import { computeTableName } from 'src/engine/utils/compute-table-name.util';
import { escapeIdentifier } from 'src/engine/workspace-manager/workspace-migration/utils/remove-sql-injection.util';
import { getCoreEntityMetadatasWithWorkspaceId } from 'src/database/commands/workspace-export/utils/get-core-entity-metadatas-with-workspace-id.util';
import { generateWorkspaceSchemaDdl } from 'src/database/commands/workspace-export/utils/generate-workspace-schema-ddl.util';
import { buildInsertPrefix } from 'src/database/commands/workspace-export/utils/build-insert-prefix.util';
import { buildWorkspaceTableColumnSets } from 'src/database/commands/workspace-export/utils/build-workspace-table-column-sets.util';
import { formatSqlValue } from 'src/database/commands/workspace-export/utils/format-sql-value.util';
import { generateInsertStatement } from 'src/database/commands/workspace-export/utils/generate-insert-statement.util';
const BATCH_SIZE = 5000;
type WorkspaceExportParams = {
workspaceId: string;
outputPath: string;
tableFilter?: string[];
};
type RowFilter = {
filterColumn: string;
filterValue: string;
};
type WriteRowsOptions = {
schemaName: string;
tableName: string;
displayName: string;
queryRunner: QueryRunner;
stream: WriteStream;
rowFilter?: RowFilter;
jsonColumns?: Set<string>;
excludedColumns?: Set<string>;
};
@Injectable()
export class WorkspaceExportService {
private readonly logger = new Logger(WorkspaceExportService.name);
constructor(
@InjectDataSource()
private readonly dataSource: DataSource,
@InjectRepository(ObjectMetadataEntity)
private readonly objectMetadataRepository: Repository<ObjectMetadataEntity>,
@InjectRepository(FieldMetadataEntity)
private readonly fieldMetadataRepository: Repository<FieldMetadataEntity>,
) {}
async exportWorkspace({
workspaceId,
outputPath,
tableFilter,
}: WorkspaceExportParams): Promise<string> {
const workspace = await this.dataSource
.getRepository(WorkspaceEntity)
.findOne({ where: { id: workspaceId } });
if (!workspace) {
throw new Error(`Workspace ${workspaceId} not found`);
}
const schemaName = getWorkspaceSchemaName(workspaceId);
this.logger.log(`Exporting workspace ${workspaceId} (${schemaName})`);
const objectMetadatas = await this.objectMetadataRepository.find({
where: { workspaceId },
});
const fieldMetadatas = await this.fieldMetadataRepository.find({
where: { workspaceId },
});
const fieldsByObjectId = new Map<string, FieldMetadataEntity[]>();
for (const fieldMetadata of fieldMetadatas) {
const objectFields =
fieldsByObjectId.get(fieldMetadata.objectMetadataId) ?? [];
objectFields.push(fieldMetadata);
fieldsByObjectId.set(fieldMetadata.objectMetadataId, objectFields);
}
mkdirSync(outputPath, { recursive: true });
const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
const filePath = `${outputPath}/${workspaceId}-${timestamp}.sql`;
const stream = createWriteStream(filePath);
const queryRunner = this.dataSource.createQueryRunner();
try {
stream.write("SET session_replication_role = 'replica';\n\n");
await this.writeCoreEntityRows(workspaceId, queryRunner, stream);
stream.write(
`\nCREATE SCHEMA IF NOT EXISTS ${escapeIdentifier(schemaName)};\n\n`,
);
this.writeWorkspaceSchemaDdl(
workspaceId,
schemaName,
objectMetadatas,
fieldsByObjectId,
stream,
);
await this.writeWorkspaceDataRows(
workspaceId,
schemaName,
objectMetadatas,
fieldsByObjectId,
tableFilter,
queryRunner,
stream,
);
stream.write("\nSET session_replication_role = 'origin';\n");
} finally {
await queryRunner.release();
stream.end();
await finished(stream);
}
return filePath;
}
private async writeCoreEntityRows(
workspaceId: string,
queryRunner: QueryRunner,
stream: WriteStream,
): Promise<void> {
const workspaceEntityMetadata = this.dataSource.entityMetadatas.find(
(entityMetadata) => entityMetadata.tableName === 'workspace',
);
if (workspaceEntityMetadata) {
await this.writeRows({
schemaName: workspaceEntityMetadata.schema || 'core',
tableName: workspaceEntityMetadata.tableName,
displayName: workspaceEntityMetadata.tableName,
queryRunner,
stream,
rowFilter: { filterColumn: 'id', filterValue: workspaceId },
jsonColumns: this.buildJsonColumnSet(workspaceEntityMetadata),
});
}
const coreEntityMetadatas = getCoreEntityMetadatasWithWorkspaceId(
this.dataSource,
);
for (const entityMetadata of coreEntityMetadatas) {
try {
await this.writeRows({
schemaName: entityMetadata.schema || 'core',
tableName: entityMetadata.tableName,
displayName: entityMetadata.tableName,
queryRunner,
stream,
rowFilter: {
filterColumn: 'workspaceId',
filterValue: workspaceId,
},
jsonColumns: this.buildJsonColumnSet(entityMetadata),
});
} catch (error) {
this.logger.warn(`${entityMetadata.tableName}: skipped`, error);
}
}
}
private buildJsonColumnSet(entityMetadata: EntityMetadata): Set<string> {
return new Set(
entityMetadata.columns
.filter((column) => column.type === 'jsonb' || column.type === 'json')
.map((column) => column.databaseName),
);
}
private async writeRows({
schemaName,
tableName,
displayName,
queryRunner,
stream,
rowFilter,
jsonColumns,
excludedColumns,
}: WriteRowsOptions): Promise<void> {
const whereClause = rowFilter
? ` WHERE "${rowFilter.filterColumn}" = $1`
: '';
const queryParameters = rowFilter ? [rowFilter.filterValue] : [];
const [{ count: totalCount }] = await queryRunner.query(
`SELECT COUNT(*)::int as count FROM "${schemaName}"."${tableName}"${whereClause}`,
queryParameters,
);
if (totalCount === 0) return;
this.logger.log(` ${displayName}: ${totalCount} rows`);
let insertPrefix: string | undefined;
for (let offset = 0; offset < totalCount; offset += BATCH_SIZE) {
const rows: Record<string, unknown>[] = await queryRunner.query(
`SELECT * FROM "${schemaName}"."${tableName}"${whereClause} ORDER BY "id" LIMIT ${BATCH_SIZE} OFFSET ${offset}`,
queryParameters,
);
const batchStatements: string[] = [];
for (const row of rows) {
const columnNames = Object.keys(row).filter(
(columnName) => !excludedColumns?.has(columnName),
);
if (!insertPrefix) {
insertPrefix = buildInsertPrefix(schemaName, tableName, columnNames);
}
const formattedValues = columnNames.map((columnName) =>
formatSqlValue(row[columnName], jsonColumns?.has(columnName)),
);
batchStatements.push(
generateInsertStatement(insertPrefix, formattedValues),
);
}
if (!stream.write(batchStatements.join(''))) {
await once(stream, 'drain');
}
}
}
private writeWorkspaceSchemaDdl(
workspaceId: string,
schemaName: string,
objectMetadatas: ObjectMetadataEntity[],
fieldsByObjectId: Map<string, FieldMetadataEntity[]>,
stream: WriteStream,
): void {
this.logger.log('Generating workspace schema DDL from metadata...');
const ddlStatements = generateWorkspaceSchemaDdl(
workspaceId,
schemaName,
objectMetadatas,
fieldsByObjectId,
);
this.logger.log(` ${ddlStatements.length} DDL statements`);
for (const statement of ddlStatements) {
stream.write(statement + '\n');
}
stream.write('\n');
}
private async writeWorkspaceDataRows(
workspaceId: string,
schemaName: string,
objectMetadatas: ObjectMetadataEntity[],
fieldsByObjectId: Map<string, FieldMetadataEntity[]>,
tableFilter: string[] | undefined,
queryRunner: QueryRunner,
stream: WriteStream,
): Promise<void> {
for (const objectMetadata of objectMetadatas) {
if (!objectMetadata.isActive) continue;
const tableName = computeTableName(
objectMetadata.nameSingular,
objectMetadata.isCustom,
);
if (tableFilter && !tableFilter.includes(objectMetadata.nameSingular)) {
continue;
}
const objectFieldMetadatas =
fieldsByObjectId.get(objectMetadata.id) ?? [];
const { jsonColumns, generatedColumns } = buildWorkspaceTableColumnSets(
workspaceId,
objectMetadata,
objectFieldMetadatas,
);
try {
await this.writeRows({
schemaName,
tableName,
displayName: objectMetadata.nameSingular,
queryRunner,
stream,
jsonColumns,
excludedColumns: generatedColumns,
});
} catch (error) {
this.logger.warn(`${objectMetadata.nameSingular}: skipped`, error);
}
}
}
}