diff --git a/packages/twenty-server/src/database/commands/database-command.module.ts b/packages/twenty-server/src/database/commands/database-command.module.ts index b06ab6db82..f2dd7cc859 100644 --- a/packages/twenty-server/src/database/commands/database-command.module.ts +++ b/packages/twenty-server/src/database/commands/database-command.module.ts @@ -5,6 +5,7 @@ import { CronRegisterAllCommand } from 'src/database/commands/cron-register-all. import { DataSeedWorkspaceCommand } from 'src/database/commands/data-seed-dev-workspace.command'; import { ListOrphanedWorkspaceEntitiesCommand } from 'src/database/commands/list-and-delete-orphaned-workspace-entities.command'; import { ConfirmationQuestion } from 'src/database/commands/questions/confirmation.question'; +import { WorkspaceExportModule } from 'src/database/commands/workspace-export/workspace-export.module'; import { UpgradeVersionCommandModule } from 'src/database/commands/upgrade-version-command/upgrade-version-command.module'; import { TypeORMModule } from 'src/database/typeorm/typeorm.module'; import { ApiKeyModule } from 'src/engine/core-modules/api-key/api-key.module'; @@ -39,6 +40,7 @@ import { AutomatedTriggerModule } from 'src/modules/workflow/workflow-trigger/au imports: [ UpgradeVersionCommandModule, TypeOrmModule.forFeature([WorkspaceEntity]), + WorkspaceExportModule, // Cron command dependencies MessagingImportManagerModule, CalendarEventImportManagerModule, diff --git a/packages/twenty-server/src/database/commands/workspace-export/utils/__tests__/format-sql-value.util.spec.ts b/packages/twenty-server/src/database/commands/workspace-export/utils/__tests__/format-sql-value.util.spec.ts new file mode 100644 index 0000000000..51bfa6db94 --- /dev/null +++ b/packages/twenty-server/src/database/commands/workspace-export/utils/__tests__/format-sql-value.util.spec.ts @@ -0,0 +1,84 @@ +import { formatSqlValue } from 'src/database/commands/workspace-export/utils/format-sql-value.util'; + +describe('formatSqlValue', () => { + it('should return NULL for null and undefined', () => { + expect(formatSqlValue(null)).toBe('NULL'); + expect(formatSqlValue(undefined)).toBe('NULL'); + }); + + it('should return unquoted TRUE/FALSE for booleans', () => { + expect(formatSqlValue(true)).toBe('TRUE'); + expect(formatSqlValue(false)).toBe('FALSE'); + }); + + it('should return unquoted numbers', () => { + expect(formatSqlValue(42)).toBe('42'); + expect(formatSqlValue(3.14)).toBe('3.14'); + expect(formatSqlValue(-1)).toBe('-1'); + expect(formatSqlValue(0)).toBe('0'); + }); + + it('should return NULL for NaN and Infinity', () => { + expect(formatSqlValue(NaN)).toBe('NULL'); + expect(formatSqlValue(Infinity)).toBe('NULL'); + expect(formatSqlValue(-Infinity)).toBe('NULL'); + }); + + it('should return unquoted bigint', () => { + expect(formatSqlValue(BigInt(9007199254740991))).toBe('9007199254740991'); + }); + + it('should escape single quotes in strings', () => { + expect(formatSqlValue("it's")).toBe("'it''s'"); + }); + + it('should handle backslashes with E-string prefix', () => { + expect(formatSqlValue('path\\to\\file')).toBe("E'path\\\\to\\\\file'"); + }); + + it('should format dates as escaped ISO strings', () => { + const date = new Date('2024-01-15T10:30:00.000Z'); + + expect(formatSqlValue(date)).toBe("'2024-01-15T10:30:00.000Z'"); + }); + + it('should JSON-serialize objects when isJsonColumn is true', () => { + const value = { key: 'value' }; + + expect(formatSqlValue(value, true)).toBe('\'{"key":"value"}\''); + }); + + it('should JSON-serialize plain objects even when isJsonColumn is false', () => { + const value = { key: 'value' }; + + expect(formatSqlValue(value, false)).toBe('\'{"key":"value"}\''); + }); + + it('should return empty PostgreSQL array literal for empty arrays', () => { + expect(formatSqlValue([])).toBe("'{}'"); + }); + + it('should format string arrays as PostgreSQL array literals', () => { + expect(formatSqlValue(['a', 'b', 'c'])).toBe('\'{"a","b","c"}\''); + }); + + it('should escape single quotes in array elements', () => { + expect(formatSqlValue(["O'Reilly"])).toBe("'{\"O''Reilly\"}'"); + }); + + it('should format arrays with null elements as PostgreSQL array literals', () => { + expect(formatSqlValue([null, 'foo', 'bar'])).toBe('\'{NULL,"foo","bar"}\''); + }); + + it('should JSON-serialize arrays of objects', () => { + const value = [{ id: 1 }, { id: 2 }]; + + expect(formatSqlValue(value)).toBe('\'[{"id":1},{"id":2}]\''); + }); + + it('should throw on strings containing null bytes', () => { + expect(() => formatSqlValue('hello\0world')).toThrow( + 'Null bytes are not allowed', + ); + }); +}); diff --git a/packages/twenty-server/src/database/commands/workspace-export/utils/build-insert-prefix.util.ts b/packages/twenty-server/src/database/commands/workspace-export/utils/build-insert-prefix.util.ts new file mode 100644 index 0000000000..60f6f9000f --- /dev/null +++ b/packages/twenty-server/src/database/commands/workspace-export/utils/build-insert-prefix.util.ts @@ -0,0 +1,11 @@ +import { escapeIdentifier } from 'src/engine/workspace-manager/workspace-migration/utils/remove-sql-injection.util'; + +export const buildInsertPrefix = ( + schemaName: string, + tableName: string, + columnNames: string[], +): string => { + const escapedColumnNames = columnNames.map(escapeIdentifier).join(', '); + + return `INSERT INTO ${escapeIdentifier(schemaName)}.${escapeIdentifier(tableName)} (${escapedColumnNames}) VALUES `; +}; diff --git a/packages/twenty-server/src/database/commands/workspace-export/utils/build-workspace-table-column-sets.util.ts b/packages/twenty-server/src/database/commands/workspace-export/utils/build-workspace-table-column-sets.util.ts new file mode 100644 index 0000000000..0fca048655 --- /dev/null +++ b/packages/twenty-server/src/database/commands/workspace-export/utils/build-workspace-table-column-sets.util.ts @@ -0,0 +1,45 @@ +import { type FieldMetadataEntity } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity'; +import { type FlatFieldMetadata } from 'src/engine/metadata-modules/flat-field-metadata/types/flat-field-metadata.type'; +import { type FlatObjectMetadata } from 'src/engine/metadata-modules/flat-object-metadata/types/flat-object-metadata.type'; +import { type ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity'; +import { generateColumnDefinitions } from 'src/engine/workspace-manager/workspace-migration/workspace-migration-runner/utils/generate-column-definitions.util'; + +const JSON_COLUMN_TYPES = new Set(['json', 'jsonb']); + +type WorkspaceTableColumnSets = { + jsonColumns: Set; + generatedColumns: Set; +}; + +export const buildWorkspaceTableColumnSets = ( + workspaceId: string, + objectMetadata: ObjectMetadataEntity, + fieldMetadatas: FieldMetadataEntity[], +): WorkspaceTableColumnSets => { + const jsonColumns = new Set(); + const generatedColumns = new Set(); + + const flatObjectMetadata = objectMetadata as unknown as FlatObjectMetadata; + + for (const fieldMetadata of fieldMetadatas) { + const flatFieldMetadata = fieldMetadata as unknown as FlatFieldMetadata; + + const columnDefinitions = generateColumnDefinitions({ + flatFieldMetadata, + flatObjectMetadata, + workspaceId, + }); + + for (const columnDefinition of columnDefinitions) { + if (JSON_COLUMN_TYPES.has(columnDefinition.type)) { + jsonColumns.add(columnDefinition.name); + } + + if (columnDefinition.type === 'tsvector') { + generatedColumns.add(columnDefinition.name); + } + } + } + + return { jsonColumns, generatedColumns }; +}; diff --git a/packages/twenty-server/src/database/commands/workspace-export/utils/format-sql-value.util.ts b/packages/twenty-server/src/database/commands/workspace-export/utils/format-sql-value.util.ts new file mode 100644 index 0000000000..0bacfff26e --- /dev/null +++ b/packages/twenty-server/src/database/commands/workspace-export/utils/format-sql-value.util.ts @@ -0,0 +1,55 @@ +import { isDefined } from 'twenty-shared/utils'; + +import { escapeLiteral } from 'src/engine/workspace-manager/workspace-migration/utils/remove-sql-injection.util'; + +export const formatSqlValue = ( + value: unknown, + isJsonColumn = false, +): string => { + if (!isDefined(value)) return 'NULL'; + + if (isJsonColumn) { + return escapeLiteral(JSON.stringify(value)); + } + + if (typeof value === 'boolean') return value ? 'TRUE' : 'FALSE'; + + if (typeof value === 'number') { + if (!Number.isFinite(value)) return 'NULL'; + + return String(value); + } + + if (typeof value === 'bigint') return String(value); + + if (value instanceof Date) return escapeLiteral(value.toISOString()); + + if (Array.isArray(value)) { + if (value.length === 0) return "'{}'"; + + if (isDefined(value[0]) && typeof value[0] === 'object') { + return escapeLiteral(JSON.stringify(value)); + } + + const formattedElements = value.map((element) => { + if (!isDefined(element)) return 'NULL'; + + const stringElement = String(element); + const escapedElement = stringElement + .replace(/\\/g, '\\\\') + .replace(/"/g, '\\"'); + + return `"${escapedElement}"`; + }); + + const arrayLiteral = `{${formattedElements.join(',')}}`; + + return `'${arrayLiteral.replace(/'/g, "''")}'`; + } + + if (typeof value === 'object') { + return escapeLiteral(JSON.stringify(value)); + } + + return escapeLiteral(String(value)); +}; diff --git a/packages/twenty-server/src/database/commands/workspace-export/utils/generate-insert-statement.util.ts b/packages/twenty-server/src/database/commands/workspace-export/utils/generate-insert-statement.util.ts new file mode 100644 index 0000000000..90fe19068c --- /dev/null +++ b/packages/twenty-server/src/database/commands/workspace-export/utils/generate-insert-statement.util.ts @@ -0,0 +1,4 @@ +export const generateInsertStatement = ( + insertPrefix: string, + formattedValues: string[], +): string => `${insertPrefix}(${formattedValues.join(', ')});\n`; diff --git a/packages/twenty-server/src/database/commands/workspace-export/utils/generate-workspace-schema-ddl.util.ts b/packages/twenty-server/src/database/commands/workspace-export/utils/generate-workspace-schema-ddl.util.ts new file mode 100644 index 0000000000..e989af8a2a --- /dev/null +++ b/packages/twenty-server/src/database/commands/workspace-export/utils/generate-workspace-schema-ddl.util.ts @@ -0,0 +1,75 @@ +import { type FieldMetadataEntity } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity'; +import { type FlatFieldMetadata } from 'src/engine/metadata-modules/flat-field-metadata/types/flat-field-metadata.type'; +import { type FlatObjectMetadata } from 'src/engine/metadata-modules/flat-object-metadata/types/flat-object-metadata.type'; +import { type ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity'; +import { buildSqlColumnDefinition } from 'src/engine/twenty-orm/workspace-schema-manager/utils/build-sql-column-definition.util'; +import { computeTableName } from 'src/engine/utils/compute-table-name.util'; +import { + escapeIdentifier, + escapeLiteral, +} from 'src/engine/workspace-manager/workspace-migration/utils/remove-sql-injection.util'; +import { generateColumnDefinitions } from 'src/engine/workspace-manager/workspace-migration/workspace-migration-runner/utils/generate-column-definitions.util'; +import { + type CreateEnumOperationSpec, + EnumOperation, + collectEnumOperationsForObject, +} from 'src/engine/workspace-manager/workspace-migration/workspace-migration-runner/utils/workspace-schema-enum-operations.util'; + +export const generateWorkspaceSchemaDdl = ( + workspaceId: string, + schemaName: string, + objectMetadatas: ObjectMetadataEntity[], + fieldsByObjectId: Map, +): string[] => { + const statements: string[] = []; + + for (const objectMetadata of objectMetadatas) { + if (!objectMetadata.isActive) continue; + + const tableName = computeTableName( + objectMetadata.nameSingular, + objectMetadata.isCustom, + ); + const fieldMetadatas = fieldsByObjectId.get(objectMetadata.id) ?? []; + + const flatFieldMetadatas = fieldMetadatas as unknown as FlatFieldMetadata[]; + const flatObjectMetadata = objectMetadata as unknown as FlatObjectMetadata; + + const enumOperations = collectEnumOperationsForObject({ + tableName, + operation: EnumOperation.CREATE, + flatFieldMetadatas, + }); + + for (const enumOperation of enumOperations) { + const createOp = enumOperation as CreateEnumOperationSpec; + const escapedValues = createOp.values.map(escapeLiteral).join(', '); + + statements.push( + `CREATE TYPE ${escapeIdentifier(schemaName)}.${escapeIdentifier(createOp.enumName)} AS ENUM (${escapedValues});`, + ); + } + + const columnDefinitions = flatFieldMetadatas.flatMap((flatFieldMetadata) => + generateColumnDefinitions({ + flatFieldMetadata, + flatObjectMetadata, + workspaceId, + }), + ); + + if (columnDefinitions.length === 0) continue; + + const columnsSql = columnDefinitions + .map( + (columnDefinition) => ` ${buildSqlColumnDefinition(columnDefinition)}`, + ) + .join(',\n'); + + statements.push( + `CREATE TABLE ${escapeIdentifier(schemaName)}.${escapeIdentifier(tableName)} (\n${columnsSql}\n);`, + ); + } + + return statements; +}; diff --git a/packages/twenty-server/src/database/commands/workspace-export/utils/get-core-entity-metadatas-with-workspace-id.util.ts b/packages/twenty-server/src/database/commands/workspace-export/utils/get-core-entity-metadatas-with-workspace-id.util.ts new file mode 100644 index 0000000000..5ae0e1d35e --- /dev/null +++ b/packages/twenty-server/src/database/commands/workspace-export/utils/get-core-entity-metadatas-with-workspace-id.util.ts @@ -0,0 +1,11 @@ +import { DataSource } from 'typeorm'; + +export const getCoreEntityMetadatasWithWorkspaceId = ( + dataSource: DataSource, +) => { + return dataSource.entityMetadatas.filter((entityMetadata) => + entityMetadata.columns.some( + (column) => column.propertyName === 'workspaceId', + ), + ); +}; diff --git a/packages/twenty-server/src/database/commands/workspace-export/workspace-export.command.ts b/packages/twenty-server/src/database/commands/workspace-export/workspace-export.command.ts new file mode 100644 index 0000000000..d015628771 --- /dev/null +++ b/packages/twenty-server/src/database/commands/workspace-export/workspace-export.command.ts @@ -0,0 +1,70 @@ +import { Logger } from '@nestjs/common'; + +import { Command, CommandRunner, Option } from 'nest-commander'; + +import { WorkspaceExportService } from 'src/database/commands/workspace-export/workspace-export.service'; + +type WorkspaceExportCommandOptions = { + workspaceId: string; + outputPath: string; + tables?: string; +}; + +@Command({ + name: 'workspace:export', + description: 'Export a workspace as SQL INSERT statements', +}) +export class WorkspaceExportCommand extends CommandRunner { + private readonly logger = new Logger(WorkspaceExportCommand.name); + + constructor(private readonly workspaceExportService: WorkspaceExportService) { + super(); + } + + @Option({ + flags: '--workspace-id ', + description: 'Workspace UUID to export', + required: true, + }) + parseWorkspaceId(value: string): string { + return value; + } + + @Option({ + flags: '--output-path ', + description: 'Directory to write the .sql file', + defaultValue: '/tmp/exports', + }) + parseOutputPath(value: string): string { + return value; + } + + @Option({ + flags: '--tables ', + description: + 'Comma-separated workspace table names to export (uses nameSingular from ObjectMetadata)', + }) + parseTables(value: string): string { + return value; + } + + async run( + _passedParams: string[], + options: WorkspaceExportCommandOptions, + ): Promise { + const tableFilter = options.tables?.split(',').map((table) => table.trim()); + + try { + const filePath = await this.workspaceExportService.exportWorkspace({ + workspaceId: options.workspaceId, + outputPath: options.outputPath, + tableFilter, + }); + + this.logger.log(`Export complete: ${filePath}`); + } catch (error) { + this.logger.error('Export failed', error); + throw error; + } + } +} diff --git a/packages/twenty-server/src/database/commands/workspace-export/workspace-export.module.ts b/packages/twenty-server/src/database/commands/workspace-export/workspace-export.module.ts new file mode 100644 index 0000000000..14b41af2b9 --- /dev/null +++ b/packages/twenty-server/src/database/commands/workspace-export/workspace-export.module.ts @@ -0,0 +1,15 @@ +import { Module } from '@nestjs/common'; +import { TypeOrmModule } from '@nestjs/typeorm'; + +import { WorkspaceExportCommand } from 'src/database/commands/workspace-export/workspace-export.command'; +import { WorkspaceExportService } from 'src/database/commands/workspace-export/workspace-export.service'; +import { FieldMetadataEntity } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity'; +import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity'; + +@Module({ + imports: [ + TypeOrmModule.forFeature([ObjectMetadataEntity, FieldMetadataEntity]), + ], + providers: [WorkspaceExportCommand, WorkspaceExportService], +}) +export class WorkspaceExportModule {} diff --git a/packages/twenty-server/src/database/commands/workspace-export/workspace-export.service.ts b/packages/twenty-server/src/database/commands/workspace-export/workspace-export.service.ts new file mode 100644 index 0000000000..5058528c32 --- /dev/null +++ b/packages/twenty-server/src/database/commands/workspace-export/workspace-export.service.ts @@ -0,0 +1,325 @@ +import { Injectable, Logger } from '@nestjs/common'; +import { InjectDataSource, InjectRepository } from '@nestjs/typeorm'; +import { once } from 'events'; +import { type WriteStream, createWriteStream, mkdirSync } from 'fs'; +import { finished } from 'stream/promises'; + +import { + DataSource, + type EntityMetadata, + type QueryRunner, + Repository, +} from 'typeorm'; + +import { FieldMetadataEntity } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity'; +import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity'; +import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity'; +import { getWorkspaceSchemaName } from 'src/engine/workspace-datasource/utils/get-workspace-schema-name.util'; +import { computeTableName } from 'src/engine/utils/compute-table-name.util'; +import { escapeIdentifier } from 'src/engine/workspace-manager/workspace-migration/utils/remove-sql-injection.util'; +import { getCoreEntityMetadatasWithWorkspaceId } from 'src/database/commands/workspace-export/utils/get-core-entity-metadatas-with-workspace-id.util'; +import { generateWorkspaceSchemaDdl } from 'src/database/commands/workspace-export/utils/generate-workspace-schema-ddl.util'; +import { buildInsertPrefix } from 'src/database/commands/workspace-export/utils/build-insert-prefix.util'; +import { buildWorkspaceTableColumnSets } from 'src/database/commands/workspace-export/utils/build-workspace-table-column-sets.util'; +import { formatSqlValue } from 'src/database/commands/workspace-export/utils/format-sql-value.util'; +import { generateInsertStatement } from 'src/database/commands/workspace-export/utils/generate-insert-statement.util'; + +const BATCH_SIZE = 5000; + +type WorkspaceExportParams = { + workspaceId: string; + outputPath: string; + tableFilter?: string[]; +}; + +type RowFilter = { + filterColumn: string; + filterValue: string; +}; + +type WriteRowsOptions = { + schemaName: string; + tableName: string; + displayName: string; + queryRunner: QueryRunner; + stream: WriteStream; + rowFilter?: RowFilter; + jsonColumns?: Set; + excludedColumns?: Set; +}; + +@Injectable() +export class WorkspaceExportService { + private readonly logger = new Logger(WorkspaceExportService.name); + + constructor( + @InjectDataSource() + private readonly dataSource: DataSource, + @InjectRepository(ObjectMetadataEntity) + private readonly objectMetadataRepository: Repository, + @InjectRepository(FieldMetadataEntity) + private readonly fieldMetadataRepository: Repository, + ) {} + + async exportWorkspace({ + workspaceId, + outputPath, + tableFilter, + }: WorkspaceExportParams): Promise { + const workspace = await this.dataSource + .getRepository(WorkspaceEntity) + .findOne({ where: { id: workspaceId } }); + + if (!workspace) { + throw new Error(`Workspace ${workspaceId} not found`); + } + + const schemaName = getWorkspaceSchemaName(workspaceId); + + this.logger.log(`Exporting workspace ${workspaceId} (${schemaName})`); + + const objectMetadatas = await this.objectMetadataRepository.find({ + where: { workspaceId }, + }); + + const fieldMetadatas = await this.fieldMetadataRepository.find({ + where: { workspaceId }, + }); + + const fieldsByObjectId = new Map(); + + for (const fieldMetadata of fieldMetadatas) { + const objectFields = + fieldsByObjectId.get(fieldMetadata.objectMetadataId) ?? []; + + objectFields.push(fieldMetadata); + fieldsByObjectId.set(fieldMetadata.objectMetadataId, objectFields); + } + + mkdirSync(outputPath, { recursive: true }); + + const timestamp = new Date().toISOString().replace(/[:.]/g, '-'); + const filePath = `${outputPath}/${workspaceId}-${timestamp}.sql`; + const stream = createWriteStream(filePath); + + const queryRunner = this.dataSource.createQueryRunner(); + + try { + stream.write("SET session_replication_role = 'replica';\n\n"); + + await this.writeCoreEntityRows(workspaceId, queryRunner, stream); + + stream.write( + `\nCREATE SCHEMA IF NOT EXISTS ${escapeIdentifier(schemaName)};\n\n`, + ); + + this.writeWorkspaceSchemaDdl( + workspaceId, + schemaName, + objectMetadatas, + fieldsByObjectId, + stream, + ); + + await this.writeWorkspaceDataRows( + workspaceId, + schemaName, + objectMetadatas, + fieldsByObjectId, + tableFilter, + queryRunner, + stream, + ); + + stream.write("\nSET session_replication_role = 'origin';\n"); + } finally { + await queryRunner.release(); + stream.end(); + await finished(stream); + } + + return filePath; + } + + private async writeCoreEntityRows( + workspaceId: string, + queryRunner: QueryRunner, + stream: WriteStream, + ): Promise { + const workspaceEntityMetadata = this.dataSource.entityMetadatas.find( + (entityMetadata) => entityMetadata.tableName === 'workspace', + ); + + if (workspaceEntityMetadata) { + await this.writeRows({ + schemaName: workspaceEntityMetadata.schema || 'core', + tableName: workspaceEntityMetadata.tableName, + displayName: workspaceEntityMetadata.tableName, + queryRunner, + stream, + rowFilter: { filterColumn: 'id', filterValue: workspaceId }, + jsonColumns: this.buildJsonColumnSet(workspaceEntityMetadata), + }); + } + + const coreEntityMetadatas = getCoreEntityMetadatasWithWorkspaceId( + this.dataSource, + ); + + for (const entityMetadata of coreEntityMetadatas) { + try { + await this.writeRows({ + schemaName: entityMetadata.schema || 'core', + tableName: entityMetadata.tableName, + displayName: entityMetadata.tableName, + queryRunner, + stream, + rowFilter: { + filterColumn: 'workspaceId', + filterValue: workspaceId, + }, + jsonColumns: this.buildJsonColumnSet(entityMetadata), + }); + } catch (error) { + this.logger.warn(`${entityMetadata.tableName}: skipped`, error); + } + } + } + + private buildJsonColumnSet(entityMetadata: EntityMetadata): Set { + return new Set( + entityMetadata.columns + .filter((column) => column.type === 'jsonb' || column.type === 'json') + .map((column) => column.databaseName), + ); + } + + private async writeRows({ + schemaName, + tableName, + displayName, + queryRunner, + stream, + rowFilter, + jsonColumns, + excludedColumns, + }: WriteRowsOptions): Promise { + const whereClause = rowFilter + ? ` WHERE "${rowFilter.filterColumn}" = $1` + : ''; + const queryParameters = rowFilter ? [rowFilter.filterValue] : []; + + const [{ count: totalCount }] = await queryRunner.query( + `SELECT COUNT(*)::int as count FROM "${schemaName}"."${tableName}"${whereClause}`, + queryParameters, + ); + + if (totalCount === 0) return; + + this.logger.log(` ${displayName}: ${totalCount} rows`); + + let insertPrefix: string | undefined; + + for (let offset = 0; offset < totalCount; offset += BATCH_SIZE) { + const rows: Record[] = await queryRunner.query( + `SELECT * FROM "${schemaName}"."${tableName}"${whereClause} ORDER BY "id" LIMIT ${BATCH_SIZE} OFFSET ${offset}`, + queryParameters, + ); + + const batchStatements: string[] = []; + + for (const row of rows) { + const columnNames = Object.keys(row).filter( + (columnName) => !excludedColumns?.has(columnName), + ); + + if (!insertPrefix) { + insertPrefix = buildInsertPrefix(schemaName, tableName, columnNames); + } + + const formattedValues = columnNames.map((columnName) => + formatSqlValue(row[columnName], jsonColumns?.has(columnName)), + ); + + batchStatements.push( + generateInsertStatement(insertPrefix, formattedValues), + ); + } + + if (!stream.write(batchStatements.join(''))) { + await once(stream, 'drain'); + } + } + } + + private writeWorkspaceSchemaDdl( + workspaceId: string, + schemaName: string, + objectMetadatas: ObjectMetadataEntity[], + fieldsByObjectId: Map, + stream: WriteStream, + ): void { + this.logger.log('Generating workspace schema DDL from metadata...'); + + const ddlStatements = generateWorkspaceSchemaDdl( + workspaceId, + schemaName, + objectMetadatas, + fieldsByObjectId, + ); + + this.logger.log(` ${ddlStatements.length} DDL statements`); + + for (const statement of ddlStatements) { + stream.write(statement + '\n'); + } + + stream.write('\n'); + } + + private async writeWorkspaceDataRows( + workspaceId: string, + schemaName: string, + objectMetadatas: ObjectMetadataEntity[], + fieldsByObjectId: Map, + tableFilter: string[] | undefined, + queryRunner: QueryRunner, + stream: WriteStream, + ): Promise { + for (const objectMetadata of objectMetadatas) { + if (!objectMetadata.isActive) continue; + + const tableName = computeTableName( + objectMetadata.nameSingular, + objectMetadata.isCustom, + ); + + if (tableFilter && !tableFilter.includes(objectMetadata.nameSingular)) { + continue; + } + + const objectFieldMetadatas = + fieldsByObjectId.get(objectMetadata.id) ?? []; + + const { jsonColumns, generatedColumns } = buildWorkspaceTableColumnSets( + workspaceId, + objectMetadata, + objectFieldMetadatas, + ); + + try { + await this.writeRows({ + schemaName, + tableName, + displayName: objectMetadata.nameSingular, + queryRunner, + stream, + jsonColumns, + excludedColumns: generatedColumns, + }); + } catch (error) { + this.logger.warn(`${objectMetadata.nameSingular}: skipped`, error); + } + } + } +}