diff --git a/packages/twenty-server/src/database/commands/upgrade-version-command/1-17/1-17-delete-all-files.command.ts b/packages/twenty-server/src/database/commands/upgrade-version-command/1-17/1-17-delete-all-files-and-update-table.command.ts similarity index 75% rename from packages/twenty-server/src/database/commands/upgrade-version-command/1-17/1-17-delete-all-files.command.ts rename to packages/twenty-server/src/database/commands/upgrade-version-command/1-17/1-17-delete-all-files-and-update-table.command.ts index 9629652eb0..aa5f2c6c05 100644 --- a/packages/twenty-server/src/database/commands/upgrade-version-command/1-17/1-17-delete-all-files.command.ts +++ b/packages/twenty-server/src/database/commands/upgrade-version-command/1-17/1-17-delete-all-files-and-update-table.command.ts @@ -6,19 +6,21 @@ import { DataSource, Repository } from 'typeorm'; import { ActiveOrSuspendedWorkspacesMigrationCommandRunner } from 'src/database/commands/command-runners/active-or-suspended-workspaces-migration.command-runner'; import { RunOnWorkspaceArgs } from 'src/database/commands/command-runners/workspaces-migration.command-runner'; +import { updateFileTableQueries } from 'src/database/typeorm/core/migrations/utils/1768572831179-updateFileTable.util'; import { FileEntity } from 'src/engine/core-modules/file/entities/file.entity'; import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity'; import { DataSourceService } from 'src/engine/metadata-modules/data-source/data-source.service'; import { GlobalWorkspaceOrmManager } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager'; @Command({ - name: 'upgrade:delete-file-records', - description: - 'Delete all file records and add unique constraint on file entity', + name: 'upgrade:delete-file-records-and-update-table', + description: 'Delete all file records and update file table schema', }) -export class DeleteFileRecordsCommand extends ActiveOrSuspendedWorkspacesMigrationCommandRunner { - protected readonly logger = new Logger(DeleteFileRecordsCommand.name); - private hasAddedConstraint = false; +export class DeleteFileRecordsAndUpdateTableCommand extends ActiveOrSuspendedWorkspacesMigrationCommandRunner { + protected readonly logger = new Logger( + DeleteFileRecordsAndUpdateTableCommand.name, + ); + private hasRunOnce = false; constructor( @InjectRepository(WorkspaceEntity) @@ -34,8 +36,33 @@ export class DeleteFileRecordsCommand extends ActiveOrSuspendedWorkspacesMigrati } override async runOnWorkspace(args: RunOnWorkspaceArgs): Promise { + if (this.hasRunOnce) { + return; + } + await this.deleteFileRecords(args); + await this.updateFileTable(); await this.addFileEntityUniqueConstraint(args); + this.hasRunOnce = true; + } + + private async updateFileTable(): Promise { + const queryRunner = this.coreDataSource.createQueryRunner(); + + await queryRunner.connect(); + await queryRunner.startTransaction(); + + try { + await updateFileTableQueries(queryRunner); + + await queryRunner.commitTransaction(); + this.logger.log('Successfully updated file table'); + } catch (error) { + await queryRunner.rollbackTransaction(); + this.logger.error(`Rolling back updateFileTable: ${error.message}`); + } finally { + await queryRunner.release(); + } } private async deleteFileRecords({ @@ -50,7 +77,6 @@ export class DeleteFileRecordsCommand extends ActiveOrSuspendedWorkspacesMigrati const files = await this.fileRepository.find({ select: ['id'], - where: { workspaceId }, withDeleted: true, }); @@ -88,10 +114,6 @@ export class DeleteFileRecordsCommand extends ActiveOrSuspendedWorkspacesMigrati private async addFileEntityUniqueConstraint({ options, }: RunOnWorkspaceArgs): Promise { - if (this.hasAddedConstraint) { - return; - } - if (options.dryRun) { return; } @@ -108,7 +130,6 @@ export class DeleteFileRecordsCommand extends ActiveOrSuspendedWorkspacesMigrati await queryRunner.commitTransaction(); this.logger.log('Successfully added file entity unique constraint'); - this.hasAddedConstraint = true; } catch (error) { await queryRunner.rollbackTransaction(); this.logger.error( diff --git a/packages/twenty-server/src/database/commands/upgrade-version-command/1-17/1-17-migrate-workflow-code-steps.command.ts b/packages/twenty-server/src/database/commands/upgrade-version-command/1-17/1-17-migrate-workflow-code-steps.command.ts index 1385695190..f1eadfda03 100644 --- a/packages/twenty-server/src/database/commands/upgrade-version-command/1-17/1-17-migrate-workflow-code-steps.command.ts +++ b/packages/twenty-server/src/database/commands/upgrade-version-command/1-17/1-17-migrate-workflow-code-steps.command.ts @@ -1,8 +1,7 @@ import { Logger } from '@nestjs/common'; import { InjectRepository } from '@nestjs/typeorm'; -import * as fs from 'fs/promises'; -import { join } from 'path'; +import crypto from 'crypto'; import { Command } from 'nest-commander'; import { FileFolder, type Sources } from 'twenty-shared/types'; @@ -26,9 +25,6 @@ import { LogicFunctionEntity } from 'src/engine/metadata-modules/logic-function/ import { LogicFunctionMetadataService } from 'src/engine/metadata-modules/logic-function/services/logic-function-metadata.service'; import { GlobalWorkspaceOrmManager } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager'; import { WorkflowVersionStatus } from 'src/modules/workflow/common/standard-objects/workflow-version.workspace-entity'; -import { LogicFunctionResourceService } from 'src/engine/core-modules/logic-function/logic-function-resource/logic-function-resource.service'; -import { logicFunctionCreateHash } from 'src/engine/metadata-modules/logic-function/utils/logic-function-create-hash.utils'; -import { streamToBuffer } from 'src/utils/stream-to-buffer'; const OLD_BUILT_FOLDER = 'built-function'; const OLD_SOURCE_FOLDER = 'serverless-function'; @@ -52,7 +48,6 @@ export class MigrateWorkflowCodeStepsCommand extends ActiveOrSuspendedWorkspaces private readonly fileStorageService: FileStorageService, private readonly applicationService: ApplicationService, private readonly logicFunctionMetadataService: LogicFunctionMetadataService, - private readonly logicFunctionResourceService: LogicFunctionResourceService, ) { super(workspaceRepository, globalWorkspaceOrmManager, dataSourceService); } @@ -187,56 +182,49 @@ export class MigrateWorkflowCodeStepsCommand extends ActiveOrSuspendedWorkspaces return null; } + const newLogicFunctionId = v4(); const applicationUniversalIdentifier = await this.getApplicationUniversalIdentifier( - oldLogicFunction.application.id, + oldLogicFunction.applicationId, ); - if (!applicationUniversalIdentifier) { - this.logger.warn( - `Logic function ${serverlessFunctionId} application ${oldLogicFunction.application.id} not found not found in workspace ${workspaceId}, skipping`, - ); - - return null; - } - - const newLogicFunctionId = v4(); - - const { tempRoot, checksum } = await this.migrateFilesFromOldPathToTemp({ + const { builtContent, sourceContent } = await this.readOldFunctionFiles( workspaceId, - applicationUniversalIdentifier, serverlessFunctionId, version, - }); + ); + + const checksum = crypto + .createHash('md5') + .update(builtContent) + .digest('hex'); + + if (isDefined(applicationUniversalIdentifier)) { + await this.uploadFunctionFiles( + workspaceId, + applicationUniversalIdentifier, + newLogicFunctionId, + { builtContent, sourceContent }, + ); + } await this.logicFunctionMetadataService.createOne({ input: { + id: newLogicFunctionId, name: oldLogicFunction.name, description: oldLogicFunction.description ?? undefined, timeoutSeconds: oldLogicFunction.timeoutSeconds ?? 300, toolInputSchema: oldLogicFunction.toolInputSchema ?? undefined, isTool: oldLogicFunction.isTool ?? false, - handlerName: 'main', - builtHandlerPath: 'src/index.mjs', - sourceHandlerPath: 'src/index.ts', + sourceHandlerPath: `${NEW_WORKFLOW_RESOURCE_PREFIX}/${newLogicFunctionId}/src/index.ts`, + builtHandlerPath: `${NEW_WORKFLOW_RESOURCE_PREFIX}/${newLogicFunctionId}/src/index.mjs`, + handlerName: oldLogicFunction.handlerName, checksum, - id: newLogicFunctionId, }, workspaceId, ownerFlatApplication: oldLogicFunction.application, }); - if (isDefined(applicationUniversalIdentifier)) { - await this.uploadTempToNewPath( - workspaceId, - applicationUniversalIdentifier, - newLogicFunctionId, - tempRoot, - ); - } - - await fs.rm(tempRoot, { recursive: true, force: true }); - this.logger.log( `Created logic function ${newLogicFunctionId} (from ${serverlessFunctionId}/${version}) and migrated files in workspace ${workspaceId}`, ); @@ -244,92 +232,69 @@ export class MigrateWorkflowCodeStepsCommand extends ActiveOrSuspendedWorkspaces return newLogicFunctionId; } - private async migrateFilesFromOldPathToTemp({ - workspaceId, - applicationUniversalIdentifier, - serverlessFunctionId, - version, - }: { - workspaceId: string; - applicationUniversalIdentifier: string; - serverlessFunctionId: string; - version: string; - }): Promise<{ tempRoot: string; checksum: string }> { + private async readOldFunctionFiles( + workspaceId: string, + serverlessFunctionId: string, + version: string, + ): Promise<{ builtContent: string; sourceContent: string }> { const workspacePrefix = `workspace-${workspaceId}`; - const oldPaths = { - built: `${workspacePrefix}/${OLD_BUILT_FOLDER}/${serverlessFunctionId}/${version}`, - source: `${workspacePrefix}/${OLD_SOURCE_FOLDER}/${serverlessFunctionId}/${version}`, - }; - - const tempRoot = await fs.mkdtemp( - `/tmp/twenty-migrate-code-step-${workspaceId}-${serverlessFunctionId}-${version}-`, - ); - const builtTempDir = join(tempRoot, 'built'); - const sourceTempDir = join(tempRoot, 'source'); - - await fs.mkdir(builtTempDir, { recursive: true }); - await fs.mkdir(sourceTempDir, { recursive: true }); const builtSources = await this.fileStorageService.readFolderLegacy( - oldPaths.built, - ); - - const builtContent = ( - await streamToBuffer( - await this.fileStorageService.readFile({ - fileFolder: FileFolder.BuiltLogicFunction, - resourcePath: 'src/index.mjs', - workspaceId, - applicationUniversalIdentifier, - }), - ) - ).toString('utf-8'); - - const checksum = logicFunctionCreateHash(builtContent); - - await this.logicFunctionResourceService.writeSourcesToLocalFolder( - builtSources as Sources, - builtTempDir, + `${workspacePrefix}/${OLD_BUILT_FOLDER}/${serverlessFunctionId}/${version}`, ); const sourceSources = await this.fileStorageService.readFolderLegacy( - oldPaths.source, + `${workspacePrefix}/${OLD_SOURCE_FOLDER}/${serverlessFunctionId}/${version}`, ); - const flattened = + + // Old source layout may nest files under a `src/` key + const sourceRoot = (sourceSources.src as Sources) ?? (sourceSources as Sources); - await this.logicFunctionResourceService.writeSourcesToLocalFolder( - flattened, - sourceTempDir, - ); + const builtContent = builtSources['index.mjs'] as string; + const sourceContent = sourceRoot['index.ts'] as string; - return { tempRoot, checksum }; + if (!isDefined(builtContent) || !isDefined(sourceContent)) { + throw new Error( + `Missing index.mjs or index.ts for serverless function ${serverlessFunctionId}/${version} in workspace ${workspaceId}`, + ); + } + + return { builtContent, sourceContent }; } - private async uploadTempToNewPath( + private async uploadFunctionFiles( workspaceId: string, applicationUniversalIdentifier: string, newLogicFunctionId: string, - tempRoot: string, + files: { builtContent: string; sourceContent: string }, ): Promise { const resourcePath = `${NEW_WORKFLOW_RESOURCE_PREFIX}/${newLogicFunctionId}/src`; - const builtTempDir = join(tempRoot, 'built'); - const sourceTempDir = join(tempRoot, 'source'); - await this.fileStorageService.uploadFolder({ + await this.fileStorageService.writeFile({ workspaceId, applicationUniversalIdentifier, fileFolder: FileFolder.BuiltLogicFunction, - resourcePath, - localPath: builtTempDir, + resourcePath: `${resourcePath}/index.mjs`, + sourceFile: Buffer.from(files.builtContent), + mimeType: 'application/javascript', + settings: { + isTemporaryFile: false, + toDelete: false, + }, }); - await this.fileStorageService.uploadFolder({ + await this.fileStorageService.writeFile({ workspaceId, applicationUniversalIdentifier, fileFolder: FileFolder.Source, - resourcePath, - localPath: sourceTempDir, + resourcePath: `${resourcePath}/index.ts`, + sourceFile: Buffer.from(files.sourceContent), + mimeType: 'application/typescript', + settings: { + isTemporaryFile: false, + toDelete: false, + }, }); } } diff --git a/packages/twenty-server/src/database/commands/upgrade-version-command/1-17/1-17-update-file-table-migration.command.ts b/packages/twenty-server/src/database/commands/upgrade-version-command/1-17/1-17-update-file-table-migration.command.ts deleted file mode 100644 index 9b7e9ba601..0000000000 --- a/packages/twenty-server/src/database/commands/upgrade-version-command/1-17/1-17-update-file-table-migration.command.ts +++ /dev/null @@ -1,66 +0,0 @@ -import { InjectDataSource, InjectRepository } from '@nestjs/typeorm'; - -import { Command } from 'nest-commander'; -import { DataSource, Repository } from 'typeorm'; - -import { ActiveOrSuspendedWorkspacesMigrationCommandRunner } from 'src/database/commands/command-runners/active-or-suspended-workspaces-migration.command-runner'; -import { RunOnWorkspaceArgs } from 'src/database/commands/command-runners/workspaces-migration.command-runner'; -import { updateFileTableQueries } from 'src/database/typeorm/core/migrations/utils/1768572831179-updateFileTable.util'; -import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity'; -import { DataSourceService } from 'src/engine/metadata-modules/data-source/data-source.service'; -import { GlobalWorkspaceOrmManager } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager'; - -@Command({ - name: 'upgrade:1-17:update-file-table-migration', - description: 'Update file table schema with applicationId and new columns', -}) -export class UpdateFileTableMigrationCommand extends ActiveOrSuspendedWorkspacesMigrationCommandRunner { - private hasRunOnce = false; - - constructor( - @InjectRepository(WorkspaceEntity) - protected readonly workspaceRepository: Repository, - protected readonly twentyORMGlobalManager: GlobalWorkspaceOrmManager, - protected readonly dataSourceService: DataSourceService, - @InjectDataSource() - private readonly coreDataSource: DataSource, - ) { - super(workspaceRepository, twentyORMGlobalManager, dataSourceService); - } - - override async runOnWorkspace({ - options, - }: RunOnWorkspaceArgs): Promise { - if (this.hasRunOnce) { - this.logger.warn( - 'Skipping has already been run once UpdateFileTableMigrationCommand', - ); - - return; - } - - if (options.dryRun) { - return; - } - - const queryRunner = this.coreDataSource.createQueryRunner(); - - await queryRunner.connect(); - await queryRunner.startTransaction(); - - try { - await updateFileTableQueries(queryRunner); - - await queryRunner.commitTransaction(); - this.logger.log('Successfully run UpdateFileTableMigrationCommand'); - this.hasRunOnce = true; - } catch (error) { - await queryRunner.rollbackTransaction(); - this.logger.error( - `Rolling back UpdateFileTableMigrationCommand: ${error.message}`, - ); - } finally { - await queryRunner.release(); - } - } -} diff --git a/packages/twenty-server/src/database/commands/upgrade-version-command/1-17/1-17-upgrade-version-command.module.ts b/packages/twenty-server/src/database/commands/upgrade-version-command/1-17/1-17-upgrade-version-command.module.ts index 0863df954c..2f3af3c198 100644 --- a/packages/twenty-server/src/database/commands/upgrade-version-command/1-17/1-17-upgrade-version-command.module.ts +++ b/packages/twenty-server/src/database/commands/upgrade-version-command/1-17/1-17-upgrade-version-command.module.ts @@ -2,7 +2,7 @@ import { Module } from '@nestjs/common'; import { TypeOrmModule } from '@nestjs/typeorm'; import { BackfillApplicationPackageFilesCommand } from 'src/database/commands/upgrade-version-command/1-17/1-17-backfill-application-package-files.command'; -import { DeleteFileRecordsCommand } from 'src/database/commands/upgrade-version-command/1-17/1-17-delete-all-files.command'; +import { DeleteFileRecordsAndUpdateTableCommand } from 'src/database/commands/upgrade-version-command/1-17/1-17-delete-all-files-and-update-table.command'; import { FixMorphRelationFieldNamesCommand } from 'src/database/commands/upgrade-version-command/1-17/1-17-fix-morph-relation-field-names.command'; import { IdentifyWebhookMetadataCommand } from 'src/database/commands/upgrade-version-command/1-17/1-17-identify-webhook-metadata.command'; import { MakeWebhookUniversalIdentifierAndApplicationIdNotNullableMigrationCommand } from 'src/database/commands/upgrade-version-command/1-17/1-17-make-webhook-universal-identifier-and-application-id-not-nullable-migration.command'; @@ -13,7 +13,6 @@ import { MigrateSendEmailRecipientsCommand } from 'src/database/commands/upgrade import { MigrateTaskTargetToMorphRelationsCommand } from 'src/database/commands/upgrade-version-command/1-17/1-17-migrate-task-target-to-morph-relations.command'; import { MigrateWorkflowCodeStepsCommand } from 'src/database/commands/upgrade-version-command/1-17/1-17-migrate-workflow-code-steps.command'; import { SeedWorkflowV1_16Command } from 'src/database/commands/upgrade-version-command/1-17/1-17-seed-workflow-v1-16.command'; -import { UpdateFileTableMigrationCommand } from 'src/database/commands/upgrade-version-command/1-17/1-17-update-file-table-migration.command'; import { ApplicationModule } from 'src/engine/core-modules/application/application.module'; import { FeatureFlagEntity } from 'src/engine/core-modules/feature-flag/feature-flag.entity'; import { FeatureFlagModule } from 'src/engine/core-modules/feature-flag/feature-flag.module'; @@ -76,12 +75,11 @@ import { TaskTargetWorkspaceEntity } from 'src/modules/task/standard-objects/tas MigrateTaskTargetToMorphRelationsCommand, IdentifyWebhookMetadataCommand, MakeWebhookUniversalIdentifierAndApplicationIdNotNullableMigrationCommand, - DeleteFileRecordsCommand, + DeleteFileRecordsAndUpdateTableCommand, MigrateSendEmailRecipientsCommand, MigrateWorkflowCodeStepsCommand, SeedWorkflowV1_16Command, BackfillApplicationPackageFilesCommand, - UpdateFileTableMigrationCommand, ], exports: [ FixMorphRelationFieldNamesCommand, @@ -91,12 +89,11 @@ import { TaskTargetWorkspaceEntity } from 'src/modules/task/standard-objects/tas MigrateTaskTargetToMorphRelationsCommand, IdentifyWebhookMetadataCommand, MakeWebhookUniversalIdentifierAndApplicationIdNotNullableMigrationCommand, - DeleteFileRecordsCommand, MigrateSendEmailRecipientsCommand, + DeleteFileRecordsAndUpdateTableCommand, MigrateWorkflowCodeStepsCommand, SeedWorkflowV1_16Command, BackfillApplicationPackageFilesCommand, - UpdateFileTableMigrationCommand, ], }) export class V1_17_UpgradeVersionCommandModule {} diff --git a/packages/twenty-server/src/database/commands/upgrade-version-command/upgrade.command.ts b/packages/twenty-server/src/database/commands/upgrade-version-command/upgrade.command.ts index bbad938852..0aae8cff0a 100644 --- a/packages/twenty-server/src/database/commands/upgrade-version-command/upgrade.command.ts +++ b/packages/twenty-server/src/database/commands/upgrade-version-command/upgrade.command.ts @@ -10,7 +10,7 @@ import { type VersionCommands, } from 'src/database/commands/command-runners/upgrade.command-runner'; import { BackfillApplicationPackageFilesCommand } from 'src/database/commands/upgrade-version-command/1-17/1-17-backfill-application-package-files.command'; -import { DeleteFileRecordsCommand } from 'src/database/commands/upgrade-version-command/1-17/1-17-delete-all-files.command'; +import { DeleteFileRecordsAndUpdateTableCommand } from 'src/database/commands/upgrade-version-command/1-17/1-17-delete-all-files-and-update-table.command'; import { FixMorphRelationFieldNamesCommand } from 'src/database/commands/upgrade-version-command/1-17/1-17-fix-morph-relation-field-names.command'; import { IdentifyWebhookMetadataCommand } from 'src/database/commands/upgrade-version-command/1-17/1-17-identify-webhook-metadata.command'; import { MakeWebhookUniversalIdentifierAndApplicationIdNotNullableMigrationCommand } from 'src/database/commands/upgrade-version-command/1-17/1-17-make-webhook-universal-identifier-and-application-id-not-nullable-migration.command'; @@ -19,7 +19,6 @@ import { MigrateFavoritesToNavigationMenuItemsCommand } from 'src/database/comma import { MigrateNoteTargetToMorphRelationsCommand } from 'src/database/commands/upgrade-version-command/1-17/1-17-migrate-note-target-to-morph-relations.command'; import { MigrateTaskTargetToMorphRelationsCommand } from 'src/database/commands/upgrade-version-command/1-17/1-17-migrate-task-target-to-morph-relations.command'; import { MigrateWorkflowCodeStepsCommand } from 'src/database/commands/upgrade-version-command/1-17/1-17-migrate-workflow-code-steps.command'; -import { UpdateFileTableMigrationCommand } from 'src/database/commands/upgrade-version-command/1-17/1-17-update-file-table-migration.command'; import { TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service'; import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity'; import { DataSourceService } from 'src/engine/metadata-modules/data-source/data-source.service'; @@ -41,7 +40,7 @@ export class UpgradeCommand extends UpgradeCommandRunner { // 1.17 Commands protected readonly backfillApplicationPackageFilesCommand: BackfillApplicationPackageFilesCommand, - protected readonly deleteFileRecordsCommand: DeleteFileRecordsCommand, + protected readonly deleteFileRecordsAndUpdateTableCommand: DeleteFileRecordsAndUpdateTableCommand, protected readonly migrateAttachmentToMorphRelationsCommand: MigrateAttachmentToMorphRelationsCommand, protected readonly migrateFavoritesToNavigationMenuItemsCommand: MigrateFavoritesToNavigationMenuItemsCommand, protected readonly migrateNoteTargetToMorphRelationsCommand: MigrateNoteTargetToMorphRelationsCommand, @@ -49,7 +48,6 @@ export class UpgradeCommand extends UpgradeCommandRunner { protected readonly identifyWebhookMetadataCommand: IdentifyWebhookMetadataCommand, protected readonly makeWebhookUniversalIdentifierAndApplicationIdNotNullableMigrationCommand: MakeWebhookUniversalIdentifierAndApplicationIdNotNullableMigrationCommand, protected readonly migrateWorkflowCodeStepsCommand: MigrateWorkflowCodeStepsCommand, - protected readonly updateFileTableMigrationCommand: UpdateFileTableMigrationCommand, protected readonly fixMorphRelationFieldNamesCommand: FixMorphRelationFieldNamesCommand, ) { super( @@ -70,10 +68,9 @@ export class UpgradeCommand extends UpgradeCommandRunner { this.identifyWebhookMetadataCommand, this .makeWebhookUniversalIdentifierAndApplicationIdNotNullableMigrationCommand, + this.deleteFileRecordsAndUpdateTableCommand, this.migrateWorkflowCodeStepsCommand, - this.deleteFileRecordsCommand, this.backfillApplicationPackageFilesCommand, - this.updateFileTableMigrationCommand, this.fixMorphRelationFieldNamesCommand, ];