Fix commands order for v1.17.0 (#17839)

Order should be 1. We delete all the file records (from core.file table) 2. We add the foreign key file / applicationId (pg constraint) 3. We further update the table structure: fullPath is deleted; path is created; unicity constraint between workspaceId/applicationId/path is created (pg constraint) 4. we migrate the workflow steps (this will create files in core.file) 5. we backfill the application package (same) --------- Co-authored-by: Charles Bochet <charles@twenty.com>
2026-02-10 18:47:59 +01:00
parent bb037e13dd
commit bc72879c70
5 changed files with 99 additions and 185 deletions
@@ -6,19 +6,21 @@ import { DataSource, Repository } from 'typeorm';

 import { ActiveOrSuspendedWorkspacesMigrationCommandRunner } from 'src/database/commands/command-runners/active-or-suspended-workspaces-migration.command-runner';
 import { RunOnWorkspaceArgs } from 'src/database/commands/command-runners/workspaces-migration.command-runner';
+import { updateFileTableQueries } from 'src/database/typeorm/core/migrations/utils/1768572831179-updateFileTable.util';
 import { FileEntity } from 'src/engine/core-modules/file/entities/file.entity';
 import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity';
 import { DataSourceService } from 'src/engine/metadata-modules/data-source/data-source.service';
 import { GlobalWorkspaceOrmManager } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager';

@Command({
-  name: 'upgrade:delete-file-records',
-  description:
-    'Delete all file records and add unique constraint on file entity',
+  name: 'upgrade:delete-file-records-and-update-table',
+  description: 'Delete all file records and update file table schema',
 })
-export class DeleteFileRecordsCommand extends ActiveOrSuspendedWorkspacesMigrationCommandRunner {
-  protected readonly logger = new Logger(DeleteFileRecordsCommand.name);
-  private hasAddedConstraint = false;
+export class DeleteFileRecordsAndUpdateTableCommand extends ActiveOrSuspendedWorkspacesMigrationCommandRunner {
+  protected readonly logger = new Logger(
+    DeleteFileRecordsAndUpdateTableCommand.name,
+  );
+  private hasRunOnce = false;

  constructor(
    @InjectRepository(WorkspaceEntity)
@@ -34,8 +36,33 @@ export class DeleteFileRecordsCommand extends ActiveOrSuspendedWorkspacesMigrati
  }

  override async runOnWorkspace(args: RunOnWorkspaceArgs): Promise<void> {
+    if (this.hasRunOnce) {
+      return;
+    }
+
    await this.deleteFileRecords(args);
+    await this.updateFileTable();
    await this.addFileEntityUniqueConstraint(args);
+    this.hasRunOnce = true;
+  }
+
+  private async updateFileTable(): Promise<void> {
+    const queryRunner = this.coreDataSource.createQueryRunner();
+
+    await queryRunner.connect();
+    await queryRunner.startTransaction();
+
+    try {
+      await updateFileTableQueries(queryRunner);
+
+      await queryRunner.commitTransaction();
+      this.logger.log('Successfully updated file table');
+    } catch (error) {
+      await queryRunner.rollbackTransaction();
+      this.logger.error(`Rolling back updateFileTable: ${error.message}`);
+    } finally {
+      await queryRunner.release();
+    }
  }

  private async deleteFileRecords({
@@ -50,7 +77,6 @@ export class DeleteFileRecordsCommand extends ActiveOrSuspendedWorkspacesMigrati

    const files = await this.fileRepository.find({
      select: ['id'],
-      where: { workspaceId },
      withDeleted: true,
    });

@@ -88,10 +114,6 @@ export class DeleteFileRecordsCommand extends ActiveOrSuspendedWorkspacesMigrati
  private async addFileEntityUniqueConstraint({
    options,
  }: RunOnWorkspaceArgs): Promise<void> {
-    if (this.hasAddedConstraint) {
-      return;
-    }
-
    if (options.dryRun) {
      return;
    }
@@ -108,7 +130,6 @@ export class DeleteFileRecordsCommand extends ActiveOrSuspendedWorkspacesMigrati

      await queryRunner.commitTransaction();
      this.logger.log('Successfully added file entity unique constraint');
-      this.hasAddedConstraint = true;
    } catch (error) {
      await queryRunner.rollbackTransaction();
      this.logger.error(
@@ -1,8 +1,7 @@
 import { Logger } from '@nestjs/common';
 import { InjectRepository } from '@nestjs/typeorm';

-import * as fs from 'fs/promises';
-import { join } from 'path';
+import crypto from 'crypto';

 import { Command } from 'nest-commander';
 import { FileFolder, type Sources } from 'twenty-shared/types';
@@ -26,9 +25,6 @@ import { LogicFunctionEntity } from 'src/engine/metadata-modules/logic-function/
 import { LogicFunctionMetadataService } from 'src/engine/metadata-modules/logic-function/services/logic-function-metadata.service';
 import { GlobalWorkspaceOrmManager } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager';
 import { WorkflowVersionStatus } from 'src/modules/workflow/common/standard-objects/workflow-version.workspace-entity';
-import { LogicFunctionResourceService } from 'src/engine/core-modules/logic-function/logic-function-resource/logic-function-resource.service';
-import { logicFunctionCreateHash } from 'src/engine/metadata-modules/logic-function/utils/logic-function-create-hash.utils';
-import { streamToBuffer } from 'src/utils/stream-to-buffer';

 const OLD_BUILT_FOLDER = 'built-function';
 const OLD_SOURCE_FOLDER = 'serverless-function';
@@ -52,7 +48,6 @@ export class MigrateWorkflowCodeStepsCommand extends ActiveOrSuspendedWorkspaces
    private readonly fileStorageService: FileStorageService,
    private readonly applicationService: ApplicationService,
    private readonly logicFunctionMetadataService: LogicFunctionMetadataService,
-    private readonly logicFunctionResourceService: LogicFunctionResourceService,
  ) {
    super(workspaceRepository, globalWorkspaceOrmManager, dataSourceService);
  }
@@ -187,56 +182,49 @@ export class MigrateWorkflowCodeStepsCommand extends ActiveOrSuspendedWorkspaces
      return null;
    }

+    const newLogicFunctionId = v4();
    const applicationUniversalIdentifier =
      await this.getApplicationUniversalIdentifier(
-        oldLogicFunction.application.id,
+        oldLogicFunction.applicationId,
      );

-    if (!applicationUniversalIdentifier) {
-      this.logger.warn(
-        `Logic function ${serverlessFunctionId} application ${oldLogicFunction.application.id} not found not found in workspace ${workspaceId}, skipping`,
-      );
-
-      return null;
-    }
-
-    const newLogicFunctionId = v4();
-
-    const { tempRoot, checksum } = await this.migrateFilesFromOldPathToTemp({
+    const { builtContent, sourceContent } = await this.readOldFunctionFiles(
      workspaceId,
-      applicationUniversalIdentifier,
      serverlessFunctionId,
      version,
-    });
+    );
+
+    const checksum = crypto
+      .createHash('md5')
+      .update(builtContent)
+      .digest('hex');
+
+    if (isDefined(applicationUniversalIdentifier)) {
+      await this.uploadFunctionFiles(
+        workspaceId,
+        applicationUniversalIdentifier,
+        newLogicFunctionId,
+        { builtContent, sourceContent },
+      );
+    }

    await this.logicFunctionMetadataService.createOne({
      input: {
+        id: newLogicFunctionId,
        name: oldLogicFunction.name,
        description: oldLogicFunction.description ?? undefined,
        timeoutSeconds: oldLogicFunction.timeoutSeconds ?? 300,
        toolInputSchema: oldLogicFunction.toolInputSchema ?? undefined,
        isTool: oldLogicFunction.isTool ?? false,
-        handlerName: 'main',
-        builtHandlerPath: 'src/index.mjs',
-        sourceHandlerPath: 'src/index.ts',
+        sourceHandlerPath: `${NEW_WORKFLOW_RESOURCE_PREFIX}/${newLogicFunctionId}/src/index.ts`,
+        builtHandlerPath: `${NEW_WORKFLOW_RESOURCE_PREFIX}/${newLogicFunctionId}/src/index.mjs`,
+        handlerName: oldLogicFunction.handlerName,
        checksum,
-        id: newLogicFunctionId,
      },
      workspaceId,
      ownerFlatApplication: oldLogicFunction.application,
    });

-    if (isDefined(applicationUniversalIdentifier)) {
-      await this.uploadTempToNewPath(
-        workspaceId,
-        applicationUniversalIdentifier,
-        newLogicFunctionId,
-        tempRoot,
-      );
-    }
-
-    await fs.rm(tempRoot, { recursive: true, force: true });
-
    this.logger.log(
      `Created logic function ${newLogicFunctionId} (from ${serverlessFunctionId}/${version}) and migrated files in workspace ${workspaceId}`,
    );
@@ -244,92 +232,69 @@ export class MigrateWorkflowCodeStepsCommand extends ActiveOrSuspendedWorkspaces
    return newLogicFunctionId;
  }

-  private async migrateFilesFromOldPathToTemp({
-    workspaceId,
-    applicationUniversalIdentifier,
-    serverlessFunctionId,
-    version,
-  }: {
-    workspaceId: string;
-    applicationUniversalIdentifier: string;
-    serverlessFunctionId: string;
-    version: string;
-  }): Promise<{ tempRoot: string; checksum: string }> {
+  private async readOldFunctionFiles(
+    workspaceId: string,
+    serverlessFunctionId: string,
+    version: string,
+  ): Promise<{ builtContent: string; sourceContent: string }> {
    const workspacePrefix = `workspace-${workspaceId}`;
-    const oldPaths = {
-      built: `${workspacePrefix}/${OLD_BUILT_FOLDER}/${serverlessFunctionId}/${version}`,
-      source: `${workspacePrefix}/${OLD_SOURCE_FOLDER}/${serverlessFunctionId}/${version}`,
-    };
-
-    const tempRoot = await fs.mkdtemp(
-      `/tmp/twenty-migrate-code-step-${workspaceId}-${serverlessFunctionId}-${version}-`,
-    );
-    const builtTempDir = join(tempRoot, 'built');
-    const sourceTempDir = join(tempRoot, 'source');
-
-    await fs.mkdir(builtTempDir, { recursive: true });
-    await fs.mkdir(sourceTempDir, { recursive: true });

    const builtSources = await this.fileStorageService.readFolderLegacy(
-      oldPaths.built,
-    );
-
-    const builtContent = (
-      await streamToBuffer(
-        await this.fileStorageService.readFile({
-          fileFolder: FileFolder.BuiltLogicFunction,
-          resourcePath: 'src/index.mjs',
-          workspaceId,
-          applicationUniversalIdentifier,
-        }),
-      )
-    ).toString('utf-8');
-
-    const checksum = logicFunctionCreateHash(builtContent);
-
-    await this.logicFunctionResourceService.writeSourcesToLocalFolder(
-      builtSources as Sources,
-      builtTempDir,
+      `${workspacePrefix}/${OLD_BUILT_FOLDER}/${serverlessFunctionId}/${version}`,
    );

    const sourceSources = await this.fileStorageService.readFolderLegacy(
-      oldPaths.source,
+      `${workspacePrefix}/${OLD_SOURCE_FOLDER}/${serverlessFunctionId}/${version}`,
    );
-    const flattened =
+
+    // Old source layout may nest files under a `src/` key
+    const sourceRoot =
      (sourceSources.src as Sources) ?? (sourceSources as Sources);

-    await this.logicFunctionResourceService.writeSourcesToLocalFolder(
-      flattened,
-      sourceTempDir,
-    );
+    const builtContent = builtSources['index.mjs'] as string;
+    const sourceContent = sourceRoot['index.ts'] as string;

-    return { tempRoot, checksum };
+    if (!isDefined(builtContent) || !isDefined(sourceContent)) {
+      throw new Error(
+        `Missing index.mjs or index.ts for serverless function ${serverlessFunctionId}/${version} in workspace ${workspaceId}`,
+      );
+    }
+
+    return { builtContent, sourceContent };
  }

-  private async uploadTempToNewPath(
+  private async uploadFunctionFiles(
    workspaceId: string,
    applicationUniversalIdentifier: string,
    newLogicFunctionId: string,
-    tempRoot: string,
+    files: { builtContent: string; sourceContent: string },
  ): Promise<void> {
    const resourcePath = `${NEW_WORKFLOW_RESOURCE_PREFIX}/${newLogicFunctionId}/src`;
-    const builtTempDir = join(tempRoot, 'built');
-    const sourceTempDir = join(tempRoot, 'source');

-    await this.fileStorageService.uploadFolder({
+    await this.fileStorageService.writeFile({
      workspaceId,
      applicationUniversalIdentifier,
      fileFolder: FileFolder.BuiltLogicFunction,
-      resourcePath,
-      localPath: builtTempDir,
+      resourcePath: `${resourcePath}/index.mjs`,
+      sourceFile: Buffer.from(files.builtContent),
+      mimeType: 'application/javascript',
+      settings: {
+        isTemporaryFile: false,
+        toDelete: false,
+      },
    });

-    await this.fileStorageService.uploadFolder({
+    await this.fileStorageService.writeFile({
      workspaceId,
      applicationUniversalIdentifier,
      fileFolder: FileFolder.Source,
-      resourcePath,
-      localPath: sourceTempDir,
+      resourcePath: `${resourcePath}/index.ts`,
+      sourceFile: Buffer.from(files.sourceContent),
+      mimeType: 'application/typescript',
+      settings: {
+        isTemporaryFile: false,
+        toDelete: false,
+      },
    });
  }
 }
@@ -1,66 +0,0 @@
-import { InjectDataSource, InjectRepository } from '@nestjs/typeorm';
-
-import { Command } from 'nest-commander';
-import { DataSource, Repository } from 'typeorm';
-
-import { ActiveOrSuspendedWorkspacesMigrationCommandRunner } from 'src/database/commands/command-runners/active-or-suspended-workspaces-migration.command-runner';
-import { RunOnWorkspaceArgs } from 'src/database/commands/command-runners/workspaces-migration.command-runner';
-import { updateFileTableQueries } from 'src/database/typeorm/core/migrations/utils/1768572831179-updateFileTable.util';
-import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity';
-import { DataSourceService } from 'src/engine/metadata-modules/data-source/data-source.service';
-import { GlobalWorkspaceOrmManager } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager';
-
-@Command({
-  name: 'upgrade:1-17:update-file-table-migration',
-  description: 'Update file table schema with applicationId and new columns',
-})
-export class UpdateFileTableMigrationCommand extends ActiveOrSuspendedWorkspacesMigrationCommandRunner {
-  private hasRunOnce = false;
-
-  constructor(
-    @InjectRepository(WorkspaceEntity)
-    protected readonly workspaceRepository: Repository<WorkspaceEntity>,
-    protected readonly twentyORMGlobalManager: GlobalWorkspaceOrmManager,
-    protected readonly dataSourceService: DataSourceService,
-    @InjectDataSource()
-    private readonly coreDataSource: DataSource,
-  ) {
-    super(workspaceRepository, twentyORMGlobalManager, dataSourceService);
-  }
-
-  override async runOnWorkspace({
-    options,
-  }: RunOnWorkspaceArgs): Promise<void> {
-    if (this.hasRunOnce) {
-      this.logger.warn(
-        'Skipping has already been run once UpdateFileTableMigrationCommand',
-      );
-
-      return;
-    }
-
-    if (options.dryRun) {
-      return;
-    }
-
-    const queryRunner = this.coreDataSource.createQueryRunner();
-
-    await queryRunner.connect();
-    await queryRunner.startTransaction();
-
-    try {
-      await updateFileTableQueries(queryRunner);
-
-      await queryRunner.commitTransaction();
-      this.logger.log('Successfully run UpdateFileTableMigrationCommand');
-      this.hasRunOnce = true;
-    } catch (error) {
-      await queryRunner.rollbackTransaction();
-      this.logger.error(
-        `Rolling back UpdateFileTableMigrationCommand: ${error.message}`,
-      );
-    } finally {
-      await queryRunner.release();
-    }
-  }
-}
@@ -2,7 +2,7 @@ import { Module } from '@nestjs/common';
 import { TypeOrmModule } from '@nestjs/typeorm';

 import { BackfillApplicationPackageFilesCommand } from 'src/database/commands/upgrade-version-command/1-17/1-17-backfill-application-package-files.command';
-import { DeleteFileRecordsCommand } from 'src/database/commands/upgrade-version-command/1-17/1-17-delete-all-files.command';
+import { DeleteFileRecordsAndUpdateTableCommand } from 'src/database/commands/upgrade-version-command/1-17/1-17-delete-all-files-and-update-table.command';
 import { FixMorphRelationFieldNamesCommand } from 'src/database/commands/upgrade-version-command/1-17/1-17-fix-morph-relation-field-names.command';
 import { IdentifyWebhookMetadataCommand } from 'src/database/commands/upgrade-version-command/1-17/1-17-identify-webhook-metadata.command';
 import { MakeWebhookUniversalIdentifierAndApplicationIdNotNullableMigrationCommand } from 'src/database/commands/upgrade-version-command/1-17/1-17-make-webhook-universal-identifier-and-application-id-not-nullable-migration.command';
@@ -13,7 +13,6 @@ import { MigrateSendEmailRecipientsCommand } from 'src/database/commands/upgrade
 import { MigrateTaskTargetToMorphRelationsCommand } from 'src/database/commands/upgrade-version-command/1-17/1-17-migrate-task-target-to-morph-relations.command';
 import { MigrateWorkflowCodeStepsCommand } from 'src/database/commands/upgrade-version-command/1-17/1-17-migrate-workflow-code-steps.command';
 import { SeedWorkflowV1_16Command } from 'src/database/commands/upgrade-version-command/1-17/1-17-seed-workflow-v1-16.command';
-import { UpdateFileTableMigrationCommand } from 'src/database/commands/upgrade-version-command/1-17/1-17-update-file-table-migration.command';
 import { ApplicationModule } from 'src/engine/core-modules/application/application.module';
 import { FeatureFlagEntity } from 'src/engine/core-modules/feature-flag/feature-flag.entity';
 import { FeatureFlagModule } from 'src/engine/core-modules/feature-flag/feature-flag.module';
@@ -76,12 +75,11 @@ import { TaskTargetWorkspaceEntity } from 'src/modules/task/standard-objects/tas
    MigrateTaskTargetToMorphRelationsCommand,
    IdentifyWebhookMetadataCommand,
    MakeWebhookUniversalIdentifierAndApplicationIdNotNullableMigrationCommand,
-    DeleteFileRecordsCommand,
+    DeleteFileRecordsAndUpdateTableCommand,
    MigrateSendEmailRecipientsCommand,
    MigrateWorkflowCodeStepsCommand,
    SeedWorkflowV1_16Command,
    BackfillApplicationPackageFilesCommand,
-    UpdateFileTableMigrationCommand,
  ],
  exports: [
    FixMorphRelationFieldNamesCommand,
@@ -91,12 +89,11 @@ import { TaskTargetWorkspaceEntity } from 'src/modules/task/standard-objects/tas
    MigrateTaskTargetToMorphRelationsCommand,
    IdentifyWebhookMetadataCommand,
    MakeWebhookUniversalIdentifierAndApplicationIdNotNullableMigrationCommand,
-    DeleteFileRecordsCommand,
    MigrateSendEmailRecipientsCommand,
+    DeleteFileRecordsAndUpdateTableCommand,
    MigrateWorkflowCodeStepsCommand,
    SeedWorkflowV1_16Command,
    BackfillApplicationPackageFilesCommand,
-    UpdateFileTableMigrationCommand,
  ],
 })
 export class V1_17_UpgradeVersionCommandModule {}
@@ -10,7 +10,7 @@ import {
  type VersionCommands,
 } from 'src/database/commands/command-runners/upgrade.command-runner';
 import { BackfillApplicationPackageFilesCommand } from 'src/database/commands/upgrade-version-command/1-17/1-17-backfill-application-package-files.command';
-import { DeleteFileRecordsCommand } from 'src/database/commands/upgrade-version-command/1-17/1-17-delete-all-files.command';
+import { DeleteFileRecordsAndUpdateTableCommand } from 'src/database/commands/upgrade-version-command/1-17/1-17-delete-all-files-and-update-table.command';
 import { FixMorphRelationFieldNamesCommand } from 'src/database/commands/upgrade-version-command/1-17/1-17-fix-morph-relation-field-names.command';
 import { IdentifyWebhookMetadataCommand } from 'src/database/commands/upgrade-version-command/1-17/1-17-identify-webhook-metadata.command';
 import { MakeWebhookUniversalIdentifierAndApplicationIdNotNullableMigrationCommand } from 'src/database/commands/upgrade-version-command/1-17/1-17-make-webhook-universal-identifier-and-application-id-not-nullable-migration.command';
@@ -19,7 +19,6 @@ import { MigrateFavoritesToNavigationMenuItemsCommand } from 'src/database/comma
 import { MigrateNoteTargetToMorphRelationsCommand } from 'src/database/commands/upgrade-version-command/1-17/1-17-migrate-note-target-to-morph-relations.command';
 import { MigrateTaskTargetToMorphRelationsCommand } from 'src/database/commands/upgrade-version-command/1-17/1-17-migrate-task-target-to-morph-relations.command';
 import { MigrateWorkflowCodeStepsCommand } from 'src/database/commands/upgrade-version-command/1-17/1-17-migrate-workflow-code-steps.command';
-import { UpdateFileTableMigrationCommand } from 'src/database/commands/upgrade-version-command/1-17/1-17-update-file-table-migration.command';
 import { TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service';
 import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity';
 import { DataSourceService } from 'src/engine/metadata-modules/data-source/data-source.service';
@@ -41,7 +40,7 @@ export class UpgradeCommand extends UpgradeCommandRunner {

    // 1.17 Commands
    protected readonly backfillApplicationPackageFilesCommand: BackfillApplicationPackageFilesCommand,
-    protected readonly deleteFileRecordsCommand: DeleteFileRecordsCommand,
+    protected readonly deleteFileRecordsAndUpdateTableCommand: DeleteFileRecordsAndUpdateTableCommand,
    protected readonly migrateAttachmentToMorphRelationsCommand: MigrateAttachmentToMorphRelationsCommand,
    protected readonly migrateFavoritesToNavigationMenuItemsCommand: MigrateFavoritesToNavigationMenuItemsCommand,
    protected readonly migrateNoteTargetToMorphRelationsCommand: MigrateNoteTargetToMorphRelationsCommand,
@@ -49,7 +48,6 @@ export class UpgradeCommand extends UpgradeCommandRunner {
    protected readonly identifyWebhookMetadataCommand: IdentifyWebhookMetadataCommand,
    protected readonly makeWebhookUniversalIdentifierAndApplicationIdNotNullableMigrationCommand: MakeWebhookUniversalIdentifierAndApplicationIdNotNullableMigrationCommand,
    protected readonly migrateWorkflowCodeStepsCommand: MigrateWorkflowCodeStepsCommand,
-    protected readonly updateFileTableMigrationCommand: UpdateFileTableMigrationCommand,
    protected readonly fixMorphRelationFieldNamesCommand: FixMorphRelationFieldNamesCommand,
  ) {
    super(
@@ -70,10 +68,9 @@ export class UpgradeCommand extends UpgradeCommandRunner {
      this.identifyWebhookMetadataCommand,
      this
        .makeWebhookUniversalIdentifierAndApplicationIdNotNullableMigrationCommand,
+      this.deleteFileRecordsAndUpdateTableCommand,
      this.migrateWorkflowCodeStepsCommand,
-      this.deleteFileRecordsCommand,
      this.backfillApplicationPackageFilesCommand,
-      this.updateFileTableMigrationCommand,
      this.fixMorphRelationFieldNamesCommand,
    ];