Upgrade command remove duplicated role target (#16281)

# Introduction On production facing a migration error with UpdateRoleTargetsUniqueConstraint1764329720503 ```ts Migration "UpdateRoleTargetsUniqueConstraint1764329720503" failed, error: could not create unique index "IDX_ROLE_TARGETS_UNIQUE_AGENT" query: ROLLBACK Error during migration run: QueryFailedError: could not create unique index "IDX_ROLE_TARGETS_UNIQUE_AGENT" at PostgresQueryRunner.query (/Users/paulrastoin/ws/twenty/node_modules/typeorm/driver/postgres/PostgresQueryRunner.js:219:19) at process.processTicksAndRejections (node:internal/process/task_queues:105:5) at async UpdateRoleTargetsUniqueConstraint1764329720503.up (/Users/paulrastoin/ws/twenty/packages/twenty-server/dist/database/typeorm/core/migrations/common/1764329720503-update-role-targets-unique-constraint.js:15:9) at async MigrationExecutor.executePendingMigrations (/Users/paulrastoin/ws/twenty/node_modules/typeorm/migration/MigrationExecutor.js:225:17) at async DataSource.runMigrations (/Users/paulrastoin/ws/twenty/node_modules/typeorm/data-source/DataSource.js:265:35) at async Object.handler (/Users/paulrastoin/ws/twenty/node_modules/typeorm/commands/MigrationRunCommand.js:68:13) { query: 'ALTER TABLE "core"."roleTargets" ADD CONSTRAINT "IDX_ROLE_TARGETS_UNIQUE_AGENT" UNIQUE ("workspaceId", "agentId")', parameters: undefined, driverError: error: could not create unique index "IDX_ROLE_TARGETS_UNIQUE_AGENT" at /Users/paulrastoin/ws/twenty/node_modules/pg/lib/client.js:526:17 at process.processTicksAndRejections (node:internal/process/task_queues:105:5) at async PostgresQueryRunner.query (/Users/paulrastoin/ws/twenty/node_modules/typeorm/driver/postgres/PostgresQueryRunner.js:184:25) at async UpdateRoleTargetsUniqueConstraint1764329720503.up (/Users/paulrastoin/ws/twenty/packages/twenty-server/dist/database/typeorm/core/migrations/common/1764329720503-update-role-targets-unique-constraint.js:15:9) at async MigrationExecutor.executePendingMigrations (/Users/paulrastoin/ws/twenty/node_modules/typeorm/migration/MigrationExecutor.js:225:17) at async DataSource.runMigrations (/Users/paulrastoin/ws/twenty/node_modules/typeorm/data-source/DataSource.js:265:35) at async Object.handler (/Users/paulrastoin/ws/twenty/node_modules/typeorm/commands/MigrationRunCommand.js:68:13) { length: 314, severity: 'ERROR', code: '23505', ``` As several agent has duplicated role target in database ```sql SELECT constraint_name, COUNT(*) AS duplicate_groups, SUM(duplicate_count - 1) AS rows_to_delete FROM ( SELECT 'IDX_ROLE_TARGET_UNIQUE_API_KEY' AS constraint_name, COUNT(*) AS duplicate_count FROM "core"."roleTargets" rt WHERE rt."apiKeyId" IS NOT NULL GROUP BY rt."workspaceId", rt."apiKeyId" HAVING COUNT(*) > 1 UNION ALL SELECT 'IDX_ROLE_TARGET_UNIQUE_AGENT', COUNT(*) FROM "core"."roleTargets" rt WHERE rt."agentId" IS NOT NULL GROUP BY rt."workspaceId", rt."agentId" HAVING COUNT(*) > 1 UNION ALL SELECT 'IDX_ROLE_TARGET_UNIQUE_USER_WORKSPACE', COUNT(*) FROM "core"."roleTargets" rt WHERE rt."userWorkspaceId" IS NOT NULL GROUP BY rt."workspaceId", rt."userWorkspaceId" HAVING COUNT(*) > 1 ) AS all_duplicates GROUP BY constraint_name; ``` with constraint name, duplicated_groups, row_to_delete `IDX_ROLE_TARGET_UNIQUE_AGENT 2507 7229` Please note that only active or suspended workspaces contains duplicated role target ## Fix Introduced an upgrade command that will only keep the latest inserted role target Swallowing migration error on typeorm atomic migration ( still required for self host new instances etc ) ```ts [Nest] 91886 - 12/03/2025, 2:56:28 PM LOG [DeduplicateRoleTargetsCommand] Running command on workspace SOME_WORKSPACE_ID 2587/2587 flatFieldMetadataMaps,flatIndexMaps,flatObjectMetadataMaps out of 298 query: SELECT version(); [Nest] 91886 - 12/03/2025, 2:56:29 PM LOG [DeduplicateRoleTargetsCommand] Command completed! ``` ## Test Tested through an extract in local, tested both the upgrade command and the swallowed migration test
2025-12-03 16:17:46 +01:00
parent 578aa9d5ca
commit 3e75e39650
6 changed files with 272 additions and 26 deletions
@@ -0,0 +1,129 @@
+import { InjectRepository } from '@nestjs/typeorm';
+
+import { Command } from 'nest-commander';
+import { In, type Repository } from 'typeorm';
+
+import { ActiveOrSuspendedWorkspacesMigrationCommandRunner } from 'src/database/commands/command-runners/active-or-suspended-workspaces-migration.command-runner';
+import { RunOnWorkspaceArgs } from 'src/database/commands/command-runners/workspaces-migration.command-runner';
+import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity';
+import { DataSourceService } from 'src/engine/metadata-modules/data-source/data-source.service';
+import { RoleTargetEntity } from 'src/engine/metadata-modules/role-target/role-target.entity';
+import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager';
+
+type DuplicateKey = 'apiKeyId' | 'agentId' | 'userWorkspaceId';
+
+type DuplicateGroup = {
+  foreignKeyId: string;
+  duplicateKey: DuplicateKey;
+};
+
+@Command({
+  name: 'upgrade:1-13:deduplicate-role-targets',
+  description:
+    'Remove duplicate roleTargets keeping only the most recently updated one for each unique constraint',
+})
+export class DeduplicateRoleTargetsCommand extends ActiveOrSuspendedWorkspacesMigrationCommandRunner {
+  constructor(
+    @InjectRepository(RoleTargetEntity)
+    private readonly roleTargetRepository: Repository<RoleTargetEntity>,
+    @InjectRepository(WorkspaceEntity)
+    protected readonly workspaceRepository: Repository<WorkspaceEntity>,
+    protected readonly twentyORMGlobalManager: TwentyORMGlobalManager,
+    protected readonly dataSourceService: DataSourceService,
+  ) {
+    super(workspaceRepository, twentyORMGlobalManager, dataSourceService);
+  }
+
+  override async runOnWorkspace({
+    options,
+    workspaceId,
+  }: RunOnWorkspaceArgs): Promise<void> {
+    const isDryRun = options.dryRun || false;
+
+    const duplicateGroups =
+      await this.findDuplicateGroupsForWorkspace(workspaceId);
+
+    if (duplicateGroups.length === 0) {
+      return;
+    }
+
+    const idsToDelete: string[] = [];
+
+    for (const duplicateGroup of duplicateGroups) {
+      const duplicateIds = await this.findDuplicateIdsToDelete(
+        workspaceId,
+        duplicateGroup,
+      );
+
+      idsToDelete.push(...duplicateIds);
+    }
+
+    if (idsToDelete.length === 0) {
+      return;
+    }
+
+    if (isDryRun) {
+      this.logger.log(
+        `DRY RUN: Would delete ${idsToDelete.length} duplicate roleTarget(s)`,
+      );
+
+      return;
+    }
+
+    await this.roleTargetRepository.delete({ id: In(idsToDelete) });
+
+    this.logger.log(`Deleted ${idsToDelete.length} duplicate roleTarget(s)`);
+  }
+
+  private async findDuplicateGroupsForWorkspace(
+    workspaceId: string,
+  ): Promise<DuplicateGroup[]> {
+    const duplicateGroups: DuplicateGroup[] = [];
+
+    const duplicateKeys: DuplicateKey[] = [
+      'apiKeyId',
+      'agentId',
+      'userWorkspaceId',
+    ];
+
+    for (const duplicateKey of duplicateKeys) {
+      const duplicates = await this.roleTargetRepository
+        .createQueryBuilder('roleTarget')
+        .select([`roleTarget.${duplicateKey} AS "foreignKeyId"`])
+        .where('roleTarget.workspaceId = :workspaceId', { workspaceId })
+        .andWhere(`roleTarget.${duplicateKey} IS NOT NULL`)
+        .groupBy(`roleTarget.${duplicateKey}`)
+        .having('COUNT(*) > 1')
+        .getRawMany<{ foreignKeyId: string }>();
+
+      for (const duplicate of duplicates) {
+        duplicateGroups.push({
+          foreignKeyId: duplicate.foreignKeyId,
+          duplicateKey,
+        });
+      }
+    }
+
+    return duplicateGroups;
+  }
+
+  private async findDuplicateIdsToDelete(
+    workspaceId: string,
+    duplicateGroup: DuplicateGroup,
+  ): Promise<string[]> {
+    const { foreignKeyId, duplicateKey } = duplicateGroup;
+
+    const roleTargets = await this.roleTargetRepository.find({
+      where: {
+        workspaceId,
+        [duplicateKey]: foreignKeyId,
+      },
+      order: {
+        updatedAt: 'DESC',
+      },
+    });
+
+    // Keep the first one (most recently updated), delete the rest
+    return roleTargets.slice(1).map((roleTarget) => roleTarget.id);
+  }
+}
@@ -0,0 +1,77 @@
+import { InjectDataSource, InjectRepository } from '@nestjs/typeorm';
+
+import { Command } from 'nest-commander';
+import { DataSource, Repository } from 'typeorm';
+
+import { ActiveOrSuspendedWorkspacesMigrationCommandRunner } from 'src/database/commands/command-runners/active-or-suspended-workspaces-migration.command-runner';
+import { RunOnWorkspaceArgs } from 'src/database/commands/command-runners/workspaces-migration.command-runner';
+import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity';
+import { DataSourceService } from 'src/engine/metadata-modules/data-source/data-source.service';
+import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager';
+
+@Command({
+  name: 'upgrade:1-13:update-role-targets-unique-constraint-migration',
+  description:
+    'Update roleTargets unique constraints from combined to separate constraints',
+})
+export class UpdateRoleTargetsUniqueConstraintMigrationCommand extends ActiveOrSuspendedWorkspacesMigrationCommandRunner {
+  private hasRunOnce = false;
+
+  constructor(
+    @InjectRepository(WorkspaceEntity)
+    protected readonly workspaceRepository: Repository<WorkspaceEntity>,
+    protected readonly twentyORMGlobalManager: TwentyORMGlobalManager,
+    protected readonly dataSourceService: DataSourceService,
+    @InjectDataSource()
+    private readonly coreDataSource: DataSource,
+  ) {
+    super(workspaceRepository, twentyORMGlobalManager, dataSourceService);
+  }
+
+  override async runOnWorkspace({
+    options,
+  }: RunOnWorkspaceArgs): Promise<void> {
+    if (this.hasRunOnce) {
+      this.logger.log(
+        'Skipping has already been run once UpdateRoleTargetsUniqueConstraintMigrationCommand',
+      );
+
+      return;
+    }
+
+    const queryRunner = this.coreDataSource.createQueryRunner();
+
+    await queryRunner.connect();
+    await queryRunner.startTransaction();
+
+    if (!options.dryRun) {
+      try {
+        await queryRunner.query(
+          `ALTER TABLE "core"."roleTarget" DROP CONSTRAINT "IDX_ROLE_TARGETS_UNIQUE"`,
+        );
+        await queryRunner.query(
+          `ALTER TABLE "core"."roleTarget" ADD CONSTRAINT "IDX_ROLE_TARGET_UNIQUE_API_KEY" UNIQUE ("workspaceId", "apiKeyId")`,
+        );
+        await queryRunner.query(
+          `ALTER TABLE "core"."roleTarget" ADD CONSTRAINT "IDX_ROLE_TARGET_UNIQUE_AGENT" UNIQUE ("workspaceId", "agentId")`,
+        );
+        await queryRunner.query(
+          `ALTER TABLE "core"."roleTarget" ADD CONSTRAINT "IDX_ROLE_TARGET_UNIQUE_USER_WORKSPACE" UNIQUE ("workspaceId", "userWorkspaceId")`,
+        );
+
+        await queryRunner.commitTransaction();
+        this.logger.log(
+          'Successfully run UpdateRoleTargetsUniqueConstraintMigrationCommand',
+        );
+        this.hasRunOnce = true;
+      } catch (error) {
+        await queryRunner.rollbackTransaction();
+        this.logger.log(
+          `Rollbacking UpdateRoleTargetsUniqueConstraintMigrationCommand: ${error.message}`,
+        );
+      } finally {
+        await queryRunner.release();
+      }
+    }
+  }
+}
@@ -3,10 +3,14 @@ import { TypeOrmModule } from '@nestjs/typeorm';

 import { BackfillViewMainGroupByFieldMetadataIdCommand } from 'src/database/commands/upgrade-version-command/1-13/1-13-backfill-view-main-group-by-field-metadata-id.command';
 import { CleanEmptyStringNullInTextFieldsCommand } from 'src/database/commands/upgrade-version-command/1-13/1-13-clean-empty-string-null-in-text-fields.command';
+import { DeduplicateRoleTargetsCommand } from 'src/database/commands/upgrade-version-command/1-13/1-13-deduplicate-role-targets.command';
+import { UpdateRoleTargetsUniqueConstraintMigrationCommand } from 'src/database/commands/upgrade-version-command/1-13/1-13-update-role-targets-unique-constraint-migration.command';
+import { FeatureFlagEntity } from 'src/engine/core-modules/feature-flag/feature-flag.entity';
 import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity';
 import { DataSourceModule } from 'src/engine/metadata-modules/data-source/data-source.module';
 import { FieldMetadataEntity } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity';
 import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity';
+import { RoleTargetEntity } from 'src/engine/metadata-modules/role-target/role-target.entity';
 import { ViewGroupEntity } from 'src/engine/metadata-modules/view-group/entities/view-group.entity';
 import { ViewEntity } from 'src/engine/metadata-modules/view/entities/view.entity';

@@ -18,16 +22,22 @@ import { ViewEntity } from 'src/engine/metadata-modules/view/entities/view.entit
      FieldMetadataEntity,
      ViewEntity,
      ViewGroupEntity,
+      FeatureFlagEntity,
+      RoleTargetEntity,
    ]),
    DataSourceModule,
  ],
  providers: [
    CleanEmptyStringNullInTextFieldsCommand,
    BackfillViewMainGroupByFieldMetadataIdCommand,
+    DeduplicateRoleTargetsCommand,
+    UpdateRoleTargetsUniqueConstraintMigrationCommand,
  ],
  exports: [
    CleanEmptyStringNullInTextFieldsCommand,
    BackfillViewMainGroupByFieldMetadataIdCommand,
+    DeduplicateRoleTargetsCommand,
+    UpdateRoleTargetsUniqueConstraintMigrationCommand,
  ],
 })
 export class V1_13_UpgradeVersionCommandModule {}
@@ -27,6 +27,8 @@ import { CleanNullEquivalentValuesCommand } from 'src/database/commands/upgrade-
 import { CreateWorkspaceCustomApplicationCommand } from 'src/database/commands/upgrade-version-command/1-12/1-12-create-workspace-custom-application.command';
 import { SetStandardApplicationNotUninstallableCommand } from 'src/database/commands/upgrade-version-command/1-12/1-12-set-standard-application-not-uninstallable.command';
 import { WorkspaceCustomApplicationIdNonNullableCommand } from 'src/database/commands/upgrade-version-command/1-12/1-12-workspace-custom-application-id-non-nullable-migration.command';
+import { DeduplicateRoleTargetsCommand } from 'src/database/commands/upgrade-version-command/1-13/1-13-deduplicate-role-targets.command';
+import { UpdateRoleTargetsUniqueConstraintMigrationCommand } from 'src/database/commands/upgrade-version-command/1-13/1-13-update-role-targets-unique-constraint-migration.command';
 import { FixLabelIdentifierPositionAndVisibilityCommand } from 'src/database/commands/upgrade-version-command/1-6/1-6-fix-label-identifier-position-and-visibility.command';
 import { BackfillWorkflowManualTriggerAvailabilityCommand } from 'src/database/commands/upgrade-version-command/1-7/1-7-backfill-workflow-manual-trigger-availability.command';
 import { DeduplicateUniqueFieldsCommand } from 'src/database/commands/upgrade-version-command/1-8/1-8-deduplicate-unique-fields.command';
@@ -92,6 +94,10 @@ export class UpgradeCommand extends UpgradeCommandRunner {
    protected readonly addMessagesImportScheduledSyncStageCommand: AddMessagesImportScheduledSyncStageCommand,
    protected readonly addCalendarEventsImportScheduledSyncStageCommand: AddCalendarEventsImportScheduledSyncStageCommand,
    protected readonly cleanNullEquivalentValuesCommand: CleanNullEquivalentValuesCommand,
+
+    // 1.13 Commands
+    protected readonly deduplicateRoleTargetsCommand: DeduplicateRoleTargetsCommand,
+    protected readonly updateRoleTargetsUniqueConstraintMigrationCommand: UpdateRoleTargetsUniqueConstraintMigrationCommand,
  ) {
    super(
      workspaceRepository,
@@ -159,6 +165,14 @@ export class UpgradeCommand extends UpgradeCommandRunner {
      afterSyncMetadata: [this.setStandardApplicationNotUninstallableCommand],
    };

+    const commands_1130: VersionCommands = {
+      beforeSyncMetadata: [
+        this.deduplicateRoleTargetsCommand,
+        this.updateRoleTargetsUniqueConstraintMigrationCommand,
+      ],
+      afterSyncMetadata: [],
+    };
+
    this.allCommands = {
      '1.6.0': commands_160,
      '1.7.0': commands_170,
@@ -166,6 +180,7 @@ export class UpgradeCommand extends UpgradeCommandRunner {
      '1.10.0': commands_1100,
      '1.11.0': commands_1110,
      '1.12.0': commands_1120,
+      '1.13.0': commands_1130,
    };
  }

@@ -6,29 +6,55 @@ export class UpdateRoleTargetsUniqueConstraint1764329720503
  name = 'UpdateRoleTargetsUniqueConstraint1764329720503';

  public async up(queryRunner: QueryRunner): Promise<void> {
-    await queryRunner.query(
-      `ALTER TABLE "core"."roleTargets" DROP CONSTRAINT "IDX_ROLE_TARGETS_UNIQUE"`,
-    );
-    await queryRunner.query(
-      `ALTER TABLE "core"."roleTargets" ADD CONSTRAINT "IDX_ROLE_TARGETS_UNIQUE_API_KEY" UNIQUE ("workspaceId", "apiKeyId")`,
-    );
-    await queryRunner.query(
-      `ALTER TABLE "core"."roleTargets" ADD CONSTRAINT "IDX_ROLE_TARGETS_UNIQUE_AGENT" UNIQUE ("workspaceId", "agentId")`,
-    );
-    await queryRunner.query(
-      `ALTER TABLE "core"."roleTargets" ADD CONSTRAINT "IDX_ROLE_TARGETS_UNIQUE_USER_WORKSPACE" UNIQUE ("workspaceId", "userWorkspaceId")`,
-    );
+    const savepointName = 'sp_update_role_targets_unique_constraint';
+
+    try {
+      await queryRunner.query(`SAVEPOINT ${savepointName}`);
+
+      await queryRunner.query(
+        `ALTER TABLE "core"."roleTargets" DROP CONSTRAINT "IDX_ROLE_TARGETS_UNIQUE"`,
+      );
+      await queryRunner.query(
+        `ALTER TABLE "core"."roleTargets" ADD CONSTRAINT "IDX_ROLE_TARGET_UNIQUE_API_KEY" UNIQUE ("workspaceId", "apiKeyId")`,
+      );
+      await queryRunner.query(
+        `ALTER TABLE "core"."roleTargets" ADD CONSTRAINT "IDX_ROLE_TARGET_UNIQUE_AGENT" UNIQUE ("workspaceId", "agentId")`,
+      );
+      await queryRunner.query(
+        `ALTER TABLE "core"."roleTargets" ADD CONSTRAINT "IDX_ROLE_TARGET_UNIQUE_USER_WORKSPACE" UNIQUE ("workspaceId", "userWorkspaceId")`,
+      );
+
+      await queryRunner.query(`RELEASE SAVEPOINT ${savepointName}`);
+    } catch (e) {
+      try {
+        await queryRunner.query(`ROLLBACK TO SAVEPOINT ${savepointName}`);
+        await queryRunner.query(`RELEASE SAVEPOINT ${savepointName}`);
+      } catch (rollbackError) {
+        // eslint-disable-next-line no-console
+        console.error(
+          'Failed to rollback to savepoint in UpdateRoleTargetsUniqueConstraint1764329720503',
+          rollbackError,
+        );
+        throw rollbackError;
+      }
+
+      // eslint-disable-next-line no-console
+      console.error(
+        'Swallowing UpdateRoleTargetsUniqueConstraint1764329720503 error',
+        e,
+      );
+    }
  }

  public async down(queryRunner: QueryRunner): Promise<void> {
    await queryRunner.query(
-      `ALTER TABLE "core"."roleTargets" DROP CONSTRAINT "IDX_ROLE_TARGETS_UNIQUE_USER_WORKSPACE"`,
+      `ALTER TABLE "core"."roleTargets" DROP CONSTRAINT "IDX_ROLE_TARGET_UNIQUE_USER_WORKSPACE"`,
    );
    await queryRunner.query(
-      `ALTER TABLE "core"."roleTargets" DROP CONSTRAINT "IDX_ROLE_TARGETS_UNIQUE_AGENT"`,
+      `ALTER TABLE "core"."roleTargets" DROP CONSTRAINT "IDX_ROLE_TARGET_UNIQUE_AGENT"`,
    );
    await queryRunner.query(
-      `ALTER TABLE "core"."roleTargets" DROP CONSTRAINT "IDX_ROLE_TARGETS_UNIQUE_API_KEY"`,
+      `ALTER TABLE "core"."roleTargets" DROP CONSTRAINT "IDX_ROLE_TARGET_UNIQUE_API_KEY"`,
    );
    await queryRunner.query(
      `ALTER TABLE "core"."roleTargets" ADD CONSTRAINT "IDX_ROLE_TARGETS_UNIQUE" UNIQUE ("workspaceId", "userWorkspaceId", "agentId", "apiKeyId")`,
@@ -9,17 +9,6 @@ export class RenameRoleTargets1764671363647 implements MigrationInterface {
      `ALTER TABLE "core"."roleTargets" RENAME TO "roleTarget"`,
    );

-    // Rename unique constraints
-    await queryRunner.query(
-      `ALTER TABLE "core"."roleTarget" RENAME CONSTRAINT "IDX_ROLE_TARGETS_UNIQUE_USER_WORKSPACE" TO "IDX_ROLE_TARGET_UNIQUE_USER_WORKSPACE"`,
-    );
-    await queryRunner.query(
-      `ALTER TABLE "core"."roleTarget" RENAME CONSTRAINT "IDX_ROLE_TARGETS_UNIQUE_AGENT" TO "IDX_ROLE_TARGET_UNIQUE_AGENT"`,
-    );
-    await queryRunner.query(
-      `ALTER TABLE "core"."roleTarget" RENAME CONSTRAINT "IDX_ROLE_TARGETS_UNIQUE_API_KEY" TO "IDX_ROLE_TARGET_UNIQUE_API_KEY"`,
-    );
-
    // Rename check constraint
    await queryRunner.query(
      `ALTER TABLE "core"."roleTarget" RENAME CONSTRAINT "CHK_role_targets_single_entity" TO "CHK_role_target_single_entity"`,