From 28e98086b082abb54b3b22811f94c4d0f8057d64 Mon Sep 17 00:00:00 2001 From: Paul Rastoin <45004772+prastoin@users.noreply.github.com> Date: Mon, 19 Jan 2026 16:36:41 +0100 Subject: [PATCH] Identify index (#17239) # Introduction Related to https://github.com/twentyhq/core-team-issues/issues/1989 1/ Migration, applicationId and universalIdentifier are required on entity ( save point migration + upgrade command fallback pattern ) 2/ Backfill using previous standard ids ## Test tested prod extract --- .../1-16-identify-index-metadata.command.ts | 341 ++++++++++++++++++ ...ation-id-not-nullable-migration.command.ts | 71 ++++ .../1-16-upgrade-version-command.module.ts | 8 + .../standard-index-field-names.constant.ts | 295 +++++++++++++++ .../upgrade.command.ts | 7 + ...alIdentifierAndApplicationIdNotNullable.ts | 64 ++++ ...ntifierAndApplicationIdNotNullable.util.ts | 23 ++ .../index-metadata/index-metadata.entity.ts | 4 +- 8 files changed, 811 insertions(+), 2 deletions(-) create mode 100644 packages/twenty-server/src/database/commands/upgrade-version-command/1-16/1-16-identify-index-metadata.command.ts create mode 100644 packages/twenty-server/src/database/commands/upgrade-version-command/1-16/1-16-make-index-metadata-universal-identifier-and-application-id-not-nullable-migration.command.ts create mode 100644 packages/twenty-server/src/database/commands/upgrade-version-command/1-16/constants/standard-index-field-names.constant.ts create mode 100644 packages/twenty-server/src/database/typeorm/core/migrations/common/1768830235328-makeIndexMetadataUniversalIdentifierAndApplicationIdNotNullable.ts create mode 100644 packages/twenty-server/src/database/typeorm/core/migrations/utils/1768830235328-makeIndexMetadataUniversalIdentifierAndApplicationIdNotNullable.util.ts diff --git a/packages/twenty-server/src/database/commands/upgrade-version-command/1-16/1-16-identify-index-metadata.command.ts b/packages/twenty-server/src/database/commands/upgrade-version-command/1-16/1-16-identify-index-metadata.command.ts new file mode 100644 index 0000000000..fb05919373 --- /dev/null +++ b/packages/twenty-server/src/database/commands/upgrade-version-command/1-16/1-16-identify-index-metadata.command.ts @@ -0,0 +1,341 @@ +import { InjectRepository } from '@nestjs/typeorm'; + +import { Command } from 'nest-commander'; +import { isDefined } from 'twenty-shared/utils'; +import { IsNull, Repository } from 'typeorm'; +import { v4 } from 'uuid'; + +import { ActiveOrSuspendedWorkspacesMigrationCommandRunner } from 'src/database/commands/command-runners/active-or-suspended-workspaces-migration.command-runner'; +import { RunOnWorkspaceArgs } from 'src/database/commands/command-runners/workspaces-migration.command-runner'; +import { ApplicationService } from 'src/engine/core-modules/application/application.service'; +import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity'; +import { DataSourceService } from 'src/engine/metadata-modules/data-source/data-source.service'; +import { WorkspaceManyOrAllFlatEntityMapsCacheService } from 'src/engine/metadata-modules/flat-entity/services/workspace-many-or-all-flat-entity-maps-cache.service'; +import { type FlatEntityMaps } from 'src/engine/metadata-modules/flat-entity/types/flat-entity-maps.type'; +import { findFlatEntityByUniversalIdentifier } from 'src/engine/metadata-modules/flat-entity/utils/find-flat-entity-by-universal-identifier.util'; +import { findManyFlatEntityByIdInFlatEntityMapsOrThrow } from 'src/engine/metadata-modules/flat-entity/utils/find-many-flat-entity-by-id-in-flat-entity-maps-or-throw.util'; +import { getMetadataFlatEntityMapsKey } from 'src/engine/metadata-modules/flat-entity/utils/get-metadata-flat-entity-maps-key.util'; +import { getMetadataRelatedMetadataNames } from 'src/engine/metadata-modules/flat-entity/utils/get-metadata-related-metadata-names.util'; +import { type FlatFieldMetadata } from 'src/engine/metadata-modules/flat-field-metadata/types/flat-field-metadata.type'; +import { type FlatIndexMetadata } from 'src/engine/metadata-modules/flat-index-metadata/types/flat-index-metadata.type'; +import { type FlatObjectMetadata } from 'src/engine/metadata-modules/flat-object-metadata/types/flat-object-metadata.type'; +import { IndexMetadataEntity } from 'src/engine/metadata-modules/index-metadata/index-metadata.entity'; +import { GlobalWorkspaceOrmManager } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager'; +import { WorkspaceCacheService } from 'src/engine/workspace-cache/services/workspace-cache.service'; +import { STANDARD_OBJECTS } from 'src/engine/workspace-manager/twenty-standard-application/constants/standard-object.constant'; + +import { STANDARD_INDEX_FIELD_UNIVERSAL_IDENTIFIERS } from './constants/standard-index-field-names.constant'; + +type StandardIndexUpdate = { + flatIndexMetadata: FlatIndexMetadata; + universalIdentifier: string; + objectNameSingular: string; + indexName: string; +}; + +@Command({ + name: 'upgrade:1-16:identify-index-metadata', + description: 'Identify standard index metadata', +}) +export class IdentifyIndexMetadataCommand extends ActiveOrSuspendedWorkspacesMigrationCommandRunner { + constructor( + @InjectRepository(WorkspaceEntity) + protected readonly workspaceRepository: Repository, + @InjectRepository(IndexMetadataEntity) + private readonly indexMetadataRepository: Repository, + protected readonly twentyORMGlobalManager: GlobalWorkspaceOrmManager, + protected readonly dataSourceService: DataSourceService, + protected readonly applicationService: ApplicationService, + protected readonly workspaceCacheService: WorkspaceCacheService, + private readonly flatEntityMapsCacheService: WorkspaceManyOrAllFlatEntityMapsCacheService, + ) { + super(workspaceRepository, twentyORMGlobalManager, dataSourceService); + } + + override async runOnWorkspace({ + workspaceId, + options, + }: RunOnWorkspaceArgs): Promise { + this.logger.log( + `Running identify standard index metadata for workspace ${workspaceId}`, + ); + + const { twentyStandardFlatApplication, workspaceCustomFlatApplication } = + await this.applicationService.findWorkspaceTwentyStandardAndCustomApplicationOrThrow( + { workspaceId }, + ); + + const { flatObjectMetadataMaps, flatFieldMetadataMaps, flatIndexMaps } = + await this.flatEntityMapsCacheService.getOrRecomputeManyOrAllFlatEntityMaps( + { + workspaceId, + flatMapsKeys: [ + 'flatObjectMetadataMaps', + 'flatFieldMetadataMaps', + 'flatIndexMaps', + ], + }, + ); + + await this.identifyStandardIndexesOrThrow({ + workspaceId, + flatObjectMetadataMaps, + flatFieldMetadataMaps, + flatIndexMaps, + twentyStandardApplicationId: twentyStandardFlatApplication.id, + dryRun: options.dryRun ?? false, + }); + + await this.identifyCustomIndexes({ + workspaceId, + flatObjectMetadataMaps, + workspaceCustomApplicationId: workspaceCustomFlatApplication.id, + dryRun: options.dryRun ?? false, + }); + + const relatedMetadataNames = getMetadataRelatedMetadataNames('index'); + const relatedCacheKeysToInvalidate = relatedMetadataNames.map( + getMetadataFlatEntityMapsKey, + ); + + this.logger.log( + `Invalidating caches: ${relatedCacheKeysToInvalidate.join(' ')}`, + ); + if (!options.dryRun) { + await this.workspaceCacheService.invalidateAndRecompute(workspaceId, [ + 'flatIndexMaps', + ...relatedCacheKeysToInvalidate, + ]); + } + } + + private async identifyStandardIndexesOrThrow({ + workspaceId, + flatObjectMetadataMaps, + flatFieldMetadataMaps, + flatIndexMaps, + twentyStandardApplicationId, + dryRun, + }: { + workspaceId: string; + flatObjectMetadataMaps: FlatEntityMaps; + flatFieldMetadataMaps: FlatEntityMaps; + flatIndexMaps: FlatEntityMaps; + twentyStandardApplicationId: string; + dryRun: boolean; + }): Promise { + const standardIndexUpdates: StandardIndexUpdate[] = []; + + for (const [objectNameSingular, objectConfig] of Object.entries( + STANDARD_OBJECTS, + )) { + const objectIndexes = + 'indexes' in objectConfig + ? (objectConfig.indexes as Record< + string, + { universalIdentifier: string } | undefined + >) + : null; + + if ( + !isDefined(objectIndexes) || + Object.keys(objectIndexes).length === 0 + ) { + continue; + } + + const flatObjectMetadata = findFlatEntityByUniversalIdentifier({ + flatEntityMaps: flatObjectMetadataMaps, + universalIdentifier: objectConfig.universalIdentifier, + }); + + if (!isDefined(flatObjectMetadata)) { + this.logger.error( + `Standard object "${objectNameSingular}" not found in workspace, this needs investigation, skipping`, + ); + continue; + } + + const objectFlatIndexMetadatas = + findManyFlatEntityByIdInFlatEntityMapsOrThrow({ + flatEntityIds: flatObjectMetadata.indexMetadataIds, + flatEntityMaps: flatIndexMaps, + }); + + for (const [indexName, indexConfig] of Object.entries(objectIndexes)) { + if (!isDefined(indexConfig)) { + continue; + } + + const indexUniversalIdentifier = indexConfig.universalIdentifier; + + if (!isDefined(indexUniversalIdentifier)) { + this.logger.warn( + `Index "${indexName}" config not found for object "${objectNameSingular}", skipping`, + ); + continue; + } + + const relatedFieldUniversalIdentifiers = + STANDARD_INDEX_FIELD_UNIVERSAL_IDENTIFIERS[objectNameSingular]?.[ + indexName + ]; + + if ( + !isDefined(relatedFieldUniversalIdentifiers) || + relatedFieldUniversalIdentifiers.length === 0 + ) { + this.logger.warn( + `No field mapping found for index "${indexName}" on object "${objectNameSingular}", skipping`, + ); + continue; + } + + const expectedFlatFieldMetadatas = relatedFieldUniversalIdentifiers + .map((fieldUniversalIdentifier) => + findFlatEntityByUniversalIdentifier({ + flatEntityMaps: flatFieldMetadataMaps, + universalIdentifier: fieldUniversalIdentifier, + }), + ) + .filter(isDefined); + + if ( + expectedFlatFieldMetadatas.length !== + relatedFieldUniversalIdentifiers.length + ) { + this.logger.warn( + `Could not resolve all field metadata for index "${indexName}" on object "${objectNameSingular}", skipping`, + ); + continue; + } + + const expectedFieldMetadataIds = new Set( + expectedFlatFieldMetadatas.map((flatField) => flatField.id), + ); + const matchingFlatIndexMetadata = objectFlatIndexMetadatas.find( + (flatIndex) => { + const indexFieldMetadataIds = new Set( + flatIndex.flatIndexFieldMetadatas.map( + (indexField) => indexField.fieldMetadataId, + ), + ); + + return this.setsEqual( + indexFieldMetadataIds, + expectedFieldMetadataIds, + ); + }, + ); + + if (!isDefined(matchingFlatIndexMetadata)) { + this.logger.warn( + `Could not find matching index for "${indexName}" on object "${objectNameSingular}", skipping`, + ); + continue; + } + + if (isDefined(matchingFlatIndexMetadata.applicationId)) { + continue; + } + + standardIndexUpdates.push({ + flatIndexMetadata: matchingFlatIndexMetadata, + universalIdentifier: indexUniversalIdentifier, + objectNameSingular: flatObjectMetadata.nameSingular, + indexName, + }); + } + } + + const standardUpdates = standardIndexUpdates.map( + ({ flatIndexMetadata, universalIdentifier }) => ({ + id: flatIndexMetadata.id, + universalIdentifier, + applicationId: twentyStandardApplicationId, + }), + ); + + this.logger.log( + `Found ${standardUpdates.length} standard index(es) to update for workspace ${workspaceId}`, + ); + + for (const { + flatIndexMetadata, + universalIdentifier, + objectNameSingular, + indexName, + } of standardIndexUpdates) { + this.logger.log( + ` - Standard index "${indexName}" on object "${objectNameSingular}" (id=${flatIndexMetadata.id}) -> universalIdentifier=${universalIdentifier}`, + ); + } + + if (!dryRun) { + await this.indexMetadataRepository.save(standardUpdates); + } + } + + private async identifyCustomIndexes({ + workspaceId, + flatObjectMetadataMaps, + workspaceCustomApplicationId, + dryRun, + }: { + workspaceId: string; + flatObjectMetadataMaps: FlatEntityMaps; + workspaceCustomApplicationId: string; + dryRun: boolean; + }): Promise { + const remainingCustomIndexes = await this.indexMetadataRepository.find({ + select: { + id: true, + universalIdentifier: true, + applicationId: true, + name: true, + objectMetadataId: true, + }, + where: { + workspaceId, + applicationId: IsNull(), + }, + }); + + const customUpdates = remainingCustomIndexes.map((indexEntity) => ({ + id: indexEntity.id, + universalIdentifier: indexEntity.universalIdentifier ?? v4(), + applicationId: workspaceCustomApplicationId, + })); + + this.logger.log( + `Found ${customUpdates.length} custom index(es) to update for workspace ${workspaceId}`, + ); + + for (const indexEntity of remainingCustomIndexes) { + const flatObjectMetadata = + flatObjectMetadataMaps.byId[indexEntity.objectMetadataId]; + + this.logger.log( + ` - Custom index "${indexEntity.name}" on object "${flatObjectMetadata?.nameSingular ?? 'unknown'}" (id=${indexEntity.id})`, + ); + } + + if (!dryRun) { + await this.indexMetadataRepository.save(customUpdates); + } + } + + private setsEqual(a: Set, b: Set): boolean { + if (a.size !== b.size) { + return false; + } + + for (const value of a) { + if (!b.has(value)) { + return false; + } + } + + return true; + } +} diff --git a/packages/twenty-server/src/database/commands/upgrade-version-command/1-16/1-16-make-index-metadata-universal-identifier-and-application-id-not-nullable-migration.command.ts b/packages/twenty-server/src/database/commands/upgrade-version-command/1-16/1-16-make-index-metadata-universal-identifier-and-application-id-not-nullable-migration.command.ts new file mode 100644 index 0000000000..eff95f11ec --- /dev/null +++ b/packages/twenty-server/src/database/commands/upgrade-version-command/1-16/1-16-make-index-metadata-universal-identifier-and-application-id-not-nullable-migration.command.ts @@ -0,0 +1,71 @@ +import { InjectDataSource, InjectRepository } from '@nestjs/typeorm'; + +import { Command } from 'nest-commander'; +import { DataSource, Repository } from 'typeorm'; + +import { ActiveOrSuspendedWorkspacesMigrationCommandRunner } from 'src/database/commands/command-runners/active-or-suspended-workspaces-migration.command-runner'; +import { RunOnWorkspaceArgs } from 'src/database/commands/command-runners/workspaces-migration.command-runner'; +import { makeIndexMetadataUniversalIdentifierAndApplicationIdNotNullableQueries } from 'src/database/typeorm/core/migrations/utils/1768830235328-makeIndexMetadataUniversalIdentifierAndApplicationIdNotNullable.util'; +import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity'; +import { DataSourceService } from 'src/engine/metadata-modules/data-source/data-source.service'; +import { GlobalWorkspaceOrmManager } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager'; + +@Command({ + name: 'upgrade:1-16:make-index-metadata-universal-identifier-and-application-id-not-nullable-migration', + description: + 'Make universalIdentifier and applicationId columns NOT NULL on indexMetadata table', +}) +export class MakeIndexMetadataUniversalIdentifierAndApplicationIdNotNullableMigrationCommand extends ActiveOrSuspendedWorkspacesMigrationCommandRunner { + private hasRunOnce = false; + + constructor( + @InjectRepository(WorkspaceEntity) + protected readonly workspaceRepository: Repository, + protected readonly twentyORMGlobalManager: GlobalWorkspaceOrmManager, + protected readonly dataSourceService: DataSourceService, + @InjectDataSource() + private readonly coreDataSource: DataSource, + ) { + super(workspaceRepository, twentyORMGlobalManager, dataSourceService); + } + + override async runOnWorkspace({ + options, + }: RunOnWorkspaceArgs): Promise { + if (this.hasRunOnce) { + this.logger.warn( + 'Skipping has already been run once MakeIndexMetadataUniversalIdentifierAndApplicationIdNotNullableMigrationCommand', + ); + + return; + } + + if (options.dryRun) { + return; + } + + const queryRunner = this.coreDataSource.createQueryRunner(); + + await queryRunner.connect(); + await queryRunner.startTransaction(); + + try { + await makeIndexMetadataUniversalIdentifierAndApplicationIdNotNullableQueries( + queryRunner, + ); + + await queryRunner.commitTransaction(); + this.logger.log( + 'Successfully run MakeIndexMetadataUniversalIdentifierAndApplicationIdNotNullableMigrationCommand', + ); + this.hasRunOnce = true; + } catch (error) { + await queryRunner.rollbackTransaction(); + this.logger.error( + `Rolling back MakeIndexMetadataUniversalIdentifierAndApplicationIdNotNullableMigrationCommand: ${error.message}`, + ); + } finally { + await queryRunner.release(); + } + } +} diff --git a/packages/twenty-server/src/database/commands/upgrade-version-command/1-16/1-16-upgrade-version-command.module.ts b/packages/twenty-server/src/database/commands/upgrade-version-command/1-16/1-16-upgrade-version-command.module.ts index 0dc4b5462b..1724092f8e 100644 --- a/packages/twenty-server/src/database/commands/upgrade-version-command/1-16/1-16-upgrade-version-command.module.ts +++ b/packages/twenty-server/src/database/commands/upgrade-version-command/1-16/1-16-upgrade-version-command.module.ts @@ -5,6 +5,7 @@ import { BackfillOpportunityOwnerFieldCommand } from 'src/database/commands/upgr import { BackfillStandardPageLayoutsCommand } from 'src/database/commands/upgrade-version-command/1-16/1-16-backfill-standard-page-layouts.command'; import { IdentifyAgentMetadataCommand } from 'src/database/commands/upgrade-version-command/1-16/1-16-identify-agent-metadata.command'; import { IdentifyFieldMetadataCommand } from 'src/database/commands/upgrade-version-command/1-16/1-16-identify-field-metadata.command'; +import { IdentifyIndexMetadataCommand } from 'src/database/commands/upgrade-version-command/1-16/1-16-identify-index-metadata.command'; import { IdentifyObjectMetadataCommand } from 'src/database/commands/upgrade-version-command/1-16/1-16-identify-object-metadata.command'; import { IdentifyRoleMetadataCommand } from 'src/database/commands/upgrade-version-command/1-16/1-16-identify-role-metadata.command'; import { IdentifyViewFieldMetadataCommand } from 'src/database/commands/upgrade-version-command/1-16/1-16-identify-view-field-metadata.command'; @@ -13,6 +14,7 @@ import { IdentifyViewGroupMetadataCommand } from 'src/database/commands/upgrade- import { IdentifyViewMetadataCommand } from 'src/database/commands/upgrade-version-command/1-16/1-16-identify-view-metadata.command'; import { MakeAgentUniversalIdentifierAndApplicationIdNotNullableMigrationCommand } from 'src/database/commands/upgrade-version-command/1-16/1-16-make-agent-universal-identifier-and-application-id-not-nullable-migration.command'; import { MakeFieldMetadataUniversalIdentifierAndApplicationIdNotNullableMigrationCommand } from 'src/database/commands/upgrade-version-command/1-16/1-16-make-field-metadata-universal-identifier-and-application-id-not-nullable-migration.command'; +import { MakeIndexMetadataUniversalIdentifierAndApplicationIdNotNullableMigrationCommand } from 'src/database/commands/upgrade-version-command/1-16/1-16-make-index-metadata-universal-identifier-and-application-id-not-nullable-migration.command'; import { MakeObjectMetadataUniversalIdentifierAndApplicationIdNotNullableMigrationCommand } from 'src/database/commands/upgrade-version-command/1-16/1-16-make-object-metadata-universal-identifier-and-application-id-not-nullable-migration.command'; import { MakeRoleUniversalIdentifierAndApplicationIdNotNullableMigrationCommand } from 'src/database/commands/upgrade-version-command/1-16/1-16-make-role-universal-identifier-and-application-id-not-nullable-migration.command'; import { MakeViewFieldUniversalIdentifierAndApplicationIdNotNullableMigrationCommand } from 'src/database/commands/upgrade-version-command/1-16/1-16-make-view-field-universal-identifier-and-application-id-not-nullable-migration.command'; @@ -27,6 +29,7 @@ import { DataSourceModule } from 'src/engine/metadata-modules/data-source/data-s import { FieldMetadataEntity } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity'; import { FieldMetadataModule } from 'src/engine/metadata-modules/field-metadata/field-metadata.module'; import { WorkspaceManyOrAllFlatEntityMapsCacheModule } from 'src/engine/metadata-modules/flat-entity/services/workspace-many-or-all-flat-entity-maps-cache.module'; +import { IndexMetadataEntity } from 'src/engine/metadata-modules/index-metadata/index-metadata.entity'; import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity'; import { RoleEntity } from 'src/engine/metadata-modules/role/role.entity'; import { ViewFieldEntity } from 'src/engine/metadata-modules/view-field/entities/view-field.entity'; @@ -44,6 +47,7 @@ import { WorkspaceMigrationModule } from 'src/engine/workspace-manager/workspace WorkspaceEntity, AgentEntity, FieldMetadataEntity, + IndexMetadataEntity, ObjectMetadataEntity, RoleEntity, ViewEntity, @@ -66,6 +70,7 @@ import { WorkspaceMigrationModule } from 'src/engine/workspace-manager/workspace BackfillStandardPageLayoutsCommand, IdentifyAgentMetadataCommand, IdentifyFieldMetadataCommand, + IdentifyIndexMetadataCommand, IdentifyObjectMetadataCommand, IdentifyRoleMetadataCommand, IdentifyViewMetadataCommand, @@ -80,6 +85,7 @@ import { WorkspaceMigrationModule } from 'src/engine/workspace-manager/workspace MakeViewFieldUniversalIdentifierAndApplicationIdNotNullableMigrationCommand, MakeViewFilterUniversalIdentifierAndApplicationIdNotNullableMigrationCommand, MakeViewGroupUniversalIdentifierAndApplicationIdNotNullableMigrationCommand, + MakeIndexMetadataUniversalIdentifierAndApplicationIdNotNullableMigrationCommand, ], exports: [ UpdateTaskOnDeleteActionCommand, @@ -87,6 +93,7 @@ import { WorkspaceMigrationModule } from 'src/engine/workspace-manager/workspace BackfillStandardPageLayoutsCommand, IdentifyAgentMetadataCommand, IdentifyFieldMetadataCommand, + IdentifyIndexMetadataCommand, IdentifyObjectMetadataCommand, IdentifyRoleMetadataCommand, IdentifyViewMetadataCommand, @@ -101,6 +108,7 @@ import { WorkspaceMigrationModule } from 'src/engine/workspace-manager/workspace MakeViewFieldUniversalIdentifierAndApplicationIdNotNullableMigrationCommand, MakeViewFilterUniversalIdentifierAndApplicationIdNotNullableMigrationCommand, MakeViewGroupUniversalIdentifierAndApplicationIdNotNullableMigrationCommand, + MakeIndexMetadataUniversalIdentifierAndApplicationIdNotNullableMigrationCommand, ], }) export class V1_16_UpgradeVersionCommandModule {} diff --git a/packages/twenty-server/src/database/commands/upgrade-version-command/1-16/constants/standard-index-field-names.constant.ts b/packages/twenty-server/src/database/commands/upgrade-version-command/1-16/constants/standard-index-field-names.constant.ts new file mode 100644 index 0000000000..f45d8c3249 --- /dev/null +++ b/packages/twenty-server/src/database/commands/upgrade-version-command/1-16/constants/standard-index-field-names.constant.ts @@ -0,0 +1,295 @@ +import { STANDARD_OBJECTS } from 'src/engine/workspace-manager/twenty-standard-application/constants/standard-object.constant'; + +// Maps indexName to its related field universal identifiers for each standard object +export const STANDARD_INDEX_FIELD_UNIVERSAL_IDENTIFIERS: Record< + string, + Record +> = { + attachment: { + taskIdIndex: [STANDARD_OBJECTS.attachment.fields.task.universalIdentifier], + noteIdIndex: [STANDARD_OBJECTS.attachment.fields.note.universalIdentifier], + personIdIndex: [ + STANDARD_OBJECTS.attachment.fields.person.universalIdentifier, + ], + companyIdIndex: [ + STANDARD_OBJECTS.attachment.fields.company.universalIdentifier, + ], + opportunityIdIndex: [ + STANDARD_OBJECTS.attachment.fields.opportunity.universalIdentifier, + ], + dashboardIdIndex: [ + STANDARD_OBJECTS.attachment.fields.dashboard.universalIdentifier, + ], + workflowIdIndex: [ + STANDARD_OBJECTS.attachment.fields.workflow.universalIdentifier, + ], + }, + blocklist: { + workspaceMemberIdIndex: [ + STANDARD_OBJECTS.blocklist.fields.workspaceMember.universalIdentifier, + ], + }, + calendarChannelEventAssociation: { + calendarChannelIdIndex: [ + STANDARD_OBJECTS.calendarChannelEventAssociation.fields.calendarChannel + .universalIdentifier, + ], + calendarEventIdIndex: [ + STANDARD_OBJECTS.calendarChannelEventAssociation.fields.calendarEvent + .universalIdentifier, + ], + }, + calendarChannel: { + connectedAccountIdIndex: [ + STANDARD_OBJECTS.calendarChannel.fields.connectedAccount + .universalIdentifier, + ], + }, + calendarEventParticipant: { + calendarEventIdIndex: [ + STANDARD_OBJECTS.calendarEventParticipant.fields.calendarEvent + .universalIdentifier, + ], + personIdIndex: [ + STANDARD_OBJECTS.calendarEventParticipant.fields.person + .universalIdentifier, + ], + workspaceMemberIdIndex: [ + STANDARD_OBJECTS.calendarEventParticipant.fields.workspaceMember + .universalIdentifier, + ], + }, + company: { + accountOwnerIdIndex: [ + STANDARD_OBJECTS.company.fields.accountOwner.universalIdentifier, + ], + domainNameUniqueIndex: [ + STANDARD_OBJECTS.company.fields.domainName.universalIdentifier, + ], + searchVectorGinIndex: [ + STANDARD_OBJECTS.company.fields.searchVector.universalIdentifier, + ], + }, + connectedAccount: { + accountOwnerIdIndex: [ + STANDARD_OBJECTS.connectedAccount.fields.accountOwner.universalIdentifier, + ], + }, + dashboard: { + searchVectorGinIndex: [ + STANDARD_OBJECTS.dashboard.fields.searchVector.universalIdentifier, + ], + }, + favorite: { + forWorkspaceMemberIdIndex: [ + STANDARD_OBJECTS.favorite.fields.forWorkspaceMember.universalIdentifier, + ], + personIdIndex: [ + STANDARD_OBJECTS.favorite.fields.person.universalIdentifier, + ], + companyIdIndex: [ + STANDARD_OBJECTS.favorite.fields.company.universalIdentifier, + ], + favoriteFolderIdIndex: [ + STANDARD_OBJECTS.favorite.fields.favoriteFolder.universalIdentifier, + ], + opportunityIdIndex: [ + STANDARD_OBJECTS.favorite.fields.opportunity.universalIdentifier, + ], + workflowIdIndex: [ + STANDARD_OBJECTS.favorite.fields.workflow.universalIdentifier, + ], + workflowVersionIdIndex: [ + STANDARD_OBJECTS.favorite.fields.workflowVersion.universalIdentifier, + ], + workflowRunIdIndex: [ + STANDARD_OBJECTS.favorite.fields.workflowRun.universalIdentifier, + ], + taskIdIndex: [STANDARD_OBJECTS.favorite.fields.task.universalIdentifier], + noteIdIndex: [STANDARD_OBJECTS.favorite.fields.note.universalIdentifier], + dashboardIdIndex: [ + STANDARD_OBJECTS.favorite.fields.dashboard.universalIdentifier, + ], + }, + messageChannelMessageAssociation: { + messageChannelIdIndex: [ + STANDARD_OBJECTS.messageChannelMessageAssociation.fields.messageChannel + .universalIdentifier, + ], + messageIdIndex: [ + STANDARD_OBJECTS.messageChannelMessageAssociation.fields.message + .universalIdentifier, + ], + messageChannelIdMessageIdUniqueIndex: [ + STANDARD_OBJECTS.messageChannelMessageAssociation.fields.messageChannel + .universalIdentifier, + STANDARD_OBJECTS.messageChannelMessageAssociation.fields.message + .universalIdentifier, + ], + }, + messageChannel: { + connectedAccountIdIndex: [ + STANDARD_OBJECTS.messageChannel.fields.connectedAccount + .universalIdentifier, + ], + }, + messageFolder: { + messageChannelIdIndex: [ + STANDARD_OBJECTS.messageFolder.fields.messageChannel.universalIdentifier, + ], + }, + messageParticipant: { + messageIdIndex: [ + STANDARD_OBJECTS.messageParticipant.fields.message.universalIdentifier, + ], + personIdIndex: [ + STANDARD_OBJECTS.messageParticipant.fields.person.universalIdentifier, + ], + workspaceMemberIdIndex: [ + STANDARD_OBJECTS.messageParticipant.fields.workspaceMember + .universalIdentifier, + ], + }, + message: { + messageThreadIdIndex: [ + STANDARD_OBJECTS.message.fields.messageThread.universalIdentifier, + ], + }, + note: { + searchVectorGinIndex: [ + STANDARD_OBJECTS.note.fields.searchVector.universalIdentifier, + ], + }, + noteTarget: { + noteIdIndex: [STANDARD_OBJECTS.noteTarget.fields.note.universalIdentifier], + personIdIndex: [ + STANDARD_OBJECTS.noteTarget.fields.person.universalIdentifier, + ], + companyIdIndex: [ + STANDARD_OBJECTS.noteTarget.fields.company.universalIdentifier, + ], + opportunityIdIndex: [ + STANDARD_OBJECTS.noteTarget.fields.opportunity.universalIdentifier, + ], + }, + opportunity: { + pointOfContactIdIndex: [ + STANDARD_OBJECTS.opportunity.fields.pointOfContact.universalIdentifier, + ], + companyIdIndex: [ + STANDARD_OBJECTS.opportunity.fields.company.universalIdentifier, + ], + stageIndex: [STANDARD_OBJECTS.opportunity.fields.stage.universalIdentifier], + searchVectorGinIndex: [ + STANDARD_OBJECTS.opportunity.fields.searchVector.universalIdentifier, + ], + }, + person: { + companyIdIndex: [ + STANDARD_OBJECTS.person.fields.company.universalIdentifier, + ], + emailsUniqueIndex: [ + STANDARD_OBJECTS.person.fields.emails.universalIdentifier, + ], + searchVectorGinIndex: [ + STANDARD_OBJECTS.person.fields.searchVector.universalIdentifier, + ], + }, + task: { + assigneeIdIndex: [ + STANDARD_OBJECTS.task.fields.assignee.universalIdentifier, + ], + searchVectorGinIndex: [ + STANDARD_OBJECTS.task.fields.searchVector.universalIdentifier, + ], + }, + taskTarget: { + taskIdIndex: [STANDARD_OBJECTS.taskTarget.fields.task.universalIdentifier], + personIdIndex: [ + STANDARD_OBJECTS.taskTarget.fields.person.universalIdentifier, + ], + companyIdIndex: [ + STANDARD_OBJECTS.taskTarget.fields.company.universalIdentifier, + ], + opportunityIdIndex: [ + STANDARD_OBJECTS.taskTarget.fields.opportunity.universalIdentifier, + ], + }, + timelineActivity: { + workspaceMemberIdIndex: [ + STANDARD_OBJECTS.timelineActivity.fields.workspaceMember + .universalIdentifier, + ], + personIdIndex: [ + STANDARD_OBJECTS.timelineActivity.fields.targetPerson.universalIdentifier, + ], + companyIdIndex: [ + STANDARD_OBJECTS.timelineActivity.fields.targetCompany + .universalIdentifier, + ], + opportunityIdIndex: [ + STANDARD_OBJECTS.timelineActivity.fields.targetOpportunity + .universalIdentifier, + ], + noteIdIndex: [ + STANDARD_OBJECTS.timelineActivity.fields.targetNote.universalIdentifier, + ], + taskIdIndex: [ + STANDARD_OBJECTS.timelineActivity.fields.targetTask.universalIdentifier, + ], + workflowIdIndex: [ + STANDARD_OBJECTS.timelineActivity.fields.targetWorkflow + .universalIdentifier, + ], + workflowVersionIdIndex: [ + STANDARD_OBJECTS.timelineActivity.fields.targetWorkflowVersion + .universalIdentifier, + ], + workflowRunIdIndex: [ + STANDARD_OBJECTS.timelineActivity.fields.targetWorkflowRun + .universalIdentifier, + ], + dashboardIdIndex: [ + STANDARD_OBJECTS.timelineActivity.fields.targetDashboard + .universalIdentifier, + ], + }, + workflow: { + searchVectorGinIndex: [ + STANDARD_OBJECTS.workflow.fields.searchVector.universalIdentifier, + ], + }, + workflowAutomatedTrigger: { + workflowIdIndex: [ + STANDARD_OBJECTS.workflowAutomatedTrigger.fields.workflow + .universalIdentifier, + ], + }, + workflowRun: { + workflowVersionIdIndex: [ + STANDARD_OBJECTS.workflowRun.fields.workflowVersion.universalIdentifier, + ], + workflowIdIndex: [ + STANDARD_OBJECTS.workflowRun.fields.workflow.universalIdentifier, + ], + searchVectorGinIndex: [ + STANDARD_OBJECTS.workflowRun.fields.searchVector.universalIdentifier, + ], + }, + workflowVersion: { + workflowIdIndex: [ + STANDARD_OBJECTS.workflowVersion.fields.workflow.universalIdentifier, + ], + searchVectorGinIndex: [ + STANDARD_OBJECTS.workflowVersion.fields.searchVector.universalIdentifier, + ], + }, + workspaceMember: { + userEmailUniqueIndex: [ + STANDARD_OBJECTS.workspaceMember.fields.userEmail.universalIdentifier, + ], + searchVectorGinIndex: [ + STANDARD_OBJECTS.workspaceMember.fields.searchVector.universalIdentifier, + ], + }, +}; diff --git a/packages/twenty-server/src/database/commands/upgrade-version-command/upgrade.command.ts b/packages/twenty-server/src/database/commands/upgrade-version-command/upgrade.command.ts index 12af9c95ab..a27e372127 100644 --- a/packages/twenty-server/src/database/commands/upgrade-version-command/upgrade.command.ts +++ b/packages/twenty-server/src/database/commands/upgrade-version-command/upgrade.command.ts @@ -26,6 +26,7 @@ import { BackfillOpportunityOwnerFieldCommand } from 'src/database/commands/upgr import { BackfillStandardPageLayoutsCommand } from 'src/database/commands/upgrade-version-command/1-16/1-16-backfill-standard-page-layouts.command'; import { IdentifyAgentMetadataCommand } from 'src/database/commands/upgrade-version-command/1-16/1-16-identify-agent-metadata.command'; import { IdentifyFieldMetadataCommand } from 'src/database/commands/upgrade-version-command/1-16/1-16-identify-field-metadata.command'; +import { IdentifyIndexMetadataCommand } from 'src/database/commands/upgrade-version-command/1-16/1-16-identify-index-metadata.command'; import { IdentifyObjectMetadataCommand } from 'src/database/commands/upgrade-version-command/1-16/1-16-identify-object-metadata.command'; import { IdentifyRoleMetadataCommand } from 'src/database/commands/upgrade-version-command/1-16/1-16-identify-role-metadata.command'; import { IdentifyViewFieldMetadataCommand } from 'src/database/commands/upgrade-version-command/1-16/1-16-identify-view-field-metadata.command'; @@ -34,6 +35,7 @@ import { IdentifyViewGroupMetadataCommand } from 'src/database/commands/upgrade- import { IdentifyViewMetadataCommand } from 'src/database/commands/upgrade-version-command/1-16/1-16-identify-view-metadata.command'; import { MakeAgentUniversalIdentifierAndApplicationIdNotNullableMigrationCommand } from 'src/database/commands/upgrade-version-command/1-16/1-16-make-agent-universal-identifier-and-application-id-not-nullable-migration.command'; import { MakeFieldMetadataUniversalIdentifierAndApplicationIdNotNullableMigrationCommand } from 'src/database/commands/upgrade-version-command/1-16/1-16-make-field-metadata-universal-identifier-and-application-id-not-nullable-migration.command'; +import { MakeIndexMetadataUniversalIdentifierAndApplicationIdNotNullableMigrationCommand } from 'src/database/commands/upgrade-version-command/1-16/1-16-make-index-metadata-universal-identifier-and-application-id-not-nullable-migration.command'; import { MakeObjectMetadataUniversalIdentifierAndApplicationIdNotNullableMigrationCommand } from 'src/database/commands/upgrade-version-command/1-16/1-16-make-object-metadata-universal-identifier-and-application-id-not-nullable-migration.command'; import { MakeRoleUniversalIdentifierAndApplicationIdNotNullableMigrationCommand } from 'src/database/commands/upgrade-version-command/1-16/1-16-make-role-universal-identifier-and-application-id-not-nullable-migration.command'; import { MakeViewFieldUniversalIdentifierAndApplicationIdNotNullableMigrationCommand } from 'src/database/commands/upgrade-version-command/1-16/1-16-make-view-field-universal-identifier-and-application-id-not-nullable-migration.command'; @@ -85,6 +87,7 @@ export class UpgradeCommand extends UpgradeCommandRunner { protected readonly backfillStandardPageLayoutsCommand: BackfillStandardPageLayoutsCommand, protected readonly identifyAgentMetadataCommand: IdentifyAgentMetadataCommand, protected readonly identifyFieldMetadataCommand: IdentifyFieldMetadataCommand, + protected readonly identifyIndexMetadataCommand: IdentifyIndexMetadataCommand, protected readonly identifyObjectMetadataCommand: IdentifyObjectMetadataCommand, protected readonly identifyRoleMetadataCommand: IdentifyRoleMetadataCommand, protected readonly identifyViewMetadataCommand: IdentifyViewMetadataCommand, @@ -99,6 +102,7 @@ export class UpgradeCommand extends UpgradeCommandRunner { protected readonly makeViewFieldUniversalIdentifierAndApplicationIdNotNullableMigrationCommand: MakeViewFieldUniversalIdentifierAndApplicationIdNotNullableMigrationCommand, protected readonly makeViewFilterUniversalIdentifierAndApplicationIdNotNullableMigrationCommand: MakeViewFilterUniversalIdentifierAndApplicationIdNotNullableMigrationCommand, protected readonly makeViewGroupUniversalIdentifierAndApplicationIdNotNullableMigrationCommand: MakeViewGroupUniversalIdentifierAndApplicationIdNotNullableMigrationCommand, + protected readonly makeIndexMetadataUniversalIdentifierAndApplicationIdNotNullableMigrationCommand: MakeIndexMetadataUniversalIdentifierAndApplicationIdNotNullableMigrationCommand, ) { super( workspaceRepository, @@ -144,6 +148,7 @@ export class UpgradeCommand extends UpgradeCommandRunner { this.identifyViewFieldMetadataCommand, this.identifyViewFilterMetadataCommand, this.identifyViewGroupMetadataCommand, + this.identifyIndexMetadataCommand, this .makeAgentUniversalIdentifierAndApplicationIdNotNullableMigrationCommand, this @@ -160,6 +165,8 @@ export class UpgradeCommand extends UpgradeCommandRunner { .makeViewFilterUniversalIdentifierAndApplicationIdNotNullableMigrationCommand, this .makeViewGroupUniversalIdentifierAndApplicationIdNotNullableMigrationCommand, + this + .makeIndexMetadataUniversalIdentifierAndApplicationIdNotNullableMigrationCommand, ]; this.allCommands = { diff --git a/packages/twenty-server/src/database/typeorm/core/migrations/common/1768830235328-makeIndexMetadataUniversalIdentifierAndApplicationIdNotNullable.ts b/packages/twenty-server/src/database/typeorm/core/migrations/common/1768830235328-makeIndexMetadataUniversalIdentifierAndApplicationIdNotNullable.ts new file mode 100644 index 0000000000..d6b2eac9fc --- /dev/null +++ b/packages/twenty-server/src/database/typeorm/core/migrations/common/1768830235328-makeIndexMetadataUniversalIdentifierAndApplicationIdNotNullable.ts @@ -0,0 +1,64 @@ +import { type MigrationInterface, type QueryRunner } from 'typeorm'; + +import { makeIndexMetadataUniversalIdentifierAndApplicationIdNotNullableQueries } from 'src/database/typeorm/core/migrations/utils/1768830235328-makeIndexMetadataUniversalIdentifierAndApplicationIdNotNullable.util'; + +export class MakeIndexMetadataUniversalIdentifierAndApplicationIdNotNullable1768830235328 + implements MigrationInterface +{ + name = + 'MakeIndexMetadataUniversalIdentifierAndApplicationIdNotNullable1768830235328'; + + public async up(queryRunner: QueryRunner): Promise { + const savepointName = + 'sp_make_index_metadata_universal_identifier_and_application_id_not_nullable'; + + try { + await queryRunner.query(`SAVEPOINT ${savepointName}`); + + await makeIndexMetadataUniversalIdentifierAndApplicationIdNotNullableQueries( + queryRunner, + ); + + await queryRunner.query(`RELEASE SAVEPOINT ${savepointName}`); + } catch (e) { + try { + await queryRunner.query(`ROLLBACK TO SAVEPOINT ${savepointName}`); + await queryRunner.query(`RELEASE SAVEPOINT ${savepointName}`); + } catch (rollbackError) { + // eslint-disable-next-line no-console + console.error( + 'Failed to rollback to savepoint in MakeIndexMetadataUniversalIdentifierAndApplicationIdNotNullable1768830235328', + rollbackError, + ); + throw rollbackError; + } + + // eslint-disable-next-line no-console + console.error( + 'Swallowing MakeIndexMetadataUniversalIdentifierAndApplicationIdNotNullable1768830235328 error', + e, + ); + } + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query( + `ALTER TABLE "core"."indexMetadata" DROP CONSTRAINT "FK_056363e1599f5b9a0e33323d9da"`, + ); + await queryRunner.query( + `DROP INDEX "core"."IDX_b27c681286ac581f81498c5d4b"`, + ); + await queryRunner.query( + `ALTER TABLE "core"."indexMetadata" ALTER COLUMN "applicationId" DROP NOT NULL`, + ); + await queryRunner.query( + `ALTER TABLE "core"."indexMetadata" ALTER COLUMN "universalIdentifier" DROP NOT NULL`, + ); + await queryRunner.query( + `CREATE UNIQUE INDEX "IDX_b27c681286ac581f81498c5d4b" ON "core"."indexMetadata" ("workspaceId", "universalIdentifier") `, + ); + await queryRunner.query( + `ALTER TABLE "core"."indexMetadata" ADD CONSTRAINT "FK_056363e1599f5b9a0e33323d9da" FOREIGN KEY ("applicationId") REFERENCES "core"."application"("id") ON DELETE CASCADE ON UPDATE NO ACTION`, + ); + } +} diff --git a/packages/twenty-server/src/database/typeorm/core/migrations/utils/1768830235328-makeIndexMetadataUniversalIdentifierAndApplicationIdNotNullable.util.ts b/packages/twenty-server/src/database/typeorm/core/migrations/utils/1768830235328-makeIndexMetadataUniversalIdentifierAndApplicationIdNotNullable.util.ts new file mode 100644 index 0000000000..a6985efb14 --- /dev/null +++ b/packages/twenty-server/src/database/typeorm/core/migrations/utils/1768830235328-makeIndexMetadataUniversalIdentifierAndApplicationIdNotNullable.util.ts @@ -0,0 +1,23 @@ +import { type QueryRunner } from 'typeorm'; + +export const makeIndexMetadataUniversalIdentifierAndApplicationIdNotNullableQueries = + async (queryRunner: QueryRunner): Promise => { + await queryRunner.query( + `ALTER TABLE "core"."indexMetadata" DROP CONSTRAINT "FK_056363e1599f5b9a0e33323d9da"`, + ); + await queryRunner.query( + `DROP INDEX "core"."IDX_b27c681286ac581f81498c5d4b"`, + ); + await queryRunner.query( + `ALTER TABLE "core"."indexMetadata" ALTER COLUMN "universalIdentifier" SET NOT NULL`, + ); + await queryRunner.query( + `ALTER TABLE "core"."indexMetadata" ALTER COLUMN "applicationId" SET NOT NULL`, + ); + await queryRunner.query( + `CREATE UNIQUE INDEX "IDX_b27c681286ac581f81498c5d4b" ON "core"."indexMetadata" ("workspaceId", "universalIdentifier") `, + ); + await queryRunner.query( + `ALTER TABLE "core"."indexMetadata" ADD CONSTRAINT "FK_056363e1599f5b9a0e33323d9da" FOREIGN KEY ("applicationId") REFERENCES "core"."application"("id") ON DELETE CASCADE ON UPDATE NO ACTION`, + ); + }; diff --git a/packages/twenty-server/src/engine/metadata-modules/index-metadata/index-metadata.entity.ts b/packages/twenty-server/src/engine/metadata-modules/index-metadata/index-metadata.entity.ts index 17e2a38e65..3883e79afc 100644 --- a/packages/twenty-server/src/engine/metadata-modules/index-metadata/index-metadata.entity.ts +++ b/packages/twenty-server/src/engine/metadata-modules/index-metadata/index-metadata.entity.ts @@ -15,7 +15,7 @@ import { import { IndexFieldMetadataEntity } from 'src/engine/metadata-modules/index-metadata/index-field-metadata.entity'; import { IndexType } from 'src/engine/metadata-modules/index-metadata/types/indexType.types'; import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity'; -import { SyncableEntity } from 'src/engine/workspace-manager/types/syncable-entity.interface'; +import { SyncableEntityRequired } from 'src/engine/workspace-manager/types/syncable-entity-required.interface'; @Unique('IDX_INDEX_METADATA_NAME_WORKSPACE_ID_OBJECT_METADATA_ID_UNIQUE', [ 'name', @@ -28,7 +28,7 @@ import { SyncableEntity } from 'src/engine/workspace-manager/types/syncable-enti ]) @Entity('indexMetadata') export class IndexMetadataEntity - extends SyncableEntity + extends SyncableEntityRequired implements Required { @PrimaryGeneratedColumn('uuid')