From ca5bd76c6a613330b5bd3dbb74c231cbca483aeb Mon Sep 17 00:00:00 2001 From: Etienne <45695613+etiennejouan@users.noreply.github.com> Date: Thu, 27 Nov 2025 14:18:56 +0100 Subject: [PATCH] Null equivalence - migration command (#16018) Awaiting https://github.com/twentyhq/twenty/pull/15926 approval, before un-drafting it --------- Co-authored-by: prastoin --- .../1-12/1-12-clean-null-equivalent-values.ts | 304 ++++++++++++++++++ .../1-12-upgrade-version-command.module.ts | 15 +- .../upgrade.command.ts | 3 + .../deprecated-generate-default-value.ts | 61 ++++ 4 files changed, 382 insertions(+), 1 deletion(-) create mode 100644 packages/twenty-server/src/database/commands/upgrade-version-command/1-12/1-12-clean-null-equivalent-values.ts create mode 100644 packages/twenty-server/src/engine/metadata-modules/field-metadata/utils/deprecated-generate-default-value.ts diff --git a/packages/twenty-server/src/database/commands/upgrade-version-command/1-12/1-12-clean-null-equivalent-values.ts b/packages/twenty-server/src/database/commands/upgrade-version-command/1-12/1-12-clean-null-equivalent-values.ts new file mode 100644 index 0000000000..b4271af8eb --- /dev/null +++ b/packages/twenty-server/src/database/commands/upgrade-version-command/1-12/1-12-clean-null-equivalent-values.ts @@ -0,0 +1,304 @@ +import { Logger } from '@nestjs/common'; +import { InjectRepository } from '@nestjs/typeorm'; + +import deepEqual from 'deep-equal'; +import { Command } from 'nest-commander'; +import { + CompositeType, + compositeTypeDefinitions, + FieldMetadataType, +} from 'twenty-shared/types'; +import { isDefined } from 'twenty-shared/utils'; +import { Repository } from 'typeorm'; + +import { ActiveOrSuspendedWorkspacesMigrationCommandRunner } from 'src/database/commands/command-runners/active-or-suspended-workspaces-migration.command-runner'; +import { RunOnWorkspaceArgs } from 'src/database/commands/command-runners/workspaces-migration.command-runner'; +import { FeatureFlagKey } from 'src/engine/core-modules/feature-flag/enums/feature-flag-key.enum'; +import { FeatureFlagEntity } from 'src/engine/core-modules/feature-flag/feature-flag.entity'; +import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity'; +import { DataSourceService } from 'src/engine/metadata-modules/data-source/data-source.service'; +import { FieldMetadataEntity } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity'; +import { computeCompositeColumnName } from 'src/engine/metadata-modules/field-metadata/utils/compute-column-name.util'; +import { deprecatedGenerateDefaultValue } from 'src/engine/metadata-modules/field-metadata/utils/deprecated-generate-default-value'; +import { isCompositeFieldMetadataType } from 'src/engine/metadata-modules/field-metadata/utils/is-composite-field-metadata-type.util'; +import { WorkspaceManyOrAllFlatEntityMapsCacheService } from 'src/engine/metadata-modules/flat-entity/services/workspace-many-or-all-flat-entity-maps-cache.service'; +import { IndexMetadataEntity } from 'src/engine/metadata-modules/index-metadata/index-metadata.entity'; +import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity'; +import { generateMigrationName } from 'src/engine/metadata-modules/workspace-migration/utils/generate-migration-name.util'; +import { + WorkspaceMigrationIndexAction, + WorkspaceMigrationIndexActionType, + WorkspaceMigrationTableAction, + WorkspaceMigrationTableActionType, +} from 'src/engine/metadata-modules/workspace-migration/workspace-migration.entity'; +import { WorkspaceMigrationService } from 'src/engine/metadata-modules/workspace-migration/workspace-migration.service'; +import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager'; +import { computeObjectTargetTable } from 'src/engine/utils/compute-object-target-table.util'; +import { getWorkspaceSchemaName } from 'src/engine/workspace-datasource/utils/get-workspace-schema-name.util'; +import { WorkspaceMigrationRunnerService } from 'src/engine/workspace-manager/workspace-migration-runner/workspace-migration-runner.service'; + +@Command({ + name: 'upgrade:1-12:clean-null-equivalent-values', + description: 'Clean up null equivalent values in the database', +}) +export class CleanNullEquivalentValuesCommand extends ActiveOrSuspendedWorkspacesMigrationCommandRunner { + protected readonly logger = new Logger(CleanNullEquivalentValuesCommand.name); + + constructor( + @InjectRepository(WorkspaceEntity) + protected readonly workspaceRepository: Repository, + protected readonly twentyORMGlobalManager: TwentyORMGlobalManager, + @InjectRepository(ObjectMetadataEntity) + protected readonly objectMetadataRepository: Repository, + @InjectRepository(FieldMetadataEntity) + protected readonly fieldMetadataRepository: Repository, + @InjectRepository(IndexMetadataEntity) + protected readonly indexMetadataRepository: Repository, + @InjectRepository(FeatureFlagEntity) + protected readonly featureFlagRepository: Repository, + protected readonly workspaceMigrationService: WorkspaceMigrationService, + protected readonly workspaceMigrationRunnerService: WorkspaceMigrationRunnerService, + protected readonly dataSourceService: DataSourceService, + protected readonly workspaceManyOrAllFlatEntityMapsCacheService: WorkspaceManyOrAllFlatEntityMapsCacheService, + ) { + super(workspaceRepository, twentyORMGlobalManager, dataSourceService); + } + + override async runOnWorkspace({ + workspaceId, + options, + dataSource, + }: RunOnWorkspaceArgs): Promise { + const isDryRun = options.dryRun || false; + + if (!isDefined(dataSource)) { + throw new Error( + `Could not find data source for workspace ${workspaceId}, should never occur`, + ); + } + + const schemaName = getWorkspaceSchemaName(workspaceId); + + if (isDryRun) { + this.logger.log('Dry run mode: No changes will be applied'); + } + + const featureFlag = await this.featureFlagRepository.findOne({ + where: { + key: FeatureFlagKey.IS_NULL_EQUIVALENCE_ENABLED, + value: true, + workspaceId, + }, + }); + + if (isDefined(featureFlag)) { + this.logger.log( + `Feature flag ${FeatureFlagKey.IS_NULL_EQUIVALENCE_ENABLED} already enabled for workspace ${workspaceId}`, + ); + + return; + } + + const objectMetadataItems = await this.objectMetadataRepository.find({ + where: { workspaceId }, + relations: [ + 'fields', + 'indexMetadatas', + 'indexMetadatas.indexFieldMetadatas', + ], + }); + + for (const objectMetadataItem of objectMetadataItems) { + const tableName = computeObjectTargetTable(objectMetadataItem); + + for (const field of objectMetadataItem.fields) { + const fieldDefaultDefaultValue = deprecatedGenerateDefaultValue( + field.type, + ); + + if ( + !( + isDefined(field.defaultValue) && + ((typeof field.defaultValue === 'string' && + field.defaultValue === fieldDefaultDefaultValue) || + this.objectEquals( + field.defaultValue, + fieldDefaultDefaultValue, + )) && + field.type !== FieldMetadataType.ACTOR + ) + ) + continue; + + this.logger.log( + `Processing field ${field.name} on object ${objectMetadataItem.nameSingular} (Table: ${tableName})`, + ); + if (!isDryRun) { + if (isCompositeFieldMetadataType(field.type)) { + const compositeType = compositeTypeDefinitions.get(field.type); + + if (isDefined(compositeType)) { + for (const property of compositeType.properties) { + const columnName = computeCompositeColumnName( + field.name, + property, + ); + + await dataSource.query( + `ALTER TABLE "${schemaName}"."${tableName}" ALTER COLUMN "${columnName}" DROP NOT NULL, ALTER COLUMN "${columnName}" DROP DEFAULT`, + [], + undefined, + { shouldBypassPermissionChecks: true }, + ); + + await dataSource.query( + `UPDATE "${schemaName}"."${tableName}" SET "${columnName}" = NULL WHERE "${columnName}" = ${field.defaultValue[property.name as keyof typeof field.defaultValue]}`, + [], + undefined, + { shouldBypassPermissionChecks: true }, + ); + } + } + } else { + await dataSource.query( + `ALTER TABLE "${schemaName}"."${tableName}" ALTER COLUMN "${field.name}" DROP NOT NULL, ALTER COLUMN "${field.name}" DROP DEFAULT`, + [], + undefined, + { shouldBypassPermissionChecks: true }, + ); + + await dataSource.query( + `UPDATE "${schemaName}"."${tableName}" SET "${field.name}" = NULL WHERE "${field.name}" = ${field.defaultValue}`, + [], + undefined, + { shouldBypassPermissionChecks: true }, + ); + } + + await this.fieldMetadataRepository.update(field.id, { + isNullable: true, + defaultValue: null, + }); + + await this.workspaceManyOrAllFlatEntityMapsCacheService.invalidateFlatEntityMaps( + { + workspaceId, + flatMapsKeys: ['flatFieldMetadataMaps'], + }, + ); + } + + const relevantIndexes = objectMetadataItem.indexMetadatas.filter( + (index) => + index.isUnique && + index.indexFieldMetadatas.some( + (ifm) => ifm.fieldMetadataId === field.id, + ), + ); + + for (const index of relevantIndexes) { + if (!isDryRun) { + this.logger.log( + `Removing where clause from index ${index.name} on ${tableName}`, + ); + + await this.indexMetadataRepository.update(index.id, { + indexWhereClause: null, + }); + + await this.workspaceManyOrAllFlatEntityMapsCacheService.invalidateFlatEntityMaps( + { + workspaceId, + flatMapsKeys: ['flatIndexMaps'], + }, + ); + + const columnNames = index.indexFieldMetadatas.flatMap( + (indexFieldMetadata) => { + const fieldMetadata = objectMetadataItem.fields.find( + (f) => f.id === indexFieldMetadata.fieldMetadataId, + ); + + if (!isDefined(fieldMetadata)) { + throw new Error( + `Field metadata not found for index field metadata ${indexFieldMetadata.id}`, + ); + } + + if (isCompositeFieldMetadataType(fieldMetadata.type)) { + const compositeType = compositeTypeDefinitions.get( + fieldMetadata.type, + ) as CompositeType; + + const uniqueCompositeProperties = + compositeType.properties.filter( + (property) => property.isIncludedInUniqueConstraint, + ); + + return uniqueCompositeProperties.map((subField) => + computeCompositeColumnName(fieldMetadata.name, subField), + ); + } + + return [fieldMetadata.name]; + }, + ); + + const dropIndexAction: WorkspaceMigrationIndexAction = { + action: WorkspaceMigrationIndexActionType.DROP, + name: index.name, + columns: [], + isUnique: true, + }; + + const createIndexAction: WorkspaceMigrationIndexAction = { + action: WorkspaceMigrationIndexActionType.CREATE, + name: index.name, + columns: columnNames, + isUnique: true, + where: null, + }; + + const migration: WorkspaceMigrationTableAction = { + name: tableName, + action: WorkspaceMigrationTableActionType.ALTER_INDEXES, + indexes: [dropIndexAction, createIndexAction], + }; + + await this.workspaceMigrationService.createCustomMigration( + generateMigrationName(`update-index-${index.name}-remove-where`), + workspaceId, + [migration], + ); + + await this.workspaceMigrationRunnerService.executeMigrationFromPendingMigrations( + workspaceId, + ); + } + } + } + } + if (!isDryRun) { + await this.featureFlagRepository.upsert( + { + key: FeatureFlagKey.IS_NULL_EQUIVALENCE_ENABLED, + value: true, + workspaceId, + }, + ['key', 'workspaceId'], + ); + + this.logger.log( + `Switched on feature flag ${FeatureFlagKey.IS_NULL_EQUIVALENCE_ENABLED} for workspace ${workspaceId}`, + ); + } + } + + private objectEquals(obj1: unknown, obj2: unknown): boolean { + if (typeof obj1 !== 'object' || typeof obj2 !== 'object') { + return false; + } + + return deepEqual(obj1, obj2, { strict: true }); + } +} diff --git a/packages/twenty-server/src/database/commands/upgrade-version-command/1-12/1-12-upgrade-version-command.module.ts b/packages/twenty-server/src/database/commands/upgrade-version-command/1-12/1-12-upgrade-version-command.module.ts index e85d77e6ce..a106bee9a1 100644 --- a/packages/twenty-server/src/database/commands/upgrade-version-command/1-12/1-12-upgrade-version-command.module.ts +++ b/packages/twenty-server/src/database/commands/upgrade-version-command/1-12/1-12-upgrade-version-command.module.ts @@ -3,16 +3,22 @@ import { TypeOrmModule } from '@nestjs/typeorm'; import { AddCalendarEventsImportScheduledSyncStageCommand } from 'src/database/commands/upgrade-version-command/1-12/1-12-add-calendar-events-import-scheduled-sync-stage.command'; import { AddMessagesImportScheduledSyncStageCommand } from 'src/database/commands/upgrade-version-command/1-12/1-12-add-messages-import-scheduled-sync-stage.command'; +import { CleanNullEquivalentValuesCommand } from 'src/database/commands/upgrade-version-command/1-12/1-12-clean-null-equivalent-values'; import { CreateWorkspaceCustomApplicationCommand } from 'src/database/commands/upgrade-version-command/1-12/1-12-create-workspace-custom-application.command'; import { SetStandardApplicationNotUninstallableCommand } from 'src/database/commands/upgrade-version-command/1-12/1-12-set-standard-application-not-uninstallable.command'; import { WorkspaceCustomApplicationIdNonNullableCommand } from 'src/database/commands/upgrade-version-command/1-12/1-12-workspace-custom-application-id-non-nullable-migration.command'; import { ApplicationModule } from 'src/engine/core-modules/application/application.module'; +import { FeatureFlagEntity } from 'src/engine/core-modules/feature-flag/feature-flag.entity'; import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity'; import { DataSourceModule } from 'src/engine/metadata-modules/data-source/data-source.module'; import { FieldMetadataEntity } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity'; import { FieldMetadataModule } from 'src/engine/metadata-modules/field-metadata/field-metadata.module'; +import { WorkspaceManyOrAllFlatEntityMapsCacheModule } from 'src/engine/metadata-modules/flat-entity/services/workspace-many-or-all-flat-entity-maps-cache.module'; +import { IndexMetadataEntity } from 'src/engine/metadata-modules/index-metadata/index-metadata.entity'; import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity'; +import { WorkspaceMigrationModule } from 'src/engine/metadata-modules/workspace-migration/workspace-migration.module'; import { WorkspaceSchemaManagerModule } from 'src/engine/twenty-orm/workspace-schema-manager/workspace-schema-manager.module'; +import { WorkspaceMigrationRunnerModule } from 'src/engine/workspace-manager/workspace-migration-runner/workspace-migration-runner.module'; @Module({ imports: [ @@ -20,11 +26,16 @@ import { WorkspaceSchemaManagerModule } from 'src/engine/twenty-orm/workspace-sc WorkspaceEntity, ObjectMetadataEntity, FieldMetadataEntity, + IndexMetadataEntity, + FeatureFlagEntity, ]), - DataSourceModule, WorkspaceSchemaManagerModule, + WorkspaceMigrationModule, + WorkspaceMigrationRunnerModule, ApplicationModule, FieldMetadataModule, + DataSourceModule, + WorkspaceManyOrAllFlatEntityMapsCacheModule, ], providers: [ AddCalendarEventsImportScheduledSyncStageCommand, @@ -32,6 +43,7 @@ import { WorkspaceSchemaManagerModule } from 'src/engine/twenty-orm/workspace-sc CreateWorkspaceCustomApplicationCommand, SetStandardApplicationNotUninstallableCommand, WorkspaceCustomApplicationIdNonNullableCommand, + CleanNullEquivalentValuesCommand, ], exports: [ AddCalendarEventsImportScheduledSyncStageCommand, @@ -39,6 +51,7 @@ import { WorkspaceSchemaManagerModule } from 'src/engine/twenty-orm/workspace-sc CreateWorkspaceCustomApplicationCommand, SetStandardApplicationNotUninstallableCommand, WorkspaceCustomApplicationIdNonNullableCommand, + CleanNullEquivalentValuesCommand, ], }) export class V1_12_UpgradeVersionCommandModule {} diff --git a/packages/twenty-server/src/database/commands/upgrade-version-command/upgrade.command.ts b/packages/twenty-server/src/database/commands/upgrade-version-command/upgrade.command.ts index ed0c4d58d3..534eef7453 100644 --- a/packages/twenty-server/src/database/commands/upgrade-version-command/upgrade.command.ts +++ b/packages/twenty-server/src/database/commands/upgrade-version-command/upgrade.command.ts @@ -23,6 +23,7 @@ import { CleanOrphanedUserWorkspacesCommand } from 'src/database/commands/upgrad import { CreateTwentyStandardApplicationCommand } from 'src/database/commands/upgrade-version-command/1-11/1-11-create-twenty-standard-application.command'; import { AddCalendarEventsImportScheduledSyncStageCommand } from 'src/database/commands/upgrade-version-command/1-12/1-12-add-calendar-events-import-scheduled-sync-stage.command'; import { AddMessagesImportScheduledSyncStageCommand } from 'src/database/commands/upgrade-version-command/1-12/1-12-add-messages-import-scheduled-sync-stage.command'; +import { CleanNullEquivalentValuesCommand } from 'src/database/commands/upgrade-version-command/1-12/1-12-clean-null-equivalent-values'; import { CreateWorkspaceCustomApplicationCommand } from 'src/database/commands/upgrade-version-command/1-12/1-12-create-workspace-custom-application.command'; import { SetStandardApplicationNotUninstallableCommand } from 'src/database/commands/upgrade-version-command/1-12/1-12-set-standard-application-not-uninstallable.command'; import { WorkspaceCustomApplicationIdNonNullableCommand } from 'src/database/commands/upgrade-version-command/1-12/1-12-workspace-custom-application-id-non-nullable-migration.command'; @@ -90,6 +91,7 @@ export class UpgradeCommand extends UpgradeCommandRunner { protected readonly workspaceCustomApplicationIdNonNullableCommand: WorkspaceCustomApplicationIdNonNullableCommand, protected readonly addMessagesImportScheduledSyncStageCommand: AddMessagesImportScheduledSyncStageCommand, protected readonly addCalendarEventsImportScheduledSyncStageCommand: AddCalendarEventsImportScheduledSyncStageCommand, + protected readonly cleanNullEquivalentValuesCommand: CleanNullEquivalentValuesCommand, ) { super( workspaceRepository, @@ -152,6 +154,7 @@ export class UpgradeCommand extends UpgradeCommandRunner { this.workspaceCustomApplicationIdNonNullableCommand, this.addMessagesImportScheduledSyncStageCommand, this.addCalendarEventsImportScheduledSyncStageCommand, + this.cleanNullEquivalentValuesCommand, ], afterSyncMetadata: [this.setStandardApplicationNotUninstallableCommand], }; diff --git a/packages/twenty-server/src/engine/metadata-modules/field-metadata/utils/deprecated-generate-default-value.ts b/packages/twenty-server/src/engine/metadata-modules/field-metadata/utils/deprecated-generate-default-value.ts new file mode 100644 index 0000000000..3ba64e4253 --- /dev/null +++ b/packages/twenty-server/src/engine/metadata-modules/field-metadata/utils/deprecated-generate-default-value.ts @@ -0,0 +1,61 @@ +import { + FieldActorSource, + type FieldMetadataDefaultValue, + FieldMetadataType, +} from 'twenty-shared/types'; + +export function deprecatedGenerateDefaultValue( + type: FieldMetadataType, +): FieldMetadataDefaultValue { + switch (type) { + case FieldMetadataType.TEXT: + return "''" satisfies FieldMetadataDefaultValue; + case FieldMetadataType.EMAILS: + return { + primaryEmail: "''", + additionalEmails: null, + } satisfies FieldMetadataDefaultValue; + case FieldMetadataType.FULL_NAME: + return { + firstName: "''", + lastName: "''", + } satisfies FieldMetadataDefaultValue; + case FieldMetadataType.ADDRESS: + return { + addressStreet1: "''", + addressStreet2: "''", + addressCity: "''", + addressState: "''", + addressCountry: "''", + addressPostcode: "''", + addressLat: null, + addressLng: null, + } satisfies FieldMetadataDefaultValue; + case FieldMetadataType.CURRENCY: + return { + amountMicros: null, + currencyCode: "''", + } satisfies FieldMetadataDefaultValue; + case FieldMetadataType.LINKS: + return { + primaryLinkLabel: "''", + primaryLinkUrl: "''", + secondaryLinks: null, + } satisfies FieldMetadataDefaultValue; + case FieldMetadataType.PHONES: + return { + primaryPhoneNumber: "''", + primaryPhoneCountryCode: "''", + primaryPhoneCallingCode: "''", + additionalPhones: null, + } satisfies FieldMetadataDefaultValue; + case FieldMetadataType.ACTOR: + return { + source: `'${FieldActorSource.MANUAL}'`, + name: "'System'", + workspaceMemberId: null, + } satisfies FieldMetadataDefaultValue; + default: + return null; + } +}