From c7c671f3e16a031125f18cd49e9358565e2f25ae Mon Sep 17 00:00:00 2001 From: "Abdullah." <125115953+mabdullahabaid@users.noreply.github.com> Date: Fri, 24 Oct 2025 15:16:23 +0500 Subject: [PATCH] Introduce a command to regenerate search vectors for standard and custom objects to fix accent issue. (#15175) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Legacy workspaces still hold the old stored expression, which omits public.unaccent_immutable, so their tsvectors remain accented and can’t match the new, unaccented queries. Metadata sync doesn’t touch asExpression, so only a targeted drop/recreate fixes the underlying column. In simpler words, the search vector should contain `mader` instead of `mäder` for the search to work properly. Therefore, this command regenerates the search vector across every object that uses `SEARCH_FIELDS_FOR_*`. Note that dashboard has a searchVector, but breaks the pattern of using `SEARCH_FIELDS_FOR_DASHBOARD`. If you look at packages/twenty-server/src/modules/dashboard/standard-objects/dashboard.workspace-entity.ts:116, the searchVector field is hard-coded as ``` asExpression: `to_tsvector('english', title)` ``` Therefore, the following code snippet. ``` const storedExpression = hasAsExpressionSetting( searchVectorFieldMetadata.settings, ) ? searchVectorFieldMetadata.settings.asExpression : undefined; if (storedExpression) { return storedExpression; } return undefined; ``` It checks whether the searchVector field already carries its own asExpression value in metadata. If the settings object includes that string, it returns it so the upgrade can reuse the existing expression for objects that aren’t in our predefined lists. If not, it returns undefined, signaling there’s no stored expression to fall back on. --------- Co-authored-by: Félix Malfait Co-authored-by: Félix Malfait --- .../1-10-regenerate-search-vectors.command.ts | 425 ++++++++++++++++++ .../1-10-upgrade-version-command.module.ts | 12 +- .../upgrade.command.ts | 7 +- .../dashboard.workspace-entity.ts | 14 +- 4 files changed, 455 insertions(+), 3 deletions(-) create mode 100644 packages/twenty-server/src/database/commands/upgrade-version-command/1-10/1-10-regenerate-search-vectors.command.ts diff --git a/packages/twenty-server/src/database/commands/upgrade-version-command/1-10/1-10-regenerate-search-vectors.command.ts b/packages/twenty-server/src/database/commands/upgrade-version-command/1-10/1-10-regenerate-search-vectors.command.ts new file mode 100644 index 0000000000..520934c030 --- /dev/null +++ b/packages/twenty-server/src/database/commands/upgrade-version-command/1-10/1-10-regenerate-search-vectors.command.ts @@ -0,0 +1,425 @@ +import { InjectDataSource, InjectRepository } from '@nestjs/typeorm'; + +import { Command } from 'nest-commander'; +import { DataSource, In, Repository, type QueryRunner } from 'typeorm'; + +import { + ActiveOrSuspendedWorkspacesMigrationCommandRunner, + type RunOnWorkspaceArgs, +} from 'src/database/commands/command-runners/active-or-suspended-workspaces-migration.command-runner'; +import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity'; +import { SEARCH_VECTOR_FIELD } from 'src/engine/metadata-modules/constants/search-vector-field.constants'; +import { IndexMetadataEntity } from 'src/engine/metadata-modules/index-metadata/index-metadata.entity'; +import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity'; +import { SEARCH_FIELDS_FOR_CUSTOM_OBJECT } from 'src/engine/twenty-orm/custom.workspace-entity'; +import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager'; +import { WorkspaceSchemaManagerService } from 'src/engine/twenty-orm/workspace-schema-manager/workspace-schema-manager.service'; +import { computeObjectTargetTable } from 'src/engine/utils/compute-object-target-table.util'; +import { getWorkspaceSchemaName } from 'src/engine/workspace-datasource/utils/get-workspace-schema-name.util'; +import { STANDARD_OBJECT_IDS } from 'src/engine/workspace-manager/workspace-sync-metadata/constants/standard-object-ids'; +import { getTsVectorColumnExpressionFromFields } from 'src/engine/workspace-manager/workspace-sync-metadata/utils/get-ts-vector-column-expression.util'; +import { type SearchableFieldType } from 'src/engine/workspace-manager/workspace-sync-metadata/utils/is-searchable-field.util'; +import { SEARCH_FIELDS_FOR_COMPANY } from 'src/modules/company/standard-objects/company.workspace-entity'; +import { SEARCH_FIELDS_FOR_DASHBOARD } from 'src/modules/dashboard/standard-objects/dashboard.workspace-entity'; +import { SEARCH_FIELDS_FOR_NOTES } from 'src/modules/note/standard-objects/note.workspace-entity'; +import { SEARCH_FIELDS_FOR_OPPORTUNITY } from 'src/modules/opportunity/standard-objects/opportunity.workspace-entity'; +import { SEARCH_FIELDS_FOR_PERSON } from 'src/modules/person/standard-objects/person.workspace-entity'; +import { SEARCH_FIELDS_FOR_TASKS } from 'src/modules/task/standard-objects/task.workspace-entity'; +import { SEARCH_FIELDS_FOR_WORKFLOW_RUNS } from 'src/modules/workflow/common/standard-objects/workflow-run.workspace-entity'; +import { SEARCH_FIELDS_FOR_WORKFLOW_VERSIONS } from 'src/modules/workflow/common/standard-objects/workflow-version.workspace-entity'; +import { SEARCH_FIELDS_FOR_WORKFLOWS } from 'src/modules/workflow/common/standard-objects/workflow.workspace-entity'; +import { SEARCH_FIELDS_FOR_WORKSPACE_MEMBER } from 'src/modules/workspace-member/standard-objects/workspace-member.workspace-entity'; + +const STANDARD_SEARCH_EXPRESSIONS = buildStandardSearchExpressions(); + +@Command({ + name: 'upgrade:1-10:regenerate-search-vectors', + description: + 'Regenerate searchVector generated columns using unaccent-aware expressions for every searchable object', +}) +export class RegenerateSearchVectorsCommand extends ActiveOrSuspendedWorkspacesMigrationCommandRunner { + constructor( + @InjectRepository(WorkspaceEntity) + protected readonly workspaceRepository: Repository, + protected readonly twentyORMGlobalManager: TwentyORMGlobalManager, + private readonly workspaceSchemaManager: WorkspaceSchemaManagerService, + @InjectRepository(ObjectMetadataEntity) + private readonly objectMetadataRepository: Repository, + @InjectRepository(IndexMetadataEntity) + private readonly indexMetadataRepository: Repository, + @InjectDataSource() + private readonly coreDataSource: DataSource, + ) { + super(workspaceRepository, twentyORMGlobalManager); + } + + override async runOnWorkspace({ + workspaceId, + options, + }: RunOnWorkspaceArgs): Promise { + await this.ensureUnaccentFunctionExists(); + + const schemaName = getWorkspaceSchemaName(workspaceId); + const isDryRun = options.dryRun || false; + + const searchableObjects = await this.fetchSearchableObjects(workspaceId); + + if (searchableObjects.length === 0) { + this.logger.log( + `No searchable objects found for workspace ${workspaceId}, skipping`, + ); + + return; + } + + const objectIds = searchableObjects.map((obj) => obj.id); + const allIndexes = await this.fetchIndexesForObjects(objectIds); + + let queryRunner: QueryRunner | undefined; + + if (isDryRun) { + queryRunner = this.coreDataSource.createQueryRunner(); + await queryRunner.connect(); + await queryRunner.startTransaction(); + } + + let successCount = 0; + let skipCount = 0; + let errorCount = 0; + + try { + for (const object of searchableObjects) { + try { + const result = await this.regenerateSearchVectorForObject( + queryRunner, + object, + allIndexes, + schemaName, + workspaceId, + ); + + if (result === 'skipped') { + skipCount++; + } else { + successCount++; + } + } catch (error) { + errorCount++; + this.logger.error( + `Failed to regenerate search vector for ${object.nameSingular} in workspace ${workspaceId}`, + error, + ); + + if (isDryRun) { + throw error; + } + } + } + + if (isDryRun && queryRunner) { + await queryRunner.rollbackTransaction(); + this.logger.log( + `DRY RUN: Would regenerate ${successCount} search vectors, skip ${skipCount}, ${errorCount} errors - rolled back all changes`, + ); + } else { + this.logger.log( + `Search vector regeneration complete for workspace ${workspaceId}: ${successCount} succeeded, ${skipCount} skipped, ${errorCount} failed out of ${searchableObjects.length} total`, + ); + } + } catch (error) { + if (queryRunner?.isTransactionActive) { + await queryRunner.rollbackTransaction(); + } + + throw error; + } finally { + if (queryRunner) { + await queryRunner.release(); + } + } + } + + private async fetchSearchableObjects( + workspaceId: string, + ): Promise { + return this.objectMetadataRepository.find({ + where: { + workspaceId, + isSearchable: true, + }, + relations: ['fields'], + }); + } + + private async fetchIndexesForObjects( + objectIds: string[], + ): Promise { + return this.indexMetadataRepository.find({ + where: { + objectMetadataId: In(objectIds), + }, + relations: ['indexFieldMetadatas'], + }); + } + + private async regenerateSearchVectorForObject( + queryRunner: QueryRunner | undefined, + object: ObjectMetadataEntity, + allIndexes: IndexMetadataEntity[], + schemaName: string, + workspaceId: string, + ): Promise<'success' | 'skipped'> { + const searchVectorField = this.findSearchVectorField(object); + + if (!searchVectorField) { + this.logger.warn( + `Search vector field not found for ${object.nameSingular} in workspace ${workspaceId}, skipping`, + ); + + return 'skipped'; + } + + const searchExpression = this.buildSearchExpression(object); + + if (!searchExpression) { + this.logger.warn( + `Cannot determine search expression for ${object.nameSingular} in workspace ${workspaceId}, skipping`, + ); + + return 'skipped'; + } + + const tableName = computeObjectTargetTable({ + nameSingular: object.nameSingular, + isCustom: object.isCustom, + }); + + const existingIndex = this.findSearchVectorIndex( + allIndexes, + object.id, + searchVectorField.id, + ); + + let localQueryRunner: QueryRunner; + const shouldRelease = !queryRunner; + + if (queryRunner) { + localQueryRunner = queryRunner; + } else { + localQueryRunner = this.coreDataSource.createQueryRunner(); + await localQueryRunner.connect(); + } + + try { + if (existingIndex) { + await this.dropIndex(localQueryRunner, schemaName, existingIndex.name); + } + + await this.dropSearchVectorColumn( + localQueryRunner, + schemaName, + tableName, + ); + + await this.createSearchVectorColumn( + localQueryRunner, + schemaName, + tableName, + searchExpression, + ); + + if (existingIndex) { + await this.recreateIndex( + localQueryRunner, + schemaName, + tableName, + existingIndex, + ); + } + + this.logger.log( + `Regenerated search vector for ${object.nameSingular} in workspace ${workspaceId}`, + ); + + return 'success'; + } finally { + if (shouldRelease) { + await localQueryRunner.release(); + } + } + } + + private findSearchVectorField( + object: ObjectMetadataEntity, + ): ObjectMetadataEntity['fields'][0] | undefined { + return object.fields.find( + (field) => field.name === SEARCH_VECTOR_FIELD.name, + ); + } + + private findSearchVectorIndex( + allIndexes: IndexMetadataEntity[], + objectId: string, + searchVectorFieldId: string, + ): IndexMetadataEntity | undefined { + return allIndexes.find( + (index) => + index.objectMetadataId === objectId && + index.indexFieldMetadatas.some( + (indexField) => indexField.fieldMetadataId === searchVectorFieldId, + ), + ); + } + + private buildSearchExpression( + object: ObjectMetadataEntity, + ): string | undefined { + if (object.standardId) { + const standardExpression = STANDARD_SEARCH_EXPRESSIONS[object.standardId]; + + if (standardExpression) { + return standardExpression; + } + } + + if (object.isCustom) { + return this.buildCustomObjectSearchExpression(object); + } + + return undefined; + } + + private buildCustomObjectSearchExpression( + object: ObjectMetadataEntity, + ): string { + const labelField = object.fields.find( + (field) => field.id === object.labelIdentifierFieldMetadataId, + ); + + if (!labelField) { + return getTsVectorColumnExpressionFromFields( + SEARCH_FIELDS_FOR_CUSTOM_OBJECT, + ); + } + + return getTsVectorColumnExpressionFromFields([ + { + name: labelField.name, + type: labelField.type as SearchableFieldType, + }, + ]); + } + + private async dropIndex( + queryRunner: QueryRunner, + schemaName: string, + indexName: string, + ): Promise { + await this.workspaceSchemaManager.indexManager.dropIndex({ + queryRunner, + schemaName, + indexName, + }); + } + + private async dropSearchVectorColumn( + queryRunner: QueryRunner, + schemaName: string, + tableName: string, + ): Promise { + await this.workspaceSchemaManager.columnManager.dropColumns({ + queryRunner, + schemaName, + tableName, + columnNames: [SEARCH_VECTOR_FIELD.name], + }); + } + + private async createSearchVectorColumn( + queryRunner: QueryRunner, + schemaName: string, + tableName: string, + expression: string, + ): Promise { + await this.workspaceSchemaManager.columnManager.addColumns({ + queryRunner, + schemaName, + tableName, + columnDefinitions: [ + { + name: SEARCH_VECTOR_FIELD.name, + type: 'tsvector', + isNullable: true, + generatedType: 'STORED', + asExpression: expression, + }, + ], + }); + } + + private async recreateIndex( + queryRunner: QueryRunner, + schemaName: string, + tableName: string, + index: IndexMetadataEntity, + ): Promise { + await this.workspaceSchemaManager.indexManager.createIndex({ + queryRunner, + schemaName, + tableName, + index: { + name: index.name, + columns: [SEARCH_VECTOR_FIELD.name], + type: index.indexType, + isUnique: index.isUnique, + where: index.indexWhereClause ?? undefined, + }, + }); + } + + private async ensureUnaccentFunctionExists(): Promise { + const result = await this.coreDataSource.query(` + SELECT EXISTS ( + SELECT 1 FROM pg_proc p + JOIN pg_namespace n ON p.pronamespace = n.oid + WHERE n.nspname = 'public' + AND p.proname = 'unaccent_immutable' + ) as function_exists + `); + + if (!result[0]?.function_exists) { + throw new Error( + 'The public.unaccent_immutable() function is required but not found. Please run database migrations first.', + ); + } + } +} + +function buildStandardSearchExpressions(): Partial> { + const standardObjectSearchFields = { + person: SEARCH_FIELDS_FOR_PERSON, + company: SEARCH_FIELDS_FOR_COMPANY, + opportunity: SEARCH_FIELDS_FOR_OPPORTUNITY, + task: SEARCH_FIELDS_FOR_TASKS, + note: SEARCH_FIELDS_FOR_NOTES, + dashboard: SEARCH_FIELDS_FOR_DASHBOARD, + workspaceMember: SEARCH_FIELDS_FOR_WORKSPACE_MEMBER, + workflow: SEARCH_FIELDS_FOR_WORKFLOWS, + workflowVersion: SEARCH_FIELDS_FOR_WORKFLOW_VERSIONS, + workflowRun: SEARCH_FIELDS_FOR_WORKFLOW_RUNS, + }; + + const expressions: Partial> = {}; + + for (const [objectKey, searchFields] of Object.entries( + standardObjectSearchFields, + )) { + const standardId = + STANDARD_OBJECT_IDS[objectKey as keyof typeof standardObjectSearchFields]; + + expressions[standardId] = + getTsVectorColumnExpressionFromFields(searchFields); + } + + return expressions; +} diff --git a/packages/twenty-server/src/database/commands/upgrade-version-command/1-10/1-10-upgrade-version-command.module.ts b/packages/twenty-server/src/database/commands/upgrade-version-command/1-10/1-10-upgrade-version-command.module.ts index 327a3e030c..ce4bbbb574 100644 --- a/packages/twenty-server/src/database/commands/upgrade-version-command/1-10/1-10-upgrade-version-command.module.ts +++ b/packages/twenty-server/src/database/commands/upgrade-version-command/1-10/1-10-upgrade-version-command.module.ts @@ -4,23 +4,33 @@ import { TypeOrmModule } from '@nestjs/typeorm'; import { AddWorkflowRunStopStatusesCommand } from 'src/database/commands/upgrade-version-command/1-10/1-10-add-workflow-run-stop-statuses.command'; import { MigrateAttachmentAuthorToCreatedByCommand } from 'src/database/commands/upgrade-version-command/1-10/1-10-migrate-attachment-author-to-created-by.command'; import { MigrateAttachmentTypeToFileCategoryCommand } from 'src/database/commands/upgrade-version-command/1-10/1-10-migrate-attachment-type-to-file-category.command'; +import { RegenerateSearchVectorsCommand } from 'src/database/commands/upgrade-version-command/1-10/1-10-regenerate-search-vectors.command'; import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity'; import { FieldMetadataEntity } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity'; +import { IndexMetadataEntity } from 'src/engine/metadata-modules/index-metadata/index-metadata.entity'; +import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity'; import { WorkspaceSchemaManagerModule } from 'src/engine/twenty-orm/workspace-schema-manager/workspace-schema-manager.module'; @Module({ imports: [ - TypeOrmModule.forFeature([WorkspaceEntity, FieldMetadataEntity]), + TypeOrmModule.forFeature([ + WorkspaceEntity, + ObjectMetadataEntity, + FieldMetadataEntity, + IndexMetadataEntity, + ]), WorkspaceSchemaManagerModule, ], providers: [ MigrateAttachmentAuthorToCreatedByCommand, MigrateAttachmentTypeToFileCategoryCommand, + RegenerateSearchVectorsCommand, AddWorkflowRunStopStatusesCommand, ], exports: [ MigrateAttachmentAuthorToCreatedByCommand, MigrateAttachmentTypeToFileCategoryCommand, + RegenerateSearchVectorsCommand, AddWorkflowRunStopStatusesCommand, ], }) diff --git a/packages/twenty-server/src/database/commands/upgrade-version-command/upgrade.command.ts b/packages/twenty-server/src/database/commands/upgrade-version-command/upgrade.command.ts index 6301015319..b4f2816b31 100644 --- a/packages/twenty-server/src/database/commands/upgrade-version-command/upgrade.command.ts +++ b/packages/twenty-server/src/database/commands/upgrade-version-command/upgrade.command.ts @@ -22,6 +22,7 @@ import { MigrateWorkflowRunStatesCommand } from 'src/database/commands/upgrade-v import { AddWorkflowRunStopStatusesCommand } from 'src/database/commands/upgrade-version-command/1-10/1-10-add-workflow-run-stop-statuses.command'; import { MigrateAttachmentAuthorToCreatedByCommand } from 'src/database/commands/upgrade-version-command/1-10/1-10-migrate-attachment-author-to-created-by.command'; import { MigrateAttachmentTypeToFileCategoryCommand } from 'src/database/commands/upgrade-version-command/1-10/1-10-migrate-attachment-type-to-file-category.command'; +import { RegenerateSearchVectorsCommand } from 'src/database/commands/upgrade-version-command/1-10/1-10-regenerate-search-vectors.command'; import { AddEnqueuedStatusToWorkflowRunV2Command } from 'src/database/commands/upgrade-version-command/1-2/1-2-add-enqueued-status-to-workflow-run-v2.command'; import { AddNextStepIdsToWorkflowVersionTriggers } from 'src/database/commands/upgrade-version-command/1-2/1-2-add-next-step-ids-to-workflow-version-triggers.command'; import { RemoveWorkflowRunsWithoutState } from 'src/database/commands/upgrade-version-command/1-2/1-2-remove-workflow-runs-without-state.command'; @@ -102,6 +103,7 @@ export class UpgradeCommand extends UpgradeCommandRunner { // 1.10 Commands protected readonly migrateAttachmentAuthorToCreatedByCommand: MigrateAttachmentAuthorToCreatedByCommand, protected readonly migrateAttachmentTypeToFileCategoryCommand: MigrateAttachmentTypeToFileCategoryCommand, + protected readonly regenerateSearchVectorsCommand: RegenerateSearchVectorsCommand, protected readonly addWorkflowRunStopStatusesCommand: AddWorkflowRunStopStatusesCommand, ) { super( @@ -208,7 +210,10 @@ export class UpgradeCommand extends UpgradeCommandRunner { }; const commands_1100: VersionCommands = { - beforeSyncMetadata: [this.addWorkflowRunStopStatusesCommand], + beforeSyncMetadata: [ + this.regenerateSearchVectorsCommand, + this.addWorkflowRunStopStatusesCommand, + ], afterSyncMetadata: [ this.migrateAttachmentAuthorToCreatedByCommand, this.migrateAttachmentTypeToFileCategoryCommand, diff --git a/packages/twenty-server/src/modules/dashboard/standard-objects/dashboard.workspace-entity.ts b/packages/twenty-server/src/modules/dashboard/standard-objects/dashboard.workspace-entity.ts index 1daf738c72..16fe2bcae9 100644 --- a/packages/twenty-server/src/modules/dashboard/standard-objects/dashboard.workspace-entity.ts +++ b/packages/twenty-server/src/modules/dashboard/standard-objects/dashboard.workspace-entity.ts @@ -22,10 +22,20 @@ import { WorkspaceRelation } from 'src/engine/twenty-orm/decorators/workspace-re import { DASHBOARD_STANDARD_FIELD_IDS } from 'src/engine/workspace-manager/workspace-sync-metadata/constants/standard-field-ids'; import { STANDARD_OBJECT_ICONS } from 'src/engine/workspace-manager/workspace-sync-metadata/constants/standard-object-icons'; import { STANDARD_OBJECT_IDS } from 'src/engine/workspace-manager/workspace-sync-metadata/constants/standard-object-ids'; +import { + type FieldTypeAndNameMetadata, + getTsVectorColumnExpressionFromFields, +} from 'src/engine/workspace-manager/workspace-sync-metadata/utils/get-ts-vector-column-expression.util'; import { AttachmentWorkspaceEntity } from 'src/modules/attachment/standard-objects/attachment.workspace-entity'; import { FavoriteWorkspaceEntity } from 'src/modules/favorite/standard-objects/favorite.workspace-entity'; import { TimelineActivityWorkspaceEntity } from 'src/modules/timeline/standard-objects/timeline-activity.workspace-entity'; +const TITLE_FIELD_NAME = 'title'; + +export const SEARCH_FIELDS_FOR_DASHBOARD: FieldTypeAndNameMetadata[] = [ + { name: TITLE_FIELD_NAME, type: FieldMetadataType.TEXT }, +]; + @WorkspaceEntity({ standardId: STANDARD_OBJECT_IDS.dashboard, namePlural: 'dashboards', @@ -123,7 +133,9 @@ export class DashboardWorkspaceEntity extends BaseWorkspaceEntity { description: SEARCH_VECTOR_FIELD.description, icon: 'IconUser', generatedType: 'STORED', - asExpression: `to_tsvector('english', title)`, + asExpression: getTsVectorColumnExpressionFromFields( + SEARCH_FIELDS_FOR_DASHBOARD, + ), }) @WorkspaceIsNullable() @WorkspaceIsSystem()