Introduce a command to regenerate search vectors for standard and custom objects to fix accent issue. (#15175)

Legacy workspaces still hold the old stored expression, which omits
public.unaccent_immutable, so their tsvectors remain accented and can’t
match the new, unaccented queries. Metadata sync doesn’t touch
asExpression, so only a targeted drop/recreate fixes the underlying
column.

In simpler words, the search vector should contain `mader` instead of
`mäder` for the search to work properly. Therefore, this command
regenerates the search vector across every object that uses
`SEARCH_FIELDS_FOR_*`.

Note that dashboard has a searchVector, but breaks the pattern of using
`SEARCH_FIELDS_FOR_DASHBOARD`. If you look at
packages/twenty-server/src/modules/dashboard/standard-objects/dashboard.workspace-entity.ts:116,
the searchVector field is hard-coded as

```
asExpression: `to_tsvector('english', title)`
```

Therefore, the following code snippet.

```
    const storedExpression = hasAsExpressionSetting(
      searchVectorFieldMetadata.settings,
    )
      ? searchVectorFieldMetadata.settings.asExpression
      : undefined;

    if (storedExpression) {
      return storedExpression;
    }

    return undefined;
```

It checks whether the searchVector field already carries its own
asExpression value in metadata. If the settings object includes that
string, it returns it so the upgrade can reuse the existing expression
for objects that aren’t in our predefined lists. If not, it returns
undefined, signaling there’s no stored expression to fall back on.

---------

Co-authored-by: Félix Malfait <felix.malfait@gmail.com>
Co-authored-by: Félix Malfait <felix@twenty.com>

packages/twenty-server/src/database/commands/upgrade-version-command/1-10/1-10-regenerate-search-vectors.command.ts

@@ -0,0 +1,425 @@
+import { InjectDataSource, InjectRepository } from '@nestjs/typeorm';
+
+import { Command } from 'nest-commander';
+import { DataSource, In, Repository, type QueryRunner } from 'typeorm';
+
+import {
+  ActiveOrSuspendedWorkspacesMigrationCommandRunner,
+  type RunOnWorkspaceArgs,
+} from 'src/database/commands/command-runners/active-or-suspended-workspaces-migration.command-runner';
+import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity';
+import { SEARCH_VECTOR_FIELD } from 'src/engine/metadata-modules/constants/search-vector-field.constants';
+import { IndexMetadataEntity } from 'src/engine/metadata-modules/index-metadata/index-metadata.entity';
+import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity';
+import { SEARCH_FIELDS_FOR_CUSTOM_OBJECT } from 'src/engine/twenty-orm/custom.workspace-entity';
+import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager';
+import { WorkspaceSchemaManagerService } from 'src/engine/twenty-orm/workspace-schema-manager/workspace-schema-manager.service';
+import { computeObjectTargetTable } from 'src/engine/utils/compute-object-target-table.util';
+import { getWorkspaceSchemaName } from 'src/engine/workspace-datasource/utils/get-workspace-schema-name.util';
+import { STANDARD_OBJECT_IDS } from 'src/engine/workspace-manager/workspace-sync-metadata/constants/standard-object-ids';
+import { getTsVectorColumnExpressionFromFields } from 'src/engine/workspace-manager/workspace-sync-metadata/utils/get-ts-vector-column-expression.util';
+import { type SearchableFieldType } from 'src/engine/workspace-manager/workspace-sync-metadata/utils/is-searchable-field.util';
+import { SEARCH_FIELDS_FOR_COMPANY } from 'src/modules/company/standard-objects/company.workspace-entity';
+import { SEARCH_FIELDS_FOR_DASHBOARD } from 'src/modules/dashboard/standard-objects/dashboard.workspace-entity';
+import { SEARCH_FIELDS_FOR_NOTES } from 'src/modules/note/standard-objects/note.workspace-entity';
+import { SEARCH_FIELDS_FOR_OPPORTUNITY } from 'src/modules/opportunity/standard-objects/opportunity.workspace-entity';
+import { SEARCH_FIELDS_FOR_PERSON } from 'src/modules/person/standard-objects/person.workspace-entity';
+import { SEARCH_FIELDS_FOR_TASKS } from 'src/modules/task/standard-objects/task.workspace-entity';
+import { SEARCH_FIELDS_FOR_WORKFLOW_RUNS } from 'src/modules/workflow/common/standard-objects/workflow-run.workspace-entity';
+import { SEARCH_FIELDS_FOR_WORKFLOW_VERSIONS } from 'src/modules/workflow/common/standard-objects/workflow-version.workspace-entity';
+import { SEARCH_FIELDS_FOR_WORKFLOWS } from 'src/modules/workflow/common/standard-objects/workflow.workspace-entity';
+import { SEARCH_FIELDS_FOR_WORKSPACE_MEMBER } from 'src/modules/workspace-member/standard-objects/workspace-member.workspace-entity';
+
+const STANDARD_SEARCH_EXPRESSIONS = buildStandardSearchExpressions();
+
+@Command({
+  name: 'upgrade:1-10:regenerate-search-vectors',
+  description:
+    'Regenerate searchVector generated columns using unaccent-aware expressions for every searchable object',
+})
+export class RegenerateSearchVectorsCommand extends ActiveOrSuspendedWorkspacesMigrationCommandRunner {
+  constructor(
+    @InjectRepository(WorkspaceEntity)
+    protected readonly workspaceRepository: Repository<WorkspaceEntity>,
+    protected readonly twentyORMGlobalManager: TwentyORMGlobalManager,
+    private readonly workspaceSchemaManager: WorkspaceSchemaManagerService,
+    @InjectRepository(ObjectMetadataEntity)
+    private readonly objectMetadataRepository: Repository<ObjectMetadataEntity>,
+    @InjectRepository(IndexMetadataEntity)
+    private readonly indexMetadataRepository: Repository<IndexMetadataEntity>,
+    @InjectDataSource()
+    private readonly coreDataSource: DataSource,
+  ) {
+    super(workspaceRepository, twentyORMGlobalManager);
+  }
+
+  override async runOnWorkspace({
+    workspaceId,
+    options,
+  }: RunOnWorkspaceArgs): Promise<void> {
+    await this.ensureUnaccentFunctionExists();
+
+    const schemaName = getWorkspaceSchemaName(workspaceId);
+    const isDryRun = options.dryRun || false;
+
+    const searchableObjects = await this.fetchSearchableObjects(workspaceId);
+
+    if (searchableObjects.length === 0) {
+      this.logger.log(
+        `No searchable objects found for workspace ${workspaceId}, skipping`,
+      );
+
+      return;
+    }
+
+    const objectIds = searchableObjects.map((obj) => obj.id);
+    const allIndexes = await this.fetchIndexesForObjects(objectIds);
+
+    let queryRunner: QueryRunner | undefined;
+
+    if (isDryRun) {
+      queryRunner = this.coreDataSource.createQueryRunner();
+      await queryRunner.connect();
+      await queryRunner.startTransaction();
+    }
+
+    let successCount = 0;
+    let skipCount = 0;
+    let errorCount = 0;
+
+    try {
+      for (const object of searchableObjects) {
+        try {
+          const result = await this.regenerateSearchVectorForObject(
+            queryRunner,
+            object,
+            allIndexes,
+            schemaName,
+            workspaceId,
+          );
+
+          if (result === 'skipped') {
+            skipCount++;
+          } else {
+            successCount++;
+          }
+        } catch (error) {
+          errorCount++;
+          this.logger.error(
+            `Failed to regenerate search vector for ${object.nameSingular} in workspace ${workspaceId}`,
+            error,
+          );
+
+          if (isDryRun) {
+            throw error;
+          }
+        }
+      }
+
+      if (isDryRun && queryRunner) {
+        await queryRunner.rollbackTransaction();
+        this.logger.log(
+          `DRY RUN: Would regenerate ${successCount} search vectors, skip ${skipCount}, ${errorCount} errors - rolled back all changes`,
+        );
+      } else {
+        this.logger.log(
+          `Search vector regeneration complete for workspace ${workspaceId}: ${successCount} succeeded, ${skipCount} skipped, ${errorCount} failed out of ${searchableObjects.length} total`,
+        );
+      }
+    } catch (error) {
+      if (queryRunner?.isTransactionActive) {
+        await queryRunner.rollbackTransaction();
+      }
+
+      throw error;
+    } finally {
+      if (queryRunner) {
+        await queryRunner.release();
+      }
+    }
+  }
+
+  private async fetchSearchableObjects(
+    workspaceId: string,
+  ): Promise<ObjectMetadataEntity[]> {
+    return this.objectMetadataRepository.find({
+      where: {
+        workspaceId,
+        isSearchable: true,
+      },
+      relations: ['fields'],
+    });
+  }
+
+  private async fetchIndexesForObjects(
+    objectIds: string[],
+  ): Promise<IndexMetadataEntity[]> {
+    return this.indexMetadataRepository.find({
+      where: {
+        objectMetadataId: In(objectIds),
+      },
+      relations: ['indexFieldMetadatas'],
+    });
+  }
+
+  private async regenerateSearchVectorForObject(
+    queryRunner: QueryRunner | undefined,
+    object: ObjectMetadataEntity,
+    allIndexes: IndexMetadataEntity[],
+    schemaName: string,
+    workspaceId: string,
+  ): Promise<'success' | 'skipped'> {
+    const searchVectorField = this.findSearchVectorField(object);
+
+    if (!searchVectorField) {
+      this.logger.warn(
+        `Search vector field not found for ${object.nameSingular} in workspace ${workspaceId}, skipping`,
+      );
+
+      return 'skipped';
+    }
+
+    const searchExpression = this.buildSearchExpression(object);
+
+    if (!searchExpression) {
+      this.logger.warn(
+        `Cannot determine search expression for ${object.nameSingular} in workspace ${workspaceId}, skipping`,
+      );
+
+      return 'skipped';
+    }
+
+    const tableName = computeObjectTargetTable({
+      nameSingular: object.nameSingular,
+      isCustom: object.isCustom,
+    });
+
+    const existingIndex = this.findSearchVectorIndex(
+      allIndexes,
+      object.id,
+      searchVectorField.id,
+    );
+
+    let localQueryRunner: QueryRunner;
+    const shouldRelease = !queryRunner;
+
+    if (queryRunner) {
+      localQueryRunner = queryRunner;
+    } else {
+      localQueryRunner = this.coreDataSource.createQueryRunner();
+      await localQueryRunner.connect();
+    }
+
+    try {
+      if (existingIndex) {
+        await this.dropIndex(localQueryRunner, schemaName, existingIndex.name);
+      }
+
+      await this.dropSearchVectorColumn(
+        localQueryRunner,
+        schemaName,
+        tableName,
+      );
+
+      await this.createSearchVectorColumn(
+        localQueryRunner,
+        schemaName,
+        tableName,
+        searchExpression,
+      );
+
+      if (existingIndex) {
+        await this.recreateIndex(
+          localQueryRunner,
+          schemaName,
+          tableName,
+          existingIndex,
+        );
+      }
+
+      this.logger.log(
+        `Regenerated search vector for ${object.nameSingular} in workspace ${workspaceId}`,
+      );
+
+      return 'success';
+    } finally {
+      if (shouldRelease) {
+        await localQueryRunner.release();
+      }
+    }
+  }
+
+  private findSearchVectorField(
+    object: ObjectMetadataEntity,
+  ): ObjectMetadataEntity['fields'][0] | undefined {
+    return object.fields.find(
+      (field) => field.name === SEARCH_VECTOR_FIELD.name,
+    );
+  }
+
+  private findSearchVectorIndex(
+    allIndexes: IndexMetadataEntity[],
+    objectId: string,
+    searchVectorFieldId: string,
+  ): IndexMetadataEntity | undefined {
+    return allIndexes.find(
+      (index) =>
+        index.objectMetadataId === objectId &&
+        index.indexFieldMetadatas.some(
+          (indexField) => indexField.fieldMetadataId === searchVectorFieldId,
+        ),
+    );
+  }
+
+  private buildSearchExpression(
+    object: ObjectMetadataEntity,
+  ): string | undefined {
+    if (object.standardId) {
+      const standardExpression = STANDARD_SEARCH_EXPRESSIONS[object.standardId];
+
+      if (standardExpression) {
+        return standardExpression;
+      }
+    }
+
+    if (object.isCustom) {
+      return this.buildCustomObjectSearchExpression(object);
+    }
+
+    return undefined;
+  }
+
+  private buildCustomObjectSearchExpression(
+    object: ObjectMetadataEntity,
+  ): string {
+    const labelField = object.fields.find(
+      (field) => field.id === object.labelIdentifierFieldMetadataId,
+    );
+
+    if (!labelField) {
+      return getTsVectorColumnExpressionFromFields(
+        SEARCH_FIELDS_FOR_CUSTOM_OBJECT,
+      );
+    }
+
+    return getTsVectorColumnExpressionFromFields([
+      {
+        name: labelField.name,
+        type: labelField.type as SearchableFieldType,
+      },
+    ]);
+  }
+
+  private async dropIndex(
+    queryRunner: QueryRunner,
+    schemaName: string,
+    indexName: string,
+  ): Promise<void> {
+    await this.workspaceSchemaManager.indexManager.dropIndex({
+      queryRunner,
+      schemaName,
+      indexName,
+    });
+  }
+
+  private async dropSearchVectorColumn(
+    queryRunner: QueryRunner,
+    schemaName: string,
+    tableName: string,
+  ): Promise<void> {
+    await this.workspaceSchemaManager.columnManager.dropColumns({
+      queryRunner,
+      schemaName,
+      tableName,
+      columnNames: [SEARCH_VECTOR_FIELD.name],
+    });
+  }
+
+  private async createSearchVectorColumn(
+    queryRunner: QueryRunner,
+    schemaName: string,
+    tableName: string,
+    expression: string,
+  ): Promise<void> {
+    await this.workspaceSchemaManager.columnManager.addColumns({
+      queryRunner,
+      schemaName,
+      tableName,
+      columnDefinitions: [
+        {
+          name: SEARCH_VECTOR_FIELD.name,
+          type: 'tsvector',
+          isNullable: true,
+          generatedType: 'STORED',
+          asExpression: expression,
+        },
+      ],
+    });
+  }
+
+  private async recreateIndex(
+    queryRunner: QueryRunner,
+    schemaName: string,
+    tableName: string,
+    index: IndexMetadataEntity,
+  ): Promise<void> {
+    await this.workspaceSchemaManager.indexManager.createIndex({
+      queryRunner,
+      schemaName,
+      tableName,
+      index: {
+        name: index.name,
+        columns: [SEARCH_VECTOR_FIELD.name],
+        type: index.indexType,
+        isUnique: index.isUnique,
+        where: index.indexWhereClause ?? undefined,
+      },
+    });
+  }
+
+  private async ensureUnaccentFunctionExists(): Promise<void> {
+    const result = await this.coreDataSource.query(`
+      SELECT EXISTS (
+        SELECT 1 FROM pg_proc p
+        JOIN pg_namespace n ON p.pronamespace = n.oid
+        WHERE n.nspname = 'public'
+        AND p.proname = 'unaccent_immutable'
+      ) as function_exists
+    `);
+
+    if (!result[0]?.function_exists) {
+      throw new Error(
+        'The public.unaccent_immutable() function is required but not found. Please run database migrations first.',
+      );
+    }
+  }
+}
+
+function buildStandardSearchExpressions(): Partial<Record<string, string>> {
+  const standardObjectSearchFields = {
+    person: SEARCH_FIELDS_FOR_PERSON,
+    company: SEARCH_FIELDS_FOR_COMPANY,
+    opportunity: SEARCH_FIELDS_FOR_OPPORTUNITY,
+    task: SEARCH_FIELDS_FOR_TASKS,
+    note: SEARCH_FIELDS_FOR_NOTES,
+    dashboard: SEARCH_FIELDS_FOR_DASHBOARD,
+    workspaceMember: SEARCH_FIELDS_FOR_WORKSPACE_MEMBER,
+    workflow: SEARCH_FIELDS_FOR_WORKFLOWS,
+    workflowVersion: SEARCH_FIELDS_FOR_WORKFLOW_VERSIONS,
+    workflowRun: SEARCH_FIELDS_FOR_WORKFLOW_RUNS,
+  };
+
+  const expressions: Partial<Record<string, string>> = {};
+
+  for (const [objectKey, searchFields] of Object.entries(
+    standardObjectSearchFields,
+  )) {
+    const standardId =
+      STANDARD_OBJECT_IDS[objectKey as keyof typeof standardObjectSearchFields];
+
+    expressions[standardId] =
+      getTsVectorColumnExpressionFromFields(searchFields);
+  }
+
+  return expressions;
+}

packages/twenty-server/src/database/commands/upgrade-version-command/1-10/1-10-upgrade-version-command.module.ts

@@ -4,23 +4,33 @@ import { TypeOrmModule } from '@nestjs/typeorm';
 import { AddWorkflowRunStopStatusesCommand } from 'src/database/commands/upgrade-version-command/1-10/1-10-add-workflow-run-stop-statuses.command';
 import { MigrateAttachmentAuthorToCreatedByCommand } from 'src/database/commands/upgrade-version-command/1-10/1-10-migrate-attachment-author-to-created-by.command';
 import { MigrateAttachmentTypeToFileCategoryCommand } from 'src/database/commands/upgrade-version-command/1-10/1-10-migrate-attachment-type-to-file-category.command';
+import { RegenerateSearchVectorsCommand } from 'src/database/commands/upgrade-version-command/1-10/1-10-regenerate-search-vectors.command';
 import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity';
 import { FieldMetadataEntity } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity';
+import { IndexMetadataEntity } from 'src/engine/metadata-modules/index-metadata/index-metadata.entity';
+import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity';
 import { WorkspaceSchemaManagerModule } from 'src/engine/twenty-orm/workspace-schema-manager/workspace-schema-manager.module';
 
 @Module({
   imports: [
-    TypeOrmModule.forFeature([WorkspaceEntity, FieldMetadataEntity]),
+    TypeOrmModule.forFeature([
+      WorkspaceEntity,
+      ObjectMetadataEntity,
+      FieldMetadataEntity,
+      IndexMetadataEntity,
+    ]),
     WorkspaceSchemaManagerModule,
   ],
   providers: [
     MigrateAttachmentAuthorToCreatedByCommand,
     MigrateAttachmentTypeToFileCategoryCommand,
+    RegenerateSearchVectorsCommand,
     AddWorkflowRunStopStatusesCommand,
   ],
   exports: [
     MigrateAttachmentAuthorToCreatedByCommand,
     MigrateAttachmentTypeToFileCategoryCommand,
+    RegenerateSearchVectorsCommand,
     AddWorkflowRunStopStatusesCommand,
   ],
 })

packages/twenty-server/src/database/commands/upgrade-version-command/upgrade.command.ts

@@ -22,6 +22,7 @@ import { MigrateWorkflowRunStatesCommand } from 'src/database/commands/upgrade-v
 import { AddWorkflowRunStopStatusesCommand } from 'src/database/commands/upgrade-version-command/1-10/1-10-add-workflow-run-stop-statuses.command';
 import { MigrateAttachmentAuthorToCreatedByCommand } from 'src/database/commands/upgrade-version-command/1-10/1-10-migrate-attachment-author-to-created-by.command';
 import { MigrateAttachmentTypeToFileCategoryCommand } from 'src/database/commands/upgrade-version-command/1-10/1-10-migrate-attachment-type-to-file-category.command';
+import { RegenerateSearchVectorsCommand } from 'src/database/commands/upgrade-version-command/1-10/1-10-regenerate-search-vectors.command';
 import { AddEnqueuedStatusToWorkflowRunV2Command } from 'src/database/commands/upgrade-version-command/1-2/1-2-add-enqueued-status-to-workflow-run-v2.command';
 import { AddNextStepIdsToWorkflowVersionTriggers } from 'src/database/commands/upgrade-version-command/1-2/1-2-add-next-step-ids-to-workflow-version-triggers.command';
 import { RemoveWorkflowRunsWithoutState } from 'src/database/commands/upgrade-version-command/1-2/1-2-remove-workflow-runs-without-state.command';
@@ -102,6 +103,7 @@ export class UpgradeCommand extends UpgradeCommandRunner {
     // 1.10 Commands
     protected readonly migrateAttachmentAuthorToCreatedByCommand: MigrateAttachmentAuthorToCreatedByCommand,
     protected readonly migrateAttachmentTypeToFileCategoryCommand: MigrateAttachmentTypeToFileCategoryCommand,
+    protected readonly regenerateSearchVectorsCommand: RegenerateSearchVectorsCommand,
     protected readonly addWorkflowRunStopStatusesCommand: AddWorkflowRunStopStatusesCommand,
   ) {
     super(
@@ -208,7 +210,10 @@ export class UpgradeCommand extends UpgradeCommandRunner {
     };
 
     const commands_1100: VersionCommands = {
-      beforeSyncMetadata: [this.addWorkflowRunStopStatusesCommand],
+      beforeSyncMetadata: [
+        this.regenerateSearchVectorsCommand,
+        this.addWorkflowRunStopStatusesCommand,
+      ],
       afterSyncMetadata: [
         this.migrateAttachmentAuthorToCreatedByCommand,
         this.migrateAttachmentTypeToFileCategoryCommand,

packages/twenty-server/src/modules/dashboard/standard-objects/dashboard.workspace-entity.ts

@@ -22,10 +22,20 @@ import { WorkspaceRelation } from 'src/engine/twenty-orm/decorators/workspace-re
 import { DASHBOARD_STANDARD_FIELD_IDS } from 'src/engine/workspace-manager/workspace-sync-metadata/constants/standard-field-ids';
 import { STANDARD_OBJECT_ICONS } from 'src/engine/workspace-manager/workspace-sync-metadata/constants/standard-object-icons';
 import { STANDARD_OBJECT_IDS } from 'src/engine/workspace-manager/workspace-sync-metadata/constants/standard-object-ids';
+import {
+  type FieldTypeAndNameMetadata,
+  getTsVectorColumnExpressionFromFields,
+} from 'src/engine/workspace-manager/workspace-sync-metadata/utils/get-ts-vector-column-expression.util';
 import { AttachmentWorkspaceEntity } from 'src/modules/attachment/standard-objects/attachment.workspace-entity';
 import { FavoriteWorkspaceEntity } from 'src/modules/favorite/standard-objects/favorite.workspace-entity';
 import { TimelineActivityWorkspaceEntity } from 'src/modules/timeline/standard-objects/timeline-activity.workspace-entity';
 
+const TITLE_FIELD_NAME = 'title';
+
+export const SEARCH_FIELDS_FOR_DASHBOARD: FieldTypeAndNameMetadata[] = [
+  { name: TITLE_FIELD_NAME, type: FieldMetadataType.TEXT },
+];
+
 @WorkspaceEntity({
   standardId: STANDARD_OBJECT_IDS.dashboard,
   namePlural: 'dashboards',
@@ -123,7 +133,9 @@ export class DashboardWorkspaceEntity extends BaseWorkspaceEntity {
     description: SEARCH_VECTOR_FIELD.description,
     icon: 'IconUser',
     generatedType: 'STORED',
-    asExpression: `to_tsvector('english', title)`,
+    asExpression: getTsVectorColumnExpressionFromFields(
+      SEARCH_FIELDS_FOR_DASHBOARD,
+    ),
   })
   @WorkspaceIsNullable()
   @WorkspaceIsSystem()