diff --git a/packages/twenty-server/src/engine/core-modules/search/services/search.service.ts b/packages/twenty-server/src/engine/core-modules/search/services/search.service.ts index ffbd026d5b..c191243572 100644 --- a/packages/twenty-server/src/engine/core-modules/search/services/search.service.ts +++ b/packages/twenty-server/src/engine/core-modules/search/services/search.service.ts @@ -25,6 +25,7 @@ import { SearchExceptionCode, } from 'src/engine/core-modules/search/exceptions/search.exception'; import { type RecordsWithObjectMetadataItem } from 'src/engine/core-modules/search/types/records-with-object-metadata-item'; +import { escapeForIlike } from 'src/engine/core-modules/search/utils/escape-for-ilike'; import { formatSearchTerms } from 'src/engine/core-modules/search/utils/format-search-terms'; import { findFlatEntityByIdInFlatEntityMaps } from 'src/engine/metadata-modules/flat-entity/utils/find-flat-entity-by-id-in-flat-entity-maps.util'; import { type FlatEntityMaps } from 'src/engine/metadata-modules/flat-entity/types/flat-entity-maps.type'; @@ -98,10 +99,11 @@ export class SearchService { return { objectMetadataItem: flatObjectMetadata, - records: await this.buildSearchQueryAndGetRecords({ + records: await this.buildSearchQueryAndGetRecordsWithFallback({ entityManager: repository, flatObjectMetadata, flatFieldMetadataMaps, + searchInput, searchTerms: formatSearchTerms(searchInput, 'and'), searchTermsOr: formatSearchTerms(searchInput, 'or'), limit: limit as number, @@ -149,6 +151,70 @@ export class SearchService { ); } + // Runs a fast tsvector query first (uses GIN index). On the first page only, + // if not enough results, falls back to an ILIKE query on the searchVector + // text representation to catch cases where tsvector tokenization fails + // (e.g. continuous CJK text). Skipped on subsequent pages since any ILIKE-only + // matches would already have appeared on page 1 with rank 0. + async buildSearchQueryAndGetRecordsWithFallback< + Entity extends ObjectLiteral, + >({ + entityManager, + flatObjectMetadata, + flatFieldMetadataMaps, + searchInput, + searchTerms, + searchTermsOr, + limit, + filter, + after, + }: { + entityManager: WorkspaceRepository; + flatObjectMetadata: FlatObjectMetadata; + flatFieldMetadataMaps: FlatEntityMaps; + searchInput: string; + searchTerms: string; + searchTermsOr: string; + limit: number; + filter: ObjectRecordFilterInput; + after?: string; + }) { + const tsvectorResults = await this.buildSearchQueryAndGetRecords({ + entityManager, + flatObjectMetadata, + flatFieldMetadataMaps, + searchTerms, + searchTermsOr, + limit, + filter, + after, + }); + + if ( + tsvectorResults.length >= limit || + !isNonEmptyString(searchInput.trim()) || + isDefined(after) + ) { + return tsvectorResults; + } + + const tsvectorRecordIds = new Set( + tsvectorResults.map((record) => record.id as string), + ); + + const fallbackResults = await this.buildIlikeFallbackQuery({ + entityManager, + flatObjectMetadata, + flatFieldMetadataMaps, + searchInput, + excludeIds: [...tsvectorRecordIds], + limit: limit + 1 - tsvectorResults.length, + filter, + }); + + return [...tsvectorResults, ...fallbackResults]; + } + async buildSearchQueryAndGetRecords({ entityManager, flatObjectMetadata, @@ -250,6 +316,74 @@ export class SearchService { .getRawMany(); } + private async buildIlikeFallbackQuery({ + entityManager, + flatObjectMetadata, + flatFieldMetadataMaps, + searchInput, + excludeIds, + limit, + filter, + }: { + entityManager: WorkspaceRepository; + flatObjectMetadata: FlatObjectMetadata; + flatFieldMetadataMaps: FlatEntityMaps; + searchInput: string; + excludeIds: string[]; + limit: number; + filter: ObjectRecordFilterInput; + }) { + const queryBuilder = entityManager.createQueryBuilder(); + + const { flatObjectMetadataMaps } = entityManager.internalContext; + + const queryParser = new GraphqlQueryParser( + flatObjectMetadata, + flatObjectMetadataMaps, + flatFieldMetadataMaps, + ); + + queryParser.applyFilterToBuilder( + queryBuilder, + flatObjectMetadata.nameSingular, + filter, + ); + + queryParser.applyDeletedAtToBuilder(queryBuilder, filter); + + const imageIdentifierField = this.getImageIdentifierColumn( + flatObjectMetadata, + flatFieldMetadataMaps, + ); + + const fieldsToSelect = [ + 'id', + ...this.getLabelIdentifierColumns( + flatObjectMetadata, + flatFieldMetadataMaps, + ), + ...(imageIdentifierField ? [imageIdentifierField] : []), + ].map((field) => `"${field}"`); + + queryBuilder + .select(fieldsToSelect) + .addSelect('0', 'tsRankCD') + .addSelect('0', 'tsRank'); + + const escapedInput = escapeForIlike(searchInput.trim()); + + queryBuilder.andWhere( + `public.unaccent_immutable("${SEARCH_VECTOR_FIELD.name}"::text) ILIKE public.unaccent_immutable(:ilikeFallbackPattern)`, + { ilikeFallbackPattern: `%${escapedInput}%` }, + ); + + if (excludeIds.length > 0) { + queryBuilder.andWhere('id NOT IN (:...excludeIds)', { excludeIds }); + } + + return await queryBuilder.orderBy('"id"', 'ASC').take(limit).getRawMany(); + } + computeCursorWhereCondition({ after, objectMetadataNameSingular, diff --git a/packages/twenty-server/src/engine/core-modules/search/utils/__tests__/escape-for-ilike.spec.ts b/packages/twenty-server/src/engine/core-modules/search/utils/__tests__/escape-for-ilike.spec.ts new file mode 100644 index 0000000000..25ba0eca83 --- /dev/null +++ b/packages/twenty-server/src/engine/core-modules/search/utils/__tests__/escape-for-ilike.spec.ts @@ -0,0 +1,27 @@ +import { escapeForIlike } from 'src/engine/core-modules/search/utils/escape-for-ilike'; + +describe('escapeForIlike', () => { + it('should escape percent signs', () => { + expect(escapeForIlike('100%')).toBe('100\\%'); + }); + + it('should escape underscores', () => { + expect(escapeForIlike('my_company')).toBe('my\\_company'); + }); + + it('should escape backslashes', () => { + expect(escapeForIlike('path\\to')).toBe('path\\\\to'); + }); + + it('should leave normal text unchanged', () => { + expect(escapeForIlike('hello world')).toBe('hello world'); + }); + + it('should handle CJK text unchanged', () => { + expect(escapeForIlike('商业线索')).toBe('商业线索'); + }); + + it('should handle multiple special characters', () => { + expect(escapeForIlike('50%_off\\deal')).toBe('50\\%\\_off\\\\deal'); + }); +}); diff --git a/packages/twenty-server/src/engine/core-modules/search/utils/escape-for-ilike.ts b/packages/twenty-server/src/engine/core-modules/search/utils/escape-for-ilike.ts new file mode 100644 index 0000000000..ad574bd950 --- /dev/null +++ b/packages/twenty-server/src/engine/core-modules/search/utils/escape-for-ilike.ts @@ -0,0 +1,2 @@ +export const escapeForIlike = (value: string): string => + value.replace(/[\\%_]/g, '\\$&'); diff --git a/packages/twenty-server/test/integration/constants/test-pet-ids.constants.ts b/packages/twenty-server/test/integration/constants/test-pet-ids.constants.ts index 0cadfb85f5..dc52aa2a75 100644 --- a/packages/twenty-server/test/integration/constants/test-pet-ids.constants.ts +++ b/packages/twenty-server/test/integration/constants/test-pet-ids.constants.ts @@ -2,3 +2,4 @@ export const TEST_PET_ID_1 = 'a4907cff-a582-4daf-8635-ad6c782c7c25'; export const TEST_PET_ID_2 = 'c4e97187-9b9b-4e1f-a3c5-b7883c590332'; export const TEST_PET_ID_3 = 'e4907cff-a582-4aaf-8635-ad6c782c7c26'; export const TEST_PET_ID_4 = 'f4907cff-a582-4aaf-8635-ad6c782c7c27'; +export const TEST_PET_ID_5 = 'd4907cff-a582-4aaf-8635-ad6c782c7c28'; diff --git a/packages/twenty-server/test/integration/graphql/suites/search/search-resolver.integration-spec.ts b/packages/twenty-server/test/integration/graphql/suites/search/search-resolver.integration-spec.ts index 4904efc8f9..582e00d0bb 100644 --- a/packages/twenty-server/test/integration/graphql/suites/search/search-resolver.integration-spec.ts +++ b/packages/twenty-server/test/integration/graphql/suites/search/search-resolver.integration-spec.ts @@ -21,6 +21,7 @@ import { TEST_PET_ID_2, TEST_PET_ID_3, TEST_PET_ID_4, + TEST_PET_ID_5, } from 'test/integration/constants/test-pet-ids.constants'; import { createManyOperation } from 'test/integration/graphql/utils/create-many-operation.util'; import { search } from 'test/integration/graphql/utils/search.util'; @@ -145,6 +146,7 @@ describe('SearchResolver', () => { { id: TEST_PET_ID_2, name: 'searchInput2' }, { id: TEST_PET_ID_3, name: 'Café' }, { id: TEST_PET_ID_4, name: 'Naïve' }, + { id: TEST_PET_ID_5, name: '示例商业线索' }, ]; const [ @@ -159,7 +161,7 @@ describe('SearchResolver', () => { multiPhonePerson, ] = persons; const [cafeCorp, naiveCorp] = companies; - const [searchInput1Pet, searchInput2Pet, cafePet, naivePet] = pets; + const [searchInput1Pet, searchInput2Pet, cafePet, naivePet, cjkPet] = pets; beforeAll(async () => { // TODO refactor not a good practice, or should at least restore afterwards @@ -236,6 +238,7 @@ describe('SearchResolver', () => { cafeCorp.id, searchInput1Pet.id, searchInput2Pet.id, + cjkPet.id, cafePet.id, naivePet.id, ], @@ -297,6 +300,7 @@ describe('SearchResolver', () => { orderedRecordIds: [ searchInput1Pet.id, searchInput2Pet.id, + cjkPet.id, cafePet.id, naivePet.id, ], @@ -331,6 +335,7 @@ describe('SearchResolver', () => { cafeCorp.id, searchInput1Pet.id, searchInput2Pet.id, + cjkPet.id, cafePet.id, naivePet.id, ], @@ -1298,6 +1303,34 @@ describe('SearchResolver', () => { }, }, }, + { + title: + 'should find CJK records via ILIKE fallback when tsvector tokenization fails', + context: { + input: { + searchInput: '商业', + excludedObjectNameSingulars: [ + 'workspaceMember', + 'employmentHistory', + 'petCareAgreement', + ], + includedObjectNameSingulars: ['pet'], + limit: 50, + }, + eval: { + orderedRecordIds: [cjkPet.id], + pageInfo: { + hasNextPage: false, + decodedEndCursor: { + lastRanks: { tsRank: 0, tsRankCD: 0 }, + lastRecordIdsPerObject: { + pet: cjkPet.id, + }, + }, + }, + }, + }, + }, ]; it.each(eachTestingContextFilter(testsUseCases))(