Fix global search for CJK and non-tokenizable text (#18030)
## Summary Fixes #12962 - Adds a two-pass ILIKE fallback to the global search service (`search.service.ts`) - **Fast path**: runs the tsvector query first (uses GIN index, sub-millisecond) - **Fallback**: only if tsvector returns fewer results than the limit, runs an ILIKE query on `searchVector::text` to catch cases where PostgreSQL's `simple` text search config fails to tokenize (continuous CJK text, etc.) - Zero performance impact for the common case (Latin text where tsvector works) - Also adds `escapeForIlike` utility to properly escape `%`, `_`, `\` in user input ### Why tsvector fails for CJK PostgreSQL's `simple` config treats continuous CJK text as a single lexeme: - `to_tsvector('simple', '示例商业线索')` → `'示例商业线索':1` - Searching `商业:*` only prefix-matches from the start, so it misses `商业` in the middle The ILIKE fallback catches these substring matches when the tsvector path can't. ### What this fixes - Global search (command menu / sidebar) - Relation picker (single and multi-object) - Morph relation picker All three use `search.service.ts` under the hood. Co-authored-by: mykh-hailo (original direction in #18021) ## Test plan - [ ] Search `示例` with records `示例商业线索` and `示例-商业-线索` → both should appear - [ ] Search `商业` → both should appear (previously only the hyphenated one did) - [ ] Search for Latin text (e.g. `john`) → same performance, results unchanged - [ ] Relation picker search with CJK text → results appear - [ ] Search input with special chars like `%` or `_` → no SQL injection, results correct Made with [Cursor](https://cursor.com) --------- Co-authored-by: mykh-hailo <mykh-hailo@users.noreply.github.com> Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
Félix Malfait
@@ -25,6 +25,7 @@ import {
|
||||
SearchExceptionCode,
|
||||
} from 'src/engine/core-modules/search/exceptions/search.exception';
|
||||
import { type RecordsWithObjectMetadataItem } from 'src/engine/core-modules/search/types/records-with-object-metadata-item';
|
||||
import { escapeForIlike } from 'src/engine/core-modules/search/utils/escape-for-ilike';
|
||||
import { formatSearchTerms } from 'src/engine/core-modules/search/utils/format-search-terms';
|
||||
import { findFlatEntityByIdInFlatEntityMaps } from 'src/engine/metadata-modules/flat-entity/utils/find-flat-entity-by-id-in-flat-entity-maps.util';
|
||||
import { type FlatEntityMaps } from 'src/engine/metadata-modules/flat-entity/types/flat-entity-maps.type';
|
||||
@@ -98,10 +99,11 @@ export class SearchService {
|
||||
|
||||
return {
|
||||
objectMetadataItem: flatObjectMetadata,
|
||||
records: await this.buildSearchQueryAndGetRecords({
|
||||
records: await this.buildSearchQueryAndGetRecordsWithFallback({
|
||||
entityManager: repository,
|
||||
flatObjectMetadata,
|
||||
flatFieldMetadataMaps,
|
||||
searchInput,
|
||||
searchTerms: formatSearchTerms(searchInput, 'and'),
|
||||
searchTermsOr: formatSearchTerms(searchInput, 'or'),
|
||||
limit: limit as number,
|
||||
@@ -149,6 +151,70 @@ export class SearchService {
|
||||
);
|
||||
}
|
||||
|
||||
// Runs a fast tsvector query first (uses GIN index). On the first page only,
|
||||
// if not enough results, falls back to an ILIKE query on the searchVector
|
||||
// text representation to catch cases where tsvector tokenization fails
|
||||
// (e.g. continuous CJK text). Skipped on subsequent pages since any ILIKE-only
|
||||
// matches would already have appeared on page 1 with rank 0.
|
||||
async buildSearchQueryAndGetRecordsWithFallback<
|
||||
Entity extends ObjectLiteral,
|
||||
>({
|
||||
entityManager,
|
||||
flatObjectMetadata,
|
||||
flatFieldMetadataMaps,
|
||||
searchInput,
|
||||
searchTerms,
|
||||
searchTermsOr,
|
||||
limit,
|
||||
filter,
|
||||
after,
|
||||
}: {
|
||||
entityManager: WorkspaceRepository<Entity>;
|
||||
flatObjectMetadata: FlatObjectMetadata;
|
||||
flatFieldMetadataMaps: FlatEntityMaps<FlatFieldMetadata>;
|
||||
searchInput: string;
|
||||
searchTerms: string;
|
||||
searchTermsOr: string;
|
||||
limit: number;
|
||||
filter: ObjectRecordFilterInput;
|
||||
after?: string;
|
||||
}) {
|
||||
const tsvectorResults = await this.buildSearchQueryAndGetRecords({
|
||||
entityManager,
|
||||
flatObjectMetadata,
|
||||
flatFieldMetadataMaps,
|
||||
searchTerms,
|
||||
searchTermsOr,
|
||||
limit,
|
||||
filter,
|
||||
after,
|
||||
});
|
||||
|
||||
if (
|
||||
tsvectorResults.length >= limit ||
|
||||
!isNonEmptyString(searchInput.trim()) ||
|
||||
isDefined(after)
|
||||
) {
|
||||
return tsvectorResults;
|
||||
}
|
||||
|
||||
const tsvectorRecordIds = new Set(
|
||||
tsvectorResults.map((record) => record.id as string),
|
||||
);
|
||||
|
||||
const fallbackResults = await this.buildIlikeFallbackQuery({
|
||||
entityManager,
|
||||
flatObjectMetadata,
|
||||
flatFieldMetadataMaps,
|
||||
searchInput,
|
||||
excludeIds: [...tsvectorRecordIds],
|
||||
limit: limit + 1 - tsvectorResults.length,
|
||||
filter,
|
||||
});
|
||||
|
||||
return [...tsvectorResults, ...fallbackResults];
|
||||
}
|
||||
|
||||
async buildSearchQueryAndGetRecords<Entity extends ObjectLiteral>({
|
||||
entityManager,
|
||||
flatObjectMetadata,
|
||||
@@ -250,6 +316,74 @@ export class SearchService {
|
||||
.getRawMany();
|
||||
}
|
||||
|
||||
private async buildIlikeFallbackQuery<Entity extends ObjectLiteral>({
|
||||
entityManager,
|
||||
flatObjectMetadata,
|
||||
flatFieldMetadataMaps,
|
||||
searchInput,
|
||||
excludeIds,
|
||||
limit,
|
||||
filter,
|
||||
}: {
|
||||
entityManager: WorkspaceRepository<Entity>;
|
||||
flatObjectMetadata: FlatObjectMetadata;
|
||||
flatFieldMetadataMaps: FlatEntityMaps<FlatFieldMetadata>;
|
||||
searchInput: string;
|
||||
excludeIds: string[];
|
||||
limit: number;
|
||||
filter: ObjectRecordFilterInput;
|
||||
}) {
|
||||
const queryBuilder = entityManager.createQueryBuilder();
|
||||
|
||||
const { flatObjectMetadataMaps } = entityManager.internalContext;
|
||||
|
||||
const queryParser = new GraphqlQueryParser(
|
||||
flatObjectMetadata,
|
||||
flatObjectMetadataMaps,
|
||||
flatFieldMetadataMaps,
|
||||
);
|
||||
|
||||
queryParser.applyFilterToBuilder(
|
||||
queryBuilder,
|
||||
flatObjectMetadata.nameSingular,
|
||||
filter,
|
||||
);
|
||||
|
||||
queryParser.applyDeletedAtToBuilder(queryBuilder, filter);
|
||||
|
||||
const imageIdentifierField = this.getImageIdentifierColumn(
|
||||
flatObjectMetadata,
|
||||
flatFieldMetadataMaps,
|
||||
);
|
||||
|
||||
const fieldsToSelect = [
|
||||
'id',
|
||||
...this.getLabelIdentifierColumns(
|
||||
flatObjectMetadata,
|
||||
flatFieldMetadataMaps,
|
||||
),
|
||||
...(imageIdentifierField ? [imageIdentifierField] : []),
|
||||
].map((field) => `"${field}"`);
|
||||
|
||||
queryBuilder
|
||||
.select(fieldsToSelect)
|
||||
.addSelect('0', 'tsRankCD')
|
||||
.addSelect('0', 'tsRank');
|
||||
|
||||
const escapedInput = escapeForIlike(searchInput.trim());
|
||||
|
||||
queryBuilder.andWhere(
|
||||
`public.unaccent_immutable("${SEARCH_VECTOR_FIELD.name}"::text) ILIKE public.unaccent_immutable(:ilikeFallbackPattern)`,
|
||||
{ ilikeFallbackPattern: `%${escapedInput}%` },
|
||||
);
|
||||
|
||||
if (excludeIds.length > 0) {
|
||||
queryBuilder.andWhere('id NOT IN (:...excludeIds)', { excludeIds });
|
||||
}
|
||||
|
||||
return await queryBuilder.orderBy('"id"', 'ASC').take(limit).getRawMany();
|
||||
}
|
||||
|
||||
computeCursorWhereCondition({
|
||||
after,
|
||||
objectMetadataNameSingular,
|
||||
|
||||
+27
@@ -0,0 +1,27 @@
|
||||
import { escapeForIlike } from 'src/engine/core-modules/search/utils/escape-for-ilike';
|
||||
|
||||
describe('escapeForIlike', () => {
|
||||
it('should escape percent signs', () => {
|
||||
expect(escapeForIlike('100%')).toBe('100\\%');
|
||||
});
|
||||
|
||||
it('should escape underscores', () => {
|
||||
expect(escapeForIlike('my_company')).toBe('my\\_company');
|
||||
});
|
||||
|
||||
it('should escape backslashes', () => {
|
||||
expect(escapeForIlike('path\\to')).toBe('path\\\\to');
|
||||
});
|
||||
|
||||
it('should leave normal text unchanged', () => {
|
||||
expect(escapeForIlike('hello world')).toBe('hello world');
|
||||
});
|
||||
|
||||
it('should handle CJK text unchanged', () => {
|
||||
expect(escapeForIlike('商业线索')).toBe('商业线索');
|
||||
});
|
||||
|
||||
it('should handle multiple special characters', () => {
|
||||
expect(escapeForIlike('50%_off\\deal')).toBe('50\\%\\_off\\\\deal');
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,2 @@
|
||||
export const escapeForIlike = (value: string): string =>
|
||||
value.replace(/[\\%_]/g, '\\$&');
|
||||
Reference in New Issue
Block a user