fix(ai-tool): make search_output a raw-text occurrence search (#22034)

## Summary

`search_output` (the spilled-output navigation tool) was built around a
JSON-centric, line-based model that breaks for the data it actually
receives. Spilled outputs are written as compact
`JSON.stringify(output)` (single line, escaped newlines), so the tool's
line-by-line matching collapsed to at most one match, and its schema
described searching "the indented JSON representation" even though it
falls back to raw text for non-JSON. It also ran arbitrary,
model-supplied regexes through the native engine with no ReDoS
protection.

This reworks the tool into a `grep -o` style search over the raw file
bytes: it finds every occurrence of a pattern regardless of newlines and
returns a character window around each hit. It works uniformly for
compact/pretty JSON, CSV, HTML, and plain text.

## Changes

- **Occurrence-based matching** (`search-output.util.ts`): search the
raw content for every match via a global-regex `exec` loop (with a
zero-width-match guard), bounded by `offset + maxMatches`. Results are
now `{ charOffset, match, context }` with a character window around each
occurrence and a centered-ellipsis cap for very long single matches. The
line model (`split`, line numbers, line context) is removed.
- **ReDoS hardening**: matching now uses `re2` (already a dependency)
with the global flag, guaranteeing linear-time matching. Unsupported
regex features (lookahead/backreferences) and invalid patterns fall back
to escaped-literal search instead of throwing.
- **No more reserialization** (`search-output-tool.ts`): the
`JSON.stringify(JSON.parse(...))` round-trip is gone; the tool searches
the exact bytes on disk, so there is no coordinate divergence with
`extract_json_paths`.
- **API** (`search-output-tool.schema.ts`): `contextLines` →
`contextChars` (default 100, max 2000); honest descriptions reflecting
raw-text occurrence search and the regex-or-literal fallback. The result
message reports occurrence counts.
- **Cleanup**: removed unused constants
(`default-search-output-context-lines`,
`search-output-max-line-length`); added
`default-search-output-context-chars` and
`search-output-max-match-length`.

`extract_json_paths` and the spill service are untouched.

## Tradeoff

Results use character offsets/windows rather than line numbers and line
context. For an LLM extracting values from a spilled blob this is more
robust (works on single-line content); the cost is no line-based context
for genuinely line-structured content.

## Test plan

- [x] `search-output.util.spec.ts` rewritten for occurrence semantics:
multiple hits on a single newline-free line, zero-width-pattern
termination, catastrophic-backtracking pattern stays fast (RE2),
lookahead/invalid-regex literal fallback, char-window clipping, offset
pagination, long-match truncation. 12/12 pass.
- [x] `npx nx typecheck twenty-server` clean.
- [x] `npx nx lint:diff-with-main twenty-server` clean (lint + format).

## Deploy note

`re2` is a native addon. It was declared in `package.json` but never
imported/built before this PR, so its binary may be absent in some
environments (local install required `npm rebuild re2`). Confirm the
install/build pipeline (CI, Docker images) compiles native modules so
the tool doesn't throw `Cannot find module 're2.node'` at runtime.

<!-- This is an auto-generated description by cubic. -->
<a
href="https://cubic.dev/pr/twentyhq/twenty/pull/22034?utm_source=github"
target="_blank" rel="noopener noreferrer"
data-no-image-dialog="true"><picture><source
media="(prefers-color-scheme: dark)"
srcset="https://www.cubic.dev/buttons/review-in-cubic-dark.svg"><source
media="(prefers-color-scheme: light)"
srcset="https://www.cubic.dev/buttons/review-in-cubic-light.svg"><img
alt="Review in cubic"
src="https://www.cubic.dev/buttons/review-in-cubic-dark.svg"></picture></a>
<!-- End of auto-generated description by cubic. -->
This commit is contained in:
Etienne
2026-06-23 19:18:46 +02:00
committed by GitHub
parent 2e9550914c
commit 0f4c4e69a9
10 changed files with 188 additions and 118 deletions
-1
View File
@@ -152,7 +152,6 @@
"pluralize": "8.0.0",
"postal-mime": "^2.6.1",
"psl": "^1.9.0",
"re2": "^1.25.0",
"react": "^19.2.0",
"react-dom": "^19.2.0",
"redis": "^4.7.0",
@@ -0,0 +1 @@
export const DEFAULT_SEARCH_OUTPUT_CONTEXT_CHARS = 100;
@@ -1 +0,0 @@
export const DEFAULT_SEARCH_OUTPUT_CONTEXT_LINES = 2;
@@ -1 +0,0 @@
export const SEARCH_OUTPUT_MAX_LINE_LENGTH = 500;
@@ -0,0 +1 @@
export const SEARCH_OUTPUT_MAX_MATCH_LENGTH = 500;
@@ -8,8 +8,9 @@ export const SearchOutputInputZodSchema = z.object({
),
pattern: z
.string()
.min(1)
.describe(
'Text or regular expression to search for. Matched line by line against the indented JSON representation of the file.',
'Text or regular expression to search for. Treated as a regular expression when valid, otherwise as a literal string. Matched against the raw file text.',
),
maxMatches: z
.number()
@@ -17,22 +18,22 @@ export const SearchOutputInputZodSchema = z.object({
.positive()
.max(100)
.optional()
.describe('Maximum number of matches to return (default 10).'),
.describe('Maximum number of occurrences to return (default 10).'),
offset: z
.number()
.int()
.nonnegative()
.optional()
.describe(
'Number of matches to skip before returning results, for stateless pagination (default 0).',
'Number of occurrences to skip before returning results, for stateless pagination (default 0).',
),
contextLines: z
contextChars: z
.number()
.int()
.nonnegative()
.max(10)
.max(2000)
.optional()
.describe(
'Lines of context to include before and after each match (default 2).',
'Number of characters of context to include before and after each occurrence (default 100).',
),
});
@@ -3,7 +3,7 @@ import { Injectable, Logger } from '@nestjs/common';
import { FileFolder } from 'twenty-shared/types';
import { FileService } from 'src/engine/core-modules/file/services/file.service';
import { DEFAULT_SEARCH_OUTPUT_CONTEXT_LINES } from 'src/engine/core-modules/tool/tools/output-navigation-tool/constants/default-search-output-context-lines.constant';
import { DEFAULT_SEARCH_OUTPUT_CONTEXT_CHARS } from 'src/engine/core-modules/tool/tools/output-navigation-tool/constants/default-search-output-context-chars.constant';
import { DEFAULT_SEARCH_OUTPUT_MAX_MATCHES } from 'src/engine/core-modules/tool/tools/output-navigation-tool/constants/default-search-output-max-matches.constant';
import { SearchOutputInputZodSchema } from 'src/engine/core-modules/tool/tools/output-navigation-tool/search-output-tool.schema';
import { searchOutput } from 'src/engine/core-modules/tool/tools/output-navigation-tool/utils/search-output.util';
@@ -13,20 +13,12 @@ import { type ToolOutput } from 'src/engine/core-modules/tool/types/tool-output.
import { type Tool } from 'src/engine/core-modules/tool/types/tool.type';
import { isDefined } from 'twenty-shared/utils';
type SearchOutputInput = {
fileId: string;
pattern: string;
maxMatches?: number;
offset?: number;
contextLines?: number;
};
@Injectable()
export class SearchOutputTool implements Tool {
private readonly logger = new Logger(SearchOutputTool.name);
description =
'Search (grep-like) within a large spilled tool output for a text or regex pattern, returning matching lines with surrounding context. Supports stateless pagination via offset. Use this to locate an error message or key in a file too large to inline.';
'Search (grep -o style) within a large spilled tool output for a text or regex pattern, returning every occurrence with surrounding characters of context. Works on raw text of any shape (CSV, HTML, stringified JSON, plain text), including single-line content. Supports stateless pagination via offset. Use this to locate an error message or key in a file too large to inline.';
inputSchema = SearchOutputInputZodSchema;
@@ -37,8 +29,23 @@ export class SearchOutputTool implements Tool {
context: ToolExecutionContext,
): Promise<ToolOutput> {
const { workspaceId } = context;
const { fileId, pattern, maxMatches, offset, contextLines } =
parameters as SearchOutputInput;
const parseResult = SearchOutputInputZodSchema.safeParse(parameters);
if (!parseResult.success) {
return {
success: false,
message: 'Invalid input for search output',
error: parseResult.error.issues
.map(
(issue) => `${issue.path.join('.') || '(root)'}: ${issue.message}`,
)
.join('; '),
};
}
const { fileId, pattern, maxMatches, offset, contextChars } =
parseResult.data;
let fileContent: { buffer: Buffer; mimeType: string } | null;
@@ -66,22 +73,14 @@ export class SearchOutputTool implements Tool {
};
}
const rawContent = fileContent.buffer.toString('utf-8');
let content: string;
try {
content = JSON.stringify(JSON.parse(rawContent), null, 2);
} catch {
content = rawContent;
}
const content = fileContent.buffer.toString('utf-8');
const result = searchOutput({
content,
pattern,
maxMatches: maxMatches ?? DEFAULT_SEARCH_OUTPUT_MAX_MATCHES,
offset: offset ?? 0,
contextLines: contextLines ?? DEFAULT_SEARCH_OUTPUT_CONTEXT_LINES,
contextChars: contextChars ?? DEFAULT_SEARCH_OUTPUT_CONTEXT_CHARS,
});
return {
@@ -89,7 +88,7 @@ export class SearchOutputTool implements Tool {
message:
result.totalMatches === 0
? `No matches for "${pattern}"`
: `Found ${result.totalMatches} match(es) for "${pattern}"`,
: `Found ${result.totalMatches} occurrence(s) for "${pattern}"`,
result,
};
}
@@ -1,6 +1,7 @@
import { SEARCH_OUTPUT_MAX_MATCH_LENGTH } from 'src/engine/core-modules/tool/tools/output-navigation-tool/constants/search-output-max-match-length.constant';
import { searchOutput } from 'src/engine/core-modules/tool/tools/output-navigation-tool/utils/search-output.util';
const content = [
const multiLineContent = [
'line 0 ok',
'line 1 error: boom',
'line 2 ok',
@@ -10,31 +11,53 @@ const content = [
].join('\n');
describe('searchOutput', () => {
it('returns matches with context and line numbers', () => {
it('returns each occurrence with a character window and offset', () => {
const result = searchOutput({
content,
content: multiLineContent,
pattern: 'error',
maxMatches: 10,
offset: 0,
contextLines: 1,
contextChars: 5,
});
expect(result.totalMatches).toBe(3);
expect(result.hasMore).toBe(false);
expect(result.matches[0]).toEqual({
lineNumber: 2,
match: 'line 1 error: boom',
context: '1: line 0 ok\n2: line 1 error: boom\n3: line 2 ok',
expect(result.matches).toHaveLength(3);
expect(result.matches[0].match).toBe('error');
expect(result.matches[0].charOffset).toBe(
multiLineContent.indexOf('error'),
);
expect(result.matches[0].context).toContain('error');
expect(result.matches[0].context.endsWith('…')).toBe(true);
});
it('finds every occurrence on a single newline-free line', () => {
const compact = '{"a":"error","b":"error","c":"error"}';
const result = searchOutput({
content: compact,
pattern: 'error',
maxMatches: 10,
offset: 0,
contextChars: 0,
});
expect(result.totalMatches).toBe(3);
expect(result.matches).toHaveLength(3);
expect(result.matches.map((match) => match.charOffset)).toEqual([
compact.indexOf('error'),
compact.indexOf('error', compact.indexOf('error') + 1),
compact.lastIndexOf('error'),
]);
});
it('caps results at maxMatches and reports hasMore', () => {
const result = searchOutput({
content,
content: multiLineContent,
pattern: 'error',
maxMatches: 2,
offset: 0,
contextLines: 0,
contextChars: 0,
});
expect(result.totalMatches).toBe(3);
@@ -44,26 +67,28 @@ describe('searchOutput', () => {
it('paginates with offset', () => {
const result = searchOutput({
content,
content: multiLineContent,
pattern: 'error',
maxMatches: 2,
offset: 2,
contextLines: 0,
contextChars: 0,
});
expect(result.totalMatches).toBe(3);
expect(result.matches).toHaveLength(1);
expect(result.matches[0].lineNumber).toBe(6);
expect(result.matches[0].charOffset).toBe(
multiLineContent.lastIndexOf('error'),
);
expect(result.hasMore).toBe(false);
});
it('returns an empty result for zero matches', () => {
const result = searchOutput({
content,
content: multiLineContent,
pattern: 'nonexistent',
maxMatches: 10,
offset: 0,
contextLines: 2,
contextChars: 2,
});
expect(result).toEqual({ matches: [], totalMatches: 0, hasMore: false });
@@ -71,26 +96,69 @@ describe('searchOutput', () => {
it('supports regex patterns', () => {
const result = searchOutput({
content,
content: multiLineContent,
pattern: 'splat|boom',
maxMatches: 10,
offset: 0,
contextLines: 0,
contextChars: 0,
});
expect(result.totalMatches).toBe(3);
});
it('terminates on a zero-width pattern instead of looping forever', () => {
const result = searchOutput({
content: 'abc',
pattern: 'x*',
maxMatches: 10,
offset: 0,
contextChars: 0,
});
expect(result.totalMatches).toBeGreaterThan(0);
expect(result.matches.every((match) => match.match === '')).toBe(true);
});
it('falls back to literal matching for invalid regex', () => {
const result = searchOutput({
content: 'a (b c\nd e f',
content: 'a (b c d e f',
pattern: '(b',
maxMatches: 10,
offset: 0,
contextLines: 0,
contextChars: 0,
});
expect(result.totalMatches).toBe(1);
expect(result.matches[0].match).toBe('a (b c');
expect(result.matches[0].match).toBe('(b');
});
it('truncates an overly long single match with a centered ellipsis', () => {
const longMatch = 'z'.repeat(SEARCH_OUTPUT_MAX_MATCH_LENGTH + 200);
const result = searchOutput({
content: longMatch,
pattern: 'z+',
maxMatches: 10,
offset: 0,
contextChars: 0,
});
expect(result.totalMatches).toBe(1);
expect(result.matches[0].match).toContain('…');
expect(result.matches[0].match.length).toBeLessThanOrEqual(
SEARCH_OUTPUT_MAX_MATCH_LENGTH,
);
});
it('throws for an empty pattern instead of matching everything', () => {
expect(() =>
searchOutput({
content: multiLineContent,
pattern: '',
maxMatches: 10,
offset: 0,
contextChars: 0,
}),
).toThrow('Search pattern must be a non-empty string.');
});
});
@@ -1,7 +1,10 @@
import { SEARCH_OUTPUT_MAX_LINE_LENGTH } from 'src/engine/core-modules/tool/tools/output-navigation-tool/constants/search-output-max-line-length.constant';
import { isNonEmptyString } from '@sniptt/guards';
import { SEARCH_OUTPUT_MAX_MATCH_LENGTH } from 'src/engine/core-modules/tool/tools/output-navigation-tool/constants/search-output-max-match-length.constant';
import { isDefined } from 'twenty-shared/utils';
export type SearchMatch = {
lineNumber: number;
charOffset: number;
match: string;
context: string;
};
@@ -17,64 +20,95 @@ const escapeRegExp = (value: string): string =>
const compilePattern = (pattern: string): RegExp => {
try {
return new RegExp(pattern);
return new RegExp(pattern, 'g');
} catch {
return new RegExp(escapeRegExp(pattern));
return new RegExp(escapeRegExp(pattern), 'g');
}
};
const truncateLine = (line: string): string =>
line.length > SEARCH_OUTPUT_MAX_LINE_LENGTH
? `${line.slice(0, SEARCH_OUTPUT_MAX_LINE_LENGTH)}`
: line;
const truncateMatch = (match: string): string => {
if (match.length <= SEARCH_OUTPUT_MAX_MATCH_LENGTH) {
return match;
}
const half = Math.floor((SEARCH_OUTPUT_MAX_MATCH_LENGTH - 1) / 2);
return `${match.slice(0, half)}${match.slice(match.length - half)}`;
};
const buildContext = ({
content,
index,
length,
contextChars,
}: {
content: string;
index: number;
length: number;
contextChars: number;
}): string => {
const start = Math.max(0, index - contextChars);
const end = Math.min(content.length, index + length + contextChars);
const prefix = start > 0 ? '…' : '';
const suffix = end < content.length ? '…' : '';
return `${prefix}${content.slice(start, end)}${suffix}`;
};
export const searchOutput = ({
content,
pattern,
maxMatches,
offset,
contextLines,
contextChars,
}: {
content: string;
pattern: string;
maxMatches: number;
offset: number;
contextLines: number;
contextChars: number;
}): SearchOutputResult => {
const lines = content.split('\n');
const regex = compilePattern(pattern);
const matchLineIndices: number[] = [];
for (let lineIndex = 0; lineIndex < lines.length; lineIndex++) {
if (regex.test(lines[lineIndex])) {
matchLineIndices.push(lineIndex);
}
if (!isNonEmptyString(pattern)) {
throw new Error('Search pattern must be a non-empty string.');
}
const totalMatches = matchLineIndices.length;
const selected = matchLineIndices.slice(offset, offset + maxMatches);
const regex = compilePattern(pattern);
const matches: SearchMatch[] = selected.map((lineIndex) => {
const start = Math.max(0, lineIndex - contextLines);
const end = Math.min(lines.length - 1, lineIndex + contextLines);
const matches: SearchMatch[] = [];
let totalMatches = 0;
const contextBlock: string[] = [];
let execResult = regex.exec(content);
for (let cursor = start; cursor <= end; cursor++) {
contextBlock.push(`${cursor + 1}: ${truncateLine(lines[cursor])}`);
while (isDefined(execResult)) {
const index = execResult.index;
const matched = execResult[0];
if (totalMatches >= offset && matches.length < maxMatches) {
matches.push({
charOffset: index,
match: truncateMatch(matched),
context: buildContext({
content,
index,
length: matched.length,
contextChars,
}),
});
}
return {
lineNumber: lineIndex + 1,
match: truncateLine(lines[lineIndex]),
context: contextBlock.join('\n'),
};
});
totalMatches += 1;
if (matched.length === 0) {
regex.lastIndex += 1;
}
execResult = regex.exec(content);
}
return {
matches,
totalMatches,
hasMore: offset + selected.length < totalMatches,
hasMore: offset + matches.length < totalMatches,
};
};
+1 -32
View File
@@ -36351,16 +36351,6 @@ __metadata:
languageName: node
linkType: hard
"install-artifact-from-github@npm:^1.6.0":
version: 1.6.0
resolution: "install-artifact-from-github@npm:1.6.0"
bin:
install-from-cache: bin/install-from-cache.js
save-to-github-cache: bin/save-to-github-cache.js
checksum: 10c0/e480c994a6230ba9c24b76e29908dcfb29977d3ee5b4a193b5c793139065aa0deb9bc5f475d11fd347c516b88ddb1a0f02935aa11fd1d3585761a4a4ccc1885c
languageName: node
linkType: hard
"internal-slot@npm:^1.1.0":
version: 1.1.0
resolution: "internal-slot@npm:1.1.0"
@@ -42148,15 +42138,6 @@ __metadata:
languageName: node
linkType: hard
"nan@npm:^2.27.0":
version: 2.27.0
resolution: "nan@npm:2.27.0"
dependencies:
node-gyp: "npm:latest"
checksum: 10c0/38eb00b06e40f0c65b6a98d75795f17d651a8b7b52f03873dff6902d0053f12e7638d7f64fc52bda6c8f8ec454d69636e3988c8a9eb2bc749c2d5c255ba55f4c
languageName: node
linkType: hard
"nan@npm:^2.4.0":
version: 2.26.2
resolution: "nan@npm:2.26.2"
@@ -42687,7 +42668,7 @@ __metadata:
languageName: node
linkType: hard
"node-gyp@npm:^13.0.0, node-gyp@npm:latest":
"node-gyp@npm:latest":
version: 13.0.0
resolution: "node-gyp@npm:13.0.0"
dependencies:
@@ -46361,17 +46342,6 @@ __metadata:
languageName: node
linkType: hard
"re2@npm:^1.25.0":
version: 1.25.0
resolution: "re2@npm:1.25.0"
dependencies:
install-artifact-from-github: "npm:^1.6.0"
nan: "npm:^2.27.0"
node-gyp: "npm:^13.0.0"
checksum: 10c0/a1b89e4d6c594d3fe675fc18f5ee3ce06ea83d66ebc5b3b2d66abf8c93c57757478969d1c934db4379ffba0575154c149d4168a567d192a936fced369d7c3ecf
languageName: node
linkType: hard
"react-aria@npm:^3.48.0":
version: 3.49.0
resolution: "react-aria@npm:3.49.0"
@@ -52860,7 +52830,6 @@ __metadata:
postal-mime: "npm:^2.6.1"
prettier: "npm:^3.1.1"
psl: "npm:^1.9.0"
re2: "npm:^1.25.0"
react: "npm:^19.2.0"
react-dom: "npm:^19.2.0"
redis: "npm:^4.7.0"