fix: prepend UTF-8 BOM to fix non-Latin characters in csv (#19246)

Fixes: #19230 

CSV exports containing Arabic, Chinese, Japanese, Korean characters
displayed as garbled text in Excel because the file lacked a UTF-8 BOM.
So I added `\uFEFF` prefix to the CSV Blob so Excel and other
spreadsheet apps correctly interpret the file as UTF-8

## Table value
<img width="370" height="277" alt="image"
src="https://github.com/user-attachments/assets/ed668f00-93f0-4991-bc87-42e3cb314dbc"
/>

## Before
<img width="206" height="160" alt="image"
src="https://github.com/user-attachments/assets/a5324dbc-bd65-4443-a7fb-78047028bf91"
/>

## After
<img width="180" height="155" alt="image"
src="https://github.com/user-attachments/assets/cae5f38d-6c91-4017-96ae-9924bb41d0e0"
/>


And those are my test data

Script | Example | Language
-- | -- | --
Arabic | مرحبا | Arabic, Persian, Urdu
Chinese | 你好 | Chinese
Japanese | こんにちは | Japanese
Korean | 안녕하세요 | Korean

---------

Co-authored-by: Claude <noreply@anthropic.com>
Co-authored-by: Félix Malfait <felix@twenty.com>
Co-authored-by: Félix Malfait <felix.malfait@gmail.com>
This commit is contained in:
BugIsGod
2026-04-02 18:23:22 +01:00
committed by GitHub
parent a7b2bec529
commit 43baf7b91c
2 changed files with 65 additions and 2 deletions
@@ -2,11 +2,17 @@ import { type FieldMetadata } from '@/object-record/record-field/ui/types/FieldM
import { type ColumnDefinition } from '@/object-record/record-table/types/ColumnDefinition';
import { CSV_INJECTION_PREVENTION_ZWJ } from '@/spreadsheet-import/constants/CsvInjectionPreventionZwj';
import { FieldMetadataType, RelationType } from '~/generated-metadata/graphql';
import {
csvDownloader,
displayedExportProgress,
generateCsv,
} from '@/object-record/record-index/export/hooks/useRecordIndexExportRecords';
import { saveAs } from 'file-saver';
import { FieldMetadataType, RelationType } from '~/generated-metadata/graphql';
jest.mock('file-saver', () => ({
saveAs: jest.fn(),
}));
jest.useFakeTimers();
@@ -63,7 +69,7 @@ describe('generateCsv', () => {
];
const csv = generateCsv({ columns, rows });
expect(csv)
.toEqual(`Id,Foo,Empty,Nested link field / Link URL,Nested link field / Secondary Links,Relation
.toEqual(`\uFEFFId,Foo,Empty,Nested link field / Link URL,Nested link field / Secondary Links,Relation
1,some field,,https://www.test.com,"[{""label"":""secondary link 1"",""url"":""https://www.test.com""},{""label"":""secondary link 2"",""url"":""https://www.test.com""}]",a relation`);
});
@@ -436,6 +442,62 @@ describe('generateCsv', () => {
});
});
describe('csvDownloader', () => {
const mockSaveAs = saveAs as jest.MockedFunction<typeof saveAs>;
beforeEach(() => {
mockSaveAs.mockClear();
});
const readBlob = async (blob: Blob): Promise<ArrayBuffer> =>
new Promise<ArrayBuffer>((resolve, reject) => {
const reader = new FileReader();
reader.onload = () => resolve(reader.result as ArrayBuffer);
reader.onerror = reject;
reader.readAsArrayBuffer(blob);
});
const columns: Pick<
ColumnDefinition<FieldMetadata>,
'size' | 'label' | 'type' | 'metadata'
>[] = [
{
label: 'Name',
size: 100,
type: FieldMetadataType.TEXT,
metadata: { fieldName: 'name' },
},
];
it('prepends UTF-8 BOM (EF BB BF) as first three bytes', async () => {
csvDownloader('export.csv', {
columns,
rows: [{ id: '1', name: 'test' }],
});
const blob = mockSaveAs.mock.calls[0][0] as Blob;
const bytes = new Uint8Array(await readBlob(blob));
expect(bytes[0]).toBe(0xef);
expect(bytes[1]).toBe(0xbb);
expect(bytes[2]).toBe(0xbf);
});
it.each([
['Arabic', 'مرحبا'],
['Chinese', '你好'],
['Japanese', 'こんにちは'],
['Korean', '안녕하세요'],
])('preserves %s characters in exported content', async (_, name) => {
csvDownloader('export.csv', { columns, rows: [{ id: '1', name }] });
const blob = mockSaveAs.mock.calls[0][0] as Blob;
const text = Buffer.from(await readBlob(blob)).toString('utf-8');
expect(text).toContain(name);
});
});
describe('displayedExportProgress', () => {
it.each([
[undefined, undefined, 'percentage', 'Export'],
@@ -114,6 +114,7 @@ export const generateCsv: GenerateExport = ({
return json2csv(sanitizedRows, {
keys,
emptyFieldValue: '',
excelBOM: true,
// Note: We handle CSV injection prevention manually with ZWJ approach above
// This preserves original which the csvSecurity option does not do
});