fix: prepend UTF-8 BOM to fix non-Latin characters in csv (#19246)
Fixes: #19230 CSV exports containing Arabic, Chinese, Japanese, Korean characters displayed as garbled text in Excel because the file lacked a UTF-8 BOM. So I added `\uFEFF` prefix to the CSV Blob so Excel and other spreadsheet apps correctly interpret the file as UTF-8 ## Table value <img width="370" height="277" alt="image" src="https://github.com/user-attachments/assets/ed668f00-93f0-4991-bc87-42e3cb314dbc" /> ## Before <img width="206" height="160" alt="image" src="https://github.com/user-attachments/assets/a5324dbc-bd65-4443-a7fb-78047028bf91" /> ## After <img width="180" height="155" alt="image" src="https://github.com/user-attachments/assets/cae5f38d-6c91-4017-96ae-9924bb41d0e0" /> And those are my test data Script | Example | Language -- | -- | -- Arabic | مرحبا | Arabic, Persian, Urdu Chinese | 你好 | Chinese Japanese | こんにちは | Japanese Korean | 안녕하세요 | Korean --------- Co-authored-by: Claude <noreply@anthropic.com> Co-authored-by: Félix Malfait <felix@twenty.com> Co-authored-by: Félix Malfait <felix.malfait@gmail.com>
This commit is contained in:
+64
-2
@@ -2,11 +2,17 @@ import { type FieldMetadata } from '@/object-record/record-field/ui/types/FieldM
|
||||
import { type ColumnDefinition } from '@/object-record/record-table/types/ColumnDefinition';
|
||||
import { CSV_INJECTION_PREVENTION_ZWJ } from '@/spreadsheet-import/constants/CsvInjectionPreventionZwj';
|
||||
|
||||
import { FieldMetadataType, RelationType } from '~/generated-metadata/graphql';
|
||||
import {
|
||||
csvDownloader,
|
||||
displayedExportProgress,
|
||||
generateCsv,
|
||||
} from '@/object-record/record-index/export/hooks/useRecordIndexExportRecords';
|
||||
import { saveAs } from 'file-saver';
|
||||
import { FieldMetadataType, RelationType } from '~/generated-metadata/graphql';
|
||||
|
||||
jest.mock('file-saver', () => ({
|
||||
saveAs: jest.fn(),
|
||||
}));
|
||||
|
||||
jest.useFakeTimers();
|
||||
|
||||
@@ -63,7 +69,7 @@ describe('generateCsv', () => {
|
||||
];
|
||||
const csv = generateCsv({ columns, rows });
|
||||
expect(csv)
|
||||
.toEqual(`Id,Foo,Empty,Nested link field / Link URL,Nested link field / Secondary Links,Relation
|
||||
.toEqual(`\uFEFFId,Foo,Empty,Nested link field / Link URL,Nested link field / Secondary Links,Relation
|
||||
1,some field,,https://www.test.com,"[{""label"":""secondary link 1"",""url"":""https://www.test.com""},{""label"":""secondary link 2"",""url"":""https://www.test.com""}]",a relation`);
|
||||
});
|
||||
|
||||
@@ -436,6 +442,62 @@ describe('generateCsv', () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe('csvDownloader', () => {
|
||||
const mockSaveAs = saveAs as jest.MockedFunction<typeof saveAs>;
|
||||
|
||||
beforeEach(() => {
|
||||
mockSaveAs.mockClear();
|
||||
});
|
||||
|
||||
const readBlob = async (blob: Blob): Promise<ArrayBuffer> =>
|
||||
new Promise<ArrayBuffer>((resolve, reject) => {
|
||||
const reader = new FileReader();
|
||||
reader.onload = () => resolve(reader.result as ArrayBuffer);
|
||||
reader.onerror = reject;
|
||||
reader.readAsArrayBuffer(blob);
|
||||
});
|
||||
|
||||
const columns: Pick<
|
||||
ColumnDefinition<FieldMetadata>,
|
||||
'size' | 'label' | 'type' | 'metadata'
|
||||
>[] = [
|
||||
{
|
||||
label: 'Name',
|
||||
size: 100,
|
||||
type: FieldMetadataType.TEXT,
|
||||
metadata: { fieldName: 'name' },
|
||||
},
|
||||
];
|
||||
|
||||
it('prepends UTF-8 BOM (EF BB BF) as first three bytes', async () => {
|
||||
csvDownloader('export.csv', {
|
||||
columns,
|
||||
rows: [{ id: '1', name: 'test' }],
|
||||
});
|
||||
|
||||
const blob = mockSaveAs.mock.calls[0][0] as Blob;
|
||||
const bytes = new Uint8Array(await readBlob(blob));
|
||||
|
||||
expect(bytes[0]).toBe(0xef);
|
||||
expect(bytes[1]).toBe(0xbb);
|
||||
expect(bytes[2]).toBe(0xbf);
|
||||
});
|
||||
|
||||
it.each([
|
||||
['Arabic', 'مرحبا'],
|
||||
['Chinese', '你好'],
|
||||
['Japanese', 'こんにちは'],
|
||||
['Korean', '안녕하세요'],
|
||||
])('preserves %s characters in exported content', async (_, name) => {
|
||||
csvDownloader('export.csv', { columns, rows: [{ id: '1', name }] });
|
||||
|
||||
const blob = mockSaveAs.mock.calls[0][0] as Blob;
|
||||
const text = Buffer.from(await readBlob(blob)).toString('utf-8');
|
||||
|
||||
expect(text).toContain(name);
|
||||
});
|
||||
});
|
||||
|
||||
describe('displayedExportProgress', () => {
|
||||
it.each([
|
||||
[undefined, undefined, 'percentage', 'Export'],
|
||||
|
||||
+1
@@ -114,6 +114,7 @@ export const generateCsv: GenerateExport = ({
|
||||
return json2csv(sanitizedRows, {
|
||||
keys,
|
||||
emptyFieldValue: '',
|
||||
excelBOM: true,
|
||||
// Note: We handle CSV injection prevention manually with ZWJ approach above
|
||||
// This preserves original which the csvSecurity option does not do
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user