From b21fb4aa6f1192fa5ac7a12b1ba65c8a4d7d4f2e Mon Sep 17 00:00:00 2001 From: neo773 <62795688+neo773@users.noreply.github.com> Date: Thu, 12 Mar 2026 16:04:24 +0530 Subject: [PATCH] Fix PDF Upload edge case (#18533) we were using an older version of `file-type` which has limited support for PDF as it's a complex spec Updated to latest version which includes support for plugins and added `@file-type/pdf` which has extensive spec compliant detection approach fixes TWENTY-SERVER-FAN --- packages/twenty-server/jest.config.mjs | 5 +- packages/twenty-server/package.json | 3 +- ...1-18-migrate-workspace-pictures.command.ts | 6 +- .../services/file-core-picture.service.ts | 6 +- .../__tests__/extract-file-info.utils.spec.ts | 262 ++++++------------ .../file/utils/extract-file-info.utils.ts | 11 +- yarn.lock | 40 ++- 7 files changed, 153 insertions(+), 180 deletions(-) diff --git a/packages/twenty-server/jest.config.mjs b/packages/twenty-server/jest.config.mjs index cb114e3aef..22d28de204 100644 --- a/packages/twenty-server/jest.config.mjs +++ b/packages/twenty-server/jest.config.mjs @@ -10,7 +10,9 @@ const jestConfig = { rootDir: './', testEnvironment: 'node', setupFilesAfterEnv: ['./setupTests.ts'], - transformIgnorePatterns: ['/node_modules/'], + transformIgnorePatterns: [ + '/node_modules/(?!(file-type|@file-type|strtok3|token-types|@borewit|@tokenizer|uint8array-extras|read-next-line)/)', + ], testRegex: '.*\\.spec\\.ts$', transform: { '^.+\\.(t|j)s$': [ @@ -42,6 +44,7 @@ const jestConfig = { moduleNameMapper: { '^src/(.*)': '/src/$1', '^test/(.*)': '/test/$1', + '^file-type$': '/node_modules/file-type/index.js', }, moduleFileExtensions: ['js', 'json', 'ts'], modulePathIgnorePatterns: ['/dist'], diff --git a/packages/twenty-server/package.json b/packages/twenty-server/package.json index 3696a9a324..c80bab474b 100644 --- a/packages/twenty-server/package.json +++ b/packages/twenty-server/package.json @@ -39,6 +39,7 @@ "@envelop/on-resolve": "4.1.0", "@esbuild-plugins/node-modules-polyfill": "^0.2.2", "@faker-js/faker": "9.8.0", + "@file-type/pdf": "^0.2.0", "@graphql-tools/schema": "10.0.4", "@graphql-tools/utils": "9.2.1", "@graphql-yoga/nestjs": "patch:@graphql-yoga/nestjs@2.1.0#./patches/@graphql-yoga+nestjs+2.1.0.patch", @@ -106,7 +107,7 @@ "dotenv": "16.4.5", "express": "4.22.1", "express-session": "^1.18.2", - "file-type": "16.5.4", + "file-type": "^21.3.1", "fuse.js": "^7.1.0", "gaxios": "5.1.3", "google-auth-library": "8.9.0", diff --git a/packages/twenty-server/src/database/commands/upgrade-version-command/1-18/1-18-migrate-workspace-pictures.command.ts b/packages/twenty-server/src/database/commands/upgrade-version-command/1-18/1-18-migrate-workspace-pictures.command.ts index cafc79ba69..70f443aedb 100644 --- a/packages/twenty-server/src/database/commands/upgrade-version-command/1-18/1-18-migrate-workspace-pictures.command.ts +++ b/packages/twenty-server/src/database/commands/upgrade-version-command/1-18/1-18-migrate-workspace-pictures.command.ts @@ -1,7 +1,8 @@ import { InjectDataSource, InjectRepository } from '@nestjs/typeorm'; import { isNonEmptyString } from '@sniptt/guards'; -import FileType from 'file-type'; +import { FileTypeParser } from 'file-type'; +import { detectPdf } from '@file-type/pdf'; import { Command } from 'nest-commander'; import { STANDARD_OBJECTS } from 'twenty-shared/metadata'; import { FileFolder } from 'twenty-shared/types'; @@ -204,7 +205,8 @@ export class MigrateWorkspacePicturesCommand extends ActiveOrSuspendedWorkspaces const httpClient = this.secureHttpClientService.getHttpClient(); const buffer = await getImageBufferFromUrl(logoUrl, httpClient); - const type = await FileType.fromBuffer(buffer); + const parser = new FileTypeParser({ customDetectors: [detectPdf] }); + const type = await parser.fromBuffer(buffer); if (!isDefined(type) || !type.mime.startsWith('image/')) { this.logger.warn( diff --git a/packages/twenty-server/src/engine/core-modules/file/file-core-picture/services/file-core-picture.service.ts b/packages/twenty-server/src/engine/core-modules/file/file-core-picture/services/file-core-picture.service.ts index 06394be2b5..808076ab82 100644 --- a/packages/twenty-server/src/engine/core-modules/file/file-core-picture/services/file-core-picture.service.ts +++ b/packages/twenty-server/src/engine/core-modules/file/file-core-picture/services/file-core-picture.service.ts @@ -4,7 +4,8 @@ import { InjectRepository } from '@nestjs/typeorm'; import { buffer as streamToBuffer } from 'node:stream/consumers'; import { isNonEmptyString } from '@sniptt/guards'; -import FileType from 'file-type'; +import { FileTypeParser } from 'file-type'; +import { detectPdf } from '@file-type/pdf'; import { FileFolder } from 'twenty-shared/types'; import { isDefined } from 'twenty-shared/utils'; import { Like, type QueryRunner, Repository } from 'typeorm'; @@ -195,7 +196,8 @@ export class FileCorePictureService { const buffer = await getImageBufferFromUrl(imageUrl, httpClient); - const type = await FileType.fromBuffer(buffer); + const parser = new FileTypeParser({ customDetectors: [detectPdf] }); + const type = await parser.fromBuffer(buffer); if (!isDefined(type) || !type.mime.startsWith('image/')) { return undefined; diff --git a/packages/twenty-server/src/engine/core-modules/file/utils/__tests__/extract-file-info.utils.spec.ts b/packages/twenty-server/src/engine/core-modules/file/utils/__tests__/extract-file-info.utils.spec.ts index efdd025e5e..8aa8fcf539 100644 --- a/packages/twenty-server/src/engine/core-modules/file/utils/__tests__/extract-file-info.utils.spec.ts +++ b/packages/twenty-server/src/engine/core-modules/file/utils/__tests__/extract-file-info.utils.spec.ts @@ -1,182 +1,104 @@ -import { extractFileInfo } from 'src/engine/core-modules/file/utils/extract-file-info.utils'; +import { extractFileInfo } from '../extract-file-info.utils'; -// Mock detectableMimeTypes to work around ESM/CommonJS interop issues in Jest -jest.mock('file-type', () => { - const actual = jest.requireActual('file-type'); - - return { - ...actual, - mimeTypes: actual.mimeTypes, - }; -}); +const pngBuffer = Buffer.from([ + 0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0x00, 0x00, 0x00, 0x0d, 0x49, + 0x48, 0x44, 0x52, +]); +const pdfBuffer = Buffer.from('%PDF-1.4\n', 'utf-8'); +const textBuffer = Buffer.from('Hello, world!', 'utf-8'); +const zipBuffer = Buffer.from([0x50, 0x4b, 0x03, 0x04]); describe('extractFileInfo', () => { - // Real PNG file header (magic numbers) - const pngBuffer = Buffer.from([ - 0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0x00, 0x00, 0x00, 0x0d, - 0x49, 0x48, 0x44, 0x52, - ]); - - // Real PDF file header - const pdfBuffer = Buffer.from('%PDF-1.4\n', 'utf-8'); - - // Plain text buffer (no magic numbers) - const textBuffer = Buffer.from('Hello, world!', 'utf-8'); - - // Real ZIP file header (for testing docx, xlsx, etc.) - const zipBuffer = Buffer.from([0x50, 0x4b, 0x03, 0x04]); - - it('should detect PNG from buffer magic numbers', async () => { - const result = await extractFileInfo({ - file: pngBuffer, + it.each([ + { + name: 'PNG', + buffer: pngBuffer, filename: 'image.png', - }); - - expect(result).toEqual({ - mimeType: 'image/png', ext: 'png', - }); - }); - - it('should detect PDF from buffer magic numbers', async () => { - const result = await extractFileInfo({ - file: pdfBuffer, + mime: 'image/png', + }, + { + name: 'PDF', + buffer: pdfBuffer, filename: 'document.pdf', - }); - - expect(result).toEqual({ - mimeType: 'application/pdf', ext: 'pdf', - }); - }); - - it('should use extension-based lookup for text files', async () => { - const result = await extractFileInfo({ - file: textBuffer, - filename: 'document.txt', - }); - - expect(result).toEqual({ - mimeType: 'text/plain', - ext: 'txt', - }); - }); - - it('should handle CSV files using extension', async () => { - const result = await extractFileInfo({ - file: textBuffer, - filename: 'data.csv', - }); - - expect(result).toEqual({ - mimeType: 'text/csv', - ext: 'csv', - }); - }); - - it('should handle JSON files using extension', async () => { - const result = await extractFileInfo({ - file: Buffer.from('{"key": "value"}'), - filename: 'config.json', - }); - - expect(result).toEqual({ - mimeType: 'application/json', - ext: 'json', - }); - }); - - it('should return application/octet-stream for unknown extensions', async () => { - const result = await extractFileInfo({ - file: textBuffer, - filename: 'file.unknown', - }); - - expect(result).toEqual({ - mimeType: 'application/octet-stream', - ext: 'unknown', - }); - }); - - it('should return application/octet-stream for files without extension', async () => { - const result = await extractFileInfo({ - file: textBuffer, - filename: 'file-without-extension', - }); - - expect(result).toEqual({ - mimeType: 'application/octet-stream', - ext: '', - }); - }); - - it('should detect ZIP files from buffer', async () => { - const result = await extractFileInfo({ - file: zipBuffer, + mime: 'application/pdf', + }, + { + name: 'ZIP', + buffer: zipBuffer, filename: 'archive.zip', - }); - - expect(result).toEqual({ - mimeType: 'application/zip', ext: 'zip', - }); - }); - - it('should throw error when PNG extension is used with non-PNG buffer', async () => { - await expect( - extractFileInfo({ - file: textBuffer, - filename: 'fake-image.png', - }), - ).rejects.toThrow( - "File content does not match its extension. The file has extension 'png' (expected mime type: image/png), but the file content could not be detected as this type. The file may be corrupted, have the wrong extension, or be a security risk.", - ); - }); - - it('should throw error when PDF extension is used with non-PDF buffer', async () => { - await expect( - extractFileInfo({ - file: textBuffer, - filename: 'fake-document.pdf', - }), - ).rejects.toThrow( - "File content does not match its extension. The file has extension 'pdf' (expected mime type: application/pdf), but the file content could not be detected as this type. The file may be corrupted, have the wrong extension, or be a security risk.", - ); - }); - - it('should handle markdown files using extension', async () => { - const result = await extractFileInfo({ - file: Buffer.from('# Heading\n\nContent'), - filename: 'README.md', - }); - - expect(result).toEqual({ - mimeType: 'text/markdown', - ext: 'md', - }); - }); - - it('should handle HTML files using extension', async () => { - const result = await extractFileInfo({ - file: Buffer.from('Test'), - filename: 'index.html', - }); - - expect(result).toEqual({ - mimeType: 'text/html', - ext: 'html', - }); - }); - - it('should prefer detected type over declared extension', async () => { - const result = await extractFileInfo({ - file: pngBuffer, + mime: 'application/zip', + }, + { + name: 'PNG (mismatched extension)', + buffer: pngBuffer, filename: 'image.txt', - }); - - expect(result).toEqual({ - mimeType: 'image/png', ext: 'png', - }); - }); + mime: 'image/png', + }, + ])( + 'should detect $name from buffer magic numbers', + async ({ buffer, filename, ext, mime }) => { + const result = await extractFileInfo({ file: buffer, filename }); + + expect(result).toEqual({ mimeType: mime, ext }); + }, + ); + + it.each([ + { name: 'text', filename: 'document.txt', ext: 'txt', mime: 'text/plain' }, + { name: 'CSV', filename: 'data.csv', ext: 'csv', mime: 'text/csv' }, + { + name: 'JSON', + filename: 'config.json', + ext: 'json', + mime: 'application/json', + }, + { + name: 'markdown', + filename: 'README.md', + ext: 'md', + mime: 'text/markdown', + }, + { name: 'HTML', filename: 'index.html', ext: 'html', mime: 'text/html' }, + { + name: 'unknown extension', + filename: 'file.unknown', + ext: 'unknown', + mime: 'application/octet-stream', + }, + { + name: 'no extension', + filename: 'file-without-extension', + ext: '', + mime: 'application/octet-stream', + }, + ])( + 'should fall back to extension for $name files', + async ({ filename, ext, mime }) => { + const result = await extractFileInfo({ file: textBuffer, filename }); + + expect(result).toEqual({ mimeType: mime, ext }); + }, + ); + + it.each([ + { ext: 'png', filename: 'fake-image.png', expectedMime: 'image/png' }, + { + ext: 'pdf', + filename: 'fake-document.pdf', + expectedMime: 'application/pdf', + }, + ])( + 'should throw when $ext extension does not match buffer content', + async ({ filename, ext, expectedMime }) => { + await expect( + extractFileInfo({ file: textBuffer, filename }), + ).rejects.toThrow( + `File content does not match its extension. The file has extension '${ext}' (expected mime type: ${expectedMime}), but the file content could not be detected as this type. The file may be corrupted, have the wrong extension, or be a security risk.`, + ); + }, + ); }); diff --git a/packages/twenty-server/src/engine/core-modules/file/utils/extract-file-info.utils.ts b/packages/twenty-server/src/engine/core-modules/file/utils/extract-file-info.utils.ts index cb1c5a4e8c..2f93f9530f 100644 --- a/packages/twenty-server/src/engine/core-modules/file/utils/extract-file-info.utils.ts +++ b/packages/twenty-server/src/engine/core-modules/file/utils/extract-file-info.utils.ts @@ -1,6 +1,6 @@ import { msg } from '@lingui/core/macro'; import { isNonEmptyString } from '@sniptt/guards'; -import FileType, { type MimeType } from 'file-type'; +import { FileTypeParser, supportedMimeTypes } from 'file-type'; import { lookup } from 'mrmime'; import { isDefined } from 'twenty-shared/utils'; @@ -9,6 +9,7 @@ import { FileStorageExceptionCode, } from 'src/engine/core-modules/file-storage/interfaces/file-storage-exception'; +import { detectPdf } from '@file-type/pdf'; import { buildFileInfo } from 'src/engine/core-modules/file/utils/build-file-info.utils'; export const extractFileInfo = async ({ @@ -20,8 +21,12 @@ export const extractFileInfo = async ({ }) => { const { ext: declaredExt } = buildFileInfo(filename); + const fileParser = new FileTypeParser({ + customDetectors: [detectPdf], + }); + const { ext: detectedExt, mime: detectedMime } = - (await FileType.fromBuffer(file)) ?? {}; + (await fileParser.fromBuffer(file)) ?? {}; if (isDefined(detectedExt) && isDefined(detectedMime)) { return { @@ -39,7 +44,7 @@ export const extractFileInfo = async ({ if ( mimeTypeFromExtension && - FileType.mimeTypes.has(mimeTypeFromExtension as MimeType) + supportedMimeTypes.has(mimeTypeFromExtension) ) { throw new FileStorageException( `File content does not match its extension. The file has extension '${ext}' (expected mime type: ${mimeTypeFromExtension}), but the file content could not be detected as this type. The file may be corrupted, have the wrong extension, or be a security risk.`, diff --git a/yarn.lock b/yarn.lock index 6059c55ef7..49051cfe03 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5873,6 +5873,17 @@ __metadata: languageName: node linkType: hard +"@file-type/pdf@npm:^0.2.0": + version: 0.2.0 + resolution: "@file-type/pdf@npm:0.2.0" + dependencies: + "@borewit/text-codec": "npm:^0.2.1" + read-next-line: "npm:^0.5.0" + sax: "npm:^1.4.1" + checksum: 10c0/9f2f446a04ca2fff4f5a02aeb6065d72f4fc95b6bce5efc6a8ec77530c5b36192f9aa718b65e00398921cf5a0d505cda5e9208d3ac81aad63f7d60a535212a89 + languageName: node + linkType: hard + "@floating-ui/core@npm:^1.6.0": version: 1.6.7 resolution: "@floating-ui/core@npm:1.6.7" @@ -36527,6 +36538,18 @@ __metadata: languageName: node linkType: hard +"file-type@npm:^21.3.1": + version: 21.3.1 + resolution: "file-type@npm:21.3.1" + dependencies: + "@tokenizer/inflate": "npm:^0.4.1" + strtok3: "npm:^10.3.4" + token-types: "npm:^6.1.1" + uint8array-extras: "npm:^1.4.0" + checksum: 10c0/66a8eda781c803c6fc372464abba88cee564cfe30f029716f06909da1564bc51d0a4e8d34654b881dc751cdb0513338b3c0747e06a608cee0dd5c5499b018d47 + languageName: node + linkType: hard + "filelist@npm:^1.0.4": version: 1.0.4 resolution: "filelist@npm:1.0.4" @@ -51234,6 +51257,13 @@ __metadata: languageName: node linkType: hard +"read-next-line@npm:^0.5.0": + version: 0.5.0 + resolution: "read-next-line@npm:0.5.0" + checksum: 10c0/a658dd647a655767a81d237474dac2cce6f15fff14e8963a41980bafb30c11298e27d6210a06031ecef62a28edf703cc2e3fce8d28ec411bb309ea71403c6992 + languageName: node + linkType: hard + "read-only-stream@npm:^2.0.0": version: 2.0.0 resolution: "read-only-stream@npm:2.0.0" @@ -52880,6 +52910,13 @@ __metadata: languageName: node linkType: hard +"sax@npm:^1.4.1": + version: 1.5.0 + resolution: "sax@npm:1.5.0" + checksum: 10c0/bc3b60a7bfecd40b18256596e96b32df2488339ae1e00a77f842b568f0831228a16c3bd357ec500241ec0b9dc7a475a1286427795c4a8c50bb8e8878f3435dd8 + languageName: node + linkType: hard + "saxes@npm:^6.0.0": version: 6.0.0 resolution: "saxes@npm:6.0.0" @@ -56594,6 +56631,7 @@ __metadata: "@envelop/on-resolve": "npm:4.1.0" "@esbuild-plugins/node-modules-polyfill": "npm:^0.2.2" "@faker-js/faker": "npm:^9.8.0" + "@file-type/pdf": "npm:^0.2.0" "@graphql-tools/schema": "npm:10.0.4" "@graphql-tools/utils": "npm:9.2.1" "@graphql-yoga/nestjs": "patch:@graphql-yoga/nestjs@2.1.0#./patches/@graphql-yoga+nestjs+2.1.0.patch" @@ -56700,7 +56738,7 @@ __metadata: dotenv: "npm:16.4.5" express: "npm:4.22.1" express-session: "npm:^1.18.2" - file-type: "npm:16.5.4" + file-type: "npm:^21.3.1" fuse.js: "npm:^7.1.0" gaxios: "npm:5.1.3" google-auth-library: "npm:8.9.0"