File - Migrate avatarUrl > avatarFile on person (data migration + logic) + Attachment data migration (#17752)

- Migration command - Check IS_FILES_FIELD_MIGRATED:false - Check or create avatarFile field - Fetch all people with avatarUrl - Move (Copy/move) file in storage - Create core.file record - Update person record - bonus : attachment migration : fullPath > file (same logic) - BE logic - Add avatarFile field on person - FE logic - Adapt logic to upload on/display avatarFile data The whole imageIdentifier logic will be done later
2026-02-11 16:07:05 +01:00
parent 5be64bf4be
commit d0c1841f0f
46 changed files with 1704 additions and 737 deletions
@@ -114,6 +114,7 @@ export class FileStorageService {
        workspaceId,
        applicationId: application.id,
        id: fileId,
+        mimeType,
        size:
          typeof sourceFile === 'string'
            ? Buffer.byteLength(sourceFile)
@@ -7,12 +7,15 @@ import { CustomException } from 'src/utils/custom-exception';
 export enum FileStorageExceptionCode {
  FILE_NOT_FOUND = 'FILE_NOT_FOUND',
  ACCESS_DENIED = 'ACCESS_DENIED',
+  INVALID_EXTENSION = 'INVALID_EXTENSION',
 }

 const getFileStorageExceptionUserFriendlyMessage = (
  code: FileStorageExceptionCode,
 ) => {
  switch (code) {
+    case FileStorageExceptionCode.INVALID_EXTENSION:
+      return msg`Wrong extension for file.`;
    case FileStorageExceptionCode.FILE_NOT_FOUND:
      return msg`File not found.`;
    case FileStorageExceptionCode.ACCESS_DENIED:
@@ -56,4 +56,11 @@ export class FileEntity extends WorkspaceRelatedEntity {

  @Column({ nullable: true, type: 'jsonb' })
  settings: FileSettings | null;
+
+  @Column({
+    nullable: false,
+    type: 'varchar',
+    default: 'application/octet-stream',
+  })
+  mimeType: string;
 }
@@ -4,6 +4,7 @@ import { InjectRepository } from '@nestjs/typeorm';
 import { Readable } from 'stream';

 import { msg } from '@lingui/core/macro';
+import { isNonEmptyString } from '@sniptt/guards';
 import { FileFolder } from 'twenty-shared/types';
 import { Repository } from 'typeorm';
 import { v4 } from 'uuid';
@@ -44,26 +45,23 @@ export class FilesFieldService {
  async uploadFile({
    file,
    filename,
-    declaredMimeType,
    workspaceId,
    fieldMetadataId,
  }: {
    file: Buffer;
    filename: string;
-    declaredMimeType: string | undefined;
    workspaceId: string;
    fieldMetadataId: string;
  }): Promise<FileEntity> {
    const { mimeType, ext } = await extractFileInfo({
      file,
-      declaredMimeType,
      filename,
    });

    const sanitizedFile = sanitizeFile({ file, ext, mimeType });

    const fileId = v4();
-    const name = `${fileId}${ext ? `.${ext}` : ''}`;
+    const name = `${fileId}${isNonEmptyString(ext) ? `.${ext}` : ''}`;

    const fieldMetadata = await this.fieldMetadataRepository.findOneOrFail({
      select: ['applicationId', 'universalIdentifier'],
@@ -30,7 +30,7 @@ export class FilesFieldResolver {
    @AuthWorkspace()
    { id: workspaceId }: WorkspaceEntity,
    @Args({ name: 'file', type: () => GraphQLUpload })
-    { createReadStream, filename, mimetype }: FileUpload,
+    { createReadStream, filename }: FileUpload,
    @Args({
      name: 'fieldMetadataId',
      type: () => String,
@@ -44,7 +44,6 @@ export class FilesFieldResolver {
    return await this.filesFieldService.uploadFile({
      file: buffer,
      filename,
-      declaredMimeType: mimetype,
      workspaceId,
      fieldMetadataId,
    });
@@ -1,28 +1,35 @@
-import FileType from 'file-type';
-
 import { extractFileInfo } from 'src/engine/core-modules/file/utils/extract-file-info.utils';

-jest.mock('file-type', () => ({
-  fromBuffer: jest.fn(),
-}));
+// Mock detectableMimeTypes to work around ESM/CommonJS interop issues in Jest
+jest.mock('file-type', () => {
+  const actual = jest.requireActual('file-type');
+
+  return {
+    ...actual,
+    mimeTypes: actual.mimeTypes,
+  };
+});

 describe('extractFileInfo', () => {
-  const mockBuffer = Buffer.from('test content');
+  // Real PNG file header (magic numbers)
+  const pngBuffer = Buffer.from([
+    0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0x00, 0x00, 0x00, 0x0d,
+    0x49, 0x48, 0x44, 0x52,
+  ]);

-  beforeEach(() => {
-    jest.clearAllMocks();
-  });
+  // Real PDF file header
+  const pdfBuffer = Buffer.from('%PDF-1.4\n', 'utf-8');

-  it('should use detected file type when available', async () => {
-    (FileType.fromBuffer as jest.Mock).mockResolvedValue({
-      mime: 'image/png',
-      ext: 'png',
-    });
+  // Plain text buffer (no magic numbers)
+  const textBuffer = Buffer.from('Hello, world!', 'utf-8');

+  // Real ZIP file header (for testing docx, xlsx, etc.)
+  const zipBuffer = Buffer.from([0x50, 0x4b, 0x03, 0x04]);
+
+  it('should detect PNG from buffer magic numbers', async () => {
    const result = await extractFileInfo({
-      file: mockBuffer,
-      declaredMimeType: 'text/plain',
-      filename: 'test.txt',
+      file: pngBuffer,
+      filename: 'image.png',
    });

    expect(result).toEqual({
@@ -31,31 +38,10 @@ describe('extractFileInfo', () => {
    });
  });

-  it('should fall back to declared values when file type is not detected', async () => {
-    (FileType.fromBuffer as jest.Mock).mockResolvedValue(undefined);
-
+  it('should detect PDF from buffer magic numbers', async () => {
    const result = await extractFileInfo({
-      file: mockBuffer,
-      declaredMimeType: 'text/plain',
-      filename: 'test.txt',
-    });
-
-    expect(result).toEqual({
-      mimeType: 'text/plain',
-      ext: 'txt',
-    });
-  });
-
-  it('should handle missing declared mime type when file type is detected', async () => {
-    (FileType.fromBuffer as jest.Mock).mockResolvedValue({
-      mime: 'application/pdf',
-      ext: 'pdf',
-    });
-
-    const result = await extractFileInfo({
-      file: mockBuffer,
-      declaredMimeType: undefined,
-      filename: 'document',
+      file: pdfBuffer,
+      filename: 'document.pdf',
    });

    expect(result).toEqual({
@@ -64,45 +50,133 @@ describe('extractFileInfo', () => {
    });
  });

-  it('should handle both mime type and extension being undefined', async () => {
-    (FileType.fromBuffer as jest.Mock).mockResolvedValue(undefined);
-
+  it('should use extension-based lookup for text files', async () => {
    const result = await extractFileInfo({
-      file: mockBuffer,
-      declaredMimeType: undefined,
+      file: textBuffer,
+      filename: 'document.txt',
+    });
+
+    expect(result).toEqual({
+      mimeType: 'text/plain',
+      ext: 'txt',
+    });
+  });
+
+  it('should handle CSV files using extension', async () => {
+    const result = await extractFileInfo({
+      file: textBuffer,
+      filename: 'data.csv',
+    });
+
+    expect(result).toEqual({
+      mimeType: 'text/csv',
+      ext: 'csv',
+    });
+  });
+
+  it('should handle JSON files using extension', async () => {
+    const result = await extractFileInfo({
+      file: Buffer.from('{"key": "value"}'),
+      filename: 'config.json',
+    });
+
+    expect(result).toEqual({
+      mimeType: 'application/json',
+      ext: 'json',
+    });
+  });
+
+  it('should return application/octet-stream for unknown extensions', async () => {
+    const result = await extractFileInfo({
+      file: textBuffer,
+      filename: 'file.unknown',
+    });
+
+    expect(result).toEqual({
+      mimeType: 'application/octet-stream',
+      ext: 'unknown',
+    });
+  });
+
+  it('should return application/octet-stream for files without extension', async () => {
+    const result = await extractFileInfo({
+      file: textBuffer,
      filename: 'file-without-extension',
    });

    expect(result).toEqual({
-      mimeType: undefined,
+      mimeType: 'application/octet-stream',
      ext: '',
    });
  });

-  it('should handle filenames with multiple dots', async () => {
-    (FileType.fromBuffer as jest.Mock).mockResolvedValue(undefined);
-
+  it('should detect ZIP files from buffer', async () => {
    const result = await extractFileInfo({
-      file: mockBuffer,
-      declaredMimeType: 'application/gzip',
-      filename: 'archive.tar.gz',
+      file: zipBuffer,
+      filename: 'archive.zip',
    });

    expect(result).toEqual({
-      mimeType: 'application/gzip',
-      ext: 'gz',
+      mimeType: 'application/zip',
+      ext: 'zip',
    });
  });

-  it('should call FileType.fromBuffer with the provided buffer', async () => {
-    (FileType.fromBuffer as jest.Mock).mockResolvedValue(undefined);
+  it('should throw error when PNG extension is used with non-PNG buffer', async () => {
+    await expect(
+      extractFileInfo({
+        file: textBuffer,
+        filename: 'fake-image.png',
+      }),
+    ).rejects.toThrow(
+      "File content does not match its extension. The file has extension 'png' (expected mime type: image/png), but the file content could not be detected as this type. The file may be corrupted, have the wrong extension, or be a security risk.",
+    );
+  });

-    await extractFileInfo({
-      file: mockBuffer,
-      declaredMimeType: 'image/png',
-      filename: 'image.png',
+  it('should throw error when PDF extension is used with non-PDF buffer', async () => {
+    await expect(
+      extractFileInfo({
+        file: textBuffer,
+        filename: 'fake-document.pdf',
+      }),
+    ).rejects.toThrow(
+      "File content does not match its extension. The file has extension 'pdf' (expected mime type: application/pdf), but the file content could not be detected as this type. The file may be corrupted, have the wrong extension, or be a security risk.",
+    );
+  });
+
+  it('should handle markdown files using extension', async () => {
+    const result = await extractFileInfo({
+      file: Buffer.from('# Heading\n\nContent'),
+      filename: 'README.md',
    });

-    expect(FileType.fromBuffer).toHaveBeenCalledWith(mockBuffer);
+    expect(result).toEqual({
+      mimeType: 'text/markdown',
+      ext: 'md',
+    });
+  });
+
+  it('should handle HTML files using extension', async () => {
+    const result = await extractFileInfo({
+      file: Buffer.from('<html><body>Test</body></html>'),
+      filename: 'index.html',
+    });
+
+    expect(result).toEqual({
+      mimeType: 'text/html',
+      ext: 'html',
+    });
+  });
+
+  it('should prefer detected type over declared extension', async () => {
+    const result = await extractFileInfo({
+      file: pngBuffer,
+      filename: 'image.txt',
+    });
+
+    expect(result).toEqual({
+      mimeType: 'image/png',
+      ext: 'png',
+    });
  });
 });
@@ -1,23 +1,57 @@
-import FileType from 'file-type';
+import { msg } from '@lingui/core/macro';
+import { isNonEmptyString } from '@sniptt/guards';
+import FileType, { type MimeType } from 'file-type';
+import { lookup } from 'mrmime';
+import { isDefined } from 'twenty-shared/utils';
+
+import {
+  FileStorageException,
+  FileStorageExceptionCode,
+} from 'src/engine/core-modules/file-storage/interfaces/file-storage-exception';

 import { buildFileInfo } from 'src/engine/core-modules/file/utils/build-file-info.utils';

 export const extractFileInfo = async ({
  file,
-  declaredMimeType,
  filename,
 }: {
  file: Buffer;
-  declaredMimeType: string | undefined;
  filename: string;
 }) => {
  const { ext: declaredExt } = buildFileInfo(filename);

-  const detectedFileType = await FileType.fromBuffer(file);
+  const { ext: detectedExt, mime: detectedMime } =
+    (await FileType.fromBuffer(file)) ?? {};

-  const mimeType = detectedFileType?.mime ?? declaredMimeType;
+  if (isDefined(detectedExt) && isDefined(detectedMime)) {
+    return {
+      mimeType: detectedMime,
+      ext: detectedExt,
+    };
+  }

-  const ext = detectedFileType?.ext ?? declaredExt;
+  const ext = declaredExt;
+
+  let mimeType: string = 'application/octet-stream';
+
+  if (isNonEmptyString(ext)) {
+    const mimeTypeFromExtension = lookup(ext);
+
+    if (
+      mimeTypeFromExtension &&
+      FileType.mimeTypes.has(mimeTypeFromExtension as MimeType)
+    ) {
+      throw new FileStorageException(
+        `File content does not match its extension. The file has extension '${ext}' (expected mime type: ${mimeTypeFromExtension}), but the file content could not be detected as this type. The file may be corrupted, have the wrong extension, or be a security risk.`,
+        FileStorageExceptionCode.INVALID_EXTENSION,
+        {
+          userFriendlyMessage: msg`The file extension doesn't match the file content. Please check that your file is not corrupted and has the correct extension.`,
+        },
+      );
+    }
+
+    mimeType = mimeTypeFromExtension ?? 'application/octet-stream';
+  }

  return {
    mimeType,
@@ -134,6 +134,14 @@ export class FilesFieldSync {
      return null;
    }

+    const isModifyingFilesField = filesFields.some((filesField) =>
+      isDefined(updatePayload[filesField.name as keyof typeof updatePayload]),
+    );
+
+    if (!isModifyingFilesField) {
+      return null;
+    }
+
    if (existingRecords.length !== 1) {
      throw new TwentyORMException(
        `Cannot update multiple records with files field at once`,
@@ -229,6 +229,7 @@ export const buildPersonStandardFlatFieldMetadatas = ({
    twentyStandardApplicationId,
    now,
  }),
+  //deprecated
  avatarUrl: createStandardFieldFlatMetadata({
    objectName,
    workspaceId,
@@ -246,6 +247,26 @@ export const buildPersonStandardFlatFieldMetadatas = ({
    twentyStandardApplicationId,
    now,
  }),
+  avatarFile: createStandardFieldFlatMetadata({
+    objectName,
+    workspaceId,
+    context: {
+      fieldName: 'avatarFile',
+      type: FieldMetadataType.FILES,
+      label: 'Avatar File',
+      description: "Contact's avatar file",
+      icon: 'IconFileUpload',
+      isSystem: true,
+      isNullable: true,
+      settings: {
+        maxNumberOfValues: 1,
+      },
+    },
+    standardObjectMetadataRelatedEntityIds,
+    dependencyFlatEntityMaps,
+    twentyStandardApplicationId,
+    now,
+  }),
  position: createStandardFieldFlatMetadata({
    objectName,
    workspaceId,