File - Migrate avatarUrl > avatarFile on person (data migration + logic) + Attachment data migration (#17752)

- Migration command
    - Check IS_FILES_FIELD_MIGRATED:false
    - Check or create avatarFile field
    - Fetch all people with avatarUrl
           - Move (Copy/move) file in storage
           - Create core.file record
           - Update person record
           
    - bonus : attachment migration  : fullPath > file (same logic)
   
- BE logic
    - Add avatarFile field on person

- FE logic 
   - Adapt logic to upload on/display avatarFile data

The whole imageIdentifier logic will be done later
This commit is contained in:
Etienne
2026-02-11 16:07:05 +01:00
committed by GitHub
parent 5be64bf4be
commit d0c1841f0f
46 changed files with 1704 additions and 737 deletions
@@ -0,0 +1,298 @@
import { InjectDataSource, InjectRepository } from '@nestjs/typeorm';
import { isNonEmptyString } from '@sniptt/guards';
import { Command } from 'nest-commander';
import { STANDARD_OBJECTS } from 'twenty-shared/metadata';
import { FieldMetadataType, FileFolder } from 'twenty-shared/types';
import {
extractFolderPathFilenameAndTypeOrThrow,
isDefined,
} from 'twenty-shared/utils';
import { DataSource, Equal, IsNull, Not, Or, Repository } from 'typeorm';
import { v4 } from 'uuid';
import { ActiveOrSuspendedWorkspacesMigrationCommandRunner } from 'src/database/commands/command-runners/active-or-suspended-workspaces-migration.command-runner';
import { RunOnWorkspaceArgs } from 'src/database/commands/command-runners/workspaces-migration.command-runner';
import { getFlatFieldsFromFlatObjectMetadata } from 'src/engine/api/graphql/workspace-schema-builder/utils/get-flat-fields-for-flat-object-metadata.util';
import { ApplicationService } from 'src/engine/core-modules/application/services/application.service';
import { FeatureFlagKey } from 'src/engine/core-modules/feature-flag/enums/feature-flag-key.enum';
import { FeatureFlagService } from 'src/engine/core-modules/feature-flag/services/feature-flag.service';
import { FileStorageService } from 'src/engine/core-modules/file-storage/file-storage.service';
import { FileEntity } from 'src/engine/core-modules/file/entities/file.entity';
import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity';
import { DataSourceService } from 'src/engine/metadata-modules/data-source/data-source.service';
import { FieldMetadataService } from 'src/engine/metadata-modules/field-metadata/services/field-metadata.service';
import { findFlatEntityByUniversalIdentifier } from 'src/engine/metadata-modules/flat-entity/utils/find-flat-entity-by-universal-identifier.util';
import { type FlatObjectMetadata } from 'src/engine/metadata-modules/flat-object-metadata/types/flat-object-metadata.type';
import { GlobalWorkspaceOrmManager } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager';
import { WorkspaceCacheService } from 'src/engine/workspace-cache/services/workspace-cache.service';
import { AttachmentWorkspaceEntity } from 'src/modules/attachment/standard-objects/attachment.workspace-entity';
@Command({
name: 'upgrade:1-18:migrate-attachment-files',
description:
'Migrate attachment files to file field: copy files and create file records',
})
export class MigrateAttachmentFilesCommand extends ActiveOrSuspendedWorkspacesMigrationCommandRunner {
constructor(
@InjectRepository(WorkspaceEntity)
protected readonly workspaceRepository: Repository<WorkspaceEntity>,
protected readonly twentyORMGlobalManager: GlobalWorkspaceOrmManager,
protected readonly dataSourceService: DataSourceService,
private readonly featureFlagService: FeatureFlagService,
private readonly fileStorageService: FileStorageService,
private readonly workspaceCacheService: WorkspaceCacheService,
private readonly fieldMetadataService: FieldMetadataService,
private readonly applicationService: ApplicationService,
@InjectDataSource()
private readonly coreDataSource: DataSource,
) {
super(workspaceRepository, twentyORMGlobalManager, dataSourceService);
}
override async runOnWorkspace({
workspaceId,
options,
}: RunOnWorkspaceArgs): Promise<void> {
const isDryRun = options.dryRun ?? false;
const isMigrated = await this.featureFlagService.isFeatureEnabled(
FeatureFlagKey.IS_FILES_FIELD_MIGRATED,
workspaceId,
);
if (isMigrated) {
this.logger.log(
`Attachment files migration already completed for workspace ${workspaceId}, skipping`,
);
return;
}
this.logger.log(
`${isDryRun ? '[DRY RUN] ' : ''}Starting attachment files migration for workspace ${workspaceId}`,
);
const { flatObjectMetadataMaps, flatFieldMetadataMaps } =
await this.workspaceCacheService.getOrRecompute(workspaceId, [
'flatObjectMetadataMaps',
'flatFieldMetadataMaps',
]);
const attachmentObjectMetadata =
findFlatEntityByUniversalIdentifier<FlatObjectMetadata>({
flatEntityMaps: flatObjectMetadataMaps,
universalIdentifier: STANDARD_OBJECTS.attachment.universalIdentifier,
});
if (!isDefined(attachmentObjectMetadata)) {
this.logger.warn(
`Attachment object metadata not found for workspace ${workspaceId}, skipping`,
);
return;
}
const { twentyStandardFlatApplication } =
await this.applicationService.findWorkspaceTwentyStandardAndCustomApplicationOrThrow(
{
workspaceId,
},
);
let attachmentFileflatFieldMetadata = findFlatEntityByUniversalIdentifier({
flatEntityMaps: flatFieldMetadataMaps,
universalIdentifier:
STANDARD_OBJECTS.attachment.fields.file.universalIdentifier,
});
if (!isDefined(attachmentFileflatFieldMetadata)) {
this.logger.log(
`File field metadata not found for attachments in workspace ${workspaceId}, creating it`,
);
if (!isDryRun) {
const attachmentFieldMetadatas = getFlatFieldsFromFlatObjectMetadata(
attachmentObjectMetadata,
flatFieldMetadataMaps,
);
const existingFieldWithSameName = attachmentFieldMetadatas.find(
(fieldMetadata) => fieldMetadata.name === 'file',
);
if (isDefined(existingFieldWithSameName)) {
this.logger.log(
`Found existing field with name 'file' (id: ${existingFieldWithSameName.id}), renaming it to 'old-file'`,
);
try {
await this.fieldMetadataService.updateOneField({
updateFieldInput: {
id: existingFieldWithSameName.id,
name: 'oldFile',
label: 'Old File',
},
workspaceId,
ownerFlatApplication: twentyStandardFlatApplication,
});
} catch (error) {
this.logger.error(
`Failed to rename existing 'file' field to 'old-file' for workspace ${workspaceId}: ${error.message}`,
);
throw error;
}
this.logger.log(
`Renamed existing 'file' field to 'oldFile' for workspace ${workspaceId}`,
);
}
try {
attachmentFileflatFieldMetadata =
await this.fieldMetadataService.createOneField({
createFieldInput: {
objectMetadataId: attachmentObjectMetadata.id,
type: FieldMetadataType.FILES,
name: 'file',
label: 'File',
description: 'Attachment file',
icon: 'IconFileUpload',
isNullable: true,
isUIReadOnly: true,
isSystem: true,
settings: {
maxNumberOfValues: 1,
},
universalIdentifier:
STANDARD_OBJECTS.attachment.fields.file.universalIdentifier,
},
workspaceId,
ownerFlatApplication: twentyStandardFlatApplication,
});
} catch (error) {
this.logger.error(
`Failed to create file field metadata for attachments in workspace ${workspaceId}: ${error.message}`,
);
throw error;
}
}
this.logger.log(
`Created file field metadata for attachments in workspace ${workspaceId}`,
);
}
if (!isDefined(attachmentFileflatFieldMetadata)) {
this.logger.error(
`File field metadata not found for attachments in workspace ${workspaceId}`,
);
return;
}
const attachmentRepository =
await this.twentyORMGlobalManager.getRepository<AttachmentWorkspaceEntity>(
workspaceId,
'attachment',
{ shouldBypassPermissionChecks: true },
);
const attachments = await attachmentRepository.find({
where: {
fullPath: Not(IsNull()),
file: Or(IsNull(), Equal([])),
},
select: ['id', 'name', 'fullPath'],
});
if (attachments.length === 0) {
this.logger.log(`No attachments to migrate for workspace ${workspaceId}`);
return;
}
this.logger.log(
`Found ${attachments.length} attachment(s) to migrate in workspace ${workspaceId}`,
);
const fileRepository = this.coreDataSource.getRepository(FileEntity);
for (const attachment of attachments) {
if (!isNonEmptyString(attachment.fullPath)) {
this.logger.warn(
`Skipping attachment ${attachment.id} - invalid fullPath`,
);
continue;
}
try {
const { type: fileExtension, filename } =
extractFolderPathFilenameAndTypeOrThrow(attachment.fullPath);
const fileId = v4();
const newFilename = `${fileId}${isNonEmptyString(fileExtension) ? `.${fileExtension}` : ''}`;
const newResourcePath = `${FileFolder.FilesField}/${attachmentFileflatFieldMetadata.universalIdentifier}/${newFilename}`;
if (!isDryRun) {
await this.fileStorageService.copyLegacy({
from: {
folderPath: `workspace-${workspaceId}`,
filename: attachment.fullPath,
},
to: {
folderPath: `${workspaceId}/${twentyStandardFlatApplication.universalIdentifier}`,
filename: newResourcePath,
},
});
const fileEntity = fileRepository.create({
id: fileId,
path: newResourcePath,
workspaceId,
applicationId: twentyStandardFlatApplication.id,
size: -1,
settings: {
isTemporaryFile: true,
toDelete: false,
},
});
await fileRepository.save(fileEntity);
await attachmentRepository.update(
{ id: attachment.id },
{
file: [
{
fileId: fileEntity.id,
label: attachment.name || filename,
},
],
},
);
}
this.logger.log(
`Migrated attachment ${attachment.id} (${attachment.name})`,
);
} catch (error) {
this.logger.error(
`Failed to migrate attachment ${attachment.id} in workspace ${workspaceId}: ${error.message}`,
);
throw error;
}
}
if (!isDryRun) {
await this.featureFlagService.enableFeatureFlags(
[FeatureFlagKey.IS_FILES_FIELD_MIGRATED],
workspaceId,
);
}
this.logger.log(
`${isDryRun ? '[DRY RUN] ' : ''}Completed attachment files migration for workspace ${workspaceId}`,
);
}
}
@@ -0,0 +1,296 @@
import { InjectDataSource, InjectRepository } from '@nestjs/typeorm';
import { isNonEmptyString } from '@sniptt/guards';
import { Command } from 'nest-commander';
import { STANDARD_OBJECTS } from 'twenty-shared/metadata';
import { FieldMetadataType, FileFolder } from 'twenty-shared/types';
import {
assertIsDefinedOrThrow,
extractFolderPathFilenameAndTypeOrThrow,
isDefined,
} from 'twenty-shared/utils';
import { DataSource, Equal, ILike, IsNull, Or, Repository } from 'typeorm';
import { v4 } from 'uuid';
import { ActiveOrSuspendedWorkspacesMigrationCommandRunner } from 'src/database/commands/command-runners/active-or-suspended-workspaces-migration.command-runner';
import { RunOnWorkspaceArgs } from 'src/database/commands/command-runners/workspaces-migration.command-runner';
import { getFlatFieldsFromFlatObjectMetadata } from 'src/engine/api/graphql/workspace-schema-builder/utils/get-flat-fields-for-flat-object-metadata.util';
import { ApplicationService } from 'src/engine/core-modules/application/services/application.service';
import { FeatureFlagKey } from 'src/engine/core-modules/feature-flag/enums/feature-flag-key.enum';
import { FeatureFlagService } from 'src/engine/core-modules/feature-flag/services/feature-flag.service';
import { FileStorageService } from 'src/engine/core-modules/file-storage/file-storage.service';
import { FileEntity } from 'src/engine/core-modules/file/entities/file.entity';
import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity';
import { DataSourceService } from 'src/engine/metadata-modules/data-source/data-source.service';
import { FieldMetadataService } from 'src/engine/metadata-modules/field-metadata/services/field-metadata.service';
import { findFlatEntityByUniversalIdentifier } from 'src/engine/metadata-modules/flat-entity/utils/find-flat-entity-by-universal-identifier.util';
import { type FlatObjectMetadata } from 'src/engine/metadata-modules/flat-object-metadata/types/flat-object-metadata.type';
import { GlobalWorkspaceOrmManager } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager';
import { WorkspaceCacheService } from 'src/engine/workspace-cache/services/workspace-cache.service';
import { PersonWorkspaceEntity } from 'src/modules/person/standard-objects/person.workspace-entity';
@Command({
name: 'upgrade:1-18:migrate-person-avatar-files',
description:
'Migrate person avatarUrl files to file field: copy files and create file records',
})
export class MigratePersonAvatarFilesCommand extends ActiveOrSuspendedWorkspacesMigrationCommandRunner {
constructor(
@InjectRepository(WorkspaceEntity)
protected readonly workspaceRepository: Repository<WorkspaceEntity>,
protected readonly twentyORMGlobalManager: GlobalWorkspaceOrmManager,
protected readonly dataSourceService: DataSourceService,
private readonly featureFlagService: FeatureFlagService,
private readonly fileStorageService: FileStorageService,
private readonly workspaceCacheService: WorkspaceCacheService,
private readonly fieldMetadataService: FieldMetadataService,
private readonly applicationService: ApplicationService,
@InjectDataSource()
private readonly coreDataSource: DataSource,
) {
super(workspaceRepository, twentyORMGlobalManager, dataSourceService);
}
override async runOnWorkspace({
workspaceId,
options,
}: RunOnWorkspaceArgs): Promise<void> {
const isDryRun = options.dryRun ?? false;
const isMigrated = await this.featureFlagService.isFeatureEnabled(
FeatureFlagKey.IS_FILES_FIELD_MIGRATED,
workspaceId,
);
if (isMigrated) {
this.logger.log(
`Person avatar files migration already completed for workspace ${workspaceId}, skipping`,
);
return;
}
this.logger.log(
`${
isDryRun ? '[DRY RUN] ' : ''
}Starting person avatar files migration for workspace ${workspaceId}`,
);
const { flatObjectMetadataMaps, flatFieldMetadataMaps } =
await this.workspaceCacheService.getOrRecompute(workspaceId, [
'flatObjectMetadataMaps',
'flatFieldMetadataMaps',
]);
const personObjectMetadata =
findFlatEntityByUniversalIdentifier<FlatObjectMetadata>({
flatEntityMaps: flatObjectMetadataMaps,
universalIdentifier: STANDARD_OBJECTS.person.universalIdentifier,
});
if (!isDefined(personObjectMetadata)) {
this.logger.warn(
`Person object metadata not found for workspace ${workspaceId}, skipping`,
);
return;
}
const { twentyStandardFlatApplication } =
await this.applicationService.findWorkspaceTwentyStandardAndCustomApplicationOrThrow(
{
workspaceId,
},
);
if (!isDefined(twentyStandardFlatApplication)) {
this.logger.error(
`Twenty standard application not found for workspace ${workspaceId}`,
);
return;
}
let avatarFileFieldMetadata = findFlatEntityByUniversalIdentifier({
flatEntityMaps: flatFieldMetadataMaps,
universalIdentifier:
STANDARD_OBJECTS.person.fields.avatarFile.universalIdentifier,
});
if (!isDefined(avatarFileFieldMetadata)) {
this.logger.log(
`Avatar file field metadata not found for workspace ${workspaceId}, creating it`,
);
if (!isDryRun) {
const personFieldMetadatas = getFlatFieldsFromFlatObjectMetadata(
personObjectMetadata,
flatFieldMetadataMaps,
);
const existingFieldWithSameName = personFieldMetadatas.find(
(fieldMetadata) => fieldMetadata.name === 'avatarFile',
);
if (isDefined(existingFieldWithSameName)) {
this.logger.log(
`Found existing field with name 'avatarFile' (id: ${existingFieldWithSameName.id}), renaming it to 'old-avatarFile'`,
);
try {
await this.fieldMetadataService.updateOneField({
updateFieldInput: {
id: existingFieldWithSameName.id,
name: 'oldAvatarFile',
label: 'Old Avatar File',
},
workspaceId,
ownerFlatApplication: twentyStandardFlatApplication,
});
} catch (error) {
this.logger.error(
`Failed to rename existing 'avatarFile' field to 'old-avatarFile' for workspace ${workspaceId}: ${error.message}`,
);
throw error;
}
this.logger.log(
`Renamed existing 'avatarFile' field to 'oldAvatarFile' for workspace ${workspaceId}`,
);
}
try {
avatarFileFieldMetadata =
await this.fieldMetadataService.createOneField({
createFieldInput: {
objectMetadataId: personObjectMetadata.id,
type: FieldMetadataType.FILES,
name: 'avatarFile',
label: 'Avatar File',
description: "Contact's avatar file",
icon: 'IconFileUpload',
isSystem: true,
isNullable: true,
settings: {
maxNumberOfValues: 1,
},
universalIdentifier:
STANDARD_OBJECTS.person.fields.avatarFile.universalIdentifier,
},
workspaceId,
ownerFlatApplication: twentyStandardFlatApplication,
});
} catch (error) {
this.logger.error(
`Failed to create avatarFile field metadata for workspace ${workspaceId}: ${error.message}`,
);
throw error;
}
}
this.logger.log(
`Created avatarFile field metadata for workspace ${workspaceId}`,
);
}
if (!isDefined(avatarFileFieldMetadata)) {
this.logger.error(
`Avatar file field metadata not found for workspace ${workspaceId}`,
);
return;
}
const personRepository =
await this.twentyORMGlobalManager.getRepository<PersonWorkspaceEntity>(
workspaceId,
'person',
{ shouldBypassPermissionChecks: true },
);
const personsWithAvatars = await personRepository.find({
where: {
avatarUrl: ILike(`%${FileFolder.PersonPicture}%`),
avatarFile: Or(IsNull(), Equal([])),
},
select: ['id', 'avatarUrl'],
});
if (personsWithAvatars.length === 0) {
this.logger.log(
`No persons with avatarUrl found in workspace ${workspaceId}`,
);
return;
}
this.logger.log(
`Found ${personsWithAvatars.length} person(s) with avatarUrl containing 'people' folder in workspace ${workspaceId}`,
);
const fileRepository = this.coreDataSource.getRepository(FileEntity);
for (const person of personsWithAvatars) {
assertIsDefinedOrThrow(person.avatarUrl);
try {
const { type: fileExtension } = extractFolderPathFilenameAndTypeOrThrow(
person.avatarUrl,
);
const fileId = v4();
const newFileName = `${fileId}${isNonEmptyString(fileExtension) ? `.${fileExtension}` : ''}`;
const newResourcePath = `${FileFolder.FilesField}/${avatarFileFieldMetadata.universalIdentifier}/${newFileName}`;
if (!isDryRun) {
await this.fileStorageService.copyLegacy({
from: {
folderPath: `workspace-${workspaceId}`,
filename: person.avatarUrl,
},
to: {
folderPath: `${workspaceId}/${twentyStandardFlatApplication.universalIdentifier}`,
filename: newResourcePath,
},
});
const fileEntity = fileRepository.create({
id: fileId,
path: newResourcePath,
workspaceId,
applicationId: twentyStandardFlatApplication.id,
size: -1,
settings: {
isTemporaryFile: true,
toDelete: false,
},
});
await fileRepository.save(fileEntity);
await personRepository.update(
{ id: person.id },
{
avatarFile: [
{
fileId: fileEntity.id,
label: newFileName,
},
],
},
);
}
this.logger.log(`Migrated avatar for person ${person.id}`);
} catch (error) {
this.logger.error(
`Failed to migrate avatar for person ${person.id} in workspace ${workspaceId}: ${error.message}`,
);
throw error;
}
}
this.logger.log(
`${isDryRun ? '[DRY RUN] ' : ''}Completed person avatar files migration for workspace ${workspaceId}`,
);
}
}
@@ -0,0 +1,37 @@
import { Module } from '@nestjs/common';
import { TypeOrmModule } from '@nestjs/typeorm';
import { MigrateAttachmentFilesCommand } from 'src/database/commands/upgrade-version-command/1-18/1-18-migrate-attachment-files.command';
import { MigratePersonAvatarFilesCommand } from 'src/database/commands/upgrade-version-command/1-18/1-18-migrate-person-avatar-files.command';
import { ApplicationModule } from 'src/engine/core-modules/application/application.module';
import { FeatureFlagEntity } from 'src/engine/core-modules/feature-flag/feature-flag.entity';
import { FeatureFlagModule } from 'src/engine/core-modules/feature-flag/feature-flag.module';
import { FileStorageModule } from 'src/engine/core-modules/file-storage/file-storage.module';
import { FileEntity } from 'src/engine/core-modules/file/entities/file.entity';
import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity';
import { DataSourceModule } from 'src/engine/metadata-modules/data-source/data-source.module';
import { FieldMetadataModule } from 'src/engine/metadata-modules/field-metadata/field-metadata.module';
import { WorkspaceCacheModule } from 'src/engine/workspace-cache/workspace-cache.module';
import { AttachmentWorkspaceEntity } from 'src/modules/attachment/standard-objects/attachment.workspace-entity';
import { PersonWorkspaceEntity } from 'src/modules/person/standard-objects/person.workspace-entity';
@Module({
imports: [
TypeOrmModule.forFeature([
WorkspaceEntity,
FeatureFlagEntity,
PersonWorkspaceEntity,
FileEntity,
AttachmentWorkspaceEntity,
]),
DataSourceModule,
FeatureFlagModule,
FileStorageModule.forRoot(),
WorkspaceCacheModule,
FieldMetadataModule,
ApplicationModule,
],
providers: [MigratePersonAvatarFilesCommand, MigrateAttachmentFilesCommand],
exports: [MigratePersonAvatarFilesCommand, MigrateAttachmentFilesCommand],
})
export class V1_18_UpgradeVersionCommandModule {}
@@ -2,6 +2,7 @@ import { Module } from '@nestjs/common';
import { TypeOrmModule } from '@nestjs/typeorm';
import { V1_17_UpgradeVersionCommandModule } from 'src/database/commands/upgrade-version-command/1-17/1-17-upgrade-version-command.module';
import { V1_18_UpgradeVersionCommandModule } from 'src/database/commands/upgrade-version-command/1-18/1-18-upgrade-version-command.module';
import { UpgradeCommand } from 'src/database/commands/upgrade-version-command/upgrade.command';
import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity';
import { DataSourceModule } from 'src/engine/metadata-modules/data-source/data-source.module';
@@ -10,6 +11,7 @@ import { DataSourceModule } from 'src/engine/metadata-modules/data-source/data-s
imports: [
TypeOrmModule.forFeature([WorkspaceEntity]),
V1_17_UpgradeVersionCommandModule,
V1_18_UpgradeVersionCommandModule,
DataSourceModule,
],
providers: [UpgradeCommand],
@@ -19,6 +19,7 @@ import { MigrateFavoritesToNavigationMenuItemsCommand } from 'src/database/comma
import { MigrateNoteTargetToMorphRelationsCommand } from 'src/database/commands/upgrade-version-command/1-17/1-17-migrate-note-target-to-morph-relations.command';
import { MigrateTaskTargetToMorphRelationsCommand } from 'src/database/commands/upgrade-version-command/1-17/1-17-migrate-task-target-to-morph-relations.command';
import { MigrateWorkflowCodeStepsCommand } from 'src/database/commands/upgrade-version-command/1-17/1-17-migrate-workflow-code-steps.command';
import { MigratePersonAvatarFilesCommand } from 'src/database/commands/upgrade-version-command/1-18/1-18-migrate-person-avatar-files.command';
import { TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service';
import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity';
import { DataSourceService } from 'src/engine/metadata-modules/data-source/data-source.service';
@@ -49,6 +50,9 @@ export class UpgradeCommand extends UpgradeCommandRunner {
protected readonly makeWebhookUniversalIdentifierAndApplicationIdNotNullableMigrationCommand: MakeWebhookUniversalIdentifierAndApplicationIdNotNullableMigrationCommand,
protected readonly migrateWorkflowCodeStepsCommand: MigrateWorkflowCodeStepsCommand,
protected readonly fixMorphRelationFieldNamesCommand: FixMorphRelationFieldNamesCommand,
// 1.18 Commands
protected readonly migratePersonAvatarFilesCommand: MigratePersonAvatarFilesCommand,
) {
super(
workspaceRepository,
@@ -74,9 +78,14 @@ export class UpgradeCommand extends UpgradeCommandRunner {
this.fixMorphRelationFieldNamesCommand,
];
const commands_1180: VersionCommands = [
this.migratePersonAvatarFilesCommand,
];
this.allCommands = {
'1.16.0': commands_1160,
'1.17.0': commands_1170,
'1.18.0': commands_1180,
};
}
@@ -0,0 +1,15 @@
import { type MigrationInterface, type QueryRunner } from 'typeorm';
export class AddMimeTypeToFileTable1770814914548 implements MigrationInterface {
name = 'AddMimeTypeToFileTable1770814914548';
public async up(queryRunner: QueryRunner): Promise<void> {
await queryRunner.query(
`ALTER TABLE "core"."file" ADD "mimeType" character varying NOT NULL DEFAULT 'application/octet-stream'`,
);
}
public async down(queryRunner: QueryRunner): Promise<void> {
await queryRunner.query(`ALTER TABLE "core"."file" DROP COLUMN "mimeType"`);
}
}
@@ -114,6 +114,7 @@ export class FileStorageService {
workspaceId,
applicationId: application.id,
id: fileId,
mimeType,
size:
typeof sourceFile === 'string'
? Buffer.byteLength(sourceFile)
@@ -7,12 +7,15 @@ import { CustomException } from 'src/utils/custom-exception';
export enum FileStorageExceptionCode {
FILE_NOT_FOUND = 'FILE_NOT_FOUND',
ACCESS_DENIED = 'ACCESS_DENIED',
INVALID_EXTENSION = 'INVALID_EXTENSION',
}
const getFileStorageExceptionUserFriendlyMessage = (
code: FileStorageExceptionCode,
) => {
switch (code) {
case FileStorageExceptionCode.INVALID_EXTENSION:
return msg`Wrong extension for file.`;
case FileStorageExceptionCode.FILE_NOT_FOUND:
return msg`File not found.`;
case FileStorageExceptionCode.ACCESS_DENIED:
@@ -56,4 +56,11 @@ export class FileEntity extends WorkspaceRelatedEntity {
@Column({ nullable: true, type: 'jsonb' })
settings: FileSettings | null;
@Column({
nullable: false,
type: 'varchar',
default: 'application/octet-stream',
})
mimeType: string;
}
@@ -4,6 +4,7 @@ import { InjectRepository } from '@nestjs/typeorm';
import { Readable } from 'stream';
import { msg } from '@lingui/core/macro';
import { isNonEmptyString } from '@sniptt/guards';
import { FileFolder } from 'twenty-shared/types';
import { Repository } from 'typeorm';
import { v4 } from 'uuid';
@@ -44,26 +45,23 @@ export class FilesFieldService {
async uploadFile({
file,
filename,
declaredMimeType,
workspaceId,
fieldMetadataId,
}: {
file: Buffer;
filename: string;
declaredMimeType: string | undefined;
workspaceId: string;
fieldMetadataId: string;
}): Promise<FileEntity> {
const { mimeType, ext } = await extractFileInfo({
file,
declaredMimeType,
filename,
});
const sanitizedFile = sanitizeFile({ file, ext, mimeType });
const fileId = v4();
const name = `${fileId}${ext ? `.${ext}` : ''}`;
const name = `${fileId}${isNonEmptyString(ext) ? `.${ext}` : ''}`;
const fieldMetadata = await this.fieldMetadataRepository.findOneOrFail({
select: ['applicationId', 'universalIdentifier'],
@@ -30,7 +30,7 @@ export class FilesFieldResolver {
@AuthWorkspace()
{ id: workspaceId }: WorkspaceEntity,
@Args({ name: 'file', type: () => GraphQLUpload })
{ createReadStream, filename, mimetype }: FileUpload,
{ createReadStream, filename }: FileUpload,
@Args({
name: 'fieldMetadataId',
type: () => String,
@@ -44,7 +44,6 @@ export class FilesFieldResolver {
return await this.filesFieldService.uploadFile({
file: buffer,
filename,
declaredMimeType: mimetype,
workspaceId,
fieldMetadataId,
});
@@ -1,28 +1,35 @@
import FileType from 'file-type';
import { extractFileInfo } from 'src/engine/core-modules/file/utils/extract-file-info.utils';
jest.mock('file-type', () => ({
fromBuffer: jest.fn(),
}));
// Mock detectableMimeTypes to work around ESM/CommonJS interop issues in Jest
jest.mock('file-type', () => {
const actual = jest.requireActual('file-type');
return {
...actual,
mimeTypes: actual.mimeTypes,
};
});
describe('extractFileInfo', () => {
const mockBuffer = Buffer.from('test content');
// Real PNG file header (magic numbers)
const pngBuffer = Buffer.from([
0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0x00, 0x00, 0x00, 0x0d,
0x49, 0x48, 0x44, 0x52,
]);
beforeEach(() => {
jest.clearAllMocks();
});
// Real PDF file header
const pdfBuffer = Buffer.from('%PDF-1.4\n', 'utf-8');
it('should use detected file type when available', async () => {
(FileType.fromBuffer as jest.Mock).mockResolvedValue({
mime: 'image/png',
ext: 'png',
});
// Plain text buffer (no magic numbers)
const textBuffer = Buffer.from('Hello, world!', 'utf-8');
// Real ZIP file header (for testing docx, xlsx, etc.)
const zipBuffer = Buffer.from([0x50, 0x4b, 0x03, 0x04]);
it('should detect PNG from buffer magic numbers', async () => {
const result = await extractFileInfo({
file: mockBuffer,
declaredMimeType: 'text/plain',
filename: 'test.txt',
file: pngBuffer,
filename: 'image.png',
});
expect(result).toEqual({
@@ -31,31 +38,10 @@ describe('extractFileInfo', () => {
});
});
it('should fall back to declared values when file type is not detected', async () => {
(FileType.fromBuffer as jest.Mock).mockResolvedValue(undefined);
it('should detect PDF from buffer magic numbers', async () => {
const result = await extractFileInfo({
file: mockBuffer,
declaredMimeType: 'text/plain',
filename: 'test.txt',
});
expect(result).toEqual({
mimeType: 'text/plain',
ext: 'txt',
});
});
it('should handle missing declared mime type when file type is detected', async () => {
(FileType.fromBuffer as jest.Mock).mockResolvedValue({
mime: 'application/pdf',
ext: 'pdf',
});
const result = await extractFileInfo({
file: mockBuffer,
declaredMimeType: undefined,
filename: 'document',
file: pdfBuffer,
filename: 'document.pdf',
});
expect(result).toEqual({
@@ -64,45 +50,133 @@ describe('extractFileInfo', () => {
});
});
it('should handle both mime type and extension being undefined', async () => {
(FileType.fromBuffer as jest.Mock).mockResolvedValue(undefined);
it('should use extension-based lookup for text files', async () => {
const result = await extractFileInfo({
file: mockBuffer,
declaredMimeType: undefined,
file: textBuffer,
filename: 'document.txt',
});
expect(result).toEqual({
mimeType: 'text/plain',
ext: 'txt',
});
});
it('should handle CSV files using extension', async () => {
const result = await extractFileInfo({
file: textBuffer,
filename: 'data.csv',
});
expect(result).toEqual({
mimeType: 'text/csv',
ext: 'csv',
});
});
it('should handle JSON files using extension', async () => {
const result = await extractFileInfo({
file: Buffer.from('{"key": "value"}'),
filename: 'config.json',
});
expect(result).toEqual({
mimeType: 'application/json',
ext: 'json',
});
});
it('should return application/octet-stream for unknown extensions', async () => {
const result = await extractFileInfo({
file: textBuffer,
filename: 'file.unknown',
});
expect(result).toEqual({
mimeType: 'application/octet-stream',
ext: 'unknown',
});
});
it('should return application/octet-stream for files without extension', async () => {
const result = await extractFileInfo({
file: textBuffer,
filename: 'file-without-extension',
});
expect(result).toEqual({
mimeType: undefined,
mimeType: 'application/octet-stream',
ext: '',
});
});
it('should handle filenames with multiple dots', async () => {
(FileType.fromBuffer as jest.Mock).mockResolvedValue(undefined);
it('should detect ZIP files from buffer', async () => {
const result = await extractFileInfo({
file: mockBuffer,
declaredMimeType: 'application/gzip',
filename: 'archive.tar.gz',
file: zipBuffer,
filename: 'archive.zip',
});
expect(result).toEqual({
mimeType: 'application/gzip',
ext: 'gz',
mimeType: 'application/zip',
ext: 'zip',
});
});
it('should call FileType.fromBuffer with the provided buffer', async () => {
(FileType.fromBuffer as jest.Mock).mockResolvedValue(undefined);
it('should throw error when PNG extension is used with non-PNG buffer', async () => {
await expect(
extractFileInfo({
file: textBuffer,
filename: 'fake-image.png',
}),
).rejects.toThrow(
"File content does not match its extension. The file has extension 'png' (expected mime type: image/png), but the file content could not be detected as this type. The file may be corrupted, have the wrong extension, or be a security risk.",
);
});
await extractFileInfo({
file: mockBuffer,
declaredMimeType: 'image/png',
filename: 'image.png',
it('should throw error when PDF extension is used with non-PDF buffer', async () => {
await expect(
extractFileInfo({
file: textBuffer,
filename: 'fake-document.pdf',
}),
).rejects.toThrow(
"File content does not match its extension. The file has extension 'pdf' (expected mime type: application/pdf), but the file content could not be detected as this type. The file may be corrupted, have the wrong extension, or be a security risk.",
);
});
it('should handle markdown files using extension', async () => {
const result = await extractFileInfo({
file: Buffer.from('# Heading\n\nContent'),
filename: 'README.md',
});
expect(FileType.fromBuffer).toHaveBeenCalledWith(mockBuffer);
expect(result).toEqual({
mimeType: 'text/markdown',
ext: 'md',
});
});
it('should handle HTML files using extension', async () => {
const result = await extractFileInfo({
file: Buffer.from('<html><body>Test</body></html>'),
filename: 'index.html',
});
expect(result).toEqual({
mimeType: 'text/html',
ext: 'html',
});
});
it('should prefer detected type over declared extension', async () => {
const result = await extractFileInfo({
file: pngBuffer,
filename: 'image.txt',
});
expect(result).toEqual({
mimeType: 'image/png',
ext: 'png',
});
});
});
@@ -1,23 +1,57 @@
import FileType from 'file-type';
import { msg } from '@lingui/core/macro';
import { isNonEmptyString } from '@sniptt/guards';
import FileType, { type MimeType } from 'file-type';
import { lookup } from 'mrmime';
import { isDefined } from 'twenty-shared/utils';
import {
FileStorageException,
FileStorageExceptionCode,
} from 'src/engine/core-modules/file-storage/interfaces/file-storage-exception';
import { buildFileInfo } from 'src/engine/core-modules/file/utils/build-file-info.utils';
export const extractFileInfo = async ({
file,
declaredMimeType,
filename,
}: {
file: Buffer;
declaredMimeType: string | undefined;
filename: string;
}) => {
const { ext: declaredExt } = buildFileInfo(filename);
const detectedFileType = await FileType.fromBuffer(file);
const { ext: detectedExt, mime: detectedMime } =
(await FileType.fromBuffer(file)) ?? {};
const mimeType = detectedFileType?.mime ?? declaredMimeType;
if (isDefined(detectedExt) && isDefined(detectedMime)) {
return {
mimeType: detectedMime,
ext: detectedExt,
};
}
const ext = detectedFileType?.ext ?? declaredExt;
const ext = declaredExt;
let mimeType: string = 'application/octet-stream';
if (isNonEmptyString(ext)) {
const mimeTypeFromExtension = lookup(ext);
if (
mimeTypeFromExtension &&
FileType.mimeTypes.has(mimeTypeFromExtension as MimeType)
) {
throw new FileStorageException(
`File content does not match its extension. The file has extension '${ext}' (expected mime type: ${mimeTypeFromExtension}), but the file content could not be detected as this type. The file may be corrupted, have the wrong extension, or be a security risk.`,
FileStorageExceptionCode.INVALID_EXTENSION,
{
userFriendlyMessage: msg`The file extension doesn't match the file content. Please check that your file is not corrupted and has the correct extension.`,
},
);
}
mimeType = mimeTypeFromExtension ?? 'application/octet-stream';
}
return {
mimeType,
@@ -134,6 +134,14 @@ export class FilesFieldSync {
return null;
}
const isModifyingFilesField = filesFields.some((filesField) =>
isDefined(updatePayload[filesField.name as keyof typeof updatePayload]),
);
if (!isModifyingFilesField) {
return null;
}
if (existingRecords.length !== 1) {
throw new TwentyORMException(
`Cannot update multiple records with files field at once`,
@@ -229,6 +229,7 @@ export const buildPersonStandardFlatFieldMetadatas = ({
twentyStandardApplicationId,
now,
}),
//deprecated
avatarUrl: createStandardFieldFlatMetadata({
objectName,
workspaceId,
@@ -246,6 +247,26 @@ export const buildPersonStandardFlatFieldMetadatas = ({
twentyStandardApplicationId,
now,
}),
avatarFile: createStandardFieldFlatMetadata({
objectName,
workspaceId,
context: {
fieldName: 'avatarFile',
type: FieldMetadataType.FILES,
label: 'Avatar File',
description: "Contact's avatar file",
icon: 'IconFileUpload',
isSystem: true,
isNullable: true,
settings: {
maxNumberOfValues: 1,
},
},
standardObjectMetadataRelatedEntityIds,
dependencyFlatEntityMaps,
twentyStandardApplicationId,
now,
}),
position: createStandardFieldFlatMetadata({
objectName,
workspaceId,
@@ -7,6 +7,7 @@ import {
type PhonesMetadata,
} from 'twenty-shared/types';
import { type FileOutput } from 'src/engine/api/common/common-args-processors/data-arg-processor/types/file-item.type';
import { BaseWorkspaceEntity } from 'src/engine/twenty-orm/base.workspace-entity';
import { type FieldTypeAndNameMetadata } from 'src/engine/workspace-manager/utils/get-ts-vector-column-expression.util';
import { type EntityRelation } from 'src/engine/workspace-manager/workspace-migration/types/entity-relation.interface';
@@ -42,7 +43,9 @@ export class PersonWorkspaceEntity extends BaseWorkspaceEntity {
phone: string | null;
phones: PhonesMetadata;
city: string | null;
/** @deprecated Use `avatarFile` field instead */
avatarUrl: string | null;
avatarFile: FileOutput[] | null;
position: number;
createdBy: ActorMetadata;
updatedBy: ActorMetadata;