From eb52976e9185cc3575a9dad7e1c61c41b81294fe Mon Sep 17 00:00:00 2001 From: Etienne <45695613+etiennejouan@users.noreply.github.com> Date: Thu, 12 Feb 2026 11:49:10 +0100 Subject: [PATCH] File v2 - Backfill mimeType and size - command (#17875) --- .../files/hooks/useUploadAttachmentFile.tsx | 12 +- .../hooks/useUploadFilesFieldFile.ts | 6 +- ...ackfill-file-size-and-mime-type.command.ts | 184 ++++++++++++++++++ .../1-18-upgrade-version-command.module.ts | 13 +- .../upgrade.command.ts | 3 + 5 files changed, 207 insertions(+), 11 deletions(-) create mode 100644 packages/twenty-server/src/database/commands/upgrade-version-command/1-18/1-18-backfill-file-size-and-mime-type.command.ts diff --git a/packages/twenty-front/src/modules/activities/files/hooks/useUploadAttachmentFile.tsx b/packages/twenty-front/src/modules/activities/files/hooks/useUploadAttachmentFile.tsx index fddf7930d6..fd95d723c7 100644 --- a/packages/twenty-front/src/modules/activities/files/hooks/useUploadAttachmentFile.tsx +++ b/packages/twenty-front/src/modules/activities/files/hooks/useUploadAttachmentFile.tsx @@ -2,26 +2,26 @@ import { type Attachment } from '@/activities/files/types/Attachment'; import { getFileType } from '@/activities/files/utils/getFileType'; import { type ActivityTargetableObject } from '@/activities/types/ActivityTargetableEntity'; import { getActivityTargetObjectFieldIdName } from '@/activities/utils/getActivityTargetObjectFieldIdName'; -import { useApolloCoreClient } from '@/object-metadata/hooks/useApolloCoreClient'; import { useObjectMetadataItem } from '@/object-metadata/hooks/useObjectMetadataItem'; import { CoreObjectNameSingular } from '@/object-metadata/types/CoreObjectNameSingular'; import { useCreateOneRecord } from '@/object-record/hooks/useCreateOneRecord'; import { useIsFeatureEnabled } from '@/workspace/hooks/useIsFeatureEnabled'; +import { useApolloClient } from '@apollo/client'; import { t } from '@lingui/core/macro'; import { assertIsDefinedOrThrow, isDefined } from 'twenty-shared/utils'; import { + FeatureFlagKey, + FieldMetadataType, FileFolder, useUploadFileMutation, useUploadFilesFieldFileMutation, - FeatureFlagKey, - FieldMetadataType, } from '~/generated-metadata/graphql'; export const useUploadAttachmentFile = () => { - const coreClient = useApolloCoreClient(); - const [uploadFile] = useUploadFileMutation({ client: coreClient }); + const apolloClient = useApolloClient(); + const [uploadFile] = useUploadFileMutation({ client: apolloClient }); const [uploadFilesFieldFile] = useUploadFilesFieldFileMutation({ - client: coreClient, + client: apolloClient, }); const isAttachmentMigrated = useIsFeatureEnabled( FeatureFlagKey.IS_ATTACHMENT_MIGRATED, diff --git a/packages/twenty-front/src/modules/object-record/record-field/ui/meta-types/hooks/useUploadFilesFieldFile.ts b/packages/twenty-front/src/modules/object-record/record-field/ui/meta-types/hooks/useUploadFilesFieldFile.ts index eb495aee9e..78983df09e 100644 --- a/packages/twenty-front/src/modules/object-record/record-field/ui/meta-types/hooks/useUploadFilesFieldFile.ts +++ b/packages/twenty-front/src/modules/object-record/record-field/ui/meta-types/hooks/useUploadFilesFieldFile.ts @@ -1,5 +1,5 @@ -import { useApolloCoreClient } from '@/object-metadata/hooks/useApolloCoreClient'; import { useSnackBar } from '@/ui/feedback/snack-bar-manager/hooks/useSnackBar'; +import { useApolloClient } from '@apollo/client'; import { useLingui } from '@lingui/react/macro'; import { isDefined } from 'twenty-shared/utils'; import { useUploadFilesFieldFileMutation } from '~/generated-metadata/graphql'; @@ -8,9 +8,9 @@ const DEFAULT_VALUE_BEFORE_SERVER_RESPONSE = 'default-value-before-server-response'; export const useUploadFilesFieldFile = () => { - const coreClient = useApolloCoreClient(); + const apolloClient = useApolloClient(); const [uploadFilesFieldFile] = useUploadFilesFieldFileMutation({ - client: coreClient, + client: apolloClient, }); const { enqueueSuccessSnackBar, enqueueErrorSnackBar } = useSnackBar(); const { t } = useLingui(); diff --git a/packages/twenty-server/src/database/commands/upgrade-version-command/1-18/1-18-backfill-file-size-and-mime-type.command.ts b/packages/twenty-server/src/database/commands/upgrade-version-command/1-18/1-18-backfill-file-size-and-mime-type.command.ts new file mode 100644 index 0000000000..7f216ab03b --- /dev/null +++ b/packages/twenty-server/src/database/commands/upgrade-version-command/1-18/1-18-backfill-file-size-and-mime-type.command.ts @@ -0,0 +1,184 @@ +import { InjectDataSource, InjectRepository } from '@nestjs/typeorm'; + +import { Command } from 'nest-commander'; +import { FileFolder } from 'twenty-shared/types'; +import { DataSource, Equal, LessThan, Repository } from 'typeorm'; + +import { ActiveOrSuspendedWorkspacesMigrationCommandRunner } from 'src/database/commands/command-runners/active-or-suspended-workspaces-migration.command-runner'; +import { RunOnWorkspaceArgs } from 'src/database/commands/command-runners/workspaces-migration.command-runner'; +import { FileStorageService } from 'src/engine/core-modules/file-storage/file-storage.service'; +import { FileEntity } from 'src/engine/core-modules/file/entities/file.entity'; +import { extractFileInfo } from 'src/engine/core-modules/file/utils/extract-file-info.utils'; +import { removeFileFolderFromFileEntityPath } from 'src/engine/core-modules/file/utils/remove-file-folder-from-file-entity-path.utils'; +import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity'; +import { DataSourceService } from 'src/engine/metadata-modules/data-source/data-source.service'; +import { GlobalWorkspaceOrmManager } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager'; +import { WorkspaceCacheService } from 'src/engine/workspace-cache/services/workspace-cache.service'; +import { streamToBuffer } from 'src/utils/stream-to-buffer'; + +@Command({ + name: 'upgrade:1-18:backfill-file-size-and-mime-type', + description: + 'Backfill file size and mime type for files with missing or default values', +}) +export class BackfillFileSizeAndMimeTypeCommand extends ActiveOrSuspendedWorkspacesMigrationCommandRunner { + constructor( + @InjectRepository(WorkspaceEntity) + protected readonly workspaceRepository: Repository, + protected readonly twentyORMGlobalManager: GlobalWorkspaceOrmManager, + protected readonly dataSourceService: DataSourceService, + private readonly fileStorageService: FileStorageService, + private readonly workspaceCacheService: WorkspaceCacheService, + @InjectDataSource() + private readonly coreDataSource: DataSource, + ) { + super(workspaceRepository, twentyORMGlobalManager, dataSourceService); + } + + override async runOnWorkspace({ + workspaceId, + options, + }: RunOnWorkspaceArgs): Promise { + const isDryRun = options.dryRun ?? false; + + this.logger.log( + `${isDryRun ? '[DRY RUN] ' : ''}Starting file size and mime type backfill for workspace ${workspaceId}`, + ); + + const { flatApplicationMaps } = + await this.workspaceCacheService.getOrRecompute(workspaceId, [ + 'flatApplicationMaps', + ]); + + const fileRepository = this.coreDataSource.getRepository(FileEntity); + + const filesToBackfill = await fileRepository.find({ + where: [ + { + workspaceId, + size: LessThan(0), + }, + { + workspaceId, + mimeType: Equal('application/octet-stream'), + }, + ], + select: [ + 'id', + 'path', + 'applicationId', + 'workspaceId', + 'size', + 'mimeType', + ], + }); + + if (filesToBackfill.length === 0) { + this.logger.log(`No files to backfill for workspace ${workspaceId}`); + + return; + } + + this.logger.log( + `Found ${filesToBackfill.length} file(s) to backfill in workspace ${workspaceId}`, + ); + + for (const fileEntity of filesToBackfill) { + try { + this.logger.log( + `Processing file ${fileEntity.id} (path: ${fileEntity.path})`, + ); + + const application = flatApplicationMaps.byId[fileEntity.applicationId]; + + if (!application) { + this.logger.error( + `Application not found for file ${fileEntity.id} (applicationId: ${fileEntity.applicationId})`, + ); + + continue; + } + + const [fileFolder] = fileEntity.path.split('/') as [FileFolder]; + + if (!Object.values(FileFolder).includes(fileFolder)) { + this.logger.error( + `Invalid file folder '${fileFolder}' for file ${fileEntity.id} (path: ${fileEntity.path})`, + ); + + continue; + } + + const fileExists = await this.fileStorageService.checkFileExists({ + fileFolder, + applicationUniversalIdentifier: application.universalIdentifier, + workspaceId, + resourcePath: removeFileFolderFromFileEntityPath(fileEntity.path), + }); + + if (!fileExists) { + this.logger.error( + `File ${fileEntity.id} (path: ${fileEntity.path}) not found in storage`, + ); + + continue; + } + + const fileStream = await this.fileStorageService.readFile({ + fileFolder, + applicationUniversalIdentifier: application.universalIdentifier, + workspaceId, + resourcePath: removeFileFolderFromFileEntityPath(fileEntity.path), + }); + + const fileBuffer = await streamToBuffer(fileStream); + + const updateData: { + size?: number; + mimeType?: string; + } = {}; + + if (fileEntity.size < 0) { + updateData.size = fileBuffer.length; + } + + if (fileEntity.mimeType === 'application/octet-stream') { + try { + const { mimeType } = await extractFileInfo({ + file: fileBuffer, + filename: fileEntity.path, + }); + + updateData.mimeType = mimeType; + } catch (error) { + this.logger.error( + `Failed to extract file mime type for file ${fileEntity.id} in workspace ${workspaceId}: ${error.message}`, + ); + + continue; + } + } + + if (!isDryRun) { + await fileRepository.update({ id: fileEntity.id }, updateData); + } + + const updateDetails = Object.entries(updateData) + .map(([key, value]) => `${key}=${value}`) + .join(', '); + + this.logger.log( + `${isDryRun ? '[DRY RUN] ' : ''}Backfilled file ${fileEntity.id}: ${updateDetails}`, + ); + } catch (error) { + this.logger.error( + `Failed to backfill file ${fileEntity.id} in workspace ${workspaceId}: ${error.message}`, + ); + } + } + + this.logger.log( + `${isDryRun ? '[DRY RUN] ' : ''}Completed file size and mime type backfill for workspace ${workspaceId}`, + ); + } +} diff --git a/packages/twenty-server/src/database/commands/upgrade-version-command/1-18/1-18-upgrade-version-command.module.ts b/packages/twenty-server/src/database/commands/upgrade-version-command/1-18/1-18-upgrade-version-command.module.ts index 469ca1ad26..03af16b46d 100644 --- a/packages/twenty-server/src/database/commands/upgrade-version-command/1-18/1-18-upgrade-version-command.module.ts +++ b/packages/twenty-server/src/database/commands/upgrade-version-command/1-18/1-18-upgrade-version-command.module.ts @@ -1,6 +1,7 @@ import { Module } from '@nestjs/common'; import { TypeOrmModule } from '@nestjs/typeorm'; +import { BackfillFileSizeAndMimeTypeCommand } from 'src/database/commands/upgrade-version-command/1-18/1-18-backfill-file-size-and-mime-type.command'; import { MigrateAttachmentFilesCommand } from 'src/database/commands/upgrade-version-command/1-18/1-18-migrate-attachment-files.command'; import { MigratePersonAvatarFilesCommand } from 'src/database/commands/upgrade-version-command/1-18/1-18-migrate-person-avatar-files.command'; import { ApplicationModule } from 'src/engine/core-modules/application/application.module'; @@ -31,7 +32,15 @@ import { PersonWorkspaceEntity } from 'src/modules/person/standard-objects/perso FieldMetadataModule, ApplicationModule, ], - providers: [MigratePersonAvatarFilesCommand, MigrateAttachmentFilesCommand], - exports: [MigratePersonAvatarFilesCommand, MigrateAttachmentFilesCommand], + providers: [ + MigratePersonAvatarFilesCommand, + MigrateAttachmentFilesCommand, + BackfillFileSizeAndMimeTypeCommand, + ], + exports: [ + MigratePersonAvatarFilesCommand, + MigrateAttachmentFilesCommand, + BackfillFileSizeAndMimeTypeCommand, + ], }) export class V1_18_UpgradeVersionCommandModule {} diff --git a/packages/twenty-server/src/database/commands/upgrade-version-command/upgrade.command.ts b/packages/twenty-server/src/database/commands/upgrade-version-command/upgrade.command.ts index 318e8918cd..cc14cf9658 100644 --- a/packages/twenty-server/src/database/commands/upgrade-version-command/upgrade.command.ts +++ b/packages/twenty-server/src/database/commands/upgrade-version-command/upgrade.command.ts @@ -19,6 +19,7 @@ import { MigrateFavoritesToNavigationMenuItemsCommand } from 'src/database/comma import { MigrateNoteTargetToMorphRelationsCommand } from 'src/database/commands/upgrade-version-command/1-17/1-17-migrate-note-target-to-morph-relations.command'; import { MigrateTaskTargetToMorphRelationsCommand } from 'src/database/commands/upgrade-version-command/1-17/1-17-migrate-task-target-to-morph-relations.command'; import { MigrateWorkflowCodeStepsCommand } from 'src/database/commands/upgrade-version-command/1-17/1-17-migrate-workflow-code-steps.command'; +import { BackfillFileSizeAndMimeTypeCommand } from 'src/database/commands/upgrade-version-command/1-18/1-18-backfill-file-size-and-mime-type.command'; import { MigratePersonAvatarFilesCommand } from 'src/database/commands/upgrade-version-command/1-18/1-18-migrate-person-avatar-files.command'; import { TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service'; import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity'; @@ -53,6 +54,7 @@ export class UpgradeCommand extends UpgradeCommandRunner { // 1.18 Commands protected readonly migratePersonAvatarFilesCommand: MigratePersonAvatarFilesCommand, + protected readonly backfillFileSizeAndMimeTypeCommand: BackfillFileSizeAndMimeTypeCommand, ) { super( workspaceRepository, @@ -80,6 +82,7 @@ export class UpgradeCommand extends UpgradeCommandRunner { const commands_1180: VersionCommands = [ this.migratePersonAvatarFilesCommand, + this.backfillFileSizeAndMimeTypeCommand, ]; this.allCommands = {