File v2 - Backfill mimeType and size - command (#17875)

packages/twenty-front/src/modules/activities/files/hooks/useUploadAttachmentFile.tsx

@@ -2,26 +2,26 @@ import { type Attachment } from '@/activities/files/types/Attachment';
 import { getFileType } from '@/activities/files/utils/getFileType';
 import { type ActivityTargetableObject } from '@/activities/types/ActivityTargetableEntity';
 import { getActivityTargetObjectFieldIdName } from '@/activities/utils/getActivityTargetObjectFieldIdName';
-import { useApolloCoreClient } from '@/object-metadata/hooks/useApolloCoreClient';
 import { useObjectMetadataItem } from '@/object-metadata/hooks/useObjectMetadataItem';
 import { CoreObjectNameSingular } from '@/object-metadata/types/CoreObjectNameSingular';
 import { useCreateOneRecord } from '@/object-record/hooks/useCreateOneRecord';
 import { useIsFeatureEnabled } from '@/workspace/hooks/useIsFeatureEnabled';
+import { useApolloClient } from '@apollo/client';
 import { t } from '@lingui/core/macro';
 import { assertIsDefinedOrThrow, isDefined } from 'twenty-shared/utils';
 import {
+  FeatureFlagKey,
+  FieldMetadataType,
   FileFolder,
   useUploadFileMutation,
   useUploadFilesFieldFileMutation,
-  FeatureFlagKey,
-  FieldMetadataType,
 } from '~/generated-metadata/graphql';
 
 export const useUploadAttachmentFile = () => {
-  const coreClient = useApolloCoreClient();
-  const [uploadFile] = useUploadFileMutation({ client: coreClient });
+  const apolloClient = useApolloClient();
+  const [uploadFile] = useUploadFileMutation({ client: apolloClient });
   const [uploadFilesFieldFile] = useUploadFilesFieldFileMutation({
-    client: coreClient,
+    client: apolloClient,
   });
   const isAttachmentMigrated = useIsFeatureEnabled(
     FeatureFlagKey.IS_ATTACHMENT_MIGRATED,

packages/twenty-front/src/modules/object-record/record-field/ui/meta-types/hooks/useUploadFilesFieldFile.ts

@@ -1,5 +1,5 @@
-import { useApolloCoreClient } from '@/object-metadata/hooks/useApolloCoreClient';
 import { useSnackBar } from '@/ui/feedback/snack-bar-manager/hooks/useSnackBar';
+import { useApolloClient } from '@apollo/client';
 import { useLingui } from '@lingui/react/macro';
 import { isDefined } from 'twenty-shared/utils';
 import { useUploadFilesFieldFileMutation } from '~/generated-metadata/graphql';
@@ -8,9 +8,9 @@ const DEFAULT_VALUE_BEFORE_SERVER_RESPONSE =
   'default-value-before-server-response';
 
 export const useUploadFilesFieldFile = () => {
-  const coreClient = useApolloCoreClient();
+  const apolloClient = useApolloClient();
   const [uploadFilesFieldFile] = useUploadFilesFieldFileMutation({
-    client: coreClient,
+    client: apolloClient,
   });
   const { enqueueSuccessSnackBar, enqueueErrorSnackBar } = useSnackBar();
   const { t } = useLingui();

packages/twenty-server/src/database/commands/upgrade-version-command/1-18/1-18-backfill-file-size-and-mime-type.command.ts

@@ -0,0 +1,184 @@
+import { InjectDataSource, InjectRepository } from '@nestjs/typeorm';
+
+import { Command } from 'nest-commander';
+import { FileFolder } from 'twenty-shared/types';
+import { DataSource, Equal, LessThan, Repository } from 'typeorm';
+
+import { ActiveOrSuspendedWorkspacesMigrationCommandRunner } from 'src/database/commands/command-runners/active-or-suspended-workspaces-migration.command-runner';
+import { RunOnWorkspaceArgs } from 'src/database/commands/command-runners/workspaces-migration.command-runner';
+import { FileStorageService } from 'src/engine/core-modules/file-storage/file-storage.service';
+import { FileEntity } from 'src/engine/core-modules/file/entities/file.entity';
+import { extractFileInfo } from 'src/engine/core-modules/file/utils/extract-file-info.utils';
+import { removeFileFolderFromFileEntityPath } from 'src/engine/core-modules/file/utils/remove-file-folder-from-file-entity-path.utils';
+import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity';
+import { DataSourceService } from 'src/engine/metadata-modules/data-source/data-source.service';
+import { GlobalWorkspaceOrmManager } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager';
+import { WorkspaceCacheService } from 'src/engine/workspace-cache/services/workspace-cache.service';
+import { streamToBuffer } from 'src/utils/stream-to-buffer';
+
+@Command({
+  name: 'upgrade:1-18:backfill-file-size-and-mime-type',
+  description:
+    'Backfill file size and mime type for files with missing or default values',
+})
+export class BackfillFileSizeAndMimeTypeCommand extends ActiveOrSuspendedWorkspacesMigrationCommandRunner {
+  constructor(
+    @InjectRepository(WorkspaceEntity)
+    protected readonly workspaceRepository: Repository<WorkspaceEntity>,
+    protected readonly twentyORMGlobalManager: GlobalWorkspaceOrmManager,
+    protected readonly dataSourceService: DataSourceService,
+    private readonly fileStorageService: FileStorageService,
+    private readonly workspaceCacheService: WorkspaceCacheService,
+    @InjectDataSource()
+    private readonly coreDataSource: DataSource,
+  ) {
+    super(workspaceRepository, twentyORMGlobalManager, dataSourceService);
+  }
+
+  override async runOnWorkspace({
+    workspaceId,
+    options,
+  }: RunOnWorkspaceArgs): Promise<void> {
+    const isDryRun = options.dryRun ?? false;
+
+    this.logger.log(
+      `${isDryRun ? '[DRY RUN] ' : ''}Starting file size and mime type backfill for workspace ${workspaceId}`,
+    );
+
+    const { flatApplicationMaps } =
+      await this.workspaceCacheService.getOrRecompute(workspaceId, [
+        'flatApplicationMaps',
+      ]);
+
+    const fileRepository = this.coreDataSource.getRepository(FileEntity);
+
+    const filesToBackfill = await fileRepository.find({
+      where: [
+        {
+          workspaceId,
+          size: LessThan(0),
+        },
+        {
+          workspaceId,
+          mimeType: Equal('application/octet-stream'),
+        },
+      ],
+      select: [
+        'id',
+        'path',
+        'applicationId',
+        'workspaceId',
+        'size',
+        'mimeType',
+      ],
+    });
+
+    if (filesToBackfill.length === 0) {
+      this.logger.log(`No files to backfill for workspace ${workspaceId}`);
+
+      return;
+    }
+
+    this.logger.log(
+      `Found ${filesToBackfill.length} file(s) to backfill in workspace ${workspaceId}`,
+    );
+
+    for (const fileEntity of filesToBackfill) {
+      try {
+        this.logger.log(
+          `Processing file ${fileEntity.id} (path: ${fileEntity.path})`,
+        );
+
+        const application = flatApplicationMaps.byId[fileEntity.applicationId];
+
+        if (!application) {
+          this.logger.error(
+            `Application not found for file ${fileEntity.id} (applicationId: ${fileEntity.applicationId})`,
+          );
+
+          continue;
+        }
+
+        const [fileFolder] = fileEntity.path.split('/') as [FileFolder];
+
+        if (!Object.values(FileFolder).includes(fileFolder)) {
+          this.logger.error(
+            `Invalid file folder '${fileFolder}' for file ${fileEntity.id} (path: ${fileEntity.path})`,
+          );
+
+          continue;
+        }
+
+        const fileExists = await this.fileStorageService.checkFileExists({
+          fileFolder,
+          applicationUniversalIdentifier: application.universalIdentifier,
+          workspaceId,
+          resourcePath: removeFileFolderFromFileEntityPath(fileEntity.path),
+        });
+
+        if (!fileExists) {
+          this.logger.error(
+            `File ${fileEntity.id} (path: ${fileEntity.path}) not found in storage`,
+          );
+
+          continue;
+        }
+
+        const fileStream = await this.fileStorageService.readFile({
+          fileFolder,
+          applicationUniversalIdentifier: application.universalIdentifier,
+          workspaceId,
+          resourcePath: removeFileFolderFromFileEntityPath(fileEntity.path),
+        });
+
+        const fileBuffer = await streamToBuffer(fileStream);
+
+        const updateData: {
+          size?: number;
+          mimeType?: string;
+        } = {};
+
+        if (fileEntity.size < 0) {
+          updateData.size = fileBuffer.length;
+        }
+
+        if (fileEntity.mimeType === 'application/octet-stream') {
+          try {
+            const { mimeType } = await extractFileInfo({
+              file: fileBuffer,
+              filename: fileEntity.path,
+            });
+
+            updateData.mimeType = mimeType;
+          } catch (error) {
+            this.logger.error(
+              `Failed to extract file mime type for file ${fileEntity.id} in workspace ${workspaceId}: ${error.message}`,
+            );
+
+            continue;
+          }
+        }
+
+        if (!isDryRun) {
+          await fileRepository.update({ id: fileEntity.id }, updateData);
+        }
+
+        const updateDetails = Object.entries(updateData)
+          .map(([key, value]) => `${key}=${value}`)
+          .join(', ');
+
+        this.logger.log(
+          `${isDryRun ? '[DRY RUN] ' : ''}Backfilled file ${fileEntity.id}: ${updateDetails}`,
+        );
+      } catch (error) {
+        this.logger.error(
+          `Failed to backfill file ${fileEntity.id} in workspace ${workspaceId}: ${error.message}`,
+        );
+      }
+    }
+
+    this.logger.log(
+      `${isDryRun ? '[DRY RUN] ' : ''}Completed file size and mime type backfill for workspace ${workspaceId}`,
+    );
+  }
+}

packages/twenty-server/src/database/commands/upgrade-version-command/1-18/1-18-upgrade-version-command.module.ts

@@ -1,6 +1,7 @@
 import { Module } from '@nestjs/common';
 import { TypeOrmModule } from '@nestjs/typeorm';
 
+import { BackfillFileSizeAndMimeTypeCommand } from 'src/database/commands/upgrade-version-command/1-18/1-18-backfill-file-size-and-mime-type.command';
 import { MigrateAttachmentFilesCommand } from 'src/database/commands/upgrade-version-command/1-18/1-18-migrate-attachment-files.command';
 import { MigratePersonAvatarFilesCommand } from 'src/database/commands/upgrade-version-command/1-18/1-18-migrate-person-avatar-files.command';
 import { ApplicationModule } from 'src/engine/core-modules/application/application.module';
@@ -31,7 +32,15 @@ import { PersonWorkspaceEntity } from 'src/modules/person/standard-objects/perso
     FieldMetadataModule,
     ApplicationModule,
   ],
-  providers: [MigratePersonAvatarFilesCommand, MigrateAttachmentFilesCommand],
-  exports: [MigratePersonAvatarFilesCommand, MigrateAttachmentFilesCommand],
+  providers: [
+    MigratePersonAvatarFilesCommand,
+    MigrateAttachmentFilesCommand,
+    BackfillFileSizeAndMimeTypeCommand,
+  ],
+  exports: [
+    MigratePersonAvatarFilesCommand,
+    MigrateAttachmentFilesCommand,
+    BackfillFileSizeAndMimeTypeCommand,
+  ],
 })
 export class V1_18_UpgradeVersionCommandModule {}

packages/twenty-server/src/database/commands/upgrade-version-command/upgrade.command.ts

@@ -19,6 +19,7 @@ import { MigrateFavoritesToNavigationMenuItemsCommand } from 'src/database/comma
 import { MigrateNoteTargetToMorphRelationsCommand } from 'src/database/commands/upgrade-version-command/1-17/1-17-migrate-note-target-to-morph-relations.command';
 import { MigrateTaskTargetToMorphRelationsCommand } from 'src/database/commands/upgrade-version-command/1-17/1-17-migrate-task-target-to-morph-relations.command';
 import { MigrateWorkflowCodeStepsCommand } from 'src/database/commands/upgrade-version-command/1-17/1-17-migrate-workflow-code-steps.command';
+import { BackfillFileSizeAndMimeTypeCommand } from 'src/database/commands/upgrade-version-command/1-18/1-18-backfill-file-size-and-mime-type.command';
 import { MigratePersonAvatarFilesCommand } from 'src/database/commands/upgrade-version-command/1-18/1-18-migrate-person-avatar-files.command';
 import { TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service';
 import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity';
@@ -53,6 +54,7 @@ export class UpgradeCommand extends UpgradeCommandRunner {
 
     // 1.18 Commands
     protected readonly migratePersonAvatarFilesCommand: MigratePersonAvatarFilesCommand,
+    protected readonly backfillFileSizeAndMimeTypeCommand: BackfillFileSizeAndMimeTypeCommand,
   ) {
     super(
       workspaceRepository,
@@ -80,6 +82,7 @@ export class UpgradeCommand extends UpgradeCommandRunner {
 
     const commands_1180: VersionCommands = [
       this.migratePersonAvatarFilesCommand,
+      this.backfillFileSizeAndMimeTypeCommand,
     ];
 
     this.allCommands = {