Files - Migrate attachments in activities (#17808)
As attachment files have migrated from fullPath to file files field, need to migrate richText logic to fit to new attachment file handling + data migration
This commit is contained in:
+472
@@ -0,0 +1,472 @@
|
||||
import { InjectDataSource, InjectRepository } from '@nestjs/typeorm';
|
||||
|
||||
import { isNonEmptyString } from '@sniptt/guards';
|
||||
import { Command } from 'nest-commander';
|
||||
import { STANDARD_OBJECTS } from 'twenty-shared/metadata';
|
||||
import { FileFolder } from 'twenty-shared/types';
|
||||
import {
|
||||
extractFolderPathFilenameAndTypeOrThrow,
|
||||
isDefined,
|
||||
} from 'twenty-shared/utils';
|
||||
import { DataSource, Repository } from 'typeorm';
|
||||
import { v4 } from 'uuid';
|
||||
|
||||
import { ActiveOrSuspendedWorkspacesMigrationCommandRunner } from 'src/database/commands/command-runners/active-or-suspended-workspaces-migration.command-runner';
|
||||
import { RunOnWorkspaceArgs } from 'src/database/commands/command-runners/workspaces-migration.command-runner';
|
||||
import { ApplicationService } from 'src/engine/core-modules/application/services/application.service';
|
||||
import { FeatureFlagKey } from 'src/engine/core-modules/feature-flag/enums/feature-flag-key.enum';
|
||||
import { FeatureFlagService } from 'src/engine/core-modules/feature-flag/services/feature-flag.service';
|
||||
import { FileStorageService } from 'src/engine/core-modules/file-storage/file-storage.service';
|
||||
import { FileEntity } from 'src/engine/core-modules/file/entities/file.entity';
|
||||
import { FilesFieldService } from 'src/engine/core-modules/file/files-field/files-field.service';
|
||||
import { extractFileIdFromUrl } from 'src/engine/core-modules/file/files-field/utils/extract-file-id-from-url.util';
|
||||
import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity';
|
||||
import { DataSourceService } from 'src/engine/metadata-modules/data-source/data-source.service';
|
||||
import { findFlatEntityByUniversalIdentifier } from 'src/engine/metadata-modules/flat-entity/utils/find-flat-entity-by-universal-identifier.util';
|
||||
import { type FlatObjectMetadata } from 'src/engine/metadata-modules/flat-object-metadata/types/flat-object-metadata.type';
|
||||
import { GlobalWorkspaceOrmManager } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager';
|
||||
import { WorkspaceRepository } from 'src/engine/twenty-orm/repository/workspace.repository';
|
||||
import { WorkspaceCacheService } from 'src/engine/workspace-cache/services/workspace-cache.service';
|
||||
import { AttachmentWorkspaceEntity } from 'src/modules/attachment/standard-objects/attachment.workspace-entity';
|
||||
import { NoteWorkspaceEntity } from 'src/modules/note/standard-objects/note.workspace-entity';
|
||||
import { TaskWorkspaceEntity } from 'src/modules/task/standard-objects/task.workspace-entity';
|
||||
|
||||
type RichTextBlock = Record<string, unknown>;
|
||||
|
||||
@Command({
|
||||
name: 'upgrade:1-18:migrate-activity-rich-text-attachment-file-ids',
|
||||
description:
|
||||
'Migrate activity rich text blocks to include attachmentFileId from attachment.file field',
|
||||
})
|
||||
export class MigrateActivityRichTextAttachmentFileIdsCommand extends ActiveOrSuspendedWorkspacesMigrationCommandRunner {
|
||||
constructor(
|
||||
@InjectRepository(WorkspaceEntity)
|
||||
protected readonly workspaceRepository: Repository<WorkspaceEntity>,
|
||||
protected readonly twentyORMGlobalManager: GlobalWorkspaceOrmManager,
|
||||
protected readonly dataSourceService: DataSourceService,
|
||||
private readonly featureFlagService: FeatureFlagService,
|
||||
private readonly fileStorageService: FileStorageService,
|
||||
private readonly workspaceCacheService: WorkspaceCacheService,
|
||||
private readonly applicationService: ApplicationService,
|
||||
private readonly filesFieldService: FilesFieldService,
|
||||
@InjectDataSource()
|
||||
private readonly coreDataSource: DataSource,
|
||||
) {
|
||||
super(workspaceRepository, twentyORMGlobalManager, dataSourceService);
|
||||
}
|
||||
|
||||
override async runOnWorkspace({
|
||||
workspaceId,
|
||||
options,
|
||||
}: RunOnWorkspaceArgs): Promise<void> {
|
||||
const isDryRun = options.dryRun ?? false;
|
||||
|
||||
const isFilesFieldMigrated = await this.featureFlagService.isFeatureEnabled(
|
||||
FeatureFlagKey.IS_FILES_FIELD_MIGRATED,
|
||||
workspaceId,
|
||||
);
|
||||
|
||||
if (isFilesFieldMigrated) {
|
||||
this.logger.log(
|
||||
`Files field already migrated for workspace ${workspaceId}, skipping`,
|
||||
);
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
this.logger.log(
|
||||
`${isDryRun ? '[DRY RUN] ' : ''}Starting activity rich text attachment file IDs migration for workspace ${workspaceId}`,
|
||||
);
|
||||
|
||||
const { flatObjectMetadataMaps, flatFieldMetadataMaps } =
|
||||
await this.workspaceCacheService.getOrRecompute(workspaceId, [
|
||||
'flatObjectMetadataMaps',
|
||||
'flatFieldMetadataMaps',
|
||||
]);
|
||||
|
||||
const attachmentObjectMetadata =
|
||||
findFlatEntityByUniversalIdentifier<FlatObjectMetadata>({
|
||||
flatEntityMaps: flatObjectMetadataMaps,
|
||||
universalIdentifier: STANDARD_OBJECTS.attachment.universalIdentifier,
|
||||
});
|
||||
|
||||
if (!isDefined(attachmentObjectMetadata)) {
|
||||
this.logger.warn(
|
||||
`Attachment object metadata not found for workspace ${workspaceId}, skipping`,
|
||||
);
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
const { twentyStandardFlatApplication } =
|
||||
await this.applicationService.findWorkspaceTwentyStandardAndCustomApplicationOrThrow(
|
||||
{
|
||||
workspaceId,
|
||||
},
|
||||
);
|
||||
|
||||
const attachmentFileFieldMetadata = findFlatEntityByUniversalIdentifier({
|
||||
flatEntityMaps: flatFieldMetadataMaps,
|
||||
universalIdentifier:
|
||||
STANDARD_OBJECTS.attachment.fields.file.universalIdentifier,
|
||||
});
|
||||
|
||||
if (!isDefined(attachmentFileFieldMetadata)) {
|
||||
this.logger.error(
|
||||
`Failed to find attachment file field metadata for workspace ${workspaceId}`,
|
||||
);
|
||||
|
||||
throw new Error(
|
||||
`Failed to find attachment file field metadata for workspace ${workspaceId}`,
|
||||
);
|
||||
}
|
||||
|
||||
const attachmentRepository =
|
||||
await this.twentyORMGlobalManager.getRepository<AttachmentWorkspaceEntity>(
|
||||
workspaceId,
|
||||
'attachment',
|
||||
{ shouldBypassPermissionChecks: true },
|
||||
);
|
||||
|
||||
const noteRepository =
|
||||
await this.twentyORMGlobalManager.getRepository<NoteWorkspaceEntity>(
|
||||
workspaceId,
|
||||
'note',
|
||||
{ shouldBypassPermissionChecks: true },
|
||||
);
|
||||
|
||||
const taskRepository =
|
||||
await this.twentyORMGlobalManager.getRepository<TaskWorkspaceEntity>(
|
||||
workspaceId,
|
||||
'task',
|
||||
{ shouldBypassPermissionChecks: true },
|
||||
);
|
||||
|
||||
const fileRepository = this.coreDataSource.getRepository(FileEntity);
|
||||
|
||||
await this.migrateActivityTable({
|
||||
activityRepository: noteRepository,
|
||||
attachmentRepository,
|
||||
fileRepository,
|
||||
activityType: 'note',
|
||||
targetColumnName: 'targetNoteId',
|
||||
workspaceId,
|
||||
twentyStandardApplication: twentyStandardFlatApplication,
|
||||
fileFieldMetadata: attachmentFileFieldMetadata,
|
||||
isDryRun,
|
||||
});
|
||||
|
||||
await this.migrateActivityTable({
|
||||
activityRepository: taskRepository,
|
||||
attachmentRepository,
|
||||
fileRepository,
|
||||
activityType: 'task',
|
||||
targetColumnName: 'targetTaskId',
|
||||
workspaceId,
|
||||
twentyStandardApplication: twentyStandardFlatApplication,
|
||||
fileFieldMetadata: attachmentFileFieldMetadata,
|
||||
isDryRun,
|
||||
});
|
||||
|
||||
if (!isDryRun) {
|
||||
await this.featureFlagService.enableFeatureFlags(
|
||||
[FeatureFlagKey.IS_FILES_FIELD_MIGRATED],
|
||||
workspaceId,
|
||||
);
|
||||
}
|
||||
|
||||
this.logger.log(
|
||||
`${isDryRun ? '[DRY RUN] ' : ''}Completed activity rich text attachment file IDs migration for workspace ${workspaceId}`,
|
||||
);
|
||||
}
|
||||
|
||||
private async migrateActivityTable({
|
||||
activityRepository,
|
||||
attachmentRepository,
|
||||
fileRepository,
|
||||
activityType,
|
||||
targetColumnName,
|
||||
workspaceId,
|
||||
twentyStandardApplication,
|
||||
fileFieldMetadata,
|
||||
isDryRun,
|
||||
}: {
|
||||
activityRepository:
|
||||
| WorkspaceRepository<NoteWorkspaceEntity>
|
||||
| WorkspaceRepository<TaskWorkspaceEntity>;
|
||||
attachmentRepository: Awaited<
|
||||
ReturnType<GlobalWorkspaceOrmManager['getRepository']>
|
||||
>;
|
||||
fileRepository: Repository<FileEntity>;
|
||||
activityType: 'note' | 'task';
|
||||
targetColumnName: 'targetNoteId' | 'targetTaskId';
|
||||
workspaceId: string;
|
||||
twentyStandardApplication: { id: string; universalIdentifier: string };
|
||||
fileFieldMetadata: { universalIdentifier: string };
|
||||
isDryRun: boolean;
|
||||
}): Promise<void> {
|
||||
this.logger.log(
|
||||
`Migrating ${activityType} rich text for workspace ${workspaceId}`,
|
||||
);
|
||||
|
||||
const activities = await activityRepository.find();
|
||||
|
||||
this.logger.log(
|
||||
`Found ${activities.length} ${activityType}(s) to process in workspace ${workspaceId}`,
|
||||
);
|
||||
|
||||
for (const activity of activities) {
|
||||
const bodyV2 = activity.bodyV2;
|
||||
|
||||
if (!isDefined(bodyV2?.blocknote)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
let blocknote: RichTextBlock[];
|
||||
|
||||
try {
|
||||
blocknote = JSON.parse(bodyV2.blocknote);
|
||||
} catch (error) {
|
||||
this.logger.warn(
|
||||
`Failed to parse bodyV2.blocknote for ${activityType} ${activity.id}: ${error.message}`,
|
||||
);
|
||||
continue;
|
||||
}
|
||||
|
||||
const needsMigration = blocknote.some((block: RichTextBlock) => {
|
||||
const props = (block.props as Record<string, unknown>) || {};
|
||||
const url = props.url as string | undefined;
|
||||
|
||||
return isDefined(url) && !isDefined(extractFileIdFromUrl(url));
|
||||
});
|
||||
|
||||
if (!needsMigration) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const urlToFileIdMap = await this.buildUrlToFileIdMap(
|
||||
attachmentRepository,
|
||||
targetColumnName,
|
||||
activity.id,
|
||||
);
|
||||
|
||||
let hasChanges = false;
|
||||
const enrichedBlocknote = [];
|
||||
|
||||
for (const block of blocknote) {
|
||||
const props = (block.props as Record<string, unknown>) || {};
|
||||
const url = props.url as string | undefined;
|
||||
|
||||
if (
|
||||
!isDefined(url) ||
|
||||
isDefined(extractFileIdFromUrl(url)) ||
|
||||
!url.includes('/files/attachment/')
|
||||
) {
|
||||
enrichedBlocknote.push(block);
|
||||
continue;
|
||||
}
|
||||
|
||||
let fileId = this.findMatchingFileId(url, urlToFileIdMap);
|
||||
|
||||
if (!isDefined(fileId) && !isDryRun) {
|
||||
const createdFileId = await this.createAttachmentFromUrl(
|
||||
url,
|
||||
block,
|
||||
attachmentRepository,
|
||||
fileRepository,
|
||||
targetColumnName,
|
||||
activity.id,
|
||||
workspaceId,
|
||||
twentyStandardApplication,
|
||||
fileFieldMetadata,
|
||||
);
|
||||
|
||||
if (isDefined(createdFileId)) {
|
||||
fileId = createdFileId;
|
||||
}
|
||||
}
|
||||
|
||||
if (isDefined(fileId)) {
|
||||
this.logger.log(
|
||||
`Enriching ${block.type} block in ${activityType} ${activity.id} with fileId: ${fileId}`,
|
||||
);
|
||||
hasChanges = true;
|
||||
|
||||
const signedUrl = this.filesFieldService.signFileUrl({
|
||||
fileId,
|
||||
workspaceId,
|
||||
});
|
||||
|
||||
enrichedBlocknote.push({
|
||||
...block,
|
||||
props: {
|
||||
...props,
|
||||
url: signedUrl,
|
||||
},
|
||||
});
|
||||
} else {
|
||||
enrichedBlocknote.push(block);
|
||||
}
|
||||
}
|
||||
|
||||
if (hasChanges && !isDryRun) {
|
||||
const updatedBodyV2 = {
|
||||
...bodyV2,
|
||||
blocknote: JSON.stringify(enrichedBlocknote),
|
||||
};
|
||||
|
||||
await activityRepository.update(
|
||||
{ id: activity.id },
|
||||
{ bodyV2: updatedBodyV2 },
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async buildUrlToFileIdMap(
|
||||
attachmentRepository: Awaited<
|
||||
ReturnType<GlobalWorkspaceOrmManager['getRepository']>
|
||||
>,
|
||||
targetColumnName: 'targetNoteId' | 'targetTaskId',
|
||||
activityId: string,
|
||||
): Promise<Map<string, string>> {
|
||||
const urlToFileIdMap = new Map<string, string>();
|
||||
|
||||
const attachments = await attachmentRepository.find({
|
||||
where: {
|
||||
[targetColumnName]: activityId,
|
||||
},
|
||||
select: ['fullPath', 'file'],
|
||||
});
|
||||
|
||||
for (const attachment of attachments) {
|
||||
if (
|
||||
isDefined(attachment.file) &&
|
||||
Array.isArray(attachment.file) &&
|
||||
attachment.file.length > 0
|
||||
) {
|
||||
const fileData = attachment.file[0];
|
||||
|
||||
if (isDefined(attachment.fullPath) && isDefined(fileData.fileId)) {
|
||||
urlToFileIdMap.set(attachment.fullPath, fileData.fileId);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return urlToFileIdMap;
|
||||
}
|
||||
|
||||
private findMatchingFileId(
|
||||
url: string,
|
||||
urlToFileIdMap: Map<string, string>,
|
||||
): string | undefined {
|
||||
const normalizedUrl = url.includes(FileFolder.FilesField)
|
||||
? this.getAttachmentNewdRelativePath(url)
|
||||
: this.getAttachmentLegacyRelativePath(url);
|
||||
|
||||
return urlToFileIdMap.get(normalizedUrl);
|
||||
}
|
||||
|
||||
private async createAttachmentFromUrl(
|
||||
url: string,
|
||||
block: RichTextBlock,
|
||||
attachmentRepository: Awaited<
|
||||
ReturnType<GlobalWorkspaceOrmManager['getRepository']>
|
||||
>,
|
||||
fileRepository: Repository<FileEntity>,
|
||||
targetColumnName: 'targetNoteId' | 'targetTaskId',
|
||||
activityId: string,
|
||||
workspaceId: string,
|
||||
twentyStandardApplication: { id: string; universalIdentifier: string },
|
||||
fileFieldMetadata: { universalIdentifier: string },
|
||||
): Promise<string | undefined> {
|
||||
try {
|
||||
const props = block.props as Record<string, unknown>;
|
||||
const blockName = (props.name as string) || '';
|
||||
|
||||
const filePath = this.getAttachmentLegacyRelativePath(url);
|
||||
|
||||
const {
|
||||
type: fileExtension,
|
||||
filename,
|
||||
folderPath,
|
||||
} = extractFolderPathFilenameAndTypeOrThrow(filePath);
|
||||
|
||||
const fileId = v4();
|
||||
const newFilename = `${fileId}${isNonEmptyString(fileExtension) ? `.${fileExtension}` : ''}`;
|
||||
const newResourcePath = `${FileFolder.FilesField}/${fileFieldMetadata.universalIdentifier}/${newFilename}`;
|
||||
|
||||
await this.fileStorageService.copyLegacy({
|
||||
from: {
|
||||
folderPath: `workspace-${workspaceId}/${folderPath}`,
|
||||
filename: filename,
|
||||
},
|
||||
to: {
|
||||
folderPath: `${workspaceId}/${twentyStandardApplication.universalIdentifier}`,
|
||||
filename: newResourcePath,
|
||||
},
|
||||
});
|
||||
|
||||
const fileEntity = fileRepository.create({
|
||||
id: fileId,
|
||||
path: newResourcePath,
|
||||
workspaceId,
|
||||
applicationId: twentyStandardApplication.id,
|
||||
size: -1,
|
||||
settings: {
|
||||
isTemporaryFile: true,
|
||||
toDelete: false,
|
||||
},
|
||||
});
|
||||
|
||||
await fileRepository.save(fileEntity);
|
||||
|
||||
const attachmentName = isNonEmptyString(blockName) ? blockName : filename;
|
||||
|
||||
await attachmentRepository.save({
|
||||
[targetColumnName]: activityId,
|
||||
name: attachmentName,
|
||||
file: [
|
||||
{
|
||||
fileId: fileEntity.id,
|
||||
label: attachmentName,
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
this.logger.log(
|
||||
`Created attachment for ${block.type} block with URL: ${url}`,
|
||||
);
|
||||
|
||||
return fileId;
|
||||
} catch (error) {
|
||||
this.logger.error(
|
||||
`Failed to create attachment from URL ${url}: ${error.message}`,
|
||||
);
|
||||
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
getAttachmentLegacyRelativePath = (url: string): string => {
|
||||
if (!url.includes('/files/attachment/')) {
|
||||
throw new Error(`Invalid attachment legacy relative path: ${url}`);
|
||||
}
|
||||
|
||||
//https://example.com/files/attachment/token/filename.jpg -> attachment/filename.jpg
|
||||
const attachmentPath = url.split('/files/attachment/')[1].split('/')[1];
|
||||
|
||||
return `${FileFolder.Attachment}/${attachmentPath}`;
|
||||
};
|
||||
|
||||
getAttachmentNewdRelativePath = (url: string): string => {
|
||||
if (!url.includes('/files-field/')) {
|
||||
throw new Error(`Invalid new attachment path: ${url}`);
|
||||
}
|
||||
|
||||
//https://example.com/files-field/123/456/789/filename.jpg -> files-field/123/456/789/filename.jpg
|
||||
const attachmentPath = url.split('/files-field/')[1];
|
||||
|
||||
return `${FileFolder.FilesField}/${attachmentPath}`;
|
||||
};
|
||||
}
|
||||
+8
-10
@@ -176,10 +176,15 @@ export class MigrateAttachmentFilesCommand extends ActiveOrSuspendedWorkspacesMi
|
||||
);
|
||||
throw error;
|
||||
}
|
||||
|
||||
this.logger.log(
|
||||
`Created file field metadata for attachments in workspace ${workspaceId}`,
|
||||
);
|
||||
} else {
|
||||
this.logger.log(
|
||||
`[DRY RUN] Would create file field metadata for attachments in workspace ${workspaceId}`,
|
||||
);
|
||||
}
|
||||
this.logger.log(
|
||||
`Created file field metadata for attachments in workspace ${workspaceId}`,
|
||||
);
|
||||
}
|
||||
|
||||
if (!isDefined(attachmentFileflatFieldMetadata)) {
|
||||
@@ -284,13 +289,6 @@ export class MigrateAttachmentFilesCommand extends ActiveOrSuspendedWorkspacesMi
|
||||
}
|
||||
}
|
||||
|
||||
if (!isDryRun) {
|
||||
await this.featureFlagService.enableFeatureFlags(
|
||||
[FeatureFlagKey.IS_FILES_FIELD_MIGRATED],
|
||||
workspaceId,
|
||||
);
|
||||
}
|
||||
|
||||
this.logger.log(
|
||||
`${isDryRun ? '[DRY RUN] ' : ''}Completed attachment files migration for workspace ${workspaceId}`,
|
||||
);
|
||||
|
||||
+8
-3
@@ -186,10 +186,15 @@ export class MigratePersonAvatarFilesCommand extends ActiveOrSuspendedWorkspaces
|
||||
);
|
||||
throw error;
|
||||
}
|
||||
|
||||
this.logger.log(
|
||||
`Created avatarFile field metadata for workspace ${workspaceId}`,
|
||||
);
|
||||
} else {
|
||||
this.logger.log(
|
||||
`[DRY RUN] Would create avatarFile field metadata for workspace ${workspaceId}`,
|
||||
);
|
||||
}
|
||||
this.logger.log(
|
||||
`Created avatarFile field metadata for workspace ${workspaceId}`,
|
||||
);
|
||||
}
|
||||
|
||||
if (!isDefined(avatarFileFieldMetadata)) {
|
||||
|
||||
+5
@@ -2,6 +2,7 @@ import { Module } from '@nestjs/common';
|
||||
import { TypeOrmModule } from '@nestjs/typeorm';
|
||||
|
||||
import { BackfillFileSizeAndMimeTypeCommand } from 'src/database/commands/upgrade-version-command/1-18/1-18-backfill-file-size-and-mime-type.command';
|
||||
import { MigrateActivityRichTextAttachmentFileIdsCommand } from 'src/database/commands/upgrade-version-command/1-18/1-18-migrate-activity-rich-text-attachment-file-ids.command';
|
||||
import { MigrateAttachmentFilesCommand } from 'src/database/commands/upgrade-version-command/1-18/1-18-migrate-attachment-files.command';
|
||||
import { MigratePersonAvatarFilesCommand } from 'src/database/commands/upgrade-version-command/1-18/1-18-migrate-person-avatar-files.command';
|
||||
import { ApplicationModule } from 'src/engine/core-modules/application/application.module';
|
||||
@@ -9,6 +10,7 @@ import { FeatureFlagEntity } from 'src/engine/core-modules/feature-flag/feature-
|
||||
import { FeatureFlagModule } from 'src/engine/core-modules/feature-flag/feature-flag.module';
|
||||
import { FileStorageModule } from 'src/engine/core-modules/file-storage/file-storage.module';
|
||||
import { FileEntity } from 'src/engine/core-modules/file/entities/file.entity';
|
||||
import { FilesFieldModule } from 'src/engine/core-modules/file/files-field/files-field.module';
|
||||
import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity';
|
||||
import { DataSourceModule } from 'src/engine/metadata-modules/data-source/data-source.module';
|
||||
import { FieldMetadataModule } from 'src/engine/metadata-modules/field-metadata/field-metadata.module';
|
||||
@@ -31,16 +33,19 @@ import { PersonWorkspaceEntity } from 'src/modules/person/standard-objects/perso
|
||||
WorkspaceCacheModule,
|
||||
FieldMetadataModule,
|
||||
ApplicationModule,
|
||||
FilesFieldModule,
|
||||
],
|
||||
providers: [
|
||||
MigratePersonAvatarFilesCommand,
|
||||
MigrateAttachmentFilesCommand,
|
||||
BackfillFileSizeAndMimeTypeCommand,
|
||||
MigrateActivityRichTextAttachmentFileIdsCommand,
|
||||
],
|
||||
exports: [
|
||||
MigratePersonAvatarFilesCommand,
|
||||
MigrateAttachmentFilesCommand,
|
||||
BackfillFileSizeAndMimeTypeCommand,
|
||||
MigrateActivityRichTextAttachmentFileIdsCommand,
|
||||
],
|
||||
})
|
||||
export class V1_18_UpgradeVersionCommandModule {}
|
||||
|
||||
+6
@@ -20,6 +20,8 @@ import { MigrateNoteTargetToMorphRelationsCommand } from 'src/database/commands/
|
||||
import { MigrateTaskTargetToMorphRelationsCommand } from 'src/database/commands/upgrade-version-command/1-17/1-17-migrate-task-target-to-morph-relations.command';
|
||||
import { MigrateWorkflowCodeStepsCommand } from 'src/database/commands/upgrade-version-command/1-17/1-17-migrate-workflow-code-steps.command';
|
||||
import { BackfillFileSizeAndMimeTypeCommand } from 'src/database/commands/upgrade-version-command/1-18/1-18-backfill-file-size-and-mime-type.command';
|
||||
import { MigrateActivityRichTextAttachmentFileIdsCommand } from 'src/database/commands/upgrade-version-command/1-18/1-18-migrate-activity-rich-text-attachment-file-ids.command';
|
||||
import { MigrateAttachmentFilesCommand } from 'src/database/commands/upgrade-version-command/1-18/1-18-migrate-attachment-files.command';
|
||||
import { MigratePersonAvatarFilesCommand } from 'src/database/commands/upgrade-version-command/1-18/1-18-migrate-person-avatar-files.command';
|
||||
import { TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service';
|
||||
import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity';
|
||||
@@ -55,6 +57,8 @@ export class UpgradeCommand extends UpgradeCommandRunner {
|
||||
// 1.18 Commands
|
||||
protected readonly migratePersonAvatarFilesCommand: MigratePersonAvatarFilesCommand,
|
||||
protected readonly backfillFileSizeAndMimeTypeCommand: BackfillFileSizeAndMimeTypeCommand,
|
||||
protected readonly migrateAttachmentFilesCommand: MigrateAttachmentFilesCommand,
|
||||
protected readonly migrateActivityRichTextAttachmentFileIdsCommand: MigrateActivityRichTextAttachmentFileIdsCommand,
|
||||
) {
|
||||
super(
|
||||
workspaceRepository,
|
||||
@@ -82,6 +86,8 @@ export class UpgradeCommand extends UpgradeCommandRunner {
|
||||
|
||||
const commands_1180: VersionCommands = [
|
||||
this.migratePersonAvatarFilesCommand,
|
||||
this.migrateAttachmentFilesCommand,
|
||||
this.migrateActivityRichTextAttachmentFileIdsCommand,
|
||||
this.backfillFileSizeAndMimeTypeCommand,
|
||||
];
|
||||
|
||||
|
||||
+8
-2
@@ -15,11 +15,12 @@ import { RichTextV2FieldQueryResultGetterHandler } from 'src/engine/api/common/c
|
||||
import { AttachmentQueryResultGetterHandler } from 'src/engine/api/graphql/workspace-query-runner/factories/query-result-getters/handlers/attachment-query-result-getter.handler';
|
||||
import { PersonQueryResultGetterHandler } from 'src/engine/api/graphql/workspace-query-runner/factories/query-result-getters/handlers/person-query-result-getter.handler';
|
||||
import { WorkspaceMemberQueryResultGetterHandler } from 'src/engine/api/graphql/workspace-query-runner/factories/query-result-getters/handlers/workspace-member-query-result-getter.handler';
|
||||
import { FeatureFlagService } from 'src/engine/core-modules/feature-flag/services/feature-flag.service';
|
||||
import { FilesFieldService } from 'src/engine/core-modules/file/files-field/files-field.service';
|
||||
import { FileService } from 'src/engine/core-modules/file/services/file.service';
|
||||
import { type FlatEntityMaps } from 'src/engine/metadata-modules/flat-entity/types/flat-entity-maps.type';
|
||||
import { findFlatEntityByIdInFlatEntityMaps } from 'src/engine/metadata-modules/flat-entity/utils/find-flat-entity-by-id-in-flat-entity-maps.util';
|
||||
import { findFlatEntityByIdInFlatEntityMapsOrThrow } from 'src/engine/metadata-modules/flat-entity/utils/find-flat-entity-by-id-in-flat-entity-maps-or-throw.util';
|
||||
import { findFlatEntityByIdInFlatEntityMaps } from 'src/engine/metadata-modules/flat-entity/utils/find-flat-entity-by-id-in-flat-entity-maps.util';
|
||||
import { type FlatFieldMetadata } from 'src/engine/metadata-modules/flat-field-metadata/types/flat-field-metadata.type';
|
||||
import {
|
||||
buildFieldMapsFromFlatObjectMetadata,
|
||||
@@ -42,6 +43,7 @@ export class CommonResultGettersService {
|
||||
constructor(
|
||||
private readonly fileService: FileService,
|
||||
private readonly filesFieldService: FilesFieldService,
|
||||
private readonly featureFlagService: FeatureFlagService,
|
||||
) {
|
||||
this.initializeObjectHandlers();
|
||||
this.initializeFieldHandlers();
|
||||
@@ -69,7 +71,11 @@ export class CommonResultGettersService {
|
||||
],
|
||||
[
|
||||
FieldMetadataType.RICH_TEXT_V2,
|
||||
new RichTextV2FieldQueryResultGetterHandler(this.fileService),
|
||||
new RichTextV2FieldQueryResultGetterHandler(
|
||||
this.fileService,
|
||||
this.filesFieldService,
|
||||
this.featureFlagService,
|
||||
),
|
||||
],
|
||||
]);
|
||||
}
|
||||
|
||||
+28
-4
@@ -1,6 +1,8 @@
|
||||
import { FieldMetadataType, type ObjectRecord } from 'twenty-shared/types';
|
||||
|
||||
import { RichTextV2FieldQueryResultGetterHandler } from 'src/engine/api/common/common-result-getters/handlers/field-handlers/rich-text-v2-field-query-result-getter.handler';
|
||||
import { type FeatureFlagService } from 'src/engine/core-modules/feature-flag/services/feature-flag.service';
|
||||
import { type FilesFieldService } from 'src/engine/core-modules/file/files-field/files-field.service';
|
||||
import { type FileService } from 'src/engine/core-modules/file/services/file.service';
|
||||
import { type FlatFieldMetadata } from 'src/engine/metadata-modules/flat-field-metadata/types/flat-field-metadata.type';
|
||||
|
||||
@@ -22,12 +24,24 @@ const mockFileService = {
|
||||
signFileUrl: jest.fn().mockReturnValue('signed-path'),
|
||||
} as unknown as FileService;
|
||||
|
||||
const mockFilesFieldService = {
|
||||
signFileUrl: jest.fn().mockReturnValue('signed-path'),
|
||||
} as unknown as FilesFieldService;
|
||||
|
||||
const mockFeatureFlagService = {
|
||||
isFeatureEnabled: jest.fn().mockReturnValue(true),
|
||||
} as unknown as FeatureFlagService;
|
||||
|
||||
describe('RichTextV2FieldQueryResultGetterHandler', () => {
|
||||
let handler: RichTextV2FieldQueryResultGetterHandler;
|
||||
|
||||
beforeEach(() => {
|
||||
process.env.SERVER_URL = 'https://my-domain.twenty.com';
|
||||
handler = new RichTextV2FieldQueryResultGetterHandler(mockFileService);
|
||||
handler = new RichTextV2FieldQueryResultGetterHandler(
|
||||
mockFileService,
|
||||
mockFilesFieldService,
|
||||
mockFeatureFlagService,
|
||||
);
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
@@ -113,7 +127,11 @@ describe('RichTextV2FieldQueryResultGetterHandler', () => {
|
||||
bodyV2: {
|
||||
markdown: null,
|
||||
blocknote: JSON.stringify([
|
||||
{ type: 'paragraph', text: 'Hello, world!' },
|
||||
{
|
||||
type: 'paragraph',
|
||||
props: {},
|
||||
children: [{ text: 'Hello, world!' }],
|
||||
},
|
||||
]),
|
||||
},
|
||||
};
|
||||
@@ -152,7 +170,11 @@ describe('RichTextV2FieldQueryResultGetterHandler', () => {
|
||||
});
|
||||
|
||||
describe('should sign internal image URLs', () => {
|
||||
it('when image block has an internal attachment URL', async () => {
|
||||
it('when image block has an internal attachment URL (legacy path)', async () => {
|
||||
jest
|
||||
.spyOn(mockFeatureFlagService, 'isFeatureEnabled')
|
||||
.mockResolvedValue(false);
|
||||
|
||||
const imageBlock = {
|
||||
type: 'image',
|
||||
props: {
|
||||
@@ -206,7 +228,9 @@ describe('RichTextV2FieldQueryResultGetterHandler', () => {
|
||||
...baseRecord,
|
||||
bodyV2: {
|
||||
markdown: null,
|
||||
blocknote: JSON.stringify([{ type: 'paragraph', text: 'Hello' }]),
|
||||
blocknote: JSON.stringify([
|
||||
{ type: 'paragraph', props: {}, children: [{ text: 'Hello' }] },
|
||||
]),
|
||||
},
|
||||
description: {
|
||||
markdown: null,
|
||||
|
||||
+83
-49
@@ -3,58 +3,16 @@ import { isDefined } from 'twenty-shared/utils';
|
||||
|
||||
import { type QueryResultGetterHandlerInterface } from 'src/engine/api/graphql/workspace-query-runner/factories/query-result-getters/interfaces/query-result-getter-handler.interface';
|
||||
|
||||
import { FeatureFlagKey } from 'src/engine/core-modules/feature-flag/enums/feature-flag-key.enum';
|
||||
import { type FeatureFlagService } from 'src/engine/core-modules/feature-flag/services/feature-flag.service';
|
||||
import { type FilesFieldService } from 'src/engine/core-modules/file/files-field/files-field.service';
|
||||
import { extractFileIdFromUrl } from 'src/engine/core-modules/file/files-field/utils/extract-file-id-from-url.util';
|
||||
import { type FileService } from 'src/engine/core-modules/file/services/file.service';
|
||||
import { type FlatFieldMetadata } from 'src/engine/metadata-modules/flat-field-metadata/types/flat-field-metadata.type';
|
||||
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
type RichTextBlock = Record<string, any>;
|
||||
|
||||
const signBlocknoteImageUrls = (
|
||||
blocknoteBlocks: RichTextBlock[],
|
||||
workspaceId: string,
|
||||
fileService: FileService,
|
||||
): RichTextBlock[] => {
|
||||
return blocknoteBlocks.map((block: RichTextBlock) => {
|
||||
if (block.type !== 'image' || !block.props?.url) {
|
||||
return block;
|
||||
}
|
||||
|
||||
let url: URL;
|
||||
|
||||
try {
|
||||
url = new URL(block.props.url);
|
||||
} catch {
|
||||
return block;
|
||||
}
|
||||
|
||||
const pathname = url.pathname;
|
||||
const isLinkExternal = !pathname.startsWith('/files/attachment/');
|
||||
|
||||
if (isLinkExternal) {
|
||||
return block;
|
||||
}
|
||||
|
||||
const fileName = pathname.match(/files\/attachment\/(?:.+)\/(.+)$/)?.[1];
|
||||
|
||||
if (!isDefined(fileName)) {
|
||||
return block;
|
||||
}
|
||||
|
||||
const signedPath = fileService.signFileUrl({
|
||||
url: `attachment/${fileName}`,
|
||||
workspaceId,
|
||||
});
|
||||
|
||||
return {
|
||||
...block,
|
||||
props: {
|
||||
...block.props,
|
||||
url: `${process.env.SERVER_URL}/files/${signedPath}`,
|
||||
},
|
||||
};
|
||||
});
|
||||
};
|
||||
|
||||
const parseBlocknoteJsonSafely = (
|
||||
blocknoteJson: string,
|
||||
): RichTextBlock[] | null => {
|
||||
@@ -74,7 +32,11 @@ const parseBlocknoteJsonSafely = (
|
||||
export class RichTextV2FieldQueryResultGetterHandler
|
||||
implements QueryResultGetterHandlerInterface
|
||||
{
|
||||
constructor(private readonly fileService: FileService) {}
|
||||
constructor(
|
||||
private readonly fileService: FileService,
|
||||
private readonly filesFieldService: FilesFieldService,
|
||||
private readonly featureFlagService: FeatureFlagService,
|
||||
) {}
|
||||
|
||||
async handle(
|
||||
record: ObjectRecord,
|
||||
@@ -89,6 +51,11 @@ export class RichTextV2FieldQueryResultGetterHandler
|
||||
return record;
|
||||
}
|
||||
|
||||
const isFilesFieldMigrated = await this.featureFlagService.isFeatureEnabled(
|
||||
FeatureFlagKey.IS_FILES_FIELD_MIGRATED,
|
||||
workspaceId,
|
||||
);
|
||||
|
||||
for (const field of richTextV2Fields) {
|
||||
const fieldValue = record[field.name];
|
||||
const blocknoteJson = fieldValue?.blocknote;
|
||||
@@ -103,10 +70,10 @@ export class RichTextV2FieldQueryResultGetterHandler
|
||||
continue;
|
||||
}
|
||||
|
||||
const signedBlocks = signBlocknoteImageUrls(
|
||||
const signedBlocks = this.signBlocknoteImageUrls(
|
||||
blocknoteBlocks,
|
||||
workspaceId,
|
||||
this.fileService,
|
||||
isFilesFieldMigrated,
|
||||
);
|
||||
|
||||
record[field.name] = {
|
||||
@@ -117,4 +84,71 @@ export class RichTextV2FieldQueryResultGetterHandler
|
||||
|
||||
return record;
|
||||
}
|
||||
|
||||
signBlocknoteImageUrls = (
|
||||
blocknoteBlocks: RichTextBlock[],
|
||||
workspaceId: string,
|
||||
isFilesFieldMigrated: boolean,
|
||||
): RichTextBlock[] => {
|
||||
return blocknoteBlocks.map((block: RichTextBlock) => {
|
||||
if (isFilesFieldMigrated && isDefined(block.props?.url)) {
|
||||
const fileIdFromUrl = extractFileIdFromUrl(block.props.url);
|
||||
|
||||
if (!isDefined(fileIdFromUrl)) {
|
||||
return block;
|
||||
}
|
||||
|
||||
const url = this.filesFieldService.signFileUrl({
|
||||
fileId: fileIdFromUrl,
|
||||
workspaceId,
|
||||
});
|
||||
|
||||
return {
|
||||
...block,
|
||||
props: {
|
||||
...block.props,
|
||||
url,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
if (block.type !== 'image' || !block.props?.url) {
|
||||
return block;
|
||||
}
|
||||
|
||||
let url: URL;
|
||||
|
||||
try {
|
||||
url = new URL(block.props.url);
|
||||
} catch {
|
||||
return block;
|
||||
}
|
||||
|
||||
const pathname = url.pathname;
|
||||
const isLinkExternal = !pathname.startsWith('/files/attachment/');
|
||||
|
||||
if (isLinkExternal) {
|
||||
return block;
|
||||
}
|
||||
|
||||
const fileName = pathname.match(/files\/attachment\/(?:.+)\/(.+)$/)?.[1];
|
||||
|
||||
if (!isDefined(fileName)) {
|
||||
return block;
|
||||
}
|
||||
|
||||
const signedPath = this.fileService.signFileUrl({
|
||||
url: `attachment/${fileName}`,
|
||||
workspaceId,
|
||||
});
|
||||
|
||||
return {
|
||||
...block,
|
||||
props: {
|
||||
...block.props,
|
||||
url: `${process.env.SERVER_URL}/files/${signedPath}`,
|
||||
},
|
||||
};
|
||||
});
|
||||
};
|
||||
}
|
||||
|
||||
+21
@@ -0,0 +1,21 @@
|
||||
import { Field, ObjectType } from '@nestjs/graphql';
|
||||
|
||||
import { UUIDScalarType } from 'src/engine/api/graphql/workspace-schema-builder/graphql-types/scalars';
|
||||
|
||||
@ObjectType('FilesFieldFile')
|
||||
export class FilesFieldFileDTO {
|
||||
@Field(() => UUIDScalarType)
|
||||
id: string;
|
||||
|
||||
@Field()
|
||||
path: string;
|
||||
|
||||
@Field()
|
||||
size: number;
|
||||
|
||||
@Field(() => Date, { nullable: false })
|
||||
createdAt: Date;
|
||||
|
||||
@Field()
|
||||
url: string;
|
||||
}
|
||||
+10
-6
@@ -6,7 +6,7 @@ import { Readable } from 'stream';
|
||||
import { msg } from '@lingui/core/macro';
|
||||
import { isNonEmptyString } from '@sniptt/guards';
|
||||
import { FileFolder } from 'twenty-shared/types';
|
||||
import { Repository } from 'typeorm';
|
||||
import { Like, Repository } from 'typeorm';
|
||||
import { v4 } from 'uuid';
|
||||
|
||||
import { ApplicationEntity } from 'src/engine/core-modules/application/application.entity';
|
||||
@@ -16,6 +16,7 @@ import {
|
||||
} from 'src/engine/core-modules/auth/types/auth-context.type';
|
||||
import { FileStorageService } from 'src/engine/core-modules/file-storage/file-storage.service';
|
||||
import { FileEntity } from 'src/engine/core-modules/file/entities/file.entity';
|
||||
import { FilesFieldFileDTO } from 'src/engine/core-modules/file/files-field/dtos/files-field-file.dto';
|
||||
import { extractFileInfo } from 'src/engine/core-modules/file/utils/extract-file-info.utils';
|
||||
import { removeFileFolderFromFileEntityPath } from 'src/engine/core-modules/file/utils/remove-file-folder-from-file-entity-path.utils';
|
||||
import { sanitizeFile } from 'src/engine/core-modules/file/utils/sanitize-file.utils';
|
||||
@@ -52,7 +53,7 @@ export class FilesFieldService {
|
||||
filename: string;
|
||||
workspaceId: string;
|
||||
fieldMetadataId: string;
|
||||
}): Promise<FileEntity> {
|
||||
}): Promise<FilesFieldFileDTO> {
|
||||
const { mimeType, ext } = await extractFileInfo({
|
||||
file,
|
||||
filename,
|
||||
@@ -78,7 +79,7 @@ export class FilesFieldService {
|
||||
},
|
||||
});
|
||||
|
||||
return await this.fileStorageService.writeFile({
|
||||
const savedFile = await this.fileStorageService.writeFile({
|
||||
sourceFile: sanitizedFile,
|
||||
resourcePath: `${fieldMetadata.universalIdentifier}/${name}`,
|
||||
mimeType,
|
||||
@@ -91,6 +92,11 @@ export class FilesFieldService {
|
||||
toDelete: false,
|
||||
},
|
||||
});
|
||||
|
||||
return {
|
||||
...savedFile,
|
||||
url: this.signFileUrl({ fileId, workspaceId }),
|
||||
};
|
||||
}
|
||||
|
||||
async deleteFilesFieldFile({
|
||||
@@ -127,10 +133,8 @@ export class FilesFieldService {
|
||||
const file = await this.fileRepository.findOneOrFail({
|
||||
where: {
|
||||
id: fileId,
|
||||
path: Like(`${FileFolder.FilesField}/%`),
|
||||
workspaceId,
|
||||
application: {
|
||||
workspaceId,
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
|
||||
+3
-3
@@ -6,7 +6,7 @@ import { PermissionFlagType } from 'twenty-shared/constants';
|
||||
|
||||
import type { FileUpload } from 'graphql-upload/processRequest.mjs';
|
||||
|
||||
import { FileDTO } from 'src/engine/core-modules/file/dtos/file.dto';
|
||||
import { FilesFieldFileDTO } from 'src/engine/core-modules/file/files-field/dtos/files-field-file.dto';
|
||||
import { FilesFieldService } from 'src/engine/core-modules/file/files-field/files-field.service';
|
||||
import { PreventNestToAutoLogGraphqlErrorsFilter } from 'src/engine/core-modules/graphql/filters/prevent-nest-to-auto-log-graphql-errors.filter';
|
||||
import { ResolverValidationPipe } from 'src/engine/core-modules/graphql/pipes/resolver-validation.pipe';
|
||||
@@ -24,7 +24,7 @@ import { streamToBuffer } from 'src/utils/stream-to-buffer';
|
||||
export class FilesFieldResolver {
|
||||
constructor(private readonly filesFieldService: FilesFieldService) {}
|
||||
|
||||
@Mutation(() => FileDTO)
|
||||
@Mutation(() => FilesFieldFileDTO)
|
||||
@UseGuards(SettingsPermissionGuard(PermissionFlagType.UPLOAD_FILE))
|
||||
async uploadFilesFieldFile(
|
||||
@AuthWorkspace()
|
||||
@@ -37,7 +37,7 @@ export class FilesFieldResolver {
|
||||
nullable: false,
|
||||
})
|
||||
fieldMetadataId: string,
|
||||
): Promise<FileDTO> {
|
||||
): Promise<FilesFieldFileDTO> {
|
||||
const stream = createReadStream();
|
||||
const buffer = await streamToBuffer(stream);
|
||||
|
||||
|
||||
+22
@@ -0,0 +1,22 @@
|
||||
import { isDefined, isValidUuid } from 'twenty-shared/utils';
|
||||
|
||||
export const extractFileIdFromUrl = (url: string): string | null => {
|
||||
let parsedUrl: URL;
|
||||
|
||||
try {
|
||||
parsedUrl = new URL(url);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
|
||||
const pathname = parsedUrl.pathname;
|
||||
const isLinkExternal = !pathname.startsWith('/files-field/');
|
||||
|
||||
if (isLinkExternal) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const fileId = pathname.match(/files-field\/([^/]+)/)?.[1];
|
||||
|
||||
return isDefined(fileId) && isValidUuid(fileId) ? fileId : null;
|
||||
};
|
||||
Reference in New Issue
Block a user