Refactor and centralize file mimeType integrity check and sanitization (#20889)
# Introduction closes https://github.com/twentyhq/private-issues/issues/484 This PR refactors the writeFile API to never expect to be passed a mimetype, its extract is done programmatically low level so any callers will pass through Same for the file sanitization ## IANA override Disclaimer for consistency we existing behavior we wanted to always have `application/typescript` - should we rather consider fallbacking to octect-steam instead ? - Any pulbic assets that has .ts will now also fallback to `application/typescript` instead of the official IANA ## Integration Added coverage
This commit is contained in:
+2
-3
@@ -20,7 +20,6 @@ import { DevelopmentApplicationDTO } from 'src/engine/core-modules/application/a
|
||||
import { GenerateApplicationTokenInput } from 'src/engine/core-modules/application/application-development/dtos/generate-application-token.input';
|
||||
import { UploadApplicationFileInput } from 'src/engine/core-modules/application/application-development/dtos/upload-application-file.input';
|
||||
import { WorkspaceMigrationDTO } from 'src/engine/core-modules/application/application-development/dtos/workspace-migration.dto';
|
||||
import { validateFilePath } from 'src/engine/core-modules/file-storage/utils/validate-file-path.util';
|
||||
import { ApplicationExceptionFilter } from 'src/engine/core-modules/application/application-exception-filter';
|
||||
import { ApplicationSyncService } from 'src/engine/core-modules/application/application-manifest/application-sync.service';
|
||||
import { resolveManifestAssetUrls } from 'src/engine/core-modules/application/application-marketplace/utils/resolve-manifest-asset-urls.util';
|
||||
@@ -35,6 +34,7 @@ import {
|
||||
import { ApplicationService } from 'src/engine/core-modules/application/application.service';
|
||||
import { ApplicationTokenService } from 'src/engine/core-modules/auth/token/services/application-token.service';
|
||||
import { FileStorageService } from 'src/engine/core-modules/file-storage/file-storage.service';
|
||||
import { validateFilePath } from 'src/engine/core-modules/file-storage/utils/validate-file-path.util';
|
||||
import { FileDTO } from 'src/engine/core-modules/file/dtos/file.dto';
|
||||
import { ResolverValidationPipe } from 'src/engine/core-modules/graphql/pipes/resolver-validation.pipe';
|
||||
import { SdkClientGenerationService } from 'src/engine/core-modules/sdk-client/sdk-client-generation.service';
|
||||
@@ -192,7 +192,7 @@ export class ApplicationDevelopmentResolver {
|
||||
async uploadApplicationFile(
|
||||
@AuthWorkspace() { id: workspaceId }: WorkspaceEntity,
|
||||
@Args({ name: 'file', type: () => GraphQLUpload })
|
||||
{ createReadStream, mimetype }: FileUpload,
|
||||
{ createReadStream }: FileUpload,
|
||||
@Args()
|
||||
{
|
||||
applicationUniversalIdentifier,
|
||||
@@ -250,7 +250,6 @@ export class ApplicationDevelopmentResolver {
|
||||
|
||||
return await this.fileStorageService.writeFile({
|
||||
sourceFile: buffer,
|
||||
mimeType: mimetype,
|
||||
fileFolder,
|
||||
applicationUniversalIdentifier,
|
||||
workspaceId,
|
||||
|
||||
+1
-10
@@ -5,8 +5,6 @@ import { promises as fs } from 'fs';
|
||||
import { resolve } from 'path';
|
||||
|
||||
import semver from 'semver';
|
||||
import { extractFileInfo } from 'src/engine/core-modules/file/utils/extract-file-info.utils';
|
||||
import { sanitizeFile } from 'src/engine/core-modules/file/utils/sanitize-file.utils';
|
||||
import { Manifest } from 'twenty-shared/application';
|
||||
import { FileFolder } from 'twenty-shared/types';
|
||||
import { isDefined } from 'twenty-shared/utils';
|
||||
@@ -493,15 +491,8 @@ export class ApplicationInstallService {
|
||||
);
|
||||
}
|
||||
|
||||
const { mimeType, ext } = await extractFileInfo({
|
||||
file: content,
|
||||
filename: relativePath,
|
||||
});
|
||||
const sanitizedContent = sanitizeFile({ file: content, ext, mimeType });
|
||||
|
||||
await this.fileStorageService.writeFile({
|
||||
sourceFile: sanitizedContent,
|
||||
mimeType,
|
||||
sourceFile: content,
|
||||
fileFolder,
|
||||
applicationUniversalIdentifier,
|
||||
workspaceId,
|
||||
|
||||
-1
@@ -190,7 +190,6 @@ export class ApplicationTarballService {
|
||||
const savedFile = await this.fileStorageService.writeFile({
|
||||
sourceFile: params.tarballBuffer,
|
||||
resourcePath: `${appRegistration.id}/app.tar.gz`,
|
||||
mimeType: 'application/gzip',
|
||||
fileFolder: FileFolder.AppTarball,
|
||||
applicationUniversalIdentifier:
|
||||
workspaceCustomFlatApplication.universalIdentifier,
|
||||
|
||||
@@ -411,7 +411,6 @@ export class ApplicationService {
|
||||
|
||||
const packageJsonFile = await this.fileStorageService.writeFile({
|
||||
sourceFile: defaultPackageFields.packageJsonContent,
|
||||
mimeType: undefined,
|
||||
fileFolder: FileFolder.Dependencies,
|
||||
applicationUniversalIdentifier: application.universalIdentifier,
|
||||
workspaceId: application.workspaceId,
|
||||
@@ -422,7 +421,6 @@ export class ApplicationService {
|
||||
|
||||
const yarnLockFile = await this.fileStorageService.writeFile({
|
||||
sourceFile: defaultPackageFields.yarnLockContent,
|
||||
mimeType: undefined,
|
||||
fileFolder: FileFolder.Dependencies,
|
||||
applicationUniversalIdentifier: application.universalIdentifier,
|
||||
workspaceId: application.workspaceId,
|
||||
|
||||
+36
@@ -0,0 +1,36 @@
|
||||
import { FileStorageExceptionFilter } from 'src/engine/core-modules/file-storage/file-storage-exception-filter';
|
||||
import {
|
||||
FileStorageException,
|
||||
FileStorageExceptionCode,
|
||||
} from 'src/engine/core-modules/file-storage/interfaces/file-storage-exception';
|
||||
import {
|
||||
ForbiddenError,
|
||||
NotFoundError,
|
||||
UserInputError,
|
||||
} from 'src/engine/core-modules/graphql/utils/graphql-errors.util';
|
||||
|
||||
describe('FileStorageExceptionFilter', () => {
|
||||
const filter = new FileStorageExceptionFilter();
|
||||
|
||||
it.each([
|
||||
{
|
||||
code: FileStorageExceptionCode.INVALID_EXTENSION,
|
||||
expectedError: UserInputError,
|
||||
},
|
||||
{
|
||||
code: FileStorageExceptionCode.ACCESS_DENIED,
|
||||
expectedError: ForbiddenError,
|
||||
},
|
||||
{
|
||||
code: FileStorageExceptionCode.FILE_NOT_FOUND,
|
||||
expectedError: NotFoundError,
|
||||
},
|
||||
])(
|
||||
'should map $code to the expected GraphQL error',
|
||||
({ code, expectedError }) => {
|
||||
const exception = new FileStorageException('test message', code);
|
||||
|
||||
expect(() => filter.catch(exception)).toThrow(expectedError);
|
||||
},
|
||||
);
|
||||
});
|
||||
+135
-2
@@ -381,7 +381,6 @@ describe('FileStorageService', () => {
|
||||
...validResourceIdentifier,
|
||||
resourcePath: context.resourcePath,
|
||||
sourceFile: Buffer.from('malicious'),
|
||||
mimeType: 'application/javascript',
|
||||
settings: { isTemporaryFile: false, toDelete: false },
|
||||
}),
|
||||
).rejects.toMatchObject({
|
||||
@@ -396,7 +395,6 @@ describe('FileStorageService', () => {
|
||||
await service.writeFile({
|
||||
...validResourceIdentifier,
|
||||
sourceFile: Buffer.from('valid content'),
|
||||
mimeType: 'application/javascript',
|
||||
settings: { isTemporaryFile: false, toDelete: false },
|
||||
});
|
||||
|
||||
@@ -406,6 +404,141 @@ describe('FileStorageService', () => {
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
describe('magic-byte backstop', () => {
|
||||
const pngBuffer = Buffer.from([
|
||||
0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0x00, 0x00, 0x00,
|
||||
0x0d, 0x49, 0x48, 0x44, 0x52,
|
||||
]);
|
||||
const textBuffer = Buffer.from('Hello, world!', 'utf-8');
|
||||
|
||||
it('should reject buffer whose magic bytes do not match the path extension', async () => {
|
||||
await expect(
|
||||
service.writeFile({
|
||||
workspaceId: 'workspace-123',
|
||||
applicationUniversalIdentifier: 'app-456',
|
||||
fileFolder: FileFolder.PublicAsset,
|
||||
resourcePath: 'assets/fake-image.png',
|
||||
sourceFile: textBuffer,
|
||||
settings: { isTemporaryFile: false, toDelete: false },
|
||||
}),
|
||||
).rejects.toMatchObject({
|
||||
code: FileStorageExceptionCode.INVALID_EXTENSION,
|
||||
});
|
||||
|
||||
expect(mockDriver.writeFile).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('should accept buffer whose magic bytes match the path extension and persist the bytes-derived mime', async () => {
|
||||
await service.writeFile({
|
||||
workspaceId: 'workspace-123',
|
||||
applicationUniversalIdentifier: 'app-456',
|
||||
fileFolder: FileFolder.PublicAsset,
|
||||
resourcePath: 'assets/photo.png',
|
||||
sourceFile: pngBuffer,
|
||||
settings: { isTemporaryFile: false, toDelete: false },
|
||||
});
|
||||
|
||||
expect(mockDriver.writeFile).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ mimeType: 'image/png' }),
|
||||
);
|
||||
expect(mockFileRepository.upsert).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ mimeType: 'image/png' }),
|
||||
expect.anything(),
|
||||
);
|
||||
});
|
||||
|
||||
it('should persist application/typescript for a TypeScript source string (TWENTY_MIME_POLICY)', async () => {
|
||||
await service.writeFile({
|
||||
workspaceId: 'workspace-123',
|
||||
applicationUniversalIdentifier: 'app-456',
|
||||
fileFolder: FileFolder.Source,
|
||||
resourcePath: 'src/index.tsx',
|
||||
sourceFile: 'export const App = () => null;',
|
||||
settings: { isTemporaryFile: false, toDelete: false },
|
||||
});
|
||||
|
||||
expect(mockDriver.writeFile).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ mimeType: 'application/typescript' }),
|
||||
);
|
||||
});
|
||||
|
||||
it('should persist application/typescript for a TypeScript source buffer (policy beats mrmime collision)', async () => {
|
||||
await service.writeFile({
|
||||
workspaceId: 'workspace-123',
|
||||
applicationUniversalIdentifier: 'app-456',
|
||||
fileFolder: FileFolder.Source,
|
||||
resourcePath: 'src/handler.ts',
|
||||
sourceFile: Buffer.from(
|
||||
'export const handler = () => null;',
|
||||
'utf-8',
|
||||
),
|
||||
settings: { isTemporaryFile: false, toDelete: false },
|
||||
});
|
||||
|
||||
expect(mockDriver.writeFile).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ mimeType: 'application/typescript' }),
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('SVG sanitization (centralized invariant)', () => {
|
||||
const maliciousSvg =
|
||||
'<svg xmlns="http://www.w3.org/2000/svg" onload="alert(1)"><script>alert(2)</script><circle r="10" /></svg>';
|
||||
|
||||
it('should strip scripts and event handlers from an SVG string upload', async () => {
|
||||
await service.writeFile({
|
||||
workspaceId: 'workspace-123',
|
||||
applicationUniversalIdentifier: 'app-456',
|
||||
fileFolder: FileFolder.PublicAsset,
|
||||
resourcePath: 'assets/icon.svg',
|
||||
sourceFile: maliciousSvg,
|
||||
settings: { isTemporaryFile: false, toDelete: false },
|
||||
});
|
||||
|
||||
const driverCall = mockDriver.writeFile.mock.calls[0][0];
|
||||
|
||||
expect(driverCall.mimeType).toBe('image/svg+xml');
|
||||
expect(typeof driverCall.sourceFile).toBe('string');
|
||||
expect(driverCall.sourceFile).not.toContain('<script>');
|
||||
expect(driverCall.sourceFile).not.toContain('onload');
|
||||
expect(driverCall.sourceFile).toContain('<circle');
|
||||
});
|
||||
|
||||
it('should strip scripts and event handlers from an SVG buffer upload', async () => {
|
||||
await service.writeFile({
|
||||
workspaceId: 'workspace-123',
|
||||
applicationUniversalIdentifier: 'app-456',
|
||||
fileFolder: FileFolder.PublicAsset,
|
||||
resourcePath: 'assets/icon.svg',
|
||||
sourceFile: Buffer.from(maliciousSvg, 'utf-8'),
|
||||
settings: { isTemporaryFile: false, toDelete: false },
|
||||
});
|
||||
|
||||
const driverCall = mockDriver.writeFile.mock.calls[0][0];
|
||||
|
||||
expect(driverCall.mimeType).toBe('image/svg+xml');
|
||||
expect(driverCall.sourceFile).not.toContain('<script>');
|
||||
expect(driverCall.sourceFile).not.toContain('onload');
|
||||
});
|
||||
|
||||
it('should leave non-SVG content untouched (sanitize is no-op)', async () => {
|
||||
const original = '{"foo": "bar"}';
|
||||
|
||||
await service.writeFile({
|
||||
workspaceId: 'workspace-123',
|
||||
applicationUniversalIdentifier: 'app-456',
|
||||
fileFolder: FileFolder.Source,
|
||||
resourcePath: 'package.json',
|
||||
sourceFile: original,
|
||||
settings: { isTemporaryFile: false, toDelete: false },
|
||||
});
|
||||
|
||||
const driverCall = mockDriver.writeFile.mock.calls[0][0];
|
||||
|
||||
expect(driverCall.sourceFile).toBe(original);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('deleteFile', () => {
|
||||
|
||||
+29
@@ -0,0 +1,29 @@
|
||||
import { Catch, ExceptionFilter } from '@nestjs/common';
|
||||
|
||||
import { assertUnreachable } from 'twenty-shared/utils';
|
||||
|
||||
import {
|
||||
FileStorageException,
|
||||
FileStorageExceptionCode,
|
||||
} from 'src/engine/core-modules/file-storage/interfaces/file-storage-exception';
|
||||
import {
|
||||
ForbiddenError,
|
||||
NotFoundError,
|
||||
UserInputError,
|
||||
} from 'src/engine/core-modules/graphql/utils/graphql-errors.util';
|
||||
|
||||
@Catch(FileStorageException)
|
||||
export class FileStorageExceptionFilter implements ExceptionFilter {
|
||||
catch(exception: FileStorageException) {
|
||||
switch (exception.code) {
|
||||
case FileStorageExceptionCode.INVALID_EXTENSION:
|
||||
throw new UserInputError(exception);
|
||||
case FileStorageExceptionCode.ACCESS_DENIED:
|
||||
throw new ForbiddenError(exception);
|
||||
case FileStorageExceptionCode.FILE_NOT_FOUND:
|
||||
throw new NotFoundError(exception);
|
||||
default:
|
||||
assertUnreachable(exception.code);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,7 +1,9 @@
|
||||
import { type DynamicModule, Global } from '@nestjs/common';
|
||||
import { APP_FILTER } from '@nestjs/core';
|
||||
import { TypeOrmModule } from '@nestjs/typeorm';
|
||||
|
||||
import { ApplicationEntity } from 'src/engine/core-modules/application/application.entity';
|
||||
import { FileStorageExceptionFilter } from 'src/engine/core-modules/file-storage/file-storage-exception-filter';
|
||||
import { FileStorageDriverFactory } from 'src/engine/core-modules/file-storage/file-storage-driver.factory';
|
||||
import { FileStorageService } from 'src/engine/core-modules/file-storage/file-storage.service';
|
||||
import { FileEntity } from 'src/engine/core-modules/file/entities/file.entity';
|
||||
@@ -16,7 +18,14 @@ export class FileStorageModule {
|
||||
TwentyConfigModule,
|
||||
TypeOrmModule.forFeature([FileEntity, ApplicationEntity]),
|
||||
],
|
||||
providers: [FileStorageDriverFactory, FileStorageService],
|
||||
providers: [
|
||||
FileStorageDriverFactory,
|
||||
FileStorageService,
|
||||
{
|
||||
provide: APP_FILTER,
|
||||
useClass: FileStorageExceptionFilter,
|
||||
},
|
||||
],
|
||||
exports: [FileStorageDriverFactory, FileStorageService],
|
||||
};
|
||||
}
|
||||
|
||||
+26
-17
@@ -13,6 +13,7 @@ import {
|
||||
FileStorageException,
|
||||
FileStorageExceptionCode,
|
||||
} from 'src/engine/core-modules/file-storage/interfaces/file-storage-exception';
|
||||
import { prepareFileForStorageOrThrow } from 'src/engine/core-modules/file-storage/utils/prepare-file-for-storage-or-throw.util';
|
||||
import { validateFilePath } from 'src/engine/core-modules/file-storage/utils/validate-file-path.util';
|
||||
import { validateFolderPath } from 'src/engine/core-modules/file-storage/utils/validate-folder-path.util';
|
||||
import { validateStoragePathIsWithinWorkspaceOrThrow } from 'src/engine/core-modules/file-storage/utils/validate-storage-path-is-within-workspace-or-throw.util';
|
||||
@@ -66,7 +67,7 @@ export class FileStorageService {
|
||||
return { onStoragePath, resourcePath };
|
||||
}
|
||||
|
||||
private validateAndBuildFileStoragePath(params: ResourceIdentifier): {
|
||||
private validateAndBuildFileStoragePathOrThrow(params: ResourceIdentifier): {
|
||||
onStorageFilePath: string;
|
||||
filePath: string;
|
||||
} {
|
||||
@@ -91,7 +92,7 @@ export class FileStorageService {
|
||||
return { onStorageFilePath: onStoragePath, filePath: resourcePath };
|
||||
}
|
||||
|
||||
private validateAndBuildFolderStoragePath(
|
||||
private validateAndBuildFolderStoragePathOrThrow(
|
||||
params: Omit<ResourceIdentifier, 'resourcePath'> & { folderPath: string },
|
||||
): { onStorageFolderPath: string; folderPath: string } {
|
||||
const validationResult = validateFolderPath({
|
||||
@@ -119,7 +120,6 @@ export class FileStorageService {
|
||||
|
||||
async writeFile({
|
||||
sourceFile,
|
||||
mimeType,
|
||||
fileFolder,
|
||||
applicationUniversalIdentifier,
|
||||
workspaceId,
|
||||
@@ -129,7 +129,6 @@ export class FileStorageService {
|
||||
queryRunner,
|
||||
}: ResourceIdentifier & {
|
||||
sourceFile: string | Buffer | Uint8Array;
|
||||
mimeType: string | undefined;
|
||||
fileId?: string;
|
||||
settings: FileSettings;
|
||||
queryRunner?: QueryRunner;
|
||||
@@ -151,17 +150,23 @@ export class FileStorageService {
|
||||
});
|
||||
|
||||
const { onStorageFilePath, filePath } =
|
||||
this.validateAndBuildFileStoragePath({
|
||||
this.validateAndBuildFileStoragePathOrThrow({
|
||||
workspaceId,
|
||||
applicationUniversalIdentifier,
|
||||
fileFolder,
|
||||
resourcePath,
|
||||
});
|
||||
|
||||
const { sourceFile: persistedSourceFile, mimeType } =
|
||||
await prepareFileForStorageOrThrow({
|
||||
sourceFile,
|
||||
resourcePath,
|
||||
});
|
||||
|
||||
await driver.writeFile({
|
||||
filePath: onStorageFilePath,
|
||||
mimeType,
|
||||
sourceFile,
|
||||
sourceFile: persistedSourceFile,
|
||||
});
|
||||
|
||||
await fileRepository.upsert(
|
||||
@@ -172,9 +177,9 @@ export class FileStorageService {
|
||||
id: fileId,
|
||||
mimeType,
|
||||
size:
|
||||
typeof sourceFile === 'string'
|
||||
? Buffer.byteLength(sourceFile)
|
||||
: sourceFile.length,
|
||||
typeof persistedSourceFile === 'string'
|
||||
? Buffer.byteLength(persistedSourceFile)
|
||||
: persistedSourceFile.length,
|
||||
settings,
|
||||
},
|
||||
['path', 'workspaceId', 'applicationId'],
|
||||
@@ -197,7 +202,8 @@ export class FileStorageService {
|
||||
},
|
||||
): Promise<string | null> {
|
||||
const driver = this.fileStorageDriverFactory.getCurrentDriver();
|
||||
const { onStorageFilePath } = this.validateAndBuildFileStoragePath(params);
|
||||
const { onStorageFilePath } =
|
||||
this.validateAndBuildFileStoragePathOrThrow(params);
|
||||
|
||||
return driver.getPresignedUrl({
|
||||
filePath: onStorageFilePath,
|
||||
@@ -210,7 +216,8 @@ export class FileStorageService {
|
||||
readFile(params: ResourceIdentifier): Promise<Readable> {
|
||||
const driver = this.fileStorageDriverFactory.getCurrentDriver();
|
||||
|
||||
const { onStorageFilePath } = this.validateAndBuildFileStoragePath(params);
|
||||
const { onStorageFilePath } =
|
||||
this.validateAndBuildFileStoragePathOrThrow(params);
|
||||
|
||||
return driver.readFile({ filePath: onStorageFilePath });
|
||||
}
|
||||
@@ -219,7 +226,8 @@ export class FileStorageService {
|
||||
params: ResourceIdentifier & { localPath: string },
|
||||
): Promise<void> {
|
||||
const driver = this.fileStorageDriverFactory.getCurrentDriver();
|
||||
const { onStorageFilePath } = this.validateAndBuildFileStoragePath(params);
|
||||
const { onStorageFilePath } =
|
||||
this.validateAndBuildFileStoragePathOrThrow(params);
|
||||
|
||||
return driver.downloadFile({
|
||||
onStoragePath: onStorageFilePath,
|
||||
@@ -256,7 +264,7 @@ export class FileStorageService {
|
||||
async deleteFile(params: ResourceIdentifier): Promise<void> {
|
||||
const driver = this.fileStorageDriverFactory.getCurrentDriver();
|
||||
const { onStorageFilePath, filePath } =
|
||||
this.validateAndBuildFileStoragePath(params);
|
||||
this.validateAndBuildFileStoragePathOrThrow(params);
|
||||
|
||||
await driver.delete({
|
||||
folderPath: dirname(onStorageFilePath),
|
||||
@@ -288,7 +296,7 @@ export class FileStorageService {
|
||||
} = params;
|
||||
|
||||
const { onStorageFolderPath, folderPath: validatedFolderPath } =
|
||||
this.validateAndBuildFolderStoragePath({
|
||||
this.validateAndBuildFolderStoragePathOrThrow({
|
||||
workspaceId,
|
||||
applicationUniversalIdentifier,
|
||||
fileFolder,
|
||||
@@ -373,9 +381,9 @@ export class FileStorageService {
|
||||
const driver = this.fileStorageDriverFactory.getCurrentDriver();
|
||||
|
||||
const { onStorageFilePath: fromPath } =
|
||||
this.validateAndBuildFileStoragePath(from);
|
||||
this.validateAndBuildFileStoragePathOrThrow(from);
|
||||
const { onStorageFilePath: toPath } =
|
||||
this.validateAndBuildFileStoragePath(to);
|
||||
this.validateAndBuildFileStoragePathOrThrow(to);
|
||||
|
||||
const isFile = await driver.checkFileExists({ filePath: fromPath });
|
||||
|
||||
@@ -394,7 +402,8 @@ export class FileStorageService {
|
||||
|
||||
checkFileExists(params: ResourceIdentifier): Promise<boolean> {
|
||||
const driver = this.fileStorageDriverFactory.getCurrentDriver();
|
||||
const { onStorageFilePath } = this.validateAndBuildFileStoragePath(params);
|
||||
const { onStorageFilePath } =
|
||||
this.validateAndBuildFileStoragePathOrThrow(params);
|
||||
|
||||
return driver.checkFileExists({ filePath: onStorageFilePath });
|
||||
}
|
||||
|
||||
+207
@@ -0,0 +1,207 @@
|
||||
import { FileStorageExceptionCode } from 'src/engine/core-modules/file-storage/interfaces/file-storage-exception';
|
||||
import { prepareFileForStorageOrThrow } from 'src/engine/core-modules/file-storage/utils/prepare-file-for-storage-or-throw.util';
|
||||
|
||||
const pngBuffer = Buffer.from([
|
||||
0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0x00, 0x00, 0x00, 0x0d, 0x49,
|
||||
0x48, 0x44, 0x52,
|
||||
]);
|
||||
const pdfBuffer = Buffer.from('%PDF-1.4\n', 'utf-8');
|
||||
const zipBuffer = Buffer.from([0x50, 0x4b, 0x03, 0x04]);
|
||||
const textBuffer = Buffer.from('Hello, world!', 'utf-8');
|
||||
|
||||
const benignSvg =
|
||||
'<svg xmlns="http://www.w3.org/2000/svg"><circle r="10" /></svg>';
|
||||
const maliciousSvg = `<svg xmlns="http://www.w3.org/2000/svg" onload="alert(1)"><script>alert(2)</script><circle r="10" /></svg>`;
|
||||
|
||||
describe('prepareFileForStorageOrThrow', () => {
|
||||
describe('mime resolution', () => {
|
||||
it.each([
|
||||
{
|
||||
name: 'TypeScript .ts string (policy)',
|
||||
sourceFile: 'export const x = 1;',
|
||||
resourcePath: 'src/index.ts',
|
||||
expectedMime: 'application/typescript',
|
||||
},
|
||||
{
|
||||
name: 'TypeScript .tsx string (policy)',
|
||||
sourceFile: 'export const App = () => null;',
|
||||
resourcePath: 'src/App.tsx',
|
||||
expectedMime: 'application/typescript',
|
||||
},
|
||||
{
|
||||
name: '.mjs string (mrmime)',
|
||||
sourceFile: 'export default 1;',
|
||||
resourcePath: 'src/handler.mjs',
|
||||
expectedMime: 'text/javascript',
|
||||
},
|
||||
{
|
||||
name: 'JSON string (mrmime)',
|
||||
sourceFile: '{"foo": "bar"}',
|
||||
resourcePath: 'package.json',
|
||||
expectedMime: 'application/json',
|
||||
},
|
||||
{
|
||||
name: 'unknown-extension string (octet-stream)',
|
||||
sourceFile: 'lock content',
|
||||
resourcePath: 'yarn.lock',
|
||||
expectedMime: 'application/octet-stream',
|
||||
},
|
||||
{
|
||||
name: 'no-extension string (octet-stream)',
|
||||
sourceFile: 'content',
|
||||
resourcePath: 'somefile',
|
||||
expectedMime: 'application/octet-stream',
|
||||
},
|
||||
{
|
||||
name: 'dot-file string (octet-stream — extname semantics)',
|
||||
sourceFile: 'content',
|
||||
resourcePath: '.gitignore',
|
||||
expectedMime: 'application/octet-stream',
|
||||
},
|
||||
])(
|
||||
'returns $expectedMime for $name',
|
||||
async ({ sourceFile, resourcePath, expectedMime }) => {
|
||||
const { mimeType } = await prepareFileForStorageOrThrow({
|
||||
sourceFile,
|
||||
resourcePath,
|
||||
});
|
||||
|
||||
expect(mimeType).toBe(expectedMime);
|
||||
},
|
||||
);
|
||||
|
||||
it.each([
|
||||
{
|
||||
name: 'PNG buffer',
|
||||
sourceFile: pngBuffer,
|
||||
resourcePath: 'images/photo.png',
|
||||
expectedMime: 'image/png',
|
||||
},
|
||||
{
|
||||
name: 'PDF buffer',
|
||||
sourceFile: pdfBuffer,
|
||||
resourcePath: 'docs/contract.pdf',
|
||||
expectedMime: 'application/pdf',
|
||||
},
|
||||
{
|
||||
name: 'ZIP buffer',
|
||||
sourceFile: zipBuffer,
|
||||
resourcePath: 'archives/data.zip',
|
||||
expectedMime: 'application/zip',
|
||||
},
|
||||
])(
|
||||
'returns the bytes-derived mime for $name',
|
||||
async ({ sourceFile, resourcePath, expectedMime }) => {
|
||||
const { mimeType } = await prepareFileForStorageOrThrow({
|
||||
sourceFile,
|
||||
resourcePath,
|
||||
});
|
||||
|
||||
expect(mimeType).toBe(expectedMime);
|
||||
},
|
||||
);
|
||||
|
||||
it('returns the bytes-derived mime for a Uint8Array source', async () => {
|
||||
const { mimeType } = await prepareFileForStorageOrThrow({
|
||||
sourceFile: new Uint8Array(pngBuffer),
|
||||
resourcePath: 'images/photo.png',
|
||||
});
|
||||
|
||||
expect(mimeType).toBe('image/png');
|
||||
});
|
||||
});
|
||||
|
||||
describe('binary-extension backstop', () => {
|
||||
it('throws when a buffer with a binary extension does not match the magic bytes', async () => {
|
||||
await expect(
|
||||
prepareFileForStorageOrThrow({
|
||||
sourceFile: textBuffer,
|
||||
resourcePath: 'fake-image.png',
|
||||
}),
|
||||
).rejects.toMatchObject({
|
||||
code: FileStorageExceptionCode.INVALID_EXTENSION,
|
||||
});
|
||||
});
|
||||
|
||||
it('throws when a string with a binary extension does not match the magic bytes', async () => {
|
||||
await expect(
|
||||
prepareFileForStorageOrThrow({
|
||||
sourceFile: 'this is not a png',
|
||||
resourcePath: 'fake-image.png',
|
||||
}),
|
||||
).rejects.toMatchObject({
|
||||
code: FileStorageExceptionCode.INVALID_EXTENSION,
|
||||
});
|
||||
});
|
||||
|
||||
it('throws on Uint8Array byte/extension mismatch', async () => {
|
||||
await expect(
|
||||
prepareFileForStorageOrThrow({
|
||||
sourceFile: new Uint8Array(textBuffer),
|
||||
resourcePath: 'fake-image.pdf',
|
||||
}),
|
||||
).rejects.toMatchObject({
|
||||
code: FileStorageExceptionCode.INVALID_EXTENSION,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('SVG sanitization', () => {
|
||||
it('sanitizes an SVG string by removing scripts and event handlers', async () => {
|
||||
const { sourceFile, mimeType } = await prepareFileForStorageOrThrow({
|
||||
sourceFile: maliciousSvg,
|
||||
resourcePath: 'assets/icon.svg',
|
||||
});
|
||||
|
||||
expect(mimeType).toBe('image/svg+xml');
|
||||
expect(typeof sourceFile).toBe('string');
|
||||
expect(sourceFile).not.toContain('<script>');
|
||||
expect(sourceFile).not.toContain('onload');
|
||||
expect(sourceFile).toContain('<circle');
|
||||
});
|
||||
|
||||
it('sanitizes an SVG buffer (DOMPurify returns a string)', async () => {
|
||||
const { sourceFile, mimeType } = await prepareFileForStorageOrThrow({
|
||||
sourceFile: Buffer.from(maliciousSvg, 'utf-8'),
|
||||
resourcePath: 'assets/icon.svg',
|
||||
});
|
||||
|
||||
expect(mimeType).toBe('image/svg+xml');
|
||||
expect(typeof sourceFile).toBe('string');
|
||||
expect(sourceFile).not.toContain('<script>');
|
||||
expect(sourceFile).not.toContain('onload');
|
||||
});
|
||||
|
||||
it('preserves a benign SVG (no destructive sanitization)', async () => {
|
||||
const { sourceFile } = await prepareFileForStorageOrThrow({
|
||||
sourceFile: benignSvg,
|
||||
resourcePath: 'assets/icon.svg',
|
||||
});
|
||||
|
||||
expect(sourceFile).toContain('<circle');
|
||||
expect(sourceFile).toContain('xmlns');
|
||||
});
|
||||
});
|
||||
|
||||
describe('non-SVG content is returned unchanged', () => {
|
||||
it('returns the original buffer reference for a non-SVG buffer', async () => {
|
||||
const { sourceFile } = await prepareFileForStorageOrThrow({
|
||||
sourceFile: pngBuffer,
|
||||
resourcePath: 'images/photo.png',
|
||||
});
|
||||
|
||||
expect(sourceFile).toBe(pngBuffer);
|
||||
});
|
||||
|
||||
it('returns the original string for a non-SVG string', async () => {
|
||||
const original = '{"foo": "bar"}';
|
||||
|
||||
const { sourceFile } = await prepareFileForStorageOrThrow({
|
||||
sourceFile: original,
|
||||
resourcePath: 'package.json',
|
||||
});
|
||||
|
||||
expect(sourceFile).toBe(original);
|
||||
});
|
||||
});
|
||||
});
|
||||
+33
@@ -0,0 +1,33 @@
|
||||
import { extractFileInfoOrThrow } from 'src/engine/core-modules/file/utils/extract-file-info-or-throw.utils';
|
||||
import { sanitizeFile } from 'src/engine/core-modules/file/utils/sanitize-file.utils';
|
||||
|
||||
export const prepareFileForStorageOrThrow = async ({
|
||||
sourceFile,
|
||||
resourcePath,
|
||||
}: {
|
||||
sourceFile: Buffer | Uint8Array | string;
|
||||
resourcePath: string;
|
||||
}): Promise<{
|
||||
sourceFile: Buffer | Uint8Array | string;
|
||||
mimeType: string;
|
||||
}> => {
|
||||
const bufferForExtract =
|
||||
typeof sourceFile === 'string'
|
||||
? Buffer.from(sourceFile, 'utf8')
|
||||
: Buffer.isBuffer(sourceFile)
|
||||
? sourceFile
|
||||
: Buffer.from(sourceFile);
|
||||
|
||||
const { mimeType, ext } = await extractFileInfoOrThrow({
|
||||
file: bufferForExtract,
|
||||
filename: resourcePath,
|
||||
});
|
||||
|
||||
const sanitizedSourceFile = sanitizeFile({
|
||||
file: sourceFile,
|
||||
ext,
|
||||
mimeType,
|
||||
});
|
||||
|
||||
return { sourceFile: sanitizedSourceFile, mimeType };
|
||||
};
|
||||
+19
@@ -0,0 +1,19 @@
|
||||
// Extensions where Twenty intentionally deviates from the IANA-standard mime
|
||||
// mapping returned by mrmime. Twenty's storage layer persists these values
|
||||
// instead of mrmime's output (or instead of throwing in the case of `.ts`).
|
||||
//
|
||||
// Why each entry exists:
|
||||
// - ts/tsx: IANA registers `.ts` as `video/mp2t` (MPEG-2 Transport Stream),
|
||||
// which predates TypeScript. `.tsx` is unregistered. Twenty uses
|
||||
// the developer-tooling convention `application/typescript`.
|
||||
//
|
||||
// Only add an entry when (1) mrmime returns the wrong mime (collision) or no
|
||||
// mime AND (2) Twenty actually writes that extension via FileStorageService AND
|
||||
// (3) there is a clear developer-tooling convention for the right mime. For
|
||||
// everything else — media, archives, documents, `.js`, `.mjs`, `.cjs`, etc. —
|
||||
// trust mrmime. This policy is reactive: extend it the day we observe a real
|
||||
// wrong-mime persisted by an exercised code path, not in anticipation.
|
||||
export const TWENTY_MIME_POLICY: Record<string, string> = {
|
||||
ts: 'application/typescript',
|
||||
tsx: 'application/typescript',
|
||||
};
|
||||
+4
-7
@@ -8,8 +8,8 @@ import { ApplicationService } from 'src/engine/core-modules/application/applicat
|
||||
import { FileStorageService } from 'src/engine/core-modules/file-storage/file-storage.service';
|
||||
import { FileWithSignedUrlDTO } from 'src/engine/core-modules/file/dtos/file-with-sign-url.dto';
|
||||
import { FileUrlService } from 'src/engine/core-modules/file/file-url/file-url.service';
|
||||
import { extractFileInfo } from 'src/engine/core-modules/file/utils/extract-file-info.utils';
|
||||
import { sanitizeFile } from 'src/engine/core-modules/file/utils/sanitize-file.utils';
|
||||
import { extractFileInfoOrThrow } from 'src/engine/core-modules/file/utils/extract-file-info-or-throw.utils';
|
||||
|
||||
@Injectable()
|
||||
export class FileAiChatService {
|
||||
constructor(
|
||||
@@ -27,13 +27,11 @@ export class FileAiChatService {
|
||||
filename: string;
|
||||
workspaceId: string;
|
||||
}): Promise<FileWithSignedUrlDTO> {
|
||||
const { mimeType, ext } = await extractFileInfo({
|
||||
const { ext } = await extractFileInfoOrThrow({
|
||||
file,
|
||||
filename,
|
||||
});
|
||||
|
||||
const sanitizedFile = sanitizeFile({ file, ext, mimeType });
|
||||
|
||||
const fileId = v4();
|
||||
const name = `${fileId}${isNonEmptyString(ext) ? `.${ext}` : ''}`;
|
||||
|
||||
@@ -45,9 +43,8 @@ export class FileAiChatService {
|
||||
);
|
||||
|
||||
const savedFile = await this.fileStorageService.writeFile({
|
||||
sourceFile: sanitizedFile,
|
||||
sourceFile: file,
|
||||
resourcePath: name,
|
||||
mimeType,
|
||||
fileFolder: FileFolder.AgentChat,
|
||||
applicationUniversalIdentifier:
|
||||
workspaceCustomFlatApplication.universalIdentifier,
|
||||
|
||||
+3
-6
@@ -19,9 +19,8 @@ import { FileStorageService } from 'src/engine/core-modules/file-storage/file-st
|
||||
import { FileWithSignedUrlDTO } from 'src/engine/core-modules/file/dtos/file-with-sign-url.dto';
|
||||
import { FileEntity } from 'src/engine/core-modules/file/entities/file.entity';
|
||||
import { FileUrlService } from 'src/engine/core-modules/file/file-url/file-url.service';
|
||||
import { extractFileInfo } from 'src/engine/core-modules/file/utils/extract-file-info.utils';
|
||||
import { extractFileInfoOrThrow } from 'src/engine/core-modules/file/utils/extract-file-info-or-throw.utils';
|
||||
import { removeFileFolderFromFileEntityPath } from 'src/engine/core-modules/file/utils/remove-file-folder-from-file-entity-path.utils';
|
||||
import { sanitizeFile } from 'src/engine/core-modules/file/utils/sanitize-file.utils';
|
||||
import { SecureHttpClientService } from 'src/engine/core-modules/secure-http-client/secure-http-client.service';
|
||||
import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity';
|
||||
import { getImageBufferFromUrl } from 'src/utils/image';
|
||||
@@ -72,8 +71,7 @@ export class FileCorePictureService {
|
||||
applicationUniversalIdentifier?: string;
|
||||
queryRunner?: QueryRunner;
|
||||
}): Promise<FileEntity> {
|
||||
const { mimeType, ext } = await extractFileInfo({ file, filename });
|
||||
const sanitizedFile = sanitizeFile({ file, ext, mimeType });
|
||||
const { ext } = await extractFileInfoOrThrow({ file, filename });
|
||||
|
||||
const fileId = v4();
|
||||
const finalName = `${fileId}${isNonEmptyString(ext) ? `.${ext}` : ''}`;
|
||||
@@ -83,9 +81,8 @@ export class FileCorePictureService {
|
||||
(await this.findCustomApplicationUniversalIdentifier(workspaceId));
|
||||
|
||||
const savedFile = await this.fileStorageService.writeFile({
|
||||
sourceFile: sanitizedFile,
|
||||
sourceFile: file,
|
||||
resourcePath: finalName,
|
||||
mimeType,
|
||||
fileFolder: FileFolder.CorePicture,
|
||||
applicationUniversalIdentifier: universalIdentifier,
|
||||
workspaceId,
|
||||
|
||||
+3
-7
@@ -8,8 +8,7 @@ import { ApplicationService } from 'src/engine/core-modules/application/applicat
|
||||
import { FileStorageService } from 'src/engine/core-modules/file-storage/file-storage.service';
|
||||
import { FileWithSignedUrlDTO } from 'src/engine/core-modules/file/dtos/file-with-sign-url.dto';
|
||||
import { FileUrlService } from 'src/engine/core-modules/file/file-url/file-url.service';
|
||||
import { extractFileInfo } from 'src/engine/core-modules/file/utils/extract-file-info.utils';
|
||||
import { sanitizeFile } from 'src/engine/core-modules/file/utils/sanitize-file.utils';
|
||||
import { extractFileInfoOrThrow } from 'src/engine/core-modules/file/utils/extract-file-info-or-throw.utils';
|
||||
|
||||
@Injectable()
|
||||
export class FileEmailAttachmentService {
|
||||
@@ -30,13 +29,11 @@ export class FileEmailAttachmentService {
|
||||
filename: string;
|
||||
workspaceId: string;
|
||||
}): Promise<FileWithSignedUrlDTO> {
|
||||
const { mimeType, ext } = await extractFileInfo({
|
||||
const { ext } = await extractFileInfoOrThrow({
|
||||
file,
|
||||
filename,
|
||||
});
|
||||
|
||||
const sanitizedFile = sanitizeFile({ file, ext, mimeType });
|
||||
|
||||
const fileId = v4();
|
||||
const name = `${fileId}${isNonEmptyString(ext) ? `.${ext}` : ''}`;
|
||||
|
||||
@@ -48,9 +45,8 @@ export class FileEmailAttachmentService {
|
||||
);
|
||||
|
||||
const savedFile = await this.fileStorageService.writeFile({
|
||||
sourceFile: sanitizedFile,
|
||||
sourceFile: file,
|
||||
resourcePath: name,
|
||||
mimeType,
|
||||
fileFolder: FileFolder.EmailAttachment,
|
||||
applicationUniversalIdentifier:
|
||||
workspaceCustomFlatApplication.universalIdentifier,
|
||||
|
||||
+3
-7
@@ -8,8 +8,7 @@ import { ApplicationService } from 'src/engine/core-modules/application/applicat
|
||||
import { FileStorageService } from 'src/engine/core-modules/file-storage/file-storage.service';
|
||||
import { FileWithSignedUrlDTO } from 'src/engine/core-modules/file/dtos/file-with-sign-url.dto';
|
||||
import { FileUrlService } from 'src/engine/core-modules/file/file-url/file-url.service';
|
||||
import { extractFileInfo } from 'src/engine/core-modules/file/utils/extract-file-info.utils';
|
||||
import { sanitizeFile } from 'src/engine/core-modules/file/utils/sanitize-file.utils';
|
||||
import { extractFileInfoOrThrow } from 'src/engine/core-modules/file/utils/extract-file-info-or-throw.utils';
|
||||
|
||||
@Injectable()
|
||||
export class FileWorkflowService {
|
||||
@@ -28,13 +27,11 @@ export class FileWorkflowService {
|
||||
filename: string;
|
||||
workspaceId: string;
|
||||
}): Promise<FileWithSignedUrlDTO> {
|
||||
const { mimeType, ext } = await extractFileInfo({
|
||||
const { ext } = await extractFileInfoOrThrow({
|
||||
file,
|
||||
filename,
|
||||
});
|
||||
|
||||
const sanitizedFile = sanitizeFile({ file, ext, mimeType });
|
||||
|
||||
const fileId = v4();
|
||||
const name = `${fileId}${isNonEmptyString(ext) ? `.${ext}` : ''}`;
|
||||
|
||||
@@ -46,9 +43,8 @@ export class FileWorkflowService {
|
||||
);
|
||||
|
||||
const savedFile = await this.fileStorageService.writeFile({
|
||||
sourceFile: sanitizedFile,
|
||||
sourceFile: file,
|
||||
resourcePath: name,
|
||||
mimeType,
|
||||
fileFolder: FileFolder.Workflow,
|
||||
applicationUniversalIdentifier:
|
||||
workspaceCustomFlatApplication.universalIdentifier,
|
||||
|
||||
+3
-7
@@ -15,8 +15,7 @@ import {
|
||||
FilesFieldException,
|
||||
FilesFieldExceptionCode,
|
||||
} from 'src/engine/core-modules/file/files-field/files-field.exception';
|
||||
import { extractFileInfo } from 'src/engine/core-modules/file/utils/extract-file-info.utils';
|
||||
import { sanitizeFile } from 'src/engine/core-modules/file/utils/sanitize-file.utils';
|
||||
import { extractFileInfoOrThrow } from 'src/engine/core-modules/file/utils/extract-file-info-or-throw.utils';
|
||||
import { FieldMetadataEntity } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity';
|
||||
|
||||
@Injectable()
|
||||
@@ -53,13 +52,11 @@ export class FilesFieldService {
|
||||
);
|
||||
}
|
||||
|
||||
const { mimeType, ext } = await extractFileInfo({
|
||||
const { ext } = await extractFileInfoOrThrow({
|
||||
file,
|
||||
filename,
|
||||
});
|
||||
|
||||
const sanitizedFile = sanitizeFile({ file, ext, mimeType });
|
||||
|
||||
const fileId = v4();
|
||||
const name = `${fileId}${isNonEmptyString(ext) ? `.${ext}` : ''}`;
|
||||
|
||||
@@ -82,9 +79,8 @@ export class FilesFieldService {
|
||||
});
|
||||
|
||||
const savedFile = await this.fileStorageService.writeFile({
|
||||
sourceFile: sanitizedFile,
|
||||
sourceFile: file,
|
||||
resourcePath: `${fieldMetadata.universalIdentifier}/${name}`,
|
||||
mimeType,
|
||||
fileFolder: FileFolder.FilesField,
|
||||
applicationUniversalIdentifier: application.universalIdentifier,
|
||||
workspaceId,
|
||||
|
||||
+25
-5
@@ -1,4 +1,4 @@
|
||||
import { extractFileInfo } from '../extract-file-info.utils';
|
||||
import { extractFileInfoOrThrow } from '../extract-file-info-or-throw.utils';
|
||||
|
||||
const pngBuffer = Buffer.from([
|
||||
0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0x00, 0x00, 0x00, 0x0d, 0x49,
|
||||
@@ -8,7 +8,7 @@ const pdfBuffer = Buffer.from('%PDF-1.4\n', 'utf-8');
|
||||
const textBuffer = Buffer.from('Hello, world!', 'utf-8');
|
||||
const zipBuffer = Buffer.from([0x50, 0x4b, 0x03, 0x04]);
|
||||
|
||||
describe('extractFileInfo', () => {
|
||||
describe('extractFileInfoOrThrow', () => {
|
||||
it.each([
|
||||
{
|
||||
name: 'PNG',
|
||||
@@ -41,7 +41,7 @@ describe('extractFileInfo', () => {
|
||||
])(
|
||||
'should detect $name from buffer magic numbers',
|
||||
async ({ buffer, filename, ext, mime }) => {
|
||||
const result = await extractFileInfo({ file: buffer, filename });
|
||||
const result = await extractFileInfoOrThrow({ file: buffer, filename });
|
||||
|
||||
expect(result).toEqual({ mimeType: mime, ext });
|
||||
},
|
||||
@@ -78,7 +78,10 @@ describe('extractFileInfo', () => {
|
||||
])(
|
||||
'should fall back to extension for $name files',
|
||||
async ({ filename, ext, mime }) => {
|
||||
const result = await extractFileInfo({ file: textBuffer, filename });
|
||||
const result = await extractFileInfoOrThrow({
|
||||
file: textBuffer,
|
||||
filename,
|
||||
});
|
||||
|
||||
expect(result).toEqual({ mimeType: mime, ext });
|
||||
},
|
||||
@@ -95,10 +98,27 @@ describe('extractFileInfo', () => {
|
||||
'should throw when $ext extension does not match buffer content',
|
||||
async ({ filename, ext, expectedMime }) => {
|
||||
await expect(
|
||||
extractFileInfo({ file: textBuffer, filename }),
|
||||
extractFileInfoOrThrow({ file: textBuffer, filename }),
|
||||
).rejects.toThrow(
|
||||
`File content does not match its extension. The file has extension '${ext}' (expected mime type: ${expectedMime}), but the file content could not be detected as this type. The file may be corrupted, have the wrong extension, or be a security risk.`,
|
||||
);
|
||||
},
|
||||
);
|
||||
|
||||
describe('TWENTY_MIME_POLICY (Twenty deviates from IANA)', () => {
|
||||
it.each([
|
||||
{ filename: 'src/index.ts', expectedMime: 'application/typescript' },
|
||||
{ filename: 'src/index.tsx', expectedMime: 'application/typescript' },
|
||||
])(
|
||||
'should return $expectedMime for $filename without throwing on IANA collision',
|
||||
async ({ filename, expectedMime }) => {
|
||||
const result = await extractFileInfoOrThrow({
|
||||
file: textBuffer,
|
||||
filename,
|
||||
});
|
||||
|
||||
expect(result.mimeType).toBe(expectedMime);
|
||||
},
|
||||
);
|
||||
});
|
||||
});
|
||||
+23
-6
@@ -10,9 +10,14 @@ import {
|
||||
} from 'src/engine/core-modules/file-storage/interfaces/file-storage-exception';
|
||||
|
||||
import { detectPdf } from '@file-type/pdf';
|
||||
import { TWENTY_MIME_POLICY } from 'src/engine/core-modules/file/constants/twenty-mime-policy.constant';
|
||||
import { buildFileInfo } from 'src/engine/core-modules/file/utils/build-file-info.utils';
|
||||
|
||||
export const extractFileInfo = async ({
|
||||
const fileTypeParser = new FileTypeParser({
|
||||
customDetectors: [detectPdf],
|
||||
});
|
||||
|
||||
export const extractFileInfoOrThrow = async ({
|
||||
file,
|
||||
filename,
|
||||
}: {
|
||||
@@ -21,12 +26,8 @@ export const extractFileInfo = async ({
|
||||
}) => {
|
||||
const { ext: declaredExt } = buildFileInfo(filename);
|
||||
|
||||
const fileParser = new FileTypeParser({
|
||||
customDetectors: [detectPdf],
|
||||
});
|
||||
|
||||
const { ext: detectedExt, mime: detectedMime } =
|
||||
(await fileParser.fromBuffer(file)) ?? {};
|
||||
(await fileTypeParser.fromBuffer(file)) ?? {};
|
||||
|
||||
if (isDefined(detectedExt) && isDefined(detectedMime)) {
|
||||
return {
|
||||
@@ -40,6 +41,22 @@ export const extractFileInfo = async ({
|
||||
let mimeType: string = 'application/octet-stream';
|
||||
|
||||
if (isNonEmptyString(ext)) {
|
||||
// Twenty policy wins over the ext-based fallback for the (small) set of
|
||||
// extensions where mrmime's IANA mapping collides with a developer-tooling
|
||||
// convention. This branch is only reached when file-type's magic-byte
|
||||
// sniff returned nothing — when the bytes actually match (e.g. a real
|
||||
// MPEG-TS video at foo.ts), we already returned above.
|
||||
//
|
||||
// For .ts/.tsx the policy is also load-bearing for correctness: without
|
||||
// it, lookup('ts') → 'video/mp2t' is in file-type's supportedMimeTypes,
|
||||
// so the check below would throw INVALID_EXTENSION on every TypeScript
|
||||
// source upload.
|
||||
const policyMime = TWENTY_MIME_POLICY[ext];
|
||||
|
||||
if (isDefined(policyMime)) {
|
||||
return { mimeType: policyMime, ext };
|
||||
}
|
||||
|
||||
const mimeTypeFromExtension = lookup(ext);
|
||||
|
||||
if (
|
||||
-4
@@ -94,7 +94,6 @@ export class LogicFunctionResourceService {
|
||||
fileFolder: FileFolder.Source,
|
||||
resourcePath: sourceHandlerPath,
|
||||
sourceFile: sourceFile.content,
|
||||
mimeType: 'application/typescript',
|
||||
settings: {
|
||||
isTemporaryFile: false,
|
||||
toDelete: false,
|
||||
@@ -107,7 +106,6 @@ export class LogicFunctionResourceService {
|
||||
fileFolder: FileFolder.BuiltLogicFunction,
|
||||
resourcePath: builtHandlerPath,
|
||||
sourceFile: builtFile.content,
|
||||
mimeType: 'application/javascript',
|
||||
settings: {
|
||||
isTemporaryFile: false,
|
||||
toDelete: false,
|
||||
@@ -139,7 +137,6 @@ export class LogicFunctionResourceService {
|
||||
resourcePath: sourceHandlerPath,
|
||||
sourceFile: sourceHandlerCode,
|
||||
settings: { isTemporaryFile: false, toDelete: false },
|
||||
mimeType: 'application/typescript',
|
||||
queryRunner,
|
||||
});
|
||||
}
|
||||
@@ -172,7 +169,6 @@ export class LogicFunctionResourceService {
|
||||
fileFolder: FileFolder.BuiltLogicFunction,
|
||||
resourcePath: builtHandlerPath,
|
||||
sourceFile: builtCode,
|
||||
mimeType: 'application/javascript',
|
||||
settings: {
|
||||
isTemporaryFile: false,
|
||||
toDelete: false,
|
||||
|
||||
-1
@@ -157,7 +157,6 @@ export class SdkClientGenerationService {
|
||||
fileFolder: FileFolder.GeneratedSdkClient,
|
||||
resourcePath: SDK_CLIENT_ARCHIVE_NAME,
|
||||
sourceFile: archiveBuffer,
|
||||
mimeType: 'application/zip',
|
||||
settings: { isTemporaryFile: false, toDelete: false },
|
||||
});
|
||||
|
||||
|
||||
-1
@@ -337,7 +337,6 @@ export class CodeInterpreterTool implements Tool {
|
||||
|
||||
const savedFile = await this.fileStorageService.writeFile({
|
||||
sourceFile: file.content,
|
||||
mimeType: file.mimeType,
|
||||
fileFolder: FileFolder.AgentChat,
|
||||
applicationUniversalIdentifier:
|
||||
workspaceCustomFlatApplication.universalIdentifier,
|
||||
|
||||
Reference in New Issue
Block a user