feat(files): direct-to-storage upload endpoints with pending file lifecycle (#22449)
<img width="1484" height="404" alt="image" src="https://github.com/user-attachments/assets/b2d363bf-d9e1-49fb-9811-8cc98041aa79" /> ## Context Uploading large files currently OOMs the server: every upload resolver buffers the whole file in memory (`streamToBuffer`) before writing it to storage. This PR is the first of a series introducing direct client-to-storage uploads. It adds the server-side endpoints and driver support only — it is non-breaking and nothing consumes the new flow yet. Follow-up PRs will migrate the frontend upload paths, add a stale-pending-file cleanup cron, and cap the legacy buffered resolvers. ## What it does **New upload flow (initiate → PUT → confirm):** - `createFileUpload(filename, size, fileFolder, fieldMetadataId?)` validates the request (folder allowlist: `FilesField`/`Workflow`, max size, extension-derived mime type), creates the file record in a new `PENDING` status, and returns an upload target: - **S3 with presign enabled** → a presigned PUT URL with `Content-Type`/`Content-Length` pinned in the signature, so the client uploads straight to the bucket; - **local storage, or S3 without presign** → a token-authenticated streaming endpoint on the server (`PUT /file-upload/:id?token=…`, new `FILE_UPLOAD` JWT type) that pipes the request body to the storage driver with constant memory usage and a declared-size cap. - `completeFileUpload(fileId)` verifies the bytes actually landed in storage (HEAD + size match against the declared size) and flips the record to `UPLOADED`. Idempotent. **Pending lifecycle safety:** - New `status` column on `core.file` (`PENDING`/`UPLOADED`, default `UPLOADED` so all existing rows and the legacy upload path are unaffected) + fast instance command. - Files are refused by the serving endpoints and by FILES-field sync while `PENDING`. **Driver support (both drivers):** - `getPresignedUploadUrl` (S3: presigned PUT; local: `null` → server-endpoint fallback) - `writeFileStream` (local: `fs` pipeline with the existing symlink/containment hardening, partial-file cleanup on error; S3: `@aws-sdk/lib-storage` `Upload` for bounded-memory streaming) - `getFileMetadata` (HEAD/stat for confirm-time verification) ## Tests - `file-upload.service.spec.ts`: initiate validation (folder allowlist, size), presigned vs fallback target, confirm verification (missing object, size mismatch, happy path, idempotency) - `local.driver.spec.ts`: `writeFileStream` (content, symlink rejection, partial-file cleanup on stream error), `getFileMetadata` - `s3.driver.spec.ts`: `getPresignedUploadUrl` (disabled → null, PUT command with signed content-type/content-length) - `direct-file-upload.integration-spec.ts`: full end-to-end flow against the local driver (initiate → PUT → complete → download), plus error paths (complete without upload, oversized PUT → 413, invalid token → 403, unsupported folder, size above max) ## Notes for reviewers - The upload-size ceiling for direct uploads is `settings.storage.maxDirectUploadFileSize` (1GB), separate from the 10MB `maxFileSize` used for pictures. - Since content can't be sniffed before it reaches storage, the mime type is derived from the file extension (with the existing `TWENTY_MIME_POLICY` override) and unknown extensions fall back to `application/octet-stream`; the serving path already forces `Content-Disposition: attachment` for anything not on the inline-safe allowlist. - Self-hosters using S3 presign will need a bucket CORS policy allowing `PUT` from the frontend origin (config variable description updated). https://claude.ai/code/session_015UH8KWmsB9zdYaog8MFG1d --- _Generated by [Claude Code](https://claude.ai/code/session_015UH8KWmsB9zdYaog8MFG1d)_ <!-- This is an auto-generated description by cubic. --> <a href="https://cubic.dev/pr/twentyhq/twenty/pull/22449?utm_source=github" target="_blank" rel="noopener noreferrer" data-no-image-dialog="true"><picture><source media="(prefers-color-scheme: dark)" srcset="https://www.cubic.dev/buttons/review-in-cubic-dark.svg"><source media="(prefers-color-scheme: light)" srcset="https://www.cubic.dev/buttons/review-in-cubic-light.svg"><img alt="Review in cubic" src="https://www.cubic.dev/buttons/review-in-cubic-dark.svg"></picture></a> <!-- End of auto-generated description by cubic. -->
This commit is contained in:
@@ -12,12 +12,19 @@ import {
|
||||
UpdateDateColumn,
|
||||
} from 'typeorm';
|
||||
|
||||
import { ADD_STATUS_TO_FILE_UPGRADE_COMMAND_NAME } from 'src/database/commands/upgrade-version-command/2-19/add-status-to-file-upgrade-command-name.constant';
|
||||
import { ApplicationEntity } from 'src/engine/core-modules/application/application.entity';
|
||||
import { FileSettings } from 'src/engine/core-modules/file/types/file-settings.types';
|
||||
import {
|
||||
FILE_STATUS,
|
||||
FileStatus,
|
||||
} from 'src/engine/core-modules/file/types/file-status.types';
|
||||
import { WasIntroducedInUpgrade } from 'src/engine/core-modules/upgrade/decorators/was-introduced-in-upgrade.decorator';
|
||||
import { WorkspaceRelatedEntity } from 'src/engine/workspace-manager/types/workspace-related-entity';
|
||||
|
||||
@Entity('file')
|
||||
@Index('IDX_FILE_WORKSPACE_ID', ['workspaceId'])
|
||||
@Index('IDX_FILE_STATUS', ['status'])
|
||||
@Unique('IDX_APPLICATION_PATH_WORKSPACE_ID_APPLICATION_ID_UNIQUE', [
|
||||
'workspaceId',
|
||||
'applicationId',
|
||||
@@ -63,4 +70,15 @@ export class FileEntity extends WorkspaceRelatedEntity {
|
||||
default: 'application/octet-stream',
|
||||
})
|
||||
mimeType: string;
|
||||
|
||||
@WasIntroducedInUpgrade({
|
||||
upgradeCommandName: ADD_STATUS_TO_FILE_UPGRADE_COMMAND_NAME,
|
||||
})
|
||||
@Column({
|
||||
type: 'enum',
|
||||
enum: Object.values(FILE_STATUS),
|
||||
nullable: false,
|
||||
default: FILE_STATUS.UPLOADED,
|
||||
})
|
||||
status: FileStatus;
|
||||
}
|
||||
|
||||
+44
@@ -0,0 +1,44 @@
|
||||
import {
|
||||
Controller,
|
||||
Param,
|
||||
Put,
|
||||
Req,
|
||||
Res,
|
||||
UseFilters,
|
||||
UseGuards,
|
||||
} from '@nestjs/common';
|
||||
|
||||
import { Request, Response } from 'express';
|
||||
|
||||
import { FileUploadApiExceptionFilter } from 'src/engine/core-modules/file/file-upload/filters/file-upload-api-exception.filter';
|
||||
import { FileUploadTokenGuard } from 'src/engine/core-modules/file/file-upload/guards/file-upload-token.guard';
|
||||
import { FileUploadService } from 'src/engine/core-modules/file/file-upload/services/file-upload.service';
|
||||
import { NoPermissionGuard } from 'src/engine/guards/no-permission.guard';
|
||||
|
||||
@Controller()
|
||||
@UseFilters(FileUploadApiExceptionFilter)
|
||||
export class FileUploadController {
|
||||
constructor(private readonly fileUploadService: FileUploadService) {}
|
||||
|
||||
// Streaming target for direct uploads when storage has no presigned upload
|
||||
// support (local driver, or S3 without presign enabled). The body is piped
|
||||
// to the storage driver without ever being buffered in memory.
|
||||
@Put('file-upload/:id')
|
||||
@UseGuards(FileUploadTokenGuard, NoPermissionGuard)
|
||||
async uploadFileById(
|
||||
@Req() req: Request,
|
||||
@Res() res: Response,
|
||||
@Param('id') fileId: string,
|
||||
) {
|
||||
// oxlint-disable-next-line typescript/no-explicit-any
|
||||
const workspaceId = (req as any)?.workspaceId;
|
||||
|
||||
await this.fileUploadService.receiveFileStream({
|
||||
workspaceId,
|
||||
fileId,
|
||||
stream: req,
|
||||
});
|
||||
|
||||
res.status(204).send();
|
||||
}
|
||||
}
|
||||
+19
@@ -0,0 +1,19 @@
|
||||
import { Field, ObjectType } from '@nestjs/graphql';
|
||||
|
||||
import { UUIDScalarType } from 'src/engine/api/graphql/workspace-schema-builder/graphql-types/scalars';
|
||||
|
||||
@ObjectType('FileUploadTarget')
|
||||
export class FileUploadTargetDTO {
|
||||
@Field(() => UUIDScalarType)
|
||||
fileId: string;
|
||||
|
||||
@Field()
|
||||
uploadUrl: string;
|
||||
|
||||
// Content-Type header the client must send when uploading to uploadUrl
|
||||
@Field()
|
||||
contentType: string;
|
||||
|
||||
@Field(() => Date, { nullable: false })
|
||||
expiresAt: Date;
|
||||
}
|
||||
+23
@@ -0,0 +1,23 @@
|
||||
import { type MessageDescriptor } from '@lingui/core';
|
||||
|
||||
import { CustomException } from 'src/utils/custom-exception';
|
||||
|
||||
export enum FileUploadExceptionCode {
|
||||
BAD_REQUEST = 'BAD_REQUEST',
|
||||
FILE_NOT_FOUND = 'FILE_NOT_FOUND',
|
||||
FILE_NOT_UPLOADED = 'FILE_NOT_UPLOADED',
|
||||
FILE_SIZE_MISMATCH = 'FILE_SIZE_MISMATCH',
|
||||
FILE_TOO_LARGE = 'FILE_TOO_LARGE',
|
||||
}
|
||||
|
||||
export class FileUploadException extends CustomException<FileUploadExceptionCode> {
|
||||
constructor(
|
||||
message: string,
|
||||
code: FileUploadExceptionCode,
|
||||
{ userFriendlyMessage }: { userFriendlyMessage: MessageDescriptor },
|
||||
) {
|
||||
super(message, code, {
|
||||
userFriendlyMessage,
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
import { Module } from '@nestjs/common';
|
||||
import { TypeOrmModule } from '@nestjs/typeorm';
|
||||
|
||||
import { ApplicationEntity } from 'src/engine/core-modules/application/application.entity';
|
||||
import { ApplicationModule } from 'src/engine/core-modules/application/application.module';
|
||||
import { FileStorageModule } from 'src/engine/core-modules/file-storage/file-storage.module';
|
||||
import { FileEntity } from 'src/engine/core-modules/file/entities/file.entity';
|
||||
import { FileUploadController } from 'src/engine/core-modules/file/file-upload/controllers/file-upload.controller';
|
||||
import { FileUploadTokenGuard } from 'src/engine/core-modules/file/file-upload/guards/file-upload-token.guard';
|
||||
import { FileUploadResolver } from 'src/engine/core-modules/file/file-upload/resolvers/file-upload.resolver';
|
||||
import { FileUploadService } from 'src/engine/core-modules/file/file-upload/services/file-upload.service';
|
||||
import { FileUrlModule } from 'src/engine/core-modules/file/file-url/file-url.module';
|
||||
import { JwtModule } from 'src/engine/core-modules/jwt/jwt.module';
|
||||
import { FieldMetadataEntity } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity';
|
||||
import { PermissionsModule } from 'src/engine/metadata-modules/permissions/permissions.module';
|
||||
import { provideWorkspaceScopedRepository } from 'src/engine/twenty-orm/workspace-scoped-repository/provide-workspace-scoped-repository';
|
||||
|
||||
@Module({
|
||||
imports: [
|
||||
JwtModule,
|
||||
TypeOrmModule.forFeature([
|
||||
FileEntity,
|
||||
ApplicationEntity,
|
||||
FieldMetadataEntity,
|
||||
]),
|
||||
PermissionsModule,
|
||||
FileStorageModule,
|
||||
FileUrlModule,
|
||||
ApplicationModule,
|
||||
],
|
||||
providers: [
|
||||
FileUploadService,
|
||||
FileUploadResolver,
|
||||
FileUploadTokenGuard,
|
||||
provideWorkspaceScopedRepository(FileEntity),
|
||||
],
|
||||
exports: [FileUploadService],
|
||||
controllers: [FileUploadController],
|
||||
})
|
||||
export class FileUploadModule {}
|
||||
+54
@@ -0,0 +1,54 @@
|
||||
import {
|
||||
type ArgumentsHost,
|
||||
Catch,
|
||||
type ExceptionFilter,
|
||||
} from '@nestjs/common';
|
||||
|
||||
import { type Response } from 'express';
|
||||
|
||||
import { HttpExceptionHandlerService } from 'src/engine/core-modules/exception-handler/http-exception-handler.service';
|
||||
import {
|
||||
FileUploadException,
|
||||
FileUploadExceptionCode,
|
||||
} from 'src/engine/core-modules/file/file-upload/file-upload.exception';
|
||||
|
||||
@Catch(FileUploadException)
|
||||
export class FileUploadApiExceptionFilter implements ExceptionFilter {
|
||||
constructor(
|
||||
private readonly httpExceptionHandlerService: HttpExceptionHandlerService,
|
||||
) {}
|
||||
|
||||
catch(exception: FileUploadException, host: ArgumentsHost) {
|
||||
const ctx = host.switchToHttp();
|
||||
const response = ctx.getResponse<Response>();
|
||||
|
||||
switch (exception.code) {
|
||||
case FileUploadExceptionCode.FILE_NOT_FOUND:
|
||||
return this.httpExceptionHandlerService.handleError(
|
||||
exception,
|
||||
response,
|
||||
404,
|
||||
);
|
||||
case FileUploadExceptionCode.FILE_TOO_LARGE:
|
||||
return this.httpExceptionHandlerService.handleError(
|
||||
exception,
|
||||
response,
|
||||
413,
|
||||
);
|
||||
case FileUploadExceptionCode.BAD_REQUEST:
|
||||
case FileUploadExceptionCode.FILE_NOT_UPLOADED:
|
||||
case FileUploadExceptionCode.FILE_SIZE_MISMATCH:
|
||||
return this.httpExceptionHandlerService.handleError(
|
||||
exception,
|
||||
response,
|
||||
400,
|
||||
);
|
||||
default:
|
||||
return this.httpExceptionHandlerService.handleError(
|
||||
exception,
|
||||
response,
|
||||
500,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
+42
@@ -0,0 +1,42 @@
|
||||
import { CanActivate, ExecutionContext, Injectable } from '@nestjs/common';
|
||||
|
||||
import { FileUploadTokenJwtPayload } from 'src/engine/core-modules/auth/types/file-upload-token-jwt-payload.type';
|
||||
import { JwtTokenTypeEnum } from 'src/engine/core-modules/auth/types/jwt-token-type.enum';
|
||||
import { JwtWrapperService } from 'src/engine/core-modules/jwt/services/jwt-wrapper.service';
|
||||
|
||||
@Injectable()
|
||||
export class FileUploadTokenGuard implements CanActivate {
|
||||
constructor(private readonly jwtWrapperService: JwtWrapperService) {}
|
||||
|
||||
async canActivate(context: ExecutionContext): Promise<boolean> {
|
||||
const request = context.switchToHttp().getRequest();
|
||||
const fileId = request.params.id;
|
||||
const uploadToken = request.query.token;
|
||||
|
||||
if (!uploadToken) {
|
||||
return false;
|
||||
}
|
||||
|
||||
let payload: FileUploadTokenJwtPayload;
|
||||
|
||||
try {
|
||||
payload = await this.jwtWrapperService.verifyJwtToken(uploadToken);
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
|
||||
// A FILE (download) token also carries workspaceId + fileId: reject
|
||||
// anything that is not explicitly an upload token.
|
||||
if (payload.type !== JwtTokenTypeEnum.FILE_UPLOAD) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if (!payload.workspaceId || payload.fileId !== fileId) {
|
||||
return false;
|
||||
}
|
||||
|
||||
request.workspaceId = payload.workspaceId;
|
||||
|
||||
return true;
|
||||
}
|
||||
}
|
||||
+68
@@ -0,0 +1,68 @@
|
||||
import { UseFilters, UseGuards, UsePipes } from '@nestjs/common';
|
||||
import { Args, Mutation } from '@nestjs/graphql';
|
||||
|
||||
import { PermissionFlagType } from 'twenty-shared/constants';
|
||||
import { FileFolder } from 'twenty-shared/types';
|
||||
|
||||
import { MetadataResolver } from 'src/engine/api/graphql/graphql-config/decorators/metadata-resolver.decorator';
|
||||
import { FileWithSignedUrlDTO } from 'src/engine/core-modules/file/dtos/file-with-sign-url.dto';
|
||||
import { FileUploadTargetDTO } from 'src/engine/core-modules/file/file-upload/dtos/file-upload-target.dto';
|
||||
import { FileUploadService } from 'src/engine/core-modules/file/file-upload/services/file-upload.service';
|
||||
import { PreventNestToAutoLogGraphqlErrorsFilter } from 'src/engine/core-modules/graphql/filters/prevent-nest-to-auto-log-graphql-errors.filter';
|
||||
import { ResolverValidationPipe } from 'src/engine/core-modules/graphql/pipes/resolver-validation.pipe';
|
||||
import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity';
|
||||
import { AuthWorkspace } from 'src/engine/decorators/auth/auth-workspace.decorator';
|
||||
import { SettingsPermissionGuard } from 'src/engine/guards/settings-permission.guard';
|
||||
import { WorkspaceAuthGuard } from 'src/engine/guards/workspace-auth.guard';
|
||||
|
||||
@UseGuards(WorkspaceAuthGuard)
|
||||
@UsePipes(ResolverValidationPipe)
|
||||
@UseFilters(PreventNestToAutoLogGraphqlErrorsFilter)
|
||||
@MetadataResolver()
|
||||
export class FileUploadResolver {
|
||||
constructor(private readonly fileUploadService: FileUploadService) {}
|
||||
|
||||
@Mutation(() => FileUploadTargetDTO)
|
||||
@UseGuards(SettingsPermissionGuard(PermissionFlagType.UPLOAD_FILE))
|
||||
async createFileUpload(
|
||||
@AuthWorkspace()
|
||||
{ id: workspaceId }: WorkspaceEntity,
|
||||
@Args({ name: 'filename', type: () => String })
|
||||
filename: string,
|
||||
@Args({ name: 'size', type: () => Number })
|
||||
size: number,
|
||||
@Args({ name: 'fileFolder', type: () => FileFolder })
|
||||
fileFolder: FileFolder,
|
||||
@Args({ name: 'fieldMetadataId', type: () => String, nullable: true })
|
||||
fieldMetadataId?: string,
|
||||
@Args({
|
||||
name: 'fieldMetadataUniversalIdentifier',
|
||||
type: () => String,
|
||||
nullable: true,
|
||||
})
|
||||
fieldMetadataUniversalIdentifier?: string,
|
||||
): Promise<FileUploadTargetDTO> {
|
||||
return await this.fileUploadService.createFileUpload({
|
||||
workspaceId,
|
||||
filename,
|
||||
size,
|
||||
fileFolder,
|
||||
fieldMetadataId,
|
||||
fieldMetadataUniversalIdentifier,
|
||||
});
|
||||
}
|
||||
|
||||
@Mutation(() => FileWithSignedUrlDTO)
|
||||
@UseGuards(SettingsPermissionGuard(PermissionFlagType.UPLOAD_FILE))
|
||||
async completeFileUpload(
|
||||
@AuthWorkspace()
|
||||
{ id: workspaceId }: WorkspaceEntity,
|
||||
@Args({ name: 'fileId', type: () => String })
|
||||
fileId: string,
|
||||
): Promise<FileWithSignedUrlDTO> {
|
||||
return await this.fileUploadService.completeFileUpload({
|
||||
workspaceId,
|
||||
fileId,
|
||||
});
|
||||
}
|
||||
}
|
||||
+309
@@ -0,0 +1,309 @@
|
||||
import { Test, type TestingModule } from '@nestjs/testing';
|
||||
import { getRepositoryToken } from '@nestjs/typeorm';
|
||||
|
||||
import { FileFolder } from 'twenty-shared/types';
|
||||
|
||||
import { ApplicationEntity } from 'src/engine/core-modules/application/application.entity';
|
||||
import { ApplicationService } from 'src/engine/core-modules/application/application.service';
|
||||
import { FileStorageService } from 'src/engine/core-modules/file-storage/file-storage.service';
|
||||
import { FileEntity } from 'src/engine/core-modules/file/entities/file.entity';
|
||||
import {
|
||||
FileUploadException,
|
||||
FileUploadExceptionCode,
|
||||
} from 'src/engine/core-modules/file/file-upload/file-upload.exception';
|
||||
import { FileUploadService } from 'src/engine/core-modules/file/file-upload/services/file-upload.service';
|
||||
import { FileUrlService } from 'src/engine/core-modules/file/file-url/file-url.service';
|
||||
import { FILE_STATUS } from 'src/engine/core-modules/file/types/file-status.types';
|
||||
import { JwtWrapperService } from 'src/engine/core-modules/jwt/services/jwt-wrapper.service';
|
||||
import { TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service';
|
||||
import { FieldMetadataEntity } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity';
|
||||
import { getWorkspaceScopedRepositoryToken } from 'src/engine/twenty-orm/workspace-scoped-repository/get-workspace-scoped-repository-token.util';
|
||||
|
||||
jest.mock('uuid', () => ({
|
||||
v4: jest.fn(() => 'mocked-file-id'),
|
||||
}));
|
||||
|
||||
describe('FileUploadService', () => {
|
||||
let service: FileUploadService;
|
||||
|
||||
const fileStorageService = {
|
||||
createPendingFile: jest.fn(),
|
||||
getPresignedUploadUrl: jest.fn(),
|
||||
getFileMetadata: jest.fn(),
|
||||
writeFileStream: jest.fn(),
|
||||
};
|
||||
|
||||
const fileUrlService = {
|
||||
signFileByIdUrl: jest.fn().mockResolvedValue('https://signed-url'),
|
||||
};
|
||||
|
||||
const jwtWrapperService = {
|
||||
signAsyncOrThrow: jest.fn().mockResolvedValue('upload-token'),
|
||||
};
|
||||
|
||||
const twentyConfigService = {
|
||||
get: jest.fn((key: string) => {
|
||||
if (key === 'STORAGE_S3_PRESIGNED_URL_EXPIRES_IN') {
|
||||
return 900;
|
||||
}
|
||||
if (key === 'SERVER_URL') {
|
||||
return 'https://server.tld';
|
||||
}
|
||||
|
||||
return undefined;
|
||||
}),
|
||||
};
|
||||
|
||||
const applicationService = {
|
||||
findWorkspaceTwentyStandardAndCustomApplicationOrThrow: jest
|
||||
.fn()
|
||||
.mockResolvedValue({
|
||||
workspaceCustomFlatApplication: {
|
||||
universalIdentifier: 'custom-app-uid',
|
||||
},
|
||||
}),
|
||||
};
|
||||
|
||||
const applicationRepository = {
|
||||
findOneOrFail: jest.fn().mockResolvedValue({
|
||||
id: 'application-id',
|
||||
universalIdentifier: 'application-uid',
|
||||
}),
|
||||
};
|
||||
|
||||
const fieldMetadataRepository = {
|
||||
findOneOrFail: jest.fn().mockResolvedValue({
|
||||
applicationId: 'application-id',
|
||||
universalIdentifier: 'field-metadata-uid',
|
||||
}),
|
||||
};
|
||||
|
||||
const fileRepository = {
|
||||
findOne: jest.fn(),
|
||||
update: jest.fn(),
|
||||
};
|
||||
|
||||
beforeEach(async () => {
|
||||
jest.clearAllMocks();
|
||||
|
||||
const module: TestingModule = await Test.createTestingModule({
|
||||
providers: [
|
||||
FileUploadService,
|
||||
{ provide: FileStorageService, useValue: fileStorageService },
|
||||
{ provide: FileUrlService, useValue: fileUrlService },
|
||||
{ provide: JwtWrapperService, useValue: jwtWrapperService },
|
||||
{ provide: TwentyConfigService, useValue: twentyConfigService },
|
||||
{ provide: ApplicationService, useValue: applicationService },
|
||||
{
|
||||
provide: getRepositoryToken(ApplicationEntity),
|
||||
useValue: applicationRepository,
|
||||
},
|
||||
{
|
||||
provide: getRepositoryToken(FieldMetadataEntity),
|
||||
useValue: fieldMetadataRepository,
|
||||
},
|
||||
{
|
||||
provide: getWorkspaceScopedRepositoryToken(FileEntity),
|
||||
useValue: fileRepository,
|
||||
},
|
||||
],
|
||||
}).compile();
|
||||
|
||||
service = module.get<FileUploadService>(FileUploadService);
|
||||
});
|
||||
|
||||
describe('createFileUpload', () => {
|
||||
it('should reject file folders without direct upload support', async () => {
|
||||
await expect(
|
||||
service.createFileUpload({
|
||||
workspaceId: 'workspace-id',
|
||||
filename: 'document.pdf',
|
||||
size: 1024,
|
||||
fileFolder: FileFolder.CorePicture,
|
||||
}),
|
||||
).rejects.toThrow(FileUploadException);
|
||||
});
|
||||
|
||||
it('should reject an invalid declared size', async () => {
|
||||
await expect(
|
||||
service.createFileUpload({
|
||||
workspaceId: 'workspace-id',
|
||||
filename: 'document.pdf',
|
||||
size: 0,
|
||||
fileFolder: FileFolder.FilesField,
|
||||
fieldMetadataId: 'field-metadata-id',
|
||||
}),
|
||||
).rejects.toThrow(FileUploadException);
|
||||
});
|
||||
|
||||
it('should create a PENDING file and return the presigned url when storage supports it', async () => {
|
||||
fileStorageService.getPresignedUploadUrl.mockResolvedValueOnce(
|
||||
'https://bucket/presigned-put',
|
||||
);
|
||||
|
||||
const result = await service.createFileUpload({
|
||||
workspaceId: 'workspace-id',
|
||||
filename: 'document.pdf',
|
||||
size: 1024,
|
||||
fileFolder: FileFolder.FilesField,
|
||||
fieldMetadataId: 'field-metadata-id',
|
||||
});
|
||||
|
||||
expect(fileStorageService.createPendingFile).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
fileId: 'mocked-file-id',
|
||||
size: 1024,
|
||||
mimeType: 'application/pdf',
|
||||
resourcePath: 'field-metadata-uid/mocked-file-id.pdf',
|
||||
settings: { isTemporaryFile: true, toDelete: false },
|
||||
}),
|
||||
);
|
||||
expect(result.uploadUrl).toBe('https://bucket/presigned-put');
|
||||
expect(result.contentType).toBe('application/pdf');
|
||||
expect(result.fileId).toBe('mocked-file-id');
|
||||
});
|
||||
|
||||
it('should fall back to the server streaming endpoint when presign is unavailable', async () => {
|
||||
fileStorageService.getPresignedUploadUrl.mockResolvedValueOnce(null);
|
||||
|
||||
const result = await service.createFileUpload({
|
||||
workspaceId: 'workspace-id',
|
||||
filename: 'archive.zip',
|
||||
size: 2048,
|
||||
fileFolder: FileFolder.Workflow,
|
||||
});
|
||||
|
||||
expect(jwtWrapperService.signAsyncOrThrow).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
workspaceId: 'workspace-id',
|
||||
fileId: 'mocked-file-id',
|
||||
}),
|
||||
{ expiresIn: 900 },
|
||||
);
|
||||
expect(result.uploadUrl).toBe(
|
||||
'https://server.tld/file-upload/mocked-file-id?token=upload-token',
|
||||
);
|
||||
expect(result.contentType).toBe('application/octet-stream');
|
||||
});
|
||||
});
|
||||
|
||||
describe('completeFileUpload', () => {
|
||||
const pendingFile = {
|
||||
id: 'file-id',
|
||||
path: 'files-field/field-metadata-uid/file-id.pdf',
|
||||
size: 1024,
|
||||
applicationId: 'application-id',
|
||||
mimeType: 'application/pdf',
|
||||
status: FILE_STATUS.PENDING,
|
||||
settings: { isTemporaryFile: true, toDelete: false },
|
||||
createdAt: new Date(),
|
||||
};
|
||||
|
||||
it('should throw when the file record does not exist', async () => {
|
||||
fileRepository.findOne.mockResolvedValueOnce(null);
|
||||
|
||||
await expect(
|
||||
service.completeFileUpload({
|
||||
workspaceId: 'workspace-id',
|
||||
fileId: 'file-id',
|
||||
}),
|
||||
).rejects.toThrow(FileUploadException);
|
||||
});
|
||||
|
||||
it('should throw when the bytes are not in storage yet', async () => {
|
||||
fileRepository.findOne.mockResolvedValueOnce(pendingFile);
|
||||
fileStorageService.getFileMetadata.mockResolvedValueOnce(null);
|
||||
|
||||
await expect(
|
||||
service.completeFileUpload({
|
||||
workspaceId: 'workspace-id',
|
||||
fileId: 'file-id',
|
||||
}),
|
||||
).rejects.toMatchObject({
|
||||
code: FileUploadExceptionCode.FILE_NOT_UPLOADED,
|
||||
});
|
||||
expect(fileRepository.update).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('should throw when the stored size does not match the declared size', async () => {
|
||||
fileRepository.findOne.mockResolvedValueOnce(pendingFile);
|
||||
fileStorageService.getFileMetadata.mockResolvedValueOnce({ size: 999 });
|
||||
|
||||
await expect(
|
||||
service.completeFileUpload({
|
||||
workspaceId: 'workspace-id',
|
||||
fileId: 'file-id',
|
||||
}),
|
||||
).rejects.toMatchObject({
|
||||
code: FileUploadExceptionCode.FILE_SIZE_MISMATCH,
|
||||
});
|
||||
expect(fileRepository.update).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('should flip the file to UPLOADED when the stored size matches', async () => {
|
||||
fileRepository.findOne.mockResolvedValueOnce(pendingFile);
|
||||
fileStorageService.getFileMetadata.mockResolvedValueOnce({ size: 1024 });
|
||||
|
||||
const result = await service.completeFileUpload({
|
||||
workspaceId: 'workspace-id',
|
||||
fileId: 'file-id',
|
||||
});
|
||||
|
||||
expect(fileRepository.update).toHaveBeenCalledWith(
|
||||
'workspace-id',
|
||||
{ id: 'file-id' },
|
||||
{ status: FILE_STATUS.UPLOADED },
|
||||
);
|
||||
expect(result.url).toBe('https://signed-url');
|
||||
});
|
||||
|
||||
it('should be idempotent when the file is already UPLOADED', async () => {
|
||||
fileRepository.findOne.mockResolvedValueOnce({
|
||||
...pendingFile,
|
||||
status: FILE_STATUS.UPLOADED,
|
||||
});
|
||||
|
||||
const result = await service.completeFileUpload({
|
||||
workspaceId: 'workspace-id',
|
||||
fileId: 'file-id',
|
||||
});
|
||||
|
||||
expect(fileStorageService.getFileMetadata).not.toHaveBeenCalled();
|
||||
expect(fileRepository.update).not.toHaveBeenCalled();
|
||||
expect(result.url).toBe('https://signed-url');
|
||||
});
|
||||
|
||||
it('should refuse confirming files that already left the upload flow', async () => {
|
||||
fileRepository.findOne.mockResolvedValueOnce({
|
||||
...pendingFile,
|
||||
status: FILE_STATUS.UPLOADED,
|
||||
settings: { isTemporaryFile: false, toDelete: false },
|
||||
});
|
||||
|
||||
await expect(
|
||||
service.completeFileUpload({
|
||||
workspaceId: 'workspace-id',
|
||||
fileId: 'file-id',
|
||||
}),
|
||||
).rejects.toMatchObject({
|
||||
code: FileUploadExceptionCode.BAD_REQUEST,
|
||||
});
|
||||
});
|
||||
|
||||
it('should refuse files outside direct-upload folders', async () => {
|
||||
fileRepository.findOne.mockResolvedValueOnce({
|
||||
...pendingFile,
|
||||
path: 'core-picture/file-id.png',
|
||||
});
|
||||
|
||||
await expect(
|
||||
service.completeFileUpload({
|
||||
workspaceId: 'workspace-id',
|
||||
fileId: 'file-id',
|
||||
}),
|
||||
).rejects.toMatchObject({
|
||||
code: FileUploadExceptionCode.FILE_NOT_FOUND,
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
+511
@@ -0,0 +1,511 @@
|
||||
import { Injectable } from '@nestjs/common';
|
||||
import { InjectRepository } from '@nestjs/typeorm';
|
||||
|
||||
import { type Readable, Transform } from 'stream';
|
||||
import { pipeline } from 'stream/promises';
|
||||
|
||||
import { msg } from '@lingui/core/macro';
|
||||
import { isNonEmptyString } from '@sniptt/guards';
|
||||
import bytes from 'bytes';
|
||||
import { lookup } from 'mrmime';
|
||||
import { FileFolder } from 'twenty-shared/types';
|
||||
import { isDefined } from 'twenty-shared/utils';
|
||||
import { Repository } from 'typeorm';
|
||||
import { v4 } from 'uuid';
|
||||
|
||||
import { settings } from 'src/engine/constants/settings';
|
||||
import { ApplicationEntity } from 'src/engine/core-modules/application/application.entity';
|
||||
import { ApplicationService } from 'src/engine/core-modules/application/application.service';
|
||||
import { FileUploadTokenJwtPayload } from 'src/engine/core-modules/auth/types/file-upload-token-jwt-payload.type';
|
||||
import { JwtTokenTypeEnum } from 'src/engine/core-modules/auth/types/jwt-token-type.enum';
|
||||
import { FileStorageService } from 'src/engine/core-modules/file-storage/file-storage.service';
|
||||
import { TWENTY_MIME_POLICY } from 'src/engine/core-modules/file/constants/twenty-mime-policy.constant';
|
||||
import { FileWithSignedUrlDTO } from 'src/engine/core-modules/file/dtos/file-with-sign-url.dto';
|
||||
import { FileEntity } from 'src/engine/core-modules/file/entities/file.entity';
|
||||
import { FileUploadTargetDTO } from 'src/engine/core-modules/file/file-upload/dtos/file-upload-target.dto';
|
||||
import {
|
||||
FileUploadException,
|
||||
FileUploadExceptionCode,
|
||||
} from 'src/engine/core-modules/file/file-upload/file-upload.exception';
|
||||
import { FileUrlService } from 'src/engine/core-modules/file/file-url/file-url.service';
|
||||
import { FILE_STATUS } from 'src/engine/core-modules/file/types/file-status.types';
|
||||
import { buildFileInfo } from 'src/engine/core-modules/file/utils/build-file-info.utils';
|
||||
import { removeFileFolderFromFileEntityPath } from 'src/engine/core-modules/file/utils/remove-file-folder-from-file-entity-path.utils';
|
||||
import { JwtWrapperService } from 'src/engine/core-modules/jwt/services/jwt-wrapper.service';
|
||||
import { TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service';
|
||||
import { FieldMetadataEntity } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity';
|
||||
import { InjectWorkspaceScopedRepository } from 'src/engine/twenty-orm/workspace-scoped-repository/inject-workspace-scoped-repository.decorator';
|
||||
import { WorkspaceScopedRepository } from 'src/engine/twenty-orm/workspace-scoped-repository/workspace-scoped-repository';
|
||||
|
||||
export const DIRECT_UPLOAD_FILE_FOLDERS = [
|
||||
FileFolder.FilesField,
|
||||
FileFolder.Workflow,
|
||||
] as const;
|
||||
|
||||
@Injectable()
|
||||
export class FileUploadService {
|
||||
constructor(
|
||||
private readonly fileStorageService: FileStorageService,
|
||||
private readonly fileUrlService: FileUrlService,
|
||||
private readonly jwtWrapperService: JwtWrapperService,
|
||||
private readonly twentyConfigService: TwentyConfigService,
|
||||
private readonly applicationService: ApplicationService,
|
||||
@InjectRepository(ApplicationEntity)
|
||||
private readonly applicationRepository: Repository<ApplicationEntity>,
|
||||
@InjectRepository(FieldMetadataEntity)
|
||||
private readonly fieldMetadataRepository: Repository<FieldMetadataEntity>,
|
||||
@InjectWorkspaceScopedRepository(FileEntity)
|
||||
private readonly fileRepository: WorkspaceScopedRepository<FileEntity>,
|
||||
) {}
|
||||
|
||||
async createFileUpload({
|
||||
workspaceId,
|
||||
filename,
|
||||
size,
|
||||
fileFolder,
|
||||
fieldMetadataId,
|
||||
fieldMetadataUniversalIdentifier,
|
||||
}: {
|
||||
workspaceId: string;
|
||||
filename: string;
|
||||
size: number;
|
||||
fileFolder: FileFolder;
|
||||
fieldMetadataId?: string;
|
||||
fieldMetadataUniversalIdentifier?: string;
|
||||
}): Promise<FileUploadTargetDTO> {
|
||||
if (
|
||||
!DIRECT_UPLOAD_FILE_FOLDERS.includes(
|
||||
fileFolder as (typeof DIRECT_UPLOAD_FILE_FOLDERS)[number],
|
||||
)
|
||||
) {
|
||||
throw new FileUploadException(
|
||||
`Direct upload is not supported for file folder ${fileFolder}`,
|
||||
FileUploadExceptionCode.BAD_REQUEST,
|
||||
{
|
||||
userFriendlyMessage: msg`Direct upload is not supported for this file type.`,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
const maxFileSize = bytes(settings.storage.maxDirectUploadFileSize) ?? 0;
|
||||
|
||||
if (!Number.isInteger(size) || size <= 0 || size > maxFileSize) {
|
||||
throw new FileUploadException(
|
||||
`Invalid file size ${size} (max ${maxFileSize} bytes)`,
|
||||
FileUploadExceptionCode.FILE_TOO_LARGE,
|
||||
{
|
||||
userFriendlyMessage: msg`The file is empty or exceeds the maximum allowed size.`,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
const { ext } = buildFileInfo(filename);
|
||||
// The file content cannot be sniffed before it reaches storage, so the
|
||||
// mime type is derived from the extension only. Anything unknown is
|
||||
// stored as octet-stream, and the serving path already forces
|
||||
// Content-Disposition: attachment for non-inline-safe mime types.
|
||||
const mimeType =
|
||||
TWENTY_MIME_POLICY[ext] ??
|
||||
(isNonEmptyString(ext) ? lookup(ext) : undefined) ??
|
||||
'application/octet-stream';
|
||||
|
||||
const fileId = v4();
|
||||
const name = `${fileId}${isNonEmptyString(ext) ? `.${ext}` : ''}`;
|
||||
|
||||
const { applicationUniversalIdentifier, resourcePath } =
|
||||
await this.resolveUploadLocation({
|
||||
workspaceId,
|
||||
fileFolder,
|
||||
name,
|
||||
fieldMetadataId,
|
||||
fieldMetadataUniversalIdentifier,
|
||||
});
|
||||
|
||||
await this.fileStorageService.createPendingFile({
|
||||
fileFolder,
|
||||
applicationUniversalIdentifier,
|
||||
workspaceId,
|
||||
resourcePath,
|
||||
fileId,
|
||||
size,
|
||||
mimeType,
|
||||
settings: {
|
||||
isTemporaryFile: true,
|
||||
toDelete: false,
|
||||
},
|
||||
});
|
||||
|
||||
const expiresInSeconds = this.twentyConfigService.get(
|
||||
'STORAGE_S3_PRESIGNED_URL_EXPIRES_IN',
|
||||
);
|
||||
const expiresAt = new Date(Date.now() + expiresInSeconds * 1000);
|
||||
|
||||
const presignedUploadUrl =
|
||||
await this.fileStorageService.getPresignedUploadUrl({
|
||||
fileFolder,
|
||||
applicationUniversalIdentifier,
|
||||
workspaceId,
|
||||
resourcePath,
|
||||
contentType: mimeType,
|
||||
contentLength: size,
|
||||
expiresInSeconds,
|
||||
});
|
||||
|
||||
if (isDefined(presignedUploadUrl)) {
|
||||
return {
|
||||
fileId,
|
||||
uploadUrl: presignedUploadUrl,
|
||||
contentType: mimeType,
|
||||
expiresAt,
|
||||
};
|
||||
}
|
||||
|
||||
// No presign support (local storage, or S3 without presign enabled):
|
||||
// fall back to the token-authenticated streaming endpoint on the server.
|
||||
const payload: FileUploadTokenJwtPayload = {
|
||||
workspaceId,
|
||||
fileId,
|
||||
sub: workspaceId,
|
||||
type: JwtTokenTypeEnum.FILE_UPLOAD,
|
||||
};
|
||||
|
||||
const token = await this.jwtWrapperService.signAsyncOrThrow(payload, {
|
||||
expiresIn: expiresInSeconds,
|
||||
});
|
||||
|
||||
const serverUrl = this.twentyConfigService.get('SERVER_URL');
|
||||
|
||||
return {
|
||||
fileId,
|
||||
uploadUrl: `${serverUrl}/file-upload/${fileId}?token=${token}`,
|
||||
// octet-stream keeps the request body away from the server's json/text
|
||||
// body parsers; the real mime type is already on the file record.
|
||||
contentType: 'application/octet-stream',
|
||||
expiresAt,
|
||||
};
|
||||
}
|
||||
|
||||
// Streams the request body straight to the storage driver, bounded by the
|
||||
// size declared at createFileUpload time. Memory usage stays constant no
|
||||
// matter how large the file is.
|
||||
async receiveFileStream({
|
||||
workspaceId,
|
||||
fileId,
|
||||
stream,
|
||||
}: {
|
||||
workspaceId: string;
|
||||
fileId: string;
|
||||
stream: Readable;
|
||||
}): Promise<void> {
|
||||
const file = await this.findFileOrThrow({ workspaceId, fileId });
|
||||
|
||||
if (file.status !== FILE_STATUS.PENDING) {
|
||||
throw new FileUploadException(
|
||||
`File ${fileId} is not awaiting an upload`,
|
||||
FileUploadExceptionCode.BAD_REQUEST,
|
||||
{
|
||||
userFriendlyMessage: msg`This file has already been uploaded.`,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
const { application, fileFolder, resourcePath } =
|
||||
await this.resolveFileLocation({ workspaceId, file });
|
||||
|
||||
const declaredSize = Number(file.size);
|
||||
let receivedBytes = 0;
|
||||
|
||||
const sizeLimiter = new Transform({
|
||||
transform: (chunk: Buffer, _encoding, callback) => {
|
||||
receivedBytes += chunk.length;
|
||||
|
||||
if (receivedBytes > declaredSize) {
|
||||
callback(
|
||||
new FileUploadException(
|
||||
`Upload exceeds declared size of ${declaredSize} bytes`,
|
||||
FileUploadExceptionCode.FILE_TOO_LARGE,
|
||||
{
|
||||
userFriendlyMessage: msg`The uploaded file is larger than declared.`,
|
||||
},
|
||||
),
|
||||
);
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
callback(null, chunk);
|
||||
},
|
||||
});
|
||||
|
||||
try {
|
||||
await Promise.all([
|
||||
pipeline(stream, sizeLimiter),
|
||||
this.fileStorageService.writeFileStream({
|
||||
fileFolder,
|
||||
applicationUniversalIdentifier: application.universalIdentifier,
|
||||
workspaceId,
|
||||
resourcePath,
|
||||
stream: sizeLimiter,
|
||||
mimeType: file.mimeType,
|
||||
}),
|
||||
]);
|
||||
} catch (error) {
|
||||
// The storage-side pipeline can lose the rejection race to the limiter:
|
||||
// surface the size violation over the resulting stream teardown error.
|
||||
if (receivedBytes > declaredSize) {
|
||||
throw new FileUploadException(
|
||||
`Upload exceeds declared size of ${declaredSize} bytes`,
|
||||
FileUploadExceptionCode.FILE_TOO_LARGE,
|
||||
{
|
||||
userFriendlyMessage: msg`The uploaded file is larger than declared.`,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
throw error;
|
||||
}
|
||||
|
||||
if (receivedBytes !== declaredSize) {
|
||||
// The short object stays in storage but the record stays PENDING, so it
|
||||
// can never be served or attached: the client retries against the same
|
||||
// upload url (overwriting it), and the pending-file cleanup cron
|
||||
// (follow-up PR) reaps whatever is abandoned.
|
||||
throw new FileUploadException(
|
||||
`Uploaded ${receivedBytes} bytes but ${declaredSize} were declared`,
|
||||
FileUploadExceptionCode.FILE_SIZE_MISMATCH,
|
||||
{
|
||||
userFriendlyMessage: msg`The uploaded file does not match the declared size. Please retry the upload.`,
|
||||
},
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// Verifies the bytes actually landed in storage with the declared size and
|
||||
// flips the file to UPLOADED. Idempotent: confirming twice is a no-op.
|
||||
async completeFileUpload({
|
||||
workspaceId,
|
||||
fileId,
|
||||
}: {
|
||||
workspaceId: string;
|
||||
fileId: string;
|
||||
}): Promise<FileWithSignedUrlDTO> {
|
||||
const file = await this.findFileOrThrow({ workspaceId, fileId });
|
||||
const [fileFolder] = file.path.split('/');
|
||||
|
||||
// Restrict to files created through createFileUpload so this mutation
|
||||
// cannot be used to mint signed download urls for arbitrary files.
|
||||
if (
|
||||
!DIRECT_UPLOAD_FILE_FOLDERS.includes(
|
||||
fileFolder as (typeof DIRECT_UPLOAD_FILE_FOLDERS)[number],
|
||||
)
|
||||
) {
|
||||
throw new FileUploadException(
|
||||
`File not found: ${fileId}`,
|
||||
FileUploadExceptionCode.FILE_NOT_FOUND,
|
||||
{
|
||||
userFriendlyMessage: msg`File not found.`,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
if (file.status === FILE_STATUS.UPLOADED) {
|
||||
// Idempotent retry of a confirm that already succeeded. Only files not
|
||||
// yet attached to a record qualify: this cannot be used to mint signed
|
||||
// urls for files that went through the legacy flow and got attached.
|
||||
if (!file.settings?.isTemporaryFile) {
|
||||
throw new FileUploadException(
|
||||
`File ${fileId} is not awaiting an upload confirmation`,
|
||||
FileUploadExceptionCode.BAD_REQUEST,
|
||||
{
|
||||
userFriendlyMessage: msg`This file upload has already been finalized.`,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
return this.toFileWithSignedUrl({
|
||||
file,
|
||||
fileFolder: fileFolder as FileFolder,
|
||||
workspaceId,
|
||||
});
|
||||
}
|
||||
|
||||
const { application, resourcePath } = await this.resolveFileLocation({
|
||||
workspaceId,
|
||||
file,
|
||||
});
|
||||
|
||||
const metadata = await this.fileStorageService.getFileMetadata({
|
||||
fileFolder: fileFolder as FileFolder,
|
||||
applicationUniversalIdentifier: application.universalIdentifier,
|
||||
workspaceId,
|
||||
resourcePath,
|
||||
});
|
||||
|
||||
if (!isDefined(metadata)) {
|
||||
throw new FileUploadException(
|
||||
`File ${fileId} has not been uploaded to storage yet`,
|
||||
FileUploadExceptionCode.FILE_NOT_UPLOADED,
|
||||
{
|
||||
userFriendlyMessage: msg`The file has not been uploaded yet. Please upload it before confirming.`,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
if (metadata.size !== Number(file.size)) {
|
||||
throw new FileUploadException(
|
||||
`File ${fileId} has ${metadata.size} bytes in storage but ${file.size} were declared`,
|
||||
FileUploadExceptionCode.FILE_SIZE_MISMATCH,
|
||||
{
|
||||
userFriendlyMessage: msg`The uploaded file does not match the declared size. Please retry the upload.`,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
await this.fileRepository.update(
|
||||
workspaceId,
|
||||
{ id: fileId },
|
||||
{ status: FILE_STATUS.UPLOADED },
|
||||
);
|
||||
|
||||
return this.toFileWithSignedUrl({
|
||||
file: { ...file, status: FILE_STATUS.UPLOADED },
|
||||
fileFolder: fileFolder as FileFolder,
|
||||
workspaceId,
|
||||
});
|
||||
}
|
||||
|
||||
private async resolveUploadLocation({
|
||||
workspaceId,
|
||||
fileFolder,
|
||||
name,
|
||||
fieldMetadataId,
|
||||
fieldMetadataUniversalIdentifier,
|
||||
}: {
|
||||
workspaceId: string;
|
||||
fileFolder: FileFolder;
|
||||
name: string;
|
||||
fieldMetadataId?: string;
|
||||
fieldMetadataUniversalIdentifier?: string;
|
||||
}): Promise<{
|
||||
applicationUniversalIdentifier: string;
|
||||
resourcePath: string;
|
||||
}> {
|
||||
if (fileFolder === FileFolder.FilesField) {
|
||||
if (!fieldMetadataId && !fieldMetadataUniversalIdentifier) {
|
||||
throw new FileUploadException(
|
||||
'fieldMetadataId or fieldMetadataUniversalIdentifier must be provided',
|
||||
FileUploadExceptionCode.BAD_REQUEST,
|
||||
{
|
||||
userFriendlyMessage: msg`fieldMetadataId or fieldMetadataUniversalIdentifier must be provided`,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
const fieldMetadata = await this.fieldMetadataRepository.findOneOrFail({
|
||||
select: ['applicationId', 'universalIdentifier'],
|
||||
where: {
|
||||
...(fieldMetadataId ? { id: fieldMetadataId } : {}),
|
||||
...(fieldMetadataUniversalIdentifier
|
||||
? { universalIdentifier: fieldMetadataUniversalIdentifier }
|
||||
: {}),
|
||||
workspaceId,
|
||||
},
|
||||
});
|
||||
|
||||
const application = await this.applicationRepository.findOneOrFail({
|
||||
where: {
|
||||
id: fieldMetadata.applicationId,
|
||||
workspaceId,
|
||||
},
|
||||
});
|
||||
|
||||
return {
|
||||
applicationUniversalIdentifier: application.universalIdentifier,
|
||||
resourcePath: `${fieldMetadata.universalIdentifier}/${name}`,
|
||||
};
|
||||
}
|
||||
|
||||
const { workspaceCustomFlatApplication } =
|
||||
await this.applicationService.findWorkspaceTwentyStandardAndCustomApplicationOrThrow(
|
||||
{
|
||||
workspaceId,
|
||||
},
|
||||
);
|
||||
|
||||
return {
|
||||
applicationUniversalIdentifier:
|
||||
workspaceCustomFlatApplication.universalIdentifier,
|
||||
resourcePath: name,
|
||||
};
|
||||
}
|
||||
|
||||
private async findFileOrThrow({
|
||||
workspaceId,
|
||||
fileId,
|
||||
}: {
|
||||
workspaceId: string;
|
||||
fileId: string;
|
||||
}): Promise<FileEntity> {
|
||||
const file = await this.fileRepository.findOne(workspaceId, {
|
||||
where: { id: fileId },
|
||||
});
|
||||
|
||||
if (!isDefined(file)) {
|
||||
throw new FileUploadException(
|
||||
`File not found: ${fileId}`,
|
||||
FileUploadExceptionCode.FILE_NOT_FOUND,
|
||||
{
|
||||
userFriendlyMessage: msg`File not found.`,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
return file;
|
||||
}
|
||||
|
||||
private async resolveFileLocation({
|
||||
workspaceId,
|
||||
file,
|
||||
}: {
|
||||
workspaceId: string;
|
||||
file: FileEntity;
|
||||
}): Promise<{
|
||||
application: ApplicationEntity;
|
||||
fileFolder: FileFolder;
|
||||
resourcePath: string;
|
||||
}> {
|
||||
const [fileFolder] = file.path.split('/');
|
||||
|
||||
const application = await this.applicationRepository.findOneOrFail({
|
||||
where: {
|
||||
id: file.applicationId,
|
||||
workspaceId,
|
||||
},
|
||||
});
|
||||
|
||||
return {
|
||||
application,
|
||||
fileFolder: fileFolder as FileFolder,
|
||||
resourcePath: removeFileFolderFromFileEntityPath(file.path),
|
||||
};
|
||||
}
|
||||
|
||||
private async toFileWithSignedUrl({
|
||||
file,
|
||||
fileFolder,
|
||||
workspaceId,
|
||||
}: {
|
||||
file: FileEntity;
|
||||
fileFolder: FileFolder;
|
||||
workspaceId: string;
|
||||
}): Promise<FileWithSignedUrlDTO> {
|
||||
return {
|
||||
...file,
|
||||
url: await this.fileUrlService.signFileByIdUrl({
|
||||
fileId: file.id,
|
||||
workspaceId,
|
||||
fileFolder,
|
||||
}),
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -15,6 +15,7 @@ import { FileController } from './controllers/file.controller';
|
||||
import { FileEntity } from './entities/file.entity';
|
||||
import { FileCorePictureModule } from './file-core-picture/file-core-picture.module';
|
||||
import { FileEmailAttachmentModule } from './file-email-attachment/file-email-attachment.module';
|
||||
import { FileUploadModule } from './file-upload/file-upload.module';
|
||||
import { FileUrlModule } from './file-url/file-url.module';
|
||||
import { FileWorkflowModule } from './file-workflow/file-workflow.module';
|
||||
import { FilesFieldModule } from './files-field/files-field.module';
|
||||
@@ -33,6 +34,7 @@ import { FileService } from './services/file.service';
|
||||
FileWorkflowModule,
|
||||
FileAiChatModule,
|
||||
FileEmailAttachmentModule,
|
||||
FileUploadModule,
|
||||
SecureHttpClientModule,
|
||||
],
|
||||
providers: [
|
||||
@@ -50,6 +52,7 @@ import { FileService } from './services/file.service';
|
||||
FileWorkflowModule,
|
||||
FileAiChatModule,
|
||||
FileEmailAttachmentModule,
|
||||
FileUploadModule,
|
||||
],
|
||||
controllers: [FileController],
|
||||
})
|
||||
|
||||
@@ -18,6 +18,7 @@ import {
|
||||
IMMUTABLE_FILE_CACHE_CONTROL,
|
||||
} from 'src/engine/core-modules/file/interfaces/file-folder.interface';
|
||||
import { type FileResponse } from 'src/engine/core-modules/file/types/file-response.type';
|
||||
import { FILE_STATUS } from 'src/engine/core-modules/file/types/file-status.types';
|
||||
import { getContentDisposition } from 'src/engine/core-modules/file/utils/get-content-disposition.utils';
|
||||
import { removeFileFolderFromFileEntityPath } from 'src/engine/core-modules/file/utils/remove-file-folder-from-file-entity-path.utils';
|
||||
import { JwtWrapperService } from 'src/engine/core-modules/jwt/services/jwt-wrapper.service';
|
||||
@@ -66,6 +67,7 @@ export class FileService {
|
||||
where: {
|
||||
path: `${fileFolder}/${filepath}`,
|
||||
applicationId,
|
||||
status: FILE_STATUS.UPLOADED,
|
||||
},
|
||||
});
|
||||
|
||||
@@ -94,6 +96,7 @@ export class FileService {
|
||||
const file = await this.fileRepository.findOne(workspaceId, {
|
||||
where: {
|
||||
id: fileId,
|
||||
status: FILE_STATUS.UPLOADED,
|
||||
},
|
||||
});
|
||||
|
||||
@@ -155,6 +158,7 @@ export class FileService {
|
||||
where: {
|
||||
id: params.fileId,
|
||||
path: Like(`${params.fileFolder}/%`),
|
||||
status: FILE_STATUS.UPLOADED,
|
||||
},
|
||||
});
|
||||
|
||||
@@ -251,6 +255,7 @@ export class FileService {
|
||||
where: {
|
||||
id: fileId,
|
||||
path: Like(`${fileFolder}/%`),
|
||||
status: FILE_STATUS.UPLOADED,
|
||||
},
|
||||
});
|
||||
|
||||
|
||||
@@ -0,0 +1,8 @@
|
||||
export const FILE_STATUS = {
|
||||
// File record exists but bytes have not been confirmed in storage yet
|
||||
// (direct upload initiated, waiting for the client to upload and confirm).
|
||||
PENDING: 'PENDING',
|
||||
UPLOADED: 'UPLOADED',
|
||||
} as const;
|
||||
|
||||
export type FileStatus = (typeof FILE_STATUS)[keyof typeof FILE_STATUS];
|
||||
Reference in New Issue
Block a user