feat(files): direct-to-storage upload endpoints with pending file lifecycle (#22449)
<img width="1484" height="404" alt="image" src="https://github.com/user-attachments/assets/b2d363bf-d9e1-49fb-9811-8cc98041aa79" /> ## Context Uploading large files currently OOMs the server: every upload resolver buffers the whole file in memory (`streamToBuffer`) before writing it to storage. This PR is the first of a series introducing direct client-to-storage uploads. It adds the server-side endpoints and driver support only — it is non-breaking and nothing consumes the new flow yet. Follow-up PRs will migrate the frontend upload paths, add a stale-pending-file cleanup cron, and cap the legacy buffered resolvers. ## What it does **New upload flow (initiate → PUT → confirm):** - `createFileUpload(filename, size, fileFolder, fieldMetadataId?)` validates the request (folder allowlist: `FilesField`/`Workflow`, max size, extension-derived mime type), creates the file record in a new `PENDING` status, and returns an upload target: - **S3 with presign enabled** → a presigned PUT URL with `Content-Type`/`Content-Length` pinned in the signature, so the client uploads straight to the bucket; - **local storage, or S3 without presign** → a token-authenticated streaming endpoint on the server (`PUT /file-upload/:id?token=…`, new `FILE_UPLOAD` JWT type) that pipes the request body to the storage driver with constant memory usage and a declared-size cap. - `completeFileUpload(fileId)` verifies the bytes actually landed in storage (HEAD + size match against the declared size) and flips the record to `UPLOADED`. Idempotent. **Pending lifecycle safety:** - New `status` column on `core.file` (`PENDING`/`UPLOADED`, default `UPLOADED` so all existing rows and the legacy upload path are unaffected) + fast instance command. - Files are refused by the serving endpoints and by FILES-field sync while `PENDING`. **Driver support (both drivers):** - `getPresignedUploadUrl` (S3: presigned PUT; local: `null` → server-endpoint fallback) - `writeFileStream` (local: `fs` pipeline with the existing symlink/containment hardening, partial-file cleanup on error; S3: `@aws-sdk/lib-storage` `Upload` for bounded-memory streaming) - `getFileMetadata` (HEAD/stat for confirm-time verification) ## Tests - `file-upload.service.spec.ts`: initiate validation (folder allowlist, size), presigned vs fallback target, confirm verification (missing object, size mismatch, happy path, idempotency) - `local.driver.spec.ts`: `writeFileStream` (content, symlink rejection, partial-file cleanup on stream error), `getFileMetadata` - `s3.driver.spec.ts`: `getPresignedUploadUrl` (disabled → null, PUT command with signed content-type/content-length) - `direct-file-upload.integration-spec.ts`: full end-to-end flow against the local driver (initiate → PUT → complete → download), plus error paths (complete without upload, oversized PUT → 413, invalid token → 403, unsupported folder, size above max) ## Notes for reviewers - The upload-size ceiling for direct uploads is `settings.storage.maxDirectUploadFileSize` (1GB), separate from the 10MB `maxFileSize` used for pictures. - Since content can't be sniffed before it reaches storage, the mime type is derived from the file extension (with the existing `TWENTY_MIME_POLICY` override) and unknown extensions fall back to `application/octet-stream`; the serving path already forces `Content-Disposition: attachment` for anything not on the inline-safe allowlist. - Self-hosters using S3 presign will need a bucket CORS policy allowing `PUT` from the frontend origin (config variable description updated). https://claude.ai/code/session_015UH8KWmsB9zdYaog8MFG1d --- _Generated by [Claude Code](https://claude.ai/code/session_015UH8KWmsB9zdYaog8MFG1d)_ <!-- This is an auto-generated description by cubic. --> <a href="https://cubic.dev/pr/twentyhq/twenty/pull/22449?utm_source=github" target="_blank" rel="noopener noreferrer" data-no-image-dialog="true"><picture><source media="(prefers-color-scheme: dark)" srcset="https://www.cubic.dev/buttons/review-in-cubic-dark.svg"><source media="(prefers-color-scheme: light)" srcset="https://www.cubic.dev/buttons/review-in-cubic-light.svg"><img alt="Review in cubic" src="https://www.cubic.dev/buttons/review-in-cubic-dark.svg"></picture></a> <!-- End of auto-generated description by cubic. -->
This commit is contained in:
@@ -1437,6 +1437,13 @@ type FileWithSignedUrl {
|
||||
url: String!
|
||||
}
|
||||
|
||||
type FileUploadTarget {
|
||||
fileId: UUID!
|
||||
uploadUrl: String!
|
||||
contentType: String!
|
||||
expiresAt: DateTime!
|
||||
}
|
||||
|
||||
type BillingSubscriptionSchedulePhaseItem {
|
||||
price: String!
|
||||
quantity: Float
|
||||
@@ -3246,9 +3253,11 @@ type Mutation {
|
||||
updateNavigationMenuItem(input: UpdateOneNavigationMenuItemInput!): NavigationMenuItem!
|
||||
deleteManyNavigationMenuItems(ids: [UUID!]!): [NavigationMenuItem!]!
|
||||
deleteNavigationMenuItem(id: UUID!): NavigationMenuItem!
|
||||
uploadEmailAttachmentFile(file: Upload!): FileWithSignedUrl!
|
||||
createFileUpload(filename: String!, size: Float!, fileFolder: FileFolder!, fieldMetadataId: String, fieldMetadataUniversalIdentifier: String): FileUploadTarget!
|
||||
completeFileUpload(fileId: String!): FileWithSignedUrl!
|
||||
refreshEnterpriseValidityToken: Boolean!
|
||||
setEnterpriseKey(enterpriseKey: String!): EnterpriseLicenseInfoDTO!
|
||||
uploadEmailAttachmentFile(file: Upload!): FileWithSignedUrl!
|
||||
uploadAiChatFile(file: Upload!): FileWithSignedUrl!
|
||||
uploadWorkflowFile(file: Upload!): FileWithSignedUrl!
|
||||
uploadWorkspaceLogo(file: Upload!): FileWithSignedUrl!
|
||||
@@ -3509,6 +3518,22 @@ input UpdateNavigationMenuItemInput {
|
||||
pageLayoutId: UUID
|
||||
}
|
||||
|
||||
enum FileFolder {
|
||||
CorePicture
|
||||
AgentChat
|
||||
BuiltLogicFunction
|
||||
BuiltFrontComponent
|
||||
PublicAsset
|
||||
Source
|
||||
FilesField
|
||||
Dependencies
|
||||
Workflow
|
||||
EmailAttachment
|
||||
AppTarball
|
||||
GeneratedSdkClient
|
||||
Dpa
|
||||
}
|
||||
|
||||
"""The `Upload` scalar type represents a file upload."""
|
||||
scalar Upload
|
||||
|
||||
@@ -4632,22 +4657,6 @@ input CreateAppTokenInput {
|
||||
expiresAt: DateTime!
|
||||
}
|
||||
|
||||
enum FileFolder {
|
||||
CorePicture
|
||||
AgentChat
|
||||
BuiltLogicFunction
|
||||
BuiltFrontComponent
|
||||
PublicAsset
|
||||
Source
|
||||
FilesField
|
||||
Dependencies
|
||||
Workflow
|
||||
EmailAttachment
|
||||
AppTarball
|
||||
GeneratedSdkClient
|
||||
Dpa
|
||||
}
|
||||
|
||||
type Subscription {
|
||||
onEventSubscription(eventStreamId: String!): EventSubscription
|
||||
logicFunctionLogs(input: LogicFunctionLogsInput!): LogicFunctionLogs!
|
||||
|
||||
@@ -1098,6 +1098,14 @@ export interface FileWithSignedUrl {
|
||||
__typename: 'FileWithSignedUrl'
|
||||
}
|
||||
|
||||
export interface FileUploadTarget {
|
||||
fileId: Scalars['UUID']
|
||||
uploadUrl: Scalars['String']
|
||||
contentType: Scalars['String']
|
||||
expiresAt: Scalars['DateTime']
|
||||
__typename: 'FileUploadTarget'
|
||||
}
|
||||
|
||||
export interface BillingSubscriptionSchedulePhaseItem {
|
||||
price: Scalars['String']
|
||||
quantity?: Scalars['Float']
|
||||
@@ -2773,9 +2781,11 @@ export interface Mutation {
|
||||
updateNavigationMenuItem: NavigationMenuItem
|
||||
deleteManyNavigationMenuItems: NavigationMenuItem[]
|
||||
deleteNavigationMenuItem: NavigationMenuItem
|
||||
uploadEmailAttachmentFile: FileWithSignedUrl
|
||||
createFileUpload: FileUploadTarget
|
||||
completeFileUpload: FileWithSignedUrl
|
||||
refreshEnterpriseValidityToken: Scalars['Boolean']
|
||||
setEnterpriseKey: EnterpriseLicenseInfoDTO
|
||||
uploadEmailAttachmentFile: FileWithSignedUrl
|
||||
uploadAiChatFile: FileWithSignedUrl
|
||||
uploadWorkflowFile: FileWithSignedUrl
|
||||
uploadWorkspaceLogo: FileWithSignedUrl
|
||||
@@ -2993,12 +3003,12 @@ export interface Mutation {
|
||||
__typename: 'Mutation'
|
||||
}
|
||||
|
||||
export type FileFolder = 'CorePicture' | 'AgentChat' | 'BuiltLogicFunction' | 'BuiltFrontComponent' | 'PublicAsset' | 'Source' | 'FilesField' | 'Dependencies' | 'Workflow' | 'EmailAttachment' | 'AppTarball' | 'GeneratedSdkClient' | 'Dpa'
|
||||
|
||||
export type WorkspaceMigrationActionType = 'delete' | 'create' | 'update'
|
||||
|
||||
export type AnalyticsType = 'PAGEVIEW' | 'TRACK'
|
||||
|
||||
export type FileFolder = 'CorePicture' | 'AgentChat' | 'BuiltLogicFunction' | 'BuiltFrontComponent' | 'PublicAsset' | 'Source' | 'FilesField' | 'Dependencies' | 'Workflow' | 'EmailAttachment' | 'AppTarball' | 'GeneratedSdkClient' | 'Dpa'
|
||||
|
||||
export interface Subscription {
|
||||
onEventSubscription?: EventSubscription
|
||||
logicFunctionLogs: LogicFunctionLogs
|
||||
@@ -4145,6 +4155,15 @@ export interface FileWithSignedUrlGenqlSelection{
|
||||
__scalar?: boolean | number
|
||||
}
|
||||
|
||||
export interface FileUploadTargetGenqlSelection{
|
||||
fileId?: boolean | number
|
||||
uploadUrl?: boolean | number
|
||||
contentType?: boolean | number
|
||||
expiresAt?: boolean | number
|
||||
__typename?: boolean | number
|
||||
__scalar?: boolean | number
|
||||
}
|
||||
|
||||
export interface BillingSubscriptionSchedulePhaseItemGenqlSelection{
|
||||
price?: boolean | number
|
||||
quantity?: boolean | number
|
||||
@@ -5958,9 +5977,11 @@ export interface MutationGenqlSelection{
|
||||
updateNavigationMenuItem?: (NavigationMenuItemGenqlSelection & { __args: {input: UpdateOneNavigationMenuItemInput} })
|
||||
deleteManyNavigationMenuItems?: (NavigationMenuItemGenqlSelection & { __args: {ids: Scalars['UUID'][]} })
|
||||
deleteNavigationMenuItem?: (NavigationMenuItemGenqlSelection & { __args: {id: Scalars['UUID']} })
|
||||
uploadEmailAttachmentFile?: (FileWithSignedUrlGenqlSelection & { __args: {file: Scalars['Upload']} })
|
||||
createFileUpload?: (FileUploadTargetGenqlSelection & { __args: {filename: Scalars['String'], size: Scalars['Float'], fileFolder: FileFolder, fieldMetadataId?: (Scalars['String'] | null), fieldMetadataUniversalIdentifier?: (Scalars['String'] | null)} })
|
||||
completeFileUpload?: (FileWithSignedUrlGenqlSelection & { __args: {fileId: Scalars['String']} })
|
||||
refreshEnterpriseValidityToken?: boolean | number
|
||||
setEnterpriseKey?: (EnterpriseLicenseInfoDTOGenqlSelection & { __args: {enterpriseKey: Scalars['String']} })
|
||||
uploadEmailAttachmentFile?: (FileWithSignedUrlGenqlSelection & { __args: {file: Scalars['Upload']} })
|
||||
uploadAiChatFile?: (FileWithSignedUrlGenqlSelection & { __args: {file: Scalars['Upload']} })
|
||||
uploadWorkflowFile?: (FileWithSignedUrlGenqlSelection & { __args: {file: Scalars['Upload']} })
|
||||
uploadWorkspaceLogo?: (FileWithSignedUrlGenqlSelection & { __args: {file: Scalars['Upload']} })
|
||||
@@ -7211,6 +7232,14 @@ export interface LogicFunctionLogsInput {applicationId?: (Scalars['UUID'] | null
|
||||
|
||||
|
||||
|
||||
const FileUploadTarget_possibleTypes: string[] = ['FileUploadTarget']
|
||||
export const isFileUploadTarget = (obj?: { __typename?: any } | null): obj is FileUploadTarget => {
|
||||
if (!obj?.__typename) throw new Error('__typename is missing in "isFileUploadTarget"')
|
||||
return FileUploadTarget_possibleTypes.includes(obj.__typename)
|
||||
}
|
||||
|
||||
|
||||
|
||||
const BillingSubscriptionSchedulePhaseItem_possibleTypes: string[] = ['BillingSubscriptionSchedulePhaseItem']
|
||||
export const isBillingSubscriptionSchedulePhaseItem = (obj?: { __typename?: any } | null): obj is BillingSubscriptionSchedulePhaseItem => {
|
||||
if (!obj?.__typename) throw new Error('__typename is missing in "isBillingSubscriptionSchedulePhaseItem"')
|
||||
@@ -9294,17 +9323,6 @@ export const enumUsageOperationType = {
|
||||
EMAIL_SEND: 'EMAIL_SEND' as const
|
||||
}
|
||||
|
||||
export const enumWorkspaceMigrationActionType = {
|
||||
delete: 'delete' as const,
|
||||
create: 'create' as const,
|
||||
update: 'update' as const
|
||||
}
|
||||
|
||||
export const enumAnalyticsType = {
|
||||
PAGEVIEW: 'PAGEVIEW' as const,
|
||||
TRACK: 'TRACK' as const
|
||||
}
|
||||
|
||||
export const enumFileFolder = {
|
||||
CorePicture: 'CorePicture' as const,
|
||||
AgentChat: 'AgentChat' as const,
|
||||
@@ -9320,3 +9338,14 @@ export const enumFileFolder = {
|
||||
GeneratedSdkClient: 'GeneratedSdkClient' as const,
|
||||
Dpa: 'Dpa' as const
|
||||
}
|
||||
|
||||
export const enumWorkspaceMigrationActionType = {
|
||||
delete: 'delete' as const,
|
||||
create: 'create' as const,
|
||||
update: 'update' as const
|
||||
}
|
||||
|
||||
export const enumAnalyticsType = {
|
||||
PAGEVIEW: 'PAGEVIEW' as const,
|
||||
TRACK: 'TRACK' as const
|
||||
}
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1924,6 +1924,14 @@ export enum FileFolder {
|
||||
Workflow = 'Workflow'
|
||||
}
|
||||
|
||||
export type FileUploadTarget = {
|
||||
__typename?: 'FileUploadTarget';
|
||||
contentType: Scalars['String']['output'];
|
||||
expiresAt: Scalars['DateTime']['output'];
|
||||
fileId: Scalars['UUID']['output'];
|
||||
uploadUrl: Scalars['String']['output'];
|
||||
};
|
||||
|
||||
export type FileWithSignedUrl = {
|
||||
__typename?: 'FileWithSignedUrl';
|
||||
createdAt: Scalars['DateTime']['output'];
|
||||
@@ -2473,6 +2481,7 @@ export type Mutation = {
|
||||
checkCustomDomainValidRecords?: Maybe<DomainValidRecords>;
|
||||
checkPublicDomainValidRecords?: Maybe<DomainValidRecords>;
|
||||
checkoutSession: BillingSession;
|
||||
completeFileUpload: FileWithSignedUrl;
|
||||
createApiKey: ApiKey;
|
||||
createApplicationRegistration: CreateApplicationRegistration;
|
||||
createApplicationRegistrationVariable: ApplicationRegistrationVariable;
|
||||
@@ -2484,6 +2493,7 @@ export type Mutation = {
|
||||
createDevelopmentApplication: DevelopmentApplication;
|
||||
createEmailGroupChannel: CreateEmailGroupChannelOutput;
|
||||
createEmailingDomain: EmailingDomain;
|
||||
createFileUpload: FileUploadTarget;
|
||||
createFrontComponent: FrontComponent;
|
||||
createManyNavigationMenuItems: Array<NavigationMenuItem>;
|
||||
createManyViewFieldGroups: Array<ViewFieldGroup>;
|
||||
@@ -2749,6 +2759,11 @@ export type MutationCheckoutSessionArgs = {
|
||||
};
|
||||
|
||||
|
||||
export type MutationCompleteFileUploadArgs = {
|
||||
fileId: Scalars['String']['input'];
|
||||
};
|
||||
|
||||
|
||||
export type MutationCreateApiKeyArgs = {
|
||||
input: CreateApiKeyInput;
|
||||
};
|
||||
@@ -2795,6 +2810,15 @@ export type MutationCreateEmailingDomainArgs = {
|
||||
};
|
||||
|
||||
|
||||
export type MutationCreateFileUploadArgs = {
|
||||
fieldMetadataId?: InputMaybe<Scalars['String']['input']>;
|
||||
fieldMetadataUniversalIdentifier?: InputMaybe<Scalars['String']['input']>;
|
||||
fileFolder: FileFolder;
|
||||
filename: Scalars['String']['input'];
|
||||
size: Scalars['Float']['input'];
|
||||
};
|
||||
|
||||
|
||||
export type MutationCreateFrontComponentArgs = {
|
||||
input: CreateFrontComponentInput;
|
||||
};
|
||||
|
||||
@@ -54,7 +54,7 @@ export const rule = defineRule({
|
||||
meta: {
|
||||
docs: {
|
||||
description:
|
||||
'REST API endpoints should have authentication guards (UserAuthGuard, WorkspaceAuthGuard, FilePathGuard, FileByIdGuard) or be explicitly marked as public (PublicEndpointGuard) and permission guards (SettingsPermissionGuard or CustomPermissionGuard) to maintain our security model.',
|
||||
'REST API endpoints should have authentication guards (UserAuthGuard, WorkspaceAuthGuard, FilePathGuard, FileByIdGuard, FileUploadTokenGuard) or be explicitly marked as public (PublicEndpointGuard) and permission guards (SettingsPermissionGuard or CustomPermissionGuard) to maintain our security model.',
|
||||
},
|
||||
messages: {
|
||||
restApiMethodsShouldBeGuarded:
|
||||
|
||||
@@ -42,7 +42,8 @@ export const typedTokenHelpers = {
|
||||
arg.name === 'WorkspaceAuthGuard' ||
|
||||
arg.name === 'PublicEndpointGuard' ||
|
||||
arg.name === 'FilePathGuard' ||
|
||||
arg.name === 'FileByIdGuard'
|
||||
arg.name === 'FileByIdGuard' ||
|
||||
arg.name === 'FileUploadTokenGuard'
|
||||
);
|
||||
}
|
||||
return false;
|
||||
|
||||
@@ -29,6 +29,7 @@
|
||||
"@aws-sdk/client-sesv2": "3.1001.0",
|
||||
"@aws-sdk/client-sts": "3.1001.0",
|
||||
"@aws-sdk/credential-providers": "3.1001.0",
|
||||
"@aws-sdk/lib-storage": "3.1001.0",
|
||||
"@aws-sdk/s3-request-presigner": "3.1001.0",
|
||||
"@azure/msal-node": "^5.2.3",
|
||||
"@blocknote/server-util": "^0.51.4",
|
||||
|
||||
+25
@@ -0,0 +1,25 @@
|
||||
import { QueryRunner } from 'typeorm';
|
||||
|
||||
import { RegisteredInstanceCommand } from 'src/engine/core-modules/upgrade/decorators/registered-instance-command.decorator';
|
||||
import { FastInstanceCommand } from 'src/engine/core-modules/upgrade/interfaces/fast-instance-command.interface';
|
||||
|
||||
@RegisteredInstanceCommand('2.19.0', 1783082964705)
|
||||
export class AddStatusToFileFastInstanceCommand implements FastInstanceCommand {
|
||||
public async up(queryRunner: QueryRunner): Promise<void> {
|
||||
await queryRunner.query(
|
||||
`DO $$ BEGIN CREATE TYPE "core"."file_status_enum" AS ENUM('PENDING', 'UPLOADED'); EXCEPTION WHEN duplicate_object THEN null; END $$`,
|
||||
);
|
||||
await queryRunner.query(
|
||||
`ALTER TABLE "core"."file" ADD COLUMN IF NOT EXISTS "status" "core"."file_status_enum" NOT NULL DEFAULT 'UPLOADED'`,
|
||||
);
|
||||
await queryRunner.query(
|
||||
`CREATE INDEX IF NOT EXISTS "IDX_FILE_STATUS" ON "core"."file" ("status")`,
|
||||
);
|
||||
}
|
||||
|
||||
public async down(queryRunner: QueryRunner): Promise<void> {
|
||||
await queryRunner.query(`DROP INDEX IF EXISTS "core"."IDX_FILE_STATUS"`);
|
||||
await queryRunner.query(`ALTER TABLE "core"."file" DROP COLUMN IF EXISTS "status"`);
|
||||
await queryRunner.query(`DROP TYPE IF EXISTS "core"."file_status_enum"`);
|
||||
}
|
||||
}
|
||||
+4
@@ -0,0 +1,4 @@
|
||||
// Referenced by @WasIntroducedInUpgrade on the file "status" column so
|
||||
// pre-2.19 upgrade steps don't SELECT it before this command adds it.
|
||||
export const ADD_STATUS_TO_FILE_UPGRADE_COMMAND_NAME =
|
||||
'2.19.0_AddStatusToFileFastInstanceCommand_1783082964705';
|
||||
+2
@@ -91,6 +91,7 @@ import { AddFolderImportToMessageFolderPendingSyncActionFastInstanceCommand } fr
|
||||
import { AddViewKanbanColumnWidthFastInstanceCommand } from './2-15/2-15-instance-command-fast-1781900000000-add-view-kanban-column-width';
|
||||
import { AddPendingQuestionMessageIdToAgentChatThreadFastInstanceCommand } from 'src/database/commands/upgrade-version-command/2-19/2-19-instance-command-fast-1782999138000-add-pending-question-to-agent-chat-thread';
|
||||
import { AddWorkspaceDiscoverabilityToWorkspaceFastInstanceCommand } from './2-19/2-19-instance-command-fast-1783004140000-add-workspace-discoverability-to-workspace';
|
||||
import { AddStatusToFileFastInstanceCommand } from './2-19/2-19-instance-command-fast-1783082964705-add-status-to-file';
|
||||
import { DropMetadataStandardOverridesColumnFastInstanceCommand } from './2-20/2-20-instance-command-fast-1825000000000-drop-metadata-standard-overrides-column';
|
||||
import { AddLogoToApplicationRegistrationFastInstanceCommand } from './2-19/2-19-instance-command-fast-1783069672191-add-logo-to-application-registration';
|
||||
import { BackfillLogoOnApplicationRegistrationSlowInstanceCommand } from './2-19/2-19-instance-command-slow-1783069673191-backfill-logo-on-application-registration';
|
||||
@@ -194,4 +195,5 @@ export const INSTANCE_COMMANDS = [
|
||||
BackfillLogoOnApplicationRegistrationSlowInstanceCommand,
|
||||
AddDisplayFieldsToApplicationRegistrationFastInstanceCommand,
|
||||
BackfillDisplayFieldsOnApplicationRegistrationSlowInstanceCommand,
|
||||
AddStatusToFileFastInstanceCommand,
|
||||
];
|
||||
|
||||
@@ -3,6 +3,10 @@ import { type Settings } from './interfaces/settings.interface';
|
||||
export const settings: Settings = {
|
||||
storage: {
|
||||
maxFileSize: '10MB',
|
||||
// Direct uploads (createFileUpload/completeFileUpload) stream to storage
|
||||
// without transiting the server memory, so they get a much higher cap
|
||||
// than multipart uploads.
|
||||
maxDirectUploadFileSize: '1GB',
|
||||
},
|
||||
minLengthOfStringForDuplicateCheck: 3,
|
||||
maxVisibleViewFields: 30,
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
export interface Settings {
|
||||
storage: {
|
||||
maxFileSize: `${number}MB`;
|
||||
maxDirectUploadFileSize: `${number}MB` | `${number}GB`;
|
||||
};
|
||||
minLengthOfStringForDuplicateCheck: number;
|
||||
maxVisibleViewFields: number;
|
||||
|
||||
+8
@@ -0,0 +1,8 @@
|
||||
import { type CommonPropertiesJwtPayload } from 'src/engine/core-modules/auth/types/common-properties-jwt-payload.type';
|
||||
import { JwtTokenTypeEnum } from 'src/engine/core-modules/auth/types/jwt-token-type.enum';
|
||||
|
||||
export type FileUploadTokenJwtPayload = CommonPropertiesJwtPayload & {
|
||||
type: JwtTokenTypeEnum.FILE_UPLOAD;
|
||||
workspaceId: string;
|
||||
fileId: string;
|
||||
};
|
||||
@@ -5,6 +5,7 @@ import { type ApplicationRefreshTokenJwtPayload } from 'src/engine/core-modules/
|
||||
import { type AppOAuthStateJwtPayload } from 'src/engine/core-modules/auth/types/app-oauth-state-jwt-payload.type';
|
||||
import { type ApprovedAccessDomainJwtPayload } from 'src/engine/core-modules/auth/types/approved-access-domain-jwt-payload.type';
|
||||
import { type FileTokenJwtPayload } from 'src/engine/core-modules/auth/types/file-token-jwt-payload.type';
|
||||
import { type FileUploadTokenJwtPayload } from 'src/engine/core-modules/auth/types/file-upload-token-jwt-payload.type';
|
||||
import { type FileTokenJwtPayloadLegacy } from 'src/engine/core-modules/auth/types/file-token-jwt-payload-legacy.type';
|
||||
import { type LoginTokenJwtPayload } from 'src/engine/core-modules/auth/types/login-token-jwt-payload.type';
|
||||
import { type PlaygroundTokenJwtPayload } from 'src/engine/core-modules/auth/types/playground-token-jwt-payload.type';
|
||||
@@ -23,6 +24,7 @@ export type JwtPayload =
|
||||
| RefreshTokenJwtPayload
|
||||
| FileTokenJwtPayload
|
||||
| FileTokenJwtPayloadLegacy
|
||||
| FileUploadTokenJwtPayload
|
||||
| AppOAuthStateJwtPayload
|
||||
| ApprovedAccessDomainJwtPayload
|
||||
| PlaygroundTokenJwtPayload;
|
||||
|
||||
@@ -4,6 +4,7 @@ export enum JwtTokenTypeEnum {
|
||||
WORKSPACE_AGNOSTIC = 'WORKSPACE_AGNOSTIC',
|
||||
LOGIN = 'LOGIN',
|
||||
FILE = 'FILE',
|
||||
FILE_UPLOAD = 'FILE_UPLOAD',
|
||||
API_KEY = 'API_KEY',
|
||||
REMOTE_SERVER = 'REMOTE_SERVER',
|
||||
KEY_ENCRYPTION_KEY = 'KEY_ENCRYPTION_KEY',
|
||||
|
||||
+114
-1
@@ -1,6 +1,15 @@
|
||||
import { mkdtemp, mkdir, readFile, rm, symlink, writeFile } from 'fs/promises';
|
||||
import {
|
||||
mkdtemp,
|
||||
mkdir,
|
||||
readFile,
|
||||
rm,
|
||||
stat,
|
||||
symlink,
|
||||
writeFile,
|
||||
} from 'fs/promises';
|
||||
import { tmpdir } from 'os';
|
||||
import path from 'path';
|
||||
import { Readable } from 'stream';
|
||||
|
||||
import { FileStorageExceptionCode } from 'src/engine/core-modules/file-storage/interfaces/file-storage-exception';
|
||||
|
||||
@@ -78,4 +87,108 @@ describe('LocalDriver security hardening', () => {
|
||||
code: FileStorageExceptionCode.ACCESS_DENIED,
|
||||
});
|
||||
});
|
||||
|
||||
describe('writeFileStream', () => {
|
||||
it('should write the streamed content to disk', async () => {
|
||||
const storagePath = await createTempDirectory('local-driver-storage-');
|
||||
const driver = new LocalDriver({ storagePath });
|
||||
|
||||
await driver.writeFileStream({
|
||||
filePath: 'workspace/app/streamed.txt',
|
||||
stream: Readable.from([
|
||||
Buffer.from('streamed-'),
|
||||
Buffer.from('content'),
|
||||
]),
|
||||
mimeType: 'text/plain',
|
||||
});
|
||||
|
||||
await expect(
|
||||
readFile(path.join(storagePath, 'workspace/app/streamed.txt'), 'utf8'),
|
||||
).resolves.toBe('streamed-content');
|
||||
});
|
||||
|
||||
it('should reject when target is a symlink', async () => {
|
||||
const storagePath = await createTempDirectory('local-driver-storage-');
|
||||
const outsidePath = await createTempDirectory('local-driver-outside-');
|
||||
const outsideFilePath = path.join(outsidePath, 'outside.txt');
|
||||
const symlinkFolderPath = path.join(storagePath, 'workspace', 'app');
|
||||
const symlinkFilePath = path.join(symlinkFolderPath, 'target.txt');
|
||||
|
||||
await mkdir(symlinkFolderPath, { recursive: true });
|
||||
await writeFile(outsideFilePath, 'outside');
|
||||
await symlink(outsideFilePath, symlinkFilePath);
|
||||
|
||||
const driver = new LocalDriver({ storagePath });
|
||||
|
||||
await expect(
|
||||
driver.writeFileStream({
|
||||
filePath: 'workspace/app/target.txt',
|
||||
stream: Readable.from([Buffer.from('new-content')]),
|
||||
mimeType: undefined,
|
||||
}),
|
||||
).rejects.toMatchObject({
|
||||
code: FileStorageExceptionCode.ACCESS_DENIED,
|
||||
});
|
||||
|
||||
await expect(readFile(outsideFilePath, 'utf8')).resolves.toBe('outside');
|
||||
});
|
||||
|
||||
it('should remove the partial file when the stream errors', async () => {
|
||||
const storagePath = await createTempDirectory('local-driver-storage-');
|
||||
const driver = new LocalDriver({ storagePath });
|
||||
|
||||
const failingStream = new Readable({
|
||||
read() {
|
||||
this.push(Buffer.from('partial'));
|
||||
this.destroy(new Error('stream interrupted'));
|
||||
},
|
||||
});
|
||||
|
||||
await expect(
|
||||
driver.writeFileStream({
|
||||
filePath: 'workspace/app/partial.txt',
|
||||
stream: failingStream,
|
||||
mimeType: undefined,
|
||||
}),
|
||||
).rejects.toThrow('stream interrupted');
|
||||
|
||||
await expect(
|
||||
stat(path.join(storagePath, 'workspace/app/partial.txt')),
|
||||
).rejects.toMatchObject({ code: 'ENOENT' });
|
||||
});
|
||||
});
|
||||
|
||||
describe('getFileMetadata', () => {
|
||||
it('should return the file size', async () => {
|
||||
const storagePath = await createTempDirectory('local-driver-storage-');
|
||||
const folderPath = path.join(storagePath, 'workspace', 'app');
|
||||
|
||||
await mkdir(folderPath, { recursive: true });
|
||||
await writeFile(path.join(folderPath, 'file.txt'), '12345');
|
||||
|
||||
const driver = new LocalDriver({ storagePath });
|
||||
|
||||
await expect(
|
||||
driver.getFileMetadata({ filePath: 'workspace/app/file.txt' }),
|
||||
).resolves.toEqual({ size: 5 });
|
||||
});
|
||||
|
||||
it('should return null when the file does not exist', async () => {
|
||||
const storagePath = await createTempDirectory('local-driver-storage-');
|
||||
const driver = new LocalDriver({ storagePath });
|
||||
|
||||
await expect(
|
||||
driver.getFileMetadata({ filePath: 'workspace/app/missing.txt' }),
|
||||
).resolves.toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe('getPresignedUploadUrl', () => {
|
||||
it('should return null so callers fall back to the server endpoint', async () => {
|
||||
const storagePath = await createTempDirectory('local-driver-storage-');
|
||||
const driver = new LocalDriver({ storagePath });
|
||||
|
||||
await expect(driver.getPresignedUploadUrl()).resolves.toBeNull();
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
+64
-1
@@ -1,4 +1,4 @@
|
||||
import { GetObjectCommand } from '@aws-sdk/client-s3';
|
||||
import { GetObjectCommand, PutObjectCommand } from '@aws-sdk/client-s3';
|
||||
import { getSignedUrl } from '@aws-sdk/s3-request-presigner';
|
||||
|
||||
import { S3Driver } from 'src/engine/core-modules/file-storage/drivers/s3.driver';
|
||||
@@ -120,3 +120,66 @@ describe('S3Driver.getPresignedUrl', () => {
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('S3Driver.getPresignedUploadUrl', () => {
|
||||
afterEach(() => {
|
||||
jest.clearAllMocks();
|
||||
});
|
||||
|
||||
it('should return null when presigning is not enabled', async () => {
|
||||
const driver = new S3Driver({
|
||||
bucketName: 'test-bucket',
|
||||
region: 'us-east-1',
|
||||
});
|
||||
|
||||
const result = await driver.getPresignedUploadUrl({
|
||||
filePath: 'some/file.pdf',
|
||||
contentType: 'application/pdf',
|
||||
contentLength: 1024,
|
||||
});
|
||||
|
||||
expect(result).toBeNull();
|
||||
expect(getSignedUrl).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('should presign a PUT with content-type and content-length in the signature', async () => {
|
||||
(getSignedUrl as jest.Mock).mockResolvedValue(
|
||||
'https://s3.us-east-1.amazonaws.com/test-bucket/some/file.pdf?X-Amz-Signature=abc',
|
||||
);
|
||||
|
||||
const driver = new S3Driver({
|
||||
bucketName: 'test-bucket',
|
||||
region: 'us-east-1',
|
||||
presignEnabled: true,
|
||||
});
|
||||
|
||||
const result = await driver.getPresignedUploadUrl({
|
||||
filePath: 'some/file.pdf',
|
||||
contentType: 'application/pdf',
|
||||
contentLength: 1024,
|
||||
expiresInSeconds: 900,
|
||||
});
|
||||
|
||||
expect(result).toBe(
|
||||
'https://s3.us-east-1.amazonaws.com/test-bucket/some/file.pdf?X-Amz-Signature=abc',
|
||||
);
|
||||
expect(getSignedUrl).toHaveBeenCalledWith(
|
||||
expect.anything(),
|
||||
expect.any(PutObjectCommand),
|
||||
{
|
||||
expiresIn: 900,
|
||||
signableHeaders: new Set(['content-type', 'content-length']),
|
||||
},
|
||||
);
|
||||
|
||||
const command = (getSignedUrl as jest.Mock).mock
|
||||
.calls[0][1] as PutObjectCommand;
|
||||
|
||||
expect(command.input).toMatchObject({
|
||||
Bucket: 'test-bucket',
|
||||
Key: 'some/file.pdf',
|
||||
ContentType: 'application/pdf',
|
||||
ContentLength: 1024,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
+87
@@ -8,6 +8,9 @@ import { ValidatedStorageDriver } from 'src/engine/core-modules/file-storage/dri
|
||||
const createMockDriver = (): jest.Mocked<StorageDriver> => ({
|
||||
readFile: jest.fn().mockResolvedValue(Readable.from([])),
|
||||
writeFile: jest.fn().mockResolvedValue(undefined),
|
||||
writeFileStream: jest.fn().mockResolvedValue(undefined),
|
||||
getFileMetadata: jest.fn().mockResolvedValue(null),
|
||||
getPresignedUploadUrl: jest.fn().mockResolvedValue(null),
|
||||
downloadFolder: jest.fn().mockResolvedValue(undefined),
|
||||
uploadFolder: jest.fn().mockResolvedValue(undefined),
|
||||
downloadFile: jest.fn().mockResolvedValue(undefined),
|
||||
@@ -138,6 +141,52 @@ describe('ValidatedStorageDriver', () => {
|
||||
responseContentDisposition: 'inline',
|
||||
});
|
||||
});
|
||||
|
||||
it('should delegate writeFileStream', async () => {
|
||||
const params = {
|
||||
filePath: 'folder/file.txt',
|
||||
stream: Readable.from([Buffer.from('data')]),
|
||||
mimeType: 'text/plain' as string | undefined,
|
||||
};
|
||||
|
||||
await driver.writeFileStream(params);
|
||||
|
||||
expect(mockDelegate.writeFileStream).toHaveBeenCalledWith(params);
|
||||
});
|
||||
|
||||
it('should delegate getFileMetadata', async () => {
|
||||
mockDelegate.getFileMetadata.mockResolvedValue({ size: 1024 });
|
||||
|
||||
const result = await driver.getFileMetadata({
|
||||
filePath: 'folder/file.txt',
|
||||
});
|
||||
|
||||
expect(result).toEqual({ size: 1024 });
|
||||
expect(mockDelegate.getFileMetadata).toHaveBeenCalledWith({
|
||||
filePath: 'folder/file.txt',
|
||||
});
|
||||
});
|
||||
|
||||
it('should delegate getPresignedUploadUrl', async () => {
|
||||
mockDelegate.getPresignedUploadUrl.mockResolvedValue(
|
||||
'https://s3.example.com/signed-put',
|
||||
);
|
||||
|
||||
const result = await driver.getPresignedUploadUrl({
|
||||
filePath: 'folder/file.txt',
|
||||
contentType: 'application/pdf',
|
||||
contentLength: 1024,
|
||||
expiresInSeconds: 900,
|
||||
});
|
||||
|
||||
expect(result).toBe('https://s3.example.com/signed-put');
|
||||
expect(mockDelegate.getPresignedUploadUrl).toHaveBeenCalledWith({
|
||||
filePath: 'folder/file.txt',
|
||||
contentType: 'application/pdf',
|
||||
contentLength: 1024,
|
||||
expiresInSeconds: 900,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('rejects path traversal attempts', () => {
|
||||
@@ -165,6 +214,44 @@ describe('ValidatedStorageDriver', () => {
|
||||
expect(mockDelegate.writeFile).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('should reject writeFileStream with traversal', async () => {
|
||||
await expect(
|
||||
driver.writeFileStream({
|
||||
filePath: '../../evil',
|
||||
stream: Readable.from([Buffer.from('x')]),
|
||||
mimeType: undefined,
|
||||
}),
|
||||
).rejects.toMatchObject({
|
||||
code: FileStorageExceptionCode.ACCESS_DENIED,
|
||||
});
|
||||
|
||||
expect(mockDelegate.writeFileStream).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('should reject getFileMetadata with traversal', async () => {
|
||||
await expect(
|
||||
driver.getFileMetadata({ filePath: '../etc/passwd' }),
|
||||
).rejects.toMatchObject({
|
||||
code: FileStorageExceptionCode.ACCESS_DENIED,
|
||||
});
|
||||
|
||||
expect(mockDelegate.getFileMetadata).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('should reject getPresignedUploadUrl with traversal', async () => {
|
||||
await expect(
|
||||
driver.getPresignedUploadUrl({
|
||||
filePath: '../etc/passwd',
|
||||
contentType: 'application/pdf',
|
||||
contentLength: 1024,
|
||||
}),
|
||||
).rejects.toMatchObject({
|
||||
code: FileStorageExceptionCode.ACCESS_DENIED,
|
||||
});
|
||||
|
||||
expect(mockDelegate.getPresignedUploadUrl).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('should reject delete with traversal in folderPath', async () => {
|
||||
await expect(
|
||||
driver.delete({ folderPath: '../secret' }),
|
||||
|
||||
+17
@@ -8,6 +8,16 @@ export interface StorageDriver {
|
||||
mimeType: string | undefined;
|
||||
}): Promise<void>;
|
||||
|
||||
writeFileStream(params: {
|
||||
filePath: string;
|
||||
stream: Readable;
|
||||
mimeType: string | undefined;
|
||||
}): Promise<void>;
|
||||
|
||||
getFileMetadata(params: {
|
||||
filePath: string;
|
||||
}): Promise<{ size: number } | null>;
|
||||
|
||||
downloadFolder(params: {
|
||||
onStoragePath: string;
|
||||
localPath: string;
|
||||
@@ -42,4 +52,11 @@ export interface StorageDriver {
|
||||
responseContentDisposition?: string;
|
||||
responseCacheControl?: string;
|
||||
}): Promise<string | null>;
|
||||
|
||||
getPresignedUploadUrl(params: {
|
||||
filePath: string;
|
||||
contentType: string;
|
||||
contentLength: number;
|
||||
expiresInSeconds?: number;
|
||||
}): Promise<string | null>;
|
||||
}
|
||||
|
||||
+79
-9
@@ -1,7 +1,13 @@
|
||||
import { createReadStream, existsSync, realpathSync } from 'fs';
|
||||
import {
|
||||
createReadStream,
|
||||
createWriteStream,
|
||||
existsSync,
|
||||
realpathSync,
|
||||
} from 'fs';
|
||||
import * as fs from 'fs/promises';
|
||||
import path, { dirname, join } from 'path';
|
||||
import { type Readable } from 'stream';
|
||||
import { pipeline } from 'stream/promises';
|
||||
|
||||
import { type StorageDriver } from 'src/engine/core-modules/file-storage/drivers/interfaces/storage-driver.interface';
|
||||
import {
|
||||
@@ -64,18 +70,18 @@ export class LocalDriver implements StorageDriver {
|
||||
}
|
||||
}
|
||||
|
||||
async writeFile(params: {
|
||||
filePath: string;
|
||||
sourceFile: Buffer | Uint8Array | string;
|
||||
mimeType: string | undefined;
|
||||
}): Promise<void> {
|
||||
const filePath = path.resolve(this.options.storagePath, params.filePath);
|
||||
const folderPath = dirname(filePath);
|
||||
// Resolves the on-disk path for a write, creating the parent folder and
|
||||
// enforcing storage containment + symlink rejection.
|
||||
private async resolveWritableRealPathOrThrow(
|
||||
filePath: string,
|
||||
): Promise<string> {
|
||||
const resolvedPath = path.resolve(this.options.storagePath, filePath);
|
||||
const folderPath = dirname(resolvedPath);
|
||||
|
||||
await this.createFolder(folderPath);
|
||||
|
||||
const realFolderPath = realpathSync(folderPath);
|
||||
const realFilePath = path.join(realFolderPath, path.basename(filePath));
|
||||
const realFilePath = path.join(realFolderPath, path.basename(resolvedPath));
|
||||
|
||||
this.assertRealPathIsWithinStorage(realFilePath);
|
||||
|
||||
@@ -94,9 +100,67 @@ export class LocalDriver implements StorageDriver {
|
||||
}
|
||||
}
|
||||
|
||||
return realFilePath;
|
||||
}
|
||||
|
||||
async writeFile(params: {
|
||||
filePath: string;
|
||||
sourceFile: Buffer | Uint8Array | string;
|
||||
mimeType: string | undefined;
|
||||
}): Promise<void> {
|
||||
const realFilePath = await this.resolveWritableRealPathOrThrow(
|
||||
params.filePath,
|
||||
);
|
||||
|
||||
await fs.writeFile(realFilePath, params.sourceFile);
|
||||
}
|
||||
|
||||
async writeFileStream(params: {
|
||||
filePath: string;
|
||||
stream: Readable;
|
||||
mimeType: string | undefined;
|
||||
}): Promise<void> {
|
||||
const realFilePath = await this.resolveWritableRealPathOrThrow(
|
||||
params.filePath,
|
||||
);
|
||||
|
||||
try {
|
||||
await pipeline(params.stream, createWriteStream(realFilePath));
|
||||
} catch (error) {
|
||||
// Remove the partial file so a failed upload can be retried cleanly
|
||||
await fs.rm(realFilePath, { force: true });
|
||||
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async getFileMetadata(params: {
|
||||
filePath: string;
|
||||
}): Promise<{ size: number } | null> {
|
||||
const joinedPath = join(this.options.storagePath, params.filePath);
|
||||
let filePath: string;
|
||||
|
||||
try {
|
||||
filePath = realpathSync(path.resolve(joinedPath));
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
|
||||
this.assertRealPathIsWithinStorage(filePath);
|
||||
|
||||
try {
|
||||
const stats = await fs.stat(filePath);
|
||||
|
||||
return { size: stats.size };
|
||||
} catch (error) {
|
||||
if ((error as NodeJS.ErrnoException).code === 'ENOENT') {
|
||||
return null;
|
||||
}
|
||||
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async downloadFile(params: {
|
||||
onStoragePath: string;
|
||||
localPath: string;
|
||||
@@ -304,6 +368,12 @@ export class LocalDriver implements StorageDriver {
|
||||
return null;
|
||||
}
|
||||
|
||||
// Local storage has no external endpoint to upload to: the caller falls
|
||||
// back to the server-side streaming upload endpoint.
|
||||
async getPresignedUploadUrl(): Promise<string | null> {
|
||||
return null;
|
||||
}
|
||||
|
||||
async checkFolderExists(params: { folderPath: string }): Promise<boolean> {
|
||||
const folderFullPath = path.resolve(
|
||||
this.options.storagePath,
|
||||
|
||||
@@ -20,6 +20,7 @@ import {
|
||||
S3,
|
||||
type S3ClientConfig,
|
||||
} from '@aws-sdk/client-s3';
|
||||
import { Upload } from '@aws-sdk/lib-storage';
|
||||
import { getSignedUrl } from '@aws-sdk/s3-request-presigner';
|
||||
import { isDefined } from 'twenty-shared/utils';
|
||||
|
||||
@@ -112,6 +113,47 @@ export class S3Driver implements StorageDriver {
|
||||
await this.s3Client.send(command);
|
||||
}
|
||||
|
||||
async writeFileStream(params: {
|
||||
filePath: string;
|
||||
stream: Readable;
|
||||
mimeType: string | undefined;
|
||||
}): Promise<void> {
|
||||
// Upload streams the body with bounded memory (multipart under the hood),
|
||||
// unlike PutObjectCommand which requires the whole payload upfront.
|
||||
const upload = new Upload({
|
||||
client: this.s3Client,
|
||||
params: {
|
||||
Bucket: this.bucketName,
|
||||
Key: params.filePath,
|
||||
Body: params.stream,
|
||||
ContentType: params.mimeType,
|
||||
},
|
||||
});
|
||||
|
||||
await upload.done();
|
||||
}
|
||||
|
||||
async getFileMetadata(params: {
|
||||
filePath: string;
|
||||
}): Promise<{ size: number } | null> {
|
||||
try {
|
||||
const head = await this.s3Client.send(
|
||||
new HeadObjectCommand({
|
||||
Bucket: this.bucketName,
|
||||
Key: params.filePath,
|
||||
}),
|
||||
);
|
||||
|
||||
return { size: head.ContentLength ?? 0 };
|
||||
} catch (error) {
|
||||
if (error instanceof NotFound) {
|
||||
return null;
|
||||
}
|
||||
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
private async createFolder(path: string) {
|
||||
return fs.mkdirSync(path, { recursive: true });
|
||||
}
|
||||
@@ -404,6 +446,31 @@ export class S3Driver implements StorageDriver {
|
||||
});
|
||||
}
|
||||
|
||||
async getPresignedUploadUrl(params: {
|
||||
filePath: string;
|
||||
contentType: string;
|
||||
contentLength: number;
|
||||
expiresInSeconds?: number;
|
||||
}): Promise<string | null> {
|
||||
if (!this.presignClient) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const command = new PutObjectCommand({
|
||||
Bucket: this.bucketName,
|
||||
Key: params.filePath,
|
||||
ContentType: params.contentType,
|
||||
ContentLength: params.contentLength,
|
||||
});
|
||||
|
||||
// Content-Type and Content-Length are part of the signature so the client
|
||||
// cannot upload a payload of a different type or size than declared.
|
||||
return getSignedUrl(this.presignClient, command, {
|
||||
expiresIn: params.expiresInSeconds ?? 900,
|
||||
signableHeaders: new Set(['content-type', 'content-length']),
|
||||
});
|
||||
}
|
||||
|
||||
async checkBucketExists(args: HeadBucketCommandInput) {
|
||||
try {
|
||||
await this.s3Client.headBucket(args);
|
||||
|
||||
+29
@@ -23,6 +23,24 @@ export class ValidatedStorageDriver implements StorageDriver {
|
||||
return this.delegate.writeFile(params);
|
||||
}
|
||||
|
||||
async writeFileStream(params: {
|
||||
filePath: string;
|
||||
stream: Readable;
|
||||
mimeType: string | undefined;
|
||||
}): Promise<void> {
|
||||
assertStoragePathIsSafe(params.filePath);
|
||||
|
||||
return this.delegate.writeFileStream(params);
|
||||
}
|
||||
|
||||
async getFileMetadata(params: {
|
||||
filePath: string;
|
||||
}): Promise<{ size: number } | null> {
|
||||
assertStoragePathIsSafe(params.filePath);
|
||||
|
||||
return this.delegate.getFileMetadata(params);
|
||||
}
|
||||
|
||||
async downloadFolder(params: {
|
||||
onStoragePath: string;
|
||||
localPath: string;
|
||||
@@ -111,6 +129,17 @@ export class ValidatedStorageDriver implements StorageDriver {
|
||||
return this.delegate.getPresignedUrl(params);
|
||||
}
|
||||
|
||||
async getPresignedUploadUrl(params: {
|
||||
filePath: string;
|
||||
contentType: string;
|
||||
contentLength: number;
|
||||
expiresInSeconds?: number;
|
||||
}): Promise<string | null> {
|
||||
assertStoragePathIsSafe(params.filePath);
|
||||
|
||||
return this.delegate.getPresignedUploadUrl(params);
|
||||
}
|
||||
|
||||
async checkFileExists(params: { filePath: string }): Promise<boolean> {
|
||||
assertStoragePathIsSafe(params.filePath);
|
||||
|
||||
|
||||
@@ -19,6 +19,7 @@ import { validateFolderPath } from 'src/engine/core-modules/file-storage/utils/v
|
||||
import { validateStoragePathIsWithinWorkspaceOrThrow } from 'src/engine/core-modules/file-storage/utils/validate-storage-path-is-within-workspace-or-throw.util';
|
||||
import { FileEntity } from 'src/engine/core-modules/file/entities/file.entity';
|
||||
import { FileSettings } from 'src/engine/core-modules/file/types/file-settings.types';
|
||||
import { FILE_STATUS } from 'src/engine/core-modules/file/types/file-status.types';
|
||||
import { removeFileFolderFromFileEntityPath } from 'src/engine/core-modules/file/utils/remove-file-folder-from-file-entity-path.utils';
|
||||
import { InjectWorkspaceScopedRepository } from 'src/engine/twenty-orm/workspace-scoped-repository/inject-workspace-scoped-repository.decorator';
|
||||
import { WorkspaceScopedRepository } from 'src/engine/twenty-orm/workspace-scoped-repository/workspace-scoped-repository';
|
||||
@@ -187,6 +188,99 @@ export class FileStorageService {
|
||||
);
|
||||
}
|
||||
|
||||
// Creates the file record ahead of a direct client upload. The bytes are
|
||||
// not in storage yet: the record stays PENDING until the upload is
|
||||
// confirmed (completeFileUpload) or reaped by the cleanup cron.
|
||||
async createPendingFile({
|
||||
fileFolder,
|
||||
applicationUniversalIdentifier,
|
||||
workspaceId,
|
||||
resourcePath,
|
||||
fileId,
|
||||
size,
|
||||
mimeType,
|
||||
settings,
|
||||
}: ResourceIdentifier & {
|
||||
fileId: string;
|
||||
size: number;
|
||||
mimeType: string;
|
||||
settings: FileSettings;
|
||||
}): Promise<FileEntity> {
|
||||
const application = await this.applicationRepository.findOneOrFail({
|
||||
where: {
|
||||
universalIdentifier: applicationUniversalIdentifier,
|
||||
workspaceId,
|
||||
},
|
||||
});
|
||||
|
||||
const { filePath } = this.validateAndBuildFileStoragePathOrThrow({
|
||||
workspaceId,
|
||||
applicationUniversalIdentifier,
|
||||
fileFolder,
|
||||
resourcePath,
|
||||
});
|
||||
|
||||
return this.fileRepository.upsertAndReturnOne(
|
||||
workspaceId,
|
||||
{
|
||||
path: filePath,
|
||||
applicationId: application.id,
|
||||
id: fileId,
|
||||
mimeType,
|
||||
size,
|
||||
settings,
|
||||
status: FILE_STATUS.PENDING,
|
||||
},
|
||||
['path', 'workspaceId', 'applicationId'],
|
||||
);
|
||||
}
|
||||
|
||||
async writeFileStream(
|
||||
params: ResourceIdentifier & {
|
||||
stream: Readable;
|
||||
mimeType: string | undefined;
|
||||
},
|
||||
): Promise<void> {
|
||||
const driver = this.fileStorageDriverFactory.getCurrentDriver();
|
||||
const { onStorageFilePath } =
|
||||
this.validateAndBuildFileStoragePathOrThrow(params);
|
||||
|
||||
return driver.writeFileStream({
|
||||
filePath: onStorageFilePath,
|
||||
stream: params.stream,
|
||||
mimeType: params.mimeType,
|
||||
});
|
||||
}
|
||||
|
||||
async getFileMetadata(
|
||||
params: ResourceIdentifier,
|
||||
): Promise<{ size: number } | null> {
|
||||
const driver = this.fileStorageDriverFactory.getCurrentDriver();
|
||||
const { onStorageFilePath } =
|
||||
this.validateAndBuildFileStoragePathOrThrow(params);
|
||||
|
||||
return driver.getFileMetadata({ filePath: onStorageFilePath });
|
||||
}
|
||||
|
||||
async getPresignedUploadUrl(
|
||||
params: ResourceIdentifier & {
|
||||
contentType: string;
|
||||
contentLength: number;
|
||||
expiresInSeconds?: number;
|
||||
},
|
||||
): Promise<string | null> {
|
||||
const driver = this.fileStorageDriverFactory.getCurrentDriver();
|
||||
const { onStorageFilePath } =
|
||||
this.validateAndBuildFileStoragePathOrThrow(params);
|
||||
|
||||
return driver.getPresignedUploadUrl({
|
||||
filePath: onStorageFilePath,
|
||||
contentType: params.contentType,
|
||||
contentLength: params.contentLength,
|
||||
expiresInSeconds: params.expiresInSeconds,
|
||||
});
|
||||
}
|
||||
|
||||
async getPresignedUrl(
|
||||
params: ResourceIdentifier & {
|
||||
expiresInSeconds?: number;
|
||||
|
||||
@@ -12,12 +12,19 @@ import {
|
||||
UpdateDateColumn,
|
||||
} from 'typeorm';
|
||||
|
||||
import { ADD_STATUS_TO_FILE_UPGRADE_COMMAND_NAME } from 'src/database/commands/upgrade-version-command/2-19/add-status-to-file-upgrade-command-name.constant';
|
||||
import { ApplicationEntity } from 'src/engine/core-modules/application/application.entity';
|
||||
import { FileSettings } from 'src/engine/core-modules/file/types/file-settings.types';
|
||||
import {
|
||||
FILE_STATUS,
|
||||
FileStatus,
|
||||
} from 'src/engine/core-modules/file/types/file-status.types';
|
||||
import { WasIntroducedInUpgrade } from 'src/engine/core-modules/upgrade/decorators/was-introduced-in-upgrade.decorator';
|
||||
import { WorkspaceRelatedEntity } from 'src/engine/workspace-manager/types/workspace-related-entity';
|
||||
|
||||
@Entity('file')
|
||||
@Index('IDX_FILE_WORKSPACE_ID', ['workspaceId'])
|
||||
@Index('IDX_FILE_STATUS', ['status'])
|
||||
@Unique('IDX_APPLICATION_PATH_WORKSPACE_ID_APPLICATION_ID_UNIQUE', [
|
||||
'workspaceId',
|
||||
'applicationId',
|
||||
@@ -63,4 +70,15 @@ export class FileEntity extends WorkspaceRelatedEntity {
|
||||
default: 'application/octet-stream',
|
||||
})
|
||||
mimeType: string;
|
||||
|
||||
@WasIntroducedInUpgrade({
|
||||
upgradeCommandName: ADD_STATUS_TO_FILE_UPGRADE_COMMAND_NAME,
|
||||
})
|
||||
@Column({
|
||||
type: 'enum',
|
||||
enum: Object.values(FILE_STATUS),
|
||||
nullable: false,
|
||||
default: FILE_STATUS.UPLOADED,
|
||||
})
|
||||
status: FileStatus;
|
||||
}
|
||||
|
||||
+44
@@ -0,0 +1,44 @@
|
||||
import {
|
||||
Controller,
|
||||
Param,
|
||||
Put,
|
||||
Req,
|
||||
Res,
|
||||
UseFilters,
|
||||
UseGuards,
|
||||
} from '@nestjs/common';
|
||||
|
||||
import { Request, Response } from 'express';
|
||||
|
||||
import { FileUploadApiExceptionFilter } from 'src/engine/core-modules/file/file-upload/filters/file-upload-api-exception.filter';
|
||||
import { FileUploadTokenGuard } from 'src/engine/core-modules/file/file-upload/guards/file-upload-token.guard';
|
||||
import { FileUploadService } from 'src/engine/core-modules/file/file-upload/services/file-upload.service';
|
||||
import { NoPermissionGuard } from 'src/engine/guards/no-permission.guard';
|
||||
|
||||
@Controller()
|
||||
@UseFilters(FileUploadApiExceptionFilter)
|
||||
export class FileUploadController {
|
||||
constructor(private readonly fileUploadService: FileUploadService) {}
|
||||
|
||||
// Streaming target for direct uploads when storage has no presigned upload
|
||||
// support (local driver, or S3 without presign enabled). The body is piped
|
||||
// to the storage driver without ever being buffered in memory.
|
||||
@Put('file-upload/:id')
|
||||
@UseGuards(FileUploadTokenGuard, NoPermissionGuard)
|
||||
async uploadFileById(
|
||||
@Req() req: Request,
|
||||
@Res() res: Response,
|
||||
@Param('id') fileId: string,
|
||||
) {
|
||||
// oxlint-disable-next-line typescript/no-explicit-any
|
||||
const workspaceId = (req as any)?.workspaceId;
|
||||
|
||||
await this.fileUploadService.receiveFileStream({
|
||||
workspaceId,
|
||||
fileId,
|
||||
stream: req,
|
||||
});
|
||||
|
||||
res.status(204).send();
|
||||
}
|
||||
}
|
||||
+19
@@ -0,0 +1,19 @@
|
||||
import { Field, ObjectType } from '@nestjs/graphql';
|
||||
|
||||
import { UUIDScalarType } from 'src/engine/api/graphql/workspace-schema-builder/graphql-types/scalars';
|
||||
|
||||
@ObjectType('FileUploadTarget')
|
||||
export class FileUploadTargetDTO {
|
||||
@Field(() => UUIDScalarType)
|
||||
fileId: string;
|
||||
|
||||
@Field()
|
||||
uploadUrl: string;
|
||||
|
||||
// Content-Type header the client must send when uploading to uploadUrl
|
||||
@Field()
|
||||
contentType: string;
|
||||
|
||||
@Field(() => Date, { nullable: false })
|
||||
expiresAt: Date;
|
||||
}
|
||||
+23
@@ -0,0 +1,23 @@
|
||||
import { type MessageDescriptor } from '@lingui/core';
|
||||
|
||||
import { CustomException } from 'src/utils/custom-exception';
|
||||
|
||||
export enum FileUploadExceptionCode {
|
||||
BAD_REQUEST = 'BAD_REQUEST',
|
||||
FILE_NOT_FOUND = 'FILE_NOT_FOUND',
|
||||
FILE_NOT_UPLOADED = 'FILE_NOT_UPLOADED',
|
||||
FILE_SIZE_MISMATCH = 'FILE_SIZE_MISMATCH',
|
||||
FILE_TOO_LARGE = 'FILE_TOO_LARGE',
|
||||
}
|
||||
|
||||
export class FileUploadException extends CustomException<FileUploadExceptionCode> {
|
||||
constructor(
|
||||
message: string,
|
||||
code: FileUploadExceptionCode,
|
||||
{ userFriendlyMessage }: { userFriendlyMessage: MessageDescriptor },
|
||||
) {
|
||||
super(message, code, {
|
||||
userFriendlyMessage,
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
import { Module } from '@nestjs/common';
|
||||
import { TypeOrmModule } from '@nestjs/typeorm';
|
||||
|
||||
import { ApplicationEntity } from 'src/engine/core-modules/application/application.entity';
|
||||
import { ApplicationModule } from 'src/engine/core-modules/application/application.module';
|
||||
import { FileStorageModule } from 'src/engine/core-modules/file-storage/file-storage.module';
|
||||
import { FileEntity } from 'src/engine/core-modules/file/entities/file.entity';
|
||||
import { FileUploadController } from 'src/engine/core-modules/file/file-upload/controllers/file-upload.controller';
|
||||
import { FileUploadTokenGuard } from 'src/engine/core-modules/file/file-upload/guards/file-upload-token.guard';
|
||||
import { FileUploadResolver } from 'src/engine/core-modules/file/file-upload/resolvers/file-upload.resolver';
|
||||
import { FileUploadService } from 'src/engine/core-modules/file/file-upload/services/file-upload.service';
|
||||
import { FileUrlModule } from 'src/engine/core-modules/file/file-url/file-url.module';
|
||||
import { JwtModule } from 'src/engine/core-modules/jwt/jwt.module';
|
||||
import { FieldMetadataEntity } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity';
|
||||
import { PermissionsModule } from 'src/engine/metadata-modules/permissions/permissions.module';
|
||||
import { provideWorkspaceScopedRepository } from 'src/engine/twenty-orm/workspace-scoped-repository/provide-workspace-scoped-repository';
|
||||
|
||||
@Module({
|
||||
imports: [
|
||||
JwtModule,
|
||||
TypeOrmModule.forFeature([
|
||||
FileEntity,
|
||||
ApplicationEntity,
|
||||
FieldMetadataEntity,
|
||||
]),
|
||||
PermissionsModule,
|
||||
FileStorageModule,
|
||||
FileUrlModule,
|
||||
ApplicationModule,
|
||||
],
|
||||
providers: [
|
||||
FileUploadService,
|
||||
FileUploadResolver,
|
||||
FileUploadTokenGuard,
|
||||
provideWorkspaceScopedRepository(FileEntity),
|
||||
],
|
||||
exports: [FileUploadService],
|
||||
controllers: [FileUploadController],
|
||||
})
|
||||
export class FileUploadModule {}
|
||||
+54
@@ -0,0 +1,54 @@
|
||||
import {
|
||||
type ArgumentsHost,
|
||||
Catch,
|
||||
type ExceptionFilter,
|
||||
} from '@nestjs/common';
|
||||
|
||||
import { type Response } from 'express';
|
||||
|
||||
import { HttpExceptionHandlerService } from 'src/engine/core-modules/exception-handler/http-exception-handler.service';
|
||||
import {
|
||||
FileUploadException,
|
||||
FileUploadExceptionCode,
|
||||
} from 'src/engine/core-modules/file/file-upload/file-upload.exception';
|
||||
|
||||
@Catch(FileUploadException)
|
||||
export class FileUploadApiExceptionFilter implements ExceptionFilter {
|
||||
constructor(
|
||||
private readonly httpExceptionHandlerService: HttpExceptionHandlerService,
|
||||
) {}
|
||||
|
||||
catch(exception: FileUploadException, host: ArgumentsHost) {
|
||||
const ctx = host.switchToHttp();
|
||||
const response = ctx.getResponse<Response>();
|
||||
|
||||
switch (exception.code) {
|
||||
case FileUploadExceptionCode.FILE_NOT_FOUND:
|
||||
return this.httpExceptionHandlerService.handleError(
|
||||
exception,
|
||||
response,
|
||||
404,
|
||||
);
|
||||
case FileUploadExceptionCode.FILE_TOO_LARGE:
|
||||
return this.httpExceptionHandlerService.handleError(
|
||||
exception,
|
||||
response,
|
||||
413,
|
||||
);
|
||||
case FileUploadExceptionCode.BAD_REQUEST:
|
||||
case FileUploadExceptionCode.FILE_NOT_UPLOADED:
|
||||
case FileUploadExceptionCode.FILE_SIZE_MISMATCH:
|
||||
return this.httpExceptionHandlerService.handleError(
|
||||
exception,
|
||||
response,
|
||||
400,
|
||||
);
|
||||
default:
|
||||
return this.httpExceptionHandlerService.handleError(
|
||||
exception,
|
||||
response,
|
||||
500,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
+42
@@ -0,0 +1,42 @@
|
||||
import { CanActivate, ExecutionContext, Injectable } from '@nestjs/common';
|
||||
|
||||
import { FileUploadTokenJwtPayload } from 'src/engine/core-modules/auth/types/file-upload-token-jwt-payload.type';
|
||||
import { JwtTokenTypeEnum } from 'src/engine/core-modules/auth/types/jwt-token-type.enum';
|
||||
import { JwtWrapperService } from 'src/engine/core-modules/jwt/services/jwt-wrapper.service';
|
||||
|
||||
@Injectable()
|
||||
export class FileUploadTokenGuard implements CanActivate {
|
||||
constructor(private readonly jwtWrapperService: JwtWrapperService) {}
|
||||
|
||||
async canActivate(context: ExecutionContext): Promise<boolean> {
|
||||
const request = context.switchToHttp().getRequest();
|
||||
const fileId = request.params.id;
|
||||
const uploadToken = request.query.token;
|
||||
|
||||
if (!uploadToken) {
|
||||
return false;
|
||||
}
|
||||
|
||||
let payload: FileUploadTokenJwtPayload;
|
||||
|
||||
try {
|
||||
payload = await this.jwtWrapperService.verifyJwtToken(uploadToken);
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
|
||||
// A FILE (download) token also carries workspaceId + fileId: reject
|
||||
// anything that is not explicitly an upload token.
|
||||
if (payload.type !== JwtTokenTypeEnum.FILE_UPLOAD) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if (!payload.workspaceId || payload.fileId !== fileId) {
|
||||
return false;
|
||||
}
|
||||
|
||||
request.workspaceId = payload.workspaceId;
|
||||
|
||||
return true;
|
||||
}
|
||||
}
|
||||
+68
@@ -0,0 +1,68 @@
|
||||
import { UseFilters, UseGuards, UsePipes } from '@nestjs/common';
|
||||
import { Args, Mutation } from '@nestjs/graphql';
|
||||
|
||||
import { PermissionFlagType } from 'twenty-shared/constants';
|
||||
import { FileFolder } from 'twenty-shared/types';
|
||||
|
||||
import { MetadataResolver } from 'src/engine/api/graphql/graphql-config/decorators/metadata-resolver.decorator';
|
||||
import { FileWithSignedUrlDTO } from 'src/engine/core-modules/file/dtos/file-with-sign-url.dto';
|
||||
import { FileUploadTargetDTO } from 'src/engine/core-modules/file/file-upload/dtos/file-upload-target.dto';
|
||||
import { FileUploadService } from 'src/engine/core-modules/file/file-upload/services/file-upload.service';
|
||||
import { PreventNestToAutoLogGraphqlErrorsFilter } from 'src/engine/core-modules/graphql/filters/prevent-nest-to-auto-log-graphql-errors.filter';
|
||||
import { ResolverValidationPipe } from 'src/engine/core-modules/graphql/pipes/resolver-validation.pipe';
|
||||
import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity';
|
||||
import { AuthWorkspace } from 'src/engine/decorators/auth/auth-workspace.decorator';
|
||||
import { SettingsPermissionGuard } from 'src/engine/guards/settings-permission.guard';
|
||||
import { WorkspaceAuthGuard } from 'src/engine/guards/workspace-auth.guard';
|
||||
|
||||
@UseGuards(WorkspaceAuthGuard)
|
||||
@UsePipes(ResolverValidationPipe)
|
||||
@UseFilters(PreventNestToAutoLogGraphqlErrorsFilter)
|
||||
@MetadataResolver()
|
||||
export class FileUploadResolver {
|
||||
constructor(private readonly fileUploadService: FileUploadService) {}
|
||||
|
||||
@Mutation(() => FileUploadTargetDTO)
|
||||
@UseGuards(SettingsPermissionGuard(PermissionFlagType.UPLOAD_FILE))
|
||||
async createFileUpload(
|
||||
@AuthWorkspace()
|
||||
{ id: workspaceId }: WorkspaceEntity,
|
||||
@Args({ name: 'filename', type: () => String })
|
||||
filename: string,
|
||||
@Args({ name: 'size', type: () => Number })
|
||||
size: number,
|
||||
@Args({ name: 'fileFolder', type: () => FileFolder })
|
||||
fileFolder: FileFolder,
|
||||
@Args({ name: 'fieldMetadataId', type: () => String, nullable: true })
|
||||
fieldMetadataId?: string,
|
||||
@Args({
|
||||
name: 'fieldMetadataUniversalIdentifier',
|
||||
type: () => String,
|
||||
nullable: true,
|
||||
})
|
||||
fieldMetadataUniversalIdentifier?: string,
|
||||
): Promise<FileUploadTargetDTO> {
|
||||
return await this.fileUploadService.createFileUpload({
|
||||
workspaceId,
|
||||
filename,
|
||||
size,
|
||||
fileFolder,
|
||||
fieldMetadataId,
|
||||
fieldMetadataUniversalIdentifier,
|
||||
});
|
||||
}
|
||||
|
||||
@Mutation(() => FileWithSignedUrlDTO)
|
||||
@UseGuards(SettingsPermissionGuard(PermissionFlagType.UPLOAD_FILE))
|
||||
async completeFileUpload(
|
||||
@AuthWorkspace()
|
||||
{ id: workspaceId }: WorkspaceEntity,
|
||||
@Args({ name: 'fileId', type: () => String })
|
||||
fileId: string,
|
||||
): Promise<FileWithSignedUrlDTO> {
|
||||
return await this.fileUploadService.completeFileUpload({
|
||||
workspaceId,
|
||||
fileId,
|
||||
});
|
||||
}
|
||||
}
|
||||
+309
@@ -0,0 +1,309 @@
|
||||
import { Test, type TestingModule } from '@nestjs/testing';
|
||||
import { getRepositoryToken } from '@nestjs/typeorm';
|
||||
|
||||
import { FileFolder } from 'twenty-shared/types';
|
||||
|
||||
import { ApplicationEntity } from 'src/engine/core-modules/application/application.entity';
|
||||
import { ApplicationService } from 'src/engine/core-modules/application/application.service';
|
||||
import { FileStorageService } from 'src/engine/core-modules/file-storage/file-storage.service';
|
||||
import { FileEntity } from 'src/engine/core-modules/file/entities/file.entity';
|
||||
import {
|
||||
FileUploadException,
|
||||
FileUploadExceptionCode,
|
||||
} from 'src/engine/core-modules/file/file-upload/file-upload.exception';
|
||||
import { FileUploadService } from 'src/engine/core-modules/file/file-upload/services/file-upload.service';
|
||||
import { FileUrlService } from 'src/engine/core-modules/file/file-url/file-url.service';
|
||||
import { FILE_STATUS } from 'src/engine/core-modules/file/types/file-status.types';
|
||||
import { JwtWrapperService } from 'src/engine/core-modules/jwt/services/jwt-wrapper.service';
|
||||
import { TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service';
|
||||
import { FieldMetadataEntity } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity';
|
||||
import { getWorkspaceScopedRepositoryToken } from 'src/engine/twenty-orm/workspace-scoped-repository/get-workspace-scoped-repository-token.util';
|
||||
|
||||
jest.mock('uuid', () => ({
|
||||
v4: jest.fn(() => 'mocked-file-id'),
|
||||
}));
|
||||
|
||||
describe('FileUploadService', () => {
|
||||
let service: FileUploadService;
|
||||
|
||||
const fileStorageService = {
|
||||
createPendingFile: jest.fn(),
|
||||
getPresignedUploadUrl: jest.fn(),
|
||||
getFileMetadata: jest.fn(),
|
||||
writeFileStream: jest.fn(),
|
||||
};
|
||||
|
||||
const fileUrlService = {
|
||||
signFileByIdUrl: jest.fn().mockResolvedValue('https://signed-url'),
|
||||
};
|
||||
|
||||
const jwtWrapperService = {
|
||||
signAsyncOrThrow: jest.fn().mockResolvedValue('upload-token'),
|
||||
};
|
||||
|
||||
const twentyConfigService = {
|
||||
get: jest.fn((key: string) => {
|
||||
if (key === 'STORAGE_S3_PRESIGNED_URL_EXPIRES_IN') {
|
||||
return 900;
|
||||
}
|
||||
if (key === 'SERVER_URL') {
|
||||
return 'https://server.tld';
|
||||
}
|
||||
|
||||
return undefined;
|
||||
}),
|
||||
};
|
||||
|
||||
const applicationService = {
|
||||
findWorkspaceTwentyStandardAndCustomApplicationOrThrow: jest
|
||||
.fn()
|
||||
.mockResolvedValue({
|
||||
workspaceCustomFlatApplication: {
|
||||
universalIdentifier: 'custom-app-uid',
|
||||
},
|
||||
}),
|
||||
};
|
||||
|
||||
const applicationRepository = {
|
||||
findOneOrFail: jest.fn().mockResolvedValue({
|
||||
id: 'application-id',
|
||||
universalIdentifier: 'application-uid',
|
||||
}),
|
||||
};
|
||||
|
||||
const fieldMetadataRepository = {
|
||||
findOneOrFail: jest.fn().mockResolvedValue({
|
||||
applicationId: 'application-id',
|
||||
universalIdentifier: 'field-metadata-uid',
|
||||
}),
|
||||
};
|
||||
|
||||
const fileRepository = {
|
||||
findOne: jest.fn(),
|
||||
update: jest.fn(),
|
||||
};
|
||||
|
||||
beforeEach(async () => {
|
||||
jest.clearAllMocks();
|
||||
|
||||
const module: TestingModule = await Test.createTestingModule({
|
||||
providers: [
|
||||
FileUploadService,
|
||||
{ provide: FileStorageService, useValue: fileStorageService },
|
||||
{ provide: FileUrlService, useValue: fileUrlService },
|
||||
{ provide: JwtWrapperService, useValue: jwtWrapperService },
|
||||
{ provide: TwentyConfigService, useValue: twentyConfigService },
|
||||
{ provide: ApplicationService, useValue: applicationService },
|
||||
{
|
||||
provide: getRepositoryToken(ApplicationEntity),
|
||||
useValue: applicationRepository,
|
||||
},
|
||||
{
|
||||
provide: getRepositoryToken(FieldMetadataEntity),
|
||||
useValue: fieldMetadataRepository,
|
||||
},
|
||||
{
|
||||
provide: getWorkspaceScopedRepositoryToken(FileEntity),
|
||||
useValue: fileRepository,
|
||||
},
|
||||
],
|
||||
}).compile();
|
||||
|
||||
service = module.get<FileUploadService>(FileUploadService);
|
||||
});
|
||||
|
||||
describe('createFileUpload', () => {
|
||||
it('should reject file folders without direct upload support', async () => {
|
||||
await expect(
|
||||
service.createFileUpload({
|
||||
workspaceId: 'workspace-id',
|
||||
filename: 'document.pdf',
|
||||
size: 1024,
|
||||
fileFolder: FileFolder.CorePicture,
|
||||
}),
|
||||
).rejects.toThrow(FileUploadException);
|
||||
});
|
||||
|
||||
it('should reject an invalid declared size', async () => {
|
||||
await expect(
|
||||
service.createFileUpload({
|
||||
workspaceId: 'workspace-id',
|
||||
filename: 'document.pdf',
|
||||
size: 0,
|
||||
fileFolder: FileFolder.FilesField,
|
||||
fieldMetadataId: 'field-metadata-id',
|
||||
}),
|
||||
).rejects.toThrow(FileUploadException);
|
||||
});
|
||||
|
||||
it('should create a PENDING file and return the presigned url when storage supports it', async () => {
|
||||
fileStorageService.getPresignedUploadUrl.mockResolvedValueOnce(
|
||||
'https://bucket/presigned-put',
|
||||
);
|
||||
|
||||
const result = await service.createFileUpload({
|
||||
workspaceId: 'workspace-id',
|
||||
filename: 'document.pdf',
|
||||
size: 1024,
|
||||
fileFolder: FileFolder.FilesField,
|
||||
fieldMetadataId: 'field-metadata-id',
|
||||
});
|
||||
|
||||
expect(fileStorageService.createPendingFile).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
fileId: 'mocked-file-id',
|
||||
size: 1024,
|
||||
mimeType: 'application/pdf',
|
||||
resourcePath: 'field-metadata-uid/mocked-file-id.pdf',
|
||||
settings: { isTemporaryFile: true, toDelete: false },
|
||||
}),
|
||||
);
|
||||
expect(result.uploadUrl).toBe('https://bucket/presigned-put');
|
||||
expect(result.contentType).toBe('application/pdf');
|
||||
expect(result.fileId).toBe('mocked-file-id');
|
||||
});
|
||||
|
||||
it('should fall back to the server streaming endpoint when presign is unavailable', async () => {
|
||||
fileStorageService.getPresignedUploadUrl.mockResolvedValueOnce(null);
|
||||
|
||||
const result = await service.createFileUpload({
|
||||
workspaceId: 'workspace-id',
|
||||
filename: 'archive.zip',
|
||||
size: 2048,
|
||||
fileFolder: FileFolder.Workflow,
|
||||
});
|
||||
|
||||
expect(jwtWrapperService.signAsyncOrThrow).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
workspaceId: 'workspace-id',
|
||||
fileId: 'mocked-file-id',
|
||||
}),
|
||||
{ expiresIn: 900 },
|
||||
);
|
||||
expect(result.uploadUrl).toBe(
|
||||
'https://server.tld/file-upload/mocked-file-id?token=upload-token',
|
||||
);
|
||||
expect(result.contentType).toBe('application/octet-stream');
|
||||
});
|
||||
});
|
||||
|
||||
describe('completeFileUpload', () => {
|
||||
const pendingFile = {
|
||||
id: 'file-id',
|
||||
path: 'files-field/field-metadata-uid/file-id.pdf',
|
||||
size: 1024,
|
||||
applicationId: 'application-id',
|
||||
mimeType: 'application/pdf',
|
||||
status: FILE_STATUS.PENDING,
|
||||
settings: { isTemporaryFile: true, toDelete: false },
|
||||
createdAt: new Date(),
|
||||
};
|
||||
|
||||
it('should throw when the file record does not exist', async () => {
|
||||
fileRepository.findOne.mockResolvedValueOnce(null);
|
||||
|
||||
await expect(
|
||||
service.completeFileUpload({
|
||||
workspaceId: 'workspace-id',
|
||||
fileId: 'file-id',
|
||||
}),
|
||||
).rejects.toThrow(FileUploadException);
|
||||
});
|
||||
|
||||
it('should throw when the bytes are not in storage yet', async () => {
|
||||
fileRepository.findOne.mockResolvedValueOnce(pendingFile);
|
||||
fileStorageService.getFileMetadata.mockResolvedValueOnce(null);
|
||||
|
||||
await expect(
|
||||
service.completeFileUpload({
|
||||
workspaceId: 'workspace-id',
|
||||
fileId: 'file-id',
|
||||
}),
|
||||
).rejects.toMatchObject({
|
||||
code: FileUploadExceptionCode.FILE_NOT_UPLOADED,
|
||||
});
|
||||
expect(fileRepository.update).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('should throw when the stored size does not match the declared size', async () => {
|
||||
fileRepository.findOne.mockResolvedValueOnce(pendingFile);
|
||||
fileStorageService.getFileMetadata.mockResolvedValueOnce({ size: 999 });
|
||||
|
||||
await expect(
|
||||
service.completeFileUpload({
|
||||
workspaceId: 'workspace-id',
|
||||
fileId: 'file-id',
|
||||
}),
|
||||
).rejects.toMatchObject({
|
||||
code: FileUploadExceptionCode.FILE_SIZE_MISMATCH,
|
||||
});
|
||||
expect(fileRepository.update).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('should flip the file to UPLOADED when the stored size matches', async () => {
|
||||
fileRepository.findOne.mockResolvedValueOnce(pendingFile);
|
||||
fileStorageService.getFileMetadata.mockResolvedValueOnce({ size: 1024 });
|
||||
|
||||
const result = await service.completeFileUpload({
|
||||
workspaceId: 'workspace-id',
|
||||
fileId: 'file-id',
|
||||
});
|
||||
|
||||
expect(fileRepository.update).toHaveBeenCalledWith(
|
||||
'workspace-id',
|
||||
{ id: 'file-id' },
|
||||
{ status: FILE_STATUS.UPLOADED },
|
||||
);
|
||||
expect(result.url).toBe('https://signed-url');
|
||||
});
|
||||
|
||||
it('should be idempotent when the file is already UPLOADED', async () => {
|
||||
fileRepository.findOne.mockResolvedValueOnce({
|
||||
...pendingFile,
|
||||
status: FILE_STATUS.UPLOADED,
|
||||
});
|
||||
|
||||
const result = await service.completeFileUpload({
|
||||
workspaceId: 'workspace-id',
|
||||
fileId: 'file-id',
|
||||
});
|
||||
|
||||
expect(fileStorageService.getFileMetadata).not.toHaveBeenCalled();
|
||||
expect(fileRepository.update).not.toHaveBeenCalled();
|
||||
expect(result.url).toBe('https://signed-url');
|
||||
});
|
||||
|
||||
it('should refuse confirming files that already left the upload flow', async () => {
|
||||
fileRepository.findOne.mockResolvedValueOnce({
|
||||
...pendingFile,
|
||||
status: FILE_STATUS.UPLOADED,
|
||||
settings: { isTemporaryFile: false, toDelete: false },
|
||||
});
|
||||
|
||||
await expect(
|
||||
service.completeFileUpload({
|
||||
workspaceId: 'workspace-id',
|
||||
fileId: 'file-id',
|
||||
}),
|
||||
).rejects.toMatchObject({
|
||||
code: FileUploadExceptionCode.BAD_REQUEST,
|
||||
});
|
||||
});
|
||||
|
||||
it('should refuse files outside direct-upload folders', async () => {
|
||||
fileRepository.findOne.mockResolvedValueOnce({
|
||||
...pendingFile,
|
||||
path: 'core-picture/file-id.png',
|
||||
});
|
||||
|
||||
await expect(
|
||||
service.completeFileUpload({
|
||||
workspaceId: 'workspace-id',
|
||||
fileId: 'file-id',
|
||||
}),
|
||||
).rejects.toMatchObject({
|
||||
code: FileUploadExceptionCode.FILE_NOT_FOUND,
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
+511
@@ -0,0 +1,511 @@
|
||||
import { Injectable } from '@nestjs/common';
|
||||
import { InjectRepository } from '@nestjs/typeorm';
|
||||
|
||||
import { type Readable, Transform } from 'stream';
|
||||
import { pipeline } from 'stream/promises';
|
||||
|
||||
import { msg } from '@lingui/core/macro';
|
||||
import { isNonEmptyString } from '@sniptt/guards';
|
||||
import bytes from 'bytes';
|
||||
import { lookup } from 'mrmime';
|
||||
import { FileFolder } from 'twenty-shared/types';
|
||||
import { isDefined } from 'twenty-shared/utils';
|
||||
import { Repository } from 'typeorm';
|
||||
import { v4 } from 'uuid';
|
||||
|
||||
import { settings } from 'src/engine/constants/settings';
|
||||
import { ApplicationEntity } from 'src/engine/core-modules/application/application.entity';
|
||||
import { ApplicationService } from 'src/engine/core-modules/application/application.service';
|
||||
import { FileUploadTokenJwtPayload } from 'src/engine/core-modules/auth/types/file-upload-token-jwt-payload.type';
|
||||
import { JwtTokenTypeEnum } from 'src/engine/core-modules/auth/types/jwt-token-type.enum';
|
||||
import { FileStorageService } from 'src/engine/core-modules/file-storage/file-storage.service';
|
||||
import { TWENTY_MIME_POLICY } from 'src/engine/core-modules/file/constants/twenty-mime-policy.constant';
|
||||
import { FileWithSignedUrlDTO } from 'src/engine/core-modules/file/dtos/file-with-sign-url.dto';
|
||||
import { FileEntity } from 'src/engine/core-modules/file/entities/file.entity';
|
||||
import { FileUploadTargetDTO } from 'src/engine/core-modules/file/file-upload/dtos/file-upload-target.dto';
|
||||
import {
|
||||
FileUploadException,
|
||||
FileUploadExceptionCode,
|
||||
} from 'src/engine/core-modules/file/file-upload/file-upload.exception';
|
||||
import { FileUrlService } from 'src/engine/core-modules/file/file-url/file-url.service';
|
||||
import { FILE_STATUS } from 'src/engine/core-modules/file/types/file-status.types';
|
||||
import { buildFileInfo } from 'src/engine/core-modules/file/utils/build-file-info.utils';
|
||||
import { removeFileFolderFromFileEntityPath } from 'src/engine/core-modules/file/utils/remove-file-folder-from-file-entity-path.utils';
|
||||
import { JwtWrapperService } from 'src/engine/core-modules/jwt/services/jwt-wrapper.service';
|
||||
import { TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service';
|
||||
import { FieldMetadataEntity } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity';
|
||||
import { InjectWorkspaceScopedRepository } from 'src/engine/twenty-orm/workspace-scoped-repository/inject-workspace-scoped-repository.decorator';
|
||||
import { WorkspaceScopedRepository } from 'src/engine/twenty-orm/workspace-scoped-repository/workspace-scoped-repository';
|
||||
|
||||
export const DIRECT_UPLOAD_FILE_FOLDERS = [
|
||||
FileFolder.FilesField,
|
||||
FileFolder.Workflow,
|
||||
] as const;
|
||||
|
||||
@Injectable()
|
||||
export class FileUploadService {
|
||||
constructor(
|
||||
private readonly fileStorageService: FileStorageService,
|
||||
private readonly fileUrlService: FileUrlService,
|
||||
private readonly jwtWrapperService: JwtWrapperService,
|
||||
private readonly twentyConfigService: TwentyConfigService,
|
||||
private readonly applicationService: ApplicationService,
|
||||
@InjectRepository(ApplicationEntity)
|
||||
private readonly applicationRepository: Repository<ApplicationEntity>,
|
||||
@InjectRepository(FieldMetadataEntity)
|
||||
private readonly fieldMetadataRepository: Repository<FieldMetadataEntity>,
|
||||
@InjectWorkspaceScopedRepository(FileEntity)
|
||||
private readonly fileRepository: WorkspaceScopedRepository<FileEntity>,
|
||||
) {}
|
||||
|
||||
async createFileUpload({
|
||||
workspaceId,
|
||||
filename,
|
||||
size,
|
||||
fileFolder,
|
||||
fieldMetadataId,
|
||||
fieldMetadataUniversalIdentifier,
|
||||
}: {
|
||||
workspaceId: string;
|
||||
filename: string;
|
||||
size: number;
|
||||
fileFolder: FileFolder;
|
||||
fieldMetadataId?: string;
|
||||
fieldMetadataUniversalIdentifier?: string;
|
||||
}): Promise<FileUploadTargetDTO> {
|
||||
if (
|
||||
!DIRECT_UPLOAD_FILE_FOLDERS.includes(
|
||||
fileFolder as (typeof DIRECT_UPLOAD_FILE_FOLDERS)[number],
|
||||
)
|
||||
) {
|
||||
throw new FileUploadException(
|
||||
`Direct upload is not supported for file folder ${fileFolder}`,
|
||||
FileUploadExceptionCode.BAD_REQUEST,
|
||||
{
|
||||
userFriendlyMessage: msg`Direct upload is not supported for this file type.`,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
const maxFileSize = bytes(settings.storage.maxDirectUploadFileSize) ?? 0;
|
||||
|
||||
if (!Number.isInteger(size) || size <= 0 || size > maxFileSize) {
|
||||
throw new FileUploadException(
|
||||
`Invalid file size ${size} (max ${maxFileSize} bytes)`,
|
||||
FileUploadExceptionCode.FILE_TOO_LARGE,
|
||||
{
|
||||
userFriendlyMessage: msg`The file is empty or exceeds the maximum allowed size.`,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
const { ext } = buildFileInfo(filename);
|
||||
// The file content cannot be sniffed before it reaches storage, so the
|
||||
// mime type is derived from the extension only. Anything unknown is
|
||||
// stored as octet-stream, and the serving path already forces
|
||||
// Content-Disposition: attachment for non-inline-safe mime types.
|
||||
const mimeType =
|
||||
TWENTY_MIME_POLICY[ext] ??
|
||||
(isNonEmptyString(ext) ? lookup(ext) : undefined) ??
|
||||
'application/octet-stream';
|
||||
|
||||
const fileId = v4();
|
||||
const name = `${fileId}${isNonEmptyString(ext) ? `.${ext}` : ''}`;
|
||||
|
||||
const { applicationUniversalIdentifier, resourcePath } =
|
||||
await this.resolveUploadLocation({
|
||||
workspaceId,
|
||||
fileFolder,
|
||||
name,
|
||||
fieldMetadataId,
|
||||
fieldMetadataUniversalIdentifier,
|
||||
});
|
||||
|
||||
await this.fileStorageService.createPendingFile({
|
||||
fileFolder,
|
||||
applicationUniversalIdentifier,
|
||||
workspaceId,
|
||||
resourcePath,
|
||||
fileId,
|
||||
size,
|
||||
mimeType,
|
||||
settings: {
|
||||
isTemporaryFile: true,
|
||||
toDelete: false,
|
||||
},
|
||||
});
|
||||
|
||||
const expiresInSeconds = this.twentyConfigService.get(
|
||||
'STORAGE_S3_PRESIGNED_URL_EXPIRES_IN',
|
||||
);
|
||||
const expiresAt = new Date(Date.now() + expiresInSeconds * 1000);
|
||||
|
||||
const presignedUploadUrl =
|
||||
await this.fileStorageService.getPresignedUploadUrl({
|
||||
fileFolder,
|
||||
applicationUniversalIdentifier,
|
||||
workspaceId,
|
||||
resourcePath,
|
||||
contentType: mimeType,
|
||||
contentLength: size,
|
||||
expiresInSeconds,
|
||||
});
|
||||
|
||||
if (isDefined(presignedUploadUrl)) {
|
||||
return {
|
||||
fileId,
|
||||
uploadUrl: presignedUploadUrl,
|
||||
contentType: mimeType,
|
||||
expiresAt,
|
||||
};
|
||||
}
|
||||
|
||||
// No presign support (local storage, or S3 without presign enabled):
|
||||
// fall back to the token-authenticated streaming endpoint on the server.
|
||||
const payload: FileUploadTokenJwtPayload = {
|
||||
workspaceId,
|
||||
fileId,
|
||||
sub: workspaceId,
|
||||
type: JwtTokenTypeEnum.FILE_UPLOAD,
|
||||
};
|
||||
|
||||
const token = await this.jwtWrapperService.signAsyncOrThrow(payload, {
|
||||
expiresIn: expiresInSeconds,
|
||||
});
|
||||
|
||||
const serverUrl = this.twentyConfigService.get('SERVER_URL');
|
||||
|
||||
return {
|
||||
fileId,
|
||||
uploadUrl: `${serverUrl}/file-upload/${fileId}?token=${token}`,
|
||||
// octet-stream keeps the request body away from the server's json/text
|
||||
// body parsers; the real mime type is already on the file record.
|
||||
contentType: 'application/octet-stream',
|
||||
expiresAt,
|
||||
};
|
||||
}
|
||||
|
||||
// Streams the request body straight to the storage driver, bounded by the
|
||||
// size declared at createFileUpload time. Memory usage stays constant no
|
||||
// matter how large the file is.
|
||||
async receiveFileStream({
|
||||
workspaceId,
|
||||
fileId,
|
||||
stream,
|
||||
}: {
|
||||
workspaceId: string;
|
||||
fileId: string;
|
||||
stream: Readable;
|
||||
}): Promise<void> {
|
||||
const file = await this.findFileOrThrow({ workspaceId, fileId });
|
||||
|
||||
if (file.status !== FILE_STATUS.PENDING) {
|
||||
throw new FileUploadException(
|
||||
`File ${fileId} is not awaiting an upload`,
|
||||
FileUploadExceptionCode.BAD_REQUEST,
|
||||
{
|
||||
userFriendlyMessage: msg`This file has already been uploaded.`,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
const { application, fileFolder, resourcePath } =
|
||||
await this.resolveFileLocation({ workspaceId, file });
|
||||
|
||||
const declaredSize = Number(file.size);
|
||||
let receivedBytes = 0;
|
||||
|
||||
const sizeLimiter = new Transform({
|
||||
transform: (chunk: Buffer, _encoding, callback) => {
|
||||
receivedBytes += chunk.length;
|
||||
|
||||
if (receivedBytes > declaredSize) {
|
||||
callback(
|
||||
new FileUploadException(
|
||||
`Upload exceeds declared size of ${declaredSize} bytes`,
|
||||
FileUploadExceptionCode.FILE_TOO_LARGE,
|
||||
{
|
||||
userFriendlyMessage: msg`The uploaded file is larger than declared.`,
|
||||
},
|
||||
),
|
||||
);
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
callback(null, chunk);
|
||||
},
|
||||
});
|
||||
|
||||
try {
|
||||
await Promise.all([
|
||||
pipeline(stream, sizeLimiter),
|
||||
this.fileStorageService.writeFileStream({
|
||||
fileFolder,
|
||||
applicationUniversalIdentifier: application.universalIdentifier,
|
||||
workspaceId,
|
||||
resourcePath,
|
||||
stream: sizeLimiter,
|
||||
mimeType: file.mimeType,
|
||||
}),
|
||||
]);
|
||||
} catch (error) {
|
||||
// The storage-side pipeline can lose the rejection race to the limiter:
|
||||
// surface the size violation over the resulting stream teardown error.
|
||||
if (receivedBytes > declaredSize) {
|
||||
throw new FileUploadException(
|
||||
`Upload exceeds declared size of ${declaredSize} bytes`,
|
||||
FileUploadExceptionCode.FILE_TOO_LARGE,
|
||||
{
|
||||
userFriendlyMessage: msg`The uploaded file is larger than declared.`,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
throw error;
|
||||
}
|
||||
|
||||
if (receivedBytes !== declaredSize) {
|
||||
// The short object stays in storage but the record stays PENDING, so it
|
||||
// can never be served or attached: the client retries against the same
|
||||
// upload url (overwriting it), and the pending-file cleanup cron
|
||||
// (follow-up PR) reaps whatever is abandoned.
|
||||
throw new FileUploadException(
|
||||
`Uploaded ${receivedBytes} bytes but ${declaredSize} were declared`,
|
||||
FileUploadExceptionCode.FILE_SIZE_MISMATCH,
|
||||
{
|
||||
userFriendlyMessage: msg`The uploaded file does not match the declared size. Please retry the upload.`,
|
||||
},
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// Verifies the bytes actually landed in storage with the declared size and
|
||||
// flips the file to UPLOADED. Idempotent: confirming twice is a no-op.
|
||||
async completeFileUpload({
|
||||
workspaceId,
|
||||
fileId,
|
||||
}: {
|
||||
workspaceId: string;
|
||||
fileId: string;
|
||||
}): Promise<FileWithSignedUrlDTO> {
|
||||
const file = await this.findFileOrThrow({ workspaceId, fileId });
|
||||
const [fileFolder] = file.path.split('/');
|
||||
|
||||
// Restrict to files created through createFileUpload so this mutation
|
||||
// cannot be used to mint signed download urls for arbitrary files.
|
||||
if (
|
||||
!DIRECT_UPLOAD_FILE_FOLDERS.includes(
|
||||
fileFolder as (typeof DIRECT_UPLOAD_FILE_FOLDERS)[number],
|
||||
)
|
||||
) {
|
||||
throw new FileUploadException(
|
||||
`File not found: ${fileId}`,
|
||||
FileUploadExceptionCode.FILE_NOT_FOUND,
|
||||
{
|
||||
userFriendlyMessage: msg`File not found.`,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
if (file.status === FILE_STATUS.UPLOADED) {
|
||||
// Idempotent retry of a confirm that already succeeded. Only files not
|
||||
// yet attached to a record qualify: this cannot be used to mint signed
|
||||
// urls for files that went through the legacy flow and got attached.
|
||||
if (!file.settings?.isTemporaryFile) {
|
||||
throw new FileUploadException(
|
||||
`File ${fileId} is not awaiting an upload confirmation`,
|
||||
FileUploadExceptionCode.BAD_REQUEST,
|
||||
{
|
||||
userFriendlyMessage: msg`This file upload has already been finalized.`,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
return this.toFileWithSignedUrl({
|
||||
file,
|
||||
fileFolder: fileFolder as FileFolder,
|
||||
workspaceId,
|
||||
});
|
||||
}
|
||||
|
||||
const { application, resourcePath } = await this.resolveFileLocation({
|
||||
workspaceId,
|
||||
file,
|
||||
});
|
||||
|
||||
const metadata = await this.fileStorageService.getFileMetadata({
|
||||
fileFolder: fileFolder as FileFolder,
|
||||
applicationUniversalIdentifier: application.universalIdentifier,
|
||||
workspaceId,
|
||||
resourcePath,
|
||||
});
|
||||
|
||||
if (!isDefined(metadata)) {
|
||||
throw new FileUploadException(
|
||||
`File ${fileId} has not been uploaded to storage yet`,
|
||||
FileUploadExceptionCode.FILE_NOT_UPLOADED,
|
||||
{
|
||||
userFriendlyMessage: msg`The file has not been uploaded yet. Please upload it before confirming.`,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
if (metadata.size !== Number(file.size)) {
|
||||
throw new FileUploadException(
|
||||
`File ${fileId} has ${metadata.size} bytes in storage but ${file.size} were declared`,
|
||||
FileUploadExceptionCode.FILE_SIZE_MISMATCH,
|
||||
{
|
||||
userFriendlyMessage: msg`The uploaded file does not match the declared size. Please retry the upload.`,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
await this.fileRepository.update(
|
||||
workspaceId,
|
||||
{ id: fileId },
|
||||
{ status: FILE_STATUS.UPLOADED },
|
||||
);
|
||||
|
||||
return this.toFileWithSignedUrl({
|
||||
file: { ...file, status: FILE_STATUS.UPLOADED },
|
||||
fileFolder: fileFolder as FileFolder,
|
||||
workspaceId,
|
||||
});
|
||||
}
|
||||
|
||||
private async resolveUploadLocation({
|
||||
workspaceId,
|
||||
fileFolder,
|
||||
name,
|
||||
fieldMetadataId,
|
||||
fieldMetadataUniversalIdentifier,
|
||||
}: {
|
||||
workspaceId: string;
|
||||
fileFolder: FileFolder;
|
||||
name: string;
|
||||
fieldMetadataId?: string;
|
||||
fieldMetadataUniversalIdentifier?: string;
|
||||
}): Promise<{
|
||||
applicationUniversalIdentifier: string;
|
||||
resourcePath: string;
|
||||
}> {
|
||||
if (fileFolder === FileFolder.FilesField) {
|
||||
if (!fieldMetadataId && !fieldMetadataUniversalIdentifier) {
|
||||
throw new FileUploadException(
|
||||
'fieldMetadataId or fieldMetadataUniversalIdentifier must be provided',
|
||||
FileUploadExceptionCode.BAD_REQUEST,
|
||||
{
|
||||
userFriendlyMessage: msg`fieldMetadataId or fieldMetadataUniversalIdentifier must be provided`,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
const fieldMetadata = await this.fieldMetadataRepository.findOneOrFail({
|
||||
select: ['applicationId', 'universalIdentifier'],
|
||||
where: {
|
||||
...(fieldMetadataId ? { id: fieldMetadataId } : {}),
|
||||
...(fieldMetadataUniversalIdentifier
|
||||
? { universalIdentifier: fieldMetadataUniversalIdentifier }
|
||||
: {}),
|
||||
workspaceId,
|
||||
},
|
||||
});
|
||||
|
||||
const application = await this.applicationRepository.findOneOrFail({
|
||||
where: {
|
||||
id: fieldMetadata.applicationId,
|
||||
workspaceId,
|
||||
},
|
||||
});
|
||||
|
||||
return {
|
||||
applicationUniversalIdentifier: application.universalIdentifier,
|
||||
resourcePath: `${fieldMetadata.universalIdentifier}/${name}`,
|
||||
};
|
||||
}
|
||||
|
||||
const { workspaceCustomFlatApplication } =
|
||||
await this.applicationService.findWorkspaceTwentyStandardAndCustomApplicationOrThrow(
|
||||
{
|
||||
workspaceId,
|
||||
},
|
||||
);
|
||||
|
||||
return {
|
||||
applicationUniversalIdentifier:
|
||||
workspaceCustomFlatApplication.universalIdentifier,
|
||||
resourcePath: name,
|
||||
};
|
||||
}
|
||||
|
||||
private async findFileOrThrow({
|
||||
workspaceId,
|
||||
fileId,
|
||||
}: {
|
||||
workspaceId: string;
|
||||
fileId: string;
|
||||
}): Promise<FileEntity> {
|
||||
const file = await this.fileRepository.findOne(workspaceId, {
|
||||
where: { id: fileId },
|
||||
});
|
||||
|
||||
if (!isDefined(file)) {
|
||||
throw new FileUploadException(
|
||||
`File not found: ${fileId}`,
|
||||
FileUploadExceptionCode.FILE_NOT_FOUND,
|
||||
{
|
||||
userFriendlyMessage: msg`File not found.`,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
return file;
|
||||
}
|
||||
|
||||
private async resolveFileLocation({
|
||||
workspaceId,
|
||||
file,
|
||||
}: {
|
||||
workspaceId: string;
|
||||
file: FileEntity;
|
||||
}): Promise<{
|
||||
application: ApplicationEntity;
|
||||
fileFolder: FileFolder;
|
||||
resourcePath: string;
|
||||
}> {
|
||||
const [fileFolder] = file.path.split('/');
|
||||
|
||||
const application = await this.applicationRepository.findOneOrFail({
|
||||
where: {
|
||||
id: file.applicationId,
|
||||
workspaceId,
|
||||
},
|
||||
});
|
||||
|
||||
return {
|
||||
application,
|
||||
fileFolder: fileFolder as FileFolder,
|
||||
resourcePath: removeFileFolderFromFileEntityPath(file.path),
|
||||
};
|
||||
}
|
||||
|
||||
private async toFileWithSignedUrl({
|
||||
file,
|
||||
fileFolder,
|
||||
workspaceId,
|
||||
}: {
|
||||
file: FileEntity;
|
||||
fileFolder: FileFolder;
|
||||
workspaceId: string;
|
||||
}): Promise<FileWithSignedUrlDTO> {
|
||||
return {
|
||||
...file,
|
||||
url: await this.fileUrlService.signFileByIdUrl({
|
||||
fileId: file.id,
|
||||
workspaceId,
|
||||
fileFolder,
|
||||
}),
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -15,6 +15,7 @@ import { FileController } from './controllers/file.controller';
|
||||
import { FileEntity } from './entities/file.entity';
|
||||
import { FileCorePictureModule } from './file-core-picture/file-core-picture.module';
|
||||
import { FileEmailAttachmentModule } from './file-email-attachment/file-email-attachment.module';
|
||||
import { FileUploadModule } from './file-upload/file-upload.module';
|
||||
import { FileUrlModule } from './file-url/file-url.module';
|
||||
import { FileWorkflowModule } from './file-workflow/file-workflow.module';
|
||||
import { FilesFieldModule } from './files-field/files-field.module';
|
||||
@@ -33,6 +34,7 @@ import { FileService } from './services/file.service';
|
||||
FileWorkflowModule,
|
||||
FileAiChatModule,
|
||||
FileEmailAttachmentModule,
|
||||
FileUploadModule,
|
||||
SecureHttpClientModule,
|
||||
],
|
||||
providers: [
|
||||
@@ -50,6 +52,7 @@ import { FileService } from './services/file.service';
|
||||
FileWorkflowModule,
|
||||
FileAiChatModule,
|
||||
FileEmailAttachmentModule,
|
||||
FileUploadModule,
|
||||
],
|
||||
controllers: [FileController],
|
||||
})
|
||||
|
||||
@@ -18,6 +18,7 @@ import {
|
||||
IMMUTABLE_FILE_CACHE_CONTROL,
|
||||
} from 'src/engine/core-modules/file/interfaces/file-folder.interface';
|
||||
import { type FileResponse } from 'src/engine/core-modules/file/types/file-response.type';
|
||||
import { FILE_STATUS } from 'src/engine/core-modules/file/types/file-status.types';
|
||||
import { getContentDisposition } from 'src/engine/core-modules/file/utils/get-content-disposition.utils';
|
||||
import { removeFileFolderFromFileEntityPath } from 'src/engine/core-modules/file/utils/remove-file-folder-from-file-entity-path.utils';
|
||||
import { JwtWrapperService } from 'src/engine/core-modules/jwt/services/jwt-wrapper.service';
|
||||
@@ -66,6 +67,7 @@ export class FileService {
|
||||
where: {
|
||||
path: `${fileFolder}/${filepath}`,
|
||||
applicationId,
|
||||
status: FILE_STATUS.UPLOADED,
|
||||
},
|
||||
});
|
||||
|
||||
@@ -94,6 +96,7 @@ export class FileService {
|
||||
const file = await this.fileRepository.findOne(workspaceId, {
|
||||
where: {
|
||||
id: fileId,
|
||||
status: FILE_STATUS.UPLOADED,
|
||||
},
|
||||
});
|
||||
|
||||
@@ -155,6 +158,7 @@ export class FileService {
|
||||
where: {
|
||||
id: params.fileId,
|
||||
path: Like(`${params.fileFolder}/%`),
|
||||
status: FILE_STATUS.UPLOADED,
|
||||
},
|
||||
});
|
||||
|
||||
@@ -251,6 +255,7 @@ export class FileService {
|
||||
where: {
|
||||
id: fileId,
|
||||
path: Like(`${fileFolder}/%`),
|
||||
status: FILE_STATUS.UPLOADED,
|
||||
},
|
||||
});
|
||||
|
||||
|
||||
@@ -0,0 +1,8 @@
|
||||
export const FILE_STATUS = {
|
||||
// File record exists but bytes have not been confirmed in storage yet
|
||||
// (direct upload initiated, waiting for the client to upload and confirm).
|
||||
PENDING: 'PENDING',
|
||||
UPLOADED: 'UPLOADED',
|
||||
} as const;
|
||||
|
||||
export type FileStatus = (typeof FILE_STATUS)[keyof typeof FILE_STATUS];
|
||||
@@ -565,7 +565,7 @@ export class ConfigVariables {
|
||||
@ConfigVariablesMetadata({
|
||||
group: ConfigVariablesGroup.STORAGE_CONFIG,
|
||||
description:
|
||||
'When enabled, file downloads are 302-redirected to S3 presigned URLs instead of being proxied through the server. Reduces server load and bandwidth.',
|
||||
'When enabled, file downloads are 302-redirected to S3 presigned URLs and direct uploads go straight to S3 via presigned PUT URLs instead of being proxied through the server. Reduces server load and bandwidth. Requires a bucket CORS policy allowing PUT from the frontend origin.',
|
||||
type: ConfigVariableType.BOOLEAN,
|
||||
})
|
||||
@ValidateIf((env) => env.STORAGE_TYPE === StorageDriverType.S_3)
|
||||
|
||||
+17
-1
@@ -19,6 +19,7 @@ import { type WorkspaceInternalContext } from 'src/engine/twenty-orm/interfaces/
|
||||
import { STANDARD_ERROR_MESSAGE } from 'src/engine/api/common/common-query-runners/errors/standard-error-message.constant';
|
||||
import { getFlatFieldsFromFlatObjectMetadata } from 'src/engine/api/graphql/workspace-schema-builder/utils/get-flat-fields-for-flat-object-metadata.util';
|
||||
import { FileEntity } from 'src/engine/core-modules/file/entities/file.entity';
|
||||
import { FILE_STATUS } from 'src/engine/core-modules/file/types/file-status.types';
|
||||
import { findFlatEntityByIdInFlatEntityMaps } from 'src/engine/metadata-modules/flat-entity/utils/find-flat-entity-by-id-in-flat-entity-maps.util';
|
||||
import { type FlatFieldMetadata } from 'src/engine/metadata-modules/flat-field-metadata/types/flat-field-metadata.type';
|
||||
import {
|
||||
@@ -486,7 +487,7 @@ export class FilesFieldSync {
|
||||
id: In([...allFileIdsToFetch, ...allFileIds.toRemove]),
|
||||
workspaceId,
|
||||
},
|
||||
select: ['id', 'path', 'settings'],
|
||||
select: ['id', 'path', 'settings', 'status'],
|
||||
});
|
||||
|
||||
const existingFileMap = new Map(
|
||||
@@ -523,6 +524,21 @@ export class FilesFieldSync {
|
||||
);
|
||||
}
|
||||
|
||||
// Direct uploads stay PENDING until the client confirms the bytes
|
||||
// landed in storage (completeFileUpload): only confirmed files can
|
||||
// be attached to a record.
|
||||
if (fileEntity.status !== FILE_STATUS.UPLOADED) {
|
||||
const fileId = file.fileId;
|
||||
|
||||
throw new TwentyORMException(
|
||||
`File ${fileId} upload has not been completed`,
|
||||
TwentyORMExceptionCode.INVALID_INPUT,
|
||||
{
|
||||
userFriendlyMessage: msg`File ${fileId} upload has not been completed. Please retry the upload.`,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
file.extension = path.extname(fileEntity.path);
|
||||
}
|
||||
|
||||
|
||||
+297
@@ -0,0 +1,297 @@
|
||||
import gql from 'graphql-tag';
|
||||
import request from 'supertest';
|
||||
import { createOneFieldMetadata } from 'test/integration/metadata/suites/field-metadata/utils/create-one-field-metadata.util';
|
||||
import { createOneObjectMetadata } from 'test/integration/metadata/suites/object-metadata/utils/create-one-object-metadata.util';
|
||||
import { deleteOneObjectMetadata } from 'test/integration/metadata/suites/object-metadata/utils/delete-one-object-metadata.util';
|
||||
import { updateOneObjectMetadata } from 'test/integration/metadata/suites/object-metadata/utils/update-one-object-metadata.util';
|
||||
import { makeMetadataAPIRequest } from 'test/integration/metadata/suites/utils/make-metadata-api-request.util';
|
||||
import { FieldMetadataType, FileFolder } from 'twenty-shared/types';
|
||||
|
||||
const createFileUploadMutation = gql`
|
||||
mutation CreateFileUpload(
|
||||
$filename: String!
|
||||
$size: Float!
|
||||
$fileFolder: FileFolder!
|
||||
$fieldMetadataId: String
|
||||
) {
|
||||
createFileUpload(
|
||||
filename: $filename
|
||||
size: $size
|
||||
fileFolder: $fileFolder
|
||||
fieldMetadataId: $fieldMetadataId
|
||||
) {
|
||||
fileId
|
||||
uploadUrl
|
||||
contentType
|
||||
expiresAt
|
||||
}
|
||||
}
|
||||
`;
|
||||
|
||||
const completeFileUploadMutation = gql`
|
||||
mutation CompleteFileUpload($fileId: String!) {
|
||||
completeFileUpload(fileId: $fileId) {
|
||||
id
|
||||
path
|
||||
size
|
||||
createdAt
|
||||
url
|
||||
}
|
||||
}
|
||||
`;
|
||||
|
||||
const deleteFileMutation = gql`
|
||||
mutation DeleteFile($fileId: UUID!) {
|
||||
deleteFile(fileId: $fileId) {
|
||||
id
|
||||
}
|
||||
}
|
||||
`;
|
||||
|
||||
describe('direct file upload (createFileUpload / completeFileUpload)', () => {
|
||||
let createdObjectMetadataId: string;
|
||||
let createdFieldMetadataId: string;
|
||||
const uploadedFileIds: string[] = [];
|
||||
|
||||
const createFileUpload = async (variables: Record<string, unknown>) => {
|
||||
return makeMetadataAPIRequest({
|
||||
query: createFileUploadMutation,
|
||||
variables,
|
||||
});
|
||||
};
|
||||
|
||||
const completeFileUpload = async (fileId: string) => {
|
||||
return makeMetadataAPIRequest({
|
||||
query: completeFileUploadMutation,
|
||||
variables: { fileId },
|
||||
});
|
||||
};
|
||||
|
||||
const putFileToUploadUrl = async (
|
||||
uploadUrl: string,
|
||||
contentType: string,
|
||||
content: Buffer,
|
||||
) => {
|
||||
// Integration tests run on the local storage driver, so the upload url
|
||||
// targets the server's streaming endpoint: replay it against the test app.
|
||||
const { pathname, search } = new URL(uploadUrl);
|
||||
|
||||
return request(global.app.getHttpServer())
|
||||
.put(`${pathname}${search}`)
|
||||
.set('Content-Type', contentType)
|
||||
.send(content);
|
||||
};
|
||||
|
||||
beforeAll(async () => {
|
||||
jest.useRealTimers();
|
||||
|
||||
const {
|
||||
data: {
|
||||
createOneObject: { id: objectMetadataId },
|
||||
},
|
||||
} = await createOneObjectMetadata({
|
||||
input: {
|
||||
nameSingular: 'directUploadTestObject',
|
||||
namePlural: 'directUploadTestObjects',
|
||||
labelSingular: 'Direct Upload Test Object',
|
||||
labelPlural: 'Direct Upload Test Objects',
|
||||
icon: 'IconFile',
|
||||
},
|
||||
});
|
||||
|
||||
createdObjectMetadataId = objectMetadataId;
|
||||
|
||||
const {
|
||||
data: {
|
||||
createOneField: { id: fieldMetadataId },
|
||||
},
|
||||
} = await createOneFieldMetadata({
|
||||
input: {
|
||||
name: 'filesField',
|
||||
label: 'Files Field',
|
||||
type: FieldMetadataType.FILES,
|
||||
objectMetadataId: createdObjectMetadataId,
|
||||
settings: { maxNumberOfValues: 5 },
|
||||
},
|
||||
gqlFields: `
|
||||
id
|
||||
`,
|
||||
});
|
||||
|
||||
createdFieldMetadataId = fieldMetadataId;
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
for (const fileId of uploadedFileIds) {
|
||||
try {
|
||||
await makeMetadataAPIRequest({
|
||||
query: deleteFileMutation,
|
||||
variables: { fileId },
|
||||
});
|
||||
} catch {
|
||||
// Cleanup is best-effort: a failed deletion must not prevent the
|
||||
// object metadata teardown below.
|
||||
}
|
||||
}
|
||||
|
||||
await updateOneObjectMetadata({
|
||||
expectToFail: false,
|
||||
input: {
|
||||
idToUpdate: createdObjectMetadataId,
|
||||
updatePayload: {
|
||||
isActive: false,
|
||||
},
|
||||
},
|
||||
});
|
||||
await deleteOneObjectMetadata({
|
||||
input: { idToDelete: createdObjectMetadataId },
|
||||
});
|
||||
|
||||
jest.useFakeTimers();
|
||||
});
|
||||
|
||||
it('should upload a file end to end: initiate, PUT bytes, complete', async () => {
|
||||
const testFileContent = Buffer.from('direct upload test content');
|
||||
|
||||
const createResponse = await createFileUpload({
|
||||
filename: 'direct-upload.txt',
|
||||
size: testFileContent.length,
|
||||
fileFolder: 'FilesField',
|
||||
fieldMetadataId: createdFieldMetadataId,
|
||||
});
|
||||
|
||||
expect(createResponse.status).toBe(200);
|
||||
expect(createResponse.body.errors).toBeUndefined();
|
||||
|
||||
const uploadTarget = createResponse.body.data.createFileUpload;
|
||||
|
||||
expect(uploadTarget.fileId).toBeDefined();
|
||||
expect(uploadTarget.uploadUrl).toContain(
|
||||
`/file-upload/${uploadTarget.fileId}?token=`,
|
||||
);
|
||||
expect(uploadTarget.contentType).toBe('application/octet-stream');
|
||||
expect(new Date(uploadTarget.expiresAt).getTime()).toBeGreaterThan(
|
||||
Date.now(),
|
||||
);
|
||||
|
||||
uploadedFileIds.push(uploadTarget.fileId);
|
||||
|
||||
const putResponse = await putFileToUploadUrl(
|
||||
uploadTarget.uploadUrl,
|
||||
uploadTarget.contentType,
|
||||
testFileContent,
|
||||
);
|
||||
|
||||
expect(putResponse.status).toBe(204);
|
||||
|
||||
const completeResponse = await completeFileUpload(uploadTarget.fileId);
|
||||
|
||||
expect(completeResponse.status).toBe(200);
|
||||
expect(completeResponse.body.errors).toBeUndefined();
|
||||
|
||||
const completedFile = completeResponse.body.data.completeFileUpload;
|
||||
|
||||
expect(completedFile.id).toBe(uploadTarget.fileId);
|
||||
expect(completedFile.path).toContain(FileFolder.FilesField);
|
||||
expect(completedFile.size).toBe(testFileContent.length);
|
||||
expect(completedFile.url).toContain(
|
||||
`/file/${FileFolder.FilesField}/${uploadTarget.fileId}?token=`,
|
||||
);
|
||||
|
||||
// The confirmed file is downloadable through the regular file endpoint
|
||||
const { pathname, search } = new URL(completedFile.url);
|
||||
const downloadResponse = await request(global.app.getHttpServer()).get(
|
||||
`${pathname}${search}`,
|
||||
);
|
||||
|
||||
expect(downloadResponse.status).toBe(200);
|
||||
expect(downloadResponse.text).toBe(testFileContent.toString());
|
||||
});
|
||||
|
||||
it('should refuse to complete an upload whose bytes never reached storage', async () => {
|
||||
const createResponse = await createFileUpload({
|
||||
filename: 'never-uploaded.txt',
|
||||
size: 100,
|
||||
fileFolder: 'FilesField',
|
||||
fieldMetadataId: createdFieldMetadataId,
|
||||
});
|
||||
|
||||
const uploadTarget = createResponse.body.data.createFileUpload;
|
||||
|
||||
uploadedFileIds.push(uploadTarget.fileId);
|
||||
|
||||
const completeResponse = await completeFileUpload(uploadTarget.fileId);
|
||||
|
||||
expect(completeResponse.body.errors).toBeDefined();
|
||||
expect(completeResponse.body.errors[0].message).toContain(
|
||||
'has not been uploaded',
|
||||
);
|
||||
});
|
||||
|
||||
it('should refuse a PUT larger than the declared size', async () => {
|
||||
const declaredSize = 10;
|
||||
const oversizedContent = Buffer.from(
|
||||
'this content is longer than ten bytes',
|
||||
);
|
||||
|
||||
const createResponse = await createFileUpload({
|
||||
filename: 'oversized.txt',
|
||||
size: declaredSize,
|
||||
fileFolder: 'FilesField',
|
||||
fieldMetadataId: createdFieldMetadataId,
|
||||
});
|
||||
|
||||
const uploadTarget = createResponse.body.data.createFileUpload;
|
||||
|
||||
uploadedFileIds.push(uploadTarget.fileId);
|
||||
|
||||
const putResponse = await putFileToUploadUrl(
|
||||
uploadTarget.uploadUrl,
|
||||
uploadTarget.contentType,
|
||||
oversizedContent,
|
||||
);
|
||||
|
||||
expect(putResponse.status).toBe(413);
|
||||
});
|
||||
|
||||
it('should refuse a PUT without a valid upload token', async () => {
|
||||
const createResponse = await createFileUpload({
|
||||
filename: 'bad-token.txt',
|
||||
size: 10,
|
||||
fileFolder: 'FilesField',
|
||||
fieldMetadataId: createdFieldMetadataId,
|
||||
});
|
||||
|
||||
const uploadTarget = createResponse.body.data.createFileUpload;
|
||||
|
||||
uploadedFileIds.push(uploadTarget.fileId);
|
||||
|
||||
const putResponse = await request(global.app.getHttpServer())
|
||||
.put(`/file-upload/${uploadTarget.fileId}?token=not-a-valid-token`)
|
||||
.set('Content-Type', 'application/octet-stream')
|
||||
.send(Buffer.from('0123456789'));
|
||||
|
||||
expect(putResponse.status).toBe(403);
|
||||
});
|
||||
|
||||
it('should reject file folders without direct upload support', async () => {
|
||||
const createResponse = await createFileUpload({
|
||||
filename: 'picture.png',
|
||||
size: 10,
|
||||
fileFolder: 'CorePicture',
|
||||
});
|
||||
|
||||
expect(createResponse.body.errors).toBeDefined();
|
||||
});
|
||||
|
||||
it('should reject a size above the direct upload maximum', async () => {
|
||||
const createResponse = await createFileUpload({
|
||||
filename: 'huge.bin',
|
||||
size: 5 * 1024 * 1024 * 1024,
|
||||
fileFolder: 'FilesField',
|
||||
fieldMetadataId: createdFieldMetadataId,
|
||||
});
|
||||
|
||||
expect(createResponse.body.errors).toBeDefined();
|
||||
});
|
||||
});
|
||||
@@ -1771,6 +1771,23 @@ __metadata:
|
||||
languageName: node
|
||||
linkType: hard
|
||||
|
||||
"@aws-sdk/lib-storage@npm:3.1001.0":
|
||||
version: 3.1001.0
|
||||
resolution: "@aws-sdk/lib-storage@npm:3.1001.0"
|
||||
dependencies:
|
||||
"@smithy/abort-controller": "npm:^4.2.10"
|
||||
"@smithy/middleware-endpoint": "npm:^4.4.21"
|
||||
"@smithy/smithy-client": "npm:^4.12.1"
|
||||
buffer: "npm:5.6.0"
|
||||
events: "npm:3.3.0"
|
||||
stream-browserify: "npm:3.0.0"
|
||||
tslib: "npm:^2.6.2"
|
||||
peerDependencies:
|
||||
"@aws-sdk/client-s3": ^3.1001.0
|
||||
checksum: 10c0/9ead85eb70c2be7ecf1b28a2e40926da170908e5fe0ac1e9d034a86c2c2b39424f4ac0c2a3560aed4469449e89be8353bab3b663a29df20fb57bcb19d7f57fe4
|
||||
languageName: node
|
||||
linkType: hard
|
||||
|
||||
"@aws-sdk/middleware-bucket-endpoint@npm:^3.972.3":
|
||||
version: 3.972.13
|
||||
resolution: "@aws-sdk/middleware-bucket-endpoint@npm:3.972.13"
|
||||
@@ -27204,7 +27221,7 @@ __metadata:
|
||||
languageName: node
|
||||
linkType: hard
|
||||
|
||||
"base64-js@npm:1.5.1, base64-js@npm:^1.1.2, base64-js@npm:^1.3.0, base64-js@npm:^1.3.1, base64-js@npm:^1.5.1":
|
||||
"base64-js@npm:1.5.1, base64-js@npm:^1.0.2, base64-js@npm:^1.1.2, base64-js@npm:^1.3.0, base64-js@npm:^1.3.1, base64-js@npm:^1.5.1":
|
||||
version: 1.5.1
|
||||
resolution: "base64-js@npm:1.5.1"
|
||||
checksum: 10c0/f23823513b63173a001030fae4f2dabe283b99a9d324ade3ad3d148e218134676f1ee8568c877cd79ec1c53158dcf2d2ba527a97c606618928ba99dd930102bf
|
||||
@@ -27738,6 +27755,16 @@ __metadata:
|
||||
languageName: node
|
||||
linkType: hard
|
||||
|
||||
"buffer@npm:5.6.0":
|
||||
version: 5.6.0
|
||||
resolution: "buffer@npm:5.6.0"
|
||||
dependencies:
|
||||
base64-js: "npm:^1.0.2"
|
||||
ieee754: "npm:^1.1.4"
|
||||
checksum: 10c0/07037a0278b07fbc779920f1ba1b473933ffb4a2e2f7b387c55daf6ac64a05b58c27da9e85730a4046e8f97a49f8acd9f7bf89605c0a4dfda88ebfb7e08bfe4a
|
||||
languageName: node
|
||||
linkType: hard
|
||||
|
||||
"buffer@npm:5.7.1, buffer@npm:^5.2.1, buffer@npm:^5.5.0":
|
||||
version: 5.7.1
|
||||
resolution: "buffer@npm:5.7.1"
|
||||
@@ -32447,7 +32474,7 @@ __metadata:
|
||||
languageName: node
|
||||
linkType: hard
|
||||
|
||||
"events@npm:^3.2.0, events@npm:^3.3.0":
|
||||
"events@npm:3.3.0, events@npm:^3.2.0, events@npm:^3.3.0":
|
||||
version: 3.3.0
|
||||
resolution: "events@npm:3.3.0"
|
||||
checksum: 10c0/d6b6f2adbccbcda74ddbab52ed07db727ef52e31a61ed26db9feb7dc62af7fc8e060defa65e5f8af9449b86b52cc1a1f6a79f2eafcf4e62add2b7a1fa4a432f6
|
||||
@@ -36187,7 +36214,7 @@ __metadata:
|
||||
languageName: node
|
||||
linkType: hard
|
||||
|
||||
"ieee754@npm:1.2.1, ieee754@npm:^1.1.13, ieee754@npm:^1.2.1":
|
||||
"ieee754@npm:1.2.1, ieee754@npm:^1.1.13, ieee754@npm:^1.1.4, ieee754@npm:^1.2.1":
|
||||
version: 1.2.1
|
||||
resolution: "ieee754@npm:1.2.1"
|
||||
checksum: 10c0/b0782ef5e0935b9f12883a2e2aa37baa75da6e66ce6515c168697b42160807d9330de9a32ec1ed73149aea02e0d822e572bca6f1e22bdcbd2149e13b050b17bb
|
||||
@@ -47486,7 +47513,7 @@ __metadata:
|
||||
languageName: node
|
||||
linkType: hard
|
||||
|
||||
"readable-stream@npm:3, readable-stream@npm:3.6.2, readable-stream@npm:^3.0.2, readable-stream@npm:^3.0.6, readable-stream@npm:^3.1.1, readable-stream@npm:^3.4.0, readable-stream@npm:^3.6.0":
|
||||
"readable-stream@npm:3, readable-stream@npm:3.6.2, readable-stream@npm:^3.0.2, readable-stream@npm:^3.0.6, readable-stream@npm:^3.1.1, readable-stream@npm:^3.4.0, readable-stream@npm:^3.5.0, readable-stream@npm:^3.6.0":
|
||||
version: 3.6.2
|
||||
resolution: "readable-stream@npm:3.6.2"
|
||||
dependencies:
|
||||
@@ -50763,6 +50790,16 @@ __metadata:
|
||||
languageName: node
|
||||
linkType: hard
|
||||
|
||||
"stream-browserify@npm:3.0.0":
|
||||
version: 3.0.0
|
||||
resolution: "stream-browserify@npm:3.0.0"
|
||||
dependencies:
|
||||
inherits: "npm:~2.0.4"
|
||||
readable-stream: "npm:^3.5.0"
|
||||
checksum: 10c0/ec3b975a4e0aa4b3dc5e70ffae3fc8fd29ac725353a14e72f213dff477b00330140ad014b163a8cbb9922dfe90803f81a5ea2b269e1bbfd8bd71511b88f889ad
|
||||
languageName: node
|
||||
linkType: hard
|
||||
|
||||
"stream-buffers@npm:~2.2.0":
|
||||
version: 2.2.0
|
||||
resolution: "stream-buffers@npm:2.2.0"
|
||||
@@ -53068,6 +53105,7 @@ __metadata:
|
||||
"@aws-sdk/client-sesv2": "npm:3.1001.0"
|
||||
"@aws-sdk/client-sts": "npm:3.1001.0"
|
||||
"@aws-sdk/credential-providers": "npm:3.1001.0"
|
||||
"@aws-sdk/lib-storage": "npm:3.1001.0"
|
||||
"@aws-sdk/s3-request-presigner": "npm:3.1001.0"
|
||||
"@azure/msal-node": "npm:^5.2.3"
|
||||
"@blocknote/server-util": "npm:^0.51.4"
|
||||
|
||||
Reference in New Issue
Block a user