1a85b88d38
<img width="1484" height="404" alt="image" src="https://github.com/user-attachments/assets/b2d363bf-d9e1-49fb-9811-8cc98041aa79" /> ## Context Uploading large files currently OOMs the server: every upload resolver buffers the whole file in memory (`streamToBuffer`) before writing it to storage. This PR is the first of a series introducing direct client-to-storage uploads. It adds the server-side endpoints and driver support only — it is non-breaking and nothing consumes the new flow yet. Follow-up PRs will migrate the frontend upload paths, add a stale-pending-file cleanup cron, and cap the legacy buffered resolvers. ## What it does **New upload flow (initiate → PUT → confirm):** - `createFileUpload(filename, size, fileFolder, fieldMetadataId?)` validates the request (folder allowlist: `FilesField`/`Workflow`, max size, extension-derived mime type), creates the file record in a new `PENDING` status, and returns an upload target: - **S3 with presign enabled** → a presigned PUT URL with `Content-Type`/`Content-Length` pinned in the signature, so the client uploads straight to the bucket; - **local storage, or S3 without presign** → a token-authenticated streaming endpoint on the server (`PUT /file-upload/:id?token=…`, new `FILE_UPLOAD` JWT type) that pipes the request body to the storage driver with constant memory usage and a declared-size cap. - `completeFileUpload(fileId)` verifies the bytes actually landed in storage (HEAD + size match against the declared size) and flips the record to `UPLOADED`. Idempotent. **Pending lifecycle safety:** - New `status` column on `core.file` (`PENDING`/`UPLOADED`, default `UPLOADED` so all existing rows and the legacy upload path are unaffected) + fast instance command. - Files are refused by the serving endpoints and by FILES-field sync while `PENDING`. **Driver support (both drivers):** - `getPresignedUploadUrl` (S3: presigned PUT; local: `null` → server-endpoint fallback) - `writeFileStream` (local: `fs` pipeline with the existing symlink/containment hardening, partial-file cleanup on error; S3: `@aws-sdk/lib-storage` `Upload` for bounded-memory streaming) - `getFileMetadata` (HEAD/stat for confirm-time verification) ## Tests - `file-upload.service.spec.ts`: initiate validation (folder allowlist, size), presigned vs fallback target, confirm verification (missing object, size mismatch, happy path, idempotency) - `local.driver.spec.ts`: `writeFileStream` (content, symlink rejection, partial-file cleanup on stream error), `getFileMetadata` - `s3.driver.spec.ts`: `getPresignedUploadUrl` (disabled → null, PUT command with signed content-type/content-length) - `direct-file-upload.integration-spec.ts`: full end-to-end flow against the local driver (initiate → PUT → complete → download), plus error paths (complete without upload, oversized PUT → 413, invalid token → 403, unsupported folder, size above max) ## Notes for reviewers - The upload-size ceiling for direct uploads is `settings.storage.maxDirectUploadFileSize` (1GB), separate from the 10MB `maxFileSize` used for pictures. - Since content can't be sniffed before it reaches storage, the mime type is derived from the file extension (with the existing `TWENTY_MIME_POLICY` override) and unknown extensions fall back to `application/octet-stream`; the serving path already forces `Content-Disposition: attachment` for anything not on the inline-safe allowlist. - Self-hosters using S3 presign will need a bucket CORS policy allowing `PUT` from the frontend origin (config variable description updated). https://claude.ai/code/session_015UH8KWmsB9zdYaog8MFG1d --- _Generated by [Claude Code](https://claude.ai/code/session_015UH8KWmsB9zdYaog8MFG1d)_ <!-- This is an auto-generated description by cubic. --> <a href="https://cubic.dev/pr/twentyhq/twenty/pull/22449?utm_source=github" target="_blank" rel="noopener noreferrer" data-no-image-dialog="true"><picture><source media="(prefers-color-scheme: dark)" srcset="https://www.cubic.dev/buttons/review-in-cubic-dark.svg"><source media="(prefers-color-scheme: light)" srcset="https://www.cubic.dev/buttons/review-in-cubic-light.svg"><img alt="Review in cubic" src="https://www.cubic.dev/buttons/review-in-cubic-dark.svg"></picture></a> <!-- End of auto-generated description by cubic. -->
80 lines
2.4 KiB
TypeScript
80 lines
2.4 KiB
TypeScript
import { defineRule } from '@oxlint/plugins';
|
|
|
|
import { typedTokenHelpers } from '../utils/typedTokenHelpers';
|
|
|
|
export const RULE_NAME = 'rest-api-methods-should-be-guarded';
|
|
|
|
export const restApiMethodsShouldBeGuarded = (node: any) => {
|
|
const hasRestApiMethodDecorator =
|
|
typedTokenHelpers.nodeHasDecoratorsNamed(node, [
|
|
'Get',
|
|
'Post',
|
|
'Put',
|
|
'Delete',
|
|
'Patch',
|
|
'Options',
|
|
'Head',
|
|
'All',
|
|
]);
|
|
|
|
const hasAuthGuards = typedTokenHelpers.nodeHasAuthGuards(node);
|
|
const hasPermissionsGuard =
|
|
typedTokenHelpers.nodeHasPermissionsGuard(node);
|
|
|
|
const findClassDeclaration = (node: any): any | null => {
|
|
if (node.type === 'ClassDeclaration') return node;
|
|
if (node.parent) return findClassDeclaration(node.parent);
|
|
return null;
|
|
};
|
|
|
|
const classNode = findClassDeclaration(node);
|
|
|
|
const hasAuthGuardsOnController = classNode
|
|
? typedTokenHelpers.nodeHasAuthGuards(classNode)
|
|
: false;
|
|
|
|
const hasPermissionsGuardOnController = classNode
|
|
? typedTokenHelpers.nodeHasPermissionsGuard(classNode)
|
|
: false;
|
|
|
|
const missingAuthGuard =
|
|
hasRestApiMethodDecorator &&
|
|
!hasAuthGuards &&
|
|
!hasAuthGuardsOnController;
|
|
|
|
const missingPermissionGuard =
|
|
hasRestApiMethodDecorator &&
|
|
!hasPermissionsGuard &&
|
|
!hasPermissionsGuardOnController;
|
|
|
|
return missingAuthGuard || missingPermissionGuard;
|
|
};
|
|
|
|
export const rule = defineRule({
|
|
meta: {
|
|
docs: {
|
|
description:
|
|
'REST API endpoints should have authentication guards (UserAuthGuard, WorkspaceAuthGuard, FilePathGuard, FileByIdGuard, FileUploadTokenGuard) or be explicitly marked as public (PublicEndpointGuard) and permission guards (SettingsPermissionGuard or CustomPermissionGuard) to maintain our security model.',
|
|
},
|
|
messages: {
|
|
restApiMethodsShouldBeGuarded:
|
|
'All REST API controller endpoints must have authentication guards (@UseGuards(...)) and permission guards (@UseGuards(..., SettingsPermissionGuard(...)), CustomPermissionGuard, or NoPermissionGuard).',
|
|
},
|
|
schema: [],
|
|
hasSuggestions: false,
|
|
type: 'suggestion',
|
|
},
|
|
create: (context) => {
|
|
return {
|
|
MethodDefinition: (node: any): void => {
|
|
if (restApiMethodsShouldBeGuarded(node)) {
|
|
context.report({
|
|
node: node,
|
|
messageId: 'restApiMethodsShouldBeGuarded',
|
|
});
|
|
}
|
|
},
|
|
};
|
|
},
|
|
});
|