From 127fb2a470bb9376efe03129a3b98e09003ba405 Mon Sep 17 00:00:00 2001 From: martmull Date: Wed, 20 May 2026 14:39:11 +0200 Subject: [PATCH] Increase size of tarball upload (#20767) - check size while reading stream instead of checking after reading all stream - move MAX_TARBALL_UPLOAD_SIZE_BYTES to config variables - increase MAX_TARBALL_UPLOAD_SIZE_BYTES default from 50Mb to 100Mb --- .../application-registration.resolver.ts | 50 +++++++++++-------- .../application-tarball.service.ts | 2 - .../twenty-config/config-variables.ts | 10 ++++ .../utils/__test__/stream-to-buffer.spec.ts | 41 +++++++++++++++ .../src/utils/stream-to-buffer.ts | 21 +++++++- 5 files changed, 101 insertions(+), 23 deletions(-) diff --git a/packages/twenty-server/src/engine/core-modules/application/application-registration/application-registration.resolver.ts b/packages/twenty-server/src/engine/core-modules/application/application-registration/application-registration.resolver.ts index 2ba7cf2d0c..d560622d3b 100644 --- a/packages/twenty-server/src/engine/core-modules/application/application-registration/application-registration.resolver.ts +++ b/packages/twenty-server/src/engine/core-modules/application/application-registration/application-registration.resolver.ts @@ -24,15 +24,8 @@ import { CreateApplicationRegistrationVariableInput } from 'src/engine/core-modu import { UpdateApplicationRegistrationVariableInput } from 'src/engine/core-modules/application/application-registration-variable/dtos/update-application-registration-variable.input'; import { ApplicationRegistrationExceptionFilter } from 'src/engine/core-modules/application/application-registration/application-registration-exception-filter'; import { ApplicationRegistrationEntity } from 'src/engine/core-modules/application/application-registration/application-registration.entity'; -import { - ApplicationRegistrationException, - ApplicationRegistrationExceptionCode, -} from 'src/engine/core-modules/application/application-registration/application-registration.exception'; import { ApplicationRegistrationService } from 'src/engine/core-modules/application/application-registration/application-registration.service'; -import { - ApplicationTarballService, - MAX_TARBALL_UPLOAD_SIZE_BYTES, -} from 'src/engine/core-modules/application/application-registration/application-tarball.service'; +import { ApplicationTarballService } from 'src/engine/core-modules/application/application-registration/application-tarball.service'; import { ApplicationRegistrationStatsDTO } from 'src/engine/core-modules/application/application-registration/dtos/application-registration-stats.dto'; import { CreateApplicationRegistrationDTO } from 'src/engine/core-modules/application/application-registration/dtos/create-application-registration.dto'; import { CreateApplicationRegistrationInput } from 'src/engine/core-modules/application/application-registration/dtos/create-application-registration.input'; @@ -53,8 +46,13 @@ import { NoPermissionGuard } from 'src/engine/guards/no-permission.guard'; import { PublicEndpointGuard } from 'src/engine/guards/public-endpoint.guard'; import { SettingsPermissionGuard } from 'src/engine/guards/settings-permission.guard'; import { WorkspaceAuthGuard } from 'src/engine/guards/workspace-auth.guard'; +import { TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service'; import { streamToBuffer } from 'src/utils/stream-to-buffer'; import { ApplicationRegistrationVariableDTO } from 'src/engine/core-modules/application/application-registration-variable/dtos/application-registration-variable.dto'; +import { + ApplicationRegistrationException, + ApplicationRegistrationExceptionCode, +} from 'src/engine/core-modules/application/application-registration/application-registration.exception'; @UsePipes(ResolverValidationPipe) @MetadataResolver(() => ApplicationRegistrationEntity) @@ -69,6 +67,7 @@ export class ApplicationRegistrationResolver { private readonly applicationRegistrationVariableService: ApplicationRegistrationVariableService, private readonly applicationTarballService: ApplicationTarballService, private readonly fileUrlService: FileUrlService, + private readonly twentyConfigService: TwentyConfigService, ) {} @UseGuards(PublicEndpointGuard, NoPermissionGuard) @@ -255,21 +254,32 @@ export class ApplicationRegistrationResolver { universalIdentifier: string | undefined, @AuthWorkspace() { id: workspaceId }: WorkspaceEntity, ): Promise { + const maxSize = this.twentyConfigService.get( + 'MAX_TARBALL_UPLOAD_SIZE_BYTES', + ); + const stream = createReadStream(); - const tarballBuffer = await streamToBuffer(stream); - if (tarballBuffer.length > MAX_TARBALL_UPLOAD_SIZE_BYTES) { - throw new ApplicationRegistrationException( - `Tarball exceeds maximum size of ${MAX_TARBALL_UPLOAD_SIZE_BYTES} bytes`, - ApplicationRegistrationExceptionCode.INVALID_INPUT, - ); + try { + const tarballBuffer = await streamToBuffer(stream, maxSize); + + return this.applicationTarballService.uploadTarball({ + tarballBuffer, + universalIdentifier, + ownerWorkspaceId: workspaceId, + }); + } catch (error) { + if ( + error instanceof Error && + error.message.includes('maximum allowed size') + ) { + throw new ApplicationRegistrationException( + `Tarball exceeds maximum size of ${maxSize} bytes`, + ApplicationRegistrationExceptionCode.INVALID_INPUT, + ); + } + throw error; } - - return this.applicationTarballService.uploadTarball({ - tarballBuffer, - universalIdentifier, - ownerWorkspaceId: workspaceId, - }); } @UseGuards( diff --git a/packages/twenty-server/src/engine/core-modules/application/application-registration/application-tarball.service.ts b/packages/twenty-server/src/engine/core-modules/application/application-registration/application-tarball.service.ts index d8623d4d07..f2ab940d41 100644 --- a/packages/twenty-server/src/engine/core-modules/application/application-registration/application-tarball.service.ts +++ b/packages/twenty-server/src/engine/core-modules/application/application-registration/application-tarball.service.ts @@ -29,8 +29,6 @@ import { FileStorageService } from 'src/engine/core-modules/file-storage/file-st import type { ApplicationManifest } from 'twenty-shared/application'; import { ApplicationRegistrationVariableService } from 'src/engine/core-modules/application/application-registration-variable/application-registration-variable.service'; -export const MAX_TARBALL_UPLOAD_SIZE_BYTES = 50 * 1024 * 1024; - @Injectable() export class ApplicationTarballService { private readonly logger = new Logger(ApplicationTarballService.name); diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/config-variables.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/config-variables.ts index 3b7299bee5..71c9c9344c 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/config-variables.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/config-variables.ts @@ -530,6 +530,16 @@ export class ConfigVariables { @IsOptional() STORAGE_S3_PRESIGNED_URL_EXPIRES_IN: number = 900; + @ConfigVariablesMetadata({ + group: ConfigVariablesGroup.STORAGE_CONFIG, + description: + 'Maximum tarball upload size in bytes for application registration', + type: ConfigVariableType.NUMBER, + }) + @CastToPositiveNumber() + @IsOptional() + MAX_TARBALL_UPLOAD_SIZE_BYTES: number = 100 * 1024 * 1024; + @ConfigVariablesMetadata({ group: ConfigVariablesGroup.LOGIC_FUNCTION_CONFIG, description: 'Type of function execution (local or Lambda)', diff --git a/packages/twenty-server/src/utils/__test__/stream-to-buffer.spec.ts b/packages/twenty-server/src/utils/__test__/stream-to-buffer.spec.ts index 131a2c647b..20e69b5105 100644 --- a/packages/twenty-server/src/utils/__test__/stream-to-buffer.spec.ts +++ b/packages/twenty-server/src/utils/__test__/stream-to-buffer.spec.ts @@ -76,4 +76,45 @@ describe('streamToBuffer', () => { await expect(promise).rejects.toThrow('Stream closed before end'); }); }); + + describe('maxSizeBytes', () => { + it('should accept stream within size limit', async () => { + const data = 'Hello, World!'; + const stream = Readable.from([Buffer.from(data)]); + + const result = await streamToBuffer(stream, 100); + + expect(result.toString()).toBe(data); + }); + + it('should reject when stream exceeds maxSizeBytes', async () => { + const stream = new PassThrough(); + + const promise = streamToBuffer(stream, 10); + + stream.write(Buffer.from('12345')); + stream.write(Buffer.from('678901')); + + await expect(promise).rejects.toThrow( + 'Stream exceeds maximum allowed size of 10 bytes', + ); + }); + + it('should reject on a single chunk exceeding maxSizeBytes', async () => { + const stream = Readable.from([Buffer.from('this is too long')]); + + await expect(streamToBuffer(stream, 5)).rejects.toThrow( + 'Stream exceeds maximum allowed size of 5 bytes', + ); + }); + + it('should accept stream exactly at maxSizeBytes', async () => { + const data = '12345'; + const stream = Readable.from([Buffer.from(data)]); + + const result = await streamToBuffer(stream, 5); + + expect(result.toString()).toBe(data); + }); + }); }); diff --git a/packages/twenty-server/src/utils/stream-to-buffer.ts b/packages/twenty-server/src/utils/stream-to-buffer.ts index 73310e301f..3479f63de2 100644 --- a/packages/twenty-server/src/utils/stream-to-buffer.ts +++ b/packages/twenty-server/src/utils/stream-to-buffer.ts @@ -1,7 +1,11 @@ import { type Readable } from 'stream'; -export const streamToBuffer = async (stream: Readable): Promise => { +export const streamToBuffer = async ( + stream: Readable, + maxSizeBytes?: number, +): Promise => { const chunks: Buffer[] = []; + let totalSize = 0; return new Promise((resolve, reject) => { if (stream.readableEnded) { @@ -27,6 +31,21 @@ export const streamToBuffer = async (stream: Readable): Promise => { const onData = (chunk: Buffer) => { if (!isResolved) { + totalSize += chunk.length; + + if (maxSizeBytes !== undefined && totalSize > maxSizeBytes) { + isResolved = true; + cleanup(); + stream.destroy(); + reject( + new Error( + `Stream exceeds maximum allowed size of ${maxSizeBytes} bytes`, + ), + ); + + return; + } + chunks.push(chunk); } };