Increase size of tarball upload (#20767)

- check size while reading stream instead of checking after reading all
stream
- move MAX_TARBALL_UPLOAD_SIZE_BYTES to config variables
- increase MAX_TARBALL_UPLOAD_SIZE_BYTES default from 50Mb to 100Mb

packages/twenty-server/src/engine/core-modules/application/application-registration/application-registration.resolver.ts

@@ -24,15 +24,8 @@ import { CreateApplicationRegistrationVariableInput } from 'src/engine/core-modu
 import { UpdateApplicationRegistrationVariableInput } from 'src/engine/core-modules/application/application-registration-variable/dtos/update-application-registration-variable.input';
 import { ApplicationRegistrationExceptionFilter } from 'src/engine/core-modules/application/application-registration/application-registration-exception-filter';
 import { ApplicationRegistrationEntity } from 'src/engine/core-modules/application/application-registration/application-registration.entity';
-import {
-  ApplicationRegistrationException,
-  ApplicationRegistrationExceptionCode,
-} from 'src/engine/core-modules/application/application-registration/application-registration.exception';
 import { ApplicationRegistrationService } from 'src/engine/core-modules/application/application-registration/application-registration.service';
-import {
-  ApplicationTarballService,
-  MAX_TARBALL_UPLOAD_SIZE_BYTES,
-} from 'src/engine/core-modules/application/application-registration/application-tarball.service';
+import { ApplicationTarballService } from 'src/engine/core-modules/application/application-registration/application-tarball.service';
 import { ApplicationRegistrationStatsDTO } from 'src/engine/core-modules/application/application-registration/dtos/application-registration-stats.dto';
 import { CreateApplicationRegistrationDTO } from 'src/engine/core-modules/application/application-registration/dtos/create-application-registration.dto';
 import { CreateApplicationRegistrationInput } from 'src/engine/core-modules/application/application-registration/dtos/create-application-registration.input';
@@ -53,8 +46,13 @@ import { NoPermissionGuard } from 'src/engine/guards/no-permission.guard';
 import { PublicEndpointGuard } from 'src/engine/guards/public-endpoint.guard';
 import { SettingsPermissionGuard } from 'src/engine/guards/settings-permission.guard';
 import { WorkspaceAuthGuard } from 'src/engine/guards/workspace-auth.guard';
+import { TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service';
 import { streamToBuffer } from 'src/utils/stream-to-buffer';
 import { ApplicationRegistrationVariableDTO } from 'src/engine/core-modules/application/application-registration-variable/dtos/application-registration-variable.dto';
+import {
+  ApplicationRegistrationException,
+  ApplicationRegistrationExceptionCode,
+} from 'src/engine/core-modules/application/application-registration/application-registration.exception';
 
 @UsePipes(ResolverValidationPipe)
 @MetadataResolver(() => ApplicationRegistrationEntity)
@@ -69,6 +67,7 @@ export class ApplicationRegistrationResolver {
     private readonly applicationRegistrationVariableService: ApplicationRegistrationVariableService,
     private readonly applicationTarballService: ApplicationTarballService,
     private readonly fileUrlService: FileUrlService,
+    private readonly twentyConfigService: TwentyConfigService,
   ) {}
 
   @UseGuards(PublicEndpointGuard, NoPermissionGuard)
@@ -255,21 +254,32 @@ export class ApplicationRegistrationResolver {
     universalIdentifier: string | undefined,
     @AuthWorkspace() { id: workspaceId }: WorkspaceEntity,
   ): Promise<ApplicationRegistrationEntity> {
+    const maxSize = this.twentyConfigService.get(
+      'MAX_TARBALL_UPLOAD_SIZE_BYTES',
+    );
+
     const stream = createReadStream();
-    const tarballBuffer = await streamToBuffer(stream);
 
-    if (tarballBuffer.length > MAX_TARBALL_UPLOAD_SIZE_BYTES) {
-      throw new ApplicationRegistrationException(
-        `Tarball exceeds maximum size of ${MAX_TARBALL_UPLOAD_SIZE_BYTES} bytes`,
-        ApplicationRegistrationExceptionCode.INVALID_INPUT,
-      );
+    try {
+      const tarballBuffer = await streamToBuffer(stream, maxSize);
+
+      return this.applicationTarballService.uploadTarball({
+        tarballBuffer,
+        universalIdentifier,
+        ownerWorkspaceId: workspaceId,
+      });
+    } catch (error) {
+      if (
+        error instanceof Error &&
+        error.message.includes('maximum allowed size')
+      ) {
+        throw new ApplicationRegistrationException(
+          `Tarball exceeds maximum size of ${maxSize} bytes`,
+          ApplicationRegistrationExceptionCode.INVALID_INPUT,
+        );
+      }
+      throw error;
     }
-
-    return this.applicationTarballService.uploadTarball({
-      tarballBuffer,
-      universalIdentifier,
-      ownerWorkspaceId: workspaceId,
-    });
   }
 
   @UseGuards(

packages/twenty-server/src/engine/core-modules/application/application-registration/application-tarball.service.ts

@@ -29,8 +29,6 @@ import { FileStorageService } from 'src/engine/core-modules/file-storage/file-st
 import type { ApplicationManifest } from 'twenty-shared/application';
 import { ApplicationRegistrationVariableService } from 'src/engine/core-modules/application/application-registration-variable/application-registration-variable.service';
 
-export const MAX_TARBALL_UPLOAD_SIZE_BYTES = 50 * 1024 * 1024;
-
 @Injectable()
 export class ApplicationTarballService {
   private readonly logger = new Logger(ApplicationTarballService.name);

packages/twenty-server/src/engine/core-modules/twenty-config/config-variables.ts

@@ -530,6 +530,16 @@ export class ConfigVariables {
   @IsOptional()
   STORAGE_S3_PRESIGNED_URL_EXPIRES_IN: number = 900;
 
+  @ConfigVariablesMetadata({
+    group: ConfigVariablesGroup.STORAGE_CONFIG,
+    description:
+      'Maximum tarball upload size in bytes for application registration',
+    type: ConfigVariableType.NUMBER,
+  })
+  @CastToPositiveNumber()
+  @IsOptional()
+  MAX_TARBALL_UPLOAD_SIZE_BYTES: number = 100 * 1024 * 1024;
+
   @ConfigVariablesMetadata({
     group: ConfigVariablesGroup.LOGIC_FUNCTION_CONFIG,
     description: 'Type of function execution (local or Lambda)',

packages/twenty-server/src/utils/__test__/stream-to-buffer.spec.ts

@@ -76,4 +76,45 @@ describe('streamToBuffer', () => {
       await expect(promise).rejects.toThrow('Stream closed before end');
     });
   });
+
+  describe('maxSizeBytes', () => {
+    it('should accept stream within size limit', async () => {
+      const data = 'Hello, World!';
+      const stream = Readable.from([Buffer.from(data)]);
+
+      const result = await streamToBuffer(stream, 100);
+
+      expect(result.toString()).toBe(data);
+    });
+
+    it('should reject when stream exceeds maxSizeBytes', async () => {
+      const stream = new PassThrough();
+
+      const promise = streamToBuffer(stream, 10);
+
+      stream.write(Buffer.from('12345'));
+      stream.write(Buffer.from('678901'));
+
+      await expect(promise).rejects.toThrow(
+        'Stream exceeds maximum allowed size of 10 bytes',
+      );
+    });
+
+    it('should reject on a single chunk exceeding maxSizeBytes', async () => {
+      const stream = Readable.from([Buffer.from('this is too long')]);
+
+      await expect(streamToBuffer(stream, 5)).rejects.toThrow(
+        'Stream exceeds maximum allowed size of 5 bytes',
+      );
+    });
+
+    it('should accept stream exactly at maxSizeBytes', async () => {
+      const data = '12345';
+      const stream = Readable.from([Buffer.from(data)]);
+
+      const result = await streamToBuffer(stream, 5);
+
+      expect(result.toString()).toBe(data);
+    });
+  });
 });

packages/twenty-server/src/utils/stream-to-buffer.ts

@@ -1,7 +1,11 @@
 import { type Readable } from 'stream';
 
-export const streamToBuffer = async (stream: Readable): Promise<Buffer> => {
+export const streamToBuffer = async (
+  stream: Readable,
+  maxSizeBytes?: number,
+): Promise<Buffer> => {
   const chunks: Buffer[] = [];
+  let totalSize = 0;
 
   return new Promise((resolve, reject) => {
     if (stream.readableEnded) {
@@ -27,6 +31,21 @@ export const streamToBuffer = async (stream: Readable): Promise<Buffer> => {
 
     const onData = (chunk: Buffer) => {
       if (!isResolved) {
+        totalSize += chunk.length;
+
+        if (maxSizeBytes !== undefined && totalSize > maxSizeBytes) {
+          isResolved = true;
+          cleanup();
+          stream.destroy();
+          reject(
+            new Error(
+              `Stream exceeds maximum allowed size of ${maxSizeBytes} bytes`,
+            ),
+          );
+
+          return;
+        }
+
         chunks.push(chunk);
       }
     };