Increase size of tarball upload (#20767)

- check size while reading stream instead of checking after reading all
stream
- move MAX_TARBALL_UPLOAD_SIZE_BYTES to config variables
- increase MAX_TARBALL_UPLOAD_SIZE_BYTES default from 50Mb to 100Mb

packages/twenty-server/src/engine/core-modules/application/application-registration/application-registration.resolver.ts

@@ -24,15 +24,8 @@ import { CreateApplicationRegistrationVariableInput } from 'src/engine/core-modu
 import { UpdateApplicationRegistrationVariableInput } from 'src/engine/core-modules/application/application-registration-variable/dtos/update-application-registration-variable.input';
 import { ApplicationRegistrationExceptionFilter } from 'src/engine/core-modules/application/application-registration/application-registration-exception-filter';
 import { ApplicationRegistrationEntity } from 'src/engine/core-modules/application/application-registration/application-registration.entity';
-import {
-  ApplicationRegistrationException,
-  ApplicationRegistrationExceptionCode,
-} from 'src/engine/core-modules/application/application-registration/application-registration.exception';
 import { ApplicationRegistrationService } from 'src/engine/core-modules/application/application-registration/application-registration.service';
-import {
-  ApplicationTarballService,
-  MAX_TARBALL_UPLOAD_SIZE_BYTES,
-} from 'src/engine/core-modules/application/application-registration/application-tarball.service';
+import { ApplicationTarballService } from 'src/engine/core-modules/application/application-registration/application-tarball.service';
 import { ApplicationRegistrationStatsDTO } from 'src/engine/core-modules/application/application-registration/dtos/application-registration-stats.dto';
 import { CreateApplicationRegistrationDTO } from 'src/engine/core-modules/application/application-registration/dtos/create-application-registration.dto';
 import { CreateApplicationRegistrationInput } from 'src/engine/core-modules/application/application-registration/dtos/create-application-registration.input';
@@ -53,8 +46,13 @@ import { NoPermissionGuard } from 'src/engine/guards/no-permission.guard';
 import { PublicEndpointGuard } from 'src/engine/guards/public-endpoint.guard';
 import { SettingsPermissionGuard } from 'src/engine/guards/settings-permission.guard';
 import { WorkspaceAuthGuard } from 'src/engine/guards/workspace-auth.guard';
+import { TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service';
 import { streamToBuffer } from 'src/utils/stream-to-buffer';
 import { ApplicationRegistrationVariableDTO } from 'src/engine/core-modules/application/application-registration-variable/dtos/application-registration-variable.dto';
+import {
+  ApplicationRegistrationException,
+  ApplicationRegistrationExceptionCode,
+} from 'src/engine/core-modules/application/application-registration/application-registration.exception';
 
 @UsePipes(ResolverValidationPipe)
 @MetadataResolver(() => ApplicationRegistrationEntity)
@@ -69,6 +67,7 @@ export class ApplicationRegistrationResolver {
     private readonly applicationRegistrationVariableService: ApplicationRegistrationVariableService,
     private readonly applicationTarballService: ApplicationTarballService,
     private readonly fileUrlService: FileUrlService,
+    private readonly twentyConfigService: TwentyConfigService,
   ) {}
 
   @UseGuards(PublicEndpointGuard, NoPermissionGuard)
@@ -255,21 +254,32 @@ export class ApplicationRegistrationResolver {
     universalIdentifier: string | undefined,
     @AuthWorkspace() { id: workspaceId }: WorkspaceEntity,
   ): Promise<ApplicationRegistrationEntity> {
+    const maxSize = this.twentyConfigService.get(
+      'MAX_TARBALL_UPLOAD_SIZE_BYTES',
+    );
+
     const stream = createReadStream();
-    const tarballBuffer = await streamToBuffer(stream);
 
-    if (tarballBuffer.length > MAX_TARBALL_UPLOAD_SIZE_BYTES) {
-      throw new ApplicationRegistrationException(
-        `Tarball exceeds maximum size of ${MAX_TARBALL_UPLOAD_SIZE_BYTES} bytes`,
-        ApplicationRegistrationExceptionCode.INVALID_INPUT,
-      );
+    try {
+      const tarballBuffer = await streamToBuffer(stream, maxSize);
+
+      return this.applicationTarballService.uploadTarball({
+        tarballBuffer,
+        universalIdentifier,
+        ownerWorkspaceId: workspaceId,
+      });
+    } catch (error) {
+      if (
+        error instanceof Error &&
+        error.message.includes('maximum allowed size')
+      ) {
+        throw new ApplicationRegistrationException(
+          `Tarball exceeds maximum size of ${maxSize} bytes`,
+          ApplicationRegistrationExceptionCode.INVALID_INPUT,
+        );
+      }
+      throw error;
     }
-
-    return this.applicationTarballService.uploadTarball({
-      tarballBuffer,
-      universalIdentifier,
-      ownerWorkspaceId: workspaceId,
-    });
   }
 
   @UseGuards(

packages/twenty-server/src/engine/core-modules/application/application-registration/application-tarball.service.ts

@@ -29,8 +29,6 @@ import { FileStorageService } from 'src/engine/core-modules/file-storage/file-st
 import type { ApplicationManifest } from 'twenty-shared/application';
 import { ApplicationRegistrationVariableService } from 'src/engine/core-modules/application/application-registration-variable/application-registration-variable.service';
 
-export const MAX_TARBALL_UPLOAD_SIZE_BYTES = 50 * 1024 * 1024;
-
 @Injectable()
 export class ApplicationTarballService {
   private readonly logger = new Logger(ApplicationTarballService.name);

packages/twenty-server/src/engine/core-modules/twenty-config/config-variables.ts

@@ -530,6 +530,16 @@ export class ConfigVariables {
   @IsOptional()
   STORAGE_S3_PRESIGNED_URL_EXPIRES_IN: number = 900;
 
+  @ConfigVariablesMetadata({
+    group: ConfigVariablesGroup.STORAGE_CONFIG,
+    description:
+      'Maximum tarball upload size in bytes for application registration',
+    type: ConfigVariableType.NUMBER,
+  })
+  @CastToPositiveNumber()
+  @IsOptional()
+  MAX_TARBALL_UPLOAD_SIZE_BYTES: number = 100 * 1024 * 1024;
+
   @ConfigVariablesMetadata({
     group: ConfigVariablesGroup.LOGIC_FUNCTION_CONFIG,
     description: 'Type of function execution (local or Lambda)',