diff --git a/packages/twenty-server/src/engine/core-modules/emailing-domain/drivers/aws-ses/providers/aws-ses-client.provider.ts b/packages/twenty-server/src/engine/core-modules/emailing-domain/drivers/aws-ses/providers/aws-ses-client.provider.ts index 232e112666..ad27eb1872 100644 --- a/packages/twenty-server/src/engine/core-modules/emailing-domain/drivers/aws-ses/providers/aws-ses-client.provider.ts +++ b/packages/twenty-server/src/engine/core-modules/emailing-domain/drivers/aws-ses/providers/aws-ses-client.provider.ts @@ -6,6 +6,7 @@ import { } from '@aws-sdk/client-sesv2'; import { TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service'; +import { buildAwsRequestHandlerOptions } from 'src/utils/aws-request-handler.util'; @Injectable() export class AwsSesClientProvider { @@ -17,6 +18,7 @@ export class AwsSesClientProvider { if (!this.sesClient) { const config: SESClientConfig = { region: this.twentyConfigService.get('AWS_SES_REGION'), + requestHandler: buildAwsRequestHandlerOptions(), }; const accessKeyId = this.twentyConfigService.get('AWS_SES_ACCESS_KEY_ID'); diff --git a/packages/twenty-server/src/engine/core-modules/file-storage/constants/s3-client-timeouts.constant.ts b/packages/twenty-server/src/engine/core-modules/file-storage/constants/s3-client-timeouts.constant.ts new file mode 100644 index 0000000000..d269f9a920 --- /dev/null +++ b/packages/twenty-server/src/engine/core-modules/file-storage/constants/s3-client-timeouts.constant.ts @@ -0,0 +1,3 @@ +export const FILE_STORAGE_S3_REQUEST_TIMEOUT_MS = 5 * 60 * 1000; +export const FILE_STORAGE_S3_CONNECTION_TIMEOUT_MS = 30 * 1000; +export const FILE_STORAGE_S3_MAX_SOCKETS = 200; diff --git a/packages/twenty-server/src/engine/core-modules/file-storage/drivers/s3.driver.ts b/packages/twenty-server/src/engine/core-modules/file-storage/drivers/s3.driver.ts index 42e0ba5543..1ced1877cc 100644 --- a/packages/twenty-server/src/engine/core-modules/file-storage/drivers/s3.driver.ts +++ b/packages/twenty-server/src/engine/core-modules/file-storage/drivers/s3.driver.ts @@ -24,11 +24,17 @@ import { Upload } from '@aws-sdk/lib-storage'; import { getSignedUrl } from '@aws-sdk/s3-request-presigner'; import { isDefined } from 'twenty-shared/utils'; +import { + FILE_STORAGE_S3_CONNECTION_TIMEOUT_MS, + FILE_STORAGE_S3_MAX_SOCKETS, + FILE_STORAGE_S3_REQUEST_TIMEOUT_MS, +} from 'src/engine/core-modules/file-storage/constants/s3-client-timeouts.constant'; import { type StorageDriver } from 'src/engine/core-modules/file-storage/drivers/interfaces/storage-driver.interface'; import { FileStorageException, FileStorageExceptionCode, } from 'src/engine/core-modules/file-storage/interfaces/file-storage-exception'; +import { buildAwsRequestHandlerOptions } from 'src/utils/aws-request-handler.util'; export interface S3DriverOptions extends S3ClientConfig { bucketName: string; @@ -58,12 +64,23 @@ export class S3Driver implements StorageDriver { return; } - this.s3Client = new S3({ ...s3Options, region, endpoint }); + const requestHandler = buildAwsRequestHandlerOptions({ + requestTimeoutMs: FILE_STORAGE_S3_REQUEST_TIMEOUT_MS, + connectionTimeoutMs: FILE_STORAGE_S3_CONNECTION_TIMEOUT_MS, + maxSockets: FILE_STORAGE_S3_MAX_SOCKETS, + }); + + this.s3Client = new S3({ ...s3Options, region, endpoint, requestHandler }); this.bucketName = bucketName; if (presignEnabled) { this.presignClient = presignEndpoint - ? new S3({ ...s3Options, region, endpoint: presignEndpoint }) + ? new S3({ + ...s3Options, + region, + endpoint: presignEndpoint, + requestHandler, + }) : this.s3Client; } } diff --git a/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-drivers/drivers/lambda/constants/lambda-driver.constant.ts b/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-drivers/drivers/lambda/constants/lambda-driver.constant.ts index bfc51baa26..db962e4402 100644 --- a/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-drivers/drivers/lambda/constants/lambda-driver.constant.ts +++ b/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-drivers/drivers/lambda/constants/lambda-driver.constant.ts @@ -16,6 +16,11 @@ export const EXECUTOR_LAMBDA_MEMORY_MB = 512; export const EXECUTOR_LAMBDA_TIMEOUT_SECONDS = 900; export const LAMBDA_EPHEMERAL_STORAGE_MB = 4096; +export const LAMBDA_CLIENT_REQUEST_TIMEOUT_MS = + (EXECUTOR_LAMBDA_TIMEOUT_SECONDS + 60) * 1000; +export const LAMBDA_CLIENT_MAX_SOCKETS = 500; +export const LAMBDA_CLIENT_CONNECTION_TIMEOUT_MS = 60_000; + export const COMMON_LAYER_NAME_PREFIX = 'twenty-common-layer'; export const YARN_INSTALL_FUNCTION_NAME_PREFIX = 'twenty-yarn-install'; export const BUILDER_FUNCTION_NAME_PREFIX = 'twenty-builder'; diff --git a/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-drivers/drivers/lambda/services/lambda-aws-client.service.ts b/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-drivers/drivers/lambda/services/lambda-aws-client.service.ts index f95f23247f..7d49ae371b 100644 --- a/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-drivers/drivers/lambda/services/lambda-aws-client.service.ts +++ b/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-drivers/drivers/lambda/services/lambda-aws-client.service.ts @@ -11,14 +11,20 @@ import { isDefined } from 'twenty-shared/utils'; import { CREDENTIALS_DURATION_IN_SECONDS, + LAMBDA_CLIENT_CONNECTION_TIMEOUT_MS, LAMBDA_CLIENT_MAX_ATTEMPTS, + LAMBDA_CLIENT_MAX_SOCKETS, + LAMBDA_CLIENT_REQUEST_TIMEOUT_MS, LAMBDA_CLIENT_RETRY_MODE, UPDATE_FUNCTION_DURATION_TIMEOUT_IN_SECONDS, } from 'src/engine/core-modules/logic-function/logic-function-drivers/drivers/lambda/constants/lambda-driver.constant'; import { type LambdaDriverOptions } from 'src/engine/core-modules/logic-function/logic-function-drivers/drivers/lambda/types/lambda-driver.type'; +import { buildAwsRequestHandlerOptions } from 'src/utils/aws-request-handler.util'; export class LambdaAwsClientService { private lambdaClient: Lambda | undefined; + private s3Client: S3Client | undefined; + private stsClient: STSClient | undefined; private assumeRoleCredentials: | { accessKeyId: string; secretAccessKey: string; sessionToken: string } | undefined; @@ -39,22 +45,40 @@ export class LambdaAwsClientService { }), maxAttempts: LAMBDA_CLIENT_MAX_ATTEMPTS, retryMode: LAMBDA_CLIENT_RETRY_MODE, + requestHandler: buildAwsRequestHandlerOptions({ + requestTimeoutMs: LAMBDA_CLIENT_REQUEST_TIMEOUT_MS, + connectionTimeoutMs: LAMBDA_CLIENT_CONNECTION_TIMEOUT_MS, + maxSockets: LAMBDA_CLIENT_MAX_SOCKETS, + }), }); } return this.lambdaClient; } + private async getS3Client(): Promise { + if ( + !isDefined(this.s3Client) || + (isDefined(this.options.subhostingRole) && + this.areAssumeRoleCredentialsExpired()) + ) { + this.s3Client = new S3Client({ + region: this.options.layerBucketRegion, + credentials: isDefined(this.options.subhostingRole) + ? await this.getAssumeRoleCredentials() + : this.options.credentials, + requestHandler: buildAwsRequestHandlerOptions(), + }); + } + + return this.s3Client; + } + async generatePresignedUploadUrl( s3Key: string, expiresIn: number = 300, ): Promise { - const s3Client = new S3Client({ - region: this.options.layerBucketRegion, - credentials: isDefined(this.options.subhostingRole) - ? await this.getAssumeRoleCredentials() - : this.options.credentials, - }); + const s3Client = await this.getS3Client(); const putCommand = new PutObjectCommand({ Bucket: this.options.layerBucket, @@ -107,7 +131,12 @@ export class LambdaAwsClientService { } private async refreshAssumeRoleCredentials() { - const stsClient = new STSClient({ region: this.options.region }); + this.stsClient ??= new STSClient({ + region: this.options.region, + requestHandler: buildAwsRequestHandlerOptions(), + }); + + const stsClient = this.stsClient; const assumeRoleCommand = new AssumeRoleCommand({ RoleArn: this.options.subhostingRole, @@ -137,6 +166,7 @@ export class LambdaAwsClientService { ); this.lambdaClient = undefined; + this.s3Client = undefined; } private async getAssumeRoleCredentials() { diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/inbound-email/providers/inbound-email-s3-client.provider.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/inbound-email/providers/inbound-email-s3-client.provider.ts index 2ea87cd54d..eccbd93b45 100644 --- a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/inbound-email/providers/inbound-email-s3-client.provider.ts +++ b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/inbound-email/providers/inbound-email-s3-client.provider.ts @@ -5,6 +5,7 @@ import { isNonEmptyString } from '@sniptt/guards'; import { StorageDriverType } from 'src/engine/core-modules/file-storage/interfaces/file-storage.interface'; import { TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service'; +import { buildAwsRequestHandlerOptions } from 'src/utils/aws-request-handler.util'; @Injectable() export class InboundEmailS3ClientProvider { @@ -54,7 +55,10 @@ export class InboundEmailS3ClientProvider { throw new Error('STORAGE_S3_REGION must be set to use email group.'); } - const config: S3ClientConfig = { region }; + const config: S3ClientConfig = { + region, + requestHandler: buildAwsRequestHandlerOptions(), + }; const endpoint = this.twentyConfigService.get('STORAGE_S3_ENDPOINT'); diff --git a/packages/twenty-server/src/utils/__tests__/aws-request-handler.util.spec.ts b/packages/twenty-server/src/utils/__tests__/aws-request-handler.util.spec.ts new file mode 100644 index 0000000000..da30e45bf1 --- /dev/null +++ b/packages/twenty-server/src/utils/__tests__/aws-request-handler.util.spec.ts @@ -0,0 +1,47 @@ +import { + AWS_DEFAULT_CONNECTION_TIMEOUT_MS, + AWS_DEFAULT_MAX_SOCKETS, + AWS_DEFAULT_REQUEST_TIMEOUT_MS, + buildAwsRequestHandlerOptions, +} from 'src/utils/aws-request-handler.util'; + +describe('buildAwsRequestHandlerOptions', () => { + it('always sets a non-zero request timeout', () => { + expect(buildAwsRequestHandlerOptions().requestTimeout).toBeGreaterThan(0); + }); + + it('always throws on request timeout rather than only warning', () => { + expect(buildAwsRequestHandlerOptions().throwOnRequestTimeout).toBe(true); + }); + + it('always sets a non-zero connection timeout', () => { + expect(buildAwsRequestHandlerOptions().connectionTimeout).toBeGreaterThan( + 0, + ); + }); + + it('applies the defaults when called with no arguments', () => { + expect(buildAwsRequestHandlerOptions()).toEqual({ + connectionTimeout: AWS_DEFAULT_CONNECTION_TIMEOUT_MS, + requestTimeout: AWS_DEFAULT_REQUEST_TIMEOUT_MS, + throwOnRequestTimeout: true, + httpsAgent: { maxSockets: AWS_DEFAULT_MAX_SOCKETS }, + }); + }); + + it('allows callers with long-running requests to raise the timeouts', () => { + const options = buildAwsRequestHandlerOptions({ + requestTimeoutMs: 960_000, + connectionTimeoutMs: 60_000, + }); + + expect(options.requestTimeout).toBe(960_000); + expect(options.connectionTimeout).toBe(60_000); + }); + + it('allows the socket pool size to be raised above the SDK default', () => { + expect( + buildAwsRequestHandlerOptions({ maxSockets: 500 }).httpsAgent.maxSockets, + ).toBe(500); + }); +}); diff --git a/packages/twenty-server/src/utils/aws-request-handler.util.ts b/packages/twenty-server/src/utils/aws-request-handler.util.ts new file mode 100644 index 0000000000..c24bdb789b --- /dev/null +++ b/packages/twenty-server/src/utils/aws-request-handler.util.ts @@ -0,0 +1,20 @@ +export const AWS_DEFAULT_CONNECTION_TIMEOUT_MS = 10_000; +export const AWS_DEFAULT_REQUEST_TIMEOUT_MS = 30_000; +export const AWS_DEFAULT_MAX_SOCKETS = 100; + +type AwsRequestHandlerOptions = { + requestTimeoutMs?: number; + connectionTimeoutMs?: number; + maxSockets?: number; +}; + +export const buildAwsRequestHandlerOptions = ({ + requestTimeoutMs = AWS_DEFAULT_REQUEST_TIMEOUT_MS, + connectionTimeoutMs = AWS_DEFAULT_CONNECTION_TIMEOUT_MS, + maxSockets = AWS_DEFAULT_MAX_SOCKETS, +}: AwsRequestHandlerOptions = {}) => ({ + connectionTimeout: connectionTimeoutMs, + requestTimeout: requestTimeoutMs, + throwOnRequestTimeout: true, + httpsAgent: { maxSockets }, +});