diff --git a/packages/twenty-server/src/database/clickHouse/clickHouse.service.spec.ts b/packages/twenty-server/src/database/clickHouse/clickHouse.service.spec.ts index 08e8f3cfa4..1f32368f4d 100644 --- a/packages/twenty-server/src/database/clickHouse/clickHouse.service.spec.ts +++ b/packages/twenty-server/src/database/clickHouse/clickHouse.service.spec.ts @@ -112,6 +112,25 @@ describe('ClickHouseService', () => { }); }); + it('should allow overriding the asynchronous insert busy timeout', async () => { + const testData = [{ id: 1, name: 'test' }]; + const result = await service.insert('test_table', testData, { + asyncInsertBusyTimeoutMaxMs: 100, + }); + + expect(result).toEqual({ success: true }); + expect(mockClickHouseClient.insert).toHaveBeenCalledWith({ + table: 'test_table', + values: testData, + format: 'JSONEachRow', + clickhouse_settings: { + async_insert: 1, + async_insert_busy_timeout_max_ms: 100, + wait_for_async_insert: 1, + }, + }); + }); + it('should return failure when clickhouse client is not defined', async () => { (service as any).mainClient = undefined; diff --git a/packages/twenty-server/src/database/clickHouse/clickHouse.service.ts b/packages/twenty-server/src/database/clickHouse/clickHouse.service.ts index 8210d19619..9a5bae2f96 100644 --- a/packages/twenty-server/src/database/clickHouse/clickHouse.service.ts +++ b/packages/twenty-server/src/database/clickHouse/clickHouse.service.ts @@ -13,6 +13,11 @@ import { import { TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service'; +export type ClickHouseInsertOptions = { + clientId?: string; + asyncInsertBusyTimeoutMaxMs?: number; +}; + @Injectable() export class ClickHouseService implements OnModuleInit, OnModuleDestroy { private mainClient: ClickHouseClient | undefined; @@ -135,11 +140,11 @@ export class ClickHouseService implements OnModuleInit, OnModuleDestroy { public async insert>( table: string, values: T[], - clientId?: string, + options: ClickHouseInsertOptions = {}, ): Promise<{ success: boolean }> { try { - const client = clientId - ? await this.connectToClient(clientId) + const client = options.clientId + ? await this.connectToClient(options.clientId) : this.mainClient; if (!client) { @@ -149,6 +154,7 @@ export class ClickHouseService implements OnModuleInit, OnModuleDestroy { await this.insertInChunks(client, table, values, { chunkSize: 1000, maxMemoryMB: 4, + asyncInsertBusyTimeoutMaxMs: options.asyncInsertBusyTimeoutMaxMs, }); return { success: true }; @@ -260,7 +266,11 @@ export class ClickHouseService implements OnModuleInit, OnModuleDestroy { client: ClickHouseClient, table: string, values: T[], - options: { chunkSize?: number; maxMemoryMB?: number } = {}, + options: { + chunkSize?: number; + maxMemoryMB?: number; + asyncInsertBusyTimeoutMaxMs?: number; + } = {}, ): Promise { const chunkSize = options.chunkSize ?? 1000; const maxMemoryMB = options.maxMemoryMB; @@ -276,6 +286,12 @@ export class ClickHouseService implements OnModuleInit, OnModuleDestroy { format: 'JSONEachRow', clickhouse_settings: { async_insert: 1, + ...(options.asyncInsertBusyTimeoutMaxMs !== undefined + ? { + async_insert_busy_timeout_max_ms: + options.asyncInsertBusyTimeoutMaxMs, + } + : {}), wait_for_async_insert: 1, }, }); diff --git a/packages/twenty-server/src/engine/core-modules/event-logs/ingest/clickhouse-event.sink.spec.ts b/packages/twenty-server/src/engine/core-modules/event-logs/ingest/clickhouse-event.sink.spec.ts index 126a8c6c44..ca51de7229 100644 --- a/packages/twenty-server/src/engine/core-modules/event-logs/ingest/clickhouse-event.sink.spec.ts +++ b/packages/twenty-server/src/engine/core-modules/event-logs/ingest/clickhouse-event.sink.spec.ts @@ -43,8 +43,14 @@ describe('ClickHouseEventSink', () => { await sink.write([first, second, applicationLog]); expect(insert).toHaveBeenCalledTimes(2); - expect(insert).toHaveBeenCalledWith('pageview', [first.row, second.row]); - expect(insert).toHaveBeenCalledWith('applicationLog', [applicationLog.row]); + expect(insert).toHaveBeenCalledWith('pageview', [first.row, second.row], { + asyncInsertBusyTimeoutMaxMs: 100, + }); + expect(insert).toHaveBeenCalledWith( + 'applicationLog', + [applicationLog.row], + undefined, + ); }); it('no-ops when ClickHouse is not configured', async () => { diff --git a/packages/twenty-server/src/engine/core-modules/event-logs/ingest/clickhouse-event.sink.ts b/packages/twenty-server/src/engine/core-modules/event-logs/ingest/clickhouse-event.sink.ts index acbbe146e5..e154fa3f43 100644 --- a/packages/twenty-server/src/engine/core-modules/event-logs/ingest/clickhouse-event.sink.ts +++ b/packages/twenty-server/src/engine/core-modules/event-logs/ingest/clickhouse-event.sink.ts @@ -1,8 +1,22 @@ import { Injectable } from '@nestjs/common'; -import { ClickHouseService } from 'src/database/clickHouse/clickHouse.service'; +import { + type ClickHouseInsertOptions, + ClickHouseService, +} from 'src/database/clickHouse/clickHouse.service'; import { type EventSink } from 'src/engine/core-modules/event-logs/ingest/event-sink'; -import { type WorkspaceEventEnvelope } from 'src/engine/core-modules/event-logs/types/workspace-event-envelope.type'; +import { + type WorkspaceEventEnvelope, + type WorkspaceEventTable, +} from 'src/engine/core-modules/event-logs/types/workspace-event-envelope.type'; + +const CLICKHOUSE_INSERT_OPTIONS_BY_TABLE: Partial< + Record +> = { + pageview: { + asyncInsertBusyTimeoutMaxMs: 100, + }, +}; @Injectable() export class ClickHouseEventSink implements EventSink { @@ -13,7 +27,10 @@ export class ClickHouseEventSink implements EventSink { return; } - const rowsByTable = new Map[]>(); + const rowsByTable = new Map< + WorkspaceEventTable, + Record[] + >(); for (const event of events) { const rows = rowsByTable.get(event.table) ?? []; @@ -24,7 +41,11 @@ export class ClickHouseEventSink implements EventSink { await Promise.all( [...rowsByTable.entries()].map(async ([table, rows]) => { - const result = await this.clickHouseService.insert(table, rows); + const result = await this.clickHouseService.insert( + table, + rows, + CLICKHOUSE_INSERT_OPTIONS_BY_TABLE[table], + ); if (!result.success) { throw new Error(