Add application-logs module with driver pattern for logic function log persistence (#19486)

## Summary - Introduces a new `application-logs` core module with a driver pattern (disabled/console/clickhouse) to capture and persist logic function execution logs - Adds a ClickHouse `applicationLog` table with per-line log storage, 30-day TTL, and `ORDER BY (workspaceId, timestamp, applicationId, logicFunctionId)` - Surfaces application logs in the existing frontend audit logs table as a new "Application Logs" source with dedicated columns (Function, Timestamp, Level, Message, Execution ID) ## Details **Write path**: `LogicFunctionExecutorService.handleExecutionResult()` parses the multi-line log string from driver output into individual `{ timestamp, level, message }` entries, generates an execution UUID, and passes them to `ApplicationLogsService.writeLogs()` which delegates to the configured driver. **Driver pattern**: Follows the exception-handler module style (Symbol injection token + `forRootAsync` dynamic module). Three drivers: - `DISABLED` (default) — no-op, prevents information leaking - `CONSOLE` — structured stdout logging with level-based `console.*` calls - `CLICKHOUSE` — inserts rows into the `applicationLog` ClickHouse table **Read path**: Extends the existing event-logs module by adding `APPLICATION_LOG` to the `EventLogTable` enum, table name mapping, and normalization logic. **Config**: New `APPLICATION_LOG_DRIVER_TYPE` environment variable (default: `DISABLED`).
2026-04-09 16:35:24 +02:00
parent 5116002ca2
commit 36fbfca069
33 changed files with 730 additions and 137 deletions
@@ -0,0 +1 @@
+export const APPLICATION_LOG_DRIVER = Symbol('APPLICATION_LOG_DRIVER');
@@ -0,0 +1,14 @@
+import { ConfigurableModuleBuilder } from '@nestjs/common';
+
+import { type ApplicationLogsModuleOptions } from 'src/engine/core-modules/application-logs/interfaces/application-logs-module-options.type';
+
+export const {
+  ConfigurableModuleClass,
+  MODULE_OPTIONS_TOKEN,
+  OPTIONS_TYPE,
+  ASYNC_OPTIONS_TYPE,
+} = new ConfigurableModuleBuilder<ApplicationLogsModuleOptions>({
+  moduleName: 'ApplicationLogsModule',
+})
+  .setClassMethodName('forRoot')
+  .build();
@@ -0,0 +1,14 @@
+import { type TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service';
+
+import { type OPTIONS_TYPE } from 'src/engine/core-modules/application-logs/application-logs.module-definition';
+import { ApplicationLogDriverType } from 'src/engine/core-modules/application-logs/interfaces/application-log-driver-type.enum';
+
+export const applicationLogsModuleFactory = async (
+  twentyConfigService: TwentyConfigService,
+): Promise<typeof OPTIONS_TYPE> => {
+  const driverType = twentyConfigService.get('APPLICATION_LOG_DRIVER_TYPE');
+
+  return {
+    type: driverType as ApplicationLogDriverType,
+  };
+};
@@ -0,0 +1,86 @@
+import { type DynamicModule, Global, Module } from '@nestjs/common';
+
+import { ClickHouseModule } from 'src/database/clickHouse/clickHouse.module';
+import { ClickHouseService } from 'src/database/clickHouse/clickHouse.service';
+import { APPLICATION_LOG_DRIVER } from 'src/engine/core-modules/application-logs/application-logs.constants';
+import {
+  type ASYNC_OPTIONS_TYPE,
+  ConfigurableModuleClass,
+  type OPTIONS_TYPE,
+} from 'src/engine/core-modules/application-logs/application-logs.module-definition';
+import { ApplicationLogsService } from 'src/engine/core-modules/application-logs/application-logs.service';
+import { ClickHouseApplicationLogDriver } from 'src/engine/core-modules/application-logs/drivers/clickhouse.driver';
+import { ConsoleApplicationLogDriver } from 'src/engine/core-modules/application-logs/drivers/console.driver';
+import { DisabledApplicationLogDriver } from 'src/engine/core-modules/application-logs/drivers/disabled.driver';
+import { ApplicationLogDriverType } from 'src/engine/core-modules/application-logs/interfaces/application-log-driver-type.enum';
+
+@Global()
+@Module({
+  imports: [ClickHouseModule],
+  providers: [ApplicationLogsService],
+  exports: [ApplicationLogsService],
+})
+export class ApplicationLogsModule extends ConfigurableModuleClass {
+  static forRoot(options: typeof OPTIONS_TYPE): DynamicModule {
+    const provider = {
+      provide: APPLICATION_LOG_DRIVER,
+      useValue: ApplicationLogsModule.createDriver(options.type),
+    };
+    const dynamicModule = super.forRoot(options);
+
+    return {
+      ...dynamicModule,
+      providers: [...(dynamicModule.providers ?? []), provider],
+    };
+  }
+
+  static forRootAsync(options: typeof ASYNC_OPTIONS_TYPE): DynamicModule {
+    const provider = {
+      provide: APPLICATION_LOG_DRIVER,
+      // oxlint-disable-next-line @typescripttypescript/no-explicit-any
+      useFactory: async (
+        clickHouseService: ClickHouseService,
+        ...args: unknown[]
+      ) => {
+        const config = await options?.useFactory?.(...args);
+
+        if (!config) {
+          return new DisabledApplicationLogDriver();
+        }
+
+        return ApplicationLogsModule.createDriver(
+          config.type,
+          clickHouseService,
+        );
+      },
+      inject: [ClickHouseService, ...(options.inject || [])],
+    };
+    const dynamicModule = super.forRootAsync(options);
+
+    return {
+      ...dynamicModule,
+      providers: [...(dynamicModule.providers ?? []), provider],
+    };
+  }
+
+  private static createDriver(
+    type: ApplicationLogDriverType,
+    clickHouseService?: ClickHouseService,
+  ) {
+    switch (type) {
+      case ApplicationLogDriverType.CONSOLE:
+        return new ConsoleApplicationLogDriver();
+      case ApplicationLogDriverType.CLICKHOUSE:
+        if (!clickHouseService) {
+          throw new Error(
+            'ClickHouseService is required for the ClickHouse application log driver',
+          );
+        }
+
+        return new ClickHouseApplicationLogDriver(clickHouseService);
+      case ApplicationLogDriverType.DISABLED:
+      default:
+        return new DisabledApplicationLogDriver();
+    }
+  }
+}
@@ -0,0 +1,17 @@
+import { Inject, Injectable } from '@nestjs/common';
+
+import { APPLICATION_LOG_DRIVER } from 'src/engine/core-modules/application-logs/application-logs.constants';
+import { type ApplicationLogDriverInterface } from 'src/engine/core-modules/application-logs/interfaces/application-log-driver.interface';
+import { type ApplicationLogEntry } from 'src/engine/core-modules/application-logs/interfaces/application-log-entry.interface';
+
+@Injectable()
+export class ApplicationLogsService {
+  constructor(
+    @Inject(APPLICATION_LOG_DRIVER)
+    private driver: ApplicationLogDriverInterface,
+  ) {}
+
+  async writeLogs(entries: ApplicationLogEntry[]): Promise<void> {
+    return this.driver.writeLogs(entries);
+  }
+}
@@ -0,0 +1,37 @@
+import { Logger } from '@nestjs/common';
+
+import { ClickHouseService } from 'src/database/clickHouse/clickHouse.service';
+import { formatDateForClickHouse } from 'src/database/clickHouse/clickHouse.util';
+import { type ApplicationLogEntry } from 'src/engine/core-modules/application-logs/interfaces/application-log-entry.interface';
+import { type ApplicationLogDriverInterface } from 'src/engine/core-modules/application-logs/interfaces/application-log-driver.interface';
+
+export class ClickHouseApplicationLogDriver
+  implements ApplicationLogDriverInterface
+{
+  private readonly logger = new Logger(ClickHouseApplicationLogDriver.name);
+
+  constructor(private readonly clickHouseService: ClickHouseService) {}
+
+  async writeLogs(entries: ApplicationLogEntry[]): Promise<void> {
+    if (entries.length === 0) {
+      return;
+    }
+
+    const rows = entries.map((entry) => ({
+      timestamp: formatDateForClickHouse(entry.timestamp),
+      workspaceId: entry.workspaceId,
+      applicationId: entry.applicationId,
+      logicFunctionId: entry.logicFunctionId,
+      logicFunctionName: entry.logicFunctionName,
+      executionId: entry.executionId,
+      level: entry.level,
+      message: entry.message,
+    }));
+
+    const result = await this.clickHouseService.insert('applicationLog', rows);
+
+    if (!result.success) {
+      this.logger.error('Failed to insert application logs into ClickHouse');
+    }
+  }
+}
@@ -0,0 +1,31 @@
+import { Logger } from '@nestjs/common';
+
+import { type ApplicationLogDriverInterface } from 'src/engine/core-modules/application-logs/interfaces/application-log-driver.interface';
+import { type ApplicationLogEntry } from 'src/engine/core-modules/application-logs/interfaces/application-log-entry.interface';
+
+export class ConsoleApplicationLogDriver
+  implements ApplicationLogDriverInterface
+{
+  private readonly logger = new Logger(ConsoleApplicationLogDriver.name);
+
+  async writeLogs(entries: ApplicationLogEntry[]): Promise<void> {
+    for (const entry of entries) {
+      const context = `${entry.logicFunctionName}:${entry.executionId}`;
+
+      switch (entry.level) {
+        case 'ERROR':
+          this.logger.error(entry.message, undefined, context);
+          break;
+        case 'WARN':
+          this.logger.warn(entry.message, context);
+          break;
+        case 'DEBUG':
+          this.logger.debug(entry.message, context);
+          break;
+        default:
+          this.logger.log(entry.message, context);
+          break;
+      }
+    }
+  }
+}
@@ -0,0 +1,9 @@
+import { type ApplicationLogDriverInterface } from 'src/engine/core-modules/application-logs/interfaces/application-log-driver.interface';
+
+export class DisabledApplicationLogDriver
+  implements ApplicationLogDriverInterface
+{
+  async writeLogs(): Promise<void> {
+    return;
+  }
+}
@@ -0,0 +1,5 @@
+export enum ApplicationLogDriverType {
+  DISABLED = 'DISABLED',
+  CONSOLE = 'CONSOLE',
+  CLICKHOUSE = 'CLICKHOUSE',
+}
@@ -0,0 +1,5 @@
+import { type ApplicationLogEntry } from 'src/engine/core-modules/application-logs/interfaces/application-log-entry.interface';
+
+export interface ApplicationLogDriverInterface {
+  writeLogs(entries: ApplicationLogEntry[]): Promise<void>;
+}
@@ -0,0 +1,10 @@
+export type ApplicationLogEntry = {
+  timestamp: Date;
+  workspaceId: string;
+  applicationId: string;
+  logicFunctionId: string;
+  logicFunctionName: string;
+  executionId: string;
+  level: string;
+  message: string;
+};
@@ -0,0 +1,5 @@
+import { type ApplicationLogDriverType } from 'src/engine/core-modules/application-logs/interfaces/application-log-driver-type.enum';
+
+export type ApplicationLogsModuleOptions = {
+  type: ApplicationLogDriverType;
+};
@@ -0,0 +1,5 @@
+export type ParsedLogLine = {
+  timestamp: Date;
+  level: string;
+  message: string;
+};
@@ -0,0 +1,125 @@
+import { parseApplicationLogLines } from './parse-application-log-lines';
+
+describe('parseApplicationLogLines', () => {
+  it('should return an empty array for empty string', () => {
+    expect(parseApplicationLogLines('')).toEqual([]);
+  });
+
+  it('should return an empty array for undefined-ish input', () => {
+    expect(parseApplicationLogLines(undefined as unknown as string)).toEqual(
+      [],
+    );
+  });
+
+  it('should parse a structured INFO log line', () => {
+    const raw = '2024-06-15T10:30:00.123Z INFO Hello world';
+
+    expect(parseApplicationLogLines(raw)).toEqual([
+      {
+        timestamp: new Date('2024-06-15T10:30:00.123Z'),
+        level: 'INFO',
+        message: 'Hello world',
+      },
+    ]);
+  });
+
+  it('should parse all supported log levels', () => {
+    const raw = [
+      '2024-01-01T00:00:00.000Z INFO info message',
+      '2024-01-01T00:00:01.000Z ERROR error message',
+      '2024-01-01T00:00:02.000Z WARN warn message',
+      '2024-01-01T00:00:03.000Z DEBUG debug message',
+    ].join('\n');
+
+    expect(parseApplicationLogLines(raw)).toEqual([
+      {
+        timestamp: new Date('2024-01-01T00:00:00.000Z'),
+        level: 'INFO',
+        message: 'info message',
+      },
+      {
+        timestamp: new Date('2024-01-01T00:00:01.000Z'),
+        level: 'ERROR',
+        message: 'error message',
+      },
+      {
+        timestamp: new Date('2024-01-01T00:00:02.000Z'),
+        level: 'WARN',
+        message: 'warn message',
+      },
+      {
+        timestamp: new Date('2024-01-01T00:00:03.000Z'),
+        level: 'DEBUG',
+        message: 'debug message',
+      },
+    ]);
+  });
+
+  it('should default unstructured lines to INFO with current timestamp', () => {
+    const now = Date.now();
+    const raw = 'some plain text without timestamp or level';
+    const result = parseApplicationLogLines(raw);
+
+    expect(result).toHaveLength(1);
+    expect(result[0].level).toBe('INFO');
+    expect(result[0].message).toBe(
+      'some plain text without timestamp or level',
+    );
+    expect(result[0].timestamp.getTime()).toBeGreaterThanOrEqual(now);
+    expect(result[0].timestamp.getTime()).toBeLessThanOrEqual(now + 1000);
+  });
+
+  it('should skip empty lines', () => {
+    const raw =
+      '2024-01-01T00:00:00.000Z INFO first\n\n\n2024-01-01T00:00:01.000Z ERROR second\n';
+
+    expect(parseApplicationLogLines(raw)).toEqual([
+      {
+        timestamp: new Date('2024-01-01T00:00:00.000Z'),
+        level: 'INFO',
+        message: 'first',
+      },
+      {
+        timestamp: new Date('2024-01-01T00:00:01.000Z'),
+        level: 'ERROR',
+        message: 'second',
+      },
+    ]);
+  });
+
+  it('should handle a mix of structured and unstructured lines', () => {
+    const raw = [
+      '2024-01-01T00:00:00.000Z INFO structured line',
+      'plain unstructured line',
+      '2024-01-01T00:00:01.000Z ERROR another structured',
+    ].join('\n');
+
+    const result = parseApplicationLogLines(raw);
+
+    expect(result).toHaveLength(3);
+    expect(result[0]).toEqual({
+      timestamp: new Date('2024-01-01T00:00:00.000Z'),
+      level: 'INFO',
+      message: 'structured line',
+    });
+    expect(result[1].level).toBe('INFO');
+    expect(result[1].message).toBe('plain unstructured line');
+    expect(result[2]).toEqual({
+      timestamp: new Date('2024-01-01T00:00:01.000Z'),
+      level: 'ERROR',
+      message: 'another structured',
+    });
+  });
+
+  it('should preserve message content including special characters', () => {
+    const raw = '2024-01-01T00:00:00.000Z INFO {"key": "value", "count": 42}';
+
+    expect(parseApplicationLogLines(raw)).toEqual([
+      {
+        timestamp: new Date('2024-01-01T00:00:00.000Z'),
+        level: 'INFO',
+        message: '{"key": "value", "count": 42}',
+      },
+    ]);
+  });
+});
@@ -0,0 +1,31 @@
+import { type ParsedLogLine } from 'src/engine/core-modules/application-logs/types/parsed-log-line.type';
+
+// Matches: 2024-01-01T00:00:00.000Z INFO some message
+const LOG_LINE_REGEX =
+  /^(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z)\s+(INFO|ERROR|WARN|DEBUG)\s+(.*)$/;
+
+export const parseApplicationLogLines = (rawLogs: string): ParsedLogLine[] => {
+  if (!rawLogs) {
+    return [];
+  }
+
+  const lines = rawLogs.split('\n').filter(Boolean);
+
+  return lines.map((line) => {
+    const match = line.match(LOG_LINE_REGEX);
+
+    if (match) {
+      return {
+        timestamp: new Date(match[1]),
+        level: match[2],
+        message: match[3],
+      };
+    }
+
+    return {
+      timestamp: new Date(),
+      level: 'INFO',
+      message: line,
+    };
+  });
+};
				`@@ -0,0 +1 @@`
				`export const APPLICATION_LOG_DRIVER = Symbol('APPLICATION_LOG_DRIVER');`