houseassassin/twenty
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add application-logs module with driver pattern for logic function log persistence (#19486)

Browse Source 
## Summary

- Introduces a new `application-logs` core module with a driver pattern
(disabled/console/clickhouse) to capture and persist logic function
execution logs
- Adds a ClickHouse `applicationLog` table with per-line log storage,
30-day TTL, and `ORDER BY (workspaceId, timestamp, applicationId,
logicFunctionId)`
- Surfaces application logs in the existing frontend audit logs table as
a new "Application Logs" source with dedicated columns (Function,
Timestamp, Level, Message, Execution ID)

## Details

**Write path**: `LogicFunctionExecutorService.handleExecutionResult()`
parses the multi-line log string from driver output into individual `{
timestamp, level, message }` entries, generates an execution UUID, and
passes them to `ApplicationLogsService.writeLogs()` which delegates to
the configured driver.

**Driver pattern**: Follows the exception-handler module style (Symbol
injection token + `forRootAsync` dynamic module). Three drivers:
- `DISABLED` (default) — no-op, prevents information leaking
- `CONSOLE` — structured stdout logging with level-based `console.*`
calls
- `CLICKHOUSE` — inserts rows into the `applicationLog` ClickHouse table

**Read path**: Extends the existing event-logs module by adding
`APPLICATION_LOG` to the `EventLogTable` enum, table name mapping, and
normalization logic.

**Config**: New `APPLICATION_LOG_DRIVER_TYPE` environment variable
(default: `DISABLED`).
This commit is contained in:
Charles Bochet
2026-04-09 16:35:24 +02:00
committed by GitHub
parent 5116002ca2
commit 36fbfca069
33 changed files with 730 additions and 137 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/ci-create-app-e2e-hello-world.yaml
+12 -4
View File
@@ -139,11 +139,19 @@ jobs:
cd /tmp/e2e-test-workspace/test-app
npx --no-install twenty --version
- name: Create test database
run: PGPASSWORD=postgres psql -h localhost -p 5432 -U postgres -d postgres -c 'CREATE DATABASE "test";'
- name: Setup server environment
run: npx nx reset:env:e2e-testing-server twenty-server
- name: Setup database and start server
run: npx nx run twenty-server:database:reset && nohup npx nx start:ci twenty-server &
- name: Create databases
run: |
PGPASSWORD=postgres psql -h localhost -p 5432 -U postgres -d postgres -c 'CREATE DATABASE "default";'
PGPASSWORD=postgres psql -h localhost -p 5432 -U postgres -d postgres -c 'CREATE DATABASE "test";'
- name: Setup database
run: npx nx run twenty-server:database:reset
- name: Start server
run: nohup npx nx start:ci twenty-server &
- name: Wait for server to be ready
run: npx wait-on http://localhost:3000/healthz --timeout 120000 --interval 1000
.github/workflows/ci-create-app-e2e-minimal.yaml
+13 -15
View File
@@ -133,11 +133,19 @@ jobs:
cd /tmp/e2e-test-workspace/test-app
npx --no-install twenty --version
- name: Create test database
run: PGPASSWORD=postgres psql -h localhost -p 5432 -U postgres -d postgres -c 'CREATE DATABASE "test";'
- name: Setup server environment
run: npx nx reset:env:e2e-testing-server twenty-server
- name: Setup database and start server
run: npx nx run twenty-server:database:reset && nohup npx nx start:ci twenty-server &
- name: Create databases
run: |
PGPASSWORD=postgres psql -h localhost -p 5432 -U postgres -d postgres -c 'CREATE DATABASE "default";'
PGPASSWORD=postgres psql -h localhost -p 5432 -U postgres -d postgres -c 'CREATE DATABASE "test";'
- name: Setup database
run: npx nx run twenty-server:database:reset
- name: Start server
run: nohup npx nx start:ci twenty-server &
- name: Wait for server to be ready
run: npx wait-on http://localhost:3000/healthz --timeout 120000 --interval 1000
@@ -147,17 +155,7 @@ jobs:
cd /tmp/e2e-test-workspace/test-app
npx --no-install twenty remote add --api-key ${{ env.TWENTY_API_KEY }} --api-url ${{ env.TWENTY_API_URL }}
- name: Deploy scaffolded app
run: |
cd /tmp/e2e-test-workspace/test-app
npx --no-install twenty deploy
- name: Install scaffolded app
run: |
cd /tmp/e2e-test-workspace/test-app
npx --no-install twenty install
- name: Run scaffolded app integration test
- name: Run scaffolded app integration test (deploys, installs, and verifies the app)
run: |
cd /tmp/e2e-test-workspace/test-app
yarn test
.github/workflows/ci-create-app-e2e-postcard.yaml
+12 -4
View File
@@ -137,11 +137,19 @@ jobs:
cd /tmp/e2e-test-workspace/test-app
npx --no-install twenty --version
- name: Create test database
run: PGPASSWORD=postgres psql -h localhost -p 5432 -U postgres -d postgres -c 'CREATE DATABASE "test";'
- name: Setup server environment
run: npx nx reset:env:e2e-testing-server twenty-server
- name: Setup database and start server
run: npx nx run twenty-server:database:reset && nohup npx nx start:ci twenty-server &
- name: Create databases
run: |
PGPASSWORD=postgres psql -h localhost -p 5432 -U postgres -d postgres -c 'CREATE DATABASE "default";'
PGPASSWORD=postgres psql -h localhost -p 5432 -U postgres -d postgres -c 'CREATE DATABASE "test";'
- name: Setup database
run: npx nx run twenty-server:database:reset
- name: Start server
run: nohup npx nx start:ci twenty-server &
- name: Wait for server to be ready
run: npx wait-on http://localhost:3000/healthz --timeout 120000 --interval 1000
.github/workflows/ci-example-app-hello-world.yaml
+12 -4
View File
@@ -62,11 +62,19 @@ jobs:
- name: Build SDK packages
run: npx nx build twenty-sdk
- name: Create test database
run: PGPASSWORD=postgres psql -h localhost -p 5432 -U postgres -d postgres -c 'CREATE DATABASE "test";'
- name: Setup server environment
run: npx nx reset:env:e2e-testing-server twenty-server
- name: Setup database and start server
run: npx nx run twenty-server:database:reset && nohup npx nx start:ci twenty-server &
- name: Create databases
run: |
PGPASSWORD=postgres psql -h localhost -p 5432 -U postgres -d postgres -c 'CREATE DATABASE "default";'
PGPASSWORD=postgres psql -h localhost -p 5432 -U postgres -d postgres -c 'CREATE DATABASE "test";'
- name: Setup database
run: npx nx run twenty-server:database:reset
- name: Start server
run: nohup npx nx start:ci twenty-server &
- name: Wait for server to be ready
run: npx wait-on http://localhost:3000/healthz --timeout 120000 --interval 1000
.github/workflows/ci-example-app-postcard.yaml
+12 -4
View File
@@ -62,11 +62,19 @@ jobs:
- name: Build SDK packages
run: npx nx build twenty-sdk
- name: Create test database
run: PGPASSWORD=postgres psql -h localhost -p 5432 -U postgres -d postgres -c 'CREATE DATABASE "test";'
- name: Setup server environment
run: npx nx reset:env:e2e-testing-server twenty-server
- name: Setup database and start server
run: npx nx run twenty-server:database:reset && nohup npx nx start:ci twenty-server &
- name: Create databases
run: |
PGPASSWORD=postgres psql -h localhost -p 5432 -U postgres -d postgres -c 'CREATE DATABASE "default";'
PGPASSWORD=postgres psql -h localhost -p 5432 -U postgres -d postgres -c 'CREATE DATABASE "test";'
- name: Setup database
run: npx nx run twenty-server:database:reset
- name: Start server
run: nohup npx nx start:ci twenty-server &
- name: Wait for server to be ready
run: npx wait-on http://localhost:3000/healthz --timeout 120000 --interval 1000
packages/twenty-client-sdk/src/metadata/generated/schema.graphql
+1
View File
@@ -3384,6 +3384,7 @@ enum EventLogTable {
PAGEVIEW
OBJECT_EVENT
USAGE_EVENT
APPLICATION_LOG
}
input EventLogFiltersInput {
packages/twenty-client-sdk/src/metadata/generated/schema.ts
+3 -2
View File
@@ -2880,7 +2880,7 @@ export type SortDirection = 'ASC' | 'DESC'
/** Sort Nulls Options */
export type SortNulls = 'NULLS_FIRST' | 'NULLS_LAST'
export type EventLogTable = 'WORKSPACE_EVENT' | 'PAGEVIEW' | 'OBJECT_EVENT' | 'USAGE_EVENT'
export type EventLogTable = 'WORKSPACE_EVENT' | 'PAGEVIEW' | 'OBJECT_EVENT' | 'USAGE_EVENT' | 'APPLICATION_LOG'
export type UsageOperationType = 'AI_CHAT_TOKEN' | 'AI_WORKFLOW_TOKEN' | 'WORKFLOW_EXECUTION' | 'CODE_EXECUTION' | 'WEB_SEARCH'
@@ -9619,7 +9619,8 @@ export const enumEventLogTable = {
WORKSPACE_EVENT: 'WORKSPACE_EVENT' as const,
PAGEVIEW: 'PAGEVIEW' as const,
OBJECT_EVENT: 'OBJECT_EVENT' as const,
USAGE_EVENT: 'USAGE_EVENT' as const
USAGE_EVENT: 'USAGE_EVENT' as const,
APPLICATION_LOG: 'APPLICATION_LOG' as const
}
export const enumUsageOperationType = {
packages/twenty-front/src/generated-metadata/graphql.ts
+1
View File
@@ -1725,6 +1725,7 @@ export type EventLogRecord = {
};
export enum EventLogTable {
APPLICATION_LOG = 'APPLICATION_LOG',
OBJECT_EVENT = 'OBJECT_EVENT',
PAGEVIEW = 'PAGEVIEW',
USAGE_EVENT = 'USAGE_EVENT',
packages/twenty-front/src/pages/settings/security/event-logs/components/EventLogResultsTable.tsx
+53 -97
View File
@@ -1,6 +1,5 @@
import { SettingsEmptyPlaceholder } from '@/settings/components/SettingsEmptyPlaceholder';
import { styled } from '@linaria/react';
import { msg } from '@lingui/core/macro';
import { Trans, useLingui } from '@lingui/react/macro';
import { useCallback, useContext, useEffect, useState } from 'react';
import { useInView } from 'react-intersection-observer';
@@ -15,14 +14,16 @@ import { useScrollWrapperHTMLElement } from '@/ui/utilities/scroll/hooks/useScro
import { ThemeContext, themeCssVariables } from 'twenty-ui/theme-constants';
import { beautifyPastDateRelativeToNow } from '~/utils/date-utils';
import { type MessageDescriptor } from '@lingui/core';
import {
type EventLogRecord,
EventLogTable,
} from '~/generated-metadata/graphql';
import { EventLogJsonCell } from './EventLogJsonCell';
import {
type ColumnConfig,
getColumnsForEventLogTable,
} from '~/pages/settings/security/event-logs/utils/getColumnsForEventLogTable';
import { EventLogJsonCell } from '~/pages/settings/security/event-logs/components/EventLogJsonCell';
type EventLogResultsTableProps = {
records: EventLogRecord[];
@@ -32,76 +33,6 @@ type EventLogResultsTableProps = {
selectedTable: EventLogTable;
};
type ColumnConfig = {
id: string;
label: MessageDescriptor;
minWidth: number;
defaultWidth: number;
};
const DEFAULT_COLUMNS: ColumnConfig[] = [
{ id: 'event', label: msg`Event`, minWidth: 100, defaultWidth: 200 },
{ id: 'timestamp', label: msg`Timestamp`, minWidth: 100, defaultWidth: 150 },
{
id: 'userId',
label: msg`User`,
minWidth: 100,
defaultWidth: 150,
},
{
id: 'properties',
label: msg`Properties`,
minWidth: 200,
defaultWidth: 400,
},
];
const OBJECT_EVENT_COLUMNS: ColumnConfig[] = [
{ id: 'event', label: msg`Event`, minWidth: 100, defaultWidth: 180 },
{ id: 'timestamp', label: msg`Timestamp`, minWidth: 100, defaultWidth: 130 },
{
id: 'userId',
label: msg`User`,
minWidth: 100,
defaultWidth: 130,
},
{ id: 'recordId', label: msg`Record ID`, minWidth: 100, defaultWidth: 130 },
{
id: 'objectMetadataId',
label: msg`Object ID`,
minWidth: 100,
defaultWidth: 130,
},
{
id: 'properties',
label: msg`Properties`,
minWidth: 150,
defaultWidth: 300,
},
];
const USAGE_EVENT_COLUMNS: ColumnConfig[] = [
{
id: 'event',
label: msg`Resource Type`,
minWidth: 100,
defaultWidth: 130,
},
{
id: 'timestamp',
label: msg`Timestamp`,
minWidth: 100,
defaultWidth: 140,
},
{ id: 'userId', label: msg`User`, minWidth: 100, defaultWidth: 130 },
{
id: 'properties',
label: msg`Details`,
minWidth: 200,
defaultWidth: 400,
},
];
const StyledScrollWrapperContainer = styled.div`
height: 100%;
overflow: hidden;
@@ -176,12 +107,9 @@ export const EventLogResultsTable = ({
const { t } = useLingui();
const showObjectEventColumns = selectedTable === EventLogTable.OBJECT_EVENT;
const baseColumns =
selectedTable === EventLogTable.OBJECT_EVENT
? OBJECT_EVENT_COLUMNS
: selectedTable === EventLogTable.USAGE_EVENT
? USAGE_EVENT_COLUMNS
: DEFAULT_COLUMNS;
const showApplicationLogColumns =
selectedTable === EventLogTable.APPLICATION_LOG;
const baseColumns = getColumnsForEventLogTable(selectedTable);
const [columnWidths, setColumnWidths] = useState<Record<string, number>>(() =>
Object.fromEntries(baseColumns.map((col) => [col.id, col.defaultWidth])),
@@ -333,38 +261,66 @@ export const EventLogResultsTable = ({
>
{beautifyPastDateRelativeToNow(record.timestamp)}
</TableCell>
<TableCell
overflow="hidden"
textOverflow="ellipsis"
whiteSpace="nowrap"
>
{record.userId ?? '-'}
</TableCell>
{showObjectEventColumns && (
{showApplicationLogColumns ? (
<>
<TableCell
overflow="hidden"
textOverflow="ellipsis"
whiteSpace="nowrap"
>
{record.recordId ?? '-'}
{record.properties?.level ?? '-'}
</TableCell>
<TableCell
overflow="hidden"
textOverflow="ellipsis"
whiteSpace="nowrap"
>
{record.objectMetadataId ?? '-'}
{record.properties?.message ?? '-'}
</TableCell>
<TableCell
overflow="hidden"
textOverflow="ellipsis"
whiteSpace="nowrap"
>
{record.properties?.executionId ?? '-'}
</TableCell>
</>
) : (
<>
<TableCell
overflow="hidden"
textOverflow="ellipsis"
whiteSpace="nowrap"
>
{record.userId ?? '-'}
</TableCell>
{showObjectEventColumns && (
<>
<TableCell
overflow="hidden"
textOverflow="ellipsis"
whiteSpace="nowrap"
>
{record.recordId ?? '-'}
</TableCell>
<TableCell
overflow="hidden"
textOverflow="ellipsis"
whiteSpace="nowrap"
>
{record.objectMetadataId ?? '-'}
</TableCell>
</>
)}
<TableCell
overflow="hidden"
textOverflow="ellipsis"
whiteSpace="nowrap"
>
<EventLogJsonCell value={record.properties} />
</TableCell>
</>
)}
<TableCell
overflow="hidden"
textOverflow="ellipsis"
whiteSpace="nowrap"
>
<EventLogJsonCell value={record.properties} />
</TableCell>
</TableRow>
))}
</Table>
packages/twenty-front/src/pages/settings/security/event-logs/components/EventLogTableSelector.tsx
+4
View File
@@ -31,6 +31,10 @@ export const EventLogTableSelector = ({
value: EventLogTable.USAGE_EVENT,
label: t`Usage Events`,
},
{
value: EventLogTable.APPLICATION_LOG,
label: t`Application Logs`,
},
];
return (
packages/twenty-front/src/pages/settings/security/event-logs/utils/getColumnsForEventLogTable.ts
+121
View File
@@ -0,0 +1,121 @@
import { msg } from '@lingui/core/macro';
import { type MessageDescriptor } from '@lingui/core';
import { EventLogTable } from '~/generated-metadata/graphql';
export type ColumnConfig = {
id: string;
label: MessageDescriptor;
minWidth: number;
defaultWidth: number;
};
const DEFAULT_COLUMNS: ColumnConfig[] = [
{ id: 'event', label: msg`Event`, minWidth: 100, defaultWidth: 200 },
{
id: 'timestamp',
label: msg`Timestamp`,
minWidth: 100,
defaultWidth: 150,
},
{ id: 'userId', label: msg`User`, minWidth: 100, defaultWidth: 150 },
{
id: 'properties',
label: msg`Properties`,
minWidth: 200,
defaultWidth: 400,
},
];
const OBJECT_EVENT_COLUMNS: ColumnConfig[] = [
{ id: 'event', label: msg`Event`, minWidth: 100, defaultWidth: 180 },
{
id: 'timestamp',
label: msg`Timestamp`,
minWidth: 100,
defaultWidth: 130,
},
{ id: 'userId', label: msg`User`, minWidth: 100, defaultWidth: 130 },
{
id: 'recordId',
label: msg`Record ID`,
minWidth: 100,
defaultWidth: 130,
},
{
id: 'objectMetadataId',
label: msg`Object ID`,
minWidth: 100,
defaultWidth: 130,
},
{
id: 'properties',
label: msg`Properties`,
minWidth: 150,
defaultWidth: 300,
},
];
const USAGE_EVENT_COLUMNS: ColumnConfig[] = [
{
id: 'event',
label: msg`Resource Type`,
minWidth: 100,
defaultWidth: 130,
},
{
id: 'timestamp',
label: msg`Timestamp`,
minWidth: 100,
defaultWidth: 140,
},
{ id: 'userId', label: msg`User`, minWidth: 100, defaultWidth: 130 },
{
id: 'properties',
label: msg`Details`,
minWidth: 200,
defaultWidth: 400,
},
];
const APPLICATION_LOG_COLUMNS: ColumnConfig[] = [
{
id: 'event',
label: msg`Function`,
minWidth: 100,
defaultWidth: 160,
},
{
id: 'timestamp',
label: msg`Timestamp`,
minWidth: 100,
defaultWidth: 140,
},
{ id: 'level', label: msg`Level`, minWidth: 60, defaultWidth: 80 },
{
id: 'message',
label: msg`Message`,
minWidth: 200,
defaultWidth: 400,
},
{
id: 'executionId',
label: msg`Execution ID`,
minWidth: 100,
defaultWidth: 140,
},
];
const COLUMNS_BY_TABLE: Record<EventLogTable, ColumnConfig[]> = {
[EventLogTable.OBJECT_EVENT]: OBJECT_EVENT_COLUMNS,
[EventLogTable.USAGE_EVENT]: USAGE_EVENT_COLUMNS,
[EventLogTable.APPLICATION_LOG]: APPLICATION_LOG_COLUMNS,
[EventLogTable.WORKSPACE_EVENT]: DEFAULT_COLUMNS,
[EventLogTable.PAGEVIEW]: DEFAULT_COLUMNS,
};
export const getColumnsForEventLogTable = (
table: EventLogTable,
): ColumnConfig[] => {
return COLUMNS_BY_TABLE[table] ?? DEFAULT_COLUMNS;
};
packages/twenty-server/src/database/clickHouse/migrations/005-create-application-log-table.sql
+15
View File
@@ -0,0 +1,15 @@
CREATE TABLE IF NOT EXISTS applicationLog
(
`timestamp` DateTime64(3) NOT NULL,
`workspaceId` String NOT NULL,
`applicationId` String DEFAULT '',
`logicFunctionId` String DEFAULT '',
`logicFunctionName` String DEFAULT '',
`executionId` String DEFAULT '',
`level` LowCardinality(String) DEFAULT 'INFO',
`message` String NOT NULL,
`properties` JSON
)
ENGINE = MergeTree
ORDER BY (workspaceId, timestamp, applicationId, logicFunctionId)
TTL timestamp + INTERVAL 30 DAY DELETE;
packages/twenty-server/src/engine/core-modules/application-logs/application-logs.constants.ts
+1
View File
@@ -0,0 +1 @@
export const APPLICATION_LOG_DRIVER = Symbol('APPLICATION_LOG_DRIVER');
packages/twenty-server/src/engine/core-modules/application-logs/application-logs.module-definition.ts
+14
View File
@@ -0,0 +1,14 @@
import { ConfigurableModuleBuilder } from '@nestjs/common';
import { type ApplicationLogsModuleOptions } from 'src/engine/core-modules/application-logs/interfaces/application-logs-module-options.type';
export const {
ConfigurableModuleClass,
MODULE_OPTIONS_TOKEN,
OPTIONS_TYPE,
ASYNC_OPTIONS_TYPE,
} = new ConfigurableModuleBuilder<ApplicationLogsModuleOptions>({
moduleName: 'ApplicationLogsModule',
})
.setClassMethodName('forRoot')
.build();
packages/twenty-server/src/engine/core-modules/application-logs/application-logs.module-factory.ts
+14
View File
@@ -0,0 +1,14 @@
import { type TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service';
import { type OPTIONS_TYPE } from 'src/engine/core-modules/application-logs/application-logs.module-definition';
import { ApplicationLogDriverType } from 'src/engine/core-modules/application-logs/interfaces/application-log-driver-type.enum';
export const applicationLogsModuleFactory = async (
twentyConfigService: TwentyConfigService,
): Promise<typeof OPTIONS_TYPE> => {
const driverType = twentyConfigService.get('APPLICATION_LOG_DRIVER_TYPE');
return {
type: driverType as ApplicationLogDriverType,
};
};
packages/twenty-server/src/engine/core-modules/application-logs/application-logs.module.ts
+86
View File
@@ -0,0 +1,86 @@
import { type DynamicModule, Global, Module } from '@nestjs/common';
import { ClickHouseModule } from 'src/database/clickHouse/clickHouse.module';
import { ClickHouseService } from 'src/database/clickHouse/clickHouse.service';
import { APPLICATION_LOG_DRIVER } from 'src/engine/core-modules/application-logs/application-logs.constants';
import {
type ASYNC_OPTIONS_TYPE,
ConfigurableModuleClass,
type OPTIONS_TYPE,
} from 'src/engine/core-modules/application-logs/application-logs.module-definition';
import { ApplicationLogsService } from 'src/engine/core-modules/application-logs/application-logs.service';
import { ClickHouseApplicationLogDriver } from 'src/engine/core-modules/application-logs/drivers/clickhouse.driver';
import { ConsoleApplicationLogDriver } from 'src/engine/core-modules/application-logs/drivers/console.driver';
import { DisabledApplicationLogDriver } from 'src/engine/core-modules/application-logs/drivers/disabled.driver';
import { ApplicationLogDriverType } from 'src/engine/core-modules/application-logs/interfaces/application-log-driver-type.enum';
@Global()
@Module({
imports: [ClickHouseModule],
providers: [ApplicationLogsService],
exports: [ApplicationLogsService],
})
export class ApplicationLogsModule extends ConfigurableModuleClass {
static forRoot(options: typeof OPTIONS_TYPE): DynamicModule {
const provider = {
provide: APPLICATION_LOG_DRIVER,
useValue: ApplicationLogsModule.createDriver(options.type),
};
const dynamicModule = super.forRoot(options);
return {
...dynamicModule,
providers: [...(dynamicModule.providers ?? []), provider],
};
}
static forRootAsync(options: typeof ASYNC_OPTIONS_TYPE): DynamicModule {
const provider = {
provide: APPLICATION_LOG_DRIVER,
// oxlint-disable-next-line @typescripttypescript/no-explicit-any
useFactory: async (
clickHouseService: ClickHouseService,
...args: unknown[]
) => {
const config = await options?.useFactory?.(...args);
if (!config) {
return new DisabledApplicationLogDriver();
}
return ApplicationLogsModule.createDriver(
config.type,
clickHouseService,
);
},
inject: [ClickHouseService, ...(options.inject || [])],
};
const dynamicModule = super.forRootAsync(options);
return {
...dynamicModule,
providers: [...(dynamicModule.providers ?? []), provider],
};
}
private static createDriver(
type: ApplicationLogDriverType,
clickHouseService?: ClickHouseService,
) {
switch (type) {
case ApplicationLogDriverType.CONSOLE:
return new ConsoleApplicationLogDriver();
case ApplicationLogDriverType.CLICKHOUSE:
if (!clickHouseService) {
throw new Error(
'ClickHouseService is required for the ClickHouse application log driver',
);
}
return new ClickHouseApplicationLogDriver(clickHouseService);
case ApplicationLogDriverType.DISABLED:
default:
return new DisabledApplicationLogDriver();
}
}
}
packages/twenty-server/src/engine/core-modules/application-logs/application-logs.service.ts
+17
View File
@@ -0,0 +1,17 @@
import { Inject, Injectable } from '@nestjs/common';
import { APPLICATION_LOG_DRIVER } from 'src/engine/core-modules/application-logs/application-logs.constants';
import { type ApplicationLogDriverInterface } from 'src/engine/core-modules/application-logs/interfaces/application-log-driver.interface';
import { type ApplicationLogEntry } from 'src/engine/core-modules/application-logs/interfaces/application-log-entry.interface';
@Injectable()
export class ApplicationLogsService {
constructor(
@Inject(APPLICATION_LOG_DRIVER)
private driver: ApplicationLogDriverInterface,
) {}
async writeLogs(entries: ApplicationLogEntry[]): Promise<void> {
return this.driver.writeLogs(entries);
}
}
packages/twenty-server/src/engine/core-modules/application-logs/drivers/clickhouse.driver.ts
+37
View File
@@ -0,0 +1,37 @@
import { Logger } from '@nestjs/common';
import { ClickHouseService } from 'src/database/clickHouse/clickHouse.service';
import { formatDateForClickHouse } from 'src/database/clickHouse/clickHouse.util';
import { type ApplicationLogEntry } from 'src/engine/core-modules/application-logs/interfaces/application-log-entry.interface';
import { type ApplicationLogDriverInterface } from 'src/engine/core-modules/application-logs/interfaces/application-log-driver.interface';
export class ClickHouseApplicationLogDriver
implements ApplicationLogDriverInterface
{
private readonly logger = new Logger(ClickHouseApplicationLogDriver.name);
constructor(private readonly clickHouseService: ClickHouseService) {}
async writeLogs(entries: ApplicationLogEntry[]): Promise<void> {
if (entries.length === 0) {
return;
}
const rows = entries.map((entry) => ({
timestamp: formatDateForClickHouse(entry.timestamp),
workspaceId: entry.workspaceId,
applicationId: entry.applicationId,
logicFunctionId: entry.logicFunctionId,
logicFunctionName: entry.logicFunctionName,
executionId: entry.executionId,
level: entry.level,
message: entry.message,
}));
const result = await this.clickHouseService.insert('applicationLog', rows);
if (!result.success) {
this.logger.error('Failed to insert application logs into ClickHouse');
}
}
}
packages/twenty-server/src/engine/core-modules/application-logs/drivers/console.driver.ts
+31
View File
@@ -0,0 +1,31 @@
import { Logger } from '@nestjs/common';
import { type ApplicationLogDriverInterface } from 'src/engine/core-modules/application-logs/interfaces/application-log-driver.interface';
import { type ApplicationLogEntry } from 'src/engine/core-modules/application-logs/interfaces/application-log-entry.interface';
export class ConsoleApplicationLogDriver
implements ApplicationLogDriverInterface
{
private readonly logger = new Logger(ConsoleApplicationLogDriver.name);
async writeLogs(entries: ApplicationLogEntry[]): Promise<void> {
for (const entry of entries) {
const context = `${entry.logicFunctionName}:${entry.executionId}`;
switch (entry.level) {
case 'ERROR':
this.logger.error(entry.message, undefined, context);
break;
case 'WARN':
this.logger.warn(entry.message, context);
break;
case 'DEBUG':
this.logger.debug(entry.message, context);
break;
default:
this.logger.log(entry.message, context);
break;
}
}
}
}
packages/twenty-server/src/engine/core-modules/application-logs/drivers/disabled.driver.ts
+9
View File
@@ -0,0 +1,9 @@
import { type ApplicationLogDriverInterface } from 'src/engine/core-modules/application-logs/interfaces/application-log-driver.interface';
export class DisabledApplicationLogDriver
implements ApplicationLogDriverInterface
{
async writeLogs(): Promise<void> {
return;
}
}
packages/twenty-server/src/engine/core-modules/application-logs/interfaces/application-log-driver-type.enum.ts
+5
View File
@@ -0,0 +1,5 @@
export enum ApplicationLogDriverType {
DISABLED = 'DISABLED',
CONSOLE = 'CONSOLE',
CLICKHOUSE = 'CLICKHOUSE',
}
packages/twenty-server/src/engine/core-modules/application-logs/interfaces/application-log-driver.interface.ts
+5
View File
@@ -0,0 +1,5 @@
import { type ApplicationLogEntry } from 'src/engine/core-modules/application-logs/interfaces/application-log-entry.interface';
export interface ApplicationLogDriverInterface {
writeLogs(entries: ApplicationLogEntry[]): Promise<void>;
}
packages/twenty-server/src/engine/core-modules/application-logs/interfaces/application-log-entry.interface.ts
+10
View File
@@ -0,0 +1,10 @@
export type ApplicationLogEntry = {
timestamp: Date;
workspaceId: string;
applicationId: string;
logicFunctionId: string;
logicFunctionName: string;
executionId: string;
level: string;
message: string;
};
packages/twenty-server/src/engine/core-modules/application-logs/interfaces/application-logs-module-options.type.ts
+5
View File
@@ -0,0 +1,5 @@
import { type ApplicationLogDriverType } from 'src/engine/core-modules/application-logs/interfaces/application-log-driver-type.enum';
export type ApplicationLogsModuleOptions = {
type: ApplicationLogDriverType;
};
packages/twenty-server/src/engine/core-modules/application-logs/types/parsed-log-line.type.ts
+5
View File
@@ -0,0 +1,5 @@
export type ParsedLogLine = {
timestamp: Date;
level: string;
message: string;
};
packages/twenty-server/src/engine/core-modules/application-logs/utils/parse-application-log-lines.spec.ts
+125
View File
@@ -0,0 +1,125 @@
import { parseApplicationLogLines } from './parse-application-log-lines';
describe('parseApplicationLogLines', () => {
it('should return an empty array for empty string', () => {
expect(parseApplicationLogLines('')).toEqual([]);
});
it('should return an empty array for undefined-ish input', () => {
expect(parseApplicationLogLines(undefined as unknown as string)).toEqual(
[],
);
});
it('should parse a structured INFO log line', () => {
const raw = '2024-06-15T10:30:00.123Z INFO Hello world';
expect(parseApplicationLogLines(raw)).toEqual([
{
timestamp: new Date('2024-06-15T10:30:00.123Z'),
level: 'INFO',
message: 'Hello world',
},
]);
});
it('should parse all supported log levels', () => {
const raw = [
'2024-01-01T00:00:00.000Z INFO info message',
'2024-01-01T00:00:01.000Z ERROR error message',
'2024-01-01T00:00:02.000Z WARN warn message',
'2024-01-01T00:00:03.000Z DEBUG debug message',
].join('\n');
expect(parseApplicationLogLines(raw)).toEqual([
{
timestamp: new Date('2024-01-01T00:00:00.000Z'),
level: 'INFO',
message: 'info message',
},
{
timestamp: new Date('2024-01-01T00:00:01.000Z'),
level: 'ERROR',
message: 'error message',
},
{
timestamp: new Date('2024-01-01T00:00:02.000Z'),
level: 'WARN',
message: 'warn message',
},
{
timestamp: new Date('2024-01-01T00:00:03.000Z'),
level: 'DEBUG',
message: 'debug message',
},
]);
});
it('should default unstructured lines to INFO with current timestamp', () => {
const now = Date.now();
const raw = 'some plain text without timestamp or level';
const result = parseApplicationLogLines(raw);
expect(result).toHaveLength(1);
expect(result[0].level).toBe('INFO');
expect(result[0].message).toBe(
'some plain text without timestamp or level',
);
expect(result[0].timestamp.getTime()).toBeGreaterThanOrEqual(now);
expect(result[0].timestamp.getTime()).toBeLessThanOrEqual(now + 1000);
});
it('should skip empty lines', () => {
const raw =
'2024-01-01T00:00:00.000Z INFO first\n\n\n2024-01-01T00:00:01.000Z ERROR second\n';
expect(parseApplicationLogLines(raw)).toEqual([
{
timestamp: new Date('2024-01-01T00:00:00.000Z'),
level: 'INFO',
message: 'first',
},
{
timestamp: new Date('2024-01-01T00:00:01.000Z'),
level: 'ERROR',
message: 'second',
},
]);
});
it('should handle a mix of structured and unstructured lines', () => {
const raw = [
'2024-01-01T00:00:00.000Z INFO structured line',
'plain unstructured line',
'2024-01-01T00:00:01.000Z ERROR another structured',
].join('\n');
const result = parseApplicationLogLines(raw);
expect(result).toHaveLength(3);
expect(result[0]).toEqual({
timestamp: new Date('2024-01-01T00:00:00.000Z'),
level: 'INFO',
message: 'structured line',
});
expect(result[1].level).toBe('INFO');
expect(result[1].message).toBe('plain unstructured line');
expect(result[2]).toEqual({
timestamp: new Date('2024-01-01T00:00:01.000Z'),
level: 'ERROR',
message: 'another structured',
});
});
it('should preserve message content including special characters', () => {
const raw = '2024-01-01T00:00:00.000Z INFO {"key": "value", "count": 42}';
expect(parseApplicationLogLines(raw)).toEqual([
{
timestamp: new Date('2024-01-01T00:00:00.000Z'),
level: 'INFO',
message: '{"key": "value", "count": 42}',
},
]);
});
});
packages/twenty-server/src/engine/core-modules/application-logs/utils/parse-application-log-lines.ts
+31
View File
@@ -0,0 +1,31 @@
import { type ParsedLogLine } from 'src/engine/core-modules/application-logs/types/parsed-log-line.type';
// Matches: 2024-01-01T00:00:00.000Z INFO some message
const LOG_LINE_REGEX =
/^(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z)\s+(INFO|ERROR|WARN|DEBUG)\s+(.*)$/;
export const parseApplicationLogLines = (rawLogs: string): ParsedLogLine[] => {
if (!rawLogs) {
return [];
}
const lines = rawLogs.split('\n').filter(Boolean);
return lines.map((line) => {
const match = line.match(LOG_LINE_REGEX);
if (match) {
return {
timestamp: new Date(match[1]),
level: match[2],
message: match[3],
};
}
return {
timestamp: new Date(),
level: 'INFO',
message: line,
};
});
};
packages/twenty-server/src/engine/core-modules/core-engine.module.ts
+6
View File
@@ -4,6 +4,8 @@ import { EventEmitterModule } from '@nestjs/event-emitter';
import { WorkspaceQueryRunnerModule } from 'src/engine/api/graphql/workspace-query-runner/workspace-query-runner.module';
import { ActorModule } from 'src/engine/core-modules/actor/actor.module';
import { ApplicationLogsModule } from 'src/engine/core-modules/application-logs/application-logs.module';
import { applicationLogsModuleFactory } from 'src/engine/core-modules/application-logs/application-logs.module-factory';
import { AdminPanelModule } from 'src/engine/core-modules/admin-panel/admin-panel.module';
import { ApiKeyModule } from 'src/engine/core-modules/api-key/api-key.module';
import { AppTokenModule } from 'src/engine/core-modules/app-token/app-token.module';
@@ -139,6 +141,10 @@ import { FileModule } from './file/file.module';
useFactory: exceptionHandlerModuleFactory,
inject: [TwentyConfigService, HttpAdapterHost],
}),
ApplicationLogsModule.forRootAsync({
useFactory: applicationLogsModuleFactory,
inject: [TwentyConfigService],
}),
EmailModule.forRoot(),
CaptchaModule.forRoot(),
EventEmitterModule.forRoot({
packages/twenty-server/src/engine/core-modules/event-logs/cleanup/services/event-log-cleanup.service.ts
+1
View File
@@ -12,6 +12,7 @@ const CLICKHOUSE_TABLE_NAMES: Record<EventLogTable, string> = {
[EventLogTable.PAGEVIEW]: 'pageview',
[EventLogTable.OBJECT_EVENT]: 'objectEvent',
[EventLogTable.USAGE_EVENT]: 'usageEvent',
[EventLogTable.APPLICATION_LOG]: 'applicationLog',
};
export type EventLogCleanupParams = {
packages/twenty-server/src/engine/core-modules/event-logs/event-logs.service.ts
+37 -3
View File
@@ -49,6 +49,17 @@ type ClickHouseUsageEventRecord = {
metadata?: Record<string, unknown>;
};
type ClickHouseApplicationLogRecord = {
timestamp: string;
applicationId?: string;
logicFunctionId?: string;
logicFunctionName?: string;
executionId?: string;
level?: string;
message?: string;
properties?: Record<string, unknown>;
};
const ALLOWED_TABLES = Object.values(EventLogTable);
const MAX_LIMIT = 10000;
@@ -57,6 +68,7 @@ const CLICKHOUSE_TABLE_NAMES: Record<EventLogTable, string> = {
[EventLogTable.PAGEVIEW]: 'pageview',
[EventLogTable.OBJECT_EVENT]: 'objectEvent',
[EventLogTable.USAGE_EVENT]: 'usageEvent',
[EventLogTable.APPLICATION_LOG]: 'applicationLog',
};
@Injectable()
@@ -85,7 +97,9 @@ export class EventLogsService {
? 'resourceType'
: input.table === EventLogTable.PAGEVIEW
? 'name'
: 'event';
: input.table === EventLogTable.APPLICATION_LOG
? 'logicFunctionName'
: 'event';
const whereClauses: string[] = ['"workspaceId" = {workspaceId:String}'];
const params: Record<string, unknown> = { workspaceId };
@@ -201,7 +215,9 @@ export class EventLogsService {
// userWorkspaceId directly which is more relevant in a workspace context.
// Consider migrating all event tables to userWorkspaceId for consistency.
if (isDefined(filters.userWorkspaceId)) {
if (table === EventLogTable.USAGE_EVENT) {
if (table === EventLogTable.APPLICATION_LOG) {
// Application logs don't have a user column
} else if (table === EventLogTable.USAGE_EVENT) {
whereClauses.push('"userWorkspaceId" = {userWorkspaceId:String}');
params.userWorkspaceId = filters.userWorkspaceId;
} else {
@@ -249,7 +265,10 @@ export class EventLogsService {
}
private normalizeRecords(
records: ClickHouseEventRecord[] | ClickHouseUsageEventRecord[],
records:
| ClickHouseEventRecord[]
| ClickHouseUsageEventRecord[]
| ClickHouseApplicationLogRecord[],
table: EventLogTable,
): EventLogRecord[] {
if (table === EventLogTable.USAGE_EVENT) {
@@ -269,6 +288,21 @@ export class EventLogsService {
}));
}
if (table === EventLogTable.APPLICATION_LOG) {
return (records as ClickHouseApplicationLogRecord[]).map((record) => ({
event: record.logicFunctionName ?? '',
timestamp: new Date(record.timestamp),
properties: {
level: record.level,
message: record.message,
executionId: record.executionId,
logicFunctionId: record.logicFunctionId,
applicationId: record.applicationId,
...(record.properties ?? {}),
},
}));
}
return (records as ClickHouseEventRecord[]).map((record) => {
const eventName =
table === EventLogTable.PAGEVIEW
packages/twenty-server/src/engine/core-modules/logic-function/logic-function-executor/logic-function-executor.service.ts
+17 -4
View File
@@ -6,6 +6,7 @@ import {
DEFAULT_APP_ACCESS_TOKEN_NAME,
} from 'twenty-shared/application';
import { isDefined } from 'twenty-shared/utils';
import { v4 } from 'uuid';
import {
type LogicFunctionExecuteResult,
@@ -15,6 +16,8 @@ import {
import { FlatApplication } from 'src/engine/core-modules/application/types/flat-application.type';
import type { FlatApplicationVariable } from 'src/engine/core-modules/application/application-variable/types/flat-application-variable.type';
import { ApplicationLogsService } from 'src/engine/core-modules/application-logs/application-logs.service';
import { parseApplicationLogLines } from 'src/engine/core-modules/application-logs/utils/parse-application-log-lines';
import { AuditService } from 'src/engine/core-modules/audit/services/audit.service';
import { LOGIC_FUNCTION_EXECUTED_EVENT } from 'src/engine/core-modules/audit/utils/events/workspace-event/logic-function/logic-function-executed';
import { ApplicationTokenService } from 'src/engine/core-modules/auth/token/services/application-token.service';
@@ -56,6 +59,7 @@ export class LogicFunctionExecutorService {
private readonly secretEncryptionService: SecretEncryptionService,
private readonly subscriptionService: SubscriptionService,
private readonly auditService: AuditService,
private readonly applicationLogsService: ApplicationLogsService,
) {}
async execute({
@@ -212,10 +216,19 @@ export class LogicFunctionExecutorService {
flatLogicFunction: FlatLogicFunction;
flatApplication: FlatApplication;
}) {
if (this.twentyConfigService.get('LOGIC_FUNCTION_LOGS_ENABLED')) {
/* oxlint-disable no-console */
console.log(result.logs);
}
const executionId = v4();
const parsedLines = parseApplicationLogLines(result.logs);
const logEntries = parsedLines.map((line) => ({
...line,
workspaceId,
applicationId: flatApplication.id,
logicFunctionId: flatLogicFunction.id,
logicFunctionName: flatLogicFunction.name,
executionId,
}));
this.applicationLogsService.writeLogs(logEntries);
await this.subscriptionService.publish({
channel: SubscriptionChannel.LOGIC_FUNCTION_LOGS_CHANNEL,
packages/twenty-server/src/engine/core-modules/twenty-config/config-variables.ts
+14
View File
@@ -17,6 +17,7 @@ import { type AwsRegion } from 'src/engine/core-modules/twenty-config/interfaces
import { NodeEnvironment } from 'src/engine/core-modules/twenty-config/interfaces/node-environment.interface';
import { SupportDriver } from 'src/engine/core-modules/twenty-config/interfaces/support.interface';
import { ApplicationLogDriverType } from 'src/engine/core-modules/application-logs/interfaces/application-log-driver-type.enum';
import { CaptchaDriverType } from 'src/engine/core-modules/captcha/interfaces';
import { CodeInterpreterDriverType } from 'src/engine/core-modules/code-interpreter/code-interpreter.interface';
import { WebSearchDriverType } from 'src/engine/core-modules/web-search/web-search.interface';
@@ -926,6 +927,19 @@ export class ConfigVariables {
@IsOptional()
SENTRY_ENVIRONMENT: string;
@ConfigVariablesMetadata({
group: ConfigVariablesGroup.LOGGING,
description:
'Driver used for application logs (Disabled, Console, or ClickHouse)',
type: ConfigVariableType.ENUM,
options: Object.values(ApplicationLogDriverType),
isEnvOnly: true,
})
@IsOptional()
@CastToUpperSnakeCase()
APPLICATION_LOG_DRIVER_TYPE: ApplicationLogDriverType =
ApplicationLogDriverType.DISABLED;
@ConfigVariablesMetadata({
group: ConfigVariablesGroup.SUPPORT_CHAT_CONFIG,
description: 'Driver used for support chat integration',
packages/twenty-shared/src/types/EventLogTable.ts
+1
View File
@@ -3,4 +3,5 @@ export enum EventLogTable {
PAGEVIEW = 'PAGEVIEW',
OBJECT_EVENT = 'OBJECT_EVENT',
USAGE_EVENT = 'USAGE_EVENT',
APPLICATION_LOG = 'APPLICATION_LOG',
}