Function trigger updates 2 (#16608)

- Improves route trigger job performances
- expose function params types

## Before

<img width="938" height="271" alt="image"
src="https://github.com/user-attachments/assets/5752ba64-f31d-44ed-974d-536e63458f2c"
/>


## After

<img width="1000" height="559" alt="image"
src="https://github.com/user-attachments/assets/b1f4927a-5f43-49f0-a606-244c72356772"
/>
This commit is contained in:
martmull
2025-12-16 18:20:26 +01:00
committed by GitHub
parent 58c85e7cca
commit 1cbbd04761
75 changed files with 391 additions and 202 deletions
@@ -1,15 +1,18 @@
import { Injectable } from '@nestjs/common';
import {
type ObjectRecordCreateEvent,
type ObjectRecordDeleteEvent,
type ObjectRecordDestroyEvent,
type ObjectRecordRestoreEvent,
type ObjectRecordUpdateEvent,
type ObjectRecordEvent,
type ObjectRecordNonDestructiveEvent,
} from 'twenty-shared/database-events';
import { OnDatabaseBatchEvent } from 'src/engine/api/graphql/graphql-query-runner/decorators/on-database-batch-event.decorator';
import { DatabaseEventAction } from 'src/engine/api/graphql/graphql-query-runner/enums/database-event-action';
import { CreateAuditLogFromInternalEvent } from 'src/engine/core-modules/audit/jobs/create-audit-log-from-internal-event';
import { type ObjectRecordCreateEvent } from 'src/engine/core-modules/event-emitter/types/object-record-create.event';
import { type ObjectRecordDeleteEvent } from 'src/engine/core-modules/event-emitter/types/object-record-delete.event';
import { type ObjectRecordDestroyEvent } from 'src/engine/core-modules/event-emitter/types/object-record-destroy.event';
import { type ObjectRecordEvent } from 'src/engine/core-modules/event-emitter/types/object-record-event.event';
import { type ObjectRecordNonDestructiveEvent } from 'src/engine/core-modules/event-emitter/types/object-record-non-destructive-event';
import { type ObjectRecordRestoreEvent } from 'src/engine/core-modules/event-emitter/types/object-record-restore.event';
import { type ObjectRecordUpdateEvent } from 'src/engine/core-modules/event-emitter/types/object-record-update.event';
import { InjectMessageQueue } from 'src/engine/core-modules/message-queue/decorators/message-queue.decorator';
import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';
import { MessageQueueService } from 'src/engine/core-modules/message-queue/services/message-queue.service';
@@ -1,10 +1,11 @@
import { Injectable } from '@nestjs/common';
import { type ObjectRecordCreateEvent } from 'twenty-shared/database-events';
import { OnCustomBatchEvent } from 'src/engine/api/graphql/graphql-query-runner/decorators/on-custom-batch-event.decorator';
import { USER_SIGNUP_EVENT_NAME } from 'src/engine/api/graphql/workspace-query-runner/constants/user-signup-event-name.constants';
import { AuditService } from 'src/engine/core-modules/audit/services/audit.service';
import { USER_SIGNUP_EVENT } from 'src/engine/core-modules/audit/utils/events/workspace-event/user/user-signup';
import { type ObjectRecordCreateEvent } from 'src/engine/core-modules/event-emitter/types/object-record-create.event';
import { TelemetryService } from 'src/engine/core-modules/telemetry/telemetry.service';
import { CustomWorkspaceEventBatch } from 'src/engine/workspace-event-emitter/types/custom-workspace-batch-event.type';
@@ -1,9 +1,10 @@
import { type ObjectRecordEvent } from 'twenty-shared/database-events';
import { AuditService } from 'src/engine/core-modules/audit/services/audit.service';
import { OBJECT_RECORD_CREATED_EVENT } from 'src/engine/core-modules/audit/utils/events/object-event/object-record-created';
import { OBJECT_RECORD_DELETED_EVENT } from 'src/engine/core-modules/audit/utils/events/object-event/object-record-delete';
import { OBJECT_RECORD_UPDATED_EVENT } from 'src/engine/core-modules/audit/utils/events/object-event/object-record-updated';
import { OBJECT_RECORD_UPSERTED_EVENT } from 'src/engine/core-modules/audit/utils/events/object-event/object-record-upserted';
import { type ObjectRecordEvent } from 'src/engine/core-modules/event-emitter/types/object-record-event.event';
import { Process } from 'src/engine/core-modules/message-queue/decorators/process.decorator';
import { Processor } from 'src/engine/core-modules/message-queue/decorators/processor.decorator';
import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';
@@ -2,13 +2,14 @@
import { Injectable } from '@nestjs/common';
import { type ObjectRecordCreateEvent } from 'twenty-shared/database-events';
import { OnDatabaseBatchEvent } from 'src/engine/api/graphql/graphql-query-runner/decorators/on-database-batch-event.decorator';
import { DatabaseEventAction } from 'src/engine/api/graphql/graphql-query-runner/enums/database-event-action';
import {
UpdateSubscriptionQuantityJob,
type UpdateSubscriptionQuantityJobData,
} from 'src/engine/core-modules/billing/jobs/update-subscription-quantity.job';
import { type ObjectRecordCreateEvent } from 'src/engine/core-modules/event-emitter/types/object-record-create.event';
import { InjectMessageQueue } from 'src/engine/core-modules/message-queue/decorators/message-queue.decorator';
import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';
import { MessageQueueService } from 'src/engine/core-modules/message-queue/services/message-queue.service';
@@ -1,9 +0,0 @@
import { ObjectRecordBaseEvent } from 'src/engine/core-modules/event-emitter/types/object-record.base.event';
export class ObjectRecordCreateEvent<
T = object,
> extends ObjectRecordBaseEvent<T> {
properties: {
after: T;
};
}
@@ -1,9 +0,0 @@
import { ObjectRecordBaseEvent } from 'src/engine/core-modules/event-emitter/types/object-record.base.event';
export class ObjectRecordDeleteEvent<
T = object,
> extends ObjectRecordBaseEvent<T> {
properties: {
before: T;
};
}
@@ -1,9 +0,0 @@
import { ObjectRecordBaseEvent } from 'src/engine/core-modules/event-emitter/types/object-record.base.event';
export class ObjectRecordDestroyEvent<
T = object,
> extends ObjectRecordBaseEvent<T> {
properties: {
before: T;
};
}
@@ -1,3 +0,0 @@
export type ObjectRecordDiff<T> = {
[K in keyof T]: { before: T[K]; after: T[K] };
};
@@ -1,14 +0,0 @@
import { type ObjectRecordCreateEvent } from 'src/engine/core-modules/event-emitter/types/object-record-create.event';
import { type ObjectRecordDeleteEvent } from 'src/engine/core-modules/event-emitter/types/object-record-delete.event';
import { type ObjectRecordDestroyEvent } from 'src/engine/core-modules/event-emitter/types/object-record-destroy.event';
import { type ObjectRecordRestoreEvent } from 'src/engine/core-modules/event-emitter/types/object-record-restore.event';
import { type ObjectRecordUpdateEvent } from 'src/engine/core-modules/event-emitter/types/object-record-update.event';
import { type ObjectRecordUpsertEvent } from 'src/engine/core-modules/event-emitter/types/object-record-upsert.event';
export type ObjectRecordEvent<T = object> =
| ObjectRecordUpdateEvent<T>
| ObjectRecordDeleteEvent<T>
| ObjectRecordCreateEvent<T>
| ObjectRecordDestroyEvent<T>
| ObjectRecordRestoreEvent<T>
| ObjectRecordUpsertEvent<T>;
@@ -1,12 +0,0 @@
import { type ObjectRecordCreateEvent } from 'src/engine/core-modules/event-emitter/types/object-record-create.event';
import { type ObjectRecordDeleteEvent } from 'src/engine/core-modules/event-emitter/types/object-record-delete.event';
import { type ObjectRecordRestoreEvent } from 'src/engine/core-modules/event-emitter/types/object-record-restore.event';
import { type ObjectRecordUpdateEvent } from 'src/engine/core-modules/event-emitter/types/object-record-update.event';
import { type ObjectRecordUpsertEvent } from 'src/engine/core-modules/event-emitter/types/object-record-upsert.event';
export type ObjectRecordNonDestructiveEvent =
| ObjectRecordCreateEvent
| ObjectRecordUpdateEvent
| ObjectRecordDeleteEvent
| ObjectRecordRestoreEvent
| ObjectRecordUpsertEvent;
@@ -1,9 +0,0 @@
import { ObjectRecordCreateEvent } from 'src/engine/core-modules/event-emitter/types/object-record-create.event';
export class ObjectRecordRestoreEvent<
T = object,
> extends ObjectRecordCreateEvent<T> {
properties: {
after: T;
};
}
@@ -1,13 +0,0 @@
import { type ObjectRecordDiff } from 'src/engine/core-modules/event-emitter/types/object-record-diff';
import { ObjectRecordBaseEvent } from 'src/engine/core-modules/event-emitter/types/object-record.base.event';
export class ObjectRecordUpdateEvent<
T = object,
> extends ObjectRecordBaseEvent<T> {
properties: {
updatedFields?: string[];
diff?: Partial<ObjectRecordDiff<T>>;
before: T;
after: T;
};
}
@@ -1,13 +0,0 @@
import { type ObjectRecordDiff } from 'src/engine/core-modules/event-emitter/types/object-record-diff';
import { ObjectRecordBaseEvent } from 'src/engine/core-modules/event-emitter/types/object-record.base.event';
export class ObjectRecordUpsertEvent<
T = object,
> extends ObjectRecordBaseEvent<T> {
properties: {
before?: T;
after: T;
diff?: Partial<ObjectRecordDiff<T>>;
updatedFields?: string[];
};
}
@@ -1,15 +0,0 @@
import { type ObjectRecordDiff } from 'src/engine/core-modules/event-emitter/types/object-record-diff';
type Properties<T> = {
updatedFields?: string[];
before?: T;
after?: T;
diff?: Partial<ObjectRecordDiff<T>>;
};
export class ObjectRecordBaseEvent<T = object> {
recordId: string;
userId?: string;
workspaceMemberId?: string;
properties: Properties<T>;
}
@@ -1,8 +1,9 @@
import { Injectable } from '@nestjs/common';
import { type ObjectRecordDestroyEvent } from 'twenty-shared/database-events';
import { OnDatabaseBatchEvent } from 'src/engine/api/graphql/graphql-query-runner/decorators/on-database-batch-event.decorator';
import { DatabaseEventAction } from 'src/engine/api/graphql/graphql-query-runner/enums/database-event-action';
import { type ObjectRecordDestroyEvent } from 'src/engine/core-modules/event-emitter/types/object-record-destroy.event';
import {
FileDeletionJob,
type FileDeletionJobData,
@@ -1,8 +1,9 @@
import { Injectable } from '@nestjs/common';
import { type ObjectRecordDestroyEvent } from 'twenty-shared/database-events';
import { OnDatabaseBatchEvent } from 'src/engine/api/graphql/graphql-query-runner/decorators/on-database-batch-event.decorator';
import { DatabaseEventAction } from 'src/engine/api/graphql/graphql-query-runner/enums/database-event-action';
import { type ObjectRecordDestroyEvent } from 'src/engine/core-modules/event-emitter/types/object-record-destroy.event';
import {
FileDeletionJob,
type FileDeletionJobData,
@@ -2,6 +2,8 @@ import { Logger } from '@nestjs/common';
import chunk from 'lodash.chunk';
import type { ObjectRecordEvent } from 'twenty-shared/database-events';
import { InjectMessageQueue } from 'src/engine/core-modules/message-queue/decorators/message-queue.decorator';
import { Process } from 'src/engine/core-modules/message-queue/decorators/process.decorator';
import { Processor } from 'src/engine/core-modules/message-queue/decorators/processor.decorator';
@@ -13,7 +15,6 @@ import {
} from 'src/engine/core-modules/webhook/jobs/call-webhook.job';
import { WebhookService } from 'src/engine/core-modules/webhook/webhook.service';
import { WorkspaceEventBatch } from 'src/engine/workspace-event-emitter/types/workspace-event-batch.type';
import type { ObjectRecordEvent } from 'src/engine/core-modules/event-emitter/types/object-record-event.event';
import { transformEventBatchToWebhookEvents } from 'src/engine/core-modules/webhook/utils/transform-event-batch-to-webhook-events';
const WEBHOOK_JOBS_CHUNK_SIZE = 20;
@@ -1,5 +1,6 @@
import type { ObjectRecordEvent } from 'twenty-shared/database-events';
import { type WorkspaceEventBatch } from 'src/engine/workspace-event-emitter/types/workspace-event-batch.type';
import type { ObjectRecordEvent } from 'src/engine/core-modules/event-emitter/types/object-record-event.event';
import type { WebhookEntity } from 'src/engine/core-modules/webhook/webhook.entity';
import { transformEventBatchToWebhookEvents } from 'src/engine/core-modules/webhook/utils/transform-event-batch-to-webhook-events';
import { type FlatObjectMetadata } from 'src/engine/metadata-modules/flat-object-metadata/types/flat-object-metadata.type';
@@ -1,5 +1,6 @@
import type { ObjectRecordEvent } from 'twenty-shared/database-events';
import { transformEventToWebhookEvent } from 'src/engine/core-modules/webhook/utils/transform-event-to-webhook-event';
import type { ObjectRecordEvent } from 'src/engine/core-modules/event-emitter/types/object-record-event.event';
describe('transformEventToWebhookEvent', () => {
it('should transform event to webhook event', () => {
@@ -1,7 +1,8 @@
import type { ObjectRecordEvent } from 'twenty-shared/database-events';
import { type WorkspaceEventBatch } from 'src/engine/workspace-event-emitter/types/workspace-event-batch.type';
import { type CallWebhookJobData } from 'src/engine/core-modules/webhook/jobs/call-webhook.job';
import { type WebhookEntity } from 'src/engine/core-modules/webhook/webhook.entity';
import type { ObjectRecordEvent } from 'src/engine/core-modules/event-emitter/types/object-record-event.event';
import { transformEventToWebhookEvent } from 'src/engine/core-modules/webhook/utils/transform-event-to-webhook-event';
export const transformEventBatchToWebhookEvents = ({
@@ -1,6 +1,7 @@
import { isDefined } from 'twenty-shared/utils';
import type { ObjectRecordEvent } from 'src/engine/core-modules/event-emitter/types/object-record-event.event';
import type { ObjectRecordEvent } from 'twenty-shared/database-events';
import { removeSecretFromWebhookRecord } from 'src/utils/remove-secret-from-webhook-record';
export const transformEventToWebhookEvent = ({
@@ -1,8 +1,10 @@
import { InjectRepository } from '@nestjs/typeorm';
import { Repository } from 'typeorm';
import chunk from 'lodash.chunk';
import type { ObjectRecordEvent } from 'twenty-shared/database-events';
import type { ObjectRecordEvent } from 'src/engine/core-modules/event-emitter/types/object-record-event.event';
import { InjectMessageQueue } from 'src/engine/core-modules/message-queue/decorators/message-queue.decorator';
import { Process } from 'src/engine/core-modules/message-queue/decorators/process.decorator';
import { Processor } from 'src/engine/core-modules/message-queue/decorators/processor.decorator';
@@ -14,6 +16,9 @@ import {
ServerlessFunctionTriggerJobData,
} from 'src/engine/metadata-modules/serverless-function/jobs/serverless-function-trigger.job';
import { WorkspaceEventBatch } from 'src/engine/workspace-event-emitter/types/workspace-event-batch.type';
import { transformEventBatchToEventPayloads } from 'src/engine/metadata-modules/database-event-trigger/utils/transform-event-batch-to-event-payloads';
const DATABASE_EVENT_JOBS_CHUNK_SIZE = 20;
@Processor(MessageQueue.triggerQueue)
export class CallDatabaseEventTriggerJobsJob {
@@ -35,29 +40,30 @@ export class CallDatabaseEventTriggerJobsJob {
relations: ['serverlessFunction'],
});
for (const databaseEventListener of databaseEventListeners) {
if (
!this.shouldTriggerJob({
const databaseEventListenersToTrigger = databaseEventListeners.filter(
(databaseEventListener) =>
this.shouldTriggerJob({
workspaceEventBatch,
eventName: databaseEventListener.settings.eventName,
})
) {
continue;
}
}),
);
const { events, ...batchEventInfo } = workspaceEventBatch;
const serverlessFunctionPayloads = transformEventBatchToEventPayloads({
databaseEventListeners: databaseEventListenersToTrigger,
workspaceEventBatch,
});
for (const event of events) {
await this.messageQueueService.add<ServerlessFunctionTriggerJobData>(
ServerlessFunctionTriggerJob.name,
{
serverlessFunctionId: databaseEventListener.serverlessFunction.id,
workspaceId: databaseEventListener.workspaceId,
payload: { ...batchEventInfo, ...event },
},
{ retryLimit: 3 },
);
}
const serverlessFunctionPayloadsChunks = chunk(
serverlessFunctionPayloads,
DATABASE_EVENT_JOBS_CHUNK_SIZE,
);
for (const serverlessFunctionPayloadsChunk of serverlessFunctionPayloadsChunks) {
await this.messageQueueService.add<ServerlessFunctionTriggerJobData[]>(
ServerlessFunctionTriggerJob.name,
serverlessFunctionPayloadsChunk,
{ retryLimit: 3 },
);
}
}
@@ -0,0 +1,34 @@
import type {
DatabaseEventPayload,
ObjectRecordEvent,
} from 'twenty-shared/database-events';
import type { WorkspaceEventBatch } from 'src/engine/workspace-event-emitter/types/workspace-event-batch.type';
import { type DatabaseEventTriggerEntity } from 'src/engine/metadata-modules/database-event-trigger/entities/database-event-trigger.entity';
import { type ServerlessFunctionTriggerJobData } from 'src/engine/metadata-modules/serverless-function/jobs/serverless-function-trigger.job';
export const transformEventBatchToEventPayloads = ({
workspaceEventBatch,
databaseEventListeners,
}: {
workspaceEventBatch: WorkspaceEventBatch<ObjectRecordEvent>;
databaseEventListeners: DatabaseEventTriggerEntity[];
}): ServerlessFunctionTriggerJobData[] => {
const result: ServerlessFunctionTriggerJobData[] = [];
for (const databaseEventListener of databaseEventListeners) {
const { events, ...batchEventInfo } = workspaceEventBatch;
for (const event of events) {
const payload: DatabaseEventPayload = { ...batchEventInfo, ...event };
result.push({
serverlessFunctionId: databaseEventListener.serverlessFunction.id,
workspaceId: databaseEventListener.workspaceId,
payload,
});
}
}
return result;
};
@@ -21,12 +21,17 @@ export class ServerlessFunctionTriggerJob {
) {}
@Process(ServerlessFunctionTriggerJob.name)
async handle(data: ServerlessFunctionTriggerJobData) {
await this.serverlessFunctionService.executeOneServerlessFunction({
id: data.serverlessFunctionId,
workspaceId: data.workspaceId,
payload: data.payload || {},
version: 'draft',
});
async handle(serverlessFunctionPayloads: ServerlessFunctionTriggerJobData[]) {
await Promise.all(
serverlessFunctionPayloads.map(
async (serverlessFunctionPayload) =>
await this.serverlessFunctionService.executeOneServerlessFunction({
id: serverlessFunctionPayload.serverlessFunctionId,
workspaceId: serverlessFunctionPayload.workspaceId,
payload: serverlessFunctionPayload.payload || {},
version: 'draft',
}),
),
);
}
}
@@ -1,8 +1,8 @@
import { Inject, Injectable } from '@nestjs/common';
import { RedisPubSub } from 'graphql-redis-subscriptions';
import { type ObjectRecordEvent } from 'twenty-shared/database-events';
import { type ObjectRecordEvent } from 'src/engine/core-modules/event-emitter/types/object-record-event.event';
import { WorkspaceEventBatch } from 'src/engine/workspace-event-emitter/types/workspace-event-batch.type';
import { transformEventToWebhookEvent } from 'src/engine/core-modules/webhook/utils/transform-event-to-webhook-event';
import { ON_DB_EVENT_TRIGGER } from 'src/engine/subscriptions/constants/on-db-event-trigger';
@@ -1,7 +1,7 @@
import { FieldMetadataType } from 'twenty-shared/types';
import { type ObjectRecordUpdateEvent } from 'twenty-shared/database-events';
import { DatabaseEventAction } from 'src/engine/api/graphql/graphql-query-runner/enums/database-event-action';
import { type ObjectRecordUpdateEvent } from 'src/engine/core-modules/event-emitter/types/object-record-update.event';
import { type FlatEntityMaps } from 'src/engine/metadata-modules/flat-entity/types/flat-entity-maps.type';
import { type FlatFieldMetadata } from 'src/engine/metadata-modules/flat-field-metadata/types/flat-field-metadata.type';
import { type FlatObjectMetadata } from 'src/engine/metadata-modules/flat-object-metadata/types/flat-object-metadata.type';
@@ -1,16 +1,18 @@
import { STANDARD_OBJECT_IDS } from 'twenty-shared/metadata';
import { isDefined } from 'twenty-shared/utils';
import {
ObjectRecordCreateEvent,
ObjectRecordDeleteEvent,
ObjectRecordDestroyEvent,
ObjectRecordUpdateEvent,
ObjectRecordUpsertEvent,
type ObjectRecordDiff,
} from 'twenty-shared/database-events';
import type { ObjectLiteral } from 'typeorm';
import { DatabaseEventAction } from 'src/engine/api/graphql/graphql-query-runner/enums/database-event-action';
import type { AuthContext } from 'src/engine/core-modules/auth/types/auth-context.type';
import { ObjectRecordCreateEvent } from 'src/engine/core-modules/event-emitter/types/object-record-create.event';
import { ObjectRecordDeleteEvent } from 'src/engine/core-modules/event-emitter/types/object-record-delete.event';
import { ObjectRecordDestroyEvent } from 'src/engine/core-modules/event-emitter/types/object-record-destroy.event';
import type { ObjectRecordDiff } from 'src/engine/core-modules/event-emitter/types/object-record-diff';
import { ObjectRecordUpdateEvent } from 'src/engine/core-modules/event-emitter/types/object-record-update.event';
import { ObjectRecordUpsertEvent } from 'src/engine/core-modules/event-emitter/types/object-record-upsert.event';
import { objectRecordChangedValues } from 'src/engine/core-modules/event-emitter/utils/object-record-changed-values';
import { type FlatEntityMaps } from 'src/engine/metadata-modules/flat-entity/types/flat-entity-maps.type';
import { type FlatFieldMetadata } from 'src/engine/metadata-modules/flat-field-metadata/types/flat-field-metadata.type';
@@ -2,14 +2,16 @@ import { Injectable } from '@nestjs/common';
import { EventEmitter2 } from '@nestjs/event-emitter';
import { isDefined } from 'twenty-shared/utils';
import {
ObjectRecordCreateEvent,
ObjectRecordDeleteEvent,
ObjectRecordDestroyEvent,
ObjectRecordUpdateEvent,
ObjectRecordUpsertEvent,
ObjectRecordRestoreEvent,
} from 'twenty-shared/database-events';
import { DatabaseEventAction } from 'src/engine/api/graphql/graphql-query-runner/enums/database-event-action';
import { ObjectRecordCreateEvent } from 'src/engine/core-modules/event-emitter/types/object-record-create.event';
import { ObjectRecordDeleteEvent } from 'src/engine/core-modules/event-emitter/types/object-record-delete.event';
import { ObjectRecordDestroyEvent } from 'src/engine/core-modules/event-emitter/types/object-record-destroy.event';
import { type ObjectRecordRestoreEvent } from 'src/engine/core-modules/event-emitter/types/object-record-restore.event';
import { ObjectRecordUpdateEvent } from 'src/engine/core-modules/event-emitter/types/object-record-update.event';
import { ObjectRecordUpsertEvent } from 'src/engine/core-modules/event-emitter/types/object-record-upsert.event';
import type { FlatObjectMetadata } from 'src/engine/metadata-modules/flat-object-metadata/types/flat-object-metadata.type';
import { type CustomEventName } from 'src/engine/workspace-event-emitter/types/custom-event-name.type';
import { CustomWorkspaceEventBatch } from 'src/engine/workspace-event-emitter/types/custom-workspace-batch-event.type';