Query cache instead of database for event listener webhook, logicFunction, triggers (#17824)
## Fix Replaced direct database queries with existing flat entity map caches for the two that already have cache infrastructure: - **CallDatabaseEventTriggerJobsJob** - now uses flatLogicFunctionMaps cache via WorkspaceCacheService.getOrRecompute(), filtering in memory for non-deleted logic functions with databaseEventTriggerSettings. - **CallWebhookJobsJob** - now uses flatWebhookMaps cache via WorkspaceCacheService.getOrRecompute(), filtering in memory for webhooks matching the event's operations. - Note: **WorkflowDatabaseEventTriggerListener** - left as-is since WorkflowAutomatedTriggerWorkspaceEntity extends BaseWorkspaceEntity (not SyncableEntity) and has no flat entity map cache infrastructure yet.
This commit is contained in:
+2
@@ -10,12 +10,14 @@ import { CallDatabaseEventTriggerJobsJob } from 'src/engine/core-modules/logic-f
|
||||
import { RouteTriggerService } from 'src/engine/core-modules/logic-function/logic-function-trigger/triggers/route/route-trigger.service';
|
||||
import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity';
|
||||
import { LogicFunctionEntity } from 'src/engine/metadata-modules/logic-function/logic-function.entity';
|
||||
import { WorkspaceCacheModule } from 'src/engine/workspace-cache/workspace-cache.module';
|
||||
|
||||
@Module({
|
||||
imports: [
|
||||
TypeOrmModule.forFeature([LogicFunctionEntity, WorkspaceEntity]),
|
||||
TokenModule,
|
||||
WorkspaceDomainsModule,
|
||||
WorkspaceCacheModule,
|
||||
],
|
||||
providers: [
|
||||
LogicFunctionTriggerJob,
|
||||
|
||||
+17
-14
@@ -1,8 +1,5 @@
|
||||
import { InjectRepository } from '@nestjs/typeorm';
|
||||
|
||||
import chunk from 'lodash.chunk';
|
||||
import { isDefined } from 'twenty-shared/utils';
|
||||
import { IsNull, Not, Repository } from 'typeorm';
|
||||
|
||||
import type { ObjectRecordEvent } from 'twenty-shared/database-events';
|
||||
|
||||
@@ -16,7 +13,7 @@ import {
|
||||
LogicFunctionTriggerJob,
|
||||
LogicFunctionTriggerJobData,
|
||||
} from 'src/engine/core-modules/logic-function/logic-function-trigger/jobs/logic-function-trigger.job';
|
||||
import { LogicFunctionEntity } from 'src/engine/metadata-modules/logic-function/logic-function.entity';
|
||||
import { WorkspaceCacheService } from 'src/engine/workspace-cache/services/workspace-cache.service';
|
||||
import { WorkspaceEventBatch } from 'src/engine/workspace-event-emitter/types/workspace-event-batch.type';
|
||||
|
||||
const DATABASE_EVENT_JOBS_CHUNK_SIZE = 20;
|
||||
@@ -26,20 +23,26 @@ export class CallDatabaseEventTriggerJobsJob {
|
||||
constructor(
|
||||
@InjectMessageQueue(MessageQueue.logicFunctionQueue)
|
||||
private readonly messageQueueService: MessageQueueService,
|
||||
@InjectRepository(LogicFunctionEntity)
|
||||
private readonly logicFunctionRepository: Repository<LogicFunctionEntity>,
|
||||
private readonly workspaceCacheService: WorkspaceCacheService,
|
||||
) {}
|
||||
|
||||
@Process(CallDatabaseEventTriggerJobsJob.name)
|
||||
async handle(workspaceEventBatch: WorkspaceEventBatch<ObjectRecordEvent>) {
|
||||
const logicFunctionsWithDatabaseEventTrigger =
|
||||
await this.logicFunctionRepository.find({
|
||||
where: {
|
||||
workspaceId: workspaceEventBatch.workspaceId,
|
||||
databaseEventTriggerSettings: Not(IsNull()),
|
||||
},
|
||||
select: ['id', 'databaseEventTriggerSettings', 'workspaceId'],
|
||||
});
|
||||
const { flatLogicFunctionMaps } =
|
||||
await this.workspaceCacheService.getOrRecompute(
|
||||
workspaceEventBatch.workspaceId,
|
||||
['flatLogicFunctionMaps'],
|
||||
);
|
||||
|
||||
const logicFunctionsWithDatabaseEventTrigger = Object.values(
|
||||
flatLogicFunctionMaps.byUniversalIdentifier,
|
||||
)
|
||||
.filter(isDefined)
|
||||
.filter(
|
||||
(logicFunction) =>
|
||||
!isDefined(logicFunction.deletedAt) &&
|
||||
isDefined(logicFunction.databaseEventTriggerSettings),
|
||||
);
|
||||
|
||||
const logicFunctionsToTrigger =
|
||||
logicFunctionsWithDatabaseEventTrigger.filter((logicFunction) =>
|
||||
|
||||
+4
-1
@@ -14,7 +14,10 @@ export const transformEventBatchToEventPayloads = ({
|
||||
logicFunctions,
|
||||
}: {
|
||||
workspaceEventBatch: WorkspaceEventBatch<ObjectRecordEvent>;
|
||||
logicFunctions: LogicFunctionEntity[];
|
||||
logicFunctions: Pick<
|
||||
LogicFunctionEntity,
|
||||
'id' | 'workspaceId' | 'databaseEventTriggerSettings'
|
||||
>[];
|
||||
}): LogicFunctionTriggerJobData[] => {
|
||||
const result: LogicFunctionTriggerJobData[] = [];
|
||||
const { events, ...batchEventInfo } = workspaceEventBatch;
|
||||
|
||||
+16
-13
@@ -1,8 +1,7 @@
|
||||
import { Logger } from '@nestjs/common';
|
||||
import { InjectRepository } from '@nestjs/typeorm';
|
||||
|
||||
import chunk from 'lodash.chunk';
|
||||
import { ArrayContains, IsNull, Repository } from 'typeorm';
|
||||
import { isDefined } from 'twenty-shared/utils';
|
||||
|
||||
import type { ObjectRecordEvent } from 'twenty-shared/database-events';
|
||||
|
||||
@@ -16,7 +15,7 @@ import {
|
||||
type CallWebhookJobData,
|
||||
} from 'src/engine/metadata-modules/webhook/jobs/call-webhook.job';
|
||||
import { transformEventBatchToWebhookEvents } from 'src/engine/metadata-modules/webhook/utils/transform-event-batch-to-webhook-events';
|
||||
import { WebhookEntity } from 'src/engine/metadata-modules/webhook/entities/webhook.entity';
|
||||
import { WorkspaceCacheService } from 'src/engine/workspace-cache/services/workspace-cache.service';
|
||||
import { WorkspaceEventBatch } from 'src/engine/workspace-event-emitter/types/workspace-event-batch.type';
|
||||
|
||||
const WEBHOOK_JOBS_CHUNK_SIZE = 20;
|
||||
@@ -27,8 +26,7 @@ export class CallWebhookJobsJob {
|
||||
constructor(
|
||||
@InjectMessageQueue(MessageQueue.webhookQueue)
|
||||
private readonly messageQueueService: MessageQueueService,
|
||||
@InjectRepository(WebhookEntity)
|
||||
private readonly webhookRepository: Repository<WebhookEntity>,
|
||||
private readonly workspaceCacheService: WorkspaceCacheService,
|
||||
) {}
|
||||
|
||||
@Process(CallWebhookJobsJob.name)
|
||||
@@ -42,20 +40,25 @@ export class CallWebhookJobsJob {
|
||||
|
||||
const [nameSingular, operation] = workspaceEventBatch.name.split('.');
|
||||
|
||||
const operations = [
|
||||
const operationsToMatch = [
|
||||
`${nameSingular}.${operation}`,
|
||||
`*.${operation}`,
|
||||
`${nameSingular}.*`,
|
||||
'*.*',
|
||||
];
|
||||
|
||||
const webhooks = await this.webhookRepository.find({
|
||||
where: operations.map((op) => ({
|
||||
workspaceId: workspaceEventBatch.workspaceId,
|
||||
operations: ArrayContains([op]),
|
||||
deletedAt: IsNull(),
|
||||
})),
|
||||
});
|
||||
const { flatWebhookMaps } = await this.workspaceCacheService.getOrRecompute(
|
||||
workspaceEventBatch.workspaceId,
|
||||
['flatWebhookMaps'],
|
||||
);
|
||||
|
||||
const webhooks = Object.values(flatWebhookMaps.byUniversalIdentifier)
|
||||
.filter(isDefined)
|
||||
.filter((webhook) =>
|
||||
operationsToMatch.some((operationToMatch) =>
|
||||
webhook.operations.includes(operationToMatch),
|
||||
),
|
||||
);
|
||||
|
||||
const webhookEvents = transformEventBatchToWebhookEvents({
|
||||
workspaceEventBatch,
|
||||
|
||||
+2
-3
@@ -1,19 +1,18 @@
|
||||
import { Module } from '@nestjs/common';
|
||||
import { TypeOrmModule } from '@nestjs/typeorm';
|
||||
|
||||
import { AuditModule } from 'src/engine/core-modules/audit/audit.module';
|
||||
import { MetricsModule } from 'src/engine/core-modules/metrics/metrics.module';
|
||||
import { SecureHttpClientModule } from 'src/engine/core-modules/secure-http-client/secure-http-client.module';
|
||||
import { WebhookEntity } from 'src/engine/metadata-modules/webhook/entities/webhook.entity';
|
||||
import { CallWebhookJobsJob } from 'src/engine/metadata-modules/webhook/jobs/call-webhook-jobs.job';
|
||||
import { CallWebhookJob } from 'src/engine/metadata-modules/webhook/jobs/call-webhook.job';
|
||||
import { WorkspaceCacheModule } from 'src/engine/workspace-cache/workspace-cache.module';
|
||||
|
||||
@Module({
|
||||
imports: [
|
||||
TypeOrmModule.forFeature([WebhookEntity]),
|
||||
AuditModule,
|
||||
MetricsModule,
|
||||
SecureHttpClientModule,
|
||||
WorkspaceCacheModule,
|
||||
],
|
||||
providers: [CallWebhookJobsJob, CallWebhookJob],
|
||||
})
|
||||
|
||||
+1
-1
@@ -10,7 +10,7 @@ export const transformEventBatchToWebhookEvents = ({
|
||||
webhooks,
|
||||
}: {
|
||||
workspaceEventBatch: WorkspaceEventBatch<ObjectRecordEvent>;
|
||||
webhooks: WebhookEntity[];
|
||||
webhooks: Pick<WebhookEntity, 'id' | 'targetUrl' | 'secret'>[];
|
||||
}): CallWebhookJobData[] => {
|
||||
const result: CallWebhookJobData[] = [];
|
||||
|
||||
|
||||
Reference in New Issue
Block a user