Separate metadata and object record publisher (#18740)
- Separate both publishers - Renaming - Removal of the old SSE endpoint --------- Co-authored-by: Charles Bochet <charles@twenty.com>
This commit is contained in:
Thomas Trompette
+121
@@ -0,0 +1,121 @@
|
||||
import { Injectable } from '@nestjs/common';
|
||||
import { EventEmitter2 } from '@nestjs/event-emitter';
|
||||
|
||||
import { AllMetadataName } from 'twenty-shared/metadata';
|
||||
import { isDefined } from 'twenty-shared/utils';
|
||||
|
||||
import { getWorkspaceAuthContext } from 'src/engine/core-modules/auth/storage/workspace-auth-context.storage';
|
||||
import { type WorkspaceAuthContext } from 'src/engine/core-modules/auth/types/workspace-auth-context.type';
|
||||
import { MetadataEventBatch } from 'src/engine/subscriptions/metadata-event/types/metadata-event-batch.type';
|
||||
import { computeMetadataEventName } from 'src/engine/subscriptions/metadata-event/utils/compute-metadata-event-name.util';
|
||||
import {
|
||||
AllMetadataEventName,
|
||||
AllMetadataEventType,
|
||||
MetadataEvent,
|
||||
} from 'src/engine/workspace-manager/workspace-migration/workspace-migration-runner/types/metadata-event';
|
||||
|
||||
type EmitMetadataEventsArgs = {
|
||||
metadataEvents: MetadataEvent[];
|
||||
workspaceId: string;
|
||||
initiatorContext?: WorkspaceAuthContext;
|
||||
};
|
||||
|
||||
@Injectable()
|
||||
export class MetadataEventEmitter {
|
||||
constructor(private readonly eventEmitter: EventEmitter2) {}
|
||||
|
||||
public emitMetadataEvents({
|
||||
metadataEvents,
|
||||
workspaceId,
|
||||
initiatorContext,
|
||||
}: EmitMetadataEventsArgs): void {
|
||||
if (metadataEvents.length === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
const resolvedInitiatorContext =
|
||||
this.resolveInitiatorContext(initiatorContext);
|
||||
|
||||
const userId =
|
||||
resolvedInitiatorContext?.type === 'user' ||
|
||||
resolvedInitiatorContext?.type === 'pendingActivationUser'
|
||||
? resolvedInitiatorContext.user.id
|
||||
: undefined;
|
||||
const apiKeyId =
|
||||
resolvedInitiatorContext?.type === 'apiKey'
|
||||
? resolvedInitiatorContext.apiKey.id
|
||||
: undefined;
|
||||
|
||||
const grouped = this.groupByMetadataNameAndAction(metadataEvents);
|
||||
|
||||
for (const { eventName, events, metadataName, type } of grouped.values()) {
|
||||
if (metadataEvents.length === 0) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const metadataEventBatch: MetadataEventBatch = {
|
||||
name: eventName,
|
||||
workspaceId,
|
||||
metadataName,
|
||||
type,
|
||||
events,
|
||||
userId,
|
||||
apiKeyId,
|
||||
};
|
||||
|
||||
this.eventEmitter.emit(eventName, metadataEventBatch);
|
||||
}
|
||||
}
|
||||
|
||||
private resolveInitiatorContext(
|
||||
initiatorContext?: WorkspaceAuthContext,
|
||||
): WorkspaceAuthContext | undefined {
|
||||
if (isDefined(initiatorContext)) {
|
||||
return initiatorContext;
|
||||
}
|
||||
|
||||
try {
|
||||
return getWorkspaceAuthContext();
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
private groupByMetadataNameAndAction(metadataEvents: MetadataEvent[]) {
|
||||
const grouped = new Map<
|
||||
AllMetadataEventName,
|
||||
{
|
||||
metadataName: AllMetadataName;
|
||||
type: AllMetadataEventType;
|
||||
eventName: AllMetadataEventName;
|
||||
events: MetadataEvent[];
|
||||
}
|
||||
>();
|
||||
|
||||
for (const metadataEvent of metadataEvents) {
|
||||
const { metadataName, type } = metadataEvent;
|
||||
const eventName = computeMetadataEventName({
|
||||
metadataName,
|
||||
type,
|
||||
});
|
||||
const occurence = grouped.get(eventName);
|
||||
|
||||
if (!isDefined(occurence)) {
|
||||
grouped.set(eventName, {
|
||||
eventName,
|
||||
metadataName,
|
||||
type,
|
||||
events: [metadataEvent],
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
grouped.set(eventName, {
|
||||
...occurence,
|
||||
events: [...occurence.events, metadataEvent],
|
||||
});
|
||||
}
|
||||
|
||||
return grouped;
|
||||
}
|
||||
}
|
||||
+250
@@ -0,0 +1,250 @@
|
||||
import { Injectable } from '@nestjs/common';
|
||||
|
||||
import { isDefined, isNonEmptyArray } from 'twenty-shared/utils';
|
||||
|
||||
import { OBJECT_METADATA_STANDARD_OVERRIDES_PROPERTIES } from 'src/engine/metadata-modules/object-metadata/constants/object-metadata-standard-overrides-properties.constant';
|
||||
import { WorkspaceManyOrAllFlatEntityMapsCacheService } from 'src/engine/metadata-modules/flat-entity/services/workspace-many-or-all-flat-entity-maps-cache.service';
|
||||
import { NavigationMenuItemRecordIdentifierService } from 'src/engine/metadata-modules/navigation-menu-item/services/navigation-menu-item-record-identifier.service';
|
||||
import { type MetadataEventBatch } from 'src/engine/subscriptions/metadata-event/types/metadata-event-batch.type';
|
||||
import { type EventStreamPayload } from 'src/engine/subscriptions/types/event-stream-payload.type';
|
||||
import { EventStreamService } from 'src/engine/subscriptions/event-stream.service';
|
||||
import { SubscriptionService } from 'src/engine/subscriptions/subscription.service';
|
||||
import { enrichFieldMetadataEventWithRelations } from 'src/engine/subscriptions/metadata-event/utils/enrich-field-metadata-event-with-relations.util';
|
||||
|
||||
@Injectable()
|
||||
export class MetadataEventPublisher {
|
||||
constructor(
|
||||
private readonly subscriptionService: SubscriptionService,
|
||||
private readonly eventStreamService: EventStreamService,
|
||||
private readonly workspaceManyOrAllFlatEntityMapsCacheService: WorkspaceManyOrAllFlatEntityMapsCacheService,
|
||||
private readonly navigationMenuItemRecordIdentifierService: NavigationMenuItemRecordIdentifierService,
|
||||
) {}
|
||||
|
||||
async publish(metadataEventBatch: MetadataEventBatch): Promise<void> {
|
||||
const workspaceId = metadataEventBatch.workspaceId;
|
||||
|
||||
const activeStreamIds =
|
||||
await this.eventStreamService.getActiveStreamIds(workspaceId);
|
||||
|
||||
if (activeStreamIds.length === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
const streamsData = await this.eventStreamService.getStreamsData(
|
||||
workspaceId,
|
||||
activeStreamIds,
|
||||
);
|
||||
|
||||
const enrichedBatch =
|
||||
await this.enrichMetadataEventBatch(metadataEventBatch);
|
||||
|
||||
const streamIdsToRemove: string[] = [];
|
||||
|
||||
for (const [streamChannelId, streamData] of streamsData) {
|
||||
if (!isDefined(streamData)) {
|
||||
streamIdsToRemove.push(streamChannelId);
|
||||
continue;
|
||||
}
|
||||
|
||||
await this.publishToStream({
|
||||
streamChannelId,
|
||||
metadataEventBatch: enrichedBatch,
|
||||
});
|
||||
}
|
||||
|
||||
await this.eventStreamService.removeFromActiveStreams(
|
||||
workspaceId,
|
||||
streamIdsToRemove,
|
||||
);
|
||||
}
|
||||
|
||||
private async enrichMetadataEventBatch(
|
||||
metadataEventBatch: MetadataEventBatch,
|
||||
): Promise<MetadataEventBatch> {
|
||||
switch (metadataEventBatch.metadataName) {
|
||||
case 'fieldMetadata':
|
||||
return this.enrichFieldMetadataEventsWithRelations(
|
||||
metadataEventBatch as MetadataEventBatch<'fieldMetadata'>,
|
||||
);
|
||||
case 'navigationMenuItem':
|
||||
return this.enrichNavigationMenuItemEventsWithTargetRecordIdentifier(
|
||||
metadataEventBatch as MetadataEventBatch<'navigationMenuItem'>,
|
||||
);
|
||||
case 'objectMetadata':
|
||||
return this.resolveObjectMetadataStandardOverrides(
|
||||
metadataEventBatch as MetadataEventBatch<'objectMetadata'>,
|
||||
);
|
||||
default:
|
||||
return metadataEventBatch;
|
||||
}
|
||||
}
|
||||
|
||||
private async publishToStream({
|
||||
streamChannelId,
|
||||
metadataEventBatch,
|
||||
}: {
|
||||
streamChannelId: string;
|
||||
metadataEventBatch: MetadataEventBatch;
|
||||
}): Promise<void> {
|
||||
if (!isNonEmptyArray(metadataEventBatch.events)) {
|
||||
return;
|
||||
}
|
||||
|
||||
const metadataEvents = metadataEventBatch.events.map((metadataEvent) => ({
|
||||
...metadataEvent,
|
||||
updatedCollectionHash: metadataEventBatch.updatedCollectionHash,
|
||||
}));
|
||||
|
||||
const payload: EventStreamPayload = {
|
||||
objectRecordEventsWithQueryIds: [],
|
||||
metadataEvents,
|
||||
};
|
||||
|
||||
await this.subscriptionService.publishToEventStream({
|
||||
workspaceId: metadataEventBatch.workspaceId,
|
||||
eventStreamChannelId: streamChannelId,
|
||||
payload,
|
||||
});
|
||||
}
|
||||
|
||||
private async enrichFieldMetadataEventsWithRelations(
|
||||
metadataEventBatch: MetadataEventBatch<'fieldMetadata'>,
|
||||
): Promise<MetadataEventBatch<'fieldMetadata'>> {
|
||||
const { flatFieldMetadataMaps, flatObjectMetadataMaps } =
|
||||
await this.workspaceManyOrAllFlatEntityMapsCacheService.getOrRecomputeManyOrAllFlatEntityMaps(
|
||||
{
|
||||
workspaceId: metadataEventBatch.workspaceId,
|
||||
flatMapsKeys: ['flatFieldMetadataMaps', 'flatObjectMetadataMaps'],
|
||||
},
|
||||
);
|
||||
|
||||
const enrichedEvents = metadataEventBatch.events.map((event) => {
|
||||
if (
|
||||
!('after' in event.properties) ||
|
||||
!isDefined(event.properties.after)
|
||||
) {
|
||||
return event;
|
||||
}
|
||||
|
||||
const enrichedAfter = enrichFieldMetadataEventWithRelations({
|
||||
record: event.properties.after as Record<string, unknown>,
|
||||
flatFieldMetadataMaps,
|
||||
flatObjectMetadataMaps,
|
||||
});
|
||||
|
||||
return {
|
||||
...event,
|
||||
properties: {
|
||||
...event.properties,
|
||||
after: enrichedAfter,
|
||||
},
|
||||
} as typeof event;
|
||||
});
|
||||
|
||||
return { ...metadataEventBatch, events: enrichedEvents };
|
||||
}
|
||||
|
||||
private async enrichNavigationMenuItemEventsWithTargetRecordIdentifier(
|
||||
metadataEventBatch: MetadataEventBatch<'navigationMenuItem'>,
|
||||
): Promise<MetadataEventBatch<'navigationMenuItem'>> {
|
||||
const enrichedEvents = await Promise.all(
|
||||
metadataEventBatch.events.map(async (event) => {
|
||||
if (
|
||||
!('after' in event.properties) ||
|
||||
!isDefined(event.properties.after)
|
||||
) {
|
||||
return event;
|
||||
}
|
||||
|
||||
const after = event.properties.after as Record<string, unknown>;
|
||||
const targetRecordId = after.targetRecordId as string | undefined;
|
||||
const targetObjectMetadataId = after.targetObjectMetadataId as
|
||||
| string
|
||||
| undefined;
|
||||
|
||||
if (!isDefined(targetRecordId) || !isDefined(targetObjectMetadataId)) {
|
||||
return event;
|
||||
}
|
||||
|
||||
const targetRecordIdentifier =
|
||||
await this.navigationMenuItemRecordIdentifierService.resolveRecordIdentifier(
|
||||
{
|
||||
targetRecordId,
|
||||
targetObjectMetadataId,
|
||||
workspaceId: metadataEventBatch.workspaceId,
|
||||
},
|
||||
);
|
||||
|
||||
const enrichedAfter: Record<string, unknown> = {
|
||||
...after,
|
||||
targetRecordIdentifier,
|
||||
};
|
||||
|
||||
return {
|
||||
...event,
|
||||
properties: {
|
||||
...event.properties,
|
||||
after: enrichedAfter,
|
||||
},
|
||||
} as typeof event;
|
||||
}),
|
||||
);
|
||||
|
||||
return { ...metadataEventBatch, events: enrichedEvents };
|
||||
}
|
||||
|
||||
private resolveObjectMetadataStandardOverrides(
|
||||
metadataEventBatch: MetadataEventBatch<'objectMetadata'>,
|
||||
): MetadataEventBatch<'objectMetadata'> {
|
||||
const enrichedEvents = metadataEventBatch.events.map((event) => {
|
||||
const enrichedProperties = { ...event.properties };
|
||||
|
||||
if (
|
||||
'before' in enrichedProperties &&
|
||||
isDefined(enrichedProperties.before)
|
||||
) {
|
||||
enrichedProperties.before =
|
||||
this.applyStandardOverridesToObjectMetadataRecord(
|
||||
enrichedProperties.before as Record<string, unknown>,
|
||||
) as typeof enrichedProperties.before;
|
||||
}
|
||||
|
||||
if (
|
||||
'after' in enrichedProperties &&
|
||||
isDefined(enrichedProperties.after)
|
||||
) {
|
||||
enrichedProperties.after =
|
||||
this.applyStandardOverridesToObjectMetadataRecord(
|
||||
enrichedProperties.after as Record<string, unknown>,
|
||||
) as typeof enrichedProperties.after;
|
||||
}
|
||||
|
||||
return { ...event, properties: enrichedProperties } as typeof event;
|
||||
});
|
||||
|
||||
return { ...metadataEventBatch, events: enrichedEvents };
|
||||
}
|
||||
|
||||
private applyStandardOverridesToObjectMetadataRecord(
|
||||
record: Record<string, unknown>,
|
||||
): Record<string, unknown> {
|
||||
const standardOverrides = record.standardOverrides as
|
||||
| Record<string, unknown>
|
||||
| null
|
||||
| undefined;
|
||||
|
||||
if (!isDefined(standardOverrides)) {
|
||||
return record;
|
||||
}
|
||||
|
||||
const resolved = { ...record };
|
||||
|
||||
for (const key of OBJECT_METADATA_STANDARD_OVERRIDES_PROPERTIES) {
|
||||
if (isDefined(standardOverrides[key])) {
|
||||
resolved[key] = standardOverrides[key];
|
||||
}
|
||||
}
|
||||
|
||||
return resolved;
|
||||
}
|
||||
}
|
||||
+77
@@ -0,0 +1,77 @@
|
||||
import { Injectable } from '@nestjs/common';
|
||||
import { OnEvent } from '@nestjs/event-emitter';
|
||||
|
||||
import { AllMetadataName } from 'twenty-shared/metadata';
|
||||
|
||||
import { InjectMessageQueue } from 'src/engine/core-modules/message-queue/decorators/message-queue.decorator';
|
||||
import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';
|
||||
import { MessageQueueService } from 'src/engine/core-modules/message-queue/services/message-queue.service';
|
||||
import { type MetadataEventBatch } from 'src/engine/subscriptions/metadata-event/types/metadata-event-batch.type';
|
||||
import { getMetadataFlatEntityMapsKey } from 'src/engine/metadata-modules/flat-entity/utils/get-metadata-flat-entity-maps-key.util';
|
||||
import { CallWebhookJobsForMetadataJob } from 'src/engine/metadata-modules/webhook/jobs/call-webhook-jobs-for-metadata.job';
|
||||
import { WorkspaceCacheService } from 'src/engine/workspace-cache/services/workspace-cache.service';
|
||||
import { type WorkspaceCacheKeyName } from 'src/engine/workspace-cache/types/workspace-cache-key.type';
|
||||
import { MetadataEventPublisher } from 'src/engine/subscriptions/metadata-event/metadata-event-publisher';
|
||||
import { type AllMetadataEventType } from 'src/engine/workspace-manager/workspace-migration/workspace-migration-runner/types/metadata-event';
|
||||
|
||||
@Injectable()
|
||||
export class MetadataEventsToDbListener {
|
||||
constructor(
|
||||
@InjectMessageQueue(MessageQueue.webhookQueue)
|
||||
private readonly webhookQueueService: MessageQueueService,
|
||||
private readonly metadataEventPublisher: MetadataEventPublisher,
|
||||
private readonly workspaceCacheService: WorkspaceCacheService,
|
||||
) {}
|
||||
|
||||
@OnEvent('metadata.*.created')
|
||||
async handleCreate(
|
||||
metadataEventBatch: MetadataEventBatch<AllMetadataName, 'created'>,
|
||||
): Promise<void> {
|
||||
return this.handleEvent(metadataEventBatch);
|
||||
}
|
||||
|
||||
@OnEvent('metadata.*.updated')
|
||||
async handleUpdate(
|
||||
metadataEventBatch: MetadataEventBatch<AllMetadataName, 'updated'>,
|
||||
): Promise<void> {
|
||||
return this.handleEvent(metadataEventBatch);
|
||||
}
|
||||
|
||||
@OnEvent('metadata.*.deleted')
|
||||
async handleDelete(
|
||||
metadataEventBatch: MetadataEventBatch<AllMetadataName, 'deleted'>,
|
||||
): Promise<void> {
|
||||
return this.handleEvent(metadataEventBatch);
|
||||
}
|
||||
|
||||
private async handleEvent(
|
||||
metadataEventBatch: MetadataEventBatch<
|
||||
AllMetadataName,
|
||||
AllMetadataEventType
|
||||
>,
|
||||
): Promise<void> {
|
||||
await this.webhookQueueService.add<
|
||||
MetadataEventBatch<AllMetadataName, AllMetadataEventType>
|
||||
>(CallWebhookJobsForMetadataJob.name, metadataEventBatch, {
|
||||
retryLimit: 3,
|
||||
});
|
||||
|
||||
if (metadataEventBatch.events.length > 0) {
|
||||
const cacheKeyName = getMetadataFlatEntityMapsKey(
|
||||
metadataEventBatch.metadataName,
|
||||
) as WorkspaceCacheKeyName;
|
||||
|
||||
const cacheHashes = await this.workspaceCacheService.getCacheHashes(
|
||||
metadataEventBatch.workspaceId,
|
||||
[cacheKeyName],
|
||||
);
|
||||
|
||||
const updatedCollectionHash = cacheHashes[cacheKeyName];
|
||||
|
||||
await this.metadataEventPublisher.publish({
|
||||
...metadataEventBatch,
|
||||
updatedCollectionHash,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
+20
@@ -0,0 +1,20 @@
|
||||
import type { AllMetadataName } from 'twenty-shared/metadata';
|
||||
|
||||
import {
|
||||
type AllMetadataEventType,
|
||||
type MetadataEvent,
|
||||
} from 'src/engine/workspace-manager/workspace-migration/workspace-migration-runner/types/metadata-event';
|
||||
|
||||
export type MetadataEventBatch<
|
||||
TMetadataName extends AllMetadataName = AllMetadataName,
|
||||
TType extends AllMetadataEventType = AllMetadataEventType,
|
||||
> = {
|
||||
name: `metadata.${TMetadataName}.${TType}`;
|
||||
workspaceId: string;
|
||||
metadataName: TMetadataName;
|
||||
type: TType;
|
||||
events: MetadataEvent[];
|
||||
userId?: string;
|
||||
apiKeyId?: string;
|
||||
updatedCollectionHash?: string;
|
||||
};
|
||||
+7
@@ -0,0 +1,7 @@
|
||||
import { type MetadataEvent } from 'src/engine/workspace-manager/workspace-migration/workspace-migration-runner/types/metadata-event';
|
||||
|
||||
export const computeMetadataEventName = ({
|
||||
metadataName,
|
||||
type,
|
||||
}: Pick<MetadataEvent, 'metadataName' | 'type'>) =>
|
||||
`metadata.${metadataName}.${type}` as const;
|
||||
+64
@@ -0,0 +1,64 @@
|
||||
import { FieldMetadataType } from 'twenty-shared/types';
|
||||
import { isDefined } from 'twenty-shared/utils';
|
||||
|
||||
import { type FlatEntityMaps } from 'src/engine/metadata-modules/flat-entity/types/flat-entity-maps.type';
|
||||
import { findFlatEntityByIdInFlatEntityMaps } from 'src/engine/metadata-modules/flat-entity/utils/find-flat-entity-by-id-in-flat-entity-maps.util';
|
||||
import { type FlatFieldMetadata } from 'src/engine/metadata-modules/flat-field-metadata/types/flat-field-metadata.type';
|
||||
import { isFlatFieldMetadataOfType } from 'src/engine/metadata-modules/flat-field-metadata/utils/is-flat-field-metadata-of-type.util';
|
||||
import { resolveMorphRelationsFromFlatFieldMetadata } from 'src/engine/metadata-modules/flat-field-metadata/utils/resolve-morph-relations-from-flat-field-metadata.util';
|
||||
import { resolveRelationFromFlatFieldMetadata } from 'src/engine/metadata-modules/flat-field-metadata/utils/resolve-relation-from-flat-field-metadata.util';
|
||||
import { type FlatObjectMetadata } from 'src/engine/metadata-modules/flat-object-metadata/types/flat-object-metadata.type';
|
||||
|
||||
type EnrichFieldMetadataEventArgs = {
|
||||
record: Record<string, unknown>;
|
||||
flatFieldMetadataMaps: FlatEntityMaps<FlatFieldMetadata>;
|
||||
flatObjectMetadataMaps: FlatEntityMaps<FlatObjectMetadata>;
|
||||
};
|
||||
|
||||
export const enrichFieldMetadataEventWithRelations = ({
|
||||
record,
|
||||
flatFieldMetadataMaps,
|
||||
flatObjectMetadataMaps,
|
||||
}: EnrichFieldMetadataEventArgs): Record<string, unknown> => {
|
||||
const flatFieldMetadata = findFlatEntityByIdInFlatEntityMaps({
|
||||
flatEntityId: record.id as string,
|
||||
flatEntityMaps: flatFieldMetadataMaps,
|
||||
});
|
||||
|
||||
if (!isDefined(flatFieldMetadata)) {
|
||||
return record;
|
||||
}
|
||||
|
||||
try {
|
||||
if (
|
||||
isFlatFieldMetadataOfType(flatFieldMetadata, FieldMetadataType.RELATION)
|
||||
) {
|
||||
const relation = resolveRelationFromFlatFieldMetadata({
|
||||
sourceFlatFieldMetadata: flatFieldMetadata,
|
||||
flatFieldMetadataMaps,
|
||||
flatObjectMetadataMaps,
|
||||
});
|
||||
|
||||
return isDefined(relation) ? { ...record, relation } : record;
|
||||
}
|
||||
|
||||
if (
|
||||
isFlatFieldMetadataOfType(
|
||||
flatFieldMetadata,
|
||||
FieldMetadataType.MORPH_RELATION,
|
||||
)
|
||||
) {
|
||||
const morphRelations = resolveMorphRelationsFromFlatFieldMetadata({
|
||||
morphFlatFieldMetadata: flatFieldMetadata,
|
||||
flatFieldMetadataMaps,
|
||||
flatObjectMetadataMaps,
|
||||
});
|
||||
|
||||
return { ...record, morphRelations };
|
||||
}
|
||||
} catch {
|
||||
return record;
|
||||
}
|
||||
|
||||
return record;
|
||||
};
|
||||
Reference in New Issue
Block a user