Move query matching before publication (#17121)

- new channel EVENT_STREAM_CHANNEL based on event stream id
- on event, perform the matching and publish only to the right streams
- store a list of active streams per workspace
- store the user id along with the queries for each stream

Bonus:
- remove onSubscriptionMatch

packages/twenty-server/src/engine/subscriptions/constants/event-stream-ttl.constant.ts

@@ -0,0 +1 @@
+export const EVENT_STREAM_TTL_MS = 30 * 60 * 1_000; // 30 minutes

packages/twenty-server/src/engine/subscriptions/dtos/subscription-matches.dto.ts

@@ -1,18 +0,0 @@
-import { Field, ObjectType } from '@nestjs/graphql';
-
-import { OnDbEventDTO } from './on-db-event.dto';
-
-@ObjectType('SubscriptionMatch')
-export class SubscriptionMatchDTO {
-  @Field(() => [String])
-  subscriptionIds: string[];
-
-  @Field(() => OnDbEventDTO)
-  event: OnDbEventDTO;
-}
-
-@ObjectType('SubscriptionMatches')
-export class SubscriptionMatchesDTO {
-  @Field(() => [SubscriptionMatchDTO])
-  matches: SubscriptionMatchDTO[];
-}

packages/twenty-server/src/engine/subscriptions/dtos/subscription.input.ts

@@ -1,15 +0,0 @@
-import { Field, InputType } from '@nestjs/graphql';
-
-import { DatabaseEventAction } from 'src/engine/api/graphql/graphql-query-runner/enums/database-event-action';
-
-@InputType()
-export class SubscriptionInput {
-  @Field()
-  id: string;
-
-  @Field()
-  query: string;
-
-  @Field(() => [DatabaseEventAction], { nullable: true })
-  selectedEventActions?: DatabaseEventAction[];
-}

packages/twenty-server/src/engine/subscriptions/enums/subscription-channel.enum.ts

@@ -1,5 +1,5 @@
 export enum SubscriptionChannel {
-  WORKSPACE_EVENT_BATCH_CHANNEL = 'WORKSPACE_EVENT_BATCH_CHANNEL',
   DATABASE_EVENT_CHANNEL = 'DATABASE_EVENT_CHANNEL',
   SERVERLESS_FUNCTION_LOGS_CHANNEL = 'SERVERLESS_FUNCTION_LOGS_CHANNEL',
+  EVENT_STREAM_CHANNEL = 'EVENT_STREAM_CHANNEL',
 }

packages/twenty-server/src/engine/subscriptions/event-stream.service.ts

@@ -1,73 +1,171 @@
 import { Injectable } from '@nestjs/common';
 
-import { type ObjectRecordEvent } from 'twenty-shared/database-events';
 import { type RecordGqlOperationSignature } from 'twenty-shared/types';
+import { isDefined } from 'twenty-shared/utils';
 
+import { type SerializableAuthContext } from 'src/engine/core-modules/auth/types/auth-context.type';
+import { CacheLockService } from 'src/engine/core-modules/cache-lock/cache-lock.service';
 import { WithLock } from 'src/engine/core-modules/cache-lock/with-lock.decorator';
 import { InjectCacheStorage } from 'src/engine/core-modules/cache-storage/decorators/cache-storage.decorator';
 import { CacheStorageService } from 'src/engine/core-modules/cache-storage/services/cache-storage.service';
 import { CacheStorageNamespace } from 'src/engine/core-modules/cache-storage/types/cache-storage-namespace.enum';
-
-export type ObjectRecordSubscriptionEvent = ObjectRecordEvent & {
-  objectNameSingular: string;
-};
+import { EVENT_STREAM_TTL_MS } from 'src/engine/subscriptions/constants/event-stream-ttl.constant';
+import { type EventStreamData } from 'src/engine/subscriptions/types/event-stream-data.type';
+import { type ObjectRecordSubscriptionEvent } from 'src/engine/subscriptions/types/object-record-subscription-event.type';
 
 @Injectable()
 export class EventStreamService {
   constructor(
     @InjectCacheStorage(CacheStorageNamespace.EngineSubscriptions)
     private readonly cacheStorageService: CacheStorageService,
+    private readonly cacheLockService: CacheLockService,
   ) {}
 
-  private getEventStreamKey(
-    workspaceId: string,
-    eventStreamId: string,
-  ): string {
-    return `eventStream:${workspaceId}:${eventStreamId}`;
+  async createEventStream({
+    workspaceId,
+    eventStreamChannelId,
+    authContext,
+  }: {
+    workspaceId: string;
+    eventStreamChannelId: string;
+    authContext: SerializableAuthContext;
+  }): Promise<void> {
+    const key = this.getEventStreamKey(workspaceId, eventStreamChannelId);
+    const streamData: EventStreamData = {
+      authContext,
+      workspaceId,
+      queries: {},
+      createdAt: Date.now(),
+    };
+
+    await this.cacheStorageService.set(key, streamData, EVENT_STREAM_TTL_MS);
+
+    const activeStreamsKey = this.getActiveStreamsKey(workspaceId);
+
+    await this.cacheLockService.withLock(async () => {
+      await this.cacheStorageService.setAdd(
+        activeStreamsKey,
+        [eventStreamChannelId],
+        EVENT_STREAM_TTL_MS,
+      );
+    }, activeStreamsKey);
   }
 
-  @WithLock('eventStreamId')
+  async destroyEventStream({
+    workspaceId,
+    eventStreamChannelId,
+  }: {
+    workspaceId: string;
+    eventStreamChannelId: string;
+  }): Promise<void> {
+    const key = this.getEventStreamKey(workspaceId, eventStreamChannelId);
+
+    await this.cacheStorageService.del(key);
+
+    const activeStreamsKey = this.getActiveStreamsKey(workspaceId);
+
+    await this.cacheLockService.withLock(async () => {
+      await this.cacheStorageService.setRemove(activeStreamsKey, [
+        eventStreamChannelId,
+      ]);
+    }, activeStreamsKey);
+  }
+
+  async getActiveStreamIds(workspaceId: string): Promise<string[]> {
+    return this.cacheStorageService.setMembers(
+      this.getActiveStreamsKey(workspaceId),
+    );
+  }
+
+  async removeFromActiveStreams(
+    workspaceId: string,
+    streamIdsToRemove: string[],
+  ): Promise<void> {
+    if (streamIdsToRemove.length === 0) {
+      return;
+    }
+
+    const activeStreamsKey = this.getActiveStreamsKey(workspaceId);
+
+    await this.cacheLockService.withLock(async () => {
+      await this.cacheStorageService.setRemove(
+        activeStreamsKey,
+        streamIdsToRemove,
+      );
+    }, activeStreamsKey);
+  }
+
+  async getStreamsData(
+    workspaceId: string,
+    streamChannelIds: string[],
+  ): Promise<Map<string, EventStreamData | undefined>> {
+    if (streamChannelIds.length === 0) {
+      return new Map();
+    }
+
+    const keys = streamChannelIds.map((id) =>
+      this.getEventStreamKey(workspaceId, id),
+    );
+    const values = await this.cacheStorageService.mget<EventStreamData>(keys);
+
+    const result = new Map<string, EventStreamData | undefined>();
+
+    streamChannelIds.forEach((id, index) => {
+      result.set(id, values[index]);
+    });
+
+    return result;
+  }
+
+  async getStreamData(
+    workspaceId: string,
+    eventStreamChannelId: string,
+  ): Promise<EventStreamData | undefined> {
+    const key = this.getEventStreamKey(workspaceId, eventStreamChannelId);
+
+    return this.cacheStorageService.get<EventStreamData>(key);
+  }
+
+  @WithLock('eventStreamChannelId')
   async addQuery({
     workspaceId,
-    eventStreamId,
+    eventStreamChannelId,
     queryId,
     operationSignature,
   }: {
     workspaceId: string;
-    eventStreamId: string;
+    eventStreamChannelId: string;
     queryId: string;
     operationSignature: RecordGqlOperationSignature;
   }): Promise<void> {
-    const key = this.getEventStreamKey(workspaceId, eventStreamId);
-    const existing =
-      (await this.cacheStorageService.get<
-        Record<string, RecordGqlOperationSignature>
-      >(key)) || {};
+    const key = this.getEventStreamKey(workspaceId, eventStreamChannelId);
+    const existing = await this.cacheStorageService.get<EventStreamData>(key);
 
-    existing[queryId] = operationSignature;
+    if (!isDefined(existing)) {
+      return;
+    }
 
-    await this.cacheStorageService.set(key, existing);
+    existing.queries[queryId] = operationSignature;
+
+    await this.cacheStorageService.set(key, existing, EVENT_STREAM_TTL_MS);
   }
 
-  @WithLock('eventStreamId')
+  @WithLock('eventStreamChannelId')
   async removeQuery({
     workspaceId,
-    eventStreamId,
+    eventStreamChannelId,
     queryId,
   }: {
     workspaceId: string;
-    eventStreamId: string;
+    eventStreamChannelId: string;
     queryId: string;
   }): Promise<void> {
-    const key = this.getEventStreamKey(workspaceId, eventStreamId);
-    const existing =
-      await this.cacheStorageService.get<
-        Record<string, RecordGqlOperationSignature>
-      >(key);
+    const key = this.getEventStreamKey(workspaceId, eventStreamChannelId);
+    const existing = await this.cacheStorageService.get<EventStreamData>(key);
 
-    if (existing && existing[queryId]) {
-      delete existing[queryId];
-      await this.cacheStorageService.set(key, existing);
+    if (isDefined(existing) && isDefined(existing.queries[queryId])) {
+      delete existing.queries[queryId];
+      await this.cacheStorageService.set(key, existing, EVENT_STREAM_TTL_MS);
     }
   }
 
@@ -75,22 +173,22 @@ export class EventStreamService {
     workspaceId: string,
     eventStreamId: string,
   ): Promise<Map<string, RecordGqlOperationSignature>> {
-    const key = this.getEventStreamKey(workspaceId, eventStreamId);
-    const data =
-      await this.cacheStorageService.get<
-        Record<string, RecordGqlOperationSignature>
-      >(key);
+    const streamData = await this.getStreamData(workspaceId, eventStreamId);
 
-    return new Map(Object.entries(data || {}));
+    if (!isDefined(streamData)) {
+      return new Map();
+    }
+
+    return new Map(Object.entries(streamData.queries));
   }
 
-  async matchQueriesWithEvent(
-    queries: Map<string, RecordGqlOperationSignature>,
+  matchQueriesWithEvent(
+    queries: Record<string, RecordGqlOperationSignature>,
     event: ObjectRecordSubscriptionEvent,
-  ): Promise<string[]> {
+  ): string[] {
     const matchedQueryIds: string[] = [];
 
-    for (const [queryId, operationSignature] of queries.entries()) {
+    for (const [queryId, operationSignature] of Object.entries(queries)) {
       if (this.isQueryMatchingEvent(operationSignature, event)) {
         matchedQueryIds.push(queryId);
       }
@@ -106,4 +204,15 @@ export class EventStreamService {
     // to be improved
     return operationSignature.objectNameSingular === event.objectNameSingular;
   }
+
+  private getEventStreamKey(
+    workspaceId: string,
+    eventStreamId: string,
+  ): string {
+    return `eventStream:${workspaceId}:${eventStreamId}`;
+  }
+
+  private getActiveStreamsKey(workspaceId: string): string {
+    return `workspace:${workspaceId}:activeStreams`;
+  }
 }

packages/twenty-server/src/engine/subscriptions/subscription.service.ts

@@ -1,20 +1,11 @@
 import { Injectable } from '@nestjs/common';
 
-import { FieldNode, OperationDefinitionNode, parse } from 'graphql';
-
 import { RedisClientService } from 'src/engine/core-modules/redis-client/redis-client.service';
-import { WorkspaceManyOrAllFlatEntityMapsCacheService } from 'src/engine/metadata-modules/flat-entity/services/workspace-many-or-all-flat-entity-maps-cache.service';
-import { FlatObjectMetadata } from 'src/engine/metadata-modules/flat-object-metadata/types/flat-object-metadata.type';
-import { OnDbEventDTO } from 'src/engine/subscriptions/dtos/on-db-event.dto';
-import { SubscriptionInput } from 'src/engine/subscriptions/dtos/subscription.input';
 import { SubscriptionChannel } from 'src/engine/subscriptions/enums/subscription-channel.enum';
 
 @Injectable()
 export class SubscriptionService {
-  constructor(
-    private readonly redisClient: RedisClientService,
-    private readonly workspaceManyOrAllFlatEntityMapsCacheService: WorkspaceManyOrAllFlatEntityMapsCacheService,
-  ) {}
+  constructor(private readonly redisClient: RedisClientService) {}
 
   private getSubscriptionChannel({
     channel,
@@ -26,6 +17,16 @@ export class SubscriptionService {
     return `${channel}:${workspaceId}`;
   }
 
+  private getEventStreamChannel({
+    workspaceId,
+    eventStreamChannelId,
+  }: {
+    workspaceId: string;
+    eventStreamChannelId: string;
+  }) {
+    return `${SubscriptionChannel.EVENT_STREAM_CHANNEL}:${workspaceId}:${eventStreamChannelId}`;
+  }
+
   async subscribe({
     channel,
     workspaceId,
@@ -40,6 +41,20 @@ export class SubscriptionService {
     );
   }
 
+  async subscribeToEventStream({
+    workspaceId,
+    eventStreamChannelId,
+  }: {
+    workspaceId: string;
+    eventStreamChannelId: string;
+  }) {
+    const client = this.redisClient.getPubSubClient();
+
+    return client.asyncIterator(
+      this.getEventStreamChannel({ workspaceId, eventStreamChannelId }),
+    );
+  }
+
   async publish<T>({
     channel,
     payload,
@@ -57,81 +72,20 @@ export class SubscriptionService {
     );
   }
 
-  public async isSubscriptionMatchingEvent(
-    subscription: SubscriptionInput,
-    event: OnDbEventDTO,
-    workspaceId: string,
-  ): Promise<boolean> {
-    const objectName = this.parseQueryObjectName(subscription.query);
+  async publishToEventStream<T>({
+    workspaceId,
+    eventStreamChannelId,
+    payload,
+  }: {
+    workspaceId: string;
+    eventStreamChannelId: string;
+    payload: T;
+  }): Promise<void> {
+    const client = this.redisClient.getPubSubClient();
 
-    if (!objectName) {
-      return false;
-    }
-
-    if (
-      subscription.selectedEventActions &&
-      !subscription.selectedEventActions.includes(event.action)
-    ) {
-      return false;
-    }
-
-    const { flatObjectMetadataMaps } =
-      await this.workspaceManyOrAllFlatEntityMapsCacheService.getOrRecomputeManyOrAllFlatEntityMaps(
-        {
-          workspaceId,
-          flatMapsKeys: ['flatObjectMetadataMaps'],
-        },
-      );
-
-    const eventObjectMetadata = Object.values(flatObjectMetadataMaps.byId).find(
-      (metadata: FlatObjectMetadata) =>
-        metadata.nameSingular === event.objectNameSingular,
+    await client.publish(
+      this.getEventStreamChannel({ workspaceId, eventStreamChannelId }),
+      payload,
     );
-
-    if (!eventObjectMetadata) {
-      return false;
-    }
-
-    const queryObjectNameLower = objectName.toLowerCase();
-    const eventNameSingularLower =
-      eventObjectMetadata.nameSingular.toLowerCase();
-    const eventNamePluralLower = eventObjectMetadata.namePlural.toLowerCase();
-
-    return (
-      queryObjectNameLower === eventNameSingularLower ||
-      queryObjectNameLower === eventNamePluralLower
-    );
-  }
-
-  private parseQueryObjectName(queryString: string): string | null {
-    try {
-      const { query } = JSON.parse(queryString) as {
-        query: string;
-        variables?: Record<string, unknown>;
-      };
-
-      const ast = parse(query);
-
-      const firstOperation = ast.definitions.find(
-        (def): def is OperationDefinitionNode =>
-          def.kind === 'OperationDefinition',
-      );
-
-      if (!firstOperation) {
-        return null;
-      }
-
-      const rootSelection = firstOperation.selectionSet.selections[0];
-
-      if (rootSelection.kind !== 'Field') {
-        return null;
-      }
-
-      const rootField = rootSelection as FieldNode;
-
-      return rootField.name.value;
-    } catch {
-      return null;
-    }
   }
 }

packages/twenty-server/src/engine/subscriptions/subscriptions.module.ts

@@ -1,18 +1,19 @@
 import { Module } from '@nestjs/common';
+import { TypeOrmModule } from '@nestjs/typeorm';
 
 import { CacheLockModule } from 'src/engine/core-modules/cache-lock/cache-lock.module';
 import { CacheStorageModule } from 'src/engine/core-modules/cache-storage/cache-storage.module';
 import { RedisClientModule } from 'src/engine/core-modules/redis-client/redis-client.module';
-import { WorkspaceManyOrAllFlatEntityMapsCacheModule } from 'src/engine/metadata-modules/flat-entity/services/workspace-many-or-all-flat-entity-maps-cache.module';
+import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity';
 import { EventStreamService } from 'src/engine/subscriptions/event-stream.service';
 import { SubscriptionService } from 'src/engine/subscriptions/subscription.service';
 
 @Module({
   imports: [
     RedisClientModule,
-    WorkspaceManyOrAllFlatEntityMapsCacheModule,
     CacheStorageModule,
     CacheLockModule,
+    TypeOrmModule.forFeature([WorkspaceEntity]),
   ],
   providers: [SubscriptionService, EventStreamService],
   exports: [SubscriptionService, EventStreamService],

packages/twenty-server/src/engine/subscriptions/types/event-stream-data.type.ts

@@ -0,0 +1,10 @@
+import { type RecordGqlOperationSignature } from 'twenty-shared/types';
+
+import { type SerializableAuthContext } from 'src/engine/core-modules/auth/types/auth-context.type';
+
+export type EventStreamData = {
+  authContext: SerializableAuthContext;
+  workspaceId: string;
+  queries: Record<string, RecordGqlOperationSignature>;
+  createdAt: number;
+};

packages/twenty-server/src/engine/subscriptions/types/object-record-subscription-event.type.ts

@@ -0,0 +1,5 @@
+import { type ObjectRecordEvent } from 'twenty-shared/database-events';
+
+export type ObjectRecordSubscriptionEvent = ObjectRecordEvent & {
+  objectNameSingular: string;
+};