Flat entity maps cache generic service + runner dynamically retrieving invalidating update cache + view service v2 refactor (#14508)

# Introduction Migrate previous runner only iterating on `flatObjectMetadataMaps` to `allFlatEntityMaps`. Refactored the optimistic to be handled inside the actions handler ## Workspace flat map cache Introducing a new service and registry, that will dynamically retrieve and or recompute requested cache when called ## Runner refactor Runner now dynamically invalidate updated cache at the end of the transaction close https://github.com/orgs/twentyhq/projects/1/views/8?pane=issue&itemId=129136356&issue=twentyhq%7Ccore-team-issues%7C1492 close https://github.com/orgs/twentyhq/projects/1/views/8?pane=issue&itemId=129136210&issue=twentyhq%7Ccore-team-issues%7C1494
2025-09-16 14:39:42 +02:00
parent 8f42c33bd4
commit 371c26bc9b
79 changed files with 1534 additions and 575 deletions
@@ -0,0 +1,7 @@
+import { type AllFlatEntityMaps } from 'src/engine/core-modules/common/types/all-flat-entity-maps.type';
+
+export const ALL_FLAT_ENTITY_MAPS_PROPERTIES = [
+  'flatObjectMetadataMaps',
+  'flatViewFieldMaps',
+  'flatViewMaps',
+] as const satisfies (keyof AllFlatEntityMaps)[];
@@ -3,7 +3,7 @@ import { type AllFlatEntityMaps } from 'src/engine/core-modules/common/types/all

 export const EMPTY_ALL_FLAT_ENTITY_MAPS = {
  flatObjectMetadataMaps: {
-    byId: {},
+    ...EMPTY_FLAT_ENTITY_MAPS,
    idByNameSingular: {},
  },
  flatViewFieldMaps: EMPTY_FLAT_ENTITY_MAPS,
@@ -0,0 +1,11 @@
+import { Module } from '@nestjs/common';
+
+import { WorkspaceManyOrAllFlatEntityMapsCacheService } from 'src/engine/core-modules/common/services/workspace-many-or-all-flat-entity-maps-cache.service.';
+import { WorkspaceFlatMapCacheModule } from 'src/engine/workspace-flat-map-cache/workspace-flat-map-cache.module';
+
+@Module({
+  imports: [WorkspaceFlatMapCacheModule],
+  providers: [WorkspaceManyOrAllFlatEntityMapsCacheService],
+  exports: [WorkspaceManyOrAllFlatEntityMapsCacheService],
+})
+export class WorkspaceManyOrAllFlatEntityMapsCacheModule {}
@@ -0,0 +1,105 @@
+import { Injectable, Logger } from '@nestjs/common';
+
+import { isDefined } from 'twenty-shared/utils';
+
+import { ALL_FLAT_ENTITY_MAPS_PROPERTIES } from 'src/engine/core-modules/common/constant/all-flat-entity-maps-properties.constant';
+import { EMPTY_ALL_FLAT_ENTITY_MAPS } from 'src/engine/core-modules/common/constant/empty-all-flat-entity-maps.constant';
+import { AllFlatEntityMaps } from 'src/engine/core-modules/common/types/all-flat-entity-maps.type';
+import {
+  WorkspaceFlatMapCacheException,
+  WorkspaceFlatMapCacheExceptionCode,
+} from 'src/engine/workspace-flat-map-cache/exceptions/workspace-flat-map-cache.exception';
+import { WorkspaceFlatMapCacheRegistryService } from 'src/engine/workspace-flat-map-cache/services/workspace-flat-map-cache-registry.service';
+
+@Injectable()
+export class WorkspaceManyOrAllFlatEntityMapsCacheService {
+  private readonly logger = new Logger(
+    WorkspaceManyOrAllFlatEntityMapsCacheService.name,
+  );
+
+  constructor(
+    private readonly cacheRegistry: WorkspaceFlatMapCacheRegistryService,
+  ) {}
+
+  public async getOrRecomputeManyOrAllFlatEntityMaps<
+    T extends (keyof AllFlatEntityMaps)[] = (keyof AllFlatEntityMaps)[],
+  >({
+    flatEntities,
+    workspaceId,
+  }: {
+    workspaceId: string;
+    flatEntities?: T;
+  }): Promise<Pick<AllFlatEntityMaps, T[number]>> {
+    const allFlatEntityMaps: AllFlatEntityMaps = structuredClone(
+      EMPTY_ALL_FLAT_ENTITY_MAPS,
+    );
+
+    for (const flatEntityName of ALL_FLAT_ENTITY_MAPS_PROPERTIES) {
+      if (isDefined(flatEntities) && !flatEntities.includes(flatEntityName)) {
+        delete allFlatEntityMaps[flatEntityName];
+        continue;
+      }
+
+      try {
+        const service = this.cacheRegistry.getCacheService(flatEntityName);
+
+        if (!isDefined(service)) {
+          throw new WorkspaceFlatMapCacheException(
+            `No cache service found for ${flatEntityName}`,
+            WorkspaceFlatMapCacheExceptionCode.INTERNAL_SERVER_ERROR,
+          );
+        }
+
+        const result = await service.getExistingOrRecomputeFlatMaps({
+          workspaceId,
+        });
+
+        // @ts-expect-error todo prastoin once refactored flat object metadata cache
+        allFlatEntityMaps[flatEntityName] = result;
+      } catch (error) {
+        this.logger.error(
+          `Failed to get flat entity maps for ${flatEntityName}`,
+          error,
+        );
+        throw error;
+      }
+    }
+
+    return allFlatEntityMaps;
+  }
+
+  public async invalidateFlatEntityMaps<
+    T extends (keyof AllFlatEntityMaps)[] = (keyof AllFlatEntityMaps)[],
+  >({
+    flatEntities,
+    workspaceId,
+  }: {
+    workspaceId: string;
+    flatEntities?: T;
+  }): Promise<void> {
+    for (const flatEntityName of ALL_FLAT_ENTITY_MAPS_PROPERTIES) {
+      if (isDefined(flatEntities) && !flatEntities.includes(flatEntityName)) {
+        continue;
+      }
+
+      try {
+        const service = this.cacheRegistry.getCacheService(flatEntityName);
+
+        if (!isDefined(service)) {
+          throw new WorkspaceFlatMapCacheException(
+            `No cache service found for ${flatEntityName}`,
+            WorkspaceFlatMapCacheExceptionCode.INTERNAL_SERVER_ERROR,
+          );
+        }
+
+        await service.invalidateCache({ workspaceId });
+      } catch (error) {
+        this.logger.error(
+          `Failed to invalidate flat entity maps for ${flatEntityName}`,
+          error,
+        );
+        throw error;
+      }
+    }
+  }
+}
@@ -0,0 +1,10 @@
+import { type FlatViewField } from 'src/engine/core-modules/view/flat-view/types/flat-view-field.type';
+import { type FlatView } from 'src/engine/core-modules/view/flat-view/types/flat-view.type';
+import { type FlatObjectMetadata } from 'src/engine/metadata-modules/flat-object-metadata/types/flat-object-metadata.type';
+
+export type AllFlatEntitiesByMetadataEngineName = {
+  //   flatFieldMetadata: FlatFieldMetadata;
+  objectMetadata: FlatObjectMetadata;
+  view: FlatView;
+  viewField: FlatViewField;
+};
@@ -1,10 +1,4 @@
-import { type FlatViewField } from 'src/engine/core-modules/view/flat-view/types/flat-view-field.type';
-import { type FlatView } from 'src/engine/core-modules/view/flat-view/types/flat-view.type';
-import { type FlatObjectMetadata } from 'src/engine/metadata-modules/flat-object-metadata/types/flat-object-metadata.type';
+import { type AllFlatEntitiesByMetadataEngineName } from 'src/engine/core-modules/common/types/all-flat-entities-by-metadata-engine-name.type';

-export type AllFlatEntitiesByMetadataEngineName = {
-  //   flatFieldMetadata: FlatFieldMetadata;
-  objectMetadata: FlatObjectMetadata;
-  view: FlatView;
-  viewField: FlatViewField;
-};
+export type AllFlatEntities =
+  AllFlatEntitiesByMetadataEngineName[keyof AllFlatEntitiesByMetadataEngineName];