Flat entity maps cache generic service + runner dynamically retrieving invalidating update cache + view service v2 refactor (#14508)

# Introduction
Migrate previous runner only iterating on `flatObjectMetadataMaps` to
`allFlatEntityMaps`.
Refactored the optimistic to be handled inside the actions handler

## Workspace flat map cache
Introducing a new service and registry, that will dynamically retrieve
and or recompute requested cache when called

## Runner refactor
Runner now dynamically invalidate updated cache at the end of the
transaction

close
https://github.com/orgs/twentyhq/projects/1/views/8?pane=issue&itemId=129136356&issue=twentyhq%7Ccore-team-issues%7C1492
close
https://github.com/orgs/twentyhq/projects/1/views/8?pane=issue&itemId=129136210&issue=twentyhq%7Ccore-team-issues%7C1494

packages/twenty-server/src/engine/workspace-flat-map-cache/decorators/workspace-flat-map-cache.decorator.ts

@@ -1,6 +1,9 @@
 import { SetMetadata } from '@nestjs/common';
 
+import { type ALL_FLAT_ENTITY_MAPS_PROPERTIES } from 'src/engine/core-modules/common/constant/all-flat-entity-maps-properties.constant';
+
 export const WORKSPACE_FLAT_MAP_CACHE_KEY = 'workspaceFlatMapCacheKey';
 
-export const WorkspaceFlatMapCache = (cacheKey: string) =>
-  SetMetadata(WORKSPACE_FLAT_MAP_CACHE_KEY, cacheKey);
+export const WorkspaceFlatMapCache = (
+  cacheKey: (typeof ALL_FLAT_ENTITY_MAPS_PROPERTIES)[number],
+) => SetMetadata(WORKSPACE_FLAT_MAP_CACHE_KEY, cacheKey);

packages/twenty-server/src/engine/workspace-flat-map-cache/services/workspace-flat-map-cache-registry.service.ts

@@ -0,0 +1,49 @@
+import { Injectable, OnModuleInit } from '@nestjs/common';
+import { DiscoveryService } from '@nestjs/core';
+
+import { AllFlatEntities } from 'src/engine/core-modules/common/types/all-flat-entities.type';
+import { AllFlatEntityMaps } from 'src/engine/core-modules/common/types/all-flat-entity-maps.type';
+import { FlatEntityMaps } from 'src/engine/core-modules/common/types/flat-entity-maps.type';
+import { WORKSPACE_FLAT_MAP_CACHE_KEY } from 'src/engine/workspace-flat-map-cache/decorators/workspace-flat-map-cache.decorator';
+import { WorkspaceFlatMapCacheService } from 'src/engine/workspace-flat-map-cache/services/workspace-flat-map-cache.service';
+
+@Injectable()
+export class WorkspaceFlatMapCacheRegistryService implements OnModuleInit {
+  private readonly cacheServiceMap = new Map<
+    keyof AllFlatEntityMaps,
+    WorkspaceFlatMapCacheService<FlatEntityMaps<AllFlatEntities>>
+  >();
+
+  constructor(private readonly discoveryService: DiscoveryService) {}
+
+  async onModuleInit() {
+    this.discoverAndRegisterCacheServices();
+  }
+
+  private discoverAndRegisterCacheServices(): void {
+    const providers = this.discoveryService.getProviders();
+
+    providers.forEach((wrapper) => {
+      const { instance, metatype } = wrapper;
+
+      if (!instance || !metatype) {
+        return;
+      }
+
+      const cacheKey = Reflect.getMetadata(
+        WORKSPACE_FLAT_MAP_CACHE_KEY,
+        metatype,
+      );
+
+      if (cacheKey && instance instanceof WorkspaceFlatMapCacheService) {
+        this.cacheServiceMap.set(cacheKey, instance);
+      }
+    });
+  }
+
+  getCacheService(
+    flatEntityName: keyof AllFlatEntityMaps,
+  ): WorkspaceFlatMapCacheService<FlatEntityMaps<AllFlatEntities>> | undefined {
+    return this.cacheServiceMap.get(flatEntityName);
+  }
+}

packages/twenty-server/src/engine/workspace-flat-map-cache/services/workspace-flat-map-cache.service.ts

@@ -99,6 +99,7 @@ export abstract class WorkspaceFlatMapCacheService<
   }: {
     workspaceId: string;
   }): Promise<void> {
+    await this.beforeInvalidateCache({ workspaceId });
     const { flatMapKey, hashKey } = this.buildRemoteCacheKeys({ workspaceId });
 
     await this.cacheStorageService.del(flatMapKey);
@@ -108,6 +109,19 @@ export abstract class WorkspaceFlatMapCacheService<
     this.localCacheHashes.delete(workspaceId);
 
     await this.recomputeAndStoreInCache({ workspaceId });
+    await this.afterInvalidateCache({ workspaceId });
+  }
+
+  protected async beforeInvalidateCache(_args: {
+    workspaceId: string;
+  }): Promise<void> {
+    return;
+  }
+
+  protected async afterInvalidateCache(_args: {
+    workspaceId: string;
+  }): Promise<void> {
+    return;
   }
 
   private getFlatMapCacheKey(): string {
@@ -116,7 +130,7 @@ export abstract class WorkspaceFlatMapCacheService<
       this.constructor,
     );
 
-    if (!cacheKey) {
+    if (!isDefined(cacheKey)) {
       throw new WorkspaceFlatMapCacheException(
         `${this.constructor.name} must be decorated with @WorkspaceFlatMapCache('cacheKey')`,
         WorkspaceFlatMapCacheExceptionCode.MISSING_DECORATOR,

packages/twenty-server/src/engine/workspace-flat-map-cache/workspace-flat-map-cache.module.ts

@@ -1,10 +1,36 @@
 import { Module } from '@nestjs/common';
+import { DiscoveryModule } from '@nestjs/core';
+import { TypeOrmModule } from '@nestjs/typeorm';
 
-import { CacheStorageModule } from 'src/engine/core-modules/cache-storage/cache-storage.module';
+import { ViewFieldEntity } from 'src/engine/core-modules/view/entities/view-field.entity';
+import { ViewEntity } from 'src/engine/core-modules/view/entities/view.entity';
+import { WorkspaceFlatViewFieldMapCacheService } from 'src/engine/core-modules/view/flat-view/services/workspace-flat-view-field-map-cache.service';
+import { WorkspaceFlatViewMapCacheService } from 'src/engine/core-modules/view/flat-view/services/workspace-flat-view-map-cache.service';
+import { WorkspaceFlatObjectMetadataMapCacheService } from 'src/engine/metadata-modules/flat-object-metadata/services/workspace-flat-object-metadata-map-cache.service';
+import { WorkspaceMetadataCacheModule } from 'src/engine/metadata-modules/workspace-metadata-cache/workspace-metadata-cache.module';
+import { WorkspaceMetadataVersionModule } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.module';
+import { WorkspacePermissionsCacheModule } from 'src/engine/metadata-modules/workspace-permissions-cache/workspace-permissions-cache.module';
+import { WorkspaceFlatMapCacheRegistryService } from 'src/engine/workspace-flat-map-cache/services/workspace-flat-map-cache-registry.service';
 
 @Module({
-  imports: [CacheStorageModule],
-  providers: [],
-  exports: [],
+  imports: [
+    DiscoveryModule,
+    WorkspaceMetadataCacheModule,
+    WorkspaceMetadataVersionModule,
+    WorkspacePermissionsCacheModule,
+    TypeOrmModule.forFeature([ViewEntity, ViewFieldEntity]),
+  ],
+  providers: [
+    WorkspaceFlatMapCacheRegistryService,
+    WorkspaceFlatObjectMetadataMapCacheService,
+    WorkspaceFlatViewMapCacheService,
+    WorkspaceFlatViewFieldMapCacheService,
+  ],
+  exports: [
+    WorkspaceFlatMapCacheRegistryService,
+    WorkspaceFlatObjectMetadataMapCacheService,
+    WorkspaceFlatViewMapCacheService,
+    WorkspaceFlatViewFieldMapCacheService,
+  ],
 })
 export class WorkspaceFlatMapCacheModule {}