Flat entity maps cache generic service + runner dynamically retrieving invalidating update cache + view service v2 refactor (#14508)

# Introduction
Migrate previous runner only iterating on `flatObjectMetadataMaps` to
`allFlatEntityMaps`.
Refactored the optimistic to be handled inside the actions handler

## Workspace flat map cache
Introducing a new service and registry, that will dynamically retrieve
and or recompute requested cache when called

## Runner refactor
Runner now dynamically invalidate updated cache at the end of the
transaction

close
https://github.com/orgs/twentyhq/projects/1/views/8?pane=issue&itemId=129136356&issue=twentyhq%7Ccore-team-issues%7C1492
close
https://github.com/orgs/twentyhq/projects/1/views/8?pane=issue&itemId=129136210&issue=twentyhq%7Ccore-team-issues%7C1494

packages/twenty-server/src/engine/metadata-modules/field-metadata/field-metadata.module.ts

@@ -9,6 +9,7 @@ import { NestjsQueryTypeOrmModule } from '@ptc-org/nestjs-query-typeorm';
 
 import { TypeORMModule } from 'src/database/typeorm/typeorm.module';
 import { ActorModule } from 'src/engine/core-modules/actor/actor.module';
+import { WorkspaceManyOrAllFlatEntityMapsCacheModule } from 'src/engine/core-modules/common/services/workspace-many-or-all-flat-entity-maps-cache.module';
 import { FeatureFlagModule } from 'src/engine/core-modules/feature-flag/feature-flag.module';
 import { CoreViewModule } from 'src/engine/core-modules/view/view.module';
 import { WorkspaceAuthGuard } from 'src/engine/guards/workspace-auth.guard';
@@ -66,6 +67,7 @@ import { FieldMetadataService } from './services/field-metadata.service';
         WorkspaceMigrationV2Module,
         FlatFieldMetadataModule,
         IndexMetadataModule,
+        WorkspaceManyOrAllFlatEntityMapsCacheModule,
       ],
       services: [
         IsFieldMetadataDefaultValue,

packages/twenty-server/src/engine/metadata-modules/field-metadata/services/field-metadata.service-v2.ts

@@ -4,6 +4,7 @@ import { InjectRepository } from '@nestjs/typeorm';
 import { isDefined } from 'twenty-shared/utils';
 import { In, Repository } from 'typeorm';
 
+import { WorkspaceManyOrAllFlatEntityMapsCacheService } from 'src/engine/core-modules/common/services/workspace-many-or-all-flat-entity-maps-cache.service.';
 import { type CreateFieldInput } from 'src/engine/metadata-modules/field-metadata/dtos/create-field.input';
 import { type DeleteOneFieldInput } from 'src/engine/metadata-modules/field-metadata/dtos/delete-field.input';
 import { FieldMetadataDTO } from 'src/engine/metadata-modules/field-metadata/dtos/field-metadata.dto';
@@ -22,7 +23,6 @@ import { addFlatFieldMetadataInFlatObjectMetadataMapsOrThrow } from 'src/engine/
 import { deleteFieldFromFlatObjectMetadataMapsOrThrow } from 'src/engine/metadata-modules/flat-object-metadata-maps/utils/delete-field-from-flat-object-metadata-maps-or-throw.util';
 import { getSubFlatObjectMetadataMapsOrThrow } from 'src/engine/metadata-modules/flat-object-metadata-maps/utils/get-sub-flat-object-metadata-maps-or-throw.util';
 import { replaceFlatFieldMetadataInFlatObjectMetadataMapsOrThrow } from 'src/engine/metadata-modules/flat-object-metadata-maps/utils/replace-flat-field-metadata-in-flat-object-metadata-maps-or-throw.util';
-import { WorkspaceMetadataCacheService } from 'src/engine/metadata-modules/workspace-metadata-cache/services/workspace-metadata-cache.service';
 import { WorkspaceMigrationBuilderExceptionV2 } from 'src/engine/workspace-manager/workspace-migration-v2/exceptions/workspace-migration-builder-exception-v2';
 import { WorkspaceMigrationValidateBuildAndRunService } from 'src/engine/workspace-manager/workspace-migration-v2/services/workspace-migration-validate-build-and-run-service';
 
@@ -31,7 +31,7 @@ export class FieldMetadataServiceV2 {
   constructor(
     @InjectRepository(FieldMetadataEntity)
     private readonly fieldMetadataRepository: Repository<FieldMetadataEntity>,
-    private readonly workspaceMetadataCacheService: WorkspaceMetadataCacheService,
+    private readonly flatEntityMapsCacheService: WorkspaceManyOrAllFlatEntityMapsCacheService,
     private readonly workspaceMigrationValidateBuildAndRunService: WorkspaceMigrationValidateBuildAndRunService,
   ) {}
 
@@ -65,9 +65,10 @@ export class FieldMetadataServiceV2 {
     workspaceId: string;
   }): Promise<FieldMetadataDTO> {
     const { flatObjectMetadataMaps: existingFlatObjectMetadataMaps } =
-      await this.workspaceMetadataCacheService.getExistingOrRecomputeFlatObjectMetadataMaps(
+      await this.flatEntityMapsCacheService.getOrRecomputeManyOrAllFlatEntityMaps(
         {
           workspaceId,
+          flatEntities: ['flatObjectMetadataMaps'],
         },
       );
 
@@ -136,8 +137,11 @@ export class FieldMetadataServiceV2 {
     workspaceId: string;
   }): Promise<FieldMetadataEntity> {
     const { flatObjectMetadataMaps: existingFlatObjectMetadataMaps } =
-      await this.workspaceMetadataCacheService.getExistingOrRecomputeFlatObjectMetadataMaps(
-        { workspaceId },
+      await this.flatEntityMapsCacheService.getOrRecomputeManyOrAllFlatEntityMaps(
+        {
+          workspaceId,
+          flatEntities: ['flatObjectMetadataMaps'],
+        },
       );
 
     const inputTranspilationResult = fromUpdateFieldInputToFlatFieldMetadata({
@@ -221,8 +225,11 @@ export class FieldMetadataServiceV2 {
     }
 
     const { flatObjectMetadataMaps: existingFlatObjectMetadataMaps } =
-      await this.workspaceMetadataCacheService.getExistingOrRecomputeFlatObjectMetadataMaps(
-        { workspaceId },
+      await this.flatEntityMapsCacheService.getOrRecomputeManyOrAllFlatEntityMaps(
+        {
+          workspaceId,
+          flatEntities: ['flatObjectMetadataMaps'],
+        },
       );
 
     const allTranspiledTranspilationInputs = [];

packages/twenty-server/src/engine/metadata-modules/flat-object-metadata-maps/constant/empty-flat-object-metadata-maps.constant.ts

@@ -3,4 +3,5 @@ import { type FlatObjectMetadataMaps } from 'src/engine/metadata-modules/flat-ob
 export const EMPTY_FLAT_OBJECT_METADATA_MAPS = {
   byId: {},
   idByNameSingular: {},
+  idByUniversalIdentifier: {},
 } as const satisfies FlatObjectMetadataMaps;

packages/twenty-server/src/engine/metadata-modules/flat-object-metadata-maps/types/flat-object-metadata-maps.type.ts

@@ -3,4 +3,5 @@ import { type FlatObjectMetadataWithFlatFieldMaps } from 'src/engine/metadata-mo
 export type FlatObjectMetadataMaps = {
   byId: Partial<Record<string, FlatObjectMetadataWithFlatFieldMaps>>;
   idByNameSingular: Partial<Record<string, string>>;
+  idByUniversalIdentifier: Record<string, string>;
 };

packages/twenty-server/src/engine/metadata-modules/flat-object-metadata-maps/utils/add-flat-field-metadata-in-flat-object-metadata-maps-or-throw.util.ts

@@ -27,6 +27,9 @@ export const addFlatFieldMetadataInFlatObjectMetadataMapsOrThrow = ({
   }
 
   return {
+    idByUniversalIdentifier: {
+      ...flatObjectMetadataMaps.idByUniversalIdentifier,
+    },
     byId: {
       ...flatObjectMetadataMaps.byId,
       [flatFieldMetadata.objectMetadataId]:

packages/twenty-server/src/engine/metadata-modules/flat-object-metadata-maps/utils/add-flat-object-metadata-to-flat-object-metadata-maps-or-throw.util.ts

@@ -24,6 +24,10 @@ export const addFlatObjectMetadataToFlatObjectMetadataMapsOrThrow = ({
   }
 
   return {
+    idByUniversalIdentifier: {
+      ...flatObjectMetadataMaps.idByUniversalIdentifier,
+      [flatObjectMetadata.universalIdentifier]: flatObjectMetadata.id,
+    },
     byId: {
       ...flatObjectMetadataMaps.byId,
       [flatObjectMetadata.id]:

packages/twenty-server/src/engine/metadata-modules/flat-object-metadata-maps/utils/add-flat-object-metadata-with-field-maps-to-flat-object-metadata-maps-or-throw.util.ts

@@ -29,6 +29,11 @@ export const addFlatObjectMetadataWithFlatFieldMapsToFlatObjectMetadataMapsOrThr
     }
 
     return {
+      idByUniversalIdentifier: {
+        ...flatObjectMetadataMaps.idByUniversalIdentifier,
+        [flatObjectMetadataWithFlatFieldMaps.universalIdentifier]:
+          flatObjectMetadataWithFlatFieldMaps.id,
+      },
       byId: {
         ...flatObjectMetadataMaps.byId,
         [flatObjectMetadataWithFlatFieldMaps.id]:

packages/twenty-server/src/engine/metadata-modules/flat-object-metadata-maps/utils/delete-field-from-flat-object-metadata-maps-or-throw.util.ts

@@ -28,6 +28,7 @@ export const deleteFieldFromFlatObjectMetadataMapsOrThrow = ({
   }
 
   return {
+    idByUniversalIdentifier: flatObjectMetadataMaps.idByUniversalIdentifier,
     byId: {
       ...flatObjectMetadataMaps.byId,
       [objectMetadataId]:

packages/twenty-server/src/engine/metadata-modules/flat-object-metadata-maps/utils/delete-object-from-flat-object-metadata-maps-or-throw.util.ts

@@ -25,7 +25,14 @@ export const deleteObjectFromFlatObjectMetadataMapsOrThrow = ({
     flatObjectMetadataMaps.idByNameSingular,
   ).filter(([_nameSingular, id]) => id !== objectMetadataId);
 
+  const updatedIdByUniversalIdentifierEntries = Object.entries(
+    flatObjectMetadataMaps.idByUniversalIdentifier,
+  ).filter(([_universalIdentifier, id]) => id !== objectMetadataId);
+
   return {
+    idByUniversalIdentifier: Object.fromEntries(
+      updatedIdByUniversalIdentifierEntries,
+    ),
     byId: removePropertiesFromRecord(flatObjectMetadataMaps.byId, [
       objectMetadataId,
     ]),

packages/twenty-server/src/engine/metadata-modules/flat-object-metadata-maps/utils/from-flat-object-metadata-to-flat-object-metadata-with-flat-field-maps.util.ts

@@ -24,6 +24,15 @@ export const fromObjectMetadataMapsToFlatObjectMetadataMaps = (
     }, initialAccumulator);
 
   return {
+    idByUniversalIdentifier: Object.values(objectMetadataMaps.byId)
+      .filter(isDefined)
+      .reduce(
+        (acc, objectMetadata) => ({
+          ...acc,
+          [objectMetadata.standardId ?? objectMetadata.id]: objectMetadata.id,
+        }),
+        {},
+      ),
     idByNameSingular: objectMetadataMaps.idByNameSingular,
     byId,
   };

packages/twenty-server/src/engine/metadata-modules/flat-object-metadata/services/workspace-flat-object-metadata-map-cache.service.ts

@@ -0,0 +1,59 @@
+import { Injectable } from '@nestjs/common';
+
+import { InjectCacheStorage } from 'src/engine/core-modules/cache-storage/decorators/cache-storage.decorator';
+import { CacheStorageService } from 'src/engine/core-modules/cache-storage/services/cache-storage.service';
+import { CacheStorageNamespace } from 'src/engine/core-modules/cache-storage/types/cache-storage-namespace.enum';
+import { FlatObjectMetadataMaps } from 'src/engine/metadata-modules/flat-object-metadata-maps/types/flat-object-metadata-maps.type';
+import { WorkspaceMetadataCacheService } from 'src/engine/metadata-modules/workspace-metadata-cache/services/workspace-metadata-cache.service';
+import { WorkspaceMetadataVersionService } from 'src/engine/metadata-modules/workspace-metadata-version/services/workspace-metadata-version.service';
+import { WorkspacePermissionsCacheService } from 'src/engine/metadata-modules/workspace-permissions-cache/workspace-permissions-cache.service';
+import { WorkspaceFlatMapCache } from 'src/engine/workspace-flat-map-cache/decorators/workspace-flat-map-cache.decorator';
+import { WorkspaceFlatMapCacheService } from 'src/engine/workspace-flat-map-cache/services/workspace-flat-map-cache.service';
+
+@Injectable()
+@WorkspaceFlatMapCache('flatObjectMetadataMaps')
+export class WorkspaceFlatObjectMetadataMapCacheService extends WorkspaceFlatMapCacheService<FlatObjectMetadataMaps> {
+  constructor(
+    @InjectCacheStorage(CacheStorageNamespace.EngineWorkspace)
+    cacheStorageService: CacheStorageService,
+    private workspaceMetadataCacheService: WorkspaceMetadataCacheService,
+    private workspaceMetadataVersionService: WorkspaceMetadataVersionService,
+    private readonly workspacePermissionsCacheService: WorkspacePermissionsCacheService,
+  ) {
+    super(cacheStorageService);
+  }
+
+  override async beforeInvalidateCache({
+    workspaceId,
+  }: {
+    workspaceId: string;
+  }): Promise<void> {
+    // Temporarily invalidating old cache too
+    await this.workspaceMetadataVersionService.incrementMetadataVersion(
+      workspaceId,
+    );
+    ///
+  }
+
+  override async afterInvalidateCache({
+    workspaceId,
+  }: {
+    workspaceId: string;
+  }): Promise<void> {
+    await this.workspacePermissionsCacheService.recomputeRolesPermissionsCache({
+      workspaceId,
+    });
+  }
+
+  protected async computeFlatMap({
+    workspaceId,
+  }: {
+    workspaceId: string;
+  }): Promise<FlatObjectMetadataMaps> {
+    return (
+      await this.workspaceMetadataCacheService.getExistingOrRecomputeFlatObjectMetadataMaps(
+        { workspaceId },
+      )
+    ).flatObjectMetadataMaps;
+  }
+}

packages/twenty-server/src/engine/metadata-modules/object-metadata/object-metadata-v2.service.ts

@@ -2,6 +2,7 @@ import { Injectable } from '@nestjs/common';
 
 import { isDefined } from 'twenty-shared/utils';
 
+import { WorkspaceManyOrAllFlatEntityMapsCacheService } from 'src/engine/core-modules/common/services/workspace-many-or-all-flat-entity-maps-cache.service.';
 import { addFlatFieldMetadataInFlatObjectMetadataMapsOrThrow } from 'src/engine/metadata-modules/flat-object-metadata-maps/utils/add-flat-field-metadata-in-flat-object-metadata-maps-or-throw.util';
 import { addFlatObjectMetadataToFlatObjectMetadataMapsOrThrow } from 'src/engine/metadata-modules/flat-object-metadata-maps/utils/add-flat-object-metadata-to-flat-object-metadata-maps-or-throw.util';
 import { deleteFieldFromFlatObjectMetadataMapsOrThrow } from 'src/engine/metadata-modules/flat-object-metadata-maps/utils/delete-field-from-flat-object-metadata-maps-or-throw.util';
@@ -23,7 +24,6 @@ import {
   ObjectMetadataException,
   ObjectMetadataExceptionCode,
 } from 'src/engine/metadata-modules/object-metadata/object-metadata.exception';
-import { WorkspaceMetadataCacheService } from 'src/engine/metadata-modules/workspace-metadata-cache/services/workspace-metadata-cache.service';
 import { WorkspacePermissionsCacheService } from 'src/engine/metadata-modules/workspace-permissions-cache/workspace-permissions-cache.service';
 import { WorkspaceMigrationBuilderExceptionV2 } from 'src/engine/workspace-manager/workspace-migration-v2/exceptions/workspace-migration-builder-exception-v2';
 import { WorkspaceMigrationValidateBuildAndRunService } from 'src/engine/workspace-manager/workspace-migration-v2/services/workspace-migration-validate-build-and-run-service';
@@ -31,7 +31,7 @@ import { WorkspaceMigrationValidateBuildAndRunService } from 'src/engine/workspa
 @Injectable()
 export class ObjectMetadataServiceV2 {
   constructor(
-    private readonly workspaceMetadataCacheService: WorkspaceMetadataCacheService,
+    private readonly flatEntityMapsCacheService: WorkspaceManyOrAllFlatEntityMapsCacheService,
     private readonly workspaceMigrationValidateBuildAndRunService: WorkspaceMigrationValidateBuildAndRunService,
     private readonly workspacePermissionsCacheService: WorkspacePermissionsCacheService,
   ) {}
@@ -44,9 +44,10 @@ export class ObjectMetadataServiceV2 {
     updateObjectInput: UpdateOneObjectInput;
   }): Promise<ObjectMetadataDTO> {
     const { flatObjectMetadataMaps: existingFlatObjectMetadataMaps } =
-      await this.workspaceMetadataCacheService.getExistingOrRecomputeFlatObjectMetadataMaps(
+      await this.flatEntityMapsCacheService.getOrRecomputeManyOrAllFlatEntityMaps(
         {
           workspaceId,
+          flatEntities: ['flatObjectMetadataMaps'],
         },
       );
 
@@ -81,6 +82,7 @@ export class ObjectMetadataServiceV2 {
         flatObjectMetadataMaps: fromFlatObjectMetadataMaps,
       }),
     );
+
     const validateAndBuildResult =
       await this.workspaceMigrationValidateBuildAndRunService.validateBuildAndRunWorkspaceMigration(
         {
@@ -106,9 +108,10 @@ export class ObjectMetadataServiceV2 {
     }
 
     const { flatObjectMetadataMaps: recomputedFlatObjectMetadataMaps } =
-      await this.workspaceMetadataCacheService.getExistingOrRecomputeFlatObjectMetadataMaps(
+      await this.flatEntityMapsCacheService.getOrRecomputeManyOrAllFlatEntityMaps(
         {
           workspaceId,
+          flatEntities: ['flatObjectMetadataMaps'],
         },
       );
 
@@ -143,9 +146,10 @@ export class ObjectMetadataServiceV2 {
     workspaceId: string;
   }): Promise<ObjectMetadataDTO> {
     const { flatObjectMetadataMaps: existingFlatObjectMetadataMaps } =
-      await this.workspaceMetadataCacheService.getExistingOrRecomputeFlatObjectMetadataMaps(
+      await this.flatEntityMapsCacheService.getOrRecomputeManyOrAllFlatEntityMaps(
         {
           workspaceId,
+          flatEntities: ['flatObjectMetadataMaps'],
         },
       );
 
@@ -225,9 +229,10 @@ export class ObjectMetadataServiceV2 {
     workspaceId: string;
   }): Promise<FlatObjectMetadata> {
     const { flatObjectMetadataMaps: existingFlatObjectMetadataMaps } =
-      await this.workspaceMetadataCacheService.getExistingOrRecomputeFlatObjectMetadataMaps(
+      await this.flatEntityMapsCacheService.getOrRecomputeManyOrAllFlatEntityMaps(
         {
           workspaceId,
+          flatEntities: ['flatObjectMetadataMaps'],
         },
       );
 
@@ -306,9 +311,10 @@ export class ObjectMetadataServiceV2 {
     }
 
     const { flatObjectMetadataMaps: recomputedFlatObjectMetadataMaps } =
-      await this.workspaceMetadataCacheService.getExistingOrRecomputeFlatObjectMetadataMaps(
+      await this.flatEntityMapsCacheService.getOrRecomputeManyOrAllFlatEntityMaps(
         {
           workspaceId,
+          flatEntities: ['flatObjectMetadataMaps'],
         },
       );
 

packages/twenty-server/src/engine/metadata-modules/object-metadata/object-metadata.module.ts

@@ -9,6 +9,7 @@ import {
 import { NestjsQueryTypeOrmModule } from '@ptc-org/nestjs-query-typeorm';
 
 import { TypeORMModule } from 'src/database/typeorm/typeorm.module';
+import { WorkspaceManyOrAllFlatEntityMapsCacheModule } from 'src/engine/core-modules/common/services/workspace-many-or-all-flat-entity-maps-cache.module';
 import { FeatureFlag } from 'src/engine/core-modules/feature-flag/feature-flag.entity';
 import { FeatureFlagModule } from 'src/engine/core-modules/feature-flag/feature-flag.module';
 import { ViewEntity } from 'src/engine/core-modules/view/entities/view.entity';
@@ -72,6 +73,7 @@ import { WorkspaceMigrationV2Module } from 'src/engine/workspace-manager/workspa
         FeatureFlagModule,
         WorkspaceMigrationV2Module,
         CoreViewModule,
+        WorkspaceManyOrAllFlatEntityMapsCacheModule,
       ],
       services: [
         ObjectMetadataService,