Migrate metadata cache (#16030)

## Context
Deprecating legacy ObjectMetadata from cache in favor of flat entities.
Introducing utils to build byName/byNameSingular/byNamePlural in
isolated cases

## Next
- I had to introduce a util to build from flat to legacy
objectMetadataMaps, we should instead use flat maps directly when needed
(datasource, schema generation, etc)
- Deprecate metadata version in the cache
- Use the new cache strategy for flat entities with permissions and
feature flags and inject in the global datasource context

packages/twenty-server/src/engine/core-modules/actor/actor.module.ts

@@ -4,14 +4,14 @@ import { TypeOrmModule } from '@nestjs/typeorm';
 import { CreatedByCreateManyPreQueryHook } from 'src/engine/core-modules/actor/query-hooks/created-by.create-many.pre-query-hook';
 import { CreatedByCreateOnePreQueryHook } from 'src/engine/core-modules/actor/query-hooks/created-by.create-one.pre-query-hook';
 import { FieldMetadataEntity } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity';
-import { WorkspaceMetadataCacheModule } from 'src/engine/metadata-modules/workspace-metadata-cache/workspace-metadata-cache.module';
+import { WorkspaceManyOrAllFlatEntityMapsCacheModule } from 'src/engine/metadata-modules/flat-entity/services/workspace-many-or-all-flat-entity-maps-cache.module';
 
 import { CreatedByFromAuthContextService } from './services/created-by-from-auth-context.service';
 
 @Module({
   imports: [
     TypeOrmModule.forFeature([FieldMetadataEntity]),
-    WorkspaceMetadataCacheModule,
+    WorkspaceManyOrAllFlatEntityMapsCacheModule,
   ],
   providers: [
     CreatedByCreateManyPreQueryHook,

packages/twenty-server/src/engine/core-modules/actor/services/__tests__/created-by-from-auth-context.service.spec.ts

@@ -13,7 +13,7 @@ import { type AuthContext } from 'src/engine/core-modules/auth/types/auth-contex
 import { type UserEntity } from 'src/engine/core-modules/user/user.entity';
 import { type WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity';
 import { FieldMetadataEntity } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity';
-import { WorkspaceMetadataCacheService } from 'src/engine/metadata-modules/workspace-metadata-cache/services/workspace-metadata-cache.service';
+import { WorkspaceManyOrAllFlatEntityMapsCacheService } from 'src/engine/metadata-modules/flat-entity/services/workspace-many-or-all-flat-entity-maps-cache.service';
 import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager';
 import { type WorkspaceMemberWorkspaceEntity } from 'src/modules/workspace-member/standard-objects/workspace-member.workspace-entity';
 
@@ -59,29 +59,30 @@ describe('CreatedByFromAuthContextService', () => {
           },
         },
         {
-          provide: WorkspaceMetadataCacheService,
+          provide: WorkspaceManyOrAllFlatEntityMapsCacheService,
           useValue: {
-            getExistingOrRecomputeMetadataMaps: jest.fn().mockResolvedValue({
-              objectMetadataMaps: {
+            getOrRecomputeManyOrAllFlatEntityMaps: jest.fn().mockResolvedValue({
+              flatObjectMetadataMaps: {
                 byId: {
                   'person-id': {
                     id: 'person-id',
                     nameSingular: 'person',
-                    fieldsById: {
-                      createdBy: {
-                        id: 'createdBy-id',
-                        name: 'createdBy',
-                      },
-                    },
-                    fieldIdByName: {
-                      createdBy: 'createdBy-id',
-                    },
+                    fieldMetadataIds: ['createdBy-id'],
                   },
                 },
-                idByNameSingular: {
-                  person: 'person-id',
+              },
+              flatFieldMetadataMaps: {
+                byId: {
+                  'createdBy-id': {
+                    id: 'createdBy-id',
+                    name: 'createdBy',
+                    objectMetadataId: 'person-id',
+                  },
                 },
               },
+              flatIndexMaps: {
+                byId: {},
+              },
             }),
           },
         },

packages/twenty-server/src/engine/core-modules/actor/services/created-by-from-auth-context.service.ts

@@ -1,17 +1,15 @@
 import { Injectable, Logger } from '@nestjs/common';
-import { InjectRepository } from '@nestjs/typeorm';
 
-import { assertIsDefinedOrThrow, isDefined } from 'twenty-shared/utils';
-import { Repository } from 'typeorm';
 import { type ActorMetadata } from 'twenty-shared/types';
+import { assertIsDefinedOrThrow, isDefined } from 'twenty-shared/utils';
 
 import { buildCreatedByFromApiKey } from 'src/engine/core-modules/actor/utils/build-created-by-from-api-key.util';
 import { buildCreatedByFromFullNameMetadata } from 'src/engine/core-modules/actor/utils/build-created-by-from-full-name-metadata.util';
 import { type AuthContext } from 'src/engine/core-modules/auth/types/auth-context.type';
 import { WorkspaceNotFoundDefaultError } from 'src/engine/core-modules/workspace/workspace.exception';
-import { FieldMetadataEntity } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity';
-import { getObjectMetadataMapItemByNameSingular } from 'src/engine/metadata-modules/utils/get-object-metadata-map-item-by-name-singular.util';
-import { WorkspaceMetadataCacheService } from 'src/engine/metadata-modules/workspace-metadata-cache/services/workspace-metadata-cache.service';
+import { WorkspaceManyOrAllFlatEntityMapsCacheService } from 'src/engine/metadata-modules/flat-entity/services/workspace-many-or-all-flat-entity-maps-cache.service';
+import { buildFieldMapsForObject } from 'src/engine/metadata-modules/flat-field-metadata/utils/build-field-maps-for-object.util';
+import { buildObjectIdByNameMaps } from 'src/engine/metadata-modules/flat-object-metadata/utils/build-object-id-by-name-maps.util';
 import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager';
 import { WorkspaceMemberWorkspaceEntity } from 'src/modules/workspace-member/standard-objects/workspace-member.workspace-entity';
 
@@ -23,10 +21,8 @@ export class CreatedByFromAuthContextService {
   private readonly logger = new Logger(CreatedByFromAuthContextService.name);
 
   constructor(
-    @InjectRepository(FieldMetadataEntity)
-    private readonly fieldMetadataRepository: Repository<FieldMetadataEntity>,
     private readonly twentyORMGlobalManager: TwentyORMGlobalManager,
-    private readonly workspaceMetadataCacheService: WorkspaceMetadataCacheService,
+    private readonly flatEntityMapsCacheService: WorkspaceManyOrAllFlatEntityMapsCacheService,
   ) {}
 
   async injectCreatedBy(
@@ -38,10 +34,11 @@ export class CreatedByFromAuthContextService {
 
     assertIsDefinedOrThrow(workspace, WorkspaceNotFoundDefaultError);
 
-    const { objectMetadataMaps } =
-      await this.workspaceMetadataCacheService.getExistingOrRecomputeMetadataMaps(
+    const { flatObjectMetadataMaps, flatFieldMetadataMaps } =
+      await this.flatEntityMapsCacheService.getOrRecomputeManyOrAllFlatEntityMaps(
         {
           workspaceId: workspace.id,
+          flatMapsKeys: ['flatObjectMetadataMaps', 'flatFieldMetadataMaps'],
         },
       );
 
@@ -49,16 +46,24 @@ export class CreatedByFromAuthContextService {
       `Injecting createdBy from auth context for object ${objectMetadataNameSingular} and workspace ${workspace.id}`,
     );
 
-    const objectMetadata = getObjectMetadataMapItemByNameSingular(
-      objectMetadataMaps,
-      objectMetadataNameSingular,
+    const { idByNameSingular } = buildObjectIdByNameMaps(
+      flatObjectMetadataMaps,
     );
+    const objectId = idByNameSingular[objectMetadataNameSingular];
+    const objectMetadata = objectId
+      ? flatObjectMetadataMaps.byId[objectId]
+      : undefined;
+
+    const fieldIdByName = objectMetadata
+      ? buildFieldMapsForObject(flatFieldMetadataMaps, objectMetadata.id)
+          .fieldIdByName
+      : {};
 
     this.logger.log(
-      `Object metadata found with fields: ${Object.keys(objectMetadata?.fieldIdByName ?? {})}`,
+      `Object metadata found with fields: ${Object.keys(fieldIdByName)}`,
     );
 
-    if (!isDefined(objectMetadata?.fieldIdByName['createdBy'])) {
+    if (!isDefined(fieldIdByName['createdBy'])) {
       this.logger.log(
         `CreatedBy field not found in object metadata, skipping injection`,
       );

packages/twenty-server/src/engine/core-modules/search/search.module.ts

@@ -3,10 +3,10 @@ import { Module } from '@nestjs/common';
 import { FileModule } from 'src/engine/core-modules/file/file.module';
 import { SearchResolver } from 'src/engine/core-modules/search/search.resolver';
 import { SearchService } from 'src/engine/core-modules/search/services/search.service';
-import { WorkspaceCacheStorageModule } from 'src/engine/workspace-cache-storage/workspace-cache-storage.module';
+import { WorkspaceManyOrAllFlatEntityMapsCacheModule } from 'src/engine/metadata-modules/flat-entity/services/workspace-many-or-all-flat-entity-maps-cache.module';
 
 @Module({
-  imports: [FileModule, WorkspaceCacheStorageModule],
+  imports: [FileModule, WorkspaceManyOrAllFlatEntityMapsCacheModule],
   providers: [SearchResolver, SearchService],
 })
 export class SearchModule {}

packages/twenty-server/src/engine/core-modules/search/search.resolver.ts

@@ -13,7 +13,10 @@ import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.ent
 import { AuthWorkspace } from 'src/engine/decorators/auth/auth-workspace.decorator';
 import { CustomPermissionGuard } from 'src/engine/guards/custom-permission.guard';
 import { WorkspaceAuthGuard } from 'src/engine/guards/workspace-auth.guard';
-import { WorkspaceCacheStorageService } from 'src/engine/workspace-cache-storage/workspace-cache-storage.service';
+import { WorkspaceManyOrAllFlatEntityMapsCacheService } from 'src/engine/metadata-modules/flat-entity/services/workspace-many-or-all-flat-entity-maps-cache.service';
+import { buildObjectIdByNameMaps } from 'src/engine/metadata-modules/flat-object-metadata/utils/build-object-id-by-name-maps.util';
+import { buildObjectMetadataItemWithFieldMaps } from 'src/engine/metadata-modules/flat-object-metadata/utils/build-object-metadata-item-with-field-maps.util';
+import { type ObjectMetadataMaps } from 'src/engine/metadata-modules/types/object-metadata-maps';
 
 @Resolver()
 @UseFilters(SearchApiExceptionFilter, PreventNestToAutoLogGraphqlErrorsFilter)
@@ -22,7 +25,7 @@ import { WorkspaceCacheStorageService } from 'src/engine/workspace-cache-storage
 export class SearchResolver {
   constructor(
     private readonly searchService: SearchService,
-    private readonly workspaceCacheStorageService: WorkspaceCacheStorageService,
+    private readonly workspaceManyOrAllFlatEntityMapsCacheService: WorkspaceManyOrAllFlatEntityMapsCacheService,
   ) {}
 
   @Query(() => SearchResultConnectionDTO)
@@ -38,11 +41,36 @@ export class SearchResolver {
       after,
     }: SearchArgs,
   ) {
-    const objectMetadataMaps =
-      await this.workspaceCacheStorageService.getObjectMetadataMapsOrThrow(
-        workspace.id,
+    const { flatObjectMetadataMaps, flatFieldMetadataMaps, flatIndexMaps } =
+      await this.workspaceManyOrAllFlatEntityMapsCacheService.getOrRecomputeManyOrAllFlatEntityMaps(
+        {
+          workspaceId: workspace.id,
+          flatMapsKeys: [
+            'flatObjectMetadataMaps',
+            'flatFieldMetadataMaps',
+            'flatIndexMaps',
+          ],
+        },
       );
 
+    const { idByNameSingular } = buildObjectIdByNameMaps(
+      flatObjectMetadataMaps,
+    );
+    const objectMetadataMaps: ObjectMetadataMaps = {
+      byId: {},
+      idByNameSingular,
+    };
+
+    for (const [id, flatObj] of Object.entries(flatObjectMetadataMaps.byId)) {
+      if (isDefined(flatObj)) {
+        objectMetadataMaps.byId[id] = buildObjectMetadataItemWithFieldMaps(
+          flatObj,
+          flatFieldMetadataMaps,
+          flatIndexMaps,
+        );
+      }
+    }
+
     const filteredObjectMetadataItems =
       this.searchService.filterObjectMetadataItems({
         objectMetadataItemWithFieldMaps: Object.values(

packages/twenty-server/src/engine/core-modules/workspace/workspace.module.ts

@@ -35,7 +35,6 @@ import { WorkspaceManyOrAllFlatEntityMapsCacheModule } from 'src/engine/metadata
 import { PermissionsModule } from 'src/engine/metadata-modules/permissions/permissions.module';
 import { RoleModule } from 'src/engine/metadata-modules/role/role.module';
 import { ViewModule } from 'src/engine/metadata-modules/view/view.module';
-import { WorkspaceMetadataCacheModule } from 'src/engine/metadata-modules/workspace-metadata-cache/workspace-metadata-cache.module';
 import { WorkspaceCacheStorageModule } from 'src/engine/workspace-cache-storage/workspace-cache-storage.module';
 import { WorkspaceManagerModule } from 'src/engine/workspace-manager/workspace-manager.module';
 
@@ -50,7 +49,6 @@ import { WorkspaceManagerModule } from 'src/engine/workspace-manager/workspace-m
         FileModule,
         TokenModule,
         FileUploadModule,
-        WorkspaceMetadataCacheModule,
         NestjsQueryTypeOrmModule.forFeature([
           UserEntity,
           WorkspaceEntity,