Migrate metadata cache (#16030)

## Context
Deprecating legacy ObjectMetadata from cache in favor of flat entities.
Introducing utils to build byName/byNameSingular/byNamePlural in
isolated cases

## Next
- I had to introduce a util to build from flat to legacy
objectMetadataMaps, we should instead use flat maps directly when needed
(datasource, schema generation, etc)
- Deprecate metadata version in the cache
- Use the new cache strategy for flat entities with permissions and
feature flags and inject in the global datasource context

packages/twenty-server/src/engine/twenty-orm/factories/workspace-datasource.factory.ts

@@ -10,8 +10,11 @@ import { type FeatureFlagMap } from 'src/engine/core-modules/feature-flag/interf
 import { TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service';
 import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity';
 import { DataSourceService } from 'src/engine/metadata-modules/data-source/data-source.service';
+import { WorkspaceManyOrAllFlatEntityMapsCacheService } from 'src/engine/metadata-modules/flat-entity/services/workspace-many-or-all-flat-entity-maps-cache.service';
+import { buildObjectIdByNameMaps } from 'src/engine/metadata-modules/flat-object-metadata/utils/build-object-id-by-name-maps.util';
+import { buildObjectMetadataItemWithFieldMaps } from 'src/engine/metadata-modules/flat-object-metadata/utils/build-object-metadata-item-with-field-maps.util';
+import { type ObjectMetadataMaps } from 'src/engine/metadata-modules/types/object-metadata-maps';
 import { WorkspaceFeatureFlagsMapCacheService } from 'src/engine/metadata-modules/workspace-feature-flags-map-cache/workspace-feature-flags-map-cache.service';
-import { WorkspaceMetadataCacheService } from 'src/engine/metadata-modules/workspace-metadata-cache/services/workspace-metadata-cache.service';
 import { WorkspacePermissionsCacheStorageService } from 'src/engine/metadata-modules/workspace-permissions-cache/workspace-permissions-cache-storage.service';
 import {
   ROLES_PERMISSIONS,
@@ -45,7 +48,7 @@ export class WorkspaceDatasourceFactory {
     private readonly dataSourceService: DataSourceService,
     private readonly twentyConfigService: TwentyConfigService,
     private readonly workspaceCacheStorageService: WorkspaceCacheStorageService,
-    private readonly workspaceMetadataCacheService: WorkspaceMetadataCacheService,
+    private readonly workspaceManyOrAllFlatEntityMapsCacheService: WorkspaceManyOrAllFlatEntityMapsCacheService,
     private readonly entitySchemaFactory: EntitySchemaFactory,
     private readonly workspacePermissionsCacheService: WorkspacePermissionsCacheService,
     private readonly workspacePermissionsCacheStorageService: WorkspacePermissionsCacheStorageService,
@@ -126,15 +129,47 @@ export class WorkspaceDatasourceFactory {
           let cachedEntitySchemas: EntitySchema[];
 
           const {
-            objectMetadataMaps: cachedObjectMetadataMaps,
-            metadataVersion: metadataVersionForFinalUpToDateCheck,
+            flatObjectMetadataMaps,
+            flatFieldMetadataMaps,
+            flatIndexMaps,
           } =
-            await this.workspaceMetadataCacheService.getExistingOrRecomputeMetadataMaps(
+            await this.workspaceManyOrAllFlatEntityMapsCacheService.getOrRecomputeManyOrAllFlatEntityMaps(
               {
                 workspaceId,
+                flatMapsKeys: [
+                  'flatObjectMetadataMaps',
+                  'flatFieldMetadataMaps',
+                  'flatIndexMaps',
+                ],
               },
             );
 
+          const { idByNameSingular } = buildObjectIdByNameMaps(
+            flatObjectMetadataMaps,
+          );
+          const cachedObjectMetadataMaps: ObjectMetadataMaps = {
+            byId: {},
+            idByNameSingular,
+          };
+
+          for (const [id, flatObj] of Object.entries(
+            flatObjectMetadataMaps.byId,
+          )) {
+            if (isDefined(flatObj)) {
+              cachedObjectMetadataMaps.byId[id] =
+                buildObjectMetadataItemWithFieldMaps(
+                  flatObj,
+                  flatFieldMetadataMaps,
+                  flatIndexMaps,
+                );
+            }
+          }
+
+          const metadataVersionForFinalUpToDateCheck =
+            await this.workspaceCacheStorageService.getMetadataVersion(
+              workspaceId,
+            );
+
           if (
             metadataVersionForFinalUpToDateCheck !== dataSourceMetadataVersion
           ) {

packages/twenty-server/src/engine/twenty-orm/global-workspace-datasource/global-workspace-datasource.module.ts

@@ -5,8 +5,8 @@ import { FeatureFlagModule } from 'src/engine/core-modules/feature-flag/feature-
 import { TwentyConfigModule } from 'src/engine/core-modules/twenty-config/twenty-config.module';
 import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity';
 import { DataSourceModule } from 'src/engine/metadata-modules/data-source/data-source.module';
+import { WorkspaceManyOrAllFlatEntityMapsCacheModule } from 'src/engine/metadata-modules/flat-entity/services/workspace-many-or-all-flat-entity-maps-cache.module';
 import { WorkspaceFeatureFlagsMapCacheModule } from 'src/engine/metadata-modules/workspace-feature-flags-map-cache/workspace-feature-flags-map-cache.module';
-import { WorkspaceMetadataCacheModule } from 'src/engine/metadata-modules/workspace-metadata-cache/workspace-metadata-cache.module';
 import { WorkspacePermissionsCacheModule } from 'src/engine/metadata-modules/workspace-permissions-cache/workspace-permissions-cache.module';
 import { EntitySchemaColumnFactory } from 'src/engine/twenty-orm/factories/entity-schema-column.factory';
 import { EntitySchemaRelationFactory } from 'src/engine/twenty-orm/factories/entity-schema-relation.factory';
@@ -22,7 +22,7 @@ import { WorkspaceEventEmitterModule } from 'src/engine/workspace-event-emitter/
     TypeOrmModule.forFeature([WorkspaceEntity]),
     DataSourceModule,
     WorkspaceCacheStorageModule,
-    WorkspaceMetadataCacheModule,
+    WorkspaceManyOrAllFlatEntityMapsCacheModule,
     WorkspacePermissionsCacheModule,
     WorkspaceFeatureFlagsMapCacheModule,
     FeatureFlagModule,

packages/twenty-server/src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager.ts

@@ -1,13 +1,15 @@
 import { Injectable, type Type } from '@nestjs/common';
 
 import { isDefined } from 'twenty-shared/utils';
-import { EntitySchema, type ObjectLiteral } from 'typeorm';
+import { EntitySchema, ObjectLiteral } from 'typeorm';
 
 import { WorkspaceAuthContext } from 'src/engine/api/common/interfaces/workspace-auth-context.interface';
 
+import { WorkspaceManyOrAllFlatEntityMapsCacheService } from 'src/engine/metadata-modules/flat-entity/services/workspace-many-or-all-flat-entity-maps-cache.service';
+import { buildObjectIdByNameMaps } from 'src/engine/metadata-modules/flat-object-metadata/utils/build-object-id-by-name-maps.util';
+import { buildObjectMetadataItemWithFieldMaps } from 'src/engine/metadata-modules/flat-object-metadata/utils/build-object-metadata-item-with-field-maps.util';
 import { ObjectMetadataMaps } from 'src/engine/metadata-modules/types/object-metadata-maps';
 import { WorkspaceFeatureFlagsMapCacheService } from 'src/engine/metadata-modules/workspace-feature-flags-map-cache/workspace-feature-flags-map-cache.service';
-import { WorkspaceMetadataCacheService } from 'src/engine/metadata-modules/workspace-metadata-cache/services/workspace-metadata-cache.service';
 import { WorkspacePermissionsCacheService } from 'src/engine/metadata-modules/workspace-permissions-cache/workspace-permissions-cache.service';
 import { EntitySchemaFactory } from 'src/engine/twenty-orm/factories/entity-schema.factory';
 import { GlobalWorkspaceDataSourceService } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-datasource.service';
@@ -24,10 +26,10 @@ import { convertClassNameToObjectMetadataName } from 'src/engine/workspace-manag
 export class GlobalWorkspaceOrmManager {
   constructor(
     private readonly globalWorkspaceDataSourceService: GlobalWorkspaceDataSourceService,
-    private readonly workspaceMetadataCacheService: WorkspaceMetadataCacheService,
+    private readonly workspaceManyOrAllFlatEntityMapsCacheService: WorkspaceManyOrAllFlatEntityMapsCacheService,
+    private readonly workspaceCacheStorageService: WorkspaceCacheStorageService,
     private readonly workspaceFeatureFlagsMapCacheService: WorkspaceFeatureFlagsMapCacheService,
     private readonly workspacePermissionsCacheService: WorkspacePermissionsCacheService,
-    private readonly workspaceCacheStorageService: WorkspaceCacheStorageService,
     private readonly entitySchemaFactory: EntitySchemaFactory,
   ) {}
 
@@ -84,13 +86,46 @@ export class GlobalWorkspaceOrmManager {
     authContext: WorkspaceAuthContext,
   ): Promise<WorkspaceContext> {
     const workspaceId = authContext.workspace.id;
-    const { objectMetadataMaps, metadataVersion } =
-      await this.workspaceMetadataCacheService.getExistingOrRecomputeMetadataMaps(
+
+    const { flatObjectMetadataMaps, flatFieldMetadataMaps, flatIndexMaps } =
+      await this.workspaceManyOrAllFlatEntityMapsCacheService.getOrRecomputeManyOrAllFlatEntityMaps(
         {
           workspaceId,
+          flatMapsKeys: [
+            'flatObjectMetadataMaps',
+            'flatFieldMetadataMaps',
+            'flatIndexMaps',
+          ],
         },
       );
 
+    const { idByNameSingular } = buildObjectIdByNameMaps(
+      flatObjectMetadataMaps,
+    );
+    const objectMetadataMaps: ObjectMetadataMaps = {
+      byId: {},
+      idByNameSingular,
+    };
+
+    for (const [id, flatObj] of Object.entries(flatObjectMetadataMaps.byId)) {
+      if (isDefined(flatObj)) {
+        objectMetadataMaps.byId[id] = buildObjectMetadataItemWithFieldMaps(
+          flatObj,
+          flatFieldMetadataMaps,
+          flatIndexMaps,
+        );
+      }
+    }
+
+    const metadataVersion =
+      await this.workspaceCacheStorageService.getMetadataVersion(workspaceId);
+
+    if (!isDefined(metadataVersion)) {
+      throw new Error(
+        `Metadata version not found for workspace ${workspaceId}`,
+      );
+    }
+
     const { data: featureFlagsMap } =
       await this.workspaceFeatureFlagsMapCacheService.getWorkspaceFeatureFlagsMapAndVersion(
         { workspaceId },

packages/twenty-server/src/engine/twenty-orm/twenty-orm.module.ts

@@ -5,11 +5,11 @@ import { FeatureFlagModule } from 'src/engine/core-modules/feature-flag/feature-
 import { TwentyConfigModule } from 'src/engine/core-modules/twenty-config/twenty-config.module';
 import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity';
 import { DataSourceModule } from 'src/engine/metadata-modules/data-source/data-source.module';
+import { WorkspaceManyOrAllFlatEntityMapsCacheModule } from 'src/engine/metadata-modules/flat-entity/services/workspace-many-or-all-flat-entity-maps-cache.module';
 import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity';
 import { PermissionsModule } from 'src/engine/metadata-modules/permissions/permissions.module';
 import { RoleTargetsEntity } from 'src/engine/metadata-modules/role/role-targets.entity';
 import { WorkspaceFeatureFlagsMapCacheModule } from 'src/engine/metadata-modules/workspace-feature-flags-map-cache/workspace-feature-flags-map-cache.module';
-import { WorkspaceMetadataCacheModule } from 'src/engine/metadata-modules/workspace-metadata-cache/workspace-metadata-cache.module';
 import { WorkspacePermissionsCacheModule } from 'src/engine/metadata-modules/workspace-permissions-cache/workspace-permissions-cache.module';
 import { entitySchemaFactories } from 'src/engine/twenty-orm/factories';
 import { EntitySchemaFactory } from 'src/engine/twenty-orm/factories/entity-schema.factory';
@@ -30,7 +30,7 @@ import { PgPoolSharedModule } from './pg-shared-pool/pg-shared-pool.module';
     ]),
     DataSourceModule,
     WorkspaceCacheStorageModule,
-    WorkspaceMetadataCacheModule,
+    WorkspaceManyOrAllFlatEntityMapsCacheModule,
     PermissionsModule,
     WorkspaceFeatureFlagsMapCacheModule,
     WorkspacePermissionsCacheModule,