Migrate metadata cache (#16030)

## Context
Deprecating legacy ObjectMetadata from cache in favor of flat entities.
Introducing utils to build byName/byNameSingular/byNamePlural in
isolated cases

## Next
- I had to introduce a util to build from flat to legacy
objectMetadataMaps, we should instead use flat maps directly when needed
(datasource, schema generation, etc)
- Deprecate metadata version in the cache
- Use the new cache strategy for flat entities with permissions and
feature flags and inject in the global datasource context

packages/twenty-server/src/engine/api/graphql/__tests__/workspace.factory.spec.ts

@@ -8,7 +8,7 @@ import { FeatureFlagService } from 'src/engine/core-modules/feature-flag/service
 import { TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service';
 import { DataSourceService } from 'src/engine/metadata-modules/data-source/data-source.service';
 import { ObjectMetadataServiceV2 } from 'src/engine/metadata-modules/object-metadata/object-metadata-v2.service';
-import { WorkspaceMetadataCacheService } from 'src/engine/metadata-modules/workspace-metadata-cache/services/workspace-metadata-cache.service';
+import { WorkspaceManyOrAllFlatEntityMapsCacheService } from 'src/engine/metadata-modules/flat-entity/services/workspace-many-or-all-flat-entity-maps-cache.service';
 import { WorkspaceCacheStorageService } from 'src/engine/workspace-cache-storage/workspace-cache-storage.service';
 
 describe('WorkspaceSchemaFactory', () => {
@@ -43,7 +43,7 @@ describe('WorkspaceSchemaFactory', () => {
           useValue: {},
         },
         {
-          provide: WorkspaceMetadataCacheService,
+          provide: WorkspaceManyOrAllFlatEntityMapsCacheService,
           useValue: {},
         },
         {

packages/twenty-server/src/engine/api/graphql/core-graphql-api.module.ts

@@ -5,7 +5,7 @@ import { WorkspaceResolverBuilderModule } from 'src/engine/api/graphql/workspace
 import { WorkspaceSchemaBuilderModule } from 'src/engine/api/graphql/workspace-schema-builder/workspace-schema-builder.module';
 import { FeatureFlagModule } from 'src/engine/core-modules/feature-flag/feature-flag.module';
 import { MetadataEngineModule } from 'src/engine/metadata-modules/metadata-engine.module';
-import { WorkspaceMetadataCacheModule } from 'src/engine/metadata-modules/workspace-metadata-cache/workspace-metadata-cache.module';
+import { WorkspaceManyOrAllFlatEntityMapsCacheModule } from 'src/engine/metadata-modules/flat-entity/services/workspace-many-or-all-flat-entity-maps-cache.module';
 import { WorkspaceCacheStorageModule } from 'src/engine/workspace-cache-storage/workspace-cache-storage.module';
 
 import { WorkspaceSchemaFactory } from './workspace-schema.factory';
@@ -16,7 +16,7 @@ import { WorkspaceSchemaFactory } from './workspace-schema.factory';
     WorkspaceSchemaBuilderModule,
     WorkspaceResolverBuilderModule,
     WorkspaceCacheStorageModule,
-    WorkspaceMetadataCacheModule,
+    WorkspaceManyOrAllFlatEntityMapsCacheModule,
     FeatureFlagModule,
   ],
   providers: [WorkspaceSchemaFactory, ScalarsExplorerService],

packages/twenty-server/src/engine/api/graphql/workspace-schema.factory.ts

@@ -17,10 +17,13 @@ import { type AuthContext } from 'src/engine/core-modules/auth/types/auth-contex
 import { FeatureFlagService } from 'src/engine/core-modules/feature-flag/services/feature-flag.service';
 import { DataSourceService } from 'src/engine/metadata-modules/data-source/data-source.service';
 import {
-  WorkspaceMetadataCacheException,
-  WorkspaceMetadataCacheExceptionCode,
-} from 'src/engine/metadata-modules/workspace-metadata-cache/exceptions/workspace-metadata-cache.exception';
-import { WorkspaceMetadataCacheService } from 'src/engine/metadata-modules/workspace-metadata-cache/services/workspace-metadata-cache.service';
+  FlatEntityMapsException,
+  FlatEntityMapsExceptionCode,
+} from 'src/engine/metadata-modules/flat-entity/exceptions/flat-entity-maps.exception';
+import { WorkspaceManyOrAllFlatEntityMapsCacheService } from 'src/engine/metadata-modules/flat-entity/services/workspace-many-or-all-flat-entity-maps-cache.service';
+import { buildObjectIdByNameMaps } from 'src/engine/metadata-modules/flat-object-metadata/utils/build-object-id-by-name-maps.util';
+import { buildObjectMetadataItemWithFieldMaps } from 'src/engine/metadata-modules/flat-object-metadata/utils/build-object-metadata-item-with-field-maps.util';
+import { type ObjectMetadataMaps } from 'src/engine/metadata-modules/types/object-metadata-maps';
 import { WorkspaceCacheStorageService } from 'src/engine/workspace-cache-storage/workspace-cache-storage.service';
 import { standardObjectMetadataDefinitions } from 'src/engine/workspace-manager/workspace-sync-metadata/standard-objects';
 import { shouldExcludeFromWorkspaceApi } from 'src/engine/workspace-manager/workspace-sync-metadata/utils/should-exclude-from-workspace-api.util';
@@ -33,7 +36,7 @@ export class WorkspaceSchemaFactory {
     private readonly workspaceGraphQLSchemaGenerator: WorkspaceGraphQLSchemaGenerator,
     private readonly workspaceResolverFactory: WorkspaceResolverFactory,
     private readonly workspaceCacheStorageService: WorkspaceCacheStorageService,
-    private readonly workspaceMetadataCacheService: WorkspaceMetadataCacheService,
+    private readonly workspaceManyOrAllFlatEntityMapsCacheService: WorkspaceManyOrAllFlatEntityMapsCacheService,
     private readonly featureFlagService: FeatureFlagService,
   ) {}
 
@@ -51,20 +54,6 @@ export class WorkspaceSchemaFactory {
       return new GraphQLSchema({});
     }
 
-    const { objectMetadataMaps, metadataVersion } =
-      await this.workspaceMetadataCacheService.getExistingOrRecomputeMetadataMaps(
-        {
-          workspaceId: authContext.workspace.id,
-        },
-      );
-
-    if (!objectMetadataMaps) {
-      throw new WorkspaceMetadataCacheException(
-        'Object metadata collection not found',
-        WorkspaceMetadataCacheExceptionCode.OBJECT_METADATA_COLLECTION_NOT_FOUND,
-      );
-    }
-
     const workspaceId = authContext.workspace.id;
 
     if (!workspaceId) {
@@ -74,9 +63,57 @@ export class WorkspaceSchemaFactory {
       );
     }
 
+    const { flatObjectMetadataMaps, flatFieldMetadataMaps, flatIndexMaps } =
+      await this.workspaceManyOrAllFlatEntityMapsCacheService.getOrRecomputeManyOrAllFlatEntityMaps(
+        {
+          workspaceId,
+          flatMapsKeys: [
+            'flatObjectMetadataMaps',
+            'flatFieldMetadataMaps',
+            'flatIndexMaps',
+          ],
+        },
+      );
+
+    if (!isDefined(flatObjectMetadataMaps)) {
+      throw new FlatEntityMapsException(
+        'Object metadata collection not found',
+        FlatEntityMapsExceptionCode.ENTITY_NOT_FOUND,
+      );
+    }
+
     const workspaceFeatureFlagsMap =
       await this.featureFlagService.getWorkspaceFeatureFlagsMap(workspaceId);
 
+    let metadataVersion =
+      await this.workspaceCacheStorageService.getMetadataVersion(workspaceId);
+
+    if (!isDefined(metadataVersion)) {
+      metadataVersion = authContext.workspace.metadataVersion ?? 0;
+      await this.workspaceCacheStorageService.setMetadataVersion(
+        workspaceId,
+        metadataVersion,
+      );
+    }
+
+    const { idByNameSingular } = buildObjectIdByNameMaps(
+      flatObjectMetadataMaps,
+    );
+    const objectMetadataMaps: ObjectMetadataMaps = {
+      byId: {},
+      idByNameSingular,
+    };
+
+    for (const [id, flatObj] of Object.entries(flatObjectMetadataMaps.byId)) {
+      if (isDefined(flatObj)) {
+        objectMetadataMaps.byId[id] = buildObjectMetadataItemWithFieldMaps(
+          flatObj,
+          flatFieldMetadataMaps,
+          flatIndexMaps,
+        );
+      }
+    }
+
     const objectMetadataCollection = Object.values(objectMetadataMaps.byId)
       .filter(isDefined)
       .map((objectMetadataItem) => ({

packages/twenty-server/src/engine/api/rest/core/handlers/rest-api-base.handler.ts

@@ -21,6 +21,9 @@ import { AccessTokenService } from 'src/engine/core-modules/auth/token/services/
 import { WorkspaceDomainsService } from 'src/engine/core-modules/domain/workspace-domains/services/workspace-domains.service';
 import { FeatureFlagService } from 'src/engine/core-modules/feature-flag/services/feature-flag.service';
 import { WorkspaceNotFoundDefaultError } from 'src/engine/core-modules/workspace/workspace.exception';
+import { WorkspaceManyOrAllFlatEntityMapsCacheService } from 'src/engine/metadata-modules/flat-entity/services/workspace-many-or-all-flat-entity-maps-cache.service';
+import { buildObjectIdByNameMaps } from 'src/engine/metadata-modules/flat-object-metadata/utils/build-object-id-by-name-maps.util';
+import { buildObjectMetadataItemWithFieldMaps } from 'src/engine/metadata-modules/flat-object-metadata/utils/build-object-metadata-item-with-field-maps.util';
 import {
   PermissionsException,
   PermissionsExceptionCode,
@@ -29,9 +32,6 @@ import {
 import { type ObjectMetadataItemWithFieldMaps } from 'src/engine/metadata-modules/types/object-metadata-item-with-field-maps';
 import { type ObjectMetadataMaps } from 'src/engine/metadata-modules/types/object-metadata-maps';
 import { UserRoleService } from 'src/engine/metadata-modules/user-role/user-role.service';
-import { getObjectMetadataMapItemByNamePlural } from 'src/engine/metadata-modules/utils/get-object-metadata-map-item-by-name-plural.util';
-import { getObjectMetadataMapItemByNameSingular } from 'src/engine/metadata-modules/utils/get-object-metadata-map-item-by-name-singular.util';
-import { WorkspaceMetadataCacheService } from 'src/engine/metadata-modules/workspace-metadata-cache/services/workspace-metadata-cache.service';
 import { WorkspacePermissionsCacheService } from 'src/engine/metadata-modules/workspace-permissions-cache/workspace-permissions-cache.service';
 import { TwentyORMManager } from 'src/engine/twenty-orm/twenty-orm.manager';
 import { WorkspaceCacheStorageService } from 'src/engine/workspace-cache-storage/workspace-cache-storage.service';
@@ -63,7 +63,7 @@ export abstract class RestApiBaseHandler {
   @Inject()
   protected readonly workspaceCacheStorageService: WorkspaceCacheStorageService;
   @Inject()
-  protected readonly workspaceMetadataCacheService: WorkspaceMetadataCacheService;
+  protected readonly workspaceManyOrAllFlatEntityMapsCacheService: WorkspaceManyOrAllFlatEntityMapsCacheService;
   @Inject()
   protected readonly apiKeyRoleService: ApiKeyRoleService;
   @Inject()
@@ -178,19 +178,31 @@ export abstract class RestApiBaseHandler {
       await this.workspaceCacheStorageService.getMetadataVersion(workspace.id);
 
     if (currentCacheVersion === undefined) {
-      await this.workspaceMetadataCacheService.recomputeMetadataCache({
-        workspaceId: workspace.id,
-      });
-
-      throw new BadRequestException('Metadata cache version not found');
+      if (isDefined(workspace.metadataVersion)) {
+        await this.workspaceCacheStorageService.setMetadataVersion(
+          workspace.id,
+          workspace.metadataVersion,
+        );
+      } else {
+        throw new BadRequestException(
+          'Workspace metadata version not found in database',
+        );
+      }
     }
-    const objectMetadataMaps =
-      await this.workspaceCacheStorageService.getObjectMetadataMaps(
-        workspace.id,
-        currentCacheVersion,
+
+    const { flatObjectMetadataMaps, flatFieldMetadataMaps, flatIndexMaps } =
+      await this.workspaceManyOrAllFlatEntityMapsCacheService.getOrRecomputeManyOrAllFlatEntityMaps(
+        {
+          workspaceId: workspace.id,
+          flatMapsKeys: [
+            'flatObjectMetadataMaps',
+            'flatFieldMetadataMaps',
+            'flatIndexMaps',
+          ],
+        },
       );
 
-    if (!objectMetadataMaps) {
+    if (!isDefined(flatObjectMetadataMaps)) {
       throw new BadRequestException(
         `No object was found for the workspace associated with this API key. You may generate a new one here ${this.workspaceDomainsService
           .buildWorkspaceURL({
@@ -201,21 +213,25 @@ export abstract class RestApiBaseHandler {
       );
     }
 
-    const objectMetadataItem = getObjectMetadataMapItemByNamePlural(
-      objectMetadataMaps,
-      parsedObject,
+    const { idByNameSingular, idByNamePlural } = buildObjectIdByNameMaps(
+      flatObjectMetadataMaps,
     );
 
-    if (!objectMetadataItem) {
-      const wrongObjectMetadataItem = getObjectMetadataMapItemByNameSingular(
-        objectMetadataMaps,
-        parsedObject,
-      );
+    let objectId = idByNamePlural[parsedObject];
+    let flatObjectMetadataItem = objectId
+      ? flatObjectMetadataMaps.byId[objectId]
+      : undefined;
+
+    if (!flatObjectMetadataItem) {
+      const wrongObjectId = idByNameSingular[parsedObject];
+      const wrongFlatObjectMetadataItem = wrongObjectId
+        ? flatObjectMetadataMaps.byId[wrongObjectId]
+        : undefined;
 
       let hint = 'eg: companies';
 
-      if (wrongObjectMetadataItem) {
-        hint = `Did you mean '${wrongObjectMetadataItem.namePlural}'?`;
+      if (wrongFlatObjectMetadataItem) {
+        hint = `Did you mean '${wrongFlatObjectMetadataItem.namePlural}'?`;
       }
 
       throw new BadRequestException(
@@ -226,6 +242,12 @@ export abstract class RestApiBaseHandler {
     const workspaceFeatureFlagsMap =
       await this.featureFlagService.getWorkspaceFeatureFlagsMap(workspace.id);
 
+    const objectMetadataItem = buildObjectMetadataItemWithFieldMaps(
+      flatObjectMetadataItem,
+      flatFieldMetadataMaps,
+      flatIndexMaps,
+    );
+
     // Check if this entity is workspace-gated and should be blocked from workspace API
     if (
       shouldExcludeFromWorkspaceApi(
@@ -239,6 +261,21 @@ export abstract class RestApiBaseHandler {
       );
     }
 
+    const objectMetadataMaps: ObjectMetadataMaps = {
+      byId: {},
+      idByNameSingular,
+    };
+
+    for (const [id, flatObj] of Object.entries(flatObjectMetadataMaps.byId)) {
+      if (isDefined(flatObj)) {
+        objectMetadataMaps.byId[id] = buildObjectMetadataItemWithFieldMaps(
+          flatObj,
+          flatFieldMetadataMaps,
+          flatIndexMaps,
+        );
+      }
+    }
+
     return {
       objectMetadataMaps,
       objectMetadataMapItem: objectMetadataItem,

packages/twenty-server/src/engine/api/rest/core/rest-api-core.module.ts

@@ -27,11 +27,12 @@ import { AuthModule } from 'src/engine/core-modules/auth/auth.module';
 import { WorkspaceDomainsModule } from 'src/engine/core-modules/domain/workspace-domains/workspace-domains.module';
 import { FeatureFlagModule } from 'src/engine/core-modules/feature-flag/feature-flag.module';
 import { RecordTransformerModule } from 'src/engine/core-modules/record-transformer/record-transformer.module';
+import { WorkspaceManyOrAllFlatEntityMapsCacheModule } from 'src/engine/metadata-modules/flat-entity/services/workspace-many-or-all-flat-entity-maps-cache.module';
 import { UserRoleModule } from 'src/engine/metadata-modules/user-role/user-role.module';
-import { WorkspaceMetadataCacheModule } from 'src/engine/metadata-modules/workspace-metadata-cache/workspace-metadata-cache.module';
 import { WorkspacePermissionsCacheModule } from 'src/engine/metadata-modules/workspace-permissions-cache/workspace-permissions-cache.module';
 import { TwentyORMModule } from 'src/engine/twenty-orm/twenty-orm.module';
 import { WorkspaceCacheStorageModule } from 'src/engine/workspace-cache-storage/workspace-cache-storage.module';
+import { WorkspaceFlatMapCacheModule } from 'src/engine/workspace-flat-map-cache/workspace-flat-map-cache.module';
 
 const restApiCoreResolvers = [
   RestApiCreateOneHandler,
@@ -62,7 +63,8 @@ const restApiCoreResolvers = [
     TwentyORMModule,
     RecordTransformerModule,
     WorkspacePermissionsCacheModule,
-    WorkspaceMetadataCacheModule,
+    WorkspaceManyOrAllFlatEntityMapsCacheModule,
+    WorkspaceFlatMapCacheModule,
     ActorModule,
     FeatureFlagModule,
     CoreCommonApiModule,