Deprecate object metadata maps in favor of flat entities (#16080)

## Context
Deprecating the old objectMetadataMap type in favour of split flat
entities to match with our new caching.
In the long run, trying to achieve:
- Better performance through caching
- Consistent data access patterns across the codebase
- Reduced database queries

Now that everything is based on flat entities, which are cached, we can
finish the refactoring of workspace context cache which should already
improve performances.
Then the last step will be to consume that new cache in the new global
datasource to get rid of the many workspace datasources stored in the
server

packages/twenty-server/src/modules/workflow/workflow-trigger/automated-trigger/listeners/__tests__/workflow-database-event-trigger.listener.spec.ts

@@ -1,13 +1,13 @@
 import { Test, type TestingModule } from '@nestjs/testing';
 
 import { MessageQueueService } from 'src/engine/core-modules/message-queue/services/message-queue.service';
+import { type FlatObjectMetadata } from 'src/engine/metadata-modules/flat-object-metadata/types/flat-object-metadata.type';
 import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager';
+import { type WorkspaceEventBatch } from 'src/engine/workspace-event-emitter/types/workspace-event-batch.type';
 import { AutomatedTriggerType } from 'src/modules/workflow/common/standard-objects/workflow-automated-trigger.workspace-entity';
 import { WorkflowCommonWorkspaceService } from 'src/modules/workflow/common/workspace-services/workflow-common.workspace-service';
 import { WorkflowDatabaseEventTriggerListener } from 'src/modules/workflow/workflow-trigger/automated-trigger/listeners/workflow-database-event-trigger.listener';
 import { WorkflowTriggerJob } from 'src/modules/workflow/workflow-trigger/jobs/workflow-trigger.job';
-import { getMockObjectMetadataEntity } from 'src/utils/__test__/get-object-metadata-entity.mock';
-import { getMockObjectMetadataItemWithFieldsMaps } from 'src/utils/__test__/get-object-metadata-item-with-fields-maps.mock';
 
 describe('WorkflowDatabaseEventTriggerListener', () => {
   let listener: WorkflowDatabaseEventTriggerListener;
@@ -18,6 +18,35 @@ describe('WorkflowDatabaseEventTriggerListener', () => {
     find: jest.fn(),
   };
 
+  const createMockFlatObjectMetadata = (
+    overrides: Partial<FlatObjectMetadata>,
+  ): FlatObjectMetadata =>
+    ({
+      id: 'test-object-metadata',
+      workspaceId: 'test-workspace',
+      nameSingular: 'testObject',
+      namePlural: 'testObjects',
+      labelSingular: 'Test Object',
+      labelPlural: 'Test Objects',
+      description: 'Test object for testing',
+      targetTableName: 'test_objects',
+      isSystem: false,
+      isCustom: false,
+      isActive: true,
+      isRemote: false,
+      isAuditLogged: true,
+      isSearchable: true,
+      createdAt: new Date(),
+      updatedAt: new Date(),
+      icon: 'Icon123',
+      universalIdentifier: 'test-object-metadata',
+      fieldMetadataIds: [],
+      indexMetadataIds: [],
+      viewIds: [],
+      applicationId: null,
+      ...overrides,
+    }) as FlatObjectMetadata;
+
   beforeEach(async () => {
     twentyORMGlobalManager = {
       getRepositoryForWorkspace: jest.fn().mockResolvedValue(mockRepository),
@@ -46,37 +75,10 @@ describe('WorkflowDatabaseEventTriggerListener', () => {
           provide: WorkflowCommonWorkspaceService,
           useValue: {
             getWorkflowById: jest.fn(),
-            getObjectMetadataItemWithFieldsMaps: jest.fn().mockResolvedValue({
-              objectMetadataMaps: {
-                byId: {
-                  'test-object-metadata': {
-                    nameSingular: 'testObject',
-                    namePlural: 'testObjects',
-                  },
-                },
-              },
-              objectMetadataItemWithFieldsMaps:
-                getMockObjectMetadataItemWithFieldsMaps({
-                  id: 'test-object-metadata',
-                  workspaceId: 'test-workspace',
-                  nameSingular: 'testObject',
-                  namePlural: 'testObjects',
-                  labelSingular: 'Test Object',
-                  labelPlural: 'Test Objects',
-                  description: 'Test object for testing',
-                  indexMetadatas: [],
-                  targetTableName: 'test_objects',
-                  isSystem: false,
-                  isCustom: false,
-                  isActive: true,
-                  isRemote: false,
-                  isAuditLogged: true,
-                  isSearchable: true,
-                  icon: 'Icon123',
-                  fieldIdByJoinColumnName: {},
-                  fieldsById: {},
-                  fieldIdByName: {},
-                }),
+            getObjectMetadataInfo: jest.fn().mockResolvedValue({
+              flatObjectMetadata: createMockFlatObjectMetadata({}),
+              flatObjectMetadataMaps: { byId: {}, byName: {} },
+              flatFieldMetadataMaps: { byId: {}, byName: {} },
             }),
           },
         },
@@ -93,30 +95,10 @@ describe('WorkflowDatabaseEventTriggerListener', () => {
     const databaseEventName = 'testEvent';
     const workflowId = 'test-workflow';
 
-    const mockPayload = {
+    const mockPayload: WorkspaceEventBatch<any> = {
       workspaceId,
       name: databaseEventName,
-      objectMetadata: getMockObjectMetadataEntity({
-        id: 'test-object-metadata',
-        workspaceId,
-        nameSingular: 'testObject',
-        namePlural: 'testObjects',
-        labelSingular: 'Test Object',
-        labelPlural: 'Test Objects',
-        description: 'Test object for testing',
-        targetTableName: 'test_objects',
-        isSystem: false,
-        isCustom: false,
-        isActive: true,
-        isRemote: false,
-        isAuditLogged: true,
-        isSearchable: true,
-        createdAt: new Date(),
-        updatedAt: new Date(),
-        fields: [],
-        indexMetadatas: [],
-        icon: 'Icon123',
-      }),
+      objectMetadata: createMockFlatObjectMetadata({}),
       events: [
         {
           recordId: 'test-record',
@@ -205,7 +187,7 @@ describe('WorkflowDatabaseEventTriggerListener', () => {
     });
 
     it('should handle create events correctly', async () => {
-      const createPayload = {
+      const createPayload: WorkspaceEventBatch<any> = {
         ...mockPayload,
         name: 'createEvent',
         events: [
@@ -242,7 +224,7 @@ describe('WorkflowDatabaseEventTriggerListener', () => {
     });
 
     it('should handle delete events correctly', async () => {
-      const deletePayload = {
+      const deletePayload: WorkspaceEventBatch<any> = {
         ...mockPayload,
         name: 'deleteEvent',
         events: [
@@ -279,7 +261,7 @@ describe('WorkflowDatabaseEventTriggerListener', () => {
     });
 
     it('should handle destroy events correctly', async () => {
-      const destroyPayload = {
+      const destroyPayload: WorkspaceEventBatch<any> = {
         ...mockPayload,
         name: 'destroyEvent',
         events: [
@@ -316,7 +298,7 @@ describe('WorkflowDatabaseEventTriggerListener', () => {
     });
 
     it('should handle multiple events in a batch', async () => {
-      const batchPayload = {
+      const batchPayload: WorkspaceEventBatch<any> = {
         ...mockPayload,
         events: [
           mockPayload.events[0],

packages/twenty-server/src/modules/workflow/workflow-trigger/automated-trigger/listeners/workflow-database-event-trigger.listener.ts

@@ -15,8 +15,11 @@ import { type ObjectRecordUpsertEvent } from 'src/engine/core-modules/event-emit
 import { InjectMessageQueue } from 'src/engine/core-modules/message-queue/decorators/message-queue.decorator';
 import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';
 import { MessageQueueService } from 'src/engine/core-modules/message-queue/services/message-queue.service';
-import { ObjectMetadataItemWithFieldMaps } from 'src/engine/metadata-modules/types/object-metadata-item-with-field-maps';
-import { ObjectMetadataMaps } from 'src/engine/metadata-modules/types/object-metadata-maps';
+import { type FlatEntityMaps } from 'src/engine/metadata-modules/flat-entity/types/flat-entity-maps.type';
+import { findFlatEntityByIdInFlatEntityMapsOrThrow } from 'src/engine/metadata-modules/flat-entity/utils/find-flat-entity-by-id-in-flat-entity-maps-or-throw.util';
+import { type FlatFieldMetadata } from 'src/engine/metadata-modules/flat-field-metadata/types/flat-field-metadata.type';
+import { buildFieldMapsFromFlatObjectMetadata } from 'src/engine/metadata-modules/flat-field-metadata/utils/build-field-maps-from-flat-object-metadata.util';
+import { type FlatObjectMetadata } from 'src/engine/metadata-modules/flat-object-metadata/types/flat-object-metadata.type';
 import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager';
 import { WorkspaceEventBatch } from 'src/engine/workspace-event-emitter/types/workspace-event-batch.type';
 import {
@@ -135,17 +138,21 @@ export class WorkflowDatabaseEventTriggerListener {
     payload: WorkspaceEventBatch<ObjectRecordCreateEvent>,
   ) {
     const workspaceId = payload.workspaceId;
-    const { objectMetadataMaps, objectMetadataItemWithFieldsMaps } =
-      await this.workflowCommonWorkspaceService.getObjectMetadataItemWithFieldsMaps(
-        payload.objectMetadata.nameSingular,
-        workspaceId,
-      );
+    const {
+      flatObjectMetadata,
+      flatObjectMetadataMaps,
+      flatFieldMetadataMaps,
+    } = await this.workflowCommonWorkspaceService.getObjectMetadataInfo(
+      payload.objectMetadata.nameSingular,
+      workspaceId,
+    );
 
     await this.enrichRecordsWithRelations({
       records: payload.events.map((event) => event.properties.after),
       workspaceId,
-      objectMetadataMaps,
-      objectMetadataItemWithFieldsMaps,
+      flatObjectMetadata,
+      flatObjectMetadataMaps,
+      flatFieldMetadataMaps,
     });
   }
 
@@ -153,23 +160,28 @@ export class WorkflowDatabaseEventTriggerListener {
     payload: WorkspaceEventBatch<ObjectRecordUpdateEvent>,
   ) {
     const workspaceId = payload.workspaceId;
-    const { objectMetadataMaps, objectMetadataItemWithFieldsMaps } =
-      await this.workflowCommonWorkspaceService.getObjectMetadataItemWithFieldsMaps(
-        payload.objectMetadata.nameSingular,
-        workspaceId,
-      );
+    const {
+      flatObjectMetadata,
+      flatObjectMetadataMaps,
+      flatFieldMetadataMaps,
+    } = await this.workflowCommonWorkspaceService.getObjectMetadataInfo(
+      payload.objectMetadata.nameSingular,
+      workspaceId,
+    );
 
     await this.enrichRecordsWithRelations({
       records: payload.events.map((event) => event.properties.before),
       workspaceId,
-      objectMetadataMaps,
-      objectMetadataItemWithFieldsMaps,
+      flatObjectMetadata,
+      flatObjectMetadataMaps,
+      flatFieldMetadataMaps,
     });
     await this.enrichRecordsWithRelations({
       records: payload.events.map((event) => event.properties.after),
       workspaceId,
-      objectMetadataMaps,
-      objectMetadataItemWithFieldsMaps,
+      flatObjectMetadata,
+      flatObjectMetadataMaps,
+      flatFieldMetadataMaps,
     });
   }
 
@@ -177,17 +189,21 @@ export class WorkflowDatabaseEventTriggerListener {
     payload: WorkspaceEventBatch<ObjectRecordDeleteEvent>,
   ) {
     const workspaceId = payload.workspaceId;
-    const { objectMetadataMaps, objectMetadataItemWithFieldsMaps } =
-      await this.workflowCommonWorkspaceService.getObjectMetadataItemWithFieldsMaps(
-        payload.objectMetadata.nameSingular,
-        workspaceId,
-      );
+    const {
+      flatObjectMetadata,
+      flatObjectMetadataMaps,
+      flatFieldMetadataMaps,
+    } = await this.workflowCommonWorkspaceService.getObjectMetadataInfo(
+      payload.objectMetadata.nameSingular,
+      workspaceId,
+    );
 
     await this.enrichRecordsWithRelations({
       records: payload.events.map((event) => event.properties.before),
       workspaceId,
-      objectMetadataMaps,
-      objectMetadataItemWithFieldsMaps,
+      flatObjectMetadata,
+      flatObjectMetadataMaps,
+      flatFieldMetadataMaps,
     });
   }
 
@@ -195,36 +211,50 @@ export class WorkflowDatabaseEventTriggerListener {
     payload: WorkspaceEventBatch<ObjectRecordDestroyEvent>,
   ) {
     const workspaceId = payload.workspaceId;
-    const { objectMetadataMaps, objectMetadataItemWithFieldsMaps } =
-      await this.workflowCommonWorkspaceService.getObjectMetadataItemWithFieldsMaps(
-        payload.objectMetadata.nameSingular,
-        workspaceId,
-      );
+    const {
+      flatObjectMetadata,
+      flatObjectMetadataMaps,
+      flatFieldMetadataMaps,
+    } = await this.workflowCommonWorkspaceService.getObjectMetadataInfo(
+      payload.objectMetadata.nameSingular,
+      workspaceId,
+    );
 
     await this.enrichRecordsWithRelations({
       records: payload.events.map((event) => event.properties.before),
-      objectMetadataMaps,
       workspaceId,
-      objectMetadataItemWithFieldsMaps,
+      flatObjectMetadata,
+      flatObjectMetadataMaps,
+      flatFieldMetadataMaps,
     });
   }
 
   private async enrichRecordsWithRelations({
     records,
     workspaceId,
-    objectMetadataMaps,
-    objectMetadataItemWithFieldsMaps,
+    flatObjectMetadata,
+    flatObjectMetadataMaps,
+    flatFieldMetadataMaps,
   }: {
     records: Partial<ObjectRecord>[];
     workspaceId: string;
-    objectMetadataMaps: ObjectMetadataMaps;
-    objectMetadataItemWithFieldsMaps: ObjectMetadataItemWithFieldMaps;
+    flatObjectMetadata: FlatObjectMetadata;
+    flatObjectMetadataMaps: FlatEntityMaps<FlatObjectMetadata>;
+    flatFieldMetadataMaps: FlatEntityMaps<FlatFieldMetadata>;
   }) {
+    const { fieldIdByJoinColumnName } = buildFieldMapsFromFlatObjectMetadata(
+      flatFieldMetadataMaps,
+      flatObjectMetadata,
+    );
+
     for (const [joinColumnName, joinFieldId] of Object.entries(
-      objectMetadataItemWithFieldsMaps.fieldIdByJoinColumnName,
+      fieldIdByJoinColumnName,
     )) {
-      const joinField =
-        objectMetadataItemWithFieldsMaps.fieldsById[joinFieldId];
+      const joinField = findFlatEntityByIdInFlatEntityMapsOrThrow({
+        flatEntityMaps: flatFieldMetadataMaps,
+        flatEntityId: joinFieldId,
+      });
+
       const joinRecordIds = records
         .map((record) => record[joinColumnName])
         .filter(isDefined);
@@ -240,7 +270,7 @@ export class WorkflowDatabaseEventTriggerListener {
       }
 
       const relatedObjectMetadataNameSingular =
-        objectMetadataMaps.byId[relatedObjectMetadataId]?.nameSingular;
+        flatObjectMetadataMaps.byId[relatedObjectMetadataId]?.nameSingular;
 
       if (!isDefined(relatedObjectMetadataNameSingular)) {
         continue;