Extract field metadata build out of object metadata build (#14763)

# Introduction

Extracting the legacy fields build and dispatch out of the object one to
follow the generic flat entity build, also update caches entries for
object

## Main tasks
- flat field map cache
- flat field builder
- refactored the dispatch matrix to return flat entity maps instead of
flat entity arrays
- orchestrator aggregator
- removing legacy code
- making universal identifier aka standardId of standard field for
custom object deterministically dynamic
- perfs debug logs for v2

## TODO
- [x] Refactor the generic entity builder to be dependency flat maps in
order to main foreign keys list in flat parent
- [x] Refactor the flat object metadata to contain the array of related
fields and avoid costy find object fields
- [ ] Improve the create field handler to handle multiple field at once
- [ ] Refactor the dispatch to embbed the comparison
- [ ] Improve perf by extracting from elements out of existing
- [ ] Fix the labelIdentifierId validators on object before field
creation ( integ tests are in failing mode )

## Debug logs snippet

```ts
[EntityBuilder fieldMetadata] matrix computation: 0.027ms
[EntityBuilder fieldMetadata] creation validation: 0.001ms
[EntityBuilder fieldMetadata] deletion validation: 0.293ms
[EntityBuilder fieldMetadata] update validation: 0.006ms
[EntityBuilder fieldMetadata] entity processing: 0.363ms
[EntityBuilder fieldMetadata] validateAndBuild: 0.455ms
[EntityBuilder index] matrix computation: 0.005ms
[EntityBuilder index] creation validation: 0.001ms
[EntityBuilder index] deletion validation: 0.146ms
[EntityBuilder index] update validation: 0.004ms
[EntityBuilder index] entity processing: 0.199ms
[EntityBuilder index] validateAndBuild: 0.228ms
[Runner] Initial cache retrieval: 0.549ms
[BaseWorkspaceMigrationRunnerActionHandlerService] delete_index executeForWorkspaceSchema: 11.665ms
[BaseWorkspaceMigrationRunnerActionHandlerService] delete_index executeForMetadata: 12.864ms
[BaseWorkspaceMigrationRunnerActionHandlerService] delete_field executeForWorkspaceSchema: 1.476ms
[BaseWorkspaceMigrationRunnerActionHandlerService] delete_field executeForMetadata: 6.816ms
[BaseWorkspaceMigrationRunnerActionHandlerService] delete_field executeForWorkspaceSchema: 0.062ms
[BaseWorkspaceMigrationRunnerActionHandlerService] delete_field executeForMetadata: 0.889ms
[Runner] Transaction execution: 23.434ms
[Runner] Cache invalidation: 316.662ms
[Runner] Total execution: 340.767ms
```
As you can see cache invalidation is way to long, we could replace the
cache by the optimistic in the end

packages/twenty-server/src/engine/core-modules/application/application-sync.service.ts

@@ -15,14 +15,13 @@ import {
   ServerlessFunctionManifest,
   ServerlessFunctionTriggerManifest,
 } from 'src/engine/core-modules/application/types/application.types';
-import { WorkspaceManyOrAllFlatEntityMapsCacheService } from 'src/engine/core-modules/common/services/workspace-many-or-all-flat-entity-maps-cache.service.';
+import { WorkspaceManyOrAllFlatEntityMapsCacheService } from 'src/engine/core-modules/common/services/workspace-many-or-all-flat-entity-maps-cache.service';
 import { AgentService } from 'src/engine/metadata-modules/agent/agent.service';
+import { CronTriggerV2Service } from 'src/engine/metadata-modules/cron-trigger/services/cron-trigger-v2.service';
+import { FlatCronTrigger } from 'src/engine/metadata-modules/cron-trigger/types/flat-cron-trigger.type';
 import { DataSourceService } from 'src/engine/metadata-modules/data-source/data-source.service';
 import { DatabaseEventTriggerV2Service } from 'src/engine/metadata-modules/database-event-trigger/services/database-event-trigger-v2.service';
 import { FlatDatabaseEventTrigger } from 'src/engine/metadata-modules/database-event-trigger/types/flat-database-event-trigger.type';
-import { CronTriggerV2Service } from 'src/engine/metadata-modules/cron-trigger/services/cron-trigger-v2.service';
-import { FlatCronTrigger } from 'src/engine/metadata-modules/cron-trigger/types/flat-cron-trigger.type';
-import type { FlatObjectMetadataWithFlatFieldMaps } from 'src/engine/metadata-modules/flat-object-metadata-maps/types/flat-object-metadata-with-flat-field-metadata-maps.type';
 import { ObjectMetadataServiceV2 } from 'src/engine/metadata-modules/object-metadata/object-metadata-v2.service';
 import { ServerlessFunctionLayerService } from 'src/engine/metadata-modules/serverless-function-layer/serverless-function-layer.service';
 import { ServerlessFunctionV2Service } from 'src/engine/metadata-modules/serverless-function/services/serverless-function-v2.service';
@@ -196,7 +195,9 @@ export class ApplicationSyncService {
       existingFlatObjectMetadataMaps.byId,
     ).filter(
       (obj) => isDefined(obj) && obj.applicationId === applicationId,
-    ) as FlatObjectMetadataWithFlatFieldMaps[];
+      // TODO handle when migrating to trinite usage
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    ) as any[];
 
     const objectsToSyncStandardIds = objectsToSync.map((obj) => obj.standardId);
 

packages/twenty-server/src/engine/core-modules/common/constant/all-flat-entity-maps-properties.constant.ts

@@ -8,4 +8,5 @@ export const ALL_FLAT_ENTITY_MAPS_PROPERTIES = [
   'flatServerlessFunctionMaps',
   'flatDatabaseEventTriggerMaps',
   'flatCronTriggerMaps',
+  'flatFieldMetadataMaps',
 ] as const satisfies (keyof AllFlatEntityMaps)[];

packages/twenty-server/src/engine/core-modules/common/constant/empty-all-flat-entity-maps.constant.ts

@@ -2,10 +2,8 @@ import { EMPTY_FLAT_ENTITY_MAPS } from 'src/engine/core-modules/common/constant/
 import { type AllFlatEntityMaps } from 'src/engine/core-modules/common/types/all-flat-entity-maps.type';
 
 export const EMPTY_ALL_FLAT_ENTITY_MAPS = {
-  flatObjectMetadataMaps: {
-    ...EMPTY_FLAT_ENTITY_MAPS,
-    idByNameSingular: {},
-  },
+  flatObjectMetadataMaps: EMPTY_FLAT_ENTITY_MAPS,
+  flatFieldMetadataMaps: EMPTY_FLAT_ENTITY_MAPS,
   flatIndexMaps: EMPTY_FLAT_ENTITY_MAPS,
   flatViewFieldMaps: EMPTY_FLAT_ENTITY_MAPS,
   flatViewMaps: EMPTY_FLAT_ENTITY_MAPS,

packages/twenty-server/src/engine/core-modules/common/services/workspace-many-or-all-flat-entity-maps-cache.module.ts

@@ -1,6 +1,6 @@
 import { Module } from '@nestjs/common';
 
-import { WorkspaceManyOrAllFlatEntityMapsCacheService } from 'src/engine/core-modules/common/services/workspace-many-or-all-flat-entity-maps-cache.service.';
+import { WorkspaceManyOrAllFlatEntityMapsCacheService } from 'src/engine/core-modules/common/services/workspace-many-or-all-flat-entity-maps-cache.service';
 import { WorkspaceFlatMapCacheModule } from 'src/engine/workspace-flat-map-cache/workspace-flat-map-cache.module';
 
 @Module({

packages/twenty-server/src/engine/core-modules/common/types/all-flat-entities-by-metadata-engine-name.type.ts

@@ -2,12 +2,13 @@ import { type FlatViewField } from 'src/engine/core-modules/view/flat-view/types
 import { type FlatView } from 'src/engine/core-modules/view/flat-view/types/flat-view.type';
 import { type FlatCronTrigger } from 'src/engine/metadata-modules/cron-trigger/types/flat-cron-trigger.type';
 import { type FlatDatabaseEventTrigger } from 'src/engine/metadata-modules/database-event-trigger/types/flat-database-event-trigger.type';
+import { type FlatFieldMetadata } from 'src/engine/metadata-modules/flat-field-metadata/types/flat-field-metadata.type';
 import { type FlatIndexMetadata } from 'src/engine/metadata-modules/flat-index-metadata/types/flat-index-metadata.type';
 import { type FlatObjectMetadata } from 'src/engine/metadata-modules/flat-object-metadata/types/flat-object-metadata.type';
 import { type FlatServerlessFunction } from 'src/engine/metadata-modules/serverless-function/types/flat-serverless-function.type';
 
 export type AllFlatEntitiesByMetadataEngineName = {
-  //   flatFieldMetadata: FlatFieldMetadata;
+  fieldMetadata: FlatFieldMetadata;
   objectMetadata: FlatObjectMetadata;
   view: FlatView;
   viewField: FlatViewField;

packages/twenty-server/src/engine/core-modules/common/types/all-flat-entity-maps.type.ts

@@ -1,14 +1,9 @@
 import { type AllFlatEntitiesByMetadataEngineName } from 'src/engine/core-modules/common/types/all-flat-entities-by-metadata-engine-name.type';
 import { type FlatEntityMaps } from 'src/engine/core-modules/common/types/flat-entity-maps.type';
-import { type FlatObjectMetadataMaps } from 'src/engine/metadata-modules/flat-object-metadata-maps/types/flat-object-metadata-maps.type';
+import { type FromMetadataEngineNameToFlatEntityMapsKey } from 'src/engine/core-modules/common/types/from-metadata-engine-name-to-flat-entity-maps-key.type';
 
-export type AllFlatEntityMaps = Omit<
-  {
-    [P in keyof AllFlatEntitiesByMetadataEngineName as `flat${Capitalize<P>}Maps`]: FlatEntityMaps<
-      AllFlatEntitiesByMetadataEngineName[P]
-    >;
-  },
-  'flatObjectMetadataMaps'
-> & {
-  flatObjectMetadataMaps: FlatObjectMetadataMaps;
+export type AllFlatEntityMaps = {
+  [P in keyof AllFlatEntitiesByMetadataEngineName as FromMetadataEngineNameToFlatEntityMapsKey<P>]: FlatEntityMaps<
+    AllFlatEntitiesByMetadataEngineName[P]
+  >;
 };

packages/twenty-server/src/engine/core-modules/common/types/from-metadata-engine-name-to-flat-entity-maps-key.type.ts

@@ -0,0 +1,5 @@
+import { type AllFlatEntitiesByMetadataEngineName } from 'src/engine/core-modules/common/types/all-flat-entities-by-metadata-engine-name.type';
+
+export type FromMetadataEngineNameToFlatEntityMapsKey<
+  P extends keyof AllFlatEntitiesByMetadataEngineName,
+> = `flat${Capitalize<P>}Maps`;

packages/twenty-server/src/engine/core-modules/logger/logger.service.ts

@@ -1,4 +1,5 @@
 import {
+  ConsoleLogger,
   Inject,
   Injectable,
   type LogLevel,
@@ -7,9 +8,15 @@ import {
 
 import { LOGGER_DRIVER } from 'src/engine/core-modules/logger/logger.constants';
 
+type LoggerDriverType = ConsoleLogger & {
+  options?: {
+    logLevels?: LogLevel[];
+  };
+};
+
 @Injectable()
 export class LoggerService implements LoggerServiceInterface {
-  constructor(@Inject(LOGGER_DRIVER) private driver: LoggerServiceInterface) {}
+  constructor(@Inject(LOGGER_DRIVER) private driver: LoggerDriverType) {}
 
   // eslint-disable-next-line @typescript-eslint/no-explicit-any
   log(message: any, category: string, ...optionalParams: any[]) {
@@ -51,4 +58,18 @@ export class LoggerService implements LoggerServiceInterface {
   setLogLevels(levels: LogLevel[]) {
     this.driver.setLogLevels?.apply(this.driver, [levels]);
   }
+
+  time(category: string, label: string) {
+    if (this.driver.options.logLevels?.includes('debug')) {
+      // eslint-disable-next-line no-console
+      console.time(`[${category}] ${label}`);
+    }
+  }
+
+  timeEnd(category: string, label: string) {
+    if (this.driver.options.logLevels?.includes('debug')) {
+      // eslint-disable-next-line no-console
+      console.timeEnd(`[${category}] ${label}`);
+    }
+  }
 }

packages/twenty-server/src/engine/core-modules/view/services/view-field-v2.service.ts

@@ -2,7 +2,7 @@ import { Injectable } from '@nestjs/common';
 
 import { isDefined } from 'twenty-shared/utils';
 
-import { WorkspaceManyOrAllFlatEntityMapsCacheService } from 'src/engine/core-modules/common/services/workspace-many-or-all-flat-entity-maps-cache.service.';
+import { WorkspaceManyOrAllFlatEntityMapsCacheService } from 'src/engine/core-modules/common/services/workspace-many-or-all-flat-entity-maps-cache.service';
 import { addFlatEntityToFlatEntityMapsOrThrow } from 'src/engine/core-modules/common/utils/add-flat-entity-to-flat-entity-maps-or-throw.util';
 import { deleteFlatEntityFromFlatEntityMapsOrThrow } from 'src/engine/core-modules/common/utils/delete-flat-entity-from-flat-entity-maps-or-throw.util';
 import { findFlatEntityByIdInFlatEntityMapsOrThrow } from 'src/engine/core-modules/common/utils/find-flat-entity-by-id-in-flat-entity-maps-or-throw.util';
@@ -35,16 +35,16 @@ export class ViewFieldV2Service {
     workspaceId: string;
   }): Promise<ViewFieldDTO> {
     const {
-      flatObjectMetadataMaps,
       flatViewFieldMaps: existingFlatViewFieldMaps,
-      flatViewMaps: existingFlatViewMaps,
+      flatViewMaps,
+      flatFieldMetadataMaps,
     } = await this.flatEntityMapsCacheService.getOrRecomputeManyOrAllFlatEntityMaps(
       {
         workspaceId,
         flatEntities: [
-          'flatObjectMetadataMaps',
           'flatViewFieldMaps',
           'flatViewMaps',
+          'flatFieldMetadataMaps',
         ],
       },
     );
@@ -70,8 +70,8 @@ export class ViewFieldV2Service {
             },
           },
           dependencyAllFlatEntityMaps: {
-            flatObjectMetadataMaps,
-            flatViewMaps: existingFlatViewMaps,
+            flatFieldMetadataMaps,
+            flatViewMaps,
           },
           buildOptions: {
             isSystemBuild: false,

packages/twenty-server/src/engine/core-modules/view/services/view-v2.service.ts

@@ -2,7 +2,7 @@ import { Injectable } from '@nestjs/common';
 
 import { isDefined } from 'twenty-shared/utils';
 
-import { WorkspaceManyOrAllFlatEntityMapsCacheService } from 'src/engine/core-modules/common/services/workspace-many-or-all-flat-entity-maps-cache.service.';
+import { WorkspaceManyOrAllFlatEntityMapsCacheService } from 'src/engine/core-modules/common/services/workspace-many-or-all-flat-entity-maps-cache.service';
 import { addFlatEntityToFlatEntityMapsOrThrow } from 'src/engine/core-modules/common/utils/add-flat-entity-to-flat-entity-maps-or-throw.util';
 import { deleteFlatEntityFromFlatEntityMapsOrThrow } from 'src/engine/core-modules/common/utils/delete-flat-entity-from-flat-entity-maps-or-throw.util';
 import { findFlatEntityByIdInFlatEntityMapsOrThrow } from 'src/engine/core-modules/common/utils/find-flat-entity-by-id-in-flat-entity-maps-or-throw.util';