Extract field metadata build out of object metadata build (#14763)

# Introduction Extracting the legacy fields build and dispatch out of the object one to follow the generic flat entity build, also update caches entries for object ## Main tasks - flat field map cache - flat field builder - refactored the dispatch matrix to return flat entity maps instead of flat entity arrays - orchestrator aggregator - removing legacy code - making universal identifier aka standardId of standard field for custom object deterministically dynamic - perfs debug logs for v2 ## TODO - [x] Refactor the generic entity builder to be dependency flat maps in order to main foreign keys list in flat parent - [x] Refactor the flat object metadata to contain the array of related fields and avoid costy find object fields - [ ] Improve the create field handler to handle multiple field at once - [ ] Refactor the dispatch to embbed the comparison - [ ] Improve perf by extracting from elements out of existing - [ ] Fix the labelIdentifierId validators on object before field creation ( integ tests are in failing mode ) ## Debug logs snippet ```ts [EntityBuilder fieldMetadata] matrix computation: 0.027ms [EntityBuilder fieldMetadata] creation validation: 0.001ms [EntityBuilder fieldMetadata] deletion validation: 0.293ms [EntityBuilder fieldMetadata] update validation: 0.006ms [EntityBuilder fieldMetadata] entity processing: 0.363ms [EntityBuilder fieldMetadata] validateAndBuild: 0.455ms [EntityBuilder index] matrix computation: 0.005ms [EntityBuilder index] creation validation: 0.001ms [EntityBuilder index] deletion validation: 0.146ms [EntityBuilder index] update validation: 0.004ms [EntityBuilder index] entity processing: 0.199ms [EntityBuilder index] validateAndBuild: 0.228ms [Runner] Initial cache retrieval: 0.549ms [BaseWorkspaceMigrationRunnerActionHandlerService] delete_index executeForWorkspaceSchema: 11.665ms [BaseWorkspaceMigrationRunnerActionHandlerService] delete_index executeForMetadata: 12.864ms [BaseWorkspaceMigrationRunnerActionHandlerService] delete_field executeForWorkspaceSchema: 1.476ms [BaseWorkspaceMigrationRunnerActionHandlerService] delete_field executeForMetadata: 6.816ms [BaseWorkspaceMigrationRunnerActionHandlerService] delete_field executeForWorkspaceSchema: 0.062ms [BaseWorkspaceMigrationRunnerActionHandlerService] delete_field executeForMetadata: 0.889ms [Runner] Transaction execution: 23.434ms [Runner] Cache invalidation: 316.662ms [Runner] Total execution: 340.767ms ``` As you can see cache invalidation is way to long, we could replace the cache by the optimistic in the end
2025-10-02 18:07:34 +02:00
parent d4b83df3f5
commit 170f6d27c5
240 changed files with 11385 additions and 17744 deletions
@@ -1,4 +1,3 @@
-import { ATTACHMENT_FLAT_FIELDS_MOCK } from 'src/engine/metadata-modules/flat-field-metadata/__mocks__/attachment-flat-fields.mock';
 import { getFlatObjectMetadataMock } from 'src/engine/metadata-modules/flat-object-metadata/__mocks__/get-flat-object-metadata.mock';

 export const ATTACHMENT_FLAT_OBJECT_MOCK = getFlatObjectMetadataMock({
@@ -24,5 +23,4 @@ export const ATTACHMENT_FLAT_OBJECT_MOCK = getFlatObjectMetadataMock({
  isLabelSyncedWithName: false,
  workspaceId: '20202020-1c25-4d02-bf25-6aeccf7ea419',
  universalIdentifier: '20202020-bd3d-4c60-8dca-571c71d4447a',
-  flatFieldMetadatas: Object.values(ATTACHMENT_FLAT_FIELDS_MOCK),
 });
@@ -1,4 +1,3 @@
-import { COMPANY_FLAT_FIELDS_MOCK } from 'src/engine/metadata-modules/flat-field-metadata/__mocks__/company-flat-fields.mock';
 import { getFlatObjectMetadataMock } from 'src/engine/metadata-modules/flat-object-metadata/__mocks__/get-flat-object-metadata.mock';

 export const COMPANY_FLAT_OBJECT_MOCK = getFlatObjectMetadataMock({
@@ -24,5 +23,4 @@ export const COMPANY_FLAT_OBJECT_MOCK = getFlatObjectMetadataMock({
  isLabelSyncedWithName: false,
  workspaceId: '20202020-1c25-4d02-bf25-6aeccf7ea419',
  universalIdentifier: '20202020-b374-4779-a561-80086cb2e17f',
-  flatFieldMetadatas: Object.values(COMPANY_FLAT_FIELDS_MOCK),
 });
@@ -1,4 +1,3 @@
-import { FAVORITE_FLAT_FIELDS_MOCK } from 'src/engine/metadata-modules/flat-field-metadata/__mocks__/favorite-flat-fields.mock';
 import { getFlatObjectMetadataMock } from 'src/engine/metadata-modules/flat-object-metadata/__mocks__/get-flat-object-metadata.mock';

 export const FAVORITE_FLAT_OBJECT_MOCK = getFlatObjectMetadataMock({
@@ -24,5 +23,4 @@ export const FAVORITE_FLAT_OBJECT_MOCK = getFlatObjectMetadataMock({
  isLabelSyncedWithName: false,
  workspaceId: '20202020-1c25-4d02-bf25-6aeccf7ea419',
  universalIdentifier: '20202020-ab56-4e05-92a3-e2414a499860',
-  flatFieldMetadatas: Object.values(FAVORITE_FLAT_FIELDS_MOCK),
 });
@@ -1,4 +1,3 @@
-import { FAVORITE_FOLDER_FLAT_FIELDS_MOCK } from 'src/engine/metadata-modules/flat-field-metadata/__mocks__/favorite-folder-flat-fields.mock';
 import { getFlatObjectMetadataMock } from 'src/engine/metadata-modules/flat-object-metadata/__mocks__/get-flat-object-metadata.mock';

 export const FAVORITE_FOLDER_FLAT_OBJECT_MOCK = getFlatObjectMetadataMock({
@@ -24,5 +23,4 @@ export const FAVORITE_FOLDER_FLAT_OBJECT_MOCK = getFlatObjectMetadataMock({
  isLabelSyncedWithName: false,
  workspaceId: '20202020-1c25-4d02-bf25-6aeccf7ea419',
  universalIdentifier: '20202020-7cf8-401f-8211-a9587d27fd2d',
-  flatFieldMetadatas: Object.values(FAVORITE_FOLDER_FLAT_FIELDS_MOCK),
 });
@@ -0,0 +1,14 @@
+import { EMPTY_FLAT_ENTITY_MAPS } from 'src/engine/core-modules/common/constant/empty-flat-entity-maps.constant';
+import { addFlatEntityToFlatEntityMapsOrThrow } from 'src/engine/core-modules/common/utils/add-flat-entity-to-flat-entity-maps-or-throw.util';
+import { ALL_FLAT_OBJECT_METADATA_MOCKS } from 'src/engine/metadata-modules/flat-object-metadata/__mocks__/all-flat-object-metadatas.mock';
+
+export const FLAT_OBJECT_METADATA_MAPS_MOCKS = [
+  ...ALL_FLAT_OBJECT_METADATA_MOCKS,
+].reduce(
+  (flatObjectMaps, flatObjectMetadata) =>
+    addFlatEntityToFlatEntityMapsOrThrow({
+      flatEntity: flatObjectMetadata,
+      flatEntityMaps: flatObjectMaps,
+    }),
+  EMPTY_FLAT_ENTITY_MAPS,
+);
@@ -12,8 +12,8 @@ export const getFlatObjectMetadataMock = (
  const createdAt = '2024-01-01T00:00:00.000Z' as unknown as Date;

  return {
-    flatFieldMetadatas: [],
-
+    indexMetadataIds: [],
+    fieldMetadataIds: [],
    description: 'default flat object metadata description',
    icon: 'icon',
    id: faker.string.uuid(),
@@ -1,4 +1,3 @@
-import { NOTE_FLAT_FIELDS_MOCK } from 'src/engine/metadata-modules/flat-field-metadata/__mocks__/note-flat-fields.mock';
 import { getFlatObjectMetadataMock } from 'src/engine/metadata-modules/flat-object-metadata/__mocks__/get-flat-object-metadata.mock';

 export const NOTE_FLAT_OBJECT_MOCK = getFlatObjectMetadataMock({
@@ -24,6 +23,4 @@ export const NOTE_FLAT_OBJECT_MOCK = getFlatObjectMetadataMock({
  isLabelSyncedWithName: false,
  workspaceId: '20202020-1c25-4d02-bf25-6aeccf7ea419',
  universalIdentifier: '20202020-0b00-45cd-b6f6-6cd806fc6804',
-
-  flatFieldMetadatas: Object.values(NOTE_FLAT_FIELDS_MOCK),
 });
@@ -1,4 +1,3 @@
-import { NOTETARGET_FLAT_FIELDS_MOCK } from 'src/engine/metadata-modules/flat-field-metadata/__mocks__/notetarget-flat-fields.mock';
 import { getFlatObjectMetadataMock } from 'src/engine/metadata-modules/flat-object-metadata/__mocks__/get-flat-object-metadata.mock';

 export const NOTE_TARGET_FLAT_OBJECT_MOCK = getFlatObjectMetadataMock({
@@ -24,6 +23,4 @@ export const NOTE_TARGET_FLAT_OBJECT_MOCK = getFlatObjectMetadataMock({
  isLabelSyncedWithName: false,
  workspaceId: '20202020-1c25-4d02-bf25-6aeccf7ea419',
  universalIdentifier: '20202020-fff0-4b44-be82-bda313884400',
-
-  flatFieldMetadatas: Object.values(NOTETARGET_FLAT_FIELDS_MOCK),
 });
@@ -1,4 +1,3 @@
-import { OPPORTUNITY_FLAT_FIELDS_MOCK } from 'src/engine/metadata-modules/flat-field-metadata/__mocks__/opportunity-flat-fields.mock';
 import { getFlatObjectMetadataMock } from 'src/engine/metadata-modules/flat-object-metadata/__mocks__/get-flat-object-metadata.mock';

 export const OPPORTUNITY_FLAT_OBJECT_MOCK = getFlatObjectMetadataMock({
@@ -24,5 +23,4 @@ export const OPPORTUNITY_FLAT_OBJECT_MOCK = getFlatObjectMetadataMock({
  isLabelSyncedWithName: false,
  workspaceId: '20202020-1c25-4d02-bf25-6aeccf7ea419',
  universalIdentifier: '20202020-9549-49dd-b2b2-883999db8938',
-  flatFieldMetadatas: Object.values(OPPORTUNITY_FLAT_FIELDS_MOCK),
 });
@@ -1,4 +1,3 @@
-import { PERSON_FLAT_FIELDS_MOCK } from 'src/engine/metadata-modules/flat-field-metadata/__mocks__/person-flat-fields.mock';
 import { getFlatObjectMetadataMock } from 'src/engine/metadata-modules/flat-object-metadata/__mocks__/get-flat-object-metadata.mock';

 export const PERSON_FLAT_OBJECT_MOCK = getFlatObjectMetadataMock({
@@ -24,6 +23,4 @@ export const PERSON_FLAT_OBJECT_MOCK = getFlatObjectMetadataMock({
  isLabelSyncedWithName: false,
  workspaceId: '20202020-1c25-4d02-bf25-6aeccf7ea419',
  universalIdentifier: '20202020-e674-48e5-a542-72570eee7213',
-
-  flatFieldMetadatas: Object.values(PERSON_FLAT_FIELDS_MOCK),
 });
@@ -1,4 +1,3 @@
-import { PET_FLAT_FIELDS_MOCK } from 'src/engine/metadata-modules/flat-field-metadata/__mocks__/pet-flat-fields.mock';
 import { getFlatObjectMetadataMock } from 'src/engine/metadata-modules/flat-object-metadata/__mocks__/get-flat-object-metadata.mock';

 export const PET_FLAT_OBJECT_MOCK = getFlatObjectMetadataMock({
@@ -24,6 +23,4 @@ export const PET_FLAT_OBJECT_MOCK = getFlatObjectMetadataMock({
  isLabelSyncedWithName: false,
  workspaceId: '20202020-1c25-4d02-bf25-6aeccf7ea419',
  universalIdentifier: 'd34e0f07-1b8c-4de0-938e-599cf05e1f7f',
-
-  flatFieldMetadatas: Object.values(PET_FLAT_FIELDS_MOCK),
 });
@@ -1,4 +1,3 @@
-import { ROCKET_FLAT_FIELDS_MOCK } from 'src/engine/metadata-modules/flat-field-metadata/__mocks__/rocket-flat-fields.mock';
 import { getFlatObjectMetadataMock } from 'src/engine/metadata-modules/flat-object-metadata/__mocks__/get-flat-object-metadata.mock';

 export const ROCKET_FLAT_OBJECT_MOCK = getFlatObjectMetadataMock({
@@ -24,6 +23,4 @@ export const ROCKET_FLAT_OBJECT_MOCK = getFlatObjectMetadataMock({
  isLabelSyncedWithName: false,
  workspaceId: '20202020-1c25-4d02-bf25-6aeccf7ea419',
  universalIdentifier: 'd78ec657-74a4-4652-a350-1f44ff62970a',
-
-  flatFieldMetadatas: Object.values(ROCKET_FLAT_FIELDS_MOCK),
 });
@@ -1,4 +1,3 @@
-import { TASK_FLAT_FIELDS_MOCK } from 'src/engine/metadata-modules/flat-field-metadata/__mocks__/task-flat-fields.mock';
 import { getFlatObjectMetadataMock } from 'src/engine/metadata-modules/flat-object-metadata/__mocks__/get-flat-object-metadata.mock';

 export const TASK_FLAT_OBJECT_MOCK = getFlatObjectMetadataMock({
@@ -24,6 +23,4 @@ export const TASK_FLAT_OBJECT_MOCK = getFlatObjectMetadataMock({
  isLabelSyncedWithName: false,
  workspaceId: '20202020-1c25-4d02-bf25-6aeccf7ea419',
  universalIdentifier: '20202020-1ba1-48ba-bc83-ef7e5990ed10',
-
-  flatFieldMetadatas: Object.values(TASK_FLAT_FIELDS_MOCK),
 });
@@ -1,4 +1,3 @@
-import { TASKTARGET_FLAT_FIELDS_MOCK } from 'src/engine/metadata-modules/flat-field-metadata/__mocks__/tasktarget-flat-fields.mock';
 import { getFlatObjectMetadataMock } from 'src/engine/metadata-modules/flat-object-metadata/__mocks__/get-flat-object-metadata.mock';

 export const TASK_TARGET_FLAT_OBJECT_MOCK = getFlatObjectMetadataMock({
@@ -24,6 +23,4 @@ export const TASK_TARGET_FLAT_OBJECT_MOCK = getFlatObjectMetadataMock({
  isLabelSyncedWithName: false,
  workspaceId: '20202020-1c25-4d02-bf25-6aeccf7ea419',
  universalIdentifier: '20202020-5a9a-44e8-95df-771cd06d0fb1',
-
-  flatFieldMetadatas: Object.values(TASKTARGET_FLAT_FIELDS_MOCK),
 });
@@ -1,4 +1,3 @@
-import { TIMELINEACTIVITY_FLAT_FIELDS_MOCK } from 'src/engine/metadata-modules/flat-field-metadata/__mocks__/timelineactivity-flat-fields.mock';
 import { getFlatObjectMetadataMock } from 'src/engine/metadata-modules/flat-object-metadata/__mocks__/get-flat-object-metadata.mock';

 export const TIMELINE_ACTIVITY_FLAT_OBJECT_MOCK = getFlatObjectMetadataMock({
@@ -24,5 +23,4 @@ export const TIMELINE_ACTIVITY_FLAT_OBJECT_MOCK = getFlatObjectMetadataMock({
  isLabelSyncedWithName: false,
  workspaceId: '20202020-1c25-4d02-bf25-6aeccf7ea419',
  universalIdentifier: '20202020-6736-4337-b5c4-8b39fae325a5',
-  flatFieldMetadatas: Object.values(TIMELINEACTIVITY_FLAT_FIELDS_MOCK),
 });