Deprecate object metadata maps in favor of flat entities (#16080)

## Context Deprecating the old objectMetadataMap type in favour of split flat entities to match with our new caching. In the long run, trying to achieve: - Better performance through caching - Consistent data access patterns across the codebase - Reduced database queries Now that everything is based on flat entities, which are cached, we can finish the refactoring of workspace context cache which should already improve performances. Then the last step will be to consume that new cache in the new global datasource to get rid of the many workspace datasources stored in the server
2025-11-27 13:43:34 +01:00
parent 978c0acb90
commit 1607aebcc6
281 changed files with 6510 additions and 6627 deletions
@@ -1,6 +1,4 @@
-import { type FieldMetadataType } from 'twenty-shared/types';
-
-import { type FieldMetadataEntity } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity';
+import { type FlatFieldMetadata } from 'src/engine/metadata-modules/flat-field-metadata/types/flat-field-metadata.type';

 export type UniqueFieldCondition = [field: string, value: string];

@@ -11,7 +9,7 @@ export type RelationConnectQueryConfig = {
  recordToConnectConditions: UniqueConstraintCondition[];
  relationFieldName: string;
  connectFieldName: string;
-  uniqueConstraintFields: FieldMetadataEntity<FieldMetadataType>[];
+  uniqueConstraintFields: FlatFieldMetadata[];
  recordToConnectConditionByEntityIndex: {
    [entityIndex: number]: UniqueConstraintCondition;
  };
@@ -1,11 +1,16 @@
-import { type ObjectsPermissions } from 'twenty-shared/types';
+import {
+  type FieldMetadataType,
+  type ObjectsPermissions,
+} from 'twenty-shared/types';
 import { EntityManager } from 'typeorm';
 import { EntityPersistExecutor } from 'typeorm/persistence/EntityPersistExecutor';
 import { PlainObjectToDatabaseEntityTransformer } from 'typeorm/query-builder/transformer/PlainObjectToDatabaseEntityTransformer';

 import { type WorkspaceInternalContext } from 'src/engine/twenty-orm/interfaces/workspace-internal-context.interface';

-import { type ObjectMetadataItemWithFieldMaps } from 'src/engine/metadata-modules/types/object-metadata-item-with-field-maps';
+import { type FlatEntityMaps } from 'src/engine/metadata-modules/flat-entity/types/flat-entity-maps.type';
+import { type FlatFieldMetadata } from 'src/engine/metadata-modules/flat-field-metadata/types/flat-field-metadata.type';
+import { type FlatObjectMetadata } from 'src/engine/metadata-modules/flat-object-metadata/types/flat-object-metadata.type';
 import { type WorkspaceDataSource } from 'src/engine/twenty-orm/datasource/workspace.datasource';
 import { validateOperationIsPermittedOrThrow } from 'src/engine/twenty-orm/repository/permissions.utils';

@@ -79,47 +84,105 @@ describe('WorkspaceEntityManager', () => {
  };

  beforeEach(() => {
+    const mockFlatObjectMetadata: FlatObjectMetadata = {
+      id: 'test-entity-id',
+      nameSingular: 'test-entity',
+      namePlural: 'test-entities',
+      labelSingular: 'Test Entity',
+      labelPlural: 'Test Entities',
+      workspaceId: 'test-workspace-id',
+      icon: 'test-icon',
+      isCustom: false,
+      isRemote: false,
+      isAuditLogged: false,
+      isSearchable: false,
+      isSystem: false,
+      isActive: true,
+      targetTableName: 'test_entity',
+      fieldMetadataIds: ['field-id'],
+      indexMetadataIds: [],
+      viewIds: [],
+      universalIdentifier: 'test-entity-id',
+      description: null,
+      imageIdentifierFieldMetadataId: null,
+      labelIdentifierFieldMetadataId: null,
+      shortcut: null,
+      standardId: null,
+      standardOverrides: null,
+      applicationId: null,
+      isLabelSyncedWithName: false,
+      isUIReadOnly: false,
+      duplicateCriteria: null,
+      createdAt: new Date(),
+      updatedAt: new Date(),
+    };
+
+    const mockFlatFieldMetadata: FlatFieldMetadata = {
+      id: 'field-id',
+      type: 'TEXT' as FieldMetadataType,
+      name: 'fieldName',
+      label: 'Field Name',
+      objectMetadataId: 'test-entity-id',
+      isNullable: true,
+      isLabelSyncedWithName: false,
+      createdAt: new Date(),
+      updatedAt: new Date(),
+      universalIdentifier: 'field-id',
+      defaultValue: null,
+      description: null,
+      icon: null,
+      isActive: true,
+      isCustom: false,
+      isSystem: false,
+      isUIReadOnly: false,
+      isUnique: false,
+      options: null,
+      settings: null,
+      standardId: null,
+      standardOverrides: null,
+      workspaceId: 'test-workspace-id',
+      viewFieldIds: [],
+      viewFilterIds: [],
+      viewGroupIds: [],
+      kanbanAggregateOperationViewIds: [],
+      calendarViewIds: [],
+      relationTargetFieldMetadataId: null,
+      relationTargetObjectMetadataId: null,
+      morphId: null,
+      applicationId: null,
+    };
+
+    const flatObjectMetadataMaps: FlatEntityMaps<FlatObjectMetadata> = {
+      byId: {
+        'test-entity-id': mockFlatObjectMetadata,
+      },
+      idByUniversalIdentifier: {
+        'test-entity-id': 'test-entity-id',
+      },
+      universalIdentifiersByApplicationId: {},
+    };
+
+    const flatFieldMetadataMaps: FlatEntityMaps<FlatFieldMetadata> = {
+      byId: {
+        'field-id': mockFlatFieldMetadata,
+      },
+      idByUniversalIdentifier: {
+        'field-id': 'field-id',
+      },
+      universalIdentifiersByApplicationId: {},
+    };
+
    mockInternalContext = {
      workspaceId: 'test-workspace-id',
-      objectMetadataMaps: {
-        byId: {
-          'test-entity-id': {
-            id: 'test-entity-id',
-            nameSingular: 'test-entity',
-            namePlural: 'test-entities',
-            labelSingular: 'Test Entity',
-            labelPlural: 'Test Entities',
-            workspaceId: 'test-workspace-id',
-            icon: 'test-icon',
-            color: 'test-color',
-            isCustom: false,
-            isRemote: false,
-            isAuditLogged: false,
-            isSearchable: false,
-            isSystem: false,
-            isActive: true,
-            targetTableName: 'test_entity',
-            indexMetadatas: [],
-            fieldsById: {
-              'field-id': {
-                id: 'field-id',
-                type: 'TEXT',
-                name: 'fieldName',
-                label: 'Field Name',
-                objectMetadataId: 'test-entity-id',
-                isNullable: true,
-                isLabelSyncedWithName: false,
-                createdAt: new Date(),
-                updatedAt: new Date(),
-              },
-            },
-            fieldIdByName: { fieldName: 'field-id' },
-            fieldIdByJoinColumnName: {},
-          } as unknown as ObjectMetadataItemWithFieldMaps,
-        },
-        idByNameSingular: {
-          'test-entity': 'test-entity-id',
-        },
+      flatObjectMetadataMaps,
+      flatFieldMetadataMaps,
+      flatIndexMaps: {
+        byId: {},
+        idByUniversalIdentifier: {},
+        universalIdentifiersByApplicationId: {},
+      },
+      objectIdByNameSingular: {
+        'test-entity': 'test-entity-id',
      },
      featureFlagsMap: {
        IS_AIRTABLE_INTEGRATION_ENABLED: false,
@@ -333,7 +396,9 @@ describe('WorkspaceEntityManager', () => {
      expect(validateOperationIsPermittedOrThrow).toHaveBeenCalledWith({
        entityName: 'test-entity',
        operationType: 'update',
-        objectMetadataMaps: mockInternalContext.objectMetadataMaps,
+        flatObjectMetadataMaps: mockInternalContext.flatObjectMetadataMaps,
+        flatFieldMetadataMaps: mockInternalContext.flatFieldMetadataMaps,
+        objectIdByNameSingular: mockInternalContext.objectIdByNameSingular,
        objectsPermissions: mockPermissionOptions.objectRecordsPermissions,
        selectedColumns: [],
        allFieldsSelected: false,
@@ -366,7 +431,9 @@ describe('WorkspaceEntityManager', () => {
      expect(validateOperationIsPermittedOrThrow).toHaveBeenCalledWith({
        entityName: 'test-entity',
        operationType: 'delete',
-        objectMetadataMaps: mockInternalContext.objectMetadataMaps,
+        flatObjectMetadataMaps: mockInternalContext.flatObjectMetadataMaps,
+        flatFieldMetadataMaps: mockInternalContext.flatFieldMetadataMaps,
+        objectIdByNameSingular: mockInternalContext.objectIdByNameSingular,
        objectsPermissions: mockPermissionOptions.objectRecordsPermissions,
        selectedColumns: [],
        allFieldsSelected: false,
@@ -39,11 +39,11 @@ import { type WorkspaceInternalContext } from 'src/engine/twenty-orm/interfaces/
 import { DatabaseEventAction } from 'src/engine/api/graphql/graphql-query-runner/enums/database-event-action';
 import { type AuthContext } from 'src/engine/core-modules/auth/types/auth-context.type';
 import { InternalServerError } from 'src/engine/core-modules/graphql/utils/graphql-errors.util';
+import { type FlatObjectMetadata } from 'src/engine/metadata-modules/flat-object-metadata/types/flat-object-metadata.type';
 import {
  PermissionsException,
  PermissionsExceptionCode,
 } from 'src/engine/metadata-modules/permissions/permissions.exception';
-import { type ObjectMetadataItemWithFieldMaps } from 'src/engine/metadata-modules/types/object-metadata-item-with-field-maps';
 import { type WorkspaceDataSource } from 'src/engine/twenty-orm/datasource/workspace.datasource';
 import { type DeepPartialWithNestedRelationFields } from 'src/engine/twenty-orm/entity-manager/types/deep-partial-entity-with-nested-relation-fields.type';
 import { type QueryDeepPartialEntityWithNestedRelationFields } from 'src/engine/twenty-orm/entity-manager/types/query-deep-partial-entity-with-nested-relation-fields.type';
@@ -442,7 +442,9 @@ export class WorkspaceEntityManager extends EntityManager {
      entityName,
      operationType,
      objectsPermissions: permissionOptions?.objectRecordsPermissions ?? {},
-      objectMetadataMaps: this.internalContext.objectMetadataMaps,
+      flatObjectMetadataMaps: this.internalContext.flatObjectMetadataMaps,
+      flatFieldMetadataMaps: this.internalContext.flatFieldMetadataMaps,
+      objectIdByNameSingular: this.internalContext.objectIdByNameSingular,
      selectedColumns,
      allFieldsSelected: false,
      updatedColumns,
@@ -956,7 +958,11 @@ export class WorkspaceEntityManager extends EntityManager {
      this.internalContext,
    );

-    const formattedEntityLike = formatData(entityLike, objectMetadataItem);
+    const formattedEntityLike = formatData(
+      entityLike,
+      objectMetadataItem,
+      this.internalContext.flatFieldMetadataMaps,
+    );

    const managerWithPermissionOptions = Object.assign(
      Object.create(Object.getPrototypeOf(this)),
@@ -1190,6 +1196,7 @@ export class WorkspaceEntityManager extends EntityManager {
      const formattedEntityOrEntities = formatData(
        entityWithConnectedRelations,
        objectMetadataItem,
+        this.internalContext.flatFieldMetadataMaps,
      );

      const updatedColumns = formattedEntityOrEntities
@@ -1221,7 +1228,8 @@ export class WorkspaceEntityManager extends EntityManager {
      let formattedResult = formatResult<Entity[]>(
        resultArray,
        objectMetadataItem,
-        this.internalContext.objectMetadataMaps,
+        this.internalContext.flatObjectMetadataMaps,
+        this.internalContext.flatFieldMetadataMaps,
      );

      const updatedEntities = formattedResult.filter(
@@ -1235,6 +1243,7 @@ export class WorkspaceEntityManager extends EntityManager {
        formatTwentyOrmEventToDatabaseBatchEvent({
          action: DatabaseEventAction.UPDATED,
          objectMetadataItem,
+          flatFieldMetadataMaps: this.internalContext.flatFieldMetadataMaps,
          workspaceId: this.internalContext.workspaceId,
          entities: updatedEntities,
          beforeEntities: updatedEntities.map(
@@ -1247,6 +1256,7 @@ export class WorkspaceEntityManager extends EntityManager {
        formatTwentyOrmEventToDatabaseBatchEvent({
          action: DatabaseEventAction.CREATED,
          objectMetadataItem,
+          flatFieldMetadataMaps: this.internalContext.flatFieldMetadataMaps,
          workspaceId: this.internalContext.workspaceId,
          entities: createdEntities,
        }),
@@ -1283,7 +1293,7 @@ export class WorkspaceEntityManager extends EntityManager {
    permissionOptionsFromArgs,
  }: {
    formattedResult: Entity[];
-    objectMetadataItem: ObjectMetadataItemWithFieldMaps;
+    objectMetadataItem: FlatObjectMetadata;
    permissionOptionsFromArgs: PermissionOptions | undefined;
  }): Entity[] {
    if (permissionOptionsFromArgs?.shouldBypassPermissionChecks === true) {
@@ -1303,15 +1313,12 @@ export class WorkspaceEntityManager extends EntityManager {
      return formattedResult;
    }

-    const objectMetadataItemWithFieldMaps =
-      this.internalContext.objectMetadataMaps.byId[objectMetadataItem.id];
-
    const restrictedFieldNames = new Set(
      Object.entries(restrictedFields)
        .filter(([_, fieldPermissions]) => fieldPermissions.canRead === false)
        .map(([fieldMetadataId]) => {
          const fieldMetadata =
-            objectMetadataItemWithFieldMaps?.fieldsById[fieldMetadataId];
+            this.internalContext.flatFieldMetadataMaps.byId[fieldMetadataId];

          if (!isDefined(fieldMetadata)) {
            throw new InternalServerError(
@@ -1407,7 +1414,11 @@ export class WorkspaceEntityManager extends EntityManager {
      this.internalContext,
    );

-    const formattedEntity = formatData(entity, objectMetadataItem);
+    const formattedEntity = formatData(
+      entity,
+      objectMetadataItem,
+      this.internalContext.flatFieldMetadataMaps,
+    );

    const result = new EntityPersistExecutor(
      this.connection,
@@ -1424,13 +1435,15 @@ export class WorkspaceEntityManager extends EntityManager {
    const formattedResult = formatResult<Entity[]>(
      result,
      objectMetadataItem,
-      this.internalContext.objectMetadataMaps,
+      this.internalContext.flatObjectMetadataMaps,
+      this.internalContext.flatFieldMetadataMaps,
    );

    this.internalContext.eventEmitterService.emitDatabaseBatchEvent(
      formatTwentyOrmEventToDatabaseBatchEvent({
        action: DatabaseEventAction.DESTROYED,
        objectMetadataItem,
+        flatFieldMetadataMaps: this.internalContext.flatFieldMetadataMaps,
        workspaceId: this.internalContext.workspaceId,
        entities: formattedResult,
      }),
@@ -1522,7 +1535,11 @@ export class WorkspaceEntityManager extends EntityManager {
      this.internalContext,
    );

-    const formattedEntity = formatData(entity, objectMetadataItem);
+    const formattedEntity = formatData(
+      entity,
+      objectMetadataItem,
+      this.internalContext.flatFieldMetadataMaps,
+    );

    const result = new EntityPersistExecutor(
      this.connection,
@@ -1539,13 +1556,15 @@ export class WorkspaceEntityManager extends EntityManager {
    const formattedResult = formatResult<Entity[]>(
      result,
      objectMetadataItem,
-      this.internalContext.objectMetadataMaps,
+      this.internalContext.flatObjectMetadataMaps,
+      this.internalContext.flatFieldMetadataMaps,
    );

    this.internalContext.eventEmitterService.emitDatabaseBatchEvent(
      formatTwentyOrmEventToDatabaseBatchEvent({
        action: DatabaseEventAction.DELETED,
        objectMetadataItem,
+        flatFieldMetadataMaps: this.internalContext.flatFieldMetadataMaps,
        workspaceId: this.internalContext.workspaceId,
        entities: formattedResult,
      }),
@@ -1633,7 +1652,11 @@ export class WorkspaceEntityManager extends EntityManager {
      this.internalContext,
    );

-    const formattedEntity = formatData(entity, objectMetadataItem);
+    const formattedEntity = formatData(
+      entity,
+      objectMetadataItem,
+      this.internalContext.flatFieldMetadataMaps,
+    );

    const result = new EntityPersistExecutor(
      this.connection,
@@ -1650,13 +1673,15 @@ export class WorkspaceEntityManager extends EntityManager {
    const formattedResult = formatResult<Entity[]>(
      result,
      objectMetadataItem,
-      this.internalContext.objectMetadataMaps,
+      this.internalContext.flatObjectMetadataMaps,
+      this.internalContext.flatFieldMetadataMaps,
    );

    this.internalContext.eventEmitterService.emitDatabaseBatchEvent(
      formatTwentyOrmEventToDatabaseBatchEvent({
        action: DatabaseEventAction.RESTORED,
        objectMetadataItem,
+        flatFieldMetadataMaps: this.internalContext.flatFieldMetadataMaps,
        workspaceId: this.internalContext.workspaceId,
        entities: formattedResult,
      }),