[REQUIRES_FULL_CACHE_FLUSH_WHEN_RELEASED] Refactor FlatEntity to be UniversalFlatEntity superset (#17452)

# Introduction In this PR we're refactoring the `FlatEntity` type to become a superset of the `UniversalFlatEntity`. Right now we're storing all the extra properties in `__universal` property, at some point it might just be sibling to other entity and we might rely on the `propertiesToCompare` constants and TypeScript allowing passing a superset type into a smaller subset type ## FromTo utils The entity to flat entity method now computes the universal information, standardized a typing and pattern to do ## Example Also strictly type ```ts "bbb019ea-6205-498c-aea5-67bc53bce8a9": { "workspaceId": "20202020-1c25-4d02-bf25-6aeccf7ea419", "universalIdentifier": "20202020-d111-4d11-8d11-da5ab0a11002", "applicationId": "d01b010d-b984-465b-b40b-370e954e5188", "id": "bbb019ea-6205-498c-aea5-67bc53bce8a9", "pageLayoutTabId": "791a512f-169f-4209-b731-aa86716668c6", "title": "Deals by Company", "type": "GRAPH", "objectMetadataId": "9e14efea-df5b-4c0e-aba9-cfe455f32397", "gridPosition": { "row": 0, "column": 6, "rowSpan": 6, "columnSpan": 6 }, "configuration": { "color": "orange", "orderBy": "FIELD_ASC", "timezone": "UTC", "displayLegend": true, "displayDataLabel": false, "showCenterMetric": true, "configurationType": "PIE_CHART", "firstDayOfTheWeek": 0, "aggregateOperation": "COUNT", "groupBySubFieldName": "name", "groupByFieldMetadataId": "6673ff18-63d2-47a1-8f85-2b9b09ca27a5", "aggregateFieldMetadataId": "8d64ee41-5dd4-4de6-945a-7c0c18399715" }, "createdAt": "2026-01-28T14:08:52.140Z", "updatedAt": "2026-01-28T14:08:52.140Z", "deletedAt": null, "__universal": { "universalIdentifier": "20202020-d111-4d11-8d11-da5ab0a11002", "applicationUniversalIdentifier": "20202020-64aa-4b6f-b003-9c74b97cee20", "pageLayoutTabUniversalIdentifier": "20202020-d011-4d11-8d11-da5ab0a01001", "objectMetadataUniversalIdentifier": "20202020-9549-49dd-b2b2-883999db8938", "gridPosition": { "row": 0, "column": 6, "rowSpan": 6, "columnSpan": 6 }, "configuration": { "color": "orange", "orderBy": "FIELD_ASC", "timezone": "UTC", "displayLegend": true, "displayDataLabel": false, "showCenterMetric": true, "configurationType": "PIE_CHART", "firstDayOfTheWeek": 0, "aggregateOperation": "COUNT", "groupBySubFieldName": "name", "aggregateFieldMetadataUniversalIdentifier": "20202020-d01a-4131-8a31-f123456789ab", "groupByFieldMetadataUniversalIdentifier": "20202020-cbac-457e-b565-adece5fc815f" } } }, ```
2026-01-28 17:46:34 +01:00
parent 81eaee81a8
commit fe9d6f34ff
90 changed files with 3058 additions and 538 deletions
@@ -0,0 +1,56 @@
+import { type AllMetadataName } from 'twenty-shared/metadata';
+
+import { type MetadataEntity } from 'src/engine/metadata-modules/flat-entity/types/metadata-entity.type';
+import { type ExtractJsonbProperties } from 'src/engine/workspace-manager/workspace-migration/universal-flat-entity/types/extract-jsonb-properties.type';
+
+export const ALL_JSONB_PROPERTIES_BY_METADATA_NAME = {
+  fieldMetadata: {
+    defaultValue: 'defaultValue',
+    standardOverrides: 'standardOverrides',
+    options: 'options',
+    settings: 'settings',
+  },
+  objectMetadata: {
+    standardOverrides: 'standardOverrides',
+    duplicateCriteria: 'duplicateCriteria',
+  },
+  view: {},
+  viewField: {},
+  viewGroup: {},
+  viewFilter: { value: 'value' },
+  viewFilterGroup: {},
+  index: {},
+  role: {},
+  roleTarget: {},
+  rowLevelPermissionPredicate: { value: 'value' },
+  rowLevelPermissionPredicateGroup: {},
+  logicFunction: {
+    cronTriggerSettings: 'cronTriggerSettings',
+    databaseEventTriggerSettings: 'databaseEventTriggerSettings',
+    httpRouteTriggerSettings: 'httpRouteTriggerSettings',
+    publishedVersions: 'publishedVersions',
+    toolInputSchema: 'toolInputSchema',
+  },
+  webhook: {},
+  agent: {
+    responseFormat: 'responseFormat',
+    modelConfiguration: 'modelConfiguration',
+  },
+  skill: {},
+  pageLayout: {},
+  pageLayoutTab: {},
+  pageLayoutWidget: {
+    gridPosition: 'gridPosition',
+    configuration: 'configuration',
+  },
+  commandMenuItem: {},
+  navigationMenuItem: {},
+  frontComponent: {},
+} as const satisfies {
+  [P in AllMetadataName]: {
+    [K in ExtractJsonbProperties<MetadataEntity<P>>]: K;
+  };
+};
+
+export type AllJsonbPropertiesForMetadataName<T extends AllMetadataName> =
+  keyof (typeof ALL_JSONB_PROPERTIES_BY_METADATA_NAME)[T];
@@ -18,24 +18,24 @@ type BrandedObjectWithRelation = JsonbProperty<ObjectWithRelation>;

 // eslint-disable-next-line unused-imports/no-unused-vars
 type ObjectAssertions = [
-  // Object with SerializedRelation: Id suffix renamed to UniversalIdentifier
+  // Object with SerializedRelation: Id suffix renamed to UniversalIdentifier, value becomes nullable
  Expect<
    Equal<
      FormatRecordSerializedRelationProperties<ObjectWithRelation>,
      {
        name: string;
-        targetFieldMetadataUniversalIdentifier: SerializedRelation;
+        targetFieldMetadataUniversalIdentifier: SerializedRelation | null;
      }
    >
  >,

-  // Branded object with SerializedRelation: renames property, preserves brand key
+  // Branded object with SerializedRelation: renames property, preserves brand key, value becomes nullable
  Expect<
    Equal<
      FormatRecordSerializedRelationProperties<BrandedObjectWithRelation>,
      {
        name: string;
-        targetFieldMetadataUniversalIdentifier: SerializedRelation;
+        targetFieldMetadataUniversalIdentifier: SerializedRelation | null;
        __JsonbPropertyBrand__?: undefined;
      }
    >
@@ -56,6 +56,13 @@ type PrimitiveAssertions = [
  Expect<Equal<FormatRecordSerializedRelationProperties<string>, string>>,
  Expect<Equal<FormatRecordSerializedRelationProperties<number>, number>>,
  Expect<Equal<FormatRecordSerializedRelationProperties<null>, null>>,
+  // SerializedRelation becomes nullable
+  Expect<
+    Equal<
+      FormatRecordSerializedRelationProperties<SerializedRelation>,
+      SerializedRelation | null
+    >
+  >,
 ];

 // eslint-disable-next-line unused-imports/no-unused-vars
@@ -66,7 +73,7 @@ type ArrayAssertions = [
      FormatRecordSerializedRelationProperties<ObjectWithRelation[]>,
      {
        name: string;
-        targetFieldMetadataUniversalIdentifier: SerializedRelation;
+        targetFieldMetadataUniversalIdentifier: SerializedRelation | null;
      }[]
    >
  >,
@@ -88,7 +95,7 @@ type ArrayAssertions = [
      FormatRecordSerializedRelationProperties<ObjectWithRelation[][]>,
      {
        name: string;
-        targetFieldMetadataUniversalIdentifier: SerializedRelation;
+        targetFieldMetadataUniversalIdentifier: SerializedRelation | null;
      }[][]
    >
  >,
@@ -102,7 +109,7 @@ type UnionAssertions = [
      FormatRecordSerializedRelationProperties<ObjectWithRelation | null>,
      {
        name: string;
-        targetFieldMetadataUniversalIdentifier: SerializedRelation;
+        targetFieldMetadataUniversalIdentifier: SerializedRelation | null;
      } | null
    >
  >,
@@ -113,7 +120,7 @@ type UnionAssertions = [
      FormatRecordSerializedRelationProperties<(ObjectWithRelation | null)[]>,
      ({
        name: string;
-        targetFieldMetadataUniversalIdentifier: SerializedRelation;
+        targetFieldMetadataUniversalIdentifier: SerializedRelation | null;
      } | null)[]
    >
  >,
@@ -128,14 +135,14 @@ type MultipleRelationsObject = {

 // eslint-disable-next-line unused-imports/no-unused-vars
 type MultipleRelationsAssertions = [
-  // Multiple SerializedRelation properties: all get renamed
+  // Multiple SerializedRelation properties: all get renamed and become nullable
  Expect<
    Equal<
      FormatRecordSerializedRelationProperties<MultipleRelationsObject>,
      {
        name: string;
-        sourceFieldUniversalIdentifier: SerializedRelation;
-        targetFieldUniversalIdentifier: SerializedRelation;
+        sourceFieldUniversalIdentifier: SerializedRelation | null;
+        targetFieldUniversalIdentifier: SerializedRelation | null;
        regularId: string;
      }
    >
@@ -190,7 +197,7 @@ type NestedObjectAssertions = [
      {
        name: string;
        nested: {
-          fieldMetadataUniversalIdentifier: SerializedRelation;
+          fieldMetadataUniversalIdentifier: SerializedRelation | null;
        };
      }
    >
@@ -204,7 +211,7 @@ type NestedObjectAssertions = [
        name: string;
        level1: {
          level2: {
-            targetUniversalIdentifier: SerializedRelation;
+            targetUniversalIdentifier: SerializedRelation | null;
          };
        };
      }
@@ -217,9 +224,9 @@ type NestedObjectAssertions = [
      FormatRecordSerializedRelationProperties<MixedNestedObject>,
      {
        name: string;
-        directRelationUniversalIdentifier: SerializedRelation;
+        directRelationUniversalIdentifier: SerializedRelation | null;
        nested: {
-          nestedRelationUniversalIdentifier: SerializedRelation;
+          nestedRelationUniversalIdentifier: SerializedRelation | null;
          plainField: string;
        };
      }
@@ -233,7 +240,7 @@ type NestedObjectAssertions = [
      {
        name: string;
        nested: {
-          relationUniversalIdentifier: SerializedRelation;
+          relationUniversalIdentifier: SerializedRelation | null;
        } | null;
      }
    >
@@ -246,9 +253,85 @@ type NestedObjectAssertions = [
      {
        name: string;
        items: {
-          itemRelationUniversalIdentifier: SerializedRelation;
+          itemRelationUniversalIdentifier: SerializedRelation | null;
        }[];
      }
    >
  >,
 ];
+
+// JSON Schema-like structures with Record<string, X> properties
+// These should NOT have their 'properties' key renamed
+type JsonSchemaLikeObject = {
+  type: 'object';
+  properties: Record<string, { type: string; description?: string }>;
+  required?: string[];
+};
+
+type JsonSchemaUnion =
+  | { type: 'text' }
+  | {
+      type: 'json';
+      schema: JsonSchemaLikeObject;
+    };
+
+type BrandedJsonSchemaUnion = JsonbProperty<JsonSchemaUnion>;
+
+// eslint-disable-next-line unused-imports/no-unused-vars
+type RecordPropertyAssertions = [
+  // Object with 'properties' key containing Record<string, X>:
+  // The 'properties' key should NOT be renamed to 'propertiesUniversalIdentifier'
+  Expect<
+    Equal<
+      FormatRecordSerializedRelationProperties<JsonSchemaLikeObject>,
+      {
+        type: 'object';
+        properties: Record<string, { type: string; description?: string }>;
+        required?: string[];
+      }
+    >
+  >,
+
+  // Union type with JSON schema: properties inside schema should stay as 'properties'
+  Expect<
+    Equal<
+      FormatRecordSerializedRelationProperties<JsonSchemaUnion>,
+      | { type: 'text' }
+      | {
+          type: 'json';
+          schema: {
+            type: 'object';
+            properties: Record<string, { type: string; description?: string }>;
+            required?: string[];
+          };
+        }
+    >
+  >,
+
+  // Branded JSON schema union: should preserve structure, only add brand
+  Expect<
+    Equal<
+      FormatRecordSerializedRelationProperties<BrandedJsonSchemaUnion>,
+      | { type: 'text'; __JsonbPropertyBrand__?: undefined }
+      | {
+          type: 'json';
+          schema: {
+            type: 'object';
+            properties: Record<string, { type: string; description?: string }>;
+            required?: string[];
+          };
+          __JsonbPropertyBrand__?: undefined;
+        }
+    >
+  >,
+
+  // Plain Record<string, X> should pass through unchanged
+  Expect<
+    Equal<
+      FormatRecordSerializedRelationProperties<
+        Record<string, { value: number }>
+      >,
+      Record<string, { value: number }>
+    >
+  >,
+];
@@ -98,7 +98,10 @@ type NarrowedExpectedResult = {
  relationType: RelationType;
  onDelete?: RelationOnDeleteAction | undefined;
  joinColumnName?: string | null | undefined;
-  junctionTargetFieldUniversalIdentifier?: SerializedRelation | undefined;
+  junctionTargetFieldUniversalIdentifier?:
+    | SerializedRelation
+    | null
+    | undefined;
  __JsonbPropertyBrand__?: undefined;
 };

@@ -111,7 +114,10 @@ type SettingsExpectedResult =
      relationType: RelationType;
      onDelete?: RelationOnDeleteAction | undefined;
      joinColumnName?: string | null | undefined;
-      junctionTargetFieldUniversalIdentifier?: SerializedRelation | undefined;
+      junctionTargetFieldUniversalIdentifier?:
+        | SerializedRelation
+        | null
+        | undefined;
      __JsonbPropertyBrand__?: undefined;
    }
  | {
@@ -1,17 +1,40 @@
-import { type ExtractSerializedRelationProperties } from 'twenty-shared/types';
+import {
+  type ExtractSerializedRelationProperties,
+  type IsSerializedRelation,
+} from 'twenty-shared/types';

 import { type RemoveSuffix } from 'src/engine/workspace-manager/workspace-migration/workspace-migration-builder/types/remove-suffix.type';

+// Determines if a property should be transformed to a universal identifier
+// A property is transformed only if:
+// 1. It matches ExtractSerializedRelationProperties (has the serialized relation brand)
+// 2. Its value does NOT have a string index signature (not a Record<string, X>)
+type ShouldTransformToUniversalIdentifier<T, P extends keyof T> = [P] extends [
+  ExtractSerializedRelationProperties<T>,
+]
+  ? string extends keyof NonNullable<T[P]>
+    ? false
+    : true
+  : false;
+
 export type FormatRecordSerializedRelationProperties<T> = T extends unknown
  ? T extends (infer U)[]
    ? FormatRecordSerializedRelationProperties<U>[]
    : T extends string
-      ? T
+      ? // By definition we assume that any SerializedRelation are not enforced at pg scope through an FK
+        // Which mean that SerializedRelation might resolve to non-existent entities ( null )
+        IsSerializedRelation<T> extends true
+        ? T | null
+        : T
      : T extends object
        ? {
-            [P in keyof T as P extends ExtractSerializedRelationProperties<T> &
-              string
-              ? `${RemoveSuffix<P, 'Id'>}UniversalIdentifier`
+            [P in keyof T as ShouldTransformToUniversalIdentifier<
+              T,
+              P
+            > extends true
+              ? P extends string
+                ? `${RemoveSuffix<P, 'Id'>}UniversalIdentifier`
+                : P
              : P]: FormatRecordSerializedRelationProperties<T[P]>;
          }
        : T
@@ -1,16 +1,38 @@
 import { type AllMetadataName } from 'twenty-shared/metadata';

+import { type AddSuffixToEntityManyToOneProperties } from 'src/engine/metadata-modules/flat-entity/types/add-suffix-to-entity-many-to-one-properties.type';
+import { type AddSuffixToEntityOneToManyProperties } from 'src/engine/metadata-modules/flat-entity/types/add-suffix-to-entity-one-to-many-properties.type';
 import { type CastRecordTypeOrmDatePropertiesToString } from 'src/engine/metadata-modules/flat-entity/types/cast-record-typeorm-date-properties-to-string.type';
-import { type ExtractEntityOneToManyEntityRelationProperties } from 'src/engine/metadata-modules/flat-entity/types/extract-entity-one-to-many-entity-relation-properties.type';
 import { type ExtractEntityRelatedEntityProperties } from 'src/engine/metadata-modules/flat-entity/types/extract-entity-related-entity-properties.type';
 import { type FromMetadataEntityToMetadataName } from 'src/engine/metadata-modules/flat-entity/types/from-metadata-entity-to-metadata-name.type';
 import { type MetadataManyToOneJoinColumn } from 'src/engine/metadata-modules/flat-entity/types/metadata-many-to-one-join-column.type';
 import { type SyncableEntity } from 'src/engine/workspace-manager/types/syncable-entity.interface';
-import { type ExtractJsonbProperties } from 'src/engine/workspace-manager/workspace-migration/universal-flat-entity/types/extract-jsonb-properties.type';
+import { type AllJsonbPropertiesForMetadataName } from 'src/engine/workspace-manager/workspace-migration/universal-flat-entity/constants/all-jsonb-properties-by-metadata-name.constant';
 import { type FormatRecordSerializedRelationProperties } from 'src/engine/workspace-manager/workspace-migration/universal-flat-entity/types/format-record-serialized-relation-properties.type';
-import { type RemoveSuffix } from 'src/engine/workspace-manager/workspace-migration/workspace-migration-builder/types/remove-suffix.type';

-// TODO Handle universal settings
+export type UniversalSyncableFlatEntity = Omit<
+  SyncableEntity,
+  'application' | 'applicationId' | 'workspace' | 'workspaceId'
+> & {
+  applicationUniversalIdentifier: string;
+};
+
+export type UniversalFlatEntityExtraProperties<
+  TEntity extends SyncableEntity,
+  TMetadataName extends
+    AllMetadataName = FromMetadataEntityToMetadataName<TEntity>,
+> = AddSuffixToEntityOneToManyProperties<TEntity, 'universalIdentifiers'> &
+  AddSuffixToEntityManyToOneProperties<
+    TEntity,
+    TMetadataName,
+    'universalIdentifier'
+  > & {
+    applicationUniversalIdentifier: string;
+  } & {
+    [P in AllJsonbPropertiesForMetadataName<TMetadataName> &
+      keyof TEntity]: FormatRecordSerializedRelationProperties<TEntity[P]>;
+  };
+
 export type UniversalFlatEntityFrom<
  TEntity extends SyncableEntity,
  TMetadataName extends
@@ -24,21 +46,7 @@ export type UniversalFlatEntityFrom<
  | ExtractEntityRelatedEntityProperties<TEntity>
  | Extract<MetadataManyToOneJoinColumn<TMetadataName>, keyof TEntity>
  | keyof CastRecordTypeOrmDatePropertiesToString<TEntity>
-  | ExtractJsonbProperties<TEntity>
+  | AllJsonbPropertiesForMetadataName<TMetadataName>
 > &
-  CastRecordTypeOrmDatePropertiesToString<TEntity> & {
-    [P in ExtractEntityOneToManyEntityRelationProperties<
-      TEntity,
-      SyncableEntity
-    > &
-      string as `${RemoveSuffix<P, 's'>}UniversalIdentifiers`]: string[];
-  } & {
-    [P in Extract<MetadataManyToOneJoinColumn<TMetadataName>, keyof TEntity> &
-      string as `${RemoveSuffix<P, 'Id'>}UniversalIdentifier`]: TEntity[P];
-  } & {
-    applicationUniversalIdentifier: string;
-  } & {
-    [P in ExtractJsonbProperties<TEntity>]: FormatRecordSerializedRelationProperties<
-      TEntity[P]
-    >;
-  };
+  CastRecordTypeOrmDatePropertiesToString<TEntity> &
+  UniversalFlatEntityExtraProperties<TEntity, TMetadataName>;
@@ -0,0 +1,4 @@
+import { type RoleEntity } from 'src/engine/metadata-modules/role/role.entity';
+import { type UniversalFlatEntityFrom } from 'src/engine/workspace-manager/workspace-migration/universal-flat-entity/types/universal-flat-entity-from.type';
+
+export type UniversalFlatRole = UniversalFlatEntityFrom<RoleEntity, 'role'>;
@@ -1,10 +1,11 @@
-// Jest Snapshot v1, https://jestjs.io/docs/snapshot-testing
+// Jest Snapshot v1, https://goo.gl/fbAQLP

 exports[`flatEntityDeletedCreatedUpdatedMatrixDispatcher It should detect a created entity 1`] = `
 {
  "createdFlatEntityMaps": {
    "byId": {
      "field-id-1": {
+        "__universal": {},
        "applicationId": "application-id-1",
        "calendarViewIds": [],
        "createdAt": "2024-01-01T00:00:00.000Z",
@@ -69,6 +70,7 @@ exports[`flatEntityDeletedCreatedUpdatedMatrixDispatcher It should detect a dele
  "deletedFlatEntityMaps": {
    "byId": {
      "field-id-1": {
+        "__universal": {},
        "applicationId": "application-id-1",
        "calendarViewIds": [],
        "createdAt": "2024-01-01T00:00:00.000Z",
@@ -151,6 +153,7 @@ exports[`flatEntityDeletedCreatedUpdatedMatrixDispatcher It should detect create
  "createdFlatEntityMaps": {
    "byId": {
      "field-id-3": {
+        "__universal": {},
        "applicationId": "application-id-1",
        "calendarViewIds": [],
        "createdAt": "2024-01-01T00:00:00.000Z",
@@ -197,6 +200,7 @@ exports[`flatEntityDeletedCreatedUpdatedMatrixDispatcher It should detect create
  "deletedFlatEntityMaps": {
    "byId": {
      "field-id-2": {
+        "__universal": {},
        "applicationId": "application-id-1",
        "calendarViewIds": [],
        "createdAt": "2024-01-01T00:00:00.000Z",
@@ -18,5 +18,6 @@ export type UpdateFieldAction =

 export type DeleteFieldAction =
  BaseDeleteWorkspaceMigrationAction<'fieldMetadata'> & {
+    // Remove this
    objectMetadataId: string;
  };