[Requires "warm" cache flush (no immediate downtime before flush)] Migrate viewGroup.fieldMetadataId -> view.mainGroupByFieldMetadataId (1/3) (#16206)

In this PR (1/3)
- introduce view.mainGroupByFieldMetadataId as the new reference
determining which fieldMetadataId is used in a grouped view, in order to
deprecate viewGroup.fieldMetadataId which creates inconsistencies.
view.mainGroupByFieldMetadataId is now filled at every view creation,
though not in use yet.
- Introduce a command to backfill view.mainGroupByFieldMetadataId for
existing views + delete all viewGroup.fieldMetadataId with a
fieldMetadataId that is not view.mainGroupByFieldMetadataId. (It should
concern 37 active workspaces)
- Temporarily disable the option to change a grouped view's
fieldMetadataId as for now it creates inconsistencies. This feature can
be reintroduced when we have done the full migration.

In a next PR
- (2/3) use view.mainGroupByFieldMetadataId instead of
viewGroup.fieldMetadataId. In FE we may keep viewGroup.fieldMetadataId
as a state (TBD). View groups will now be created / deleted as a side
effect of view's mainGroupByFieldMetadataId update.
- (3/3) remove viewGroup.fieldMetadataId

---------

Co-authored-by: Charles Bochet <charles@twenty.com>

packages/twenty-server/src/engine/metadata-modules/field-metadata/field-metadata.entity.ts

@@ -1,7 +1,7 @@
 import {
-  FieldMetadataType,
-  FieldMetadataSettings,
   FieldMetadataOptions,
+  FieldMetadataSettings,
+  FieldMetadataType,
 } from 'twenty-shared/types';
 import {
   Check,
@@ -212,4 +212,7 @@ export class FieldMetadataEntity<
 
   @OneToMany(() => ViewEntity, (view) => view.calendarFieldMetadata)
   calendarViews: Relation<ViewEntity[]>;
+
+  @OneToMany(() => ViewEntity, (view) => view.mainGroupByFieldMetadata)
+  mainGroupByFieldMetadataViews: Relation<ViewEntity[]>;
 }

packages/twenty-server/src/engine/metadata-modules/flat-entity/constant/all-metadata-many-to-one-relations.constant.ts

@@ -41,6 +41,10 @@ export const ALL_METADATA_RELATED_METADATA_BY_FOREIGN_KEY = {
       metadataName: 'fieldMetadata',
       flatEntityForeignKeyAggregator: 'calendarViewIds',
     },
+    mainGroupByFieldMetadataId: {
+      metadataName: 'fieldMetadata',
+      flatEntityForeignKeyAggregator: 'mainGroupByFieldMetadataViewIds',
+    },
     objectMetadataId: {
       metadataName: 'objectMetadata',
       flatEntityForeignKeyAggregator: 'viewIds',

packages/twenty-server/src/engine/metadata-modules/flat-field-metadata/__mocks__/get-flat-field-metadata.mock.ts

@@ -25,6 +25,7 @@ export const getFlatFieldMetadataMock = <T extends FieldMetadataType>(
     kanbanAggregateOperationViewIds: [],
     viewFieldIds: [],
     createdAt,
+    mainGroupByFieldMetadataViewIds: [],
     updatedAt: createdAt,
     defaultValue: null,
     options: null,

packages/twenty-server/src/engine/metadata-modules/flat-field-metadata/__mocks__/get-morph-or-relation-target-flat-field-metadata-mock.ts

@@ -33,6 +33,7 @@ export const getRelationTargetFlatFieldMetadataMock = ({
 
   return {
     calendarViewIds: [],
+    mainGroupByFieldMetadataViewIds: [],
     viewFilterIds: [],
     viewGroupIds: [],
     viewFieldIds: [],

packages/twenty-server/src/engine/metadata-modules/flat-field-metadata/__tests__/__snapshots__/morph-relation-from-create-field-input-to-flat-field-metadatas-to-create.spec.ts.snap

@@ -120,6 +120,7 @@ exports[`fromCreateFieldInputToFlatFieldMetadatasToCreate MORPH_RELATION test su
         "isUnique": null,
         "kanbanAggregateOperationViewIds": [],
         "label": "newFieldLabel",
+        "mainGroupByFieldMetadataViewIds": [],
         "morphId": Any<String>,
         "name": "newFieldPets",
         "objectMetadataId": Any<String>,
@@ -156,6 +157,7 @@ exports[`fromCreateFieldInputToFlatFieldMetadatasToCreate MORPH_RELATION test su
         "isUnique": null,
         "kanbanAggregateOperationViewIds": [],
         "label": "Pet",
+        "mainGroupByFieldMetadataViewIds": [],
         "morphId": null,
         "name": "pet",
         "objectMetadataId": Any<String>,
@@ -194,6 +196,7 @@ exports[`fromCreateFieldInputToFlatFieldMetadatasToCreate MORPH_RELATION test su
         "isUnique": null,
         "kanbanAggregateOperationViewIds": [],
         "label": "newFieldLabel",
+        "mainGroupByFieldMetadataViewIds": [],
         "morphId": Any<String>,
         "name": "newFieldCompanies",
         "objectMetadataId": Any<String>,
@@ -230,6 +233,7 @@ exports[`fromCreateFieldInputToFlatFieldMetadatasToCreate MORPH_RELATION test su
         "isUnique": null,
         "kanbanAggregateOperationViewIds": [],
         "label": "Company",
+        "mainGroupByFieldMetadataViewIds": [],
         "morphId": null,
         "name": "company",
         "objectMetadataId": Any<String>,

packages/twenty-server/src/engine/metadata-modules/flat-field-metadata/services/workspace-flat-field-metadata-map-cache.service.ts

@@ -68,6 +68,7 @@ export class WorkspaceFlatFieldMetadataMapCacheService extends WorkspaceCachePro
             'id',
             'kanbanAggregateOperationFieldMetadataId',
             'calendarFieldMetadataId',
+            'mainGroupByFieldMetadataId',
           ],
           withDeleted: true,
         }),
@@ -79,6 +80,7 @@ export class WorkspaceFlatFieldMetadataMapCacheService extends WorkspaceCachePro
       viewGroupsByFieldId,
       calendarViewsByFieldId,
       kanbanViewsByFieldId,
+      mainGroupByFieldMetadataViewsByFieldId,
     ] = (
       [
         {
@@ -101,6 +103,10 @@ export class WorkspaceFlatFieldMetadataMapCacheService extends WorkspaceCachePro
           entities: views,
           foreignKey: 'kanbanAggregateOperationFieldMetadataId',
         },
+        {
+          entities: views,
+          foreignKey: 'mainGroupByFieldMetadataId',
+        },
       ] as const
     ).map(regroupEntitiesByRelatedEntityId);
 
@@ -115,6 +121,9 @@ export class WorkspaceFlatFieldMetadataMapCacheService extends WorkspaceCachePro
         kanbanAggregateOperationViews:
           kanbanViewsByFieldId.get(fieldMetadataEntity.id) || [],
         calendarViews: calendarViewsByFieldId.get(fieldMetadataEntity.id) || [],
+        mainGroupByFieldMetadataViews:
+          mainGroupByFieldMetadataViewsByFieldId.get(fieldMetadataEntity.id) ||
+          [],
       } as FieldMetadataEntity);
 
       addFlatEntityToFlatEntityMapsThroughMutationOrThrow({

packages/twenty-server/src/engine/metadata-modules/flat-field-metadata/types/flat-field-metadata.type.ts

@@ -11,9 +11,10 @@ export const FIELD_METADATA_RELATION_PROPERTIES = [
   'viewFields',
   'application',
   'viewFilters',
-  'viewGroups',
   'kanbanAggregateOperationViews',
   'calendarViews',
+  'mainGroupByFieldMetadataViews',
+  'viewGroups',
 ] as const satisfies (keyof FieldMetadataEntity)[];
 
 export type FieldMetadataEntityRelationProperties =
@@ -24,7 +25,8 @@ export type FlatFieldMetadata<T extends FieldMetadataType = FieldMetadataType> =
     universalIdentifier: string;
     viewFieldIds: string[];
     viewFilterIds: string[];
-    viewGroupIds: string[];
     kanbanAggregateOperationViewIds: string[];
     calendarViewIds: string[];
+    mainGroupByFieldMetadataViewIds: string[];
+    viewGroupIds: string[];
   };

packages/twenty-server/src/engine/metadata-modules/flat-field-metadata/utils/from-field-metadata-entity-to-flat-field-metadata.util.ts

@@ -23,6 +23,9 @@ export const fromFieldMetadataEntityToFlatFieldMetadata = <
     kanbanAggregateOperationViewIds:
       fieldMetadataEntity.kanbanAggregateOperationViews.map(({ id }) => id),
     calendarViewIds: fieldMetadataEntity.calendarViews.map(({ id }) => id),
+    mainGroupByFieldMetadataViewIds:
+      fieldMetadataEntity.mainGroupByFieldMetadataViews?.map(({ id }) => id) ??
+      [],
     viewGroupIds: fieldMetadataEntity.viewGroups.map(({ id }) => id),
     viewFieldIds: fieldMetadataEntity.viewFields.map(({ id }) => id),
     viewFilterIds: fieldMetadataEntity.viewFilters.map(({ id }) => id),

packages/twenty-server/src/engine/metadata-modules/flat-field-metadata/utils/get-default-flat-field-metadata-from-create-field-input.util.ts

@@ -26,6 +26,7 @@ export const getDefaultFlatFieldMetadata = ({
 
   return {
     calendarViewIds: [],
+    mainGroupByFieldMetadataViewIds: [],
     viewFieldIds: [],
     description: createFieldInput.description ?? null,
     id: fieldMetadataId,

packages/twenty-server/src/engine/metadata-modules/flat-field-metadata/utils/recompute-view-groups-on-flat-field-metadata-options-update.util.ts

@@ -1,5 +1,5 @@
-import { isDefined } from 'class-validator';
 import { type EnumFieldMetadataType } from 'twenty-shared/types';
+import { isDefined } from 'twenty-shared/utils';
 import { v4 } from 'uuid';
 
 import {
@@ -63,7 +63,8 @@ export const recomputeViewGroupsOnFlatFieldMetadataOptionsUpdate = ({
   const flatViewGroupsToUpdate = updatedFieldMetadataOptions.flatMap(
     ({ from: fromOption, to: toOption }) =>
       flatViewGroups.flatMap((flatViewGroup) =>
-        flatViewGroup.fieldValue === fromOption.value
+        flatViewGroup.fieldValue === fromOption.value &&
+        flatViewGroup.fieldMetadataId === fromFlatFieldMetadata.id
           ? { ...flatViewGroup, fieldValue: toOption.value }
           : [],
       ),

packages/twenty-server/src/engine/metadata-modules/flat-view/constants/flat-view-editable-properties.constant.ts

@@ -13,4 +13,5 @@ export const FLAT_VIEW_EDITABLE_PROPERTIES = [
   'calendarLayout',
   'calendarFieldMetadataId',
   'visibility',
+  'mainGroupByFieldMetadataId',
 ] as const satisfies (keyof FlatView)[];

packages/twenty-server/src/engine/metadata-modules/flat-view/utils/from-create-view-input-to-flat-view-to-create.util.ts

@@ -44,6 +44,8 @@ export const fromCreateViewInputToFlatViewToCreate = ({
     kanbanAggregateOperation: createViewInput.kanbanAggregateOperation ?? null,
     kanbanAggregateOperationFieldMetadataId:
       createViewInput.kanbanAggregateOperationFieldMetadataId ?? null,
+    mainGroupByFieldMetadataId:
+      createViewInput.mainGroupByFieldMetadataId ?? null,
     key: createViewInput.key ?? null,
     openRecordIn: createViewInput.openRecordIn ?? ViewOpenRecordIn.SIDE_PANEL,
     position: createViewInput.position ?? 0,

packages/twenty-server/src/engine/metadata-modules/object-metadata/utils/build-default-flat-field-metadatas-for-custom-object.util.ts

@@ -31,6 +31,7 @@ export const buildDefaultFlatFieldMetadatasForCustomObject = ({
     viewGroupIds: [],
     kanbanAggregateOperationViewIds: [],
     calendarViewIds: [],
+    mainGroupByFieldMetadataViewIds: [],
     isLabelSyncedWithName: false,
     isUnique: true,
     objectMetadataId,
@@ -68,6 +69,7 @@ export const buildDefaultFlatFieldMetadatasForCustomObject = ({
     viewGroupIds: [],
     kanbanAggregateOperationViewIds: [],
     calendarViewIds: [],
+    mainGroupByFieldMetadataViewIds: [],
     isLabelSyncedWithName: false,
     isUnique: false,
     objectMetadataId,
@@ -105,6 +107,7 @@ export const buildDefaultFlatFieldMetadatasForCustomObject = ({
     viewGroupIds: [],
     kanbanAggregateOperationViewIds: [],
     calendarViewIds: [],
+    mainGroupByFieldMetadataViewIds: [],
     isLabelSyncedWithName: false,
     isUnique: false,
     objectMetadataId,
@@ -142,6 +145,7 @@ export const buildDefaultFlatFieldMetadatasForCustomObject = ({
     viewGroupIds: [],
     kanbanAggregateOperationViewIds: [],
     calendarViewIds: [],
+    mainGroupByFieldMetadataViewIds: [],
     isLabelSyncedWithName: false,
     isUnique: false,
     objectMetadataId,
@@ -179,6 +183,7 @@ export const buildDefaultFlatFieldMetadatasForCustomObject = ({
     viewGroupIds: [],
     kanbanAggregateOperationViewIds: [],
     calendarViewIds: [],
+    mainGroupByFieldMetadataViewIds: [],
     isLabelSyncedWithName: false,
     isUnique: false,
     objectMetadataId,
@@ -216,6 +221,7 @@ export const buildDefaultFlatFieldMetadatasForCustomObject = ({
     viewGroupIds: [],
     kanbanAggregateOperationViewIds: [],
     calendarViewIds: [],
+    mainGroupByFieldMetadataViewIds: [],
     isLabelSyncedWithName: false,
     isUnique: false,
     objectMetadataId,
@@ -252,6 +258,7 @@ export const buildDefaultFlatFieldMetadatasForCustomObject = ({
     viewGroupIds: [],
     kanbanAggregateOperationViewIds: [],
     calendarViewIds: [],
+    mainGroupByFieldMetadataViewIds: [],
     isLabelSyncedWithName: false,
     isUnique: false,
     objectMetadataId,
@@ -284,6 +291,7 @@ export const buildDefaultFlatFieldMetadatasForCustomObject = ({
   const searchVectorFieldId = v4();
   const searchVectorField: FlatFieldMetadata<FieldMetadataType.TS_VECTOR> = {
     type: FieldMetadataType.TS_VECTOR,
+    mainGroupByFieldMetadataViewIds: [],
     viewFieldIds: [],
     viewGroupIds: [],
     kanbanAggregateOperationViewIds: [],

packages/twenty-server/src/engine/metadata-modules/object-metadata/utils/build-default-relation-flat-field-metadatas-for-custom-object.util.ts

@@ -61,6 +61,7 @@ const generateSourceFlatFieldMetadata = ({
 
   return {
     calendarViewIds: [],
+    mainGroupByFieldMetadataViewIds: [],
     kanbanAggregateOperationViewIds: [],
     viewFilterIds: [],
     viewFieldIds: [],
@@ -128,6 +129,7 @@ const generateTargetFlatFieldMetadata = ({
   return {
     morphId: null,
     calendarViewIds: [],
+    mainGroupByFieldMetadataViewIds: [],
     viewFieldIds: [],
     kanbanAggregateOperationViewIds: [],
     viewFilterIds: [],

packages/twenty-server/src/engine/metadata-modules/view-filter/services/view-filter.service.ts

@@ -69,7 +69,7 @@ export class ViewFilterService {
         workspaceCustomApplicationId: workspaceCustomFlatApplication.id,
       });
 
-    const validateAndBuildResult =
+    const buildAndRunResult =
       await this.workspaceMigrationValidateBuildAndRunService.validateBuildAndRunWorkspaceMigration(
         {
           fromToAllFlatEntityMaps: {
@@ -92,13 +92,18 @@ export class ViewFilterService {
         },
       );
 
-    if (isDefined(validateAndBuildResult)) {
+    if (isDefined(buildAndRunResult)) {
       throw new WorkspaceMigrationBuilderExceptionV2(
-        validateAndBuildResult,
+        buildAndRunResult,
         'Multiple validation errors occurred while creating view filter',
       );
     }
 
+    this.flatEntityMapsCacheService.invalidateFlatEntityMaps({
+      workspaceId,
+      flatMapsKeys: ['flatViewFilterMaps'],
+    });
+
     const { flatViewFilterMaps: recomputedExistingFlatViewFilterMaps } =
       await this.flatEntityMapsCacheService.getOrRecomputeManyOrAllFlatEntityMaps(
         {

packages/twenty-server/src/engine/metadata-modules/view/dtos/inputs/create-view.input.ts

@@ -92,6 +92,11 @@ export class CreateViewInput {
   @Field(() => UUIDScalarType, { nullable: true })
   calendarFieldMetadataId?: string;
 
+  @IsOptional()
+  @IsUUID()
+  @Field(() => UUIDScalarType, { nullable: true })
+  mainGroupByFieldMetadataId?: string;
+
   @IsOptional()
   @IsEnum(ViewVisibility)
   @Field(() => ViewVisibility, {

packages/twenty-server/src/engine/metadata-modules/view/dtos/inputs/update-view.input.ts

@@ -87,4 +87,9 @@ export class UpdateViewInput {
   @IsEnum(ViewVisibility)
   @Field(() => ViewVisibility, { nullable: true })
   visibility?: ViewVisibility;
+
+  @IsOptional()
+  @IsUUID()
+  @Field(() => UUIDScalarType, { nullable: true })
+  mainGroupByFieldMetadataId?: string;
 }

packages/twenty-server/src/engine/metadata-modules/view/dtos/view.dto.ts

@@ -62,6 +62,9 @@ export class ViewDTO {
   @Field(() => UUIDScalarType, { nullable: true })
   kanbanAggregateOperationFieldMetadataId?: string | null;
 
+  @Field(() => UUIDScalarType, { nullable: true })
+  mainGroupByFieldMetadataId?: string | null;
+
   @Field(() => UUIDScalarType, { nullable: true })
   calendarFieldMetadataId?: string | null;
 

packages/twenty-server/src/engine/metadata-modules/view/entities/view.entity.ts

@@ -138,6 +138,20 @@ export class ViewEntity extends SyncableEntity implements Required<ViewEntity> {
   @JoinColumn({ name: 'calendarFieldMetadataId' })
   calendarFieldMetadata: Relation<FieldMetadataEntity>;
 
+  @Column({ nullable: true, type: 'uuid' })
+  mainGroupByFieldMetadataId: string | null;
+
+  @ManyToOne(
+    () => FieldMetadataEntity,
+    (fieldMetadata) => fieldMetadata.mainGroupByFieldMetadataViews,
+    {
+      onDelete: 'CASCADE',
+      nullable: true,
+    },
+  )
+  @JoinColumn({ name: 'mainGroupByFieldMetadataId' })
+  mainGroupByFieldMetadata: Relation<FieldMetadataEntity>;
+
   @Column({ nullable: false, type: 'uuid' })
   workspaceId: string;