Reduce complexity on groupBy query (#16803)

fixes https://github.com/twentyhq/core-team-issues/issues/2009

packages/twenty-front/src/modules/object-record/object-options-dropdown/components/ObjectOptionsDropdownHiddenRecordGroupsContent.tsx

@@ -6,6 +6,7 @@ import { useObjectOptionsDropdown } from '@/object-record/object-options-dropdow
 import { RecordGroupsVisibilityDropdownSection } from '@/object-record/record-group/components/RecordGroupsVisibilityDropdownSection';
 import { useRecordGroupVisibility } from '@/object-record/record-group/hooks/useRecordGroupVisibility';
 import { hiddenRecordGroupIdsComponentSelector } from '@/object-record/record-group/states/selectors/hiddenRecordGroupIdsComponentSelector';
+import { visibleRecordGroupIdsComponentFamilySelector } from '@/object-record/record-group/states/selectors/visibleRecordGroupIdsComponentFamilySelector';
 import { recordIndexGroupFieldMetadataItemComponentState } from '@/object-record/record-index/states/recordIndexGroupFieldMetadataComponentState';
 import { DropdownContent } from '@/ui/layout/dropdown/components/DropdownContent';
 import { DropdownMenuHeader } from '@/ui/layout/dropdown/components/DropdownMenuHeader/DropdownMenuHeader';
@@ -13,10 +14,12 @@ import { DropdownMenuHeaderLeftComponent } from '@/ui/layout/dropdown/components
 import { DropdownMenuItemsContainer } from '@/ui/layout/dropdown/components/DropdownMenuItemsContainer';
 import { DropdownMenuSeparator } from '@/ui/layout/dropdown/components/DropdownMenuSeparator';
 import { navigationMemorizedUrlState } from '@/ui/navigation/states/navigationMemorizedUrlState';
+import { useRecoilComponentFamilyValue } from '@/ui/utilities/state/component-state/hooks/useRecoilComponentFamilyValue';
 import { useRecoilComponentValue } from '@/ui/utilities/state/component-state/hooks/useRecoilComponentValue';
 import { useLingui } from '@lingui/react/macro';
 import { useLocation } from 'react-router-dom';
 import { useSetRecoilState } from 'recoil';
+import { VIEW_GROUP_VISIBLE_OPTIONS_MAX } from 'twenty-shared/constants';
 import { SettingsPath } from 'twenty-shared/types';
 import { getSettingsPath } from 'twenty-shared/utils';
 import { IconChevronLeft, IconSettings } from 'twenty-ui/display';
@@ -29,6 +32,7 @@ export const ObjectOptionsDropdownHiddenRecordGroupsContent = () => {
     objectMetadataItem,
     onContentChange,
     closeDropdown,
+    viewType,
   } = useObjectOptionsDropdown();
 
   const recordGroupFieldMetadata = useRecoilComponentValue(
@@ -39,6 +43,14 @@ export const ObjectOptionsDropdownHiddenRecordGroupsContent = () => {
     hiddenRecordGroupIdsComponentSelector,
   );
 
+  const visibleRecordGroupIds = useRecoilComponentFamilyValue(
+    visibleRecordGroupIdsComponentFamilySelector,
+    viewType,
+  );
+
+  const isVisibleLimitReached =
+    visibleRecordGroupIds.length >= VIEW_GROUP_VISIBLE_OPTIONS_MAX;
+
   const { objectNamePlural } = useObjectNamePluralFromSingular({
     objectNameSingular: objectMetadataItem.nameSingular,
   });
@@ -84,6 +96,7 @@ export const ObjectOptionsDropdownHiddenRecordGroupsContent = () => {
         isDraggable={false}
         showSubheader={false}
         showDragGrip={false}
+        isVisibleLimitReached={isVisibleLimitReached}
       />
       <DropdownMenuSeparator />
       <UndecoratedLink

packages/twenty-front/src/modules/object-record/record-group/components/RecordGroupMenuItemDraggable.tsx

@@ -3,8 +3,8 @@ import {
   type RecordGroupDefinition,
   RecordGroupDefinitionType,
 } from '@/object-record/record-group/types/RecordGroupDefinition';
-import { useRecoilValue } from 'recoil';
 import { t } from '@lingui/core/macro';
+import { useRecoilValue } from 'recoil';
 import { isDefined } from 'twenty-shared/utils';
 import { Tag } from 'twenty-ui/components';
 import { IconEye, IconEyeOff } from 'twenty-ui/display';
@@ -15,6 +15,7 @@ type RecordGroupMenuItemDraggableProps = {
   showDragGrip?: boolean;
   isDraggable?: boolean;
   onVisibilityChange: (recordGroup: RecordGroupDefinition) => void;
+  isVisibleLimitReached?: boolean;
 };
 
 export const RecordGroupMenuItemDraggable = ({
@@ -22,6 +23,7 @@ export const RecordGroupMenuItemDraggable = ({
   showDragGrip,
   isDraggable,
   onVisibilityChange,
+  isVisibleLimitReached = false,
 }: RecordGroupMenuItemDraggableProps) => {
   const recordGroup = useRecoilValue(
     recordGroupDefinitionFamilyState(recordGroupId),
@@ -35,6 +37,11 @@ export const RecordGroupMenuItemDraggable = ({
 
   const getIconButtons = (recordGroup: RecordGroupDefinition) => {
     const groupValue = recordGroup.value;
+
+    if (!recordGroup.isVisible && isVisibleLimitReached) {
+      return undefined;
+    }
+
     const iconButtons = [
       {
         Icon: recordGroup.isVisible ? IconEyeOff : IconEye,

packages/twenty-front/src/modules/object-record/record-group/components/RecordGroupsVisibilityDropdownSection.tsx

@@ -20,6 +20,7 @@ type RecordGroupsVisibilityDropdownSectionProps = {
   title: string;
   showSubheader?: boolean;
   showDragGrip: boolean;
+  isVisibleLimitReached?: boolean;
 };
 
 export const RecordGroupsVisibilityDropdownSection = ({
@@ -30,6 +31,7 @@ export const RecordGroupsVisibilityDropdownSection = ({
   title,
   showSubheader = true,
   showDragGrip,
+  isVisibleLimitReached = false,
 }: RecordGroupsVisibilityDropdownSectionProps) => {
   const handleOnDrag = (result: DropResult, provided: ResponderProvided) => {
     onDragEnd?.(result, provided);
@@ -53,6 +55,7 @@ export const RecordGroupsVisibilityDropdownSection = ({
                   onVisibilityChange={onVisibilityChange}
                   showDragGrip={showDragGrip}
                   isDraggable={isDraggable}
+                  isVisibleLimitReached={isVisibleLimitReached}
                 />
               ))
             ) : (
@@ -71,6 +74,7 @@ export const RecordGroupsVisibilityDropdownSection = ({
                             onVisibilityChange={onVisibilityChange}
                             showDragGrip={showDragGrip}
                             isDraggable={isDraggable}
+                            isVisibleLimitReached={isVisibleLimitReached}
                           />
                         }
                       />

packages/twenty-front/src/modules/object-record/record-index/hooks/useRecordIndexGroupCommonQueryVariables.ts

@@ -4,6 +4,7 @@ import { currentRecordFilterGroupsComponentState } from '@/object-record/record-
 import { useFilterValueDependencies } from '@/object-record/record-filter/hooks/useFilterValueDependencies';
 import { anyFieldFilterValueComponentState } from '@/object-record/record-filter/states/anyFieldFilterValueComponentState';
 import { currentRecordFiltersComponentState } from '@/object-record/record-filter/states/currentRecordFiltersComponentState';
+import { recordGroupDefinitionsComponentSelector } from '@/object-record/record-group/states/selectors/recordGroupDefinitionsComponentSelector';
 import { useRecordIndexContextOrThrow } from '@/object-record/record-index/contexts/RecordIndexContext';
 import { recordIndexGroupFieldMetadataItemComponentState } from '@/object-record/record-index/states/recordIndexGroupFieldMetadataComponentState';
 import { currentRecordSortsComponentState } from '@/object-record/record-sort/states/currentRecordSortsComponentState';
@@ -11,6 +12,7 @@ import { useRecoilComponentValue } from '@/ui/utilities/state/component-state/ho
 import {
   combineFilters,
   computeRecordGqlOperationFilter,
+  isDefined,
   turnAnyFieldFilterIntoRecordGqlFilter,
 } from 'twenty-shared/utils';
 
@@ -59,11 +61,38 @@ export const useRecordIndexGroupCommonQueryVariables = () => {
     additionalFieldMetadataId: recordGroupFieldMetadata?.id,
   });
 
-  const combinedFilters = combineFilters([anyFieldFilter, requestFilters]);
+  const recordGroupDefinitions = useRecoilComponentValue(
+    recordGroupDefinitionsComponentSelector,
+  );
+
+  const visibleRecordGroupDefinitions = recordGroupDefinitions.filter(
+    (recordGroupDefinition) => recordGroupDefinition.isVisible,
+  );
+
+  const recordGroupValues = visibleRecordGroupDefinitions.map(
+    (recordGroupDefinition) => recordGroupDefinition.value,
+  );
+
+  const recordGroupOptionsFilter = isDefined(recordGroupFieldMetadata)
+    ? {
+        [recordGroupFieldMetadata.name]: {
+          in: recordGroupValues,
+        },
+      }
+    : {};
+
+  const combinedFilters = combineFilters([
+    anyFieldFilter,
+    requestFilters,
+    recordGroupOptionsFilter,
+  ]);
+
+  const recordGroupsLimit = visibleRecordGroupDefinitions.length;
 
   return {
     combinedFilters,
     recordGqlFields,
     orderBy,
+    recordGroupsLimit,
   };
 };

packages/twenty-front/src/modules/object-record/record-index/hooks/useRecordIndexGroupsRecordsLazyGroupBy.ts

@@ -31,7 +31,7 @@ export const useRecordIndexGroupsRecordsLazyGroupBy = ({
   const { objectMetadataItems } = useObjectMetadataItems();
   const apolloCoreClient = useApolloCoreClient();
 
-  const { combinedFilters, orderBy, recordGqlFields } =
+  const { combinedFilters, orderBy, recordGqlFields, recordGroupsLimit } =
     useRecordIndexGroupCommonQueryVariables();
 
   const { objectPermissionsByObjectMetadataId } = useObjectPermissions();
@@ -55,11 +55,12 @@ export const useRecordIndexGroupsRecordsLazyGroupBy = ({
       recordIndexGroupsRecordGroupsGroupByQuery,
       {
         variables: {
-          filter: { ...combinedFilters },
+          filter: combinedFilters,
           groupBy: {
             ...groupByGqlInput,
           },
           orderByForRecords: orderBy,
+          limit: recordGroupsLimit,
         },
         client: apolloCoreClient,
       },

packages/twenty-server/src/engine/metadata-modules/flat-field-metadata/utils/recompute-view-groups-on-flat-field-metadata-options-update.util.ts

@@ -1,3 +1,4 @@
+import { VIEW_GROUP_VISIBLE_OPTIONS_MAX } from 'twenty-shared/constants';
 import { type EnumFieldMetadataType } from 'twenty-shared/types';
 import { isDefined } from 'twenty-shared/utils';
 import { v4 } from 'uuid';
@@ -77,16 +78,29 @@ export const recomputeViewGroupsOnFlatFieldMetadataOptionsUpdate = ({
       ),
   );
 
+  const remainingFlatViewGroups = flatViewGroups.filter(
+    (flatViewGroup) =>
+      !flatViewGroupsToDelete.some(
+        (flatViewGroupToDelete) =>
+          flatViewGroupToDelete.id === flatViewGroup.id,
+      ),
+  );
+
   const viewGroupsByViewId = reduceFlatViewGroupsByViewId({
-    flatViewGroups: flatViewGroups.filter(
-      (flatViewGroup) =>
-        !flatViewGroupsToDelete.some(
-          (flatViewGroupToDelete) =>
-            flatViewGroupToDelete.id === flatViewGroup.id,
-        ),
-    ),
+    flatViewGroups: remainingFlatViewGroups,
   });
 
+  // Count visible view groups per view to enforce the limit
+  const visibleViewGroupCountByViewId = remainingFlatViewGroups.reduce<
+    Record<string, number>
+  >((acc, flatViewGroup) => {
+    if (flatViewGroup.isVisible) {
+      acc[flatViewGroup.viewId] = (acc[flatViewGroup.viewId] ?? 0) + 1;
+    }
+
+    return acc;
+  }, {});
+
   const viewIds = Object.keys(viewGroupsByViewId.flatViewGroupRecordByViewId);
 
   const createdAt = new Date().toISOString();
@@ -103,6 +117,14 @@ export const recomputeViewGroupsOnFlatFieldMetadataOptionsUpdate = ({
           );
         }
 
+        const currentVisibleCount = visibleViewGroupCountByViewId[viewId] ?? 0;
+        const isVisible = currentVisibleCount < VIEW_GROUP_VISIBLE_OPTIONS_MAX;
+
+        // Increment the count for future iterations if this group will be visible
+        if (isVisible) {
+          visibleViewGroupCountByViewId[viewId] = currentVisibleCount + 1;
+        }
+
         const viewGroupId = v4();
 
         return {
@@ -114,7 +136,7 @@ export const recomputeViewGroupsOnFlatFieldMetadataOptionsUpdate = ({
           updatedAt: createdAt,
           deletedAt: null,
           universalIdentifier: viewGroupId,
-          isVisible: true,
+          isVisible,
           fieldValue: option.value,
           position: viewGroupHighestPosition + createdOptionIndex + 1,
           applicationId: fromFlatFieldMetadata.applicationId,

packages/twenty-server/src/engine/metadata-modules/flat-view-group/utils/compute-flat-view-groups-on-view-create.util.ts

@@ -1,3 +1,4 @@
+import { VIEW_GROUP_VISIBLE_OPTIONS_MAX } from 'twenty-shared/constants';
 import { isDefined } from 'twenty-shared/utils';
 import { v4 } from 'uuid';
 
@@ -44,7 +45,7 @@ export const computeFlatViewGroupsOnViewCreate = ({
       updatedAt: createdAt,
       deletedAt: null,
       universalIdentifier: viewGroupId,
-      isVisible: true,
+      isVisible: index < VIEW_GROUP_VISIBLE_OPTIONS_MAX,
       fieldValue: option.value,
       position: index,
       applicationId: mainGroupByFieldMetadata.applicationId,
@@ -55,6 +56,7 @@ export const computeFlatViewGroupsOnViewCreate = ({
 
   if (mainGroupByFieldMetadata.isNullable === true) {
     const emptyGroupId = v4();
+    const emptyGroupPosition = flatViewGroupsFromOptions.length;
 
     flatViewGroups.push({
       id: emptyGroupId,
@@ -64,9 +66,9 @@ export const computeFlatViewGroupsOnViewCreate = ({
       updatedAt: createdAt,
       deletedAt: null,
       universalIdentifier: emptyGroupId,
-      isVisible: true,
+      isVisible: emptyGroupPosition < VIEW_GROUP_VISIBLE_OPTIONS_MAX,
       fieldValue: '',
-      position: flatViewGroupsFromOptions.length,
+      position: emptyGroupPosition,
       applicationId: mainGroupByFieldMetadata.applicationId,
     });
   }

packages/twenty-shared/src/constants/ViewGroupVisibleOptionsMax.ts

@@ -0,0 +1 @@
+export const VIEW_GROUP_VISIBLE_OPTIONS_MAX = 40;

packages/twenty-shared/src/constants/index.ts

@@ -33,3 +33,4 @@ export { RELATION_NESTED_QUERY_KEYWORDS } from './RelationNestedQueriesKeyword';
 export { STANDARD_OBJECT_RECORDS_UNDER_OBJECT_RECORDS_PERMISSIONS } from './StandardObjectRecordsUnderObjectRecordsPermissions';
 export { TWENTY_COMPANIES_BASE_URL } from './TwentyCompaniesBaseUrl';
 export { TWENTY_ICONS_BASE_URL } from './TwentyIconsBaseUrl';
+export { VIEW_GROUP_VISIBLE_OPTIONS_MAX } from './ViewGroupVisibleOptionsMax';