Improve board experience 🖼️ (#16063)

This PR improves the general UX and DX of boards, by modifying the query effect to only use paged group by queries. In this PR we implement two more things in the backend for group by queries : - Fixed ORDER BY in the PARTITION BY sub-query (this wasn't working because it was applied in the main query, so it sorted randomly picked records, which was a correct sort on an incorrect dataset returned by the sub-query) - Added offset paging in PARTITION BY Miscellaneous, various bug fixes and improvements along the way : - Throttled loading of cards to avoid React freeze - Handling of drag & drop - Handling of create / delete / update - Reworked skeleton (the library slows down a lot with hundreds of skeleton for a spinning effect that is hardly noticed) - Fixed refetch of aggregate queries (I included the new group by aggregates query we use in the existing refetch mechanism) - Re-trigger queries on filters and sorts changes - Unselect all record ids when deleting / restoring / detroying - Fetch only groups that still have records to lighten the group by query. # What remains to be done This is still a naïve fetch more implementation that will work for a few fetch more rounds, but if you scroll and load say 200 cards per column on a board, React will re-render all 200 cards of each column each time. We would probably need to virtualize the board with paged queries as we did for the table, this could be done after this PR but seems less urgent. What's nice is that this new query pattern is well designed for virtualization also, drawing from our experience with table virtualization, and adapted to a multi-column request pattern, like a 2:2 matrix of records, for our boards. So the remaining work would be to design a UI solution for virtualizing this matrix of records, which could be quite different from our table virtualization mechanism.
2025-12-02 11:09:29 +01:00
parent 68c429a54a
commit 2691222d5f
86 changed files with 1782 additions and 612 deletions
@@ -1,3 +1,5 @@
+import { isNonEmptyString } from '@sniptt/guards';
+import { isDefined } from 'class-validator';
 import { type OrderByWithGroupBy } from 'twenty-shared/types';
 import { type FindOptionsWhere, type ObjectLiteral } from 'typeorm';

@@ -115,6 +117,36 @@ export class GraphqlQueryParser {
    return queryBuilder.orderBy(parsedOrderBys);
  }

+  public getOrderByRawSQL(
+    orderBy: ObjectRecordOrderBy | OrderByWithGroupBy,
+    objectNameSingular: string,
+    isForwardPagination = true,
+  ): string {
+    const parsedOrderBys = this.orderFieldParser.parse(
+      orderBy as ObjectRecordOrderBy,
+      objectNameSingular,
+      isForwardPagination,
+    );
+
+    const orderByRawSQLClauseArray = Object.entries(parsedOrderBys).map(
+      ([orderByField, orderByCondition]) => {
+        const nullsCondition = isDefined(orderByCondition.nulls)
+          ? ` ${orderByCondition.nulls}`
+          : '';
+
+        return `${orderByField} ${orderByCondition.order}${nullsCondition}`;
+      },
+    );
+
+    const orderByRawSQLString = orderByRawSQLClauseArray.join(', ');
+
+    const orderByCompleteSQLClause = isNonEmptyString(orderByRawSQLString)
+      ? `ORDER BY ${orderByRawSQLString}`
+      : '';
+
+    return orderByCompleteSQLClause;
+  }
+
  public applyGroupByOrderToBuilder(
    // eslint-disable-next-line @typescript-eslint/no-explicit-any
    queryBuilder: WorkspaceSelectQueryBuilder<any>,
@@ -1,5 +1,6 @@
 import { Inject, Injectable } from '@nestjs/common';

+import { isNonEmptyString } from '@sniptt/guards';
 import isEmpty from 'lodash.isempty';
 import { ObjectRecord } from 'twenty-shared/types';
 import { isDefined } from 'twenty-shared/utils';
@@ -44,6 +45,7 @@ export class GroupByWithRecordsService {
    queryRunnerContext,
    orderByForRecords,
    groupLimit,
+    offsetForRecords,
  }: {
    queryBuilderWithGroupBy: WorkspaceSelectQueryBuilder<ObjectLiteral>;
    queryBuilderWithFiltersAndWithoutGroupBy: WorkspaceSelectQueryBuilder<ObjectLiteral>;
@@ -52,6 +54,7 @@ export class GroupByWithRecordsService {
    queryRunnerContext: CommonExtendedQueryRunnerContext;
    orderByForRecords: ObjectRecordOrderBy;
    groupLimit?: number;
+    offsetForRecords?: number;
  }): Promise<CommonGroupByOutputItem[]> {
    const effectiveGroupLimit = getGroupLimit(groupLimit);

@@ -91,16 +94,21 @@ export class GroupByWithRecordsService {
      flatObjectMetadata,
      flatObjectMetadataMaps,
      flatFieldMetadataMaps,
+      offsetForRecords,
    });

    const recordsResult = await queryBuilderWithPartitionBy.getRawMany();

+    const allRecords = recordsResult
+      .flatMap((group) => group.records)
+      .filter(isDefined);
+
    if (!isEmpty(selectedFieldsResult.relations)) {
      await this.processNestedRelationsHelper.processNestedRelations({
        flatObjectMetadataMaps,
        flatFieldMetadataMaps,
        parentObjectMetadataItem: flatObjectMetadata,
-        parentObjectRecords: recordsResult.flatMap((group) => group.records),
+        parentObjectRecords: allRecords,
        parentObjectRecordsAggregatedValues: {},
        relations: selectedFieldsResult.relations,
        aggregate: selectedFieldsResult.aggregate,
@@ -141,6 +149,7 @@ export class GroupByWithRecordsService {
    flatObjectMetadata,
    flatObjectMetadataMaps,
    flatFieldMetadataMaps,
+    offsetForRecords = 0,
  }: {
    queryBuilderForSubQuery: WorkspaceSelectQueryBuilder<ObjectLiteral>;
    columnsToSelect: Record<string, boolean>;
@@ -151,11 +160,8 @@ export class GroupByWithRecordsService {
    flatObjectMetadata: FlatObjectMetadata;
    flatObjectMetadataMaps: FlatEntityMaps<FlatObjectMetadata>;
    flatFieldMetadataMaps: FlatEntityMaps<FlatFieldMetadata>;
+    offsetForRecords?: number;
  }): WorkspaceSelectQueryBuilder<ObjectLiteral> {
-    const groupByExpressions = groupByDefinitions
-      .map((def) => def.expression)
-      .join(', ');
-
    const groupByAliases = groupByDefinitions
      .map((def) => `"${def.alias}"`)
      .join(', ');
@@ -179,9 +185,17 @@ export class GroupByWithRecordsService {
    const subQuery = queryBuilderForSubQuery
      .select(recordSelectWithAlias)
      .addSelect(groupBySelectWithAlias)
-      .addSelect(`ROW_NUMBER() OVER (PARTITION BY ${groupByExpressions})`, 'rn')
      .andWhere(groupConditions);

+    this.applyPartitionByToBuilder({
+      groupByDefinitions,
+      flatObjectMetadata,
+      flatObjectMetadataMaps,
+      flatFieldMetadataMaps,
+      orderByForRecords,
+      queryBuilder: subQuery,
+    });
+
    if (!isEmpty(orderByForRecords)) {
      const graphqlQueryParser = new GraphqlQueryParser(
        flatObjectMetadata,
@@ -198,16 +212,17 @@ export class GroupByWithRecordsService {

    let mainQueryQueryBuilder = repository.createQueryBuilder();

+    const pageStart = offsetForRecords;
+    const pageEnd = offsetForRecords + RECORDS_PER_GROUP_LIMIT;
+
    const mainQuery = mainQueryQueryBuilder
      .from(`(${subQuery.getQuery()})`, 'ranked_records')
      .setParameters(queryBuilderForSubQuery.expressionMap.parameters)
-      .where('rn <= :recordsPerGroupLimit', {
-        recordsPerGroupLimit: RECORDS_PER_GROUP_LIMIT,
-      })
+
      .select(groupByAliases)
      .addSelect(
        `JSON_AGG(
-        CASE WHEN rn <= ${RECORDS_PER_GROUP_LIMIT} THEN
+        CASE WHEN record_row_number > ${pageStart} AND record_row_number <= ${pageEnd} THEN
          JSON_BUILD_OBJECT(
            ${[
              ...Object.keys(columnsToSelect).map(
@@ -219,7 +234,7 @@ export class GroupByWithRecordsService {
            ].join(',\n              ')}
          )
        END
-      ) FILTER (WHERE rn <= ${RECORDS_PER_GROUP_LIMIT})`,
+      ) FILTER (WHERE record_row_number > ${pageStart} AND record_row_number <= ${pageEnd})`,
        'records',
      )
      .groupBy(groupByAliases);
@@ -232,6 +247,53 @@ export class GroupByWithRecordsService {
    return mainQuery as WorkspaceSelectQueryBuilder<ObjectLiteral>;
  }

+  private applyPartitionByToBuilder({
+    groupByDefinitions,
+    flatObjectMetadata,
+    flatObjectMetadataMaps,
+    flatFieldMetadataMaps,
+    orderByForRecords,
+    queryBuilder,
+  }: {
+    queryBuilder: WorkspaceSelectQueryBuilder<ObjectLiteral>;
+    groupByDefinitions: GroupByDefinition[];
+    orderByForRecords: ObjectRecordOrderBy;
+    flatObjectMetadata: FlatObjectMetadata;
+    flatObjectMetadataMaps: FlatEntityMaps<FlatObjectMetadata>;
+    flatFieldMetadataMaps: FlatEntityMaps<FlatFieldMetadata>;
+  }) {
+    const groupByExpressions = groupByDefinitions
+      .map((def) => def.expression)
+      .join(', ');
+
+    const hasOrderByForRecords = !isEmpty(orderByForRecords);
+
+    if (hasOrderByForRecords) {
+      const graphqlQueryParser = new GraphqlQueryParser(
+        flatObjectMetadata,
+        flatObjectMetadataMaps,
+        flatFieldMetadataMaps,
+      );
+
+      const orderByRawSQL = graphqlQueryParser.getOrderByRawSQL(
+        orderByForRecords,
+        flatObjectMetadata.nameSingular,
+      );
+
+      if (isNonEmptyString(orderByRawSQL)) {
+        return queryBuilder.addSelect(
+          `ROW_NUMBER() OVER (PARTITION BY ${groupByExpressions} ${orderByRawSQL})`,
+          'record_row_number',
+        );
+      }
+    }
+
+    return queryBuilder.addSelect(
+      `ROW_NUMBER() OVER (PARTITION BY ${groupByExpressions})`,
+      'record_row_number',
+    );
+  }
+
  private buildGroupConditions(
    groupsResult: Array<Record<string, unknown>>,
    groupByDefinitions: GroupByDefinition[],
@@ -64,6 +64,7 @@ export interface GroupByResolverArgs<Filter = ObjectRecordFilter> {
  viewId?: string;
  orderBy?: OrderByWithGroupBy;
  orderByForRecords?: ObjectRecordOrderBy;
+  offsetForRecords?: number;
  limit?: number;
 }

@@ -190,6 +190,10 @@ export const getResolverArgs = (
          type: GraphQLInt,
          isNullable: true,
        },
+        offsetForRecords: {
+          type: GraphQLInt,
+          isNullable: true,
+        },
      };
    default:
      throw new Error(`Unknown resolver type: ${type}`);