Aggregated queries #1 (#8345)

First step of https://github.com/twentyhq/twenty/issues/6868

Adds min.., max.. queries for DATETIME fields
adds min.., max.., avg.., sum.. queries for NUMBER fields 

(count distinct operation and composite fields such as CURRENCY handling
will be dealt with in a future PR)

<img width="1422" alt="Capture d’écran 2024-11-06 à 15 48 46"
src="https://github.com/user-attachments/assets/4bcdece0-ad3e-4536-9720-fe4044a36719">

---------

Co-authored-by: Charles Bochet <charles@twenty.com>
Co-authored-by: Weiko <corentin@twenty.com>

packages/twenty-server/src/engine/api/graphql/graphql-query-runner/graphql-query-parsers/graphql-query-filter/graphql-query-filter-condition.parser.ts

@@ -5,27 +5,27 @@ import {
   WhereExpressionBuilder,
 } from 'typeorm';
 
-import { RecordFilter } from 'src/engine/api/graphql/workspace-query-builder/interfaces/record.interface';
+import { ObjectRecordFilter } from 'src/engine/api/graphql/workspace-query-builder/interfaces/object-record.interface';
 
-import { FieldMetadataMap } from 'src/engine/metadata-modules/utils/generate-object-metadata-map.util';
+import { FieldMetadataMap } from 'src/engine/metadata-modules/types/field-metadata-map';
 
 import { GraphqlQueryFilterFieldParser } from './graphql-query-filter-field.parser';
 
 export class GraphqlQueryFilterConditionParser {
-  private fieldMetadataMap: FieldMetadataMap;
+  private fieldMetadataMapByName: FieldMetadataMap;
   private queryFilterFieldParser: GraphqlQueryFilterFieldParser;
 
-  constructor(fieldMetadataMap: FieldMetadataMap) {
-    this.fieldMetadataMap = fieldMetadataMap;
+  constructor(fieldMetadataMapByName: FieldMetadataMap) {
+    this.fieldMetadataMapByName = fieldMetadataMapByName;
     this.queryFilterFieldParser = new GraphqlQueryFilterFieldParser(
-      this.fieldMetadataMap,
+      this.fieldMetadataMapByName,
     );
   }
 
   public parse(
     queryBuilder: SelectQueryBuilder<any>,
     objectNameSingular: string,
-    filter: Partial<RecordFilter>,
+    filter: Partial<ObjectRecordFilter>,
   ): SelectQueryBuilder<any> {
     if (!filter || Object.keys(filter).length === 0) {
       return queryBuilder;
@@ -50,7 +50,7 @@ export class GraphqlQueryFilterConditionParser {
     switch (key) {
       case 'and': {
         const andWhereCondition = new Brackets((qb) => {
-          value.forEach((filter: RecordFilter, index: number) => {
+          value.forEach((filter: ObjectRecordFilter, index: number) => {
             const whereCondition = new Brackets((qb2) => {
               Object.entries(filter).forEach(
                 ([subFilterkey, subFilterValue], index) => {
@@ -82,7 +82,7 @@ export class GraphqlQueryFilterConditionParser {
       }
       case 'or': {
         const orWhereCondition = new Brackets((qb) => {
-          value.forEach((filter: RecordFilter, index: number) => {
+          value.forEach((filter: ObjectRecordFilter, index: number) => {
             const whereCondition = new Brackets((qb2) => {
               Object.entries(filter).forEach(
                 ([subFilterkey, subFilterValue], index) => {

packages/twenty-server/src/engine/api/graphql/graphql-query-runner/graphql-query-parsers/graphql-query-filter/graphql-query-filter-field.parser.ts

@@ -9,17 +9,17 @@ import {
 import { computeWhereConditionParts } from 'src/engine/api/graphql/graphql-query-runner/utils/compute-where-condition-parts';
 import { compositeTypeDefinitions } from 'src/engine/metadata-modules/field-metadata/composite-types';
 import { isCompositeFieldMetadataType } from 'src/engine/metadata-modules/field-metadata/utils/is-composite-field-metadata-type.util';
-import { FieldMetadataMap } from 'src/engine/metadata-modules/utils/generate-object-metadata-map.util';
+import { FieldMetadataMap } from 'src/engine/metadata-modules/types/field-metadata-map';
 import { CompositeFieldMetadataType } from 'src/engine/metadata-modules/workspace-migration/factories/composite-column-action.factory';
 import { capitalize } from 'src/utils/capitalize';
 
 const ARRAY_OPERATORS = ['in', 'contains', 'not_contains'];
 
 export class GraphqlQueryFilterFieldParser {
-  private fieldMetadataMap: FieldMetadataMap;
+  private fieldMetadataMapByName: FieldMetadataMap;
 
-  constructor(fieldMetadataMap: FieldMetadataMap) {
-    this.fieldMetadataMap = fieldMetadataMap;
+  constructor(fieldMetadataMapByName: FieldMetadataMap) {
+    this.fieldMetadataMapByName = fieldMetadataMapByName;
   }
 
   public parse(
@@ -29,7 +29,7 @@ export class GraphqlQueryFilterFieldParser {
     filterValue: any,
     isFirst = false,
   ): void {
-    const fieldMetadata = this.fieldMetadataMap[`${key}`];
+    const fieldMetadata = this.fieldMetadataMapByName[`${key}`];
 
     if (!fieldMetadata) {
       throw new Error(`Field metadata not found for field: ${key}`);

packages/twenty-server/src/engine/api/graphql/graphql-query-runner/graphql-query-parsers/graphql-query-order/graphql-query-order.parser.ts

@@ -1,7 +1,7 @@
 import {
+  ObjectRecordOrderBy,
   OrderByDirection,
-  RecordOrderBy,
-} from 'src/engine/api/graphql/workspace-query-builder/interfaces/record.interface';
+} from 'src/engine/api/graphql/workspace-query-builder/interfaces/object-record.interface';
 import { FieldMetadataInterface } from 'src/engine/metadata-modules/field-metadata/interfaces/field-metadata.interface';
 
 import {
@@ -10,25 +10,25 @@ import {
 } from 'src/engine/api/graphql/graphql-query-runner/errors/graphql-query-runner.exception';
 import { compositeTypeDefinitions } from 'src/engine/metadata-modules/field-metadata/composite-types';
 import { isCompositeFieldMetadataType } from 'src/engine/metadata-modules/field-metadata/utils/is-composite-field-metadata-type.util';
-import { FieldMetadataMap } from 'src/engine/metadata-modules/utils/generate-object-metadata-map.util';
+import { FieldMetadataMap } from 'src/engine/metadata-modules/types/field-metadata-map';
 import { CompositeFieldMetadataType } from 'src/engine/metadata-modules/workspace-migration/factories/composite-column-action.factory';
 import { capitalize } from 'src/utils/capitalize';
 export class GraphqlQueryOrderFieldParser {
-  private fieldMetadataMap: FieldMetadataMap;
+  private fieldMetadataMapByName: FieldMetadataMap;
 
-  constructor(fieldMetadataMap: FieldMetadataMap) {
-    this.fieldMetadataMap = fieldMetadataMap;
+  constructor(fieldMetadataMapByName: FieldMetadataMap) {
+    this.fieldMetadataMapByName = fieldMetadataMapByName;
   }
 
   parse(
-    orderBy: RecordOrderBy,
+    orderBy: ObjectRecordOrderBy,
     objectNameSingular: string,
     isForwardPagination = true,
   ): Record<string, string> {
     return orderBy.reduce(
       (acc, item) => {
         Object.entries(item).forEach(([key, value]) => {
-          const fieldMetadata = this.fieldMetadataMap[key];
+          const fieldMetadata = this.fieldMetadataMapByName[key];
 
           if (!fieldMetadata || value === undefined) {
             throw new GraphqlQueryRunnerException(

packages/twenty-server/src/engine/api/graphql/graphql-query-runner/graphql-query-parsers/graphql-query-selected-fields/graphql-selected-fields-aggregate.parser.ts

@@ -0,0 +1,30 @@
+import { FieldMetadataInterface } from 'src/engine/metadata-modules/field-metadata/interfaces/field-metadata.interface';
+
+import { GraphqlQuerySelectedFieldsResult } from 'src/engine/api/graphql/graphql-query-runner/graphql-query-parsers/graphql-query-selected-fields/graphql-selected-fields.parser';
+import {
+  AggregationField,
+  getAvailableAggregationsFromObjectFields,
+} from 'src/engine/api/graphql/workspace-schema-builder/utils/get-available-aggregations-from-object-fields.util';
+
+export class GraphqlQuerySelectedFieldsAggregateParser {
+  parse(
+    graphqlSelectedFields: Partial<Record<string, any>>,
+    fieldMetadataMapByName: Record<string, FieldMetadataInterface>,
+    accumulator: GraphqlQuerySelectedFieldsResult,
+  ): void {
+    const availableAggregations: Record<string, AggregationField> =
+      getAvailableAggregationsFromObjectFields(
+        Object.values(fieldMetadataMapByName),
+      );
+
+    for (const selectedField of Object.keys(graphqlSelectedFields)) {
+      const selectedAggregation = availableAggregations[selectedField];
+
+      if (!selectedAggregation) {
+        continue;
+      }
+
+      accumulator.aggregate[selectedField] = selectedAggregation;
+    }
+  }
+}

packages/twenty-server/src/engine/api/graphql/graphql-query-runner/graphql-query-parsers/graphql-query-selected-fields/graphql-selected-fields-relation.parser.ts

@@ -1,43 +1,47 @@
 import { FieldMetadataInterface } from 'src/engine/metadata-modules/field-metadata/interfaces/field-metadata.interface';
 
-import { GraphqlQuerySelectedFieldsParser } from 'src/engine/api/graphql/graphql-query-runner/graphql-query-parsers/graphql-query-selected-fields/graphql-selected-fields.parser';
+import {
+  GraphqlQuerySelectedFieldsParser,
+  GraphqlQuerySelectedFieldsResult,
+} from 'src/engine/api/graphql/graphql-query-runner/graphql-query-parsers/graphql-query-selected-fields/graphql-selected-fields.parser';
 import { getRelationObjectMetadata } from 'src/engine/api/graphql/graphql-query-runner/utils/get-relation-object-metadata.util';
-import { ObjectMetadataMap } from 'src/engine/metadata-modules/utils/generate-object-metadata-map.util';
+import { ObjectMetadataMaps } from 'src/engine/metadata-modules/types/object-metadata-maps';
 
 export class GraphqlQuerySelectedFieldsRelationParser {
-  private objectMetadataMap: ObjectMetadataMap;
+  private objectMetadataMaps: ObjectMetadataMaps;
 
-  constructor(objectMetadataMap: ObjectMetadataMap) {
-    this.objectMetadataMap = objectMetadataMap;
+  constructor(objectMetadataMaps: ObjectMetadataMaps) {
+    this.objectMetadataMaps = objectMetadataMaps;
   }
 
   parseRelationField(
     fieldMetadata: FieldMetadataInterface,
     fieldKey: string,
     fieldValue: any,
-    result: { select: Record<string, any>; relations: Record<string, any> },
+    accumulator: GraphqlQuerySelectedFieldsResult,
   ): void {
     if (!fieldValue || typeof fieldValue !== 'object') {
       return;
     }
 
-    result.relations[fieldKey] = true;
+    accumulator.relations[fieldKey] = true;
 
     const referencedObjectMetadata = getRelationObjectMetadata(
       fieldMetadata,
-      this.objectMetadataMap,
+      this.objectMetadataMaps,
     );
 
-    const relationFields = referencedObjectMetadata.fields;
+    const relationFields = referencedObjectMetadata.fieldsByName;
     const fieldParser = new GraphqlQuerySelectedFieldsParser(
-      this.objectMetadataMap,
+      this.objectMetadataMaps,
     );
-    const subResult = fieldParser.parse(fieldValue, relationFields);
+    const relationAccumulator = fieldParser.parse(fieldValue, relationFields);
 
-    result.select[fieldKey] = {
+    accumulator.select[fieldKey] = {
       id: true,
-      ...subResult.select,
+      ...relationAccumulator.select,
     };
-    result.relations[fieldKey] = subResult.relations;
+    accumulator.relations[fieldKey] = relationAccumulator.relations;
+    accumulator.aggregate[fieldKey] = relationAccumulator.aggregate;
   }
 }

packages/twenty-server/src/engine/api/graphql/graphql-query-runner/graphql-query-parsers/graphql-query-selected-fields/graphql-selected-fields.parser.ts

@@ -1,59 +1,71 @@
 import { FieldMetadataInterface } from 'src/engine/metadata-modules/field-metadata/interfaces/field-metadata.interface';
 
-import {
-  GraphqlQueryRunnerException,
-  GraphqlQueryRunnerExceptionCode,
-} from 'src/engine/api/graphql/graphql-query-runner/errors/graphql-query-runner.exception';
+import { GraphqlQuerySelectedFieldsAggregateParser } from 'src/engine/api/graphql/graphql-query-runner/graphql-query-parsers/graphql-query-selected-fields/graphql-selected-fields-aggregate.parser';
 import { GraphqlQuerySelectedFieldsRelationParser } from 'src/engine/api/graphql/graphql-query-runner/graphql-query-parsers/graphql-query-selected-fields/graphql-selected-fields-relation.parser';
 import { compositeTypeDefinitions } from 'src/engine/metadata-modules/field-metadata/composite-types';
 import { isCompositeFieldMetadataType } from 'src/engine/metadata-modules/field-metadata/utils/is-composite-field-metadata-type.util';
-import { ObjectMetadataMap } from 'src/engine/metadata-modules/utils/generate-object-metadata-map.util';
+import { ObjectMetadataMaps } from 'src/engine/metadata-modules/types/object-metadata-maps';
 import { CompositeFieldMetadataType } from 'src/engine/metadata-modules/workspace-migration/factories/composite-column-action.factory';
 import { isRelationFieldMetadataType } from 'src/engine/utils/is-relation-field-metadata-type.util';
 import { capitalize } from 'src/utils/capitalize';
-import { isPlainObject } from 'src/utils/is-plain-object';
+
+export type GraphqlQuerySelectedFieldsResult = {
+  select: Record<string, any>;
+  relations: Record<string, any>;
+  aggregate: Record<string, any>;
+};
 
 export class GraphqlQuerySelectedFieldsParser {
   private graphqlQuerySelectedFieldsRelationParser: GraphqlQuerySelectedFieldsRelationParser;
+  private aggregateParser: GraphqlQuerySelectedFieldsAggregateParser;
 
-  constructor(objectMetadataMap: ObjectMetadataMap) {
+  constructor(objectMetadataMaps: ObjectMetadataMaps) {
     this.graphqlQuerySelectedFieldsRelationParser =
-      new GraphqlQuerySelectedFieldsRelationParser(objectMetadataMap);
+      new GraphqlQuerySelectedFieldsRelationParser(objectMetadataMaps);
+    this.aggregateParser = new GraphqlQuerySelectedFieldsAggregateParser();
   }
 
   parse(
     graphqlSelectedFields: Partial<Record<string, any>>,
-    fieldMetadataMap: Record<string, FieldMetadataInterface>,
-  ): { select: Record<string, any>; relations: Record<string, any> } {
-    const result: {
-      select: Record<string, any>;
-      relations: Record<string, any>;
-    } = {
+    fieldMetadataMapByName: Record<string, FieldMetadataInterface>,
+  ): GraphqlQuerySelectedFieldsResult {
+    const accumulator: GraphqlQuerySelectedFieldsResult = {
       select: {},
       relations: {},
+      aggregate: {},
     };
 
+    if (this.isRootConnection(graphqlSelectedFields)) {
+      this.parseConnectionField(
+        graphqlSelectedFields,
+        fieldMetadataMapByName,
+        accumulator,
+      );
+
+      return accumulator;
+    }
+
+    this.parseRecordField(
+      graphqlSelectedFields,
+      fieldMetadataMapByName,
+      accumulator,
+    );
+
+    return accumulator;
+  }
+
+  private parseRecordField(
+    graphqlSelectedFields: Partial<Record<string, any>>,
+    fieldMetadataMapByName: Record<string, FieldMetadataInterface>,
+    accumulator: GraphqlQuerySelectedFieldsResult,
+  ): void {
     for (const [fieldKey, fieldValue] of Object.entries(
       graphqlSelectedFields,
     )) {
-      if (this.shouldNotParseField(fieldKey)) {
-        continue;
-      }
-      if (this.isConnectionField(fieldKey, fieldValue)) {
-        const subResult = this.parse(fieldValue, fieldMetadataMap);
-
-        Object.assign(result.select, subResult.select);
-        Object.assign(result.relations, subResult.relations);
-        continue;
-      }
-
-      const fieldMetadata = fieldMetadataMap[fieldKey];
+      const fieldMetadata = fieldMetadataMapByName[fieldKey];
 
       if (!fieldMetadata) {
-        throw new GraphqlQueryRunnerException(
-          `Field "${fieldKey}" does not exist or is not selectable`,
-          GraphqlQueryRunnerExceptionCode.FIELD_NOT_FOUND,
-        );
+        continue;
       }
 
       if (isRelationFieldMetadataType(fieldMetadata.type)) {
@@ -61,7 +73,7 @@ export class GraphqlQuerySelectedFieldsParser {
           fieldMetadata,
           fieldKey,
           fieldValue,
-          result,
+          accumulator,
         );
       } else if (isCompositeFieldMetadataType(fieldMetadata.type)) {
         const compositeResult = this.parseCompositeField(
@@ -69,23 +81,33 @@ export class GraphqlQuerySelectedFieldsParser {
           fieldValue,
         );
 
-        Object.assign(result.select, compositeResult);
+        Object.assign(accumulator.select, compositeResult);
       } else {
-        result.select[fieldKey] = true;
+        accumulator.select[fieldKey] = true;
       }
     }
-
-    return result;
   }
 
-  private isConnectionField(fieldKey: string, fieldValue: any): boolean {
-    return ['edges', 'node'].includes(fieldKey) && isPlainObject(fieldValue);
-  }
-
-  private shouldNotParseField(fieldKey: string): boolean {
-    return ['__typename', 'totalCount', 'pageInfo', 'cursor'].includes(
-      fieldKey,
+  private parseConnectionField(
+    graphqlSelectedFields: Partial<Record<string, any>>,
+    fieldMetadataMapByName: Record<string, FieldMetadataInterface>,
+    accumulator: GraphqlQuerySelectedFieldsResult,
+  ): void {
+    this.aggregateParser.parse(
+      graphqlSelectedFields,
+      fieldMetadataMapByName,
+      accumulator,
     );
+
+    const node = graphqlSelectedFields.edges.node;
+
+    this.parseRecordField(node, fieldMetadataMapByName, accumulator);
+  }
+
+  private isRootConnection(
+    graphqlSelectedFields: Partial<Record<string, any>>,
+  ): boolean {
+    return Object.keys(graphqlSelectedFields).includes('edges');
   }
 
   private parseCompositeField(

packages/twenty-server/src/engine/api/graphql/graphql-query-runner/graphql-query-parsers/graphql-query.parser.ts

@@ -6,43 +6,44 @@ import {
 } from 'typeorm';
 
 import {
-  RecordFilter,
-  RecordOrderBy,
-} from 'src/engine/api/graphql/workspace-query-builder/interfaces/record.interface';
+  ObjectRecordFilter,
+  ObjectRecordOrderBy,
+} from 'src/engine/api/graphql/workspace-query-builder/interfaces/object-record.interface';
 
 import { GraphqlQueryFilterConditionParser } from 'src/engine/api/graphql/graphql-query-runner/graphql-query-parsers/graphql-query-filter/graphql-query-filter-condition.parser';
 import { GraphqlQueryOrderFieldParser } from 'src/engine/api/graphql/graphql-query-runner/graphql-query-parsers/graphql-query-order/graphql-query-order.parser';
-import { GraphqlQuerySelectedFieldsParser } from 'src/engine/api/graphql/graphql-query-runner/graphql-query-parsers/graphql-query-selected-fields/graphql-selected-fields.parser';
 import {
-  FieldMetadataMap,
-  ObjectMetadataMap,
-  ObjectMetadataMapItem,
-} from 'src/engine/metadata-modules/utils/generate-object-metadata-map.util';
+  GraphqlQuerySelectedFieldsParser,
+  GraphqlQuerySelectedFieldsResult,
+} from 'src/engine/api/graphql/graphql-query-runner/graphql-query-parsers/graphql-query-selected-fields/graphql-selected-fields.parser';
+import { FieldMetadataMap } from 'src/engine/metadata-modules/types/field-metadata-map';
+import { ObjectMetadataItemWithFieldMaps } from 'src/engine/metadata-modules/types/object-metadata-item-with-field-maps';
+import { ObjectMetadataMaps } from 'src/engine/metadata-modules/types/object-metadata-maps';
 
 export class GraphqlQueryParser {
-  private fieldMetadataMap: FieldMetadataMap;
-  private objectMetadataMap: ObjectMetadataMap;
+  private fieldMetadataMapByName: FieldMetadataMap;
+  private objectMetadataMaps: ObjectMetadataMaps;
   private filterConditionParser: GraphqlQueryFilterConditionParser;
   private orderFieldParser: GraphqlQueryOrderFieldParser;
 
   constructor(
-    fieldMetadataMap: FieldMetadataMap,
-    objectMetadataMap: ObjectMetadataMap,
+    fieldMetadataMapByName: FieldMetadataMap,
+    objectMetadataMaps: ObjectMetadataMaps,
   ) {
-    this.objectMetadataMap = objectMetadataMap;
-    this.fieldMetadataMap = fieldMetadataMap;
+    this.objectMetadataMaps = objectMetadataMaps;
+    this.fieldMetadataMapByName = fieldMetadataMapByName;
     this.filterConditionParser = new GraphqlQueryFilterConditionParser(
-      this.fieldMetadataMap,
+      this.fieldMetadataMapByName,
     );
     this.orderFieldParser = new GraphqlQueryOrderFieldParser(
-      this.fieldMetadataMap,
+      this.fieldMetadataMapByName,
     );
   }
 
   public applyFilterToBuilder(
     queryBuilder: SelectQueryBuilder<any>,
     objectNameSingular: string,
-    recordFilter: Partial<RecordFilter>,
+    recordFilter: Partial<ObjectRecordFilter>,
   ): SelectQueryBuilder<any> {
     return this.filterConditionParser.parse(
       queryBuilder,
@@ -53,7 +54,7 @@ export class GraphqlQueryParser {
 
   public applyDeletedAtToBuilder(
     queryBuilder: SelectQueryBuilder<any>,
-    recordFilter: RecordFilter,
+    recordFilter: ObjectRecordFilter,
   ): SelectQueryBuilder<any> {
     if (this.checkForDeletedAtFilter(recordFilter)) {
       queryBuilder.withDeleted();
@@ -90,7 +91,7 @@ export class GraphqlQueryParser {
 
   public applyOrderToBuilder(
     queryBuilder: SelectQueryBuilder<any>,
-    orderBy: RecordOrderBy,
+    orderBy: ObjectRecordOrderBy,
     objectNameSingular: string,
     isForwardPagination = true,
   ): SelectQueryBuilder<any> {
@@ -104,11 +105,12 @@ export class GraphqlQueryParser {
   }
 
   public parseSelectedFields(
-    parentObjectMetadata: ObjectMetadataMapItem,
+    parentObjectMetadata: ObjectMetadataItemWithFieldMaps,
     graphqlSelectedFields: Partial<Record<string, any>>,
-  ): { select: Record<string, any>; relations: Record<string, any> } {
+  ): GraphqlQuerySelectedFieldsResult {
     const parentFields =
-      this.objectMetadataMap[parentObjectMetadata.nameSingular]?.fields;
+      this.objectMetadataMaps.byNameSingular[parentObjectMetadata.nameSingular]
+        ?.fieldsByName;
 
     if (!parentFields) {
       throw new Error(
@@ -117,7 +119,7 @@ export class GraphqlQueryParser {
     }
 
     const selectedFieldsParser = new GraphqlQuerySelectedFieldsParser(
-      this.objectMetadataMap,
+      this.objectMetadataMaps,
     );
 
     return selectedFieldsParser.parse(graphqlSelectedFields, parentFields);