From d7bda9576f48b554fcaf65a36cae0d657de31194 Mon Sep 17 00:00:00 2001 From: Marie <51697796+ijreilly@users.noreply.github.com> Date: Fri, 24 Oct 2025 17:30:57 +0200 Subject: [PATCH] [groupBy] Load record within groups (without relations) (#15307) First step of https://github.com/twentyhq/core-team-issues/issues/1726. I will handle relations in another PR. Another ticket is planned to allow for sorting among the records. When querying records we have set the number of maximum groups to 50, and of maximum records par group to 10. image --- .../common-group-by-query-runner.service.ts | 125 +++++-- .../api/common/core-common-api.module.ts | 2 + .../types/common-group-by-output-item.type.ts | 10 +- .../common/types/common-query-args.type.ts | 2 + .../graphql-query-runner.module.ts | 2 - ...graphql-query-group-by-resolver.service.ts | 249 ------------- .../types/group-by-definition.types.ts | 8 + ...ult-with-group-by-dimension-values.util.ts | 104 ++++-- .../utils/parse-group-by-args.util.ts | 4 + .../services/group-by-with-records.service.ts | 168 +++++++++ .../factories/group-by-resolver.factory.ts | 73 ++-- .../handlers/rest-api-group-by.handler.ts | 38 +- .../core/interfaces/rest-api-base.handler.ts | 3 +- .../path-parsers/parse-core-path.utils.ts | 2 +- .../core/services/rest-api-core.service.ts | 10 +- ...nclude-records-sample-rest-request.util.ts | 13 + .../repository/permissions.utils.ts | 35 +- ... => group-by-resolver.integration-spec.ts} | 2 +- ...-with-records-resolver.integration-spec.ts | 352 ++++++++++++++++++ 19 files changed, 840 insertions(+), 362 deletions(-) delete mode 100644 packages/twenty-server/src/engine/api/graphql/graphql-query-runner/group-by/resolvers/graphql-query-group-by-resolver.service.ts create mode 100644 packages/twenty-server/src/engine/api/graphql/graphql-query-runner/group-by/resolvers/types/group-by-definition.types.ts create mode 100644 packages/twenty-server/src/engine/api/graphql/graphql-query-runner/group-by/services/group-by-with-records.service.ts create mode 100644 packages/twenty-server/src/engine/api/rest/input-request-parsers/group-by-with-records/parse-include-records-sample-rest-request.util.ts rename packages/twenty-server/test/integration/graphql/suites/{group-by-resolvers.integration-spec.ts => group-by-resolver.integration-spec.ts} (99%) create mode 100644 packages/twenty-server/test/integration/graphql/suites/group-by-with-records-resolver.integration-spec.ts diff --git a/packages/twenty-server/src/engine/api/common/common-query-runners/common-group-by-query-runner.service.ts b/packages/twenty-server/src/engine/api/common/common-query-runners/common-group-by-query-runner.service.ts index e3cea1ad65..2da54996c4 100644 --- a/packages/twenty-server/src/engine/api/common/common-query-runners/common-group-by-query-runner.service.ts +++ b/packages/twenty-server/src/engine/api/common/common-query-runners/common-group-by-query-runner.service.ts @@ -11,9 +11,9 @@ import { computeRecordGqlOperationFilter, convertViewFilterValueToString, getFilterTypeFromFieldType, - isDefined, turnAnyFieldFilterIntoRecordGqlFilter, } from 'twenty-shared/utils'; +import { ObjectLiteral } from 'typeorm'; import { WorkspaceAuthContext } from 'src/engine/api/common/interfaces/workspace-auth-context.interface'; import { ObjectRecordFilter } from 'src/engine/api/graphql/workspace-query-builder/interfaces/object-record.interface'; @@ -28,11 +28,15 @@ import { CommonQueryNames, GroupByQueryArgs, } from 'src/engine/api/common/types/common-query-args.type'; +import { GraphqlQuerySelectedFieldsResult } from 'src/engine/api/graphql/graphql-query-runner/graphql-query-parsers/graphql-query-selected-fields/graphql-selected-fields.parser'; +import { GraphqlQueryParser } from 'src/engine/api/graphql/graphql-query-runner/graphql-query-parsers/graphql-query.parser'; +import { GroupByDefinition } from 'src/engine/api/graphql/graphql-query-runner/group-by/resolvers/types/group-by-definition.types'; import { formatResultWithGroupByDimensionValues } from 'src/engine/api/graphql/graphql-query-runner/group-by/resolvers/utils/format-result-with-group-by-dimension-values.util'; import { getGroupByExpression } from 'src/engine/api/graphql/graphql-query-runner/group-by/resolvers/utils/get-group-by-expression.util'; import { isGroupByDateField } from 'src/engine/api/graphql/graphql-query-runner/group-by/resolvers/utils/is-group-by-date-field.util'; import { parseGroupByArgs } from 'src/engine/api/graphql/graphql-query-runner/group-by/resolvers/utils/parse-group-by-args.util'; import { removeQuotes } from 'src/engine/api/graphql/graphql-query-runner/group-by/resolvers/utils/remove-quote.util'; +import { GroupByWithRecordsService } from 'src/engine/api/graphql/graphql-query-runner/group-by/services/group-by-with-records.service'; import { ProcessAggregateHelper } from 'src/engine/api/graphql/graphql-query-runner/helpers/process-aggregate.helper'; import { ObjectMetadataItemWithFieldMaps } from 'src/engine/metadata-modules/types/object-metadata-item-with-field-maps'; import { ObjectMetadataMaps } from 'src/engine/metadata-modules/types/object-metadata-maps'; @@ -40,6 +44,7 @@ import { ViewFilterGroupService } from 'src/engine/metadata-modules/view-filter- import { ViewFilterService } from 'src/engine/metadata-modules/view-filter/services/view-filter.service'; import { ViewEntity } from 'src/engine/metadata-modules/view/entities/view.entity'; import { ViewService } from 'src/engine/metadata-modules/view/services/view.service'; +import { WorkspaceSelectQueryBuilder } from 'src/engine/twenty-orm/repository/workspace-select-query-builder'; import { formatColumnNamesFromCompositeFieldAndSubfields } from 'src/engine/twenty-orm/utils/format-column-names-from-composite-field-and-subfield.util'; @Injectable() @@ -51,6 +56,7 @@ export class CommonGroupByQueryRunnerService extends CommonBaseQueryRunnerServic private readonly viewFilterService: ViewFilterService, private readonly viewFilterGroupService: ViewFilterGroupService, private readonly viewService: ViewService, + private readonly groupByWithRecordsService: GroupByWithRecordsService, ) { super(); } @@ -66,6 +72,7 @@ export class CommonGroupByQueryRunnerService extends CommonBaseQueryRunnerServic commonQueryParser, objectMetadataItemWithFieldMaps, authContext, + objectMetadataMaps, } = queryRunnerContext; const objectMetadataNameSingular = @@ -77,22 +84,14 @@ export class CommonGroupByQueryRunnerService extends CommonBaseQueryRunnerServic let appliedFilters = args.filter ?? ({} as ObjectRecordFilter); - if (isDefined(args.viewId)) { - appliedFilters = await this.addFiltersFromView({ - args, - authContext, - objectMetadataItemWithFieldMaps, - appliedFilters, - }); - } - - commonQueryParser.applyFilterToBuilder( - queryBuilder, - objectMetadataNameSingular, + await this.addFiltersToQueryBuilder({ + args, appliedFilters, - ); - - commonQueryParser.applyDeletedAtToBuilder(queryBuilder, appliedFilters); + queryBuilder, + objectMetadataItemWithFieldMaps, + workspaceId: authContext.workspace.id, + commonQueryParser, + }); ProcessAggregateHelper.addSelectedAggregatedFieldsQueriesToQueryBuilder({ selectedAggregatedFields: args.selectedFieldsResult.aggregate, @@ -147,13 +146,37 @@ export class CommonGroupByQueryRunnerService extends CommonBaseQueryRunnerServic groupByFields, ); - const result = await queryBuilder.getRawMany(); + const shouldIncludeRecords = args.includeRecords ?? false; - return formatResultWithGroupByDimensionValues( - result, + if (shouldIncludeRecords) { + const queryBuilderWithFiltersAndWithoutGroupBy = + repository.createQueryBuilder(objectMetadataNameSingular); + + await this.addFiltersToQueryBuilder({ + args, + objectMetadataItemWithFieldMaps, + workspaceId: authContext.workspace.id, + commonQueryParser, + appliedFilters, + queryBuilder: queryBuilderWithFiltersAndWithoutGroupBy, + }); + + return this.groupByWithRecordsService.resolveWithRecords({ + queryBuilderWithFiltersAndWithoutGroupBy, + queryBuilderWithGroupBy: queryBuilder, + groupByDefinitions, + objectMetadataItemWithFieldMaps, + objectMetadataMaps, + repository, + selectedFieldsResult: args.selectedFieldsResult, + }); + } + + return this.resolveWithoutRecords({ + queryBuilder, groupByDefinitions, - Object.keys(args.selectedFieldsResult.aggregate), - ); + selectedFieldsResult: args.selectedFieldsResult, + }); } async processQueryResult( @@ -169,17 +192,15 @@ export class CommonGroupByQueryRunnerService extends CommonBaseQueryRunnerServic args, objectMetadataItemWithFieldMaps, appliedFilters, - authContext, + workspaceId, }: { args: GroupByQueryArgs; - authContext: WorkspaceAuthContext; objectMetadataItemWithFieldMaps: ObjectMetadataItemWithFieldMaps; appliedFilters: ObjectRecordFilter; + workspaceId: string; }): Promise { assertIsDefinedOrThrow(args.viewId); - const workspaceId = authContext.workspace.id; - const viewFilters = await this.viewFilterService.findByViewId( workspaceId, args.viewId, @@ -253,6 +274,60 @@ export class CommonGroupByQueryRunnerService extends CommonBaseQueryRunnerServic return appliedFilters; } + private async addFiltersToQueryBuilder({ + args, + appliedFilters, + queryBuilder, + objectMetadataItemWithFieldMaps, + workspaceId, + commonQueryParser, + }: { + args: GroupByQueryArgs; + appliedFilters: ObjectRecordFilter; + queryBuilder: WorkspaceSelectQueryBuilder; + objectMetadataItemWithFieldMaps: ObjectMetadataItemWithFieldMaps; + workspaceId: string; + commonQueryParser: GraphqlQueryParser; + }): Promise { + const objectMetadataNameSingular = + objectMetadataItemWithFieldMaps.nameSingular; + + if (args.viewId) { + appliedFilters = await this.addFiltersFromView({ + args, + objectMetadataItemWithFieldMaps, + appliedFilters, + workspaceId, + }); + } + + commonQueryParser.applyFilterToBuilder( + queryBuilder, + objectMetadataNameSingular, + appliedFilters, + ); + + commonQueryParser.applyDeletedAtToBuilder(queryBuilder, appliedFilters); + } + + private async resolveWithoutRecords({ + queryBuilder, + groupByDefinitions, + selectedFieldsResult, + }: { + queryBuilder: WorkspaceSelectQueryBuilder; + groupByDefinitions: GroupByDefinition[]; + selectedFieldsResult: GraphqlQuerySelectedFieldsResult; + }): Promise { + const result = await queryBuilder.getRawMany(); + + return formatResultWithGroupByDimensionValues({ + groupsResult: result, + groupByDefinitions, + aggregateFieldNames: Object.keys(selectedFieldsResult.aggregate), + }); + } + async validate( _args: CommonInput, _queryRunnerContext: CommonBaseQueryRunnerContext, diff --git a/packages/twenty-server/src/engine/api/common/core-common-api.module.ts b/packages/twenty-server/src/engine/api/common/core-common-api.module.ts index 75097824a7..7bf6714df1 100644 --- a/packages/twenty-server/src/engine/api/common/core-common-api.module.ts +++ b/packages/twenty-server/src/engine/api/common/core-common-api.module.ts @@ -4,6 +4,7 @@ import { TypeOrmModule } from '@nestjs/typeorm'; import { CommonArgsHandlers } from 'src/engine/api/common/common-args-handlers/common-query-selected-fields/common-arg-handlers'; import { CommonQueryRunners } from 'src/engine/api/common/common-query-runners/common-query-runners'; import { CommonResultGettersService } from 'src/engine/api/common/common-result-getters/common-result-getters.service'; +import { GroupByWithRecordsService } from 'src/engine/api/graphql/graphql-query-runner/group-by/services/group-by-with-records.service'; import { ProcessAggregateHelper } from 'src/engine/api/graphql/graphql-query-runner/helpers/process-aggregate.helper'; import { ProcessNestedRelationsV2Helper } from 'src/engine/api/graphql/graphql-query-runner/helpers/process-nested-relations-v2.helper'; import { ProcessNestedRelationsHelper } from 'src/engine/api/graphql/graphql-query-runner/helpers/process-nested-relations.helper'; @@ -40,6 +41,7 @@ import { WorkspacePermissionsCacheModule } from 'src/engine/metadata-modules/wor ProcessAggregateHelper, ...CommonQueryRunners, CommonResultGettersService, + GroupByWithRecordsService, ], exports: [...CommonQueryRunners], }) diff --git a/packages/twenty-server/src/engine/api/common/types/common-group-by-output-item.type.ts b/packages/twenty-server/src/engine/api/common/types/common-group-by-output-item.type.ts index ba2c8d8e70..93eac372bc 100644 --- a/packages/twenty-server/src/engine/api/common/types/common-group-by-output-item.type.ts +++ b/packages/twenty-server/src/engine/api/common/types/common-group-by-output-item.type.ts @@ -1,3 +1,5 @@ +import { type ObjectRecord } from 'twenty-shared/types'; + type AggregateValues = { [key: string]: string; }; @@ -6,4 +8,10 @@ type GroupByDimensionValues = { groupByDimensionValues: string[]; }; -export type CommonGroupByOutputItem = AggregateValues & GroupByDimensionValues; +type Records = { + records?: ObjectRecord[]; +}; + +export type CommonGroupByOutputItem = GroupByDimensionValues & + AggregateValues & + Records; diff --git a/packages/twenty-server/src/engine/api/common/types/common-query-args.type.ts b/packages/twenty-server/src/engine/api/common/types/common-query-args.type.ts index 2e4d8bfb24..30b51d20cf 100644 --- a/packages/twenty-server/src/engine/api/common/types/common-query-args.type.ts +++ b/packages/twenty-server/src/engine/api/common/types/common-query-args.type.ts @@ -66,6 +66,8 @@ export interface GroupByQueryArgs { orderBy?: OrderByWithGroupBy; groupBy: ObjectRecordGroupBy; viewId?: string; + includeRecords?: boolean; + selectedFields: CommonSelectedFields; } export interface DestroyOneQueryArgs { id: string; diff --git a/packages/twenty-server/src/engine/api/graphql/graphql-query-runner/graphql-query-runner.module.ts b/packages/twenty-server/src/engine/api/graphql/graphql-query-runner/graphql-query-runner.module.ts index 7126f4e0a3..ed6540e41d 100644 --- a/packages/twenty-server/src/engine/api/graphql/graphql-query-runner/graphql-query-runner.module.ts +++ b/packages/twenty-server/src/engine/api/graphql/graphql-query-runner/graphql-query-runner.module.ts @@ -1,7 +1,6 @@ import { Module } from '@nestjs/common'; import { TypeOrmModule } from '@nestjs/typeorm'; -import { GraphqlQueryGroupByResolverService } from 'src/engine/api/graphql/graphql-query-runner/group-by/resolvers/graphql-query-group-by-resolver.service'; import { ProcessAggregateHelper } from 'src/engine/api/graphql/graphql-query-runner/helpers/process-aggregate.helper'; import { ProcessNestedRelationsV2Helper } from 'src/engine/api/graphql/graphql-query-runner/helpers/process-nested-relations-v2.helper'; import { ProcessNestedRelationsHelper } from 'src/engine/api/graphql/graphql-query-runner/helpers/process-nested-relations.helper'; @@ -39,7 +38,6 @@ const graphqlQueryResolvers = [ GraphqlQueryFindDuplicatesResolverService, GraphqlQueryFindManyResolverService, GraphqlQueryFindOneResolverService, - GraphqlQueryGroupByResolverService, GraphqlQueryMergeManyResolverService, GraphqlQueryRestoreManyResolverService, GraphqlQueryRestoreOneResolverService, diff --git a/packages/twenty-server/src/engine/api/graphql/graphql-query-runner/group-by/resolvers/graphql-query-group-by-resolver.service.ts b/packages/twenty-server/src/engine/api/graphql/graphql-query-runner/group-by/resolvers/graphql-query-group-by-resolver.service.ts deleted file mode 100644 index 8668cfad0c..0000000000 --- a/packages/twenty-server/src/engine/api/graphql/graphql-query-runner/group-by/resolvers/graphql-query-group-by-resolver.service.ts +++ /dev/null @@ -1,249 +0,0 @@ -import { Injectable } from '@nestjs/common'; - -import { - CompositeFieldSubFieldName, - ObjectRecord, - PartialFieldMetadataItemOption, - RecordFilterGroupLogicalOperator, -} from 'twenty-shared/types'; -import { - assertIsDefinedOrThrow, - combineFilters, - computeRecordGqlOperationFilter, - convertViewFilterValueToString, - getFilterTypeFromFieldType, - turnAnyFieldFilterIntoRecordGqlFilter, -} from 'twenty-shared/utils'; - -import { - GraphqlQueryBaseResolverService, - GraphqlQueryResolverExecutionArgs, -} from 'src/engine/api/graphql/graphql-query-runner/interfaces/base-resolver-service'; -import { ObjectRecordFilter } from 'src/engine/api/graphql/workspace-query-builder/interfaces/object-record.interface'; -import { IEdge } from 'src/engine/api/graphql/workspace-query-runner/interfaces/edge.interface'; -import { IGroupByConnection } from 'src/engine/api/graphql/workspace-query-runner/interfaces/group-by-connection.interface'; -import { type WorkspaceQueryRunnerOptions } from 'src/engine/api/graphql/workspace-query-runner/interfaces/query-runner-option.interface'; -import { GroupByResolverArgs } from 'src/engine/api/graphql/workspace-resolver-builder/interfaces/workspace-resolvers-builder.interface'; - -import { formatResultWithGroupByDimensionValues } from 'src/engine/api/graphql/graphql-query-runner/group-by/resolvers/utils/format-result-with-group-by-dimension-values.util'; -import { getGroupByExpression } from 'src/engine/api/graphql/graphql-query-runner/group-by/resolvers/utils/get-group-by-expression.util'; -import { isGroupByDateField } from 'src/engine/api/graphql/graphql-query-runner/group-by/resolvers/utils/is-group-by-date-field.util'; -import { parseGroupByArgs } from 'src/engine/api/graphql/graphql-query-runner/group-by/resolvers/utils/parse-group-by-args.util'; -import { removeQuotes } from 'src/engine/api/graphql/graphql-query-runner/group-by/resolvers/utils/remove-quote.util'; -import { ProcessAggregateHelper } from 'src/engine/api/graphql/graphql-query-runner/helpers/process-aggregate.helper'; -import { FeatureFlagKey } from 'src/engine/core-modules/feature-flag/enums/feature-flag-key.enum'; -import { WorkspaceNotFoundDefaultError } from 'src/engine/core-modules/workspace/workspace.exception'; -import { ObjectMetadataItemWithFieldMaps } from 'src/engine/metadata-modules/types/object-metadata-item-with-field-maps'; -import { ViewFilterGroupService } from 'src/engine/metadata-modules/view-filter-group/services/view-filter-group.service'; -import { ViewFilterService } from 'src/engine/metadata-modules/view-filter/services/view-filter.service'; -import { ViewEntity } from 'src/engine/metadata-modules/view/entities/view.entity'; -import { ViewService } from 'src/engine/metadata-modules/view/services/view.service'; -import { formatColumnNamesFromCompositeFieldAndSubfields } from 'src/engine/twenty-orm/utils/format-column-names-from-composite-field-and-subfield.util'; - -@Injectable() -export class GraphqlQueryGroupByResolverService extends GraphqlQueryBaseResolverService< - GroupByResolverArgs, - IGroupByConnection>[] -> { - constructor( - private readonly viewFilterService: ViewFilterService, - private readonly viewFilterGroupService: ViewFilterGroupService, - private readonly viewService: ViewService, - ) { - super(); - } - - async resolve( - executionArgs: GraphqlQueryResolverExecutionArgs, - _featureFlagsMap: Record, - ): Promise>[]> { - const { objectMetadataItemWithFieldMaps } = executionArgs.options; - - const objectMetadataNameSingular = - objectMetadataItemWithFieldMaps.nameSingular; - - let queryBuilder = executionArgs.repository.createQueryBuilder( - objectMetadataNameSingular, - ); - - let appliedFilters = - executionArgs.args.filter ?? ({} as ObjectRecordFilter); - - if (executionArgs.args.viewId) { - appliedFilters = await this.addFiltersFromView({ - executionArgs, - objectMetadataItemWithFieldMaps, - appliedFilters, - }); - } - - executionArgs.graphqlQueryParser.applyFilterToBuilder( - queryBuilder, - objectMetadataNameSingular, - appliedFilters, - ); - - executionArgs.graphqlQueryParser.applyDeletedAtToBuilder( - queryBuilder, - appliedFilters, - ); - - ProcessAggregateHelper.addSelectedAggregatedFieldsQueriesToQueryBuilder({ - selectedAggregatedFields: - executionArgs.graphqlQuerySelectedFieldsResult.aggregate, - queryBuilder, - objectMetadataNameSingular, - }); - - const groupByFields = parseGroupByArgs( - executionArgs.args, - objectMetadataItemWithFieldMaps, - ); - - const groupByDefinitions = groupByFields.map((groupByField) => { - const columnNameWithQuotes = `"${ - formatColumnNamesFromCompositeFieldAndSubfields( - groupByField.fieldMetadata.name, - groupByField.subFieldName ? [groupByField.subFieldName] : undefined, - )[0] - }"`; - const alias = - removeQuotes(columnNameWithQuotes) + - (isGroupByDateField(groupByField) - ? `_${groupByField.dateGranularity}` - : ''); - - return { - columnNameWithQuotes, - expression: getGroupByExpression({ - groupByField, - columnNameWithQuotes, - }), - alias, - dateGranularity: isGroupByDateField(groupByField) - ? groupByField.dateGranularity - : undefined, - }; - }); - - groupByDefinitions.forEach((groupByColumn, index) => { - queryBuilder.addSelect(groupByColumn.expression, groupByColumn.alias); - - if (index === 0) { - queryBuilder.groupBy(groupByColumn.expression); - } else { - queryBuilder.addGroupBy(groupByColumn.expression); - } - }); - - executionArgs.graphqlQueryParser.applyGroupByOrderToBuilder( - queryBuilder, - executionArgs.args.orderBy ?? [], - groupByFields, - ); - - const result = await queryBuilder.getRawMany(); - - return formatResultWithGroupByDimensionValues( - result, - groupByDefinitions, - Object.keys(executionArgs.graphqlQuerySelectedFieldsResult.aggregate), - ); - } - - private async addFiltersFromView({ - executionArgs, - objectMetadataItemWithFieldMaps, - appliedFilters, - }: { - executionArgs: GraphqlQueryResolverExecutionArgs; - objectMetadataItemWithFieldMaps: ObjectMetadataItemWithFieldMaps; - appliedFilters: ObjectRecordFilter; - }): Promise { - assertIsDefinedOrThrow(executionArgs.args.viewId); - - const workspaceId = executionArgs.options.authContext.workspace?.id; - - assertIsDefinedOrThrow(workspaceId, WorkspaceNotFoundDefaultError); - - const viewFilters = await this.viewFilterService.findByViewId( - workspaceId, - executionArgs.args.viewId, - ); - - const viewFilterGroups = await this.viewFilterGroupService.findByViewId( - workspaceId, - executionArgs.args.viewId, - ); - - const recordFilters = viewFilters.map((viewFilter) => { - const fieldMetadataItem = - objectMetadataItemWithFieldMaps.fieldsById[viewFilter.fieldMetadataId]; - - return { - id: viewFilter.id, - fieldMetadataId: viewFilter.fieldMetadataId, - value: convertViewFilterValueToString(viewFilter.value), - type: getFilterTypeFromFieldType(fieldMetadataItem.type), - operand: viewFilter.operand, - recordFilterGroupId: viewFilter.viewFilterGroupId, - positionInRecordFilterGroup: viewFilter.positionInViewFilterGroup, - subFieldName: viewFilter.subFieldName as CompositeFieldSubFieldName, - }; - }); - - const recordFilterGroups = viewFilterGroups.map((viewFilterGroup) => { - return { - id: viewFilterGroup.id, - logicalOperator: - viewFilterGroup.logicalOperator as unknown as RecordFilterGroupLogicalOperator, // TODO - https://github.com/twentyhq/twenty/issues/14746 - parentRecordFilterGroupId: viewFilterGroup.parentViewFilterGroupId, - }; - }); - - const fields = Object.values( - objectMetadataItemWithFieldMaps.fieldsById, - ).map((field) => ({ - id: field.id, - name: field.name, - type: field.type, - label: field.label, - options: field.options as PartialFieldMetadataItemOption[], - })); - - const filtersFromView = computeRecordGqlOperationFilter({ - recordFilters, - recordFilterGroups: recordFilterGroups, - fields, - filterValueDependencies: {}, - }); - - let view: ViewEntity | null = viewFilters[0]?.view; - - if (!view) { - view = await this.viewService.findById( - executionArgs.args.viewId, - workspaceId, - ); - } - - const { recordGqlOperationFilter: anyFieldFilter } = - turnAnyFieldFilterIntoRecordGqlFilter({ - fields, - filterValue: view?.anyFieldFilterValue ?? '', - }); - - appliedFilters = combineFilters([ - appliedFilters, - filtersFromView, - anyFieldFilter, - ]); - - return appliedFilters; - } - - async validate( - _args: GroupByResolverArgs, - _options: WorkspaceQueryRunnerOptions, - ): Promise {} -} diff --git a/packages/twenty-server/src/engine/api/graphql/graphql-query-runner/group-by/resolvers/types/group-by-definition.types.ts b/packages/twenty-server/src/engine/api/graphql/graphql-query-runner/group-by/resolvers/types/group-by-definition.types.ts new file mode 100644 index 0000000000..6da7cc5c06 --- /dev/null +++ b/packages/twenty-server/src/engine/api/graphql/graphql-query-runner/group-by/resolvers/types/group-by-definition.types.ts @@ -0,0 +1,8 @@ +import { type ObjectRecordGroupByDateGranularity } from 'twenty-shared/types'; + +export type GroupByDefinition = { + columnNameWithQuotes: string; + expression: string; + alias: string; + dateGranularity?: ObjectRecordGroupByDateGranularity; +}; diff --git a/packages/twenty-server/src/engine/api/graphql/graphql-query-runner/group-by/resolvers/utils/format-result-with-group-by-dimension-values.util.ts b/packages/twenty-server/src/engine/api/graphql/graphql-query-runner/group-by/resolvers/utils/format-result-with-group-by-dimension-values.util.ts index 561fdaf2ec..281409ffd9 100644 --- a/packages/twenty-server/src/engine/api/graphql/graphql-query-runner/group-by/resolvers/utils/format-result-with-group-by-dimension-values.util.ts +++ b/packages/twenty-server/src/engine/api/graphql/graphql-query-runner/group-by/resolvers/utils/format-result-with-group-by-dimension-values.util.ts @@ -1,33 +1,61 @@ import { t } from '@lingui/core/macro'; -import { - ObjectRecordGroupByDateGranularity, - type ObjectRecord, -} from 'twenty-shared/types'; - -import { type IEdge } from 'src/engine/api/graphql/workspace-query-runner/interfaces/edge.interface'; -import { type IGroupByConnection } from 'src/engine/api/graphql/workspace-query-runner/interfaces/group-by-connection.interface'; +import { ObjectRecordGroupByDateGranularity } from 'twenty-shared/types'; +import { isDefined } from 'twenty-shared/utils'; import { type CommonGroupByOutputItem } from 'src/engine/api/common/types/common-group-by-output-item.type'; +import { type GroupByDefinition } from 'src/engine/api/graphql/graphql-query-runner/group-by/resolvers/types/group-by-definition.types'; +import { type ObjectMetadataItemWithFieldMaps } from 'src/engine/metadata-modules/types/object-metadata-item-with-field-maps'; +import { type ObjectMetadataMaps } from 'src/engine/metadata-modules/types/object-metadata-maps'; +import { formatResult } from 'src/engine/twenty-orm/utils/format-result.util'; -export const formatResultWithGroupByDimensionValues = < - T extends - | IGroupByConnection> - | CommonGroupByOutputItem, ->( - result: Record[], - groupByColumnsWithQuotes: { - columnNameWithQuotes: string; - alias: string; - dateGranularity?: ObjectRecordGroupByDateGranularity; - }[], - aggregateFieldNames: string[], -): T[] => { - let formattedResult: T[] = []; +export const formatResultWithGroupByDimensionValues = ({ + groupsResult, + groupByDefinitions, + aggregateFieldNames, + recordsResult, + objectMetadataItemWithFieldMaps, + objectMetadataMaps, +}: { + groupsResult: Record[]; + groupByDefinitions: GroupByDefinition[]; + aggregateFieldNames: string[]; + recordsResult?: Array>; + objectMetadataItemWithFieldMaps?: ObjectMetadataItemWithFieldMaps; + objectMetadataMaps?: ObjectMetadataMaps; +}): CommonGroupByOutputItem[] => { + const formattedResult: CommonGroupByOutputItem[] = []; - result.forEach((group) => { - let dimensionValues: string[] = []; + const recordsByGroupKey = new Map>>(); - for (const groupByColumn of groupByColumnsWithQuotes) { + if (isDefined(recordsResult)) { + if ( + !isDefined(objectMetadataItemWithFieldMaps) || + !isDefined(objectMetadataMaps) + ) { + throw new Error('Metadata are required to format result'); + } + + recordsResult.forEach((entry) => { + const groupKey = createGroupKey(entry, groupByDefinitions); + + const records: Record[] = ( + (entry.records as Array>) ?? [] + ).map((record) => { + return formatResult( + record, + objectMetadataItemWithFieldMaps, + objectMetadataMaps, + ); + }); + + recordsByGroupKey.set(groupKey, records); + }); + } + + groupsResult.forEach((group) => { + const dimensionValues: unknown[] = []; + + for (const groupByColumn of groupByDefinitions) { dimensionValues.push( getTranslatedValueIfApplicable( group[groupByColumn.alias], @@ -44,23 +72,37 @@ export const formatResultWithGroupByDimensionValues = < return acc; }, - {} as Record, + {} as Record, ); + const groupKey = createGroupKey(group, groupByDefinitions); + const records = recordsByGroupKey.get(groupKey) || []; + formattedResult.push({ - groupByDimensionValues: dimensionValues, ...aggregateValues, - //TODO: Refacto-common - remove generic type - } as T); + groupByDimensionValues: dimensionValues, + ...(isDefined(recordsResult) ? { records } : {}), + } as CommonGroupByOutputItem); }); return formattedResult; }; -const getTranslatedValueIfApplicable = ( - value: string, - dateGranularity?: ObjectRecordGroupByDateGranularity, +const createGroupKey = ( + group: Record, + groupByDefinitions: GroupByDefinition[], ): string => { + return groupByDefinitions.map((def) => String(group[def.alias])).join('|'); +}; + +const getTranslatedValueIfApplicable = ( + value: T, + dateGranularity?: ObjectRecordGroupByDateGranularity, +): T | string => { + if (typeof value !== 'string') { + return value; + } + switch (dateGranularity) { case ObjectRecordGroupByDateGranularity.DAY_OF_THE_WEEK: switch (value) { diff --git a/packages/twenty-server/src/engine/api/graphql/graphql-query-runner/group-by/resolvers/utils/parse-group-by-args.util.ts b/packages/twenty-server/src/engine/api/graphql/graphql-query-runner/group-by/resolvers/utils/parse-group-by-args.util.ts index 93431d3d85..7ebf07f566 100644 --- a/packages/twenty-server/src/engine/api/graphql/graphql-query-runner/group-by/resolvers/utils/parse-group-by-args.util.ts +++ b/packages/twenty-server/src/engine/api/graphql/graphql-query-runner/group-by/resolvers/utils/parse-group-by-args.util.ts @@ -64,6 +64,10 @@ export const parseGroupByArgs = ( const fieldMetadata = objectMetadataItemWithFieldMaps.fieldsById[fieldMetadataId]; + if (!isDefined(fieldMetadata) || !isDefined(fieldMetadataId)) { + throw new Error(`Unidentified field in groupBy: ${fieldName}`); + } + if ( fieldMetadata.type === FieldMetadataType.DATE || fieldMetadata.type === FieldMetadataType.DATE_TIME diff --git a/packages/twenty-server/src/engine/api/graphql/graphql-query-runner/group-by/services/group-by-with-records.service.ts b/packages/twenty-server/src/engine/api/graphql/graphql-query-runner/group-by/services/group-by-with-records.service.ts new file mode 100644 index 0000000000..c41c56d166 --- /dev/null +++ b/packages/twenty-server/src/engine/api/graphql/graphql-query-runner/group-by/services/group-by-with-records.service.ts @@ -0,0 +1,168 @@ +import { isDefined } from 'class-validator'; +import { type ObjectLiteral } from 'typeorm'; + +import { type CommonGroupByOutputItem } from 'src/engine/api/common/types/common-group-by-output-item.type'; +import { type GraphqlQuerySelectedFieldsResult } from 'src/engine/api/graphql/graphql-query-runner/graphql-query-parsers/graphql-query-selected-fields/graphql-selected-fields.parser'; +import { type GroupByDefinition } from 'src/engine/api/graphql/graphql-query-runner/group-by/resolvers/types/group-by-definition.types'; +import { formatResultWithGroupByDimensionValues } from 'src/engine/api/graphql/graphql-query-runner/group-by/resolvers/utils/format-result-with-group-by-dimension-values.util'; +import { buildColumnsToSelect } from 'src/engine/api/graphql/graphql-query-runner/utils/build-columns-to-select'; +import { type ObjectMetadataItemWithFieldMaps } from 'src/engine/metadata-modules/types/object-metadata-item-with-field-maps'; +import { type ObjectMetadataMaps } from 'src/engine/metadata-modules/types/object-metadata-maps'; +import { type WorkspaceSelectQueryBuilder } from 'src/engine/twenty-orm/repository/workspace-select-query-builder'; +import { type WorkspaceRepository } from 'src/engine/twenty-orm/repository/workspace.repository'; + +const GROUPS_LIMIT = 50; +const RECORDS_PER_GROUP_LIMIT = 10; +const SUB_QUERY_PREFIX = 'sub_query_'; + +export class GroupByWithRecordsService { + constructor() {} + + public async resolveWithRecords({ + queryBuilderWithGroupBy, + queryBuilderWithFiltersAndWithoutGroupBy, + groupByDefinitions, + objectMetadataItemWithFieldMaps, + selectedFieldsResult, + objectMetadataMaps, + repository, + }: { + queryBuilderWithGroupBy: WorkspaceSelectQueryBuilder; + queryBuilderWithFiltersAndWithoutGroupBy: WorkspaceSelectQueryBuilder; + groupByDefinitions: GroupByDefinition[]; + objectMetadataItemWithFieldMaps: ObjectMetadataItemWithFieldMaps; + selectedFieldsResult: GraphqlQuerySelectedFieldsResult; + objectMetadataMaps: ObjectMetadataMaps; + repository: WorkspaceRepository; + }): Promise { + const groupsResult = await queryBuilderWithGroupBy + .limit(GROUPS_LIMIT) + .getRawMany(); + + if (groupsResult.length === 0) { + return []; + } + + const columnsToSelect = buildColumnsToSelect({ + select: selectedFieldsResult.select, + relations: selectedFieldsResult.relations, // TODO - not handled for now + objectMetadataItemWithFieldMaps, + objectMetadataMaps: objectMetadataMaps, + }); + + const queryBuilderWithPartitionBy = this.addPartitionByToQueryBuilder({ + queryBuilderForSubQuery: queryBuilderWithFiltersAndWithoutGroupBy, + columnsToSelect, + groupsResult, + groupByDefinitions, + repository, + }); + + const recordsResult = await queryBuilderWithPartitionBy.getRawMany(); + + return formatResultWithGroupByDimensionValues({ + groupsResult, + recordsResult, + groupByDefinitions, + aggregateFieldNames: Object.keys(selectedFieldsResult.aggregate), + objectMetadataItemWithFieldMaps, + objectMetadataMaps, + }); + } + + private addPartitionByToQueryBuilder({ + queryBuilderForSubQuery, + columnsToSelect, + groupsResult, + groupByDefinitions, + repository, + }: { + queryBuilderForSubQuery: WorkspaceSelectQueryBuilder; + columnsToSelect: Record; + groupsResult: Array>; + groupByDefinitions: GroupByDefinition[]; + repository: WorkspaceRepository; + }): WorkspaceSelectQueryBuilder { + const groupByExpressions = groupByDefinitions + .map((def) => def.expression) + .join(', '); + + const groupByAliases = groupByDefinitions + .map((def) => `"${def.alias}"`) + .join(', '); + + const groupConditions = this.buildGroupConditions( + groupsResult, + groupByDefinitions, + queryBuilderForSubQuery, + ); + + const recordSelectWithAlias = Object.keys(columnsToSelect) + .map((col) => `"${col}" as "${SUB_QUERY_PREFIX}${col}"`) + .join(', '); + + const groupBySelectWithAlias = groupByDefinitions + .map((def) => `${def.expression} as "${def.alias}"`) + .join(', '); + + const subQuery = queryBuilderForSubQuery + .select(recordSelectWithAlias) + .addSelect(groupBySelectWithAlias) + .addSelect(`ROW_NUMBER() OVER (PARTITION BY ${groupByExpressions})`, 'rn') + .andWhere(groupConditions); + + let mainQueryQueryBuilder = repository.createQueryBuilder(); + + const mainQuery = mainQueryQueryBuilder + .from(`(${subQuery.getQuery()})`, 'ranked_records') + .setParameters(queryBuilderForSubQuery.expressionMap.parameters) + .select(groupByAliases) + .addSelect( + `JSON_AGG( + CASE WHEN rn <= ${RECORDS_PER_GROUP_LIMIT} THEN + JSON_BUILD_OBJECT( + ${[ + ...Object.keys(columnsToSelect).map( + (col) => `'${col}', "${SUB_QUERY_PREFIX}${col}"`, + ), + ...groupByDefinitions.map( + (def) => `'${def.alias}', "${def.alias}"`, + ), + ].join(',\n ')} + ) + END + ) FILTER (WHERE rn <= ${RECORDS_PER_GROUP_LIMIT})`, + 'records', + ) + .groupBy(groupByAliases); + + // Remove initial "from" condition (typeOrm limitation) + mainQuery.expressionMap.aliases = mainQuery.expressionMap.aliases.filter( + (alias) => isDefined(alias.subQuery), + ); + + return mainQuery as WorkspaceSelectQueryBuilder; + } + + private buildGroupConditions( + groupsResult: Array>, + groupByDefinitions: GroupByDefinition[], + queryBuilder: WorkspaceSelectQueryBuilder, + ): string { + const groupConditions = groupsResult.map((group, groupIndex) => { + const conditions = groupByDefinitions + .map((def, defIndex) => { + const paramName = `groupValue_${groupIndex}_${defIndex}`; + + queryBuilder.setParameter(paramName, group[def.alias]); + + return `${def.expression} = :${paramName}`; + }) + .join(' AND '); + + return `(${conditions})`; + }); + + return `(${groupConditions.join(' OR ')})`; + } +} diff --git a/packages/twenty-server/src/engine/api/graphql/workspace-resolver-builder/factories/group-by-resolver.factory.ts b/packages/twenty-server/src/engine/api/graphql/workspace-resolver-builder/factories/group-by-resolver.factory.ts index fd1718e85f..133edc6c70 100644 --- a/packages/twenty-server/src/engine/api/graphql/workspace-resolver-builder/factories/group-by-resolver.factory.ts +++ b/packages/twenty-server/src/engine/api/graphql/workspace-resolver-builder/factories/group-by-resolver.factory.ts @@ -1,8 +1,8 @@ import { Injectable } from '@nestjs/common'; +import { isDefined } from 'class-validator'; import graphqlFields from 'graphql-fields'; -import { type WorkspaceQueryRunnerOptions } from 'src/engine/api/graphql/workspace-query-runner/interfaces/query-runner-option.interface'; import { type WorkspaceResolverBuilderFactoryInterface } from 'src/engine/api/graphql/workspace-resolver-builder/interfaces/workspace-resolver-builder-factory.interface'; import { GroupByResolverArgs, @@ -11,11 +11,9 @@ import { import { WorkspaceSchemaBuilderContext } from 'src/engine/api/graphql/workspace-schema-builder/interfaces/workspace-schema-builder-context.interface'; import { CommonGroupByQueryRunnerService } from 'src/engine/api/common/common-query-runners/common-group-by-query-runner.service'; -import { GraphqlQueryGroupByResolverService } from 'src/engine/api/graphql/graphql-query-runner/group-by/resolvers/graphql-query-group-by-resolver.service'; +import { ObjectRecordsToGraphqlConnectionHelper } from 'src/engine/api/graphql/graphql-query-runner/helpers/object-records-to-graphql-connection.helper'; import { workspaceQueryRunnerGraphqlApiExceptionHandler } from 'src/engine/api/graphql/workspace-query-runner/utils/workspace-query-runner-graphql-api-exception-handler.util'; import { RESOLVER_METHOD_NAMES } from 'src/engine/api/graphql/workspace-resolver-builder/constants/resolver-method-names'; -import { FeatureFlagKey } from 'src/engine/core-modules/feature-flag/enums/feature-flag-key.enum'; -import { FeatureFlagService } from 'src/engine/core-modules/feature-flag/services/feature-flag.service'; @Injectable() export class GroupByResolverFactory @@ -24,9 +22,7 @@ export class GroupByResolverFactory public static methodName = RESOLVER_METHOD_NAMES.GROUP_BY; constructor( - private readonly graphqlQueryRunnerService: GraphqlQueryGroupByResolverService, private readonly commonGroupByQueryRunnerService: CommonGroupByQueryRunnerService, - private readonly featureFlagService: FeatureFlagService, ) {} create( @@ -35,37 +31,48 @@ export class GroupByResolverFactory const internalContext = context; return async (_source, args, _context, info) => { - const isCommonApiEnabled = await this.featureFlagService.isFeatureEnabled( - FeatureFlagKey.IS_COMMON_API_ENABLED, - internalContext.authContext.workspace?.id as string, - ); + const selectedFields = graphqlFields(info); - if (isCommonApiEnabled) { - const selectedFields = graphqlFields(info); + const shouldIncludeRecords = + isDefined(selectedFields.edges?.node) && + Object.keys(selectedFields.edges?.node).length > 0; - try { - return await this.commonGroupByQueryRunnerService.execute( - { ...args, selectedFields }, - internalContext, + try { + const typeORMObjectRecordsParser = + new ObjectRecordsToGraphqlConnectionHelper( + internalContext.objectMetadataMaps, ); - } catch (error) { - workspaceQueryRunnerGraphqlApiExceptionHandler(error); - } + + const results = await this.commonGroupByQueryRunnerService.execute( + { ...args, selectedFields, includeRecords: shouldIncludeRecords }, + internalContext, + ); + + const formattedResults = results.map((group) => { + const formattedRecords = typeORMObjectRecordsParser.createConnection({ + objectRecords: group.records ?? [], + objectName: + internalContext.objectMetadataItemWithFieldMaps.nameSingular, + objectRecordsAggregatedValues: {}, + selectedAggregatedFields: {}, + take: group.records?.length || 0, + totalCount: Number(group.totalCount ?? 0), + hasNextPage: false, + hasPreviousPage: false, + }); + + const { _records, ...groupWithoutRecords } = group; + + return { + ...groupWithoutRecords, + ...formattedRecords, + }; + }); + + return formattedResults; + } catch (error) { + workspaceQueryRunnerGraphqlApiExceptionHandler(error); } - - const options: WorkspaceQueryRunnerOptions = { - authContext: internalContext.authContext, - info, - objectMetadataMaps: internalContext.objectMetadataMaps, - objectMetadataItemWithFieldMaps: - internalContext.objectMetadataItemWithFieldMaps, - }; - - return await this.graphqlQueryRunnerService.execute( - args, - options, - GroupByResolverFactory.methodName, - ); }; } } diff --git a/packages/twenty-server/src/engine/api/rest/core/handlers/rest-api-group-by.handler.ts b/packages/twenty-server/src/engine/api/rest/core/handlers/rest-api-group-by.handler.ts index bb29b2b532..6ecb88997d 100644 --- a/packages/twenty-server/src/engine/api/rest/core/handlers/rest-api-group-by.handler.ts +++ b/packages/twenty-server/src/engine/api/rest/core/handlers/rest-api-group-by.handler.ts @@ -6,6 +6,7 @@ import { CommonGroupByQueryRunnerService } from 'src/engine/api/common/common-qu import { parseAggregateFieldsRestRequest } from 'src/engine/api/rest/input-request-parsers/aggregate-fields-parser-utils/parse-aggregate-fields-rest-request.util'; import { parseFilterRestRequest } from 'src/engine/api/rest/input-request-parsers/filter-parser-utils/parse-filter-rest-request.util'; import { parseGroupByRestRequest } from 'src/engine/api/rest/input-request-parsers/group-by-parser-utils/parse-group-by-rest-request.util'; +import { parseIncludeRecordsSampleRestRequest } from 'src/engine/api/rest/input-request-parsers/group-by-with-records/parse-include-records-sample-rest-request.util'; import { parseOrderByWithGroupByRestRequest } from 'src/engine/api/rest/input-request-parsers/order-by-with-group-by-parser-utils/parse-order-by-with-group-by-rest-request.util'; import { parseViewIdRestRequest } from 'src/engine/api/rest/input-request-parsers/view-id-parser-utils/parse-view-id-rest-request.util'; import { AuthenticatedRequest } from 'src/engine/api/rest/types/authenticated-request'; @@ -22,13 +23,16 @@ export class RestApiGroupByHandler extends RestApiBaseHandler { async handle(request: AuthenticatedRequest) { try { const { + filter, + orderBy, + viewId, + groupBy, + selectedFields, authContext, - objectMetadataItemWithFieldMaps, objectMetadataMaps, - } = await this.buildCommonOptions(request); - - const { filter, orderBy, viewId, groupBy, selectedFields } = - this.parseRequestArgs(request); + objectMetadataItemWithFieldMaps, + includeRecords, + } = await this.parseRequestArgs(request); return await this.commonGroupByQueryRunnerService.execute( { @@ -37,6 +41,7 @@ export class RestApiGroupByHandler extends RestApiBaseHandler { viewId, groupBy, selectedFields, + includeRecords, }, { authContext, @@ -49,20 +54,39 @@ export class RestApiGroupByHandler extends RestApiBaseHandler { } } - private parseRequestArgs(request: AuthenticatedRequest) { + private async parseRequestArgs(request: AuthenticatedRequest) { + const { authContext, objectMetadataItemWithFieldMaps, objectMetadataMaps } = + await this.buildCommonOptions(request); + const orderByWithGroupBy = parseOrderByWithGroupByRestRequest(request); const filter = parseFilterRestRequest(request); const viewId = parseViewIdRestRequest(request); const groupBy = parseGroupByRestRequest(request); + const includeRecords = parseIncludeRecordsSampleRestRequest(request); const aggregateFields = parseAggregateFieldsRestRequest(request); - const selectedFields = { ...aggregateFields, groupByDimensionValues: true }; + let selectedFields = { ...aggregateFields, groupByDimensionValues: true }; + + if (includeRecords) { + const selectableFields = await this.computeSelectedFields({ + depth: 0, + objectMetadataMapItem: objectMetadataItemWithFieldMaps, + objectMetadataMaps, + authContext, + }); + + selectedFields = { ...selectedFields, ...selectableFields }; + } return { + authContext, + objectMetadataItemWithFieldMaps, + objectMetadataMaps, filter, orderBy: orderByWithGroupBy, viewId, groupBy, selectedFields, + includeRecords, }; } } diff --git a/packages/twenty-server/src/engine/api/rest/core/interfaces/rest-api-base.handler.ts b/packages/twenty-server/src/engine/api/rest/core/interfaces/rest-api-base.handler.ts index 2ffd9547d5..b0d4bb47aa 100644 --- a/packages/twenty-server/src/engine/api/rest/core/interfaces/rest-api-base.handler.ts +++ b/packages/twenty-server/src/engine/api/rest/core/interfaces/rest-api-base.handler.ts @@ -14,6 +14,7 @@ import { WorkspaceAuthContext } from 'src/engine/api/common/interfaces/workspace import { type ObjectRecordFilter } from 'src/engine/api/graphql/workspace-query-builder/interfaces/object-record.interface'; import { CommonGroupByOutputItem } from 'src/engine/api/common/types/common-group-by-output-item.type'; +import { CommonSelectedFields } from 'src/engine/api/common/types/common-selected-fields-result.type'; import { GraphqlQueryParser } from 'src/engine/api/graphql/graphql-query-runner/graphql-query-parsers/graphql-query.parser'; import { encodeCursor } from 'src/engine/api/graphql/graphql-query-runner/utils/cursors.util'; import { CoreQueryBuilderFactory } from 'src/engine/api/rest/core/query-builder/core-query-builder.factory'; @@ -596,7 +597,7 @@ export abstract class RestApiBaseHandler { depth?: Depth | undefined; objectMetadataMapItem: ObjectMetadataItemWithFieldMaps; objectMetadataMaps: ObjectMetadataMaps; - }) { + }): Promise { const { objectsPermissions } = await this.getObjectsPermissions(authContext); diff --git a/packages/twenty-server/src/engine/api/rest/core/query-builder/utils/path-parsers/parse-core-path.utils.ts b/packages/twenty-server/src/engine/api/rest/core/query-builder/utils/path-parsers/parse-core-path.utils.ts index f4bb56fe06..da69d296de 100644 --- a/packages/twenty-server/src/engine/api/rest/core/query-builder/utils/path-parsers/parse-core-path.utils.ts +++ b/packages/twenty-server/src/engine/api/rest/core/query-builder/utils/path-parsers/parse-core-path.utils.ts @@ -37,7 +37,7 @@ export const parseCorePath = ( if ( queryAction[1] === 'duplicates' || - queryAction[1] === 'group' || + queryAction[1] === 'groupBy' || queryAction[1] === 'merge' ) { return { object: queryAction[0] }; diff --git a/packages/twenty-server/src/engine/api/rest/core/services/rest-api-core.service.ts b/packages/twenty-server/src/engine/api/rest/core/services/rest-api-core.service.ts index e8a617dc54..3fa8056d2c 100644 --- a/packages/twenty-server/src/engine/api/rest/core/services/rest-api-core.service.ts +++ b/packages/twenty-server/src/engine/api/rest/core/services/rest-api-core.service.ts @@ -122,15 +122,7 @@ export class RestApiCoreService { } async groupBy(request: AuthenticatedRequest) { - const isCommonApiEnabled = await this.isCommonApiEnabled(request); - - if (isCommonApiEnabled) { - return await this.restApiGroupByHandler.handle(request); - } else { - throw new BadRequestException( - 'Activate feature flag to use GroupBy in the REST API', - ); - } + return await this.restApiGroupByHandler.handle(request); } async delete(request: AuthenticatedRequest) { diff --git a/packages/twenty-server/src/engine/api/rest/input-request-parsers/group-by-with-records/parse-include-records-sample-rest-request.util.ts b/packages/twenty-server/src/engine/api/rest/input-request-parsers/group-by-with-records/parse-include-records-sample-rest-request.util.ts new file mode 100644 index 0000000000..9323e79095 --- /dev/null +++ b/packages/twenty-server/src/engine/api/rest/input-request-parsers/group-by-with-records/parse-include-records-sample-rest-request.util.ts @@ -0,0 +1,13 @@ +import { isDefined } from 'twenty-shared/utils'; + +import { type AuthenticatedRequest } from 'src/engine/api/rest/types/authenticated-request'; + +export const parseIncludeRecordsSampleRestRequest = ( + request: AuthenticatedRequest, +): boolean => { + if (!isDefined(request.query.include_records_sample)) { + return false; + } + + return request.query.include_records_sample === 'true'; +}; diff --git a/packages/twenty-server/src/engine/twenty-orm/repository/permissions.utils.ts b/packages/twenty-server/src/engine/twenty-orm/repository/permissions.utils.ts index 357d303a8e..0503fc3874 100644 --- a/packages/twenty-server/src/engine/twenty-orm/repository/permissions.utils.ts +++ b/packages/twenty-server/src/engine/twenty-orm/repository/permissions.utils.ts @@ -17,7 +17,34 @@ import { import { type ObjectMetadataMaps } from 'src/engine/metadata-modules/types/object-metadata-maps'; import { getColumnNameToFieldMetadataIdMap } from 'src/engine/twenty-orm/utils/get-column-name-to-field-metadata-id.util'; -const getTargetEntityAndOperationType = (expressionMap: QueryExpressionMap) => { +const getTargetEntityAndOperationType = ( + expressionMap: QueryExpressionMap, +): + | { + isSubQuery: true; + mainEntity?: undefined; + operationType?: undefined; + } + | { + isSubQuery?: undefined; + mainEntity: string; + operationType: + | 'select' + | 'insert' + | 'update' + | 'delete' + | 'restore' + | 'soft-delete' + | 'relation'; + } => { + const isSubQuery = expressionMap.aliases[0].subQuery; + + if (isSubQuery) { + return { + isSubQuery: true, // will bypass permission checks because subQuery permissions will be evaluated when it is executed. This is valid for groupBy with records usecase. If your usecase is different, make sure permission checks are run. + }; + } + const mainEntity = expressionMap.aliases[0].metadata.name; const operationType = expressionMap.queryType; @@ -181,9 +208,13 @@ export const validateQueryIsPermittedOrThrow = ({ return; } - const { mainEntity, operationType } = + const { mainEntity, operationType, isSubQuery } = getTargetEntityAndOperationType(expressionMap); + if (isSubQuery) { + return; + } + const allFieldsSelected = expressionMap.selects.some( (select) => select.selection === mainEntity, ); diff --git a/packages/twenty-server/test/integration/graphql/suites/group-by-resolvers.integration-spec.ts b/packages/twenty-server/test/integration/graphql/suites/group-by-resolver.integration-spec.ts similarity index 99% rename from packages/twenty-server/test/integration/graphql/suites/group-by-resolvers.integration-spec.ts rename to packages/twenty-server/test/integration/graphql/suites/group-by-resolver.integration-spec.ts index a49a284e98..5f8497ff7a 100644 --- a/packages/twenty-server/test/integration/graphql/suites/group-by-resolvers.integration-spec.ts +++ b/packages/twenty-server/test/integration/graphql/suites/group-by-resolver.integration-spec.ts @@ -16,7 +16,7 @@ import { type FieldMetadataDTO } from 'src/engine/metadata-modules/field-metadat import { type ObjectMetadataDTO } from 'src/engine/metadata-modules/object-metadata/dtos/object-metadata.dto'; import { ViewFilterGroupLogicalOperator } from 'src/modules/view/standard-objects/view-filter-group.workspace-entity'; -describe('group-by resolvers (integration)', () => { +describe('group-by resolver (integration)', () => { describe('standard case', () => { const testPersonId = randomUUID(); const testPerson2Id = randomUUID(); diff --git a/packages/twenty-server/test/integration/graphql/suites/group-by-with-records-resolver.integration-spec.ts b/packages/twenty-server/test/integration/graphql/suites/group-by-with-records-resolver.integration-spec.ts new file mode 100644 index 0000000000..0afb3c08e6 --- /dev/null +++ b/packages/twenty-server/test/integration/graphql/suites/group-by-with-records-resolver.integration-spec.ts @@ -0,0 +1,352 @@ +import { randomUUID } from 'crypto'; + +import gql from 'graphql-tag'; +import { COMPANY_GQL_FIELDS } from 'test/integration/constants/company-gql-fields.constants'; +import { createOneOperationFactory } from 'test/integration/graphql/utils/create-one-operation-factory.util'; +import { destroyOneOperationFactory } from 'test/integration/graphql/utils/destroy-one-operation-factory.util'; +import { makeGraphqlAPIRequest } from 'test/integration/graphql/utils/make-graphql-api-request.util'; + +const OPPORTUNITY_GQL_FIELDS = ` + id + stage + amount { + amountMicros + } + companyId + createdAt + closeDate +`; + +// used not to mix records with the seeded ones +const FILTER_2020 = { + and: [ + { + createdAt: { + gte: '2020-01-01T00:00:00.000Z', + }, + }, + { + createdAt: { + lte: '2020-03-03T23:59:59.999Z', + }, + }, + ], +}; + +describe('basic group-by with records', () => { + const testOpportunityId1 = randomUUID(); + const testOpportunityId2 = randomUUID(); + const testOpportunityId3 = randomUUID(); + const testOpportunityId4 = randomUUID(); + const testCompanyId1 = randomUUID(); + const testCompanyId2 = randomUUID(); + + beforeAll(async () => { + // Create test companies + await makeGraphqlAPIRequest( + createOneOperationFactory({ + objectMetadataSingularName: 'company', + gqlFields: COMPANY_GQL_FIELDS, + data: { + id: testCompanyId1, + name: 'Company 1', + employees: 10, + }, + }), + ); + + await makeGraphqlAPIRequest( + createOneOperationFactory({ + objectMetadataSingularName: 'company', + gqlFields: COMPANY_GQL_FIELDS, + data: { + id: testCompanyId2, + name: 'Company 2', + employees: 20, + }, + }), + ); + + // Create test opportunities with different stages and dates + await makeGraphqlAPIRequest( + createOneOperationFactory({ + objectMetadataSingularName: 'opportunity', + gqlFields: OPPORTUNITY_GQL_FIELDS, + data: { + id: testOpportunityId1, + stage: 'NEW', + name: 'Opportunity 1', + amount: { amountMicros: 1000000000000 }, // 1000 + companyId: testCompanyId1, + closeDate: '2025-02-05T08:00:00.000Z', // Wednesday + createdAt: '2020-02-05T08:00:00.000Z', + }, + }), + ); + + await makeGraphqlAPIRequest( + createOneOperationFactory({ + objectMetadataSingularName: 'opportunity', + gqlFields: OPPORTUNITY_GQL_FIELDS, + data: { + id: testOpportunityId2, + stage: 'NEW', + name: 'Opportunity 2', + amount: { amountMicros: 2000000000000 }, // 2000 + companyId: testCompanyId1, + closeDate: '2025-02-06T08:00:00.000Z', // Thursday + createdAt: '2020-02-05T08:00:00.000Z', + }, + }), + ); + + await makeGraphqlAPIRequest( + createOneOperationFactory({ + objectMetadataSingularName: 'opportunity', + gqlFields: OPPORTUNITY_GQL_FIELDS, + data: { + id: testOpportunityId3, + stage: 'NEW', + name: 'Opportunity 3', + amount: { amountMicros: 3000000000000 }, // 3000 + companyId: testCompanyId2, + closeDate: '2025-02-06T08:00:00.000Z', // Thursday + createdAt: '2020-02-05T08:00:00.000Z', + }, + }), + ); + + await makeGraphqlAPIRequest( + createOneOperationFactory({ + objectMetadataSingularName: 'opportunity', + gqlFields: OPPORTUNITY_GQL_FIELDS, + data: { + id: testOpportunityId4, + stage: 'SCREENING', + name: 'Opportunity 4', + amount: { amountMicros: 4000000000000 }, // 4000 + companyId: testCompanyId2, + closeDate: '2025-02-06T08:00:00.000Z', // Thursday + createdAt: '2020-02-05T08:00:00.000Z', + }, + }), + ); + }); + + afterAll(async () => { + // Cleanup created opportunities + for (const id of [ + testOpportunityId1, + testOpportunityId2, + testOpportunityId3, + testOpportunityId4, + ]) { + await makeGraphqlAPIRequest( + destroyOneOperationFactory({ + objectMetadataSingularName: 'opportunity', + gqlFields: 'id', + recordId: id, + }), + ); + } + + // Cleanup created companies + for (const id of [testCompanyId1, testCompanyId2]) { + await makeGraphqlAPIRequest( + destroyOneOperationFactory({ + objectMetadataSingularName: 'company', + gqlFields: 'id', + recordId: id, + }), + ); + } + }); + + it('groups by stage and createdAt with records', async () => { + const response = await makeGraphqlAPIRequest({ + query: gql` + query OpportunitiesGroupBy( + $groupBy: [OpportunityGroupByInput!] + $filter: OpportunityFilterInput + ) { + opportunitiesGroupBy(groupBy: $groupBy, filter: $filter) { + minCloseDate + groupByDimensionValues + sumAmountAmountMicros + __typename + edges { + node { + stage + closeDate + name + amount { + amountMicros + } + } + } + } + } + `, + variables: { + groupBy: [ + { + closeDate: { + granularity: 'DAY_OF_THE_WEEK', + }, + }, + { + stage: true, + }, + ], + filter: FILTER_2020, + }, + }); + + expect(response.body.errors).toBeUndefined(); + expect(response.body.data).toBeDefined(); + + const groups = response.body.data.opportunitiesGroupBy; + + expect(groups).toBeDefined(); + expect(groups).toHaveLength(3); + + // Check that each group has the expected structure + groups.forEach((group: any) => { + expect(group.groupByDimensionValues).toBeDefined(); + expect(Array.isArray(group.groupByDimensionValues)).toBe(true); + expect(group.edges).toBeDefined(); + expect(Array.isArray(group.edges)).toBe(true); + }); + + // Find specific groups and verify their content + const wednesdayNewGroup = groups.find( + (group: any) => + group.groupByDimensionValues.includes('Wednesday') && + group.groupByDimensionValues.includes('NEW'), + ); + + expect(wednesdayNewGroup).toBeDefined(); + expect(wednesdayNewGroup.edges).toHaveLength(1); + expect(wednesdayNewGroup.edges[0].node.name).toBe('Opportunity 1'); + expect(wednesdayNewGroup.edges[0].node.stage).toBe('NEW'); + expect(wednesdayNewGroup.edges[0].node.amount.amountMicros).toBe( + 1000000000000, + ); + expect(wednesdayNewGroup.sumAmountAmountMicros).toBe(1000000000000); + + const thursdayNewGroup = groups.find( + (group: any) => + group.groupByDimensionValues.includes('Thursday') && + group.groupByDimensionValues.includes('NEW'), + ); + + expect(thursdayNewGroup).toBeDefined(); + expect(thursdayNewGroup.edges).toHaveLength(2); + expect(thursdayNewGroup.sumAmountAmountMicros).toBe(5000000000000); + for (const edge of thursdayNewGroup.edges) { + expect(edge.node.stage).toBe('NEW'); + expect(edge.node.name).toBeDefined(); + expect( + edge.node.name === 'Opportunity 2' || + edge.node.name === 'Opportunity 3', + ).toBe(true); + } + + const thursdayScreeningGroup = groups.find( + (group: any) => + group.groupByDimensionValues.includes('Thursday') && + group.groupByDimensionValues.includes('SCREENING'), + ); + + expect(thursdayScreeningGroup).toBeDefined(); + expect(thursdayScreeningGroup.edges).toHaveLength(1); + expect(thursdayScreeningGroup.edges[0].node.amount.amountMicros).toBe( + 4000000000000, + ); + expect(thursdayScreeningGroup.edges[0].node.stage).toBe('SCREENING'); + expect(thursdayScreeningGroup.edges[0].node.name).toBe('Opportunity 4'); + expect(thursdayScreeningGroup.sumAmountAmountMicros).toBe(4000000000000); + }); + + it('groups by stage and createdAt with records and filters', async () => { + // Test with filter to only include NEW stage opportunities + const response = await makeGraphqlAPIRequest({ + query: gql` + query OpportunitiesGroupBy( + $groupBy: [OpportunityGroupByInput!] + $filter: OpportunityFilterInput + ) { + opportunitiesGroupBy(groupBy: $groupBy, filter: $filter) { + minCloseDate + groupByDimensionValues + sumAmountAmountMicros + __typename + edges { + node { + stage + createdAt + amount { + amountMicros + } + } + } + } + } + `, + variables: { + groupBy: [ + { + closeDate: { + granularity: 'DAY_OF_THE_WEEK', + }, + }, + { + stage: true, + }, + ], + filter: { + and: [ + ...FILTER_2020.and, + { + stage: { + eq: 'NEW', + }, + }, + ], + }, + }, + }); + + expect(response.body.errors).toBeUndefined(); + expect(response.body.data).toBeDefined(); + + const groups = response.body.data.opportunitiesGroupBy; + + expect(groups).toHaveLength(2); + const wednesdayNewGroup = groups.find( + (group: any) => + group.groupByDimensionValues.includes('Wednesday') && + group.groupByDimensionValues.includes('NEW'), + ); + + expect(wednesdayNewGroup.groupByDimensionValues).toHaveLength(2); + expect(wednesdayNewGroup.groupByDimensionValues).toContain('NEW'); + expect(wednesdayNewGroup.groupByDimensionValues).toContain('Wednesday'); + expect(wednesdayNewGroup.edges).toHaveLength(1); + expect(wednesdayNewGroup.edges[0].node.stage).toBe('NEW'); + expect(wednesdayNewGroup.edges[0].node.amount.amountMicros).toBe( + 1000000000000, + ); + + const thursdayNewGroup = groups.find( + (group: any) => + group.groupByDimensionValues.includes('Thursday') && + group.groupByDimensionValues.includes('NEW'), + ); + + expect(thursdayNewGroup.groupByDimensionValues).toHaveLength(2); + expect(thursdayNewGroup.groupByDimensionValues).toContain('NEW'); + expect(thursdayNewGroup.groupByDimensionValues).toContain('Thursday'); + expect(thursdayNewGroup.edges).toHaveLength(2); + }); +});