Cap nested relation query concurrency (#23252)

## Context

Common API queries load selected relations after fetching the root
records.

Relation loading is batched: one query pipeline loads a relation for all
parent records, so this is not an N+1 problem. However, every sibling
relation currently starts concurrently through `Promise.all`.

Nested relations repeat the same behavior recursively. A wide selection
can therefore submit many independent relation query pipelines at once.

Existing query complexity and record limits restrict what can be
requested, but they do not limit how much database work starts
concurrently.

## What this changes

This PR adds a request-local FIFO concurrency limiter for nested
relation loading.

- At most four `findRelations` pipelines execute concurrently.
- One limiter is created for the outer relation-loading call.
- The same limiter is shared by every recursive level.
- Queued work starts as permits become available.
- Permits are released in `finally`, including when a query fails.

Conceptually:

```text
Before:
all sibling relations -> database concurrently
nested siblings       -> more database work concurrently

After:
all sibling relations -> FIFO queue -> at most 4 database pipelines
nested siblings       -> same FIFO queue and same limit
```

Note: Also addressing
https://github.com/twentyhq/twenty/pull/23251#discussion_r3644510597
This commit is contained in:
Weiko
2026-07-24 15:27:26 +02:00
committed by GitHub
parent 9390c28cb6
commit abe4d7491c
5 changed files with 272 additions and 67 deletions
@@ -59,16 +59,23 @@ describe('DatabasePoolMetricsService', () => {
let service: DatabasePoolMetricsService;
let gaugeCallbacks: Map<string, GaugeCallback>;
let histogramRecord: jest.Mock;
let counterAdd: jest.Mock;
let createCounter: jest.Mock;
beforeEach(() => {
gaugeCallbacks = new Map();
histogramRecord = jest.fn();
counterAdd = jest.fn();
createCounter = jest.fn().mockReturnValue({
add: counterAdd,
});
const metricsService = {
getMeter: jest.fn().mockReturnValue({
createHistogram: jest.fn().mockReturnValue({
record: histogramRecord,
}),
createCounter,
}),
createMultiObservableGauge: jest
.fn()
@@ -191,6 +198,7 @@ describe('DatabasePoolMetricsService', () => {
expect(histogramRecord).toHaveBeenCalledWith(0.25, {
pool: DatabasePoolName.WorkspacePrimary,
});
expect(counterAdd).not.toHaveBeenCalled();
});
it('records failed connection acquisitions', async () => {
@@ -204,12 +212,21 @@ describe('DatabasePoolMetricsService', () => {
dataSource: dataSource.dataSource,
});
await expect(dataSource.driver.obtainMasterConnection()).rejects.toThrow(
await expect(dataSource.driver.obtainMasterConnection()).rejects.toBe(
error,
);
expect(histogramRecord).toHaveBeenCalledWith(0, {
pool: DatabasePoolName.Core,
});
expect(counterAdd).toHaveBeenCalledWith(1, {
pool: DatabasePoolName.Core,
});
expect(createCounter).toHaveBeenCalledWith(
'twenty_database_pool_acquisition_failures',
{
description: 'Number of failed PostgreSQL pool connection acquisitions',
},
);
});
it('does not instrument a data source more than once', async () => {
@@ -1,6 +1,6 @@
import { Injectable } from '@nestjs/common';
import { type Histogram } from '@opentelemetry/api';
import { type Counter, type Histogram } from '@opentelemetry/api';
import { type Pool } from 'pg';
import { type DataSource } from 'typeorm';
import { type PostgresDriver } from 'typeorm/driver/postgres/PostgresDriver';
@@ -46,18 +46,28 @@ export class DatabasePoolMetricsService {
private readonly pools = new Map<DatabasePoolName, Pool>();
private readonly instrumentedDrivers = new WeakSet<PostgresDriver>();
private readonly acquisitionDurationHistogram: Histogram;
private readonly acquisitionFailureCounter: Counter;
constructor(private readonly metricsService: MetricsService) {
this.acquisitionDurationHistogram = this.metricsService
.getMeter()
.createHistogram('twenty_database_pool_acquisition_duration_seconds', {
const meter = this.metricsService.getMeter();
this.acquisitionDurationHistogram = meter.createHistogram(
'twenty_database_pool_acquisition_duration_seconds',
{
description:
'Time spent acquiring a connection from the PostgreSQL pool',
unit: 's',
advice: {
explicitBucketBoundaries: ACQUISITION_DURATION_BUCKETS_SECONDS,
},
});
},
);
this.acquisitionFailureCounter = meter.createCounter(
'twenty_database_pool_acquisition_failures',
{
description: 'Number of failed PostgreSQL pool connection acquisitions',
},
);
for (const gauge of POOL_GAUGES) {
this.metricsService.createMultiObservableGauge({
@@ -99,6 +109,12 @@ export class DatabasePoolMetricsService {
try {
return await obtainMasterConnection();
} catch (error) {
this.acquisitionFailureCounter.add(1, {
pool: poolName,
});
throw error;
} finally {
this.acquisitionDurationHistogram.record(
(performance.now() - start) / 1000,