Cap nested relation query concurrency (#23252)
## Context Common API queries load selected relations after fetching the root records. Relation loading is batched: one query pipeline loads a relation for all parent records, so this is not an N+1 problem. However, every sibling relation currently starts concurrently through `Promise.all`. Nested relations repeat the same behavior recursively. A wide selection can therefore submit many independent relation query pipelines at once. Existing query complexity and record limits restrict what can be requested, but they do not limit how much database work starts concurrently. ## What this changes This PR adds a request-local FIFO concurrency limiter for nested relation loading. - At most four `findRelations` pipelines execute concurrently. - One limiter is created for the outer relation-loading call. - The same limiter is shared by every recursive level. - Queued work starts as permits become available. - Permits are released in `finally`, including when a query fails. Conceptually: ```text Before: all sibling relations -> database concurrently nested siblings -> more database work concurrently After: all sibling relations -> FIFO queue -> at most 4 database pipelines nested siblings -> same FIFO queue and same limit ``` Note: Also addressing https://github.com/twentyhq/twenty/pull/23251#discussion_r3644510597
This commit is contained in:
@@ -59,16 +59,23 @@ describe('DatabasePoolMetricsService', () => {
|
||||
let service: DatabasePoolMetricsService;
|
||||
let gaugeCallbacks: Map<string, GaugeCallback>;
|
||||
let histogramRecord: jest.Mock;
|
||||
let counterAdd: jest.Mock;
|
||||
let createCounter: jest.Mock;
|
||||
|
||||
beforeEach(() => {
|
||||
gaugeCallbacks = new Map();
|
||||
histogramRecord = jest.fn();
|
||||
counterAdd = jest.fn();
|
||||
createCounter = jest.fn().mockReturnValue({
|
||||
add: counterAdd,
|
||||
});
|
||||
|
||||
const metricsService = {
|
||||
getMeter: jest.fn().mockReturnValue({
|
||||
createHistogram: jest.fn().mockReturnValue({
|
||||
record: histogramRecord,
|
||||
}),
|
||||
createCounter,
|
||||
}),
|
||||
createMultiObservableGauge: jest
|
||||
.fn()
|
||||
@@ -191,6 +198,7 @@ describe('DatabasePoolMetricsService', () => {
|
||||
expect(histogramRecord).toHaveBeenCalledWith(0.25, {
|
||||
pool: DatabasePoolName.WorkspacePrimary,
|
||||
});
|
||||
expect(counterAdd).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('records failed connection acquisitions', async () => {
|
||||
@@ -204,12 +212,21 @@ describe('DatabasePoolMetricsService', () => {
|
||||
dataSource: dataSource.dataSource,
|
||||
});
|
||||
|
||||
await expect(dataSource.driver.obtainMasterConnection()).rejects.toThrow(
|
||||
await expect(dataSource.driver.obtainMasterConnection()).rejects.toBe(
|
||||
error,
|
||||
);
|
||||
expect(histogramRecord).toHaveBeenCalledWith(0, {
|
||||
pool: DatabasePoolName.Core,
|
||||
});
|
||||
expect(counterAdd).toHaveBeenCalledWith(1, {
|
||||
pool: DatabasePoolName.Core,
|
||||
});
|
||||
expect(createCounter).toHaveBeenCalledWith(
|
||||
'twenty_database_pool_acquisition_failures',
|
||||
{
|
||||
description: 'Number of failed PostgreSQL pool connection acquisitions',
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
it('does not instrument a data source more than once', async () => {
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { Injectable } from '@nestjs/common';
|
||||
|
||||
import { type Histogram } from '@opentelemetry/api';
|
||||
import { type Counter, type Histogram } from '@opentelemetry/api';
|
||||
import { type Pool } from 'pg';
|
||||
import { type DataSource } from 'typeorm';
|
||||
import { type PostgresDriver } from 'typeorm/driver/postgres/PostgresDriver';
|
||||
@@ -46,18 +46,28 @@ export class DatabasePoolMetricsService {
|
||||
private readonly pools = new Map<DatabasePoolName, Pool>();
|
||||
private readonly instrumentedDrivers = new WeakSet<PostgresDriver>();
|
||||
private readonly acquisitionDurationHistogram: Histogram;
|
||||
private readonly acquisitionFailureCounter: Counter;
|
||||
|
||||
constructor(private readonly metricsService: MetricsService) {
|
||||
this.acquisitionDurationHistogram = this.metricsService
|
||||
.getMeter()
|
||||
.createHistogram('twenty_database_pool_acquisition_duration_seconds', {
|
||||
const meter = this.metricsService.getMeter();
|
||||
|
||||
this.acquisitionDurationHistogram = meter.createHistogram(
|
||||
'twenty_database_pool_acquisition_duration_seconds',
|
||||
{
|
||||
description:
|
||||
'Time spent acquiring a connection from the PostgreSQL pool',
|
||||
unit: 's',
|
||||
advice: {
|
||||
explicitBucketBoundaries: ACQUISITION_DURATION_BUCKETS_SECONDS,
|
||||
},
|
||||
});
|
||||
},
|
||||
);
|
||||
this.acquisitionFailureCounter = meter.createCounter(
|
||||
'twenty_database_pool_acquisition_failures',
|
||||
{
|
||||
description: 'Number of failed PostgreSQL pool connection acquisitions',
|
||||
},
|
||||
);
|
||||
|
||||
for (const gauge of POOL_GAUGES) {
|
||||
this.metricsService.createMultiObservableGauge({
|
||||
@@ -99,6 +109,12 @@ export class DatabasePoolMetricsService {
|
||||
|
||||
try {
|
||||
return await obtainMasterConnection();
|
||||
} catch (error) {
|
||||
this.acquisitionFailureCounter.add(1, {
|
||||
pool: poolName,
|
||||
});
|
||||
|
||||
throw error;
|
||||
} finally {
|
||||
this.acquisitionDurationHistogram.record(
|
||||
(performance.now() - start) / 1000,
|
||||
|
||||
Reference in New Issue
Block a user