houseassassin/twenty
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Extract field metadata build out of object metadata build (#14763)

Browse Source 
# Introduction

Extracting the legacy fields build and dispatch out of the object one to
follow the generic flat entity build, also update caches entries for
object

## Main tasks
- flat field map cache
- flat field builder
- refactored the dispatch matrix to return flat entity maps instead of
flat entity arrays
- orchestrator aggregator
- removing legacy code
- making universal identifier aka standardId of standard field for
custom object deterministically dynamic
- perfs debug logs for v2

## TODO
- [x] Refactor the generic entity builder to be dependency flat maps in
order to main foreign keys list in flat parent
- [x] Refactor the flat object metadata to contain the array of related
fields and avoid costy find object fields
- [ ] Improve the create field handler to handle multiple field at once
- [ ] Refactor the dispatch to embbed the comparison
- [ ] Improve perf by extracting from elements out of existing
- [ ] Fix the labelIdentifierId validators on object before field
creation ( integ tests are in failing mode )

## Debug logs snippet

```ts
[EntityBuilder fieldMetadata] matrix computation: 0.027ms
[EntityBuilder fieldMetadata] creation validation: 0.001ms
[EntityBuilder fieldMetadata] deletion validation: 0.293ms
[EntityBuilder fieldMetadata] update validation: 0.006ms
[EntityBuilder fieldMetadata] entity processing: 0.363ms
[EntityBuilder fieldMetadata] validateAndBuild: 0.455ms
[EntityBuilder index] matrix computation: 0.005ms
[EntityBuilder index] creation validation: 0.001ms
[EntityBuilder index] deletion validation: 0.146ms
[EntityBuilder index] update validation: 0.004ms
[EntityBuilder index] entity processing: 0.199ms
[EntityBuilder index] validateAndBuild: 0.228ms
[Runner] Initial cache retrieval: 0.549ms
[BaseWorkspaceMigrationRunnerActionHandlerService] delete_index executeForWorkspaceSchema: 11.665ms
[BaseWorkspaceMigrationRunnerActionHandlerService] delete_index executeForMetadata: 12.864ms
[BaseWorkspaceMigrationRunnerActionHandlerService] delete_field executeForWorkspaceSchema: 1.476ms
[BaseWorkspaceMigrationRunnerActionHandlerService] delete_field executeForMetadata: 6.816ms
[BaseWorkspaceMigrationRunnerActionHandlerService] delete_field executeForWorkspaceSchema: 0.062ms
[BaseWorkspaceMigrationRunnerActionHandlerService] delete_field executeForMetadata: 0.889ms
[Runner] Transaction execution: 23.434ms
[Runner] Cache invalidation: 316.662ms
[Runner] Total execution: 340.767ms
```
As you can see cache invalidation is way to long, we could replace the
cache by the optimistic in the end
This commit is contained in:
Paul Rastoin
2025-10-02 18:07:34 +02:00
committed by GitHub
parent d4b83df3f5
commit 170f6d27c5
240 changed files with 11385 additions and 17744 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/twenty-server/src/engine/dataloaders/dataloader.module.ts
+6 -1
View File
@@ -1,11 +1,16 @@
import { Module } from '@nestjs/common';
import { WorkspaceManyOrAllFlatEntityMapsCacheModule } from 'src/engine/core-modules/common/services/workspace-many-or-all-flat-entity-maps-cache.module';
import { DataloaderService } from 'src/engine/dataloaders/dataloader.service';
import { FieldMetadataModule } from 'src/engine/metadata-modules/field-metadata/field-metadata.module';
import { WorkspaceMetadataCacheModule } from 'src/engine/metadata-modules/workspace-metadata-cache/workspace-metadata-cache.module';
@Module({
imports: [FieldMetadataModule, WorkspaceMetadataCacheModule],
imports: [
FieldMetadataModule,
WorkspaceMetadataCacheModule,
WorkspaceManyOrAllFlatEntityMapsCacheModule,
],
providers: [DataloaderService],
exports: [DataloaderService],
})
packages/twenty-server/src/engine/dataloaders/dataloader.service.ts
+113 -60
View File
@@ -7,6 +7,8 @@ import { isDefined } from 'twenty-shared/utils';
import { type IndexMetadataInterface } from 'src/engine/metadata-modules/index-metadata/interfaces/index-metadata.interface';
import { WorkspaceManyOrAllFlatEntityMapsCacheService } from 'src/engine/core-modules/common/services/workspace-many-or-all-flat-entity-maps-cache.service';
import { findFlatEntityByIdInFlatEntityMapsOrThrow } from 'src/engine/core-modules/common/utils/find-flat-entity-by-id-in-flat-entity-maps-or-throw.util';
import { I18nService } from 'src/engine/core-modules/i18n/i18n.service';
import { type IDataloaders } from 'src/engine/dataloaders/dataloader.interface';
import { filterMorphRelationDuplicateFields } from 'src/engine/dataloaders/utils/filter-morph-relation-duplicate-fields.util';
@@ -16,11 +18,11 @@ import { RelationDTO } from 'src/engine/metadata-modules/field-metadata/dtos/rel
import { type FieldMetadataEntity } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity';
import { resolveFieldMetadataStandardOverride } from 'src/engine/metadata-modules/field-metadata/utils/resolve-field-metadata-standard-override.util';
import { findAllOthersMorphRelationFlatFieldMetadatasOrThrow } from 'src/engine/metadata-modules/flat-field-metadata/utils/find-all-others-morph-relation-flat-field-metadatas-or-throw.util';
import { findObjectFieldsInFlatFieldMetadataMapsOrThrow } from 'src/engine/metadata-modules/flat-field-metadata/utils/find-object-fields-in-flat-field-metadata-maps-or-throw.util';
import { fromFlatFieldMetadataToFieldMetadataDto } from 'src/engine/metadata-modules/flat-field-metadata/utils/from-flat-field-metadata-to-field-metadata-dto.util';
import { fromMorphOrRelationFlatFieldMetadataToRelationDto } from 'src/engine/metadata-modules/flat-field-metadata/utils/from-morph-or-relation-flat-field-metadata-to-relation-dto.util';
import { isFlatFieldMetadataOfType } from 'src/engine/metadata-modules/flat-field-metadata/utils/is-flat-field-metadata-of-type.util';
import { findFlatFieldMetadataInFlatObjectMetadataMapsOrThrow } from 'src/engine/metadata-modules/flat-object-metadata-maps/utils/find-flat-field-metadata-in-flat-object-metadata-maps-or-throw.util';
import { findFlatObjectMetadataInFlatObjectMetadataMapsOrThrow } from 'src/engine/metadata-modules/flat-object-metadata-maps/utils/find-flat-object-metadata-in-flat-object-metadata-maps-or-throw.util';
import { isMorphOrRelationFlatFieldMetadata } from 'src/engine/metadata-modules/flat-field-metadata/utils/is-morph-or-relation-flat-field-metadata.util';
import { type IndexFieldMetadataDTO } from 'src/engine/metadata-modules/index-metadata/dtos/index-field-metadata.dto';
import { type IndexMetadataDTO } from 'src/engine/metadata-modules/index-metadata/dtos/index-metadata.dto';
import { type ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity';
@@ -70,6 +72,7 @@ export type ObjectMetadataLoaderPayload = {
export class DataloaderService {
constructor(
private readonly i18nService: I18nService,
private readonly flatEntityMapsCacheService: WorkspaceManyOrAllFlatEntityMapsCacheService,
private readonly workspaceMetadataCacheService: WorkspaceMetadataCacheService,
) {}
@@ -96,19 +99,19 @@ export class DataloaderService {
async (dataLoaderParams: RelationLoaderPayload[]) => {
const relationDtos: Array<RelationDTO | null> = [];
const workspaceId = dataLoaderParams[0].workspaceId;
const { flatObjectMetadataMaps: existingFlatObjectMetadataMaps } =
await this.workspaceMetadataCacheService.getExistingOrRecomputeFlatObjectMetadataMaps(
const { flatFieldMetadataMaps, flatObjectMetadataMaps } =
await this.flatEntityMapsCacheService.getOrRecomputeManyOrAllFlatEntityMaps(
{
workspaceId,
flatEntities: ['flatFieldMetadataMaps', 'flatObjectMetadataMaps'],
},
);
for (const { fieldMetadataId, objectMetadataId } of dataLoaderParams) {
for (const { fieldMetadataId } of dataLoaderParams) {
const sourceFlatFieldMetadata =
findFlatFieldMetadataInFlatObjectMetadataMapsOrThrow({
fieldMetadataId,
flatObjectMetadataMaps: existingFlatObjectMetadataMaps,
objectMetadataId,
findFlatEntityByIdInFlatEntityMapsOrThrow({
flatEntityId: fieldMetadataId,
flatEntityMaps: flatFieldMetadataMaps,
});
if (
@@ -122,32 +125,48 @@ export class DataloaderService {
}
const sourceFlatObjectMetadata =
findFlatObjectMetadataInFlatObjectMetadataMapsOrThrow({
flatObjectMetadataMaps: existingFlatObjectMetadataMaps,
objectMetadataId: sourceFlatFieldMetadata.objectMetadataId,
findFlatEntityByIdInFlatEntityMapsOrThrow({
flatEntityId: sourceFlatFieldMetadata.objectMetadataId,
flatEntityMaps: flatObjectMetadataMaps,
});
const targetFlatFieldMetadata =
findFlatEntityByIdInFlatEntityMapsOrThrow({
flatEntityId:
sourceFlatFieldMetadata.relationTargetFieldMetadataId,
flatEntityMaps: flatFieldMetadataMaps,
});
if (!isMorphOrRelationFlatFieldMetadata(targetFlatFieldMetadata)) {
relationDtos.push(null);
continue;
}
const targetFlatObjectMetadata =
findFlatEntityByIdInFlatEntityMapsOrThrow({
flatEntityId: targetFlatFieldMetadata.objectMetadataId,
flatEntityMaps: flatObjectMetadataMaps,
});
if (
isFlatFieldMetadataOfType(
sourceFlatFieldMetadata.flatRelationTargetFieldMetadata,
targetFlatFieldMetadata,
FieldMetadataType.MORPH_RELATION,
)
) {
const allMorphFlatFieldMetadatas =
findAllOthersMorphRelationFlatFieldMetadatasOrThrow({
flatFieldMetadata:
sourceFlatFieldMetadata.flatRelationTargetFieldMetadata,
flatObjectMetadataMaps: existingFlatObjectMetadataMaps,
flatFieldMetadata: targetFlatFieldMetadata,
flatFieldMetadataMaps,
flatObjectMetadata: targetFlatObjectMetadata,
}).sort((a, b) => (a.id > b.id ? 1 : -1));
relationDtos.push(
fromMorphOrRelationFlatFieldMetadataToRelationDto({
flatFieldMetadata: {
...sourceFlatFieldMetadata,
flatRelationTargetFieldMetadata:
allMorphFlatFieldMetadatas[0],
},
sourceFlatFieldMetadata,
sourceFlatObjectMetadata,
targetFlatFieldMetadata: allMorphFlatFieldMetadatas[0],
targetFlatObjectMetadata,
}),
);
continue;
@@ -155,7 +174,9 @@ export class DataloaderService {
relationDtos.push(
fromMorphOrRelationFlatFieldMetadataToRelationDto({
flatFieldMetadata: sourceFlatFieldMetadata,
sourceFlatFieldMetadata,
targetFlatFieldMetadata,
targetFlatObjectMetadata,
sourceFlatObjectMetadata,
}),
);
@@ -170,20 +191,20 @@ export class DataloaderService {
return new DataLoader<MorphRelationLoaderPayload, RelationDTO[] | null>(
async (dataLoaderParams: MorphRelationLoaderPayload[]) => {
const workspaceId = dataLoaderParams[0].workspaceId;
const { flatObjectMetadataMaps: existingFlatObjectMetadataMaps } =
await this.workspaceMetadataCacheService.getExistingOrRecomputeFlatObjectMetadataMaps(
const { flatFieldMetadataMaps, flatObjectMetadataMaps } =
await this.flatEntityMapsCacheService.getOrRecomputeManyOrAllFlatEntityMaps(
{
workspaceId,
flatEntities: ['flatFieldMetadataMaps', 'flatObjectMetadataMaps'],
},
);
const relationDtos: Array<RelationDTO[] | null> = [];
for (const { fieldMetadataId, objectMetadataId } of dataLoaderParams) {
for (const { fieldMetadataId } of dataLoaderParams) {
const morphFlatFieldMetadata =
findFlatFieldMetadataInFlatObjectMetadataMapsOrThrow({
fieldMetadataId,
flatObjectMetadataMaps: existingFlatObjectMetadataMaps,
objectMetadataId,
findFlatEntityByIdInFlatEntityMapsOrThrow({
flatEntityId: fieldMetadataId,
flatEntityMaps: flatFieldMetadataMaps,
});
if (
@@ -197,15 +218,16 @@ export class DataloaderService {
}
const sourceFlatObjectMetadata =
findFlatObjectMetadataInFlatObjectMetadataMapsOrThrow({
flatObjectMetadataMaps: existingFlatObjectMetadataMaps,
objectMetadataId: morphFlatFieldMetadata.objectMetadataId,
findFlatEntityByIdInFlatEntityMapsOrThrow({
flatEntityMaps: flatObjectMetadataMaps,
flatEntityId: morphFlatFieldMetadata.objectMetadataId,
});
const relatedMorphFlatFieldMetadatas =
findAllOthersMorphRelationFlatFieldMetadatasOrThrow({
flatFieldMetadata: morphFlatFieldMetadata,
flatObjectMetadataMaps: existingFlatObjectMetadataMaps,
flatFieldMetadataMaps,
flatObjectMetadata: sourceFlatObjectMetadata,
});
const allMorphFlatFieldMetadatas = [
morphFlatFieldMetadata,
@@ -213,12 +235,34 @@ export class DataloaderService {
];
relationDtos.push(
allMorphFlatFieldMetadatas.map((flatFieldMetadata) =>
fromMorphOrRelationFlatFieldMetadataToRelationDto({
flatFieldMetadata,
allMorphFlatFieldMetadatas.flatMap((sourceFlatFieldMetadata) => {
const targetFlatFieldMetadata =
findFlatEntityByIdInFlatEntityMapsOrThrow({
flatEntityId:
sourceFlatFieldMetadata.relationTargetFieldMetadataId,
flatEntityMaps: flatFieldMetadataMaps,
});
if (
!isMorphOrRelationFlatFieldMetadata(targetFlatFieldMetadata)
) {
return [];
}
const targetFlatObjectMetadata =
findFlatEntityByIdInFlatEntityMapsOrThrow({
flatEntityId:
sourceFlatFieldMetadata.relationTargetObjectMetadataId,
flatEntityMaps: flatObjectMetadataMaps,
});
return fromMorphOrRelationFlatFieldMetadataToRelationDto({
sourceFlatFieldMetadata,
targetFlatFieldMetadata,
targetFlatObjectMetadata,
sourceFlatObjectMetadata,
}),
),
});
}),
);
}
@@ -279,22 +323,29 @@ export class DataloaderService {
(dataLoaderParam) => dataLoaderParam.objectMetadata.id,
);
const { flatObjectMetadataMaps: existingFlatObjectMetadataMaps } =
await this.workspaceMetadataCacheService.getExistingOrRecomputeFlatObjectMetadataMaps(
{ workspaceId },
const { flatFieldMetadataMaps, flatObjectMetadataMaps } =
await this.flatEntityMapsCacheService.getOrRecomputeManyOrAllFlatEntityMaps(
{
workspaceId,
flatEntities: ['flatFieldMetadataMaps', 'flatObjectMetadataMaps'],
},
);
const fieldMetadataCollection = objectMetadataIds.map((id) => {
const flatObjectMetadataWithFlatFieldMaps =
existingFlatObjectMetadataMaps.byId[id];
const fieldMetadataCollection = objectMetadataIds.map(
(objectMetadataId) => {
const flatObjectMetadata =
findFlatEntityByIdInFlatEntityMapsOrThrow({
flatEntityId: objectMetadataId,
flatEntityMaps: flatObjectMetadataMaps,
});
const { objectFlatFieldMetadatas } =
findObjectFieldsInFlatFieldMetadataMapsOrThrow({
flatFieldMetadataMaps: flatFieldMetadataMaps,
flatObjectMetadata,
});
if (!isDefined(flatObjectMetadataWithFlatFieldMaps)) {
return [];
}
const overriddenFieldMetadataEntities =
flatObjectMetadataWithFlatFieldMaps.flatFieldMetadatas.map(
(flatFieldMetadata) => {
const overriddenFieldMetadataEntities =
objectFlatFieldMetadatas.map((flatFieldMetadata) => {
return FIELD_METADATA_STANDARD_OVERRIDES_PROPERTIES.reduce(
(acc, property) => ({
...acc,
@@ -314,16 +365,18 @@ export class DataloaderService {
}),
flatFieldMetadata,
);
},
});
const filteredFieldMetadataEntities =
filterMorphRelationDuplicateFields(
overriddenFieldMetadataEntities,
);
return filteredFieldMetadataEntities.map(
fromFlatFieldMetadataToFieldMetadataDto,
);
const filteredFieldMetadataEntities =
filterMorphRelationDuplicateFields(overriddenFieldMetadataEntities);
return filteredFieldMetadataEntities.map(
fromFlatFieldMetadataToFieldMetadataDto,
);
});
},
);
return fieldMetadataCollection;
},