houseassassin/twenty
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Deprecate object metadata maps in favor of flat entities (#16080)

Browse Source 
## Context
Deprecating the old objectMetadataMap type in favour of split flat
entities to match with our new caching.
In the long run, trying to achieve:
- Better performance through caching
- Consistent data access patterns across the codebase
- Reduced database queries

Now that everything is based on flat entities, which are cached, we can
finish the refactoring of workspace context cache which should already
improve performances.
Then the last step will be to consume that new cache in the new global
datasource to get rid of the many workspace datasources stored in the
server
This commit is contained in:
Weiko
2025-11-27 13:43:34 +01:00
committed by GitHub
parent 978c0acb90
commit 1607aebcc6
281 changed files with 6510 additions and 6627 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/twenty-server/src/engine/api/common/common-args-processors/data-arg-processor/data-arg.processor.ts
+28 -13
View File
@@ -52,8 +52,10 @@ import { transformLinksValue } from 'src/engine/core-modules/record-transformer/
import { transformPhonesValue } from 'src/engine/core-modules/record-transformer/utils/transform-phones-value.util';
import { transformRichTextV2Value } from 'src/engine/core-modules/record-transformer/utils/transform-rich-text-v2.util';
import { WorkspaceNotFoundDefaultError } from 'src/engine/core-modules/workspace/workspace.exception';
import { FieldMetadataEntity } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity';
import { ObjectMetadataItemWithFieldMaps } from 'src/engine/metadata-modules/types/object-metadata-item-with-field-maps';
import { FlatEntityMaps } from 'src/engine/metadata-modules/flat-entity/types/flat-entity-maps.type';
import { FlatFieldMetadata } from 'src/engine/metadata-modules/flat-field-metadata/types/flat-field-metadata.type';
import { buildFieldMapsFromFlatObjectMetadata } from 'src/engine/metadata-modules/flat-field-metadata/utils/build-field-maps-from-flat-object-metadata.util';
import { FlatObjectMetadata } from 'src/engine/metadata-modules/flat-object-metadata/types/flat-object-metadata.type';
@Injectable()
export class DataArgProcessor {
@@ -65,12 +67,14 @@ export class DataArgProcessor {
async process({
partialRecordInputs,
authContext,
objectMetadataItemWithFieldMaps,
flatObjectMetadata,
flatFieldMetadataMaps,
shouldBackfillPositionIfUndefined = true,
}: {
partialRecordInputs: Partial<ObjectRecord>[] | undefined;
authContext: AuthContext;
objectMetadataItemWithFieldMaps: ObjectMetadataItemWithFieldMaps;
flatObjectMetadata: FlatObjectMetadata;
flatFieldMetadataMaps: FlatEntityMaps<FlatFieldMetadata>;
shouldBackfillPositionIfUndefined?: boolean;
}): Promise<Partial<ObjectRecord>[]> {
if (!isDefined(partialRecordInputs)) {
@@ -87,6 +91,12 @@ export class DataArgProcessor {
workspace.id,
);
const { fieldIdByName, fieldIdByJoinColumnName } =
buildFieldMapsFromFlatObjectMetadata(
flatFieldMetadataMaps,
flatObjectMetadata,
);
const processedRecords: Partial<ObjectRecord>[] = [];
for (const record of partialRecordInputs) {
@@ -94,18 +104,23 @@ export class DataArgProcessor {
for (const [key, value] of Object.entries(record)) {
const fieldMetadataId =
objectMetadataItemWithFieldMaps.fieldIdByName[key] ||
objectMetadataItemWithFieldMaps.fieldIdByJoinColumnName[key];
fieldIdByName[key] || fieldIdByJoinColumnName[key];
if (!isDefined(fieldMetadataId)) {
throw new CommonQueryRunnerException(
`Object ${objectMetadataItemWithFieldMaps.nameSingular} doesn't have any "${key}" field.`,
`Object ${flatObjectMetadata.nameSingular} doesn't have any "${key}" field.`,
CommonQueryRunnerExceptionCode.INVALID_ARGS_DATA,
);
}
const fieldMetadata =
objectMetadataItemWithFieldMaps.fieldsById[fieldMetadataId];
const fieldMetadata = flatFieldMetadataMaps.byId[fieldMetadataId];
if (!fieldMetadata) {
throw new CommonQueryRunnerException(
`Field metadata not found for field ${key}`,
CommonQueryRunnerExceptionCode.INVALID_ARGS_DATA,
);
}
if (
!isDefined(fieldMetadata.defaultValue) &&
@@ -137,9 +152,9 @@ export class DataArgProcessor {
partialRecordInputs: processedRecords,
workspaceId: workspace.id,
objectMetadata: {
isCustom: objectMetadataItemWithFieldMaps.isCustom,
nameSingular: objectMetadataItemWithFieldMaps.nameSingular,
fieldIdByName: objectMetadataItemWithFieldMaps.fieldIdByName,
isCustom: flatObjectMetadata.isCustom,
nameSingular: flatObjectMetadata.nameSingular,
fieldIdByName,
},
shouldBackfillPositionIfUndefined,
});
@@ -148,7 +163,7 @@ export class DataArgProcessor {
}
private async processField(
fieldMetadata: FieldMetadataEntity,
fieldMetadata: FlatFieldMetadata,
key: string,
value: unknown,
isNullEquivalenceEnabled: boolean,
packages/twenty-server/src/engine/api/common/common-args-processors/query-runner-args.factory.ts
+28 -15
View File
@@ -5,15 +5,15 @@ import { isDefined } from 'twenty-shared/utils';
import { type ObjectRecordFilter } from 'src/engine/api/graphql/workspace-query-builder/interfaces/object-record.interface';
import { RecordPositionService } from 'src/engine/core-modules/record-position/services/record-position.service';
import { RecordInputTransformerService } from 'src/engine/core-modules/record-transformer/services/record-input-transformer.service';
import { type FieldMetadataMap } from 'src/engine/metadata-modules/types/field-metadata-map';
import { type ObjectMetadataItemWithFieldMaps } from 'src/engine/metadata-modules/types/object-metadata-item-with-field-maps';
import { FlatEntityMaps } from 'src/engine/metadata-modules/flat-entity/types/flat-entity-maps.type';
import { FlatFieldMetadata } from 'src/engine/metadata-modules/flat-field-metadata/types/flat-field-metadata.type';
import { buildFieldMapsFromFlatObjectMetadata } from 'src/engine/metadata-modules/flat-field-metadata/utils/build-field-maps-from-flat-object-metadata.util';
import { FlatObjectMetadata } from 'src/engine/metadata-modules/flat-object-metadata/types/flat-object-metadata.type';
@Injectable()
export class QueryRunnerArgsFactory {
constructor(
private readonly recordPositionService: RecordPositionService,
private readonly recordInputTransformerService: RecordInputTransformerService,
) {}
@@ -21,12 +21,18 @@ export class QueryRunnerArgsFactory {
T extends ObjectRecordFilter | undefined,
>(
filter: T,
objectMetadataItemWithFieldMaps: ObjectMetadataItemWithFieldMaps,
flatObjectMetadata: FlatObjectMetadata,
flatFieldMetadataMaps: FlatEntityMaps<FlatFieldMetadata>,
): T {
if (!isDefined(filter)) {
return filter;
}
const { fieldIdByName } = buildFieldMapsFromFlatObjectMetadata(
flatFieldMetadataMaps,
flatObjectMetadata,
);
const overrideFilter = (filterObject: ObjectRecordFilter) => {
return Object.entries(filterObject).reduce((acc, [key, value]) => {
if (key === 'and' || key === 'or') {
@@ -42,7 +48,8 @@ export class QueryRunnerArgsFactory {
acc[key] = this.transformFilterValueByType(
key,
value,
objectMetadataItemWithFieldMaps,
fieldIdByName,
flatFieldMetadataMaps,
);
}
@@ -57,11 +64,13 @@ export class QueryRunnerArgsFactory {
key: string,
// eslint-disable-next-line @typescript-eslint/no-explicit-any
value: any,
objectMetadataItemWithFieldMaps: ObjectMetadataItemWithFieldMaps,
fieldIdByName: Record<string, string>,
flatFieldMetadataMaps: FlatEntityMaps<FlatFieldMetadata>,
) {
const fieldMetadataId = objectMetadataItemWithFieldMaps.fieldIdByName[key];
const fieldMetadata =
objectMetadataItemWithFieldMaps.fieldsById[fieldMetadataId];
const fieldMetadataId = fieldIdByName[key];
const fieldMetadata = fieldMetadataId
? flatFieldMetadataMaps.byId[fieldMetadataId]
: undefined;
if (!fieldMetadata) {
return value;
@@ -90,18 +99,22 @@ export class QueryRunnerArgsFactory {
key: string,
// eslint-disable-next-line @typescript-eslint/no-explicit-any
value: any,
fieldMetadataMapByName: FieldMetadataMap,
objectMetadataItemWithFieldMaps: ObjectMetadataItemWithFieldMaps,
fieldIdByName: Record<string, string>,
flatObjectMetadata: FlatObjectMetadata,
flatFieldMetadataMaps: FlatEntityMaps<FlatFieldMetadata>,
) {
const fieldMetadata = fieldMetadataMapByName[key];
const fieldMetadata = flatFieldMetadataMaps.byId[fieldIdByName[key]];
if (!fieldMetadata) {
return value;
}
return this.recordInputTransformerService.process({
const processed = await this.recordInputTransformerService.process({
recordInput: { [key]: value },
objectMetadataMapItem: objectMetadataItemWithFieldMaps,
flatObjectMetadata,
flatFieldMetadataMaps,
});
return processed[key] ?? value;
}
}