houseassassin/twenty
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Optimize EntityMetadata caching in GlobalWorkspaceDataSource (#16146)

Browse Source 
## Context
EntityMetadata was being rebuilt from scratch on every
findMetadata()/getMetadata() call (~20 times per request). This involved
running EntitySchemaTransformer.transform() and
EntityMetadataBuilder.build() repeatedly, causing unnecessary CPU
overhead.

## Implementation
Cache entityMetadatas in ORMWorkspaceContext: Build EntityMetadata once
during workspace context initialization instead of on every metadata
lookup
Remove redundant entitySchemas caching: Since flatMetadata is already
cached, the additional Redis cache for entitySchemaOptions was
unnecessary overhead
Remove WorkspaceEntitiesStorage: Replaced with direct lookup from
FlatObjectMetadataMap
Simplify getObjectMetadataFromEntityTarget: Now only accepts string
targets, using flat metadata maps directly

Also:
Removed unused injections in some services
This commit is contained in:
Weiko
2025-11-28 10:09:52 +01:00
committed by GitHub
parent fd9ea2f5ee
commit 9620a4b0ba
11 changed files with 45 additions and 148 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/twenty-server/src/engine/metadata-modules/field-metadata/services/field-metadata-relation.service.ts
+1 -4
View File
@@ -30,7 +30,6 @@ import { validateFieldNameAvailabilityOrThrow } from 'src/engine/metadata-module
import { validateMetadataNameOrThrow } from 'src/engine/metadata-modules/utils/validate-metadata-name-or-throw.utils';
import { computeMetadataNameFromLabel } from 'src/engine/metadata-modules/utils/validate-name-and-label-are-sync-or-throw.util';
import { isFieldMetadataEntityOfType } from 'src/engine/utils/is-field-metadata-of-type.util';
import { WorkspaceCacheStorageService } from 'src/engine/workspace-cache-storage/workspace-cache-storage.service';
export class RelationCreationPayloadValidation {
@IsUUID()
@@ -58,9 +57,7 @@ type ValidateFieldMetadataArgs<T extends UpdateFieldInput | CreateFieldInput> =
@Injectable()
export class FieldMetadataRelationService {
constructor(
private readonly workspaceCacheStorageService: WorkspaceCacheStorageService,
) {}
constructor() {}
async createRelationFieldMetadataItems({
fieldMetadataInput,
packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/foreign-table/foreign-table.service.ts
-2
View File
@@ -24,7 +24,6 @@ import {
WorkspaceMigrationTableActionType,
} from 'src/engine/metadata-modules/workspace-migration/workspace-migration.entity';
import { WorkspaceMigrationService } from 'src/engine/metadata-modules/workspace-migration/workspace-migration.service';
import { WorkspaceDataSourceService } from 'src/engine/workspace-datasource/workspace-datasource.service';
import { WorkspaceMigrationRunnerService } from 'src/engine/workspace-manager/workspace-migration-runner/workspace-migration-runner.service';
@Injectable()
@@ -32,7 +31,6 @@ export class ForeignTableService {
constructor(
private readonly workspaceMigrationService: WorkspaceMigrationService,
private readonly workspaceMigrationRunnerService: WorkspaceMigrationRunnerService,
private readonly workspaceDataSourceService: WorkspaceDataSourceService,
private readonly workspaceMetadataVersionService: WorkspaceMetadataVersionService,
@InjectDataSource()
private readonly coreDataSource: DataSource,
packages/twenty-server/src/engine/twenty-orm/factories/entity-schema.factory.ts
-7
View File
@@ -7,7 +7,6 @@ import { type FlatFieldMetadata } from 'src/engine/metadata-modules/flat-field-m
import { type FlatObjectMetadata } from 'src/engine/metadata-modules/flat-object-metadata/types/flat-object-metadata.type';
import { EntitySchemaColumnFactory } from 'src/engine/twenty-orm/factories/entity-schema-column.factory';
import { EntitySchemaRelationFactory } from 'src/engine/twenty-orm/factories/entity-schema-relation.factory';
import { WorkspaceEntitiesStorage } from 'src/engine/twenty-orm/storage/workspace-entities.storage';
import { computeTableName } from 'src/engine/utils/compute-table-name.util';
import { getWorkspaceSchemaName } from 'src/engine/workspace-datasource/utils/get-workspace-schema-name.util';
@@ -48,12 +47,6 @@ export class EntitySchemaFactory {
schema: schemaName,
});
WorkspaceEntitiesStorage.setEntitySchema(
workspaceId,
flatObjectMetadata.nameSingular,
entitySchema,
);
return entitySchema;
}
}
packages/twenty-server/src/engine/twenty-orm/global-workspace-datasource/global-workspace-datasource.service.ts
-2
View File
@@ -6,7 +6,6 @@ import {
} from '@nestjs/common';
import { TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service';
import { EntitySchemaFactory } from 'src/engine/twenty-orm/factories/entity-schema.factory';
import { GlobalWorkspaceDataSource } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-datasource';
import { WorkspaceEventEmitter } from 'src/engine/workspace-event-emitter/workspace-event-emitter';
@@ -19,7 +18,6 @@ export class GlobalWorkspaceDataSourceService
constructor(
private readonly twentyConfigService: TwentyConfigService,
private readonly entitySchemaFactory: EntitySchemaFactory,
private readonly workspaceEventEmitter: WorkspaceEventEmitter,
) {}
packages/twenty-server/src/engine/twenty-orm/global-workspace-datasource/global-workspace-datasource.ts
+3 -24
View File
@@ -4,7 +4,6 @@ import {
DataSource,
type DataSourceOptions,
type EntityMetadata,
type EntitySchema,
type EntityTarget,
type ObjectLiteral,
type QueryRunner,
@@ -12,9 +11,7 @@ import {
type SelectQueryBuilder,
} from 'typeorm';
import { EntityManagerFactory } from 'typeorm/entity-manager/EntityManagerFactory';
import { EntitySchemaTransformer } from 'typeorm/entity-schema/EntitySchemaTransformer';
import { EntityMetadataNotFoundError } from 'typeorm/error/EntityMetadataNotFoundError';
import { EntityMetadataBuilder } from 'typeorm/metadata-builder/EntityMetadataBuilder';
import { type WorkspaceAuthContext } from 'src/engine/api/common/interfaces/workspace-auth-context.interface';
import { type FeatureFlagMap } from 'src/engine/core-modules/feature-flag/interfaces/feature-flag-map.interface';
@@ -26,7 +23,7 @@ import {
import { WorkspaceEntityManager } from 'src/engine/twenty-orm/entity-manager/workspace-entity-manager';
import { type WorkspaceQueryRunner } from 'src/engine/twenty-orm/query-runner/workspace-query-runner';
import { type WorkspaceRepository } from 'src/engine/twenty-orm/repository/workspace.repository';
import { getWorkspaceContext } from 'src/engine/twenty-orm/storage/workspace-context.storage';
import { getWorkspaceContext } from 'src/engine/twenty-orm/storage/orm-workspace-context.storage';
import { type RolePermissionConfig } from 'src/engine/twenty-orm/types/role-permission-config';
import { type WorkspaceEventEmitter } from 'src/engine/workspace-event-emitter/workspace-event-emitter';
@@ -83,11 +80,9 @@ export class GlobalWorkspaceDataSource extends DataSource {
target: EntityTarget<ObjectLiteral>,
): EntityMetadata | undefined {
const context = getWorkspaceContext();
const { entitySchemas } = context;
const { entityMetadatas } = context;
const entityMetadata = this.buildMetadatasFromSchemas(entitySchemas);
return entityMetadata.find((metadata) => metadata.target === target);
return entityMetadatas.find((metadata) => metadata.target === target);
}
override getMetadata(target: EntityTarget<ObjectLiteral>): EntityMetadata {
@@ -264,20 +259,4 @@ export class GlobalWorkspaceDataSource extends DataSource {
return super.query(query, parameters, queryRunner);
}
private buildMetadatasFromSchemas(
entitySchemas: EntitySchema[],
): EntityMetadata[] {
const transformer = new EntitySchemaTransformer();
const metadataArgsStorage = transformer.transform(entitySchemas);
const entityMetadataBuilder = new EntityMetadataBuilder(
this,
metadataArgsStorage,
);
const entityMetadatas = entityMetadataBuilder.build();
return entityMetadatas;
}
}
packages/twenty-server/src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager.ts
+28 -48
View File
@@ -2,6 +2,8 @@ import { Injectable, type Type } from '@nestjs/common';
import { isDefined } from 'twenty-shared/utils';
import { EntitySchema, ObjectLiteral } from 'typeorm';
import { EntitySchemaTransformer } from 'typeorm/entity-schema/EntitySchemaTransformer';
import { EntityMetadataBuilder } from 'typeorm/metadata-builder/EntityMetadataBuilder';
import { WorkspaceAuthContext } from 'src/engine/api/common/interfaces/workspace-auth-context.interface';
@@ -16,9 +18,9 @@ import { EntitySchemaFactory } from 'src/engine/twenty-orm/factories/entity-sche
import { GlobalWorkspaceDataSourceService } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-datasource.service';
import { type WorkspaceRepository } from 'src/engine/twenty-orm/repository/workspace.repository';
import {
type WorkspaceContext,
type ORMWorkspaceContext,
withWorkspaceContext,
} from 'src/engine/twenty-orm/storage/workspace-context.storage';
} from 'src/engine/twenty-orm/storage/orm-workspace-context.storage';
import { type RolePermissionConfig } from 'src/engine/twenty-orm/types/role-permission-config';
import { WorkspaceCacheStorageService } from 'src/engine/workspace-cache-storage/workspace-cache-storage.service';
import { convertClassNameToObjectMetadataName } from 'src/engine/workspace-manager/workspace-sync-metadata/utils/convert-class-to-object-metadata-name.util';
@@ -85,7 +87,7 @@ export class GlobalWorkspaceOrmManager {
private async loadWorkspaceContext(
authContext: WorkspaceAuthContext,
): Promise<WorkspaceContext> {
): Promise<ORMWorkspaceContext> {
const workspaceId = authContext.workspace.id;
const { flatObjectMetadataMaps, flatFieldMetadataMaps, flatIndexMaps } =
@@ -103,15 +105,6 @@ export class GlobalWorkspaceOrmManager {
const { idByNameSingular: objectIdByNameSingular } =
buildObjectIdByNameMaps(flatObjectMetadataMaps);
const metadataVersion =
await this.workspaceCacheStorageService.getMetadataVersion(workspaceId);
if (!isDefined(metadataVersion)) {
throw new Error(
`Metadata version not found for workspace ${workspaceId}`,
);
}
const { data: featureFlagsMap } =
await this.workspaceFeatureFlagsMapCacheService.getWorkspaceFeatureFlagsMapAndVersion(
{ workspaceId },
@@ -122,65 +115,52 @@ export class GlobalWorkspaceOrmManager {
workspaceId,
});
const entitySchemas = await this.buildEntitySchemas(
const entitySchemas = this.buildEntitySchemas(
workspaceId,
metadataVersion,
flatObjectMetadataMaps,
flatFieldMetadataMaps,
);
const entityMetadatas = this.buildEntityMetadatas(entitySchemas);
return {
authContext,
flatObjectMetadataMaps,
flatFieldMetadataMaps,
flatIndexMaps,
objectIdByNameSingular,
metadataVersion,
featureFlagsMap,
permissionsPerRoleId,
entitySchemas,
entityMetadatas,
};
}
private async buildEntitySchemas(
private buildEntitySchemas(
workspaceId: string,
dataSourceMetadataVersion: number,
flatObjectMetadataMaps: FlatEntityMaps<FlatObjectMetadata>,
flatFieldMetadataMaps: FlatEntityMaps<FlatFieldMetadata>,
) {
const cachedEntitySchemaOptions =
await this.workspaceCacheStorageService.getORMEntitySchema(
workspaceId,
dataSourceMetadataVersion,
return Object.values(flatObjectMetadataMaps.byId)
.filter(isDefined)
.map((flatObjectMetadata) =>
this.entitySchemaFactory.create(
workspaceId,
flatObjectMetadata,
flatObjectMetadataMaps,
flatFieldMetadataMaps,
),
);
}
let cachedEntitySchemas: EntitySchema[];
private buildEntityMetadatas(entitySchemas: EntitySchema[]) {
const transformer = new EntitySchemaTransformer();
const metadataArgsStorage = transformer.transform(entitySchemas);
if (cachedEntitySchemaOptions) {
cachedEntitySchemas = cachedEntitySchemaOptions.map(
(option) => new EntitySchema(option),
);
} else {
const entitySchemas = Object.values(flatObjectMetadataMaps.byId)
.filter(isDefined)
.map((flatObjectMetadata) =>
this.entitySchemaFactory.create(
workspaceId,
flatObjectMetadata,
flatObjectMetadataMaps,
flatFieldMetadataMaps,
),
);
const entityMetadataBuilder = new EntityMetadataBuilder(
this.globalWorkspaceDataSourceService.getGlobalWorkspaceDataSource(),
metadataArgsStorage,
);
await this.workspaceCacheStorageService.setORMEntitySchema(
workspaceId,
dataSourceMetadataVersion,
entitySchemas.map((entitySchema) => entitySchema.options),
);
cachedEntitySchemas = entitySchemas;
}
return cachedEntitySchemas;
return entityMetadataBuilder.build();
}
}
packages/twenty-server/src/engine/twenty-orm/storage/workspace-context.storage.ts → packages/twenty-server/src/engine/twenty-orm/storage/orm-workspace-context.storage.ts
+7 -8
View File
@@ -1,7 +1,7 @@
import { AsyncLocalStorage } from 'async_hooks';
import { type ObjectsPermissionsByRoleId } from 'twenty-shared/types';
import { type EntitySchema } from 'typeorm';
import { type EntityMetadata } from 'typeorm';
import { type WorkspaceAuthContext } from 'src/engine/api/common/interfaces/workspace-auth-context.interface';
@@ -11,22 +11,21 @@ import { type FlatFieldMetadata } from 'src/engine/metadata-modules/flat-field-m
import { type FlatIndexMetadata } from 'src/engine/metadata-modules/flat-index-metadata/types/flat-index-metadata.type';
import { type FlatObjectMetadata } from 'src/engine/metadata-modules/flat-object-metadata/types/flat-object-metadata.type';
export type WorkspaceContext = {
export type ORMWorkspaceContext = {
authContext: WorkspaceAuthContext;
flatObjectMetadataMaps: FlatEntityMaps<FlatObjectMetadata>;
flatFieldMetadataMaps: FlatEntityMaps<FlatFieldMetadata>;
flatIndexMaps: FlatEntityMaps<FlatIndexMetadata>;
objectIdByNameSingular: Record<string, string>;
metadataVersion: number;
featureFlagsMap: Record<FeatureFlagKey, boolean>;
permissionsPerRoleId: ObjectsPermissionsByRoleId;
entitySchemas: EntitySchema[];
entityMetadatas: EntityMetadata[];
};
export const workspaceContextStorage =
new AsyncLocalStorage<WorkspaceContext>();
new AsyncLocalStorage<ORMWorkspaceContext>();
export const getWorkspaceContext = (): WorkspaceContext => {
export const getWorkspaceContext = (): ORMWorkspaceContext => {
const context = workspaceContextStorage.getStore();
if (!context) {
@@ -39,12 +38,12 @@ export const getWorkspaceContext = (): WorkspaceContext => {
};
export const withWorkspaceContext = <T>(
context: WorkspaceContext,
context: ORMWorkspaceContext,
fn: () => T | Promise<T>,
): T | Promise<T> => {
return workspaceContextStorage.run(context, fn);
};
export const setWorkspaceContext = (context: WorkspaceContext): void => {
export const setWorkspaceContext = (context: ORMWorkspaceContext): void => {
workspaceContextStorage.enterWith(context);
};
packages/twenty-server/src/engine/twenty-orm/storage/workspace-entities.storage.ts
-32
View File
@@ -1,32 +0,0 @@
import { type EntitySchema } from 'typeorm';
export class WorkspaceEntitiesStorage {
private static workspaceEntities = new Map<
string,
Map<string, EntitySchema>
>();
static setEntitySchema(
workspaceId: string,
objectMetadataName: string,
schema: EntitySchema,
): void {
if (!this.workspaceEntities.has(workspaceId)) {
this.workspaceEntities.set(workspaceId, new Map<string, EntitySchema>());
}
const workspace = this.workspaceEntities.get(workspaceId);
workspace?.set(objectMetadataName, schema);
}
static getObjectMetadataName(
workspaceId: string,
target: EntitySchema,
): string | undefined {
const workspace = this.workspaceEntities.get(workspaceId);
return Array.from(workspace?.entries() || []).find(
([, schema]) => schema.options.name === target.options.name,
)?.[0];
}
}
packages/twenty-server/src/engine/twenty-orm/utils/get-object-metadata-from-entity-target.util.ts
+5 -16
View File
@@ -1,8 +1,4 @@
import {
type EntitySchema,
type EntityTarget,
type ObjectLiteral,
} from 'typeorm';
import { type EntityTarget, type ObjectLiteral } from 'typeorm';
import { type WorkspaceInternalContext } from 'src/engine/twenty-orm/interfaces/workspace-internal-context.interface';
@@ -11,27 +7,20 @@ import {
TwentyORMException,
TwentyORMExceptionCode,
} from 'src/engine/twenty-orm/exceptions/twenty-orm.exception';
import { WorkspaceEntitiesStorage } from 'src/engine/twenty-orm/storage/workspace-entities.storage';
export const getObjectMetadataFromEntityTarget = <T extends ObjectLiteral>(
entityTarget: EntityTarget<T>,
internalContext: WorkspaceInternalContext,
): FlatObjectMetadata => {
const objectMetadataName =
typeof entityTarget === 'string'
? entityTarget
: WorkspaceEntitiesStorage.getObjectMetadataName(
internalContext.workspaceId,
entityTarget as EntitySchema,
);
if (!objectMetadataName) {
if (typeof entityTarget !== 'string') {
throw new TwentyORMException(
'Object metadata name is missing',
'Entity target must be a string',
TwentyORMExceptionCode.MALFORMED_METADATA,
);
}
const objectMetadataName = entityTarget;
const objectMetadataId =
internalContext.objectIdByNameSingular[objectMetadataName];
packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-process-group-email-actions.service.ts
-2
View File
@@ -4,7 +4,6 @@ import { isDefined } from 'twenty-shared/utils';
import { type WorkspaceEntityManager } from 'src/engine/twenty-orm/entity-manager/workspace-entity-manager';
import { TwentyORMManager } from 'src/engine/twenty-orm/twenty-orm.manager';
import { MessageChannelSyncStatusService } from 'src/modules/messaging/common/services/message-channel-sync-status.service';
import {
MessageChannelPendingGroupEmailsAction,
MessageChannelWorkspaceEntity,
@@ -22,7 +21,6 @@ export class MessagingProcessGroupEmailActionsService {
private readonly twentyORMManager: TwentyORMManager,
private readonly messagingDeleteGroupEmailMessagesService: MessagingDeleteGroupEmailMessagesService,
private readonly messagingClearCursorsService: MessagingClearCursorsService,
private readonly messageChannelSyncStatusService: MessageChannelSyncStatusService,
) {}
async markMessageChannelAsPendingGroupEmailsAction(
packages/twenty-server/src/modules/messaging/monitoring/services/messaging-monitoring.service.ts
+1 -3
View File
@@ -1,7 +1,5 @@
import { Injectable } from '@nestjs/common';
import { AuditService } from 'src/engine/core-modules/audit/services/audit.service';
type MessagingMonitoringTrackInput = {
eventName: string;
workspaceId?: string;
@@ -13,7 +11,7 @@ type MessagingMonitoringTrackInput = {
@Injectable()
export class MessagingMonitoringService {
constructor(private readonly _auditService: AuditService) {}
constructor() {}
public async track({
eventName: _eventName,