From 9620a4b0baf2c6b79b66b0b357b2ed330df11d1f Mon Sep 17 00:00:00 2001 From: Weiko Date: Fri, 28 Nov 2025 10:09:52 +0100 Subject: [PATCH] Optimize EntityMetadata caching in GlobalWorkspaceDataSource (#16146) ## Context EntityMetadata was being rebuilt from scratch on every findMetadata()/getMetadata() call (~20 times per request). This involved running EntitySchemaTransformer.transform() and EntityMetadataBuilder.build() repeatedly, causing unnecessary CPU overhead. ## Implementation Cache entityMetadatas in ORMWorkspaceContext: Build EntityMetadata once during workspace context initialization instead of on every metadata lookup Remove redundant entitySchemas caching: Since flatMetadata is already cached, the additional Redis cache for entitySchemaOptions was unnecessary overhead Remove WorkspaceEntitiesStorage: Replaced with direct lookup from FlatObjectMetadataMap Simplify getObjectMetadataFromEntityTarget: Now only accepts string targets, using flat metadata maps directly Also: Removed unused injections in some services --- .../field-metadata-relation.service.ts | 5 +- .../foreign-table/foreign-table.service.ts | 2 - .../factories/entity-schema.factory.ts | 7 -- .../global-workspace-datasource.service.ts | 2 - .../global-workspace-datasource.ts | 27 +------ .../global-workspace-orm.manager.ts | 76 +++++++------------ ...ge.ts => orm-workspace-context.storage.ts} | 15 ++-- .../storage/workspace-entities.storage.ts | 32 -------- ...object-metadata-from-entity-target.util.ts | 21 ++--- ...ing-process-group-email-actions.service.ts | 2 - .../services/messaging-monitoring.service.ts | 4 +- 11 files changed, 45 insertions(+), 148 deletions(-) rename packages/twenty-server/src/engine/twenty-orm/storage/{workspace-context.storage.ts => orm-workspace-context.storage.ts} (82%) delete mode 100644 packages/twenty-server/src/engine/twenty-orm/storage/workspace-entities.storage.ts diff --git a/packages/twenty-server/src/engine/metadata-modules/field-metadata/services/field-metadata-relation.service.ts b/packages/twenty-server/src/engine/metadata-modules/field-metadata/services/field-metadata-relation.service.ts index 817ead6e5b..3520b76bc2 100644 --- a/packages/twenty-server/src/engine/metadata-modules/field-metadata/services/field-metadata-relation.service.ts +++ b/packages/twenty-server/src/engine/metadata-modules/field-metadata/services/field-metadata-relation.service.ts @@ -30,7 +30,6 @@ import { validateFieldNameAvailabilityOrThrow } from 'src/engine/metadata-module import { validateMetadataNameOrThrow } from 'src/engine/metadata-modules/utils/validate-metadata-name-or-throw.utils'; import { computeMetadataNameFromLabel } from 'src/engine/metadata-modules/utils/validate-name-and-label-are-sync-or-throw.util'; import { isFieldMetadataEntityOfType } from 'src/engine/utils/is-field-metadata-of-type.util'; -import { WorkspaceCacheStorageService } from 'src/engine/workspace-cache-storage/workspace-cache-storage.service'; export class RelationCreationPayloadValidation { @IsUUID() @@ -58,9 +57,7 @@ type ValidateFieldMetadataArgs = @Injectable() export class FieldMetadataRelationService { - constructor( - private readonly workspaceCacheStorageService: WorkspaceCacheStorageService, - ) {} + constructor() {} async createRelationFieldMetadataItems({ fieldMetadataInput, diff --git a/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/foreign-table/foreign-table.service.ts b/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/foreign-table/foreign-table.service.ts index 54b0dfa22b..f6d8b1a0f6 100644 --- a/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/foreign-table/foreign-table.service.ts +++ b/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/foreign-table/foreign-table.service.ts @@ -24,7 +24,6 @@ import { WorkspaceMigrationTableActionType, } from 'src/engine/metadata-modules/workspace-migration/workspace-migration.entity'; import { WorkspaceMigrationService } from 'src/engine/metadata-modules/workspace-migration/workspace-migration.service'; -import { WorkspaceDataSourceService } from 'src/engine/workspace-datasource/workspace-datasource.service'; import { WorkspaceMigrationRunnerService } from 'src/engine/workspace-manager/workspace-migration-runner/workspace-migration-runner.service'; @Injectable() @@ -32,7 +31,6 @@ export class ForeignTableService { constructor( private readonly workspaceMigrationService: WorkspaceMigrationService, private readonly workspaceMigrationRunnerService: WorkspaceMigrationRunnerService, - private readonly workspaceDataSourceService: WorkspaceDataSourceService, private readonly workspaceMetadataVersionService: WorkspaceMetadataVersionService, @InjectDataSource() private readonly coreDataSource: DataSource, diff --git a/packages/twenty-server/src/engine/twenty-orm/factories/entity-schema.factory.ts b/packages/twenty-server/src/engine/twenty-orm/factories/entity-schema.factory.ts index 064c41fabd..74259c2715 100644 --- a/packages/twenty-server/src/engine/twenty-orm/factories/entity-schema.factory.ts +++ b/packages/twenty-server/src/engine/twenty-orm/factories/entity-schema.factory.ts @@ -7,7 +7,6 @@ import { type FlatFieldMetadata } from 'src/engine/metadata-modules/flat-field-m import { type FlatObjectMetadata } from 'src/engine/metadata-modules/flat-object-metadata/types/flat-object-metadata.type'; import { EntitySchemaColumnFactory } from 'src/engine/twenty-orm/factories/entity-schema-column.factory'; import { EntitySchemaRelationFactory } from 'src/engine/twenty-orm/factories/entity-schema-relation.factory'; -import { WorkspaceEntitiesStorage } from 'src/engine/twenty-orm/storage/workspace-entities.storage'; import { computeTableName } from 'src/engine/utils/compute-table-name.util'; import { getWorkspaceSchemaName } from 'src/engine/workspace-datasource/utils/get-workspace-schema-name.util'; @@ -48,12 +47,6 @@ export class EntitySchemaFactory { schema: schemaName, }); - WorkspaceEntitiesStorage.setEntitySchema( - workspaceId, - flatObjectMetadata.nameSingular, - entitySchema, - ); - return entitySchema; } } diff --git a/packages/twenty-server/src/engine/twenty-orm/global-workspace-datasource/global-workspace-datasource.service.ts b/packages/twenty-server/src/engine/twenty-orm/global-workspace-datasource/global-workspace-datasource.service.ts index cf585832ec..df70e4b94c 100644 --- a/packages/twenty-server/src/engine/twenty-orm/global-workspace-datasource/global-workspace-datasource.service.ts +++ b/packages/twenty-server/src/engine/twenty-orm/global-workspace-datasource/global-workspace-datasource.service.ts @@ -6,7 +6,6 @@ import { } from '@nestjs/common'; import { TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service'; -import { EntitySchemaFactory } from 'src/engine/twenty-orm/factories/entity-schema.factory'; import { GlobalWorkspaceDataSource } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-datasource'; import { WorkspaceEventEmitter } from 'src/engine/workspace-event-emitter/workspace-event-emitter'; @@ -19,7 +18,6 @@ export class GlobalWorkspaceDataSourceService constructor( private readonly twentyConfigService: TwentyConfigService, - private readonly entitySchemaFactory: EntitySchemaFactory, private readonly workspaceEventEmitter: WorkspaceEventEmitter, ) {} diff --git a/packages/twenty-server/src/engine/twenty-orm/global-workspace-datasource/global-workspace-datasource.ts b/packages/twenty-server/src/engine/twenty-orm/global-workspace-datasource/global-workspace-datasource.ts index b8854fea89..cb4f2e0212 100644 --- a/packages/twenty-server/src/engine/twenty-orm/global-workspace-datasource/global-workspace-datasource.ts +++ b/packages/twenty-server/src/engine/twenty-orm/global-workspace-datasource/global-workspace-datasource.ts @@ -4,7 +4,6 @@ import { DataSource, type DataSourceOptions, type EntityMetadata, - type EntitySchema, type EntityTarget, type ObjectLiteral, type QueryRunner, @@ -12,9 +11,7 @@ import { type SelectQueryBuilder, } from 'typeorm'; import { EntityManagerFactory } from 'typeorm/entity-manager/EntityManagerFactory'; -import { EntitySchemaTransformer } from 'typeorm/entity-schema/EntitySchemaTransformer'; import { EntityMetadataNotFoundError } from 'typeorm/error/EntityMetadataNotFoundError'; -import { EntityMetadataBuilder } from 'typeorm/metadata-builder/EntityMetadataBuilder'; import { type WorkspaceAuthContext } from 'src/engine/api/common/interfaces/workspace-auth-context.interface'; import { type FeatureFlagMap } from 'src/engine/core-modules/feature-flag/interfaces/feature-flag-map.interface'; @@ -26,7 +23,7 @@ import { import { WorkspaceEntityManager } from 'src/engine/twenty-orm/entity-manager/workspace-entity-manager'; import { type WorkspaceQueryRunner } from 'src/engine/twenty-orm/query-runner/workspace-query-runner'; import { type WorkspaceRepository } from 'src/engine/twenty-orm/repository/workspace.repository'; -import { getWorkspaceContext } from 'src/engine/twenty-orm/storage/workspace-context.storage'; +import { getWorkspaceContext } from 'src/engine/twenty-orm/storage/orm-workspace-context.storage'; import { type RolePermissionConfig } from 'src/engine/twenty-orm/types/role-permission-config'; import { type WorkspaceEventEmitter } from 'src/engine/workspace-event-emitter/workspace-event-emitter'; @@ -83,11 +80,9 @@ export class GlobalWorkspaceDataSource extends DataSource { target: EntityTarget, ): EntityMetadata | undefined { const context = getWorkspaceContext(); - const { entitySchemas } = context; + const { entityMetadatas } = context; - const entityMetadata = this.buildMetadatasFromSchemas(entitySchemas); - - return entityMetadata.find((metadata) => metadata.target === target); + return entityMetadatas.find((metadata) => metadata.target === target); } override getMetadata(target: EntityTarget): EntityMetadata { @@ -264,20 +259,4 @@ export class GlobalWorkspaceDataSource extends DataSource { return super.query(query, parameters, queryRunner); } - - private buildMetadatasFromSchemas( - entitySchemas: EntitySchema[], - ): EntityMetadata[] { - const transformer = new EntitySchemaTransformer(); - const metadataArgsStorage = transformer.transform(entitySchemas); - - const entityMetadataBuilder = new EntityMetadataBuilder( - this, - metadataArgsStorage, - ); - - const entityMetadatas = entityMetadataBuilder.build(); - - return entityMetadatas; - } } diff --git a/packages/twenty-server/src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager.ts b/packages/twenty-server/src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager.ts index b684159a8d..cd3cc85cd4 100644 --- a/packages/twenty-server/src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager.ts +++ b/packages/twenty-server/src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager.ts @@ -2,6 +2,8 @@ import { Injectable, type Type } from '@nestjs/common'; import { isDefined } from 'twenty-shared/utils'; import { EntitySchema, ObjectLiteral } from 'typeorm'; +import { EntitySchemaTransformer } from 'typeorm/entity-schema/EntitySchemaTransformer'; +import { EntityMetadataBuilder } from 'typeorm/metadata-builder/EntityMetadataBuilder'; import { WorkspaceAuthContext } from 'src/engine/api/common/interfaces/workspace-auth-context.interface'; @@ -16,9 +18,9 @@ import { EntitySchemaFactory } from 'src/engine/twenty-orm/factories/entity-sche import { GlobalWorkspaceDataSourceService } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-datasource.service'; import { type WorkspaceRepository } from 'src/engine/twenty-orm/repository/workspace.repository'; import { - type WorkspaceContext, + type ORMWorkspaceContext, withWorkspaceContext, -} from 'src/engine/twenty-orm/storage/workspace-context.storage'; +} from 'src/engine/twenty-orm/storage/orm-workspace-context.storage'; import { type RolePermissionConfig } from 'src/engine/twenty-orm/types/role-permission-config'; import { WorkspaceCacheStorageService } from 'src/engine/workspace-cache-storage/workspace-cache-storage.service'; import { convertClassNameToObjectMetadataName } from 'src/engine/workspace-manager/workspace-sync-metadata/utils/convert-class-to-object-metadata-name.util'; @@ -85,7 +87,7 @@ export class GlobalWorkspaceOrmManager { private async loadWorkspaceContext( authContext: WorkspaceAuthContext, - ): Promise { + ): Promise { const workspaceId = authContext.workspace.id; const { flatObjectMetadataMaps, flatFieldMetadataMaps, flatIndexMaps } = @@ -103,15 +105,6 @@ export class GlobalWorkspaceOrmManager { const { idByNameSingular: objectIdByNameSingular } = buildObjectIdByNameMaps(flatObjectMetadataMaps); - const metadataVersion = - await this.workspaceCacheStorageService.getMetadataVersion(workspaceId); - - if (!isDefined(metadataVersion)) { - throw new Error( - `Metadata version not found for workspace ${workspaceId}`, - ); - } - const { data: featureFlagsMap } = await this.workspaceFeatureFlagsMapCacheService.getWorkspaceFeatureFlagsMapAndVersion( { workspaceId }, @@ -122,65 +115,52 @@ export class GlobalWorkspaceOrmManager { workspaceId, }); - const entitySchemas = await this.buildEntitySchemas( + const entitySchemas = this.buildEntitySchemas( workspaceId, - metadataVersion, flatObjectMetadataMaps, flatFieldMetadataMaps, ); + const entityMetadatas = this.buildEntityMetadatas(entitySchemas); + return { authContext, flatObjectMetadataMaps, flatFieldMetadataMaps, flatIndexMaps, objectIdByNameSingular, - metadataVersion, featureFlagsMap, permissionsPerRoleId, - entitySchemas, + entityMetadatas, }; } - private async buildEntitySchemas( + private buildEntitySchemas( workspaceId: string, - dataSourceMetadataVersion: number, flatObjectMetadataMaps: FlatEntityMaps, flatFieldMetadataMaps: FlatEntityMaps, ) { - const cachedEntitySchemaOptions = - await this.workspaceCacheStorageService.getORMEntitySchema( - workspaceId, - dataSourceMetadataVersion, + return Object.values(flatObjectMetadataMaps.byId) + .filter(isDefined) + .map((flatObjectMetadata) => + this.entitySchemaFactory.create( + workspaceId, + flatObjectMetadata, + flatObjectMetadataMaps, + flatFieldMetadataMaps, + ), ); + } - let cachedEntitySchemas: EntitySchema[]; + private buildEntityMetadatas(entitySchemas: EntitySchema[]) { + const transformer = new EntitySchemaTransformer(); + const metadataArgsStorage = transformer.transform(entitySchemas); - if (cachedEntitySchemaOptions) { - cachedEntitySchemas = cachedEntitySchemaOptions.map( - (option) => new EntitySchema(option), - ); - } else { - const entitySchemas = Object.values(flatObjectMetadataMaps.byId) - .filter(isDefined) - .map((flatObjectMetadata) => - this.entitySchemaFactory.create( - workspaceId, - flatObjectMetadata, - flatObjectMetadataMaps, - flatFieldMetadataMaps, - ), - ); + const entityMetadataBuilder = new EntityMetadataBuilder( + this.globalWorkspaceDataSourceService.getGlobalWorkspaceDataSource(), + metadataArgsStorage, + ); - await this.workspaceCacheStorageService.setORMEntitySchema( - workspaceId, - dataSourceMetadataVersion, - entitySchemas.map((entitySchema) => entitySchema.options), - ); - - cachedEntitySchemas = entitySchemas; - } - - return cachedEntitySchemas; + return entityMetadataBuilder.build(); } } diff --git a/packages/twenty-server/src/engine/twenty-orm/storage/workspace-context.storage.ts b/packages/twenty-server/src/engine/twenty-orm/storage/orm-workspace-context.storage.ts similarity index 82% rename from packages/twenty-server/src/engine/twenty-orm/storage/workspace-context.storage.ts rename to packages/twenty-server/src/engine/twenty-orm/storage/orm-workspace-context.storage.ts index fa71149d6f..487f7fcb5d 100644 --- a/packages/twenty-server/src/engine/twenty-orm/storage/workspace-context.storage.ts +++ b/packages/twenty-server/src/engine/twenty-orm/storage/orm-workspace-context.storage.ts @@ -1,7 +1,7 @@ import { AsyncLocalStorage } from 'async_hooks'; import { type ObjectsPermissionsByRoleId } from 'twenty-shared/types'; -import { type EntitySchema } from 'typeorm'; +import { type EntityMetadata } from 'typeorm'; import { type WorkspaceAuthContext } from 'src/engine/api/common/interfaces/workspace-auth-context.interface'; @@ -11,22 +11,21 @@ import { type FlatFieldMetadata } from 'src/engine/metadata-modules/flat-field-m import { type FlatIndexMetadata } from 'src/engine/metadata-modules/flat-index-metadata/types/flat-index-metadata.type'; import { type FlatObjectMetadata } from 'src/engine/metadata-modules/flat-object-metadata/types/flat-object-metadata.type'; -export type WorkspaceContext = { +export type ORMWorkspaceContext = { authContext: WorkspaceAuthContext; flatObjectMetadataMaps: FlatEntityMaps; flatFieldMetadataMaps: FlatEntityMaps; flatIndexMaps: FlatEntityMaps; objectIdByNameSingular: Record; - metadataVersion: number; featureFlagsMap: Record; permissionsPerRoleId: ObjectsPermissionsByRoleId; - entitySchemas: EntitySchema[]; + entityMetadatas: EntityMetadata[]; }; export const workspaceContextStorage = - new AsyncLocalStorage(); + new AsyncLocalStorage(); -export const getWorkspaceContext = (): WorkspaceContext => { +export const getWorkspaceContext = (): ORMWorkspaceContext => { const context = workspaceContextStorage.getStore(); if (!context) { @@ -39,12 +38,12 @@ export const getWorkspaceContext = (): WorkspaceContext => { }; export const withWorkspaceContext = ( - context: WorkspaceContext, + context: ORMWorkspaceContext, fn: () => T | Promise, ): T | Promise => { return workspaceContextStorage.run(context, fn); }; -export const setWorkspaceContext = (context: WorkspaceContext): void => { +export const setWorkspaceContext = (context: ORMWorkspaceContext): void => { workspaceContextStorage.enterWith(context); }; diff --git a/packages/twenty-server/src/engine/twenty-orm/storage/workspace-entities.storage.ts b/packages/twenty-server/src/engine/twenty-orm/storage/workspace-entities.storage.ts deleted file mode 100644 index 53f3db668f..0000000000 --- a/packages/twenty-server/src/engine/twenty-orm/storage/workspace-entities.storage.ts +++ /dev/null @@ -1,32 +0,0 @@ -import { type EntitySchema } from 'typeorm'; - -export class WorkspaceEntitiesStorage { - private static workspaceEntities = new Map< - string, - Map - >(); - - static setEntitySchema( - workspaceId: string, - objectMetadataName: string, - schema: EntitySchema, - ): void { - if (!this.workspaceEntities.has(workspaceId)) { - this.workspaceEntities.set(workspaceId, new Map()); - } - const workspace = this.workspaceEntities.get(workspaceId); - - workspace?.set(objectMetadataName, schema); - } - - static getObjectMetadataName( - workspaceId: string, - target: EntitySchema, - ): string | undefined { - const workspace = this.workspaceEntities.get(workspaceId); - - return Array.from(workspace?.entries() || []).find( - ([, schema]) => schema.options.name === target.options.name, - )?.[0]; - } -} diff --git a/packages/twenty-server/src/engine/twenty-orm/utils/get-object-metadata-from-entity-target.util.ts b/packages/twenty-server/src/engine/twenty-orm/utils/get-object-metadata-from-entity-target.util.ts index bd08b43526..8e764f0d10 100644 --- a/packages/twenty-server/src/engine/twenty-orm/utils/get-object-metadata-from-entity-target.util.ts +++ b/packages/twenty-server/src/engine/twenty-orm/utils/get-object-metadata-from-entity-target.util.ts @@ -1,8 +1,4 @@ -import { - type EntitySchema, - type EntityTarget, - type ObjectLiteral, -} from 'typeorm'; +import { type EntityTarget, type ObjectLiteral } from 'typeorm'; import { type WorkspaceInternalContext } from 'src/engine/twenty-orm/interfaces/workspace-internal-context.interface'; @@ -11,27 +7,20 @@ import { TwentyORMException, TwentyORMExceptionCode, } from 'src/engine/twenty-orm/exceptions/twenty-orm.exception'; -import { WorkspaceEntitiesStorage } from 'src/engine/twenty-orm/storage/workspace-entities.storage'; export const getObjectMetadataFromEntityTarget = ( entityTarget: EntityTarget, internalContext: WorkspaceInternalContext, ): FlatObjectMetadata => { - const objectMetadataName = - typeof entityTarget === 'string' - ? entityTarget - : WorkspaceEntitiesStorage.getObjectMetadataName( - internalContext.workspaceId, - entityTarget as EntitySchema, - ); - - if (!objectMetadataName) { + if (typeof entityTarget !== 'string') { throw new TwentyORMException( - 'Object metadata name is missing', + 'Entity target must be a string', TwentyORMExceptionCode.MALFORMED_METADATA, ); } + const objectMetadataName = entityTarget; + const objectMetadataId = internalContext.objectIdByNameSingular[objectMetadataName]; diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-process-group-email-actions.service.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-process-group-email-actions.service.ts index 48689bb04a..674f63d77a 100644 --- a/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-process-group-email-actions.service.ts +++ b/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-process-group-email-actions.service.ts @@ -4,7 +4,6 @@ import { isDefined } from 'twenty-shared/utils'; import { type WorkspaceEntityManager } from 'src/engine/twenty-orm/entity-manager/workspace-entity-manager'; import { TwentyORMManager } from 'src/engine/twenty-orm/twenty-orm.manager'; -import { MessageChannelSyncStatusService } from 'src/modules/messaging/common/services/message-channel-sync-status.service'; import { MessageChannelPendingGroupEmailsAction, MessageChannelWorkspaceEntity, @@ -22,7 +21,6 @@ export class MessagingProcessGroupEmailActionsService { private readonly twentyORMManager: TwentyORMManager, private readonly messagingDeleteGroupEmailMessagesService: MessagingDeleteGroupEmailMessagesService, private readonly messagingClearCursorsService: MessagingClearCursorsService, - private readonly messageChannelSyncStatusService: MessageChannelSyncStatusService, ) {} async markMessageChannelAsPendingGroupEmailsAction( diff --git a/packages/twenty-server/src/modules/messaging/monitoring/services/messaging-monitoring.service.ts b/packages/twenty-server/src/modules/messaging/monitoring/services/messaging-monitoring.service.ts index 8eaeb34294..9b7c7fb7da 100644 --- a/packages/twenty-server/src/modules/messaging/monitoring/services/messaging-monitoring.service.ts +++ b/packages/twenty-server/src/modules/messaging/monitoring/services/messaging-monitoring.service.ts @@ -1,7 +1,5 @@ import { Injectable } from '@nestjs/common'; -import { AuditService } from 'src/engine/core-modules/audit/services/audit.service'; - type MessagingMonitoringTrackInput = { eventName: string; workspaceId?: string; @@ -13,7 +11,7 @@ type MessagingMonitoringTrackInput = { @Injectable() export class MessagingMonitoringService { - constructor(private readonly _auditService: AuditService) {} + constructor() {} public async track({ eventName: _eventName,