From a21d58f43ddad010c1bdc63b370c9ea626f34fd8 Mon Sep 17 00:00:00 2001 From: Weiko Date: Fri, 21 Nov 2025 15:07:46 +0100 Subject: [PATCH] Add entitySchemas to workspace context and cache (#15966) --- .../global-workspace-datasource.service.ts | 1 - .../global-workspace-datasource.ts | 100 +----------------- .../global-workspace-orm.manager.ts | 72 ++++++++++--- .../storage/workspace-context.storage.ts | 2 + 4 files changed, 61 insertions(+), 114 deletions(-) diff --git a/packages/twenty-server/src/engine/twenty-orm/global-workspace-datasource/global-workspace-datasource.service.ts b/packages/twenty-server/src/engine/twenty-orm/global-workspace-datasource/global-workspace-datasource.service.ts index 9146597e30..cf585832ec 100644 --- a/packages/twenty-server/src/engine/twenty-orm/global-workspace-datasource/global-workspace-datasource.service.ts +++ b/packages/twenty-server/src/engine/twenty-orm/global-workspace-datasource/global-workspace-datasource.service.ts @@ -43,7 +43,6 @@ export class GlobalWorkspaceDataSourceService }, }, this.workspaceEventEmitter, - this.entitySchemaFactory, ); await this.globalWorkspaceDataSource.initialize(); diff --git a/packages/twenty-server/src/engine/twenty-orm/global-workspace-datasource/global-workspace-datasource.ts b/packages/twenty-server/src/engine/twenty-orm/global-workspace-datasource/global-workspace-datasource.ts index d77707a7d6..c6dd77c4d2 100644 --- a/packages/twenty-server/src/engine/twenty-orm/global-workspace-datasource/global-workspace-datasource.ts +++ b/packages/twenty-server/src/engine/twenty-orm/global-workspace-datasource/global-workspace-datasource.ts @@ -23,9 +23,7 @@ import { PermissionsException, PermissionsExceptionCode, } from 'src/engine/metadata-modules/permissions/permissions.exception'; -import { type ObjectMetadataMaps } from 'src/engine/metadata-modules/types/object-metadata-maps'; import { WorkspaceEntityManager } from 'src/engine/twenty-orm/entity-manager/workspace-entity-manager'; -import { type EntitySchemaFactory } from 'src/engine/twenty-orm/factories/entity-schema.factory'; import { type WorkspaceQueryRunner } from 'src/engine/twenty-orm/query-runner/workspace-query-runner'; import { type WorkspaceRepository } from 'src/engine/twenty-orm/repository/workspace.repository'; import { getWorkspaceContext } from 'src/engine/twenty-orm/storage/workspace-context.storage'; @@ -36,29 +34,17 @@ type CreateQueryBuilderOptions = { calledByWorkspaceEntityManager?: boolean; }; -const ENTITY_METADATA_CACHE_TTL_MS = 60 * 60 * 1000; // 1 hour - -type CachedEntityMetadata = { - entityMetadataMap: Map, EntityMetadata>; - timestamp: number; -}; - export class GlobalWorkspaceDataSource extends DataSource { - private entityMetadataCache: Map; private eventEmitterService: WorkspaceEventEmitter; - private entitySchemaFactory: EntitySchemaFactory; private _isConstructing = true; dataSourceWithOverridenCreateQueryBuilder: GlobalWorkspaceDataSource; constructor( options: DataSourceOptions, eventEmitterService: WorkspaceEventEmitter, - entitySchemaFactory: EntitySchemaFactory, ) { super(options); this.eventEmitterService = eventEmitterService; - this.entitySchemaFactory = entitySchemaFactory; - this.entityMetadataCache = new Map(); this._isConstructing = false; Object.defineProperty(this, 'manager', { @@ -97,17 +83,11 @@ export class GlobalWorkspaceDataSource extends DataSource { target: EntityTarget, ): EntityMetadata | undefined { const context = getWorkspaceContext(); - const { authContext, metadataVersion } = context; - const workspaceId = authContext.workspace.id; - const cacheKey = `${workspaceId}-${metadataVersion}`; + const { entitySchemas } = context; - const cachedEntityMetadata = this.getCachedEntityMetadata(cacheKey); + const entityMetadata = this.buildMetadatasFromSchemas(entitySchemas); - if (!cachedEntityMetadata) { - return undefined; - } - - return cachedEntityMetadata.entityMetadataMap.get(target); + return entityMetadata.find((metadata) => metadata.target === target); } override getMetadata(target: EntityTarget): EntityMetadata { @@ -282,53 +262,6 @@ export class GlobalWorkspaceDataSource extends DataSource { return super.query(query, parameters, queryRunner); } - async buildWorkspaceMetadata( - workspaceId: string, - metadataVersion: number, - objectMetadataMaps: ObjectMetadataMaps, - ): Promise { - const cacheKey = `${workspaceId}-${metadataVersion}`; - - if (this.getCachedEntityMetadata(cacheKey)) { - return; - } - - this.clearWorkspaceEntityMetadataCache(workspaceId); - - const entitySchemas = await Promise.all( - Object.values(objectMetadataMaps.byId) - .filter(isDefined) - .map((objectMetadata) => - this.entitySchemaFactory.create( - workspaceId, - objectMetadata, - objectMetadataMaps, - ), - ), - ); - - const entityMetadatas = this.buildMetadatasFromSchemas(entitySchemas); - - const metadataMap = new Map, EntityMetadata>(); - - for (const metadata of entityMetadatas) { - metadataMap.set(metadata.target, metadata); - } - - this.entityMetadataCache.set(cacheKey, { - entityMetadataMap: metadataMap, - timestamp: Date.now(), - }); - } - - private clearWorkspaceEntityMetadataCache(workspaceId: string): void { - for (const key of this.entityMetadataCache.keys()) { - if (key.startsWith(`${workspaceId}-`)) { - this.entityMetadataCache.delete(key); - } - } - } - private buildMetadatasFromSchemas( entitySchemas: EntitySchema[], ): EntityMetadata[] { @@ -344,31 +277,4 @@ export class GlobalWorkspaceDataSource extends DataSource { return entityMetadatas; } - - hasWorkspaceEntityMetadataCacheForVersion( - workspaceId: string, - metadataVersion: number, - ): boolean { - const cacheKey = `${workspaceId}-${metadataVersion}`; - - return !!this.getCachedEntityMetadata(cacheKey); - } - - private getCachedEntityMetadata( - cacheKey: string, - ): CachedEntityMetadata | undefined { - const cached = this.entityMetadataCache.get(cacheKey); - - if (!cached) { - return undefined; - } - - if (Date.now() - cached.timestamp > ENTITY_METADATA_CACHE_TTL_MS) { - this.entityMetadataCache.delete(cacheKey); - - return undefined; - } - - return cached; - } } diff --git a/packages/twenty-server/src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager.ts b/packages/twenty-server/src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager.ts index f8c6f179c9..1a35d1f049 100644 --- a/packages/twenty-server/src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager.ts +++ b/packages/twenty-server/src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager.ts @@ -1,12 +1,15 @@ import { Injectable, type Type } from '@nestjs/common'; -import { type ObjectLiteral } from 'typeorm'; +import { isDefined } from 'twenty-shared/utils'; +import { EntitySchema, type ObjectLiteral } from 'typeorm'; import { WorkspaceAuthContext } from 'src/engine/api/common/interfaces/workspace-auth-context.interface'; +import { ObjectMetadataMaps } from 'src/engine/metadata-modules/types/object-metadata-maps'; import { WorkspaceFeatureFlagsMapCacheService } from 'src/engine/metadata-modules/workspace-feature-flags-map-cache/workspace-feature-flags-map-cache.service'; import { WorkspaceMetadataCacheService } from 'src/engine/metadata-modules/workspace-metadata-cache/services/workspace-metadata-cache.service'; import { WorkspacePermissionsCacheService } from 'src/engine/metadata-modules/workspace-permissions-cache/workspace-permissions-cache.service'; +import { EntitySchemaFactory } from 'src/engine/twenty-orm/factories/entity-schema.factory'; import { GlobalWorkspaceDataSourceService } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-datasource.service'; import { type WorkspaceRepository } from 'src/engine/twenty-orm/repository/workspace.repository'; import { @@ -14,6 +17,7 @@ import { withWorkspaceContext, } from 'src/engine/twenty-orm/storage/workspace-context.storage'; import { type RolePermissionConfig } from 'src/engine/twenty-orm/types/role-permission-config'; +import { WorkspaceCacheStorageService } from 'src/engine/workspace-cache-storage/workspace-cache-storage.service'; import { convertClassNameToObjectMetadataName } from 'src/engine/workspace-manager/workspace-sync-metadata/utils/convert-class-to-object-metadata-name.util'; @Injectable() @@ -23,6 +27,8 @@ export class GlobalWorkspaceOrmManager { private readonly workspaceMetadataCacheService: WorkspaceMetadataCacheService, private readonly workspaceFeatureFlagsMapCacheService: WorkspaceFeatureFlagsMapCacheService, private readonly workspacePermissionsCacheService: WorkspacePermissionsCacheService, + private readonly workspaceCacheStorageService: WorkspaceCacheStorageService, + private readonly entitySchemaFactory: EntitySchemaFactory, ) {} async getRepository( @@ -70,21 +76,6 @@ export class GlobalWorkspaceOrmManager { fn: () => T | Promise, ): Promise { const context = await this.loadWorkspaceContext(authContext); - const globalDataSource = - this.globalWorkspaceDataSourceService.getGlobalWorkspaceDataSource(); - - if ( - !globalDataSource.hasWorkspaceEntityMetadataCacheForVersion( - authContext.workspace.id, - context.metadataVersion, - ) - ) { - await globalDataSource.buildWorkspaceMetadata( - authContext.workspace.id, - context.metadataVersion, - context.objectMetadataMaps, - ); - } return withWorkspaceContext(context, fn); } @@ -110,12 +101,61 @@ export class GlobalWorkspaceOrmManager { workspaceId, }); + const entitySchemas = await this.buildEntitySchemas( + workspaceId, + metadataVersion, + objectMetadataMaps, + ); + return { authContext, objectMetadataMaps, metadataVersion, featureFlagsMap, permissionsPerRoleId, + entitySchemas, }; } + + private async buildEntitySchemas( + workspaceId: string, + dataSourceMetadataVersion: number, + objectMetadataMaps: ObjectMetadataMaps, + ) { + const cachedEntitySchemaOptions = + await this.workspaceCacheStorageService.getORMEntitySchema( + workspaceId, + dataSourceMetadataVersion, + ); + + let cachedEntitySchemas: EntitySchema[]; + + if (cachedEntitySchemaOptions) { + cachedEntitySchemas = cachedEntitySchemaOptions.map( + (option) => new EntitySchema(option), + ); + } else { + const entitySchemas = await Promise.all( + Object.values(objectMetadataMaps.byId) + .filter(isDefined) + .map((objectMetadata) => + this.entitySchemaFactory.create( + workspaceId, + objectMetadata, + objectMetadataMaps, + ), + ), + ); + + await this.workspaceCacheStorageService.setORMEntitySchema( + workspaceId, + dataSourceMetadataVersion, + entitySchemas.map((entitySchema) => entitySchema.options), + ); + + cachedEntitySchemas = entitySchemas; + } + + return cachedEntitySchemas; + } } diff --git a/packages/twenty-server/src/engine/twenty-orm/storage/workspace-context.storage.ts b/packages/twenty-server/src/engine/twenty-orm/storage/workspace-context.storage.ts index fc77c58407..97ed0d55ca 100644 --- a/packages/twenty-server/src/engine/twenty-orm/storage/workspace-context.storage.ts +++ b/packages/twenty-server/src/engine/twenty-orm/storage/workspace-context.storage.ts @@ -1,6 +1,7 @@ import { AsyncLocalStorage } from 'async_hooks'; import { type ObjectsPermissionsByRoleId } from 'twenty-shared/types'; +import { type EntitySchema } from 'typeorm'; import { type WorkspaceAuthContext } from 'src/engine/api/common/interfaces/workspace-auth-context.interface'; @@ -13,6 +14,7 @@ export type WorkspaceContext = { metadataVersion: number; featureFlagsMap: Record; permissionsPerRoleId: ObjectsPermissionsByRoleId; + entitySchemas: EntitySchema[]; }; export const workspaceContextStorage =