Add local only cache to cache service and cache typeorm entity metadata (#16287)

## Problem buildEntityMetadatas in GlobalWorkspaceOrmManager is computationally expensive and was running on every executeInWorkspaceContext call. This method uses TypeORM's EntitySchemaTransformer and EntityMetadataBuilder to build metadata for all workspace entities (30-50+ objects with many fields each). The resulting EntityMetadata[] is not serialisable which means it cannot be cached in Redis because they contain: - Circular references - Functions/methods - References to the DataSource instance ## Solution Extended the workspace cache system to support local-only caching, then created a cache provider for entityMetadatas. ## Implementation details Updated @WorkspaceCache decorator (workspace-cache.decorator.ts) - Added localOnly?: boolean option to skip Redis storage for non-serializable data Created WorkspaceEntityMetadatasCacheService - Computes entity metadatas from DB to avoid race condition, this is acceptable Simplified GlobalWorkspaceOrmManager - Now fetches entityMetadatas from cache instead of rebuilding on every call Updated Workspace migration runner - the only entry point where metadata can change - Now invalidate the new 'entityMetadata' local cache when shouldIncrementMetadataGraphqlSchemaVersion is true (== field/object mutations)
2025-12-03 19:50:40 +01:00
parent 9c334b5f03
commit 3d95d6ca00
20 changed files with 775 additions and 533 deletions
@@ -5,13 +5,16 @@ import { FeatureFlagModule } from 'src/engine/core-modules/feature-flag/feature-
 import { TwentyConfigModule } from 'src/engine/core-modules/twenty-config/twenty-config.module';
 import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity';
 import { DataSourceModule } from 'src/engine/metadata-modules/data-source/data-source.module';
+import { FieldMetadataEntity } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity';
 import { WorkspaceManyOrAllFlatEntityMapsCacheModule } from 'src/engine/metadata-modules/flat-entity/services/workspace-many-or-all-flat-entity-maps-cache.module';
+import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity';
 import { WorkspaceFeatureFlagsMapCacheModule } from 'src/engine/metadata-modules/workspace-feature-flags-map-cache/workspace-feature-flags-map-cache.module';
 import { EntitySchemaColumnFactory } from 'src/engine/twenty-orm/factories/entity-schema-column.factory';
 import { EntitySchemaRelationFactory } from 'src/engine/twenty-orm/factories/entity-schema-relation.factory';
 import { EntitySchemaFactory } from 'src/engine/twenty-orm/factories/entity-schema.factory';
 import { GlobalWorkspaceDataSourceService } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-datasource.service';
 import { GlobalWorkspaceOrmManager } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager';
+import { WorkspaceORMEntityMetadatasCacheService } from 'src/engine/twenty-orm/global-workspace-datasource/workspace-orm-entity-metadatas-cache.service';
 import { WorkspaceCacheStorageModule } from 'src/engine/workspace-cache-storage/workspace-cache-storage.module';
 import { WorkspaceCacheModule } from 'src/engine/workspace-cache/workspace-cache.module';
 import { WorkspaceEventEmitterModule } from 'src/engine/workspace-event-emitter/workspace-event-emitter.module';
@@ -19,7 +22,11 @@ import { WorkspaceEventEmitterModule } from 'src/engine/workspace-event-emitter/
@Global()
@Module({
  imports: [
-    TypeOrmModule.forFeature([WorkspaceEntity]),
+    TypeOrmModule.forFeature([
+      WorkspaceEntity,
+      ObjectMetadataEntity,
+      FieldMetadataEntity,
+    ]),
    DataSourceModule,
    WorkspaceCacheStorageModule,
    WorkspaceManyOrAllFlatEntityMapsCacheModule,
@@ -35,6 +42,7 @@ import { WorkspaceEventEmitterModule } from 'src/engine/workspace-event-emitter/
    EntitySchemaFactory,
    EntitySchemaColumnFactory,
    EntitySchemaRelationFactory,
+    WorkspaceORMEntityMetadatasCacheService,
  ],
  exports: [GlobalWorkspaceDataSourceService, GlobalWorkspaceOrmManager],
 })
@@ -1,18 +1,10 @@
 import { Injectable, type Type } from '@nestjs/common';

-import { isDefined } from 'twenty-shared/utils';
-import { EntitySchema, ObjectLiteral } from 'typeorm';
-import { EntitySchemaTransformer } from 'typeorm/entity-schema/EntitySchemaTransformer';
-import { EntityMetadataBuilder } from 'typeorm/metadata-builder/EntityMetadataBuilder';
+import { ObjectLiteral } from 'typeorm';

 import { WorkspaceAuthContext } from 'src/engine/api/common/interfaces/workspace-auth-context.interface';

-import { WorkspaceManyOrAllFlatEntityMapsCacheService } from 'src/engine/metadata-modules/flat-entity/services/workspace-many-or-all-flat-entity-maps-cache.service';
-import { type FlatEntityMaps } from 'src/engine/metadata-modules/flat-entity/types/flat-entity-maps.type';
-import { type FlatFieldMetadata } from 'src/engine/metadata-modules/flat-field-metadata/types/flat-field-metadata.type';
-import { type FlatObjectMetadata } from 'src/engine/metadata-modules/flat-object-metadata/types/flat-object-metadata.type';
 import { buildObjectIdByNameMaps } from 'src/engine/metadata-modules/flat-object-metadata/utils/build-object-id-by-name-maps.util';
-import { EntitySchemaFactory } from 'src/engine/twenty-orm/factories/entity-schema.factory';
 import { GlobalWorkspaceDataSourceService } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-datasource.service';
 import { type WorkspaceRepository } from 'src/engine/twenty-orm/repository/workspace.repository';
 import {
@@ -27,8 +19,6 @@ import { convertClassNameToObjectMetadataName } from 'src/engine/workspace-manag
 export class GlobalWorkspaceOrmManager {
  constructor(
    private readonly globalWorkspaceDataSourceService: GlobalWorkspaceDataSourceService,
-    private readonly workspaceManyOrAllFlatEntityMapsCacheService: WorkspaceManyOrAllFlatEntityMapsCacheService,
-    private readonly entitySchemaFactory: EntitySchemaFactory,
    private readonly workspaceCacheService: WorkspaceCacheService,
  ) {}

@@ -92,26 +82,19 @@ export class GlobalWorkspaceOrmManager {
      flatIndexMaps,
      featureFlagsMap,
      rolesPermissions: permissionsPerRoleId,
+      ORMEntityMetadatas: entityMetadatas,
    } = await this.workspaceCacheService.getOrRecompute(workspaceId, [
      'flatObjectMetadataMaps',
      'flatFieldMetadataMaps',
      'flatIndexMaps',
      'featureFlagsMap',
      'rolesPermissions',
-      'userWorkspaceRoleMap',
+      'ORMEntityMetadatas',
    ]);

    const { idByNameSingular: objectIdByNameSingular } =
      buildObjectIdByNameMaps(flatObjectMetadataMaps);

-    const entitySchemas = this.buildEntitySchemas(
-      workspaceId,
-      flatObjectMetadataMaps,
-      flatFieldMetadataMaps,
-    );
-
-    const entityMetadatas = this.buildEntityMetadatas(entitySchemas);
-
    return {
      authContext,
      flatObjectMetadataMaps,
@@ -123,33 +106,4 @@ export class GlobalWorkspaceOrmManager {
      entityMetadatas,
    };
  }
-
-  private buildEntitySchemas(
-    workspaceId: string,
-    flatObjectMetadataMaps: FlatEntityMaps<FlatObjectMetadata>,
-    flatFieldMetadataMaps: FlatEntityMaps<FlatFieldMetadata>,
-  ) {
-    return Object.values(flatObjectMetadataMaps.byId)
-      .filter(isDefined)
-      .map((flatObjectMetadata) =>
-        this.entitySchemaFactory.create(
-          workspaceId,
-          flatObjectMetadata,
-          flatObjectMetadataMaps,
-          flatFieldMetadataMaps,
-        ),
-      );
-  }
-
-  private buildEntityMetadatas(entitySchemas: EntitySchema[]) {
-    const transformer = new EntitySchemaTransformer();
-    const metadataArgsStorage = transformer.transform(entitySchemas);
-
-    const entityMetadataBuilder = new EntityMetadataBuilder(
-      this.globalWorkspaceDataSourceService.getGlobalWorkspaceDataSource(),
-      metadataArgsStorage,
-    );
-
-    return entityMetadataBuilder.build();
-  }
 }
@@ -0,0 +1,85 @@
+import { type FieldMetadataType } from 'twenty-shared/types';
+
+import { type FieldMetadataEntity } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity';
+import { type ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity';
+
+export type EntitySchemaObjectMetadata = Pick<
+  ObjectMetadataEntity,
+  'id' | 'nameSingular' | 'isCustom'
+> & {
+  fieldMetadataIds: string[];
+};
+
+export type EntitySchemaFieldMetadata<
+  TFieldMetadataType extends FieldMetadataType = FieldMetadataType,
+> = Pick<
+  FieldMetadataEntity<TFieldMetadataType>,
+  | 'id'
+  | 'name'
+  | 'type'
+  | 'settings'
+  | 'isNullable'
+  | 'defaultValue'
+  | 'options'
+  | 'objectMetadataId'
+  | 'relationTargetObjectMetadataId'
+  | 'relationTargetFieldMetadataId'
+>;
+
+export type EntitySchemaObjectMetadataMaps = {
+  byId: Partial<Record<string, EntitySchemaObjectMetadata>>;
+};
+
+export type EntitySchemaFieldMetadataMaps = {
+  byId: Partial<Record<string, EntitySchemaFieldMetadata>>;
+};
+
+export const buildEntitySchemaMetadataMaps = (
+  objectMetadatas: ObjectMetadataEntity[],
+  fieldMetadatas: FieldMetadataEntity[],
+): {
+  objectMetadataMaps: EntitySchemaObjectMetadataMaps;
+  fieldMetadataMaps: EntitySchemaFieldMetadataMaps;
+} => {
+  const fieldIdsByObjectId = new Map<string, string[]>();
+
+  for (const field of fieldMetadatas) {
+    const existing = fieldIdsByObjectId.get(field.objectMetadataId);
+
+    if (existing) {
+      existing.push(field.id);
+    } else {
+      fieldIdsByObjectId.set(field.objectMetadataId, [field.id]);
+    }
+  }
+
+  const objectMetadataMaps: EntitySchemaObjectMetadataMaps = { byId: {} };
+
+  for (const object of objectMetadatas) {
+    objectMetadataMaps.byId[object.id] = {
+      id: object.id,
+      nameSingular: object.nameSingular,
+      isCustom: object.isCustom,
+      fieldMetadataIds: fieldIdsByObjectId.get(object.id) ?? [],
+    };
+  }
+
+  const fieldMetadataMaps: EntitySchemaFieldMetadataMaps = { byId: {} };
+
+  for (const field of fieldMetadatas) {
+    fieldMetadataMaps.byId[field.id] = {
+      id: field.id,
+      name: field.name,
+      type: field.type,
+      settings: field.settings,
+      isNullable: field.isNullable,
+      defaultValue: field.defaultValue,
+      options: field.options,
+      objectMetadataId: field.objectMetadataId,
+      relationTargetObjectMetadataId: field.relationTargetObjectMetadataId,
+      relationTargetFieldMetadataId: field.relationTargetFieldMetadataId,
+    };
+  }
+
+  return { objectMetadataMaps, fieldMetadataMaps };
+};
@@ -0,0 +1,76 @@
+import { Injectable } from '@nestjs/common';
+import { InjectRepository } from '@nestjs/typeorm';
+
+import { isDefined } from 'twenty-shared/utils';
+import { type EntityMetadata, EntitySchema, Repository } from 'typeorm';
+import { EntitySchemaTransformer } from 'typeorm/entity-schema/EntitySchemaTransformer';
+import { EntityMetadataBuilder } from 'typeorm/metadata-builder/EntityMetadataBuilder';
+
+import { WorkspaceCacheProvider } from 'src/engine/workspace-cache/interfaces/workspace-cache-provider.service';
+
+import { FieldMetadataEntity } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity';
+import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity';
+import { EntitySchemaFactory } from 'src/engine/twenty-orm/factories/entity-schema.factory';
+import { GlobalWorkspaceDataSourceService } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-datasource.service';
+import { buildEntitySchemaMetadataMaps } from 'src/engine/twenty-orm/global-workspace-datasource/types/entity-schema-metadata.type';
+import { WorkspaceCache } from 'src/engine/workspace-cache/decorators/workspace-cache.decorator';
+
+@Injectable()
+@WorkspaceCache('ORMEntityMetadatas', { localDataOnly: true })
+export class WorkspaceORMEntityMetadatasCacheService extends WorkspaceCacheProvider<
+  EntityMetadata[]
+> {
+  constructor(
+    @InjectRepository(ObjectMetadataEntity)
+    private readonly objectMetadataRepository: Repository<ObjectMetadataEntity>,
+    @InjectRepository(FieldMetadataEntity)
+    private readonly fieldMetadataRepository: Repository<FieldMetadataEntity>,
+    private readonly entitySchemaFactory: EntitySchemaFactory,
+    private readonly globalWorkspaceDataSourceService: GlobalWorkspaceDataSourceService,
+  ) {
+    super();
+  }
+
+  async computeForCache(workspaceId: string): Promise<EntityMetadata[]> {
+    const [objectMetadatas, fieldMetadatas] = await Promise.all([
+      this.objectMetadataRepository.find({
+        where: { workspaceId },
+        withDeleted: true,
+      }),
+      this.fieldMetadataRepository.find({
+        where: { workspaceId },
+        withDeleted: true,
+      }),
+    ]);
+
+    const { objectMetadataMaps, fieldMetadataMaps } =
+      buildEntitySchemaMetadataMaps(objectMetadatas, fieldMetadatas);
+
+    const entitySchemas = Object.values(objectMetadataMaps.byId)
+      .filter(isDefined)
+      .map((objectMetadata) =>
+        this.entitySchemaFactory.create(
+          workspaceId,
+          objectMetadata,
+          objectMetadataMaps,
+          fieldMetadataMaps,
+        ),
+      );
+
+    return this.buildEntityMetadatas(entitySchemas);
+  }
+
+  private buildEntityMetadatas(
+    entitySchemas: EntitySchema[],
+  ): EntityMetadata[] {
+    const transformer = new EntitySchemaTransformer();
+    const metadataArgsStorage = transformer.transform(entitySchemas);
+
+    const entityMetadataBuilder = new EntityMetadataBuilder(
+      this.globalWorkspaceDataSourceService.getGlobalWorkspaceDataSource(),
+      metadataArgsStorage,
+    );
+
+    return entityMetadataBuilder.build();
+  }
+}