Refactor workspace cache service (#16208)

## Context We've recently introduced a new workspace cache service which now acts as a cache access and local storage for all workspace related data, deprecating the individual specific services. - Better performance through multiple caching/fetching strategies - Consistent data access patterns across the codebase - Reduced redis queries through MGET/MSET/PIPELINE with multiple cache keys
2025-12-01 17:08:21 +01:00
parent 4e0545ebc5
commit 1eb2e44058
77 changed files with 1306 additions and 1863 deletions
@@ -1,6 +1,8 @@
 import { Injectable, Logger } from '@nestjs/common';
 import { InjectRepository } from '@nestjs/typeorm';

+import crypto from 'crypto';
+
 import { type ObjectsPermissionsByRoleId } from 'twenty-shared/types';
 import { isDefined } from 'twenty-shared/utils';
 import { EntitySchema, Repository } from 'typeorm';
@@ -12,12 +14,6 @@ import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.ent
 import { DataSourceService } from 'src/engine/metadata-modules/data-source/data-source.service';
 import { WorkspaceManyOrAllFlatEntityMapsCacheService } from 'src/engine/metadata-modules/flat-entity/services/workspace-many-or-all-flat-entity-maps-cache.service';
 import { buildObjectIdByNameMaps } from 'src/engine/metadata-modules/flat-object-metadata/utils/build-object-id-by-name-maps.util';
-import { WorkspaceFeatureFlagsMapCacheService } from 'src/engine/metadata-modules/workspace-feature-flags-map-cache/workspace-feature-flags-map-cache.service';
-import { WorkspacePermissionsCacheStorageService } from 'src/engine/metadata-modules/workspace-permissions-cache/workspace-permissions-cache-storage.service';
-import {
-  ROLES_PERMISSIONS,
-  WorkspacePermissionsCacheService,
-} from 'src/engine/metadata-modules/workspace-permissions-cache/workspace-permissions-cache.service';
 import { WorkspaceDataSource } from 'src/engine/twenty-orm/datasource/workspace.datasource';
 import {
  TwentyORMException,
@@ -26,15 +22,10 @@ import {
 import { EntitySchemaFactory } from 'src/engine/twenty-orm/factories/entity-schema.factory';
 import { PromiseMemoizer } from 'src/engine/twenty-orm/storage/promise-memoizer.storage';
 import { type CacheKey } from 'src/engine/twenty-orm/storage/types/cache-key.type';
-import { GetDataFromCacheWithRecomputeService } from 'src/engine/workspace-cache-storage/services/get-data-from-cache-with-recompute.service';
 import { WorkspaceCacheStorageService } from 'src/engine/workspace-cache-storage/workspace-cache-storage.service';
+import { WorkspaceCacheService } from 'src/engine/workspace-cache/services/workspace-cache.service';
 import { WorkspaceEventEmitter } from 'src/engine/workspace-event-emitter/workspace-event-emitter';

-type CacheResult<T, U> = {
-  version: T;
-  data: U;
-};
-
 const TWENTY_MINUTES_IN_MS = 120_000;

@Injectable()
@@ -48,16 +39,10 @@ export class WorkspaceDatasourceFactory {
    private readonly workspaceCacheStorageService: WorkspaceCacheStorageService,
    private readonly workspaceManyOrAllFlatEntityMapsCacheService: WorkspaceManyOrAllFlatEntityMapsCacheService,
    private readonly entitySchemaFactory: EntitySchemaFactory,
-    private readonly workspacePermissionsCacheService: WorkspacePermissionsCacheService,
-    private readonly workspacePermissionsCacheStorageService: WorkspacePermissionsCacheStorageService,
-    private readonly workspaceFeatureFlagsMapCacheService: WorkspaceFeatureFlagsMapCacheService,
+    private readonly workspaceCacheService: WorkspaceCacheService,
    @InjectRepository(WorkspaceEntity)
    private readonly workspaceRepository: Repository<WorkspaceEntity>,
    private readonly workspaceEventEmitter: WorkspaceEventEmitter,
-    private readonly getFromCacheWithRecomputeService: GetDataFromCacheWithRecomputeService<
-      string,
-      ObjectsPermissionsByRoleId
-    >,
  ) {}

  private async safelyDestroyDataSource(
@@ -88,17 +73,23 @@ export class WorkspaceDatasourceFactory {
    const dataSourceMetadataVersion =
      await this.getWorkspaceMetadataVersionFromCacheOrFromDB(workspaceId);

-    const { data: cachedFeatureFlagMap, version: cachedFeatureFlagMapVersion } =
-      await this.workspaceFeatureFlagsMapCacheService.getWorkspaceFeatureFlagsMapAndVersion(
-        { workspaceId },
-      );
-
    const {
-      data: cachedRolesPermissions,
-      version: cachedRolesPermissionsVersion,
-    } = await this.getRolesPermissionsFromCache({
-      workspaceId,
-    });
+      featureFlagsMap: cachedFeatureFlagMap,
+      rolesPermissions: cachedRolesPermissions,
+    } = await this.workspaceCacheService.getOrRecompute(workspaceId, [
+      'featureFlagsMap',
+      'rolesPermissions',
+    ]);
+
+    const cachedFeatureFlagMapVersion = crypto
+      .createHash('sha256')
+      .update(JSON.stringify(cachedFeatureFlagMap))
+      .digest('hex');
+
+    const cachedRolesPermissionsVersion = crypto
+      .createHash('sha256')
+      .update(JSON.stringify(cachedRolesPermissions))
+      .digest('hex');

    const cacheKey: CacheKey = `${workspaceId}-${dataSourceMetadataVersion}`;

@@ -251,30 +242,6 @@ export class WorkspaceDatasourceFactory {
    return workspaceDataSource;
  }

-  private async getRolesPermissionsFromCache({
-    workspaceId,
-  }: {
-    workspaceId: string;
-  }): Promise<CacheResult<string, ObjectsPermissionsByRoleId>> {
-    return this.getFromCacheWithRecomputeService.getFromCacheWithRecompute({
-      workspaceId,
-      getCacheData: () =>
-        this.workspacePermissionsCacheStorageService.getRolesPermissions(
-          workspaceId,
-        ),
-      getCacheVersion: () =>
-        this.workspacePermissionsCacheStorageService.getRolesPermissionsVersion(
-          workspaceId,
-        ),
-      recomputeCache: () =>
-        this.workspacePermissionsCacheService.recomputeRolesPermissionsCache({
-          workspaceId,
-        }),
-      cachedEntityName: ROLES_PERMISSIONS,
-      exceptionCode: TwentyORMExceptionCode.ROLES_PERMISSIONS_VERSION_NOT_FOUND,
-    });
-  }
-
  private updateWorkspaceDataSourceIfNeeded<T>({
    workspaceDataSource,
    currentVersion,
@@ -14,6 +14,7 @@ import { EntitySchemaFactory } from 'src/engine/twenty-orm/factories/entity-sche
 import { GlobalWorkspaceDataSourceService } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-datasource.service';
 import { GlobalWorkspaceOrmManager } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager';
 import { WorkspaceCacheStorageModule } from 'src/engine/workspace-cache-storage/workspace-cache-storage.module';
+import { WorkspaceCacheModule } from 'src/engine/workspace-cache/workspace-cache.module';
 import { WorkspaceEventEmitterModule } from 'src/engine/workspace-event-emitter/workspace-event-emitter.module';

@Global()
@@ -28,6 +29,7 @@ import { WorkspaceEventEmitterModule } from 'src/engine/workspace-event-emitter/
    FeatureFlagModule,
    TwentyConfigModule,
    WorkspaceEventEmitterModule,
+    WorkspaceCacheModule,
  ],
  providers: [
    GlobalWorkspaceDataSourceService,
@@ -12,8 +12,6 @@ import { type FlatEntityMaps } from 'src/engine/metadata-modules/flat-entity/typ
 import { type FlatFieldMetadata } from 'src/engine/metadata-modules/flat-field-metadata/types/flat-field-metadata.type';
 import { type FlatObjectMetadata } from 'src/engine/metadata-modules/flat-object-metadata/types/flat-object-metadata.type';
 import { buildObjectIdByNameMaps } from 'src/engine/metadata-modules/flat-object-metadata/utils/build-object-id-by-name-maps.util';
-import { WorkspaceFeatureFlagsMapCacheService } from 'src/engine/metadata-modules/workspace-feature-flags-map-cache/workspace-feature-flags-map-cache.service';
-import { WorkspacePermissionsCacheService } from 'src/engine/metadata-modules/workspace-permissions-cache/workspace-permissions-cache.service';
 import { EntitySchemaFactory } from 'src/engine/twenty-orm/factories/entity-schema.factory';
 import { GlobalWorkspaceDataSourceService } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-datasource.service';
 import { type WorkspaceRepository } from 'src/engine/twenty-orm/repository/workspace.repository';
@@ -22,7 +20,7 @@ import {
  withWorkspaceContext,
 } from 'src/engine/twenty-orm/storage/orm-workspace-context.storage';
 import { type RolePermissionConfig } from 'src/engine/twenty-orm/types/role-permission-config';
-import { WorkspaceCacheStorageService } from 'src/engine/workspace-cache-storage/workspace-cache-storage.service';
+import { WorkspaceCacheService } from 'src/engine/workspace-cache/services/workspace-cache.service';
 import { convertClassNameToObjectMetadataName } from 'src/engine/workspace-manager/workspace-sync-metadata/utils/convert-class-to-object-metadata-name.util';

@Injectable()
@@ -30,10 +28,8 @@ export class GlobalWorkspaceOrmManager {
  constructor(
    private readonly globalWorkspaceDataSourceService: GlobalWorkspaceDataSourceService,
    private readonly workspaceManyOrAllFlatEntityMapsCacheService: WorkspaceManyOrAllFlatEntityMapsCacheService,
-    private readonly workspaceCacheStorageService: WorkspaceCacheStorageService,
-    private readonly workspaceFeatureFlagsMapCacheService: WorkspaceFeatureFlagsMapCacheService,
-    private readonly workspacePermissionsCacheService: WorkspacePermissionsCacheService,
    private readonly entitySchemaFactory: EntitySchemaFactory,
+    private readonly workspaceCacheService: WorkspaceCacheService,
  ) {}

  async getRepository<T extends ObjectLiteral>(
@@ -90,31 +86,24 @@ export class GlobalWorkspaceOrmManager {
  ): Promise<ORMWorkspaceContext> {
    const workspaceId = authContext.workspace.id;

-    const { flatObjectMetadataMaps, flatFieldMetadataMaps, flatIndexMaps } =
-      await this.workspaceManyOrAllFlatEntityMapsCacheService.getOrRecomputeManyOrAllFlatEntityMaps(
-        {
-          workspaceId,
-          flatMapsKeys: [
-            'flatObjectMetadataMaps',
-            'flatFieldMetadataMaps',
-            'flatIndexMaps',
-          ],
-        },
-      );
+    const {
+      flatObjectMetadataMaps,
+      flatFieldMetadataMaps,
+      flatIndexMaps,
+      featureFlagsMap,
+      rolesPermissions: permissionsPerRoleId,
+    } = await this.workspaceCacheService.getOrRecompute(workspaceId, [
+      'flatObjectMetadataMaps',
+      'flatFieldMetadataMaps',
+      'flatIndexMaps',
+      'featureFlagsMap',
+      'rolesPermissions',
+      'userWorkspaceRoleMap',
+    ]);

    const { idByNameSingular: objectIdByNameSingular } =
      buildObjectIdByNameMaps(flatObjectMetadataMaps);

-    const { data: featureFlagsMap } =
-      await this.workspaceFeatureFlagsMapCacheService.getWorkspaceFeatureFlagsMapAndVersion(
-        { workspaceId },
-      );
-
-    const { data: permissionsPerRoleId } =
-      await this.workspacePermissionsCacheService.getRolesPermissionsFromCache({
-        workspaceId,
-      });
-
    const entitySchemas = this.buildEntitySchemas(
      workspaceId,
      flatObjectMetadataMaps,
@@ -96,6 +96,12 @@ export class PromiseMemoizer<T> {
        await this.clearKey(cacheKey, onDelete);
      }
    }
+
+    for (const cacheKey of [...this.pending.keys()]) {
+      if (cacheKey.startsWith(cacheKeyPrefix)) {
+        this.pending.delete(cacheKey);
+      }
+    }
  }

  async clearAll(onDelete?: (value: T) => Promise<void> | void): Promise<void> {
@@ -17,6 +17,7 @@ import { ScopedWorkspaceContextFactory } from 'src/engine/twenty-orm/factories/s
 import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager';
 import { TwentyORMManager } from 'src/engine/twenty-orm/twenty-orm.manager';
 import { WorkspaceCacheStorageModule } from 'src/engine/workspace-cache-storage/workspace-cache-storage.module';
+import { WorkspaceCacheModule } from 'src/engine/workspace-cache/workspace-cache.module';

 import { PgPoolSharedModule } from './pg-shared-pool/pg-shared-pool.module';

@@ -37,6 +38,7 @@ import { PgPoolSharedModule } from './pg-shared-pool/pg-shared-pool.module';
    FeatureFlagModule,
    TwentyConfigModule,
    PgPoolSharedModule,
+    WorkspaceCacheModule,
  ],
  providers: [
    ...entitySchemaFactories,