Add flat map cache service (#14418)

## Context Adding a new service that provides an abstract caching system for FlatEntityMaps. This takes care of cache invalidation and storing local and remote cache for the map with retrieval after a comparison with map hash between local and remote (redis). ## Implementation Remote (redis) keys - Flat map data: `engine:workspace:flat-maps:{flatMapKey}:{workspaceId}:flat-map` - Content hash: `engine:workspace:flat-maps:{flatMapKey}:{workspaceId}:hash` **Local Cache Hit**: If local hash matches Redis hash, return local data **Remote Cache Hit**: If Redis has data with different hash, update local cache **Remote Cache Miss**: Recompute from database, store in Remote and locally **Invalidation**: Remove from Remote, triggering recomputation on next access ## Usage ```typescript @WorkspaceFlatMapCache('view') // redis key export class WorkspaceFlatViewMapCacheService extends WorkspaceFlatMapCacheService<FlatViewMaps> { constructor( @InjectCacheStorage(CacheStorageNamespace.EngineWorkspace) cacheStorageService: CacheStorageService, @InjectRepository(ViewEntity) private readonly viewRepository: Repository<ViewEntity>, ) { super(cacheStorageService); } // only method to implement public async computeFlatMap(workspaceId: string): Promise<FlatViewMaps> { const views = await this.viewRepository.find({ where: { workspaceId }, relations: ['viewFields'], select: { viewFields: { id: true } }, }); return generateFlatViewMaps(views); } } ``` ```typescript // 2 public methods, getExistingOrRecomputeFlatMaps to fetch the map and invalidateCache after a mutation await this.workspaceFlatViewMapCacheService.invalidateCache( viewData.workspaceId, ); const flatViewMaps = await this.workspaceFlatViewMapCacheService.getExistingOrRecomputeFlatMaps( workspaceId, ); ``` ## Multi-Pod Synchronization - Each pod maintains local cache for performance - SHA256 hash of flatMap content used for version comparison - `invalidateCache()` reset Redis data, forcing other pods to refresh when calling getExistingOrRecomputeFlatMaps --- <img width="1072" height="325" alt="Screenshot 2025-09-11 at 15 04 48" src="https://github.com/user-attachments/assets/ed6ce82c-db35-4a1b-8a4e-247b694e2ddf" /> <img width="1030" height="328" alt="Screenshot 2025-09-11 at 15 04 40" src="https://github.com/user-attachments/assets/43a15789-7f27-4104-a9bb-bef19908a5cd" /> Next step: Implement a locking mechanism by reusing existing WithLock decorator
2025-09-11 16:54:21 +02:00
parent a4036e0370
commit d9a6e7e0b1
7 changed files with 276 additions and 0 deletions
@@ -0,0 +1,6 @@
+import { SetMetadata } from '@nestjs/common';
+
+export const WORKSPACE_FLAT_MAP_CACHE_KEY = 'workspaceFlatMapCacheKey';
+
+export const WorkspaceFlatMapCache = (cacheKey: string) =>
+  SetMetadata(WORKSPACE_FLAT_MAP_CACHE_KEY, cacheKey);
@@ -0,0 +1,12 @@
+import {
+  appendCommonExceptionCode,
+  CustomException,
+} from 'src/utils/custom-exception';
+
+export class WorkspaceFlatMapCacheException extends CustomException<
+  keyof typeof WorkspaceFlatMapCacheExceptionCode
+> {}
+
+export const WorkspaceFlatMapCacheExceptionCode = appendCommonExceptionCode({
+  MISSING_DECORATOR: 'MISSING_DECORATOR',
+} as const);
@@ -0,0 +1,188 @@
+import { Injectable, Logger } from '@nestjs/common';
+import { Reflector } from '@nestjs/core';
+
+import crypto from 'crypto';
+
+import { isDefined } from 'twenty-shared/utils';
+
+import { InjectCacheStorage } from 'src/engine/core-modules/cache-storage/decorators/cache-storage.decorator';
+import { CacheStorageService } from 'src/engine/core-modules/cache-storage/services/cache-storage.service';
+import { CacheStorageNamespace } from 'src/engine/core-modules/cache-storage/types/cache-storage-namespace.enum';
+import { type FlatEntityMaps } from 'src/engine/core-modules/common/types/flat-entity-maps.type';
+import { type FlatEntity } from 'src/engine/core-modules/common/types/flat-entity.type';
+import { WORKSPACE_FLAT_MAP_CACHE_KEY } from 'src/engine/workspace-flat-map-cache/decorators/workspace-flat-map-cache.decorator';
+import {
+  WorkspaceFlatMapCacheException,
+  WorkspaceFlatMapCacheExceptionCode,
+} from 'src/engine/workspace-flat-map-cache/exceptions/workspace-flat-map-cache.exception';
+
+@Injectable()
+export abstract class WorkspaceFlatMapCacheService<
+  T extends FlatEntityMaps<FlatEntity>,
+> {
+  protected readonly logger = new Logger(this.constructor.name);
+
+  private readonly localCacheFlatMaps = new Map<string, T>();
+  private readonly localCacheHashes = new Map<string, string>();
+  private readonly reflector = new Reflector();
+
+  constructor(
+    @InjectCacheStorage(CacheStorageNamespace.EngineWorkspace)
+    private readonly cacheStorageService: CacheStorageService,
+  ) {}
+
+  protected abstract computeFlatMap({
+    workspaceId,
+  }: {
+    workspaceId: string;
+  }): Promise<T>;
+
+  async getExistingOrRecomputeFlatMaps({
+    workspaceId,
+  }: {
+    workspaceId: string;
+  }): Promise<T> {
+    const localCacheHash = this.localCacheHashes.get(workspaceId);
+    const remoteCacheHash = await this.getHashFromRemoteCache({ workspaceId });
+
+    if (
+      isDefined(localCacheHash) &&
+      isDefined(remoteCacheHash) &&
+      localCacheHash === remoteCacheHash
+    ) {
+      const localCacheFlatMap = this.localCacheFlatMaps.get(workspaceId);
+
+      if (localCacheFlatMap) {
+        return localCacheFlatMap;
+      }
+    }
+
+    if (remoteCacheHash) {
+      const remoteCacheFlatMap = await this.getFlatMapFromRemoteCache({
+        workspaceId,
+      });
+
+      if (remoteCacheFlatMap) {
+        this.localCacheFlatMaps.set(workspaceId, remoteCacheFlatMap);
+        this.localCacheHashes.set(workspaceId, remoteCacheHash);
+
+        return remoteCacheFlatMap;
+      }
+    }
+
+    const freshFlatMap = await this.recomputeAndStoreInCache({
+      workspaceId,
+    });
+
+    return freshFlatMap;
+  }
+
+  async recomputeAndStoreInCache({
+    workspaceId,
+  }: {
+    workspaceId: string;
+  }): Promise<T> {
+    const freshFlatMap = await this.computeFlatMap({ workspaceId });
+    const newHash = this.generateHash({ flatMap: freshFlatMap });
+
+    await this.setFlatMapInRemoteCache({ workspaceId, flatMap: freshFlatMap });
+    await this.setHashInRemoteCache({ workspaceId, hash: newHash });
+
+    this.localCacheFlatMaps.set(workspaceId, freshFlatMap);
+    this.localCacheHashes.set(workspaceId, newHash);
+
+    return freshFlatMap;
+  }
+
+  async invalidateCache({
+    workspaceId,
+  }: {
+    workspaceId: string;
+  }): Promise<void> {
+    const { flatMapKey, hashKey } = this.buildRemoteCacheKeys({ workspaceId });
+
+    await this.cacheStorageService.del(flatMapKey);
+    await this.cacheStorageService.del(hashKey);
+
+    this.localCacheFlatMaps.delete(workspaceId);
+    this.localCacheHashes.delete(workspaceId);
+
+    await this.recomputeAndStoreInCache({ workspaceId });
+  }
+
+  private getFlatMapCacheKey(): string {
+    const cacheKey = this.reflector.get<string>(
+      WORKSPACE_FLAT_MAP_CACHE_KEY,
+      this.constructor,
+    );
+
+    if (!cacheKey) {
+      throw new WorkspaceFlatMapCacheException(
+        `${this.constructor.name} must be decorated with @WorkspaceFlatMapCache('cacheKey')`,
+        WorkspaceFlatMapCacheExceptionCode.MISSING_DECORATOR,
+      );
+    }
+
+    return cacheKey;
+  }
+
+  private buildRemoteCacheKeys({ workspaceId }: { workspaceId: string }) {
+    const cacheKey = this.getFlatMapCacheKey();
+
+    return {
+      flatMapKey: `flat-maps:${cacheKey}:${workspaceId}:flat-map`,
+      hashKey: `flat-maps:${cacheKey}:${workspaceId}:hash`,
+    };
+  }
+
+  private generateHash({ flatMap }: { flatMap: T }): string {
+    return crypto
+      .createHash('sha256')
+      .update(JSON.stringify(flatMap))
+      .digest('hex');
+  }
+
+  private async getHashFromRemoteCache({
+    workspaceId,
+  }: {
+    workspaceId: string;
+  }): Promise<string | undefined> {
+    const { hashKey } = this.buildRemoteCacheKeys({ workspaceId });
+
+    return this.cacheStorageService.get<string>(hashKey);
+  }
+
+  private async setHashInRemoteCache({
+    workspaceId,
+    hash,
+  }: {
+    workspaceId: string;
+    hash: string;
+  }): Promise<void> {
+    const { hashKey } = this.buildRemoteCacheKeys({ workspaceId });
+
+    await this.cacheStorageService.set(hashKey, hash);
+  }
+
+  private async getFlatMapFromRemoteCache({
+    workspaceId,
+  }: {
+    workspaceId: string;
+  }): Promise<T | undefined> {
+    const { flatMapKey } = this.buildRemoteCacheKeys({ workspaceId });
+
+    return this.cacheStorageService.get<T>(flatMapKey);
+  }
+
+  private async setFlatMapInRemoteCache({
+    workspaceId,
+    flatMap,
+  }: {
+    workspaceId: string;
+    flatMap: T;
+  }): Promise<void> {
+    const { flatMapKey } = this.buildRemoteCacheKeys({ workspaceId });
+
+    await this.cacheStorageService.set(flatMapKey, flatMap);
+  }
+}
@@ -0,0 +1,10 @@
+import { Module } from '@nestjs/common';
+
+import { CacheStorageModule } from 'src/engine/core-modules/cache-storage/cache-storage.module';
+
+@Module({
+  imports: [CacheStorageModule],
+  providers: [],
+  exports: [],
+})
+export class WorkspaceFlatMapCacheModule {}