Add flat map cache service (#14418)
## Context
Adding a new service that provides an abstract caching system for
FlatEntityMaps.
This takes care of cache invalidation and storing local and remote cache
for the map with retrieval after a comparison with map hash between
local and remote (redis).
## Implementation
Remote (redis) keys
- Flat map data:
`engine:workspace:flat-maps:{flatMapKey}:{workspaceId}:flat-map`
- Content hash:
`engine:workspace:flat-maps:{flatMapKey}:{workspaceId}:hash`
**Local Cache Hit**: If local hash matches Redis hash, return local data
**Remote Cache Hit**: If Redis has data with different hash, update
local cache
**Remote Cache Miss**: Recompute from database, store in Remote and
locally
**Invalidation**: Remove from Remote, triggering recomputation on next
access
## Usage
```typescript
@WorkspaceFlatMapCache('view') // redis key
export class WorkspaceFlatViewMapCacheService extends WorkspaceFlatMapCacheService<FlatViewMaps> {
constructor(
@InjectCacheStorage(CacheStorageNamespace.EngineWorkspace)
cacheStorageService: CacheStorageService,
@InjectRepository(ViewEntity)
private readonly viewRepository: Repository<ViewEntity>,
) {
super(cacheStorageService);
}
// only method to implement
public async computeFlatMap(workspaceId: string): Promise<FlatViewMaps> {
const views = await this.viewRepository.find({
where: { workspaceId },
relations: ['viewFields'],
select: { viewFields: { id: true } },
});
return generateFlatViewMaps(views);
}
}
```
```typescript
// 2 public methods, getExistingOrRecomputeFlatMaps to fetch the map and invalidateCache after a mutation
await this.workspaceFlatViewMapCacheService.invalidateCache(
viewData.workspaceId,
);
const flatViewMaps =
await this.workspaceFlatViewMapCacheService.getExistingOrRecomputeFlatMaps(
workspaceId,
);
```
## Multi-Pod Synchronization
- Each pod maintains local cache for performance
- SHA256 hash of flatMap content used for version comparison
- `invalidateCache()` reset Redis data, forcing other pods to refresh
when calling getExistingOrRecomputeFlatMaps
---
<img width="1072" height="325" alt="Screenshot 2025-09-11 at 15 04 48"
src="https://github.com/user-attachments/assets/ed6ce82c-db35-4a1b-8a4e-247b694e2ddf"
/>
<img width="1030" height="328" alt="Screenshot 2025-09-11 at 15 04 40"
src="https://github.com/user-attachments/assets/43a15789-7f27-4104-a9bb-bef19908a5cd"
/>
Next step: Implement a locking mechanism by reusing existing WithLock
decorator
This commit is contained in:
@@ -0,0 +1,12 @@
|
||||
import { Module } from '@nestjs/common';
|
||||
import { TypeOrmModule } from '@nestjs/typeorm';
|
||||
|
||||
import { ViewEntity } from 'src/engine/core-modules/view/entities/view.entity';
|
||||
import { WorkspaceFlatViewMapCacheService } from 'src/engine/core-modules/view/flat-view/services/workspace-flat-view-map-cache.service';
|
||||
|
||||
@Module({
|
||||
imports: [TypeOrmModule.forFeature([ViewEntity])],
|
||||
providers: [WorkspaceFlatViewMapCacheService],
|
||||
exports: [WorkspaceFlatViewMapCacheService],
|
||||
})
|
||||
export class FlatViewModule {}
|
||||
+46
@@ -0,0 +1,46 @@
|
||||
import { Injectable } from '@nestjs/common';
|
||||
import { InjectRepository } from '@nestjs/typeorm';
|
||||
|
||||
import { Repository } from 'typeorm';
|
||||
|
||||
import { InjectCacheStorage } from 'src/engine/core-modules/cache-storage/decorators/cache-storage.decorator';
|
||||
import { CacheStorageService } from 'src/engine/core-modules/cache-storage/services/cache-storage.service';
|
||||
import { CacheStorageNamespace } from 'src/engine/core-modules/cache-storage/types/cache-storage-namespace.enum';
|
||||
import { ViewEntity } from 'src/engine/core-modules/view/entities/view.entity';
|
||||
import { type FlatViewMaps } from 'src/engine/core-modules/view/flat-view/types/flat-view-maps.type';
|
||||
import { generateFlatViewMaps } from 'src/engine/core-modules/view/flat-view/utils/generate-flat-view-maps.util';
|
||||
import { WorkspaceFlatMapCache } from 'src/engine/workspace-flat-map-cache/decorators/workspace-flat-map-cache.decorator';
|
||||
import { WorkspaceFlatMapCacheService } from 'src/engine/workspace-flat-map-cache/services/workspace-flat-map-cache.service';
|
||||
|
||||
@Injectable()
|
||||
@WorkspaceFlatMapCache('view')
|
||||
export class WorkspaceFlatViewMapCacheService extends WorkspaceFlatMapCacheService<FlatViewMaps> {
|
||||
constructor(
|
||||
@InjectCacheStorage(CacheStorageNamespace.EngineWorkspace)
|
||||
cacheStorageService: CacheStorageService,
|
||||
@InjectRepository(ViewEntity)
|
||||
private readonly viewRepository: Repository<ViewEntity>,
|
||||
) {
|
||||
super(cacheStorageService);
|
||||
}
|
||||
|
||||
protected async computeFlatMap({
|
||||
workspaceId,
|
||||
}: {
|
||||
workspaceId: string;
|
||||
}): Promise<FlatViewMaps> {
|
||||
const views = await this.viewRepository.find({
|
||||
where: {
|
||||
workspaceId,
|
||||
},
|
||||
relations: ['viewFields'],
|
||||
select: {
|
||||
viewFields: {
|
||||
id: true,
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
return generateFlatViewMaps(views);
|
||||
}
|
||||
}
|
||||
@@ -15,6 +15,7 @@ import { ViewFilterEntity } from 'src/engine/core-modules/view/entities/view-fil
|
||||
import { ViewGroupEntity } from 'src/engine/core-modules/view/entities/view-group.entity';
|
||||
import { ViewSortEntity } from 'src/engine/core-modules/view/entities/view-sort.entity';
|
||||
import { ViewEntity } from 'src/engine/core-modules/view/entities/view.entity';
|
||||
import { FlatViewModule } from 'src/engine/core-modules/view/flat-view/flat-view.module';
|
||||
import { ViewFieldResolver } from 'src/engine/core-modules/view/resolvers/view-field.resolver';
|
||||
import { ViewFilterGroupResolver } from 'src/engine/core-modules/view/resolvers/view-filter-group.resolver';
|
||||
import { ViewFilterResolver } from 'src/engine/core-modules/view/resolvers/view-filter.resolver';
|
||||
@@ -48,6 +49,7 @@ import { WorkspaceMigrationV2Module } from 'src/engine/workspace-manager/workspa
|
||||
WorkspaceMetadataCacheModule,
|
||||
WorkspaceMigrationV2Module,
|
||||
ViewCacheModule,
|
||||
FlatViewModule,
|
||||
],
|
||||
controllers: [
|
||||
ViewController,
|
||||
|
||||
Reference in New Issue
Block a user