Improve performances (#14869)

## Improvements

- Add logs to all gql operations and rest calls to help debug CPU issues
on the backend. These are temporary and should be removed
- Remove nested relations from workflowVersions load (used to add manual
triggers in the side bar). On some workspaces this call result in a
response of 4MB which is heavy on CPU
- Investigated Redis Usage ==> made a few improvements, we are should
still migrate to the new cache service once available
- investigated db calls in messaging / calendar fetch list + workflow
enqueue run cron jobs. Everything seems to be properly batched

packages/twenty-server/src/engine/metadata-modules/workspace-feature-flags-map-cache/workspace-feature-flags-map-cache.module.ts

@@ -10,6 +10,7 @@ import { WorkspaceCacheStorageModule } from 'src/engine/workspace-cache-storage/
   imports: [
     TypeOrmModule.forFeature([Workspace, FeatureFlag]),
     WorkspaceCacheStorageModule,
+    WorkspaceCacheStorageModule,
   ],
   providers: [WorkspaceFeatureFlagsMapCacheService],
   exports: [WorkspaceFeatureFlagsMapCacheService],

packages/twenty-server/src/engine/metadata-modules/workspace-feature-flags-map-cache/workspace-feature-flags-map-cache.service.ts

@@ -7,7 +7,7 @@ import { type FeatureFlagMap } from 'src/engine/core-modules/feature-flag/interf
 
 import { FeatureFlag } from 'src/engine/core-modules/feature-flag/feature-flag.entity';
 import { TwentyORMExceptionCode } from 'src/engine/twenty-orm/exceptions/twenty-orm.exception';
-import { getFromCacheWithRecompute } from 'src/engine/utils/get-data-from-cache-with-recompute.util';
+import { GetDataFromCacheWithRecomputeService } from 'src/engine/workspace-cache-storage/services/get-data-from-cache-with-recompute.service';
 import { WorkspaceCacheStorageService } from 'src/engine/workspace-cache-storage/workspace-cache-storage.service';
 
 const FEATURE_FLAG_MAP = 'Feature flag map';
@@ -20,6 +20,10 @@ export class WorkspaceFeatureFlagsMapCacheService {
     private readonly workspaceCacheStorageService: WorkspaceCacheStorageService,
     @InjectRepository(FeatureFlag)
     private readonly featureFlagRepository: Repository<FeatureFlag>,
+    private readonly getFromCacheWithRecomputeService: GetDataFromCacheWithRecomputeService<
+      string,
+      FeatureFlagMap
+    >,
   ) {}
 
   async getWorkspaceFeatureFlagsMap({
@@ -38,7 +42,7 @@ export class WorkspaceFeatureFlagsMapCacheService {
   }: {
     workspaceId: string;
   }) {
-    return getFromCacheWithRecompute<string, FeatureFlagMap>({
+    return this.getFromCacheWithRecomputeService.getFromCacheWithRecompute({
       workspaceId,
       getCacheData: () =>
         this.workspaceCacheStorageService.getFeatureFlagsMap(workspaceId),
@@ -49,7 +53,6 @@ export class WorkspaceFeatureFlagsMapCacheService {
       recomputeCache: (params) => this.recomputeFeatureFlagsMapCache(params),
       cachedEntityName: FEATURE_FLAG_MAP,
       exceptionCode: TwentyORMExceptionCode.FEATURE_FLAG_MAP_VERSION_NOT_FOUND,
-      logger: this.logger,
     });
   }
 

packages/twenty-server/src/engine/metadata-modules/workspace-permissions-cache/workspace-permissions-cache-storage.service.ts

@@ -98,6 +98,14 @@ export class WorkspacePermissionsCacheStorageService {
     );
   }
 
+  getUserWorkspaceRoleMapVersion(
+    workspaceId: string,
+  ): Promise<string | undefined> {
+    return this.cacheStorageService.get<string>(
+      `${WorkspaceCacheKeys.MetadataPermissionsUserWorkspaceRoleMapVersion}:${workspaceId}`,
+    );
+  }
+
   removeUserWorkspaceRoleMap(workspaceId: string) {
     return this.cacheStorageService.del(
       `${WorkspaceCacheKeys.MetadataPermissionsUserWorkspaceRoleMap}:${workspaceId}`,
@@ -126,6 +134,14 @@ export class WorkspacePermissionsCacheStorageService {
     );
   }
 
+  async getApiKeyRoleMapVersion(
+    workspaceId: string,
+  ): Promise<string | undefined> {
+    return this.cacheStorageService.get<string>(
+      `${WorkspaceCacheKeys.MetadataPermissionsApiKeyRoleMapVersion}:${workspaceId}`,
+    );
+  }
+
   async removeApiKeyRoleMap(workspaceId: string): Promise<void> {
     await Promise.all([
       this.cacheStorageService.del(

packages/twenty-server/src/engine/metadata-modules/workspace-permissions-cache/workspace-permissions-cache.service.ts

@@ -1,6 +1,7 @@
 import { Injectable, Logger } from '@nestjs/common';
 import { InjectRepository } from '@nestjs/typeorm';
 
+import { Record } from 'cloudflare/core';
 import {
   ObjectsPermissions,
   type ObjectsPermissionsByRoleIdDeprecated,
@@ -14,10 +15,10 @@ import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadat
 import { PermissionFlagType } from 'src/engine/metadata-modules/permissions/constants/permission-flag-type.constants';
 import { RoleTargetsEntity } from 'src/engine/metadata-modules/role/role-targets.entity';
 import { RoleEntity } from 'src/engine/metadata-modules/role/role.entity';
-import { type UserWorkspaceRoleMap } from 'src/engine/metadata-modules/workspace-permissions-cache/types/user-workspace-role-map.type';
+import { UserWorkspaceRoleMap } from 'src/engine/metadata-modules/workspace-permissions-cache/types/user-workspace-role-map.type';
 import { WorkspacePermissionsCacheStorageService } from 'src/engine/metadata-modules/workspace-permissions-cache/workspace-permissions-cache-storage.service';
 import { TwentyORMExceptionCode } from 'src/engine/twenty-orm/exceptions/twenty-orm.exception';
-import { getFromCacheWithRecompute } from 'src/engine/utils/get-data-from-cache-with-recompute.util';
+import { GetDataFromCacheWithRecomputeService } from 'src/engine/workspace-cache-storage/services/get-data-from-cache-with-recompute.service';
 import { STANDARD_OBJECT_IDS } from 'src/engine/workspace-manager/workspace-sync-metadata/constants/standard-object-ids';
 
 type CacheResult<T, U> = {
@@ -45,6 +46,18 @@ export class WorkspacePermissionsCacheService {
     @InjectRepository(RoleTargetsEntity)
     private readonly roleTargetsRepository: Repository<RoleTargetsEntity>,
     private readonly workspacePermissionsCacheStorageService: WorkspacePermissionsCacheStorageService,
+    private readonly getRolesPermissionsFromCacheWithRecomputeService: GetDataFromCacheWithRecomputeService<
+      string,
+      ObjectsPermissionsByRoleIdDeprecated
+    >,
+    private readonly getUserWorkspaceRoleMapFromCacheWithRecomputeService: GetDataFromCacheWithRecomputeService<
+      string,
+      UserWorkspaceRoleMap
+    >,
+    private readonly getApiKeyRoleMapFromCacheWithRecomputeService: GetDataFromCacheWithRecomputeService<
+      string,
+      Record<string, string>
+    >,
   ) {}
 
   async recomputeRolesPermissionsCache({
@@ -109,44 +122,48 @@ export class WorkspacePermissionsCacheService {
   }: {
     workspaceId: string;
   }): Promise<CacheResult<string, ObjectsPermissionsByRoleIdDeprecated>> {
-    return getFromCacheWithRecompute<
-      string,
-      ObjectsPermissionsByRoleIdDeprecated
-    >({
-      workspaceId,
-      getCacheData: () =>
-        this.workspacePermissionsCacheStorageService.getRolesPermissions(
-          workspaceId,
-        ),
-      getCacheVersion: () =>
-        this.workspacePermissionsCacheStorageService.getRolesPermissionsVersion(
-          workspaceId,
-        ),
-      recomputeCache: (params) => this.recomputeRolesPermissionsCache(params),
-      cachedEntityName: ROLES_PERMISSIONS,
-      exceptionCode: TwentyORMExceptionCode.ROLES_PERMISSIONS_VERSION_NOT_FOUND,
-      logger: this.logger,
-    });
+    return this.getRolesPermissionsFromCacheWithRecomputeService.getFromCacheWithRecompute(
+      {
+        workspaceId,
+        getCacheData: () =>
+          this.workspacePermissionsCacheStorageService.getRolesPermissions(
+            workspaceId,
+          ),
+        getCacheVersion: () =>
+          this.workspacePermissionsCacheStorageService.getRolesPermissionsVersion(
+            workspaceId,
+          ),
+        recomputeCache: (params) => this.recomputeRolesPermissionsCache(params),
+        cachedEntityName: ROLES_PERMISSIONS,
+        exceptionCode:
+          TwentyORMExceptionCode.ROLES_PERMISSIONS_VERSION_NOT_FOUND,
+      },
+    );
   }
 
   async getUserWorkspaceRoleMapFromCache({
     workspaceId,
   }: {
     workspaceId: string;
-  }): Promise<CacheResult<undefined, UserWorkspaceRoleMap>> {
-    return getFromCacheWithRecompute<undefined, UserWorkspaceRoleMap>({
-      workspaceId,
-      getCacheData: () =>
-        this.workspacePermissionsCacheStorageService.getUserWorkspaceRoleMap(
-          workspaceId,
-        ),
-      recomputeCache: (params) =>
-        this.recomputeUserWorkspaceRoleMapCache(params),
-      cachedEntityName: USER_WORKSPACE_ROLE_MAP,
-      exceptionCode:
-        TwentyORMExceptionCode.USER_WORKSPACE_ROLE_MAP_VERSION_NOT_FOUND,
-      logger: this.logger,
-    });
+  }): Promise<CacheResult<string, UserWorkspaceRoleMap>> {
+    return this.getUserWorkspaceRoleMapFromCacheWithRecomputeService.getFromCacheWithRecompute(
+      {
+        workspaceId,
+        getCacheData: () =>
+          this.workspacePermissionsCacheStorageService.getUserWorkspaceRoleMap(
+            workspaceId,
+          ),
+        getCacheVersion: () =>
+          this.workspacePermissionsCacheStorageService.getUserWorkspaceRoleMapVersion(
+            workspaceId,
+          ),
+        recomputeCache: (params) =>
+          this.recomputeUserWorkspaceRoleMapCache(params),
+        cachedEntityName: USER_WORKSPACE_ROLE_MAP,
+        exceptionCode:
+          TwentyORMExceptionCode.USER_WORKSPACE_ROLE_MAP_VERSION_NOT_FOUND,
+      },
+    );
   }
 
   async getRoleIdFromUserWorkspaceId({
@@ -364,18 +381,24 @@ export class WorkspacePermissionsCacheService {
     workspaceId,
   }: {
     workspaceId: string;
-  }): Promise<CacheResult<undefined, Record<string, string>>> {
-    return getFromCacheWithRecompute<undefined, Record<string, string>>({
-      workspaceId,
-      getCacheData: () =>
-        this.workspacePermissionsCacheStorageService.getApiKeyRoleMap(
-          workspaceId,
-        ),
-      recomputeCache: (params) => this.recomputeApiKeyRoleMapCache(params),
-      cachedEntityName: 'API_KEY_ROLE_MAP',
-      exceptionCode: TwentyORMExceptionCode.API_KEY_ROLE_MAP_VERSION_NOT_FOUND,
-      logger: this.logger,
-    });
+  }): Promise<CacheResult<string, Record<string, string>>> {
+    return this.getApiKeyRoleMapFromCacheWithRecomputeService.getFromCacheWithRecompute(
+      {
+        workspaceId,
+        getCacheData: () =>
+          this.workspacePermissionsCacheStorageService.getApiKeyRoleMap(
+            workspaceId,
+          ),
+        getCacheVersion: () =>
+          this.workspacePermissionsCacheStorageService.getApiKeyRoleMapVersion(
+            workspaceId,
+          ),
+        recomputeCache: (params) => this.recomputeApiKeyRoleMapCache(params),
+        cachedEntityName: 'API_KEY_ROLE_MAP',
+        exceptionCode:
+          TwentyORMExceptionCode.API_KEY_ROLE_MAP_VERSION_NOT_FOUND,
+      },
+    );
   }
 
   private async getApiKeyRoleMapFromDatabase({