From d3a7241b6fe67055239e2108f8652af7fab6390f Mon Sep 17 00:00:00 2001 From: Paul Rastoin <45004772+prastoin@users.noreply.github.com> Date: Fri, 3 Oct 2025 17:51:24 +0200 Subject: [PATCH] Cache flush after database reset (#14873) When starting the app on a fresh database reset the cache would be filled with empty flat field metadata maps Because the reset command hack through the repository directly in order to create views and stuff implemented an iso flush as the one existing initially added it to a workspace deletion tambien ``` { byId: {], universalIdById: {} } ``` --- .../application/application-sync.service.ts | 10 +- ...y-or-all-flat-entity-maps-cache.service.ts | 128 +++++++++--------- .../view/services/view-field-v2.service.ts | 14 +- .../view/services/view-v2.service.ts | 14 +- .../__tests__/workspace.service.spec.ts | 11 +- .../workspace/services/workspace.service.ts | 17 ++- .../workspace/workspace.module.ts | 20 +-- .../engine/dataloaders/dataloader.service.ts | 6 +- .../services/cron-trigger-v2.service.ts | 10 +- .../database-event-trigger-v2.service.ts | 10 +- .../services/field-metadata.service-v2.ts | 6 +- .../services/field-metadata.service.ts | 6 +- .../object-metadata-v2.service.ts | 10 +- .../object-metadata.service.ts | 8 +- .../services/route-trigger-v2.service.ts | 10 +- .../serverless-function-v2.service.ts | 14 +- ...rkspace-flat-map-cache-registry.service.ts | 40 ++++-- .../workspace-flat-map-cache.service.ts | 21 +-- .../dev-seeder/dev-seeder.module.ts | 2 + .../dev-seeder/services/dev-seeder.service.ts | 5 + ...runner-action-handler-service.interface.ts | 2 +- .../workspace-migration-runner-v2.service.ts | 6 +- .../workspace-sync-metadata.service.ts | 2 +- 23 files changed, 210 insertions(+), 162 deletions(-) diff --git a/packages/twenty-server/src/engine/core-modules/application/application-sync.service.ts b/packages/twenty-server/src/engine/core-modules/application/application-sync.service.ts index d33c7f69d3..3df1e1fb34 100644 --- a/packages/twenty-server/src/engine/core-modules/application/application-sync.service.ts +++ b/packages/twenty-server/src/engine/core-modules/application/application-sync.service.ts @@ -190,7 +190,7 @@ export class ApplicationSyncService { await this.flatEntityMapsCacheService.getOrRecomputeManyOrAllFlatEntityMaps( { workspaceId, - flatEntities: ['flatObjectMetadataMaps'], + flatMapsKeys: ['flatObjectMetadataMaps'], }, ); @@ -303,7 +303,7 @@ export class ApplicationSyncService { await this.flatEntityMapsCacheService.getOrRecomputeManyOrAllFlatEntityMaps( { workspaceId, - flatEntities: ['flatServerlessFunctionMaps'], + flatMapsKeys: ['flatServerlessFunctionMaps'], }, ); @@ -456,7 +456,7 @@ export class ApplicationSyncService { await this.flatEntityMapsCacheService.getOrRecomputeManyOrAllFlatEntityMaps( { workspaceId, - flatEntities: ['flatDatabaseEventTriggerMaps'], + flatMapsKeys: ['flatDatabaseEventTriggerMaps'], }, ); @@ -572,7 +572,7 @@ export class ApplicationSyncService { await this.flatEntityMapsCacheService.getOrRecomputeManyOrAllFlatEntityMaps( { workspaceId, - flatEntities: ['flatCronTriggerMaps'], + flatMapsKeys: ['flatCronTriggerMaps'], }, ); @@ -685,7 +685,7 @@ export class ApplicationSyncService { await this.flatEntityMapsCacheService.getOrRecomputeManyOrAllFlatEntityMaps( { workspaceId, - flatEntities: ['flatRouteTriggerMaps'], + flatMapsKeys: ['flatRouteTriggerMaps'], }, ); diff --git a/packages/twenty-server/src/engine/core-modules/common/services/workspace-many-or-all-flat-entity-maps-cache.service.ts b/packages/twenty-server/src/engine/core-modules/common/services/workspace-many-or-all-flat-entity-maps-cache.service.ts index 2a4f029158..a3d66e6761 100644 --- a/packages/twenty-server/src/engine/core-modules/common/services/workspace-many-or-all-flat-entity-maps-cache.service.ts +++ b/packages/twenty-server/src/engine/core-modules/common/services/workspace-many-or-all-flat-entity-maps-cache.service.ts @@ -3,13 +3,9 @@ import { Injectable, Logger } from '@nestjs/common'; import { isDefined } from 'twenty-shared/utils'; import { ALL_FLAT_ENTITY_MAPS_PROPERTIES } from 'src/engine/core-modules/common/constant/all-flat-entity-maps-properties.constant'; -import { EMPTY_ALL_FLAT_ENTITY_MAPS } from 'src/engine/core-modules/common/constant/empty-all-flat-entity-maps.constant'; import { AllFlatEntityMaps } from 'src/engine/core-modules/common/types/all-flat-entity-maps.type'; -import { - WorkspaceFlatMapCacheException, - WorkspaceFlatMapCacheExceptionCode, -} from 'src/engine/workspace-flat-map-cache/exceptions/workspace-flat-map-cache.exception'; import { WorkspaceFlatMapCacheRegistryService } from 'src/engine/workspace-flat-map-cache/services/workspace-flat-map-cache-registry.service'; +import { WorkspaceFlatMapCacheService } from 'src/engine/workspace-flat-map-cache/services/workspace-flat-map-cache.service'; @Injectable() export class WorkspaceManyOrAllFlatEntityMapsCacheService { @@ -21,85 +17,95 @@ export class WorkspaceManyOrAllFlatEntityMapsCacheService { private readonly cacheRegistry: WorkspaceFlatMapCacheRegistryService, ) {} - public async getOrRecomputeManyOrAllFlatEntityMaps< - T extends (keyof AllFlatEntityMaps)[] = (keyof AllFlatEntityMaps)[], + private async executeActionForManyOrAllFlatEntity< + K extends (keyof AllFlatEntityMaps)[] = (keyof AllFlatEntityMaps)[], >({ - flatEntities, - workspaceId, + action, + flatMapsKeys, }: { - workspaceId: string; - flatEntities?: T; - }): Promise> { - const allFlatEntityMaps: AllFlatEntityMaps = structuredClone( - EMPTY_ALL_FLAT_ENTITY_MAPS, - ); - - for (const flatEntityName of ALL_FLAT_ENTITY_MAPS_PROPERTIES) { - if (isDefined(flatEntities) && !flatEntities.includes(flatEntityName)) { - delete allFlatEntityMaps[flatEntityName]; - continue; - } + flatMapsKeys: K | undefined; + action: (args: { + service: WorkspaceFlatMapCacheService; + flatMapKey: K[number]; + }) => Promise; + }): Promise { + const keysToProcess = isDefined(flatMapsKeys) + ? flatMapsKeys + : ALL_FLAT_ENTITY_MAPS_PROPERTIES; + for (const flatMapKey of keysToProcess) { try { - const service = this.cacheRegistry.getCacheService(flatEntityName); + const service = this.cacheRegistry.getCacheServiceOrThrow( + flatMapKey as K[number], + ); - if (!isDefined(service)) { - throw new WorkspaceFlatMapCacheException( - `No cache service found for ${flatEntityName}`, - WorkspaceFlatMapCacheExceptionCode.INTERNAL_SERVER_ERROR, - ); - } - - const result = await service.getExistingOrRecomputeFlatMaps({ - workspaceId, + await action({ + flatMapKey: flatMapKey, + service, }); - - // @ts-expect-error todo prastoin once refactored flat object metadata cache - allFlatEntityMaps[flatEntityName] = result; } catch (error) { this.logger.error( - `Failed to get flat entity maps for ${flatEntityName}`, + `Failed to run action on flat entity maps of ${flatMapKey}`, error, ); throw error; } } + } - return allFlatEntityMaps; + public async getOrRecomputeManyOrAllFlatEntityMaps< + T extends (keyof AllFlatEntityMaps)[] = (keyof AllFlatEntityMaps)[], + >({ + flatMapsKeys, + workspaceId, + }: { + workspaceId: string; + flatMapsKeys?: T; + }): Promise> { + let pickedFlatEntityMaps = {} as Pick; + + await this.executeActionForManyOrAllFlatEntity({ + action: async ({ service, flatMapKey }) => { + const cacheResult = await service.getExistingOrRecomputeFlatMaps({ + workspaceId, + }); + + pickedFlatEntityMaps[flatMapKey] = cacheResult; + }, + flatMapsKeys, + }); + + return pickedFlatEntityMaps; } public async invalidateFlatEntityMaps< T extends (keyof AllFlatEntityMaps)[] = (keyof AllFlatEntityMaps)[], >({ - flatEntities, + flatMapsKeys, workspaceId, }: { workspaceId: string; - flatEntities?: T; + flatMapsKeys?: T; }): Promise { - for (const flatEntityName of ALL_FLAT_ENTITY_MAPS_PROPERTIES) { - if (isDefined(flatEntities) && !flatEntities.includes(flatEntityName)) { - continue; - } + await this.executeActionForManyOrAllFlatEntity({ + action: async ({ service }) => + await service.invalidateCache({ workspaceId }), + flatMapsKeys, + }); + } - try { - const service = this.cacheRegistry.getCacheService(flatEntityName); - - if (!isDefined(service)) { - throw new WorkspaceFlatMapCacheException( - `No cache service found for ${flatEntityName}`, - WorkspaceFlatMapCacheExceptionCode.INTERNAL_SERVER_ERROR, - ); - } - - await service.invalidateCache({ workspaceId }); - } catch (error) { - this.logger.error( - `Failed to invalidate flat entity maps for ${flatEntityName}`, - error, - ); - throw error; - } - } + public async flushFlatEntityMaps< + T extends (keyof AllFlatEntityMaps)[] = (keyof AllFlatEntityMaps)[], + >({ + flatMapsKeys, + workspaceId, + }: { + workspaceId: string; + flatMapsKeys?: T; + }): Promise { + await this.executeActionForManyOrAllFlatEntity({ + action: async ({ service }) => await service.flushCache({ workspaceId }), + flatMapsKeys, + }); } } diff --git a/packages/twenty-server/src/engine/core-modules/view/services/view-field-v2.service.ts b/packages/twenty-server/src/engine/core-modules/view/services/view-field-v2.service.ts index be4e974275..be63db7701 100644 --- a/packages/twenty-server/src/engine/core-modules/view/services/view-field-v2.service.ts +++ b/packages/twenty-server/src/engine/core-modules/view/services/view-field-v2.service.ts @@ -41,7 +41,7 @@ export class ViewFieldV2Service { } = await this.flatEntityMapsCacheService.getOrRecomputeManyOrAllFlatEntityMaps( { workspaceId, - flatEntities: [ + flatMapsKeys: [ 'flatViewFieldMaps', 'flatViewMaps', 'flatFieldMetadataMaps', @@ -92,7 +92,7 @@ export class ViewFieldV2Service { await this.flatEntityMapsCacheService.getOrRecomputeManyOrAllFlatEntityMaps( { workspaceId, - flatEntities: ['flatViewFieldMaps'], + flatMapsKeys: ['flatViewFieldMaps'], }, ); @@ -113,7 +113,7 @@ export class ViewFieldV2Service { await this.flatEntityMapsCacheService.getOrRecomputeManyOrAllFlatEntityMaps( { workspaceId, - flatEntities: ['flatViewFieldMaps'], + flatMapsKeys: ['flatViewFieldMaps'], }, ); @@ -160,7 +160,7 @@ export class ViewFieldV2Service { await this.flatEntityMapsCacheService.getOrRecomputeManyOrAllFlatEntityMaps( { workspaceId, - flatEntities: ['flatViewFieldMaps'], + flatMapsKeys: ['flatViewFieldMaps'], }, ); @@ -181,7 +181,7 @@ export class ViewFieldV2Service { await this.flatEntityMapsCacheService.getOrRecomputeManyOrAllFlatEntityMaps( { workspaceId, - flatEntities: ['flatViewFieldMaps'], + flatMapsKeys: ['flatViewFieldMaps'], }, ); @@ -224,7 +224,7 @@ export class ViewFieldV2Service { await this.flatEntityMapsCacheService.getOrRecomputeManyOrAllFlatEntityMaps( { workspaceId, - flatEntities: ['flatViewFieldMaps'], + flatMapsKeys: ['flatViewFieldMaps'], }, ); @@ -248,7 +248,7 @@ export class ViewFieldV2Service { await this.flatEntityMapsCacheService.getOrRecomputeManyOrAllFlatEntityMaps( { workspaceId, - flatEntities: ['flatViewFieldMaps', 'flatViewMaps'], + flatMapsKeys: ['flatViewFieldMaps', 'flatViewMaps'], }, ); diff --git a/packages/twenty-server/src/engine/core-modules/view/services/view-v2.service.ts b/packages/twenty-server/src/engine/core-modules/view/services/view-v2.service.ts index 87350db898..ae23a2e7b8 100644 --- a/packages/twenty-server/src/engine/core-modules/view/services/view-v2.service.ts +++ b/packages/twenty-server/src/engine/core-modules/view/services/view-v2.service.ts @@ -38,7 +38,7 @@ export class ViewV2Service { await this.flatEntityMapsCacheService.getOrRecomputeManyOrAllFlatEntityMaps( { workspaceId, - flatEntities: ['flatObjectMetadataMaps', 'flatViewMaps'], + flatMapsKeys: ['flatObjectMetadataMaps', 'flatViewMaps'], }, ); @@ -83,7 +83,7 @@ export class ViewV2Service { await this.flatEntityMapsCacheService.getOrRecomputeManyOrAllFlatEntityMaps( { workspaceId, - flatEntities: ['flatViewMaps'], + flatMapsKeys: ['flatViewMaps'], }, ); @@ -104,7 +104,7 @@ export class ViewV2Service { await this.flatEntityMapsCacheService.getOrRecomputeManyOrAllFlatEntityMaps( { workspaceId, - flatEntities: ['flatViewMaps'], + flatMapsKeys: ['flatViewMaps'], }, ); @@ -151,7 +151,7 @@ export class ViewV2Service { await this.flatEntityMapsCacheService.getOrRecomputeManyOrAllFlatEntityMaps( { workspaceId, - flatEntities: ['flatViewMaps'], + flatMapsKeys: ['flatViewMaps'], }, ); @@ -172,7 +172,7 @@ export class ViewV2Service { await this.flatEntityMapsCacheService.getOrRecomputeManyOrAllFlatEntityMaps( { workspaceId, - flatEntities: ['flatViewMaps'], + flatMapsKeys: ['flatViewMaps'], }, ); @@ -218,7 +218,7 @@ export class ViewV2Service { await this.flatEntityMapsCacheService.getOrRecomputeManyOrAllFlatEntityMaps( { workspaceId, - flatEntities: ['flatViewMaps'], + flatMapsKeys: ['flatViewMaps'], }, ); @@ -239,7 +239,7 @@ export class ViewV2Service { await this.flatEntityMapsCacheService.getOrRecomputeManyOrAllFlatEntityMaps( { workspaceId, - flatEntities: ['flatViewMaps'], + flatMapsKeys: ['flatViewMaps'], }, ); diff --git a/packages/twenty-server/src/engine/core-modules/workspace/__tests__/workspace.service.spec.ts b/packages/twenty-server/src/engine/core-modules/workspace/__tests__/workspace.service.spec.ts index bbe9aea1fe..edda22f5fc 100644 --- a/packages/twenty-server/src/engine/core-modules/workspace/__tests__/workspace.service.spec.ts +++ b/packages/twenty-server/src/engine/core-modules/workspace/__tests__/workspace.service.spec.ts @@ -6,6 +6,8 @@ import { type Repository } from 'typeorm'; import { AuditService } from 'src/engine/core-modules/audit/services/audit.service'; import { BillingSubscriptionService } from 'src/engine/core-modules/billing/services/billing-subscription.service'; import { BillingService } from 'src/engine/core-modules/billing/services/billing.service'; +import { WorkspaceManyOrAllFlatEntityMapsCacheService } from 'src/engine/core-modules/common/services/workspace-many-or-all-flat-entity-maps-cache.service'; +import { DnsManagerService } from 'src/engine/core-modules/dns-manager/services/dns-manager.service'; import { DomainManagerService } from 'src/engine/core-modules/domain-manager/services/domain-manager.service'; import { EmailService } from 'src/engine/core-modules/email/email.service'; import { ExceptionHandlerService } from 'src/engine/core-modules/exception-handler/exception-handler.service'; @@ -14,6 +16,7 @@ import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queu import { type MessageQueueService } from 'src/engine/core-modules/message-queue/services/message-queue.service'; import { getQueueToken } from 'src/engine/core-modules/message-queue/utils/get-queue-token.util'; import { OnboardingService } from 'src/engine/core-modules/onboarding/onboarding.service'; +import { PublicDomain } from 'src/engine/core-modules/public-domain/public-domain.entity'; import { TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service'; import { UserWorkspace } from 'src/engine/core-modules/user-workspace/user-workspace.entity'; import { UserWorkspaceService } from 'src/engine/core-modules/user-workspace/user-workspace.service'; @@ -25,8 +28,6 @@ import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity'; import { PermissionsService } from 'src/engine/metadata-modules/permissions/permissions.service'; import { WorkspaceCacheStorageService } from 'src/engine/workspace-cache-storage/workspace-cache-storage.service'; import { WorkspaceManagerService } from 'src/engine/workspace-manager/workspace-manager.service'; -import { DnsManagerService } from 'src/engine/core-modules/dns-manager/services/dns-manager.service'; -import { PublicDomain } from 'src/engine/core-modules/public-domain/public-domain.entity'; describe('WorkspaceService', () => { let service: WorkspaceService; @@ -112,6 +113,12 @@ describe('WorkspaceService', () => { flush: jest.fn(), }, }, + { + provide: WorkspaceManyOrAllFlatEntityMapsCacheService, + useValue: { + flushFlatEntityMaps: jest.fn(), + }, + }, { provide: getQueueToken(MessageQueue.deleteCascadeQueue), useValue: { diff --git a/packages/twenty-server/src/engine/core-modules/workspace/services/workspace.service.ts b/packages/twenty-server/src/engine/core-modules/workspace/services/workspace.service.ts index e142f6d84b..6397a0be17 100644 --- a/packages/twenty-server/src/engine/core-modules/workspace/services/workspace.service.ts +++ b/packages/twenty-server/src/engine/core-modules/workspace/services/workspace.service.ts @@ -3,15 +3,20 @@ import { InjectRepository } from '@nestjs/typeorm'; import assert from 'assert'; +import { t } from '@lingui/core/macro'; import { TypeOrmQueryService } from '@ptc-org/nestjs-query-typeorm'; import { assertIsDefinedOrThrow, isDefined } from 'twenty-shared/utils'; import { WorkspaceActivationStatus } from 'twenty-shared/workspace'; import { Repository } from 'typeorm'; -import { t } from '@lingui/core/macro'; +import { AuditService } from 'src/engine/core-modules/audit/services/audit.service'; +import { CUSTOM_DOMAIN_ACTIVATED_EVENT } from 'src/engine/core-modules/audit/utils/events/workspace-event/custom-domain/custom-domain-activated'; +import { CUSTOM_DOMAIN_DEACTIVATED_EVENT } from 'src/engine/core-modules/audit/utils/events/workspace-event/custom-domain/custom-domain-deactivated'; import { BillingEntitlementKey } from 'src/engine/core-modules/billing/enums/billing-entitlement-key.enum'; import { BillingSubscriptionService } from 'src/engine/core-modules/billing/services/billing-subscription.service'; import { BillingService } from 'src/engine/core-modules/billing/services/billing.service'; +import { WorkspaceManyOrAllFlatEntityMapsCacheService } from 'src/engine/core-modules/common/services/workspace-many-or-all-flat-entity-maps-cache.service'; +import { DnsManagerService } from 'src/engine/core-modules/dns-manager/services/dns-manager.service'; import { ExceptionHandlerService } from 'src/engine/core-modules/exception-handler/exception-handler.service'; import { FeatureFlagService } from 'src/engine/core-modules/feature-flag/services/feature-flag.service'; import { @@ -21,6 +26,7 @@ import { import { InjectMessageQueue } from 'src/engine/core-modules/message-queue/decorators/message-queue.decorator'; import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants'; import { MessageQueueService } from 'src/engine/core-modules/message-queue/services/message-queue.service'; +import { PublicDomain } from 'src/engine/core-modules/public-domain/public-domain.entity'; import { TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service'; import { UserWorkspace } from 'src/engine/core-modules/user-workspace/user-workspace.entity'; import { UserWorkspaceService } from 'src/engine/core-modules/user-workspace/user-workspace.service'; @@ -43,11 +49,6 @@ import { WorkspaceCacheStorageService } from 'src/engine/workspace-cache-storage import { WorkspaceManagerService } from 'src/engine/workspace-manager/workspace-manager.service'; import { DEFAULT_FEATURE_FLAGS } from 'src/engine/workspace-manager/workspace-sync-metadata/constants/default-feature-flags'; import { extractVersionMajorMinorPatch } from 'src/utils/version/extract-version-major-minor-patch'; -import { DnsManagerService } from 'src/engine/core-modules/dns-manager/services/dns-manager.service'; -import { CUSTOM_DOMAIN_ACTIVATED_EVENT } from 'src/engine/core-modules/audit/utils/events/workspace-event/custom-domain/custom-domain-activated'; -import { CUSTOM_DOMAIN_DEACTIVATED_EVENT } from 'src/engine/core-modules/audit/utils/events/workspace-event/custom-domain/custom-domain-deactivated'; -import { AuditService } from 'src/engine/core-modules/audit/services/audit.service'; -import { PublicDomain } from 'src/engine/core-modules/public-domain/public-domain.entity'; @Injectable() // eslint-disable-next-line @nx/workspace-inject-workspace-repository @@ -73,6 +74,7 @@ export class WorkspaceService extends TypeOrmQueryService { private readonly exceptionHandlerService: ExceptionHandlerService, private readonly permissionsService: PermissionsService, private readonly dnsManagerService: DnsManagerService, + private readonly flatEntityMapsCacheService: WorkspaceManyOrAllFlatEntityMapsCacheService, private readonly workspaceCacheStorageService: WorkspaceCacheStorageService, private readonly auditService: AuditService, @InjectMessageQueue(MessageQueue.deleteCascadeQueue) @@ -341,6 +343,9 @@ export class WorkspaceService extends TypeOrmQueryService { workspace.id, workspace.metadataVersion, ); + await this.flatEntityMapsCacheService.flushFlatEntityMaps({ + workspaceId: workspace.id, + }); this.logger.log(`workspace ${id} cache flushed`); if (softDelete) { diff --git a/packages/twenty-server/src/engine/core-modules/workspace/workspace.module.ts b/packages/twenty-server/src/engine/core-modules/workspace/workspace.module.ts index d46662dead..40b9d8e042 100644 --- a/packages/twenty-server/src/engine/core-modules/workspace/workspace.module.ts +++ b/packages/twenty-server/src/engine/core-modules/workspace/workspace.module.ts @@ -5,18 +5,28 @@ import { NestjsQueryGraphQLModule } from '@ptc-org/nestjs-query-graphql'; import { NestjsQueryTypeOrmModule } from '@ptc-org/nestjs-query-typeorm'; import { TypeORMModule } from 'src/database/typeorm/typeorm.module'; +import { AuditModule } from 'src/engine/core-modules/audit/audit.module'; import { TokenModule } from 'src/engine/core-modules/auth/token/token.module'; import { BillingModule } from 'src/engine/core-modules/billing/billing.module'; import { BillingSubscription } from 'src/engine/core-modules/billing/entities/billing-subscription.entity'; +import { WorkspaceManyOrAllFlatEntityMapsCacheModule } from 'src/engine/core-modules/common/services/workspace-many-or-all-flat-entity-maps-cache.module'; +import { DnsManagerModule } from 'src/engine/core-modules/dns-manager/dns-manager.module'; +import { DomainManagerModule } from 'src/engine/core-modules/domain-manager/domain-manager.module'; import { FeatureFlagModule } from 'src/engine/core-modules/feature-flag/feature-flag.module'; import { FileUploadModule } from 'src/engine/core-modules/file/file-upload/file-upload.module'; import { FileModule } from 'src/engine/core-modules/file/file.module'; import { OnboardingModule } from 'src/engine/core-modules/onboarding/onboarding.module'; +import { PublicDomain } from 'src/engine/core-modules/public-domain/public-domain.entity'; import { UserWorkspace } from 'src/engine/core-modules/user-workspace/user-workspace.entity'; import { UserWorkspaceModule } from 'src/engine/core-modules/user-workspace/user-workspace.module'; import { User } from 'src/engine/core-modules/user/user.entity'; import { CoreViewModule } from 'src/engine/core-modules/view/view.module'; +import { CheckCustomDomainValidRecordsCronCommand } from 'src/engine/core-modules/workspace/crons/commands/check-custom-domain-valid-records.cron.command'; +import { CheckCustomDomainValidRecordsCronJob } from 'src/engine/core-modules/workspace/crons/jobs/check-custom-domain-valid-records.cron.job'; +import { WorkspaceService } from 'src/engine/core-modules/workspace/services/workspace.service'; import { WorkspaceWorkspaceMemberListener } from 'src/engine/core-modules/workspace/workspace-workspace-member.listener'; +import { workspaceAutoResolverOpts } from 'src/engine/core-modules/workspace/workspace.auto-resolver-opts'; +import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity'; import { WorkspaceResolver } from 'src/engine/core-modules/workspace/workspace.resolver'; import { AgentModule } from 'src/engine/metadata-modules/agent/agent.module'; import { DataSourceModule } from 'src/engine/metadata-modules/data-source/data-source.module'; @@ -25,15 +35,6 @@ import { RoleModule } from 'src/engine/metadata-modules/role/role.module'; import { WorkspaceMetadataCacheModule } from 'src/engine/metadata-modules/workspace-metadata-cache/workspace-metadata-cache.module'; import { WorkspaceCacheStorageModule } from 'src/engine/workspace-cache-storage/workspace-cache-storage.module'; import { WorkspaceManagerModule } from 'src/engine/workspace-manager/workspace-manager.module'; -import { DnsManagerModule } from 'src/engine/core-modules/dns-manager/dns-manager.module'; -import { DomainManagerModule } from 'src/engine/core-modules/domain-manager/domain-manager.module'; -import { CheckCustomDomainValidRecordsCronJob } from 'src/engine/core-modules/workspace/crons/jobs/check-custom-domain-valid-records.cron.job'; -import { CheckCustomDomainValidRecordsCronCommand } from 'src/engine/core-modules/workspace/crons/commands/check-custom-domain-valid-records.cron.command'; -import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity'; -import { workspaceAutoResolverOpts } from 'src/engine/core-modules/workspace/workspace.auto-resolver-opts'; -import { WorkspaceService } from 'src/engine/core-modules/workspace/services/workspace.service'; -import { AuditModule } from 'src/engine/core-modules/audit/audit.module'; -import { PublicDomain } from 'src/engine/core-modules/public-domain/public-domain.entity'; @Module({ imports: [ @@ -66,6 +67,7 @@ import { PublicDomain } from 'src/engine/core-modules/public-domain/public-domai DnsManagerModule, DomainManagerModule, CoreViewModule, + WorkspaceManyOrAllFlatEntityMapsCacheModule, ], services: [WorkspaceService], resolvers: workspaceAutoResolverOpts, diff --git a/packages/twenty-server/src/engine/dataloaders/dataloader.service.ts b/packages/twenty-server/src/engine/dataloaders/dataloader.service.ts index 2a37718a63..9be6698ff5 100644 --- a/packages/twenty-server/src/engine/dataloaders/dataloader.service.ts +++ b/packages/twenty-server/src/engine/dataloaders/dataloader.service.ts @@ -103,7 +103,7 @@ export class DataloaderService { await this.flatEntityMapsCacheService.getOrRecomputeManyOrAllFlatEntityMaps( { workspaceId, - flatEntities: ['flatFieldMetadataMaps', 'flatObjectMetadataMaps'], + flatMapsKeys: ['flatFieldMetadataMaps', 'flatObjectMetadataMaps'], }, ); @@ -195,7 +195,7 @@ export class DataloaderService { await this.flatEntityMapsCacheService.getOrRecomputeManyOrAllFlatEntityMaps( { workspaceId, - flatEntities: ['flatFieldMetadataMaps', 'flatObjectMetadataMaps'], + flatMapsKeys: ['flatFieldMetadataMaps', 'flatObjectMetadataMaps'], }, ); const relationDtos: Array = []; @@ -327,7 +327,7 @@ export class DataloaderService { await this.flatEntityMapsCacheService.getOrRecomputeManyOrAllFlatEntityMaps( { workspaceId, - flatEntities: ['flatFieldMetadataMaps', 'flatObjectMetadataMaps'], + flatMapsKeys: ['flatFieldMetadataMaps', 'flatObjectMetadataMaps'], }, ); diff --git a/packages/twenty-server/src/engine/metadata-modules/cron-trigger/services/cron-trigger-v2.service.ts b/packages/twenty-server/src/engine/metadata-modules/cron-trigger/services/cron-trigger-v2.service.ts index 4a2b772968..af3dbe7d63 100644 --- a/packages/twenty-server/src/engine/metadata-modules/cron-trigger/services/cron-trigger-v2.service.ts +++ b/packages/twenty-server/src/engine/metadata-modules/cron-trigger/services/cron-trigger-v2.service.ts @@ -36,7 +36,7 @@ export class CronTriggerV2Service { await this.flatEntityMapsCacheService.getOrRecomputeManyOrAllFlatEntityMaps( { workspaceId, - flatEntities: ['flatCronTriggerMaps', 'flatServerlessFunctionMaps'], + flatMapsKeys: ['flatCronTriggerMaps', 'flatServerlessFunctionMaps'], }, ); @@ -86,7 +86,7 @@ export class CronTriggerV2Service { await this.flatEntityMapsCacheService.getOrRecomputeManyOrAllFlatEntityMaps( { workspaceId, - flatEntities: ['flatCronTriggerMaps', 'flatServerlessFunctionMaps'], + flatMapsKeys: ['flatCronTriggerMaps', 'flatServerlessFunctionMaps'], }, ); @@ -104,7 +104,7 @@ export class CronTriggerV2Service { await this.flatEntityMapsCacheService.getOrRecomputeManyOrAllFlatEntityMaps( { workspaceId, - flatEntities: ['flatCronTriggerMaps', 'flatServerlessFunctionMaps'], + flatMapsKeys: ['flatCronTriggerMaps', 'flatServerlessFunctionMaps'], }, ); @@ -157,7 +157,7 @@ export class CronTriggerV2Service { await this.flatEntityMapsCacheService.getOrRecomputeManyOrAllFlatEntityMaps( { workspaceId, - flatEntities: ['flatCronTriggerMaps'], + flatMapsKeys: ['flatCronTriggerMaps'], }, ); @@ -181,7 +181,7 @@ export class CronTriggerV2Service { await this.flatEntityMapsCacheService.getOrRecomputeManyOrAllFlatEntityMaps( { workspaceId, - flatEntities: ['flatCronTriggerMaps', 'flatServerlessFunctionMaps'], + flatMapsKeys: ['flatCronTriggerMaps', 'flatServerlessFunctionMaps'], }, ); diff --git a/packages/twenty-server/src/engine/metadata-modules/database-event-trigger/services/database-event-trigger-v2.service.ts b/packages/twenty-server/src/engine/metadata-modules/database-event-trigger/services/database-event-trigger-v2.service.ts index 0bf295062a..3e9827ad4c 100644 --- a/packages/twenty-server/src/engine/metadata-modules/database-event-trigger/services/database-event-trigger-v2.service.ts +++ b/packages/twenty-server/src/engine/metadata-modules/database-event-trigger/services/database-event-trigger-v2.service.ts @@ -36,7 +36,7 @@ export class DatabaseEventTriggerV2Service { await this.flatEntityMapsCacheService.getOrRecomputeManyOrAllFlatEntityMaps( { workspaceId, - flatEntities: [ + flatMapsKeys: [ 'flatDatabaseEventTriggerMaps', 'flatServerlessFunctionMaps', ], @@ -94,7 +94,7 @@ export class DatabaseEventTriggerV2Service { await this.flatEntityMapsCacheService.getOrRecomputeManyOrAllFlatEntityMaps( { workspaceId, - flatEntities: ['flatDatabaseEventTriggerMaps'], + flatMapsKeys: ['flatDatabaseEventTriggerMaps'], }, ); @@ -112,7 +112,7 @@ export class DatabaseEventTriggerV2Service { await this.flatEntityMapsCacheService.getOrRecomputeManyOrAllFlatEntityMaps( { workspaceId, - flatEntities: [ + flatMapsKeys: [ 'flatDatabaseEventTriggerMaps', 'flatServerlessFunctionMaps', ], @@ -175,7 +175,7 @@ export class DatabaseEventTriggerV2Service { await this.flatEntityMapsCacheService.getOrRecomputeManyOrAllFlatEntityMaps( { workspaceId, - flatEntities: ['flatDatabaseEventTriggerMaps'], + flatMapsKeys: ['flatDatabaseEventTriggerMaps'], }, ); @@ -199,7 +199,7 @@ export class DatabaseEventTriggerV2Service { await this.flatEntityMapsCacheService.getOrRecomputeManyOrAllFlatEntityMaps( { workspaceId, - flatEntities: [ + flatMapsKeys: [ 'flatDatabaseEventTriggerMaps', 'flatServerlessFunctionMaps', ], diff --git a/packages/twenty-server/src/engine/metadata-modules/field-metadata/services/field-metadata.service-v2.ts b/packages/twenty-server/src/engine/metadata-modules/field-metadata/services/field-metadata.service-v2.ts index cb1991b3a2..d7bf6144b2 100644 --- a/packages/twenty-server/src/engine/metadata-modules/field-metadata/services/field-metadata.service-v2.ts +++ b/packages/twenty-server/src/engine/metadata-modules/field-metadata/services/field-metadata.service-v2.ts @@ -72,7 +72,7 @@ export class FieldMetadataServiceV2 { } = await this.flatEntityMapsCacheService.getOrRecomputeManyOrAllFlatEntityMaps( { workspaceId, - flatEntities: [ + flatMapsKeys: [ 'flatObjectMetadataMaps', 'flatIndexMaps', 'flatFieldMetadataMaps', @@ -180,7 +180,7 @@ export class FieldMetadataServiceV2 { } = await this.flatEntityMapsCacheService.getOrRecomputeManyOrAllFlatEntityMaps( { workspaceId, - flatEntities: [ + flatMapsKeys: [ 'flatObjectMetadataMaps', 'flatIndexMaps', 'flatFieldMetadataMaps', @@ -288,7 +288,7 @@ export class FieldMetadataServiceV2 { } = await this.flatEntityMapsCacheService.getOrRecomputeManyOrAllFlatEntityMaps( { workspaceId, - flatEntities: [ + flatMapsKeys: [ 'flatObjectMetadataMaps', 'flatIndexMaps', 'flatFieldMetadataMaps', diff --git a/packages/twenty-server/src/engine/metadata-modules/field-metadata/services/field-metadata.service.ts b/packages/twenty-server/src/engine/metadata-modules/field-metadata/services/field-metadata.service.ts index a0b433af9a..398f7d93f6 100644 --- a/packages/twenty-server/src/engine/metadata-modules/field-metadata/services/field-metadata.service.ts +++ b/packages/twenty-server/src/engine/metadata-modules/field-metadata/services/field-metadata.service.ts @@ -402,7 +402,7 @@ export class FieldMetadataService extends TypeOrmQueryService> - >(); + private readonly cacheServiceRecord: { + [P in keyof AllFlatEntityMaps]?: WorkspaceFlatMapCacheService< + AllFlatEntityMaps[P] + >; + } = {}; constructor(private readonly discoveryService: DiscoveryService) {} @@ -30,20 +35,31 @@ export class WorkspaceFlatMapCacheRegistryService implements OnModuleInit { return; } - const cacheKey = Reflect.getMetadata( + const cacheKey: keyof AllFlatEntityMaps | undefined = Reflect.getMetadata( WORKSPACE_FLAT_MAP_CACHE_KEY, metatype, ); if (cacheKey && instance instanceof WorkspaceFlatMapCacheService) { - this.cacheServiceMap.set(cacheKey, instance); + this.cacheServiceRecord[cacheKey] = instance; } }); } - getCacheService( - flatEntityName: keyof AllFlatEntityMaps, - ): WorkspaceFlatMapCacheService> | undefined { - return this.cacheServiceMap.get(flatEntityName); + getCacheService(flatMapsKey: K) { + return this.cacheServiceRecord[flatMapsKey]; + } + + getCacheServiceOrThrow(flatMapsKey: K) { + const service = this.getCacheService(flatMapsKey); + + if (!isDefined(service)) { + throw new WorkspaceFlatMapCacheException( + `No cache service found for ${flatMapsKey}`, + WorkspaceFlatMapCacheExceptionCode.INTERNAL_SERVER_ERROR, + ); + } + + return service; } } diff --git a/packages/twenty-server/src/engine/workspace-flat-map-cache/services/workspace-flat-map-cache.service.ts b/packages/twenty-server/src/engine/workspace-flat-map-cache/services/workspace-flat-map-cache.service.ts index 7db297e773..6383f3e7b8 100644 --- a/packages/twenty-server/src/engine/workspace-flat-map-cache/services/workspace-flat-map-cache.service.ts +++ b/packages/twenty-server/src/engine/workspace-flat-map-cache/services/workspace-flat-map-cache.service.ts @@ -99,14 +99,9 @@ export abstract class WorkspaceFlatMapCacheService< }: { workspaceId: string; }): Promise { - const { flatMapKey, hashKey } = this.buildRemoteCacheKeys({ workspaceId }); - - await this.cacheStorageService.del(flatMapKey); - await this.cacheStorageService.del(hashKey); - - this.localCacheFlatMaps.delete(workspaceId); - this.localCacheHashes.delete(workspaceId); - + await this.flushCache({ + workspaceId, + }); await this.recomputeAndStoreInCache({ workspaceId }); } @@ -185,4 +180,14 @@ export abstract class WorkspaceFlatMapCacheService< await this.cacheStorageService.set(flatMapKey, flatMap); } + + async flushCache({ workspaceId }: { workspaceId: string }): Promise { + const { flatMapKey, hashKey } = this.buildRemoteCacheKeys({ workspaceId }); + + await this.cacheStorageService.del(flatMapKey); + await this.cacheStorageService.del(hashKey); + + this.localCacheFlatMaps.delete(workspaceId); + this.localCacheHashes.delete(workspaceId); + } } diff --git a/packages/twenty-server/src/engine/workspace-manager/dev-seeder/dev-seeder.module.ts b/packages/twenty-server/src/engine/workspace-manager/dev-seeder/dev-seeder.module.ts index 1b2d335d9b..b336d54587 100644 --- a/packages/twenty-server/src/engine/workspace-manager/dev-seeder/dev-seeder.module.ts +++ b/packages/twenty-server/src/engine/workspace-manager/dev-seeder/dev-seeder.module.ts @@ -3,6 +3,7 @@ import { TypeOrmModule } from '@nestjs/typeorm'; import { TypeORMModule } from 'src/database/typeorm/typeorm.module'; import { ApiKeyModule } from 'src/engine/core-modules/api-key/api-key.module'; +import { WorkspaceManyOrAllFlatEntityMapsCacheModule } from 'src/engine/core-modules/common/services/workspace-many-or-all-flat-entity-maps-cache.module'; import { FeatureFlagModule } from 'src/engine/core-modules/feature-flag/feature-flag.module'; import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity'; import { DataSourceModule } from 'src/engine/metadata-modules/data-source/data-source.module'; @@ -39,6 +40,7 @@ import { WorkspaceSyncMetadataModule } from 'src/engine/workspace-manager/worksp TypeOrmModule.forFeature([Workspace, ObjectMetadataEntity]), ObjectPermissionModule, WorkspacePermissionsCacheModule, + WorkspaceManyOrAllFlatEntityMapsCacheModule, WorkspaceMetadataCacheModule, ], exports: [DevSeederService], diff --git a/packages/twenty-server/src/engine/workspace-manager/dev-seeder/services/dev-seeder.service.ts b/packages/twenty-server/src/engine/workspace-manager/dev-seeder/services/dev-seeder.service.ts index b59780427d..4e74301204 100644 --- a/packages/twenty-server/src/engine/workspace-manager/dev-seeder/services/dev-seeder.service.ts +++ b/packages/twenty-server/src/engine/workspace-manager/dev-seeder/services/dev-seeder.service.ts @@ -3,6 +3,7 @@ import { InjectDataSource } from '@nestjs/typeorm'; import { DataSource } from 'typeorm'; +import { WorkspaceManyOrAllFlatEntityMapsCacheService } from 'src/engine/core-modules/common/services/workspace-many-or-all-flat-entity-maps-cache.service'; import { FeatureFlagService } from 'src/engine/core-modules/feature-flag/services/feature-flag.service'; import { TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service'; import { DataSourceService } from 'src/engine/metadata-modules/data-source/data-source.service'; @@ -29,6 +30,7 @@ export class DevSeederService { private readonly workspaceSyncMetadataService: WorkspaceSyncMetadataService, private readonly devSeederMetadataService: DevSeederMetadataService, private readonly devSeederPermissionsService: DevSeederPermissionsService, + private readonly flatEntityMapsCacheService: WorkspaceManyOrAllFlatEntityMapsCacheService, private readonly devSeederDataService: DevSeederDataService, @InjectDataSource() private readonly coreDataSource: DataSource, @@ -101,5 +103,8 @@ export class DevSeederService { }); await this.workspaceCacheStorageService.flush(workspaceId, undefined); + await this.flatEntityMapsCacheService.flushFlatEntityMaps({ + workspaceId, + }); } } diff --git a/packages/twenty-server/src/engine/workspace-manager/workspace-migration-v2/workspace-migration-runner-v2/interfaces/workspace-migration-runner-action-handler-service.interface.ts b/packages/twenty-server/src/engine/workspace-manager/workspace-migration-v2/workspace-migration-runner-v2/interfaces/workspace-migration-runner-action-handler-service.interface.ts index d04ba33a92..42a05ff1ad 100644 --- a/packages/twenty-server/src/engine/workspace-manager/workspace-migration-v2/workspace-migration-runner-v2/interfaces/workspace-migration-runner-action-handler-service.interface.ts +++ b/packages/twenty-server/src/engine/workspace-manager/workspace-migration-v2/workspace-migration-runner-v2/interfaces/workspace-migration-runner-action-handler-service.interface.ts @@ -1,7 +1,7 @@ import { Inject, SetMetadata } from '@nestjs/common'; -import { LoggerService } from 'src/engine/core-modules/logger/logger.service'; import { type AllFlatEntityMaps } from 'src/engine/core-modules/common/types/all-flat-entity-maps.type'; +import { LoggerService } from 'src/engine/core-modules/logger/logger.service'; import { type ExtractAction, type WorkspaceMigrationActionTypeV2, diff --git a/packages/twenty-server/src/engine/workspace-manager/workspace-migration-v2/workspace-migration-runner-v2/services/workspace-migration-runner-v2.service.ts b/packages/twenty-server/src/engine/workspace-manager/workspace-migration-v2/workspace-migration-runner-v2/services/workspace-migration-runner-v2.service.ts index 4b2ec31adf..eec43356b0 100644 --- a/packages/twenty-server/src/engine/workspace-manager/workspace-migration-v2/workspace-migration-runner-v2/services/workspace-migration-runner-v2.service.ts +++ b/packages/twenty-server/src/engine/workspace-manager/workspace-migration-v2/workspace-migration-runner-v2/services/workspace-migration-runner-v2.service.ts @@ -3,13 +3,13 @@ import { InjectDataSource } from '@nestjs/typeorm'; import { DataSource } from 'typeorm'; -import { LoggerService } from 'src/engine/core-modules/logger/logger.service'; import { WorkspaceQueryRunnerException, WorkspaceQueryRunnerExceptionCode, } from 'src/engine/api/graphql/workspace-query-runner/workspace-query-runner.exception'; import { WorkspaceManyOrAllFlatEntityMapsCacheService } from 'src/engine/core-modules/common/services/workspace-many-or-all-flat-entity-maps-cache.service'; import { AllFlatEntityMaps } from 'src/engine/core-modules/common/types/all-flat-entity-maps.type'; +import { LoggerService } from 'src/engine/core-modules/logger/logger.service'; import { WorkspaceMetadataVersionService } from 'src/engine/metadata-modules/workspace-metadata-version/services/workspace-metadata-version.service'; import { WorkspacePermissionsCacheService } from 'src/engine/metadata-modules/workspace-permissions-cache/workspace-permissions-cache.service'; import { WorkspaceMigrationV2 } from 'src/engine/workspace-manager/workspace-migration-v2/workspace-migration-builder-v2/types/workspace-migration-v2'; @@ -41,7 +41,7 @@ export class WorkspaceMigrationRunnerV2Service { await this.flatEntityMapsCacheService.getOrRecomputeManyOrAllFlatEntityMaps( { workspaceId, - flatEntities: relatedFlatEntityMapsKeys, + flatMapsKeys: relatedFlatEntityMapsKeys, }, ); @@ -111,7 +111,7 @@ export class WorkspaceMigrationRunnerV2Service { await this.flatEntityMapsCacheService.invalidateFlatEntityMaps({ workspaceId, - flatEntities: [ + flatMapsKeys: [ ...new Set([ ...flatEntityMapsToInvalidate, ...(relatedFlatEntityMapsKeys ?? []), diff --git a/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/workspace-sync-metadata.service.ts b/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/workspace-sync-metadata.service.ts index 6c2447a520..175ef1303b 100644 --- a/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/workspace-sync-metadata.service.ts +++ b/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/workspace-sync-metadata.service.ts @@ -264,7 +264,7 @@ export class WorkspaceSyncMetadataService { ); await this.flatEntityMapsCacheService.invalidateFlatEntityMaps({ workspaceId: context.workspaceId, - flatEntities: ['flatObjectMetadataMaps', 'flatFieldMetadataMaps'], + flatMapsKeys: ['flatObjectMetadataMaps', 'flatFieldMetadataMaps'], }); }