Cache flush after database reset (#14873)

When starting the app on a fresh database reset the cache would be
filled with empty flat field metadata maps
Because the reset command hack through the repository directly in order
to create views and stuff
implemented an iso flush as the one existing initially
added it to a workspace deletion tambien
```
{
   byId: {],
   universalIdById: {}
}
```
This commit is contained in:
Paul Rastoin
2025-10-03 17:51:24 +02:00
committed by GitHub
parent 81e9a02101
commit d3a7241b6f
23 changed files with 210 additions and 162 deletions
@@ -190,7 +190,7 @@ export class ApplicationSyncService {
await this.flatEntityMapsCacheService.getOrRecomputeManyOrAllFlatEntityMaps(
{
workspaceId,
flatEntities: ['flatObjectMetadataMaps'],
flatMapsKeys: ['flatObjectMetadataMaps'],
},
);
@@ -303,7 +303,7 @@ export class ApplicationSyncService {
await this.flatEntityMapsCacheService.getOrRecomputeManyOrAllFlatEntityMaps(
{
workspaceId,
flatEntities: ['flatServerlessFunctionMaps'],
flatMapsKeys: ['flatServerlessFunctionMaps'],
},
);
@@ -456,7 +456,7 @@ export class ApplicationSyncService {
await this.flatEntityMapsCacheService.getOrRecomputeManyOrAllFlatEntityMaps(
{
workspaceId,
flatEntities: ['flatDatabaseEventTriggerMaps'],
flatMapsKeys: ['flatDatabaseEventTriggerMaps'],
},
);
@@ -572,7 +572,7 @@ export class ApplicationSyncService {
await this.flatEntityMapsCacheService.getOrRecomputeManyOrAllFlatEntityMaps(
{
workspaceId,
flatEntities: ['flatCronTriggerMaps'],
flatMapsKeys: ['flatCronTriggerMaps'],
},
);
@@ -685,7 +685,7 @@ export class ApplicationSyncService {
await this.flatEntityMapsCacheService.getOrRecomputeManyOrAllFlatEntityMaps(
{
workspaceId,
flatEntities: ['flatRouteTriggerMaps'],
flatMapsKeys: ['flatRouteTriggerMaps'],
},
);
@@ -3,13 +3,9 @@ import { Injectable, Logger } from '@nestjs/common';
import { isDefined } from 'twenty-shared/utils';
import { ALL_FLAT_ENTITY_MAPS_PROPERTIES } from 'src/engine/core-modules/common/constant/all-flat-entity-maps-properties.constant';
import { EMPTY_ALL_FLAT_ENTITY_MAPS } from 'src/engine/core-modules/common/constant/empty-all-flat-entity-maps.constant';
import { AllFlatEntityMaps } from 'src/engine/core-modules/common/types/all-flat-entity-maps.type';
import {
WorkspaceFlatMapCacheException,
WorkspaceFlatMapCacheExceptionCode,
} from 'src/engine/workspace-flat-map-cache/exceptions/workspace-flat-map-cache.exception';
import { WorkspaceFlatMapCacheRegistryService } from 'src/engine/workspace-flat-map-cache/services/workspace-flat-map-cache-registry.service';
import { WorkspaceFlatMapCacheService } from 'src/engine/workspace-flat-map-cache/services/workspace-flat-map-cache.service';
@Injectable()
export class WorkspaceManyOrAllFlatEntityMapsCacheService {
@@ -21,85 +17,95 @@ export class WorkspaceManyOrAllFlatEntityMapsCacheService {
private readonly cacheRegistry: WorkspaceFlatMapCacheRegistryService,
) {}
public async getOrRecomputeManyOrAllFlatEntityMaps<
T extends (keyof AllFlatEntityMaps)[] = (keyof AllFlatEntityMaps)[],
private async executeActionForManyOrAllFlatEntity<
K extends (keyof AllFlatEntityMaps)[] = (keyof AllFlatEntityMaps)[],
>({
flatEntities,
workspaceId,
action,
flatMapsKeys,
}: {
workspaceId: string;
flatEntities?: T;
}): Promise<Pick<AllFlatEntityMaps, T[number]>> {
const allFlatEntityMaps: AllFlatEntityMaps = structuredClone(
EMPTY_ALL_FLAT_ENTITY_MAPS,
);
for (const flatEntityName of ALL_FLAT_ENTITY_MAPS_PROPERTIES) {
if (isDefined(flatEntities) && !flatEntities.includes(flatEntityName)) {
delete allFlatEntityMaps[flatEntityName];
continue;
}
flatMapsKeys: K | undefined;
action: (args: {
service: WorkspaceFlatMapCacheService<AllFlatEntityMaps[K[number]]>;
flatMapKey: K[number];
}) => Promise<void>;
}): Promise<void> {
const keysToProcess = isDefined(flatMapsKeys)
? flatMapsKeys
: ALL_FLAT_ENTITY_MAPS_PROPERTIES;
for (const flatMapKey of keysToProcess) {
try {
const service = this.cacheRegistry.getCacheService(flatEntityName);
const service = this.cacheRegistry.getCacheServiceOrThrow(
flatMapKey as K[number],
);
if (!isDefined(service)) {
throw new WorkspaceFlatMapCacheException(
`No cache service found for ${flatEntityName}`,
WorkspaceFlatMapCacheExceptionCode.INTERNAL_SERVER_ERROR,
);
}
const result = await service.getExistingOrRecomputeFlatMaps({
workspaceId,
await action({
flatMapKey: flatMapKey,
service,
});
// @ts-expect-error todo prastoin once refactored flat object metadata cache
allFlatEntityMaps[flatEntityName] = result;
} catch (error) {
this.logger.error(
`Failed to get flat entity maps for ${flatEntityName}`,
`Failed to run action on flat entity maps of ${flatMapKey}`,
error,
);
throw error;
}
}
}
return allFlatEntityMaps;
public async getOrRecomputeManyOrAllFlatEntityMaps<
T extends (keyof AllFlatEntityMaps)[] = (keyof AllFlatEntityMaps)[],
>({
flatMapsKeys,
workspaceId,
}: {
workspaceId: string;
flatMapsKeys?: T;
}): Promise<Pick<AllFlatEntityMaps, T[number]>> {
let pickedFlatEntityMaps = {} as Pick<AllFlatEntityMaps, T[number]>;
await this.executeActionForManyOrAllFlatEntity({
action: async ({ service, flatMapKey }) => {
const cacheResult = await service.getExistingOrRecomputeFlatMaps({
workspaceId,
});
pickedFlatEntityMaps[flatMapKey] = cacheResult;
},
flatMapsKeys,
});
return pickedFlatEntityMaps;
}
public async invalidateFlatEntityMaps<
T extends (keyof AllFlatEntityMaps)[] = (keyof AllFlatEntityMaps)[],
>({
flatEntities,
flatMapsKeys,
workspaceId,
}: {
workspaceId: string;
flatEntities?: T;
flatMapsKeys?: T;
}): Promise<void> {
for (const flatEntityName of ALL_FLAT_ENTITY_MAPS_PROPERTIES) {
if (isDefined(flatEntities) && !flatEntities.includes(flatEntityName)) {
continue;
}
await this.executeActionForManyOrAllFlatEntity({
action: async ({ service }) =>
await service.invalidateCache({ workspaceId }),
flatMapsKeys,
});
}
try {
const service = this.cacheRegistry.getCacheService(flatEntityName);
if (!isDefined(service)) {
throw new WorkspaceFlatMapCacheException(
`No cache service found for ${flatEntityName}`,
WorkspaceFlatMapCacheExceptionCode.INTERNAL_SERVER_ERROR,
);
}
await service.invalidateCache({ workspaceId });
} catch (error) {
this.logger.error(
`Failed to invalidate flat entity maps for ${flatEntityName}`,
error,
);
throw error;
}
}
public async flushFlatEntityMaps<
T extends (keyof AllFlatEntityMaps)[] = (keyof AllFlatEntityMaps)[],
>({
flatMapsKeys,
workspaceId,
}: {
workspaceId: string;
flatMapsKeys?: T;
}): Promise<void> {
await this.executeActionForManyOrAllFlatEntity({
action: async ({ service }) => await service.flushCache({ workspaceId }),
flatMapsKeys,
});
}
}
@@ -41,7 +41,7 @@ export class ViewFieldV2Service {
} = await this.flatEntityMapsCacheService.getOrRecomputeManyOrAllFlatEntityMaps(
{
workspaceId,
flatEntities: [
flatMapsKeys: [
'flatViewFieldMaps',
'flatViewMaps',
'flatFieldMetadataMaps',
@@ -92,7 +92,7 @@ export class ViewFieldV2Service {
await this.flatEntityMapsCacheService.getOrRecomputeManyOrAllFlatEntityMaps(
{
workspaceId,
flatEntities: ['flatViewFieldMaps'],
flatMapsKeys: ['flatViewFieldMaps'],
},
);
@@ -113,7 +113,7 @@ export class ViewFieldV2Service {
await this.flatEntityMapsCacheService.getOrRecomputeManyOrAllFlatEntityMaps(
{
workspaceId,
flatEntities: ['flatViewFieldMaps'],
flatMapsKeys: ['flatViewFieldMaps'],
},
);
@@ -160,7 +160,7 @@ export class ViewFieldV2Service {
await this.flatEntityMapsCacheService.getOrRecomputeManyOrAllFlatEntityMaps(
{
workspaceId,
flatEntities: ['flatViewFieldMaps'],
flatMapsKeys: ['flatViewFieldMaps'],
},
);
@@ -181,7 +181,7 @@ export class ViewFieldV2Service {
await this.flatEntityMapsCacheService.getOrRecomputeManyOrAllFlatEntityMaps(
{
workspaceId,
flatEntities: ['flatViewFieldMaps'],
flatMapsKeys: ['flatViewFieldMaps'],
},
);
@@ -224,7 +224,7 @@ export class ViewFieldV2Service {
await this.flatEntityMapsCacheService.getOrRecomputeManyOrAllFlatEntityMaps(
{
workspaceId,
flatEntities: ['flatViewFieldMaps'],
flatMapsKeys: ['flatViewFieldMaps'],
},
);
@@ -248,7 +248,7 @@ export class ViewFieldV2Service {
await this.flatEntityMapsCacheService.getOrRecomputeManyOrAllFlatEntityMaps(
{
workspaceId,
flatEntities: ['flatViewFieldMaps', 'flatViewMaps'],
flatMapsKeys: ['flatViewFieldMaps', 'flatViewMaps'],
},
);
@@ -38,7 +38,7 @@ export class ViewV2Service {
await this.flatEntityMapsCacheService.getOrRecomputeManyOrAllFlatEntityMaps(
{
workspaceId,
flatEntities: ['flatObjectMetadataMaps', 'flatViewMaps'],
flatMapsKeys: ['flatObjectMetadataMaps', 'flatViewMaps'],
},
);
@@ -83,7 +83,7 @@ export class ViewV2Service {
await this.flatEntityMapsCacheService.getOrRecomputeManyOrAllFlatEntityMaps(
{
workspaceId,
flatEntities: ['flatViewMaps'],
flatMapsKeys: ['flatViewMaps'],
},
);
@@ -104,7 +104,7 @@ export class ViewV2Service {
await this.flatEntityMapsCacheService.getOrRecomputeManyOrAllFlatEntityMaps(
{
workspaceId,
flatEntities: ['flatViewMaps'],
flatMapsKeys: ['flatViewMaps'],
},
);
@@ -151,7 +151,7 @@ export class ViewV2Service {
await this.flatEntityMapsCacheService.getOrRecomputeManyOrAllFlatEntityMaps(
{
workspaceId,
flatEntities: ['flatViewMaps'],
flatMapsKeys: ['flatViewMaps'],
},
);
@@ -172,7 +172,7 @@ export class ViewV2Service {
await this.flatEntityMapsCacheService.getOrRecomputeManyOrAllFlatEntityMaps(
{
workspaceId,
flatEntities: ['flatViewMaps'],
flatMapsKeys: ['flatViewMaps'],
},
);
@@ -218,7 +218,7 @@ export class ViewV2Service {
await this.flatEntityMapsCacheService.getOrRecomputeManyOrAllFlatEntityMaps(
{
workspaceId,
flatEntities: ['flatViewMaps'],
flatMapsKeys: ['flatViewMaps'],
},
);
@@ -239,7 +239,7 @@ export class ViewV2Service {
await this.flatEntityMapsCacheService.getOrRecomputeManyOrAllFlatEntityMaps(
{
workspaceId,
flatEntities: ['flatViewMaps'],
flatMapsKeys: ['flatViewMaps'],
},
);
@@ -6,6 +6,8 @@ import { type Repository } from 'typeorm';
import { AuditService } from 'src/engine/core-modules/audit/services/audit.service';
import { BillingSubscriptionService } from 'src/engine/core-modules/billing/services/billing-subscription.service';
import { BillingService } from 'src/engine/core-modules/billing/services/billing.service';
import { WorkspaceManyOrAllFlatEntityMapsCacheService } from 'src/engine/core-modules/common/services/workspace-many-or-all-flat-entity-maps-cache.service';
import { DnsManagerService } from 'src/engine/core-modules/dns-manager/services/dns-manager.service';
import { DomainManagerService } from 'src/engine/core-modules/domain-manager/services/domain-manager.service';
import { EmailService } from 'src/engine/core-modules/email/email.service';
import { ExceptionHandlerService } from 'src/engine/core-modules/exception-handler/exception-handler.service';
@@ -14,6 +16,7 @@ import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queu
import { type MessageQueueService } from 'src/engine/core-modules/message-queue/services/message-queue.service';
import { getQueueToken } from 'src/engine/core-modules/message-queue/utils/get-queue-token.util';
import { OnboardingService } from 'src/engine/core-modules/onboarding/onboarding.service';
import { PublicDomain } from 'src/engine/core-modules/public-domain/public-domain.entity';
import { TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service';
import { UserWorkspace } from 'src/engine/core-modules/user-workspace/user-workspace.entity';
import { UserWorkspaceService } from 'src/engine/core-modules/user-workspace/user-workspace.service';
@@ -25,8 +28,6 @@ import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity';
import { PermissionsService } from 'src/engine/metadata-modules/permissions/permissions.service';
import { WorkspaceCacheStorageService } from 'src/engine/workspace-cache-storage/workspace-cache-storage.service';
import { WorkspaceManagerService } from 'src/engine/workspace-manager/workspace-manager.service';
import { DnsManagerService } from 'src/engine/core-modules/dns-manager/services/dns-manager.service';
import { PublicDomain } from 'src/engine/core-modules/public-domain/public-domain.entity';
describe('WorkspaceService', () => {
let service: WorkspaceService;
@@ -112,6 +113,12 @@ describe('WorkspaceService', () => {
flush: jest.fn(),
},
},
{
provide: WorkspaceManyOrAllFlatEntityMapsCacheService,
useValue: {
flushFlatEntityMaps: jest.fn(),
},
},
{
provide: getQueueToken(MessageQueue.deleteCascadeQueue),
useValue: {
@@ -3,15 +3,20 @@ import { InjectRepository } from '@nestjs/typeorm';
import assert from 'assert';
import { t } from '@lingui/core/macro';
import { TypeOrmQueryService } from '@ptc-org/nestjs-query-typeorm';
import { assertIsDefinedOrThrow, isDefined } from 'twenty-shared/utils';
import { WorkspaceActivationStatus } from 'twenty-shared/workspace';
import { Repository } from 'typeorm';
import { t } from '@lingui/core/macro';
import { AuditService } from 'src/engine/core-modules/audit/services/audit.service';
import { CUSTOM_DOMAIN_ACTIVATED_EVENT } from 'src/engine/core-modules/audit/utils/events/workspace-event/custom-domain/custom-domain-activated';
import { CUSTOM_DOMAIN_DEACTIVATED_EVENT } from 'src/engine/core-modules/audit/utils/events/workspace-event/custom-domain/custom-domain-deactivated';
import { BillingEntitlementKey } from 'src/engine/core-modules/billing/enums/billing-entitlement-key.enum';
import { BillingSubscriptionService } from 'src/engine/core-modules/billing/services/billing-subscription.service';
import { BillingService } from 'src/engine/core-modules/billing/services/billing.service';
import { WorkspaceManyOrAllFlatEntityMapsCacheService } from 'src/engine/core-modules/common/services/workspace-many-or-all-flat-entity-maps-cache.service';
import { DnsManagerService } from 'src/engine/core-modules/dns-manager/services/dns-manager.service';
import { ExceptionHandlerService } from 'src/engine/core-modules/exception-handler/exception-handler.service';
import { FeatureFlagService } from 'src/engine/core-modules/feature-flag/services/feature-flag.service';
import {
@@ -21,6 +26,7 @@ import {
import { InjectMessageQueue } from 'src/engine/core-modules/message-queue/decorators/message-queue.decorator';
import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';
import { MessageQueueService } from 'src/engine/core-modules/message-queue/services/message-queue.service';
import { PublicDomain } from 'src/engine/core-modules/public-domain/public-domain.entity';
import { TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service';
import { UserWorkspace } from 'src/engine/core-modules/user-workspace/user-workspace.entity';
import { UserWorkspaceService } from 'src/engine/core-modules/user-workspace/user-workspace.service';
@@ -43,11 +49,6 @@ import { WorkspaceCacheStorageService } from 'src/engine/workspace-cache-storage
import { WorkspaceManagerService } from 'src/engine/workspace-manager/workspace-manager.service';
import { DEFAULT_FEATURE_FLAGS } from 'src/engine/workspace-manager/workspace-sync-metadata/constants/default-feature-flags';
import { extractVersionMajorMinorPatch } from 'src/utils/version/extract-version-major-minor-patch';
import { DnsManagerService } from 'src/engine/core-modules/dns-manager/services/dns-manager.service';
import { CUSTOM_DOMAIN_ACTIVATED_EVENT } from 'src/engine/core-modules/audit/utils/events/workspace-event/custom-domain/custom-domain-activated';
import { CUSTOM_DOMAIN_DEACTIVATED_EVENT } from 'src/engine/core-modules/audit/utils/events/workspace-event/custom-domain/custom-domain-deactivated';
import { AuditService } from 'src/engine/core-modules/audit/services/audit.service';
import { PublicDomain } from 'src/engine/core-modules/public-domain/public-domain.entity';
@Injectable()
// eslint-disable-next-line @nx/workspace-inject-workspace-repository
@@ -73,6 +74,7 @@ export class WorkspaceService extends TypeOrmQueryService<Workspace> {
private readonly exceptionHandlerService: ExceptionHandlerService,
private readonly permissionsService: PermissionsService,
private readonly dnsManagerService: DnsManagerService,
private readonly flatEntityMapsCacheService: WorkspaceManyOrAllFlatEntityMapsCacheService,
private readonly workspaceCacheStorageService: WorkspaceCacheStorageService,
private readonly auditService: AuditService,
@InjectMessageQueue(MessageQueue.deleteCascadeQueue)
@@ -341,6 +343,9 @@ export class WorkspaceService extends TypeOrmQueryService<Workspace> {
workspace.id,
workspace.metadataVersion,
);
await this.flatEntityMapsCacheService.flushFlatEntityMaps({
workspaceId: workspace.id,
});
this.logger.log(`workspace ${id} cache flushed`);
if (softDelete) {
@@ -5,18 +5,28 @@ import { NestjsQueryGraphQLModule } from '@ptc-org/nestjs-query-graphql';
import { NestjsQueryTypeOrmModule } from '@ptc-org/nestjs-query-typeorm';
import { TypeORMModule } from 'src/database/typeorm/typeorm.module';
import { AuditModule } from 'src/engine/core-modules/audit/audit.module';
import { TokenModule } from 'src/engine/core-modules/auth/token/token.module';
import { BillingModule } from 'src/engine/core-modules/billing/billing.module';
import { BillingSubscription } from 'src/engine/core-modules/billing/entities/billing-subscription.entity';
import { WorkspaceManyOrAllFlatEntityMapsCacheModule } from 'src/engine/core-modules/common/services/workspace-many-or-all-flat-entity-maps-cache.module';
import { DnsManagerModule } from 'src/engine/core-modules/dns-manager/dns-manager.module';
import { DomainManagerModule } from 'src/engine/core-modules/domain-manager/domain-manager.module';
import { FeatureFlagModule } from 'src/engine/core-modules/feature-flag/feature-flag.module';
import { FileUploadModule } from 'src/engine/core-modules/file/file-upload/file-upload.module';
import { FileModule } from 'src/engine/core-modules/file/file.module';
import { OnboardingModule } from 'src/engine/core-modules/onboarding/onboarding.module';
import { PublicDomain } from 'src/engine/core-modules/public-domain/public-domain.entity';
import { UserWorkspace } from 'src/engine/core-modules/user-workspace/user-workspace.entity';
import { UserWorkspaceModule } from 'src/engine/core-modules/user-workspace/user-workspace.module';
import { User } from 'src/engine/core-modules/user/user.entity';
import { CoreViewModule } from 'src/engine/core-modules/view/view.module';
import { CheckCustomDomainValidRecordsCronCommand } from 'src/engine/core-modules/workspace/crons/commands/check-custom-domain-valid-records.cron.command';
import { CheckCustomDomainValidRecordsCronJob } from 'src/engine/core-modules/workspace/crons/jobs/check-custom-domain-valid-records.cron.job';
import { WorkspaceService } from 'src/engine/core-modules/workspace/services/workspace.service';
import { WorkspaceWorkspaceMemberListener } from 'src/engine/core-modules/workspace/workspace-workspace-member.listener';
import { workspaceAutoResolverOpts } from 'src/engine/core-modules/workspace/workspace.auto-resolver-opts';
import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity';
import { WorkspaceResolver } from 'src/engine/core-modules/workspace/workspace.resolver';
import { AgentModule } from 'src/engine/metadata-modules/agent/agent.module';
import { DataSourceModule } from 'src/engine/metadata-modules/data-source/data-source.module';
@@ -25,15 +35,6 @@ import { RoleModule } from 'src/engine/metadata-modules/role/role.module';
import { WorkspaceMetadataCacheModule } from 'src/engine/metadata-modules/workspace-metadata-cache/workspace-metadata-cache.module';
import { WorkspaceCacheStorageModule } from 'src/engine/workspace-cache-storage/workspace-cache-storage.module';
import { WorkspaceManagerModule } from 'src/engine/workspace-manager/workspace-manager.module';
import { DnsManagerModule } from 'src/engine/core-modules/dns-manager/dns-manager.module';
import { DomainManagerModule } from 'src/engine/core-modules/domain-manager/domain-manager.module';
import { CheckCustomDomainValidRecordsCronJob } from 'src/engine/core-modules/workspace/crons/jobs/check-custom-domain-valid-records.cron.job';
import { CheckCustomDomainValidRecordsCronCommand } from 'src/engine/core-modules/workspace/crons/commands/check-custom-domain-valid-records.cron.command';
import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity';
import { workspaceAutoResolverOpts } from 'src/engine/core-modules/workspace/workspace.auto-resolver-opts';
import { WorkspaceService } from 'src/engine/core-modules/workspace/services/workspace.service';
import { AuditModule } from 'src/engine/core-modules/audit/audit.module';
import { PublicDomain } from 'src/engine/core-modules/public-domain/public-domain.entity';
@Module({
imports: [
@@ -66,6 +67,7 @@ import { PublicDomain } from 'src/engine/core-modules/public-domain/public-domai
DnsManagerModule,
DomainManagerModule,
CoreViewModule,
WorkspaceManyOrAllFlatEntityMapsCacheModule,
],
services: [WorkspaceService],
resolvers: workspaceAutoResolverOpts,
@@ -103,7 +103,7 @@ export class DataloaderService {
await this.flatEntityMapsCacheService.getOrRecomputeManyOrAllFlatEntityMaps(
{
workspaceId,
flatEntities: ['flatFieldMetadataMaps', 'flatObjectMetadataMaps'],
flatMapsKeys: ['flatFieldMetadataMaps', 'flatObjectMetadataMaps'],
},
);
@@ -195,7 +195,7 @@ export class DataloaderService {
await this.flatEntityMapsCacheService.getOrRecomputeManyOrAllFlatEntityMaps(
{
workspaceId,
flatEntities: ['flatFieldMetadataMaps', 'flatObjectMetadataMaps'],
flatMapsKeys: ['flatFieldMetadataMaps', 'flatObjectMetadataMaps'],
},
);
const relationDtos: Array<RelationDTO[] | null> = [];
@@ -327,7 +327,7 @@ export class DataloaderService {
await this.flatEntityMapsCacheService.getOrRecomputeManyOrAllFlatEntityMaps(
{
workspaceId,
flatEntities: ['flatFieldMetadataMaps', 'flatObjectMetadataMaps'],
flatMapsKeys: ['flatFieldMetadataMaps', 'flatObjectMetadataMaps'],
},
);
@@ -36,7 +36,7 @@ export class CronTriggerV2Service {
await this.flatEntityMapsCacheService.getOrRecomputeManyOrAllFlatEntityMaps(
{
workspaceId,
flatEntities: ['flatCronTriggerMaps', 'flatServerlessFunctionMaps'],
flatMapsKeys: ['flatCronTriggerMaps', 'flatServerlessFunctionMaps'],
},
);
@@ -86,7 +86,7 @@ export class CronTriggerV2Service {
await this.flatEntityMapsCacheService.getOrRecomputeManyOrAllFlatEntityMaps(
{
workspaceId,
flatEntities: ['flatCronTriggerMaps', 'flatServerlessFunctionMaps'],
flatMapsKeys: ['flatCronTriggerMaps', 'flatServerlessFunctionMaps'],
},
);
@@ -104,7 +104,7 @@ export class CronTriggerV2Service {
await this.flatEntityMapsCacheService.getOrRecomputeManyOrAllFlatEntityMaps(
{
workspaceId,
flatEntities: ['flatCronTriggerMaps', 'flatServerlessFunctionMaps'],
flatMapsKeys: ['flatCronTriggerMaps', 'flatServerlessFunctionMaps'],
},
);
@@ -157,7 +157,7 @@ export class CronTriggerV2Service {
await this.flatEntityMapsCacheService.getOrRecomputeManyOrAllFlatEntityMaps(
{
workspaceId,
flatEntities: ['flatCronTriggerMaps'],
flatMapsKeys: ['flatCronTriggerMaps'],
},
);
@@ -181,7 +181,7 @@ export class CronTriggerV2Service {
await this.flatEntityMapsCacheService.getOrRecomputeManyOrAllFlatEntityMaps(
{
workspaceId,
flatEntities: ['flatCronTriggerMaps', 'flatServerlessFunctionMaps'],
flatMapsKeys: ['flatCronTriggerMaps', 'flatServerlessFunctionMaps'],
},
);
@@ -36,7 +36,7 @@ export class DatabaseEventTriggerV2Service {
await this.flatEntityMapsCacheService.getOrRecomputeManyOrAllFlatEntityMaps(
{
workspaceId,
flatEntities: [
flatMapsKeys: [
'flatDatabaseEventTriggerMaps',
'flatServerlessFunctionMaps',
],
@@ -94,7 +94,7 @@ export class DatabaseEventTriggerV2Service {
await this.flatEntityMapsCacheService.getOrRecomputeManyOrAllFlatEntityMaps(
{
workspaceId,
flatEntities: ['flatDatabaseEventTriggerMaps'],
flatMapsKeys: ['flatDatabaseEventTriggerMaps'],
},
);
@@ -112,7 +112,7 @@ export class DatabaseEventTriggerV2Service {
await this.flatEntityMapsCacheService.getOrRecomputeManyOrAllFlatEntityMaps(
{
workspaceId,
flatEntities: [
flatMapsKeys: [
'flatDatabaseEventTriggerMaps',
'flatServerlessFunctionMaps',
],
@@ -175,7 +175,7 @@ export class DatabaseEventTriggerV2Service {
await this.flatEntityMapsCacheService.getOrRecomputeManyOrAllFlatEntityMaps(
{
workspaceId,
flatEntities: ['flatDatabaseEventTriggerMaps'],
flatMapsKeys: ['flatDatabaseEventTriggerMaps'],
},
);
@@ -199,7 +199,7 @@ export class DatabaseEventTriggerV2Service {
await this.flatEntityMapsCacheService.getOrRecomputeManyOrAllFlatEntityMaps(
{
workspaceId,
flatEntities: [
flatMapsKeys: [
'flatDatabaseEventTriggerMaps',
'flatServerlessFunctionMaps',
],
@@ -72,7 +72,7 @@ export class FieldMetadataServiceV2 {
} = await this.flatEntityMapsCacheService.getOrRecomputeManyOrAllFlatEntityMaps(
{
workspaceId,
flatEntities: [
flatMapsKeys: [
'flatObjectMetadataMaps',
'flatIndexMaps',
'flatFieldMetadataMaps',
@@ -180,7 +180,7 @@ export class FieldMetadataServiceV2 {
} = await this.flatEntityMapsCacheService.getOrRecomputeManyOrAllFlatEntityMaps(
{
workspaceId,
flatEntities: [
flatMapsKeys: [
'flatObjectMetadataMaps',
'flatIndexMaps',
'flatFieldMetadataMaps',
@@ -288,7 +288,7 @@ export class FieldMetadataServiceV2 {
} = await this.flatEntityMapsCacheService.getOrRecomputeManyOrAllFlatEntityMaps(
{
workspaceId,
flatEntities: [
flatMapsKeys: [
'flatObjectMetadataMaps',
'flatIndexMaps',
'flatFieldMetadataMaps',
@@ -402,7 +402,7 @@ export class FieldMetadataService extends TypeOrmQueryService<FieldMetadataEntit
await this.flatEntityMapsCacheService.invalidateFlatEntityMaps({
workspaceId,
flatEntities: ['flatFieldMetadataMaps'],
flatMapsKeys: ['flatFieldMetadataMaps'],
});
return updatedFieldMetadata;
@@ -590,7 +590,7 @@ export class FieldMetadataService extends TypeOrmQueryService<FieldMetadataEntit
await this.flatEntityMapsCacheService.invalidateFlatEntityMaps({
workspaceId,
flatEntities: ['flatObjectMetadataMaps', 'flatFieldMetadataMaps'],
flatMapsKeys: ['flatObjectMetadataMaps', 'flatFieldMetadataMaps'],
});
return fieldMetadata;
@@ -786,7 +786,7 @@ export class FieldMetadataService extends TypeOrmQueryService<FieldMetadataEntit
await this.flatEntityMapsCacheService.invalidateFlatEntityMaps({
workspaceId,
flatEntities: ['flatObjectMetadataMaps', 'flatFieldMetadataMaps'],
flatMapsKeys: ['flatObjectMetadataMaps', 'flatFieldMetadataMaps'],
});
return createdFieldMetadatas;
@@ -58,7 +58,7 @@ export class ObjectMetadataServiceV2 {
} = await this.flatEntityMapsCacheService.getOrRecomputeManyOrAllFlatEntityMaps(
{
workspaceId,
flatEntities: [
flatMapsKeys: [
'flatObjectMetadataMaps',
'flatIndexMaps',
'flatFieldMetadataMaps',
@@ -142,7 +142,7 @@ export class ObjectMetadataServiceV2 {
await this.flatEntityMapsCacheService.getOrRecomputeManyOrAllFlatEntityMaps(
{
workspaceId,
flatEntities: ['flatObjectMetadataMaps'],
flatMapsKeys: ['flatObjectMetadataMaps'],
},
);
@@ -185,7 +185,7 @@ export class ObjectMetadataServiceV2 {
} = await this.flatEntityMapsCacheService.getOrRecomputeManyOrAllFlatEntityMaps(
{
workspaceId,
flatEntities: [
flatMapsKeys: [
'flatObjectMetadataMaps',
'flatIndexMaps',
'flatFieldMetadataMaps',
@@ -305,7 +305,7 @@ export class ObjectMetadataServiceV2 {
} = await this.flatEntityMapsCacheService.getOrRecomputeManyOrAllFlatEntityMaps(
{
workspaceId,
flatEntities: [
flatMapsKeys: [
'flatObjectMetadataMaps',
'flatViewMaps',
'flatViewFieldMaps',
@@ -427,7 +427,7 @@ export class ObjectMetadataServiceV2 {
await this.flatEntityMapsCacheService.getOrRecomputeManyOrAllFlatEntityMaps(
{
workspaceId,
flatEntities: ['flatObjectMetadataMaps'],
flatMapsKeys: ['flatObjectMetadataMaps'],
},
);
@@ -14,6 +14,7 @@ import {
type QueryRunner,
} from 'typeorm';
import { WorkspaceManyOrAllFlatEntityMapsCacheService } from 'src/engine/core-modules/common/services/workspace-many-or-all-flat-entity-maps-cache.service';
import { FeatureFlagKey } from 'src/engine/core-modules/feature-flag/enums/feature-flag-key.enum';
import { FeatureFlagService } from 'src/engine/core-modules/feature-flag/services/feature-flag.service';
import { DataSourceService } from 'src/engine/metadata-modules/data-source/data-source.service';
@@ -52,7 +53,6 @@ import { isFieldMetadataEntityOfType } from 'src/engine/utils/is-field-metadata-
import { WorkspaceMigrationRunnerService } from 'src/engine/workspace-manager/workspace-migration-runner/workspace-migration-runner.service';
import { CUSTOM_OBJECT_STANDARD_FIELD_IDS } from 'src/engine/workspace-manager/workspace-sync-metadata/constants/standard-field-ids';
import { isSearchableFieldType } from 'src/engine/workspace-manager/workspace-sync-metadata/utils/is-searchable-field.util';
import { WorkspaceManyOrAllFlatEntityMapsCacheService } from 'src/engine/core-modules/common/services/workspace-many-or-all-flat-entity-maps-cache.service';
import { ObjectMetadataEntity } from './object-metadata.entity';
@@ -288,7 +288,7 @@ export class ObjectMetadataService extends TypeOrmQueryService<ObjectMetadataEnt
await this.flatEntityMapsCacheService.invalidateFlatEntityMaps({
workspaceId,
flatEntities: ['flatFieldMetadataMaps', 'flatObjectMetadataMaps'],
flatMapsKeys: ['flatFieldMetadataMaps', 'flatObjectMetadataMaps'],
});
return createdObjectMetadata;
@@ -475,7 +475,7 @@ export class ObjectMetadataService extends TypeOrmQueryService<ObjectMetadataEnt
await this.flatEntityMapsCacheService.invalidateFlatEntityMaps({
workspaceId,
flatEntities: ['flatObjectMetadataMaps'],
flatMapsKeys: ['flatObjectMetadataMaps'],
});
const formattedUpdatedObject = {
@@ -592,7 +592,7 @@ export class ObjectMetadataService extends TypeOrmQueryService<ObjectMetadataEnt
await this.flatEntityMapsCacheService.invalidateFlatEntityMaps({
workspaceId,
flatEntities: ['flatObjectMetadataMaps', 'flatFieldMetadataMaps'],
flatMapsKeys: ['flatObjectMetadataMaps', 'flatFieldMetadataMaps'],
});
await this.workspacePermissionsCacheService.recomputeRolesPermissionsCache(
@@ -36,7 +36,7 @@ export class RouteTriggerV2Service {
await this.flatEntityMapsCacheService.getOrRecomputeManyOrAllFlatEntityMaps(
{
workspaceId,
flatEntities: ['flatRouteTriggerMaps', 'flatServerlessFunctionMaps'],
flatMapsKeys: ['flatRouteTriggerMaps', 'flatServerlessFunctionMaps'],
},
);
@@ -85,7 +85,7 @@ export class RouteTriggerV2Service {
await this.flatEntityMapsCacheService.getOrRecomputeManyOrAllFlatEntityMaps(
{
workspaceId,
flatEntities: ['flatRouteTriggerMaps'],
flatMapsKeys: ['flatRouteTriggerMaps'],
},
);
@@ -103,7 +103,7 @@ export class RouteTriggerV2Service {
await this.flatEntityMapsCacheService.getOrRecomputeManyOrAllFlatEntityMaps(
{
workspaceId,
flatEntities: ['flatRouteTriggerMaps', 'flatServerlessFunctionMaps'],
flatMapsKeys: ['flatRouteTriggerMaps', 'flatServerlessFunctionMaps'],
},
);
@@ -156,7 +156,7 @@ export class RouteTriggerV2Service {
await this.flatEntityMapsCacheService.getOrRecomputeManyOrAllFlatEntityMaps(
{
workspaceId,
flatEntities: ['flatRouteTriggerMaps'],
flatMapsKeys: ['flatRouteTriggerMaps'],
},
);
@@ -180,7 +180,7 @@ export class RouteTriggerV2Service {
await this.flatEntityMapsCacheService.getOrRecomputeManyOrAllFlatEntityMaps(
{
workspaceId,
flatEntities: ['flatRouteTriggerMaps', 'flatServerlessFunctionMaps'],
flatMapsKeys: ['flatRouteTriggerMaps', 'flatServerlessFunctionMaps'],
},
);
@@ -36,7 +36,7 @@ export class ServerlessFunctionV2Service {
await this.flatEntityMapsCacheService.getOrRecomputeManyOrAllFlatEntityMaps(
{
workspaceId,
flatEntities: ['flatServerlessFunctionMaps'],
flatMapsKeys: ['flatServerlessFunctionMaps'],
},
);
@@ -84,7 +84,7 @@ export class ServerlessFunctionV2Service {
await this.flatEntityMapsCacheService.getOrRecomputeManyOrAllFlatEntityMaps(
{
workspaceId,
flatEntities: ['flatServerlessFunctionMaps'],
flatMapsKeys: ['flatServerlessFunctionMaps'],
},
);
@@ -102,7 +102,7 @@ export class ServerlessFunctionV2Service {
await this.flatEntityMapsCacheService.getOrRecomputeManyOrAllFlatEntityMaps(
{
workspaceId,
flatEntities: ['flatServerlessFunctionMaps'],
flatMapsKeys: ['flatServerlessFunctionMaps'],
},
);
@@ -155,7 +155,7 @@ export class ServerlessFunctionV2Service {
await this.flatEntityMapsCacheService.getOrRecomputeManyOrAllFlatEntityMaps(
{
workspaceId,
flatEntities: ['flatServerlessFunctionMaps'],
flatMapsKeys: ['flatServerlessFunctionMaps'],
},
);
@@ -178,7 +178,7 @@ export class ServerlessFunctionV2Service {
await this.flatEntityMapsCacheService.getOrRecomputeManyOrAllFlatEntityMaps(
{
workspaceId,
flatEntities: ['flatServerlessFunctionMaps'],
flatMapsKeys: ['flatServerlessFunctionMaps'],
},
);
@@ -233,7 +233,7 @@ export class ServerlessFunctionV2Service {
await this.flatEntityMapsCacheService.getOrRecomputeManyOrAllFlatEntityMaps(
{
workspaceId,
flatEntities: ['flatServerlessFunctionMaps'],
flatMapsKeys: ['flatServerlessFunctionMaps'],
},
);
@@ -256,7 +256,7 @@ export class ServerlessFunctionV2Service {
await this.flatEntityMapsCacheService.getOrRecomputeManyOrAllFlatEntityMaps(
{
workspaceId,
flatEntities: ['flatServerlessFunctionMaps'],
flatMapsKeys: ['flatServerlessFunctionMaps'],
},
);
@@ -1,18 +1,23 @@
import { Injectable, OnModuleInit } from '@nestjs/common';
import { DiscoveryService } from '@nestjs/core';
import { AllFlatEntities } from 'src/engine/core-modules/common/types/all-flat-entities.type';
import { isDefined } from 'twenty-shared/utils';
import { AllFlatEntityMaps } from 'src/engine/core-modules/common/types/all-flat-entity-maps.type';
import { FlatEntityMaps } from 'src/engine/core-modules/common/types/flat-entity-maps.type';
import { WORKSPACE_FLAT_MAP_CACHE_KEY } from 'src/engine/workspace-flat-map-cache/decorators/workspace-flat-map-cache.decorator';
import {
WorkspaceFlatMapCacheException,
WorkspaceFlatMapCacheExceptionCode,
} from 'src/engine/workspace-flat-map-cache/exceptions/workspace-flat-map-cache.exception';
import { WorkspaceFlatMapCacheService } from 'src/engine/workspace-flat-map-cache/services/workspace-flat-map-cache.service';
@Injectable()
export class WorkspaceFlatMapCacheRegistryService implements OnModuleInit {
private readonly cacheServiceMap = new Map<
keyof AllFlatEntityMaps,
WorkspaceFlatMapCacheService<FlatEntityMaps<AllFlatEntities>>
>();
private readonly cacheServiceRecord: {
[P in keyof AllFlatEntityMaps]?: WorkspaceFlatMapCacheService<
AllFlatEntityMaps[P]
>;
} = {};
constructor(private readonly discoveryService: DiscoveryService) {}
@@ -30,20 +35,31 @@ export class WorkspaceFlatMapCacheRegistryService implements OnModuleInit {
return;
}
const cacheKey = Reflect.getMetadata(
const cacheKey: keyof AllFlatEntityMaps | undefined = Reflect.getMetadata(
WORKSPACE_FLAT_MAP_CACHE_KEY,
metatype,
);
if (cacheKey && instance instanceof WorkspaceFlatMapCacheService) {
this.cacheServiceMap.set(cacheKey, instance);
this.cacheServiceRecord[cacheKey] = instance;
}
});
}
getCacheService(
flatEntityName: keyof AllFlatEntityMaps,
): WorkspaceFlatMapCacheService<FlatEntityMaps<AllFlatEntities>> | undefined {
return this.cacheServiceMap.get(flatEntityName);
getCacheService<K extends keyof AllFlatEntityMaps>(flatMapsKey: K) {
return this.cacheServiceRecord[flatMapsKey];
}
getCacheServiceOrThrow<K extends keyof AllFlatEntityMaps>(flatMapsKey: K) {
const service = this.getCacheService(flatMapsKey);
if (!isDefined(service)) {
throw new WorkspaceFlatMapCacheException(
`No cache service found for ${flatMapsKey}`,
WorkspaceFlatMapCacheExceptionCode.INTERNAL_SERVER_ERROR,
);
}
return service;
}
}
@@ -99,14 +99,9 @@ export abstract class WorkspaceFlatMapCacheService<
}: {
workspaceId: string;
}): Promise<void> {
const { flatMapKey, hashKey } = this.buildRemoteCacheKeys({ workspaceId });
await this.cacheStorageService.del(flatMapKey);
await this.cacheStorageService.del(hashKey);
this.localCacheFlatMaps.delete(workspaceId);
this.localCacheHashes.delete(workspaceId);
await this.flushCache({
workspaceId,
});
await this.recomputeAndStoreInCache({ workspaceId });
}
@@ -185,4 +180,14 @@ export abstract class WorkspaceFlatMapCacheService<
await this.cacheStorageService.set(flatMapKey, flatMap);
}
async flushCache({ workspaceId }: { workspaceId: string }): Promise<void> {
const { flatMapKey, hashKey } = this.buildRemoteCacheKeys({ workspaceId });
await this.cacheStorageService.del(flatMapKey);
await this.cacheStorageService.del(hashKey);
this.localCacheFlatMaps.delete(workspaceId);
this.localCacheHashes.delete(workspaceId);
}
}
@@ -3,6 +3,7 @@ import { TypeOrmModule } from '@nestjs/typeorm';
import { TypeORMModule } from 'src/database/typeorm/typeorm.module';
import { ApiKeyModule } from 'src/engine/core-modules/api-key/api-key.module';
import { WorkspaceManyOrAllFlatEntityMapsCacheModule } from 'src/engine/core-modules/common/services/workspace-many-or-all-flat-entity-maps-cache.module';
import { FeatureFlagModule } from 'src/engine/core-modules/feature-flag/feature-flag.module';
import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity';
import { DataSourceModule } from 'src/engine/metadata-modules/data-source/data-source.module';
@@ -39,6 +40,7 @@ import { WorkspaceSyncMetadataModule } from 'src/engine/workspace-manager/worksp
TypeOrmModule.forFeature([Workspace, ObjectMetadataEntity]),
ObjectPermissionModule,
WorkspacePermissionsCacheModule,
WorkspaceManyOrAllFlatEntityMapsCacheModule,
WorkspaceMetadataCacheModule,
],
exports: [DevSeederService],
@@ -3,6 +3,7 @@ import { InjectDataSource } from '@nestjs/typeorm';
import { DataSource } from 'typeorm';
import { WorkspaceManyOrAllFlatEntityMapsCacheService } from 'src/engine/core-modules/common/services/workspace-many-or-all-flat-entity-maps-cache.service';
import { FeatureFlagService } from 'src/engine/core-modules/feature-flag/services/feature-flag.service';
import { TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service';
import { DataSourceService } from 'src/engine/metadata-modules/data-source/data-source.service';
@@ -29,6 +30,7 @@ export class DevSeederService {
private readonly workspaceSyncMetadataService: WorkspaceSyncMetadataService,
private readonly devSeederMetadataService: DevSeederMetadataService,
private readonly devSeederPermissionsService: DevSeederPermissionsService,
private readonly flatEntityMapsCacheService: WorkspaceManyOrAllFlatEntityMapsCacheService,
private readonly devSeederDataService: DevSeederDataService,
@InjectDataSource()
private readonly coreDataSource: DataSource,
@@ -101,5 +103,8 @@ export class DevSeederService {
});
await this.workspaceCacheStorageService.flush(workspaceId, undefined);
await this.flatEntityMapsCacheService.flushFlatEntityMaps({
workspaceId,
});
}
}
@@ -1,7 +1,7 @@
import { Inject, SetMetadata } from '@nestjs/common';
import { LoggerService } from 'src/engine/core-modules/logger/logger.service';
import { type AllFlatEntityMaps } from 'src/engine/core-modules/common/types/all-flat-entity-maps.type';
import { LoggerService } from 'src/engine/core-modules/logger/logger.service';
import {
type ExtractAction,
type WorkspaceMigrationActionTypeV2,
@@ -3,13 +3,13 @@ import { InjectDataSource } from '@nestjs/typeorm';
import { DataSource } from 'typeorm';
import { LoggerService } from 'src/engine/core-modules/logger/logger.service';
import {
WorkspaceQueryRunnerException,
WorkspaceQueryRunnerExceptionCode,
} from 'src/engine/api/graphql/workspace-query-runner/workspace-query-runner.exception';
import { WorkspaceManyOrAllFlatEntityMapsCacheService } from 'src/engine/core-modules/common/services/workspace-many-or-all-flat-entity-maps-cache.service';
import { AllFlatEntityMaps } from 'src/engine/core-modules/common/types/all-flat-entity-maps.type';
import { LoggerService } from 'src/engine/core-modules/logger/logger.service';
import { WorkspaceMetadataVersionService } from 'src/engine/metadata-modules/workspace-metadata-version/services/workspace-metadata-version.service';
import { WorkspacePermissionsCacheService } from 'src/engine/metadata-modules/workspace-permissions-cache/workspace-permissions-cache.service';
import { WorkspaceMigrationV2 } from 'src/engine/workspace-manager/workspace-migration-v2/workspace-migration-builder-v2/types/workspace-migration-v2';
@@ -41,7 +41,7 @@ export class WorkspaceMigrationRunnerV2Service {
await this.flatEntityMapsCacheService.getOrRecomputeManyOrAllFlatEntityMaps(
{
workspaceId,
flatEntities: relatedFlatEntityMapsKeys,
flatMapsKeys: relatedFlatEntityMapsKeys,
},
);
@@ -111,7 +111,7 @@ export class WorkspaceMigrationRunnerV2Service {
await this.flatEntityMapsCacheService.invalidateFlatEntityMaps({
workspaceId,
flatEntities: [
flatMapsKeys: [
...new Set([
...flatEntityMapsToInvalidate,
...(relatedFlatEntityMapsKeys ?? []),
@@ -264,7 +264,7 @@ export class WorkspaceSyncMetadataService {
);
await this.flatEntityMapsCacheService.invalidateFlatEntityMaps({
workspaceId: context.workspaceId,
flatEntities: ['flatObjectMetadataMaps', 'flatFieldMetadataMaps'],
flatMapsKeys: ['flatObjectMetadataMaps', 'flatFieldMetadataMaps'],
});
}