From 9b382542563f33b3d8e4c7175dd26aed4de40bb6 Mon Sep 17 00:00:00 2001 From: Paul Rastoin <45004772+prastoin@users.noreply.github.com> Date: Sat, 3 Jan 2026 02:51:07 +0100 Subject: [PATCH] Refactor runner dynamic cache invalidation (#16913) # Introduction closes https://github.com/twentyhq/core-team-issues/issues/1792 Refactoring the cache to invalidate to be more precise and prevent any corrupted cache occurrences ## Next The load dependency cache from the workspace migration, that's not optimal it should be smart enough to infer it from the actions themselves. For the moment their definition isn't granular enough and inferring such info would be very dirty --- .../types/flat-entity-maps-key-to-metadata.ts | 8 + ...one-to-many-related-metadata-names.type.ts | 13 ++ ...a-related-metadata-names.util.spec.ts.snap | 145 ++++++++++++++++++ ...tadata-related-metadata-names.util.spec.ts | 34 ++++ ...ata-name-from-flat-entity-maps-key.util.ts | 12 ++ ...et-metadata-related-metadata-names.util.ts | 26 ++++ .../types/workspace-migration-v2.ts | 1 + .../workspace-migration-runner-v2.service.ts | 24 +-- 8 files changed, 252 insertions(+), 11 deletions(-) create mode 100644 packages/twenty-server/src/engine/metadata-modules/flat-entity/types/flat-entity-maps-key-to-metadata.ts create mode 100644 packages/twenty-server/src/engine/metadata-modules/flat-entity/types/metadata-one-to-many-related-metadata-names.type.ts create mode 100644 packages/twenty-server/src/engine/metadata-modules/flat-entity/utils/__tests__/__snapshots__/get-metadata-related-metadata-names.util.spec.ts.snap create mode 100644 packages/twenty-server/src/engine/metadata-modules/flat-entity/utils/__tests__/get-metadata-related-metadata-names.util.spec.ts create mode 100644 packages/twenty-server/src/engine/metadata-modules/flat-entity/utils/get-metadata-name-from-flat-entity-maps-key.util.ts create mode 100644 packages/twenty-server/src/engine/metadata-modules/flat-entity/utils/get-metadata-related-metadata-names.util.ts diff --git a/packages/twenty-server/src/engine/metadata-modules/flat-entity/types/flat-entity-maps-key-to-metadata.ts b/packages/twenty-server/src/engine/metadata-modules/flat-entity/types/flat-entity-maps-key-to-metadata.ts new file mode 100644 index 0000000000..1aa5809519 --- /dev/null +++ b/packages/twenty-server/src/engine/metadata-modules/flat-entity/types/flat-entity-maps-key-to-metadata.ts @@ -0,0 +1,8 @@ +import { type AllMetadataName } from 'twenty-shared/metadata'; + +export type FlatEntityMapsKeyToMetadata = + T extends `flat${infer Name}Maps` + ? Uncapitalize extends AllMetadataName + ? Uncapitalize + : never + : never; diff --git a/packages/twenty-server/src/engine/metadata-modules/flat-entity/types/metadata-one-to-many-related-metadata-names.type.ts b/packages/twenty-server/src/engine/metadata-modules/flat-entity/types/metadata-one-to-many-related-metadata-names.type.ts new file mode 100644 index 0000000000..13b295d498 --- /dev/null +++ b/packages/twenty-server/src/engine/metadata-modules/flat-entity/types/metadata-one-to-many-related-metadata-names.type.ts @@ -0,0 +1,13 @@ +import { type AllMetadataName } from 'twenty-shared/metadata'; + +import { type ALL_METADATA_RELATIONS } from 'src/engine/metadata-modules/flat-entity/constant/all-metadata-relations.constant'; + +type ExtractMetadataNames = { + [K in keyof T]: T[K] extends { metadataName: infer M } ? M : never; +}[keyof T]; + +export type MetadataOneToManyRelatedMetadataNames = + Extract< + ExtractMetadataNames<(typeof ALL_METADATA_RELATIONS)[T]['oneToMany']>, + AllMetadataName + >; diff --git a/packages/twenty-server/src/engine/metadata-modules/flat-entity/utils/__tests__/__snapshots__/get-metadata-related-metadata-names.util.spec.ts.snap b/packages/twenty-server/src/engine/metadata-modules/flat-entity/utils/__tests__/__snapshots__/get-metadata-related-metadata-names.util.spec.ts.snap new file mode 100644 index 0000000000..76a4fe9663 --- /dev/null +++ b/packages/twenty-server/src/engine/metadata-modules/flat-entity/utils/__tests__/__snapshots__/get-metadata-related-metadata-names.util.spec.ts.snap @@ -0,0 +1,145 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`getMetadataRelatedMetadataNames should return related metadata names for agent 1`] = `[]`; + +exports[`getMetadataRelatedMetadataNames should return related metadata names for cronTrigger 1`] = ` +[ + "serverlessFunction", +] +`; + +exports[`getMetadataRelatedMetadataNames should return related metadata names for databaseEventTrigger 1`] = ` +[ + "serverlessFunction", +] +`; + +exports[`getMetadataRelatedMetadataNames should return related metadata names for fieldMetadata 1`] = ` +[ + "objectMetadata", + "fieldMetadata", + "viewField", + "viewFilter", + "view", +] +`; + +exports[`getMetadataRelatedMetadataNames should return related metadata names for index 1`] = ` +[ + "objectMetadata", +] +`; + +exports[`getMetadataRelatedMetadataNames should return related metadata names for objectMetadata 1`] = ` +[ + "fieldMetadata", + "index", + "view", +] +`; + +exports[`getMetadataRelatedMetadataNames should return related metadata names for pageLayout 1`] = ` +[ + "objectMetadata", + "pageLayoutTab", +] +`; + +exports[`getMetadataRelatedMetadataNames should return related metadata names for pageLayoutTab 1`] = ` +[ + "pageLayout", + "pageLayoutWidget", +] +`; + +exports[`getMetadataRelatedMetadataNames should return related metadata names for pageLayoutWidget 1`] = ` +[ + "pageLayoutTab", + "objectMetadata", +] +`; + +exports[`getMetadataRelatedMetadataNames should return related metadata names for role 1`] = ` +[ + "roleTarget", +] +`; + +exports[`getMetadataRelatedMetadataNames should return related metadata names for roleTarget 1`] = ` +[ + "role", +] +`; + +exports[`getMetadataRelatedMetadataNames should return related metadata names for routeTrigger 1`] = ` +[ + "serverlessFunction", +] +`; + +exports[`getMetadataRelatedMetadataNames should return related metadata names for rowLevelPermissionPredicate 1`] = ` +[ + "role", + "fieldMetadata", + "objectMetadata", + "rowLevelPermissionPredicateGroup", +] +`; + +exports[`getMetadataRelatedMetadataNames should return related metadata names for rowLevelPermissionPredicateGroup 1`] = ` +[ + "role", + "rowLevelPermissionPredicateGroup", + "rowLevelPermissionPredicate", +] +`; + +exports[`getMetadataRelatedMetadataNames should return related metadata names for serverlessFunction 1`] = ` +[ + "cronTrigger", + "databaseEventTrigger", + "routeTrigger", +] +`; + +exports[`getMetadataRelatedMetadataNames should return related metadata names for skill 1`] = `[]`; + +exports[`getMetadataRelatedMetadataNames should return related metadata names for view 1`] = ` +[ + "objectMetadata", + "fieldMetadata", + "viewField", + "viewFilter", + "viewFilterGroup", + "viewGroup", +] +`; + +exports[`getMetadataRelatedMetadataNames should return related metadata names for viewField 1`] = ` +[ + "fieldMetadata", + "view", +] +`; + +exports[`getMetadataRelatedMetadataNames should return related metadata names for viewFilter 1`] = ` +[ + "fieldMetadata", + "view", + "viewFilterGroup", +] +`; + +exports[`getMetadataRelatedMetadataNames should return related metadata names for viewFilterGroup 1`] = ` +[ + "viewFilterGroup", + "view", + "viewFilter", +] +`; + +exports[`getMetadataRelatedMetadataNames should return related metadata names for viewGroup 1`] = ` +[ + "view", +] +`; diff --git a/packages/twenty-server/src/engine/metadata-modules/flat-entity/utils/__tests__/get-metadata-related-metadata-names.util.spec.ts b/packages/twenty-server/src/engine/metadata-modules/flat-entity/utils/__tests__/get-metadata-related-metadata-names.util.spec.ts new file mode 100644 index 0000000000..7c5cca3cab --- /dev/null +++ b/packages/twenty-server/src/engine/metadata-modules/flat-entity/utils/__tests__/get-metadata-related-metadata-names.util.spec.ts @@ -0,0 +1,34 @@ +import { + ALL_METADATA_NAME, + type AllMetadataName, +} from 'twenty-shared/metadata'; +import { + type EachTestingContext, + eachTestingContextFilter, +} from 'twenty-shared/testing'; + +import { getMetadataRelatedMetadataNames } from 'src/engine/metadata-modules/flat-entity/utils/get-metadata-related-metadata-names.util'; + +type TestContext = { + input: AllMetadataName; +}; + +const testCases: EachTestingContext[] = ( + Object.keys(ALL_METADATA_NAME) as (keyof typeof ALL_METADATA_NAME)[] +).map((metadataName) => ({ + title: `should return related metadata names for ${metadataName}`, + context: { + input: metadataName, + }, +})); + +describe('getMetadataRelatedMetadataNames', () => { + test.each(eachTestingContextFilter(testCases))( + '$title', + ({ context: { input } }) => { + const result = getMetadataRelatedMetadataNames(input); + + expect(result).toMatchSnapshot(); + }, + ); +}); diff --git a/packages/twenty-server/src/engine/metadata-modules/flat-entity/utils/get-metadata-name-from-flat-entity-maps-key.util.ts b/packages/twenty-server/src/engine/metadata-modules/flat-entity/utils/get-metadata-name-from-flat-entity-maps-key.util.ts new file mode 100644 index 0000000000..cdc62dc8bb --- /dev/null +++ b/packages/twenty-server/src/engine/metadata-modules/flat-entity/utils/get-metadata-name-from-flat-entity-maps-key.util.ts @@ -0,0 +1,12 @@ +import { uncapitalize } from 'twenty-shared/utils'; + +import { type FlatEntityMapsKeyToMetadata } from 'src/engine/metadata-modules/flat-entity/types/flat-entity-maps-key-to-metadata'; + +export const getMetadataNameFromFlatEntityMapsKey = ( + flatEntityMapsKey: T, +): FlatEntityMapsKeyToMetadata => { + const withoutPrefix = flatEntityMapsKey.replace(/^flat/, ''); + const withoutSuffix = withoutPrefix.replace(/Maps$/, ''); + + return uncapitalize(withoutSuffix) as FlatEntityMapsKeyToMetadata; +}; diff --git a/packages/twenty-server/src/engine/metadata-modules/flat-entity/utils/get-metadata-related-metadata-names.util.ts b/packages/twenty-server/src/engine/metadata-modules/flat-entity/utils/get-metadata-related-metadata-names.util.ts new file mode 100644 index 0000000000..3f49220442 --- /dev/null +++ b/packages/twenty-server/src/engine/metadata-modules/flat-entity/utils/get-metadata-related-metadata-names.util.ts @@ -0,0 +1,26 @@ +import { type AllMetadataName } from 'twenty-shared/metadata'; + +import { ALL_METADATA_RELATIONS } from 'src/engine/metadata-modules/flat-entity/constant/all-metadata-relations.constant'; +import { type MetadataManyToOneRelatedMetadataNames } from 'src/engine/metadata-modules/flat-entity/types/metadata-many-to-one-related-metadata-names.type'; +import { type MetadataOneToManyRelatedMetadataNames } from 'src/engine/metadata-modules/flat-entity/types/metadata-one-to-many-related-metadata-names.type'; + +export const getMetadataRelatedMetadataNames = ( + metadataName: T, +) => { + const relationProperties = ALL_METADATA_RELATIONS[metadataName]; + + const manyToOneMetadataNames = Object.values(relationProperties.manyToOne) + .filter((relation) => relation !== null) + .map((relation) => relation.metadataName); + + const oneToManyMetadataNames = Object.values(relationProperties.oneToMany) + .filter((relation) => relation !== null) + .map((relation) => relation.metadataName); + + return [ + ...new Set([...manyToOneMetadataNames, ...oneToManyMetadataNames]), + ] as ( + | MetadataOneToManyRelatedMetadataNames + | MetadataManyToOneRelatedMetadataNames + )[]; +}; diff --git a/packages/twenty-server/src/engine/workspace-manager/workspace-migration-v2/workspace-migration-builder-v2/types/workspace-migration-v2.ts b/packages/twenty-server/src/engine/workspace-manager/workspace-migration-v2/workspace-migration-builder-v2/types/workspace-migration-v2.ts index 59482292fb..b1f9dcbca9 100644 --- a/packages/twenty-server/src/engine/workspace-manager/workspace-migration-v2/workspace-migration-builder-v2/types/workspace-migration-v2.ts +++ b/packages/twenty-server/src/engine/workspace-manager/workspace-migration-v2/workspace-migration-builder-v2/types/workspace-migration-v2.ts @@ -6,5 +6,6 @@ export type WorkspaceMigrationV2< > = { actions: TActions[]; workspaceId: string; + // TODO remove from workspaceMigration once we've refactored the actions to have metadata and action type grain relatedFlatEntityMapsKeys?: (keyof AllFlatEntityMaps)[]; }; diff --git a/packages/twenty-server/src/engine/workspace-manager/workspace-migration-v2/workspace-migration-runner-v2/services/workspace-migration-runner-v2.service.ts b/packages/twenty-server/src/engine/workspace-manager/workspace-migration-v2/workspace-migration-runner-v2/services/workspace-migration-runner-v2.service.ts index 6f890923e0..4a59813941 100644 --- a/packages/twenty-server/src/engine/workspace-manager/workspace-migration-v2/workspace-migration-runner-v2/services/workspace-migration-runner-v2.service.ts +++ b/packages/twenty-server/src/engine/workspace-manager/workspace-migration-v2/workspace-migration-runner-v2/services/workspace-migration-runner-v2.service.ts @@ -10,6 +10,9 @@ import { import { LoggerService } from 'src/engine/core-modules/logger/logger.service'; import { WorkspaceManyOrAllFlatEntityMapsCacheService } from 'src/engine/metadata-modules/flat-entity/services/workspace-many-or-all-flat-entity-maps-cache.service'; import { AllFlatEntityMaps } from 'src/engine/metadata-modules/flat-entity/types/all-flat-entity-maps.type'; +import { getMetadataFlatEntityMapsKey } from 'src/engine/metadata-modules/flat-entity/utils/get-metadata-flat-entity-maps-key.util'; +import { getMetadataNameFromFlatEntityMapsKey } from 'src/engine/metadata-modules/flat-entity/utils/get-metadata-name-from-flat-entity-maps-key.util'; +import { getMetadataRelatedMetadataNames } from 'src/engine/metadata-modules/flat-entity/utils/get-metadata-related-metadata-names.util'; import { FIND_ALL_CORE_VIEWS_GRAPHQL_OPERATION } from 'src/engine/metadata-modules/view/constants/find-all-core-views-graphql-operation.constant'; import { WorkspaceMetadataVersionService } from 'src/engine/metadata-modules/workspace-metadata-version/services/workspace-metadata-version.service'; import { WorkspaceCacheStorageService } from 'src/engine/workspace-cache-storage/workspace-cache-storage.service'; @@ -33,7 +36,7 @@ export class WorkspaceMigrationRunnerV2Service { private getLegacyCacheInvalidationPromises({ workspaceMigration: { actions, workspaceId }, }: { - workspaceMigration: WorkspaceMigrationV2; + workspaceMigration: Omit; }): Promise[] { const asyncOperations: Promise[] = []; const shouldIncrementMetadataGraphqlSchemaVersion = actions.some( @@ -178,8 +181,10 @@ export class WorkspaceMigrationRunnerV2Service { ) as (keyof AllFlatEntityMaps)[]; flatEntityMapsToInvalidate = [ - ...optimisticallyUpdatedFlatEntityMapsKeys, - ...flatEntityMapsToInvalidate, + ...new Set([ + ...optimisticallyUpdatedFlatEntityMapsKeys, + ...flatEntityMapsToInvalidate, + ]), ]; allFlatEntityMaps = { @@ -195,7 +200,10 @@ export class WorkspaceMigrationRunnerV2Service { const flatEntitiesCacheToInvalidate = [ ...new Set([ ...flatEntityMapsToInvalidate, - ...(relatedFlatEntityMapsKeys ?? []), + ...flatEntityMapsToInvalidate + .map(getMetadataNameFromFlatEntityMapsKey) + .flatMap(getMetadataRelatedMetadataNames) + .map(getMetadataFlatEntityMapsKey), ]), ]; @@ -207,18 +215,12 @@ export class WorkspaceMigrationRunnerV2Service { const invalidationResults = await Promise.allSettled([ this.flatEntityMapsCacheService.invalidateFlatEntityMaps({ workspaceId, - flatMapsKeys: [ - ...new Set([ - ...flatEntityMapsToInvalidate, - ...(relatedFlatEntityMapsKeys ?? []), - ]), - ], + flatMapsKeys: flatEntitiesCacheToInvalidate, }), ...this.getLegacyCacheInvalidationPromises({ workspaceMigration: { actions, workspaceId, - relatedFlatEntityMapsKeys, }, }), ]);