houseassassin/twenty
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactor runner dynamic cache invalidation (#16913)

Browse Source 
# Introduction
closes https://github.com/twentyhq/core-team-issues/issues/1792
Refactoring the cache to invalidate to be more precise and prevent any
corrupted cache occurrences

## Next
The load dependency cache from the workspace migration, that's not
optimal it should be smart enough to infer it from the actions
themselves. For the moment their definition isn't granular enough and
inferring such info would be very dirty
This commit is contained in:
Paul Rastoin
2026-01-03 02:51:07 +01:00
committed by GitHub
parent 42c9ae1ebc
commit 9b38254256
8 changed files with 252 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/twenty-server/src/engine/metadata-modules/flat-entity/types/flat-entity-maps-key-to-metadata.ts
+8
View File
@@ -0,0 +1,8 @@
import { type AllMetadataName } from 'twenty-shared/metadata';
export type FlatEntityMapsKeyToMetadata<T extends string> =
T extends `flat${infer Name}Maps`
? Uncapitalize<Name> extends AllMetadataName
? Uncapitalize<Name>
: never
: never;
packages/twenty-server/src/engine/metadata-modules/flat-entity/types/metadata-one-to-many-related-metadata-names.type.ts
+13
View File
@@ -0,0 +1,13 @@
import { type AllMetadataName } from 'twenty-shared/metadata';
import { type ALL_METADATA_RELATIONS } from 'src/engine/metadata-modules/flat-entity/constant/all-metadata-relations.constant';
type ExtractMetadataNames<T> = {
[K in keyof T]: T[K] extends { metadataName: infer M } ? M : never;
}[keyof T];
export type MetadataOneToManyRelatedMetadataNames<T extends AllMetadataName> =
Extract<
ExtractMetadataNames<(typeof ALL_METADATA_RELATIONS)[T]['oneToMany']>,
AllMetadataName
>;
packages/twenty-server/src/engine/metadata-modules/flat-entity/utils/__tests__/__snapshots__/get-metadata-related-metadata-names.util.spec.ts.snap
+145
View File
@@ -0,0 +1,145 @@
// Jest Snapshot v1, https://goo.gl/fbAQLP
exports[`getMetadataRelatedMetadataNames should return related metadata names for agent 1`] = `[]`;
exports[`getMetadataRelatedMetadataNames should return related metadata names for cronTrigger 1`] = `
[
"serverlessFunction",
]
`;
exports[`getMetadataRelatedMetadataNames should return related metadata names for databaseEventTrigger 1`] = `
[
"serverlessFunction",
]
`;
exports[`getMetadataRelatedMetadataNames should return related metadata names for fieldMetadata 1`] = `
[
"objectMetadata",
"fieldMetadata",
"viewField",
"viewFilter",
"view",
]
`;
exports[`getMetadataRelatedMetadataNames should return related metadata names for index 1`] = `
[
"objectMetadata",
]
`;
exports[`getMetadataRelatedMetadataNames should return related metadata names for objectMetadata 1`] = `
[
"fieldMetadata",
"index",
"view",
]
`;
exports[`getMetadataRelatedMetadataNames should return related metadata names for pageLayout 1`] = `
[
"objectMetadata",
"pageLayoutTab",
]
`;
exports[`getMetadataRelatedMetadataNames should return related metadata names for pageLayoutTab 1`] = `
[
"pageLayout",
"pageLayoutWidget",
]
`;
exports[`getMetadataRelatedMetadataNames should return related metadata names for pageLayoutWidget 1`] = `
[
"pageLayoutTab",
"objectMetadata",
]
`;
exports[`getMetadataRelatedMetadataNames should return related metadata names for role 1`] = `
[
"roleTarget",
]
`;
exports[`getMetadataRelatedMetadataNames should return related metadata names for roleTarget 1`] = `
[
"role",
]
`;
exports[`getMetadataRelatedMetadataNames should return related metadata names for routeTrigger 1`] = `
[
"serverlessFunction",
]
`;
exports[`getMetadataRelatedMetadataNames should return related metadata names for rowLevelPermissionPredicate 1`] = `
[
"role",
"fieldMetadata",
"objectMetadata",
"rowLevelPermissionPredicateGroup",
]
`;
exports[`getMetadataRelatedMetadataNames should return related metadata names for rowLevelPermissionPredicateGroup 1`] = `
[
"role",
"rowLevelPermissionPredicateGroup",
"rowLevelPermissionPredicate",
]
`;
exports[`getMetadataRelatedMetadataNames should return related metadata names for serverlessFunction 1`] = `
[
"cronTrigger",
"databaseEventTrigger",
"routeTrigger",
]
`;
exports[`getMetadataRelatedMetadataNames should return related metadata names for skill 1`] = `[]`;
exports[`getMetadataRelatedMetadataNames should return related metadata names for view 1`] = `
[
"objectMetadata",
"fieldMetadata",
"viewField",
"viewFilter",
"viewFilterGroup",
"viewGroup",
]
`;
exports[`getMetadataRelatedMetadataNames should return related metadata names for viewField 1`] = `
[
"fieldMetadata",
"view",
]
`;
exports[`getMetadataRelatedMetadataNames should return related metadata names for viewFilter 1`] = `
[
"fieldMetadata",
"view",
"viewFilterGroup",
]
`;
exports[`getMetadataRelatedMetadataNames should return related metadata names for viewFilterGroup 1`] = `
[
"viewFilterGroup",
"view",
"viewFilter",
]
`;
exports[`getMetadataRelatedMetadataNames should return related metadata names for viewGroup 1`] = `
[
"view",
]
`;
packages/twenty-server/src/engine/metadata-modules/flat-entity/utils/__tests__/get-metadata-related-metadata-names.util.spec.ts
+34
View File
@@ -0,0 +1,34 @@
import {
ALL_METADATA_NAME,
type AllMetadataName,
} from 'twenty-shared/metadata';
import {
type EachTestingContext,
eachTestingContextFilter,
} from 'twenty-shared/testing';
import { getMetadataRelatedMetadataNames } from 'src/engine/metadata-modules/flat-entity/utils/get-metadata-related-metadata-names.util';
type TestContext = {
input: AllMetadataName;
};
const testCases: EachTestingContext<TestContext>[] = (
Object.keys(ALL_METADATA_NAME) as (keyof typeof ALL_METADATA_NAME)[]
).map((metadataName) => ({
title: `should return related metadata names for ${metadataName}`,
context: {
input: metadataName,
},
}));
describe('getMetadataRelatedMetadataNames', () => {
test.each(eachTestingContextFilter(testCases))(
'$title',
({ context: { input } }) => {
const result = getMetadataRelatedMetadataNames(input);
expect(result).toMatchSnapshot();
},
);
});
packages/twenty-server/src/engine/metadata-modules/flat-entity/utils/get-metadata-name-from-flat-entity-maps-key.util.ts
+12
View File
@@ -0,0 +1,12 @@
import { uncapitalize } from 'twenty-shared/utils';
import { type FlatEntityMapsKeyToMetadata } from 'src/engine/metadata-modules/flat-entity/types/flat-entity-maps-key-to-metadata';
export const getMetadataNameFromFlatEntityMapsKey = <T extends string>(
flatEntityMapsKey: T,
): FlatEntityMapsKeyToMetadata<T> => {
const withoutPrefix = flatEntityMapsKey.replace(/^flat/, '');
const withoutSuffix = withoutPrefix.replace(/Maps$/, '');
return uncapitalize(withoutSuffix) as FlatEntityMapsKeyToMetadata<T>;
};
packages/twenty-server/src/engine/metadata-modules/flat-entity/utils/get-metadata-related-metadata-names.util.ts
+26
View File
@@ -0,0 +1,26 @@
import { type AllMetadataName } from 'twenty-shared/metadata';
import { ALL_METADATA_RELATIONS } from 'src/engine/metadata-modules/flat-entity/constant/all-metadata-relations.constant';
import { type MetadataManyToOneRelatedMetadataNames } from 'src/engine/metadata-modules/flat-entity/types/metadata-many-to-one-related-metadata-names.type';
import { type MetadataOneToManyRelatedMetadataNames } from 'src/engine/metadata-modules/flat-entity/types/metadata-one-to-many-related-metadata-names.type';
export const getMetadataRelatedMetadataNames = <T extends AllMetadataName>(
metadataName: T,
) => {
const relationProperties = ALL_METADATA_RELATIONS[metadataName];
const manyToOneMetadataNames = Object.values(relationProperties.manyToOne)
.filter((relation) => relation !== null)
.map((relation) => relation.metadataName);
const oneToManyMetadataNames = Object.values(relationProperties.oneToMany)
.filter((relation) => relation !== null)
.map((relation) => relation.metadataName);
return [
...new Set([...manyToOneMetadataNames, ...oneToManyMetadataNames]),
] as (
| MetadataOneToManyRelatedMetadataNames<T>
| MetadataManyToOneRelatedMetadataNames<T>
)[];
};
packages/twenty-server/src/engine/workspace-manager/workspace-migration-v2/workspace-migration-builder-v2/types/workspace-migration-v2.ts
+1
View File
@@ -6,5 +6,6 @@ export type WorkspaceMigrationV2<
> = {
actions: TActions[];
workspaceId: string;
// TODO remove from workspaceMigration once we've refactored the actions to have metadata and action type grain
relatedFlatEntityMapsKeys?: (keyof AllFlatEntityMaps)[];
};
packages/twenty-server/src/engine/workspace-manager/workspace-migration-v2/workspace-migration-runner-v2/services/workspace-migration-runner-v2.service.ts
+13 -11
View File
@@ -10,6 +10,9 @@ import {
import { LoggerService } from 'src/engine/core-modules/logger/logger.service';
import { WorkspaceManyOrAllFlatEntityMapsCacheService } from 'src/engine/metadata-modules/flat-entity/services/workspace-many-or-all-flat-entity-maps-cache.service';
import { AllFlatEntityMaps } from 'src/engine/metadata-modules/flat-entity/types/all-flat-entity-maps.type';
import { getMetadataFlatEntityMapsKey } from 'src/engine/metadata-modules/flat-entity/utils/get-metadata-flat-entity-maps-key.util';
import { getMetadataNameFromFlatEntityMapsKey } from 'src/engine/metadata-modules/flat-entity/utils/get-metadata-name-from-flat-entity-maps-key.util';
import { getMetadataRelatedMetadataNames } from 'src/engine/metadata-modules/flat-entity/utils/get-metadata-related-metadata-names.util';
import { FIND_ALL_CORE_VIEWS_GRAPHQL_OPERATION } from 'src/engine/metadata-modules/view/constants/find-all-core-views-graphql-operation.constant';
import { WorkspaceMetadataVersionService } from 'src/engine/metadata-modules/workspace-metadata-version/services/workspace-metadata-version.service';
import { WorkspaceCacheStorageService } from 'src/engine/workspace-cache-storage/workspace-cache-storage.service';
@@ -33,7 +36,7 @@ export class WorkspaceMigrationRunnerV2Service {
private getLegacyCacheInvalidationPromises({
workspaceMigration: { actions, workspaceId },
}: {
workspaceMigration: WorkspaceMigrationV2;
workspaceMigration: Omit<WorkspaceMigrationV2, 'relatedFlatEntityMapsKeys'>;
}): Promise<void>[] {
const asyncOperations: Promise<void>[] = [];
const shouldIncrementMetadataGraphqlSchemaVersion = actions.some(
@@ -178,8 +181,10 @@ export class WorkspaceMigrationRunnerV2Service {
) as (keyof AllFlatEntityMaps)[];
flatEntityMapsToInvalidate = [
...optimisticallyUpdatedFlatEntityMapsKeys,
...flatEntityMapsToInvalidate,
...new Set([
...optimisticallyUpdatedFlatEntityMapsKeys,
...flatEntityMapsToInvalidate,
]),
];
allFlatEntityMaps = {
@@ -195,7 +200,10 @@ export class WorkspaceMigrationRunnerV2Service {
const flatEntitiesCacheToInvalidate = [
...new Set([
...flatEntityMapsToInvalidate,
...(relatedFlatEntityMapsKeys ?? []),
...flatEntityMapsToInvalidate
.map(getMetadataNameFromFlatEntityMapsKey)
.flatMap(getMetadataRelatedMetadataNames)
.map(getMetadataFlatEntityMapsKey),
]),
];
@@ -207,18 +215,12 @@ export class WorkspaceMigrationRunnerV2Service {
const invalidationResults = await Promise.allSettled([
this.flatEntityMapsCacheService.invalidateFlatEntityMaps({
workspaceId,
flatMapsKeys: [
...new Set([
...flatEntityMapsToInvalidate,
...(relatedFlatEntityMapsKeys ?? []),
]),
],
flatMapsKeys: flatEntitiesCacheToInvalidate,
}),
...this.getLegacyCacheInvalidationPromises({
workspaceMigration: {
actions,
workspaceId,
relatedFlatEntityMapsKeys,
},
}),
]);