Add local only cache to cache service and cache typeorm entity metadata (#16287)
## Problem buildEntityMetadatas in GlobalWorkspaceOrmManager is computationally expensive and was running on every executeInWorkspaceContext call. This method uses TypeORM's EntitySchemaTransformer and EntityMetadataBuilder to build metadata for all workspace entities (30-50+ objects with many fields each). The resulting EntityMetadata[] is not serialisable which means it cannot be cached in Redis because they contain: - Circular references - Functions/methods - References to the DataSource instance ## Solution Extended the workspace cache system to support local-only caching, then created a cache provider for entityMetadatas. ## Implementation details Updated @WorkspaceCache decorator (workspace-cache.decorator.ts) - Added localOnly?: boolean option to skip Redis storage for non-serializable data Created WorkspaceEntityMetadatasCacheService - Computes entity metadatas from DB to avoid race condition, this is acceptable Simplified GlobalWorkspaceOrmManager - Now fetches entityMetadatas from cache instead of rebuilding on every call Updated Workspace migration runner - the only entry point where metadata can change - Now invalidate the new 'entityMetadata' local cache when shouldIncrementMetadataGraphqlSchemaVersion is true (== field/object mutations)
This commit is contained in:
+27
-106
@@ -30,7 +30,6 @@ import { WorkspacePreQueryHookPayload } from 'src/engine/api/graphql/workspace-q
|
||||
import { WorkspaceQueryHookService } from 'src/engine/api/graphql/workspace-query-runner/workspace-query-hook/workspace-query-hook.service';
|
||||
import { ApiKeyRoleService } from 'src/engine/core-modules/api-key/services/api-key-role.service';
|
||||
import { AuthContext } from 'src/engine/core-modules/auth/types/auth-context.type';
|
||||
import { FeatureFlagKey } from 'src/engine/core-modules/feature-flag/enums/feature-flag-key.enum';
|
||||
import { FeatureFlagService } from 'src/engine/core-modules/feature-flag/services/feature-flag.service';
|
||||
import { MetricsService } from 'src/engine/core-modules/metrics/metrics.service';
|
||||
import { MetricsKeys } from 'src/engine/core-modules/metrics/types/metrics-keys.type';
|
||||
@@ -132,32 +131,15 @@ export abstract class CommonBaseQueryRunnerService<
|
||||
commonQueryParser,
|
||||
);
|
||||
|
||||
const isGlobalDatasourceEnabled =
|
||||
await this.featureFlagService.isFeatureEnabled(
|
||||
FeatureFlagKey.IS_GLOBAL_WORKSPACE_DATASOURCE_ENABLED,
|
||||
authContext.workspace.id,
|
||||
);
|
||||
|
||||
if (isGlobalDatasourceEnabled) {
|
||||
return this.globalWorkspaceOrmManager.executeInWorkspaceContext(
|
||||
authContext,
|
||||
async () =>
|
||||
this.executeQueryAndEnrichResults(
|
||||
processedArgs,
|
||||
authContext,
|
||||
queryRunnerContext,
|
||||
commonQueryParser,
|
||||
isGlobalDatasourceEnabled,
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
return this.executeQueryAndEnrichResults(
|
||||
processedArgs,
|
||||
return this.globalWorkspaceOrmManager.executeInWorkspaceContext(
|
||||
authContext,
|
||||
queryRunnerContext,
|
||||
commonQueryParser,
|
||||
isGlobalDatasourceEnabled,
|
||||
async () =>
|
||||
this.executeQueryAndEnrichResults(
|
||||
processedArgs,
|
||||
authContext,
|
||||
queryRunnerContext,
|
||||
commonQueryParser,
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
@@ -217,17 +199,12 @@ export abstract class CommonBaseQueryRunnerService<
|
||||
authContext: WorkspaceAuthContext,
|
||||
queryRunnerContext: CommonBaseQueryRunnerContext,
|
||||
commonQueryParser: GraphqlQueryParser,
|
||||
isGlobalDatasourceEnabled: boolean,
|
||||
): Promise<Output> {
|
||||
const extendedQueryRunnerContext = isGlobalDatasourceEnabled
|
||||
? await this.prepareExtendedQueryRunnerContextWithGlobalDatasource(
|
||||
authContext,
|
||||
queryRunnerContext,
|
||||
)
|
||||
: await this.prepareExtendedQueryRunnerContext(
|
||||
authContext,
|
||||
queryRunnerContext,
|
||||
);
|
||||
const extendedQueryRunnerContext =
|
||||
await this.prepareExtendedQueryRunnerContextWithGlobalDatasource(
|
||||
authContext,
|
||||
queryRunnerContext,
|
||||
);
|
||||
|
||||
const results = await this.run(processedArgs, {
|
||||
...extendedQueryRunnerContext,
|
||||
@@ -312,81 +289,28 @@ export abstract class CommonBaseQueryRunnerService<
|
||||
}
|
||||
}
|
||||
|
||||
private async getRoleIdAndObjectsPermissions(
|
||||
private async getRoleIdOrThrow(
|
||||
authContext: AuthContext,
|
||||
workspaceId: string,
|
||||
) {
|
||||
let roleId: string;
|
||||
|
||||
if (
|
||||
!isDefined(authContext.apiKey) &&
|
||||
!isDefined(authContext.userWorkspaceId)
|
||||
) {
|
||||
throw new PermissionsException(
|
||||
PermissionsExceptionMessage.NO_AUTHENTICATION_CONTEXT,
|
||||
PermissionsExceptionCode.NO_AUTHENTICATION_CONTEXT,
|
||||
);
|
||||
}
|
||||
|
||||
): Promise<string> {
|
||||
if (isDefined(authContext.apiKey)) {
|
||||
roleId = await this.apiKeyRoleService.getRoleIdForApiKey(
|
||||
return this.apiKeyRoleService.getRoleIdForApiKey(
|
||||
authContext.apiKey.id,
|
||||
workspaceId,
|
||||
);
|
||||
} else {
|
||||
const userWorkspaceRoleId =
|
||||
await this.userRoleService.getRoleIdForUserWorkspace({
|
||||
userWorkspaceId: authContext.userWorkspaceId,
|
||||
workspaceId,
|
||||
});
|
||||
|
||||
if (!isDefined(userWorkspaceRoleId)) {
|
||||
throw new PermissionsException(
|
||||
PermissionsExceptionMessage.NO_ROLE_FOUND_FOR_USER_WORKSPACE,
|
||||
PermissionsExceptionCode.NO_ROLE_FOUND_FOR_USER_WORKSPACE,
|
||||
);
|
||||
}
|
||||
|
||||
roleId = userWorkspaceRoleId;
|
||||
}
|
||||
|
||||
const { rolesPermissions } =
|
||||
await this.workspaceCacheService.getOrRecompute(workspaceId, [
|
||||
'rolesPermissions',
|
||||
]);
|
||||
if (!isDefined(authContext.userWorkspaceId)) {
|
||||
throw new CommonQueryRunnerException(
|
||||
'Invalid auth context',
|
||||
CommonQueryRunnerExceptionCode.INVALID_AUTH_CONTEXT,
|
||||
);
|
||||
}
|
||||
|
||||
return { roleId, objectsPermissions: rolesPermissions[roleId] };
|
||||
}
|
||||
|
||||
private async prepareExtendedQueryRunnerContext(
|
||||
authContext: WorkspaceAuthContext,
|
||||
queryRunnerContext: CommonBaseQueryRunnerContext,
|
||||
): Promise<Omit<CommonExtendedQueryRunnerContext, 'commonQueryParser'>> {
|
||||
const workspaceDataSource =
|
||||
await this.twentyORMGlobalManager.getDataSourceForWorkspace({
|
||||
workspaceId: authContext.workspace.id,
|
||||
});
|
||||
|
||||
const { roleId } = await this.getRoleIdAndObjectsPermissions(
|
||||
authContext,
|
||||
authContext.workspace.id,
|
||||
);
|
||||
|
||||
const rolePermissionConfig = { unionOf: [roleId] };
|
||||
|
||||
const repository = workspaceDataSource.getRepository(
|
||||
queryRunnerContext.flatObjectMetadata.nameSingular,
|
||||
rolePermissionConfig,
|
||||
authContext,
|
||||
);
|
||||
|
||||
return {
|
||||
...queryRunnerContext,
|
||||
authContext,
|
||||
workspaceDataSource,
|
||||
rolePermissionConfig,
|
||||
repository,
|
||||
};
|
||||
return this.userRoleService.getRoleIdForUserWorkspace({
|
||||
userWorkspaceId: authContext.userWorkspaceId,
|
||||
workspaceId,
|
||||
});
|
||||
}
|
||||
|
||||
private async prepareExtendedQueryRunnerContextWithGlobalDatasource(
|
||||
@@ -395,10 +319,7 @@ export abstract class CommonBaseQueryRunnerService<
|
||||
): Promise<Omit<CommonExtendedQueryRunnerContext, 'commonQueryParser'>> {
|
||||
const workspaceId = authContext.workspace.id;
|
||||
|
||||
const { roleId } = await this.getRoleIdAndObjectsPermissions(
|
||||
authContext,
|
||||
workspaceId,
|
||||
);
|
||||
const roleId = await this.getRoleIdOrThrow(authContext, workspaceId);
|
||||
|
||||
const rolePermissionConfig = { unionOf: [roleId] };
|
||||
|
||||
|
||||
+3
-1
@@ -156,7 +156,9 @@ describe('McpProtocolService', () => {
|
||||
});
|
||||
|
||||
it('should throw when role ID is missing', async () => {
|
||||
userRoleService.getRoleIdForUserWorkspace.mockResolvedValue(undefined);
|
||||
userRoleService.getRoleIdForUserWorkspace.mockResolvedValue(
|
||||
undefined as unknown as string,
|
||||
);
|
||||
|
||||
await expect(
|
||||
service.getRoleId('workspace-1', 'user-workspace-1'),
|
||||
|
||||
@@ -99,6 +99,13 @@ export abstract class RestApiBaseHandler {
|
||||
authContext.workspace.id,
|
||||
);
|
||||
} else {
|
||||
if (!isDefined(authContext.userWorkspaceId)) {
|
||||
throw new PermissionsException(
|
||||
'No user workspace ID found in authentication context',
|
||||
PermissionsExceptionCode.NO_AUTHENTICATION_CONTEXT,
|
||||
);
|
||||
}
|
||||
|
||||
const userWorkspaceRoleId =
|
||||
await this.userRoleService.getRoleIdForUserWorkspace({
|
||||
userWorkspaceId: authContext.userWorkspaceId,
|
||||
|
||||
+2
-1
@@ -1,6 +1,7 @@
|
||||
import { Injectable } from '@nestjs/common';
|
||||
import { InjectRepository } from '@nestjs/typeorm';
|
||||
|
||||
import { isDefined } from 'twenty-shared/utils';
|
||||
import { In, IsNull, Not, Repository } from 'typeorm';
|
||||
|
||||
import { ApiKeyEntity } from 'src/engine/core-modules/api-key/api-key.entity';
|
||||
@@ -69,7 +70,7 @@ export class ApiKeyRoleService {
|
||||
|
||||
const roleId = apiKeyRoleMap[apiKeyId];
|
||||
|
||||
if (!roleId) {
|
||||
if (!isDefined(roleId)) {
|
||||
throw new ApiKeyException(
|
||||
`API key ${apiKeyId} has no role assigned`,
|
||||
ApiKeyExceptionCode.API_KEY_NO_ROLE_ASSIGNED,
|
||||
|
||||
@@ -63,18 +63,23 @@ export class UserRoleService {
|
||||
userWorkspaceId,
|
||||
}: {
|
||||
workspaceId: string;
|
||||
userWorkspaceId?: string;
|
||||
}): Promise<string | undefined> {
|
||||
if (!isDefined(userWorkspaceId)) {
|
||||
return;
|
||||
}
|
||||
|
||||
userWorkspaceId: string;
|
||||
}): Promise<string> {
|
||||
const { userWorkspaceRoleMap } =
|
||||
await this.workspaceCacheService.getOrRecompute(workspaceId, [
|
||||
'userWorkspaceRoleMap',
|
||||
]);
|
||||
|
||||
return userWorkspaceRoleMap[userWorkspaceId];
|
||||
const roleId = userWorkspaceRoleMap[userWorkspaceId];
|
||||
|
||||
if (!isDefined(roleId)) {
|
||||
throw new PermissionsException(
|
||||
`User workspace ${userWorkspaceId} has no role assigned`,
|
||||
PermissionsExceptionCode.NO_ROLE_FOUND_FOR_USER_WORKSPACE,
|
||||
);
|
||||
}
|
||||
|
||||
return roleId;
|
||||
}
|
||||
|
||||
public async getRolesByUserWorkspaces({
|
||||
|
||||
+11
-11
@@ -9,19 +9,20 @@ import { type ColumnType, type EntitySchemaColumnOptions } from 'typeorm';
|
||||
|
||||
import { RelationType } from 'src/engine/metadata-modules/field-metadata/interfaces/relation-type.interface';
|
||||
|
||||
import { getFlatFieldsFromFlatObjectMetadata } from 'src/engine/api/graphql/workspace-schema-builder/utils/get-flat-fields-for-flat-object-metadata.util';
|
||||
import { computeCompositeColumnName } from 'src/engine/metadata-modules/field-metadata/utils/compute-column-name.util';
|
||||
import { isCompositeFieldMetadataType } from 'src/engine/metadata-modules/field-metadata/utils/is-composite-field-metadata-type.util';
|
||||
import { isEnumFieldMetadataType } from 'src/engine/metadata-modules/field-metadata/utils/is-enum-field-metadata-type.util';
|
||||
import { serializeDefaultValue } from 'src/engine/metadata-modules/field-metadata/utils/serialize-default-value';
|
||||
import { type FlatEntityMaps } from 'src/engine/metadata-modules/flat-entity/types/flat-entity-maps.type';
|
||||
import { type FlatFieldMetadata } from 'src/engine/metadata-modules/flat-field-metadata/types/flat-field-metadata.type';
|
||||
import { type FlatObjectMetadata } from 'src/engine/metadata-modules/flat-object-metadata/types/flat-object-metadata.type';
|
||||
import { fieldMetadataTypeToColumnType } from 'src/engine/metadata-modules/workspace-migration/utils/field-metadata-type-to-column-type.util';
|
||||
import {
|
||||
TwentyORMException,
|
||||
TwentyORMExceptionCode,
|
||||
} from 'src/engine/twenty-orm/exceptions/twenty-orm.exception';
|
||||
import {
|
||||
type EntitySchemaFieldMetadata,
|
||||
type EntitySchemaFieldMetadataMaps,
|
||||
type EntitySchemaObjectMetadata,
|
||||
} from 'src/engine/twenty-orm/global-workspace-datasource/types/entity-schema-metadata.type';
|
||||
import { isFieldMetadataEntityOfType } from 'src/engine/utils/is-field-metadata-of-type.util';
|
||||
|
||||
type EntitySchemaColumnMap = {
|
||||
@@ -31,15 +32,14 @@ type EntitySchemaColumnMap = {
|
||||
@Injectable()
|
||||
export class EntitySchemaColumnFactory {
|
||||
create(
|
||||
flatObjectMetadata: FlatObjectMetadata,
|
||||
flatFieldMetadataMaps: FlatEntityMaps<FlatFieldMetadata>,
|
||||
objectMetadata: EntitySchemaObjectMetadata,
|
||||
fieldMetadataMaps: EntitySchemaFieldMetadataMaps,
|
||||
): EntitySchemaColumnMap {
|
||||
let entitySchemaColumnMap: EntitySchemaColumnMap = {};
|
||||
|
||||
const fieldMetadataCollection = getFlatFieldsFromFlatObjectMetadata(
|
||||
flatObjectMetadata,
|
||||
flatFieldMetadataMaps,
|
||||
);
|
||||
const fieldMetadataCollection = objectMetadata.fieldMetadataIds
|
||||
.map((fieldId) => fieldMetadataMaps.byId[fieldId])
|
||||
.filter(isDefined);
|
||||
|
||||
for (const fieldMetadata of fieldMetadataCollection) {
|
||||
const key = fieldMetadata.name;
|
||||
@@ -121,7 +121,7 @@ export class EntitySchemaColumnFactory {
|
||||
}
|
||||
|
||||
private createCompositeColumns(
|
||||
fieldMetadata: FlatFieldMetadata,
|
||||
fieldMetadata: EntitySchemaFieldMetadata,
|
||||
): EntitySchemaColumnMap {
|
||||
const entitySchemaColumnMap: EntitySchemaColumnMap = {};
|
||||
const compositeType = compositeTypeDefinitions.get(fieldMetadata.type);
|
||||
|
||||
+38
-40
@@ -1,70 +1,68 @@
|
||||
import { Injectable } from '@nestjs/common';
|
||||
|
||||
import { FieldMetadataType } from 'twenty-shared/types';
|
||||
import { isDefined } from 'twenty-shared/utils';
|
||||
import { type EntitySchemaRelationOptions } from 'typeorm';
|
||||
|
||||
import { getFlatFieldsFromFlatObjectMetadata } from 'src/engine/api/graphql/workspace-schema-builder/utils/get-flat-fields-for-flat-object-metadata.util';
|
||||
import { type FlatEntityMaps } from 'src/engine/metadata-modules/flat-entity/types/flat-entity-maps.type';
|
||||
import { type FlatFieldMetadata } from 'src/engine/metadata-modules/flat-field-metadata/types/flat-field-metadata.type';
|
||||
import { isMorphOrRelationFlatFieldMetadata } from 'src/engine/metadata-modules/flat-field-metadata/utils/is-morph-or-relation-flat-field-metadata.util';
|
||||
import { type FlatObjectMetadata } from 'src/engine/metadata-modules/flat-object-metadata/types/flat-object-metadata.type';
|
||||
import {
|
||||
type EntitySchemaFieldMetadata,
|
||||
type EntitySchemaFieldMetadataMaps,
|
||||
type EntitySchemaObjectMetadata,
|
||||
type EntitySchemaObjectMetadataMaps,
|
||||
} from 'src/engine/twenty-orm/global-workspace-datasource/types/entity-schema-metadata.type';
|
||||
import { determineSchemaRelationDetails } from 'src/engine/twenty-orm/utils/determine-schema-relation-details.util';
|
||||
|
||||
type EntitySchemaRelationMap = {
|
||||
[key: string]: EntitySchemaRelationOptions;
|
||||
};
|
||||
|
||||
type RelationFieldMetadata = EntitySchemaFieldMetadata<
|
||||
FieldMetadataType.RELATION | FieldMetadataType.MORPH_RELATION
|
||||
>;
|
||||
|
||||
@Injectable()
|
||||
export class EntitySchemaRelationFactory {
|
||||
constructor() {}
|
||||
|
||||
create(
|
||||
flatObjectMetadata: FlatObjectMetadata,
|
||||
flatObjectMetadataMaps: FlatEntityMaps<FlatObjectMetadata>,
|
||||
flatFieldMetadataMaps: FlatEntityMaps<FlatFieldMetadata>,
|
||||
objectMetadata: EntitySchemaObjectMetadata,
|
||||
objectMetadataMaps: EntitySchemaObjectMetadataMaps,
|
||||
fieldMetadataMaps: EntitySchemaFieldMetadataMaps,
|
||||
): EntitySchemaRelationMap {
|
||||
const entitySchemaRelationMap: EntitySchemaRelationMap = {};
|
||||
|
||||
const flatFieldMetadatas = getFlatFieldsFromFlatObjectMetadata(
|
||||
flatObjectMetadata,
|
||||
flatFieldMetadataMaps,
|
||||
);
|
||||
const fieldMetadatas = objectMetadata.fieldMetadataIds
|
||||
.map((fieldId) => fieldMetadataMaps.byId[fieldId])
|
||||
.filter(isDefined);
|
||||
|
||||
for (const flatFieldMetadata of flatFieldMetadatas) {
|
||||
if (!isMorphOrRelationFlatFieldMetadata(flatFieldMetadata)) {
|
||||
for (const fieldMetadata of fieldMetadatas) {
|
||||
if (!this.isRelationField(fieldMetadata)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!flatFieldMetadata.settings) {
|
||||
throw new Error(
|
||||
`Field metadata settings are missing for field ${flatFieldMetadata.name}`,
|
||||
);
|
||||
}
|
||||
|
||||
const schemaRelationDetails = determineSchemaRelationDetails(
|
||||
flatFieldMetadata,
|
||||
flatObjectMetadataMaps,
|
||||
flatFieldMetadataMaps,
|
||||
const relationDetails = determineSchemaRelationDetails(
|
||||
fieldMetadata,
|
||||
objectMetadataMaps,
|
||||
fieldMetadataMaps,
|
||||
);
|
||||
|
||||
const targetObjectMetadata =
|
||||
flatObjectMetadataMaps.byId[
|
||||
flatFieldMetadata.relationTargetObjectMetadataId
|
||||
];
|
||||
|
||||
if (!targetObjectMetadata) {
|
||||
throw new Error(
|
||||
`Target object metadata not found for field ${flatFieldMetadata.name}`,
|
||||
);
|
||||
}
|
||||
|
||||
entitySchemaRelationMap[flatFieldMetadata.name] = {
|
||||
type: schemaRelationDetails.relationType,
|
||||
target: schemaRelationDetails.target,
|
||||
inverseSide: schemaRelationDetails.inverseSide,
|
||||
joinColumn: schemaRelationDetails.joinColumn,
|
||||
entitySchemaRelationMap[fieldMetadata.name] = {
|
||||
type: relationDetails.relationType,
|
||||
target: relationDetails.target,
|
||||
inverseSide: relationDetails.inverseSide,
|
||||
joinColumn: relationDetails.joinColumn,
|
||||
} satisfies EntitySchemaRelationOptions;
|
||||
}
|
||||
|
||||
return entitySchemaRelationMap;
|
||||
}
|
||||
|
||||
private isRelationField(
|
||||
fieldMetadata: EntitySchemaFieldMetadata,
|
||||
): fieldMetadata is RelationFieldMetadata {
|
||||
return (
|
||||
fieldMetadata.type === FieldMetadataType.RELATION ||
|
||||
fieldMetadata.type === FieldMetadataType.MORPH_RELATION
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2,11 +2,13 @@ import { Injectable } from '@nestjs/common';
|
||||
|
||||
import { EntitySchema } from 'typeorm';
|
||||
|
||||
import { type FlatEntityMaps } from 'src/engine/metadata-modules/flat-entity/types/flat-entity-maps.type';
|
||||
import { type FlatFieldMetadata } from 'src/engine/metadata-modules/flat-field-metadata/types/flat-field-metadata.type';
|
||||
import { type FlatObjectMetadata } from 'src/engine/metadata-modules/flat-object-metadata/types/flat-object-metadata.type';
|
||||
import { EntitySchemaColumnFactory } from 'src/engine/twenty-orm/factories/entity-schema-column.factory';
|
||||
import { EntitySchemaRelationFactory } from 'src/engine/twenty-orm/factories/entity-schema-relation.factory';
|
||||
import {
|
||||
type EntitySchemaFieldMetadataMaps,
|
||||
type EntitySchemaObjectMetadata,
|
||||
type EntitySchemaObjectMetadataMaps,
|
||||
} from 'src/engine/twenty-orm/global-workspace-datasource/types/entity-schema-metadata.type';
|
||||
import { computeTableName } from 'src/engine/utils/compute-table-name.util';
|
||||
import { getWorkspaceSchemaName } from 'src/engine/workspace-datasource/utils/get-workspace-schema-name.util';
|
||||
|
||||
@@ -19,28 +21,28 @@ export class EntitySchemaFactory {
|
||||
|
||||
create(
|
||||
workspaceId: string,
|
||||
flatObjectMetadata: FlatObjectMetadata,
|
||||
flatObjectMetadataMaps: FlatEntityMaps<FlatObjectMetadata>,
|
||||
flatFieldMetadataMaps: FlatEntityMaps<FlatFieldMetadata>,
|
||||
objectMetadata: EntitySchemaObjectMetadata,
|
||||
objectMetadataMaps: EntitySchemaObjectMetadataMaps,
|
||||
fieldMetadataMaps: EntitySchemaFieldMetadataMaps,
|
||||
): EntitySchema {
|
||||
const columns = this.entitySchemaColumnFactory.create(
|
||||
flatObjectMetadata,
|
||||
flatFieldMetadataMaps,
|
||||
objectMetadata,
|
||||
fieldMetadataMaps,
|
||||
);
|
||||
|
||||
const relations = this.entitySchemaRelationFactory.create(
|
||||
flatObjectMetadata,
|
||||
flatObjectMetadataMaps,
|
||||
flatFieldMetadataMaps,
|
||||
objectMetadata,
|
||||
objectMetadataMaps,
|
||||
fieldMetadataMaps,
|
||||
);
|
||||
|
||||
const schemaName = getWorkspaceSchemaName(workspaceId);
|
||||
|
||||
const entitySchema = new EntitySchema({
|
||||
name: flatObjectMetadata.nameSingular,
|
||||
name: objectMetadata.nameSingular,
|
||||
tableName: computeTableName(
|
||||
flatObjectMetadata.nameSingular,
|
||||
flatObjectMetadata.isCustom,
|
||||
objectMetadata.nameSingular,
|
||||
objectMetadata.isCustom,
|
||||
),
|
||||
columns,
|
||||
relations,
|
||||
|
||||
+9
-1
@@ -5,13 +5,16 @@ import { FeatureFlagModule } from 'src/engine/core-modules/feature-flag/feature-
|
||||
import { TwentyConfigModule } from 'src/engine/core-modules/twenty-config/twenty-config.module';
|
||||
import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity';
|
||||
import { DataSourceModule } from 'src/engine/metadata-modules/data-source/data-source.module';
|
||||
import { FieldMetadataEntity } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity';
|
||||
import { WorkspaceManyOrAllFlatEntityMapsCacheModule } from 'src/engine/metadata-modules/flat-entity/services/workspace-many-or-all-flat-entity-maps-cache.module';
|
||||
import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity';
|
||||
import { WorkspaceFeatureFlagsMapCacheModule } from 'src/engine/metadata-modules/workspace-feature-flags-map-cache/workspace-feature-flags-map-cache.module';
|
||||
import { EntitySchemaColumnFactory } from 'src/engine/twenty-orm/factories/entity-schema-column.factory';
|
||||
import { EntitySchemaRelationFactory } from 'src/engine/twenty-orm/factories/entity-schema-relation.factory';
|
||||
import { EntitySchemaFactory } from 'src/engine/twenty-orm/factories/entity-schema.factory';
|
||||
import { GlobalWorkspaceDataSourceService } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-datasource.service';
|
||||
import { GlobalWorkspaceOrmManager } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager';
|
||||
import { WorkspaceORMEntityMetadatasCacheService } from 'src/engine/twenty-orm/global-workspace-datasource/workspace-orm-entity-metadatas-cache.service';
|
||||
import { WorkspaceCacheStorageModule } from 'src/engine/workspace-cache-storage/workspace-cache-storage.module';
|
||||
import { WorkspaceCacheModule } from 'src/engine/workspace-cache/workspace-cache.module';
|
||||
import { WorkspaceEventEmitterModule } from 'src/engine/workspace-event-emitter/workspace-event-emitter.module';
|
||||
@@ -19,7 +22,11 @@ import { WorkspaceEventEmitterModule } from 'src/engine/workspace-event-emitter/
|
||||
@Global()
|
||||
@Module({
|
||||
imports: [
|
||||
TypeOrmModule.forFeature([WorkspaceEntity]),
|
||||
TypeOrmModule.forFeature([
|
||||
WorkspaceEntity,
|
||||
ObjectMetadataEntity,
|
||||
FieldMetadataEntity,
|
||||
]),
|
||||
DataSourceModule,
|
||||
WorkspaceCacheStorageModule,
|
||||
WorkspaceManyOrAllFlatEntityMapsCacheModule,
|
||||
@@ -35,6 +42,7 @@ import { WorkspaceEventEmitterModule } from 'src/engine/workspace-event-emitter/
|
||||
EntitySchemaFactory,
|
||||
EntitySchemaColumnFactory,
|
||||
EntitySchemaRelationFactory,
|
||||
WorkspaceORMEntityMetadatasCacheService,
|
||||
],
|
||||
exports: [GlobalWorkspaceDataSourceService, GlobalWorkspaceOrmManager],
|
||||
})
|
||||
|
||||
+3
-49
@@ -1,18 +1,10 @@
|
||||
import { Injectable, type Type } from '@nestjs/common';
|
||||
|
||||
import { isDefined } from 'twenty-shared/utils';
|
||||
import { EntitySchema, ObjectLiteral } from 'typeorm';
|
||||
import { EntitySchemaTransformer } from 'typeorm/entity-schema/EntitySchemaTransformer';
|
||||
import { EntityMetadataBuilder } from 'typeorm/metadata-builder/EntityMetadataBuilder';
|
||||
import { ObjectLiteral } from 'typeorm';
|
||||
|
||||
import { WorkspaceAuthContext } from 'src/engine/api/common/interfaces/workspace-auth-context.interface';
|
||||
|
||||
import { WorkspaceManyOrAllFlatEntityMapsCacheService } from 'src/engine/metadata-modules/flat-entity/services/workspace-many-or-all-flat-entity-maps-cache.service';
|
||||
import { type FlatEntityMaps } from 'src/engine/metadata-modules/flat-entity/types/flat-entity-maps.type';
|
||||
import { type FlatFieldMetadata } from 'src/engine/metadata-modules/flat-field-metadata/types/flat-field-metadata.type';
|
||||
import { type FlatObjectMetadata } from 'src/engine/metadata-modules/flat-object-metadata/types/flat-object-metadata.type';
|
||||
import { buildObjectIdByNameMaps } from 'src/engine/metadata-modules/flat-object-metadata/utils/build-object-id-by-name-maps.util';
|
||||
import { EntitySchemaFactory } from 'src/engine/twenty-orm/factories/entity-schema.factory';
|
||||
import { GlobalWorkspaceDataSourceService } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-datasource.service';
|
||||
import { type WorkspaceRepository } from 'src/engine/twenty-orm/repository/workspace.repository';
|
||||
import {
|
||||
@@ -27,8 +19,6 @@ import { convertClassNameToObjectMetadataName } from 'src/engine/workspace-manag
|
||||
export class GlobalWorkspaceOrmManager {
|
||||
constructor(
|
||||
private readonly globalWorkspaceDataSourceService: GlobalWorkspaceDataSourceService,
|
||||
private readonly workspaceManyOrAllFlatEntityMapsCacheService: WorkspaceManyOrAllFlatEntityMapsCacheService,
|
||||
private readonly entitySchemaFactory: EntitySchemaFactory,
|
||||
private readonly workspaceCacheService: WorkspaceCacheService,
|
||||
) {}
|
||||
|
||||
@@ -92,26 +82,19 @@ export class GlobalWorkspaceOrmManager {
|
||||
flatIndexMaps,
|
||||
featureFlagsMap,
|
||||
rolesPermissions: permissionsPerRoleId,
|
||||
ORMEntityMetadatas: entityMetadatas,
|
||||
} = await this.workspaceCacheService.getOrRecompute(workspaceId, [
|
||||
'flatObjectMetadataMaps',
|
||||
'flatFieldMetadataMaps',
|
||||
'flatIndexMaps',
|
||||
'featureFlagsMap',
|
||||
'rolesPermissions',
|
||||
'userWorkspaceRoleMap',
|
||||
'ORMEntityMetadatas',
|
||||
]);
|
||||
|
||||
const { idByNameSingular: objectIdByNameSingular } =
|
||||
buildObjectIdByNameMaps(flatObjectMetadataMaps);
|
||||
|
||||
const entitySchemas = this.buildEntitySchemas(
|
||||
workspaceId,
|
||||
flatObjectMetadataMaps,
|
||||
flatFieldMetadataMaps,
|
||||
);
|
||||
|
||||
const entityMetadatas = this.buildEntityMetadatas(entitySchemas);
|
||||
|
||||
return {
|
||||
authContext,
|
||||
flatObjectMetadataMaps,
|
||||
@@ -123,33 +106,4 @@ export class GlobalWorkspaceOrmManager {
|
||||
entityMetadatas,
|
||||
};
|
||||
}
|
||||
|
||||
private buildEntitySchemas(
|
||||
workspaceId: string,
|
||||
flatObjectMetadataMaps: FlatEntityMaps<FlatObjectMetadata>,
|
||||
flatFieldMetadataMaps: FlatEntityMaps<FlatFieldMetadata>,
|
||||
) {
|
||||
return Object.values(flatObjectMetadataMaps.byId)
|
||||
.filter(isDefined)
|
||||
.map((flatObjectMetadata) =>
|
||||
this.entitySchemaFactory.create(
|
||||
workspaceId,
|
||||
flatObjectMetadata,
|
||||
flatObjectMetadataMaps,
|
||||
flatFieldMetadataMaps,
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
private buildEntityMetadatas(entitySchemas: EntitySchema[]) {
|
||||
const transformer = new EntitySchemaTransformer();
|
||||
const metadataArgsStorage = transformer.transform(entitySchemas);
|
||||
|
||||
const entityMetadataBuilder = new EntityMetadataBuilder(
|
||||
this.globalWorkspaceDataSourceService.getGlobalWorkspaceDataSource(),
|
||||
metadataArgsStorage,
|
||||
);
|
||||
|
||||
return entityMetadataBuilder.build();
|
||||
}
|
||||
}
|
||||
|
||||
+85
@@ -0,0 +1,85 @@
|
||||
import { type FieldMetadataType } from 'twenty-shared/types';
|
||||
|
||||
import { type FieldMetadataEntity } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity';
|
||||
import { type ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity';
|
||||
|
||||
export type EntitySchemaObjectMetadata = Pick<
|
||||
ObjectMetadataEntity,
|
||||
'id' | 'nameSingular' | 'isCustom'
|
||||
> & {
|
||||
fieldMetadataIds: string[];
|
||||
};
|
||||
|
||||
export type EntitySchemaFieldMetadata<
|
||||
TFieldMetadataType extends FieldMetadataType = FieldMetadataType,
|
||||
> = Pick<
|
||||
FieldMetadataEntity<TFieldMetadataType>,
|
||||
| 'id'
|
||||
| 'name'
|
||||
| 'type'
|
||||
| 'settings'
|
||||
| 'isNullable'
|
||||
| 'defaultValue'
|
||||
| 'options'
|
||||
| 'objectMetadataId'
|
||||
| 'relationTargetObjectMetadataId'
|
||||
| 'relationTargetFieldMetadataId'
|
||||
>;
|
||||
|
||||
export type EntitySchemaObjectMetadataMaps = {
|
||||
byId: Partial<Record<string, EntitySchemaObjectMetadata>>;
|
||||
};
|
||||
|
||||
export type EntitySchemaFieldMetadataMaps = {
|
||||
byId: Partial<Record<string, EntitySchemaFieldMetadata>>;
|
||||
};
|
||||
|
||||
export const buildEntitySchemaMetadataMaps = (
|
||||
objectMetadatas: ObjectMetadataEntity[],
|
||||
fieldMetadatas: FieldMetadataEntity[],
|
||||
): {
|
||||
objectMetadataMaps: EntitySchemaObjectMetadataMaps;
|
||||
fieldMetadataMaps: EntitySchemaFieldMetadataMaps;
|
||||
} => {
|
||||
const fieldIdsByObjectId = new Map<string, string[]>();
|
||||
|
||||
for (const field of fieldMetadatas) {
|
||||
const existing = fieldIdsByObjectId.get(field.objectMetadataId);
|
||||
|
||||
if (existing) {
|
||||
existing.push(field.id);
|
||||
} else {
|
||||
fieldIdsByObjectId.set(field.objectMetadataId, [field.id]);
|
||||
}
|
||||
}
|
||||
|
||||
const objectMetadataMaps: EntitySchemaObjectMetadataMaps = { byId: {} };
|
||||
|
||||
for (const object of objectMetadatas) {
|
||||
objectMetadataMaps.byId[object.id] = {
|
||||
id: object.id,
|
||||
nameSingular: object.nameSingular,
|
||||
isCustom: object.isCustom,
|
||||
fieldMetadataIds: fieldIdsByObjectId.get(object.id) ?? [],
|
||||
};
|
||||
}
|
||||
|
||||
const fieldMetadataMaps: EntitySchemaFieldMetadataMaps = { byId: {} };
|
||||
|
||||
for (const field of fieldMetadatas) {
|
||||
fieldMetadataMaps.byId[field.id] = {
|
||||
id: field.id,
|
||||
name: field.name,
|
||||
type: field.type,
|
||||
settings: field.settings,
|
||||
isNullable: field.isNullable,
|
||||
defaultValue: field.defaultValue,
|
||||
options: field.options,
|
||||
objectMetadataId: field.objectMetadataId,
|
||||
relationTargetObjectMetadataId: field.relationTargetObjectMetadataId,
|
||||
relationTargetFieldMetadataId: field.relationTargetFieldMetadataId,
|
||||
};
|
||||
}
|
||||
|
||||
return { objectMetadataMaps, fieldMetadataMaps };
|
||||
};
|
||||
+76
@@ -0,0 +1,76 @@
|
||||
import { Injectable } from '@nestjs/common';
|
||||
import { InjectRepository } from '@nestjs/typeorm';
|
||||
|
||||
import { isDefined } from 'twenty-shared/utils';
|
||||
import { type EntityMetadata, EntitySchema, Repository } from 'typeorm';
|
||||
import { EntitySchemaTransformer } from 'typeorm/entity-schema/EntitySchemaTransformer';
|
||||
import { EntityMetadataBuilder } from 'typeorm/metadata-builder/EntityMetadataBuilder';
|
||||
|
||||
import { WorkspaceCacheProvider } from 'src/engine/workspace-cache/interfaces/workspace-cache-provider.service';
|
||||
|
||||
import { FieldMetadataEntity } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity';
|
||||
import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity';
|
||||
import { EntitySchemaFactory } from 'src/engine/twenty-orm/factories/entity-schema.factory';
|
||||
import { GlobalWorkspaceDataSourceService } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-datasource.service';
|
||||
import { buildEntitySchemaMetadataMaps } from 'src/engine/twenty-orm/global-workspace-datasource/types/entity-schema-metadata.type';
|
||||
import { WorkspaceCache } from 'src/engine/workspace-cache/decorators/workspace-cache.decorator';
|
||||
|
||||
@Injectable()
|
||||
@WorkspaceCache('ORMEntityMetadatas', { localDataOnly: true })
|
||||
export class WorkspaceORMEntityMetadatasCacheService extends WorkspaceCacheProvider<
|
||||
EntityMetadata[]
|
||||
> {
|
||||
constructor(
|
||||
@InjectRepository(ObjectMetadataEntity)
|
||||
private readonly objectMetadataRepository: Repository<ObjectMetadataEntity>,
|
||||
@InjectRepository(FieldMetadataEntity)
|
||||
private readonly fieldMetadataRepository: Repository<FieldMetadataEntity>,
|
||||
private readonly entitySchemaFactory: EntitySchemaFactory,
|
||||
private readonly globalWorkspaceDataSourceService: GlobalWorkspaceDataSourceService,
|
||||
) {
|
||||
super();
|
||||
}
|
||||
|
||||
async computeForCache(workspaceId: string): Promise<EntityMetadata[]> {
|
||||
const [objectMetadatas, fieldMetadatas] = await Promise.all([
|
||||
this.objectMetadataRepository.find({
|
||||
where: { workspaceId },
|
||||
withDeleted: true,
|
||||
}),
|
||||
this.fieldMetadataRepository.find({
|
||||
where: { workspaceId },
|
||||
withDeleted: true,
|
||||
}),
|
||||
]);
|
||||
|
||||
const { objectMetadataMaps, fieldMetadataMaps } =
|
||||
buildEntitySchemaMetadataMaps(objectMetadatas, fieldMetadatas);
|
||||
|
||||
const entitySchemas = Object.values(objectMetadataMaps.byId)
|
||||
.filter(isDefined)
|
||||
.map((objectMetadata) =>
|
||||
this.entitySchemaFactory.create(
|
||||
workspaceId,
|
||||
objectMetadata,
|
||||
objectMetadataMaps,
|
||||
fieldMetadataMaps,
|
||||
),
|
||||
);
|
||||
|
||||
return this.buildEntityMetadatas(entitySchemas);
|
||||
}
|
||||
|
||||
private buildEntityMetadatas(
|
||||
entitySchemas: EntitySchema[],
|
||||
): EntityMetadata[] {
|
||||
const transformer = new EntitySchemaTransformer();
|
||||
const metadataArgsStorage = transformer.transform(entitySchemas);
|
||||
|
||||
const entityMetadataBuilder = new EntityMetadataBuilder(
|
||||
this.globalWorkspaceDataSourceService.getGlobalWorkspaceDataSource(),
|
||||
metadataArgsStorage,
|
||||
);
|
||||
|
||||
return entityMetadataBuilder.build();
|
||||
}
|
||||
}
|
||||
@@ -1,8 +1,6 @@
|
||||
import { type Milliseconds } from 'cache-manager';
|
||||
import { isDefined } from 'twenty-shared/utils';
|
||||
|
||||
import { NodeEnvironment } from 'src/engine/core-modules/twenty-config/interfaces/node-environment.interface';
|
||||
|
||||
import { type CacheKey } from 'src/engine/twenty-orm/storage/types/cache-key.type';
|
||||
|
||||
type AsyncFactoryCallback<T> = () => Promise<T | null>;
|
||||
@@ -39,13 +37,6 @@ export class PromiseMemoizer<T> {
|
||||
return existingPromise;
|
||||
}
|
||||
|
||||
if (process.env.NODE_ENV !== NodeEnvironment.TEST) {
|
||||
// eslint-disable-next-line no-console
|
||||
console.log(
|
||||
`Computing new Datasource for cacheKey: ${cacheKey} out of ${this.cache.size}`,
|
||||
);
|
||||
}
|
||||
|
||||
const newPromise = (async () => {
|
||||
try {
|
||||
const value = await factory();
|
||||
|
||||
+11
-11
@@ -1,13 +1,15 @@
|
||||
import { type FieldMetadataType } from 'twenty-shared/types';
|
||||
import { type RelationType } from 'typeorm/metadata/types/RelationTypes';
|
||||
|
||||
import { type FlatEntityMaps } from 'src/engine/metadata-modules/flat-entity/types/flat-entity-maps.type';
|
||||
import { type FlatFieldMetadata } from 'src/engine/metadata-modules/flat-field-metadata/types/flat-field-metadata.type';
|
||||
import { type FlatObjectMetadata } from 'src/engine/metadata-modules/flat-object-metadata/types/flat-object-metadata.type';
|
||||
import {
|
||||
RelationException,
|
||||
RelationExceptionCode,
|
||||
} from 'src/engine/twenty-orm/exceptions/relation.exception';
|
||||
import {
|
||||
type EntitySchemaFieldMetadata,
|
||||
type EntitySchemaFieldMetadataMaps,
|
||||
type EntitySchemaObjectMetadataMaps,
|
||||
} from 'src/engine/twenty-orm/global-workspace-datasource/types/entity-schema-metadata.type';
|
||||
import { converRelationTypeToTypeORMRelationType } from 'src/engine/twenty-orm/utils/convert-relation-type-to-typeorm-relation-type.util';
|
||||
|
||||
interface RelationDetails {
|
||||
@@ -18,11 +20,11 @@ interface RelationDetails {
|
||||
}
|
||||
|
||||
export function determineSchemaRelationDetails(
|
||||
fieldMetadata: FlatFieldMetadata<
|
||||
fieldMetadata: EntitySchemaFieldMetadata<
|
||||
FieldMetadataType.RELATION | FieldMetadataType.MORPH_RELATION
|
||||
>,
|
||||
flatObjectMetadataMaps: FlatEntityMaps<FlatObjectMetadata>,
|
||||
flatFieldMetadataMaps: FlatEntityMaps<FlatFieldMetadata>,
|
||||
objectMetadataMaps: EntitySchemaObjectMetadataMaps,
|
||||
fieldMetadataMaps: EntitySchemaFieldMetadataMaps,
|
||||
): RelationDetails {
|
||||
if (!fieldMetadata.settings) {
|
||||
throw new Error('Field metadata settings are missing');
|
||||
@@ -39,12 +41,10 @@ export function determineSchemaRelationDetails(
|
||||
);
|
||||
}
|
||||
|
||||
const sourceObjectMetadata =
|
||||
flatObjectMetadataMaps.byId[fieldMetadata.objectMetadataId];
|
||||
const targetObjectMetadata =
|
||||
flatObjectMetadataMaps.byId[fieldMetadata.relationTargetObjectMetadataId];
|
||||
objectMetadataMaps.byId[fieldMetadata.relationTargetObjectMetadataId];
|
||||
|
||||
if (!sourceObjectMetadata || !targetObjectMetadata) {
|
||||
if (!targetObjectMetadata) {
|
||||
throw new RelationException(
|
||||
`Object metadata not found for field ${fieldMetadata.id}`,
|
||||
RelationExceptionCode.RELATION_OBJECT_METADATA_NOT_FOUND,
|
||||
@@ -59,7 +59,7 @@ export function determineSchemaRelationDetails(
|
||||
}
|
||||
|
||||
const targetFieldMetadata =
|
||||
flatFieldMetadataMaps.byId[fieldMetadata.relationTargetFieldMetadataId];
|
||||
fieldMetadataMaps.byId[fieldMetadata.relationTargetFieldMetadataId];
|
||||
|
||||
if (!targetFieldMetadata) {
|
||||
throw new Error('Target field metadata not found');
|
||||
|
||||
+15
-3
@@ -2,7 +2,19 @@ import { SetMetadata } from '@nestjs/common';
|
||||
|
||||
import { type WorkspaceCacheKeyName } from 'src/engine/workspace-cache/types/workspace-cache-key.type';
|
||||
|
||||
export const WORKSPACE_CACHE_KEY = 'WORKSPACE_CACHE_KEY';
|
||||
export type WorkspaceCacheOptions = {
|
||||
localDataOnly?: boolean;
|
||||
};
|
||||
|
||||
export const WorkspaceCache = (workspaceCacheKeyName: WorkspaceCacheKeyName) =>
|
||||
SetMetadata(WORKSPACE_CACHE_KEY, workspaceCacheKeyName);
|
||||
export const WORKSPACE_CACHE_KEY = 'WORKSPACE_CACHE_KEY';
|
||||
export const WORKSPACE_CACHE_OPTIONS = 'WORKSPACE_CACHE_OPTIONS';
|
||||
|
||||
export const WorkspaceCache = (
|
||||
workspaceCacheKeyName: WorkspaceCacheKeyName,
|
||||
options?: WorkspaceCacheOptions,
|
||||
): ClassDecorator => {
|
||||
return (target) => {
|
||||
SetMetadata(WORKSPACE_CACHE_KEY, workspaceCacheKeyName)(target);
|
||||
SetMetadata(WORKSPACE_CACHE_OPTIONS, options ?? {})(target);
|
||||
};
|
||||
};
|
||||
|
||||
+172
-4
@@ -164,10 +164,13 @@ describe('WorkspaceCacheService', () => {
|
||||
|
||||
it('should return data from redis when available', async () => {
|
||||
const cachedData = { FLAG_A: true, FLAG_B: false };
|
||||
const cachedHash = 'some-hash-from-redis';
|
||||
|
||||
cacheStorageService.mget
|
||||
// First call: validateLocalHashAgainstRedisHash checks hash
|
||||
.mockResolvedValueOnce([undefined])
|
||||
.mockResolvedValueOnce([cachedData]);
|
||||
// Second call: fetchDataFromRedis fetches data and hash atomically
|
||||
.mockResolvedValueOnce([cachedData, cachedHash]);
|
||||
|
||||
const result = await service.getOrRecompute(WORKSPACE_ID, [
|
||||
'featureFlagsMap',
|
||||
@@ -223,7 +226,9 @@ describe('WorkspaceCacheService', () => {
|
||||
|
||||
await service.getOrRecompute(WORKSPACE_ID, ['featureFlagsMap']);
|
||||
|
||||
// Each getOrRecompute call triggers 2 mget calls (hash check + data fetch)
|
||||
// Verify Redis was rechecked after TTL expired
|
||||
// Each getOrRecompute triggers: 1 hash check + 1 atomic data/hash fetch = 2 mget calls
|
||||
// Total: 4 mget calls (2 per getOrRecompute)
|
||||
expect(cacheStorageService.mget).toHaveBeenCalledTimes(4);
|
||||
});
|
||||
});
|
||||
@@ -248,7 +253,7 @@ describe('WorkspaceCacheService', () => {
|
||||
await service.onModuleInit();
|
||||
});
|
||||
|
||||
it('should delete from redis and local cache, then recompute', async () => {
|
||||
it('should delete from redis, mark local cache as stale, and recompute', async () => {
|
||||
cacheStorageService.mdel.mockResolvedValue(undefined);
|
||||
cacheStorageService.mset.mockResolvedValue(undefined);
|
||||
|
||||
@@ -313,10 +318,44 @@ describe('WorkspaceCacheService', () => {
|
||||
]),
|
||||
);
|
||||
});
|
||||
|
||||
it('should keep old versions in local cache after invalidation for race condition safety', async () => {
|
||||
cacheStorageService.mget.mockResolvedValue([undefined]);
|
||||
cacheStorageService.mset.mockResolvedValue(undefined);
|
||||
cacheStorageService.mdel.mockResolvedValue(undefined);
|
||||
|
||||
const initialData = { testData: 'initial-value' };
|
||||
const recomputedData = { testData: 'recomputed-value' };
|
||||
|
||||
jest
|
||||
.spyOn(mockProvider, 'computeForCache')
|
||||
.mockResolvedValue(initialData);
|
||||
|
||||
// First, populate the cache
|
||||
const firstResult = await service.getOrRecompute(WORKSPACE_ID, [
|
||||
'featureFlagsMap',
|
||||
]);
|
||||
|
||||
expect(firstResult).toEqual({ featureFlagsMap: initialData });
|
||||
|
||||
// Now invalidate and recompute
|
||||
jest
|
||||
.spyOn(mockProvider, 'computeForCache')
|
||||
.mockResolvedValue(recomputedData);
|
||||
|
||||
await service.invalidateAndRecompute(WORKSPACE_ID, ['featureFlagsMap']);
|
||||
|
||||
// The new value should be returned
|
||||
const secondResult = await service.getOrRecompute(WORKSPACE_ID, [
|
||||
'featureFlagsMap',
|
||||
]);
|
||||
|
||||
expect(secondResult).toEqual({ featureFlagsMap: recomputedData });
|
||||
});
|
||||
});
|
||||
|
||||
describe('flush', () => {
|
||||
it('should delete from redis and local cache', async () => {
|
||||
it('should delete from redis and mark local cache as stale', async () => {
|
||||
cacheStorageService.mdel.mockResolvedValue(undefined);
|
||||
|
||||
await service.flush(WORKSPACE_ID, ['featureFlagsMap']);
|
||||
@@ -334,5 +373,134 @@ describe('WorkspaceCacheService', () => {
|
||||
|
||||
expect(cacheStorageService.mdel).toHaveBeenCalledWith([]);
|
||||
});
|
||||
|
||||
it('should force staleness on local cache entries without deleting them', async () => {
|
||||
cacheStorageService.mget.mockResolvedValue([undefined]);
|
||||
cacheStorageService.mset.mockResolvedValue(undefined);
|
||||
cacheStorageService.mdel.mockResolvedValue(undefined);
|
||||
|
||||
jest.spyOn(mockProvider, 'computeForCache').mockResolvedValue({
|
||||
testData: 'computed-value',
|
||||
});
|
||||
|
||||
discoveryService.getProviders.mockReturnValue([
|
||||
{ instance: mockProvider },
|
||||
] as any);
|
||||
|
||||
reflector.get.mockImplementation((key, target) => {
|
||||
if (
|
||||
key === WORKSPACE_CACHE_KEY &&
|
||||
target === MockFeatureFlagsCacheProvider
|
||||
) {
|
||||
return 'featureFlagsMap';
|
||||
}
|
||||
|
||||
return undefined;
|
||||
});
|
||||
|
||||
await service.onModuleInit();
|
||||
|
||||
// Populate the cache
|
||||
await service.getOrRecompute(WORKSPACE_ID, ['featureFlagsMap']);
|
||||
|
||||
// Flush the cache (marks as stale, doesn't delete)
|
||||
await service.flush(WORKSPACE_ID, ['featureFlagsMap']);
|
||||
|
||||
// Advance time slightly (but still within memoizer TTL)
|
||||
jest.advanceTimersByTime(50);
|
||||
|
||||
// Next call should check Redis since local cache is marked stale
|
||||
cacheStorageService.mget.mockResolvedValue(['some-hash']);
|
||||
|
||||
await service.getOrRecompute(WORKSPACE_ID, ['featureFlagsMap']);
|
||||
|
||||
// Should have made additional mget calls to check Redis
|
||||
expect(cacheStorageService.mget.mock.calls.length).toBeGreaterThan(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe('versioning behavior', () => {
|
||||
beforeEach(async () => {
|
||||
discoveryService.getProviders.mockReturnValue([
|
||||
{ instance: mockProvider },
|
||||
] as any);
|
||||
|
||||
reflector.get.mockImplementation((key, target) => {
|
||||
if (
|
||||
key === WORKSPACE_CACHE_KEY &&
|
||||
target === MockFeatureFlagsCacheProvider
|
||||
) {
|
||||
return 'featureFlagsMap';
|
||||
}
|
||||
|
||||
return undefined;
|
||||
});
|
||||
|
||||
await service.onModuleInit();
|
||||
});
|
||||
|
||||
it('should store multiple versions when data is recomputed', async () => {
|
||||
cacheStorageService.mget.mockResolvedValue([undefined]);
|
||||
cacheStorageService.mset.mockResolvedValue(undefined);
|
||||
cacheStorageService.mdel.mockResolvedValue(undefined);
|
||||
|
||||
const firstData = { testData: 'first-value' };
|
||||
const secondData = { testData: 'second-value' };
|
||||
|
||||
jest.spyOn(mockProvider, 'computeForCache').mockResolvedValue(firstData);
|
||||
|
||||
// First computation
|
||||
await service.getOrRecompute(WORKSPACE_ID, ['featureFlagsMap']);
|
||||
|
||||
// Invalidate and recompute with new data
|
||||
jest.spyOn(mockProvider, 'computeForCache').mockResolvedValue(secondData);
|
||||
|
||||
await service.invalidateAndRecompute(WORKSPACE_ID, ['featureFlagsMap']);
|
||||
|
||||
// Should return the latest version
|
||||
const result = await service.getOrRecompute(WORKSPACE_ID, [
|
||||
'featureFlagsMap',
|
||||
]);
|
||||
|
||||
expect(result).toEqual({ featureFlagsMap: secondData });
|
||||
});
|
||||
|
||||
it('should cleanup stale versions after TTL expires', async () => {
|
||||
cacheStorageService.mget.mockResolvedValue([undefined]);
|
||||
cacheStorageService.mset.mockResolvedValue(undefined);
|
||||
cacheStorageService.mdel.mockResolvedValue(undefined);
|
||||
|
||||
jest.spyOn(mockProvider, 'computeForCache').mockResolvedValue({
|
||||
testData: 'value-1',
|
||||
});
|
||||
|
||||
// Create first version
|
||||
await service.getOrRecompute(WORKSPACE_ID, ['featureFlagsMap']);
|
||||
|
||||
// Create multiple versions by invalidating
|
||||
for (let i = 2; i <= 4; i++) {
|
||||
jest.spyOn(mockProvider, 'computeForCache').mockResolvedValue({
|
||||
testData: `value-${i}`,
|
||||
});
|
||||
await service.invalidateAndRecompute(WORKSPACE_ID, ['featureFlagsMap']);
|
||||
}
|
||||
|
||||
// Advance time past the stale version TTL (5000ms)
|
||||
jest.advanceTimersByTime(6_000);
|
||||
|
||||
// Advance past memoizer TTL as well
|
||||
jest.advanceTimersByTime(15_000);
|
||||
|
||||
// Trigger a read which should cleanup stale versions
|
||||
jest.spyOn(mockProvider, 'computeForCache').mockResolvedValue({
|
||||
testData: 'latest-value',
|
||||
});
|
||||
|
||||
const result = await service.getOrRecompute(WORKSPACE_ID, [
|
||||
'featureFlagsMap',
|
||||
]);
|
||||
|
||||
expect(result).toEqual({ featureFlagsMap: { testData: 'latest-value' } });
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
+276
-273
@@ -1,4 +1,4 @@
|
||||
import { Injectable, OnModuleInit } from '@nestjs/common';
|
||||
import { Injectable, Logger, OnModuleInit } from '@nestjs/common';
|
||||
import { DiscoveryService, Reflector } from '@nestjs/core';
|
||||
|
||||
import crypto from 'crypto';
|
||||
@@ -11,7 +11,11 @@ import { InjectCacheStorage } from 'src/engine/core-modules/cache-storage/decora
|
||||
import { CacheStorageService } from 'src/engine/core-modules/cache-storage/services/cache-storage.service';
|
||||
import { CacheStorageNamespace } from 'src/engine/core-modules/cache-storage/types/cache-storage-namespace.enum';
|
||||
import { PromiseMemoizer } from 'src/engine/twenty-orm/storage/promise-memoizer.storage';
|
||||
import { WORKSPACE_CACHE_KEY } from 'src/engine/workspace-cache/decorators/workspace-cache.decorator';
|
||||
import {
|
||||
WORKSPACE_CACHE_KEY,
|
||||
WORKSPACE_CACHE_OPTIONS,
|
||||
WorkspaceCacheOptions,
|
||||
} from 'src/engine/workspace-cache/decorators/workspace-cache.decorator';
|
||||
import {
|
||||
WorkspaceCacheException,
|
||||
WorkspaceCacheExceptionCode,
|
||||
@@ -24,8 +28,10 @@ import {
|
||||
} from 'src/engine/workspace-cache/types/workspace-cache-key.type';
|
||||
import { type WorkspaceLocalCacheEntry } from 'src/engine/workspace-cache/types/workspace-local-cache-entry.type';
|
||||
|
||||
const LOCAL_STALENESS_TTL_MS = 100;
|
||||
const LOCAL_TTL_MS = 100;
|
||||
const MEMOIZER_TTL_MS = 10_000;
|
||||
const STALE_VERSION_TTL_MS = 5_000;
|
||||
const MAX_LOCAL_STALE_VERSIONS = 5;
|
||||
|
||||
type CacheDataType = WorkspaceCacheDataMap[WorkspaceCacheKeyName];
|
||||
|
||||
@@ -39,10 +45,13 @@ export class WorkspaceCacheService implements OnModuleInit {
|
||||
WorkspaceCacheKeyName,
|
||||
WorkspaceCacheProvider<CacheDataType>
|
||||
>();
|
||||
private readonly localDataOnlyKeys = new Set<WorkspaceCacheKeyName>();
|
||||
private readonly memoizer = new PromiseMemoizer<
|
||||
Partial<WorkspaceCacheDataMap>
|
||||
>(MEMOIZER_TTL_MS);
|
||||
|
||||
private readonly logger = new Logger(WorkspaceCacheService.name);
|
||||
|
||||
constructor(
|
||||
@InjectCacheStorage(CacheStorageNamespace.EngineWorkspace)
|
||||
private readonly cacheStorage: CacheStorageService,
|
||||
@@ -70,17 +79,27 @@ export class WorkspaceCacheService implements OnModuleInit {
|
||||
instance instanceof WorkspaceCacheProvider
|
||||
) {
|
||||
this.workspaceCacheProviders.set(workspaceCacheKeyName, instance);
|
||||
|
||||
const options: WorkspaceCacheOptions | undefined =
|
||||
this.reflector.get<WorkspaceCacheOptions>(
|
||||
WORKSPACE_CACHE_OPTIONS,
|
||||
instance.constructor,
|
||||
);
|
||||
|
||||
if (options?.localDataOnly) {
|
||||
this.localDataOnlyKeys.add(workspaceCacheKeyName);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public async getOrRecompute<const K extends WorkspaceCacheKeyName[]>(
|
||||
workspaceId: string,
|
||||
workspaceCacheKeyNames: K,
|
||||
cacheKeyNames: K,
|
||||
): Promise<WorkspaceCacheResult<K>> {
|
||||
if (
|
||||
!isDefined(workspaceId) ||
|
||||
workspaceCacheKeyNames.length === 0 ||
|
||||
cacheKeyNames.length === 0 ||
|
||||
!isValidUuid(workspaceId)
|
||||
) {
|
||||
throw new WorkspaceCacheException(
|
||||
@@ -90,42 +109,46 @@ export class WorkspaceCacheService implements OnModuleInit {
|
||||
}
|
||||
|
||||
const memoKey =
|
||||
`${workspaceId}-${[...workspaceCacheKeyNames].sort().join(',')}` as const;
|
||||
`${workspaceId}-${[...cacheKeyNames].sort().join(',')}` as const;
|
||||
|
||||
const result = await this.memoizer.memoizePromiseAndExecute(
|
||||
memoKey,
|
||||
async () => {
|
||||
const freshResult: Partial<WorkspaceCacheDataMap> = {};
|
||||
|
||||
const { freshKeys, staleKeys } = this.partitionKeysByTTLStaleness(
|
||||
// Stage 1: Check local TTL
|
||||
const { freshKeys, staleKeys } = this.checkLocalTTL(
|
||||
workspaceId,
|
||||
workspaceCacheKeyNames,
|
||||
cacheKeyNames,
|
||||
);
|
||||
|
||||
for (const workspaceCacheKeyName of freshKeys) {
|
||||
const localKey = this.buildCacheKey(
|
||||
workspaceId,
|
||||
workspaceCacheKeyName,
|
||||
);
|
||||
const cached = this.localCache.get(localKey);
|
||||
|
||||
if (isDefined(cached)) {
|
||||
Object.assign(freshResult, {
|
||||
[workspaceCacheKeyName]: cached.data,
|
||||
});
|
||||
}
|
||||
}
|
||||
const freshData = this.getFromLocalCache(workspaceId, freshKeys);
|
||||
|
||||
if (staleKeys.length === 0) {
|
||||
return freshResult;
|
||||
return freshData;
|
||||
}
|
||||
|
||||
const staleResults = await this.resolveStaleKeys(
|
||||
// Stage 2: Validate ttl stale keys against Redis hash
|
||||
const { validKeys, keysNeedingDataFromRedis, keysNeedingRecompute } =
|
||||
await this.validateLocalHashAgainstRedisHash(workspaceId, staleKeys);
|
||||
const validatedData = this.getFromLocalCache(workspaceId, validKeys);
|
||||
|
||||
// Stage 3: Fetch data from Redis
|
||||
const { redisData, missingInRedis } = await this.fetchDataFromRedis(
|
||||
workspaceId,
|
||||
staleKeys,
|
||||
keysNeedingDataFromRedis,
|
||||
);
|
||||
|
||||
return { ...freshResult, ...staleResults };
|
||||
// Stage 4: Recompute remaining
|
||||
const keysToRecompute = [...keysNeedingRecompute, ...missingInRedis];
|
||||
const recomputedData = await this.recomputeDataFromProvider(
|
||||
workspaceId,
|
||||
keysToRecompute,
|
||||
);
|
||||
|
||||
return {
|
||||
...freshData,
|
||||
...validatedData,
|
||||
...redisData,
|
||||
...recomputedData,
|
||||
};
|
||||
},
|
||||
);
|
||||
|
||||
@@ -134,39 +157,213 @@ export class WorkspaceCacheService implements OnModuleInit {
|
||||
|
||||
public async invalidateAndRecompute(
|
||||
workspaceId: string,
|
||||
workspaceCacheKeys: WorkspaceCacheKeyName[],
|
||||
cacheKeyNames: WorkspaceCacheKeyName[],
|
||||
): Promise<void> {
|
||||
await this.memoizer.clearKeys(`${workspaceId}-`);
|
||||
|
||||
await this.flush(workspaceId, workspaceCacheKeys);
|
||||
await this.recomputeCache(workspaceId, workspaceCacheKeys);
|
||||
await this.flush(workspaceId, cacheKeyNames);
|
||||
await this.recomputeDataFromProvider(workspaceId, cacheKeyNames);
|
||||
}
|
||||
|
||||
public async flush(
|
||||
workspaceId: string,
|
||||
workspaceCacheKeys: WorkspaceCacheKeyName[],
|
||||
cacheKeyNames: WorkspaceCacheKeyName[],
|
||||
): Promise<void> {
|
||||
await this.deleteFromRedis(workspaceId, workspaceCacheKeys);
|
||||
this.deleteFromLocalCache(workspaceId, workspaceCacheKeys);
|
||||
await this.deleteFromRedis(workspaceId, cacheKeyNames);
|
||||
|
||||
this.deleteFromLocalCache(workspaceId, cacheKeyNames);
|
||||
}
|
||||
|
||||
private checkLocalTTL<K extends WorkspaceCacheKeyName>(
|
||||
workspaceId: string,
|
||||
cacheKeyNames: readonly K[],
|
||||
): { freshKeys: K[]; staleKeys: K[] } {
|
||||
const freshKeys: K[] = [];
|
||||
const staleKeys: K[] = [];
|
||||
const now = Date.now();
|
||||
|
||||
for (const keyName of cacheKeyNames) {
|
||||
const localKey = this.buildCacheKey(workspaceId, keyName);
|
||||
const cached = this.localCache.get(localKey);
|
||||
|
||||
if (isDefined(cached) && now - cached.lastHashCheckedAt < LOCAL_TTL_MS) {
|
||||
freshKeys.push(keyName);
|
||||
} else {
|
||||
staleKeys.push(keyName);
|
||||
}
|
||||
}
|
||||
|
||||
return { freshKeys, staleKeys };
|
||||
}
|
||||
|
||||
private async validateLocalHashAgainstRedisHash(
|
||||
workspaceId: string,
|
||||
cacheKeyNames: WorkspaceCacheKeyName[],
|
||||
): Promise<{
|
||||
validKeys: WorkspaceCacheKeyName[];
|
||||
keysNeedingDataFromRedis: WorkspaceCacheKeyName[];
|
||||
keysNeedingRecompute: WorkspaceCacheKeyName[];
|
||||
}> {
|
||||
const validKeys: WorkspaceCacheKeyName[] = [];
|
||||
const keysNeedingDataFromRedis: WorkspaceCacheKeyName[] = [];
|
||||
const keysNeedingRecompute: WorkspaceCacheKeyName[] = [];
|
||||
|
||||
if (cacheKeyNames.length === 0) {
|
||||
return { validKeys, keysNeedingDataFromRedis, keysNeedingRecompute };
|
||||
}
|
||||
|
||||
const hashKeys = cacheKeyNames.map(
|
||||
(keyName) => `${this.buildCacheKey(workspaceId, keyName)}:hash`,
|
||||
);
|
||||
|
||||
const redisHashes = await this.cacheStorage.mget<string>(hashKeys);
|
||||
|
||||
for (const [index, keyName] of cacheKeyNames.entries()) {
|
||||
const redisHash = redisHashes[index];
|
||||
const localKey = this.buildCacheKey(workspaceId, keyName);
|
||||
const localEntry = this.localCache.get(localKey);
|
||||
|
||||
if (
|
||||
isDefined(localEntry) &&
|
||||
isDefined(redisHash) &&
|
||||
localEntry.latestHash === redisHash
|
||||
) {
|
||||
localEntry.lastHashCheckedAt = Date.now();
|
||||
validKeys.push(keyName);
|
||||
} else if (this.localDataOnlyKeys.has(keyName)) {
|
||||
keysNeedingRecompute.push(keyName);
|
||||
} else {
|
||||
keysNeedingDataFromRedis.push(keyName);
|
||||
}
|
||||
}
|
||||
|
||||
return { validKeys, keysNeedingDataFromRedis, keysNeedingRecompute };
|
||||
}
|
||||
|
||||
private async fetchDataFromRedis(
|
||||
workspaceId: string,
|
||||
cacheKeyNames: WorkspaceCacheKeyName[],
|
||||
): Promise<{
|
||||
redisData: Partial<WorkspaceCacheDataMap>;
|
||||
missingInRedis: WorkspaceCacheKeyName[];
|
||||
}> {
|
||||
const redisData: Partial<WorkspaceCacheDataMap> = {};
|
||||
const missingInRedis: WorkspaceCacheKeyName[] = [];
|
||||
|
||||
if (cacheKeyNames.length === 0) {
|
||||
return { redisData, missingInRedis };
|
||||
}
|
||||
|
||||
// Interleave data and hash keys for atomic fetch: [data1, hash1, data2, hash2, ...]
|
||||
const allKeys = cacheKeyNames.flatMap((keyName) => {
|
||||
const baseKey = this.buildCacheKey(workspaceId, keyName);
|
||||
|
||||
return [`${baseKey}:data`, `${baseKey}:hash`];
|
||||
});
|
||||
|
||||
const allValues = await this.cacheStorage.mget<CacheDataType | string>(
|
||||
allKeys,
|
||||
);
|
||||
|
||||
for (const [index, keyName] of cacheKeyNames.entries()) {
|
||||
const data = allValues[index * 2] as CacheDataType | undefined;
|
||||
const hash = allValues[index * 2 + 1] as string | undefined;
|
||||
|
||||
if (isDefined(data) && isDefined(hash)) {
|
||||
Object.assign(redisData, { [keyName]: data });
|
||||
this.setInLocalCache(workspaceId, keyName, data, hash);
|
||||
} else {
|
||||
missingInRedis.push(keyName);
|
||||
}
|
||||
}
|
||||
|
||||
return { redisData, missingInRedis };
|
||||
}
|
||||
|
||||
private async recomputeDataFromProvider(
|
||||
workspaceId: string,
|
||||
cacheKeyNames: WorkspaceCacheKeyName[],
|
||||
): Promise<Partial<WorkspaceCacheDataMap>> {
|
||||
const result: Partial<WorkspaceCacheDataMap> = {};
|
||||
|
||||
if (cacheKeyNames.length === 0) {
|
||||
return result;
|
||||
}
|
||||
|
||||
const computePromises = cacheKeyNames.map(async (keyName) => {
|
||||
const provider = this.getProviderOrThrow(keyName);
|
||||
const data = await provider.computeForCache(workspaceId);
|
||||
const hash = crypto.randomUUID();
|
||||
|
||||
return { keyName, data, hash };
|
||||
});
|
||||
|
||||
const computed = await Promise.all(computePromises);
|
||||
|
||||
const redisEntries: Array<{ key: string; value: unknown }> = [];
|
||||
|
||||
for (const { keyName, data, hash } of computed) {
|
||||
Object.assign(result, { [keyName]: data });
|
||||
|
||||
const baseKey = this.buildCacheKey(workspaceId, keyName);
|
||||
|
||||
redisEntries.push({ key: `${baseKey}:hash`, value: hash });
|
||||
|
||||
if (!this.localDataOnlyKeys.has(keyName)) {
|
||||
redisEntries.push({ key: `${baseKey}:data`, value: data });
|
||||
}
|
||||
|
||||
this.setInLocalCache(workspaceId, keyName, data, hash);
|
||||
}
|
||||
|
||||
if (redisEntries.length > 0) {
|
||||
await this.cacheStorage.mset(redisEntries);
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
private getFromLocalCache(
|
||||
workspaceId: string,
|
||||
workspaceCacheKeyNames: WorkspaceCacheKeyName[],
|
||||
): Partial<WorkspaceCacheDataMap> {
|
||||
const result: Partial<WorkspaceCacheDataMap> = {};
|
||||
|
||||
for (const keyName of workspaceCacheKeyNames) {
|
||||
const localKey = this.buildCacheKey(workspaceId, keyName);
|
||||
const entry = this.localCache.get(localKey);
|
||||
const version = entry?.versions.get(entry.latestHash);
|
||||
|
||||
if (isDefined(entry) && isDefined(version)) {
|
||||
version.lastReadAt = Date.now();
|
||||
Object.assign(result, { [keyName]: version.data });
|
||||
this.cleanupStaleVersions(entry);
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
private deleteFromLocalCache(
|
||||
workspaceId: string,
|
||||
workspaceCacheKeys: WorkspaceCacheKeyName[],
|
||||
cacheKeyNames: WorkspaceCacheKeyName[],
|
||||
): void {
|
||||
for (const workspaceCacheKeyName of workspaceCacheKeys) {
|
||||
const localKey = this.buildCacheKey(workspaceId, workspaceCacheKeyName);
|
||||
for (const keyName of cacheKeyNames) {
|
||||
const localKey = this.buildCacheKey(workspaceId, keyName);
|
||||
const entry = this.localCache.get(localKey);
|
||||
|
||||
this.localCache.delete(localKey);
|
||||
if (isDefined(entry)) {
|
||||
entry.lastHashCheckedAt = 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async deleteFromRedis(
|
||||
workspaceId: string,
|
||||
workspaceCacheKeys: WorkspaceCacheKeyName[],
|
||||
cacheKeyNames: WorkspaceCacheKeyName[],
|
||||
): Promise<void> {
|
||||
const keysToDelete = workspaceCacheKeys.flatMap((workspaceCacheKeyName) => {
|
||||
const baseKey = this.buildCacheKey(workspaceId, workspaceCacheKeyName);
|
||||
const keysToDelete = cacheKeyNames.flatMap((keyName) => {
|
||||
const baseKey = this.buildCacheKey(workspaceId, keyName);
|
||||
|
||||
return [`${baseKey}:data`, `${baseKey}:hash`];
|
||||
});
|
||||
@@ -174,267 +371,73 @@ export class WorkspaceCacheService implements OnModuleInit {
|
||||
await this.cacheStorage.mdel(keysToDelete);
|
||||
}
|
||||
|
||||
private async recomputeCache(
|
||||
private setInLocalCache(
|
||||
workspaceId: string,
|
||||
workspaceCacheKeys: WorkspaceCacheKeyName[],
|
||||
): Promise<void> {
|
||||
const computed = await this.computeAndStoreInRedis(
|
||||
workspaceId,
|
||||
workspaceCacheKeys,
|
||||
);
|
||||
keyName: WorkspaceCacheKeyName,
|
||||
data: CacheDataType,
|
||||
hash: string,
|
||||
): void {
|
||||
const localKey = this.buildCacheKey(workspaceId, keyName);
|
||||
let entry = this.localCache.get(localKey);
|
||||
|
||||
for (const { workspaceCacheKeyName, data, hash } of computed) {
|
||||
this.setInLocalCache(workspaceId, workspaceCacheKeyName, data, hash);
|
||||
if (!isDefined(entry)) {
|
||||
entry = { versions: new Map(), latestHash: '', lastHashCheckedAt: 0 };
|
||||
this.localCache.set(localKey, entry);
|
||||
}
|
||||
|
||||
entry.versions.set(hash, { data, lastReadAt: Date.now() });
|
||||
entry.latestHash = hash;
|
||||
entry.lastHashCheckedAt = Date.now();
|
||||
}
|
||||
|
||||
private partitionKeysByTTLStaleness<K extends WorkspaceCacheKeyName>(
|
||||
workspaceId: string,
|
||||
workspaceCacheKeys: readonly K[],
|
||||
): { freshKeys: K[]; staleKeys: K[] } {
|
||||
const freshKeys: K[] = [];
|
||||
const staleKeys: K[] = [];
|
||||
private cleanupStaleVersions(
|
||||
entry: WorkspaceLocalCacheEntry<CacheDataType>,
|
||||
): void {
|
||||
const now = Date.now();
|
||||
|
||||
for (const workspaceCacheKeyName of workspaceCacheKeys) {
|
||||
const localKey = this.buildCacheKey(workspaceId, workspaceCacheKeyName);
|
||||
const cached = this.localCache.get(localKey);
|
||||
|
||||
for (const [hash, version] of entry.versions) {
|
||||
if (
|
||||
isDefined(cached) &&
|
||||
now - cached.lastCheckedAt < LOCAL_STALENESS_TTL_MS
|
||||
hash !== entry.latestHash &&
|
||||
now - version.lastReadAt > STALE_VERSION_TTL_MS
|
||||
) {
|
||||
freshKeys.push(workspaceCacheKeyName);
|
||||
} else {
|
||||
staleKeys.push(workspaceCacheKeyName);
|
||||
entry.versions.delete(hash);
|
||||
}
|
||||
}
|
||||
|
||||
return { freshKeys, staleKeys };
|
||||
}
|
||||
if (entry.versions.size >= MAX_LOCAL_STALE_VERSIONS) {
|
||||
const sorted = [...entry.versions.entries()]
|
||||
.filter(([hash]) => hash !== entry.latestHash)
|
||||
.sort((entryA, entryB) => entryA[1].lastReadAt - entryB[1].lastReadAt);
|
||||
|
||||
private async resolveStaleKeys(
|
||||
workspaceId: string,
|
||||
workspaceCacheKeys: WorkspaceCacheKeyName[],
|
||||
): Promise<Partial<WorkspaceCacheDataMap>> {
|
||||
const result: Partial<WorkspaceCacheDataMap> = {};
|
||||
|
||||
const { validFromLocal, needsRedisCheck } =
|
||||
await this.partitionKeysByLocalStaleness(workspaceId, workspaceCacheKeys);
|
||||
|
||||
for (const workspaceCacheKeyName of validFromLocal) {
|
||||
const localKey = this.buildCacheKey(workspaceId, workspaceCacheKeyName);
|
||||
const localEntry = this.localCache.get(localKey);
|
||||
|
||||
if (!isDefined(localEntry)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
Object.assign(result, { [workspaceCacheKeyName]: localEntry.data });
|
||||
this.setInLocalCache(
|
||||
workspaceId,
|
||||
workspaceCacheKeyName,
|
||||
localEntry.data,
|
||||
localEntry.hash,
|
||||
);
|
||||
}
|
||||
|
||||
if (needsRedisCheck.length === 0) {
|
||||
return result;
|
||||
}
|
||||
|
||||
const {
|
||||
validDataFromRedis: validFromRedis,
|
||||
cacheKeysToRecomputeFromProviders: needsCompute,
|
||||
} = await this.fetchDataFromRedis(workspaceId, needsRedisCheck);
|
||||
|
||||
for (const { workspaceCacheKeyName, data, hash } of validFromRedis) {
|
||||
Object.assign(result, { [workspaceCacheKeyName]: data });
|
||||
this.setInLocalCache(workspaceId, workspaceCacheKeyName, data, hash);
|
||||
}
|
||||
|
||||
if (needsCompute.length === 0) {
|
||||
return result;
|
||||
}
|
||||
|
||||
const computed = await this.computeAndStoreInRedis(
|
||||
workspaceId,
|
||||
needsCompute,
|
||||
);
|
||||
|
||||
for (const { workspaceCacheKeyName, data, hash } of computed) {
|
||||
Object.assign(result, { [workspaceCacheKeyName]: data });
|
||||
this.setInLocalCache(workspaceId, workspaceCacheKeyName, data, hash);
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
private async partitionKeysByLocalStaleness(
|
||||
workspaceId: string,
|
||||
workspaceCacheKeys: WorkspaceCacheKeyName[],
|
||||
): Promise<{
|
||||
validFromLocal: WorkspaceCacheKeyName[];
|
||||
needsRedisCheck: WorkspaceCacheKeyName[];
|
||||
}> {
|
||||
const validFromLocal: WorkspaceCacheKeyName[] = [];
|
||||
const needsRedisCheck: WorkspaceCacheKeyName[] = [];
|
||||
|
||||
const hashKeys = workspaceCacheKeys.map(
|
||||
(workspaceCacheKeyName) =>
|
||||
`${this.buildCacheKey(workspaceId, workspaceCacheKeyName)}:hash`,
|
||||
);
|
||||
|
||||
const redisHashes = await this.cacheStorage.mget<string>(hashKeys);
|
||||
|
||||
for (const [index, workspaceCacheKeyName] of workspaceCacheKeys.entries()) {
|
||||
const redisHash = redisHashes[index];
|
||||
const localKey = this.buildCacheKey(workspaceId, workspaceCacheKeyName);
|
||||
const localEntry = this.localCache.get(localKey);
|
||||
|
||||
if (
|
||||
isDefined(localEntry) &&
|
||||
isDefined(redisHash) &&
|
||||
localEntry.hash === redisHash
|
||||
while (
|
||||
entry.versions.size >= MAX_LOCAL_STALE_VERSIONS &&
|
||||
sorted.length > 0
|
||||
) {
|
||||
validFromLocal.push(workspaceCacheKeyName);
|
||||
} else {
|
||||
needsRedisCheck.push(workspaceCacheKeyName);
|
||||
const oldestEntry = sorted.shift();
|
||||
|
||||
if (isDefined(oldestEntry)) {
|
||||
entry.versions.delete(oldestEntry[0]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return { validFromLocal, needsRedisCheck };
|
||||
}
|
||||
|
||||
private async fetchDataFromRedis(
|
||||
workspaceId: string,
|
||||
workspaceCacheKeys: WorkspaceCacheKeyName[],
|
||||
): Promise<{
|
||||
validDataFromRedis: Array<{
|
||||
workspaceCacheKeyName: WorkspaceCacheKeyName;
|
||||
data: CacheDataType;
|
||||
hash: string;
|
||||
}>;
|
||||
cacheKeysToRecomputeFromProviders: WorkspaceCacheKeyName[];
|
||||
}> {
|
||||
const validDataFromRedis: Array<{
|
||||
workspaceCacheKeyName: WorkspaceCacheKeyName;
|
||||
data: CacheDataType;
|
||||
hash: string;
|
||||
}> = [];
|
||||
const cacheKeysToRecomputeFromProviders: WorkspaceCacheKeyName[] = [];
|
||||
|
||||
const dataKeys = workspaceCacheKeys.map(
|
||||
(workspaceCacheKeyName) =>
|
||||
`${this.buildCacheKey(workspaceId, workspaceCacheKeyName)}:data`,
|
||||
);
|
||||
|
||||
const redisData = await this.cacheStorage.mget<CacheDataType>(dataKeys);
|
||||
|
||||
for (const [index, workspaceCacheKeyName] of workspaceCacheKeys.entries()) {
|
||||
const data = redisData[index];
|
||||
|
||||
if (isDefined(data)) {
|
||||
const hash = this.generateHash(data);
|
||||
|
||||
validDataFromRedis.push({ workspaceCacheKeyName, data, hash });
|
||||
} else {
|
||||
cacheKeysToRecomputeFromProviders.push(workspaceCacheKeyName);
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
validDataFromRedis,
|
||||
cacheKeysToRecomputeFromProviders,
|
||||
};
|
||||
}
|
||||
|
||||
private getProviderOrThrow(
|
||||
workspaceCacheKeyName: WorkspaceCacheKeyName,
|
||||
keyName: WorkspaceCacheKeyName,
|
||||
): WorkspaceCacheProvider<CacheDataType> {
|
||||
const provider = this.workspaceCacheProviders.get(workspaceCacheKeyName);
|
||||
const provider = this.workspaceCacheProviders.get(keyName);
|
||||
|
||||
if (!isDefined(provider)) {
|
||||
throw new Error(
|
||||
`Cache provider with key name "${workspaceCacheKeyName}" not found`,
|
||||
);
|
||||
throw new Error(`Cache provider with key name "${keyName}" not found`);
|
||||
}
|
||||
|
||||
return provider;
|
||||
}
|
||||
|
||||
private async computeAndStoreInRedis(
|
||||
workspaceId: string,
|
||||
workspaceCacheKeys: WorkspaceCacheKeyName[],
|
||||
): Promise<
|
||||
Array<{
|
||||
workspaceCacheKeyName: WorkspaceCacheKeyName;
|
||||
data: CacheDataType;
|
||||
hash: string;
|
||||
}>
|
||||
> {
|
||||
const computePromises = workspaceCacheKeys.map(
|
||||
async (workspaceCacheKeyName) => {
|
||||
const provider = this.getProviderOrThrow(workspaceCacheKeyName);
|
||||
|
||||
const data = await provider.computeForCache(workspaceId);
|
||||
|
||||
return { workspaceCacheKeyName, data };
|
||||
},
|
||||
);
|
||||
|
||||
const computed = await Promise.all(computePromises);
|
||||
|
||||
const redisEntries: Array<{ key: string; value: unknown }> = [];
|
||||
|
||||
for (const { workspaceCacheKeyName, data } of computed) {
|
||||
const hash = this.generateHash(data);
|
||||
|
||||
redisEntries.push({
|
||||
key: `${this.buildCacheKey(workspaceId, workspaceCacheKeyName)}:data`,
|
||||
value: data,
|
||||
});
|
||||
redisEntries.push({
|
||||
key: `${this.buildCacheKey(workspaceId, workspaceCacheKeyName)}:hash`,
|
||||
value: hash,
|
||||
});
|
||||
}
|
||||
|
||||
await this.cacheStorage.mset(redisEntries);
|
||||
|
||||
return computed.map(({ workspaceCacheKeyName, data }) => ({
|
||||
workspaceCacheKeyName,
|
||||
data,
|
||||
hash: this.generateHash(data),
|
||||
}));
|
||||
}
|
||||
|
||||
private setInLocalCache(
|
||||
workspaceId: string,
|
||||
workspaceCacheKeyName: WorkspaceCacheKeyName,
|
||||
data: CacheDataType,
|
||||
hash: string,
|
||||
): void {
|
||||
const localKey = this.buildCacheKey(workspaceId, workspaceCacheKeyName);
|
||||
|
||||
this.localCache.set(localKey, {
|
||||
data,
|
||||
hash,
|
||||
lastCheckedAt: Date.now(),
|
||||
});
|
||||
}
|
||||
|
||||
private buildCacheKey(
|
||||
workspaceId: string,
|
||||
workspaceCacheKeyName: WorkspaceCacheKeyName,
|
||||
keyName: WorkspaceCacheKeyName,
|
||||
): string {
|
||||
return `${WORKSPACE_CACHE_KEYS_V2[workspaceCacheKeyName]}:${workspaceId}`;
|
||||
}
|
||||
|
||||
private generateHash(data: unknown): string {
|
||||
return crypto
|
||||
.createHash('sha256')
|
||||
.update(JSON.stringify(data))
|
||||
.digest('hex');
|
||||
return `${WORKSPACE_CACHE_KEYS_V2[keyName]}:${workspaceId}`;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { type ObjectsPermissionsByRoleId } from 'twenty-shared/types';
|
||||
import { type EntityMetadata } from 'typeorm';
|
||||
|
||||
import { type FlatApplicationCacheMaps } from 'src/engine/core-modules/application/types/flat-application-cache-maps.type';
|
||||
import { type FeatureFlagKey } from 'src/engine/core-modules/feature-flag/enums/feature-flag-key.enum';
|
||||
@@ -25,6 +26,7 @@ export const WORKSPACE_CACHE_KEYS_V2 = {
|
||||
flatApplicationMaps: 'flat-maps:flatApplicationMaps',
|
||||
flatRoleMaps: 'flat-maps:role',
|
||||
flatRoleTargetMaps: 'flat-maps:role-target',
|
||||
ORMEntityMetadatas: 'orm:entity-metadatas',
|
||||
flatAgentMaps: 'flat-maps:agent',
|
||||
flatRoleTargetByAgentIdMaps: 'flat-maps:flatRoleTargetByAgentId',
|
||||
} as const satisfies Record<WorkspaceCacheKeyName, string>;
|
||||
@@ -35,6 +37,7 @@ type AdditionalCacheDataMap = {
|
||||
userWorkspaceRoleMap: UserWorkspaceRoleMap;
|
||||
apiKeyRoleMap: Record<string, string>;
|
||||
flatApplicationMaps: FlatApplicationCacheMaps;
|
||||
ORMEntityMetadatas: EntityMetadata[];
|
||||
flatRoleTargetByAgentIdMaps: FlatRoleTargetByAgentIdMaps;
|
||||
};
|
||||
|
||||
|
||||
+8
-3
@@ -1,5 +1,10 @@
|
||||
export type WorkspaceLocalCacheEntry<T> = {
|
||||
export type VersionEntry<T> = {
|
||||
data: T;
|
||||
hash: string;
|
||||
lastCheckedAt: number;
|
||||
lastReadAt: number;
|
||||
};
|
||||
|
||||
export type WorkspaceLocalCacheEntry<T> = {
|
||||
versions: Map<string, VersionEntry<T>>;
|
||||
latestHash: string;
|
||||
lastHashCheckedAt: number;
|
||||
};
|
||||
|
||||
+1
@@ -124,6 +124,7 @@ export class WorkspaceMigrationRunnerV2Service {
|
||||
'userWorkspaceRoleMap',
|
||||
'flatRoleTargetMaps',
|
||||
'apiKeyRoleMap',
|
||||
'ORMEntityMetadatas',
|
||||
'flatRoleTargetByAgentIdMaps',
|
||||
]),
|
||||
);
|
||||
|
||||
Reference in New Issue
Block a user