Deprecate object metadata maps in favor of flat entities (#16080)
## Context Deprecating the old objectMetadataMap type in favour of split flat entities to match with our new caching. In the long run, trying to achieve: - Better performance through caching - Consistent data access patterns across the codebase - Reduced database queries Now that everything is based on flat entities, which are cached, we can finish the refactoring of workspace context cache which should already improve performances. Then the last step will be to consume that new cache in the new global datasource to get rid of the many workspace datasources stored in the server
This commit is contained in:
Weiko
+19
-11
@@ -28,13 +28,12 @@ import { AgentEntity } from 'src/engine/metadata-modules/ai/ai-agent/entities/ag
|
||||
import { AgentActorContextService } from 'src/engine/metadata-modules/ai/ai-agent/services/agent-actor-context.service';
|
||||
import { AgentModelConfigService } from 'src/engine/metadata-modules/ai/ai-agent/services/agent-model-config.service';
|
||||
import { AgentToolGeneratorService } from 'src/engine/metadata-modules/ai/ai-agent/services/agent-tool-generator.service';
|
||||
import { type RecordIdsByObjectMetadataNameSingularType } from 'src/engine/metadata-modules/ai/ai-agent/types/recordIdsByObjectMetadataNameSingular.type';
|
||||
import { RecordIdsByObjectMetadataNameSingularType } from 'src/engine/metadata-modules/ai/ai-agent/types/recordIdsByObjectMetadataNameSingular.type';
|
||||
import { repairToolCall } from 'src/engine/metadata-modules/ai/ai-agent/utils/repair-tool-call.util';
|
||||
import { AIBillingService } from 'src/engine/metadata-modules/ai/ai-billing/services/ai-billing.service';
|
||||
import { type ToolHints } from 'src/engine/metadata-modules/ai/ai-chat-router/types/tool-hints.interface';
|
||||
import { ToolHints } from 'src/engine/metadata-modules/ai/ai-chat-router/types/tool-hints.interface';
|
||||
import { AI_TELEMETRY_CONFIG } from 'src/engine/metadata-modules/ai/ai-models/constants/ai-telemetry.const';
|
||||
import { AiModelRegistryService } from 'src/engine/metadata-modules/ai/ai-models/services/ai-model-registry.service';
|
||||
import { getObjectMetadataMapItemByNameSingular } from 'src/engine/metadata-modules/utils/get-object-metadata-map-item-by-name-singular.util';
|
||||
import { WorkspacePermissionsCacheService } from 'src/engine/metadata-modules/workspace-permissions-cache/workspace-permissions-cache.service';
|
||||
import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager';
|
||||
|
||||
@@ -188,8 +187,12 @@ export class AgentExecutionService {
|
||||
workspaceId: workspace.id,
|
||||
});
|
||||
|
||||
const objectMetadataMaps =
|
||||
workspaceDataSource.internalContext.objectMetadataMaps;
|
||||
const flatObjectMetadataMaps =
|
||||
workspaceDataSource.internalContext.flatObjectMetadataMaps;
|
||||
const flatFieldMetadataMaps =
|
||||
workspaceDataSource.internalContext.flatFieldMetadataMaps;
|
||||
const objectIdByNameSingular =
|
||||
workspaceDataSource.internalContext.objectIdByNameSingular;
|
||||
const objectMetadataPermissions = workspaceDataSource.permissionsPerRoleId;
|
||||
|
||||
const contextObject = (
|
||||
@@ -200,11 +203,13 @@ export class AgentExecutionService {
|
||||
return [];
|
||||
}
|
||||
|
||||
const objectMetadataMapItem =
|
||||
getObjectMetadataMapItemByNameSingular(
|
||||
objectMetadataMaps,
|
||||
recordsWithObjectMetadataNameSingular.objectMetadataNameSingular,
|
||||
);
|
||||
const objectMetadataId =
|
||||
objectIdByNameSingular[
|
||||
recordsWithObjectMetadataNameSingular.objectMetadataNameSingular
|
||||
];
|
||||
const objectMetadataMapItem = objectMetadataId
|
||||
? flatObjectMetadataMaps.byId[objectMetadataId]
|
||||
: undefined;
|
||||
|
||||
if (!objectMetadataMapItem) {
|
||||
this.logger.warn(
|
||||
@@ -230,7 +235,10 @@ export class AgentExecutionService {
|
||||
const selectOptions = hasRestrictedFields
|
||||
? getAllSelectableColumnNames({
|
||||
restrictedFields,
|
||||
objectMetadata: { objectMetadataMapItem },
|
||||
objectMetadata: {
|
||||
objectMetadataMapItem,
|
||||
flatFieldMetadataMaps,
|
||||
},
|
||||
})
|
||||
: undefined;
|
||||
|
||||
|
||||
+3
-5
@@ -1,7 +1,5 @@
|
||||
import { STANDARD_OBJECT_IDS } from 'twenty-shared/metadata';
|
||||
|
||||
import { type ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity';
|
||||
|
||||
// All workflow-related standard object IDs that should be filtered out from agent access
|
||||
const WORKFLOW_STANDARD_OBJECT_IDS = [
|
||||
STANDARD_OBJECT_IDS.workflow,
|
||||
@@ -10,9 +8,9 @@ const WORKFLOW_STANDARD_OBJECT_IDS = [
|
||||
STANDARD_OBJECT_IDS.workflowAutomatedTrigger,
|
||||
] as const;
|
||||
|
||||
export const isWorkflowRelatedObject = (
|
||||
objectMetadata: ObjectMetadataEntity,
|
||||
): boolean => {
|
||||
export const isWorkflowRelatedObject = (objectMetadata: {
|
||||
standardId: string | null;
|
||||
}): boolean => {
|
||||
return (
|
||||
objectMetadata.standardId !== null &&
|
||||
WORKFLOW_STANDARD_OBJECT_IDS.includes(
|
||||
|
||||
@@ -10,7 +10,7 @@ import { ToolModule } from 'src/engine/core-modules/tool/tool.module';
|
||||
import { SearchArticlesTool } from 'src/engine/core-modules/tool/tools/search-articles-tool/search-articles-tool';
|
||||
import { ToolAdapterService } from 'src/engine/metadata-modules/ai/ai-tools/services/tool-adapter.service';
|
||||
import { ToolService } from 'src/engine/metadata-modules/ai/ai-tools/services/tool.service';
|
||||
import { ObjectMetadataModule } from 'src/engine/metadata-modules/object-metadata/object-metadata.module';
|
||||
import { WorkspaceManyOrAllFlatEntityMapsCacheModule } from 'src/engine/metadata-modules/flat-entity/services/workspace-many-or-all-flat-entity-maps-cache.module';
|
||||
import { PermissionsModule } from 'src/engine/metadata-modules/permissions/permissions.module';
|
||||
import { RoleEntity } from 'src/engine/metadata-modules/role/role.entity';
|
||||
import { UserRoleModule } from 'src/engine/metadata-modules/user-role/user-role.module';
|
||||
@@ -27,7 +27,7 @@ import { MessagingModule } from 'src/modules/messaging/messaging.module';
|
||||
TokenModule,
|
||||
FeatureFlagModule,
|
||||
RecordCrudModule,
|
||||
ObjectMetadataModule,
|
||||
WorkspaceManyOrAllFlatEntityMapsCacheModule,
|
||||
WorkspacePermissionsCacheModule,
|
||||
WorkspaceCacheStorageModule,
|
||||
UserRoleModule,
|
||||
|
||||
+15
-3
@@ -6,7 +6,7 @@ import { FindRecordsService } from 'src/engine/core-modules/record-crud/services
|
||||
import { UpdateRecordService } from 'src/engine/core-modules/record-crud/services/update-record.service';
|
||||
import { RecordInputTransformerService } from 'src/engine/core-modules/record-transformer/services/record-input-transformer.service';
|
||||
import { ToolService } from 'src/engine/metadata-modules/ai/ai-tools/services/tool.service';
|
||||
import { ObjectMetadataServiceV2 } from 'src/engine/metadata-modules/object-metadata/object-metadata-v2.service';
|
||||
import { WorkspaceManyOrAllFlatEntityMapsCacheService } from 'src/engine/metadata-modules/flat-entity/services/workspace-many-or-all-flat-entity-maps-cache.service';
|
||||
import { WorkspacePermissionsCacheService } from 'src/engine/metadata-modules/workspace-permissions-cache/workspace-permissions-cache.service';
|
||||
import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager';
|
||||
import { WorkspaceCacheStorageService } from 'src/engine/workspace-cache-storage/workspace-cache-storage.service';
|
||||
@@ -57,9 +57,21 @@ describe('ToolService', () => {
|
||||
},
|
||||
},
|
||||
{
|
||||
provide: ObjectMetadataServiceV2,
|
||||
provide: WorkspaceManyOrAllFlatEntityMapsCacheService,
|
||||
useValue: {
|
||||
findManyWithinWorkspace: jest.fn().mockResolvedValue([testObject]),
|
||||
getOrRecomputeManyOrAllFlatEntityMaps: jest.fn().mockResolvedValue({
|
||||
flatObjectMetadataMaps: {
|
||||
byId: {
|
||||
[testObject.id]: {
|
||||
...testObject,
|
||||
fieldMetadataIds: [],
|
||||
},
|
||||
},
|
||||
},
|
||||
flatFieldMetadataMaps: {
|
||||
byId: {},
|
||||
},
|
||||
}),
|
||||
},
|
||||
},
|
||||
{
|
||||
|
||||
+25
-12
@@ -2,7 +2,9 @@ import { Injectable, Logger } from '@nestjs/common';
|
||||
|
||||
import { type ToolSet } from 'ai';
|
||||
import { type ActorMetadata } from 'twenty-shared/types';
|
||||
import { isDefined } from 'twenty-shared/utils';
|
||||
|
||||
import { getFlatFieldsFromFlatObjectMetadata } from 'src/engine/api/graphql/workspace-schema-builder/utils/get-flat-fields-for-flat-object-metadata.util';
|
||||
import { CreateRecordService } from 'src/engine/core-modules/record-crud/services/create-record.service';
|
||||
import { DeleteRecordService } from 'src/engine/core-modules/record-crud/services/delete-record.service';
|
||||
import { FindRecordsService } from 'src/engine/core-modules/record-crud/services/find-records.service';
|
||||
@@ -15,13 +17,13 @@ import { generateFindToolInputSchema } from 'src/engine/core-modules/record-crud
|
||||
import { SoftDeleteToolInputSchema } from 'src/engine/core-modules/record-crud/zod-schemas/soft-delete-tool.zod-schema';
|
||||
import { isWorkflowRelatedObject } from 'src/engine/metadata-modules/ai/ai-agent/utils/is-workflow-related-object.util';
|
||||
import {
|
||||
type ToolHints,
|
||||
type ToolOperation,
|
||||
ToolHints,
|
||||
ToolOperation,
|
||||
} from 'src/engine/metadata-modules/ai/ai-chat-router/types/tool-hints.interface';
|
||||
import { ObjectMetadataServiceV2 } from 'src/engine/metadata-modules/object-metadata/object-metadata-v2.service';
|
||||
import { WorkspaceManyOrAllFlatEntityMapsCacheService } from 'src/engine/metadata-modules/flat-entity/services/workspace-many-or-all-flat-entity-maps-cache.service';
|
||||
import { WorkspacePermissionsCacheService } from 'src/engine/metadata-modules/workspace-permissions-cache/workspace-permissions-cache.service';
|
||||
import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager';
|
||||
import { type RolePermissionConfig } from 'src/engine/twenty-orm/types/role-permission-config';
|
||||
import { RolePermissionConfig } from 'src/engine/twenty-orm/types/role-permission-config';
|
||||
import { computePermissionIntersection } from 'src/engine/twenty-orm/utils/compute-permission-intersection.util';
|
||||
|
||||
@Injectable()
|
||||
@@ -30,8 +32,8 @@ export class ToolService {
|
||||
|
||||
constructor(
|
||||
private readonly twentyORMGlobalManager: TwentyORMGlobalManager,
|
||||
private readonly objectMetadataService: ObjectMetadataServiceV2,
|
||||
protected readonly workspacePermissionsCacheService: WorkspacePermissionsCacheService,
|
||||
private readonly flatEntityMapsCacheService: WorkspaceManyOrAllFlatEntityMapsCacheService,
|
||||
private readonly createRecordService: CreateRecordService,
|
||||
private readonly updateRecordService: UpdateRecordService,
|
||||
private readonly deleteRecordService: DeleteRecordService,
|
||||
@@ -78,14 +80,25 @@ export class ToolService {
|
||||
return tools;
|
||||
}
|
||||
|
||||
const allObjectMetadata =
|
||||
await this.objectMetadataService.findManyWithinWorkspace(workspaceId, {
|
||||
where: {
|
||||
isActive: true,
|
||||
isSystem: false,
|
||||
const { flatObjectMetadataMaps, flatFieldMetadataMaps } =
|
||||
await this.flatEntityMapsCacheService.getOrRecomputeManyOrAllFlatEntityMaps(
|
||||
{
|
||||
workspaceId,
|
||||
flatMapsKeys: ['flatObjectMetadataMaps', 'flatFieldMetadataMaps'],
|
||||
},
|
||||
relations: ['fields'],
|
||||
});
|
||||
);
|
||||
|
||||
const allFlatObjects = Object.values(flatObjectMetadataMaps.byId)
|
||||
.filter(isDefined)
|
||||
.filter((obj) => obj.isActive && !obj.isSystem);
|
||||
|
||||
const allObjectMetadata = allFlatObjects.map((flatObject) => ({
|
||||
...flatObject,
|
||||
fields: getFlatFieldsFromFlatObjectMetadata(
|
||||
flatObject,
|
||||
flatFieldMetadataMaps,
|
||||
),
|
||||
}));
|
||||
|
||||
let filteredObjectMetadata = allObjectMetadata.filter(
|
||||
(objectMetadata) => !isWorkflowRelatedObject(objectMetadata),
|
||||
|
||||
Reference in New Issue
Block a user