AI tools to create a demo workspace (#18236)

This PR adds the necessary tool to create a demo workspace with : relevant custom objects and fields, mock data and a real dashboard with graph widgets. It is still a bit under-optimized and slow but it works. This PR also adds an AI tool that allows to see what happens in real time, it navigates the app and waits when necessary. --------- Co-authored-by: Félix Malfait <felix.malfait@gmail.com>
2026-03-05 11:39:31 +01:00
parent a2f80d882b
commit 2a82df7073
84 changed files with 2449 additions and 315 deletions
@@ -3,4 +3,5 @@ export enum ToolType {
  SEND_EMAIL = 'SEND_EMAIL',
  SEARCH_HELP_CENTER = 'SEARCH_HELP_CENTER',
  CODE_INTERPRETER = 'CODE_INTERPRETER',
+  NAVIGATE_APP = 'NAVIGATE_APP',
 }
@@ -12,7 +12,12 @@ import { DraftEmailTool } from 'src/engine/core-modules/tool/tools/email-tool/dr
 import { EmailComposerService } from 'src/engine/core-modules/tool/tools/email-tool/email-composer.service';
 import { SendEmailTool } from 'src/engine/core-modules/tool/tools/email-tool/send-email-tool';
 import { HttpTool } from 'src/engine/core-modules/tool/tools/http-tool/http-tool';
+import { NavigateAppTool } from 'src/engine/core-modules/tool/tools/navigate-tool/navigate-app-tool';
 import { SearchHelpCenterTool } from 'src/engine/core-modules/tool/tools/search-help-center-tool/search-help-center-tool';
+import { WorkspaceManyOrAllFlatEntityMapsCacheModule } from 'src/engine/metadata-modules/flat-entity/services/workspace-many-or-all-flat-entity-maps-cache.module';
+import { NavigationMenuItemModule } from 'src/engine/metadata-modules/navigation-menu-item/navigation-menu-item.module';
+import { ObjectMetadataModule } from 'src/engine/metadata-modules/object-metadata/object-metadata.module';
+import { ViewModule } from 'src/engine/metadata-modules/view/view.module';
 import { MessagingImportManagerModule } from 'src/modules/messaging/message-import-manager/messaging-import-manager.module';
 import { MessagingSendManagerModule } from 'src/modules/messaging/message-outbound-manager/messaging-send-manager.module';

@@ -26,6 +31,10 @@ import { MessagingSendManagerModule } from 'src/modules/messaging/message-outbou
    FileModule,
    JwtModule,
    SecureHttpClientModule,
+    ObjectMetadataModule,
+    ViewModule,
+    NavigationMenuItemModule,
+    WorkspaceManyOrAllFlatEntityMapsCacheModule,
  ],
  providers: [
    HttpTool,
@@ -34,6 +43,7 @@ import { MessagingSendManagerModule } from 'src/modules/messaging/message-outbou
    EmailComposerService,
    SearchHelpCenterTool,
    CodeInterpreterTool,
+    NavigateAppTool,
  ],
  exports: [
    HttpTool,
@@ -42,6 +52,7 @@ import { MessagingSendManagerModule } from 'src/modules/messaging/message-outbou
    EmailComposerService,
    SearchHelpCenterTool,
    CodeInterpreterTool,
+    NavigateAppTool,
  ],
 })
 export class ToolModule {}
@@ -0,0 +1,62 @@
+import { z } from 'zod';
+
+export const NavigateAppInputZodSchema = z.discriminatedUnion('type', [
+  z.object({
+    type: z
+      .literal('navigateToView')
+      .describe(
+        'Navigate to a specific view by name. ONLY use this type when the user explicitly mentions the word "view" (e.g. "go to the My Companies view", "open view All People"). Do NOT use this for general navigation requests.',
+      ),
+    viewName: z
+      .string()
+      .describe(
+        'The name of the view to navigate to (e.g. "My Companies", "All People")',
+      ),
+  }),
+  z.object({
+    type: z
+      .literal('navigateToObject')
+      .describe(
+        'Navigate to the default view for an object. This is the PREFERRED and DEFAULT type for all navigation requests unless the user explicitly mentions the word "view".',
+      ),
+    objectNameSingular: z
+      .string()
+      .describe(
+        'The singular name of the object to navigate to (e.g. "company", "person", "opportunity")',
+      ),
+  }),
+  z.object({
+    type: z
+      .literal('navigateToRecord')
+      .describe(
+        'Navigate to a specific record page. Use this when the user wants to go to a particular record by name (e.g. "go to the company Acme", "open the person John Doe", "show me the deal Enterprise Plan").',
+      ),
+    objectNameSingular: z
+      .string()
+      .describe(
+        'The singular name of the object type (e.g. "company", "person", "opportunity")',
+      ),
+    recordName: z
+      .string()
+      .describe(
+        'The name or label of the record to navigate to (e.g. "Acme", "John Doe", "Enterprise Plan")',
+      ),
+  }),
+  z.object({
+    type: z
+      .literal('wait')
+      .describe(
+        'Wait for a specified duration in milliseconds before continuing. Useful when you need the page to fully load after a navigation before taking further actions (e.g. 2000 for 2 seconds).',
+      ),
+    durationMs: z
+      .number()
+      .int()
+      .min(0)
+      .max(30000)
+      .describe(
+        'The duration in milliseconds to wait (e.g. 2000 for 2 seconds). Maximum 30000 (30 seconds).',
+      ),
+  }),
+]);
+
+export type NavigateAppInput = z.infer<typeof NavigateAppInputZodSchema>;
@@ -0,0 +1,357 @@
+import { Injectable } from '@nestjs/common';
+
+import { sleep } from 'cloudflare/core';
+import Fuse from 'fuse.js';
+import { NavigateAppToolOutput } from 'twenty-shared/ai';
+import { FieldMetadataType, type ObjectRecord } from 'twenty-shared/types';
+import { isDefined } from 'twenty-shared/utils';
+
+import {
+  type NavigateAppInput,
+  NavigateAppInputZodSchema,
+} from 'src/engine/core-modules/tool/tools/navigate-tool/navigate-app-tool.schema';
+import { type ToolInput } from 'src/engine/core-modules/tool/types/tool-input.type';
+import { ToolOutput } from 'src/engine/core-modules/tool/types/tool-output.type';
+import {
+  type Tool,
+  type ToolExecutionContext,
+} from 'src/engine/core-modules/tool/types/tool.type';
+import { WorkspaceManyOrAllFlatEntityMapsCacheService } from 'src/engine/metadata-modules/flat-entity/services/workspace-many-or-all-flat-entity-maps-cache.service';
+import { findFlatEntityByIdInFlatEntityMaps } from 'src/engine/metadata-modules/flat-entity/utils/find-flat-entity-by-id-in-flat-entity-maps.util';
+import { type FlatObjectMetadata } from 'src/engine/metadata-modules/flat-object-metadata/types/flat-object-metadata.type';
+import { NavigationMenuItemService } from 'src/engine/metadata-modules/navigation-menu-item/navigation-menu-item.service';
+import { ViewService } from 'src/engine/metadata-modules/view/services/view.service';
+import { GlobalWorkspaceOrmManager } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager';
+import { buildSystemAuthContext } from 'src/engine/twenty-orm/utils/build-system-auth-context.util';
+
+@Injectable()
+export class NavigateAppTool implements Tool {
+  description = `Navigate the application.
+    Use navigateToRecord when the user wants to go to a specific record by name.
+    Default to navigateToObject for all other navigation requests.
+    Only use navigateToView when the user explicitly mentions the word "view" in their request.
+    If the user asks to wait, use the wait tool with the specified duration.`;
+
+  inputSchema = NavigateAppInputZodSchema;
+
+  constructor(
+    private readonly navigationMenuItemService: NavigationMenuItemService,
+    private readonly viewService: ViewService,
+    private readonly workspaceManyOrAllFlatEntityMapsCacheService: WorkspaceManyOrAllFlatEntityMapsCacheService,
+    private readonly globalWorkspaceOrmManager: GlobalWorkspaceOrmManager,
+  ) {}
+
+  async execute(
+    parameters: ToolInput,
+    context: ToolExecutionContext,
+  ): Promise<ToolOutput> {
+    const parseResult = NavigateAppInputZodSchema.safeParse(parameters);
+
+    if (!parseResult.success) {
+      return {
+        success: false,
+        message: 'Invalid navigation input',
+        error: parseResult.error.message,
+      };
+    }
+
+    const input: NavigateAppInput = parseResult.data;
+
+    switch (input.type) {
+      case 'navigateToView':
+        return this.navigateToView(
+          input.viewName,
+          context.workspaceId,
+          context.userWorkspaceId,
+        );
+      case 'navigateToObject':
+        return this.navigateToObject(
+          input.objectNameSingular,
+          context.workspaceId,
+        );
+      case 'navigateToRecord':
+        return this.navigateToRecord(
+          input.objectNameSingular,
+          input.recordName,
+          context.workspaceId,
+        );
+      case 'wait':
+        return this.wait(input.durationMs);
+    }
+  }
+
+  private async wait(
+    durationMs: number,
+  ): Promise<ToolOutput<NavigateAppToolOutput>> {
+    await sleep(durationMs);
+
+    return {
+      success: true,
+      message: `Waited  for ${durationMs}ms`,
+      result: {
+        action: 'wait',
+        durationMs,
+      },
+    };
+  }
+
+  private async navigateToView(
+    viewName: string,
+    workspaceId: string,
+    userWorkspaceId?: string,
+  ): Promise<ToolOutput<NavigateAppToolOutput>> {
+    const views = await this.viewService.findByWorkspaceId(
+      workspaceId,
+      userWorkspaceId,
+    );
+
+    const fuse = new Fuse(views, {
+      keys: ['name'],
+      threshold: 0.4,
+    });
+
+    const results = fuse.search(viewName);
+    const matchingView = results[0]?.item;
+
+    if (!matchingView) {
+      const availableViewNames = views.map((view) => view.name).join(', ');
+
+      return {
+        success: false,
+        message: `View "${viewName}" not found`,
+        error: `No view matching "${viewName}" was found in this workspace. Available views: ${availableViewNames}`,
+      };
+    }
+
+    return {
+      success: true,
+      message: `Navigating to view "${matchingView.name}"`,
+      result: {
+        action: 'navigateToView',
+        viewName: matchingView.name,
+      },
+    };
+  }
+
+  private async navigateToObject(
+    objectNameSingular: string,
+    workspaceId: string,
+  ): Promise<ToolOutput<NavigateAppToolOutput>> {
+    const navigationMenuItems = await this.navigationMenuItemService.findAll({
+      workspaceId,
+    });
+
+    const { flatObjectMetadataMaps, flatViewMaps } =
+      await this.workspaceManyOrAllFlatEntityMapsCacheService.getOrRecomputeManyOrAllFlatEntityMaps(
+        {
+          workspaceId,
+          flatMapsKeys: [
+            'flatObjectMetadataMaps',
+            'flatViewMaps',
+            'flatNavigationMenuItemMaps',
+          ],
+        },
+      );
+
+    const availableObjectNames = navigationMenuItems
+      .map((navigationMenuItem) => {
+        if (isDefined(navigationMenuItem.viewId)) {
+          const correspondingViewUniversalIdentifier =
+            flatViewMaps.universalIdentifierById[navigationMenuItem.viewId];
+
+          if (isDefined(correspondingViewUniversalIdentifier)) {
+            const correspondingView =
+              flatViewMaps.byUniversalIdentifier[
+                correspondingViewUniversalIdentifier
+              ];
+
+            if (isDefined(correspondingView)) {
+              const correspondingObjectMetadataUniversalIdentifier =
+                flatObjectMetadataMaps.universalIdentifierById[
+                  correspondingView.objectMetadataId
+                ];
+
+              if (isDefined(correspondingObjectMetadataUniversalIdentifier)) {
+                const correspondingObjectMetadata =
+                  flatObjectMetadataMaps.byUniversalIdentifier[
+                    correspondingObjectMetadataUniversalIdentifier
+                  ];
+
+                if (isDefined(correspondingObjectMetadata)) {
+                  const correspondingObjectNameSingular =
+                    correspondingObjectMetadata.nameSingular;
+
+                  return correspondingObjectNameSingular;
+                }
+              }
+            }
+          }
+        }
+
+        return null;
+      })
+      .filter(isDefined);
+
+    const fuse = new Fuse(availableObjectNames, {
+      threshold: 0.6,
+    });
+
+    const results = fuse.search(objectNameSingular.replace(/\s/g, ''));
+    const firstMatchingNavigationItemLabel = results[0]?.item;
+
+    if (!isDefined(firstMatchingNavigationItemLabel)) {
+      return {
+        success: false,
+        message: `Object "${objectNameSingular}" not found`,
+        error: `No object with singular name "${objectNameSingular}" was found in this workspace. Available objects: ${availableObjectNames}`,
+      };
+    }
+
+    return {
+      success: true,
+      message: `Navigating to ${firstMatchingNavigationItemLabel} default view`,
+      result: {
+        action: 'navigateToObject',
+        objectNameSingular: firstMatchingNavigationItemLabel,
+      },
+    };
+  }
+
+  private async navigateToRecord(
+    objectNameSingular: string,
+    recordName: string,
+    workspaceId: string,
+  ): Promise<ToolOutput<NavigateAppToolOutput>> {
+    const { flatObjectMetadataMaps, flatFieldMetadataMaps } =
+      await this.workspaceManyOrAllFlatEntityMapsCacheService.getOrRecomputeManyOrAllFlatEntityMaps(
+        {
+          workspaceId,
+          flatMapsKeys: ['flatObjectMetadataMaps', 'flatFieldMetadataMaps'],
+        },
+      );
+
+    const flatObjectMetadata = Object.values(
+      flatObjectMetadataMaps.byUniversalIdentifier,
+    ).find(
+      (metadata): metadata is FlatObjectMetadata =>
+        isDefined(metadata) &&
+        metadata.nameSingular === objectNameSingular &&
+        metadata.isActive,
+    );
+
+    if (!isDefined(flatObjectMetadata)) {
+      const availableObjectNames = Object.values(
+        flatObjectMetadataMaps.byUniversalIdentifier,
+      )
+        .filter(
+          (metadata): metadata is FlatObjectMetadata =>
+            isDefined(metadata) && metadata.isActive,
+        )
+        .map((metadata) => metadata.nameSingular)
+        .join(', ');
+
+      return {
+        success: false,
+        message: `Object "${objectNameSingular}" not found`,
+        error: `No object with singular name "${objectNameSingular}" was found. Available objects: ${availableObjectNames}`,
+      };
+    }
+
+    if (!isDefined(flatObjectMetadata.labelIdentifierFieldMetadataId)) {
+      return {
+        success: false,
+        message: `Object "${objectNameSingular}" has no label identifier field`,
+        error: `Cannot search records by name for object "${objectNameSingular}" because it has no label identifier field configured.`,
+      };
+    }
+
+    const labelIdentifierField = findFlatEntityByIdInFlatEntityMaps({
+      flatEntityId: flatObjectMetadata.labelIdentifierFieldMetadataId,
+      flatEntityMaps: flatFieldMetadataMaps,
+    });
+
+    if (!isDefined(labelIdentifierField)) {
+      return {
+        success: false,
+        message: `Label identifier field not found for object "${objectNameSingular}"`,
+        error: `The label identifier field metadata could not be resolved for object "${objectNameSingular}".`,
+      };
+    }
+
+    const isFullName =
+      labelIdentifierField.type === FieldMetadataType.FULL_NAME;
+
+    const selectColumns = isFullName
+      ? [
+          'id',
+          `${labelIdentifierField.name}FirstName`,
+          `${labelIdentifierField.name}LastName`,
+        ]
+      : ['id', labelIdentifierField.name];
+
+    const authContext = buildSystemAuthContext(workspaceId);
+
+    const records =
+      await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
+        async () => {
+          const repository =
+            await this.globalWorkspaceOrmManager.getRepository<ObjectRecord>(
+              workspaceId,
+              objectNameSingular,
+              { shouldBypassPermissionChecks: true },
+            );
+
+          return repository.find({
+            select: selectColumns,
+          });
+        },
+        authContext,
+      );
+
+    const recordsWithDisplayName = records.map((record) => {
+      let displayName: string;
+
+      if (isFullName) {
+        const firstName =
+          (record[`${labelIdentifierField.name}FirstName`] as string) ?? '';
+        const lastName =
+          (record[`${labelIdentifierField.name}LastName`] as string) ?? '';
+
+        displayName = `${firstName} ${lastName}`.trim();
+      } else {
+        displayName = String(record[labelIdentifierField.name] ?? '');
+      }
+
+      return {
+        id: record.id as string,
+        displayName,
+      };
+    });
+
+    const fuse = new Fuse(recordsWithDisplayName, {
+      keys: ['displayName'],
+      threshold: 0.4,
+    });
+
+    const results = fuse.search(recordName);
+    const matchingRecord = results[0]?.item;
+
+    if (!isDefined(matchingRecord)) {
+      return {
+        success: false,
+        message: `Record "${recordName}" not found in ${objectNameSingular}`,
+        error: `No ${objectNameSingular} record matching "${recordName}" was found.`,
+      };
+    }
+
+    return {
+      success: true,
+      message: `Navigating to ${objectNameSingular} record "${matchingRecord.displayName}"`,
+      result: {
+        action: 'navigateToRecord',
+        objectNameSingular,
+        recordId: matchingRecord.id,
+      },
+    };
+  }
+}