1541 extensibility twenty cli use workspace migration v2 to synchronize application objects fields views (#14706)

- synchronize objects https://github.com/user-attachments/assets/257317bc-2881-4b98-a3d4-6ae52bd72aa0
2025-09-25 11:22:11 +02:00
parent 71216b4db8
commit edb331d68b
34 changed files with 596 additions and 683 deletions
@@ -0,0 +1,236 @@
+import { Injectable, Logger } from '@nestjs/common';
+
+import { isDefined } from 'twenty-shared/utils';
+
+import {
+  AgentManifest,
+  AppManifest,
+  ObjectManifest,
+} from 'src/engine/core-modules/application/types/application.types';
+import { ApplicationService } from 'src/engine/core-modules/application/application.service';
+import { WorkspaceManyOrAllFlatEntityMapsCacheService } from 'src/engine/core-modules/common/services/workspace-many-or-all-flat-entity-maps-cache.service.';
+import type { FlatObjectMetadataWithFlatFieldMaps } from 'src/engine/metadata-modules/flat-object-metadata-maps/types/flat-object-metadata-with-flat-field-metadata-maps.type';
+import { ObjectMetadataServiceV2 } from 'src/engine/metadata-modules/object-metadata/object-metadata-v2.service';
+import { DataSourceService } from 'src/engine/metadata-modules/data-source/data-source.service';
+import { AgentService } from 'src/engine/metadata-modules/agent/agent.service';
+import {
+  ApplicationException,
+  ApplicationExceptionCode,
+} from 'src/engine/core-modules/application/application.exception';
+
+@Injectable()
+export class ApplicationSyncService {
+  private readonly logger = new Logger(ApplicationSyncService.name);
+
+  constructor(
+    private readonly applicationService: ApplicationService,
+    private readonly objectMetadataServiceV2: ObjectMetadataServiceV2,
+    private readonly flatEntityMapsCacheService: WorkspaceManyOrAllFlatEntityMapsCacheService,
+    private readonly dataSourceService: DataSourceService,
+    private readonly agentService: AgentService,
+  ) {}
+
+  public async synchronizeFromManifest(
+    workspaceId: string,
+    manifest: AppManifest,
+  ) {
+    const applicationId = await this.syncApplication(manifest, workspaceId);
+
+    await this.syncAgents({
+      agentsToSync: manifest.agents,
+      workspaceId,
+      applicationId,
+    });
+
+    await this.syncObjects({
+      objectsToSync: manifest.objects,
+      workspaceId,
+      applicationId,
+    });
+
+    this.logger.log('✅ Application sync from manifest completed');
+  }
+
+  private async syncApplication(
+    applicationToSync: AppManifest,
+    workspaceId: string,
+  ): Promise<string> {
+    const application = await this.applicationService.findByStandardId(
+      applicationToSync.standardId,
+      workspaceId,
+    );
+
+    if (!isDefined(application)) {
+      const createdApplication = await this.applicationService.create({
+        standardId: applicationToSync.standardId,
+        label: applicationToSync.label,
+        description: applicationToSync.description,
+        version: applicationToSync.version,
+        sourcePath: 'cli-sync', // Placeholder for CLI-synced apps
+        workspaceId,
+      });
+
+      return createdApplication.id;
+    }
+
+    await this.applicationService.update(application.id, {
+      label: applicationToSync.label,
+      description: applicationToSync.description,
+      version: applicationToSync.version,
+    });
+
+    return application.id;
+  }
+
+  private async syncAgents({
+    agentsToSync,
+    workspaceId,
+    applicationId,
+  }: {
+    agentsToSync: AgentManifest[];
+    workspaceId: string;
+    applicationId: string;
+  }) {
+    for (const agentToSync of agentsToSync) {
+      const existingAgent =
+        await this.agentService.findOneByApplicationAndStandardId({
+          workspaceId,
+          applicationId,
+          standardId: agentToSync.standardId,
+        });
+
+      if (isDefined(existingAgent)) {
+        await this.agentService.updateOneAgent(
+          { id: existingAgent.id, ...agentToSync },
+          workspaceId,
+        );
+
+        return;
+      }
+
+      await this.agentService.createOneAgent(
+        {
+          name: agentToSync.name,
+          label: agentToSync.label,
+          description: agentToSync.description,
+          icon: agentToSync.icon,
+          prompt: agentToSync.prompt,
+          modelId: agentToSync.modelId,
+          standardId: agentToSync.standardId,
+          isCustom: true,
+          applicationId,
+        },
+        workspaceId,
+      );
+    }
+  }
+
+  private async syncObjects({
+    objectsToSync,
+    workspaceId,
+    applicationId,
+  }: {
+    objectsToSync: ObjectManifest[];
+    workspaceId: string;
+    applicationId: string;
+  }) {
+    const { flatObjectMetadataMaps: existingFlatObjectMetadataMaps } =
+      await this.flatEntityMapsCacheService.getOrRecomputeManyOrAllFlatEntityMaps(
+        {
+          workspaceId,
+          flatEntities: ['flatObjectMetadataMaps'],
+        },
+      );
+
+    const applicationObjects = Object.values(
+      existingFlatObjectMetadataMaps.byId,
+    ).filter(
+      (obj) => isDefined(obj) && obj.applicationId === applicationId,
+    ) as FlatObjectMetadataWithFlatFieldMaps[];
+
+    const objectsToSyncStandardIds = objectsToSync.map((obj) => obj.standardId);
+
+    const applicationObjectsStandardIds = applicationObjects.map(
+      (obj) => obj.standardId,
+    );
+
+    const objectsToDelete = applicationObjects.filter(
+      (obj) =>
+        isDefined(obj.standardId) &&
+        !objectsToSyncStandardIds.includes(obj.standardId),
+    );
+
+    const objectsToUpdate = applicationObjects.filter(
+      (obj) =>
+        isDefined(obj.standardId) &&
+        objectsToSyncStandardIds.includes(obj.standardId),
+    );
+
+    const objectsToCreate = objectsToSync.filter(
+      (objectToSync) =>
+        !applicationObjectsStandardIds.includes(objectToSync.standardId),
+    );
+
+    for (const objectToDelete of objectsToDelete) {
+      await this.objectMetadataServiceV2.deleteOne({
+        deleteObjectInput: { id: objectToDelete.id },
+        workspaceId,
+        isSystemBuild: true,
+      });
+    }
+
+    for (const objectToUpdate of objectsToUpdate) {
+      const objectToSync = objectsToSync.find(
+        (obj) => obj.standardId === objectToUpdate.standardId,
+      );
+
+      if (!objectToSync) {
+        throw new ApplicationException(
+          `Failed to find object to sync with standardId ${objectToUpdate.standardId}`,
+          ApplicationExceptionCode.OBJECT_NOT_FOUND,
+        );
+      }
+
+      const updateObjectInput = {
+        id: objectToUpdate.id,
+        update: {
+          nameSingular: objectToSync.nameSingular,
+          namePlural: objectToSync.namePlural,
+          labelSingular: objectToSync.labelSingular,
+          labelPlural: objectToSync.labelPlural,
+          icon: objectToSync.icon || undefined,
+          description: objectToSync.description || undefined,
+        },
+      };
+
+      await this.objectMetadataServiceV2.updateOne({
+        updateObjectInput,
+        workspaceId,
+      });
+    }
+
+    const dataSourceMetadata =
+      await this.dataSourceService.getLastDataSourceMetadataFromWorkspaceIdOrFail(
+        workspaceId,
+      );
+
+    for (const objectToCreate of objectsToCreate) {
+      const createObjectInput = {
+        nameSingular: objectToCreate.nameSingular,
+        namePlural: objectToCreate.namePlural,
+        labelSingular: objectToCreate.labelSingular,
+        labelPlural: objectToCreate.labelPlural,
+        icon: objectToCreate.icon || undefined,
+        description: objectToCreate.description || undefined,
+        standardId: objectToCreate.standardId || undefined,
+        dataSourceId: dataSourceMetadata.id,
+        applicationId,
+      };
+
+      await this.objectMetadataServiceV2.createOne({
+        createObjectInput,
+        workspaceId,
+      });
+    }
+  }
+}
@@ -0,0 +1,7 @@
+import { CustomException } from 'src/utils/custom-exception';
+
+export class ApplicationException extends CustomException<ApplicationExceptionCode> {}
+
+export enum ApplicationExceptionCode {
+  OBJECT_NOT_FOUND = 'OBJECT_NOT_FOUND',
+}
@@ -4,23 +4,22 @@ import { TypeOrmModule } from '@nestjs/typeorm';
 import { ApplicationEntity } from 'src/engine/core-modules/application/application.entity';
 import { ApplicationResolver } from 'src/engine/core-modules/application/application.resolver';
 import { ApplicationService } from 'src/engine/core-modules/application/application.service';
-import { LocalApplicationSourceProvider } from 'src/engine/core-modules/application/providers/local-application-source.provider';
-import { ApplicationSyncAgentService } from 'src/engine/core-modules/application/services/application-sync-agent.service';
-import { ApplicationSyncService } from 'src/engine/core-modules/application/services/application-sync.service';
+import { ApplicationSyncService } from 'src/engine/core-modules/application/application-sync.service';
 import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity';
 import { AgentEntity } from 'src/engine/metadata-modules/agent/agent.entity';
+import { WorkspaceManyOrAllFlatEntityMapsCacheModule } from 'src/engine/core-modules/common/services/workspace-many-or-all-flat-entity-maps-cache.module';
+import { ObjectMetadataModule } from 'src/engine/metadata-modules/object-metadata/object-metadata.module';
+import { DataSourceModule } from 'src/engine/metadata-modules/data-source/data-source.module';
+import { AgentModule } from 'src/engine/metadata-modules/agent/agent.module';

@Module({
  imports: [
    TypeOrmModule.forFeature([ApplicationEntity, AgentEntity, Workspace]),
+    WorkspaceManyOrAllFlatEntityMapsCacheModule,
+    ObjectMetadataModule,
+    DataSourceModule,
+    AgentModule,
  ],
-  providers: [
-    ApplicationResolver,
-    ApplicationService,
-    ApplicationSyncService,
-    ApplicationSyncAgentService,
-    LocalApplicationSourceProvider,
-  ],
-  exports: [ApplicationService, ApplicationSyncService],
+  providers: [ApplicationResolver, ApplicationService, ApplicationSyncService],
 })
 export class ApplicationModule {}
@@ -3,13 +3,11 @@ import { Args, Mutation, Resolver } from '@nestjs/graphql';

 import GraphQLJSON from 'graphql-type-json';

+import { AppManifest } from 'src/engine/core-modules/application/types/application.types';
 import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity';
 import { AuthWorkspace } from 'src/engine/decorators/auth/auth-workspace.decorator';
 import { WorkspaceAuthGuard } from 'src/engine/guards/workspace-auth.guard';
-
-import { ApplicationDTO } from './dtos/application.dto';
-import { ApplicationSyncService } from './services/application-sync.service';
-import { ApplicationManifest } from './types/application-manifest.type';
+import { ApplicationSyncService } from 'src/engine/core-modules/application/application-sync.service';

@UseGuards(WorkspaceAuthGuard)
@Resolver()
@@ -18,18 +16,17 @@ export class ApplicationResolver {
    private readonly applicationSyncService: ApplicationSyncService,
  ) {}

-  @Mutation(() => ApplicationDTO)
+  @Mutation(() => Boolean)
  async syncApplication(
    @Args('manifest', { type: () => GraphQLJSON })
-    manifest: ApplicationManifest,
+    manifest: AppManifest,
    @AuthWorkspace() { id: workspaceId }: Workspace,
-  ): Promise<ApplicationDTO> {
-    const application =
-      await this.applicationSyncService.synchronizeFromManifest(
-        workspaceId,
-        manifest,
-      );
+  ) {
+    await this.applicationSyncService.synchronizeFromManifest(
+      workspaceId,
+      manifest,
+    );

-    return application;
+    return true;
  }
 }
@@ -18,11 +18,8 @@ export class ApplicationService {
    });
  }

-  async findByStandardId(
-    standardId: string,
-    workspaceId: string,
-  ): Promise<ApplicationEntity[]> {
-    return this.applicationRepository.find({
+  async findByStandardId(standardId: string, workspaceId: string) {
+    return this.applicationRepository.findOne({
      where: {
        standardId,
        workspaceId,
@@ -1,58 +0,0 @@
-import { Field, ObjectType } from '@nestjs/graphql';
-
-import {
-  IsDateString,
-  IsNotEmpty,
-  IsOptional,
-  IsString,
-  IsUUID,
-} from 'class-validator';
-
-import { UUIDScalarType } from 'src/engine/api/graphql/workspace-schema-builder/graphql-types/scalars';
-
-@ObjectType('Application')
-export class ApplicationDTO {
-  @IsUUID()
-  @IsNotEmpty()
-  @Field(() => UUIDScalarType)
-  id: string;
-
-  @IsOptional()
-  @IsUUID()
-  @Field(() => UUIDScalarType, { nullable: true })
-  standardId?: string;
-
-  @IsString()
-  @Field(() => String)
-  label: string;
-
-  @IsOptional()
-  @IsString()
-  @Field(() => String, { nullable: true })
-  description?: string | null;
-
-  @IsOptional()
-  @IsString()
-  @Field(() => String, { nullable: true })
-  version?: string | null;
-
-  @IsString()
-  @Field(() => String)
-  sourceType: string;
-
-  @IsString()
-  @Field(() => String)
-  sourcePath: string;
-
-  @IsUUID()
-  @Field(() => UUIDScalarType)
-  workspaceId: string;
-
-  @IsDateString()
-  @Field(() => Date)
-  createdAt: Date;
-
-  @IsDateString()
-  @Field(() => Date)
-  updatedAt: Date;
-}
@@ -1,35 +0,0 @@
-import { Injectable } from '@nestjs/common';
-
-import { promises as fs } from 'fs';
-import * as path from 'path';
-
-import { ApplicationManifest } from 'src/engine/core-modules/application/types/application-manifest.type';
-
-@Injectable()
-export class LocalApplicationSourceProvider {
-  async fetchManifest(localPath: string): Promise<ApplicationManifest> {
-    const manifestPath = path.join(localPath, 'twenty-app.json');
-
-    try {
-      const manifestContent = await fs.readFile(manifestPath, 'utf-8');
-
-      return JSON.parse(manifestContent) as ApplicationManifest;
-    } catch (error) {
-      throw new Error(
-        `Failed to read manifest from ${manifestPath}: ${error.message}`,
-      );
-    }
-  }
-
-  async validateSource(localPath: string): Promise<boolean> {
-    const manifestPath = path.join(localPath, 'twenty-app.json');
-
-    try {
-      await fs.access(manifestPath);
-
-      return true;
-    } catch {
-      return false;
-    }
-  }
-}
@@ -1,72 +0,0 @@
-import { Injectable, Logger } from '@nestjs/common';
-import { InjectRepository } from '@nestjs/typeorm';
-
-import { Repository } from 'typeorm';
-
-import { ApplicationSyncContext } from 'src/engine/core-modules/application/services/application-sync.service';
-import { AgentEntity } from 'src/engine/metadata-modules/agent/agent.entity';
-import { FlatAgent } from 'src/engine/metadata-modules/flat-agent/types/flat-agent.type';
-
-@Injectable()
-export class ApplicationSyncAgentService {
-  private readonly logger = new Logger(ApplicationSyncAgentService.name);
-
-  constructor(
-    @InjectRepository(AgentEntity)
-    private readonly agentRepository: Repository<AgentEntity>,
-  ) {}
-
-  async synchronize(
-    context: ApplicationSyncContext,
-    agents: FlatAgent[],
-  ): Promise<void> {
-    if (!agents || agents.length === 0) {
-      this.logger.log('No agents to synchronize');
-
-      return;
-    }
-
-    for (const agentDefinition of agents) {
-      this.logger.log(`Syncing agent: ${agentDefinition.label}`);
-
-      // Check if agent already exists
-      const existingAgent = await this.agentRepository.findOne({
-        where: {
-          workspaceId: context.workspaceId,
-          applicationId: context.applicationId,
-          name: agentDefinition.name,
-        },
-      });
-
-      if (existingAgent) {
-        // Update existing agent
-        await this.agentRepository.update(
-          { id: existingAgent.id },
-          {
-            label: agentDefinition.label,
-            description: agentDefinition.description,
-            icon: agentDefinition.icon,
-            prompt: agentDefinition.prompt,
-            modelId: agentDefinition.modelId,
-          },
-        );
-        this.logger.log(`Updated agent: ${agentDefinition.label}`);
-      } else {
-        // Create new agent
-        const newAgent = this.agentRepository.create({
-          name: agentDefinition.name,
-          label: agentDefinition.label,
-          description: agentDefinition.description,
-          icon: agentDefinition.icon,
-          prompt: agentDefinition.prompt,
-          modelId: agentDefinition.modelId,
-          workspaceId: context.workspaceId,
-          applicationId: context.applicationId,
-        });
-
-        await this.agentRepository.save(newAgent);
-        this.logger.log(`Created agent: ${agentDefinition.label}`);
-      }
-    }
-  }
-}
@@ -1,120 +0,0 @@
-import { Injectable, Logger } from '@nestjs/common';
-import { InjectRepository } from '@nestjs/typeorm';
-
-import { Repository } from 'typeorm';
-
-import { ApplicationEntity } from 'src/engine/core-modules/application/application.entity';
-import { ApplicationService } from 'src/engine/core-modules/application/application.service';
-import { LocalApplicationSourceProvider } from 'src/engine/core-modules/application/providers/local-application-source.provider';
-import { ApplicationSyncAgentService } from 'src/engine/core-modules/application/services/application-sync-agent.service';
-import { FlatAgent } from 'src/engine/metadata-modules/flat-agent/types/flat-agent.type';
-
-export interface ApplicationSyncContext {
-  workspaceId: string;
-  featureFlags: Record<string, boolean>;
-  applicationId: string;
-}
-
-interface ApplicationManifest {
-  standardId: string;
-  label: string;
-  description?: string;
-  version?: string;
-  agents?: FlatAgent[];
-}
-
-@Injectable()
-export class ApplicationSyncService {
-  private readonly logger = new Logger(ApplicationSyncService.name);
-
-  constructor(
-    @InjectRepository(ApplicationEntity)
-    private readonly applicationRepository: Repository<ApplicationEntity>,
-    private readonly localSourceProvider: LocalApplicationSourceProvider,
-    private readonly applicationSyncAgentService: ApplicationSyncAgentService,
-    private readonly applicationService: ApplicationService,
-  ) {}
-
-  public async synchronize(context: ApplicationSyncContext): Promise<void> {
-    this.logger.log(`Syncing agents for application: ${context.applicationId}`);
-
-    const application = await this.applicationRepository.findOne({
-      where: { id: context.applicationId },
-    });
-
-    if (!application) {
-      throw new Error(`Application with ID ${context.applicationId} not found`);
-    }
-
-    const manifest = await this.localSourceProvider.fetchManifest(
-      application.sourcePath,
-    );
-
-    this.logger.log(`Syncing application: ${manifest.label}`);
-
-    await this.applicationSyncAgentService.synchronize(
-      context,
-      manifest.agents,
-    );
-
-    this.logger.log('✅ Agent sync completed');
-  }
-
-  public async synchronizeFromManifest(
-    workspaceId: string,
-    manifest: ApplicationManifest,
-  ): Promise<ApplicationEntity> {
-    this.logger.log(`Syncing application from manifest: ${manifest.label}`);
-
-    // Find or create application
-    let application = await this.applicationService.findByStandardId(
-      manifest.standardId,
-      workspaceId,
-    );
-
-    if (application.length === 0) {
-      // Create new application
-      application = [
-        await this.applicationService.create({
-          standardId: manifest.standardId,
-          label: manifest.label,
-          description: manifest.description,
-          version: manifest.version,
-          sourcePath: 'cli-sync', // Placeholder for CLI-synced apps
-          workspaceId,
-        }),
-      ];
-      this.logger.log(`Created new application: ${manifest.label}`);
-    } else {
-      // Update existing application
-      const existingApp = application[0];
-
-      await this.applicationService.update(existingApp.id, {
-        label: manifest.label,
-        description: manifest.description,
-        version: manifest.version,
-      });
-      this.logger.log(`Updated existing application: ${manifest.label}`);
-    }
-
-    const app = application[0];
-
-    // Sync agents
-    if (manifest.agents && manifest.agents.length > 0) {
-      const context: ApplicationSyncContext = {
-        workspaceId,
-        featureFlags: {}, // TODO: Get actual feature flags
-        applicationId: app.id,
-      };
-
-      await this.applicationSyncAgentService.synchronize(
-        context,
-        manifest.agents,
-      );
-    }
-
-    this.logger.log('✅ Application sync from manifest completed');
-
-    return app;
-  }
-}
@@ -1,10 +0,0 @@
-import { type FlatAgent } from 'src/engine/metadata-modules/flat-agent/types/flat-agent.type';
-
-export type ApplicationManifest = {
-  standardId: string;
-  label: string;
-  description?: string;
-  icon?: string;
-  version: string;
-  agents: FlatAgent[];
-};
@@ -0,0 +1,43 @@
+export type PackageJson = {
+  $schema?: string;
+  standardId: string;
+  label: string;
+  description?: string;
+  icon?: string;
+  version: string;
+  dependencies?: object;
+  devDependencies?: object;
+};
+
+export type AppManifest = PackageJson & {
+  agents: AgentManifest[];
+  objects: ObjectManifest[];
+};
+
+export type ObjectManifest = {
+  $schema?: string;
+  standardId: string;
+  nameSingular: string;
+  namePlural: string;
+  labelSingular: string;
+  labelPlural: string;
+  description?: string;
+  icon?: string;
+};
+
+interface AgentResponseFormat {
+  type: 'json' | 'text';
+  schema?: Record<string, unknown>;
+}
+
+export type AgentManifest = {
+  $schema?: string;
+  standardId: string;
+  name: string;
+  label: string;
+  description?: string;
+  icon?: string;
+  prompt: string;
+  modelId: string;
+  responseFormat?: AgentResponseFormat;
+};