Refactor validate build and run for configuration to be less verbose and more reliable (#16343)

# Introduction Refactored the api `validateBuildAndRunWorkspaceMigration` to be require less configuration but to infer required args dynamically depending on provided metadata maps to compare ## `inferDeletionFromMissingEntities` Is not dynamically computed avoiding any miss configuration issue and any missleading devxp ## Maps computation Making only one call to redis to build both dependency and to be compared entity maps. It does not matter to avoid passing a about to compared flat entity maps to could also be a depedency, it's handled directly in the builder setup optimistic cache logic Please note that the flat maps used for the service input transpilation might differ from the one that we will dynamically compute and inject in the builder. Leading to do 2 redis calls but also race condition prone validation error We prefer that this occurs at the builder rather than at the runner level as the pg instance is not cache and reflect the real state of a given workspace In a nutshell, there's a possible race condition between cache invalidation and computation in both service input transpilers and builder but we're totally ok with that
2025-12-05 16:41:56 +01:00
parent fee3e6b7a1
commit c8f541618d
22 changed files with 554 additions and 976 deletions
@@ -1,8 +1,8 @@
 import { type FlatObjectMetadata } from 'src/engine/metadata-modules/flat-object-metadata/types/flat-object-metadata.type';
 import { STANDARD_OBJECTS } from 'src/engine/workspace-manager/twenty-standard-application/constants/standard-object.constant';
 import { type AllStandardObjectName } from 'src/engine/workspace-manager/twenty-standard-application/types/all-standard-object-name.type';
-import { type StandardFieldMetadataIdByObjectAndFieldName } from 'src/engine/workspace-manager/twenty-standard-application/utils/get-standard-field-metadata-id-by-object-and-field-name.util';
 import { createStandardObjectFlatMetadata } from 'src/engine/workspace-manager/twenty-standard-application/utils/create-standard-object-flat-metadata.util';
+import { type StandardFieldMetadataIdByObjectAndFieldName } from 'src/engine/workspace-manager/twenty-standard-application/utils/get-standard-field-metadata-id-by-object-and-field-name.util';

 export type BuildStandardFlatObjectMetadatasArgs = {
  createdAt: Date;
@@ -14,11 +14,6 @@ export type StandardFlatObjectMetadataBuilder = (
  args: BuildStandardFlatObjectMetadatasArgs,
 ) => FlatObjectMetadata;

-/**
- * Record mapping each standard object name to its corresponding object metadata builder.
- * Using `satisfies` ensures type safety: if a new standard object is added to STANDARD_OBJECTS,
- * TypeScript will error until a builder is added here.
- */
 export const STANDARD_FLAT_OBJECT_METADATA_BUILDERS_BY_OBJECT_NAME = {
  attachment: (args: BuildStandardFlatObjectMetadatasArgs) =>
    createStandardObjectFlatMetadata({
@@ -15,12 +15,12 @@ import {
 } from 'src/engine/workspace-manager/workspace-migration-v2/types/workspace-migration-orchestrator.type';
 import { aggregateOrchestratorActionsReport } from 'src/engine/workspace-manager/workspace-migration-v2/utils/aggregate-orchestrator-actions-report.util';
 import { WorkspaceMigrationV2AgentActionsBuilderService } from 'src/engine/workspace-manager/workspace-migration-v2/workspace-migration-builder-v2/builders/agent/workspace-migration-v2-agent-actions-builder.service';
-import { WorkspaceMigrationV2PageLayoutTabActionsBuilderService } from 'src/engine/workspace-manager/workspace-migration-v2/workspace-migration-builder-v2/builders/page-layout-tab/workspace-migration-v2-page-layout-tab-actions-builder.service';
 import { WorkspaceMigrationV2CronTriggerActionsBuilderService } from 'src/engine/workspace-manager/workspace-migration-v2/workspace-migration-builder-v2/builders/cron-trigger/workspace-migration-v2-cron-trigger-action-builder.service';
 import { WorkspaceMigrationV2DatabaseEventTriggerActionsBuilderService } from 'src/engine/workspace-manager/workspace-migration-v2/workspace-migration-builder-v2/builders/database-event-trigger/workspace-migration-v2-database-event-trigger-actions-builder.service';
 import { WorkspaceMigrationV2FieldActionsBuilderService } from 'src/engine/workspace-manager/workspace-migration-v2/workspace-migration-builder-v2/builders/field/workspace-migration-v2-field-actions-builder.service';
 import { WorkspaceMigrationV2IndexActionsBuilderService } from 'src/engine/workspace-manager/workspace-migration-v2/workspace-migration-builder-v2/builders/index/workspace-migration-v2-index-actions-builder.service';
 import { WorkspaceMigrationV2ObjectActionsBuilderService } from 'src/engine/workspace-manager/workspace-migration-v2/workspace-migration-builder-v2/builders/object/workspace-migration-v2-object-actions-builder.service';
+import { WorkspaceMigrationV2PageLayoutTabActionsBuilderService } from 'src/engine/workspace-manager/workspace-migration-v2/workspace-migration-builder-v2/builders/page-layout-tab/workspace-migration-v2-page-layout-tab-actions-builder.service';
 import { WorkspaceMigrationV2RoleTargetActionsBuilderService } from 'src/engine/workspace-manager/workspace-migration-v2/workspace-migration-builder-v2/builders/role-target/workspace-migration-v2-role-target-actions-builder.service';
 import { WorkspaceMigrationV2RoleActionsBuilderService } from 'src/engine/workspace-manager/workspace-migration-v2/workspace-migration-builder-v2/builders/role/workspace-migration-v2-role-actions-builder.service';
 import { WorkspaceMigrationV2RouteTriggerActionsBuilderService } from 'src/engine/workspace-manager/workspace-migration-v2/workspace-migration-builder-v2/builders/route-trigger/workspace-migration-v2-route-trigger-actions-builder.service';
@@ -65,7 +65,10 @@ export class WorkspaceMigrationBuildOrchestratorService {
      (allFlatEntityMaps, currFlatMaps) => {
        const fromToOccurence = fromToAllFlatEntityMaps[currFlatMaps];

-        if (!isDefined(fromToOccurence)) {
+        if (
+          !isDefined(fromToOccurence) ||
+          isDefined(allFlatEntityMaps[currFlatMaps])
+        ) {
          return allFlatEntityMaps;
        }

@@ -1,15 +1,35 @@
 import { Injectable, Logger } from '@nestjs/common';

-import { WorkspaceMigrationV2ExceptionCode } from 'twenty-shared/metadata';
+import {
+  AllMetadataName,
+  WorkspaceMigrationV2ExceptionCode,
+} from 'twenty-shared/metadata';
+import { isDefined } from 'twenty-shared/utils';

+import { ALL_METADATA_REQUIRED_METADATA_FOR_VALIDATION } from 'src/engine/metadata-modules/flat-entity/constant/all-metadata-required-metadata-for-validation.constant';
+import { createEmptyFlatEntityMaps } from 'src/engine/metadata-modules/flat-entity/constant/create-empty-flat-entity-maps.constant';
+import { AllFlatEntityMaps } from 'src/engine/metadata-modules/flat-entity/types/all-flat-entity-maps.type';
+import { FlatEntityToCreateDeleteUpdate } from 'src/engine/metadata-modules/flat-entity/types/flat-entity-to-create-delete-update.type';
+import { computeFlatEntityMapsFromTo } from 'src/engine/metadata-modules/flat-entity/utils/compute-flat-entity-maps-from-to.util';
+import { getMetadataFlatEntityMapsKey } from 'src/engine/metadata-modules/flat-entity/utils/get-metadata-flat-entity-maps-key.util';
+import { WorkspaceCacheService } from 'src/engine/workspace-cache/services/workspace-cache.service';
 import { WorkspaceMigrationBuildOrchestratorService } from 'src/engine/workspace-manager/workspace-migration-v2/services/workspace-migration-build-orchestrator.service';
 import {
  WorkspaceMigrationOrchestratorBuildArgs,
  WorkspaceMigrationOrchestratorFailedResult,
 } from 'src/engine/workspace-manager/workspace-migration-v2/types/workspace-migration-orchestrator.type';
+import { InferDeletionFromMissingEntities } from 'src/engine/workspace-manager/workspace-migration-v2/workspace-migration-builder-v2/types/infer-deletion-from-missing-entities.type';
 import { WorkspaceMigrationRunnerV2Service } from 'src/engine/workspace-manager/workspace-migration-v2/workspace-migration-runner-v2/services/workspace-migration-runner-v2.service';
 import { WorkspaceMigrationV2Exception } from 'src/engine/workspace-manager/workspace-migration.exception';

+type ValidateBuildAndRunWorkspaceMigrationFromMatriceArgs = {
+  workspaceId: string;
+  allFlatEntityOperationByMetadataName: {
+    [P in AllMetadataName]?: FlatEntityToCreateDeleteUpdate<P>;
+  };
+  isSystemBuild?: boolean;
+};
+
@Injectable()
 export class WorkspaceMigrationValidateBuildAndRunService {
  private readonly logger = new Logger(
@@ -19,14 +39,144 @@ export class WorkspaceMigrationValidateBuildAndRunService {
  constructor(
    private readonly workspaceMigrationRunnerV2Service: WorkspaceMigrationRunnerV2Service,
    private readonly workspaceMigrationBuildOrchestratorService: WorkspaceMigrationBuildOrchestratorService,
+    private readonly workspaceCacheService: WorkspaceCacheService,
  ) {}

-  public async validateBuildAndRunWorkspaceMigration(
-    builderArgs: WorkspaceMigrationOrchestratorBuildArgs,
-  ): Promise<WorkspaceMigrationOrchestratorFailedResult | undefined> {
+  private async computeAllRelatedFlatEntityMaps({
+    allFlatEntityOperationByMetadataName,
+    workspaceId,
+  }: ValidateBuildAndRunWorkspaceMigrationFromMatriceArgs) {
+    const allMetadataNameToCompare = Object.keys(
+      allFlatEntityOperationByMetadataName,
+    ) as AllMetadataName[];
+    const allDependencyMetadataName = allMetadataNameToCompare.flatMap(
+      (metadataName) =>
+        Object.keys(
+          ALL_METADATA_REQUIRED_METADATA_FOR_VALIDATION[metadataName],
+        ) as AllMetadataName[],
+    );
+    const allMetadataNameCacheToCompute = [
+      ...new Set([...allMetadataNameToCompare, ...allDependencyMetadataName]),
+    ];
+    const allFlatEntityMapsCacheKeysToCompute =
+      allMetadataNameCacheToCompute.map(getMetadataFlatEntityMapsKey);
+
+    const allRelatedFlatEntityMaps =
+      await this.workspaceCacheService.getOrRecompute(
+        workspaceId,
+        allFlatEntityMapsCacheKeysToCompute,
+      );
+
+    const initialAccumulator = allDependencyMetadataName.reduce<
+      Partial<AllFlatEntityMaps>
+    >(
+      (allFlatEntityMaps, metadataName) => ({
+        ...allFlatEntityMaps,
+        [getMetadataFlatEntityMapsKey(metadataName)]:
+          createEmptyFlatEntityMaps(),
+      }),
+      {},
+    );
+    const dependencyAllFlatEntityMaps = allDependencyMetadataName.reduce(
+      (allFlatEntityMaps, metadataName) => {
+        const metadataFlatEntityMapsKey =
+          getMetadataFlatEntityMapsKey(metadataName);
+
+        return {
+          ...allFlatEntityMaps,
+          [metadataFlatEntityMapsKey]:
+            allRelatedFlatEntityMaps[metadataFlatEntityMapsKey],
+        };
+      },
+      initialAccumulator,
+    );
+
+    return {
+      allRelatedFlatEntityMaps,
+      dependencyAllFlatEntityMaps,
+    };
+  }
+
+  private async computeFromToAllFlatEntityMapsAndBuildOptions({
+    allFlatEntityOperationByMetadataName,
+    workspaceId,
+  }: ValidateBuildAndRunWorkspaceMigrationFromMatriceArgs): Promise<{
+    fromToAllFlatEntityMaps: WorkspaceMigrationOrchestratorBuildArgs['fromToAllFlatEntityMaps'];
+    inferDeletionFromMissingEntities: InferDeletionFromMissingEntities;
+    dependencyAllFlatEntityMaps: Partial<AllFlatEntityMaps>;
+  }> {
+    const { allRelatedFlatEntityMaps, dependencyAllFlatEntityMaps } =
+      await this.computeAllRelatedFlatEntityMaps({
+        allFlatEntityOperationByMetadataName,
+        workspaceId,
+      });
+
+    const fromToAllFlatEntityMaps: WorkspaceMigrationOrchestratorBuildArgs['fromToAllFlatEntityMaps'] =
+      {};
+    const inferDeletionFromMissingEntities: InferDeletionFromMissingEntities =
+      {};
+    const allMetadataNameToCompare = Object.keys(
+      allFlatEntityOperationByMetadataName,
+    ) as AllMetadataName[];
+
+    for (const metadataName of allMetadataNameToCompare) {
+      const tmp = allFlatEntityOperationByMetadataName[metadataName];
+
+      if (!isDefined(tmp)) {
+        throw new Error('Should never occurs');
+      }
+      const { flatEntityToCreate, flatEntityToDelete, flatEntityToUpdate } =
+        tmp;
+      const flatEntityMapsKey = getMetadataFlatEntityMapsKey(metadataName);
+      const flatEntityMaps = allRelatedFlatEntityMaps[flatEntityMapsKey];
+
+      // @ts-expect-error Metadata flat entity maps cache key and metadataName colliding
+      fromToAllFlatEntityMaps[flatEntityMapsKey] = computeFlatEntityMapsFromTo({
+        flatEntityMaps,
+        flatEntityToCreate,
+        flatEntityToDelete,
+        flatEntityToUpdate,
+      });
+
+      if (flatEntityToDelete.length > 0) {
+        inferDeletionFromMissingEntities[metadataName] = true;
+      }
+    }
+
+    return {
+      fromToAllFlatEntityMaps,
+      inferDeletionFromMissingEntities,
+      dependencyAllFlatEntityMaps,
+    };
+  }
+
+  public async validateBuildAndRunWorkspaceMigration({
+    allFlatEntityOperationByMetadataName: allFlatEntities,
+    workspaceId,
+    isSystemBuild = false,
+  }: ValidateBuildAndRunWorkspaceMigrationFromMatriceArgs): Promise<
+    WorkspaceMigrationOrchestratorFailedResult | undefined
+  > {
+    const {
+      fromToAllFlatEntityMaps,
+      inferDeletionFromMissingEntities,
+      dependencyAllFlatEntityMaps,
+    } = await this.computeFromToAllFlatEntityMapsAndBuildOptions({
+      allFlatEntityOperationByMetadataName: allFlatEntities,
+      workspaceId,
+    });
+
    const validateAndBuildResult =
      await this.workspaceMigrationBuildOrchestratorService
-        .buildWorkspaceMigration(builderArgs)
+        .buildWorkspaceMigration({
+          buildOptions: {
+            isSystemBuild,
+            inferDeletionFromMissingEntities,
+          },
+          fromToAllFlatEntityMaps,
+          workspaceId,
+          dependencyAllFlatEntityMaps,
+        })
        .catch((error) => {
          this.logger.error(error);
          throw new WorkspaceMigrationV2Exception(
@@ -0,0 +1,6 @@
+import { type AllMetadataName } from 'twenty-shared/metadata';
+
+export type InferDeletionFromMissingEntities =
+  | true
+  | Partial<Record<AllMetadataName, boolean>>
+  | undefined;
@@ -1,8 +1,6 @@
-import { type AllMetadataName } from 'twenty-shared/metadata';
+import { type InferDeletionFromMissingEntities } from 'src/engine/workspace-manager/workspace-migration-v2/workspace-migration-builder-v2/types/infer-deletion-from-missing-entities.type';

 export type WorkspaceMigrationBuilderOptions = {
-  inferDeletionFromMissingEntities?:
-    | true
-    | Partial<Record<AllMetadataName, boolean>>;
+  inferDeletionFromMissingEntities?: InferDeletionFromMissingEntities;
  isSystemBuild: boolean;
 };
@@ -1,5 +1,6 @@
 import { Module } from '@nestjs/common';

+import { WorkspaceCacheModule } from 'src/engine/workspace-cache/workspace-cache.module';
 import { WorkspaceMigrationBuildOrchestratorService } from 'src/engine/workspace-manager/workspace-migration-v2/services/workspace-migration-build-orchestrator.service';
 import { WorkspaceMigrationValidateBuildAndRunService } from 'src/engine/workspace-manager/workspace-migration-v2/services/workspace-migration-validate-build-and-run-service';
 import { WorkspaceMigrationBuilderV2Module } from 'src/engine/workspace-manager/workspace-migration-v2/workspace-migration-builder-v2/workspace-migration-builder-v2.module';
@@ -9,6 +10,7 @@ import { WorkspaceMigrationRunnerV2Module } from 'src/engine/workspace-manager/w
  imports: [
    WorkspaceMigrationBuilderV2Module,
    WorkspaceMigrationRunnerV2Module,
+    WorkspaceCacheModule,
  ],
  providers: [
    WorkspaceMigrationValidateBuildAndRunService,