Index v2 side effects (#14567)

# Introduction Honestly this implem is a mess, discussing a potential side effect handler with @weiko before the build and run that would handle each side effect per entity and operation Handling: - [ ] unique index is generated when a field is updated with the `isUnique` - [x] an index is generated when a relation is created - [x] search vector index creation on custom object creation - [x] renaming a field metadata or an object should re-create all related indexes which are composed by their namings - [x] delete object should remove any related indexes - [x] delete field should update related indexes ( if index ends up empty it should be removed ) - [ ] on object renaming that contains morph fields -> triggers update field -> trigger index recompute - [x] on update name renaming should recompute all related indexes ## TODO - [x] Integration testing - [ ] Refactor the index maps cache to be storing a `idsByObjectMetadataId` - [x] Refactor deterministic name to use order sorting - [x] Remove flat index from flat object ## What's next Will handle morph indexes in a new dedicated PR for the moment will stick to this Same for the cache improvement and uniqueness
2025-09-23 10:57:11 +02:00
parent d0b9c7f076
commit fbd3286792
63 changed files with 2591 additions and 1323 deletions
@@ -4,6 +4,8 @@ import { isDefined } from 'twenty-shared/utils';

 import { WorkspaceManyOrAllFlatEntityMapsCacheService } from 'src/engine/core-modules/common/services/workspace-many-or-all-flat-entity-maps-cache.service.';
 import { addFlatEntityToFlatEntityMapsOrThrow } from 'src/engine/core-modules/common/utils/add-flat-entity-to-flat-entity-maps-or-throw.util';
+import { deleteFlatEntityFromFlatEntityMapsOrThrow } from 'src/engine/core-modules/common/utils/delete-flat-entity-from-flat-entity-maps-or-throw.util';
+import { replaceFlatEntityInFlatEntityMapsOrThrow } from 'src/engine/core-modules/common/utils/replace-flat-entity-in-flat-entity-maps-or-throw.util';
 import { ViewKey } from 'src/engine/core-modules/view/enums/view-key.enum';
 import { ViewType } from 'src/engine/core-modules/view/enums/view-type.enum';
 import { FlatView } from 'src/engine/core-modules/view/flat-view/types/flat-view.type';
@@ -53,20 +55,25 @@ export class ObjectMetadataServiceV2 {
    workspaceId: string;
    updateObjectInput: UpdateOneObjectInput;
  }): Promise<ObjectMetadataDTO> {
-    const { flatObjectMetadataMaps: existingFlatObjectMetadataMaps } =
+    const {
+      flatObjectMetadataMaps: existingFlatObjectMetadataMaps,
+      flatIndexMaps: existingFlatIndexMaps,
+    } =
      await this.flatEntityMapsCacheService.getOrRecomputeManyOrAllFlatEntityMaps(
        {
          workspaceId,
-          flatEntities: ['flatObjectMetadataMaps'],
+          flatEntities: ['flatObjectMetadataMaps', 'flatIndexMaps'],
        },
      );

    const {
      flatObjectMetadata: optimisticallyUpdatedFlatObjectMetadata,
-      otherObjectFlatFieldMetadatas,
+      otherObjectFlatFieldMetadataToUpdate: otherObjectFlatFieldMetadatas,
+      flatIndexMetadataToUpdate,
    } = fromUpdateObjectInputToFlatObjectMetadata({
      existingFlatObjectMetadataMaps,
      updateObjectInput,
+      flatIndexMaps: existingFlatIndexMaps,
    });

    const impactedObjectMetadataIds = [
@@ -93,6 +100,15 @@ export class ObjectMetadataServiceV2 {
      }),
    );

+    const toFlatIndexMaps = flatIndexMetadataToUpdate.reduce(
+      (flatIndexMaps, flatIndexMetadata) =>
+        replaceFlatEntityInFlatEntityMapsOrThrow({
+          flatEntity: flatIndexMetadata,
+          flatEntityMaps: flatIndexMaps,
+        }),
+      existingFlatIndexMaps,
+    );
+
    const validateAndBuildResult =
      await this.workspaceMigrationValidateBuildAndRunService.validateBuildAndRunWorkspaceMigration(
        {
@@ -101,6 +117,10 @@ export class ObjectMetadataServiceV2 {
              from: fromFlatObjectMetadataMaps,
              to: toFlatObjectMetadataMaps,
            },
+            flatIndexMaps: {
+              from: existingFlatIndexMaps,
+              to: toFlatIndexMaps,
+            },
          },
          buildOptions: {
            isSystemBuild: false,
@@ -155,19 +175,26 @@ export class ObjectMetadataServiceV2 {
    deleteObjectInput: DeleteOneObjectInput;
    workspaceId: string;
  }): Promise<ObjectMetadataDTO> {
-    const { flatObjectMetadataMaps: existingFlatObjectMetadataMaps } =
+    const {
+      flatObjectMetadataMaps: existingFlatObjectMetadataMaps,
+      flatIndexMaps: existingFlatIndexMaps,
+    } =
      await this.flatEntityMapsCacheService.getOrRecomputeManyOrAllFlatEntityMaps(
        {
          workspaceId,
-          flatEntities: ['flatObjectMetadataMaps'],
+          flatEntities: ['flatObjectMetadataMaps', 'flatIndexMaps'],
        },
      );

-    const { flatFieldMetadatasToDelete, flatObjectMetadataToDelete } =
-      fromDeleteObjectInputToFlatFieldMetadatasToDelete({
-        deleteObjectInput,
-        existingFlatObjectMetadataMaps,
-      });
+    const {
+      flatFieldMetadatasToDelete,
+      flatObjectMetadataToDelete,
+      flatIndexToDelete,
+    } = fromDeleteObjectInputToFlatFieldMetadatasToDelete({
+      existingFlatIndexMaps,
+      deleteObjectInput,
+      existingFlatObjectMetadataMaps,
+    });
    const { id: objectMetadataToDeleteId } = flatObjectMetadataToDelete;

    const impactedObjectMetadataIds = Array.from(
@@ -202,6 +229,15 @@ export class ObjectMetadataServiceV2 {
        }),
      );

+    const toFlatIndexMaps = flatIndexToDelete.reduce(
+      (flatIndexMaps, flatIndex) =>
+        deleteFlatEntityFromFlatEntityMapsOrThrow({
+          entityToDeleteId: flatIndex.id,
+          flatEntityMaps: flatIndexMaps,
+        }),
+      existingFlatIndexMaps,
+    );
+
    const validateAndBuildResult =
      await this.workspaceMigrationValidateBuildAndRunService.validateBuildAndRunWorkspaceMigration(
        {
@@ -210,6 +246,10 @@ export class ObjectMetadataServiceV2 {
              from: fromFlatObjectMetadataMaps,
              to: toFlatObjectMetadataMaps,
            },
+            flatIndexMaps: {
+              from: existingFlatIndexMaps,
+              to: toFlatIndexMaps,
+            },
          },
          buildOptions: {
            inferDeletionFromMissingEntities: true,
@@ -242,6 +282,7 @@ export class ObjectMetadataServiceV2 {
      flatObjectMetadataMaps: existingFlatObjectMetadataMaps,
      flatViewMaps: existingFlatViewMaps,
      flatViewFieldMaps: existingFlatViewFieldMaps,
+      flatIndexMaps: existingFlatIndexMaps,
    } = await this.flatEntityMapsCacheService.getOrRecomputeManyOrAllFlatEntityMaps(
      {
        workspaceId,
@@ -249,19 +290,23 @@ export class ObjectMetadataServiceV2 {
          'flatObjectMetadataMaps',
          'flatViewMaps',
          'flatViewFieldMaps',
+          'flatIndexMaps',
        ],
      },
    );

-    const { flatObjectMetadataToCreate, relationTargetFlatFieldMetadatas } =
-      fromCreateObjectInputToFlatObjectMetadataAndFlatFieldMetadatasToCreate({
-        createObjectInput,
-        workspaceId,
-        existingFlatObjectMetadataMaps,
-      });
+    const {
+      flatObjectMetadataToCreate,
+      relationTargetFlatFieldMetadataToCreate,
+      flatIndexMetadataToCreate,
+    } = fromCreateObjectInputToFlatObjectMetadataAndFlatFieldMetadatasToCreate({
+      createObjectInput,
+      workspaceId,
+      existingFlatObjectMetadataMaps,
+    });

    const existingFlatObjectMetadataMapsWithTargetRelationFlatFieldMetadatas =
-      relationTargetFlatFieldMetadatas.reduce(
+      relationTargetFlatFieldMetadataToCreate.reduce(
        (flatObjectMetadataMaps, flatFieldMetadata) =>
          addFlatFieldMetadataInFlatObjectMetadataMapsOrThrow({
            flatFieldMetadata,
@@ -272,7 +317,7 @@ export class ObjectMetadataServiceV2 {

    const flatObjectMetadataMapsWithTargetRelationFlatFieldMetadatas =
      getSubFlatObjectMetadataMapsOutOfFlatFieldMetadatasOrThrow({
-        flatFieldMetadatas: relationTargetFlatFieldMetadatas,
+        flatFieldMetadatas: relationTargetFlatFieldMetadataToCreate,
        flatObjectMetadataMaps:
          existingFlatObjectMetadataMapsWithTargetRelationFlatFieldMetadatas,
      });
@@ -286,7 +331,7 @@ export class ObjectMetadataServiceV2 {

    const impactedObjectMetadataIds = [
      ...new Set(
-        relationTargetFlatFieldMetadatas.map(
+        relationTargetFlatFieldMetadataToCreate.map(
          ({ objectMetadataId }) => objectMetadataId,
        ),
      ),
@@ -329,6 +374,15 @@ export class ObjectMetadataServiceV2 {
      existingFlatViewFieldMaps,
    );

+    const toFlatIndexMaps = flatIndexMetadataToCreate.reduce(
+      (flatIndexMaps, flatIndexMetadata) =>
+        addFlatEntityToFlatEntityMapsOrThrow({
+          flatEntity: flatIndexMetadata,
+          flatEntityMaps: flatIndexMaps,
+        }),
+      existingFlatIndexMaps,
+    );
+
    const validateAndBuildResult =
      await this.workspaceMigrationValidateBuildAndRunService.validateBuildAndRunWorkspaceMigration(
        {
@@ -345,6 +399,10 @@ export class ObjectMetadataServiceV2 {
              from: existingFlatViewFieldMaps,
              to: toFlatViewFieldMaps,
            },
+            flatIndexMaps: {
+              from: existingFlatIndexMaps,
+              to: toFlatIndexMaps,
+            },
          },
          buildOptions: {
            isSystemBuild: false,
@@ -2,6 +2,7 @@ import { FieldMetadataType } from 'twenty-shared/types';
 import { v4 } from 'uuid';

 import { type FlatFieldMetadata } from 'src/engine/metadata-modules/flat-field-metadata/types/flat-field-metadata.type';
+import { type FlatObjectMetadata } from 'src/engine/metadata-modules/flat-object-metadata/types/flat-object-metadata.type';
 import {
  BASE_OBJECT_STANDARD_FIELD_IDS,
  CUSTOM_OBJECT_STANDARD_FIELD_IDS,
@@ -10,12 +11,16 @@ import { getTsVectorColumnExpressionFromFields } from 'src/engine/workspace-mana

 type BuildDefaultFlatFieldMetadataForCustomObjectArgs = {
  workspaceId: string;
-  objectMetadataId: string;
+  flatObjectMetadata: Pick<FlatObjectMetadata, 'id'>;
 };

+export type DefaultFlatFieldForCustomObjectMaps = ReturnType<
+  typeof buildDefaultFlatFieldMetadatasForCustomObject
+>;
+// This could be replaced totally by an import schema + its transpilation when it's ready
 export const buildDefaultFlatFieldMetadatasForCustomObject = ({
  workspaceId,
-  objectMetadataId,
+  flatObjectMetadata: { id: objectMetadataId },
 }: BuildDefaultFlatFieldMetadataForCustomObjectArgs) => {
  const createdAt = new Date();
  const idField: FlatFieldMetadata<FieldMetadataType.UUID> = {
@@ -278,13 +283,17 @@ export const buildDefaultFlatFieldMetadatasForCustomObject = ({
  };

  return {
-    idField,
-    nameField,
-    createdAtField,
-    updatedAtField,
-    deletedAtField,
-    createdByField,
-    positionField,
-    searchVectorField,
-  } as const;
+    fields: {
+      idField,
+      nameField,
+      createdAtField,
+      updatedAtField,
+      deletedAtField,
+      createdByField,
+      positionField,
+      searchVectorField,
+    },
+  } as const satisfies {
+    fields: Record<string, FlatFieldMetadata>;
+  };
 };
@@ -0,0 +1,52 @@
+import { v4 } from 'uuid';
+
+import { type FlatIndexMetadata } from 'src/engine/metadata-modules/flat-index-metadata/types/flat-index-metadata.type';
+import { type FlatObjectMetadata } from 'src/engine/metadata-modules/flat-object-metadata/types/flat-object-metadata.type';
+import { IndexType } from 'src/engine/metadata-modules/index-metadata/types/indexType.types';
+import { generateFlatIndexMetadataWithNameOrThrow } from 'src/engine/metadata-modules/index-metadata/utils/generate-flat-index.util';
+import { type DefaultFlatFieldForCustomObjectMaps } from 'src/engine/metadata-modules/object-metadata/utils/build-default-flat-field-metadatas-for-custom-object.util';
+
+export const buildDefaultIndexesForCustomObject = ({
+  workspaceId,
+  flatObjectMetadata,
+  defaultFlatFieldForCustomObjectMaps,
+}: {
+  workspaceId: string;
+  flatObjectMetadata: FlatObjectMetadata;
+  defaultFlatFieldForCustomObjectMaps: DefaultFlatFieldForCustomObjectMaps;
+}) => {
+  const tsFlatVectorIndexId = v4();
+  const createdAt = new Date();
+  const tsVectorFlatIndex = generateFlatIndexMetadataWithNameOrThrow({
+    flatIndex: {
+      createdAt,
+      flatIndexFieldMetadatas: [
+        {
+          createdAt,
+          fieldMetadataId:
+            defaultFlatFieldForCustomObjectMaps.fields.searchVectorField.id,
+          id: v4(),
+          indexMetadataId: tsFlatVectorIndexId,
+          order: 0,
+          updatedAt: createdAt,
+        },
+      ],
+      id: tsFlatVectorIndexId,
+      indexType: IndexType.GIN,
+      indexWhereClause: null,
+      isCustom: false,
+      isUnique: false,
+      objectMetadataId: flatObjectMetadata.id,
+      universalIdentifier: tsFlatVectorIndexId,
+      updatedAt: createdAt,
+      workspaceId,
+    },
+    flatObjectMetadata,
+  });
+
+  return {
+    indexes: {
+      tsVectorFlatIndex,
+    },
+  } as const satisfies { indexes: Record<string, FlatIndexMetadata> };
+};