Index v2 runner (#14537)

## Introduction https://github.com/twentyhq/twenty/pull/14363 In this PR we're creating builder and runner for index metadata Removing its previous integration within object builder ## Next: - implem index impact on field and object metadata api - Add integration testing coverage add integration test on index - Implement uniqueness toggle in field metadata api - add integration test on uniqueness related to https://github.com/twentyhq/core-team-issues/issues/1344
2025-09-17 15:02:43 +02:00
parent 6e47c1b5c6
commit 2d8669ae71
55 changed files with 1692 additions and 1064 deletions
@@ -24,6 +24,7 @@ export const throwOnFieldInputTranspilationsError: ThrowOnFieldInputTranspilatio
      throw new WorkspaceMigrationBuilderExceptionV2(
        {
          report: {
+            index: [],
            view: [],
            viewField: [],
            objectMetadata: [
@@ -1,24 +0,0 @@
-import { faker } from '@faker-js/faker';
-
-import { type FlatIndexFieldMetadata } from 'src/engine/workspace-manager/workspace-migration-v2/types/flat-index-field-metadata';
-
-type FlatIndexFieldMetadataOverrides = Required<
-  Pick<
-    FlatIndexFieldMetadata,
-    'fieldMetadataId' | 'indexMetadataId' | 'universalIdentifier'
-  >
-> &
-  Partial<FlatIndexFieldMetadata>;
-export const getFlatIndexFieldMetadataMock = (
-  overrides: FlatIndexFieldMetadataOverrides,
-): FlatIndexFieldMetadata => {
-  const createdAt = '2024-01-01T00:00:00.000Z' as unknown as Date;
-
-  return {
-    createdAt,
-    id: faker.string.uuid(),
-    order: faker.number.int(),
-    updatedAt: createdAt,
-    ...overrides,
-  };
-};
@@ -17,7 +17,7 @@ export const getFlatIndexMetadataMock = (
    createdAt,
    id: faker.string.uuid(),
    indexType: IndexType.BTREE,
-    indexWhereClause: undefined,
+    indexWhereClause: null,
    isCustom: false,
    isUnique: false,
    name: 'defaultFlatIndexMetadataName',
@@ -0,0 +1,9 @@
+import { type FlatIndexMetadata } from 'src/engine/metadata-modules/flat-index-metadata/types/flat-index-metadata.type';
+
+export const FLAT_INDEX_METADATA_PROPERTIES_TO_COMPARE = [
+  'indexType',
+  'indexWhereClause',
+  'flatIndexFieldMetadatas',
+  'isUnique',
+  'name',
+] as const satisfies (keyof FlatIndexMetadata)[];
@@ -0,0 +1,6 @@
+import { type IndexMetadataRelationProperties } from 'src/engine/metadata-modules/flat-index-metadata/types/flat-index-metadata.type';
+
+export const INDEX_METADATA_ENTITY_RELATION_PROPERTIES = [
+  'objectMetadata',
+  'indexFieldMetadatas',
+] as const satisfies readonly IndexMetadataRelationProperties[];
@@ -0,0 +1,10 @@
+export enum IndexExceptionCode {
+  INDEX_EMPTY_FIELDS = 'INDEX_EMPTY_FIELDS',
+  INDEX_FIELD_NOT_FOUND = 'INDEX_FIELD_NOT_FOUND',
+  INDEX_FIELD_WRONG_OBJECT = 'INDEX_FIELD_WRONG_OBJECT',
+  INDEX_FIELD_INVALID_REFERENCE = 'INDEX_FIELD_INVALID_REFERENCE',
+  INDEX_FIELD_ID_DUPLICATE = 'INDEX_FIELD_ID_DUPLICATE',
+  INDEX_ALREADY_EXISTS = 'INDEX_ALREADY_EXISTS',
+  INDEX_NOT_FOUND = 'INDEX_NOT_FOUND',
+  INDEX_OBJECT_NOT_FOUND = 'INDEX_OBJECT_NOT_FOUND',
+}
@@ -0,0 +1,64 @@
+import { Injectable } from '@nestjs/common';
+import { InjectRepository } from '@nestjs/typeorm';
+
+import { Repository } from 'typeorm';
+
+import { InjectCacheStorage } from 'src/engine/core-modules/cache-storage/decorators/cache-storage.decorator';
+import { CacheStorageService } from 'src/engine/core-modules/cache-storage/services/cache-storage.service';
+import { CacheStorageNamespace } from 'src/engine/core-modules/cache-storage/types/cache-storage-namespace.enum';
+import { EMPTY_FLAT_ENTITY_MAPS } from 'src/engine/core-modules/common/constant/empty-flat-entity-maps.constant';
+import { FlatEntityMaps } from 'src/engine/core-modules/common/types/flat-entity-maps.type';
+import { FlatIndexMetadata } from 'src/engine/metadata-modules/flat-index-metadata/types/flat-index-metadata.type';
+import { fromIndexMetadataEntityToFlatIndexMetadata } from 'src/engine/metadata-modules/flat-index-metadata/utils/from-index-metadata-entity-to-flat-index-metadata.util';
+import { IndexMetadataEntity } from 'src/engine/metadata-modules/index-metadata/index-metadata.entity';
+import { WorkspaceFlatMapCache } from 'src/engine/workspace-flat-map-cache/decorators/workspace-flat-map-cache.decorator';
+import { WorkspaceFlatMapCacheService } from 'src/engine/workspace-flat-map-cache/services/workspace-flat-map-cache.service';
+
+@Injectable()
+@WorkspaceFlatMapCache('flatIndexMaps')
+export class WorkspaceFlatIndexMapCacheService extends WorkspaceFlatMapCacheService<
+  FlatEntityMaps<FlatIndexMetadata>
+> {
+  constructor(
+    @InjectCacheStorage(CacheStorageNamespace.EngineWorkspace)
+    cacheStorageService: CacheStorageService,
+    @InjectRepository(IndexMetadataEntity)
+    private readonly indexMetadataRepository: Repository<IndexMetadataEntity>,
+  ) {
+    super(cacheStorageService);
+  }
+
+  protected async computeFlatMap({
+    workspaceId,
+  }: {
+    workspaceId: string;
+  }): Promise<FlatEntityMaps<FlatIndexMetadata>> {
+    const indexes = await this.indexMetadataRepository.find({
+      where: {
+        workspaceId,
+      },
+      withDeleted: true,
+      relations: ['indexFieldMetadatas'],
+    });
+
+    const flatIndexMaps = indexes.reduce<FlatEntityMaps<FlatIndexMetadata>>(
+      (flatEntityMaps, index) => {
+        const flatIndex = fromIndexMetadataEntityToFlatIndexMetadata(index);
+
+        return {
+          byId: {
+            ...flatEntityMaps.byId,
+            [flatIndex.id]: flatIndex,
+          },
+          idByUniversalIdentifier: {
+            ...flatEntityMaps.idByUniversalIdentifier,
+            [flatIndex.universalIdentifier]: flatIndex.id,
+          },
+        };
+      },
+      EMPTY_FLAT_ENTITY_MAPS,
+    );
+
+    return flatIndexMaps;
+  }
+}
@@ -1,15 +0,0 @@
-import { type IndexFieldMetadataEntity } from 'src/engine/metadata-modules/index-metadata/index-field-metadata.entity';
-import { type ExtractRecordTypeOrmRelationProperties } from 'src/engine/workspace-manager/workspace-migration-v2/types/extract-record-typeorm-relation-properties.type';
-import { type MetadataEntitiesRelationTarget } from 'src/engine/workspace-manager/workspace-migration-v2/types/metadata-entities-relation-targets.type';
-
-type IndexFieldMetadataEntityRelationProperties =
-  ExtractRecordTypeOrmRelationProperties<
-    IndexFieldMetadataEntity,
-    MetadataEntitiesRelationTarget
-  >;
-
-export type FlatIndexFieldMetadata = Partial<
-  Omit<IndexFieldMetadataEntity, IndexFieldMetadataEntityRelationProperties>
-> & {
-  universalIdentifier: string;
-};
@@ -0,0 +1,4 @@
+import { type FLAT_INDEX_METADATA_PROPERTIES_TO_COMPARE } from 'src/engine/metadata-modules/flat-index-metadata/constants/flat-index-metadata-properties-to-compare.constant';
+
+export type FlatIndexMetadataPropertiesToCompare =
+  (typeof FLAT_INDEX_METADATA_PROPERTIES_TO_COMPARE)[number];
@@ -1,18 +1,24 @@
+import { type IndexFieldMetadataEntity } from 'src/engine/metadata-modules/index-metadata/index-field-metadata.entity';
 import { type IndexMetadataEntity } from 'src/engine/metadata-modules/index-metadata/index-metadata.entity';
 import { type ExtractRecordTypeOrmRelationProperties } from 'src/engine/workspace-manager/workspace-migration-v2/types/extract-record-typeorm-relation-properties.type';
-import { type FlatIndexFieldMetadata } from 'src/engine/workspace-manager/workspace-migration-v2/types/flat-index-field-metadata';
 import { type MetadataEntitiesRelationTarget } from 'src/engine/workspace-manager/workspace-migration-v2/types/metadata-entities-relation-targets.type';

-type IndexMetadataRelationProperties = ExtractRecordTypeOrmRelationProperties<
-  IndexMetadataEntity,
-  MetadataEntitiesRelationTarget
->;
+export type IndexMetadataRelationProperties =
+  ExtractRecordTypeOrmRelationProperties<
+    IndexMetadataEntity,
+    MetadataEntitiesRelationTarget
+  >;

-// TODO prastoin refactor FlatIndexMetadata to not be a Partial extension of IndexMetadataEntity
-export type FlatIndexMetadata = Partial<
-  Omit<IndexMetadataEntity, IndexMetadataRelationProperties>
+export type FlatIndexMetadata = Omit<
+  IndexMetadataEntity,
+  IndexMetadataRelationProperties
 > & {
-  id: string;
-  flatIndexFieldMetadatas: FlatIndexFieldMetadata[];
  universalIdentifier: string;
+  flatIndexFieldMetadatas: Omit<
+    IndexFieldMetadataEntity,
+    ExtractRecordTypeOrmRelationProperties<
+      IndexFieldMetadataEntity,
+      MetadataEntitiesRelationTarget
+    >
+  >[];
 };
@@ -1,37 +1,32 @@
 import diff from 'microdiff';
 import { type FromTo } from 'twenty-shared/types';
+import { parseJson } from 'twenty-shared/utils';

+import { FLAT_INDEX_METADATA_PROPERTIES_TO_COMPARE } from 'src/engine/metadata-modules/flat-index-metadata/constants/flat-index-metadata-properties-to-compare.constant';
+import { type FlatIndexMetadataPropertiesToCompare } from 'src/engine/metadata-modules/flat-index-metadata/types/flat-index-metadata-properties-to-compare.type';
 import { type FlatIndexMetadata } from 'src/engine/metadata-modules/flat-index-metadata/types/flat-index-metadata.type';
 import { transformMetadataForComparison } from 'src/engine/workspace-manager/workspace-sync-metadata/comparators/utils/transform-metadata-for-comparison.util';

-const flatIndexMetadataPropertiesToCompare = [
-  'flatIndexFieldMetadatas', // Comparing this as whole ? should iterate on each keys ? => TBD should only map over cols as before ?
-  'indexType',
-  'indexWhereClause',
-  'isUnique',
-  'name',
-] as const satisfies (keyof FlatIndexMetadata)[];
-
-type FlatIndexMetadataPropertiesToCompare =
-  (typeof flatIndexMetadataPropertiesToCompare)[number];
-
-// Should also handle indexFieldMetadata comparison ?
-/**
- * This comparator handles update on colliding universalIdentifier flatIndexMetadata
- */
 export const compareTwoFlatIndexMetadata = ({
-  from,
-  to,
-}: FromTo<FlatIndexMetadata>) => {
+  fromFlatIndexMetadata,
+  toFlatIndexMetadata,
+}: FromTo<FlatIndexMetadata, 'flatIndexMetadata'>) => {
  const transformOptions = {
+    propertiesToStringify: ['flatIndexFieldMetadatas'] as const,
    shouldIgnoreProperty: (property: string) =>
-      !flatIndexMetadataPropertiesToCompare.includes(
+      !FLAT_INDEX_METADATA_PROPERTIES_TO_COMPARE.includes(
        property as FlatIndexMetadataPropertiesToCompare,
      ),
  };

-  const fromCompare = transformMetadataForComparison(from, transformOptions);
-  const toCompare = transformMetadataForComparison(to, transformOptions);
+  const fromCompare = transformMetadataForComparison(
+    fromFlatIndexMetadata,
+    transformOptions,
+  );
+  const toCompare = transformMetadataForComparison(
+    toFlatIndexMetadata,
+    transformOptions,
+  );

  const flatIndexeDifferences = diff(fromCompare, toCompare);

@@ -40,17 +35,20 @@ export const compareTwoFlatIndexMetadata = ({
      switch (difference.type) {
        case 'CHANGE': {
          const { oldValue, path, value } = difference;
+          const property = path[0] as FlatIndexMetadataPropertiesToCompare;

-          const property = path[0];
-
-          if (typeof property === 'number') {
-            return [];
+          if (property === 'flatIndexFieldMetadatas') {
+            return {
+              from: parseJson(oldValue),
+              to: parseJson(value),
+              property,
+            };
          }

          return {
            from: oldValue,
-            property,
            to: value,
+            property,
          };
        }
        case 'CREATE':
@@ -0,0 +1,28 @@
+import { removePropertiesFromRecord } from 'twenty-shared/utils';
+
+import { INDEX_METADATA_ENTITY_RELATION_PROPERTIES } from 'src/engine/metadata-modules/flat-index-metadata/constants/index-metadata-entity-relation-properties.constant';
+import { type FlatIndexMetadata } from 'src/engine/metadata-modules/flat-index-metadata/types/flat-index-metadata.type';
+import { type IndexMetadataEntity } from 'src/engine/metadata-modules/index-metadata/index-metadata.entity';
+
+export const fromIndexMetadataEntityToFlatIndexMetadata = (
+  indexMetadataEntity: IndexMetadataEntity,
+): FlatIndexMetadata => {
+  const indexMetadataEntityWithoutRelations = removePropertiesFromRecord(
+    indexMetadataEntity,
+    [...INDEX_METADATA_ENTITY_RELATION_PROPERTIES],
+  );
+
+  return {
+    ...indexMetadataEntityWithoutRelations,
+    universalIdentifier:
+      indexMetadataEntityWithoutRelations.universalIdentifier ??
+      indexMetadataEntityWithoutRelations.id,
+    flatIndexFieldMetadatas: indexMetadataEntity.indexFieldMetadatas.map(
+      (indexFieldMetadata) =>
+        removePropertiesFromRecord(indexFieldMetadata, [
+          'indexMetadata',
+          'fieldMetadata',
+        ]),
+    ),
+  };
+};
@@ -14,7 +14,9 @@ import { FieldMetadataEntity } from 'src/engine/metadata-modules/field-metadata/
 import { IndexMetadataEntity } from 'src/engine/metadata-modules/index-metadata/index-metadata.entity';

@Entity('indexFieldMetadata')
-export class IndexFieldMetadataEntity {
+export class IndexFieldMetadataEntity
+  implements Required<IndexFieldMetadataEntity>
+{
  @PrimaryGeneratedColumn('uuid')
  id: string;

@@ -12,6 +12,8 @@ import {
  UpdateDateColumn,
 } from 'typeorm';

+import { SyncableEntity } from 'src/engine/workspace-manager/workspace-sync/interfaces/syncable-entity.interface';
+
 import { IndexFieldMetadataEntity } from 'src/engine/metadata-modules/index-metadata/index-field-metadata.entity';
 import { IndexType } from 'src/engine/metadata-modules/index-metadata/types/indexType.types';
 import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity';
@@ -26,7 +28,10 @@ import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadat
  'objectMetadataId',
 ])
@Entity('indexMetadata')
-export class IndexMetadataEntity {
+export class IndexMetadataEntity
+  extends SyncableEntity
+  implements Required<IndexMetadataEntity>
+{
  @PrimaryGeneratedColumn('uuid')
  id: string;