feat: migration can be applied on a specific schema & some enhancements (#2998)

* fix: remove old metadata seed files

* feat: wip standard to core relation

* fix: lint

* fix: merge

* fix: remove debug files

* feat: add feature flag for core object metadata

* fix: remove debug

* feat: always disable the standard core relation

* fix: missing feature flag

* fix: remove debug

* fix: feature flag doesn't seems to disable relation

* fix: delete .vscode folder, change this in another PR

* Update packages/twenty-server/src/workspace/workspace-sync-metadata/reflective-metadata.factory.ts

Co-authored-by: Weiko <corentin@twenty.com>

* Update packages/twenty-server/src/workspace/workspace-sync-metadata/reflective-metadata.factory.ts

Co-authored-by: Weiko <corentin@twenty.com>

* Update packages/twenty-server/src/workspace/workspace-sync-metadata/workspace-sync.metadata.service.ts

Co-authored-by: Weiko <corentin@twenty.com>

* fix: remove optional fields from metadata entities

* fix: renamed variable

* fix: put back CursorScalarType

* fix: delete test command

* fix: remove unused workspace standard migration command

* fix: drop core object metadata declaration

* fix: rename variable

* fix: drop creation of core datasource

* fix: remove feature flag

* fix: drop support of standard to core relations

* feat: add user email field on workspace-member standard object

* fix: update seed accordingly

* fix: missing remove command file

* fix: datasource label should remain nullable

* fix: better asserts

* Remove unused code

* Remove unused code

---------

Co-authored-by: Weiko <corentin@twenty.com>
Co-authored-by: Charles Bochet <charles@twenty.com>

packages/twenty-server/src/workspace/workspace-sync-metadata/utils/convert-class-to-object-metadata-name.util.ts

@@ -0,0 +1,12 @@
+import { camelCase } from 'src/utils/camel-case';
+
+export const convertClassNameToObjectMetadataName = (name: string): string => {
+  const classSuffix = 'ObjectMetadata';
+  let objectName = camelCase(name);
+
+  if (objectName.endsWith(classSuffix)) {
+    objectName = objectName.slice(0, -classSuffix.length);
+  }
+
+  return objectName;
+};

packages/twenty-server/src/workspace/workspace-sync-metadata/utils/is-gate-and-not-enabled.util.ts

@@ -0,0 +1,10 @@
+export const isGatedAndNotEnabled = (
+  metadata,
+  workspaceFeatureFlagsMap: Record<string, boolean>,
+): boolean => {
+  const featureFlagValue =
+    metadata.gate?.featureFlag &&
+    workspaceFeatureFlagsMap[metadata.gate.featureFlag];
+
+  return metadata.gate?.featureFlag !== undefined && !featureFlagValue;
+};

packages/twenty-server/src/workspace/workspace-sync-metadata/utils/metadata.parser.ts

@@ -1,164 +0,0 @@
-import assert from 'assert';
-
-import { FieldMetadataEntity } from 'src/metadata/field-metadata/field-metadata.entity';
-import { ObjectMetadataEntity } from 'src/metadata/object-metadata/object-metadata.entity';
-import { BaseObjectMetadata } from 'src/workspace/workspace-sync-metadata/standard-objects/base.object-metadata';
-
-export class MetadataParser {
-  static parseMetadata(
-    metadata: typeof BaseObjectMetadata,
-    workspaceId: string,
-    dataSourceId: string,
-    workspaceFeatureFlagsMap: Record<string, boolean>,
-  ): ObjectMetadataEntity | undefined {
-    const objectMetadata = Reflect.getMetadata('objectMetadata', metadata);
-    const fieldMetadata = Reflect.getMetadata('fieldMetadata', metadata);
-
-    if (!objectMetadata) {
-      throw new Error(
-        `Object metadata decorator not found, can\'t parse ${metadata.name}`,
-      );
-    }
-
-    if (isGatedAndNotEnabled(objectMetadata, workspaceFeatureFlagsMap)) {
-      return undefined;
-    }
-
-    const fields = Object.values(fieldMetadata).filter(
-      (field) => !isGatedAndNotEnabled(field, workspaceFeatureFlagsMap),
-    );
-
-    return {
-      ...objectMetadata,
-      workspaceId,
-      dataSourceId,
-      fields: fields.map((field: FieldMetadataEntity) => ({
-        ...field,
-        workspaceId,
-        isSystem: objectMetadata.isSystem || field.isSystem,
-        defaultValue: field.defaultValue || null,
-        options: field.options || null,
-      })),
-    };
-  }
-
-  static parseAllMetadata(
-    metadataCollection: (typeof BaseObjectMetadata)[],
-    workspaceId: string,
-    dataSourceId: string,
-    workspaceFeatureFlagsMap: Record<string, boolean>,
-  ): ObjectMetadataEntity[] {
-    return metadataCollection
-      .map((metadata) =>
-        MetadataParser.parseMetadata(
-          metadata,
-          workspaceId,
-          dataSourceId,
-          workspaceFeatureFlagsMap,
-        ),
-      )
-      .filter(
-        (metadata): metadata is ObjectMetadataEntity => metadata !== undefined,
-      );
-  }
-
-  static parseRelationMetadata(
-    metadata: typeof BaseObjectMetadata,
-    workspaceId: string,
-    objectMetadataFromDB: Record<string, ObjectMetadataEntity>,
-    workspaceFeatureFlagsMap: Record<string, boolean>,
-  ) {
-    const objectMetadata = Reflect.getMetadata('objectMetadata', metadata);
-    const relationMetadata = Reflect.getMetadata('relationMetadata', metadata);
-
-    if (!relationMetadata) return [];
-
-    if (!objectMetadata) {
-      throw new Error(
-        `Object metadata decorator not found, can\'t parse ${metadata.name}`,
-      );
-    }
-
-    if (isGatedAndNotEnabled(objectMetadata, workspaceFeatureFlagsMap)) {
-      return [];
-    }
-
-    return relationMetadata
-      .filter(
-        (relation) => !isGatedAndNotEnabled(relation, workspaceFeatureFlagsMap),
-      )
-      .map((relation) => {
-        const fromObjectMetadata =
-          objectMetadataFromDB[relation.fromObjectNameSingular];
-
-        assert(
-          fromObjectMetadata,
-          `Object ${relation.fromObjectNameSingular} not found in DB 
-        for fromRelation defined in class ${objectMetadata.nameSingular}`,
-        );
-
-        const toObjectMetadata =
-          objectMetadataFromDB[relation.toObjectNameSingular];
-
-        assert(
-          toObjectMetadata,
-          `Object ${relation.toObjectNameSingular} not found in DB
-        for toRelation defined in class ${objectMetadata.nameSingular}`,
-        );
-
-        const fromFieldMetadata =
-          fromObjectMetadata?.fields[relation.fromFieldMetadataName];
-
-        assert(
-          fromFieldMetadata,
-          `Field ${relation.fromFieldMetadataName} not found in object ${relation.fromObjectNameSingular}
-        for fromRelation defined in class ${objectMetadata.nameSingular}`,
-        );
-
-        const toFieldMetadata =
-          toObjectMetadata?.fields[relation.toFieldMetadataName];
-
-        assert(
-          toFieldMetadata,
-          `Field ${relation.toFieldMetadataName} not found in object ${relation.toObjectNameSingular}
-        for toRelation defined in class ${objectMetadata.nameSingular}`,
-        );
-
-        return {
-          relationType: relation.type,
-          fromObjectMetadataId: fromObjectMetadata?.id,
-          toObjectMetadataId: toObjectMetadata?.id,
-          fromFieldMetadataId: fromFieldMetadata?.id,
-          toFieldMetadataId: toFieldMetadata?.id,
-          workspaceId,
-        };
-      });
-  }
-
-  static parseAllRelations(
-    metadataCollection: (typeof BaseObjectMetadata)[],
-    workspaceId: string,
-    objectMetadataFromDB: Record<string, ObjectMetadataEntity>,
-    workspaceFeatureFlagsMap: Record<string, boolean>,
-  ) {
-    return metadataCollection.flatMap((metadata) =>
-      MetadataParser.parseRelationMetadata(
-        metadata,
-        workspaceId,
-        objectMetadataFromDB,
-        workspaceFeatureFlagsMap,
-      ),
-    );
-  }
-}
-
-function isGatedAndNotEnabled(
-  metadata,
-  workspaceFeatureFlagsMap: Record<string, boolean>,
-): boolean {
-  const featureFlagValue =
-    metadata.gate?.featureFlag &&
-    workspaceFeatureFlagsMap[metadata.gate.featureFlag];
-
-  return metadata.gate?.featureFlag !== undefined && !featureFlagValue;
-}

packages/twenty-server/src/workspace/workspace-sync-metadata/utils/sync-metadata.util.ts

@@ -1,4 +1,6 @@
-import { ObjectMetadataEntity } from 'src/metadata/object-metadata/object-metadata.entity';
+import { FieldMetadataDefaultValue } from 'src/metadata/field-metadata/interfaces/field-metadata-default-value.interface';
+import { FieldMetadataOptions } from 'src/metadata/field-metadata/interfaces/field-metadata-options.interface';
+import { FieldMetadataTargetColumnMap } from 'src/metadata/field-metadata/interfaces/field-metadata-target-column-map.interface';
 
 /**
  * This utility function filters out properties from an object based on a list of properties to ignore.
@@ -28,9 +30,12 @@ export const filterIgnoredProperties = (
  * @param arr - The array of ObjectMetadataEntity objects to convert.
  * @returns A map of object metadata, with nameSingular as the key and the object as the value.
  */
-export const mapObjectMetadataByUniqueIdentifier = (
-  arr: ObjectMetadataEntity[],
-) => {
+export const mapObjectMetadataByUniqueIdentifier = <
+  T extends { nameSingular: string; fields: U[] },
+  U extends { name: string },
+>(
+  arr: T[],
+): Record<string, Omit<T, 'fields'> & { fields: Record<string, U> }> => {
   return arr.reduce((acc, curr) => {
     acc[curr.nameSingular] = {
       ...curr,
@@ -38,29 +43,41 @@ export const mapObjectMetadataByUniqueIdentifier = (
         acc[curr.name] = curr;
 
         return acc;
-      }, {}),
+      }, {} as Record<string, U>),
     };
 
     return acc;
-  }, {});
+  }, {} as Record<string, Omit<T, 'fields'> & { fields: Record<string, U> }>);
 };
 
-export const convertStringifiedFieldsToJSON = (fieldMetadata) => {
+export const convertStringifiedFieldsToJSON = <
+  T extends {
+    targetColumnMap?: string | null;
+    defaultValue?: string | null;
+    options?: string | null;
+  },
+>(
+  fieldMetadata: T,
+): T & {
+  targetColumnMap?: FieldMetadataTargetColumnMap;
+  defaultValue?: FieldMetadataDefaultValue;
+  options?: FieldMetadataOptions;
+} => {
   if (fieldMetadata.targetColumnMap) {
-    fieldMetadata.targetColumnMap = JSON.parse(
-      fieldMetadata.targetColumnMap as unknown as string,
-    );
-  }
-  if (fieldMetadata.defaultValue) {
-    fieldMetadata.defaultValue = JSON.parse(
-      fieldMetadata.defaultValue as unknown as string,
-    );
-  }
-  if (fieldMetadata.options) {
-    fieldMetadata.options = JSON.parse(
-      fieldMetadata.options as unknown as string,
-    );
+    fieldMetadata.targetColumnMap = JSON.parse(fieldMetadata.targetColumnMap);
   }
 
-  return fieldMetadata;
+  if (fieldMetadata.defaultValue) {
+    fieldMetadata.defaultValue = JSON.parse(fieldMetadata.defaultValue);
+  }
+
+  if (fieldMetadata.options) {
+    fieldMetadata.options = JSON.parse(fieldMetadata.options);
+  }
+
+  return fieldMetadata as T & {
+    targetColumnMap?: FieldMetadataTargetColumnMap;
+    defaultValue?: FieldMetadataDefaultValue;
+    options?: FieldMetadataOptions;
+  };
 };